Merge pull request #2 from atlanhq/atlan-v0.6.cache

PLT-191: support cache feature
atlanhq · Dec 11, 2023 · e7209e1 · e7209e1
2 parents d88e2a4 + 1f75484
commit e7209e1
Show file tree

Hide file tree

Showing 32 changed files with 57 additions and 30 deletions.
diff --git a/janusgraph-all/pom.xml b/janusgraph-all/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.01</version>
+        <version>0.6.02</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-all</artifactId>

diff --git a/janusgraph-backend-testutils/pom.xml b/janusgraph-backend-testutils/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.01</version>
+        <version>0.6.02</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-backend-testutils</artifactId>

diff --git a/janusgraph-berkeleyje/pom.xml b/janusgraph-berkeleyje/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.01</version>
+        <version>0.6.02</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-berkeleyje</artifactId>

diff --git a/janusgraph-bigtable/pom.xml b/janusgraph-bigtable/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.01</version>
+        <version>0.6.02</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-bigtable</artifactId>

diff --git a/janusgraph-core/pom.xml b/janusgraph-core/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.01</version>
+        <version>0.6.02</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-core</artifactId>

diff --git a/...c/main/java/org/janusgraph/diskstorage/keycolumnvalue/cache/ExpirationKCVSRedisCache.java b/...c/main/java/org/janusgraph/diskstorage/keycolumnvalue/cache/ExpirationKCVSRedisCache.java
@@ -69,6 +69,7 @@ public ExpirationKCVSRedisCache(final KeyColumnValueStore store, String metricsN
         redisCache = redissonClient.getMapCache(String.join("-", configuration.get(CACHE_KEYSPACE_PREFIX), metricsName));
         redisIndexKeys = redissonClient.getMapCache(String.join("-", configuration.get(CACHE_KEYSPACE_PREFIX), REDIS_INDEX_CACHE_PREFIX, metricsName));
         redisCache.setMaxSize(configuration.get(REDIS_MAX_CACHE_SIZE), EvictionMode.LFU);
+        redisIndexKeys.setMaxSize(configuration.get(REDIS_MAX_CACHE_SIZE), EvictionMode.LFU);
         logger.info("********************** Configurations are loaded **********************");
     }
 
@@ -87,7 +88,7 @@ public EntryList getSlice(final KeySliceQuery query, final StoreTransaction txh)
         }
     }
 
-    private EntryList get(KeySliceQuery query, Callable<EntryList> valueLoader) {
+    private EntryList get(KeySliceQuery query, Callable<EntryList> valueLoader) throws Exception {
         byte[] bytQuery = redisCache.get(query);
         EntryList entries = bytQuery != null ? (EntryList) fastConf.asObject(bytQuery) : null;
         if (entries == null) {
@@ -97,23 +98,25 @@ private EntryList get(KeySliceQuery query, Callable<EntryList> valueLoader) {
                 if (entries == null) {
                     throw new CacheLoader.InvalidCacheLoadException("valueLoader must not return null, key=" + query);
                 } else {
-                    redisCache.fastPutAsync(query, fastConf.asByteArray(entries), this.cacheTimeMS,TimeUnit.MILLISECONDS);
+                    redisCache.fastPutAsync(query, fastConf.asByteArray(entries), this.cacheTimeMS, TimeUnit.MILLISECONDS);
                     RLock lock = redisIndexKeys.getLock(query.getKey());
                     try {
                         lock.tryLock(1, 3, TimeUnit.SECONDS);
                         ArrayList<KeySliceQuery> queryList = redisIndexKeys.get(query.getKey());
                         if (queryList == null)
                             queryList = new ArrayList<>();
                         queryList.add(query);
-                        redisIndexKeys.fastPutAsync(query.getKey(), queryList, this.cacheTimeMS,TimeUnit.MILLISECONDS);
+                        redisIndexKeys.fastPutAsync(query.getKey(), queryList, this.cacheTimeMS, TimeUnit.MILLISECONDS);
                     } catch (InterruptedException e) {
                         e.printStackTrace();
+                        throw e;
                     } finally {
                         lock.unlock();
                     }
                 }
             } catch (Exception e) {
                 e.printStackTrace();
+                throw e;
             }
         }
         return entries;

diff --git a/janusgraph-core/src/main/java/org/janusgraph/diskstorage/keycolumnvalue/cache/KCVSCache.java b/janusgraph-core/src/main/java/org/janusgraph/diskstorage/keycolumnvalue/cache/KCVSCache.java
@@ -39,6 +39,7 @@ public abstract class KCVSCache extends KCVSProxy {
 
     private final String metricsName;
 
+    protected String getMetricsName(){return this.metricsName;}
     protected KCVSCache(KeyColumnValueStore store, String metricsName) {
         super(store);
         this.metricsName = metricsName;

diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/StandardJanusGraph.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/database/StandardJanusGraph.java
@@ -83,11 +83,11 @@
 import org.janusgraph.graphdb.tinkerpop.JanusGraphFeatures;
 import org.janusgraph.graphdb.tinkerpop.optimize.strategy.AdjacentVertexFilterOptimizerStrategy;
 import org.janusgraph.graphdb.tinkerpop.optimize.strategy.AdjacentVertexHasIdOptimizerStrategy;
-import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphMixedIndexCountStrategy;
 import org.janusgraph.graphdb.tinkerpop.optimize.strategy.AdjacentVertexHasUniquePropertyOptimizerStrategy;
 import org.janusgraph.graphdb.tinkerpop.optimize.strategy.AdjacentVertexIsOptimizerStrategy;
 import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphIoRegistrationStrategy;
 import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphLocalQueryOptimizerStrategy;
+import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphMixedIndexCountStrategy;
 import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphMultiQueryStrategy;
 import org.janusgraph.graphdb.tinkerpop.optimize.strategy.JanusGraphStepStrategy;
 import org.janusgraph.graphdb.transaction.StandardJanusGraphTx;
@@ -485,11 +485,17 @@ public void remove() {
 
     public EntryList edgeQuery(long vid, SliceQuery query, BackendTransaction tx) {
         Preconditions.checkArgument(vid > 0);
+        if (!this.isCacheEnabled()) {
+            tx.disableCache();
+        }
         return tx.edgeStoreQuery(new KeySliceQuery(idManager.getKey(vid), query));
     }
 
     public List<EntryList> edgeMultiQuery(LongArrayList vertexIdsAsLongs, SliceQuery query, BackendTransaction tx) {
         Preconditions.checkArgument(vertexIdsAsLongs != null && !vertexIdsAsLongs.isEmpty());
+        if (!this.isCacheEnabled()) {
+            tx.disableCache();
+        }
         final List<StaticBuffer> vertexIds = new ArrayList<>(vertexIdsAsLongs.size());
         for (int i = 0; i < vertexIdsAsLongs.size(); i++) {
             Preconditions.checkArgument(vertexIdsAsLongs.get(i) > 0);

diff --git a/...sgraph-core/src/main/java/org/janusgraph/graphdb/log/StandardTransactionLogProcessor.java b/...sgraph-core/src/main/java/org/janusgraph/graphdb/log/StandardTransactionLogProcessor.java
@@ -393,12 +393,14 @@ protected void waitCondition() throws InterruptedException {
 
         @Override
         protected void action() {
+            graph.setEnableCache(false);
             lastInvocation = times.getTime();
             txCache.cleanUp();
         }
 
         @Override
         protected void cleanup() {
+            graph.setEnableCache(false);
             txCache.cleanUp();
         }
     }

diff --git a/...sgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/JanusGraphBlueprintsGraph.java b/...sgraph-core/src/main/java/org/janusgraph/graphdb/tinkerpop/JanusGraphBlueprintsGraph.java
@@ -70,6 +70,8 @@ public abstract class JanusGraphBlueprintsGraph implements JanusGraph {
 
     private ThreadLocal<JanusGraphBlueprintsTransaction> txs = ThreadLocal.withInitial(() -> null);
 
+    private final ThreadLocal<Boolean> cacheEnabled = ThreadLocal.withInitial(()-> true);
+
     public abstract JanusGraphTransaction newThreadBoundTransaction();
 
     private JanusGraphBlueprintsTransaction getAutoStartTx() {
@@ -94,6 +96,13 @@ public JanusGraphTransaction getCurrentThreadTx() {
         return getAutoStartTx();
     }
 
+    public void setEnableCache(boolean enableCache) {
+        this.cacheEnabled.set(enableCache);
+    }
+
+    public boolean isCacheEnabled() {
+        return this.cacheEnabled.get();
+    }
 
     @Override
     public synchronized void close() {

diff --git a/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/StandardJanusGraphTx.java b/janusgraph-core/src/main/java/org/janusgraph/graphdb/transaction/StandardJanusGraphTx.java
@@ -1202,6 +1202,9 @@ public JanusGraphMultiVertexQuery multiQuery(Collection<JanusGraphVertex> vertic
     }
 
     public void executeMultiQuery(final Collection<InternalVertex> vertices, final SliceQuery sq, final QueryProfiler profiler) {
+        if(!getGraph().isCacheEnabled()) {
+            txHandle.disableCache();
+        }
         LongArrayList vertexIds = new LongArrayList(vertices.size());
         for (InternalVertex v : vertices) {
             if (!v.isNew() && v.hasId() && (v instanceof CacheVertex) && !v.hasLoadedRelations(sq)) vertexIds.add(v.longId());
@@ -1409,6 +1412,9 @@ else if (query.getResultType() == ElementCategory.VERTEX) {
         @Override
         public Iterator<JanusGraphElement> execute(final GraphCentricQuery query, final JointIndexQuery indexQuery, final Object exeInfo, final QueryProfiler profiler) {
             Iterator<JanusGraphElement> iterator;
+            if(!getGraph().isCacheEnabled()) {
+                txHandle.disableCache();
+            }
             if (!indexQuery.isEmpty()) {
                 final List<QueryUtil.IndexCall<Object>> retrievals = new ArrayList<>();
                 // Leave first index for streaming, and prepare the rest for intersecting and lookup

diff --git a/janusgraph-cql/pom.xml b/janusgraph-cql/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.01</version>
+        <version>0.6.02</version>
     </parent>
 
     <artifactId>janusgraph-cql</artifactId>

diff --git a/janusgraph-dist/pom.xml b/janusgraph-dist/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.01</version>
+        <version>0.6.02</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <packaging>pom</packaging>

diff --git a/janusgraph-doc/pom.xml b/janusgraph-doc/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.01</version>
+        <version>0.6.02</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <packaging>pom</packaging>

diff --git a/janusgraph-driver/pom.xml b/janusgraph-driver/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.01</version>
+        <version>0.6.02</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-driver</artifactId>

diff --git a/janusgraph-es/pom.xml b/janusgraph-es/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.01</version>
+        <version>0.6.02</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-es</artifactId>

diff --git a/janusgraph-examples/example-berkeleyje/pom.xml b/janusgraph-examples/example-berkeleyje/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph-examples</artifactId>
-        <version>0.6.01</version>
+        <version>0.6.02</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>example-berkeleyje</artifactId>

diff --git a/janusgraph-examples/example-common/pom.xml b/janusgraph-examples/example-common/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph-examples</artifactId>
-        <version>0.6.01</version>
+        <version>0.6.02</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>example-common</artifactId>

diff --git a/janusgraph-examples/example-cql/pom.xml b/janusgraph-examples/example-cql/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph-examples</artifactId>
-        <version>0.6.01</version>
+        <version>0.6.02</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>example-cql</artifactId>

diff --git a/janusgraph-examples/example-hbase/pom.xml b/janusgraph-examples/example-hbase/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph-examples</artifactId>
-        <version>0.6.01</version>
+        <version>0.6.02</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>example-hbase</artifactId>

diff --git a/janusgraph-examples/example-remotegraph/pom.xml b/janusgraph-examples/example-remotegraph/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph-examples</artifactId>
-        <version>0.6.01</version>
+        <version>0.6.02</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>example-remotegraph</artifactId>

diff --git a/janusgraph-examples/example-tinkergraph/pom.xml b/janusgraph-examples/example-tinkergraph/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph-examples</artifactId>
-        <version>0.6.01</version>
+        <version>0.6.02</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>example-tinkergraph</artifactId>

diff --git a/janusgraph-examples/pom.xml b/janusgraph-examples/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.01</version>
+        <version>0.6.02</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-examples</artifactId>

diff --git a/janusgraph-grpc/pom.xml b/janusgraph-grpc/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.01</version>
+        <version>0.6.02</version>
     </parent>
     <artifactId>janusgraph-grpc</artifactId>
     <name>JanusGraph-gRPC: gRPC Components for JanusGraph</name>

diff --git a/janusgraph-hadoop/pom.xml b/janusgraph-hadoop/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.01</version>
+        <version>0.6.02</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-hadoop</artifactId>

diff --git a/janusgraph-hbase/pom.xml b/janusgraph-hbase/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.01</version>
+        <version>0.6.02</version>
     </parent>
     <artifactId>janusgraph-hbase</artifactId>
     <name>JanusGraph-HBase: Distributed Graph Database</name>

diff --git a/janusgraph-inmemory/pom.xml b/janusgraph-inmemory/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.01</version>
+        <version>0.6.02</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-inmemory</artifactId>

diff --git a/janusgraph-lucene/pom.xml b/janusgraph-lucene/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.01</version>
+        <version>0.6.02</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-lucene</artifactId>

diff --git a/janusgraph-server/pom.xml b/janusgraph-server/pom.xml
@@ -4,7 +4,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.01</version>
+        <version>0.6.02</version>
     </parent>
     <artifactId>janusgraph-server</artifactId>
     <name>JanusGraph-Server: Server Components for JanusGraph</name>

diff --git a/janusgraph-solr/pom.xml b/janusgraph-solr/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.01</version>
+        <version>0.6.02</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-solr</artifactId>

diff --git a/janusgraph-test/pom.xml b/janusgraph-test/pom.xml
@@ -3,7 +3,7 @@
     <parent>
         <groupId>org.janusgraph</groupId>
         <artifactId>janusgraph</artifactId>
-        <version>0.6.01</version>
+        <version>0.6.02</version>
         <relativePath>../pom.xml</relativePath>
     </parent>
     <artifactId>janusgraph-test</artifactId>

diff --git a/pom.xml b/pom.xml
@@ -4,7 +4,7 @@
     <modelVersion>4.0.0</modelVersion>
     <groupId>org.janusgraph</groupId>
     <artifactId>janusgraph</artifactId>
-    <version>0.6.01</version>
+    <version>0.6.02</version>
     <packaging>pom</packaging>
     <prerequisites>
         <maven>3.0.0</maven>