Polydata4 caching improvements (#2)

* Add ehcache xml * polydata 4 caching * Cache usage improvements * Imports cleanup
unidev-polydata · Mar 25, 2023 · 65a0d86 · 65a0d86
1 parent 6206bab
commit 65a0d86
Show file tree

Hide file tree

Showing 8 changed files with 131 additions and 11 deletions.
diff --git a/build.gradle b/build.gradle
@@ -54,6 +54,8 @@ subprojects {
         targetCompatibility = JavaVersion.VERSION_17
     }
 
+    tasks.withType(Copy).all { duplicatesStrategy 'exclude' }
+
     test {
         useJUnitPlatform()
     }

diff --git a/gradle.properties b/gradle.properties
@@ -16,4 +16,5 @@ commonsCodecVersion=1.15
 commonsIoVersion=2.11.0
 sqliteJdbcVersion=3.40.0.0
 flywayCoreVersion=9.10.2
-commonsTextVersion=1.10.0
+commonsTextVersion=1.10.0
+unidevTemplatesVersion=4.0.4
diff --git a/polydata-factory/build.gradle b/polydata-factory/build.gradle
@@ -9,6 +9,10 @@ dependencies {
     testImplementation(
             "org.ehcache:ehcache:$testEhcacheVersion"
     )
+
+    testImplementation("com.unidev:unidev-template-freemarker:$unidevTemplatesVersion") {
+        exclude group: "ch.qos.logback", module: "logback-classic"
+    }
 }
 
 sourceSets {

diff --git a/polydata-factory/src/itest/java/com/unidev/polydata4/IntegrationTest.java b/polydata-factory/src/itest/java/com/unidev/polydata4/IntegrationTest.java
@@ -1,10 +1,12 @@
 package com.unidev.polydata4;
 
+import com.unidev.platform.template.TemplateBuilder;
 import com.unidev.polydata4.api.Polydata;
 import com.unidev.polydata4.domain.BasicPoly;
 import com.unidev.polydata4.domain.BasicPolyList;
 import com.unidev.polydata4.domain.BasicPolyQuery;
 import com.unidev.polydata4.domain.InsertRequest;
+import freemarker.template.Template;
 import org.junit.jupiter.api.Test;
 
 import java.util.*;
@@ -55,6 +57,7 @@ void operationsById() {
         data.put("test-key", "test-value");
         polydata.insert(poly, Set.of(InsertRequest.builder().data(data).build()));
         BasicPolyList list = polydata.read(poly, Set.of("test-id"));
+        list = polydata.read(poly, Set.of("test-id"));
         assertNotNull(list);
         assertEquals(1, list.list().size());
         assertTrue(list.hasPoly("test-id"));
@@ -285,5 +288,14 @@ String createPoly() {
         return poly;
     }
 
+    public static String renderTemplate(String content, Map<String, Object> variables) {
+        try {
+            Template template = TemplateBuilder.newTemplate(content).build().get();
+            String renderedFile = TemplateBuilder.evaluate(template, variables).get();
+            return renderedFile;
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
 
 }
diff --git a/polydata-factory/src/itest/java/com/unidev/polydata4/MongodbCacheIntegrationTest.java b/polydata-factory/src/itest/java/com/unidev/polydata4/MongodbCacheIntegrationTest.java
@@ -0,0 +1,58 @@
+package com.unidev.polydata4;
+
+import com.unidev.polydata4.domain.BasicPoly;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.junit.jupiter.api.BeforeEach;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.Charset;
+import java.util.Map;
+
+/**
+ * Mongodb integration tests with ehcache.
+ */
+@Testcontainers
+public class MongodbCacheIntegrationTest extends IntegrationTest {
+
+    @Container
+    private final GenericContainer mongodb = new GenericContainer("mongo:6.0.2-focal")
+            .withExposedPorts(27017);
+
+    @Container
+    private final GenericContainer redis = new GenericContainer("redis:7.0.5")
+            .withExposedPorts(6379);
+
+
+    @BeforeEach
+    public void setup() throws IOException {
+
+        InputStream resourceAsStream = getClass().getResourceAsStream("/cache/ehcache.xml");
+        String resource = IOUtils.toString(resourceAsStream, Charset.defaultCharset());
+        int random = (int) (Math.random() * 10000);
+        resource = renderTemplate(resource, Map.of("random", random));
+        new File("/tmp/ehcache-" + random).mkdirs();
+        // write stream to temp file
+        File tempFile = File.createTempFile("ehcache", ".xml");
+        FileUtils.writeStringToFile(tempFile, resource, Charset.defaultCharset());
+        BasicPoly config = BasicPoly.newPoly()
+                .with("type", "mongodb")
+                .with("uri", "mongodb://localhost:" + mongodb.getMappedPort(27017) + "/polydata4")
+                .with("cache", BasicPoly.newPoly()
+                        .with("type", "jcache")
+                        .with("provider", "org.ehcache.jsr107.EhcacheCachingProvider")
+                        .with("name", "polydata")
+                        .with("implementationUri", "file://" + tempFile.getAbsolutePath())
+                )
+                ;
+
+        create(config);
+    }
+
+
+}
diff --git a/polydata-factory/src/itest/resources/cache/ehcache.xml b/polydata-factory/src/itest/resources/cache/ehcache.xml
@@ -0,0 +1,26 @@
+<#setting number_format="computer">
+<!-- https://github.com/anthonydahanne/ehcache3-disk-persistence-demo/blob/master/src/main/resources/ehcache.xml -->
+<!-- https://www.ehcache.org/documentation/3.6/xml.html -->
+<config
+        xmlns:jsr107='http://www.ehcache.org/v3/jsr107'
+        xmlns='http://www.ehcache.org/v3'>
+
+    <service>
+        <jsr107:defaults enable-management="true" enable-statistics="true"/>
+
+    </service>
+
+    <persistence directory="/tmp/ehcache-${random}"></persistence>
+
+    <cache alias="polydata" uses-template="heap-cache"/>
+
+    <cache-template name="heap-cache">
+
+        <resources>
+            <heap unit="entries">1024</heap>
+            <offheap unit="MB">1</offheap>
+            <disk persistent="true" unit="MB">100</disk>
+        </resources>
+    </cache-template>
+
+</config>
diff --git a/polydata-factory/src/test/java/com/unidev/polydata4/PolydataFactoryTest.java b/polydata-factory/src/test/java/com/unidev/polydata4/PolydataFactoryTest.java
@@ -6,6 +6,7 @@
 import com.unidev.polydata4.flatfiles.PolydataYaml;
 import org.junit.jupiter.api.Test;
 
+import java.io.File;
 import java.util.Optional;
 
 import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -33,7 +34,8 @@ void mongodbCache() {
 
         config.put("cache", BasicPoly.newPoly()
                 .with("type", "jcache")
-                .with("name", "org.ehcache.jsr107.EhcacheCachingProvider")
+                .with("provider", "org.ehcache.jsr107.EhcacheCachingProvider")
+                .with("name", "polydata")
         );
 
         Optional<Polydata> polydata = polydataFactory.create(config);
@@ -62,9 +64,11 @@ void flatFileFactory() {
 
     @Test
     void flatFileSingleJson() {
+        new File("/tmp/flat-file-json").mkdirs();
+
         BasicPoly config = new BasicPoly();
         config.put("type", "flat-file-json");
-        config.put("root", "/tmp");
+        config.put("root", "/tmp/flat-file-json");
 
         Optional<Polydata> polydata = polydataFactory.create(config);
         assertTrue(polydata.isPresent());

diff --git a/polydata-mongodb/src/main/java/com/unidev/polydata4/mongodb/PolydataMongodb.java b/polydata-mongodb/src/main/java/com/unidev/polydata4/mongodb/PolydataMongodb.java
@@ -190,11 +190,11 @@ public BasicPolyList update(String poly, Collection<InsertRequest> insertRequest
     @Override
     public BasicPolyList read(String poly, Set<String> ids) {
         Map<String, BasicPoly> cachedPolys = ifCache(cache -> {
-            List<String> cachedIds = new ArrayList<>();
+            Set<String> cachedIds = new HashSet<>();
             for (String id : ids) {
                 cachedIds.add(poly + "-read-" + id);
             }
-            return cache.getAll(ids);
+            return cache.getAll(cachedIds);
         });
 
         List<String> idsToQuery = new ArrayList<>(ids);
@@ -209,8 +209,12 @@ public BasicPolyList read(String poly, Set<String> ids) {
             }
         }
 
+        if (idsToQuery.isEmpty()) {
+            return list;
+        }
+
         final BasicPolyList dbPolys = new BasicPolyList();
-        Bson query = Filters.in(_ID, ids);
+        Bson query = Filters.in(_ID, idsToQuery);
         try (MongoCursor<Document> cursor = collection(poly).find(query).iterator()) {
             cursor.forEachRemaining(document -> {
                 BasicPoly polyData = toPoly(document);
@@ -219,9 +223,11 @@ public BasicPolyList read(String poly, Set<String> ids) {
         }
         if (cache.isPresent()) {
             Cache cacheInstance = cache.get();
-            for (BasicPoly polyData : dbPolys.list()) {
-                cacheInstance.put(poly + "-read-" + polyData._id(), polyData);
+            Map<String, BasicPoly> cacheMap = new HashMap<>();
+            for (BasicPoly data : dbPolys.list()) {
+                cacheMap.put(poly + "-read-" + data._id(), data);
             }
+            cacheInstance.putAll(cacheMap);
         }
         list.list().addAll(dbPolys.list());
 
@@ -230,12 +236,19 @@ public BasicPolyList read(String poly, Set<String> ids) {
 
     @Override
     public BasicPolyList remove(String poly, Set<String> ids) {
-        //TODO: update cache
-        BasicPolyList basicPolyList = read(poly, ids);
+        BasicPolyList list = read(poly, ids);
         // delete by id
         collection(poly).deleteMany(Filters.in(_ID, ids));
         recalculateIndex(poly);
-        return basicPolyList;
+        if (cache.isPresent()) {
+            Cache cacheInstance = cache.get();
+            Set<String> keysToRemove = new HashSet<>();
+            for (BasicPoly data : list.list()) {
+                keysToRemove.add(poly + "-read-" + data._id());
+            }
+            cacheInstance.removeAll(keysToRemove);
+        }
+        return list;
     }
 
     @Override