Merge pull request #99 from kbase/dev-mongo7_upgrade

mongo7 upgrade

.github/workflows/test.yml

@@ -25,8 +25,8 @@ jobs:
           # the current production setup
           - java: '8'
             mongo: 'mongodb-linux-x86_64-3.6.13'
-            wired_tiger: 'false'
-            gradle_test: 'test'
+          - java: '11'
+            mongo: 'mongodb-linux-x86_64-ubuntu2204-7.0.4'
     steps:
       - uses: actions/checkout@v3
 
@@ -57,8 +57,7 @@ jobs:
 
       - name: Run tests
         shell: bash
-        run: |
-          ./gradlew ${{matrix.gradle_test}}
+        run: ./gradlew test
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v3

build.gradle

@@ -180,7 +180,10 @@ dependencies {
 	implementation 'org.ini4j:ini4j:0.5.2'
 	implementation 'com.fasterxml.jackson.core:jackson-annotations:2.2.3'
 	implementation 'com.fasterxml.jackson.core:jackson-databind:2.2.3'
-	implementation 'org.mongodb:mongo-java-driver:3.10.1'
+	implementation 'org.mongodb:mongodb-driver-core:4.11.1'
+	implementation 'org.mongodb:mongodb-driver-sync:4.11.1'
+	implementation 'org.mongodb:bson-record-codec:4.11.1'
+	implementation 'org.mongodb:bson:4.11.1'
 	implementation 'org.yaml:snakeyaml:1.11'
 	implementation 'org.apache.velocity:velocity:1.7'
 	implementation 'commons-io:commons-io:2.4'
@@ -193,6 +196,9 @@ dependencies {
 	testImplementation 'junit:junit:4.12'
 	testImplementation 'org.mockito:mockito-core:3.0.0'
 	testImplementation 'org.apache.commons:commons-lang3:3.5'
+	testImplementation('com.github.kbase:java_test_utilities:0.1.0') {
+		exclude group: 'com.fasterxml.jackson.core'  // upgrading breaks stuff
+	}
 }
 
 task showTestClassPath {

src/main/java/us/kbase/narrativemethodstore/db/mongo/MongoUtils.java

@@ -5,14 +5,12 @@
 import java.nio.charset.Charset;
 import java.security.MessageDigest;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 
-import org.bson.BSONObject;
-import org.bson.LazyBSONList;
-import org.bson.types.BasicBSONList;
+import com.mongodb.client.MongoCursor;
+import com.mongodb.client.model.Projections;
 
 import us.kbase.common.service.UObject;
 import us.kbase.narrativemethodstore.exceptions.NarrativeMethodStoreException;
@@ -21,10 +19,8 @@
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.introspect.VisibilityChecker;
-import com.mongodb.BasicDBObject;
-import com.mongodb.DBCollection;
-import com.mongodb.DBCursor;
-import com.mongodb.DBObject;
+import com.mongodb.client.MongoCollection;
+import org.bson.Document;
 
 public class MongoUtils {
     private static final String HEXES = "0123456789abcdef";
@@ -37,17 +33,21 @@ public class MongoUtils {
 
     @SuppressWarnings({ "unchecked" })
     public static <T> List<T> getProjection(
-            final DBCollection infos,
-            final DBObject whereCondition,
+            final MongoCollection<Document> infos,
+            final Document whereCondition,
             final String selectField,
             final Class<T> type)
             throws NarrativeMethodStoreException {
-        final DBCursor cur = infos.find(whereCondition, new BasicDBObject(selectField, 1));
+        MongoCursor<Document> cursor = infos.find(whereCondition)
+                .projection(Projections.include(selectField))
+                .iterator();
+
         final List<Map<String, Object>> data = new LinkedList<>();
-        for (final DBObject dbo: cur) {
-            data.add(toMap(dbo));
+        while (cursor.hasNext()) {
+            final Map<String, Object> map = cursor.next();
+            data.add(map);
         }
-        
+
         List<T> ret = new ArrayList<T>();
         for (Map<?,?> item : data) {
             Object value = item.get(selectField);
@@ -58,81 +58,30 @@ public static <T> List<T> getProjection(
         return ret;
     }
 
-    /** Map an object to a MongoDB {@link DBObject}. The object must be serializable by
+    /**
+     * Map an object to a MongoDB {@link Document}. The object must be serializable by
      * an {@link ObjectMapper} configured so private fields are visible.
+     *
      * @param obj the object to map.
-     * @return the new mongo compatible object.
+     * @return the new mongo document.
      */
-    public static DBObject toDBObject(final Object obj) {
-        return new BasicDBObject(objToMap(obj));
+    public static Document toDocument(final Object obj) {
+        return new Document(objToMap(obj));
     }
 
     private static Map<String, Object> objToMap(final Object obj) {
         return MAPPER.convertValue(obj, new TypeReference<Map<String, Object>>() {});
     }
-
-
-    /** Map a MongoDB {@link DBObject} to a class.
-     * This method expects that all maps and lists in the objects are implemented as
-     * {@link BSONObject}s or derived classes, not standard maps, lists, or other classes.
+
+    /** Map a MongoDB {@link Document} to a class.
      * The object must be deserializable by an {@link ObjectMapper} configured so private
      * fields are visible.
-     * @param dbo the MongoDB object to transform.
+     * @param doc the MongoDB document to transform.
      * @param clazz the class to which the object will be transformed.
      * @return the transformed object.
      */
-    public static <T> T toObject(final DBObject dbo, final Class<T> clazz) {
-        return dbo == null ? null : MAPPER.convertValue(toMap(dbo), clazz);
-    }
-
-    /** Map a MongoDB {@link BSONObject} to a standard map.
-     * This method expects that all maps and lists in the objects are implemented as
-     * {@link BSONObject}s or derived classes, not standard maps, lists, or other classes.
-     * @param dbo the MongoDB object to transform to a standard map.
-     * @return the transformed object, or null if the argument was null.
-     */
-    public static Map<String, Object> toMap(final BSONObject dbo) {
-        @SuppressWarnings("unchecked")
-        final Map<String, Object> ret = (Map<String, Object>) cleanObject(dbo);
-        return ret;
-    }
-
-    // this assumes there aren't BSONObjects embedded in standard objects, which should
-    // be the case for stuff returned from mongo
-
-    // Unimplemented error for dbo.toMap()
-    // dbo is read only
-    // can't call ObjectMapper.convertValue() on dbo since it has a 'size' field outside of
-    // the internal map
-    // and just weird shit happens when you do anyway
-    private static Object cleanObject(final Object dbo) {
-        // sometimes it's lazy, sometimes it's basic. Not sure when or why.
-        if (dbo instanceof LazyBSONList) {
-            final List<Object> ret = new LinkedList<>();
-            // don't stream, sometimes has issues with nulls
-            for (final Object obj: (LazyBSONList) dbo) {
-                ret.add(cleanObject(obj));
-            }
-            return ret;
-        } else if (dbo instanceof BasicBSONList) {
-            final List<Object> ret = new LinkedList<>();
-            // don't stream, sometimes has issues with nulls
-            for (final Object obj: (BasicBSONList) dbo) {
-                ret.add(cleanObject(obj));
-            }
-        } else if (dbo instanceof BSONObject) {
-            // can't stream because streams don't like null values at HashMap.merge()
-            final BSONObject m = (BSONObject) dbo;
-            final Map<String, Object> ret = new HashMap<>();
-            for (final String k: m.keySet()) {
-                if (!k.equals("_id")) {
-                    final Object v = m.get(k);
-                    ret.put(k, cleanObject(v));
-                }
-            }
-            return ret;
-        }
-        return dbo;
+    public static <T> T toObject(final Document doc, final Class<T> clazz) {
+        return doc == null ? null : MAPPER.convertValue(doc, clazz);
     }
 
     public static String stringToHex(String text) {

src/test/java/us/kbase/test/narrativemethodstore/db/mongo/MongoDBHelper.java

@@ -4,10 +4,8 @@
 import java.io.IOException;
 import java.net.ServerSocket;
 import java.nio.file.Files;
-import java.util.Arrays;
 
-import us.kbase.common.utils.ProcessHelper;
-import us.kbase.narrativemethodstore.util.TextUtils;
+import us.kbase.testutils.controllers.mongo.MongoController;
 
 public class MongoDBHelper {
     private String tempDirName = "test/temp";
@@ -17,6 +15,8 @@ public class MongoDBHelper {
     private File mongoDir = null;
     private int mongoPort = -1;
 
+    private static MongoController mongo;
+
     public MongoDBHelper(String testName) {
         this.testName = testName;
     }
@@ -41,7 +41,7 @@ public void startup(String mongoExePath) throws Exception {
     }
 
     public void shutdown(boolean deleteTempDir) throws Exception {
-        killPid(mongoDir);
+        destroy(mongoDir);
         if (deleteTempDir && mongoDir.exists())
             deleteRecursively(mongoDir);
     }
@@ -51,58 +51,20 @@ private static int startupMongo(String mongodExePath, File dir) throws Exception
             mongodExePath = "mongod";
         if (!dir.exists())
             dir.mkdirs();
-        File dataDir = new File(dir, "data");
-        dataDir.mkdir();
-        File logFile = new File(dir, "mongodb.log");
-        int port = findFreePort();
-        File configFile = new File(dir, "mongod.conf");
-        TextUtils.writeLines(Arrays.asList(
-                "dbpath=" + dataDir.getAbsolutePath(),
-                "logpath=" + logFile.getAbsolutePath(),
-                "logappend=true",
-                "port=" + port,
-                "bind_ip=127.0.0.1"
-                ), configFile);
-        File scriptFile = new File(dir, "start_mongo.sh");
-        TextUtils.writeLines(Arrays.asList(
-                "#!/bin/bash",
-                "cd " + dir.getAbsolutePath(),
-                mongodExePath + " --nojournal --config " + configFile.getAbsolutePath() + " >out.txt 2>err.txt & pid=$!",
-                "echo $pid > pid.txt"
-                ), scriptFile);
-        ProcessHelper.cmd("bash", scriptFile.getCanonicalPath()).exec(dir);
-        boolean ready = false;
-        int waitSec = 120;
-        for (int n = 0; n < waitSec; n++) {
-            Thread.sleep(1000);
-            if (logFile.exists()) {
-                if (TextUtils.grep(TextUtils.lines(logFile), "waiting for connections on port " + port).size() > 0) {
-                    ready = true;
-                    break;
-                }
-            }
-        }
-        if (!ready) {
-            if (logFile.exists())
-                for (String l : TextUtils.lines(logFile))
-                    System.err.println("MongoDB log: " + l);
-            throw new IllegalStateException("MongoDB couldn't startup in " + waitSec + " seconds");
-        }
-        System.out.println(dir.getName() + " was started up");
-        return port;
+        mongo = new MongoController(mongodExePath, dir.toPath(), true);
+        System.out.println(String.format("Testing against mongo executable %s on port %s",
+                mongodExePath, mongo.getServerPort()));
+        return mongo.getServerPort();
     }
 
-    private static void killPid(File dir) {
+    private static void destroy(File dir) {
         if (dir == null)
             return;
-        try {
-            File pidFile = new File(dir, "pid.txt");
-            if (pidFile.exists()) {
-                String pid = TextUtils.lines(pidFile).get(0).trim();
-                ProcessHelper.cmd("kill", pid).exec(dir);
-                System.out.println(dir.getName() + " was stopped");
-            }
-        } catch (Exception ignore) {}
+        if (mongo != null) {
+            try {
+                mongo.destroy(true);
+            } catch (Exception ignore) {}
+        }
     }
 
     private static int findFreePort() {

src/test/java/us/kbase/test/narrativemethodstore/db/mongo/MongoDynamicRepoDB2Test.java

@@ -25,9 +25,10 @@
 import org.junit.BeforeClass;
 import org.junit.Test;
 
-import com.mongodb.BasicDBObject;
-import com.mongodb.DB;
-import com.mongodb.MongoClient;
+import com.mongodb.client.MongoDatabase;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
+import org.bson.Document;
 
 import us.kbase.narrativemethodstore.db.DynamicRepoDB.FileProvider;
 import us.kbase.narrativemethodstore.db.DynamicRepoDB.RepoState;
@@ -76,12 +77,12 @@ public static void afterClass() throws Exception {
 
     @Before
     public void before() throws Exception {
-        final MongoClient mc = new MongoClient("localhost:" + MONGO.getMongoPort());
-        final DB db = mc.getDB(DB_NAME);
-        for (final String name: db.getCollectionNames()) {
-            if (!name.startsWith("system.")) {
-                // dropping collection also drops indexes
-                db.getCollection(name).remove(new BasicDBObject());
+        try (final MongoClient mc = MongoClients.create("mongodb://localhost:" + MONGO.getMongoPort())) {
+            final MongoDatabase db = mc.getDatabase(DB_NAME);
+            for (final String collectionName: db.listCollectionNames()) {
+                if (!collectionName.startsWith("system.")) {
+                    db.getCollection(collectionName).deleteMany(new Document()); // Delete all documents without dropping the indexes
+                }
             }
         }
     }

src/test/java/us/kbase/test/narrativemethodstore/db/mongo/MongoDynamicRepoDBTest.java

@@ -22,8 +22,8 @@
 import org.junit.Test;
 
 import com.fasterxml.jackson.databind.JsonNode;
-import com.mongodb.DB;
-import com.mongodb.MongoClient;
+import com.mongodb.client.MongoClient;
+import com.mongodb.client.MongoClients;
 
 import us.kbase.common.service.UObject;
 import us.kbase.narrativemethodstore.MethodBriefInfo;
@@ -272,10 +272,9 @@ private static boolean diffFiles(File f1, File f2, int bufferSize) throws Except
     @Before 
     @After
     public void cleanup() throws Exception {
-        String host = "localhost:" + dbHelper.getMongoPort();
-        final MongoClient mc = new MongoClient(host);
-        final DB db = mc.getDB(dbName);
-        db.dropDatabase();
-        mc.close();
+        final String host = "mongodb://localhost:" + dbHelper.getMongoPort();
+        try (final MongoClient mc = MongoClients.create(host)) {
+            mc.getDatabase(dbName).drop();
+        }
     }
 }