Mongo7 upgrade

.github/workflows/test.yaml

@@ -24,6 +24,9 @@ jobs:
           - python: '3.9.19'
             mongo: 'mongodb-linux-x86_64-3.6.23'
             wired_tiger: 'false'
+          - python: '3.9.19'
+            mongo: 'mongodb-linux-x86_64-ubuntu2204-7.0.4'
+            wired_tiger: 'true'
 
     steps:
     - name: Check out GitHub repo

Pipfile

@@ -6,7 +6,7 @@ name = "pypi"
 [packages]
 cachetools = "==4.2.2"
 jsonrpcbase = "==0.2.0"
-pymongo = "==3.8.0"
+pymongo = "==4.7.2"
 requests = "==2.31.0"
 uwsgi = "==2.0.22"
 

deployment/conf/.templates/deployment.cfg.templ

@@ -5,6 +5,7 @@ mongo-database = {{ default .Env.mongo_database "handle_db" }}
 mongo-authmechanism = {{ default .Env.mongo_authmechanism "DEFAULT" }}
 mongo-user = {{ default .Env.mongo_user "" }}
 mongo-password = {{ default .Env.mongo_password "" }}
+mongo-retrywrites={{ default .Env.mongo_retrywrites "false" }}
 
 start-local-mongo = {{ default .Env.start_local_mongo "0" }}
 

lib/AbstractHandle/Utils/MongoUtil.py

@@ -13,7 +13,8 @@ class MongoUtil:
     _HID_COUNTER_ID = 'hid_counter'
 
     def _get_collection(self, mongo_host, mongo_port, mongo_database, mongo_collection,
-                        mongo_user=None, mongo_password=None, mongo_authmechanism='DEFAULT'):
+                        mongo_user=None, mongo_password=None, mongo_authmechanism='DEFAULT',
+                        mongo_retrywrites=False):
         """
         connect Mongo server and return a collection
         """
@@ -23,10 +24,11 @@ def _get_collection(self, mongo_host, mongo_port, mongo_database, mongo_collecti
             my_client = MongoClient(mongo_host, mongo_port,
                                     username=mongo_user, password=mongo_password,
                                     authSource=mongo_database,
-                                    authMechanism=mongo_authmechanism)
+                                    authMechanism=mongo_authmechanism,
+                                    retryWrites=mongo_retrywrites)
         else:
             logging.info('no mongo-user found in config file, connecting without auth')
-            my_client = MongoClient(mongo_host, mongo_port)
+            my_client = MongoClient(mongo_host, mongo_port, retryWrites=mongo_retrywrites)
 
         try:
             my_client.server_info()  # force a call to server
@@ -60,34 +62,26 @@ def __init__(self, config):
         self.mongo_user = config['mongo-user']
         self.mongo_pass = config['mongo-password']
         self.mongo_authmechanism = config['mongo-authmechanism']
+        self.mongo_retrywrites = config.get('mongo-retrywrites') == "true"
 
         self.handle_collection = self._get_collection(self.mongo_host, self.mongo_port,
                                                       self.mongo_database, MONGO_COLLECTION,
                                                       self.mongo_user, self.mongo_pass,
-                                                      self.mongo_authmechanism)
+                                                      self.mongo_authmechanism,
+                                                      self.mongo_retrywrites)
         # create index on startup to speed up fetching
         self.handle_collection.create_index('hid', unique=True)
 
         self.hid_counter_collection = self._get_collection(self.mongo_host, self.mongo_port,
                                                            self.mongo_database,
                                                            MONGO_HID_COUNTER_COLLECTION,
                                                            self.mongo_user, self.mongo_pass,
-                                                           self.mongo_authmechanism)
+                                                           self.mongo_authmechanism,
+                                                           self.mongo_retrywrites)
 
         logging.basicConfig(format='%(created)s %(levelname)s: %(message)s',
                             level=logging.INFO)
 
-    def get_hid_counter(self):
-        """
-        get current handle id counter
-        """
-        counter = self.hid_counter_collection.find({'_id': {'$eq': self._HID_COUNTER_ID}})
-
-        if counter.count():
-            return counter.next().get('hid_counter')
-        else:
-            return 0
-
     def find_in(self, elements, field_name, projection={'_id': False}, batch_size=1000):
         """
         return cursor that contains docs which field column is in elements
@@ -104,8 +98,6 @@ def find_in(self, elements, field_name, projection={'_id': False}, batch_size=10
                             ''.join(traceback.format_exception(None, e, e.__traceback__)))
             raise ValueError(error_msg)
 
-        logging.info('returned {} results'.format(result.count()))
-
         return result
 
     def insert_one(self, doc):

test.cfg.example

@@ -42,4 +42,8 @@ mongo-database=handle_db
 mongo-user=
 
 # If the mongo database is authenticated, the password for the given username.
-mongo-password=
+mongo-password=
+
+# Whether to enable ('true') the MongoDB retryWrites parameter or not (anything other than 'true').
+# See https://www.mongodb.com/docs/manual/core/retryable-writes/
+mongo-retrywrites=false

test/AbstractHandle_server_test.py

@@ -17,6 +17,7 @@
 
 import mongo_util
 from mongo_controller import MongoController
+from test_helper import get_hid_counter
 
 
 class handle_serviceTest(unittest.TestCase):
@@ -59,6 +60,7 @@ def setUpClass(cls):
         mongo_util.create_test_db(cls.mongo_controller, db=cls.cfg['mongo-database'])
         cls.serviceImpl = AbstractHandle(cls.cfg)
         cls.mongo_util = MongoUtil.MongoUtil(cls.cfg)
+        cls.hid_counter_collection = cls.mongo_util.hid_counter_collection
         cls.shock_ids_to_delete = list()
 
     @classmethod
@@ -169,9 +171,9 @@ def test_persist_handle_ok(self):
                   'type': 'shock',
                   'url': 'http://ci.kbase.us:7044/'}
         # testing persist_handle with non-existing handle (inserting a handle)
-        counter = self.mongo_util.get_hid_counter()
+        counter = get_hid_counter(self.hid_counter_collection)
         hid = handler.persist_handle(self.ctx, handle)[0]
-        new_counter = self.mongo_util.get_hid_counter()
+        new_counter = get_hid_counter(self.hid_counter_collection)
         self.assertEqual(new_counter, counter + 1)
         handles = handler.fetch_handles_by(self.ctx, {'elements': [hid], 'field_name': 'hid'})[0]
         self.assertEqual(len(handles), 1)
@@ -184,7 +186,7 @@ def test_persist_handle_ok(self):
 
         # testing persist_handle a second handle
         hid = handler.persist_handle(self.ctx, handle)[0]
-        new_counter = self.mongo_util.get_hid_counter()
+        new_counter = get_hid_counter(self.hid_counter_collection)
         self.assertEqual(hid, 'KBH_' + str(new_counter))
         self.assertEqual(new_counter, counter + 2)
 
@@ -206,7 +208,7 @@ def test_persist_multiple_handles_ok(self):
                   'type': 'shock',
                   'url': 'http://ci.kbase.us:7044/'}
 
-        counter = self.mongo_util.get_hid_counter()
+        counter = get_hid_counter(self.hid_counter_collection)
 
         thread_count = 257
 
@@ -226,7 +228,7 @@ def test_persist_multiple_handles_ok(self):
             result = que.get()
             hids.append(int(result[0].split('KBH_')[-1]))
 
-        new_counter = self.mongo_util.get_hid_counter()
+        new_counter = get_hid_counter(self.hid_counter_collection)
         self.assertEqual(counter + thread_count, new_counter)
 
         self.assertEqual(len(set(hids)), thread_count)

test/Handler_test.py

@@ -13,6 +13,7 @@
 
 import mongo_util
 from mongo_controller import MongoController
+from test_helper import get_hid_counter
 
 class HandlerTest(unittest.TestCase):
 
@@ -174,9 +175,9 @@ def test_persist_handle_okay(self):
                   'type': 'shock',
                   'url': 'http://ci.kbase.us:7044/'}
         # testing persist_handle with non-existing handle (inserting a handle)
-        counter = handler.mongo_util.get_hid_counter()
+        counter = get_hid_counter(handler.mongo_util.hid_counter_collection)
         hid = handler.persist_handle(handle, self.user_id)
-        new_counter = handler.mongo_util.get_hid_counter()
+        new_counter = get_hid_counter(handler.mongo_util.hid_counter_collection)
         self.assertEqual(new_counter, counter + 1)  # counter should increment
         handles = handler.fetch_handles_by({'elements': [hid], 'field_name': 'hid'})
         self.assertEqual(len(handles), 1)

test/MongoUtil_test.py

@@ -1,5 +1,4 @@
 # -*- coding: utf-8 -*-
-import os
 import unittest
 import inspect
 import copy
@@ -11,6 +10,7 @@
 
 import mongo_util
 from mongo_controller import MongoController
+from test_helper import get_hid_counter
 
 
 # TODO switch all tests to pytest, see sample_service for an example
@@ -34,6 +34,7 @@ def setUpClass(cls):
         cls.cfg["mongo-port"] = cls.mongo_controller.port
         mongo_util.create_test_db(cls.mongo_controller, db=cls.cfg['mongo-database'])
         cls.mongo_util = MongoUtil(cls.cfg)
+        cls.hid_counter_collection = cls.mongo_util.hid_counter_collection
 
     @classmethod
     def tearDownClass(cls):
@@ -48,6 +49,12 @@ def start_test(self):
         testname = inspect.stack()[1][3]
         print('\n*** starting test: ' + testname + ' **')
 
+    def test_retryWrites(self):
+        new_cfg = dict(self.cfg)
+        new_cfg['mongo-retrywrites'] = "true"
+        mu = MongoUtil(new_cfg)
+        self.assertEqual(mu.mongo_retrywrites, True)
+
     def test_get_collection(self):
         self.start_test()
         mongo_util = self.getMongoUtil()
@@ -64,6 +71,7 @@ def test_init_ok(self):
         ]
         mongo_util = self.getMongoUtil()
         self.assertTrue(set(class_attri) <= set(mongo_util.__dict__.keys()))
+        self.assertEqual(mongo_util.mongo_retrywrites, False)
 
         handle_collection = mongo_util.handle_collection
         self.assertEqual(handle_collection.name, 'handle')
@@ -85,25 +93,25 @@ def test_find_in_ok(self):
         # test query 'hid' field
         elements = [68020, 68022, 0]
         docs = mongo_util.find_in(elements, 'hid')
-        self.assertEqual(docs.count(), 2)
+        self.assertEqual(len(list(docs)), 2)
 
         # test query 'hid' field with empty data
         elements = [0]
         docs = mongo_util.find_in(elements, 'hid')
-        self.assertEqual(docs.count(), 0)
+        self.assertEqual(len(list(docs)), 0)
 
         # test query 'id' field
         elements = ['b753774f-0bbd-4b96-9202-89b0c70bf31c']
         docs = mongo_util.find_in(elements, 'id')
-        self.assertEqual(docs.count(), 1)
+        self.assertEqual(len(list(docs.clone())), 1)
         doc = docs.next()
         self.assertFalse('_id' in doc.keys())
         self.assertEqual(doc.get('hid'), 68020)
 
         # test null projection
         elements = ['b753774f-0bbd-4b96-9202-89b0c70bf31c']
         docs = mongo_util.find_in(elements, 'id', projection=None)
-        self.assertEqual(docs.count(), 1)
+        self.assertEqual(len(list(docs.clone())), 1)
         doc = docs.next()
         self.assertEqual(doc.get('_id'), 68020)
         self.assertEqual(doc.get('hid'), 68020)
@@ -114,7 +122,7 @@ def test_update_one_ok(self):
 
         elements = ['b753774f-0bbd-4b96-9202-89b0c70bf31c']
         docs = mongo_util.find_in(elements, 'id', projection=None)
-        self.assertEqual(docs.count(), 1)
+        self.assertEqual(len(list(docs.clone())), 1)
         doc = docs.next()
         self.assertEqual(doc.get('created_by'), 'tgu2')
 
@@ -133,29 +141,29 @@ def test_update_one_ok(self):
     def test_insert_one_ok(self):
         self.start_test()
         mongo_util = self.getMongoUtil()
-        self.assertEqual(mongo_util.handle_collection.find().count(), 10)
+        self.assertEqual(mongo_util.handle_collection.count_documents({}), 10)
 
         doc = {'_id': 9999, 'hid': 9999, 'file_name': 'fake_file'}
-        counter = mongo_util.get_hid_counter()
+        counter = get_hid_counter(self.hid_counter_collection)
         mongo_util.insert_one(doc)
-        new_counter = mongo_util.get_hid_counter()
+        new_counter = get_hid_counter(self.hid_counter_collection)
         self.assertEqual(new_counter, counter)
 
-        self.assertEqual(mongo_util.handle_collection.find().count(), 11)
+        self.assertEqual(mongo_util.handle_collection.count_documents({}), 11)
         elements = [9999]
         docs = mongo_util.find_in(elements, 'hid', projection=None)
-        self.assertEqual(docs.count(), 1)
+        self.assertEqual(len(list(docs.clone())), 1)
         doc = docs.next()
         self.assertEqual(doc.get('hid'), 9999)
         self.assertEqual(doc.get('file_name'), 'fake_file')
 
         mongo_util.delete_one(doc)
-        self.assertEqual(mongo_util.handle_collection.find().count(), 10)
+        self.assertEqual(mongo_util.handle_collection.count_documents({}), 10)
 
     def test_increase_counter_with_multi_threads(self):
 
         mongo_util = self.getMongoUtil()
-        counter = mongo_util.get_hid_counter()
+        counter = get_hid_counter(self.hid_counter_collection)
 
         thread_count = 329
 
@@ -173,7 +181,7 @@ def test_increase_counter_with_multi_threads(self):
         while not que.empty():
             hids.append(que.get())
 
-        new_counter = mongo_util.get_hid_counter()
+        new_counter = get_hid_counter(self.hid_counter_collection)
         self.assertEqual(counter + thread_count, new_counter)
 
         self.assertEqual(len(set(hids)), thread_count)
@@ -193,42 +201,42 @@ def test_delete_one_ok(self):
         self.start_test()
         mongo_util = self.getMongoUtil()
         docs = mongo_util.handle_collection.find()
-        self.assertEqual(docs.count(), 10)
+        self.assertEqual(len(list(docs.clone())), 10)
 
         doc = docs.next()
         hid = doc.get('hid')
         mongo_util.delete_one(doc)
-        self.assertEqual(mongo_util.handle_collection.find().count(), 9)
+        self.assertEqual(mongo_util.handle_collection.count_documents({}), 9)
 
         docs = mongo_util.find_in([hid], 'hid', projection=None)
-        self.assertEqual(docs.count(), 0)
+        self.assertEqual(len(list(docs)), 0)
 
         mongo_util.insert_one(doc)
-        self.assertEqual(mongo_util.handle_collection.find().count(), 10)
+        self.assertEqual(mongo_util.handle_collection.count_documents({}), 10)
         docs = mongo_util.find_in([hid], 'hid', projection=None)
-        self.assertEqual(docs.count(), 1)
+        self.assertEqual(len(list(docs)), 1)
 
     def test_delete_many_ok(self):
         self.start_test()
         mongo_util = self.getMongoUtil()
         docs = mongo_util.handle_collection.find()
-        self.assertEqual(docs.count(), 10)
+        self.assertEqual(len(list(docs.clone())), 10)
 
         docs_to_delete = list()
         docs_to_delete.append(docs.next())
         docs_to_delete.append(docs.next())
         docs_to_delete = docs_to_delete * 2  # test delete duplicate items
         deleted_count = mongo_util.delete_many(docs_to_delete)
         self.assertEqual(deleted_count, 2)
-        self.assertEqual(mongo_util.handle_collection.find().count(), 8)
+        self.assertEqual(mongo_util.handle_collection.count_documents({}), 8)
         docs = mongo_util.find_in([doc.get('hid') for doc in docs_to_delete], 'hid')
-        self.assertEqual(docs.count(), 0)
+        self.assertEqual(len(list(docs)), 0)
 
         for doc in docs_to_delete:
             try:
                 mongo_util.insert_one(doc)
             except Exception:
                 pass
-        self.assertEqual(mongo_util.handle_collection.find().count(), 10)
+        self.assertEqual(mongo_util.handle_collection.count_documents({}), 10)
         docs = mongo_util.find_in([doc.get('hid') for doc in docs_to_delete], 'hid')
-        self.assertEqual(docs.count(), 2)
+        self.assertEqual(len(list(docs)), 2)