speed up parallel

pyxtal/db.py

@@ -630,9 +630,11 @@ class database_topology:
         ltol (float): lattice tolerance
         stol (float): site tolerance
         atol (float): angle tolerance
+        log_file (str): log_file
     """
 
-    def __init__(self, db_name, rank=0, ltol=0.05, stol=0.05, atol=3):
+    def __init__(self, db_name, rank=0, ltol=0.05, stol=0.05, atol=3,
+                 log_file='db.log'):
         self.rank = rank
         self.db_name = db_name
         self.db = connect(db_name, serial=True)
@@ -661,7 +663,7 @@ def __init__(self, db_name, rank=0, ltol=0.05, stol=0.05, atol=3):
             ltol=ltol, stol=stol, angle_tol=atol)
 
         # Define logfile
-        self.log_file = 'db.log'
+        self.log_file = log_file
         logging.getLogger().handlers.clear()
         logging.basicConfig(format="%(asctime)s| %(message)s",
                             filename=self.log_file,
@@ -1164,6 +1166,7 @@ def update_row_energy(
         steps=250,
         use_relaxed=None,
         cmd=None,
+        calc_folder=None,
     ):
         """
         Update the row energy in the database for a given calculator.
@@ -1180,6 +1183,7 @@ def update_row_energy(
             steps (int): Number of optimization steps for DFTB (default is 250).
             use_relaxed (str, optional): Use relaxed structures (e.g. 'ff_relaxed')
             cmd (str, optional): Command for VASP calculations.
+            calc_folder (str, optional): calc_folder for GULP/VASP calculations
 
         Functionality:
             Based on the selected calculator, it updates the energy rows of the
@@ -1195,7 +1199,8 @@ def update_row_energy(
         """
 
         label = calculator.lower() + "_energy"
-        calc_folder = calculator.lower() + "_calc"
+        if calc_folder is None:
+            calc_folder = calculator.lower() + "_calc"
         os.makedirs(calc_folder, exist_ok=True)
 
         # Generate structures for calculation

pyxtal/lego/builder.py

@@ -572,7 +572,7 @@ def __init__(self, elements, composition, dim=3, prefix='mof',
             self.db_file = db_file
         else:
             self.db_file = self.prefix + '.db'
-        self.db = database_topology(self.db_file)
+        self.db = database_topology(self.db_file, log_file=self.log_file)
 
     def __str__(self):
 
@@ -831,8 +831,7 @@ def optimize_reps_mproc(self, reps, ncpu, args):
         xtals_opt = deque()
 
         # Split the input structures to minibatches
-        N_rep = 4
-        N_batches = N_rep * ncpu
+        N_batches = 8 * ncpu
         for _i, i in enumerate(range(0, len(reps), N_batches)):
             start, end = i, min([i+N_batches, len(reps)])
             ids = list(range(start, end))
@@ -859,7 +858,9 @@ def generate_args():
                            early_quit, minimizers)
             # Use the generator to pass args to reduce memory usage
             _xtal, _xs = None, None
-            for result in pool.imap_unordered(minimize_from_x_par, generate_args()):
+            for result in pool.imap_unordered(minimize_from_x_par,
+                                              generate_args(),
+                                              chunksize=1):
                 if result is not None:
                     (_xtals, _xs) = result
                     valid_xtals = self.process_xtals(
@@ -896,8 +897,7 @@ def optimize_xtals_mproc(self, xtals, ncpu, args):
         xtals_opt = deque()
 
         # Split the input structures to minibatches
-        N_rep = 4
-        N_batches = N_rep * ncpu
+        N_batches = 8 * ncpu
         for _i, i in enumerate(range(0, len(xtals), N_batches)):
             start, end = i, min([i+N_batches, len(xtals)])
             ids = list(range(start, end))
@@ -921,7 +921,7 @@ def generate_args():
                            early_quit, minimizers)
             # Use the generator to pass args to reduce memory usage
             _xtal, _xs = None, None
-            for result in pool.imap_unordered(minimize_from_x_par, generate_args()):
+            for result in pool.imap_unordered(minimize_from_x_par, generate_args(), chunksize=1):
                 if result is not None:
                     (_xtals, _xs) = result
                     valid_xtals = self.process_xtals(