From 2d9c4994c3adc4f5576a0d35014bf8a6818099f6 Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Fri, 18 Nov 2022 19:50:41 -0800 Subject: [PATCH] add note how to avoid OOM errors to BaseFeaturizer.set_n_jobs() doc str --- matminer/featurizers/base.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/matminer/featurizers/base.py b/matminer/featurizers/base.py index 182178763..2bccda91b 100644 --- a/matminer/featurizers/base.py +++ b/matminer/featurizers/base.py @@ -138,8 +138,14 @@ class BaseFeaturizer(BaseEstimator, TransformerMixin, ABC): to reproduce the feature, just to get an idea what it is. """ - def set_n_jobs(self, n_jobs): - """Set the number of threads for this.""" + def set_n_jobs(self, n_jobs: int) -> None: + """Set the number of concurrent jobs to spawn during featurization. + + Args: + n_jobs (int): Number of threads in multiprocessing pool. + + Note: It seems multiprocessing can be the cause of out-of-memory (OOM) errors, especially when trying to featurize large structures on HPC nodes with strict memory limits. Using featurizer.set_n_jobs(1) has been known to help as a workaround. + """ self._n_jobs = n_jobs @property