From d7fa2b14e3ca6d3f36b323173b4c3bfe5331c47c Mon Sep 17 00:00:00 2001 From: Kojobu Date: Tue, 25 Jun 2024 20:20:35 +0200 Subject: [PATCH 1/7] 1) fix colormaps.register for new mpl version 2) fix mulitprocessing, Pool by adding mp_context --- src/py21cmmc/mcmc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/py21cmmc/mcmc.py b/src/py21cmmc/mcmc.py index f6c43a0f..528abca4 100644 --- a/src/py21cmmc/mcmc.py +++ b/src/py21cmmc/mcmc.py @@ -579,7 +579,7 @@ def prior(p): else: pool = mcmc_options.pop( "pool", - ProcessPoolExecutor(max_workers=mcmc_options.get("threadCount", 1)), + ProcessPoolExecutor(max_workers=mcmc_options.get("threadCount", 1), mp_context='spawn'), ) sampler = sampler_cls( continue_sampling=continue_sampling, @@ -594,4 +594,4 @@ def prior(p): # The sampler writes to file, so no need to save anything ourselves. sampler.startSampling() - return sampler + return sampler \ No newline at end of file From 458cc4cc7333fd2089bd11d8c0915ce0265d3222 Mon Sep 17 00:00:00 2001 From: Kojobu Date: Tue, 25 Jun 2024 20:22:16 +0200 Subject: [PATCH 2/7] 1) fix colormaps.register for new mpl version 2) fix mulitprocessing, Pool by adding mp_context --- src/py21cmmc/mcmc.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/py21cmmc/mcmc.py b/src/py21cmmc/mcmc.py index 528abca4..29e34ad8 100644 --- a/src/py21cmmc/mcmc.py +++ b/src/py21cmmc/mcmc.py @@ -577,6 +577,7 @@ def prior(p): return sampler, result else: + # didn't spawn Pools in my case, setting mp_context (either spawn or fork) fixed it pool = mcmc_options.pop( "pool", ProcessPoolExecutor(max_workers=mcmc_options.get("threadCount", 1), mp_context='spawn'), From 42bf1d7a89e85a9c1fb5f6eca3b4a62669bfb738 Mon Sep 17 00:00:00 2001 From: Kojobu Date: Wed, 26 Jun 2024 10:11:19 +0200 Subject: [PATCH 3/7] change to schwimmbad for better escape behavoir --- src/py21cmmc/mcmc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/py21cmmc/mcmc.py b/src/py21cmmc/mcmc.py index 29e34ad8..55b8a54e 100644 --- a/src/py21cmmc/mcmc.py +++ b/src/py21cmmc/mcmc.py @@ -3,7 +3,8 @@ import numpy as np import scipy.stats as stats from cmath import log -from concurrent.futures import ProcessPoolExecutor +#from concurrent.futures import ProcessPoolExecutor +from schwimmbad import MultiPool from os import mkdir, path from py21cmfast import yaml from py21cmfast._utils import ParameterError @@ -580,7 +581,7 @@ def prior(p): # didn't spawn Pools in my case, setting mp_context (either spawn or fork) fixed it pool = mcmc_options.pop( "pool", - ProcessPoolExecutor(max_workers=mcmc_options.get("threadCount", 1), mp_context='spawn'), + MultiPool(processes=mcmc_options.get("threadCount", 1), mp_context='spawn'), ) sampler = sampler_cls( continue_sampling=continue_sampling, From febc4264baef8332f6f419a9e7135efbb70f61a5 Mon Sep 17 00:00:00 2001 From: Kojobu Date: Wed, 26 Jun 2024 10:23:12 +0200 Subject: [PATCH 4/7] fix mp --- src/py21cmmc/mcmc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py21cmmc/mcmc.py b/src/py21cmmc/mcmc.py index 55b8a54e..f36d7f2c 100644 --- a/src/py21cmmc/mcmc.py +++ b/src/py21cmmc/mcmc.py @@ -581,7 +581,7 @@ def prior(p): # didn't spawn Pools in my case, setting mp_context (either spawn or fork) fixed it pool = mcmc_options.pop( "pool", - MultiPool(processes=mcmc_options.get("threadCount", 1), mp_context='spawn'), + MultiPool(processes=mcmc_options.get("threadCount", 1)), ) sampler = sampler_cls( continue_sampling=continue_sampling, From 2814ca0d6d8b903bf98440bbded3088a335f51c5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 26 Jun 2024 08:24:16 +0000 Subject: [PATCH 5/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/py21cmmc/mcmc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/py21cmmc/mcmc.py b/src/py21cmmc/mcmc.py index f36d7f2c..831713e2 100644 --- a/src/py21cmmc/mcmc.py +++ b/src/py21cmmc/mcmc.py @@ -3,12 +3,13 @@ import numpy as np import scipy.stats as stats from cmath import log -#from concurrent.futures import ProcessPoolExecutor -from schwimmbad import MultiPool from os import mkdir, path from py21cmfast import yaml from py21cmfast._utils import ParameterError +# from concurrent.futures import ProcessPoolExecutor +from schwimmbad import MultiPool + from .cosmoHammer import ( CosmoHammerSampler, HDFStorageUtil, @@ -596,4 +597,4 @@ def prior(p): # The sampler writes to file, so no need to save anything ourselves. sampler.startSampling() - return sampler \ No newline at end of file + return sampler From 668049886cbe2fefabfd69e5b79ec83936133e5b Mon Sep 17 00:00:00 2001 From: Kojobu Date: Wed, 26 Jun 2024 10:45:01 +0200 Subject: [PATCH 6/7] rechange --- src/py21cmmc/mcmc.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/py21cmmc/mcmc.py b/src/py21cmmc/mcmc.py index f36d7f2c..01d9501d 100644 --- a/src/py21cmmc/mcmc.py +++ b/src/py21cmmc/mcmc.py @@ -3,8 +3,7 @@ import numpy as np import scipy.stats as stats from cmath import log -#from concurrent.futures import ProcessPoolExecutor -from schwimmbad import MultiPool +from concurrent.futures import ProcessPoolExecutor from os import mkdir, path from py21cmfast import yaml from py21cmfast._utils import ParameterError @@ -581,7 +580,7 @@ def prior(p): # didn't spawn Pools in my case, setting mp_context (either spawn or fork) fixed it pool = mcmc_options.pop( "pool", - MultiPool(processes=mcmc_options.get("threadCount", 1)), + ProcessPoolExecutor(max_workers=mcmc_options.get("threadCount", 1), max_tasks_per_child=1, mp_context='spawn'), ) sampler = sampler_cls( continue_sampling=continue_sampling, From dfb3d9e57eadd2a4ee6dc7aa8bd8f14ce10a9465 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 26 Jun 2024 08:46:50 +0000 Subject: [PATCH 7/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/py21cmmc/mcmc.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/py21cmmc/mcmc.py b/src/py21cmmc/mcmc.py index b034591b..beddbdec 100644 --- a/src/py21cmmc/mcmc.py +++ b/src/py21cmmc/mcmc.py @@ -583,7 +583,11 @@ def prior(p): # didn't spawn Pools in my case, setting mp_context (either spawn or fork) fixed it pool = mcmc_options.pop( "pool", - ProcessPoolExecutor(max_workers=mcmc_options.get("threadCount", 1), max_tasks_per_child=1, mp_context='spawn'), + ProcessPoolExecutor( + max_workers=mcmc_options.get("threadCount", 1), + max_tasks_per_child=1, + mp_context="spawn", + ), ) sampler = sampler_cls( continue_sampling=continue_sampling,