diff --git a/examples/2D_isentropicvortex/case.py b/examples/2D_isentropicvortex/case.py index 76606f4d9..4e96520ce 100644 --- a/examples/2D_isentropicvortex/case.py +++ b/examples/2D_isentropicvortex/case.py @@ -49,8 +49,8 @@ 'x_domain%end' : 3, 'y_domain%beg' : -3, 'y_domain%end' : 3, - 'stretch_x' : True, - 'stretch_y' : True, + 'stretch_x' : 'T', + 'stretch_y' : 'T', 'loops_x' : 2, 'loops_y' : 2, 'a_x' : 1.03, diff --git a/toolchain/mfc/build.py b/toolchain/mfc/build.py index a960be5ec..d24b7a1ae 100644 --- a/toolchain/mfc/build.py +++ b/toolchain/mfc/build.py @@ -1,5 +1,6 @@ import os, typing, hashlib, dataclasses, shutil +from .case import Case from .printer import cons from .common import MFC_ROOTDIR, MFCException, system, delete_directory, create_directory, \ format_list_to_string @@ -31,7 +32,7 @@ def compute(self) -> typing.Set: def __hash__(self) -> int: return hash(self.name) - def get_slug(self, case: input.MFCInputFile) -> str: + def get_slug(self, case: Case) -> str: if self.isDependency: return self.name @@ -43,7 +44,7 @@ def get_slug(self, case: input.MFCInputFile) -> str: return m.hexdigest()[:10] # Get path to directory that will store the build files - def get_staging_dirpath(self, case: input.MFCInputFile) -> str: + def get_staging_dirpath(self, case: Case) -> str: return os.sep.join([os.getcwd(), "build", "staging", self.get_slug(case) ]) # Get the directory that contains the target's CMakeLists.txt @@ -56,7 +57,7 @@ def get_cmake_dirpath(self) -> str: os.sep.join(["toolchain", "dependencies"]) if self.isDependency else "", ]) - def get_install_dirpath(self, case: input.MFCInputFile) -> str: + def get_install_dirpath(self, case: Case) -> str: # The install directory is located: # Regular: /build/install/ # Dependency: /build/install/dependencies (shared) @@ -67,18 +68,18 @@ def get_install_dirpath(self, case: input.MFCInputFile) -> str: 'dependencies' if self.isDependency else self.get_slug(case), ]) - def get_install_binpath(self, case: input.MFCInputFile) -> str: + def get_install_binpath(self, case: Case) -> str: # /install//bin/ return os.sep.join([self.get_install_dirpath(case), "bin", self.name]) - def is_configured(self, case: input.MFCInputFile) -> bool: + def is_configured(self, case: Case) -> bool: # We assume that if the CMakeCache.txt file exists, then the target is # configured. (this isn't perfect, but it's good enough for now) return os.path.isfile( os.sep.join([self.get_staging_dirpath(case), "CMakeCache.txt"]) ) - def get_configuration_txt(self, case: input.MFCInputFile) -> typing.Optional[dict]: + def get_configuration_txt(self, case: Case) -> typing.Optional[dict]: if not self.is_configured(case): return None @@ -272,6 +273,8 @@ def build(targets = None, case: input.MFCInputFile = None, history: typing.Set[s history = set() if targets is None: targets = ARG("targets") + elif isinstance(targets, (MFCTarget, str)): + targets = [targets] targets = get_targets(list(REQUIRED_TARGETS) + targets) case = case or input.load(ARG("input"), ARG("--"), {}) diff --git a/toolchain/mfc/common.py b/toolchain/mfc/common.py index d91b68cc0..895c44c9e 100644 --- a/toolchain/mfc/common.py +++ b/toolchain/mfc/common.py @@ -5,12 +5,13 @@ from .printer import cons -MFC_ROOTDIR = abspath(normpath(f"{dirname(realpath(__file__))}/../..")) -MFC_TESTDIR = abspath(f"{MFC_ROOTDIR}/tests") -MFC_SUBDIR = abspath(f"{MFC_ROOTDIR}/build") -MFC_TEMPLATEDIR = abspath(f"{MFC_ROOTDIR}/toolchain/templates") -MFC_LOCK_FILEPATH = abspath(f"{MFC_SUBDIR}/lock.yaml") -MFC_BENCH_FILEPATH = abspath(f"{MFC_ROOTDIR}/toolchain/bench.yaml") +MFC_ROOTDIR = abspath(normpath(f"{dirname(realpath(__file__))}/../..")) +MFC_TESTDIR = abspath(f"{MFC_ROOTDIR}/tests") +MFC_SUBDIR = abspath(f"{MFC_ROOTDIR}/build") +MFC_TEMPLATEDIR = abspath(f"{MFC_ROOTDIR}/toolchain/templates") +MFC_LOCK_FILEPATH = abspath(f"{MFC_SUBDIR}/lock.yaml") +MFC_BENCH_FILEPATH = abspath(f"{MFC_ROOTDIR}/toolchain/bench.yaml") +MFC_EXAMPLE_DIRPATH = abspath(f"{MFC_ROOTDIR}/examples") MFC_LOGO = """\ .=++*: -+*+=. diff --git a/toolchain/mfc/run/input.py b/toolchain/mfc/run/input.py index 27110e1a2..bc791c9bf 100644 --- a/toolchain/mfc/run/input.py +++ b/toolchain/mfc/run/input.py @@ -48,7 +48,6 @@ def generate(self, target) -> None: cons.print() self.generate_fpp(target) - def clean(self, _targets) -> None: targets = [build.get_target(target) for target in _targets] diff --git a/toolchain/mfc/test/case.py b/toolchain/mfc/test/case.py index e772cf13b..6621a882f 100644 --- a/toolchain/mfc/test/case.py +++ b/toolchain/mfc/test/case.py @@ -1,4 +1,8 @@ -import os, glob, typing, hashlib, binascii, subprocess, itertools, dataclasses +import os, glob, hashlib, binascii, subprocess, itertools, dataclasses + +from typing import List, Set, Union, Callable + +from ..run.input import MFCInputFile from .. import case, common from ..state import ARG @@ -98,17 +102,15 @@ def trace_to_uuid(trace: str) -> str: @dataclasses.dataclass(init=False) class TestCase(case.Case): - ppn: int - trace: str - rebuild: bool + ppn: int + trace: str - def __init__(self, trace: str, mods: dict, ppn: int = None, rebuild: bool = None) -> None: - self.trace = trace - self.ppn = ppn or 1 - self.rebuild = rebuild or False + def __init__(self, trace: str, mods: dict, ppn: int = None) -> None: + self.trace = trace + self.ppn = ppn or 1 super().__init__({**BASE_CFG.copy(), **mods}) - def run(self, targets: typing.List[typing.Union[str, MFCTarget]], gpus: typing.Set[int]) -> subprocess.CompletedProcess: + def run(self, targets: List[Union[str, MFCTarget]], gpus: Set[int]) -> subprocess.CompletedProcess: if gpus is not None and len(gpus) != 0: gpus_select = ["--gpus"] + [str(_) for _ in gpus] else: @@ -118,14 +120,13 @@ def run(self, targets: typing.List[typing.Union[str, MFCTarget]], gpus: typing.S tasks = ["-n", str(self.ppn)] jobs = ["-j", str(ARG("jobs"))] if ARG("case_optimization") else [] case_optimization = ["--case-optimization"] if ARG("case_optimization") else [] - rebuild = [] if self.rebuild or ARG("case_optimization") else ["--no-build"] mfc_script = ".\\mfc.bat" if os.name == 'nt' else "./mfc.sh" target_names = [ get_target(t).name for t in targets ] command = [ - mfc_script, "run", filepath, *rebuild, *tasks, *case_optimization, + mfc_script, "run", filepath, "--no-build", *tasks, *case_optimization, *jobs, "-t", *target_names, *gpus_select, *ARG("--") ] @@ -207,6 +208,9 @@ def create_directory(self): print(json.dumps({{**case, **mods}})) """) + def to_MFCInputFile(self) -> MFCInputFile: + return MFCInputFile(self.get_filepath(), self.get_dirpath(), self.params) + def __str__(self) -> str: return f"tests/[bold magenta]{self.get_uuid()}[/bold magenta]: {self.trace}" @@ -232,16 +236,16 @@ def compute_tolerance(self) -> float: class TestCaseBuilder: trace: str mods: dict - path: str - args: typing.List[str] + path: str | None + args: List[str] | None ppn: int - rebuild: bool + functor: Callable | None def get_uuid(self) -> str: return trace_to_uuid(self.trace) def to_case(self) -> TestCase: - dictionary = self.mods.copy() + dictionary = {} if self.path: dictionary.update(input.load(self.path, self.args).params) @@ -253,9 +257,14 @@ def to_case(self) -> TestCase: path = os.path.abspath(path) if os.path.exists(path): dictionary[key] = path - break - return TestCase(self.trace, dictionary, self.ppn, self.rebuild) + dictionary.update(self.mods) + + if self.functor: + self.functor(dictionary) + print(dictionary) + + return TestCase(self.trace, dictionary, self.ppn) @dataclasses.dataclass @@ -277,11 +286,12 @@ def pop(self) -> None: return (self.mods.pop(), self.trace.pop()) -def define_case_f(trace: str, path: str, args: typing.List[str] = None, ppn: int = None, rebuild: bool = None) -> TestCaseBuilder: - return TestCaseBuilder(trace, {}, path, args or [], ppn, rebuild) +# pylint: disable=too-many-arguments +def define_case_f(trace: str, path: str, args: List[str] = None, newMods: dict = None, ppn: int = None, functor: Callable = None) -> TestCaseBuilder: + return TestCaseBuilder(trace, newMods or {}, path, args or [], ppn, functor) -def define_case_d(stack: CaseGeneratorStack, newTrace: str, newMods: dict, ppn: int = None, rebuild: bool = None) -> TestCaseBuilder: +def define_case_d(stack: CaseGeneratorStack, newTrace: str, newMods: dict, ppn: int = None, functor: Callable = None) -> TestCaseBuilder: mods: dict = {} for mod in stack.mods: @@ -297,4 +307,4 @@ def define_case_d(stack: CaseGeneratorStack, newTrace: str, newMods: dict, ppn: if not common.isspace(trace): traces.append(trace) - return TestCaseBuilder(' -> '.join(traces), mods, None, None, ppn, rebuild) + return TestCaseBuilder(' -> '.join(traces), mods, None, [], ppn, functor) diff --git a/toolchain/mfc/test/cases.py b/toolchain/mfc/test/cases.py index 8a3f018f1..bedfa066b 100644 --- a/toolchain/mfc/test/cases.py +++ b/toolchain/mfc/test/cases.py @@ -1,7 +1,10 @@ -import typing, itertools +import os, typing, itertools from mfc import common -from .case import define_case_d, CaseGeneratorStack, TestCaseBuilder +from .case import ( + define_case_d, define_case_f, + CaseGeneratorStack, TestCaseBuilder +) def get_bc_mods(bc: int, dimInfo): params = {} @@ -576,7 +579,25 @@ def foreach_dimension(): stack.pop() stack.pop() + def foreach_example(): + for path in os.listdir(common.MFC_EXAMPLE_DIRPATH): + if path == "scaling": + continue + + name = f"{path.split('_')[0]} -> Example -> {'_'.join(path.split('_')[1:])}" + path = os.path.join(common.MFC_EXAMPLE_DIRPATH, path, "case.py") + if not os.path.isfile(path): + continue + + def modify_example_case(case: dict): + case['t_step_save'] = 10 + + cases.append(define_case_f(name, path, [], { + 't_step_stop': 100, + }, functor=modify_example_case)) + foreach_dimension() + foreach_example() # Sanity Check 1 if stack.size() != 0: diff --git a/toolchain/mfc/test/test.py b/toolchain/mfc/test/test.py index 825c4689b..032e3894e 100644 --- a/toolchain/mfc/test/test.py +++ b/toolchain/mfc/test/test.py @@ -11,7 +11,7 @@ from .. import sched from ..run.input import MFCInputFile from ..common import MFCException, does_command_exist, format_list_to_string, get_program_output -from ..build import build, HDF5, PRE_PROCESS, SIMULATION, POST_PROCESS +from ..build import build, HDF5, PRE_PROCESS, SIMULATION, POST_PROCESS, REQUIRED_TARGETS from ..packer import tol as packtol from ..packer import packer @@ -93,13 +93,18 @@ def test(): return - codes = [PRE_PROCESS, SIMULATION] + ([POST_PROCESS] if ARG('test_all') else []) - if not ARG("case_optimization"): - build(codes) + codes = list(REQUIRED_TARGETS) + [PRE_PROCESS, SIMULATION] + ([POST_PROCESS] if ARG('test_all') else []) + built_slugs = set() for case in cases: - if case.rebuild: - build(codes, MFCInputFile(os.path.basename(case.get_dirpath()), case.get_dirpath(), case.params)) + case.delete_output() + case.create_directory() + + for code in codes: + slug = code.get_slug(case) + if slug not in built_slugs: + build(code, case.to_MFCInputFile()) + built_slugs.add(slug) cons.print() @@ -139,10 +144,6 @@ def _handle_case(case: TestCase, devices: typing.Set[int]): start_time = time.time() tol = case.compute_tolerance() - - case.delete_output() - case.create_directory() - cmd = case.run([PRE_PROCESS, SIMULATION], gpus=devices) out_filepath = os.path.join(case.get_dirpath(), "out_pre_sim.txt")