diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index d0fb445f2..6dc8bdb12 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -23,10 +23,15 @@ jobs: - name: Install Sphinx and Theme via Pip run: | pip install -r .github/workflows/requirements/docs.txt + # create a dummy workspace to get a working ramble + bin/benchpark setup saxpy/openmp nosite-x86_64 /tmp/workspace + # this is gross and we should better setup ramble for people or make it a Spack package + pip install -r /tmp/workspace/ramble/requirements.txt - name: Build with sphinx run: | - sphinx-build docs/ _build + cd docs + make html WORKSPACE_PATH=/tmp/workspace - name: Check for Typos using Codespell run: | diff --git a/.github/workflows/requirements/docs.txt b/.github/workflows/requirements/docs.txt index 6078a7a31..00c7b1674 100644 --- a/.github/workflows/requirements/docs.txt +++ b/.github/workflows/requirements/docs.txt @@ -2,3 +2,6 @@ sphinx==7.2.6 sphinx-rtd-theme==2.0.0 codespell==2.2.6 +pandas==2.2.0 +pyyaml==6.0.1 +sphinxcontrib-programoutput==0.17 diff --git a/bin/benchpark b/bin/benchpark index 20f2f7fd4..1cfb4f654 100755 --- a/bin/benchpark +++ b/bin/benchpark @@ -39,6 +39,7 @@ def main(): actions = {} benchpark_list(subparsers, actions) benchpark_setup(subparsers, actions) + benchpark_tags(subparsers, actions) args = parser.parse_args() no_args = True if len(sys.argv) == 1 else False @@ -62,7 +63,6 @@ def main(): def get_version(): benchpark_version = __version__ - return benchpark_version @@ -89,6 +89,15 @@ def benchpark_benchmarks(): return benchmarks +def benchpark_experiments(): + source_dir = source_location() + experiments = [] + experiments_dir = source_dir / "experiments" + for x in os.listdir(experiments_dir): + experiments.append(f"{x}") + return experiments + + def benchpark_systems(): source_dir = source_location() systems = [] @@ -97,6 +106,28 @@ def benchpark_systems(): return systems +def benchpark_get_tags(): + f = source_location() / "tags.yaml" + tags = [] + + with open(f, "r") as stream: + try: + data = yaml.safe_load(stream) + except yaml.YAMLError as exc: + print(exc) + + for k0, v0 in data.items(): + if k0 == "benchpark-tags": + for k, v in v0.items(): + if isinstance(v, list): + for i in v: + tags.append(i) + else: + print("ERROR file does not contain benchpark-tags") + + return tags + + def benchpark_list_handler(args): source_dir = source_location() sublist = args.sublist @@ -137,6 +168,17 @@ def benchpark_check_benchmark(arg_str): return found +def benchpark_check_experiment(arg_str): + experiments = benchpark_experiments() + found = arg_str in experiments + if not found: + out_str = f'Invalid benchmark/experiment "{arg_str}" - must choose one of: ' + for experiment in experiments: + out_str += f"\n\t{experiment}" + raise ValueError(out_str) + return found + + def benchpark_check_system(arg_str): systems = benchpark_systems() found = arg_str in systems @@ -148,6 +190,17 @@ def benchpark_check_system(arg_str): return found +def benchpark_check_tag(arg_str): + tags = benchpark_get_tags() + found = arg_str in tags + if not found: + out_str = f'Invalid tag "{arg_str}" - must choose one of: ' + for tag in tags: + out_str += f"\n\t{tag}" + raise ValueError(out_str) + return found + + def benchpark_setup(subparsers, actions_dict): create_parser = subparsers.add_parser( "setup", help="Set up an experiment and prepare it to build/run" @@ -175,7 +228,7 @@ def benchpark_setup(subparsers, actions_dict): def run_command(command_str, env=None): - subprocess.run( + result = subprocess.run( shlex.split(command_str), env=env, check=True, @@ -184,6 +237,30 @@ def run_command(command_str, env=None): text=True, ) + return (result.stdout, result.stderr) + + +def benchpark_tags(subparsers, actions_dict): + create_parser = subparsers.add_parser("tags", help="Tags in Benchpark experiments") + create_parser.add_argument( + "experiments_root", + type=str, + help="The experiments_root you specified during Benchpark setup.", + ) + create_parser.add_argument( + "-a", + "--application", + action="store", + help="The application for which to find Benchpark tags", + ) + create_parser.add_argument( + "-t", + "--tag", + action="store", + help="The tag for which to search in Benchpark experiments", + ) + actions_dict["tags"] = benchpark_tags_handler + # Note: it would be nice to vendor spack.llnl.util.link_tree, but that # involves pulling in most of llnl/util/ and spack/util/ @@ -375,5 +452,54 @@ Further steps are needed to build the experiments (ramble -P -D {ramble_workspac print(instructions) +def helper_experiments_tags(ramble_exe, experiments): + # find all tags in Ramble applications (both in Ramble built-in and in Benchpark/repo) + (tags_stdout, tags_stderr) = run_command(f"{ramble_exe} attributes --tags --all") + ramble_applications_tags = {} + lines = tags_stdout.splitlines() + + for line in lines: + key_value = line.split(":") + ramble_applications_tags[key_value[0]] = key_value[1].strip().split(",") + + benchpark_experiments_tags = {} + for exp in experiments: + benchpark_experiments_tags[exp] = ramble_applications_tags[exp] + return benchpark_experiments_tags + + +def benchpark_tags_handler(args): + """ + Filter ramble tags by benchpark experiments + """ + source_dir = source_location() + experiments_root = pathlib.Path(os.path.abspath(args.experiments_root)) + ramble_location = experiments_root / "ramble" + ramble_exe = ramble_location / "bin" / "ramble" + experiments = benchpark_experiments() + + if args.tag: + if benchpark_check_tag(args.tag): + # find all applications in Ramble that have a given tag (both in Ramble built-in and in Benchpark/repo) + (tag_stdout, tag_stderr) = run_command(f"{ramble_exe} list -t {args.tag}") + lines = tag_stdout.splitlines() + + for line in lines: + if line in experiments: + print(line) + + elif args.application: + if benchpark_check_experiment(args.application): + benchpark_experiments_tags = helper_experiments_tags( + ramble_exe, experiments + ) + print(benchpark_experiments_tags[args.application]) + else: + benchpark_experiments_tags = helper_experiments_tags(ramble_exe, experiments) + print("All tags that exist in Benchpark experiments:") + for k, v in benchpark_experiments_tags.items(): + print(k) + + if __name__ == "__main__": main() diff --git a/docs/2-benchpark-list.rst b/docs/2-benchpark-list.rst index 90cb5f6a4..29f03f084 100644 --- a/docs/2-benchpark-list.rst +++ b/docs/2-benchpark-list.rst @@ -3,14 +3,43 @@ SPDX-License-Identifier: Apache-2.0 -============== -Benchpark List -============== +================ +Search Benchpark +================ -To list all benchmarks and systems available in Benchpark:: +The user can search for available system and experiment specifications in Benchpark. + +.. list-table:: Searching for specifications in Benchpark + :widths: 25 25 50 + :header-rows: 1 + + * - Command + - Description + - Listing in the docs + * - benchpark list + - Lists all benchmarks and systems specified in Benchpark + - + * - benchpark list systems + - Lists all system specified in Benchpark + - :doc:`available-system-specs` + * - benchmark list benchmarks + - Lists all benchmarks specified in Benchpark + - + * - benchpark list systems + - Lists all system specified in Benchpark + - :doc:`available-system-specs` + * - benchpark tags workspace + - Lists all tags specified in Benchpark + - + * - benchpark tags -a application workspace + - Lists all tags specified for a given application in Benchpark + - + * - benchpark tags -t tag workspace + - Lists all experiments in Benchpark with a given tag + - - cd bin - benchpark list Once you have decided on a ``system`` you will use, and the ``benchmark/ProgrammingModel`` to run, you can proceed to :doc:`4-benchpark-setup`. + +For a complete list of options, see the help menu in :doc:`benchpark-help`. diff --git a/docs/Makefile b/docs/Makefile index 5f3bbf02d..75eff50de 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -7,6 +7,7 @@ SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = . BUILDDIR = _build +WORKSPACE_PATH ?= # Put it first so that "make" without argument is like "make help". help: @@ -15,7 +16,10 @@ help: systemconfigs: ./generate-sys-defs-list.py -.PHONY: help sysconfigs Makefile +tags: + ./generate-benchmark-list.py $(WORKSPACE_PATH) + +.PHONY: help systemconfigs tags Makefile # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). diff --git a/docs/available-system-specs.rst b/docs/available-system-specs.rst index 131025eae..bf483911a 100644 --- a/docs/available-system-specs.rst +++ b/docs/available-system-specs.rst @@ -13,6 +13,6 @@ use as the ``system`` parameter in Benchpark setup. See :doc:`4-benchpark-setup` for more details. .. csv-table:: Current system definitions in Benchpark. - :file: tables/current-system-definitions.csv + :file: current-system-definitions.csv :header-rows: 1 :align: left diff --git a/docs/benchmark-list.rst b/docs/benchmark-list.rst new file mode 100644 index 000000000..5ab06ab25 --- /dev/null +++ b/docs/benchmark-list.rst @@ -0,0 +1,13 @@ +.. Copyright 2023 Lawrence Livermore National Security, LLC and other + Benchpark Project Developers. See the top-level COPYRIGHT file for details. + + SPDX-License-Identifier: Apache-2.0 + +============== +Benchmark List +============== + +.. csv-table:: Current Benchpark tags by benchmark and tag groups. + :file: benchmark-list.csv + :header-rows: 1 + :align: left diff --git a/docs/benchpark-help.rst b/docs/benchpark-help.rst new file mode 100644 index 000000000..344beede4 --- /dev/null +++ b/docs/benchpark-help.rst @@ -0,0 +1,14 @@ +.. Copyright 2023 Lawrence Livermore National Security, LLC and other + Benchpark Project Developers. See the top-level COPYRIGHT file for details. + + SPDX-License-Identifier: Apache-2.0 + +=================== +Benchpark Help Menu +=================== + +Benchpark help menu:: + + $ benchpark --help + +.. program-output:: ../bin/benchpark --help diff --git a/docs/conf.py b/docs/conf.py index 5f7555c7a..e89a8ec39 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -15,6 +15,13 @@ ] ) +subprocess.call( + [ + "make", + "tags", + ] +) + project = "Benchpark" copyright = "2023, LLNS LLC" author = "Olga Pearce, Alec Scott, Peter Scheibel, Greg Becker, Riyaz Haque, and Nathan Hanford" @@ -24,6 +31,7 @@ extensions = [ "sphinx_rtd_theme", + "sphinxcontrib.programoutput", ] exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", ".spack-env"] diff --git a/docs/generate-benchmark-list.py b/docs/generate-benchmark-list.py new file mode 100755 index 000000000..391604a68 --- /dev/null +++ b/docs/generate-benchmark-list.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 + +import pandas as pd +import yaml +import os +import sys +import subprocess + + +def construct_tag_groups(tag_groups, tag_dicts, dictionary): + # everything is a dict + for k, v in dictionary.items(): + if isinstance(v, list): + tag_groups.append(k) + tag_dicts[k] = v + else: + print("ERROR in construct_tag_groups") + + +def benchpark_benchmarks(benchmarks): + experiments_dir = "../experiments" + for x in os.listdir(experiments_dir): + benchmarks.append(f"{x}") + return benchmarks + + +def main(workspace): + benchmarks = list() + benchpark_benchmarks(benchmarks) + + f = "../tags.yaml" + with open(f, "r") as stream: + try: + data = yaml.safe_load(stream) + except yaml.YAMLError as exc: + print(exc) + + tag_groups = [] + tag_dicts = {} + for k, v in data.items(): + if k == "benchpark-tags": + construct_tag_groups(tag_groups, tag_dicts, v) + else: + print("ERROR in top level construct_tag_groups") + + main = dict() + + tags_taggroups = {} + for bmark in benchmarks: + tags_taggroups[bmark] = {} + for k, v in tag_dicts.items(): + tags_taggroups[bmark][k] = [] + + for bmark in benchmarks: + # call benchpark tags -a bmark workspace + cmd = ["../bin/benchpark", "tags", "-a", bmark, workspace] + byte_data = subprocess.run(cmd, capture_output=True) + tags = str(byte_data.stdout, "utf-8") + tags = ( + tags.replace("[", "") + .replace("]", "") + .replace("'", "") + .replace(" ", "") + .replace("\n", "") + .split(",") + ) + for t in tags: + for k, v in tag_dicts.items(): + if t in v: + print("appending", t, "at key", k) + tags_taggroups[bmark][k].append(t) + main[bmark] = tags_taggroups[bmark] + + df = pd.DataFrame(main) + df.to_csv("benchmark-list.csv") + + ################# + # Tables + # columns: benchmarks (i.e., amg2023) + # rows: tag groups (i.e., application domain). Each cell should hopefully have a tag - and some might have multiple + + +if __name__ == "__main__": + try: + workspace = sys.argv[1] + except IndexError: + print("Usage: " + os.path.basename(__file__) + " ") + sys.exit(1) + + main(workspace) diff --git a/docs/generate-sys-defs-list.py b/docs/generate-sys-defs-list.py index f96cce0c4..028affacc 100755 --- a/docs/generate-sys-defs-list.py +++ b/docs/generate-sys-defs-list.py @@ -68,7 +68,7 @@ def main(): df_tpose.set_index([""], inplace=True) # Write out current system definitions to CSV format - df_tpose.to_csv("tables/current-system-definitions.csv") + df_tpose.to_csv("current-system-definitions.csv") if __name__ == "__main__": diff --git a/docs/index.rst b/docs/index.rst index 8c4bf657f..605b44009 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -21,9 +21,12 @@ .. toctree:: :maxdepth: 1 - :caption: Available Specifications + :caption: Catalogue + catalogue-of-benchmarks available-system-specs + benchmark-list + benchpark-help .. toctree:: :maxdepth: 1 diff --git a/docs/tables/current-system-definitions.csv b/docs/tables/current-system-definitions.csv deleted file mode 100644 index 38c574238..000000000 --- a/docs/tables/current-system-definitions.csv +++ /dev/null @@ -1,20 +0,0 @@ -,Sierra,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,Magma,Piz Daint,Fugaku,Pascal,Eiger,LUMI,Tioga,x86_64 -**site**,LLNL,,,LLNL,CSCS,RIKEN Center for Computational Science,LLNL,CSCS,CSC,LLNL, -**system**,IBM-power9-V100-Infiniband,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,Penguin-icelake-OmniPath,HPECray-haswell-P100-Infiniband,Fujitsu-A64FX-TofuD,Penguin-broadwell-P100-OmniPath,HPECray-zen2-Slingshot,HPECray-zen3-MI250X-Slingshot,HPECray-zen3-MI250X-Slingshot,x86_64 -**integrator.vendor**,IBM,AWS,HPECray,PenguinComputing,HPECray,Fujitsu,Penguin,HPECray,HPECray,HPECray, -**integrator.name**,AC922,ParallelCluster3.7.2-Hpc7a,EX235a,RelionCluster,,FX1000,,,EX235a,EX235a, -**processor.vendor**,IBM,AMD,AMD,Intel,Intel,Fujitsu,Intel,AMD,AMD,AMD, -**processor.name**,POWER9,EPYC-Zen4,EPYC-Zen3,XeonPlatinum924248C,Xeon-E5-2650v3,A64FX,Xeon_E5-2695_v4,EPYC-7742,EPYC-Zen3,EPYC-Zen3, -**processor.ISA**,ppc64le,x86_64,x86_64,x86_64,x86_64,Armv8.2-A-SVE,x86_64,x86_64,x86_64,x86_64,x86_64 -**processor.uArch**,power9,zen4,zen3,icelake,haswell,aarch64,broadwell,zen2,zen3,zen3, -**accelerator.vendor**,NVIDIA,,AMD,,NVIDIA,,NVIDIA,,AMD,AMD, -**accelerator.name**,V100,,MI250X,,P100,,P100,,MI250X,MI250X, -**accelerator.ISA**,PTX,,GCN,,PTX,,PTX,,GCN,GCN, -**accelerator.uArch**,sm_70,,gfx90a,,sm_60,,sm_56,,gfx90a,gfx90a, -**interconnect.vendor**,Mellanox,AWS,HPECray,Intel,HPECray,Fujitsu,Cornelis,HPECray,HPECray,HPECray, -**interconnect.name**,EDR-Infiniband,EFA,Slingshot11,OmniPath,Aries,TofuInterconnectD,OmniPath,Slingshot,Slingshot11,Slingshot11, -**system-tested.site**,LLNL,AWS,LLNL,LLNL,CSCS,R-CCS,LLNL,CSCS,CSC,LLNL, -**system-tested.name**,lassen,,tioga,magma,daint,Fugaku,pascal,daint,LUMI,tioga, -**system-tested.installation-year**,2018,,2022,2019,2017,2020,2018,2017,2023,2022, -**system-tested.description**,`top500 `_,`aws/hpc7a `_,`top500 `_,`top500 `_,`top500 `_,`top500 `_,,`top500 `_,`top500 `_,`top500 `_, -**top500-system-instances**,Sierra (LLNL),,"Frontier (ORNL), LUMI (CSC), Tioga (LLNL)",Magma (LLNL),Piz Daint (CSCS),Fugaku (R-CCS),,,"Frontier (ORNL), LUMI (CSC), Tioga (LLNL)","Frontier (ORNL), LUMI (CSC), Tioga (LLNL)", diff --git a/experiments/lbann/cuda/execute_experiment.tpl b/experiments/lbann/cuda/execute_experiment.tpl deleted file mode 100755 index 1e2ea813e..000000000 --- a/experiments/lbann/cuda/execute_experiment.tpl +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -#!/bin/bash -{batch_nodes} -{batch_ranks} -{batch_timeout} - -cd {experiment_run_dir} - -{command} diff --git a/experiments/lbann/cuda/ramble.yaml b/experiments/lbann/cuda/ramble.yaml deleted file mode 100644 index 027481554..000000000 --- a/experiments/lbann/cuda/ramble.yaml +++ /dev/null @@ -1,56 +0,0 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -ramble: - include: - - ./configs/spack.yaml - - ./configs/variables.yaml - - config: - deprecated: true - spack_flags: - install: '--add --keep-stage' - concretize: '-U -f' - - applications: - lbann: - workloads: - problem1: - env_vars: - set: - OMP_NUM_THREADS: '{omp_num_threads}' - variables: - n_ranks: '{processes_per_node} * {n_nodes}' - p: 2 - px: '{p}' - py: '{p}' - pz: '{p}' - n: ['55', '110'] - nx: '{n}' - ny: '{n}' - nz: '{n}' - processes_per_node: ['8', '4'] - n_nodes: ['1', '2'] - threads_per_node_core: ['8', '10', '13'] #TODO: Specify n_threads according to available n_nodes and n_ranks - omp_num_threads: '{threads_per_node_core} * {n_nodes}' - experiments: - lbann_omp_problem1_{n_nodes}_{omp_num_threads}_{px}_{py}_{pz}_{nx}_{ny}_{nz}: - variables: - env_name: amg2023-omp - matrices: - - size_threads: - - n # TODO: Filter matrix - - threads_per_node_core # TODO: Filter matrix - spack: - concretized: true - packages: - lbann-cuda: - spack_spec: lbann@develop +mpi+cuda - compiler: default-compiler - environments: - amg2023-omp: - packages: - - default-mpi - - lbann-cuda diff --git a/experiments/lbann/rocm/execute_experiment.tpl b/experiments/lbann/rocm/execute_experiment.tpl deleted file mode 100755 index 1e2ea813e..000000000 --- a/experiments/lbann/rocm/execute_experiment.tpl +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -#!/bin/bash -{batch_nodes} -{batch_ranks} -{batch_timeout} - -cd {experiment_run_dir} - -{command} diff --git a/experiments/stream/openmp/execute_experiment.tpl b/experiments/streamc/openmp/execute_experiment.tpl similarity index 100% rename from experiments/stream/openmp/execute_experiment.tpl rename to experiments/streamc/openmp/execute_experiment.tpl diff --git a/experiments/stream/openmp/ramble.yaml b/experiments/streamc/openmp/ramble.yaml similarity index 100% rename from experiments/stream/openmp/ramble.yaml rename to experiments/streamc/openmp/ramble.yaml diff --git a/pyproject.toml b/pyproject.toml index f787664d9..bc919dd96 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,5 +31,5 @@ skip_gitignore = true color_output = true [tool.codespell] -skip = './_build,./docs/_static' +skip = './docs/_build,./docs/_static' ignore-words-list = 'fom' diff --git a/repo/amg2023/application.py b/repo/amg2023/application.py index 0c8cc3d80..442df30e9 100644 --- a/repo/amg2023/application.py +++ b/repo/amg2023/application.py @@ -12,7 +12,12 @@ class Amg2023(SpackApplication): """AMG2023 benchmark""" name = "amg2023" - tags = ["amg2023"] + tags = ['asc','engineering','hypre','solver','sparse-linear-algebra', + 'large-scale','multi-node','single-node','sub-node', + 'high-branching','high-memory-bandwidth','large-memory-footprint', + 'regular-memory-access','irregular-memory-access','mixed-precision', + 'mpi','network-latency-bound','network-collectives','block-structured-grid', + 'c','cuda','hip','openmp'] executable('p1', 'amg' + ' -P {px} {py} {pz}' + diff --git a/repo/lbann/application.py b/repo/lbann/application.py deleted file mode 100644 index 4d87357bc..000000000 --- a/repo/lbann/application.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2023 Lawrence Livermore National Security, LLC and other -# Benchpark Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: Apache-2.0 - -import sys - -from ramble.appkit import * - - -class LBANN(SpackApplication): - """LBANN benchmark""" - name = "lbann" - - tags = ["lbann"] - - executable('p1', 'lbann' + - ' -P {px} {py} {pz}' + - ' -n {nx} {ny} {nz}' + - ' -problem 1' + - ' -keepT', use_mpi=True) - - executable('p2', 'lbann' + - ' -P {px} {py} {pz}' + - ' -n {nx} {ny} {nz}' + - ' -problem 2' + - ' -keepT', use_mpi=True) - - workload('problem1', executables=['p1']) - workload('problem2', executables=['p2']) - - workload_variable('px', default='2', - description='px', - workloads=['problem1', 'problem2']) - workload_variable('py', default='2', - description='py', - workloads=['problem1', 'problem2']) - workload_variable('pz', default='2', - description='pz', - workloads=['problem1', 'problem2']) - workload_variable('nx', default='220', - description='nx', - workloads=['problem1', 'problem2']) - workload_variable('ny', default='220', - description='ny', - workloads=['problem1', 'problem2']) - workload_variable('nz', default='220', - description='nz', - workloads=['problem1', 'problem2']) - - figure_of_merit('Figure of Merit (FOM)', log_file='{experiment_run_dir}/{experiment_name}.out', fom_regex=r'Figure of Merit \(FOM\):\s+(?P[0-9]+\.[0-9]*(e^[0-9]*)?)', group_name='fom', units='') - - #TODO: Fix the FOM success_criteria(...) - success_criteria('pass', mode='string', match=r'Figure of Merit \(FOM\)', file='{experiment_run_dir}/{experiment_name}.out') - - def evaluate_success(self): - return True diff --git a/repo/raja-perf/application.py b/repo/raja-perf/application.py index 6d028a275..682d355d0 100644 --- a/repo/raja-perf/application.py +++ b/repo/raja-perf/application.py @@ -12,7 +12,11 @@ class RajaPerf(SpackApplication): """RAJA Performance suite""" name = "raja-perf" - tags = ["raja-perf"] + tags = ['asc','single-node','sub-node','structured-grid', + 'atomics','simd','vectorization','register-pressure', + 'high-memory-bandwidth','regular-memory-access', + 'mpi','network-point-to-point','network-latency-bound', + 'c++','raja','cuda','hip','openmp','sycl'] executable('run', 'raja-perf.exe', use_mpi=True) diff --git a/repo/saxpy/application.py b/repo/saxpy/application.py index 0fd12db01..14385b391 100644 --- a/repo/saxpy/application.py +++ b/repo/saxpy/application.py @@ -12,7 +12,9 @@ class Saxpy(SpackApplication): """saxpy benchmark""" name = "saxpy" - tags = ["saxpy"] + tags = ['single-node','high-memory-bandwidth', + 'regular-memory-access', + 'c++','cuda','hip','openmp'] executable('p', 'saxpy -n {n}', use_mpi=True) diff --git a/tags.yaml b/tags.yaml new file mode 100644 index 000000000..423d71bae --- /dev/null +++ b/tags.yaml @@ -0,0 +1,109 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +benchpark-tags: + application-domain: + - asc # Advanced Simulation and Computing + - astrophysics + - cfd # computational fluid dynamics + - chemistry + - climate + - dft # density functional theory + - engineering + - fusion + - material-science + - molecular-dynamics + benchmark-scale: + - large-scale + - multi-node + - single-node + - sub-node + communication: + - mpi + - nccl + - nvsmem + - rccl + - shmem + - upc + communication-performance-characteristics: + - network-bandwidth-bound + - network-bisection-bandwidth-bound + - network-collectives + - network-latency-bound + - network-point-to-point + compute-performance-characteristics: + - atomics + - high-branching + - high-fp + - i-o + - mixed-precision + - register-pressure + - simd + - vectorization + math-libraries: + - blas + - cublas + - hypre + - lapack + - mfem + - rocblas + - rocsolver + memory-access-characteristics: + - high-memory-bandwidth + - irregular-memory-access + - large-memory-footprint + - regular-memory-access + mesh-representation: + - amr + - block-structured-grid + - meshfree + - structured-grid + - unstructured-grid + method-type: + - ai # AI, ML, DL + - ale # arbitrary lagrangian-eulerian + - dense-linear-algebra + - deterministic + - direct-solve + - eulerian + - explicit-differentiation + - explicit-timestepping + - finite-element + - fft # fast fourier transform + - full-assembly + - high-order + - hydrodynamics + - implicit-differentiation + - implicit-timestepping + - lagrangian + - low-order + - montecarlo + - nbody + - ode # ordinary differential equations + - partial-assembly + - particles + - solver + - sph # smoothed particle hydrodynamics + - sparse-linear-algebra + - spatial-discretization + - task-parallelism + - time-dependent + - transport + programming-language: + - c + - c++ + - fortran + - julia + - python + - rust + programming-model: + - charm++ + - cuda + - hip + - kokkos + - oneapi + - openmp + - raja + - sycl