diff --git a/.github/ISSUE_TEMPLATE/new_release.md b/.github/ISSUE_TEMPLATE/new_release.md new file mode 100644 index 0000000000..1406f04e30 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/new_release.md @@ -0,0 +1,53 @@ + +Instructions for performing an ADIOS2 release: + +- [ ] Make sure that the milestone for @VERSION@ has no pending issues/PRs. +- [ ] Update your remotes +`` +git fetch origin +git fetch github #if needed +`` +- [ ] Create a branch that updates the version + +``` +git checkout -b bump-release-version origin/release_@MAJOR@@MINOR@ +``` + +``` +git checkout -b bump-release-version origin/master +``` + +- [ ] Add Commit that updates the version in the repo +``` +git grep --name-only @OLD_RELEASE@ | xargs -n1 sed -i 's/@OLD_RELEASE@/@VERSION@/g' +git commit -am 'Bump version to v@VERSION@' +git push +``` +- [ ] Create PR (BASE to master if release_@MAJOR@@MINOR@ does not exists; otherwise release_@MAJOR@@MINOR@) +- [ ] Ask for review +- [ ] Merge PR +- [ ] Create Tag commit `git tag -a v@VERSION@ the_merge_commit` +- [ ] Create Release in GitHub page + - Use the following script for getting the PR of this release + - `./scripts/developer/create-changelog.sh v@VERSION@ v@OLD_RELEASE@` + - Copy its content to the release description + +- [ ] Create the release_@MAJOR@@MINOR@ branch +``` +git fetch origin +git checkout -b release_@MAJOR@@MINOR@ origin/master +# Use the following command with care +git push origin +``` + +- [ ] Create PR that merges release_@MAJOR@@MINOR@ into master +- [ ] Submit a PR in Spack that adds this new version of ADIOS (if not RC mark this new version as preferred) +- [ ] Write an announcement in the ADIOS-ECP mail-list + (https://groups.google.com/a/kitware.com/g/adios-ecp) diff --git a/.github/workflows/everything.yml b/.github/workflows/everything.yml index 914bb5ff26..435d9851e5 100644 --- a/.github/workflows/everything.yml +++ b/.github/workflows/everything.yml @@ -82,6 +82,9 @@ jobs: - name: Python working-directory: source run: ../gha/scripts/ci/scripts/run-flake8.sh + - name: Shell + working-directory: source + run: ../gha/scripts/ci/scripts/run-shellcheck.sh ####################################### @@ -145,7 +148,11 @@ jobs: linux_emu: needs: [format, git_checks] - if: false # needs.git_checks.outputs.num_code_changes > 0 + if: needs.git_checks.outputs.num_code_changes > 0 && + ( + contains('refs/heads/master', github.ref) || + contains('refs/heads/release_', github.ref) + ) runs-on: ubuntu-latest container: @@ -164,7 +171,7 @@ jobs: cpu: [power8] os: [el7] compiler: [xl] - parallel: [serial, mpi] + parallel: [serial] include: - cpu: power8 os: el7 @@ -172,12 +179,6 @@ jobs: parallel: serial container: ornladios/adios2:ci-x86_64-power8-el7-xl arch: ppc64le - - cpu: power8 - os: el7 - compiler: xl - parallel: mpi - container: ornladios/adios2:ci-x86_64-power8-el7-xl-smpi - arch: ppc64le steps: - name: Emulation Setup diff --git a/.gitlab/config/SpackCIBridge.py b/.gitlab/config/SpackCIBridge.py new file mode 100755 index 0000000000..72c189e480 --- /dev/null +++ b/.gitlab/config/SpackCIBridge.py @@ -0,0 +1,692 @@ +#!/usr/bin/env python3 + +import argparse +import atexit +import base64 +from datetime import datetime, timedelta, timezone +import dateutil.parser +from github import Github +import json +import os +import re +import subprocess +import sys +import tempfile +import urllib.parse +import urllib.request + + +class SpackCIBridge(object): + + def __init__(self, gitlab_repo="", gitlab_host="", gitlab_project="", github_project="", + disable_status_post=True, sync_draft_prs=False, + main_branch=None, prereq_checks=[]): + self.gitlab_repo = gitlab_repo + self.github_project = github_project + github_token = os.environ.get('GITHUB_TOKEN') + self.github_repo = "https://{0}@github.com/{1}.git".format(github_token, self.github_project) + self.py_github = Github(github_token) + self.py_gh_repo = self.py_github.get_repo(self.github_project, lazy=True) + + self.merge_msg_regex = re.compile(r"Merge\s+([^\s]+)\s+into\s+([^\s]+)") + self.unmergeable_shas = [] + + self.post_status = not disable_status_post + self.sync_draft_prs = sync_draft_prs + self.main_branch = main_branch + self.currently_running_sha = None + self.latest_tested_main_commit = None + + self.prereq_checks = prereq_checks + + dt = datetime.now(timezone.utc) + timedelta(minutes=-60) + self.time_threshold_brief = urllib.parse.quote_plus(dt.isoformat(timespec="seconds")) + + self.pipeline_api_template = gitlab_host + self.pipeline_api_template += "/api/v4/projects/" + self.pipeline_api_template += urllib.parse.quote_plus(gitlab_project) + self.pipeline_api_template += "/pipelines?ref={0}" + + self.commit_api_template = gitlab_host + self.commit_api_template += "/api/v4/projects/" + self.commit_api_template += urllib.parse.quote_plus(gitlab_project) + self.commit_api_template += "/repository/commits/{0}" + + self.cached_commits = {} + + @atexit.register + def cleanup(): + """Shutdown ssh-agent upon program termination.""" + if "SSH_AGENT_PID" in os.environ: + print(" Shutting down ssh-agent({0})".format(os.environ["SSH_AGENT_PID"])) + subprocess.run(["ssh-agent", "-k"], check=True) + + def setup_ssh(self, ssh_key_base64): + """Start the ssh agent.""" + print("Starting ssh-agent") + output = subprocess.run(["ssh-agent", "-s"], check=True, stdout=subprocess.PIPE).stdout + + # Search for PID in output. + pid_regexp = re.compile(r"SSH_AGENT_PID=([0-9]+)") + match = pid_regexp.search(output.decode("utf-8")) + if match is None: + print("WARNING: could not detect ssh-agent PID.", file=sys.stderr) + print("ssh-agent will not be killed upon program termination", file=sys.stderr) + else: + pid = match.group(1) + os.environ["SSH_AGENT_PID"] = pid + self.cleanup_ssh_agent = True + + # Search for socket in output. + socket_regexp = re.compile(r"SSH_AUTH_SOCK=([^;]+);") + match = socket_regexp.search(output.decode("utf-8")) + if match is None: + print("WARNING: could not detect ssh-agent socket.", file=sys.stderr) + print("Key will be added to caller's ssh-agent (if any)", file=sys.stderr) + else: + socket = match.group(1) + os.environ["SSH_AUTH_SOCK"] = socket + + # Add the key. + ssh_key = base64.b64decode(ssh_key_base64) + ssh_key = ssh_key.replace(b"\r", b"") + with tempfile.NamedTemporaryFile() as fp: + fp.write(ssh_key) + fp.seek(0) + subprocess.run(["ssh-add", fp.name], check=True) + + def get_commit(self, commit): + """ Check our cache for a commit on GitHub. + If we don't have it yet, use the GitHub API to retrieve it.""" + if commit not in self.cached_commits: + self.cached_commits[commit] = self.py_gh_repo.get_commit(sha=commit) + return self.cached_commits[commit] + + def list_github_prs(self): + """ Return two dicts of data about open PRs on GitHub: + one for all open PRs, and one for open PRs that are not up-to-date on GitLab.""" + pr_dict = {} + pulls = self.py_gh_repo.get_pulls(state="open") + print("Rate limit after get_pulls(): {}".format(self.py_github.rate_limiting[0])) + for pull in pulls: + backlogged = False + push = True + if pull.draft and not self.sync_draft_prs: + print("Skipping draft PR {0} ({1})".format(pull.number, pull.head.ref)) + backlogged = "draft" + push = False + + pr_string = "pr{0}_{1}".format(pull.number, pull.head.ref) + + if push and pull.updated_at < datetime.now() + timedelta(minutes=-2880): + # Skip further analysis of this PR if it hasn't been updated in 48 hours. + # This helps us avoid wasting our rate limit on PRs with merge conflicts. + print("Skip pushing stale PR {0}".format(pr_string)) + backlogged = "stale" + push = False + + if push: + # Determine if this PR still needs to be pushed to GitLab. This happens in one of two cases: + # 1) we have never pushed it before + # 2) we have pushed it before, but the HEAD sha has changed since we pushed it last + log_args = ["git", "log", "--pretty=%s", "gitlab/{0}".format(pr_string)] + try: + merge_commit_msg = subprocess.run( + log_args, check=True, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).stdout + match = self.merge_msg_regex.match(merge_commit_msg.decode("utf-8")) + if match and (match.group(1) == pull.head.sha or match.group(2) == pull.head.sha): + print("Skip pushing {0} because GitLab already has HEAD {1}".format(pr_string, pull.head.sha)) + push = False + except subprocess.CalledProcessError: + # This occurs when it's a new PR that hasn't been pushed to GitLab yet. + pass + + if push: + # Check the PRs-to-be-pushed to see if any of them should be considered "backlogged". + # We currently recognize three types of backlogged PRs: + # 1) Some required "prerequisite checks" have not yet completed successfully. + # 2) The PR is based on a version of the "main branch" that has not yet been tested + # 3) Draft PRs. Handled earlier in this function. + if not backlogged and self.prereq_checks: + checks_desc = "waiting for {} check to succeed" + checks_to_verify = self.prereq_checks.copy() + pr_check_runs = self.get_commit(pull.head.sha).get_check_runs() + for check in pr_check_runs: + if check.name in checks_to_verify: + checks_to_verify.remove(check.name) + if check.conclusion != "success": + backlogged = checks_desc.format(check.name) + push = False + break + if not backlogged and checks_to_verify: + backlogged = checks_desc.format(checks_to_verify[0]) + push = False + if backlogged: + print("Skip pushing {0} because of {1}".format(pr_string, backlogged)) + + if not backlogged: + if self.main_branch and pull.base.ref == self.main_branch: + # Check if we should defer pushing/testing this PR because it is based on "too new" of a commit + # of the main branch. + tmp_pr_branch = f"temporary_{pr_string}" + subprocess.run(["git", "fetch", "--unshallow", "github", + f"refs/pull/{pull.number}/head:{tmp_pr_branch}"], check=True) + # Get the merge base between this PR and the main branch. + try: + merge_base_sha = subprocess.run( + ["git", "merge-base", tmp_pr_branch, f"github/{self.main_branch}"], + check=True, stdout=subprocess.PIPE).stdout.strip() + except subprocess.CalledProcessError: + print(f"'git merge-base {tmp_pr_branch} github/{self.main_branch}' " + "returned non-zero. Skipping") + self.unmergeable_shas.append(pull.head.sha) + continue + + repo_head_sha = subprocess.run( + ["git", "rev-parse", tmp_pr_branch], + check=True, stdout=subprocess.PIPE).stdout.decode("utf-8").strip() + + if pull.head.sha != repo_head_sha: + # If gh repo and api don't agree on what the head sha is, don't + # push. Instead log an error message and backlog the PR. + a_sha, r_sha = pull.head.sha[:7], repo_head_sha[:7] + print(f"Skip pushing {pr_string} because api says HEAD is {a_sha}, " + f"while repo says HEAD is {r_sha}") + backlogged = f"GitHub HEAD shas out of sync (repo={r_sha}, API={a_sha})" + push = False + # Check if our PR's merge base is an ancestor of the latest tested main branch commit. + elif subprocess.run( + ["git", "merge-base", "--is-ancestor", merge_base_sha, self.latest_tested_main_commit] + ).returncode == 0: + print(f"{tmp_pr_branch}'s merge base IS an ancestor of latest_tested_main " + f"{merge_base_sha} vs. {self.latest_tested_main_commit}") + try: + subprocess.run(["git", "checkout", self.latest_tested_main_commit], check=True) + subprocess.run(["git", "checkout", "-b", pr_string], check=True) + commit_msg = f"Merge {pull.head.sha} into {self.latest_tested_main_commit}" + subprocess.run( + ["git", "merge", "--no-ff", "-m", commit_msg, tmp_pr_branch], + check=True) + print(f"Merge succeeded, ready to push {pr_string} to GitLab for CI pipeline testing") + except subprocess.CalledProcessError: + print(f"Failed to merge PR {pull.number} ({pull.head.ref}) with latest tested " + f"{self.main_branch} ({self.latest_tested_main_commit}). Skipping") + self.unmergeable_shas.append(pull.head.sha) + subprocess.run(["git", "merge", "--abort"]) + backlogged = "merge conflicts with {}".format(self.main_branch) + push = False + continue + else: + print(f"Skip pushing {pr_string} because its merge base is NOT an ancestor of " + f"latest_tested_main {merge_base_sha} vs. {self.latest_tested_main_commit}") + backlogged = "base" + push = False + else: + # If the --main-branch CLI argument wasn't passed, or if this PR doesn't target that branch, + # then we will push the merge commit that was automatically created by GitHub to GitLab + # where it will kick off a CI pipeline. + try: + subprocess.run(["git", "fetch", "--unshallow", "github", + f"{pull.merge_commit_sha}:{pr_string}"], check=True) + except subprocess.CalledProcessError: + print("Failed to locally checkout PR {0} ({1}). Skipping" + .format(pull.number, pull.merge_commit_sha)) + backlogged = "GitLab failed to checkout this branch" + push = False + continue + + pr_dict[pr_string] = { + 'base_sha': pull.base.sha, + 'head_sha': pull.head.sha, + 'push': push, + 'backlogged': backlogged, + } + + def listify_dict(d): + pr_strings = sorted(d.keys()) + base_shas = [d[s]['base_sha'] for s in pr_strings] + head_shas = [d[s]['head_sha'] for s in pr_strings] + b_logged = [d[s]['backlogged'] for s in pr_strings] + return { + "pr_strings": pr_strings, + "base_shas": base_shas, + "head_shas": head_shas, + "backlogged": b_logged, + } + all_open_prs = listify_dict(pr_dict) + filtered_pr_dict = {k: v for (k, v) in pr_dict.items() if v['push']} + filtered_open_prs = listify_dict(filtered_pr_dict) + print("All Open PRs:") + for pr_string in all_open_prs['pr_strings']: + print(" {0}".format(pr_string)) + print("Filtered Open PRs:") + for pr_string in filtered_open_prs['pr_strings']: + print(" {0}".format(pr_string)) + print("Rate limit at the end of list_github_prs(): {}".format(self.py_github.rate_limiting[0])) + return [all_open_prs, filtered_open_prs] + + def list_github_protected_branches(self): + """ Return a list of protected branch names from GitHub.""" + branches = self.py_gh_repo.get_branches() + print("Rate limit after get_branches(): {}".format(self.py_github.rate_limiting[0])) + protected_branches = [br.name for br in branches if br.protected] + protected_branches = sorted(protected_branches) + if self.currently_running_sha: + print("Skip pushing {0} because it already has a pipeline running ({1})" + .format(self.main_branch, self.currently_running_sha)) + protected_branches.remove(self.main_branch) + print("Protected branches:") + for protected_branch in protected_branches: + print(" {0}".format(protected_branch)) + return protected_branches + + def list_github_tags(self): + """ Return a list of tag names from GitHub.""" + tag_list = self.py_gh_repo.get_tags() + print("Rate limit after get_tags(): {}".format(self.py_github.rate_limiting[0])) + tags = sorted([tag.name for tag in tag_list]) + print("Tags:") + for tag in tags: + print(" {0}".format(tag)) + return tags + + def setup_git_repo(self): + """Initialize a bare git repository with two remotes: + one for GitHub and one for GitLab. + If main_branch was specified, we also fetch that branch from GitHub. + """ + subprocess.run(["git", "init"], check=True) + subprocess.run(["git", "config", "user.email", "noreply@spack.io"], check=True) + subprocess.run(["git", "config", "user.name", "spackbot"], check=True) + subprocess.run(["git", "config", "advice.detachedHead", "false"], check=True) + subprocess.run(["git", "remote", "add", "github", self.github_repo], check=True) + subprocess.run(["git", "remote", "add", "gitlab", self.gitlab_repo], check=True) + + # Shallow fetch from GitLab. + self.gitlab_shallow_fetch() + + if self.main_branch: + subprocess.run(["git", "fetch", "--unshallow", "github", self.main_branch], check=True) + + def get_gitlab_pr_branches(self): + """Query GitLab for branches that have already been copied over from GitHub PRs. + Return the string output of `git branch --remotes --list gitlab/pr*`. + """ + branch_args = ["git", "branch", "--remotes", "--list", "gitlab/pr*"] + self.gitlab_pr_output = \ + subprocess.run(branch_args, check=True, stdout=subprocess.PIPE).stdout + + def gitlab_shallow_fetch(self): + """Perform a shallow fetch from GitLab""" + fetch_args = ["git", "fetch", "-q", "--depth=1", "gitlab"] + subprocess.run(fetch_args, check=True, stdout=subprocess.PIPE).stdout + + def get_open_refspecs(self, open_prs): + """Return a list of refspecs to push given a list of open PRs.""" + print("Building initial lists of refspecs to fetch and push") + pr_strings = open_prs["pr_strings"] + base_shas = open_prs["base_shas"] + backlogged = open_prs["backlogged"] + open_refspecs = [] + for open_pr, base_sha, backlog in zip(pr_strings, base_shas, backlogged): + open_refspecs.append("{0}:{0}".format(open_pr)) + print(" pushing {0} (based on {1})".format(open_pr, base_sha)) + return open_refspecs + + def update_refspecs_for_protected_branches(self, protected_branches, open_refspecs, fetch_refspecs): + """Update our refspecs lists for protected branches from GitHub.""" + for protected_branch in protected_branches: + fetch_refspecs.append("+refs/heads/{0}:refs/remotes/{0}".format(protected_branch)) + open_refspecs.append("refs/heads/{0}:refs/heads/{0}".format(protected_branch)) + return open_refspecs, fetch_refspecs + + def update_refspecs_for_tags(self, tags, open_refspecs, fetch_refspecs): + """Update our refspecs lists for tags from GitHub.""" + for tag in tags: + fetch_refspecs.append("+refs/tags/{0}:refs/tags/{0}".format(tag)) + open_refspecs.append("refs/tags/{0}:refs/tags/{0}".format(tag)) + return open_refspecs, fetch_refspecs + + def fetch_github_branches(self, fetch_refspecs): + """Perform `git fetch` for a given list of refspecs.""" + print("Fetching GitHub refs for open PRs") + fetch_args = ["git", "fetch", "-q", "--unshallow", "github"] + fetch_refspecs + subprocess.run(fetch_args, check=True) + + def build_local_branches(self, protected_branches): + """Create local branches for a list of protected branches.""" + print("Building local branches for protected branches") + for branch in protected_branches: + local_branch_name = "{0}".format(branch) + remote_branch_name = "refs/remotes/{0}".format(branch) + subprocess.run(["git", "branch", "-q", local_branch_name, remote_branch_name], check=True) + + def make_status_for_pipeline(self, pipeline): + """Generate POST data to create a GitHub status from a GitLab pipeline + API response + """ + post_data = {} + if "status" not in pipeline: + return post_data + + if pipeline["status"] == "created": + post_data["state"] = "pending" + post_data["description"] = "Pipeline has been created" + + elif pipeline["status"] == "waiting_for_resource": + post_data["state"] = "pending" + post_data["description"] = "Pipeline is waiting for resources" + + elif pipeline["status"] == "preparing": + post_data["state"] = "pending" + post_data["description"] = "Pipeline is preparing" + + elif pipeline["status"] == "pending": + post_data["state"] = "pending" + post_data["description"] = "Pipeline is pending" + + elif pipeline["status"] == "running": + post_data["state"] = "pending" + post_data["description"] = "Pipeline is running" + + elif pipeline["status"] == "manual": + post_data["state"] = "pending" + post_data["description"] = "Pipeline is running manually" + + elif pipeline["status"] == "scheduled": + post_data["state"] = "pending" + post_data["description"] = "Pipeline is scheduled" + + elif pipeline["status"] == "failed": + post_data["state"] = "error" + post_data["description"] = "Pipeline failed" + + elif pipeline["status"] == "canceled": + # Do not post canceled pipeline status to GitHub, it's confusing to our users. + # This usually happens when a PR gets force-pushed. The next time the sync script runs + # it will post a status for the newly force-pushed commit. + return {} + + elif pipeline["status"] == "skipped": + post_data["state"] = "failure" + post_data["description"] = "Pipeline was skipped" + + elif pipeline["status"] == "success": + post_data["state"] = "success" + post_data["description"] = "Pipeline succeeded" + + post_data["target_url"] = pipeline["web_url"] + return post_data + + def dedupe_pipelines(self, api_response): + """Prune pipelines API response to only include the most recent result for each SHA""" + pipelines = {} + for response in api_response: + sha = response['sha'] + if sha not in pipelines: + pipelines[sha] = response + else: + existing_datetime = dateutil.parser.parse(pipelines[sha]['updated_at']) + current_datetime = dateutil.parser.parse(response['updated_at']) + if current_datetime > existing_datetime: + pipelines[sha] = response + return pipelines + + def find_pr_sha(self, tested_sha): + api_url = self.commit_api_template.format(tested_sha) + + try: + request = urllib.request.Request(api_url) + if "GITLAB_TOKEN" in os.environ: + request.add_header("Authorization", "Bearer %s" % os.environ["GITLAB_TOKEN"]) + response = urllib.request.urlopen(request) + except OSError: + print('Failed to fetch commit for tested sha {0}'.format(tested_sha)) + return None + + response_data = response.read() + + try: + tested_commit_info = json.loads(response_data) + except json.decoder.JSONDecodeError: + print('Failed to parse response as json ({0})'.format(response_data)) + return None + + if 'title' not in tested_commit_info: + print('Returned commit object missing "Title" field') + return None + + merge_commit_msg = tested_commit_info['title'] + m = self.merge_msg_regex.match(merge_commit_msg) + + if m is None: + print('Failed to find pr_sha in merge commit message') + return None + + return m.group(1) + + def get_pipelines_for_branch(self, branch, time_threshold=None): + # Use gitlab's API to get pipeline results for the corresponding ref. + api_url = self.pipeline_api_template.format( + urllib.parse.quote_plus(branch) + ) + + # Optionally constrain the query with the provided time_threshold + if time_threshold: + api_url = "{0}&updated_after={1}".format(api_url, time_threshold) + + try: + request = urllib.request.Request(api_url) + if "GITLAB_TOKEN" in os.environ: + request.add_header("Authorization", "Bearer %s" % os.environ["GITLAB_TOKEN"]) + response = urllib.request.urlopen(request) + except OSError as inst: + print("GitLab API request error accessing {0}".format(api_url)) + print(inst) + return None + try: + pipelines = json.loads(response.read()) + except json.decoder.JSONDecodeError as inst: + print("Error parsing response to {0}".format(api_url)) + print(inst) + return None + + return self.dedupe_pipelines(pipelines) + + def post_pipeline_status(self, open_prs, protected_branches): + print("Rate limit at the beginning of post_pipeline_status(): {}".format(self.py_github.rate_limiting[0])) + pipeline_branches = [] + backlog_branches = [] + # Split up the open_prs branches into two piles: branches we force-pushed to gitlab + # and branches we deferred pushing. + for pr_branch, base_sha, head_sha, backlog in zip(open_prs["pr_strings"], + open_prs["base_shas"], + open_prs["head_shas"], + open_prs["backlogged"]): + if not backlog: + pipeline_branches.append(pr_branch) + else: + backlog_branches.append((pr_branch, head_sha, backlog)) + + pipeline_branches.extend(protected_branches) + + print('Querying pipelines to post status for:') + for branch in pipeline_branches: + # Post status to GitHub for each pipeline found. + pipelines = self.get_pipelines_for_branch(branch, self.time_threshold_brief) + if not pipelines: + continue + for sha, pipeline in pipelines.items(): + post_data = self.make_status_for_pipeline(pipeline) + if not post_data: + continue + # TODO: associate shas with protected branches, so we do not have to + # hit an endpoint here, but just use the sha we already know just like + # we do below for backlogged PR statuses. + pr_sha = self.find_pr_sha(sha) + if not pr_sha: + print('Could not find github PR sha for tested commit: {0}'.format(sha)) + print('Using tested commit to post status') + pr_sha = sha + self.create_status_for_commit(pr_sha, + branch, + post_data["state"], + post_data["target_url"], + post_data["description"]) + + # Post a status of pending/backlogged for branches we deferred pushing + print("Posting backlogged status to the following:") + base_backlog_desc = \ + "This branch's merge-base with {} is newer than the latest commit tested by GitLab".format(self.main_branch) + for branch, head_sha, reason in backlog_branches: + if reason == "stale": + print("Skip posting status for {} because it has not been updated recently".format(branch)) + continue + elif reason == "base": + desc = base_backlog_desc + url = "https://github.com/spack/spack-infrastructure/blob/main/docs/deferred_pipelines.md" + elif reason == "draft": + desc = "GitLab CI is disabled for draft PRs" + url = "" + else: + desc = reason + url = "" + self.create_status_for_commit(head_sha, branch, "pending", url, desc) + + # Post errors to any PRs that we couldn't merge to latest_tested_main_commit. + print('Posting unmergeable status to the following:') + for sha in self.unmergeable_shas: + print(' {0}'.format(sha)) + self.create_status_for_commit(sha, "", "error", "", f"PR could not be merged with {self.main_branch}") + print("Rate limit at the end of post_pipeline_status(): {}".format(self.py_github.rate_limiting[0])) + + def create_status_for_commit(self, sha, branch, state, target_url, description): + context = "OLCF Ascent (Summit)" + commit = self.get_commit(sha) + existing_statuses = commit.get_combined_status() + for status in existing_statuses.statuses: + if (status.context == context and + status.state == state and + status.description == description and + status.target_url == target_url): + print("Not posting duplicate status to {} / {}".format(branch, sha)) + return + try: + status_response = self.get_commit(sha).create_status( + state=state, + target_url=target_url, + description=description, + context=context + ) + if status_response.state != state: + print("Expected CommitStatus state {0}, got {1}".format( + state, status_response.state)) + except Exception as e_inst: + print('Caught exception posting status for {0}/{1}'.format(branch, sha)) + print(e_inst) + print(" {0} -> {1}".format(branch, sha)) + + def sync(self): + """Synchronize pull requests from GitHub as branches on GitLab.""" + + print("Initial rate limit: {}".format(self.py_github.rate_limiting[0])) + reset_time = datetime.utcfromtimestamp(self.py_github.rate_limiting_resettime).strftime('%Y-%m-%d %H:%M:%S') + print("Rate limit will refresh at: {} UTC".format(reset_time)) + + # Setup SSH command for communicating with GitLab. + os.environ["GIT_SSH_COMMAND"] = "ssh -F /dev/null -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" + + # Work inside a temporary directory that will be deleted when this script terminates. + with tempfile.TemporaryDirectory() as tmpdirname: + os.chdir(tmpdirname) + + # Setup the local repo with two remotes. + self.setup_git_repo() + + if self.main_branch: + # Find the currently running main branch pipeline, if any, and get the sha. + # Also get the latest commit on the main branch that has a completed pipeline. + main_branch_pipelines = self.get_pipelines_for_branch(self.main_branch) + for sha, pipeline in main_branch_pipelines.items(): + if self.latest_tested_main_commit is None and \ + (pipeline['status'] == "success" or pipeline['status'] == "failed"): + self.latest_tested_main_commit = sha + + if self.currently_running_sha is None and pipeline['status'] == "running": + self.currently_running_sha = sha + + if self.latest_tested_main_commit and self.currently_running_sha: + break + + print("Latest completed {0} pipeline: {1}".format(self.main_branch, self.latest_tested_main_commit)) + print("Currently running {0} pipeline: {1}".format(self.main_branch, self.currently_running_sha)) + + # Retrieve open PRs from GitHub. + all_open_prs, open_prs = self.list_github_prs() + + # Get protected branches on GitHub. + protected_branches = self.list_github_protected_branches() + + # Get tags on GitHub. + tags = self.list_github_tags() + + # Get refspecs for open PRs and protected branches. + open_refspecs = self.get_open_refspecs(open_prs) + fetch_refspecs = [] + self.update_refspecs_for_protected_branches(protected_branches, open_refspecs, fetch_refspecs) + self.update_refspecs_for_tags(tags, open_refspecs, fetch_refspecs) + + # Sync open GitHub PRs and protected branches to GitLab. + self.fetch_github_branches(fetch_refspecs) + self.build_local_branches(protected_branches) + if open_refspecs: + print("Syncing to GitLab") + push_args = ["git", "push", "--porcelain", "-f", "gitlab"] + open_refspecs + subprocess.run(push_args, check=True) + + # Post pipeline status to GitHub for each open PR, if enabled + if self.post_status: + print('Posting pipeline status for open PRs and protected branches') + self.post_pipeline_status(all_open_prs, protected_branches) + + +if __name__ == "__main__": + # Parse command-line arguments. + parser = argparse.ArgumentParser(description="Sync GitHub PRs to GitLab") + parser.add_argument("github_project", help="GitHub project (org/repo or user/repo)") + parser.add_argument("gitlab_repo", help="Full clone URL for GitLab") + parser.add_argument("gitlab_host", help="GitLab web host") + parser.add_argument("gitlab_project", help="GitLab project (org/repo or user/repo)") + parser.add_argument("--disable-status-post", action="store_true", default=False, + help="Do not post pipeline status to each GitHub PR") + parser.add_argument("--sync-draft-prs", action="store_true", default=False, + help="Copy draft PRs from GitHub to GitLab") + parser.add_argument("--pr-mirror-bucket", default=None, + help="Delete mirrors for closed PRs from the specified S3 bucket") + parser.add_argument("--main-branch", default=None, + help="""If provided, we check if there is a currently running +pipeline for this branch. If so, we defer syncing any subsequent commits in an effort +to not interrupt this pipeline. We also defer pushing any PR branches that are based +on a commit of the main branch that is newer than the latest commit tested by GitLab.""") + parser.add_argument("--prereq-check", nargs="+", default=False, + help="Only push branches that have already passed this GitHub check") + + args = parser.parse_args() + + ssh_key_base64 = os.getenv("GITLAB_SSH_KEY_BASE64") + if ssh_key_base64 is None: + raise Exception("GITLAB_SSH_KEY_BASE64 environment is not set") + + if "GITHUB_TOKEN" not in os.environ: + raise Exception("GITHUB_TOKEN environment is not set") + + bridge = SpackCIBridge(gitlab_repo=args.gitlab_repo, + gitlab_host=args.gitlab_host, + gitlab_project=args.gitlab_project, + github_project=args.github_project, + disable_status_post=args.disable_status_post, + sync_draft_prs=args.sync_draft_prs, + main_branch=args.main_branch, + prereq_checks=args.prereq_check) + bridge.setup_ssh(ssh_key_base64) + bridge.sync() diff --git a/.gitlab/config/ccache.cmake b/.gitlab/config/ccache.cmake new file mode 100644 index 0000000000..5e5f90c66b --- /dev/null +++ b/.gitlab/config/ccache.cmake @@ -0,0 +1,61 @@ +##============================================================================ +## Copyright (c) Kitware, Inc. +## All rights reserved. +## See LICENSE.txt for details. +## +## This software is distributed WITHOUT ANY WARRANTY; without even +## the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR +## PURPOSE. See the above copyright notice for more information. +##============================================================================ + +cmake_minimum_required(VERSION 3.0 FATAL_ERROR) + +set(version 4.6.1) +set(arch x86_64) + +if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux") + set(sha256sum da1e1781bc1c4b019216fa16391af3e1daaee7e7f49a8ec9b0cdc8a1d05c50e2) + set(base_url https://github.com/ccache/ccache/releases/download) + set(platform linux) + set(extension tar.xz) +elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin") + set(sha256sum 3e36ba8c80fbf7f2b95fe0227b9dd1ca6143d721aab052caf0d5729769138059) + set(full_url https://gitlab.kitware.com/utils/ci-utilities/-/package_files/534/download) + set(filename ccache) + set(extension tar.gz) +elseif(CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows") + set(sha256sum a6c6311973aa3d2aae22424895f2f968e5d661be003b25f1bd854a5c0cd57563) + set(base_url https://github.com/ccache/ccache/releases/download) + set(platform windows) + set(extension zip) +else() + message(FATAL_ERROR "Unrecognized platform ${CMAKE_HOST_SYSTEM_NAME}") +endif() + +if(NOT DEFINED filename) + set(filename "ccache-${version}-${platform}-${arch}") +endif() + +set(tarball "${filename}.${extension}") + +if(NOT DEFINED full_url) + set(full_url "${base_url}/v${version}/${tarball}") +endif() + +file(DOWNLOAD + "${full_url}" $ENV{CCACHE_INSTALL_DIR}/${tarball} + EXPECTED_HASH SHA256=${sha256sum} + SHOW_PROGRESS + ) + +execute_process( + COMMAND ${CMAKE_COMMAND} -E tar xf ${tarball} + WORKING_DIRECTORY $ENV{CCACHE_INSTALL_DIR} + RESULT_VARIABLE extract_results + ) + +if(extract_results) + message(FATAL_ERROR "Extracting `${tarball}` failed: ${extract_results}.") +endif() + +file(RENAME $ENV{CCACHE_INSTALL_DIR}/${filename} $ENV{CCACHE_INSTALL_DIR}/ccache) diff --git a/.gitlab/config/dynamic_pipeline.yml.in b/.gitlab/config/dynamic_pipeline.yml.in new file mode 100644 index 0000000000..b1a1ba01c9 --- /dev/null +++ b/.gitlab/config/dynamic_pipeline.yml.in @@ -0,0 +1,9 @@ +child_pipeline_{branch}: + variables: + DOWNSTREAM_COMMIT_SHA: '{commit}' + DOWNSTREAM_BRANCH_REF: '{branch}' + trigger: + include: + - project: 'ci/csc303_crusher/dev/adios2' + ref: '{branch}' + file: '.gitlab/gitlab-ci-crusher.yml' diff --git a/.gitlab/config/generate_pipelines.py b/.gitlab/config/generate_pipelines.py new file mode 100755 index 0000000000..17b8068b99 --- /dev/null +++ b/.gitlab/config/generate_pipelines.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 + +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +# +# generate_pipeline.py +# +# Created: May 19, 2023 +# Author: Vicente Adolfo Bolea Sanchez + +from datetime import datetime +import argparse +import requests +import time +import re +import urllib3 +# Remove annoying warning about insecure connection (self-signed cert). +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + +def is_date_after(date, days): + deadline_sec = int(time.time()) - (days * 86400) + utc_dt = datetime.strptime(date, '%Y-%m-%dT%H:%M:%SZ') + timestamp_sec = (utc_dt - datetime(1970, 1, 1)).total_seconds() + return timestamp_sec > deadline_sec + + +def request_dict(url): + r = requests.get(url + '?per_page=100', verify=False) + return r.json() + + +parser = argparse.ArgumentParser( + prog='generate_pipeline.py', + description='Generate Dynamic pipelines for Gitlab') +parser.add_argument( + '-u', '--gl-url', required=True, + help='Base URL for Gitlab remote. Ex: https://code.olcf.ornl.gov/') +parser.add_argument( + '-n', '--gh-name', required=True, + help='Full name of the GitHub project. Ex: ornladios/ADIOS2') +parser.add_argument( + '-c', '--gh-context', default='OLCF Crusher (Frontier)', + help='Name of the status in GitHub (A.K.A context)') +parser.add_argument( + '-p', '--project_id', required=True, + help='Gitlab internal project ID of the project.') +parser.add_argument( + '-d', '--days', type=int, default=1, + help='How many days back to search for commits') +parser.add_argument( + '-m', '--max', type=int, default=2, + help='Maximum amount of pipelines computed') +parser.add_argument( + '-f', '--template_file', required=True, + help='Template file of the pipeline `{branch}` will be substituted') +args = parser.parse_args() + + +with open(args.template_file, "r") as fd: + template_str = fd.read() + gl_url = args.gl_url + "/api/v4/projects/" + str(args.project_id) + gh_url = 'https://api.github.com/repos/' + args.gh_name + branches = request_dict(gl_url + "/repository/branches") + num_pipeline = 0 + for branch in branches: + # Convert to ISO 8601 date format. + date_stamp = branch['commit']['committed_date'].split('.')[0] + "Z" + if num_pipeline < args.max and is_date_after(date_stamp, args.days): + commit_sha = branch['commit']['id'] + # Backported branches use the merge head + gh_commit_sha = commit_sha + if re.fullmatch(r'^pr\d+_.*$', branch['name']): + gh_commit_sha = branch['commit']['parent_ids'][1] + + # Quit if GitHub does not have the commit + if 'sha' not in request_dict(gh_url + "/commits/" + gh_commit_sha): + continue + + # Query GitHub for the status of this commit + commit = request_dict(gh_url + "/commits/" + + gh_commit_sha + "/status") + status_found = False + for status in commit['statuses']: + if status['context'] == args.gh_context: + status_found = True + if not status_found: + num_pipeline += 1 + print(template_str.format( + branch=branch['name'], commit=commit_sha)) diff --git a/.gitlab/config/kokkos.sh b/.gitlab/config/kokkos.sh new file mode 100755 index 0000000000..2504b90637 --- /dev/null +++ b/.gitlab/config/kokkos.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -x + +WORKDIR="$1" +VERSION="$2" + +shift 2 + +if [ ! -d "$WORKDIR" ] || [ -z "$VERSION" ] +then + echo "[E] missing args: Invoke as .gitlab/ci/config/kokkos.sh [extra_args]" + exit 1 +fi + +# Build and install Kokkos +curl -L "https://github.com/kokkos/kokkos/archive/refs/tags/$VERSION.tar.gz" \ + | tar -C "$WORKDIR" -xzf - + +cmake -S "$WORKDIR/kokkos-$VERSION" -B "$WORKDIR/kokkos_build" \ + "-DBUILD_SHARED_LIBS=ON" \ + "-DCMAKE_BUILD_TYPE:STRING=release" \ + "-DCMAKE_CXX_COMPILER_LAUNCHER=ccache" \ + "-DCMAKE_CXX_STANDARD:STRING=17" \ + "-DCMAKE_CXX_EXTENSIONS:BOOL=OFF" \ + "-DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON" \ + $* + +cmake --build "$WORKDIR/kokkos_build" +cmake --install "$WORKDIR/kokkos_build" diff --git a/.gitlab/gitlab-ci-ascent.yml b/.gitlab/gitlab-ci-ascent.yml new file mode 100644 index 0000000000..c4e453df0d --- /dev/null +++ b/.gitlab/gitlab-ci-ascent.yml @@ -0,0 +1,134 @@ +# Ad-hoc build that runs in the ECP Hardware, concretely in OLCF Ascent. +.setup_env_ecpci: &setup_env_ecpci | + module purge + module load ${JOB_MODULES} + module list + export PATH="/gpfs/wolf/csc303/scratch/vbolea/ci/utils:$PATH" + +.ascent-common: + except: + - schedules + tags: + - batch + interruptible: true + variables: + CCACHE_BASEDIR: "/gpfs/wolf/" + CCACHE_DIR: "/gpfs/wolf/csc303/scratch/vbolea/ci/ccache" + # -isystem= is not affected by CCACHE_BASEDIR, thus we must ignore it + CCACHE_IGNOREOPTIONS: "-isystem=*" + CCACHE_NOHASHDIR: "true" + + CUDAHOSTCXX: "g++" + CUSTOM_CI_BUILDS_DIR: "/gpfs/wolf/csc303/scratch/vbolea/ci/adios2" + GITLAB_SITE: "OLCF Ascent" + CI_BIN_DIR: "$CI_PROJECT_DIR/build" + SCHEDULER_PARAMETERS: -P CSC303 -W 1:00 -nnodes 1 -alloc_flags gpudefault + before_script: + - *setup_env_ecpci + - ccache -z + script: + - bash scripts/ci/gitlab-ci/run.sh update + - bash scripts/ci/gitlab-ci/run.sh configure + - jsrun -n1 -a1 -g1 -c40 -bpacked:40 bash scripts/ci/gitlab-ci/run.sh build + - jsrun -n1 -a1 -g1 -c2 bash scripts/ci/gitlab-ci/run.sh test + after_script: + - *setup_env_ecpci + - bash scripts/ci/gitlab-ci/run.sh submit + - ccache -s + +ascent-cuda: + variables: + # Order matters + JOB_MODULES: >- + DefApps + zstd + cuda/11.4.2 + git + gcc/10.2.0 + ninja + spectrum-mpi + lsf-tools + libffi + hdf5 + cmake + extends: + - .ascent-common + +ascent-kokkos-cuda: + variables: + # Order matters + JOB_MODULES: >- + DefApps + zstd + cuda/11.4.2 + git + gcc/10.2.0 + ninja + spectrum-mpi + lsf-tools + libffi + hdf5 + cmake + KOKKOS_VER: 3.7.01 + Kokkos_DIR: "$CI_PROJECT_DIR/deps/kokkos_install" + # Cmake would not install a RPATH inside the source dir + LD_LIBRARY_PATH: "$Kokkos_DIR/lib64/:$LD_LIBRARY_PATH" + KOKKOS_OPTS: >- + -DKokkos_ARCH_POWER9=ON + -DKokkos_ARCH_VOLTA70=ON + -DKokkos_ENABLE_CUDA=ON + -DKokkos_ENABLE_CUDA_LAMBDA=ON + -DCMAKE_INSTALL_PREFIX:PATH=$Kokkos_DIR + -DCMAKE_CXX_COMPILER:STRING=$CI_PROJECT_DIR/deps/kokkos-$KOKKOS_VER/bin/nvcc_wrapper + before_script: + - *setup_env_ecpci + - mkdir -p "$CI_PROJECT_DIR/deps" + - ccache -z + - .gitlab/config/kokkos.sh "$CI_PROJECT_DIR/deps" "$KOKKOS_VER" $KOKKOS_OPTS + extends: + - .ascent-common + +ascent-nvhpc: + variables: + # Order matters + JOB_MODULES: >- + DefApps + zstd + nvhpc + git + spectrum-mpi + lsf-tools + libffi + hdf5 + cmake + extends: + - .ascent-common + +ascent-xl: + variables: + # Order matters + JOB_MODULES: >- + DefApps + zstd + cuda/11.4.2 + git + xl + ninja + spectrum-mpi + lsf-tools + libffi + hdf5 + cmake + extends: + - .ascent-common + +sync-github-prs: + tags: + - nobatch + only: + - schedules + variables: + CUSTOM_CI_BUILDS_DIR: "/gpfs/wolf/csc303/scratch/vbolea/ci/adios2" + script: + - export PATH="/gpfs/wolf/csc303/scratch/vbolea/ci/utils:$PATH" + - .gitlab/config/SpackCIBridge.py ornladios/ADIOS2 git@code.ornl.gov:ecpcitest/adios2.git https://code.ornl.gov/ ecpcitest/adios2 --prereq-check=format --prereq-check=git_checks diff --git a/.gitlab/gitlab-ci-crusher.yml b/.gitlab/gitlab-ci-crusher.yml new file mode 100644 index 0000000000..3e07a59794 --- /dev/null +++ b/.gitlab/gitlab-ci-crusher.yml @@ -0,0 +1,223 @@ +# Ad-hoc build that runs in the ECP Hardware, concretely in OLCF Crusher. + +stages: + - pre + - setup + - build + - post + +.setup_env_ecpci: &setup_env_ecpci | + git fetch + source scripts/ci/gitlab-ci/setup-vars.sh + git checkout "$CI_COMMIT_REF" + module purge + module load ${JOB_MODULES} + module list + export PATH="${CCACHE_INSTALL_DIR}/ccache:$PATH" + +.install_ccache: &install_ccache | + mkdir -p "$CCACHE_INSTALL_DIR" + cmake --version + cmake -VV -P .gitlab/config/ccache.cmake + ccache -z + ccache -s + +.crusher-common: + rules: + - if: $CI_PIPELINE_SOURCE =~ /parent_pipeline|web/ + interruptible: true + variables: + CCACHE_BASEDIR: "/lustre/orion/csc303/scratch/" + CCACHE_DIR: "/lustre/orion/csc303/scratch/vbolea/ci/adios2/ccache" + CUSTOM_CI_BUILDS_DIR: "/lustre/orion/csc303/scratch/vbolea/ci/adios2/runtime" + + # -isystem= is not affected by CCACHE_BASEDIR, thus we must ignore it + CCACHE_IGNOREOPTIONS: "-isystem=*" + CCACHE_NOHASHDIR: "true" + CCACHE_INSTALL_DIR: "$CI_PROJECT_DIR/deps/ccache_install" + + CMAKE_BUILD_TYPE: "RelWithDebInfo" + CMAKE_GENERATOR: "Ninja" + CMAKE_PREFIX_PATH: "$CI_PROJECT_DIR/deps/kokkos_install" + + # We do not want to use the user's ~/.gitconfig + GIT_CONFIG_GLOBAL: "true" + GITLAB_SITE: "OLCF Crusher" + CI_BIN_DIR: "$CI_PROJECT_DIR/build" + +.setup-common: + stage: setup + tags: [ shell ] + before_script: + - *setup_env_ecpci + - *install_ccache + script: + - bash scripts/ci/gitlab-ci/run.sh update + artifacts: + expire_in: 24 hours + when: always + paths: + - deps/*install/ + - build/ + +.build-common: + stage: build + tags: [ slurm ] + variables: + SCHEDULER_PARAMETERS: "-ACSC303_crusher -t30 --nice=0 -c32 --gpus=4 -N 1" + before_script: + - *setup_env_ecpci + script: + - bash scripts/ci/gitlab-ci/run.sh configure + - bash scripts/ci/gitlab-ci/run.sh build + - bash scripts/ci/gitlab-ci/run.sh test + after_script: + - *setup_env_ecpci + - bash scripts/ci/gitlab-ci/run.sh submit + - ccache -s + +.kokkos-hip-common: + variables: + Kokkos_DIR: "$CI_PROJECT_DIR/deps/kokkos_install" + # Cmake would not install a RPATH inside the source dir + LD_LIBRARY_PATH: "$Kokkos_DIR/lib64/:$LD_LIBRARY_PATH" + # Order matters + JOB_MODULES: >- + craype-accel-amd-gfx90a + gcc/12 + cmake + rocm/5.4.3 + git + ninja + libffi + hdf5 + zstd + +setup:crusher-kokkos-hip: + variables: + KOKKOS_VER: 3.7.01 + KOKKOS_OPTS: >- + -DCMAKE_INSTALL_PREFIX:PATH=$Kokkos_DIR + -DCMAKE_CXX_COMPILER:FILEPATH=/opt/rocm-5.4.3/hip/bin/hipcc + -DKokkos_ARCH_VEGA90A:BOOL=ON + -DKokkos_ENABLE_HIP:BOOL=ON + -DKokkos_ENABLE_HIP_RELOCATABLE_DEVICE_CODE:BOOL=OFF + -DKokkos_ENABLE_SERIAL:BOOL=ON + extends: + - .crusher-common + - .setup-common + - .kokkos-hip-common + before_script: + - *setup_env_ecpci + - *install_ccache + - .gitlab/config/kokkos.sh "$CI_PROJECT_DIR/deps" "$KOKKOS_VER" $KOKKOS_OPTS + +build:crusher-kokkos-hip: + extends: + - .crusher-common + - .build-common + - .kokkos-hip-common + before_script: + - *setup_env_ecpci + needs: + - setup:crusher-kokkos-hip + dependencies: + - setup:crusher-kokkos-hip + +.cray-common: + variables: + # Order matters + JOB_MODULES: >- + PrgEnv-cray + cmake + git + ninja + libffi + zstd + DefApps + extends: + - .crusher-common + +setup:crusher-cray: + extends: + - .setup-common + - .cray-common + +build:crusher-cray: + extends: + - .build-common + - .cray-common + needs: + - setup:crusher-cray + dependencies: + - setup:crusher-cray + +.report-status: + rules: + - if: $CI_PIPELINE_SOURCE =~ /parent_pipeline|web/ + tags: [ shell ] + variables: + STATUS_PROJECT: ornladios/ADIOS2 + STATUS_NAME: OLCF Crusher (Frontier) + before_script: | + git fetch + source scripts/ci/gitlab-ci/setup-vars.sh + git checkout "$CI_COMMIT_REF" + script: > + curl -X POST -H @${GITHUB_CURL_HEADERS} + "https://api.github.com/repos/${STATUS_PROJECT}/statuses/${CI_ORIGINAL_SHA}" + -d "{\"state\":\"${CI_JOB_NAME}\", \"context\":\"${STATUS_NAME}\",\"target_url\":\"${CI_PIPELINE_URL}\",\"description\":\"${STATUS_DESC}\"}" + environment: + name: report-$DOWNSTREAM_COMMIT_SHA + +pending: + stage: pre + variables: + STATUS_DESC: Pipeline is running + extends: + - .report-status +success: + stage: post + variables: + STATUS_DESC: Pipeline succeeded + extends: + - .report-status + dependencies: + - build:crusher-kokkos-hip + - build:crusher-cray +failure: + stage: post + rules: + - if: $CI_PIPELINE_SOURCE =~ /parent_pipeline|web/ + when: on_failure + variables: + STATUS_DESC: Pipeline failed + extends: + - .report-status + dependencies: + - build:crusher-kokkos-hip + - build:crusher-cray + +generate_pipelines: + stage: setup + tags: [ shell ] + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" + variables: + CUSTOM_CI_BUILDS_DIR: "/lustre/orion/csc303/scratch/vbolea/ci/adios2/runtime" + script: + - .gitlab/config/generate_pipelines.py -u "https://code.olcf.ornl.gov/" -p 78 -n ornladios/ADIOS2 -f .gitlab/config/dynamic_pipeline.yml.in > generated_pipelines.yml + artifacts: + paths: + - generated_pipelines.yml + +launch_pipelines: + stage: build + rules: + - if: $CI_PIPELINE_SOURCE == "schedule" + variables: + CUSTOM_CI_BUILDS_DIR: "/lustre/orion/csc303/scratch/vbolea/ci/adios2/runtime" + trigger: + include: + - artifact: generated_pipelines.yml + job: generate_pipelines diff --git a/.gitlab/gitlab-ci-gitlabdotcom.yml b/.gitlab/gitlab-ci-gitlabdotcom.yml deleted file mode 100644 index eba4958546..0000000000 --- a/.gitlab/gitlab-ci-gitlabdotcom.yml +++ /dev/null @@ -1,6 +0,0 @@ -sync-github-prs: - tags: linux - only: - - schedules - script: - - scripts/ci/scripts/github-prs-to-gitlab.sh ornladios/adios2 code.ornl.gov ecpcitest/adios2 diff --git a/.gitlab/gitlab-ci-olcf.yml b/.gitlab/gitlab-ci-olcf.yml deleted file mode 100644 index ac593185a1..0000000000 --- a/.gitlab/gitlab-ci-olcf.yml +++ /dev/null @@ -1,28 +0,0 @@ -.all-steps: - except: - - schedules - variables: - GITLAB_SITE: "OLCF GitLab" - CMAKE_ENV_MODULE: "cmake" - script: - - bash scripts/ci/gitlab-ci/run.sh update - - bash scripts/ci/gitlab-ci/run.sh configure - - bash scripts/ci/gitlab-ci/run.sh build - - bash scripts/ci/gitlab-ci/run.sh test - -ascent-xl: - extends: - - .all-steps - tags: [nobatch] - -ascent-gcc: - extends: - - .all-steps - tags: [nobatch] - -#ascent-xl-smpi: -# extends: -# - .all-steps -# tags: [batch] -# variables: -# SCHEDULER_PARAMETERS: "-P CSC303 -W 1:00 -nnodes 1" diff --git a/.shellcheck_exclude_paths b/.shellcheck_exclude_paths new file mode 100644 index 0000000000..1bd1cb33a0 --- /dev/null +++ b/.shellcheck_exclude_paths @@ -0,0 +1,52 @@ +scripts/ci/azure/linux-setup.sh +scripts/ci/azure/macos-setup.sh +scripts/ci/azure/run.sh +scripts/ci/circle/postCDashStatus.sh +scripts/ci/circle/run.sh +scripts/ci/gh-actions/check-branch-name.sh +scripts/ci/gh-actions/get-changed-files.sh +scripts/ci/gh-actions/linux-setup.sh +scripts/ci/gh-actions/macos-setup.sh +scripts/ci/gh-actions/run.sh +scripts/ci/images-v2/build-base.sh +scripts/ci/images-v2/build-clang-base.sh +scripts/ci/images-v2/build-clang.sh +scripts/ci/images-v2/build-cuda.sh +scripts/ci/images-v2/build-functions.sh +scripts/ci/images-v2/build-gcc-base.sh +scripts/ci/images-v2/build-gcc.sh +scripts/ci/images-v2/build-intel-base.sh +scripts/ci/images-v2/build-intel.sh +scripts/ci/images-v2/build-mpich.sh +scripts/ci/images-v2/build-nvhpc.sh +scripts/ci/images-v2/build.sh +scripts/ci/images/build-emu-power8-image.sh +scripts/ci/images/build-native-images.sh +scripts/ci/images/emu-el7/entrypoint.sh +scripts/ci/images/emu-el7/qemu-binfmt-conf.sh +scripts/ci/images/emu-el7/register.sh +scripts/ci/scripts/github-prs-to-gitlab.sh +scripts/ci/scripts/run-clang-format.sh +scripts/ci/scripts/run-flake8.sh +scripts/ci/setup-run/ci-el8-gcc10.sh +scripts/ci/setup-run/ci-el8-gcc11.sh +scripts/ci/setup-run/ci-el8-gcc9.sh +scripts/ci/setup-run/ci-el8-icc.sh +scripts/ci/setup-run/ci-el8-nvhpc222-mpi.sh +scripts/ci/setup-run/ci-el8-nvhpc222.sh +scripts/ci/setup-run/ci-el8-oneapi.sh +scripts/ci/setup/ci-power8-el7-xl-serial.sh +scripts/dashboard/nightly/aaargh.sh +scripts/dashboard/nightly/cori.sh +scripts/dashboard/nightly/summitdev.sh +scripts/developer/git/setup-aliases +scripts/developer/git/setup-hooks +scripts/developer/git/setup-remotes +scripts/developer/merge-pr-from-release.sh +scripts/developer/setup.sh +scripts/docker/setup-user.sh +scripts/runconf/runconf.sh +scripts/runconf/runconf_olcf.sh +scripts/travis/run-docker.sh +scripts/travis/run-format.sh +scripts/travis/run.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index ff352b268d..93d76a790a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -237,7 +237,7 @@ endif() set(ADIOS2_CONFIG_OPTS BP5 DataMan DataSpaces HDF5 HDF5_VOL MHS SST Fortran MPI Python Blosc2 BZip2 LIBPRESSIO MGARD PNG SZ ZFP DAOS IME O_DIRECT Sodium Catalyst SysVShMem UCX ZeroMQ - Profiling Endian_Reverse AWSSDK GPU_Support CUDA Kokkos Kokkos_CUDA Kokkos_HIP + Profiling Endian_Reverse AWSSDK GPU_Support CUDA Kokkos Kokkos_CUDA Kokkos_HIP Kokkos_SYCL ) GenerateADIOSHeaderConfig(${ADIOS2_CONFIG_OPTS}) diff --git a/CODEOWNERS b/CODEOWNERS index ef9d75102a..2d2c4cd680 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,17 +1,25 @@ # KW's stuff -**/CMakeLists.txt @vicentebolea @caitlinross -*.cmake @vicentebolea @caitlinross +CMakeLists.txt @vicentebolea @caitlinross + +# Caitlin's stuff +plugins/ @caitlinross # Vicente's stuff -*.sh @vicentebolea *.bash @vicentebolea -*.in @vicentebolea -*.yml @vicentebolea -*.yaml @vicentebolea +*.cmake @vicentebolea *.in @vicentebolea *.json @vicentebolea +*.sh @vicentebolea *.txt @vicentebolea +*.yaml @vicentebolea +*.yml @vicentebolea +cmake/ @vicentebolea scripts/ @vicentebolea .github/ @vicentebolea .circleci/ @vicentebolea source/adios2/toolkit/sst/dp/mpi_dp.c @vicentebolea + +# GPU-aware specific files +source/adios2/helper/kokkos/ @anagainaru +source/adios2/helper/adiosCUDA.* @anagainaru +source/adios2/helper/adiosGPUFunctions.h @anagainaru diff --git a/CTestCustom.cmake.in b/CTestCustom.cmake.in index 5dcb601175..e118eb3e31 100644 --- a/CTestCustom.cmake.in +++ b/CTestCustom.cmake.in @@ -22,6 +22,8 @@ list(APPEND CTEST_CUSTOM_WARNING_EXCEPTION "warning: template parameter ... is not used in declaring the parameter types of function template" "warning: command-line option '.*' is valid for Fortran but not for C" "Warning #20208-D: '.*' is treated as '.*' in device code" + "Warning: '.*' is treated as '.*' in device code" + ".*was specified as both a system and non-system include directory.*" ) list(APPEND CTEST_CUSTOM_COVERAGE_EXCLUDE ".*/thirdparty/.*" @@ -29,6 +31,14 @@ list(APPEND CTEST_CUSTOM_COVERAGE_EXCLUDE ".*/examples/.*" ) +if (@CMAKE_SYSTEM_PROCESSOR@ STREQUAL "ppc64le") + # Exclude flaky tests for now list(APPEND CTEST_CUSTOM_TESTS_IGNORE + "Interface.ADIOSInterface.ADIOSNoMpi.Serial" + "Interface.ADIOS2_CXX11_API_MultiBlock/*.Put.Serial" + "Interface.ADIOS2_CXX11_API_IO.EngineDefault.Serial" + "Interface.ADIOS2_CXX11_API_IO.Engine.Serial" ) + +endif() diff --git a/bindings/C/CMakeLists.txt b/bindings/C/CMakeLists.txt index 24cf7102e0..52eba4f1e1 100644 --- a/bindings/C/CMakeLists.txt +++ b/bindings/C/CMakeLists.txt @@ -46,7 +46,7 @@ set_target_properties( adios2_c ${maybe_adios2_c_mpi} PROPERTIES VERSION ${ADIOS2_LIBRARY_VERSION} - SOVERSION ${ADIOS2_VERSION_MAJOR} + SOVERSION ${ADIOS2_LIBRARY_SOVERSION} ) install(TARGETS adios2_c ${maybe_adios2_c_mpi} EXPORT adios2CExports diff --git a/bindings/C/adios2/c/adios2_c_engine.cpp b/bindings/C/adios2/c/adios2_c_engine.cpp index c933c96a41..46eb42422b 100644 --- a/bindings/C/adios2/c/adios2_c_engine.cpp +++ b/bindings/C/adios2/c/adios2_c_engine.cpp @@ -199,6 +199,27 @@ adios2_error adios2_engine_openmode(adios2_mode *mode, } } +adios2_error adios2_engine_get_metadata(adios2_engine *engine, char **md, + size_t *size) +{ + try + { + adios2::helper::CheckForNullptr( + engine, "for const adios2_engine, in call to adios2_get_metadata"); + + adios2::core::Engine *engineCpp = + reinterpret_cast(engine); + + engineCpp->GetMetadata(md, size); + return adios2_error_none; + } + catch (...) + { + return static_cast( + adios2::helper::ExceptionToError("adios2_get_metadata")); + } +} + adios2_error adios2_begin_step(adios2_engine *engine, const adios2_step_mode mode, const float timeout_seconds, diff --git a/bindings/C/adios2/c/adios2_c_engine.h b/bindings/C/adios2/c/adios2_c_engine.h index 984591dc7f..e4b018f326 100644 --- a/bindings/C/adios2/c/adios2_c_engine.h +++ b/bindings/C/adios2/c/adios2_c_engine.h @@ -56,6 +56,15 @@ adios2_error adios2_engine_get_type(char *type, size_t *size, adios2_error adios2_engine_openmode(adios2_mode *mode, const adios2_engine *engine); +/** Serialize all metadata right after engine is created, which can be + * delivered to other processes to open the same file for reading without + * opening and reading in metadata again. + * @return metadata (pointer to allocated memory) and size of metadata + * the pointer must be deallocated by user using free() + */ +adios2_error adios2_engine_get_metadata(adios2_engine *engine, char **md, + size_t *size); + /** * @brief Begin a logical adios2 step stream * Check each engine documentation for MPI collective/non-collective diff --git a/bindings/C/adios2/c/adios2_c_io.cpp b/bindings/C/adios2/c/adios2_c_io.cpp index 839f6a3a86..05aa5c5d88 100644 --- a/bindings/C/adios2/c/adios2_c_io.cpp +++ b/bindings/C/adios2/c/adios2_c_io.cpp @@ -947,6 +947,24 @@ adios2_engine *adios2_open(adios2_io *io, const char *name, return engine; } +adios2_engine *adios2_open_with_metadata(adios2_io *io, const char *name, + const char *md, const size_t mdsize) +{ + adios2_engine *engine = nullptr; + try + { + adios2::helper::CheckForNullptr( + io, "for adios2_io, in call to adios2_open_with_metadata"); + engine = reinterpret_cast( + &reinterpret_cast(io)->Open(name, md, mdsize)); + } + catch (...) + { + adios2::helper::ExceptionToError("adios2_open_with_metadata"); + } + return engine; +} + adios2_error adios2_flush_all_engines(adios2_io *io) { try diff --git a/bindings/C/adios2/c/adios2_c_io.h b/bindings/C/adios2/c/adios2_c_io.h index 271c2d87c6..18e2f0df22 100644 --- a/bindings/C/adios2/c/adios2_c_io.h +++ b/bindings/C/adios2/c/adios2_c_io.h @@ -325,6 +325,20 @@ adios2_error adios2_remove_all_attributes(adios2_io *io); adios2_engine *adios2_open(adios2_io *io, const char *name, const adios2_mode mode); +/** + * Open an Engine to start heavy-weight input/output operations. + * This function is for opening a file (not stream) with ReadRandomAccess mode + * and supplying the metadata already in memory. The metadata should be + * retrieved by another program calling adios2_engine_get_metadata() after + * opening the file. + * @param io engine owner + * @param name unique engine identifier + * @param md file metadata residing in memory + * @return success: handler, failure: NULL + */ +adios2_engine *adios2_open_with_metadata(adios2_io *io, const char *name, + const char *md, const size_t mdsize); + #if ADIOS2_USE_MPI /** * Open an Engine to start heavy-weight input/output operations. diff --git a/bindings/CXX11/CMakeLists.txt b/bindings/CXX11/CMakeLists.txt index 8eef3b6a54..2be90fb64e 100644 --- a/bindings/CXX11/CMakeLists.txt +++ b/bindings/CXX11/CMakeLists.txt @@ -58,7 +58,7 @@ set_target_properties( adios2_cxx11 ${maybe_adios2_cxx11_mpi} PROPERTIES VERSION ${ADIOS2_LIBRARY_VERSION} - SOVERSION ${ADIOS2_VERSION_MAJOR} + SOVERSION ${ADIOS2_LIBRARY_SOVERSION} ) install(TARGETS adios2_cxx11 ${maybe_adios2_cxx11_mpi} EXPORT adios2CXX11Exports diff --git a/bindings/CXX11/adios2/cxx11/Engine.cpp b/bindings/CXX11/adios2/cxx11/Engine.cpp index 748af4e041..e3f3eb690e 100644 --- a/bindings/CXX11/adios2/cxx11/Engine.cpp +++ b/bindings/CXX11/adios2/cxx11/Engine.cpp @@ -54,6 +54,12 @@ Mode Engine::OpenMode() const return m_Engine->OpenMode(); } +void Engine::GetMetadata(char **md, size_t *size) const +{ + helper::CheckForNullptr(m_Engine, "in call to Engine::GetMetadata"); + m_Engine->GetMetadata(md, size); +} + StepStatus Engine::BeginStep() { helper::CheckForNullptr(m_Engine, "in call to Engine::BeginStep"); diff --git a/bindings/CXX11/adios2/cxx11/Engine.h b/bindings/CXX11/adios2/cxx11/Engine.h index 1043315e91..24b173d3d2 100644 --- a/bindings/CXX11/adios2/cxx11/Engine.h +++ b/bindings/CXX11/adios2/cxx11/Engine.h @@ -75,6 +75,14 @@ class Engine */ Mode OpenMode() const; + /** Serialize all metadata right after engine is created, which can be + * delivered to other processes to open the same file for reading without + * opening and reading in metadata again. + * @return metadata (pointer to allocated memory) and size of metadata + * the pointer must be deallocated by user using free() + */ + void GetMetadata(char **md, size_t *size) const; + /** * Begin a logical adios2 step, overloaded version with timeoutSeconds = 0 * and mode = Read diff --git a/bindings/CXX11/adios2/cxx11/IO.cpp b/bindings/CXX11/adios2/cxx11/IO.cpp index e1eee598b6..42bad31918 100644 --- a/bindings/CXX11/adios2/cxx11/IO.cpp +++ b/bindings/CXX11/adios2/cxx11/IO.cpp @@ -109,6 +109,14 @@ Engine IO::Open(const std::string &name, const Mode mode) "for engine " + name + ", in call to IO::Open"); return Engine(&m_IO->Open(name, mode)); } + +Engine IO::Open(const std::string &name, const char *md, const size_t mdsize) +{ + helper::CheckForNullptr(m_IO, + "for engine " + name + ", in call to IO::Open"); + return Engine(&m_IO->Open(name, md, mdsize)); +} + Group IO::InquireGroup(char delimiter) { return Group(&m_IO->CreateGroup(delimiter)); diff --git a/bindings/CXX11/adios2/cxx11/IO.h b/bindings/CXX11/adios2/cxx11/IO.h index cb8a44a2e9..16d162c43b 100644 --- a/bindings/CXX11/adios2/cxx11/IO.h +++ b/bindings/CXX11/adios2/cxx11/IO.h @@ -307,6 +307,21 @@ class IO Engine Open(const std::string &name, const Mode mode, MPI_Comm comm); #endif + /** + * Overloaded version that is specifically for a serial program + * opening a file (not stream) with ReadRandomAccess mode and + * supplying the metadata already in memory. The metadata + * should be retrieved by another program calling engine.GetMetadata() + * after opening the file. + * @param name unique engine identifier within IO object + * (file name in case of File transports) + * @param md file metadata residing in memory + * @return a reference to a derived object of the Engine class + * @exception std::invalid_argument if Engine with unique name is already + * created with another Open + */ + Engine Open(const std::string &name, const char *md, const size_t mdsize); + /** Flushes all engines created with this IO with the Open function */ void FlushAll(); diff --git a/bindings/CXX11/adios2/cxx11/KokkosView.h b/bindings/CXX11/adios2/cxx11/KokkosView.h index 70aea00e8d..f092e51c9f 100644 --- a/bindings/CXX11/adios2/cxx11/KokkosView.h +++ b/bindings/CXX11/adios2/cxx11/KokkosView.h @@ -9,36 +9,17 @@ namespace detail { template -struct memspace_kokkos_to_adios2; - -template <> -struct memspace_kokkos_to_adios2 -{ - static constexpr adios2::MemorySpace value = adios2::MemorySpace::Host; -}; - -#if defined(KOKKOS_ENABLE_CUDA) && defined(ADIOS2_HAVE_GPU_SUPPORT) - -template <> -struct memspace_kokkos_to_adios2 +struct memspace_kokkos_to_adios2 { static constexpr adios2::MemorySpace value = adios2::MemorySpace::GPU; }; template <> -struct memspace_kokkos_to_adios2 -{ - static constexpr adios2::MemorySpace value = adios2::MemorySpace::GPU; -}; - -template <> -struct memspace_kokkos_to_adios2 +struct memspace_kokkos_to_adios2 { - static constexpr adios2::MemorySpace value = adios2::MemorySpace::GPU; + static constexpr adios2::MemorySpace value = adios2::MemorySpace::Host; }; -#endif - } // namespace detail template diff --git a/bindings/Fortran/CMakeLists.txt b/bindings/Fortran/CMakeLists.txt index a9fd9b825f..2e0cb2c1b3 100644 --- a/bindings/Fortran/CMakeLists.txt +++ b/bindings/Fortran/CMakeLists.txt @@ -78,7 +78,7 @@ endif() # Set library version information set_target_properties(adios2_fortran PROPERTIES VERSION ${ADIOS2_LIBRARY_VERSION} - SOVERSION ${ADIOS2_VERSION_MAJOR} + SOVERSION ${ADIOS2_LIBRARY_SOVERSION} ) add_library(adios2::fortran ALIAS adios2_fortran) @@ -109,7 +109,7 @@ if(ADIOS2_HAVE_MPI) target_compile_definitions(adios2_fortran_mpi PRIVATE "$<$:ADIOS2_HAVE_FORTRAN_SUBMODULES;ADIOS2_HAVE_MPI_F>") set_target_properties(adios2_fortran_mpi PROPERTIES VERSION ${ADIOS2_LIBRARY_VERSION} - SOVERSION ${ADIOS2_VERSION_MAJOR} + SOVERSION ${ADIOS2_LIBRARY_SOVERSION} ) set_property(TARGET adios2_fortran_mpi PROPERTY OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}_fortran_mpi) else() diff --git a/cmake/ADIOSFunctions.cmake b/cmake/ADIOSFunctions.cmake index 6e5332cc59..36257e34d6 100644 --- a/cmake/ADIOSFunctions.cmake +++ b/cmake/ADIOSFunctions.cmake @@ -38,6 +38,9 @@ function(setup_version BASE) endif() set(ADIOS2_LIBRARY_VERSION ${BASE} PARENT_SCOPE) + + string(REGEX MATCH "^([0-9]+\.[0-9]+)" ignore ${BASE}) + set(ADIOS2_LIBRARY_SOVERSION ${CMAKE_MATCH_1} PARENT_SCOPE) endfunction() function(adios_option name description default) @@ -224,3 +227,19 @@ program main end program ]] ${var} SRC_EXT F90) endmacro() + +# Set VERSION/SOVERSION of every shared library target in the given directory +# to be the same as the ADIOS VERSION/SOVERSION. This is important for the +# third-party libraries bundled with ADIOS2. +function(setup_libversion_dir dir) + get_directory_property(DIR_TARGETS DIRECTORY "${dir}" BUILDSYSTEM_TARGETS) + foreach(target ${DIR_TARGETS}) + get_target_property(type ${target} TYPE) + if (${type} STREQUAL "SHARED_LIBRARY") + set_target_properties(${target} PROPERTIES + VERSION ${ADIOS2_LIBRARY_VERSION} + SOVERSION ${ADIOS2_LIBRARY_SOVERSION} + ) + endif() + endforeach() +endfunction() diff --git a/cmake/DetectOptions.cmake b/cmake/DetectOptions.cmake index 9876538043..b3c271c647 100644 --- a/cmake/DetectOptions.cmake +++ b/cmake/DetectOptions.cmake @@ -177,9 +177,9 @@ endif() # Kokkos if(ADIOS2_USE_Kokkos) if(ADIOS2_USE_Kokkos STREQUAL AUTO) - find_package(Kokkos 3.7...<4.0 QUIET) + find_package(Kokkos 3.7 QUIET) else() - find_package(Kokkos 3.7...<4.0 REQUIRED) + find_package(Kokkos 3.7 REQUIRED) endif() if(Kokkos_FOUND) set(ADIOS2_HAVE_Kokkos TRUE) @@ -192,6 +192,9 @@ if(ADIOS2_USE_Kokkos) set(ADIOS2_HAVE_Kokkos_HIP TRUE) enable_language(HIP) endif() + if(Kokkos_ENABLE_SYCL) + set(ADIOS2_HAVE_Kokkos_SYCL TRUE) + endif() set(ADIOS2_HAVE_GPU_Support TRUE) endif() endif() diff --git a/cmake/install/post/generate-adios2-config.sh.in b/cmake/install/post/generate-adios2-config.sh.in index 18ce48ca73..4603420ffd 100755 --- a/cmake/install/post/generate-adios2-config.sh.in +++ b/cmake/install/post/generate-adios2-config.sh.in @@ -86,7 +86,10 @@ then export FC="@MPI_Fortran_COMPILER@" fi -export adios2_DIR="${PREFIX}/@CMAKE_INSTALL_CMAKEDIR@" +if [ -z "$adios2_DIR" ] +then + export adios2_DIR="${PREFIX}/@CMAKE_INSTALL_CMAKEDIR@" +fi # Sometimes package managers build in root jails if [ ! -d "${adios2_DIR}" ] diff --git a/docs/requirements.txt b/docs/requirements.txt index 542cffd5aa..4e74cf0aab 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -25,7 +25,7 @@ Pygments==2.14.0 pyOpenSSL==23.0.0 PySocks==1.7.1 pytz==2022.7 -requests==2.28.1 +requests==2.31.0 setuptools==65.6.3 snowballstemmer==2.2.0 Sphinx==4.5.0 diff --git a/docs/user_guide/source/advanced/gpu_aware.rst b/docs/user_guide/source/advanced/gpu_aware.rst index de25bf1f08..ee4c600180 100644 --- a/docs/user_guide/source/advanced/gpu_aware.rst +++ b/docs/user_guide/source/advanced/gpu_aware.rst @@ -5,7 +5,7 @@ The ``Put`` and ``Get`` functions in the BP4 and BP5 engines can receive user buffers allocated on the host or the device in both Sync and Deferred modes. .. note:: - Currently only CUDA and HIP allocated buffers are supported for device data. + CUDA, HIP and SYCL allocated buffers are supported for device data. If ADIOS2 is built without GPU support, only buffers allocated on the host are supported. If ADIOS2 is built with any GPU support, by default, the library will automatically detect where does the buffer memory physically resides. @@ -37,7 +37,7 @@ When building ADIOS2 with CUDA enabled, the user is responsible with setting the Building with Kokkos enabled -------------------------- -The Kokkos library can be used to enable GPU within ADIOS2. Based on how Kokkos is build, either the CUDA or HIP backend will be enabled. Building with Kokkos requires ``-DADIOS2_USE_Kokkos=ON``. The user is responsible to set the ``CMAKE_CUDA_ARCHITECTURES`` to the same architecture used when configuring the Kokkos library it links against. +The Kokkos library can be used to enable GPU within ADIOS2. Based on how Kokkos is build, either the CUDA, HIP or SYCL backend will be enabled. Building with Kokkos requires ``-DADIOS2_USE_Kokkos=ON``. The ``CMAKE_CUDA_ARCHITECTURES`` is set automanically to point to the same architecture used when configuring the Kokkos library. .. note:: Kokkos version >= 3.7 is required to enable the GPU backend in ADIOS2 @@ -71,7 +71,7 @@ If the ``SetMemorySpace`` function is used, the ADIOS2 library will not detect a .. code-block:: c++ - variable.SetMemorySpace(adios2::MemorySpace::CUDA); + data.SetMemorySpace(adios2::MemorySpace::GPU); for (size_t step = 0; step < nSteps; ++step) { bpWriter.BeginStep(); @@ -79,10 +79,14 @@ If the ``SetMemorySpace`` function is used, the ADIOS2 library will not detect a bpWriter.EndStep(); } -Underneath, ADIOS2 uses the backend used at build time to transfer the data. If ADIOS2 was build with CUDA, only CUDA buffers can be provided. If ADIOS2 was build with Kokkos (with CUDA enabled) only CUDA buffers can be provided. If ADIOS2 was build with Kokkos (with HIP enabled) only HIP buffers can be provided. +Underneath, ADIOS2 relies on the backend used at build time to transfer the data. If ADIOS2 was build with CUDA, only CUDA buffers can be provided. If ADIOS2 was build with Kokkos (with CUDA enabled) only CUDA buffers can be provided. If ADIOS2 was build with Kokkos (with HIP enabled) only HIP buffers can be provided. + +.. note:: + The SYCL backend in Kokkos can be used to run on Nvida, AMD and Intel GPUs -Using Kokkos buffers --------------------------- + +Kokkos applications +-------------------- ADIOS2 supports GPU buffers provided in the form of ``Kokkos::View`` directly in the Put/Get calls. The memory space can be automatically detected or provided by the user, in the same way as in the CUDA example. @@ -92,3 +96,13 @@ ADIOS2 supports GPU buffers provided in the form of ``Kokkos::View`` directly in bpWriter.Put(data, gpuSimData); If the CUDA backend is being used (and not Kokkos) to enable GPU support in ADIOS2, Kokkos applications can still directly pass ``Kokkos::View`` as long as the correct external header is included: ``#include ``. + +*************** +Build scripts +*************** + +The `scripts/build_scripts` folder contains scripts for building ADIOS2 with CUDA or Kokkos backends for several DOE system: Summit (OLCF Nvidia), Crusher (OLCFi AMD), Perlmutter (NERSC Nvidia), Polaris (ALCF Nvidia). + +.. note:: + Perlmutter requires Kokkos >= 4.0 + diff --git a/docs/user_guide/source/introduction/whatsnew.rst b/docs/user_guide/source/introduction/whatsnew.rst index af8aae7c08..39cefb7509 100644 --- a/docs/user_guide/source/introduction/whatsnew.rst +++ b/docs/user_guide/source/introduction/whatsnew.rst @@ -1,26 +1,34 @@ -################## -What's new in 2.8? -################## +================== +What's new in 2.9? +================== -Important changes to the API +Summary +======= - * **adios2::Mode::ReadRandomAccess** mode is introduced for reading files with access to all steps. - BeginStep/EndStep calls are *NOT allowed*. SetStepSelection() can be used to access specific steps in the file. - * **adios2::Mode::Read** mode now requires using BeginStep/EndStep loop to access steps serially one by one. Variable inquiry - fails outside BeginStep/EndStep sections. You need to modify your Open() statement to use the random-access mode if your - code wants to access all steps in any order in an existing file. - * **adios2::ADIOS::EnterComputationBlock()**, **adios2::ADIOS::ExitComputationBlock()** are hints to ADIOS that a process is in a computing (i.e. non-communicating) phase. BP5 asynchronous I/O operations can schedule writing during such phases to avoid interfering with the application's own communication. - * GPU-aware I/O supports passing device-memory data pointers to the ADIOS2 `Put()/Get()` functions, and ADIOS2 will automatically download/upload data from/to the device during I/O. Alternatively, an extra member function of the Variable class, **SetMemorySpace(const adios2::MemorySpace mem)** can explicitly tell ADIOS2 whether the pointer points to device memory or host memory. +This is a major release with new features and lots of bug fixes. -New features +General +------- - * **BP5** data format and engine. This new engine optimizes for many variables and many steps at large scale. - It is also more memory efficient than previous engines, see :ref:`BP5`. - * **Plugin** architecture to support external *engines* and *operators* outside the ADIOS2 installation, see :ref:`Plugins` - * **GPU-Aware I/O** for reading/writing data to/from device memory, using CUDA (NVidia GPUs only), see :ref:`GPU-aware I/O` +- GPU-Aware I/O enabled by using Kokkos. Device pointers can be passed to Put()/Get() calls directly. Kokkos 3.7.x required for this release. Works with CUDA, HIP and Kokkos applications. https://adios2.readthedocs.io/en/latest/advanced/gpu_aware.html#gpu-aware-i-o +- GPU-compression. MGARD and ZFP operators can compress data on GPU if they are built for GPU. MGARD operator can be fed with host/device pointers and will move data automaticaly. ZFP operator requires matching data and compressor location. +- Joined Array concept (besides Global Array and Local Array), which lets writers dump Local Arrays (no offsets no global shape) that are put together into a Global Array by the reader. One dimension of the arrays is selected for this join operation, while other dimensions must be the same for all writers. https://adios2.readthedocs.io/en/latest/components/components.html?highlight=Joined#shapes -Other changes +File I/O +-------- - * SST scales better for large N-to-1 staging, by managing the limits of outstanding remote direct memory access requests. - Of course one still introduces a literal bottleneck with such a pattern into an in situ workflow. +- Default File engine is now BP5. If for some reason this causes problems, manually specify using "BP4" for your application. +- BP5 engine supports multithreaded reading to accelerate read performance for low-core counts. +- BP5 Two level metadata aggregation and reduction reduced memory impact of collecting metadata and therefore is more scalable in terms of numbers of variables and writers than BP4. +- Uses Blosc-2 instead of Blosc for lossless compression. The new compression operator is backward compatible with old files compressed with blosc. The name of the operator remains "blosc". +Staging +------- + +- UCX dataplane added for SST staging engine to support networks under the UCX consortium +- MPI dataplane added for SST staging engine. It relies on MPI intercommunicators to connect multiple independent MPI applications for staging purposes. Applications must enable multithreaded MPI for this dataplane. + +Experimental features +--------------------- + +- Preliminary support for data structs. A struct can have single variables of basic types, and 1D fixed size arrays of basic types. Supported by BP5, SST and SSC engines. diff --git a/examples/useCases/CMakeLists.txt b/examples/useCases/CMakeLists.txt index f5de8aeacf..645d0a7954 100644 --- a/examples/useCases/CMakeLists.txt +++ b/examples/useCases/CMakeLists.txt @@ -4,4 +4,5 @@ #------------------------------------------------------------------------------# add_subdirectory(insituGlobalArrays) +add_subdirectory(ensembleRead) diff --git a/examples/useCases/ensembleRead/CMakeLists.txt b/examples/useCases/ensembleRead/CMakeLists.txt new file mode 100644 index 0000000000..35d25d482d --- /dev/null +++ b/examples/useCases/ensembleRead/CMakeLists.txt @@ -0,0 +1,9 @@ +#------------------------------------------------------------------------------# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +#------------------------------------------------------------------------------# + +if(ADIOS2_HAVE_MPI) + add_executable(ensembleRead ensembleRead.cpp) + target_link_libraries(ensembleRead adios2::cxx11_mpi MPI::MPI_C) +endif() diff --git a/examples/useCases/ensembleRead/ensembleRead.cpp b/examples/useCases/ensembleRead/ensembleRead.cpp new file mode 100644 index 0000000000..6fdc66afd1 --- /dev/null +++ b/examples/useCases/ensembleRead/ensembleRead.cpp @@ -0,0 +1,237 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * A Use Case for In Situ visulization frameworks (Conduit, SENSEI) + * + * Read in the variables that the Writer wrote. + * Every process should read only what the corresponding Writer wrote + * This is an N to N case + * + * Created on: Jul 11, 2017 + * Author: pnorbert + */ + +#include // std::transform +#include +#include +#include +#include +#include +#include //std::accumulate +#include // sleep_for +#include + +#include + +#include + +typedef std::chrono::duration Seconds; +typedef std::chrono::time_point< + std::chrono::steady_clock, + std::chrono::duration> + TimePoint; + +inline TimePoint Now() { return std::chrono::steady_clock::now(); } + +struct VarInfo +{ + std::string varName; + std::string type; + adios2::Dims shape; + adios2::ShapeID shapeID; + size_t nSteps; + std::vector data; + VarInfo(const std::string &name, const std::string &type, + const adios2::Dims shape, const adios2::ShapeID shapeID, + const size_t nsteps) + : varName(name), type(type), shape(shape), shapeID(shapeID), + nSteps(nsteps){}; +}; + +std::string DimsToString(adios2::Dims &dims) +{ + std::string s = ""; + for (size_t i = 0; i < dims.size(); i++) + { + if (i > 0) + { + s += "x"; + } + s += std::to_string(dims[i]); + } + s += ""; + return s; +} + +size_t GetTotalSize(adios2::Dims &dimensions, size_t elementSize = 1) +{ + return std::accumulate(dimensions.begin(), dimensions.end(), elementSize, + std::multiplies()); +} + +template +void ReadVariable(int rank, const std::string &name, const std::string &type, + adios2::Engine &reader, adios2::IO &io, + std::vector &varinfos) +{ + adios2::Variable variable = io.InquireVariable(name); + varinfos.push_back( + VarInfo(name, type, variable.Shape(), variable.ShapeID(), 1)); + auto vit = varinfos.rbegin(); + vit->nSteps = variable.Steps(); + if (vit->shapeID == adios2::ShapeID::GlobalArray) + { + size_t n = vit->nSteps * GetTotalSize(vit->shape, sizeof(T)); + vit->data.resize(n); + adios2::Dims start(vit->shape.size()); + variable.SetSelection({start, vit->shape}); + variable.SetStepSelection({0, vit->nSteps}); + T *dataptr = reinterpret_cast(vit->data.data()); + reader.Get(variable, dataptr); + } + else if (vit->shapeID == adios2::ShapeID::GlobalValue) + { + size_t n = vit->nSteps * sizeof(T); + vit->data.resize(n); + variable.SetStepSelection({0, vit->nSteps}); + T *dataptr = reinterpret_cast(vit->data.data()); + reader.Get(variable, dataptr); + } +} + +std::vector ReadFileContent(int rank, adios2::Engine &reader, + adios2::IO &io) +{ + std::map varNameList = io.AvailableVariables(); + std::vector varinfos; + for (auto &var : varNameList) + { + const std::string &name(var.first); + auto it = var.second.find("Type"); + const std::string &type = it->second; + if (type == "struct") + { + // not supported + } +#define declare_template_instantiation(T) \ + else if (type == adios2::GetType()) \ + { \ + ReadVariable(rank, name, type, reader, io, varinfos); \ + } + ADIOS2_FOREACH_STDTYPE_1ARG(declare_template_instantiation) +#undef declare_template_instantiation + } + + reader.PerformGets(); + return varinfos; +} + +void ProcessFile(int rank, adios2::Engine &reader, adios2::IO &io, + Seconds opentime) +{ + auto now = Now(); + std::vector varinfos = ReadFileContent(rank, reader, io); + Seconds readtime = Now() - now; + + std::cout << "File info on rank " << rank << ":" << std::endl; + std::cout << " Open time: " << opentime.count() << "s" << std::endl; + std::cout << " Read time: " << readtime.count() << "s" << std::endl; + std::cout << " Steps in file: " << reader.Steps() << std::endl; + std::cout << " Total number of variables = " << varinfos.size() + << std::endl; + for (auto &vi : varinfos) + { + std::cout << " Name: " << vi.varName + << " dimensions = " << DimsToString(vi.shape) + << " steps = " << vi.nSteps << " size = " << vi.data.size() + << " bytes" << std::endl; + } +} + +int main(int argc, char *argv[]) +{ + if (argc < 2) + { + std::cout << "Usage: " << argv[0] << " BP-file" << std::endl; + return -1; + } + std::string fname = argv[1]; + + int rank = 0, nproc = 1; + MPI_Init(&argc, &argv); + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nproc); + + /* Each process is acting as a serial program */ + adios2::ADIOS adios; // independent ADIOS object for each single process + + adios2::IO io = adios.DeclareIO("Input"); + char *fileMetadata; + size_t fileMetadataSize; + + if (!rank) + { + std::cout << "First process opens file " << fname << std::endl; + adios2::Engine reader; + auto now = Now(); + reader = io.Open(fname, adios2::Mode::ReadRandomAccess); + Seconds opentime = Now() - now; + reader.GetMetadata(&fileMetadata, &fileMetadataSize); + std::cout << "Serialized metadata size = " << fileMetadataSize + << std::endl; + ProcessFile(rank, reader, io, opentime); + reader.Close(); + std::cout << "\n===== End of first process file processing =====\n" + << std::endl; + } + + /* Send metadata to all processes via MPI + * (Note limitation to 2GB due MPI int) + */ + MPI_Bcast(&fileMetadataSize, 1, MPI_INT64_T, 0, MPI_COMM_WORLD); + if (fileMetadataSize > (size_t)std::numeric_limits::max()) + { + if (!rank) + std::cout << "ERROR: metadata size is >2GB, not supported by " + "MPI_BCast" + << std::endl; + MPI_Abort(MPI_COMM_WORLD, 1); + } + + if (rank) + { + fileMetadata = (char *)malloc(fileMetadataSize); + } + int mdsize = (int)fileMetadataSize; + MPI_Bcast(fileMetadata, mdsize, MPI_CHAR, 0, MPI_COMM_WORLD); + + /* "Open" data by passing metadata to the adios engine */ + auto now = Now(); + adios2::Engine reader = io.Open(fname, fileMetadata, fileMetadataSize); + Seconds opentime = Now() - now; + free(fileMetadata); + + /* Process file in a sequentialized order only for pretty printing */ + MPI_Status status; + int token = 0; + if (rank > 0) + { + MPI_Recv(&token, 1, MPI_INT, rank - 1, 0, MPI_COMM_WORLD, &status); + } + + ProcessFile(rank, reader, io, opentime); + + if (rank < nproc - 1) + { + std::chrono::milliseconds timespan(100); + std::this_thread::sleep_for(timespan); + MPI_Send(&token, 1, MPI_INT, rank + 1, 0, MPI_COMM_WORLD); + } + + // Called once: indicate that we are done with this output for the run + reader.Close(); + + MPI_Finalize(); + return 0; +} diff --git a/flake8.cfg b/flake8.cfg index 5525546057..39b0361140 100644 --- a/flake8.cfg +++ b/flake8.cfg @@ -3,4 +3,4 @@ max-line-length = 80 max-complexity = 1000 format = pylint ignore = E302,F401,F403,F405,F999,W504 -exclude = thirdparty/ +exclude = thirdparty/,.gitlab/config/SpackCIBridge.py diff --git a/scripts/build_scripts/build-adios2-cuda-perlmutter.sh b/scripts/build_scripts/build-adios2-cuda-perlmutter.sh new file mode 100644 index 0000000000..4198f18fb1 --- /dev/null +++ b/scripts/build_scripts/build-adios2-cuda-perlmutter.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# shellcheck disable=SC2191 + +module load cudatoolkit/11.7 +module load gcc/11.2.0 +module load cmake/3.24.3 +module refresh + +######## User Configurations ######## +ADIOS2_HOME=$(pwd) +BUILD_DIR=${ADIOS2_HOME}/build-cuda-perlmutter +INSTALL_DIR=${ADIOS2_HOME}/install-cuda-perlmutter + +num_build_procs=4 + +######## ADIOS2 ######## +mkdir -p "${BUILD_DIR}" +rm -f "${BUILD_DIR}/CMakeCache.txt" +rm -rf "${BUILD_DIR}/CMakeFiles" + +ARGS_ADIOS=( + -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + -D BUILD_TESTING=OFF + -D ADIOS2_BUILD_EXAMPLES=OFF + -D CMAKE_CXX_COMPILER=g++ + -D CMAKE_C_COMPILER=gcc + + -D ADIOS2_USE_CUDA=ON + + -D CMAKE_POSITION_INDEPENDENT_CODE=TRUE + -D BUILD_SHARED_LIBS=ON + -D ADIOS2_USE_Fortran=OFF +) +cmake "${ARGS_ADIOS[@]}" -S "${ADIOS2_HOME}" -B "${BUILD_DIR}" +cmake --build "${BUILD_DIR}" -j${num_build_procs} +cmake --install "${BUILD_DIR}" diff --git a/scripts/build_scripts/build-adios2-cuda-summit.sh b/scripts/build_scripts/build-adios2-cuda-summit.sh new file mode 100644 index 0000000000..37fafade7b --- /dev/null +++ b/scripts/build_scripts/build-adios2-cuda-summit.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# shellcheck disable=SC2191 + +module load gcc/10.2 +module load cuda/11.5 +module load cmake/3.23 +module refresh + +######## User Configurations ######## +ADIOS2_HOME=$(pwd) +BUILD_DIR=${ADIOS2_HOME}/build-cuda-summit +INSTALL_DIR=${ADIOS2_HOME}/install-cuda-summit + +num_build_procs=4 + +######## ADIOS2 ######## +mkdir -p "${BUILD_DIR}" +rm -f "${BUILD_DIR}/CMakeCache.txt" +rm -rf "${BUILD_DIR}/CMakeFiles" + +ARGS_ADIOS=( + -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + -D BUILD_TESTING=OFF + -D ADIOS2_BUILD_EXAMPLES=OFF + -D CMAKE_CXX_COMPILER=g++ + -D CMAKE_C_COMPILER=gcc + + -D ADIOS2_USE_CUDA=ON + + -D CMAKE_POSITION_INDEPENDENT_CODE=TRUE + -D BUILD_SHARED_LIBS=ON +) +cmake "${ARGS_ADIOS[@]}" -S "${ADIOS2_HOME}" -B "${BUILD_DIR}" +cmake --build "${BUILD_DIR}" -j${num_build_procs} +cmake --install "${BUILD_DIR}" diff --git a/scripts/build_scripts/build-adios2-kokkos-crusher.sh b/scripts/build_scripts/build-adios2-kokkos-crusher.sh new file mode 100644 index 0000000000..4dce4fc682 --- /dev/null +++ b/scripts/build_scripts/build-adios2-kokkos-crusher.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# shellcheck disable=SC2191 + +module load rocm/5.4.0 +module load gcc/11.2.0 +module load cmake/3.23.2 +module load craype-accel-amd-gfx90a + +######## User Configurations ######## +Kokkos_HOME=$HOME/kokkos/kokkos +ADIOS2_HOME=$(pwd) +BUILD_DIR=${ADIOS2_HOME}/build-kokkos-crusher +INSTALL_DIR=${ADIOS2_HOME}/install-kokkos-crusher + +num_build_procs=4 + +######## Kokkos ######## +mkdir -p "${BUILD_DIR}/kokkos" +rm -f "${BUILD_DIR}/kokkos/CMakeCache.txt" +rm -rf "${BUILD_DIR}/kokkos/CMakeFiles" + +ARGS=( + -D CMAKE_BUILD_TYPE=RelWithDebInfo + -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + -D CMAKE_CXX_COMPILER=hipcc + + -D Kokkos_ENABLE_SERIAL=ON + -D Kokkos_ARCH_ZEN3=ON + -D Kokkos_ENABLE_HIP=ON + -D Kokkos_ARCH_VEGA90A=ON + + -D CMAKE_CXX_STANDARD=17 + -D CMAKE_CXX_EXTENSIONS=OFF + -D CMAKE_POSITION_INDEPENDENT_CODE=TRUE + -D BUILD_SHARED_LIBS=ON +) +cmake "${ARGS[@]}" -S "${Kokkos_HOME}" -B "${BUILD_DIR}/kokkos" +cmake --build "${BUILD_DIR}/kokkos" -j${num_build_procs} +cmake --install "${BUILD_DIR}/kokkos" + +######## ADIOS2 ######## +mkdir -p "${BUILD_DIR}/adios2" +rm -f "${BUILD_DIR}/adios2/CMakeCache.txt" +rm -rf "${BUILD_DIR}/adios2/CMakeFiles" + +ARGS_ADIOS=( + -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + -D BUILD_TESTING=OFF + -D ADIOS2_BUILD_EXAMPLES=OFF + -D CMAKE_CXX_COMPILER=g++ + -D CMAKE_C_COMPILER=gcc + + -D ADIOS2_USE_Kokkos=ON + -D Kokkos_ROOT="${INSTALL_DIR}" + + -D CMAKE_POSITION_INDEPENDENT_CODE=TRUE + -D BUILD_SHARED_LIBS=ON +) +cmake "${ARGS_ADIOS[@]}" -S "${ADIOS2_HOME}" -B "${BUILD_DIR}"/adios2 +cmake --build "${BUILD_DIR}/adios2" -j${num_build_procs} +cmake --install "${BUILD_DIR}/adios2" diff --git a/scripts/build_scripts/build-adios2-kokkos-perlmutter.sh b/scripts/build_scripts/build-adios2-kokkos-perlmutter.sh new file mode 100644 index 0000000000..1fc547a173 --- /dev/null +++ b/scripts/build_scripts/build-adios2-kokkos-perlmutter.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# shellcheck disable=SC2191 + +module load cudatoolkit/11.7 +module load gcc/11.2.0 +module load cmake/3.24.3 +module refresh + +######## User Configurations ######## +Kokkos_HOME=$HOME/kokkos/kokkos +ADIOS2_HOME=$(pwd) +BUILD_DIR=${ADIOS2_HOME}/build-kokkos-perlmutter +INSTALL_DIR=${ADIOS2_HOME}/install-kokkos-perlmutter + +num_build_procs=4 + +######## Kokkos ######## +mkdir -p "${BUILD_DIR}/kokkos" +rm -f "${BUILD_DIR}/kokkos/CMakeCache.txt" +rm -rf "${BUILD_DIR}/kokkos/CMakeFiles" + +ARGS=( + -D CMAKE_BUILD_TYPE=RelWithDebInfo + -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + -D CMAKE_CXX_COMPILER="${Kokkos_HOME}/bin/nvcc_wrapper" + + -D Kokkos_ENABLE_SERIAL=ON + -D Kokkos_ARCH_ZEN3=ON + -D Kokkos_ENABLE_CUDA=ON + -D Kokkos_ENABLE_CUDA_LAMBDA=ON # from Kokkos 4.0 this is not needed + -D Kokkos_ARCH_AMPERE80=ON + + -D CMAKE_CXX_STANDARD=17 + -D CMAKE_CXX_EXTENSIONS=OFF + -D CMAKE_POSITION_INDEPENDENT_CODE=TRUE + -D BUILD_SHARED_LIBS=ON +) +cmake "${ARGS[@]}" -S "${Kokkos_HOME}" -B "${BUILD_DIR}/kokkos" +cmake --build "${BUILD_DIR}/kokkos" -j${num_build_procs} +cmake --install "${BUILD_DIR}/kokkos" + +######## ADIOS2 ######## +mkdir -p "${BUILD_DIR}/adios2" +rm -f "${BUILD_DIR}/adios2/CMakeCache.txt" +rm -rf "${BUILD_DIR}/adios2/CMakeFiles" + +ARGS_ADIOS=( + -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + -D BUILD_TESTING=OFF + -D ADIOS2_BUILD_EXAMPLES=OFF + -D CMAKE_CXX_COMPILER=g++ + -D CMAKE_C_COMPILER=gcc + + -D ADIOS2_USE_Kokkos=ON + -D Kokkos_ROOT="${INSTALL_DIR}" + + -D CMAKE_POSITION_INDEPENDENT_CODE=TRUE + -D BUILD_SHARED_LIBS=ON + -D ADIOS2_USE_Fortran=OFF +) +cmake "${ARGS_ADIOS[@]}" -S "${ADIOS2_HOME}" -B "${BUILD_DIR}"/adios2 +cmake --build "${BUILD_DIR}/adios2" -j${num_build_procs} +cmake --install "${BUILD_DIR}/adios2" diff --git a/scripts/build_scripts/build-adios2-kokkos-summit.sh b/scripts/build_scripts/build-adios2-kokkos-summit.sh new file mode 100644 index 0000000000..84af193327 --- /dev/null +++ b/scripts/build_scripts/build-adios2-kokkos-summit.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# shellcheck disable=SC2191 + +module load gcc/10.2 +module load cuda/11.5 +module load cmake/3.23 +module refresh + +######## User Configurations ######## +Kokkos_HOME=$HOME/kokkos/kokkos +ADIOS2_HOME=$(pwd) +BUILD_DIR=${ADIOS2_HOME}/build-kokkos-summit +INSTALL_DIR=${ADIOS2_HOME}/install-kokkos-summit + +num_build_procs=4 + +######## Kokkos ######## +mkdir -p "${BUILD_DIR}/kokkos" +rm -f "${BUILD_DIR}/kokkos/CMakeCache.txt" +rm -rf "${BUILD_DIR}/kokkos/CMakeFiles" + +ARGS=( + -D CMAKE_BUILD_TYPE=RelWithDebInfo + -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + -D CMAKE_CXX_COMPILER="${Kokkos_HOME}/bin/nvcc_wrapper" + + -D Kokkos_ENABLE_SERIAL=ON + -D Kokkos_ARCH_POWER9=ON + -D Kokkos_ENABLE_CUDA=ON + -D Kokkos_ENABLE_CUDA_LAMBDA=ON # from Kokkos 4.0 this is not needed + -D Kokkos_ARCH_VOLTA70=ON + + -D CMAKE_CXX_STANDARD=17 + -D CMAKE_CXX_EXTENSIONS=OFF + -D CMAKE_POSITION_INDEPENDENT_CODE=TRUE + -D BUILD_SHARED_LIBS=ON +) +cmake "${ARGS[@]}" -S "${Kokkos_HOME}" -B "${BUILD_DIR}/kokkos" +cmake --build "${BUILD_DIR}/kokkos" -j${num_build_procs} +cmake --install "${BUILD_DIR}/kokkos" + +######## ADIOS2 ######## +mkdir -p "${BUILD_DIR}/adios2" +rm -f "${BUILD_DIR}/adios2/CMakeCache.txt" +rm -rf "${BUILD_DIR}/adios2/CMakeFiles" + +ARGS_ADIOS=( + -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + -D BUILD_TESTING=OFF + -D ADIOS2_BUILD_EXAMPLES=OFF + -D CMAKE_CXX_COMPILER=g++ + -D CMAKE_C_COMPILER=gcc + + -D ADIOS2_USE_Kokkos=ON + -D Kokkos_ROOT="${INSTALL_DIR}" + + -D CMAKE_POSITION_INDEPENDENT_CODE=TRUE + -D BUILD_SHARED_LIBS=ON +) +cmake "${ARGS_ADIOS[@]}" -S "${ADIOS2_HOME}" -B "${BUILD_DIR}"/adios2 +cmake --build "${BUILD_DIR}/adios2" -j${num_build_procs} +cmake --install "${BUILD_DIR}/adios2" diff --git a/scripts/build_scripts/build-adios2-sycl-polaris.sh b/scripts/build_scripts/build-adios2-sycl-polaris.sh new file mode 100644 index 0000000000..34487c98c0 --- /dev/null +++ b/scripts/build_scripts/build-adios2-sycl-polaris.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# shellcheck disable=SC2191 + +module load oneapi +module load cmake/3.23.2 +module refresh + +######## User Configurations ######## +Kokkos_HOME=$HOME/kokkos/kokkos +ADIOS2_HOME=$(pwd) +BUILD_DIR=${ADIOS2_HOME}/build-kokkos-polaris +INSTALL_DIR=${ADIOS2_HOME}/install-kokkos-polaris + +num_build_procs=4 + +######## Kokkos ######## +mkdir -p "${BUILD_DIR}/kokkos" +rm -f "${BUILD_DIR}/kokkos/CMakeCache.txt" +rm -rf "${BUILD_DIR}/kokkos/CMakeFiles" + +ARGS=( + -D CMAKE_BUILD_TYPE=RelWithDebInfo + -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + -D CMAKE_CXX_COMPILER=clang++ + + -D Kokkos_ENABLE_SERIAL=ON + -D Kokkos_ARCH_ZEN3=ON + -D Kokkos_ENABLE_SYCL=ON + -D Kokkos_ARCH_AMPERE80=ON + -D Kokkos_ENABLE_UNSUPPORTED_ARCHS=ON + + -D CMAKE_CXX_STANDARD=17 + -D CMAKE_CXX_EXTENSIONS=OFF + -D CMAKE_POSITION_INDEPENDENT_CODE=TRUE + -D BUILD_SHARED_LIBS=ON +) +cmake "${ARGS[@]}" -S "${Kokkos_HOME}" -B "${BUILD_DIR}/kokkos" +cmake --build "${BUILD_DIR}/kokkos" -j${num_build_procs} +cmake --install "${BUILD_DIR}/kokkos" + +######## ADIOS2 ######## +mkdir -p "${BUILD_DIR}/adios2" +rm -f "${BUILD_DIR}/adios2/CMakeCache.txt" +rm -rf "${BUILD_DIR}/adios2/CMakeFiles" + +ARGS_ADIOS=( + -D CMAKE_INSTALL_PREFIX="${INSTALL_DIR}" + -D BUILD_TESTING=OFF + #-D ADIOS2_BUILD_EXAMPLES=OFF + -D CMAKE_CXX_COMPILER=g++ + -D CMAKE_C_COMPILER=gcc + + -D ADIOS2_USE_SST=OFF + -D ADIOS2_USE_Kokkos=ON + -D Kokkos_ROOT="${INSTALL_DIR}" + + -D CMAKE_CXX_STANDARD=17 + -D CMAKE_CXX_EXTENSIONS=OFF + -D CMAKE_POSITION_INDEPENDENT_CODE=TRUE + -D BUILD_SHARED_LIBS=ON + -D ADIOS2_USE_Fortran=OFF +) +cmake "${ARGS_ADIOS[@]}" -S "${ADIOS2_HOME}" -B "${BUILD_DIR}"/adios2 +cmake --build "${BUILD_DIR}/adios2" -j${num_build_procs} +cmake --install "${BUILD_DIR}/adios2" diff --git a/scripts/ci/cmake-v2/ci-ascent-cuda.cmake b/scripts/ci/cmake-v2/ci-ascent-cuda.cmake new file mode 100644 index 0000000000..93febd6614 --- /dev/null +++ b/scripts/ci/cmake-v2/ci-ascent-cuda.cmake @@ -0,0 +1,27 @@ +# Client maintainer: vicente.bolea@kitware.com + +set(dashboard_cache " +ADIOS2_USE_BZip2:BOOL=OFF +ADIOS2_USE_CUDA:BOOL=ON +ADIOS2_USE_DataMan:BOOL=ON +ADIOS2_USE_Fortran:BOOL=ON +ADIOS2_USE_MPI:BOOL=OFF +ADIOS2_USE_PNG:BOOL=OFF +ADIOS2_USE_Python:BOOL=OFF +ADIOS2_USE_SST:BOOL=ON + +CMAKE_C_COMPILER_LAUNCHER=ccache +CMAKE_CXX_COMPILER_LAUNCHER=ccache +CMAKE_CUDA_COMPILER_LAUNCHER=ccache +CMAKE_DISABLE_FIND_PACKAGE_BISON=ON +CMAKE_DISABLE_FIND_PACKAGE_FLEX=ON +CMAKE_Fortran_FLAGS:STRING=-Wall +") + +set(CTEST_TEST_ARGS + PARALLEL_LEVEL 8 + EXCLUDE ".*/BPWRCUDA.ADIOS2BPCUDAWrong/.*BP4.Serial|.*/BPWRCUDA.ADIOS2BPCUDAMemSel/.*BP4.Serial" + ) +set(CTEST_CMAKE_GENERATOR "Ninja") +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/scripts/ci/cmake-v2/ci-ascent-kokkos-cuda.cmake b/scripts/ci/cmake-v2/ci-ascent-kokkos-cuda.cmake new file mode 100644 index 0000000000..b906bed2e6 --- /dev/null +++ b/scripts/ci/cmake-v2/ci-ascent-kokkos-cuda.cmake @@ -0,0 +1,32 @@ +# Client maintainer: vicente.bolea@kitware.com + +set(kokkos_install_path $ENV{Kokkos_DIR}) + +set(ENV{CC} gcc) +set(ENV{CXX} g++) +set(ENV{FC} gfortran) + +set(dashboard_cache " +ADIOS2_USE_BZip2:BOOL=OFF +ADIOS2_USE_DataMan:BOOL=ON +ADIOS2_USE_Fortran:BOOL=OFF +ADIOS2_USE_MPI:BOOL=OFF +ADIOS2_USE_PNG:BOOL=OFF +ADIOS2_USE_Python:BOOL=OFF +ADIOS2_USE_SST:BOOL=ON +ADIOS2_USE_Kokkos=ON + +CMAKE_C_COMPILER_LAUNCHER=ccache +CMAKE_CXX_COMPILER_LAUNCHER=ccache +CMAKE_CUDA_COMPILER_LAUNCHER=ccache +CMAKE_DISABLE_FIND_PACKAGE_BISON=ON +CMAKE_DISABLE_FIND_PACKAGE_FLEX=ON +") + +set(CTEST_TEST_ARGS + PARALLEL_LEVEL 8 + EXCLUDE ".*/BPWRCUDA.ADIOS2BPCUDAWrong/.*BP4.Serial|.*/BPWRCUDA.ADIOS2BPCUDAMemSel/.*BP4.Serial|Engine.Staging.TestThreads.*" + ) +set(CTEST_CMAKE_GENERATOR "Ninja") +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/scripts/ci/cmake-v2/ci-ascent-nvhpc.cmake b/scripts/ci/cmake-v2/ci-ascent-nvhpc.cmake new file mode 100644 index 0000000000..c6aa75d3b9 --- /dev/null +++ b/scripts/ci/cmake-v2/ci-ascent-nvhpc.cmake @@ -0,0 +1,30 @@ +# Client maintainer: vicente.bolea@kitware.com + +set(ENV{CC} nvc) +set(ENV{CXX} nvc++) +set(ENV{FC} nvfortran) + +set(dashboard_cache " +ADIOS2_USE_BZip2:BOOL=OFF +ADIOS2_USE_CUDA:BOOL=ON +ADIOS2_USE_DataMan:BOOL=ON +ADIOS2_USE_Fortran:BOOL=ON +ADIOS2_USE_MPI:BOOL=OFF +ADIOS2_USE_PNG:BOOL=OFF +ADIOS2_USE_Python:BOOL=OFF +ADIOS2_USE_SST:BOOL=ON + +CMAKE_C_COMPILER_LAUNCHER=ccache +CMAKE_CXX_COMPILER_LAUNCHER=ccache +CMAKE_DISABLE_FIND_PACKAGE_BISON=ON +CMAKE_DISABLE_FIND_PACKAGE_FLEX=ON +CMAKE_NINJA_FORCE_RESPONSE_FILE=OFF +") + +set(CTEST_TEST_ARGS + PARALLEL_LEVEL 8 + EXCLUDE ".*/BPWRCUDA.ADIOS2BPCUDAWrong/.*BP4.Serial|.*/BPWRCUDA.ADIOS2BPCUDAMemSel/.*BP4.Serial|Install.*Fortran" + ) +set(CTEST_CMAKE_GENERATOR "Unix Makefiles") +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/scripts/ci/cmake/ci-ascent-xl.cmake b/scripts/ci/cmake-v2/ci-ascent-xl.cmake similarity index 53% rename from scripts/ci/cmake/ci-ascent-xl.cmake rename to scripts/ci/cmake-v2/ci-ascent-xl.cmake index fec3729755..9f73bab0de 100644 --- a/scripts/ci/cmake/ci-ascent-xl.cmake +++ b/scripts/ci/cmake-v2/ci-ascent-xl.cmake @@ -1,15 +1,4 @@ -# Client maintainer: chuck.atkins@kitware.com - -find_package(EnvModules REQUIRED) - -env_module(purge) -env_module(load git) -env_module(load xl) -env_module(load hdf5) -env_module(load libfabric) -env_module(load python/3.7.0) -env_module(load zfp) -env_module(load zeromq) +# Client maintainer: vicente.bolea@kitware.com set(ENV{CC} xlc) set(ENV{CXX} xlc++) @@ -17,20 +6,25 @@ set(ENV{FC} xlf) set(dashboard_cache " ADIOS2_USE_BZip2:BOOL=OFF -ADIOS2_USE_Blosc:BOOL=OFF ADIOS2_USE_DataMan:BOOL=ON -ADIOS2_USE_Fortran:BOOL=ON -ADIOS2_USE_HDF5:BOOL=ON +ADIOS2_USE_Fortran:BOOL=OFF ADIOS2_USE_MPI:BOOL=OFF +ADIOS2_USE_PNG:BOOL=OFF ADIOS2_USE_Python:BOOL=OFF ADIOS2_USE_SST:BOOL=ON -ADIOS2_USE_SZ:BOOL=OFF -ADIOS2_USE_ZeroMQ:STRING=ON +ADIOS2_USE_ZeroMQ:STRING=OFF ADIOS2_USE_ZFP:BOOL=OFF +ADIOS2_USE_SZ:BOOL=OFF +ADIOS2_USE_Blosc:BOOL=OFF + +CMAKE_C_COMPILER_LAUNCHER=ccache +CMAKE_CXX_COMPILER_LAUNCHER=ccache +CMAKE_DISABLE_FIND_PACKAGE_BISON=ON +CMAKE_DISABLE_FIND_PACKAGE_FLEX=ON ") set(NCPUS 4) set(CTEST_TEST_ARGS PARALLEL_LEVEL 8) -set(CTEST_CMAKE_GENERATOR "Unix Makefiles") +set(CTEST_CMAKE_GENERATOR "Ninja") list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/scripts/ci/cmake-v2/ci-crusher-cray.cmake b/scripts/ci/cmake-v2/ci-crusher-cray.cmake new file mode 100644 index 0000000000..a148430830 --- /dev/null +++ b/scripts/ci/cmake-v2/ci-crusher-cray.cmake @@ -0,0 +1,32 @@ +# Client maintainer: vicente.bolea@kitware.com + +set(ENV{CC} craycc) +set(ENV{CXX} craycxx) + +set(dashboard_cache " +ADIOS2_USE_BZip2:BOOL=OFF +ADIOS2_USE_DataMan:BOOL=ON +ADIOS2_USE_Fortran:BOOL=OFF +ADIOS2_USE_MPI:BOOL=OFF +ADIOS2_USE_HDF5:BOOL=OFF +ADIOS2_USE_PNG:BOOL=OFF +ADIOS2_USE_Python:BOOL=OFF +ADIOS2_USE_SST:BOOL=ON +ADIOS2_USE_ZeroMQ:STRING=OFF +ADIOS2_USE_ZFP:BOOL=OFF +ADIOS2_USE_SZ:BOOL=OFF +ADIOS2_USE_Blosc:BOOL=OFF + +CMAKE_C_COMPILER_LAUNCHER=ccache +CMAKE_CXX_COMPILER_LAUNCHER=ccache +CMAKE_DISABLE_FIND_PACKAGE_BISON=ON +CMAKE_DISABLE_FIND_PACKAGE_FLEX=ON +") + +set(CTEST_TEST_ARGS + PARALLEL_LEVEL 8 + EXCLUDE "Engine.Staging.TestThreads.*" + ) +set(CTEST_CMAKE_GENERATOR "Ninja") +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/scripts/ci/cmake-v2/ci-crusher-kokkos-hip.cmake b/scripts/ci/cmake-v2/ci-crusher-kokkos-hip.cmake new file mode 100644 index 0000000000..afe52c92e6 --- /dev/null +++ b/scripts/ci/cmake-v2/ci-crusher-kokkos-hip.cmake @@ -0,0 +1,31 @@ +# Client maintainer: vicente.bolea@kitware.com + +set(ENV{CC} gcc) +set(ENV{CXX} g++) +set(ENV{FC} gfortran) + +set(dashboard_cache " +ADIOS2_USE_BZip2:BOOL=OFF +ADIOS2_USE_DataMan:BOOL=ON +ADIOS2_USE_Fortran:BOOL=OFF +ADIOS2_USE_MPI:BOOL=OFF +ADIOS2_USE_HDF5:BOOL=OFF +ADIOS2_USE_PNG:BOOL=OFF +ADIOS2_USE_Python:BOOL=OFF +ADIOS2_USE_SST:BOOL=ON +ADIOS2_USE_Kokkos=ON + +CMAKE_C_COMPILER_LAUNCHER=ccache +CMAKE_CXX_COMPILER_LAUNCHER=ccache +CMAKE_CUDA_COMPILER_LAUNCHER=ccache +CMAKE_DISABLE_FIND_PACKAGE_BISON=ON +CMAKE_DISABLE_FIND_PACKAGE_FLEX=ON +") + +set(CTEST_TEST_ARGS + PARALLEL_LEVEL 8 + EXCLUDE "Engine.Staging.TestThreads.*" + ) +set(CTEST_CMAKE_GENERATOR "Ninja") +list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") +include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/scripts/ci/cmake-v2/ci-power8-el7-xl-mpi.cmake b/scripts/ci/cmake-v2/ci-power8-el7-xl-mpi.cmake deleted file mode 100644 index 3f2bf9ad4a..0000000000 --- a/scripts/ci/cmake-v2/ci-power8-el7-xl-mpi.cmake +++ /dev/null @@ -1,36 +0,0 @@ -# Client maintainer: chuck.atkins@kitware.com - -set(ENV{CC} xlc) -set(ENV{CXX} xlc++) -set(ENV{FC} xlf) - -find_program(MPICC mpixlc) -find_program(MPICXX mpixlC) -find_program(MPIFC mpixlf) - -set(dashboard_cache " -ADIOS2_USE_BZip2:BOOL=OFF -ADIOS2_USE_Blosc:BOOL=OFF -ADIOS2_USE_DataMan:BOOL=OFF -ADIOS2_USE_Fortran:BOOL=ON -ADIOS2_USE_HDF5:BOOL=ON -ADIOS2_USE_MPI:BOOL=ON -ADIOS2_USE_Python:BOOL=OFF -ADIOS2_USE_SZ:BOOL=ON -ADIOS2_USE_ZeroMQ:STRING=OFF -ADIOS2_USE_ZFP:BOOL=ON - -MPI_C_COMPILER:FILEPATH=${MPICC} -MPI_CXX_COMPILER:FILEPATH=${MPICXX} -MPI_Fortran_COMPILER:FILEPATH=${MPIFC} -MPIEXEC_EXTRA_FLAGS:STRING=--allow-run-as-root --oversubscribe -tcp -") - -set(CTEST_TEST_ARGS - PARALLEL_LEVEL 1 - INCLUDE "^(Interface|Install)" -) -set(CTEST_TEST_TIMEOUT 300) -set(CTEST_CMAKE_GENERATOR "Unix Makefiles") -list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") -include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/scripts/ci/cmake-v2/ci-power8-el7-xl-serial.cmake b/scripts/ci/cmake-v2/ci-power8-el7-xl-serial.cmake index 69ca398d64..1904740233 100644 --- a/scripts/ci/cmake-v2/ci-power8-el7-xl-serial.cmake +++ b/scripts/ci/cmake-v2/ci-power8-el7-xl-serial.cmake @@ -1,4 +1,4 @@ -# Client maintainer: chuck.atkins@kitware.com +# Client maintainer: vicente.bolea@kitware.com set(ENV{CC} xlc) set(ENV{CXX} xlc++) @@ -20,6 +20,7 @@ ADIOS2_USE_ZFP:BOOL=ON set(CTEST_TEST_ARGS INCLUDE "^(Interface|Install)" ) + set(CTEST_TEST_TIMEOUT 300) set(CTEST_CMAKE_GENERATOR "Unix Makefiles") list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") diff --git a/scripts/ci/cmake/ci-ascent-gcc.cmake b/scripts/ci/cmake/ci-ascent-gcc.cmake deleted file mode 100644 index ebf1fd0743..0000000000 --- a/scripts/ci/cmake/ci-ascent-gcc.cmake +++ /dev/null @@ -1,38 +0,0 @@ -# Client maintainer: chuck.atkins@kitware.com - -find_package(EnvModules REQUIRED) - -env_module(purge) -env_module(load git) -env_module(load gcc/8.1.1) -env_module(load hdf5) -env_module(load libfabric) -env_module(load python/3.7.0) -env_module(load zfp) -env_module(load zeromq) - -set(ENV{CC} gcc) -set(ENV{CXX} g++) -set(ENV{FC} gfortran) - -set(dashboard_cache " -ADIOS2_USE_BZip2:BOOL=OFF -ADIOS2_USE_Blosc:BOOL=OFF -ADIOS2_USE_DataMan:BOOL=ON -ADIOS2_USE_Fortran:BOOL=ON -ADIOS2_USE_HDF5:BOOL=ON -ADIOS2_USE_MPI:BOOL=OFF -ADIOS2_USE_Python:BOOL=OFF -ADIOS2_USE_SST:BOOL=ON -ADIOS2_USE_SZ:BOOL=OFF -ADIOS2_USE_ZeroMQ:STRING=ON -ADIOS2_USE_ZFP:BOOL=OFF - -CMAKE_Fortran_FLAGS:STRING=-Wall -") - -set(NCPUS 4) -set(CTEST_TEST_ARGS PARALLEL_LEVEL 8) -set(CTEST_CMAKE_GENERATOR "Unix Makefiles") -list(APPEND CTEST_UPDATE_NOTES_FILES "${CMAKE_CURRENT_LIST_FILE}") -include(${CMAKE_CURRENT_LIST_DIR}/ci-common.cmake) diff --git a/scripts/ci/gh-actions/linux-setup.sh b/scripts/ci/gh-actions/linux-setup.sh index d71c301f21..d4ef4b632b 100755 --- a/scripts/ci/gh-actions/linux-setup.sh +++ b/scripts/ci/gh-actions/linux-setup.sh @@ -3,7 +3,7 @@ set -ex export CI_ROOT_DIR="${GITHUB_WORKSPACE}/.." -export CI_SOURCE_DIR="${GITHUB_WORKSPACE}" +export CI_SOURCE_DIR="${GITHUB_WORKSPACE}/source" declare -r local_scripts_dir="$(dirname -- $0)/config" diff --git a/scripts/ci/gh-actions/windows-setup.ps1 b/scripts/ci/gh-actions/windows-setup.ps1 index 3f96c7060c..18b9248e58 100644 --- a/scripts/ci/gh-actions/windows-setup.ps1 +++ b/scripts/ci/gh-actions/windows-setup.ps1 @@ -10,38 +10,39 @@ Write-Host "::endgroup::" if($Env:GH_YML_MATRIX_PARALLEL -eq "mpi") { - $rooturl = "https://github.com/microsoft/Microsoft-MPI/releases/download" - $version = "10.1.1" - $baseurl = "$rooturl/v$version" + # This is taken from the MSMPI VCPKG + $baseurl = "https://download.microsoft.com/download/a/5/2/a5207ca5-1203-491a-8fb8-906fd68ae623" + $version = "10.1.12498" - $tempdir = $Env:RUNNER_TEMP - $msmpisdk = Join-Path $tempdir msmpisdk.msi - $msmpisetup = Join-Path $tempdir msmpisetup.exe + $tempdir = $Env:RUNNER_TEMP + $msmpisdk = Join-Path $tempdir msmpisdk.msi + $msmpisetup = Join-Path $tempdir msmpisetup.exe - Write-Host "::group::Downloading Microsoft MPI SDK $version" - Invoke-WebRequest "$baseurl/msmpisdk.msi" -OutFile $msmpisdk - Write-Host "::endgroup::" - Write-Host "::group::Installing Microsoft MPI SDK $version" - Start-Process msiexec.exe -ArgumentList "/quiet /passive /qn /i $msmpisdk" -Wait - Write-Host "::endgroup::" + Write-Host "::group::Downloading Microsoft MPI SDK $version" + Invoke-WebRequest "$baseurl/msmpisdk.msi" -OutFile $msmpisdk + Write-Host "::endgroup::" + Write-Host "::group::Installing Microsoft MPI SDK $version" + Start-Process msiexec.exe -ArgumentList "/quiet /passive /qn /i $msmpisdk" -Wait + Write-Host "::endgroup::" - Write-Host "::group::Downloading Microsoft MPI Runtime $version" - Invoke-WebRequest "$baseurl/msmpisetup.exe" -OutFile $msmpisetup - Write-Host "::endgroup::" - Write-Host "::group::Installing Microsoft MPI Runtime $version" - Start-Process $msmpisetup -ArgumentList "-unattend" -Wait - Write-Host "::endgroup::" + Write-Host "::group::Downloading Microsoft MPI Runtime $version" - if ($Env:GITHUB_ENV) { - Write-Host '::group::Adding environment variables to $GITHUB_ENV' - $envlist = @("MSMPI_BIN", "MSMPI_INC", "MSMPI_LIB32", "MSMPI_LIB64") - foreach ($name in $envlist) { - $value = [Environment]::GetEnvironmentVariable($name, "Machine") - Write-Host "$name=$value" - Add-Content $Env:GITHUB_ENV "$name=$value" - } - Write-Host "::endgroup::" - } + Invoke-WebRequest "$baseurl/msmpisetup.exe" -OutFile $msmpisetup + Write-Host "::endgroup::" + Write-Host "::group::Installing Microsoft MPI Runtime $version" + Start-Process $msmpisetup -ArgumentList "-unattend" -Wait + Write-Host "::endgroup::" + + if ($Env:GITHUB_ENV) { + Write-Host '::group::Adding environment variables to $GITHUB_ENV' + $envlist = @("MSMPI_BIN", "MSMPI_INC", "MSMPI_LIB32", "MSMPI_LIB64") + foreach ($name in $envlist) { + $value = [Environment]::GetEnvironmentVariable($name, "Machine") + Write-Host "$name=$value" + Add-Content $Env:GITHUB_ENV "$name=$value" + } + Write-Host "::endgroup::" + } if ($Env:GITHUB_PATH) { Write-Host '::group::Adding $MSMPI_BIN to $GITHUB_PATH' diff --git a/scripts/ci/gitlab-ci/run.sh b/scripts/ci/gitlab-ci/run.sh index 05ccc39c9a..1ddfb0c9dc 100755 --- a/scripts/ci/gitlab-ci/run.sh +++ b/scripts/ci/gitlab-ci/run.sh @@ -1,48 +1,42 @@ #!/bin/bash --login +# shellcheck disable=SC1091 +set -e -if [ -n "${GITLAB_SITE}" ] +source scripts/ci/gitlab-ci/setup-vars.sh + +readonly CTEST_SCRIPT=scripts/ci/cmake-v2/ci-${CI_JOB_NAME}.cmake +if [ ! -f "$CTEST_SCRIPT" ] then - export CI_SITE_NAME="${GITLAB_SITE}" -else - export CI_SITE_NAME="GitLab CI" + echo "[E] Variable files does not exits: $CTEST_SCRIPT" + exit 1 fi -export CI_BUILD_NAME="${CI_COMMIT_BRANCH#github/}_${CI_JOB_NAME}" -export CI_SOURCE_DIR="${CI_PROJECT_DIR}" -export CI_ROOT_DIR="${CI_PROJECT_DIR}/.." -export CI_BIN_DIR="${CI_ROOT_DIR}/${CI_BUILD_NAME}" -export CI_COMMIT_REF=${CI_COMMIT_SHA} - -STEP=$1 -CTEST_SCRIPT=scripts/ci/cmake/ci-${CI_JOB_NAME}.cmake - -# Update and Test steps enable an extra step -CTEST_STEP_ARGS="" -case ${STEP} in - test) CTEST_STEP_ARGS="${CTEST_STEP_ARGS} -Ddashboard_do_end=ON" ;; -esac -CTEST_STEP_ARGS="${CTEST_STEP_ARGS} -Ddashboard_do_${STEP}=ON" - -if [ -n "${CMAKE_ENV_MODULE}" ] +readonly STEP=$1 +if [ -z "$STEP" ] then - module load ${CMAKE_ENV_MODULE} - - echo "**********module avail Begin************" - module avail - echo "**********module avail End**************" + echo "[E] No argument given: $*" + exit 2 fi -CTEST=ctest - -echo "**********Env Begin**********" -env | sort -echo "**********Env End************" +declare -a CTEST_STEP_ARGS=("-Ddashboard_full=OFF") +case ${STEP} in + update) CTEST_STEP_ARGS+=("${CI_UPDATE_ARGS}") ;; + configure) CTEST_STEP_ARGS+=("-Ddashboard_do_submit=OFF") ;; + build) CTEST_STEP_ARGS+=("-Ddashboard_do_submit=OFF") ;; + test) CTEST_STEP_ARGS+=("-Ddashboard_do_submit=OFF") ;; + submit) CTEST_STEP_ARGS+=("-Ddashboard_do_submit_only=ON" "-Ddashboard_do_configure=ON" "-Ddashboard_do_build=ON" "-Ddashboard_do_test=ON") ;; +esac +CTEST_STEP_ARGS+=("-Ddashboard_do_${STEP}=ON") echo "**********CTest Begin**********" -${CTEST} --version -echo ${CTEST} -VV -S ${CTEST_SCRIPT} -Ddashboard_full=OFF ${CTEST_STEP_ARGS} -${CTEST} -VV -S ${CTEST_SCRIPT} -Ddashboard_full=OFF ${CTEST_STEP_ARGS} +echo "ctest -VV -S ${CTEST_SCRIPT} ${CTEST_STEP_ARGS[*]}" +ctest -VV -S "${CTEST_SCRIPT}" "${CTEST_STEP_ARGS[@]}" RET=$? echo "**********CTest End************" -exit ${RET} +# EC: 0-127 this script errors, 128-INF ctest errors +if [ $RET -ne 0 ] +then + (( RET += 127 )) +fi +exit $RET diff --git a/scripts/ci/gitlab-ci/setup-vars.sh b/scripts/ci/gitlab-ci/setup-vars.sh new file mode 100755 index 0000000000..6ca88e616c --- /dev/null +++ b/scripts/ci/gitlab-ci/setup-vars.sh @@ -0,0 +1,43 @@ +#!/bin/bash --login +set -e + +# Strip the job name prefix +export CI_JOB_NAME="${CI_JOB_NAME#*:}" +export CI_BUILD_NAME="${CI_COMMIT_BRANCH#github/}_${CI_JOB_NAME}" +export CI_ROOT_DIR="${CI_PROJECT_DIR}/.." +export CI_SITE_NAME="${GITLAB_SITE}" +export CI_SOURCE_DIR="${CI_PROJECT_DIR}" + +if [ -z "$DOWNSTREAM_COMMIT_SHA" ] +then + export CI_COMMIT_REF="${CI_COMMIT_SHA}" +else + export CI_COMMIT_REF="${DOWNSTREAM_COMMIT_SHA}" +fi + +if [ -z "$DOWNSTREAM_BRANCH_REF" ] +then + export CI_BRANCH_REF="${CI_COMMIT_REF_NAME}" +else + export CI_BRANCH_REF="${DOWNSTREAM_BRANCH_REF}" +fi + +# In OLCF Crusher we must fix the build directory in the yml. +if [ -z "$CI_BIN_DIR" ] +then + export CI_BIN_DIR="${CI_ROOT_DIR}/${CI_BUILD_NAME}" +fi + +# In OLCF Gitlab our PRs branches tip commit is not the head commit of the PR, +# it is instead the so called merged_commit_sha as described in the GitHub Rest +# API for pull requests. We need to report to the CDASH the original commit +# thus, we set it here using the CTEST_UPDATE_VERSION_OVERRIDE CMake variable +if [[ ${CI_BRANCH_REF} =~ ^pr[0-9]+_.*$ ]] +then + # Original commit it is always its 2nd parent + ci_original_sha=$(git rev-parse "${CI_COMMIT_REF}^2") + export CI_ORIGINAL_SHA="$ci_original_sha" + export CI_UPDATE_ARGS="-DCTEST_UPDATE_VERSION_OVERRIDE=${CI_ORIGINAL_SHA}" +else + export CI_ORIGINAL_SHA="${CI_COMMIT_REF}" +fi diff --git a/scripts/ci/images/power8-el7-xl-base/.gitignore b/scripts/ci/images/.gitignore similarity index 65% rename from scripts/ci/images/power8-el7-xl-base/.gitignore rename to scripts/ci/images/.gitignore index 0195d2d90e..2496ef415f 100644 --- a/scripts/ci/images/power8-el7-xl-base/.gitignore +++ b/scripts/ci/images/.gitignore @@ -1,2 +1,3 @@ +*-stamp *.tar.gz *.rpm diff --git a/scripts/ci/images/build-emu-power8-image.sh b/scripts/ci/images/build-emu-power8-image.sh new file mode 100755 index 0000000000..c13949ed44 --- /dev/null +++ b/scripts/ci/images/build-emu-power8-image.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -ex + +######################################## +# ppc64le CentOS 7 emulation base image +######################################## +docker build --squash \ + --build-arg TARGET_ARCH_SYSTEM=ppc64le \ + --build-arg TARGET_ARCH_DOCKER=ppc64le \ + --build-arg TARGET_CPU=power8 \ + -t ornladios/adios2:ci-x86_64-power8-el7 \ + emu-el7 +echo "" +echo "Pushing ornladios/adios2:ci-x86_64-power8-el7" +echo "" +docker push ornladios/adios2:ci-x86_64-power8-el7 + +###################################### + ppc64le CI base image +###################################### +docker build \ + --build-arg TARGET_CPU=power8 \ + -t ornladios/adios2:ci-x86_64-power8-el7-base \ + emu-el7-base +docker-squash \ + -f ornladios/adios2:ci-x86_64-power8-el7 \ + -t ornladios/adios2:ci-x86_64-power8-el7-base \ + ornladios/adios2:ci-x86_64-power8-el7-base +echo "" +echo "Pushing ornladios/adios2:ci-x86_64-power8-el7-base" +echo "" +docker push ornladios/adios2:ci-x86_64-power8-el7-base diff --git a/scripts/ci/images/build-emu-power8-images.sh b/scripts/ci/images/build-emu-power8-images.sh deleted file mode 100755 index f138e55ee1..0000000000 --- a/scripts/ci/images/build-emu-power8-images.sh +++ /dev/null @@ -1,125 +0,0 @@ -#!/bin/bash - -######################################## -# ppc64le CentOS 7 emulation base image -######################################## -docker build --squash \ - --build-arg TARGET_ARCH_SYSTEM=ppc64le \ - --build-arg TARGET_ARCH_DOCKER=ppc64le \ - --build-arg TARGET_CPU=power8 \ - -t ornladios/adios2:ci-x86_64-power8-el7 \ - emu-el7 -echo "" -echo "Pushing ornladios/adios2:ci-x86_64-power8-el7" -echo "" -docker push ornladios/adios2:ci-x86_64-power8-el7 - -######################################## -# ppc64le CI base image -######################################## -docker build --squash \ - --build-arg TARGET_CPU=power8 \ - -t ornladios/adios2:ci-x86_64-power8-el7-base \ - emu-el7-base -docker-squash \ - -f ornladios/adios2:ci-x86_64-power8-el7 \ - -t ornladios/adios2:ci-x86_64-power8-el7-base \ - ornladios/adios2:ci-x86_64-power8-el7-base -echo "" -echo "Pushing ornladios/adios2:ci-x86_64-power8-el7-base" -echo "" -docker push ornladios/adios2:ci-x86_64-power8-el7-base - -######################################## -# XL base image -######################################## -docker build \ - -t ornladios/adios2:ci-x86_64-power8-el7-xl-base \ - power8-el7-xl-base -docker-squash \ - -f ornladios/adios2:ci-x86_64-power8-el7-base \ - -t ornladios/adios2:ci-x86_64-power8-el7-xl-base \ - ornladios/adios2:ci-x86_64-power8-el7-xl-base -echo "" -echo "Pushing ornladios/adios2:ci-x86_64-power8-el7-xl-base" -echo "" -docker push ornladios/adios2:ci-x86_64-power8-el7-xl-base - -######################################## -# XL builder image -######################################## -docker build \ - -t ornladios/adios2:ci-x86_64-power8-el7-xl \ - --build-arg COMPILER=xl \ - power8-el7-leaf -docker-squash \ - -f ornladios/adios2:ci-x86_64-power8-el7-xl-base \ - -t ornladios/adios2:ci-x86_64-power8-el7-xl \ - ornladios/adios2:ci-x86_64-power8-el7-xl -echo "" -echo "Pushing ornladios/adios2:ci-x86_64-power8-el7-xl" -echo "" -docker push ornladios/adios2:ci-x86_64-power8-el7-xl - -######################################## -# XL + MPI builder image -######################################## -docker build \ - -t ornladios/adios2:ci-x86_64-power8-el7-xl-smpi \ - --build-arg COMPILER=xl \ - power8-el7-leaf-smpi -docker-squash \ - -f ornladios/adios2:ci-x86_64-power8-el7-xl-base \ - -t ornladios/adios2:ci-x86_64-power8-el7-xl-smpi \ - ornladios/adios2:ci-x86_64-power8-el7-xl-smpi -echo "" -echo "Pushing ornladios/adios2:ci-x86_64-power8-el7-xl-smpi" -echo "" -docker push ornladios/adios2:ci-x86_64-power8-el7-xl-smpi - -######################################## -# PGI base image -######################################## -docker build \ - -t ornladios/adios2:ci-x86_64-power8-el7-pgi-base \ - power8-el7-pgi-base -docker-squash \ - -f ornladios/adios2:ci-x86_64-power8-el7-base \ - -t ornladios/adios2:ci-x86_64-power8-el7-pgi-base \ - ornladios/adios2:ci-x86_64-power8-el7-pgi-base -echo "" -echo "Pushing ornladios/adios2:ci-x86_64-power8-el7-pgi-base" -echo "" -docker push ornladios/adios2:ci-x86_64-power8-el7-pgi-base - -######################################## -# PGI builder image -######################################## -docker build \ - -t ornladios/adios2:ci-x86_64-power8-el7-pgi \ - --build-arg COMPILER=pgi \ - power8-el7-leaf -docker-squash \ - -f ornladios/adios2:ci-x86_64-power8-el7-pgi-base \ - -t ornladios/adios2:ci-x86_64-power8-el7-pgi \ - ornladios/adios2:ci-x86_64-power8-el7-pgi -echo "" -echo "Pushing ornladios/adios2:ci-x86_64-power8-el7-pgi" -echo "" -docker push ornladios/adios2:ci-x86_64-power8-el7-pgi - -######################################## -# PGI + MPI builder image -######################################## -docker build \ - -t ornladios/adios2:ci-x86_64-power8-el7-pgi-smpi \ - --build-arg COMPILER=pgi --build-arg HDF5_ARGS="-DMPI_C_COMPILER=mpipgicc" \ - power8-el7-leaf-smpi -docker-squash \ - -f ornladios/adios2:ci-x86_64-power8-el7-pgi-base \ - -t ornladios/adios2:ci-x86_64-power8-el7-pgi-smpi \ - ornladios/adios2:ci-x86_64-power8-el7-pgi-smpi -echo "" -echo "Pushing ornladios/adios2:ci-x86_64-power8-el7-pgi-smpi" -echo "" -docker push ornladios/adios2:ci-x86_64-power8-el7-pgi-smpi diff --git a/scripts/ci/images/emu-el7-base/Dockerfile b/scripts/ci/images/emu-el7-base/Dockerfile index 370815c529..68a751182a 100644 --- a/scripts/ci/images/emu-el7-base/Dockerfile +++ b/scripts/ci/images/emu-el7-base/Dockerfile @@ -3,50 +3,80 @@ FROM ornladios/adios2:ci-x86_64-${TARGET_CPU}-el7 # Install core dev packages RUN yum upgrade -y && \ - yum -y install make curl file valgrind vim bison flex sudo gdb \ - pkgconfig bison flex pkgconfig gcc gcc-c++ gcc-gfortran \ - zlib zlib-devel bzip2 bzip2-libs bzip2-devel libpng-devel \ - libfabric-devel libffi-devel -RUN yum -y install epel-release && \ - yum -y install zeromq-devel blosc-devel libzstd-devel + yum -y install centos-release-scl && \ + yum-config-manager --enable rhel-server-rhscl-7-rpms && \ + yum -y install \ + bison \ + bzip2 \ + bzip2-libs \ + curl \ + devtoolset-7 \ + devtoolset-7-gcc \ + devtoolset-7-gcc-c++ \ + devtoolset-7-gcc-gdb \ + devtoolset-7-gcc-gfortran \ + file \ + flex \ + gcc \ + gcc-c++ \ + gcc-gfortran \ + libfabric-devel \ + libffi-devel \ + libpng-devel \ + make \ + pkgconfig \ + sudo \ + valgrind \ + vim \ + zlib \ + yum -y install epel-release && \ + yum -y install \ + blosc-devel \ + bzip2-devel \ + curl-devel \ + expat-devel \ + gettext \ + libcurl-devel \ + libzstd-devel \ + openssl-devel \ + rhash-devel \ + xz-devel \ + zeromq-devel \ + zlib-devel && \ + yum clean all && \ + rm -rfv /tmp/* /var/cache/yum + +ENV LAUNCHER="/usr/bin/scl enable devtoolset-7 --" # Install and setup newer version of git WORKDIR /opt/git -RUN yum install -y gettext openssl-devel curl-devel expat-devel && \ - mkdir tmp && \ +RUN mkdir tmp && \ cd tmp && \ curl -L https://mirrors.edge.kernel.org/pub/software/scm/git/git-2.26.0.tar.gz | tar -xz && \ cd git-2.26.0 && \ - make -j$(grep -c '^processor' /proc/cpuinfo) prefix=/opt/git/2.26.0 all && \ - make prefix=/opt/git/2.26.0 install && \ + $LAUNCHER make -j$(grep -c '^processor' /proc/cpuinfo) prefix=/opt/git/2.26.0 all && \ + $LAUNCHER make prefix=/opt/git/2.26.0 install && \ cd ../.. && \ rm -rf tmp ENV PATH=/opt/git/2.26.0/bin:${PATH} # Install the most recent CMake from source WORKDIR /opt/cmake -RUN yum install -y \ - bzip2-devel libcurl-devel expat-devel \ - xz-devel rhash-devel zlib-devel libzstd-devel && \ - mkdir tmp && \ +RUN mkdir tmp && \ cd tmp && \ - curl -L https://github.com/Kitware/CMake/releases/download/v3.18.0/cmake-3.18.0.tar.gz | \ + curl -L https://github.com/Kitware/CMake/releases/download/v3.26.0/cmake-3.26.0.tar.gz | \ tar -xz && \ mkdir build && \ cd build && \ - ../cmake-3.17.0/bootstrap \ + $LAUNCHER ../cmake-3.26.0/bootstrap \ --system-libs \ --no-qt-gui \ --no-system-libarchive \ --no-system-libuv \ --no-system-jsoncpp \ - --prefix=/opt/cmake/3.18.0 \ + --prefix=/opt/cmake/3.26.0 \ --parallel=$(grep -c '^processor' /proc/cpuinfo) && \ make -j$(grep -c '^processor' /proc/cpuinfo) install && \ cd ../.. && \ - rm -rf tmp -ENV PATH=/opt/cmake/3.18.0/bin:${PATH} - -# Misc cleanup of unneeded files -RUN yum clean all && \ - rm -rfv /tmp/* /var/cache/yum + rm -rf tmp +ENV PATH=/opt/cmake/3.26.0/bin:${PATH} diff --git a/scripts/ci/images/emu-el7/Dockerfile b/scripts/ci/images/emu-el7/Dockerfile index 887c7aa3a0..d954fe7f0d 100644 --- a/scripts/ci/images/emu-el7/Dockerfile +++ b/scripts/ci/images/emu-el7/Dockerfile @@ -35,7 +35,7 @@ FROM ${TARGET_ARCH_DOCKER}/centos:centos7 # Grab a fully static node.js binary # to run various CI actions ######################################## -COPY --from=ornladios/adios2:node12-static /node /x86_64/bin/node +COPY --from=ornladios/adios2:node12-static /node /x86_64/bin/node ######################################## # Build up a minimal busybox shell diff --git a/scripts/ci/images/formatting/Dockerfile b/scripts/ci/images/formatting/Dockerfile index b5a35a7106..9ee5ef0b6c 100644 --- a/scripts/ci/images/formatting/Dockerfile +++ b/scripts/ci/images/formatting/Dockerfile @@ -1,10 +1,15 @@ FROM ubuntu:20.04 -RUN apt-get update && \ - apt-get install -y apt-utils && \ - apt-get install -y curl git flake8 libtinfo5 && \ +RUN apt update && \ + DEBIAN_FRONTEND="noninteractive" apt upgrade -y --no-install-recommends && \ + DEBIAN_FRONTEND="noninteractive" apt install -y --no-install-recommends \ + apt-utils \ + ca-certificates \ + clang-format-7 \ + curl \ + flake8 \ + git \ + libtinfo5 \ + shellcheck \ + && \ apt-get clean - -RUN curl -L https://github.com/llvm/llvm-project/releases/download/llvmorg-7.1.0/clang+llvm-7.1.0-x86_64-linux-gnu-ubuntu-14.04.tar.xz | tar -C /opt -xJv && \ - mv /opt/clang* /opt/llvm-7.1.0 -ENV PATH=/opt/llvm-7.1.0/bin:$PATH diff --git a/scripts/ci/images/power8-el7-leaf-smpi/.gitignore b/scripts/ci/images/power8-el7-leaf-smpi/.gitignore deleted file mode 100644 index e7a9c1347b..0000000000 --- a/scripts/ci/images/power8-el7-leaf-smpi/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.rpm diff --git a/scripts/ci/images/power8-el7-leaf-smpi/Dockerfile b/scripts/ci/images/power8-el7-leaf-smpi/Dockerfile deleted file mode 100644 index 4e1147b826..0000000000 --- a/scripts/ci/images/power8-el7-leaf-smpi/Dockerfile +++ /dev/null @@ -1,44 +0,0 @@ -ARG COMPILER -FROM ornladios/adios2:ci-x86_64-power8-el7-${COMPILER}-base - -# Install SpectrumMPI -WORKDIR /tmp -COPY ibm_smpi*.rpm /tmp/ -RUN yum install -y ibm_smpi*.rpm && \ - /bin/env IBM_SPECTRUM_MPI_LICENSE_ACCEPT=yes \ - /opt/ibm/spectrum_mpi/lap_ce/bin/accept_spectrum_mpi_license.sh -ENV MPI_ROOT=/opt/ibm/spectrum_mpi \ - MPI_HOME=/opt/ibm/spectrum_mpi \ - PATH=/opt/ibm/spectrum_mpi/bin:${PATH} \ - LD_LIBRARY_PATH=/opt/ibm/spectrum_mpi/lib:${PATH} - -# Install HDF5 1.13.0 -WORKDIR /opt/hdf5 -ARG HDF5_ARGS -RUN curl -L https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.13/hdf5-1.13.0/src/hdf5-1.13.0.tar.bz2 | \ - tar -xvj && \ - mkdir build && \ - cd build && \ - cmake ${HDF5_ARGS} \ - -DCMAKE_INSTALL_PREFIX=/opt/hdf5/1.13.0 \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_STATIC_LIBS=OFF \ - -DCMAKE_BUILD_TYPE=Release \ - -DHDF5_BUILD_CPP_LIB=OFF \ - -DHDF5_BUILD_EXAMPLES=OFF \ - -DBUILD_TESTING=OFF \ - -DHDF5_BUILD_TOOLS=ON \ - -DHDF5_ENABLE_PARALLEL=ON \ - ../hdf5-1.13.0 && \ - make -j$(grep -c '^processor' /proc/cpuinfo) install && \ - cd .. && \ - rm -rf hdf5-1.13.0 build -ENV PATH=/opt/hdf5/1.13.0/bin:${PATH} \ - LD_LIBRARY_PATH=/opt/hdf5/1.13.0/lib:${LD_LIBRARY_PATH} \ - CMAKE_PREFIX_PATH=/opt/hdf5/1.13.0:${CMAKE_PREFIX_PATH} - -# Misc cleanup of unneeded files -RUN rm -rf /tmp/* && \ - yum clean all - -WORKDIR /root diff --git a/scripts/ci/images/power8-el7-leaf/Dockerfile b/scripts/ci/images/power8-el7-leaf/Dockerfile deleted file mode 100644 index 2dde2ca878..0000000000 --- a/scripts/ci/images/power8-el7-leaf/Dockerfile +++ /dev/null @@ -1,32 +0,0 @@ -ARG COMPILER -FROM ornladios/adios2:ci-x86_64-power8-el7-${COMPILER}-base - -# Install HDF5 1.13.0 -WORKDIR /opt/hdf5 -RUN curl -L https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.13/hdf5-1.13.0/src/hdf5-1.13.0.tar.bz2 | \ - tar -xvj && \ - mkdir build && \ - cd build && \ - cmake \ - -DCMAKE_INSTALL_PREFIX=/opt/hdf5/1.13.0 \ - -DBUILD_SHARED_LIBS=ON \ - -DBUILD_STATIC_LIBS=OFF \ - -DCMAKE_BUILD_TYPE=Release \ - -DHDF5_ENABLE_PARALLEL=OFF \ - -DHDF5_BUILD_CPP_LIB=OFF\ - -DHDF5_BUILD_EXAMPLES=OFF \ - -DBUILD_TESTING=OFF \ - -DHDF5_BUILD_TOOLS=OFF \ - ../hdf5-1.13.0 && \ - make -j$(grep -c '^processor' /proc/cpuinfo) install && \ - cd .. && \ - rm -rf hdf5-1.13.0 build -ENV PATH=/opt/hdf5/1.13.0/bin:${PATH} \ - LD_LIBRARY_PATH=/opt/hdf5/1.13.0/lib:${LD_LIBRARY_PATH} \ - CMAKE_PREFIX_PATH=/opt/hdf5/1.13.0:${CMAKE_PREFIX_PATH} - -# Misc cleanup of unneeded files -RUN rm -rf /tmp/* && \ - yum clean all - -WORKDIR /root diff --git a/scripts/ci/images/power8-el7-pgi-base/.gitignore b/scripts/ci/images/power8-el7-pgi-base/.gitignore deleted file mode 100644 index 335ec9573d..0000000000 --- a/scripts/ci/images/power8-el7-pgi-base/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.tar.gz diff --git a/scripts/ci/images/power8-el7-pgi-base/Dockerfile b/scripts/ci/images/power8-el7-pgi-base/Dockerfile deleted file mode 100644 index 554b7bf0ba..0000000000 --- a/scripts/ci/images/power8-el7-pgi-base/Dockerfile +++ /dev/null @@ -1,70 +0,0 @@ -FROM ornladios/adios2:ci-x86_64-power8-el7-base - -# -# Install PGI -COPY pgilinux-*-ppc64le.tar.gz /tmp -WORKDIR /tmp -RUN mkdir pgi && \ - cd pgi && \ - tar -xf ../pgilinux-*-ppc64le.tar.gz && \ - export \ - PGI_SILENT=true \ - PGI_ACCEPT_EULA=accept \ - PGI_INSTALL_DIR=/opt/pgi \ - PGI_INSTALL_NVIDIA=false \ - PGI_INSTALL_JAVA=false \ - PGI_INSTALL_MPI=false \ - PGI_MPI_GPU_SUPPORT=false && \ - ./install && \ - rm -rf /opt/pgi/linuxpower/20[0-9][0-9] && \ - ln -s /opt/pgi/linuxpower/*.*/ /opt/pgi/linuxpower/latest -ENV PGI=/opt/pgi \ - CC=/opt/pgi/linuxpower/latest/bin/pgcc \ - CXX=/opt/pgi/linuxpower/latest/bin/pgc++ \ - FC=/opt/pgi/linuxpower/latest/bin/pgfortran \ - F90=/opt/pgi/linuxpower/latest/bin/pgf90 \ - F77=/opt/pgi/linuxpower/latest/bin/pgf77 \ - CPP=/bin/cpp \ - PATH=/opt/pgi/linuxpower/latest/bin:${PATH} \ - LD_LIBRARY_PATH=/opt/pgi/linuxpower/latest/lib:${LD_LIBRARY_PATH} - -# Install ZFP -WORKDIR /opt/zfp -RUN curl -L https://github.com/LLNL/zfp/releases/download/0.5.5/zfp-0.5.5.tar.gz | tar -xvz && \ - mkdir build && \ - cd build && \ - cmake \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_CXX_STANDARD=11 \ - -DCMAKE_INSTALL_PREFIX=/opt/zfp/0.5.5 \ - ../zfp-0.5.5 && \ - make -j$(grep -c '^processor' /proc/cpuinfo) install && \ - cd .. && \ - rm -rf zfp-0.5.5 build -ENV PATH=/opt/zfp/0.5.5/bin:${PATH} \ - LD_LIBRARY_PATH=/opt/zfp/0.5.5/lib64:${LD_LIBRARY_PATH} \ - CMAKE_PREFIX_PATH=/opt/zfp/0.5.5:${CMAKE_PREFIX_PATH} - -# Install SZ -WORKDIR /opt/sz -RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.3.tar.gz | tar -xvz && \ - mkdir build && \ - cd build && \ - cmake \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.3 \ - ../SZ-2.1.8.3 && \ - make -j$(grep -c '^processor' /proc/cpuinfo) install && \ - cd .. && \ - rm -rf SZ-2.1.8.3 build -ENV PATH=/opt/sz/2.1.8.3/bin:${PATH} \ - LD_LIBRARY_PATH=/opt/sz/2.1.8.3/lib64:${LD_LIBRARY_PATH} \ - CMAKE_PREFIX_PATH=/opt/sz/2.1.8.3:${CMAKE_PREFIX_PATH} - -# Misc cleanup of unneeded files -RUN rm -rf /tmp/* - -# Install Spectrum MPI -# /bin/env IBM_SPECTRUM_MPI_LICENSE_ACCEPT=yes /opt/ibm/spectrum_mpi/lap_ce/bin/accept_spectrum_mpi_license.sh diff --git a/scripts/ci/images/power8-el7-xl.Makefile b/scripts/ci/images/power8-el7-xl.Makefile new file mode 100644 index 0000000000..797d04b14a --- /dev/null +++ b/scripts/ci/images/power8-el7-xl.Makefile @@ -0,0 +1,27 @@ +# vim: ft=make : +# How to invoke this: +# +# - Copy the IBM_XL binaries to scripts/ci/images directory: +# - Binaries are located at KW SMB dir `Downloads/Compilers/IBM/` or IBM site +# - Run this: make -f power8-el7-xl.Makefile +# - push built images +TARGETS = config_binfmt-stamp emu-stamp xl-stamp +BUILD_ARGS = --squash +all: $(TARGETS) + +config_binfmt-stamp: + touch $@ + sudo docker run --rm --privileged multiarch/qemu-user-static:register --reset + +emu-stamp: BUILD_ARGS+= --build-arg TARGET_CPU=power8 +emu-stamp: emu-el7-base/Dockerfile config_binfmt-stamp + touch $@ + sudo docker build $(BUILD_ARGS) -f $< -t ornladios/adios2:ci-x86_64-power8-el7-base . || (rm $@; false) + +xl-stamp: BUILD_ARGS+= --build-arg COMPILER=xl +xl-stamp: power8-el7/power8-el7-xl.dockerfile emu-stamp + touch $@ + sudo docker build $(BUILD_ARGS) -f $< -t ornladios/adios2:ci-x86_64-power8-el7-xl . || (rm $@; false) + +clean: + rm $(TARGETS) || true diff --git a/scripts/ci/images/power8-el7-xl-base/Dockerfile b/scripts/ci/images/power8-el7/power8-el7-xl.dockerfile similarity index 50% rename from scripts/ci/images/power8-el7-xl-base/Dockerfile rename to scripts/ci/images/power8-el7/power8-el7-xl.dockerfile index fe70270954..439106cfc2 100644 --- a/scripts/ci/images/power8-el7-xl-base/Dockerfile +++ b/scripts/ci/images/power8-el7/power8-el7-xl.dockerfile @@ -1,10 +1,15 @@ +# vim: ft=dockerfile: FROM ornladios/adios2:ci-x86_64-power8-el7-base -# Install XL compilers +# Install packages of XL compilers COPY IBM_XL_C_CPP_*_LINUX_COMMUNITY.tar.gz /tmp COPY IBM_XL_FORTRAN_*_LINUX_COMMUNITY.tar.gz /tmp +# Upgrades of XL compilers +COPY IBM_XL_C_CPP_*_LINUX.tar.gz /tmp +COPY IBM_XL_FORTRAN_*_LINUX.tar.gz /tmp WORKDIR /tmp -RUN mkdir xlc && \ +RUN source /opt/rh/devtoolset-7/enable && \ + mkdir xlc && \ cd xlc && \ tar -xf ../IBM_XL_C_CPP_*_LINUX_COMMUNITY.tar.gz && \ yes 1 | ./install && \ @@ -14,17 +19,33 @@ RUN mkdir xlc && \ tar -xf ../IBM_XL_FORTRAN_*_LINUX_COMMUNITY.tar.gz && \ yes 1 | ./install && \ cd .. && \ - rm -rf IBM* xlc xlf + mkdir xlc_p && \ + cd xlc_p && \ + tar -xf ../IBM_XL_C_CPP_*_LINUX.tar.gz && \ + yes 1 | ./install && \ + cd .. && \ + mkdir xlf_p && \ + cd xlf_p && \ + tar -xf ../IBM_XL_FORTRAN_*_LINUX.tar.gz && \ + yes 1 | ./install && \ + cd .. && \ + rm -rf /tmp/* && \ + yum -y clean all ENV CC=/usr/bin/xlc \ CXX=/usr/bin/xlc++ \ FC=/usr/bin/xlf +RUN source /opt/rh/devtoolset-7/enable && \ + /opt/ibm/xlC/16.1.1/bin/xlc_configure -gcc /opt/rh/devtoolset-7/root/usr/ + +ENV XLC_USR_CONFIG=/opt/ibm/xlC/16.1.1/etc/xlc.cfg.centos.7.gcc.7.3.1 + # Install ZFP WORKDIR /opt/zfp RUN curl -L https://github.com/LLNL/zfp/releases/download/0.5.5/zfp-0.5.5.tar.gz | tar -xvz && \ mkdir build && \ cd build && \ - cmake \ + $LAUNCHER cmake \ -DBUILD_SHARED_LIBS=ON \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_CXX_STANDARD=11 \ @@ -42,7 +63,7 @@ WORKDIR /opt/sz RUN curl -L https://github.com/disheng222/SZ/archive/v2.1.8.3.tar.gz | tar -xvz && \ mkdir build && \ cd build && \ - cmake \ + $LAUNCHER cmake \ -DBUILD_SHARED_LIBS=ON \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX=/opt/sz/2.1.8.3 \ @@ -54,8 +75,26 @@ ENV PATH=/opt/sz/2.1.8.3/bin:${PATH} \ LD_LIBRARY_PATH=/opt/sz/2.1.8.3/lib64:${LD_LIBRARY_PATH} \ CMAKE_PREFIX_PATH=/opt/sz/2.1.8.3:${CMAKE_PREFIX_PATH} -# Misc cleanup of unneeded files -RUN rm -rf /tmp/* - -# Install Spectrum MPI -# /bin/env IBM_SPECTRUM_MPI_LICENSE_ACCEPT=yes /opt/ibm/spectrum_mpi/lap_ce/bin/accept_spectrum_mpi_license.sh +# Install HDF5 1.13.0 +WORKDIR /opt/hdf5 +RUN curl -L https://github.com/HDFGroup/hdf5/archive/refs/tags/hdf5-1_13_0.tar.gz | tar -xvz && \ + mkdir build && \ + cd build && \ + $LAUNCHER cmake \ + -DCMAKE_INSTALL_PREFIX=/opt/hdf5/1.13.0 \ + -DBUILD_SHARED_LIBS=ON \ + -DBUILD_STATIC_LIBS=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DHDF5_ENABLE_PARALLEL=OFF \ + -DHDF5_BUILD_CPP_LIB=OFF\ + -DHDF5_BUILD_EXAMPLES=OFF \ + -DBUILD_TESTING=OFF \ + -DHDF5_BUILD_TOOLS=OFF \ + ../hdf5-hdf5-1_13_0 && \ + make -j$(grep -c '^processor' /proc/cpuinfo) install && \ + cd .. && \ + rm -rf hdf5-hdf5-1_13_0 build +ENV PATH=/opt/hdf5/1.13.0/bin:${PATH} \ + LD_LIBRARY_PATH=/opt/hdf5/1.13.0/lib:${LD_LIBRARY_PATH} \ + CMAKE_PREFIX_PATH=/opt/hdf5/1.13.0:${CMAKE_PREFIX_PATH} +WORKDIR /root diff --git a/scripts/ci/scripts/run-shellcheck.sh b/scripts/ci/scripts/run-shellcheck.sh new file mode 100755 index 0000000000..c987a5bea0 --- /dev/null +++ b/scripts/ci/scripts/run-shellcheck.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# We need the same sort order +export LC_ALL=C +EXCLUDE_SCRIPTS=.shellcheck_exclude_paths + +echo "---------- Begin ENV ----------" +env | sort +echo "---------- End ENV ----------" + +if [ -n "${SOURCE_DIR}" ] +then + cd "${SOURCE_DIR}" || exit +fi + +# Give me a sorted list of the project scripts +found_scripts="$({ + find scripts -regextype posix-extended -iregex '.*\.(sh|bash)' -print; + grep -rnlE -e '#!/(/usr)?/bin/(bash|sh)' -e '#!(/usr)?/bin/env\s+(bash|sh)' scripts; +} | sort -u)" + +echo "[I] Found the following files:" +echo "$found_scripts" + +# Give me the list of scripts without $EXCLUDE_SCRIPTS +if [ -f "$EXCLUDE_SCRIPTS" ] +then + if ! sort -uc "$EXCLUDE_SCRIPTS" + then + echo "[E] file: $EXCLUDE_SCRIPTS is not sorted." + echo " To fix this sort with: LC_ALL=C sort -u $EXCLUDE_SCRIPTS" + exit 1 + fi + + check_scripts=$(comm -2 -3 <(echo "$found_scripts") "$EXCLUDE_SCRIPTS" ) +fi + +echo "[I] Checking the following files:" +echo "$check_scripts" + +if ! xargs -n1 shellcheck <<<"$check_scripts" +then + echo "[E] shellcheck:" + echo " Code format checks failed." + echo " Please run shellcheck on your changes before committing." + exit 2 +fi + +exit 0 diff --git a/scripts/ci/setup/ci-power8-el7-xl-serial.sh b/scripts/ci/setup/ci-power8-el7-xl-serial.sh new file mode 100755 index 0000000000..48cae5afae --- /dev/null +++ b/scripts/ci/setup/ci-power8-el7-xl-serial.sh @@ -0,0 +1,9 @@ +#!/bin/bash +cat << EOF >> /etc/profile +set +u +source /opt/rh/devtoolset-7/enable + +export C_INCLUDE_DIRS="/opt/rh/devtoolset-7/root/usr/include/c++/7:${C_INCLUDE_DIRS}" +export CPP_INCLUDE_DIRS="/opt/rh/devtoolset-7/root/usr/include/c++/7:${CPP_INCLUDE_DIRS}" +set -u +EOF diff --git a/scripts/developer/create-changelog.sh b/scripts/developer/create-changelog.sh new file mode 100755 index 0000000000..ceb6f7173e --- /dev/null +++ b/scripts/developer/create-changelog.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# author: Vicente Bolea + +function require_dependency() +{ + if ! command -v "$1" &> /dev/null + then + echo "[E] Missing dependencies: $1" + exit 1 + fi +} + +require_dependency "jq" +require_dependency "gh" +require_dependency "csvlook" + +if [ "$#" != "2" ] +then + echo "[E] Wrong arguments. Invoke as:" + echo "scripts/developer/create-changelog.sh " + exit 2 +fi + +new_release="$1" +old_release="$2" + +prs="$(git log --oneline --grep='Merge pull request ' "^${old_release}" "${new_release}" | grep -Po '\s#\K[0-9]+')" +num_prs="$(wc -w <<<"$prs")" + +echo "[I] Found $num_prs PRs" + +# Header first +output=("PR, Title") +i=0 +for pr in ${prs} +do + printf "\r[I] Processing: PR=%06d progress=%05d/%05d" "$pr" "$i" "$num_prs" + output+=("$(gh api "/repos/ornladios/ADIOS2/pulls/$pr" | jq -r '["#\(.number)", .title] | @csv')") + ((i++)) +done +echo "" + +printf '%s\n' "${output[@]}" | csvlook diff --git a/source/adios2/CMakeLists.txt b/source/adios2/CMakeLists.txt index 77899e8ecc..cdd3a8d327 100644 --- a/source/adios2/CMakeLists.txt +++ b/source/adios2/CMakeLists.txt @@ -126,28 +126,14 @@ endif() set(maybe_adios2_core_kokkos) if(ADIOS2_HAVE_Kokkos) - add_library(adios2_core_kokkos helper/adiosKokkos.h helper/adiosKokkos.cpp) - - set_target_properties(adios2_core_kokkos PROPERTIES - VISIBILITY_INLINES_HIDDEN ON - INCLUDE_DIRECTORIES "$;$" - EXPORT_NAME core_kokkos - OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}_core_kokkos - ) - - kokkos_compilation(SOURCE helper/adiosKokkos.cpp) - if(Kokkos_ENABLE_CUDA) - set_property(SOURCE helper/adiosKokkos.cpp PROPERTY LANGUAGE CUDA) - set_property(SOURCE helper/adiosKokkos.cpp APPEND PROPERTY COMPILE_FLAGS "--extended-lambda") - set_target_properties(adios2_core_kokkos PROPERTIES - CUDA_VISIBILITY_PRESET hidden - ) - target_compile_features(adios2_core_kokkos PRIVATE cuda_std_17) - endif() - - target_link_libraries(adios2_core_kokkos PRIVATE Kokkos::kokkos) + # Kokkos imposes us to set our CMAKE_CXX_COMPILER to Kokkos_CXX_COMPILER. + # The problem is that we do not want this for the whole project and with + # CMake we cannot set the CXX_COMPILER for a single target. The solution is + # to move the adios2 module that uses Kokkos to its independent subdir and + # set there CMAKE_CXX_COMPILER, which is possible (and scoped to that subdir) + # in cmake. + add_subdirectory(helper/kokkos) target_link_libraries(adios2_core PRIVATE adios2_core_kokkos) - set(maybe_adios2_core_kokkos adios2_core_kokkos) endif() @@ -189,7 +175,13 @@ if (ADIOS2_HAVE_BP5 OR ADIOS2_HAVE_SST) endif() if(ADIOS2_HAVE_DAOS) - target_sources(adios2_core PRIVATE toolkit/transport/file/FileDaos.cpp) + target_sources(adios2_core PRIVATE toolkit/transport/file/FileDaos.cpp + engine/daos/DaosEngine.cpp + engine/daos/DaosWriter_EveryoneWrites_Async.cpp + engine/daos/DaosWriter_TwoLevelShm_Async.cpp + engine/daos/DaosWriter_TwoLevelShm.cpp + engine/daos/DaosReader.cpp engine/daos/DaosReader.tcc + engine/daos/DaosWriter.cpp engine/daos/DaosWriter.tcc) target_link_libraries(adios2_core PRIVATE DAOS::DAOS) endif() @@ -402,7 +394,7 @@ set_target_properties( adios2_core ${maybe_adios2_core_mpi} PROPERTIES VERSION ${ADIOS2_LIBRARY_VERSION} - SOVERSION ${ADIOS2_VERSION_MAJOR} + SOVERSION ${ADIOS2_LIBRARY_SOVERSION} ) install(FILES common/ADIOSMacros.h common/ADIOSTypes.h common/ADIOSTypes.inl diff --git a/source/adios2/core/Engine.cpp b/source/adios2/core/Engine.cpp index ab9c1706d8..bb0ea33012 100644 --- a/source/adios2/core/Engine.cpp +++ b/source/adios2/core/Engine.cpp @@ -29,6 +29,15 @@ Engine::Engine(const std::string engineType, IO &io, const std::string &name, m_FailVerbose = (m_Comm.Rank() == 0); } +Engine::Engine(const std::string engineType, IO &io, const std::string &name, + const Mode openMode, helper::Comm comm, const char *md, + const size_t mdsize) +: m_EngineType(engineType), m_IO(io), m_Name(name), m_OpenMode(openMode), + m_Comm(std::move(comm)) +{ + ThrowUp("Engine with metadata in memory"); +} + Engine::~Engine() { if (m_IsOpen) @@ -43,6 +52,13 @@ IO &Engine::GetIO() noexcept { return m_IO; } Mode Engine::OpenMode() const noexcept { return m_OpenMode; } +void Engine::GetMetadata(char **md, size_t *size) +{ + ThrowUp("GetMetadata"); + *md = nullptr; + *size = 0; +} + StepStatus Engine::BeginStep() { if (m_OpenMode == Mode::Read) diff --git a/source/adios2/core/Engine.h b/source/adios2/core/Engine.h index f91e264b68..635e088260 100644 --- a/source/adios2/core/Engine.h +++ b/source/adios2/core/Engine.h @@ -71,6 +71,19 @@ class Engine Engine(const std::string engineType, IO &io, const std::string &name, const Mode mode, helper::Comm comm); + /** + * Unique Base class constructor + * @param engineType derived class identifier + * @param io object that generates this Engine + * @param name unique engine name within IO class object + * @param mode open mode from ADIOSTypes.h Mode + * @param comm communicator passed at Open or from ADIOS class + * @param md Metadata already in memory + */ + Engine(const std::string engineType, IO &io, const std::string &name, + const Mode mode, helper::Comm comm, const char *md, + const size_t mdsize); + virtual ~Engine(); explicit operator bool() const noexcept; @@ -87,6 +100,14 @@ class Engine */ Mode OpenMode() const noexcept; + /** Serialize all metadata right after engine is created, which can be + * delivered to other processes to open the same file for reading without + * opening and reading in metadata again. + * @return metadata (pointer to allocated memory) and size of metadata + * the pointer must be deallocated by user using free() + */ + virtual void GetMetadata(char **md, size_t *size); + StepStatus BeginStep(); /** diff --git a/source/adios2/core/IO.cpp b/source/adios2/core/IO.cpp index c8ef73e3e6..50acff2b13 100644 --- a/source/adios2/core/IO.cpp +++ b/source/adios2/core/IO.cpp @@ -37,6 +37,7 @@ #include "adios2/engine/skeleton/SkeletonWriter.h" #include "adios2/helper/adiosComm.h" +#include "adios2/helper/adiosCommDummy.h" #include "adios2/helper/adiosFunctions.h" //BuildParametersMap #include "adios2/helper/adiosString.h" #include // FileIsDirectory() @@ -51,6 +52,11 @@ #include "adios2/engine/sst/SstWriter.h" #endif +#ifdef ADIOS2_HAVE_DAOS // external dependencies +#include "adios2/engine/daos/DaosReader.h" +#include "adios2/engine/daos/DaosWriter.h" +#endif + namespace adios2 { namespace core @@ -67,7 +73,8 @@ std::unordered_map Factory = { {IO::MakeEngine, IO::MakeEngine}}, {"bp5", #ifdef ADIOS2_HAVE_BP5 - {IO::MakeEngine, IO::MakeEngine} + {IO::MakeEngine, IO::MakeEngine, + IO::MakeEngineWithMD} #else IO::NoEngineEntry("ERROR: this version didn't compile with " "BP5 library, can't use BP5 engine\n") @@ -98,6 +105,14 @@ std::unordered_map Factory = { #else IO::NoEngineEntry("ERROR: this version didn't compile with " "Sst library, can't use Sst engine\n") +#endif + }, + {"daos", +#ifdef ADIOS2_HAVE_DAOS + {IO::MakeEngine, IO::MakeEngine} +#else + IO::NoEngineEntry("ERROR: this version didn't compile with " + "DAOS library, can't use DAOS engine\n") #endif }, {"effis", @@ -503,7 +518,8 @@ void IO::AddOperation(const std::string &variable, m_VarOpsPlaceholder[variable].push_back({operatorType, parameters}); } -Engine &IO::Open(const std::string &name, const Mode mode, helper::Comm comm) +Engine &IO::Open(const std::string &name, const Mode mode, helper::Comm comm, + const char *md, const size_t mdsize) { PERFSTUBS_SCOPED_TIMER("IO::Open"); auto itEngineFound = m_Engines.find(name); @@ -671,8 +687,13 @@ Engine &IO::Open(const std::string &name, const Mode mode, helper::Comm comm) auto f = FactoryLookup(engineTypeLC); if (f != Factory.end()) { - if ((mode_to_use == Mode::Read) || - (mode_to_use == Mode::ReadRandomAccess)) + if (md && mode_to_use == Mode::ReadRandomAccess) + { + engine = f->second.MakeReaderWithMD(*this, name, mode_to_use, + std::move(comm), md, mdsize); + } + else if ((mode_to_use == Mode::Read) || + (mode_to_use == Mode::ReadRandomAccess)) { engine = f->second.MakeReader(*this, name, mode_to_use, std::move(comm)); @@ -705,6 +726,15 @@ Engine &IO::Open(const std::string &name, const Mode mode) { return Open(name, mode, m_ADIOS.GetComm().Duplicate()); } + +Engine &IO::Open(const std::string &name, const char *md, const size_t mdsize) +{ + const Mode mode = Mode::ReadRandomAccess; + // helper::Comm comm; + // std::cout << "Open comm rank = " << comm.Rank(); + return Open(name, mode, helper::CommDummy(), md, mdsize); +} + Group &IO::CreateGroup(char delimiter) { m_Gr = std::make_shared("", delimiter, *this); diff --git a/source/adios2/core/IO.h b/source/adios2/core/IO.h index 8a0acac023..3db93c421d 100644 --- a/source/adios2/core/IO.h +++ b/source/adios2/core/IO.h @@ -395,7 +395,8 @@ class IO * @exception std::invalid_argument if Engine with unique name is already * created with another Open */ - Engine &Open(const std::string &name, const Mode mode, helper::Comm comm); + Engine &Open(const std::string &name, const Mode mode, helper::Comm comm, + const char *md = nullptr, const size_t mdsize = 0); /** * Overloaded version that reuses the MPI_Comm object passed @@ -409,6 +410,21 @@ class IO */ Engine &Open(const std::string &name, const Mode mode); + /** + * Overloaded version that is specifically for a serial program + * opening a file (not stream) with ReadRandomAccess mode and + * supplying the metadata already in memory. The metadata + * should be retrieved by another program calling engine.GetMetadata() + * after opening the file. + * @param name unique engine identifier within IO object + * (file name in case of File transports) + * @param md file metadata residing in memory + * @return a reference to a derived object of the Engine class + * @exception std::invalid_argument if Engine with unique name is already + * created with another Open + */ + Engine &Open(const std::string &name, const char *md, const size_t mdsize); + /** * Retrieve an engine by name */ @@ -455,10 +471,14 @@ class IO using MakeEngineFunc = std::function( IO &, const std::string &, const Mode, helper::Comm)>; + using MakeEngineWithMDFunc = std::function( + IO &, const std::string &, const Mode, helper::Comm, const char *, + const size_t)>; struct EngineFactoryEntry { MakeEngineFunc MakeReader; MakeEngineFunc MakeWriter; + MakeEngineWithMDFunc MakeReaderWithMD; }; /** @@ -487,6 +507,18 @@ class IO return std::make_shared(io, name, mode, std::move(comm)); } + /** + * Create an engine of type T. This is intended to be used when + * creating instances of EngineFactoryEntry for RegisterEngine. + */ + template + static std::shared_ptr + MakeEngineWithMD(IO &io, const std::string &name, const Mode mode, + helper::Comm comm, const char *md, const size_t mdsize) + { + return std::make_shared(io, name, mode, std::move(comm), md, mdsize); + } + /** * Register an engine factory entry to create a reader or writer * for an engine of the given engine type (named in lower case). diff --git a/source/adios2/engine/bp4/BP4Reader.cpp b/source/adios2/engine/bp4/BP4Reader.cpp index f4e1f5c939..0ff729ba19 100644 --- a/source/adios2/engine/bp4/BP4Reader.cpp +++ b/source/adios2/engine/bp4/BP4Reader.cpp @@ -15,6 +15,7 @@ #include #include +#include namespace adios2 { @@ -38,6 +39,20 @@ BP4Reader::BP4Reader(IO &io, const std::string &name, const Mode mode, m_IsOpen = true; } +BP4Reader::BP4Reader(IO &io, const std::string &name, const Mode mode, + helper::Comm comm, const char *md, const size_t mdsize) +: Engine("BP4Reader", io, name, mode, std::move(comm)), + m_BP4Deserializer(m_Comm), m_MDFileManager(io, m_Comm), + m_DataFileManager(io, m_Comm), m_MDIndexFileManager(io, m_Comm), + m_ActiveFlagFileManager(io, m_Comm) +{ + PERFSTUBS_SCOPED_TIMER("BP4Reader::Open"); + readMetadataFromFile = false; + Init(); + // ProcessMetadataFromMemory(md); + m_IsOpen = true; +} + BP4Reader::~BP4Reader() { if (m_IsOpen) @@ -47,6 +62,23 @@ BP4Reader::~BP4Reader() m_IsOpen = false; } +void BP4Reader::GetMetadata(char **md, size_t *size) +{ + uint64_t sizes[2] = {m_BP4Deserializer.m_Metadata.m_Buffer.size(), + m_BP4Deserializer.m_MetadataIndex.m_Buffer.size()}; + + size_t mdsize = sizes[0] + sizes[1] + 2 * sizeof(uint64_t); + *md = (char *)malloc(mdsize); + *size = mdsize; + char *p = *md; + memcpy(p, sizes, sizeof(sizes)); + p += sizeof(sizes); + memcpy(p, m_BP4Deserializer.m_Metadata.m_Buffer.data(), sizes[0]); + p += sizes[0]; + memcpy(p, m_BP4Deserializer.m_MetadataIndex.m_Buffer.data(), sizes[1]); + p += sizes[1]; +} + StepStatus BP4Reader::BeginStep(StepMode mode, const float timeoutSeconds) { PERFSTUBS_SCOPED_TIMER("BP4Reader::BeginStep"); @@ -195,26 +227,28 @@ void BP4Reader::Init() InitTransports(); helper::RaiseLimitNoFile(); - - /* Do a collective wait for the file(s) to appear within timeout. - Make sure every process comes to the same conclusion */ - const Seconds timeoutSeconds( - m_BP4Deserializer.m_Parameters.OpenTimeoutSecs); - - Seconds pollSeconds( - m_BP4Deserializer.m_Parameters.BeginStepPollingFrequencySecs); - if (pollSeconds > timeoutSeconds) + if (readMetadataFromFile) { - pollSeconds = timeoutSeconds; - } + /* Do a collective wait for the file(s) to appear within timeout. + Make sure every process comes to the same conclusion */ + const Seconds timeoutSeconds( + m_BP4Deserializer.m_Parameters.OpenTimeoutSecs); + + Seconds pollSeconds( + m_BP4Deserializer.m_Parameters.BeginStepPollingFrequencySecs); + if (pollSeconds > timeoutSeconds) + { + pollSeconds = timeoutSeconds; + } - TimePoint timeoutInstant = Now() + timeoutSeconds; + TimePoint timeoutInstant = Now() + timeoutSeconds; - OpenFiles(timeoutInstant, pollSeconds, timeoutSeconds); - if (!m_BP4Deserializer.m_Parameters.StreamReader) - { - /* non-stream reader gets as much steps as available now */ - InitBuffer(timeoutInstant, pollSeconds / 10, timeoutSeconds); + OpenFiles(timeoutInstant, pollSeconds, timeoutSeconds); + if (!m_BP4Deserializer.m_Parameters.StreamReader) + { + /* non-stream reader gets as much steps as available now */ + InitBuffer(timeoutInstant, pollSeconds / 10, timeoutSeconds); + } } } @@ -561,6 +595,41 @@ void BP4Reader::InitBuffer(const TimePoint &timeoutInstant, } } +void BP4Reader::ProcessMetadataFromMemory(const char *md) +{ + uint64_t size_mdidx, size_md; + const char *p = md; + memcpy(&size_md, p, sizeof(uint64_t)); + p = p + sizeof(uint64_t); + memcpy(&size_mdidx, p, sizeof(uint64_t)); + p = p + sizeof(uint64_t); + + std::string hint("when processing metadata from memory"); + size_t pos = 0; + + m_BP4Deserializer.m_Metadata.Resize(size_md, hint); + helper::CopyToBuffer(m_BP4Deserializer.m_Metadata.m_Buffer, pos, p, + size_md); + p = p + size_md; + + pos = 0; + m_BP4Deserializer.m_MetadataIndex.Resize(size_mdidx, hint); + helper::CopyToBuffer(m_BP4Deserializer.m_MetadataIndex.m_Buffer, pos, p, + size_mdidx); + p = p + size_mdidx; + + /* Parse metadata index table */ + m_BP4Deserializer.ParseMetadataIndex(m_BP4Deserializer.m_MetadataIndex, 0, + true, false); + // now we are sure the index header has been parsed, first step parsing + // done + m_IdxHeaderParsed = true; + + // fills IO with Variables and Attributes + m_MDFileProcessedSize = m_BP4Deserializer.ParseMetadata( + m_BP4Deserializer.m_Metadata, *this, true); +} + size_t BP4Reader::UpdateBuffer(const TimePoint &timeoutInstant, const Seconds &pollSeconds) { diff --git a/source/adios2/engine/bp4/BP4Reader.h b/source/adios2/engine/bp4/BP4Reader.h index 0d89668168..39858e07cc 100644 --- a/source/adios2/engine/bp4/BP4Reader.h +++ b/source/adios2/engine/bp4/BP4Reader.h @@ -39,8 +39,13 @@ class BP4Reader : public Engine BP4Reader(IO &io, const std::string &name, const Mode mode, helper::Comm comm); + BP4Reader(IO &io, const std::string &name, const Mode mode, + helper::Comm comm, const char *md, const size_t mdsize); + virtual ~BP4Reader(); + void GetMetadata(char **md, size_t *size) final; + StepStatus BeginStep(StepMode mode = StepMode::Read, const float timeoutSeconds = -1.0) final; @@ -89,8 +94,11 @@ class BP4Reader : public Engine int m_Verbosity = 0; + bool readMetadataFromFile = true; + void Init(); void InitTransports(); + void ProcessMetadataFromMemory(const char *md); /* Sleep up to pollSeconds time if we have not reached timeoutInstant. * Return true if slept diff --git a/source/adios2/engine/bp5/BP5Reader.cpp b/source/adios2/engine/bp5/BP5Reader.cpp index 1e8b6abf00..2afd0b85ce 100644 --- a/source/adios2/engine/bp5/BP5Reader.cpp +++ b/source/adios2/engine/bp5/BP5Reader.cpp @@ -40,6 +40,20 @@ BP5Reader::BP5Reader(IO &io, const std::string &name, const Mode mode, m_IsOpen = true; } +BP5Reader::BP5Reader(IO &io, const std::string &name, const Mode mode, + helper::Comm comm, const char *md, const size_t mdsize) +: Engine("BP5Reader", io, name, mode, std::move(comm)), + m_MDFileManager(io, m_Comm), m_DataFileManager(io, m_Comm), + m_MDIndexFileManager(io, m_Comm), m_FileMetaMetadataManager(io, m_Comm), + m_ActiveFlagFileManager(io, m_Comm) +{ + PERFSTUBS_SCOPED_TIMER("BP5Reader::Open"); + readMetadataFromFile = false; + Init(); + ProcessMetadataFromMemory(md); + m_IsOpen = true; +} + BP5Reader::~BP5Reader() { if (m_BP5Deserializer) @@ -58,6 +72,99 @@ void BP5Reader::DestructorClose(bool Verbose) noexcept m_IsOpen = false; } +void BP5Reader::GetMetadata(char **md, size_t *size) +{ + uint64_t sizes[3] = {m_Metadata.m_Buffer.size(), + m_MetaMetadata.m_Buffer.size(), + m_MetadataIndex.m_Buffer.size()}; + + /* BP5 modifies the metadata block in memory during processing + so we have to read it from file again + */ + auto currentPos = m_MDFileManager.CurrentPos(0); + std::vector mdbuf(sizes[0]); + m_MDFileManager.ReadFile(mdbuf.data(), sizes[0], 0); + m_MDFileManager.SeekTo(currentPos, 0); + + size_t mdsize = sizes[0] + sizes[1] + sizes[2] + 3 * sizeof(uint64_t); + *md = (char *)malloc(mdsize); + *size = mdsize; + char *p = *md; + memcpy(p, sizes, sizeof(sizes)); + p += sizeof(sizes); + memcpy(p, mdbuf.data(), sizes[0]); + p += sizes[0]; + memcpy(p, m_MetaMetadata.m_Buffer.data(), sizes[1]); + p += sizes[1]; + memcpy(p, m_MetadataIndex.m_Buffer.data(), sizes[2]); + p += sizes[2]; +} + +void BP5Reader::ProcessMetadataFromMemory(const char *md) +{ + uint64_t size_mdidx, size_md, size_mmd; + const char *p = md; + memcpy(&size_md, p, sizeof(uint64_t)); + p = p + sizeof(uint64_t); + memcpy(&size_mmd, p, sizeof(uint64_t)); + p = p + sizeof(uint64_t); + memcpy(&size_mdidx, p, sizeof(uint64_t)); + p = p + sizeof(uint64_t); + + std::string hint("when processing metadata from memory"); + size_t pos = 0; + + m_Metadata.Resize(size_md, hint); + helper::CopyToBuffer(m_Metadata.m_Buffer, pos, p, size_md); + p = p + size_md; + + pos = 0; + m_MetaMetadata.Resize(size_mmd, hint); + helper::CopyToBuffer(m_MetaMetadata.m_Buffer, pos, p, size_mmd); + p = p + size_mmd; + + pos = 0; + m_MetadataIndex.Resize(size_mdidx, hint); + helper::CopyToBuffer(m_MetadataIndex.m_Buffer, pos, p, size_mdidx); + p = p + size_mdidx; + + size_t parsedIdxSize = 0; + const auto stepsBefore = m_StepsCount; + + parsedIdxSize = ParseMetadataIndex(m_MetadataIndex, 0, true); + + // cut down the index buffer by throwing away the read but unprocessed + // steps + m_MetadataIndex.m_Buffer.resize(parsedIdxSize); + // next time read index file from this position + m_MDIndexFileAlreadyReadSize += parsedIdxSize; + + // At this point first in time we learned the writer's major and we can + // create the serializer object + if (!m_BP5Deserializer) + { + m_BP5Deserializer = + new format::BP5Deserializer(m_WriterIsRowMajor, m_ReaderIsRowMajor, + (m_OpenMode == Mode::ReadRandomAccess)); + m_BP5Deserializer->m_Engine = this; + } + + if (m_StepsCount > stepsBefore) + { + InstallMetaMetaData(m_MetaMetadata); + + if (m_OpenMode == Mode::ReadRandomAccess) + { + for (size_t Step = 0; Step < m_MetadataIndexTable.size(); Step++) + { + m_BP5Deserializer->SetupForStep( + Step, m_WriterMap[m_WriterMapIndex[Step]].WriterCount); + InstallMetadataForTimestep(Step); + } + } + } +} + void BP5Reader::InstallMetadataForTimestep(size_t Step) { size_t pgstart = m_MetadataIndexTable[Step][0]; @@ -455,19 +562,23 @@ void BP5Reader::Init() m_SelectedSteps.ParseSelection(m_Parameters.SelectSteps); } - /* Do a collective wait for the file(s) to appear within timeout. - Make sure every process comes to the same conclusion */ - const Seconds timeoutSeconds = Seconds(m_Parameters.OpenTimeoutSecs); - - Seconds pollSeconds = Seconds(m_Parameters.BeginStepPollingFrequencySecs); - if (pollSeconds > timeoutSeconds) + if (readMetadataFromFile) { - pollSeconds = timeoutSeconds; - } + /* Do a collective wait for the file(s) to appear within timeout. + Make sure every process comes to the same conclusion */ + const Seconds timeoutSeconds = Seconds(m_Parameters.OpenTimeoutSecs); + + Seconds pollSeconds = + Seconds(m_Parameters.BeginStepPollingFrequencySecs); + if (pollSeconds > timeoutSeconds) + { + pollSeconds = timeoutSeconds; + } - TimePoint timeoutInstant = Now() + timeoutSeconds; - OpenFiles(timeoutInstant, pollSeconds, timeoutSeconds); - UpdateBuffer(timeoutInstant, pollSeconds / 10, timeoutSeconds); + TimePoint timeoutInstant = Now() + timeoutSeconds; + OpenFiles(timeoutInstant, pollSeconds, timeoutSeconds); + UpdateBuffer(timeoutInstant, pollSeconds / 10, timeoutSeconds); + } } void BP5Reader::InitParameters() diff --git a/source/adios2/engine/bp5/BP5Reader.h b/source/adios2/engine/bp5/BP5Reader.h index d159ce83d5..45d803f208 100644 --- a/source/adios2/engine/bp5/BP5Reader.h +++ b/source/adios2/engine/bp5/BP5Reader.h @@ -45,8 +45,13 @@ class BP5Reader : public BP5Engine, public Engine BP5Reader(IO &io, const std::string &name, const Mode mode, helper::Comm comm); + BP5Reader(IO &io, const std::string &name, const Mode mode, + helper::Comm comm, const char *md, const size_t mdsize); + ~BP5Reader(); + void GetMetadata(char **md, size_t *size) final; + StepStatus BeginStep(StepMode mode = StepMode::Read, const float timeoutSeconds = -1.0) final; @@ -114,9 +119,12 @@ class BP5Reader : public BP5Engine, public Engine Minifooter m_Minifooter; + bool readMetadataFromFile = true; + void Init(); void InitParameters(); void InitTransports(); + void ProcessMetadataFromMemory(const char *md); /* Sleep up to pollSeconds time if we have not reached timeoutInstant. * Return true if slept diff --git a/source/adios2/engine/bp5/BP5Writer.cpp b/source/adios2/engine/bp5/BP5Writer.cpp index 7c5af59723..6c8e9293e6 100644 --- a/source/adios2/engine/bp5/BP5Writer.cpp +++ b/source/adios2/engine/bp5/BP5Writer.cpp @@ -547,12 +547,14 @@ void BP5Writer::EndStep() m_Profiler.Start("AWD"); // TSInfo destructor would delete the DataBuffer so we need to save it // for async IO and let the writer free it up when not needed anymore - adios2::format::BufferV *databuf = TSInfo.DataBuffer; - TSInfo.DataBuffer = NULL; m_AsyncWriteLock.lock(); m_flagRush = false; m_AsyncWriteLock.unlock(); - WriteData(databuf); + + // WriteData will free TSInfo.DataBuffer + WriteData(TSInfo.DataBuffer); + TSInfo.DataBuffer = NULL; + m_Profiler.Stop("AWD"); /* diff --git a/source/adios2/engine/daos/BP5Writer_TwoLevelShm.cpp b/source/adios2/engine/daos/BP5Writer_TwoLevelShm.cpp new file mode 100644 index 0000000000..9ef11dd74b --- /dev/null +++ b/source/adios2/engine/daos/BP5Writer_TwoLevelShm.cpp @@ -0,0 +1,298 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * DaosWriter.cpp + * + */ + +#include "DaosWriter.h" + +#include "adios2/common/ADIOSMacros.h" +#include "adios2/core/IO.h" +#include "adios2/helper/adiosFunctions.h" //CheckIndexRange, PaddingToAlignOffset +#include "adios2/toolkit/format/buffer/chunk/ChunkV.h" +#include "adios2/toolkit/format/buffer/malloc/MallocV.h" +#include "adios2/toolkit/shm/TokenChain.h" +#include "adios2/toolkit/transport/file/FileFStream.h" +#include + +#include +#include +#include + +namespace adios2 +{ +namespace core +{ +namespace engine +{ + +using namespace adios2::format; + +void DaosWriter::WriteData_TwoLevelShm(format::BufferV *Data) +{ + aggregator::MPIShmChain *a = + dynamic_cast(m_Aggregator); + + // new step writing starts at offset m_DataPos on master aggregator + // other aggregators to the same file will need to wait for the position + // to arrive from the rank below + + // align to PAGE_SIZE (only valid on master aggregator at this point) + m_DataPos += + helper::PaddingToAlignOffset(m_DataPos, m_Parameters.StripeSize); + + // Each aggregator needs to know the total size they write + // This calculation is valid on aggregators only + std::vector mySizes = a->m_Comm.GatherValues(Data->Size()); + uint64_t myTotalSize = 0; + uint64_t maxSize = 0; + for (auto s : mySizes) + { + myTotalSize += s; + if (s > maxSize) + { + maxSize = s; + } + } + + if (a->m_Comm.Size() > 1) + { + size_t alignment_size = sizeof(max_align_t); + if (m_Parameters.DirectIO) + { + alignment_size = m_Parameters.DirectIOAlignOffset; + } + a->CreateShm(static_cast(maxSize), m_Parameters.MaxShmSize, + alignment_size); + } + + shm::TokenChain tokenChain(&a->m_Comm); + + if (a->m_IsAggregator) + { + // In each aggregator chain, send from master down the line + // these total sizes, so every aggregator knows where to start + if (a->m_AggregatorChainComm.Rank() > 0) + { + a->m_AggregatorChainComm.Recv( + &m_DataPos, 1, a->m_AggregatorChainComm.Rank() - 1, 0, + "AggregatorChain token in DaosWriter::WriteData_TwoLevelShm"); + // align to PAGE_SIZE + m_DataPos += helper::PaddingToAlignOffset(m_DataPos, + m_Parameters.StripeSize); + } + m_StartDataPos = m_DataPos; // metadata needs this info + if (a->m_AggregatorChainComm.Rank() < + a->m_AggregatorChainComm.Size() - 1) + { + uint64_t nextWriterPos = m_DataPos + myTotalSize; + a->m_AggregatorChainComm.Isend( + &nextWriterPos, 1, a->m_AggregatorChainComm.Rank() + 1, 0, + "Chain token in DaosWriter::WriteData"); + } + else if (a->m_AggregatorChainComm.Size() > 1) + { + // send back final position from last aggregator in file to master + // aggregator + uint64_t nextWriterPos = m_DataPos + myTotalSize; + a->m_AggregatorChainComm.Isend( + &nextWriterPos, 1, 0, 0, + "Chain token in DaosWriter::WriteData"); + } + + /*std::cout << "Rank " << m_Comm.Rank() + << " aggregator start writing step " << m_WriterStep + << " to subfile " << a->m_SubStreamIndex << " at pos " + << m_DataPos << " totalsize " << myTotalSize << std::endl;*/ + + // Send token to first non-aggregator to start filling shm + // Also informs next process its starting offset (for correct metadata) + uint64_t nextWriterPos = m_DataPos + Data->Size(); + tokenChain.SendToken(nextWriterPos); + + WriteMyOwnData(Data); + + /* Write from shm until every non-aggr sent all data */ + if (a->m_Comm.Size() > 1) + { + WriteOthersData(myTotalSize - Data->Size()); + } + + // Master aggregator needs to know where the last writing ended by the + // last aggregator in the chain, so that it can start from the correct + // position at the next output step + if (a->m_AggregatorChainComm.Size() > 1 && + !a->m_AggregatorChainComm.Rank()) + { + a->m_AggregatorChainComm.Recv( + &m_DataPos, 1, a->m_AggregatorChainComm.Size() - 1, 0, + "Chain token in DaosWriter::WriteData"); + } + } + else + { + // non-aggregators fill shared buffer in marching order + // they also receive their starting offset this way + m_StartDataPos = tokenChain.RecvToken(); + + /*std::cout << "Rank " << m_Comm.Rank() + << " non-aggregator recv token to fill shm = " + << m_StartDataPos << std::endl;*/ + + SendDataToAggregator(Data); + + uint64_t nextWriterPos = m_StartDataPos + Data->Size(); + tokenChain.SendToken(nextWriterPos); + } + + if (a->m_Comm.Size() > 1) + { + a->DestroyShm(); + } +} + +void DaosWriter::WriteMyOwnData(format::BufferV *Data) +{ + std::vector DataVec = Data->DataVec(); + m_StartDataPos = m_DataPos; + m_FileDataManager.WriteFileAt(DataVec.data(), DataVec.size(), + m_StartDataPos); + m_DataPos += Data->Size(); +} + +/*std::string DoubleBufferToString(const double *b, int n) +{ + std::ostringstream out; + out.precision(1); + out << std::fixed << "["; + char s[32]; + + for (int i = 0; i < n; ++i) + { + snprintf(s, sizeof(s), "%g", b[i]); + out << s; + if (i < n - 1) + { + out << ", "; + } + } + out << "]"; + return out.str(); +}*/ + +void DaosWriter::SendDataToAggregator(format::BufferV *Data) +{ + /* Only one process is running this function at once + See shmFillerToken in the caller function + + In a loop, copy the local data into the shared memory, alternating + between the two segments. + */ + + aggregator::MPIShmChain *a = + dynamic_cast(m_Aggregator); + + std::vector DataVec = Data->DataVec(); + size_t nBlocks = DataVec.size(); + + // size_t sent = 0; + size_t block = 0; + size_t temp_offset = 0; + while (block < nBlocks) + { + // potentially blocking call waiting on Aggregator + aggregator::MPIShmChain::ShmDataBuffer *b = a->LockProducerBuffer(); + // b->max_size: how much we can copy + // b->actual_size: how much we actually copy + b->actual_size = 0; + while (true) + { + /* Copy n bytes from the current block, current offset to shm + making sure to use up to shm_size bytes + */ + size_t n = DataVec[block].iov_len - temp_offset; + if (n > (b->max_size - b->actual_size)) + { + n = b->max_size - b->actual_size; + } + std::memcpy(&b->buf[b->actual_size], + (const char *)DataVec[block].iov_base + temp_offset, n); + b->actual_size += n; + + /* Have we processed the entire block or staying with it? */ + if (n + temp_offset < DataVec[block].iov_len) + { + temp_offset += n; + } + else + { + temp_offset = 0; + ++block; + } + + /* Have we reached the max allowed shm size ?*/ + if (b->actual_size >= b->max_size) + { + break; + } + if (block >= nBlocks) + { + break; + } + } + // sent += b->actual_size; + + /*if (m_RankMPI >= 42) + { + std::cout << "Rank " << m_Comm.Rank() + << " filled shm, data_size = " << b->actual_size + << " block = " << block + << " temp offset = " << temp_offset << " sent = " << sent + << " buf = " << static_cast(b->buf) << " = " + << DoubleBufferToString((double *)b->buf, + b->actual_size / sizeof(double)) + << std::endl; + }*/ + + a->UnlockProducerBuffer(); + } +} +void DaosWriter::WriteOthersData(size_t TotalSize) +{ + /* Only an Aggregator calls this function */ + aggregator::MPIShmChain *a = + dynamic_cast(m_Aggregator); + + size_t wrote = 0; + while (wrote < TotalSize) + { + // potentially blocking call waiting on some non-aggr process + aggregator::MPIShmChain::ShmDataBuffer *b = a->LockConsumerBuffer(); + + /*std::cout << "Rank " << m_Comm.Rank() + << " write from shm, data_size = " << b->actual_size + << " total so far = " << wrote + << " buf = " << static_cast(b->buf) << " = " + << DoubleBufferToString((double *)b->buf, + b->actual_size / sizeof(double)) + << std::endl;*/ + /*<< " buf = " << static_cast(b->buf) << " = [" + << (int)b->buf[0] << (int)b->buf[1] << "..." + << (int)b->buf[b->actual_size - 2] + << (int)b->buf[b->actual_size - 1] << "]" << std::endl;*/ + + // b->actual_size: how much we need to write + m_FileDataManager.WriteFiles(b->buf, b->actual_size); + + wrote += b->actual_size; + + a->UnlockConsumerBuffer(); + } + m_DataPos += TotalSize; +} + +} // end namespace engine +} // end namespace core +} // end namespace adios2 diff --git a/source/adios2/engine/daos/DaosEngine.cpp b/source/adios2/engine/daos/DaosEngine.cpp new file mode 100644 index 0000000000..5b0dd0d37d --- /dev/null +++ b/source/adios2/engine/daos/DaosEngine.cpp @@ -0,0 +1,377 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * DaosEngine.cpp + * + */ + +#include "DaosEngine.h" + +#include "adios2/common/ADIOSMacros.h" +#include "adios2/common/ADIOSTypes.h" //PathSeparator +#include "adios2/core/IO.h" +#include "adios2/helper/adiosFunctions.h" //CreateDirectory, StringToTimeUnit, + +#include +#include + +namespace adios2 +{ +namespace core +{ +namespace engine +{ + +std::vector +DaosEngine::GetBPMetadataFileNames(const std::vector &names) const + noexcept +{ + std::vector metadataFileNames; + metadataFileNames.reserve(names.size()); + for (const auto &name : names) + { + metadataFileNames.push_back(GetBPMetadataFileName(name)); + } + return metadataFileNames; +} + +std::vector DaosEngine::GetBPMetaMetadataFileNames( + const std::vector &names) const noexcept +{ + std::vector metaMetadataFileNames; + metaMetadataFileNames.reserve(names.size()); + for (const auto &name : names) + { + metaMetadataFileNames.push_back(GetBPMetaMetadataFileName(name)); + } + return metaMetadataFileNames; +} + +std::string DaosEngine::GetBPMetadataFileName(const std::string &name) const + noexcept +{ + const std::string bpName = helper::RemoveTrailingSlash(name); + const size_t index = 0; // global metadata file is generated by rank 0 + /* the name of the metadata file is "md.0" */ + const std::string bpMetaDataRankName(bpName + PathSeparator + "md." + + std::to_string(index)); + return bpMetaDataRankName; +} + +std::string DaosEngine::GetBPMetaMetadataFileName(const std::string &name) const + noexcept +{ + const std::string bpName = helper::RemoveTrailingSlash(name); + const size_t index = 0; // global metadata file is generated by rank 0 + /* the name of the metadata file is "md.0" */ + const std::string bpMetaMetaDataRankName(bpName + PathSeparator + "mmd." + + std::to_string(index)); + return bpMetaMetaDataRankName; +} + +std::vector DaosEngine::GetBPMetadataIndexFileNames( + const std::vector &names) const noexcept +{ + std::vector metadataIndexFileNames; + metadataIndexFileNames.reserve(names.size()); + for (const auto &name : names) + { + metadataIndexFileNames.push_back(GetBPMetadataIndexFileName(name)); + } + return metadataIndexFileNames; +} + +std::string +DaosEngine::GetBPMetadataIndexFileName(const std::string &name) const noexcept +{ + const std::string bpName = helper::RemoveTrailingSlash(name); + /* the name of the metadata index file is "md.idx" */ + const std::string bpMetaDataIndexRankName(bpName + PathSeparator + + "md.idx"); + return bpMetaDataIndexRankName; +} + +std::vector +DaosEngine::GetBPVersionFileNames(const std::vector &names) const + noexcept +{ + std::vector versionFileNames; + versionFileNames.reserve(names.size()); + for (const auto &name : names) + { + versionFileNames.push_back(GetBPVersionFileName(name)); + } + return versionFileNames; +} + +std::string DaosEngine::GetBPVersionFileName(const std::string &name) const + noexcept +{ + const std::string bpName = helper::RemoveTrailingSlash(name); + /* the name of the version file is ".bpversion" */ + const std::string bpVersionFileName(bpName + PathSeparator + ".bpversion"); + return bpVersionFileName; +} + +std::string DaosEngine::GetBPSubStreamName(const std::string &name, + const size_t id, + const bool hasSubFiles, + const bool isReader) const noexcept +{ + if (!hasSubFiles) + { + return name; + } + + const std::string bpName = helper::RemoveTrailingSlash(name); + /* the name of a data file starts with "data." */ + const std::string bpRankName(bpName + PathSeparator + "data." + + std::to_string(id)); + return bpRankName; +} + +std::vector +DaosEngine::GetBPSubStreamNames(const std::vector &names, + size_t subFileIndex) const noexcept +{ + std::vector bpNames; + bpNames.reserve(names.size()); + for (const auto &name : names) + { + bpNames.push_back(GetBPSubStreamName(name, subFileIndex)); + } + return bpNames; +} + +void DaosEngine::ParseParams(IO &io, struct DAOSParams &Params) +{ + adios2::Params params_lowercase; + for (auto &p : io.m_Parameters) + { + const std::string key = helper::LowerCase(p.first); + const std::string value = helper::LowerCase(p.second); + params_lowercase[key] = value; + } + + auto lf_SetBoolParameter = [&](const std::string key, bool ¶meter, + bool def) { + const std::string lkey = helper::LowerCase(std::string(key)); + auto itKey = params_lowercase.find(lkey); + parameter = def; + if (itKey != params_lowercase.end()) + { + std::string value = itKey->second; + std::transform(value.begin(), value.end(), value.begin(), + ::tolower); + if (value == "yes" || value == "true" || value == "on") + { + parameter = true; + } + else if (value == "no" || value == "false" || value == "off") + { + parameter = false; + } + else + { + helper::Throw( + "Engine", "DaosEngine", "ParseParams", + "Unknown BP5 Boolean parameter '" + value + "'"); + } + } + }; + + auto lf_SetFloatParameter = [&](const std::string key, float ¶meter, + float def) { + const std::string lkey = helper::LowerCase(std::string(key)); + auto itKey = params_lowercase.find(lkey); + parameter = def; + if (itKey != params_lowercase.end()) + { + std::string value = itKey->second; + parameter = + helper::StringTo(value, " in Parameter key=" + key); + } + }; + + auto lf_SetSizeBytesParameter = [&](const std::string key, + size_t ¶meter, size_t def) { + const std::string lkey = helper::LowerCase(std::string(key)); + auto itKey = params_lowercase.find(lkey); + parameter = def; + if (itKey != params_lowercase.end()) + { + std::string value = itKey->second; + parameter = helper::StringToByteUnits( + value, "for Parameter key=" + key + "in call to Open"); + } + }; + + auto lf_SetIntParameter = [&](const std::string key, int ¶meter, + int def) { + const std::string lkey = helper::LowerCase(std::string(key)); + auto itKey = params_lowercase.find(lkey); + parameter = def; + if (itKey != params_lowercase.end()) + { + parameter = std::stoi(itKey->second); + return true; + } + return false; + }; + + auto lf_SetUIntParameter = [&](const std::string key, + unsigned int ¶meter, unsigned int def) { + const std::string lkey = helper::LowerCase(std::string(key)); + auto itKey = params_lowercase.find(lkey); + parameter = def; + if (itKey != params_lowercase.end()) + { + unsigned long result = std::stoul(itKey->second); + if (result > std::numeric_limits::max()) + { + result = std::numeric_limits::max(); + } + parameter = static_cast(result); + return true; + } + return false; + }; + + auto lf_SetStringParameter = [&](const std::string key, + std::string ¶meter, const char *def) { + const std::string lkey = helper::LowerCase(std::string(key)); + auto itKey = params_lowercase.find(lkey); + parameter = def; + if (itKey != params_lowercase.end()) + { + parameter = itKey->second; + return true; + } + return false; + }; + + auto lf_SetBufferVTypeParameter = [&](const std::string key, int ¶meter, + int def) { + const std::string lkey = helper::LowerCase(std::string(key)); + auto itKey = params_lowercase.find(lkey); + parameter = def; + if (itKey != params_lowercase.end()) + { + std::string value = itKey->second; + std::transform(value.begin(), value.end(), value.begin(), + ::tolower); + if (value == "malloc") + { + parameter = (int)BufferVType::MallocVType; + } + else if (value == "chunk") + { + parameter = (int)BufferVType::ChunkVType; + } + else + { + helper::Throw( + "Engine", "DaosEngine", "ParseParams", + "Unknown BP5 BufferVType parameter \"" + value + + "\" (must be \"malloc\" or \"chunk\""); + } + } + }; + + auto lf_SetAggregationTypeParameter = [&](const std::string key, + int ¶meter, int def) { + const std::string lkey = helper::LowerCase(std::string(key)); + auto itKey = params_lowercase.find(lkey); + parameter = def; + if (itKey != params_lowercase.end()) + { + std::string value = itKey->second; + std::transform(value.begin(), value.end(), value.begin(), + ::tolower); + if (value == "everyonewrites" || value == "auto") + { + parameter = (int)AggregationType::EveryoneWrites; + } + else if (value == "everyonewritesserial") + { + parameter = (int)AggregationType::EveryoneWritesSerial; + } + else if (value == "twolevelshm") + { + parameter = (int)AggregationType::TwoLevelShm; + } + else + { + helper::Throw( + "Engine", "DaosEngine", "ParseParams", + "Unknown BP5 AggregationType parameter \"" + value + + "\" (must be \"auto\", \"everyonewrites\" or " + "\"twolevelshm\""); + } + } + }; + + auto lf_SetAsyncWriteParameter = [&](const std::string key, int ¶meter, + int def) { + const std::string lkey = helper::LowerCase(std::string(key)); + auto itKey = params_lowercase.find(lkey); + parameter = def; + if (itKey != params_lowercase.end()) + { + std::string value = itKey->second; + std::transform(value.begin(), value.end(), value.begin(), + ::tolower); + if (value == "guided" || value == "auto" || value == "on" || + value == "true") + { + parameter = (int)AsyncWrite::Guided; + } + else if (value == "sync" || value == "off" || value == "false") + { + parameter = (int)AsyncWrite::Sync; + } + else if (value == "naive") + { + parameter = (int)AsyncWrite::Naive; + } + else + { + helper::Throw( + "Engine", "DaosEngine", "ParseParams", + "Unknown BP5 AsyncWriteMode parameter \"" + value + + "\" (must be \"auto\", \"sync\", \"naive\", " + "\"throttled\" " + "or \"guided\""); + } + } + }; + +#define get_params(Param, Type, Typedecl, Default) \ + lf_Set##Type##Parameter(#Param, Params.Param, Default); + + DAOS_FOREACH_PARAMETER_TYPE_4ARGS(get_params); +#undef get_params + + if (Params.verbose > 0 && !m_RankMPI) + { + std::cout << "---------------- " << io.m_EngineType + << " engine parameters --------------\n"; +#define print_params(Param, Type, Typedecl, Default) \ + lf_Set##Type##Parameter(#Param, Params.Param, Default); \ + if (!m_RankMPI) \ + { \ + std::cout << " " << std::string(#Param) << " = " << Params.Param \ + << " default = " << Default << std::endl; \ + } + + DAOS_FOREACH_PARAMETER_TYPE_4ARGS(print_params); +#undef print_params + std::cout << "-----------------------------------------------------" + << std::endl; + } +}; + +} // namespace engine +} // namespace core +} // namespace adios2 diff --git a/source/adios2/engine/daos/DaosEngine.h b/source/adios2/engine/daos/DaosEngine.h new file mode 100644 index 0000000000..b4801fab1a --- /dev/null +++ b/source/adios2/engine/daos/DaosEngine.h @@ -0,0 +1,224 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * DaosEngine.h + * + */ + +#ifndef ADIOS2_ENGINE_DAOS_DAOSENGINE_H_ +#define ADIOS2_ENGINE_DAOS_DAOSENGINE_H_ + +#include "adios2/common/ADIOSConfig.h" +#include "adios2/core/Engine.h" +#include "adios2/helper/adiosComm.h" +#include "adios2/toolkit/burstbuffer/FileDrainerSingleThread.h" +#include "adios2/toolkit/format/bp5/BP5Serializer.h" +#include "adios2/toolkit/transportman/TransportMan.h" + +namespace adios2 +{ +namespace core +{ +namespace engine +{ + +class DaosEngine +{ +public: + int m_RankMPI = 0; + /* metadata index table + 0: pos in memory for step (after filtered read) + 1: size of metadata + 2: flush count + 3: pos in index where data offsets are enumerated + 4: abs. pos in metadata File for step + */ + std::unordered_map> m_MetadataIndexTable; + + struct Minifooter + { + std::string VersionTag; + uint64_t PGIndexStart = 0; + uint64_t VarsIndexStart = 0; + uint64_t AttributesIndexStart = 0; + int8_t Version = -1; + bool IsLittleEndian = true; + bool HasSubFiles = false; + }; + + format::BufferSTL m_MetadataIndex; + + /** Positions of flags in Index Table Header that Reader uses */ + static constexpr size_t m_IndexHeaderSize = 64; + static constexpr size_t m_EndianFlagPosition = 36; + static constexpr size_t m_BPVersionPosition = 37; + static constexpr size_t m_BPMinorVersionPosition = 38; + static constexpr size_t m_ActiveFlagPosition = 39; + static constexpr size_t m_ColumnMajorFlagPosition = 40; + static constexpr size_t m_VersionTagPosition = 0; + static constexpr size_t m_VersionTagLength = 32; + + static constexpr uint8_t m_BP5MinorVersion = 2; + + /** Index record types */ + enum IndexRecord + { + StepRecord = 's', + WriterMapRecord = 'w', + }; + + std::vector + GetBPSubStreamNames(const std::vector &names, + size_t subFileIndex) const noexcept; + + std::vector + GetBPMetadataFileNames(const std::vector &names) const + noexcept; + std::vector + GetBPMetaMetadataFileNames(const std::vector &names) const + noexcept; + std::string GetBPMetadataFileName(const std::string &name) const noexcept; + std::string GetBPMetaMetadataFileName(const std::string &name) const + noexcept; + std::vector + GetBPMetadataIndexFileNames(const std::vector &names) const + noexcept; + + std::string GetBPMetadataIndexFileName(const std::string &name) const + noexcept; + + std::string GetBPSubStreamName(const std::string &name, const size_t id, + const bool hasSubFiles = true, + const bool isReader = false) const noexcept; + + std::vector + GetBPVersionFileNames(const std::vector &names) const noexcept; + + std::string GetBPVersionFileName(const std::string &name) const noexcept; + + enum class BufferVType + { + MallocVType, + ChunkVType, + Auto + }; + + BufferVType UseBufferV = BufferVType::ChunkVType; + + enum class AggregationType + { + EveryoneWrites, + EveryoneWritesSerial, + TwoLevelShm, + Auto + }; + + enum class AsyncWrite + { + Sync = 0, // enable using AsyncWriteMode as bool expression + Naive, + Guided + }; + + /** + * sub-block size for min/max calculation of large arrays in number of + * elements (not bytes). The default big number per Put() default will + * result in the original single min/max value-pair per block + */ + const size_t DefaultStatsBlockSize = 1125899906842624ULL; + +#define DAOS_FOREACH_PARAMETER_TYPE_4ARGS(MACRO) \ + MACRO(OpenTimeoutSecs, Float, float, -1.0f) \ + MACRO(BeginStepPollingFrequencySecs, Float, float, 1.0f) \ + MACRO(StreamReader, Bool, bool, false) \ + MACRO(BurstBufferDrain, Bool, bool, true) \ + MACRO(BurstBufferPath, String, std::string, "") \ + MACRO(NodeLocal, Bool, bool, false) \ + MACRO(verbose, Int, int, 0) \ + MACRO(CollectiveMetadata, Bool, bool, true) \ + MACRO(NumAggregators, UInt, unsigned int, 0) \ + MACRO(AggregatorRatio, UInt, unsigned int, 0) \ + MACRO(NumSubFiles, UInt, unsigned int, 0) \ + MACRO(StripeSize, UInt, unsigned int, 4096) \ + MACRO(DirectIO, Bool, bool, false) \ + MACRO(DirectIOAlignOffset, UInt, unsigned int, 512) \ + MACRO(DirectIOAlignBuffer, UInt, unsigned int, 0) \ + MACRO(AggregationType, AggregationType, int, \ + (int)AggregationType::TwoLevelShm) \ + MACRO(AsyncOpen, Bool, bool, true) \ + MACRO(AsyncWrite, AsyncWrite, int, (int)AsyncWrite::Sync) \ + MACRO(GrowthFactor, Float, float, DefaultBufferGrowthFactor) \ + MACRO(InitialBufferSize, SizeBytes, size_t, DefaultInitialBufferSize) \ + MACRO(MinDeferredSize, SizeBytes, size_t, DefaultMinDeferredSize) \ + MACRO(BufferChunkSize, SizeBytes, size_t, DefaultBufferChunkSize) \ + MACRO(MaxShmSize, SizeBytes, size_t, DefaultMaxShmSize) \ + MACRO(BufferVType, BufferVType, int, (int)BufferVType::ChunkVType) \ + MACRO(AppendAfterSteps, Int, int, INT_MAX) \ + MACRO(SelectSteps, String, std::string, "") \ + MACRO(ReaderShortCircuitReads, Bool, bool, false) \ + MACRO(StatsLevel, UInt, unsigned int, 1) \ + MACRO(StatsBlockSize, SizeBytes, size_t, DefaultStatsBlockSize) \ + MACRO(Threads, UInt, unsigned int, 0) \ + MACRO(UseOneTimeAttributes, Bool, bool, true) \ + MACRO(MaxOpenFilesAtOnce, UInt, unsigned int, UINT_MAX) + + struct DAOSParams + { +#define declare_struct(Param, Type, Typedecl, Default) Typedecl Param; + DAOS_FOREACH_PARAMETER_TYPE_4ARGS(declare_struct) +#undef declare_struct + }; + + void ParseParams(IO &io, DAOSParams &Params); + DAOSParams m_Parameters; + +private: +}; + +} // namespace engine +} // namespace core +} // namespace adios2 +#endif + +/* + * Data Formats: + * MetadataIndex file (md.idx) + * BP5 header for "Index Table" (64 bytes) + * for each Writer, what aggregator writes its data + * uint16_t [ WriterCount] + * for each timestep: (size (WriterCount + 2 ) 64-bit ints + * uint64_t 0 : CombinedMetaDataPos + * uint64_t 1 : CombinedMetaDataSize + * uint64_t 2 : FlushCount + * for each Writer + * for each flush before the last: + * uint64_t DataPos (in the file above) + * uint64_t DataSize + * for the final flush: + * uint64_t DataPos (in the file above) + * So, each timestep takes sizeof(uint64_t)* (3 + ((FlushCount-1)*2 + + *1) * WriterCount) bytes + * + * MetaMetadata file (mmd.0) contains FFS format information + * for each meta metadata item: + * uint64_t MetaMetaIDLen + * uint64_t MetaMetaInfoLen + * char[MeatMetaIDLen] MetaMetaID + * char[MetaMetaInfoLen] MetaMetanfo + * Notes: This file should be quite small, with size dependent upon the + *number of different "formats" written by any rank. + * + * + * MetaData file (md.0) contains encoded metadata/attribute data + * BP5 header for "Metadata" (64 bytes) + * for each timestep: + * uint64_t : TotalSize of this metadata block + * uint64_t[WriterCount] : Length of each rank's metadata + * uint64_t[WriterCount] : Length of each rank's attribute + * FFS-encoded metadata block of the length above + * FFS-encoded attribute data block of the length above + * + * Data file (data.x) contains a block of data for each timestep, for each + *rank + */ diff --git a/source/adios2/engine/daos/DaosReader.cpp b/source/adios2/engine/daos/DaosReader.cpp new file mode 100644 index 0000000000..f0bc8af734 --- /dev/null +++ b/source/adios2/engine/daos/DaosReader.cpp @@ -0,0 +1,1430 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * DaosReader.cpp + * + */ + +#include "DaosReader.h" +#include "DaosReader.tcc" + +#include "adios2/helper/adiosMath.h" // SetWithinLimit +#include + +#include +#include +#include +#include +#include + +using TP = std::chrono::high_resolution_clock::time_point; +#define NOW() std::chrono::high_resolution_clock::now(); +#define DURATION(T1, T2) static_cast((T2 - T1).count()) / 1000000000.0; + +namespace adios2 +{ +namespace core +{ +namespace engine +{ + +DaosReader::DaosReader(IO &io, const std::string &name, const Mode mode, + helper::Comm comm) +: Engine("DaosReader", io, name, mode, std::move(comm)), + m_MDFileManager(io, m_Comm), m_DataFileManager(io, m_Comm), + m_MDIndexFileManager(io, m_Comm), m_FileMetaMetadataManager(io, m_Comm), + m_ActiveFlagFileManager(io, m_Comm) +{ + PERFSTUBS_SCOPED_TIMER("DaosReader::Open"); + Init(); + m_IsOpen = true; +} + +DaosReader::~DaosReader() +{ + if (m_BP5Deserializer) + delete m_BP5Deserializer; + if (m_IsOpen) + { + DestructorClose(m_FailVerbose); + } + m_IsOpen = false; +} + +void DaosReader::DestructorClose(bool Verbose) noexcept +{ + // Nothing special needs to be done to "close" a BP5 reader during shutdown + // if it hasn't already been Closed + m_IsOpen = false; +} + +void DaosReader::InstallMetadataForTimestep(size_t Step) +{ + size_t pgstart = m_MetadataIndexTable[Step][0]; + size_t Position = pgstart + sizeof(uint64_t); // skip total data size + const uint64_t WriterCount = + m_WriterMap[m_WriterMapIndex[Step]].WriterCount; + size_t MDPosition = Position + 2 * sizeof(uint64_t) * WriterCount; + for (size_t WriterRank = 0; WriterRank < WriterCount; WriterRank++) + { + // variable metadata for timestep + // size_t ThisMDSize = helper::ReadValue( + // m_Metadata.m_Buffer, Position, m_Minifooter.IsLittleEndian); + // char *ThisMD = m_Metadata.m_Buffer.data() + MDPosition; + size_t ThisMDSize; + char *ThisMD; + + // DAOS temporary declarations + char key[1000]; + int rc; + + // Query size of a writer rank's metadata + sprintf(key, "step%d-rank%d", Step, WriterRank); + std::cout << __FILE__ << "::" << __func__ << "(), step: " << Step + << std::endl; + std::cout << "key = " << key << std::endl; + rc = daos_kv_get(oh, DAOS_TX_NONE, 0, key, &ThisMDSize, NULL, NULL); + ASSERT(rc == 0, "daos_kv_get() failed to get size with %d", rc); + std::cout << "WriterRank = " << WriterRank + << ", ThisMDSize = " << ThisMDSize << std::endl; + + // Allocate memory + ThisMD = new char[ThisMDSize]; + + // Read writer's metadata + rc = daos_kv_get(oh, DAOS_TX_NONE, 0, key, &ThisMDSize, ThisMD, NULL); + ASSERT(rc == 0, "daos_kv_get() failed to read metadata with %d", rc); + + std::cout << "Printing the first 10 bytes of Metadata" << std::endl; + char *data = reinterpret_cast(ThisMD); + for (int i = 0; i < 10; i++) + { + // std::cout << std::hex << std::setw(2) << std::setfill('0') << + // static_cast(data[i]) << " "; + std::cout << static_cast(data[i]) << " "; + } + std::cout << std::endl; + + if (m_OpenMode == Mode::ReadRandomAccess) + { + m_BP5Deserializer->InstallMetaData(ThisMD, ThisMDSize, WriterRank, + Step); + } + else + { + m_BP5Deserializer->InstallMetaData(ThisMD, ThisMDSize, WriterRank); + } + // MDPosition += ThisMDSize; + + // delete[] ThisMD; + } + // for (size_t WriterRank = 0; WriterRank < WriterCount; WriterRank++) + //{ + // // attribute metadata for timestep + // size_t ThisADSize = helper::ReadValue( + // m_Metadata.m_Buffer, Position, m_Minifooter.IsLittleEndian); + // char *ThisAD = m_Metadata.m_Buffer.data() + MDPosition; + // if (ThisADSize > 0) + // m_BP5Deserializer->InstallAttributeData(ThisAD, ThisADSize); + // MDPosition += ThisADSize; + //} +} + +StepStatus DaosReader::BeginStep(StepMode mode, const float timeoutSeconds) +{ + PERFSTUBS_SCOPED_TIMER("DaosReader::BeginStep"); + + if (m_OpenMode == Mode::ReadRandomAccess) + { + helper::Throw( + "Engine", "DaosReader", "BeginStep", + "BeginStep called in random access mode"); + } + if (m_BetweenStepPairs) + { + helper::Throw("Engine", "DaosReader", "BeginStep", + "BeginStep() is called a second time " + "without an intervening EndStep()"); + } + + if (mode != StepMode::Read) + { + helper::Throw( + "Engine", "DaosReader", "BeginStep", + "mode is not supported yet, only Read is valid for engine " + "DaosReader, in call to BeginStep"); + } + + StepStatus status = StepStatus::OK; + if (m_FirstStep) + { + if (!m_StepsCount) + { + // not steps was found in Open/Init, check for new steps now + status = CheckForNewSteps(Seconds(timeoutSeconds)); + } + } + else + { + if (m_CurrentStep + 1 >= m_StepsCount) + { + // we processed steps in memory, check for new steps now + status = CheckForNewSteps(Seconds(timeoutSeconds)); + } + } + + if (status == StepStatus::OK) + { + m_BetweenStepPairs = true; + if (m_FirstStep) + { + m_FirstStep = false; + } + else + { + ++m_CurrentStep; + } + + m_IO.m_EngineStep = m_CurrentStep; + // SstBlock AttributeBlockList = + // SstGetAttributeData(m_Input, SstCurrentStep(m_Input)); + // i = 0; + // while (AttributeBlockList && AttributeBlockList[i].BlockData) + // { + // m_IO.RemoveAllAttributes(); + // m_BP5Deserializer->InstallAttributeData( + // AttributeBlockList[i].BlockData, + // AttributeBlockList[i].BlockSize); + // i++; + // } + + m_BP5Deserializer->SetupForStep( + m_CurrentStep, + m_WriterMap[m_WriterMapIndex[m_CurrentStep]].WriterCount); + + /* Remove all existing variables from previous steps + It seems easier than trying to update them */ + // m_IO.RemoveAllVariables(); + + InstallMetadataForTimestep(m_CurrentStep); + m_IO.ResetVariablesStepSelection(false, + "in call to BP5 Reader BeginStep"); + + // caches attributes for each step + // if a variable name is a prefix + // e.g. var prefix = {var/v1, var/v2, var/v3} + m_IO.SetPrefixedNames(true); + } + + return status; +} + +size_t DaosReader::CurrentStep() const { return m_CurrentStep; } + +void DaosReader::EndStep() +{ + if (m_OpenMode == Mode::ReadRandomAccess) + { + helper::Throw("Engine", "DaosReader", "EndStep", + "EndStep called in random access mode"); + } + if (!m_BetweenStepPairs) + { + helper::Throw( + "Engine", "DaosReader", "EndStep", + "EndStep() is called without a successful BeginStep()"); + } + m_BetweenStepPairs = false; + PERFSTUBS_SCOPED_TIMER("DaosReader::EndStep"); + PerformGets(); +} + +std::pair +DaosReader::ReadData(adios2::transportman::TransportMan &FileManager, + const size_t maxOpenFiles, const size_t WriterRank, + const size_t Timestep, const size_t StartOffset, + const size_t Length, char *Destination) +{ + /* + * Warning: this function is called by multiple threads + */ + size_t FlushCount = m_MetadataIndexTable[Timestep][2]; + size_t DataPosPos = m_MetadataIndexTable[Timestep][3]; + size_t SubfileNum = static_cast( + m_WriterMap[m_WriterMapIndex[Timestep]].RankToSubfile[WriterRank]); + + // check if subfile is already opened + TP startSubfile = NOW(); + if (FileManager.m_Transports.count(SubfileNum) == 0) + { + const std::string subFileName = GetBPSubStreamName( + m_Name, SubfileNum, m_Minifooter.HasSubFiles, true); + if (FileManager.m_Transports.size() >= maxOpenFiles) + { + auto m = FileManager.m_Transports.begin(); + FileManager.CloseFiles((int)m->first); + } + FileManager.OpenFileID(subFileName, SubfileNum, Mode::Read, + m_IO.m_TransportsParameters[0], + /*{{"transport", "File"}},*/ false); + } + TP endSubfile = NOW(); + double timeSubfile = DURATION(startSubfile, endSubfile); + + /* Each block is in exactly one flush. The StartOffset was calculated + as if all the flushes were in a single contiguous block in file. + */ + TP startRead = NOW(); + size_t InfoStartPos = + DataPosPos + (WriterRank * (2 * FlushCount + 1) * sizeof(uint64_t)); + size_t SumDataSize = 0; // count in contiguous space + for (size_t flush = 0; flush < FlushCount; flush++) + { + size_t ThisDataPos = + helper::ReadValue(m_MetadataIndex.m_Buffer, InfoStartPos, + m_Minifooter.IsLittleEndian); + size_t ThisDataSize = + helper::ReadValue(m_MetadataIndex.m_Buffer, InfoStartPos, + m_Minifooter.IsLittleEndian); + + if (StartOffset < SumDataSize + ThisDataSize) + { + // discount offsets of skipped flushes + size_t Offset = StartOffset - SumDataSize; + FileManager.ReadFile(Destination, Length, ThisDataPos + Offset, + SubfileNum); + TP endRead = NOW(); + double timeRead = DURATION(startRead, endRead); + return std::make_pair(timeSubfile, timeRead); + } + SumDataSize += ThisDataSize; + } + + size_t ThisDataPos = helper::ReadValue( + m_MetadataIndex.m_Buffer, InfoStartPos, m_Minifooter.IsLittleEndian); + size_t Offset = StartOffset - SumDataSize; + FileManager.ReadFile(Destination, Length, ThisDataPos + Offset, SubfileNum); + + TP endRead = NOW(); + double timeRead = DURATION(startRead, endRead); + return std::make_pair(timeSubfile, timeRead); +} + +void DaosReader::PerformGets() +{ + auto lf_CompareReqSubfile = + [&](adios2::format::BP5Deserializer::ReadRequest &r1, + adios2::format::BP5Deserializer::ReadRequest &r2) -> bool { + return (m_WriterMap[m_WriterMapIndex[r1.Timestep]] + .RankToSubfile[r1.WriterRank] < + m_WriterMap[m_WriterMapIndex[r2.Timestep]] + .RankToSubfile[r2.WriterRank]); + }; + + // TP start = NOW(); + PERFSTUBS_SCOPED_TIMER("DaosReader::PerformGets"); + size_t maxReadSize; + + // TP startGenerate = NOW(); + auto ReadRequests = + m_BP5Deserializer->GenerateReadRequests(false, &maxReadSize); + size_t nRequest = ReadRequests.size(); + // TP endGenerate = NOW(); + // double generateTime = DURATION(startGenerate, endGenerate); + + size_t nextRequest = 0; + std::mutex mutexReadRequests; + + auto lf_GetNextRequest = [&]() -> size_t { + std::lock_guard lockGuard(mutexReadRequests); + size_t reqidx = MaxSizeT; + if (nextRequest < nRequest) + { + reqidx = nextRequest; + ++nextRequest; + } + return reqidx; + }; + + auto lf_Reader = [&](const int FileManagerID, const size_t maxOpenFiles) + -> std::tuple { + double copyTotal = 0.0; + double readTotal = 0.0; + double subfileTotal = 0.0; + size_t nReads = 0; + std::vector buf(maxReadSize); + + while (true) + { + const auto reqidx = lf_GetNextRequest(); + if (reqidx > nRequest) + { + break; + } + auto &Req = ReadRequests[reqidx]; + if (!Req.DestinationAddr) + { + Req.DestinationAddr = buf.data(); + } + std::pair t = + ReadData(fileManagers[FileManagerID], maxOpenFiles, + Req.WriterRank, Req.Timestep, Req.StartOffset, + Req.ReadLength, Req.DestinationAddr); + + TP startCopy = NOW(); + m_BP5Deserializer->FinalizeGet(Req, false); + TP endCopy = NOW(); + subfileTotal += t.first; + readTotal += t.second; + copyTotal += DURATION(startCopy, endCopy); + ++nReads; + } + return std::make_tuple(subfileTotal, readTotal, copyTotal, nReads); + }; + + // TP startRead = NOW(); + // double sortTime = 0.0; + if (m_Threads > 1 && nRequest > 1) + { + // TP startSort = NOW(); + std::sort(ReadRequests.begin(), ReadRequests.end(), + lf_CompareReqSubfile); + // TP endSort = NOW(); + // sortTime = DURATION(startSort, endSort); + size_t nThreads = (m_Threads < nRequest ? m_Threads : nRequest); + + size_t maxOpenFiles = helper::SetWithinLimit( + (size_t)m_Parameters.MaxOpenFilesAtOnce / nThreads, (size_t)1, + MaxSizeT); + + std::vector>> + futures(nThreads - 1); + + // launch Threads-1 threads to process subsets of requests, + // then main thread process the last subset + for (size_t tid = 0; tid < nThreads - 1; ++tid) + { + futures[tid] = std::async(std::launch::async, lf_Reader, tid + 1, + maxOpenFiles); + } + // main thread runs last subset of reads + /*auto tMain = */ lf_Reader(0, maxOpenFiles); + /*{ + double tSubfile = std::get<0>(tMain); + double tRead = std::get<1>(tMain); + double tCopy = std::get<2>(tMain); + size_t nReads = std::get<3>(tMain); + std::cout << " -> PerformGets() thread MAIN total = " + << tSubfile + tRead + tCopy << "s, subfile = " << tSubfile + << "s, read = " << tRead << "s, copy = " << tCopy + << ", nReads = " << nReads << std::endl; + }*/ + + // wait for all async threads + int tid = 1; + for (auto &f : futures) + { + /*auto t = */ f.get(); + /*double tSubfile = std::get<0>(t); + double tRead = std::get<1>(t); + double tCopy = std::get<2>(t); + size_t nReads = std::get<3>(t); + std::cout << " -> PerformGets() thread " << tid + << " total = " << tSubfile + tRead + tCopy + << "s, subfile = " << tSubfile << "s, read = " << tRead + << "s, copy = " << tCopy << ", nReads = " << nReads + << std::endl;*/ + ++tid; + } + } + else + { + size_t maxOpenFiles = helper::SetWithinLimit( + (size_t)m_Parameters.MaxOpenFilesAtOnce, (size_t)1, MaxSizeT); + std::vector buf(maxReadSize); + for (auto &Req : ReadRequests) + { + if (!Req.DestinationAddr) + { + Req.DestinationAddr = buf.data(); + } + ReadData(m_DataFileManager, maxOpenFiles, Req.WriterRank, + Req.Timestep, Req.StartOffset, Req.ReadLength, + Req.DestinationAddr); + m_BP5Deserializer->FinalizeGet(Req, false); + } + } + + // clear pending requests inside deserializer + { + std::vector empty; + m_BP5Deserializer->FinalizeGets(empty); + } + + /*TP end = NOW(); + double t1 = DURATION(start, end); + double t2 = DURATION(startRead, end); + std::cout << " -> PerformGets() total = " << t1 << "s, Read loop = " << t2 + << "s, sort = " << sortTime << "s, generate = " << generateTime + << ", nRequests = " << nRequest << std::endl;*/ +} + +// PRIVATE +void DaosReader::Init() +{ + if ((m_OpenMode != Mode::Read) && (m_OpenMode != Mode::ReadRandomAccess)) + { + helper::Throw( + "Engine", "DaosReader", "Init", + "BPFileReader only supports OpenMode::Read or " + "OpenMode::ReadRandomAccess from" + + m_Name); + } + + // if IO was involved in reading before this flag may be true now + m_IO.m_ReadStreaming = false; + m_ReaderIsRowMajor = (m_IO.m_ArrayOrder == ArrayOrdering::RowMajor); + InitParameters(); + InitTransports(); + InitDAOS(); + if (!m_Parameters.SelectSteps.empty()) + { + m_SelectedSteps.ParseSelection(m_Parameters.SelectSteps); + } + + /* Do a collective wait for the file(s) to appear within timeout. + Make sure every process comes to the same conclusion */ + const Seconds timeoutSeconds = Seconds(m_Parameters.OpenTimeoutSecs); + + Seconds pollSeconds = Seconds(m_Parameters.BeginStepPollingFrequencySecs); + if (pollSeconds > timeoutSeconds) + { + pollSeconds = timeoutSeconds; + } + + TimePoint timeoutInstant = Now() + timeoutSeconds; + OpenFiles(timeoutInstant, pollSeconds, timeoutSeconds); + UpdateBuffer(timeoutInstant, pollSeconds / 10, timeoutSeconds); +} + +void DaosReader::InitParameters() +{ + ParseParams(m_IO, m_Parameters); + if (m_Parameters.OpenTimeoutSecs < 0.0f) + { + if (m_OpenMode == Mode::ReadRandomAccess) + { + m_Parameters.OpenTimeoutSecs = 0.0f; + } + else + { + m_Parameters.OpenTimeoutSecs = 3600.0f; + } + } + + m_Threads = m_Parameters.Threads; + if (m_Threads == 0) + { + helper::Comm m_NodeComm = + m_Comm.GroupByShm("creating per-node comm at BP5 Open(read)"); + unsigned int NodeSize = static_cast(m_NodeComm.Size()); + unsigned int NodeThreadSize = helper::NumHardwareThreadsPerNode(); + if (NodeThreadSize > 0) + { + m_Threads = + helper::SetWithinLimit(NodeThreadSize / NodeSize, 1U, 16U); + } + else + { + m_Threads = helper::SetWithinLimit(8U / NodeSize, 1U, 8U); + } + } + + // Create m_Threads-1 extra file managers to be used by threads + // The main thread uses the DataFileManager pushed here to vector[0] + fileManagers.push_back(m_DataFileManager); + for (unsigned int i = 0; i < m_Threads - 1; ++i) + { + fileManagers.push_back(transportman::TransportMan( + transportman::TransportMan(m_IO, singleComm))); + } + + size_t limit = helper::RaiseLimitNoFile(); + if (m_Parameters.MaxOpenFilesAtOnce > limit - 8) + { + m_Parameters.MaxOpenFilesAtOnce = limit - 8; + } +} + +bool DaosReader::SleepOrQuit(const TimePoint &timeoutInstant, + const Seconds &pollSeconds) +{ + auto now = Now(); + if (now >= timeoutInstant) + { + return false; + } + auto remainderTime = timeoutInstant - now; + auto sleepTime = pollSeconds; + if (remainderTime < sleepTime) + { + sleepTime = remainderTime; + } + std::this_thread::sleep_for(sleepTime); + return true; +} + +size_t DaosReader::OpenWithTimeout(transportman::TransportMan &tm, + const std::vector &fileNames, + const TimePoint &timeoutInstant, + const Seconds &pollSeconds, + std::string &lasterrmsg /*INOUT*/) +{ + size_t flag = 1; // 0 = OK, opened file, 1 = timeout, 2 = error + do + { + try + { + errno = 0; + const bool profile = + false; // m_BP4Deserializer.m_Profiler.m_IsActive; + tm.OpenFiles(fileNames, adios2::Mode::Read, + m_IO.m_TransportsParameters, profile); + flag = 0; // found file + break; + } + catch (std::ios_base::failure &e) + { + lasterrmsg = + std::string("errno=" + std::to_string(errno) + ": " + e.what()); + if (errno == ENOENT) + { + flag = 1; // timeout + } + else + { + flag = 2; // fatal error + break; + } + } + } while (SleepOrQuit(timeoutInstant, pollSeconds)); + return flag; +} + +void DaosReader::OpenFiles(TimePoint &timeoutInstant, + const Seconds &pollSeconds, + const Seconds &timeoutSeconds) +{ + /* Poll */ + size_t flag = 1; // 0 = OK, opened file, 1 = timeout, 2 = error + std::string lasterrmsg; + if (m_Comm.Rank() == 0) + { + /* Open the metadata index table */ + const std::string metadataIndexFile(GetBPMetadataIndexFileName(m_Name)); + + flag = OpenWithTimeout(m_MDIndexFileManager, {metadataIndexFile}, + timeoutInstant, pollSeconds, lasterrmsg); + if (flag == 0) + { + /* Open the metadata file */ + const std::string metadataFile(GetBPMetadataFileName(m_Name)); + + /* We found md.idx. If we don't find md.0 immediately we should + * wait a little bit hoping for the file system to catch up. + * This slows down finding the error in file reading mode but + * it will be more robust in streaming mode + */ + if (timeoutSeconds == Seconds(0.0)) + { + timeoutInstant += Seconds(5.0); + } + + flag = OpenWithTimeout(m_MDFileManager, {metadataFile}, + timeoutInstant, pollSeconds, lasterrmsg); + if (flag != 0) + { + /* Close the metadata index table */ + m_MDIndexFileManager.CloseFiles(); + } + else + { + /* Open the metametadata file */ + const std::string metametadataFile( + GetBPMetaMetadataFileName(m_Name)); + + /* We found md.idx. If we don't find md.0 immediately we should + * wait a little bit hoping for the file system to catch up. + * This slows down finding the error in file reading mode but + * it will be more robust in streaming mode + */ + if (timeoutSeconds == Seconds(0.0)) + { + timeoutInstant += Seconds(5.0); + } + + flag = OpenWithTimeout(m_FileMetaMetadataManager, + {metametadataFile}, timeoutInstant, + pollSeconds, lasterrmsg); + if (flag != 0) + { + /* Close the metametadata index table */ + m_MDIndexFileManager.CloseFiles(); + m_MDFileManager.CloseFiles(); + } + } + } + } + + flag = m_Comm.BroadcastValue(flag, 0); + if (flag == 2) + { + if (m_Comm.Rank() == 0 && !lasterrmsg.empty()) + { + helper::Throw( + "Engine", "DaosReader", "OpenFiles", + "File " + m_Name + " cannot be opened: " + lasterrmsg); + } + else + { + helper::Throw( + "Engine", "DaosReader", "OpenFiles", + "File " + m_Name + " cannot be opened"); + } + } + else if (flag == 1) + { + if (m_Comm.Rank() == 0) + { + helper::Throw( + "Engine", "DaosReader", "OpenFiles", + "File " + m_Name + " could not be found within the " + + std::to_string(timeoutSeconds.count()) + + "s timeout: " + lasterrmsg); + } + else + { + helper::Throw( + "Engine", "DaosReader", "OpenFiles", + "File " + m_Name + " could not be found within the " + + std::to_string(timeoutSeconds.count()) + "s timeout"); + } + } + + /* At this point we may have an empty index table. + * The writer has created the file but no content may have been stored yet. + */ +} + +MinVarInfo *DaosReader::MinBlocksInfo(const VariableBase &Var, + const size_t Step) const +{ + return m_BP5Deserializer->MinBlocksInfo(Var, Step); +} + +bool DaosReader::VarShape(const VariableBase &Var, const size_t Step, + Dims &Shape) const +{ + return m_BP5Deserializer->VarShape(Var, Step, Shape); +} + +bool DaosReader::VariableMinMax(const VariableBase &Var, const size_t Step, + MinMaxStruct &MinMax) +{ + return m_BP5Deserializer->VariableMinMax(Var, Step, MinMax); +} + +void DaosReader::InitTransports() +{ + if (m_IO.m_TransportsParameters.empty()) + { + Params defaultTransportParameters; + defaultTransportParameters["transport"] = "File"; + m_IO.m_TransportsParameters.push_back(defaultTransportParameters); + } +} + +void DaosReader::InitDAOS() +{ + // Rank 0 - Connect to DAOS pool, and open container + int rc; + rc = daos_init(); + ASSERT(rc == 0, "daos_init failed with %d", rc); + + std::cout << __func__ << std::endl; + + rc = gethostname(node, sizeof(node)); + ASSERT(rc == 0, "buffer for hostname too small"); + if (m_Comm.Rank() == 0) + { + /** connect to the just created DAOS pool */ + rc = daos_pool_connect(pool_label, DSS_PSETID, + // DAOS_PC_EX , + DAOS_PC_RW /* read write access */, + &poh /* returned pool handle */, + NULL /* returned pool info */, NULL /* event */); + ASSERT(rc == 0, "pool connect failed with %d", rc); + } + + /** share pool handle with peer tasks */ + daos_handle_share(&poh, DaosReader::HANDLE_POOL); + + if (m_Comm.Rank() == 0) + { + /** open container */ + rc = daos_cont_open(poh, cont_label, DAOS_COO_RW, &coh, NULL, NULL); + ASSERT(rc == 0, "container open failed with %d", rc); + } + + /** share container handle with peer tasks */ + daos_handle_share(&coh, HANDLE_CO); + + if (m_Comm.Rank() == 0) + { + FILE *fp = fopen("./share/oid.txt", "r"); + if (fp == NULL) + { + perror("fopen"); + exit(1); + } + if (fscanf(fp, "%" SCNu64 "\n%" SCNu64 "\n", &oid.hi, &oid.lo) != 2) + { + fprintf(stderr, "Error reading OID from file\n"); + exit(1); + } + fclose(fp); + } + + // Rank 0 will broadcast the DAOS KV OID + MPI_Bcast(&oid.hi, 1, MPI_UNSIGNED_LONG, 0, MPI_COMM_WORLD); + MPI_Bcast(&oid.lo, 1, MPI_UNSIGNED_LONG, 0, MPI_COMM_WORLD); + + // Open KV object + rc = daos_kv_open(coh, oid, 0, &oh, NULL); + ASSERT(rc == 0, "daos_kv_open failed with %d", rc); +} + +void DaosReader::InstallMetaMetaData(format::BufferSTL buffer) +{ + size_t Position = m_MetaMetaDataFileAlreadyProcessedSize; + while (Position < buffer.m_Buffer.size()) + { + format::BP5Base::MetaMetaInfoBlock MMI; + + MMI.MetaMetaIDLen = helper::ReadValue( + buffer.m_Buffer, Position, m_Minifooter.IsLittleEndian); + MMI.MetaMetaInfoLen = helper::ReadValue( + buffer.m_Buffer, Position, m_Minifooter.IsLittleEndian); + MMI.MetaMetaID = buffer.Data() + Position; + MMI.MetaMetaInfo = buffer.Data() + Position + MMI.MetaMetaIDLen; + m_BP5Deserializer->InstallMetaMetaData(MMI); + Position += MMI.MetaMetaIDLen + MMI.MetaMetaInfoLen; + } + m_MetaMetaDataFileAlreadyProcessedSize = Position; +} + +void DaosReader::UpdateBuffer(const TimePoint &timeoutInstant, + const Seconds &pollSeconds, + const Seconds &timeoutSeconds) +{ + size_t newIdxSize = 0; + m_MetadataIndex.Reset(true, false); + if (m_Comm.Rank() == 0) + { + /* Read metadata index table into memory */ + const size_t metadataIndexFileSize = + m_MDIndexFileManager.GetFileSize(0); + newIdxSize = metadataIndexFileSize - m_MDIndexFileAlreadyReadSize; + if (metadataIndexFileSize > m_MDIndexFileAlreadyReadSize) + { + m_MetadataIndex.m_Buffer.resize(newIdxSize); + m_MDIndexFileManager.ReadFile(m_MetadataIndex.m_Buffer.data(), + newIdxSize, + m_MDIndexFileAlreadyReadSize); + } + else + { + m_MetadataIndex.m_Buffer.resize(0); + } + } + + // broadcast metadata index buffer to all ranks from zero + m_Comm.BroadcastVector(m_MetadataIndex.m_Buffer); + newIdxSize = m_MetadataIndex.m_Buffer.size(); + + size_t parsedIdxSize = 0; + const auto stepsBefore = m_StepsCount; + if (newIdxSize > 0) + { + /* Parse metadata index table */ + const bool hasHeader = (!m_MDIndexFileAlreadyReadSize); + parsedIdxSize = ParseMetadataIndex(m_MetadataIndex, 0, hasHeader); + // now we are sure the index header has been parsed, + // first step parsing done + // m_FilteredMetadataInfo is created + + // cut down the index buffer by throwing away the read but unprocessed + // steps + m_MetadataIndex.m_Buffer.resize(parsedIdxSize); + // next time read index file from this position + m_MDIndexFileAlreadyReadSize += parsedIdxSize; + + // At this point first in time we learned the writer's major and we can + // create the serializer object + if (!m_BP5Deserializer) + { + m_BP5Deserializer = new format::BP5Deserializer( + m_WriterIsRowMajor, m_ReaderIsRowMajor, + (m_OpenMode == Mode::ReadRandomAccess)); + m_BP5Deserializer->m_Engine = this; + } + } + + if (m_StepsCount > stepsBefore) + { + m_Metadata.Reset(true, false); + m_MetaMetadata.Reset(true, false); + if (m_Comm.Rank() == 0) + { + // // How much metadata do we need to read? + // size_t fileFilteredSize = 0; + // for (auto p : m_FilteredMetadataInfo) + // { + // fileFilteredSize += p.second; + // } + // + // /* Read metadata file into memory but first make sure + // * it has the content that the index table refers to + // */ + // auto p = m_FilteredMetadataInfo.back(); + // uint64_t expectedMinFileSize = p.first + p.second; + // size_t actualFileSize = 0; + // do + // { + // actualFileSize = m_MDFileManager.GetFileSize(0); + // if (actualFileSize >= expectedMinFileSize) + // { + // break; + // } + // } while (SleepOrQuit(timeoutInstant, pollSeconds)); + // + // if (actualFileSize >= expectedMinFileSize) + // { + // m_Metadata.Resize(fileFilteredSize, + // "allocating metadata buffer, " + // "in call to DaosReader Open"); + // size_t mempos = 0; + // for (auto p : m_FilteredMetadataInfo) + // { + // m_MDFileManager.ReadFile( + // m_Metadata.m_Buffer.data() + mempos, + // p.second, p.first); + // mempos += p.second; + // } + // m_MDFileAlreadyReadSize = expectedMinFileSize; + // } + // else + // { + // helper::Throw( + // "Engine", "DaosReader", "UpdateBuffer", + // "File " + m_Name + + // " was found with an index file but md.0 " + // "has not contained enough data within " + // "the specified timeout of " + + // std::to_string(timeoutSeconds.count()) + + // " seconds. index size = " + + // std::to_string(newIdxSize) + " metadata + // size = " + std::to_string(actualFileSize) + // + " expected size = " + + // std::to_string(expectedMinFileSize) + + // ". One reason could be if the reader finds + // old " "data " "while " "the writer is + // creating the new files."); + // } + + /* Read new meta-meta-data into memory and append to existing one in + * memory */ + const size_t metametadataFileSize = + m_FileMetaMetadataManager.GetFileSize(0); + if (metametadataFileSize > m_MetaMetaDataFileAlreadyReadSize) + { + const size_t newMMDSize = + metametadataFileSize - m_MetaMetaDataFileAlreadyReadSize; + m_MetaMetadata.Resize(metametadataFileSize, + "(re)allocating meta-meta-data buffer, " + "in call to DaosReader Open"); + m_FileMetaMetadataManager.ReadFile( + m_MetaMetadata.m_Buffer.data() + + m_MetaMetaDataFileAlreadyReadSize, + newMMDSize, m_MetaMetaDataFileAlreadyReadSize); + m_MetaMetaDataFileAlreadyReadSize += newMMDSize; + } + } + + // broadcast buffer to all ranks from zero + // m_Comm.BroadcastVector(m_Metadata.m_Buffer); + + // broadcast metadata index buffer to all ranks from zero + m_Comm.BroadcastVector(m_MetaMetadata.m_Buffer); + + InstallMetaMetaData(m_MetaMetadata); + + if (m_OpenMode == Mode::ReadRandomAccess) + { + for (size_t Step = 0; Step < m_MetadataIndexTable.size(); Step++) + { + m_BP5Deserializer->SetupForStep( + Step, m_WriterMap[m_WriterMapIndex[Step]].WriterCount); + InstallMetadataForTimestep(Step); + } + } + } +} + +size_t DaosReader::ParseMetadataIndex(format::BufferSTL &bufferSTL, + const size_t absoluteStartPos, + const bool hasHeader) +{ + const auto &buffer = bufferSTL.m_Buffer; + size_t &position = bufferSTL.m_Position; + + if (hasHeader) + { + // Read header (64 bytes) + // long version string + position = m_VersionTagPosition; + m_Minifooter.VersionTag.assign(&buffer[position], m_VersionTagLength); + + position = m_EndianFlagPosition; + const uint8_t endianness = helper::ReadValue(buffer, position); + m_Minifooter.IsLittleEndian = (endianness == 0) ? true : false; +#ifndef ADIOS2_HAVE_ENDIAN_REVERSE + if (helper::IsLittleEndian() != m_Minifooter.IsLittleEndian) + { + helper::Throw( + "Engine", "DaosReader", "ParseMetadataIndex", + "reader found BigEndian bp file, " + "this version of ADIOS2 wasn't compiled " + "with the cmake flag -DADIOS2_USE_Endian_Reverse=ON " + "explicitly, in call to Open"); + } +#endif + + // This has no flag in BP5 header. Always true + m_Minifooter.HasSubFiles = true; + + // BP version + position = m_BPVersionPosition; + m_Minifooter.Version = helper::ReadValue( + buffer, position, m_Minifooter.IsLittleEndian); + if (m_Minifooter.Version != 5) + { + helper::Throw( + "Engine", "DaosReader", "ParseMetadataIndex", + "ADIOS2 BP5 Engine only supports bp format " + "version 5, found " + + std::to_string(m_Minifooter.Version) + " version"); + } + + // BP minor version, unused + position = m_BPMinorVersionPosition; + const uint8_t minorversion = helper::ReadValue( + buffer, position, m_Minifooter.IsLittleEndian); + if (minorversion != m_BP5MinorVersion) + { + helper::Throw( + "Engine", "DaosReader", "ParseMetadataIndex", + "Current ADIOS2 BP5 Engine only supports version 5." + + std::to_string(m_BP5MinorVersion) + ", found 5." + + std::to_string(minorversion) + " version"); + } + + // Writer active flag + position = m_ActiveFlagPosition; + const char activeChar = helper::ReadValue( + buffer, position, m_Minifooter.IsLittleEndian); + m_WriterIsActive = (activeChar == '\1' ? true : false); + + position = m_ColumnMajorFlagPosition; + const uint8_t val = helper::ReadValue( + buffer, position, m_Minifooter.IsLittleEndian); + m_WriterIsRowMajor = val == 'n'; + // move position to first row + position = m_IndexHeaderSize; + } + + // set a limit for metadata size in streaming mode + size_t maxMetadataSizeInMemory = adios2::MaxSizeT; + if (m_OpenMode == Mode::Read) + { + maxMetadataSizeInMemory = 16777216; // 16MB + } + size_t metadataSizeToRead = 0; + + // Read each record now + uint64_t MetadataPosTotalSkip = 0; + m_MetadataIndexTable.clear(); + m_FilteredMetadataInfo.clear(); + uint64_t minfo_pos = 0; + uint64_t minfo_size = 0; + int n = 0; // a loop counter for current run4 + int nrec = 0; // number of records in current run + + while (position < buffer.size() && + metadataSizeToRead < maxMetadataSizeInMemory) + { + + const unsigned char recordID = helper::ReadValue( + buffer, position, m_Minifooter.IsLittleEndian); + const uint64_t recordLength = helper::ReadValue( + buffer, position, m_Minifooter.IsLittleEndian); + const size_t dbgRecordStartPosition = position; + + switch (recordID) + { + case IndexRecord::WriterMapRecord: + { + auto p = m_WriterMap.emplace(m_StepsCount, WriterMapStruct()); + auto &s = p.first->second; + s.WriterCount = helper::ReadValue( + buffer, position, m_Minifooter.IsLittleEndian); + s.AggregatorCount = helper::ReadValue( + buffer, position, m_Minifooter.IsLittleEndian); + s.SubfileCount = helper::ReadValue( + buffer, position, m_Minifooter.IsLittleEndian); + // Get the process -> subfile map + s.RankToSubfile.reserve(s.WriterCount); + for (uint64_t i = 0; i < s.WriterCount; i++) + { + const uint64_t subfileIdx = helper::ReadValue( + buffer, position, m_Minifooter.IsLittleEndian); + s.RankToSubfile.push_back(subfileIdx); + } + m_LastMapStep = m_StepsCount; + m_LastWriterCount = s.WriterCount; + break; + } + case IndexRecord::StepRecord: + { + std::vector ptrs; + const uint64_t MetadataPos = helper::ReadValue( + buffer, position, m_Minifooter.IsLittleEndian); + const uint64_t MetadataSize = helper::ReadValue( + buffer, position, m_Minifooter.IsLittleEndian); + const uint64_t FlushCount = helper::ReadValue( + buffer, position, m_Minifooter.IsLittleEndian); + + if (!n) + { + minfo_pos = MetadataPos; // initialize minfo_pos properly + MetadataPosTotalSkip = MetadataPos; + } + + if (m_SelectedSteps.IsSelected(m_AbsStepsInFile)) + { + m_WriterMapIndex.push_back(m_LastMapStep); + + // pos in metadata in memory + ptrs.push_back(MetadataPos - MetadataPosTotalSkip); + ptrs.push_back(MetadataSize); + ptrs.push_back(FlushCount); + ptrs.push_back(position); + // absolute pos in file before read + ptrs.push_back(MetadataPos); + m_MetadataIndexTable[m_StepsCount] = ptrs; +#ifdef DUMPDATALOCINFO + for (uint64_t i = 0; i < m_WriterCount; i++) + { + size_t DataPosPos = ptrs[3]; + std::cout << "Writer " << i << " data at "; + for (uint64_t j = 0; j < FlushCount; j++) + { + const uint64_t DataPos = helper::ReadValue( + buffer, DataPosPos, m_Minifooter.IsLittleEndian); + const uint64_t DataSize = helper::ReadValue( + buffer, DataPosPos, m_Minifooter.IsLittleEndian); + std::cout << "loc:" << DataPos << " siz:" << DataSize + << "; "; + } + const uint64_t DataPos = helper::ReadValue( + buffer, DataPosPos, m_Minifooter.IsLittleEndian); + std::cout << "loc:" << DataPos << std::endl; + } +#endif + minfo_size += MetadataSize; + metadataSizeToRead += MetadataSize; + m_StepsCount++; + } + else + { + MetadataPosTotalSkip += MetadataSize; + if (minfo_size > 0) + { + m_FilteredMetadataInfo.push_back( + std::make_pair(minfo_pos, minfo_size)); + } + minfo_pos = MetadataPos + MetadataSize; + minfo_size = 0; + } + + // skip over the writer -> data file offset records + position += + sizeof(uint64_t) * m_LastWriterCount * ((2 * FlushCount) + 1); + ++m_AbsStepsInFile; + ++n; + break; + } + } + // dbg + if ((position - dbgRecordStartPosition) != (size_t)recordLength) + { + helper::Throw( + "Engine", "DaosReader", "ParseMetadataIndex", + "Record " + std::to_string(nrec) + " (id = " + + std::to_string(recordID) + ") has invalid length " + + std::to_string(recordLength) + ". We parsed " + + std::to_string(position - dbgRecordStartPosition) + + " bytes for this record" + + ); + } + ++nrec; + } + if (minfo_size > 0) + { + m_FilteredMetadataInfo.push_back(std::make_pair(minfo_pos, minfo_size)); + } + + return position; +} + +bool DaosReader::ReadActiveFlag(std::vector &buffer) +{ + if (buffer.size() < m_ActiveFlagPosition) + { + helper::Throw( + "Engine", "DaosReader", "ReadActiveFlag", + "called with a buffer smaller than required"); + } + // Writer active flag + size_t position = m_ActiveFlagPosition; + const char activeChar = helper::ReadValue( + buffer, position, m_Minifooter.IsLittleEndian); + m_WriterIsActive = (activeChar == '\1' ? true : false); + return m_WriterIsActive; +} + +bool DaosReader::CheckWriterActive() +{ + size_t flag = 1; + if (m_Comm.Rank() == 0) + { + auto fsize = m_MDIndexFileManager.GetFileSize(0); + if (fsize >= m_IndexHeaderSize) + { + std::vector header(m_IndexHeaderSize, '\0'); + m_MDIndexFileManager.ReadFile(header.data(), m_IndexHeaderSize, 0, + 0); + bool active = ReadActiveFlag(header); + flag = (active ? 1 : 0); + } + } + flag = m_Comm.BroadcastValue(flag, 0); + m_WriterIsActive = (flag > 0); + return m_WriterIsActive; +} + +StepStatus DaosReader::CheckForNewSteps(Seconds timeoutSeconds) +{ + /* Do a collective wait for a step within timeout. + Make sure every reader comes to the same conclusion */ + StepStatus retval = StepStatus::OK; + + if (timeoutSeconds < Seconds::zero()) + { + timeoutSeconds = Seconds(999999999); // max 1 billion seconds wait + } + const TimePoint timeoutInstant = Now() + timeoutSeconds; + + auto pollSeconds = Seconds(m_Parameters.BeginStepPollingFrequencySecs); + if (pollSeconds > timeoutSeconds) + { + pollSeconds = timeoutSeconds; + } + + /* Poll */ + const auto stepsBefore = m_StepsCount; + do + { + UpdateBuffer(timeoutInstant, pollSeconds / 10, timeoutSeconds); + if (m_StepsCount > stepsBefore) + { + break; + } + if (!CheckWriterActive()) + { + /* Race condition: When checking data in UpdateBuffer, new + * step(s) may have not arrived yet. When checking active flag, + * the writer may have completed write and terminated. So we may + * have missed a step or two. */ + UpdateBuffer(timeoutInstant, pollSeconds / 10, timeoutSeconds); + break; + } + } while (SleepOrQuit(timeoutInstant, pollSeconds)); + + if (m_StepsCount > stepsBefore) + { + /* we have got new steps and new metadata in memory */ + retval = StepStatus::OK; + } + else + { + m_IO.m_ReadStreaming = false; + if (m_WriterIsActive) + { + retval = StepStatus::NotReady; + } + else + { + retval = StepStatus::EndOfStream; + } + } + return retval; +} + +void DaosReader::DoGetAbsoluteSteps(const VariableBase &variable, + std::vector &keys) const +{ + m_BP5Deserializer->GetAbsoluteSteps(variable, keys); + return; +} + +#define declare_type(T) \ + void DaosReader::DoGetSync(Variable &variable, T *data) \ + { \ + PERFSTUBS_SCOPED_TIMER("DaosReader::Get"); \ + GetSyncCommon(variable, data); \ + } \ + void DaosReader::DoGetDeferred(Variable &variable, T *data) \ + { \ + PERFSTUBS_SCOPED_TIMER("DaosReader::Get"); \ + GetDeferredCommon(variable, data); \ + } +ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + +void DaosReader::DoGetStructSync(VariableStruct &variable, void *data) +{ + PERFSTUBS_SCOPED_TIMER("DaosReader::Get"); + GetSyncCommon(variable, data); +} + +void DaosReader::DoGetStructDeferred(VariableStruct &variable, void *data) +{ + PERFSTUBS_SCOPED_TIMER("DaosReader::Get"); + GetDeferredCommon(variable, data); +} + +void DaosReader::DoClose(const int transportIndex) +{ + PERFSTUBS_SCOPED_TIMER("DaosReader::Close"); + if (m_OpenMode == Mode::ReadRandomAccess) + { + PerformGets(); + } + else if (m_BetweenStepPairs) + { + EndStep(); + } + m_DataFileManager.CloseFiles(); + m_MDFileManager.CloseFiles(); + m_MDIndexFileManager.CloseFiles(); + m_FileMetaMetadataManager.CloseFiles(); + for (unsigned int i = 1; i < m_Threads; ++i) + { + fileManagers[i].CloseFiles(); + } +} + +size_t DaosReader::DoSteps() const { return m_StepsCount; } + +void DaosReader::NotifyEngineNoVarsQuery() +{ + if (!m_BetweenStepPairs) + { + helper::Throw( + "Engine", "DaosReader", "NotifyEngineNoVarsQuery", + "You've called InquireVariable() when the IO is empty and " + "outside a BeginStep/EndStep pair. If this is code that is " + "newly " + "transititioning to the BP5 file engine, you may be relying " + "upon " + "deprecated behaviour. If you intend to use ADIOS using the " + "Begin/EndStep interface, move all InquireVariable calls " + "inside " + "the BeginStep/EndStep pair. If intending to use " + "random-access " + "file mode, change your Open() mode parameter to " + "Mode::ReadRandomAccess."); + } +} + +void DaosReader::daos_handle_share(daos_handle_t *hdl, int type) +{ + d_iov_t ghdl = {NULL, 0, 0}; + int rc; + + if (m_Comm.Rank() == 0) + { + /** fetch size of global handle */ + if (type == DaosReader::HANDLE_POOL) + rc = daos_pool_local2global(*hdl, &ghdl); + else + rc = daos_cont_local2global(*hdl, &ghdl); + ASSERT(rc == 0, "local2global failed with %d", rc); + } + + /** broadcast size of global handle to all peers */ + MPI_Bcast(&ghdl.iov_buf_len, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD); + + /** allocate buffer for global pool handle */ + ghdl.iov_buf = malloc(ghdl.iov_buf_len); + ghdl.iov_len = ghdl.iov_buf_len; + + if (m_Comm.Rank() == 0) + { + /** generate actual global handle to share with peer tasks */ + if (type == DaosReader::HANDLE_POOL) + rc = daos_pool_local2global(*hdl, &ghdl); + else + rc = daos_cont_local2global(*hdl, &ghdl); + ASSERT(rc == 0, "local2global failed with %d", rc); + } + + /** broadcast global handle to all peers */ + MPI_Bcast(ghdl.iov_buf, ghdl.iov_len, MPI_BYTE, 0, MPI_COMM_WORLD); + + if (m_Comm.Rank() != 0) + { + /** unpack global handle */ + if (type == DaosReader::HANDLE_POOL) + { + /* NB: Only pool_global2local are different */ + rc = daos_pool_global2local(ghdl, hdl); + } + else + { + rc = daos_cont_global2local(poh, ghdl, hdl); + } + ASSERT(rc == 0, "global2local failed with %d", rc); + } + + free(ghdl.iov_buf); + + MPI_Barrier(MPI_COMM_WORLD); +} + +} // end namespace engine +} // end namespace core +} // end namespace adios2 diff --git a/source/adios2/engine/daos/DaosReader.h b/source/adios2/engine/daos/DaosReader.h new file mode 100644 index 0000000000..57e955e8ae --- /dev/null +++ b/source/adios2/engine/daos/DaosReader.h @@ -0,0 +1,309 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * DaosReader.h + * + */ + +#ifndef ADIOS2_ENGINE_DAOS_DAOSREADER_H_ +#define ADIOS2_ENGINE_DAOS_DAOSREADER_H_ +#define DSS_PSETID "daos_server" + +#include "adios2/common/ADIOSConfig.h" +#include "adios2/core/CoreTypes.h" +#include "adios2/core/Engine.h" +#include "adios2/engine/daos/DaosEngine.h" +#include "adios2/helper/adiosComm.h" +#include "adios2/helper/adiosRangeFilter.h" +#include "adios2/toolkit/format/bp5/BP5Deserializer.h" +#include "adios2/toolkit/transportman/TransportMan.h" + +#include +#include +#include +#include +#include +#include + +#define FAIL(fmt, ...) \ + do \ + { \ + fprintf(stderr, "Process %d(%s): " fmt " aborting\n", m_Comm.Rank(), \ + node, ##__VA_ARGS__); \ + MPI_Abort(MPI_COMM_WORLD, 1); \ + } while (0) +#define ASSERT(cond, ...) \ + do \ + { \ + if (!(cond)) \ + FAIL(__VA_ARGS__); \ + } while (0) + +namespace adios2 +{ +namespace core +{ +namespace engine +{ + +class DaosReader : public DaosEngine, public Engine +{ + +public: + /** + * Unique constructor + * @param io + * @param name + * @param openMode only read + * @param comm + */ + DaosReader(IO &io, const std::string &name, const Mode mode, + helper::Comm comm); + + ~DaosReader(); + + StepStatus BeginStep(StepMode mode = StepMode::Read, + const float timeoutSeconds = -1.0) final; + + size_t CurrentStep() const final; + + void EndStep() final; + + void PerformGets() final; + + MinVarInfo *MinBlocksInfo(const VariableBase &, const size_t Step) const; + bool VarShape(const VariableBase &Var, const size_t Step, + Dims &Shape) const; + bool VariableMinMax(const VariableBase &, const size_t Step, + MinMaxStruct &MinMax); + +private: + format::BP5Deserializer *m_BP5Deserializer = nullptr; + /* transport manager for metadata file */ + transportman::TransportMan m_MDFileManager; + /* How many bytes of metadata have we already read in? */ + size_t m_MDFileAlreadyReadSize = 0; + /* How many bytes of metadata have we already processed? + * It is <= m_MDFileAlreadyReadSize, at = we need to read more */ + size_t m_MDFileProcessedSize = 0; + /* The file position of the first byte that is currently + * residing in memory. Needed for skewing positions when + * processing metadata index. + */ + size_t m_MDFileAbsolutePos = 0; + /* m_MDFileAbsolutePos <= m_MDFileProcessedSize <= m_MDFileAlreadyReadSize + */ + + /* transport manager for managing data file(s) */ + transportman::TransportMan m_DataFileManager; + + /* transport manager for managing the metadata index file */ + transportman::TransportMan m_MDIndexFileManager; + /* transport manager for managing the metadata index file */ + transportman::TransportMan m_FileMetaMetadataManager; + /* How many bytes of metadata index have we already read in? */ + size_t m_MDIndexFileAlreadyReadSize = 0; + + /* How many bytes of meta-metadata have we already read in? */ + size_t m_MetaMetaDataFileAlreadyReadSize = 0; + /* How many bytes of meta-metadata have we already processed? */ + size_t m_MetaMetaDataFileAlreadyProcessedSize = 0; + + /* transport manager for managing the active flag file */ + transportman::TransportMan m_ActiveFlagFileManager; + bool m_WriterIsActive = true; + + /* DAOS declarations */ + + uuid_t pool_uuid, cont_uuid; + char *pool_label = "pool_ranjansv"; + char *cont_label = "adios-daos-engine-cont"; + + /* Declare variables for pool and container handles */ + daos_handle_t poh, coh; + + enum DAOS_handleType + { + HANDLE_POOL, + HANDLE_CO, + }; + + /* Declare variables for the KV object */ + daos_handle_t oh; + daos_obj_id_t oid; + + char node[128] = "unknown"; + + /** used for per-step reads, TODO: to be moved to BP5Deserializer */ + size_t m_CurrentStep = 0; + size_t m_StepsCount = 0; + size_t m_AbsStepsInFile = 0; // all steps parsed including unselected + uint64_t m_LastMapStep = 0; // remember last step that had writer map + uint64_t m_LastWriterCount = 0; // remember writer count in that step + bool m_FirstStep = true; + + /** used to filter steps */ + helper::RangeFilter m_SelectedSteps; + + // offset/size pairs to read sections of metadata from file in InitBuffer + std::vector> m_FilteredMetadataInfo; + + Minifooter m_Minifooter; + + void Init(); + void InitParameters(); + void InitTransports(); + + /** DAOS pool connection and container opening */ + void InitDAOS(); + + /* Sleep up to pollSeconds time if we have not reached timeoutInstant. + * Return true if slept + * return false if sleep was not needed because it was overtime + */ + bool SleepOrQuit(const TimePoint &timeoutInstant, + const Seconds &pollSeconds); + /** Open one category of files within timeout. + * @return: 0 = OK, 1 = timeout, 2 = error + * lasterrmsg contains the error message in case of error + */ + size_t OpenWithTimeout(transportman::TransportMan &tm, + const std::vector &fileNames, + const TimePoint &timeoutInstant, + const Seconds &pollSeconds, + std::string &lasterrmsg /*INOUT*/); + + /** Open files within timeout. + * @return True if files are opened, False in case of timeout + */ + void OpenFiles(TimePoint &timeoutInstant, const Seconds &pollSeconds, + const Seconds &timeoutSeconds); + + /** Read in metadata if exist (throwing away old). + * It reads and parses metadata-index, and reads metadata into memory. + * In streaming mode, only a limited size of metadata is read in. + * Changes in m_StepsCount before and after calling can be used to + * track if new steps (after filtering with SelectSteps) are read in + * and are ready to be processed. + */ + void UpdateBuffer(const TimePoint &timeoutInstant, + const Seconds &pollSeconds, + const Seconds &timeoutSeconds); + + bool ReadActiveFlag(std::vector &buffer); + + /* Parse metadata. + * + * Return the size of metadataindex where parsing stopped. In streaming mode + * parsing is limited to read only a certain size of metadata at once. + * + * As a side effect, the following variables are filled out: + * m_MetadataIndexTable + * m_WriterMapIndex + * m_FilteredMetadataInfo + */ + size_t ParseMetadataIndex(format::BufferSTL &bufferSTL, + const size_t absoluteStartPos, + const bool hasHeader); + + /** Process the new metadata coming in (in UpdateBuffer) + * @param newIdxSize: the size of the new content from Index Table + */ + void ProcessMetadataForNewSteps(const size_t newIdxSize); + + /** Check the active status of the writer. + * @return true if writer is still active. + * It sets m_WriterIsActive. + */ + bool CheckWriterActive(); + + /** Check for a step that is already in memory but haven't + * been processed yet. + * @return true: if new step has been found and processed, false otherwise + * Used by CheckForNewSteps() to get the next step from memory if there is + * one. + */ + bool ProcessNextStepInMemory(); + + /** Check for new steps withing timeout and only if writer is active. + * @return the status flag + * Used by BeginStep() to get new steps from file when it reaches the + * end of steps in memory. + */ + StepStatus CheckForNewSteps(Seconds timeoutSeconds); + + /** Notify the engine when InquireVariable is called when the IO is empty. + * Called from IO.tcc + */ + void NotifyEngineNoVarsQuery(); + +#define declare_type(T) \ + void DoGetSync(Variable &, T *) final; \ + void DoGetDeferred(Variable &, T *) final; + ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + + void DoClose(const int transportIndex = -1) final; + + void GetSyncCommon(VariableBase &variable, void *data); + + void GetDeferredCommon(VariableBase &variable, void *data); + + void DoGetStructSync(VariableStruct &, void *); + void DoGetStructDeferred(VariableStruct &, void *); + + template + void ReadVariableBlocks(Variable &variable); + + size_t DoSteps() const final; + + void DoGetAbsoluteSteps(const VariableBase &variable, + std::vector &keys) const final; + + uint32_t m_WriterColumnMajor = 0; + bool m_ReaderIsRowMajor = true; + bool m_WriterIsRowMajor = true; + + format::BufferSTL m_MetadataIndex; + format::BufferSTL m_MetaMetadata; + format::BufferSTL m_Metadata; + + void InstallMetaMetaData(format::BufferSTL MetaMetadata); + void InstallMetadataForTimestep(size_t Step); + std::pair + ReadData(adios2::transportman::TransportMan &FileManager, + const size_t maxOpenFiles, const size_t WriterRank, + const size_t Timestep, const size_t StartOffset, + const size_t Length, char *Destination); + + struct WriterMapStruct + { + uint32_t WriterCount = 0; + uint32_t AggregatorCount = 0; + uint32_t SubfileCount = 0; + std::vector RankToSubfile; // size WriterCount + }; + + // step -> writermap but not for all steps + std::map m_WriterMap; + // step -> writermap index (for all steps) + std::vector m_WriterMapIndex; + + void DestructorClose(bool Verbose) noexcept; + + /* Communicator connecting ranks on each Compute Node. + Only used to calculate the number of threads available for reading */ + helper::Comm m_NodeComm; + helper::Comm singleComm; + unsigned int m_Threads; + std::vector fileManagers; // manager per thread + + void daos_handle_share(daos_handle_t *, int); +}; + +} // end namespace engine +} // end namespace core +} // end namespace adios2 + +#endif /* ADIOS2_ENGINE_DAOS_DAOSREADER_H_ */ diff --git a/source/adios2/engine/daos/DaosReader.tcc b/source/adios2/engine/daos/DaosReader.tcc new file mode 100644 index 0000000000..b4d47c9a76 --- /dev/null +++ b/source/adios2/engine/daos/DaosReader.tcc @@ -0,0 +1,39 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * DaosReader.tcc + * + */ + +#ifndef ADIOS2_ENGINE_DAOS_DAOSREADER_TCC_ +#define ADIOS2_ENGINE_DAOS_DAOSREADER_TCC_ + +#include "DaosReader.h" + +#include "adios2/helper/adiosFunctions.h" + +namespace adios2 +{ +namespace core +{ +namespace engine +{ + +inline void DaosReader::GetSyncCommon(VariableBase &variable, void *data) +{ + bool need_sync = m_BP5Deserializer->QueueGet(variable, data); + if (need_sync) + PerformGets(); +} + +void DaosReader::GetDeferredCommon(VariableBase &variable, void *data) +{ + (void)m_BP5Deserializer->QueueGet(variable, data); +} + +} // end namespace engine +} // end namespace core +} // end namespace adios2 + +#endif /* ADIOS2_ENGINE_DAOS_DAOSREADER_TCC_ */ diff --git a/source/adios2/engine/daos/DaosWriter.cpp b/source/adios2/engine/daos/DaosWriter.cpp new file mode 100644 index 0000000000..7adb0a169c --- /dev/null +++ b/source/adios2/engine/daos/DaosWriter.cpp @@ -0,0 +1,2014 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * DaosWriter.cpp + * + */ + +#include "DaosWriter.h" +#include "DaosWriter.tcc" + +#include "adios2/common/ADIOSMacros.h" +#include "adios2/core/IO.h" +#include "adios2/helper/adiosFunctions.h" //CheckIndexRange +#include "adios2/helper/adiosMath.h" // SetWithinLimit +#include "adios2/helper/adiosMemory.h" // NdCopy +#include "adios2/toolkit/format/buffer/chunk/ChunkV.h" +#include "adios2/toolkit/format/buffer/malloc/MallocV.h" +#include "adios2/toolkit/transport/file/FileFStream.h" +#include + +#include +#include // setw +#include +#include // make_shared + +namespace adios2 +{ +namespace core +{ +namespace engine +{ + +using namespace adios2::format; + +DaosWriter::DaosWriter(IO &io, const std::string &name, const Mode mode, + helper::Comm comm) +: Engine("DaosWriter", io, name, mode, std::move(comm)), m_BP5Serializer(), + m_FileDataManager(io, m_Comm), m_FileMetadataManager(io, m_Comm), + m_FileMetadataIndexManager(io, m_Comm), m_FileMetaMetadataManager(io, m_Comm), + m_Profiler(m_Comm) +{ + m_EngineStart = Now(); + PERFSTUBS_SCOPED_TIMER("DaosWriter::Open"); + m_IO.m_ReadStreaming = false; + + Init(); + m_IsOpen = true; +} + +StepStatus DaosWriter::BeginStep(StepMode mode, const float timeoutSeconds) +{ + if (m_BetweenStepPairs) + { + helper::Throw("Engine", "DaosWriter", "BeginStep", + "BeginStep() is called a second time " + "without an intervening EndStep()"); + } + + Seconds ts = Now() - m_EngineStart; + // std::cout << "BEGIN STEP starts at: " << ts.count() << std::endl; + m_BetweenStepPairs = true; + + if (m_WriterStep > 0) + { + m_LastTimeBetweenSteps = Now() - m_EndStepEnd; + m_TotalTimeBetweenSteps += m_LastTimeBetweenSteps; + m_AvgTimeBetweenSteps = m_TotalTimeBetweenSteps / m_WriterStep; + m_ExpectedTimeBetweenSteps = m_LastTimeBetweenSteps; + if (m_ExpectedTimeBetweenSteps > m_AvgTimeBetweenSteps) + { + m_ExpectedTimeBetweenSteps = m_AvgTimeBetweenSteps; + } + } + + if ((m_WriterStep == 0) && m_Parameters.UseOneTimeAttributes) + { + const auto &attributes = m_IO.GetAttributes(); + + for (const auto &attributePair : attributes) + { + m_BP5Serializer.OnetimeMarshalAttribute(*(attributePair.second)); + } + } + + if (m_Parameters.AsyncWrite) + { + m_AsyncWriteLock.lock(); + m_flagRush = true; + m_AsyncWriteLock.unlock(); + TimePoint wait_start = Now(); + if (m_WriteFuture.valid()) + { + m_Profiler.Start("WaitOnAsync"); + m_WriteFuture.get(); + m_Comm.Barrier(); + AsyncWriteDataCleanup(); + Seconds wait = Now() - wait_start; + if (m_Comm.Rank() == 0) + { + WriteMetadataFileIndex(m_LatestMetaDataPos, + m_LatestMetaDataSize); + if (m_Parameters.verbose > 0) + { + std::cout << "BeginStep, wait on async write was = " + << wait.count() << " time since EndStep was = " + << m_LastTimeBetweenSteps.count() + << " expect next one to be = " + << m_ExpectedTimeBetweenSteps.count() + << std::endl; + } + } + m_Profiler.Stop("WaitOnAsync"); + } + } + + if (m_Parameters.BufferVType == (int)BufferVType::MallocVType) + { + m_BP5Serializer.InitStep(new MallocV( + "DaosWriter", false, m_BP5Serializer.m_BufferAlign, + m_BP5Serializer.m_BufferBlockSize, m_Parameters.InitialBufferSize, + m_Parameters.GrowthFactor)); + } + else + { + m_BP5Serializer.InitStep(new ChunkV( + "DaosWriter", false, m_BP5Serializer.m_BufferAlign, + m_BP5Serializer.m_BufferBlockSize, m_Parameters.BufferChunkSize)); + } + m_ThisTimestepDataSize = 0; + + ts = Now() - m_EngineStart; + // std::cout << "BEGIN STEP ended at: " << ts.count() << std::endl; + return StepStatus::OK; +} + +size_t DaosWriter::CurrentStep() const { return m_WriterStep; } + +void DaosWriter::PerformPuts() +{ + PERFSTUBS_SCOPED_TIMER("DaosWriter::PerformPuts"); + m_Profiler.Start("PP"); + m_BP5Serializer.PerformPuts(m_Parameters.AsyncWrite || + m_Parameters.DirectIO); + m_Profiler.Stop("PP"); + return; +} + +void DaosWriter::WriteMetaMetadata( + const std::vector MetaMetaBlocks) +{ + for (auto &b : MetaMetaBlocks) + { + m_FileMetaMetadataManager.WriteFiles((char *)&b.MetaMetaIDLen, + sizeof(size_t)); + m_FileMetaMetadataManager.WriteFiles((char *)&b.MetaMetaInfoLen, + sizeof(size_t)); + m_FileMetaMetadataManager.WriteFiles((char *)b.MetaMetaID, + b.MetaMetaIDLen); + m_FileMetaMetadataManager.WriteFiles((char *)b.MetaMetaInfo, + b.MetaMetaInfoLen); + } +} + +uint64_t +DaosWriter::WriteMetadata(const std::vector &MetaDataBlocks, + const std::vector &AttributeBlocks) +{ + uint64_t MDataTotalSize = 0; + uint64_t MetaDataSize = 0; + std::vector SizeVector; + std::vector AttrSizeVector; + SizeVector.reserve(MetaDataBlocks.size()); + for (auto &b : MetaDataBlocks) + { + MDataTotalSize += sizeof(uint64_t) + b.iov_len; + SizeVector.push_back(b.iov_len); + } + for (auto &b : AttributeBlocks) + { + MDataTotalSize += sizeof(uint64_t) + b.iov_len; + AttrSizeVector.push_back(b.iov_len); + } + MetaDataSize = 0; + m_FileMetadataManager.WriteFiles((char *)&MDataTotalSize, sizeof(uint64_t)); + MetaDataSize += sizeof(uint64_t); + m_FileMetadataManager.WriteFiles((char *)SizeVector.data(), + sizeof(uint64_t) * SizeVector.size()); + MetaDataSize += sizeof(uint64_t) * AttrSizeVector.size(); + m_FileMetadataManager.WriteFiles((char *)AttrSizeVector.data(), + sizeof(uint64_t) * AttrSizeVector.size()); + MetaDataSize += sizeof(uint64_t) * AttrSizeVector.size(); + for (auto &b : MetaDataBlocks) + { + if (!b.iov_base) + continue; + m_FileMetadataManager.WriteFiles((char *)b.iov_base, b.iov_len); + MetaDataSize += b.iov_len; + } + + for (auto &b : AttributeBlocks) + { + if (!b.iov_base) + continue; + m_FileMetadataManager.WriteFiles((char *)b.iov_base, b.iov_len); + MetaDataSize += b.iov_len; + } + + m_MetaDataPos += MetaDataSize; + return MetaDataSize; +} + +void DaosWriter::AsyncWriteDataCleanup() +{ + if (m_Parameters.AsyncWrite) + { + switch (m_Parameters.AggregationType) + { + case (int)AggregationType::EveryoneWrites: + case (int)AggregationType::EveryoneWritesSerial: + AsyncWriteDataCleanup_EveryoneWrites(); + break; + case (int)AggregationType::TwoLevelShm: + AsyncWriteDataCleanup_TwoLevelShm(); + break; + default: + break; + } + } +} + +void DaosWriter::WriteData(format::BufferV *Data) +{ + if (m_Parameters.AsyncWrite) + { + switch (m_Parameters.AggregationType) + { + case (int)AggregationType::EveryoneWrites: + WriteData_EveryoneWrites_Async(Data, false); + break; + case (int)AggregationType::EveryoneWritesSerial: + WriteData_EveryoneWrites_Async(Data, true); + break; + case (int)AggregationType::TwoLevelShm: + WriteData_TwoLevelShm_Async(Data); + break; + default: + helper::Throw( + "Engine", "DaosWriter", "WriteData", + "Aggregation method " + + std::to_string(m_Parameters.AggregationType) + + "is not supported in BP5"); + } + } + else + { + switch (m_Parameters.AggregationType) + { + case (int)AggregationType::EveryoneWrites: + WriteData_EveryoneWrites(Data, false); + break; + case (int)AggregationType::EveryoneWritesSerial: + WriteData_EveryoneWrites(Data, true); + break; + case (int)AggregationType::TwoLevelShm: + WriteData_TwoLevelShm(Data); + break; + default: + helper::Throw( + "Engine", "DaosWriter", "WriteData", + "Aggregation method " + + std::to_string(m_Parameters.AggregationType) + + "is not supported in BP5"); + } + delete Data; + } +} + +void DaosWriter::WriteData_EveryoneWrites(format::BufferV *Data, + bool SerializedWriters) +{ + const aggregator::MPIChain *a = + dynamic_cast(m_Aggregator); + + // new step writing starts at offset m_DataPos on aggregator + // others will wait for the position to arrive from the rank below + + if (a->m_Comm.Rank() > 0) + { + a->m_Comm.Recv(&m_DataPos, 1, a->m_Comm.Rank() - 1, 0, + "Chain token in DaosWriter::WriteData"); + } + + // align to PAGE_SIZE + m_DataPos += + helper::PaddingToAlignOffset(m_DataPos, m_Parameters.StripeSize); + m_StartDataPos = m_DataPos; + + if (!SerializedWriters && a->m_Comm.Rank() < a->m_Comm.Size() - 1) + { + /* Send the token before writing so everyone can start writing asap */ + uint64_t nextWriterPos = m_DataPos + Data->Size(); + a->m_Comm.Isend(&nextWriterPos, 1, a->m_Comm.Rank() + 1, 0, + "Chain token in DaosWriter::WriteData"); + } + + m_DataPos += Data->Size(); + std::vector DataVec = Data->DataVec(); + m_FileDataManager.WriteFileAt(DataVec.data(), DataVec.size(), + m_StartDataPos); + + if (SerializedWriters && a->m_Comm.Rank() < a->m_Comm.Size() - 1) + { + /* send token now, effectively serializing the writers in the chain */ + uint64_t nextWriterPos = m_DataPos; + a->m_Comm.Isend(&nextWriterPos, 1, a->m_Comm.Rank() + 1, 0, + "Chain token in DaosWriter::WriteData"); + } + + if (a->m_Comm.Size() > 1) + { + // at the end, last rank sends back the final data pos to first rank + // so it can update its data pos + if (a->m_Comm.Rank() == a->m_Comm.Size() - 1) + { + a->m_Comm.Isend(&m_DataPos, 1, 0, 0, + "Final chain token in DaosWriter::WriteData"); + } + if (a->m_Comm.Rank() == 0) + { + a->m_Comm.Recv(&m_DataPos, 1, a->m_Comm.Size() - 1, 0, + "Chain token in DaosWriter::WriteData"); + } + } +} + +void DaosWriter::WriteMetadataFileIndex(uint64_t MetaDataPos, + uint64_t MetaDataSize) +{ + m_FileMetadataManager.FlushFiles(); + + // bufsize: Step record + size_t bufsize = + 1 + (4 + ((FlushPosSizeInfo.size() * 2) + 1) * m_Comm.Size()) * + sizeof(uint64_t); + if (MetaDataPos == 0) + { + // First time, write the headers + bufsize += m_IndexHeaderSize; + } + if (!m_WriterSubfileMap.empty()) + { + // WriterMap record + bufsize += 1 + (4 + m_Comm.Size()) * sizeof(uint64_t); + } + + std::vector buf(bufsize); + size_t pos = 0; + uint64_t d; + unsigned char record; + + if (MetaDataPos == 0) + { + // First time, write the headers + MakeHeader(buf, pos, "Index Table", true); + } + + // WriterMap record + if (!m_WriterSubfileMap.empty()) + { + record = WriterMapRecord; + helper::CopyToBuffer(buf, pos, &record, 1); // record type + d = (3 + m_Comm.Size()) * sizeof(uint64_t); + helper::CopyToBuffer(buf, pos, &d, 1); // record length + d = static_cast(m_Comm.Size()); + helper::CopyToBuffer(buf, pos, &d, 1); + d = static_cast(m_Aggregator->m_NumAggregators); + helper::CopyToBuffer(buf, pos, &d, 1); + d = static_cast(m_Aggregator->m_SubStreams); + helper::CopyToBuffer(buf, pos, &d, 1); + helper::CopyToBuffer(buf, pos, m_WriterSubfileMap.data(), + m_Comm.Size()); + m_WriterSubfileMap.clear(); + } + + // Step record + record = StepRecord; + helper::CopyToBuffer(buf, pos, &record, 1); // record type + d = (3 + ((FlushPosSizeInfo.size() * 2) + 1) * m_Comm.Size()) * + sizeof(uint64_t); + helper::CopyToBuffer(buf, pos, &d, 1); // record length + helper::CopyToBuffer(buf, pos, &MetaDataPos, 1); + helper::CopyToBuffer(buf, pos, &MetaDataSize, 1); + d = static_cast(FlushPosSizeInfo.size()); + helper::CopyToBuffer(buf, pos, &d, 1); + + for (int writer = 0; writer < m_Comm.Size(); writer++) + { + for (size_t flushNum = 0; flushNum < FlushPosSizeInfo.size(); + flushNum++) + { + // add two numbers here + helper::CopyToBuffer(buf, pos, + &FlushPosSizeInfo[flushNum][2 * writer], 2); + } + helper::CopyToBuffer(buf, pos, &m_WriterDataPos[writer], 1); + } + + m_FileMetadataIndexManager.WriteFiles((char *)buf.data(), buf.size()); + +#ifdef DUMPDATALOCINFO + std::cout << "Flush count is :" << FlushPosSizeInfo.size() << std::endl; + std::cout << "Write Index positions = {" << std::endl; + + for (size_t i = 0; i < m_Comm.Size(); ++i) + { + std::cout << "Writer " << i << " has data at: " << std::endl; + uint64_t eachWriterSize = FlushPosSizeInfo.size() * 2 + 1; + for (size_t j = 0; j < FlushPosSizeInfo.size(); ++j) + { + std::cout << "loc:" << buf[3 + eachWriterSize * i + j * 2] + << " siz:" << buf[3 + eachWriterSize * i + j * 2 + 1] + << std::endl; + } + std::cout << "loc:" << buf[3 + eachWriterSize * (i + 1) - 1] + << std::endl; + } + std::cout << "}" << std::endl; +#endif + /* reset for next timestep */ + FlushPosSizeInfo.clear(); +} + +void DaosWriter::NotifyEngineAttribute(std::string name, DataType type) noexcept +{ + helper::Throw( + "DaosWriter", "Engine", "ThrowUp", + "Engine does not support NotifyEngineAttribute"); +} + +void DaosWriter::NotifyEngineAttribute(std::string name, AttributeBase *Attr, + void *data) noexcept +{ + if (!m_Parameters.UseOneTimeAttributes) + { + m_MarshalAttributesNecessary = true; + return; + } + + m_BP5Serializer.OnetimeMarshalAttribute(*Attr); + m_MarshalAttributesNecessary = false; +} + +void DaosWriter::MarshalAttributes() +{ + PERFSTUBS_SCOPED_TIMER_FUNC(); + const auto &attributes = m_IO.GetAttributes(); + + // if there are no new attributes, nothing to do + if (!m_MarshalAttributesNecessary) + { + return; + } + m_MarshalAttributesNecessary = false; + + for (const auto &attributePair : attributes) + { + const std::string name(attributePair.first); + auto baseAttr = &attributePair.second; + const DataType type((*baseAttr)->m_Type); + int element_count = -1; + + if (!attributePair.second->m_IsSingleValue) + { + element_count = (*baseAttr)->m_Elements; + } + + if (type == DataType::None) + { + } + else if (type == helper::GetDataType()) + { + core::Attribute &attribute = + *m_IO.InquireAttribute(name); + void *data_addr; + if (attribute.m_IsSingleValue) + { + data_addr = (void *)attribute.m_DataSingleValue.c_str(); + } + else + { + const char **tmp = + (const char **)malloc(sizeof(char *) * element_count); + for (int i = 0; i < element_count; i++) + { + auto str = &attribute.m_DataArray[i]; + tmp[i] = str->c_str(); + } + // tmp will be free'd after final attribute marshalling + data_addr = (void *)tmp; + } + + m_BP5Serializer.MarshalAttribute(name.c_str(), type, sizeof(char *), + element_count, data_addr); + } +#define declare_type(T) \ + else if (type == helper::GetDataType()) \ + { \ + core::Attribute &attribute = *m_IO.InquireAttribute(name); \ + int element_count = -1; \ + void *data_addr = &attribute.m_DataSingleValue; \ + if (!attribute.m_IsSingleValue) \ + { \ + element_count = attribute.m_Elements; \ + data_addr = attribute.m_DataArray.data(); \ + } \ + m_BP5Serializer.MarshalAttribute(attribute.m_Name.c_str(), type, \ + sizeof(T), element_count, data_addr); \ + } + + ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type) +#undef declare_type + } +} + +void DaosWriter::EndStep() +{ + /* Seconds ts = Now() - m_EngineStart; + std::cout << "END STEP starts at: " << ts.count() << std::endl; */ + m_BetweenStepPairs = false; + PERFSTUBS_SCOPED_TIMER("DaosWriter::EndStep"); + m_Profiler.Start("endstep"); + + m_Profiler.Start("close_ts"); + MarshalAttributes(); + + // true: advances step + auto TSInfo = m_BP5Serializer.CloseTimestep( + m_WriterStep, m_Parameters.AsyncWrite || m_Parameters.DirectIO); + + /* TSInfo includes NewMetaMetaBlocks, the MetaEncodeBuffer, the + * AttributeEncodeBuffer and the data encode Vector */ + + m_ThisTimestepDataSize += TSInfo.DataBuffer->Size(); + m_Profiler.Stop("close_ts"); + + m_Profiler.Start("AWD"); + // TSInfo destructor would delete the DataBuffer so we need to save it + // for async IO and let the writer free it up when not needed anymore + adios2::format::BufferV *databuf = TSInfo.DataBuffer; + TSInfo.DataBuffer = NULL; + m_AsyncWriteLock.lock(); + m_flagRush = false; + m_AsyncWriteLock.unlock(); + WriteData(databuf); + m_Profiler.Stop("AWD"); + + /* + * Two-step metadata aggregation + */ + m_Profiler.Start("meta_lvl1"); + std::vector MetaBuffer; + // core::iovec m{TSInfo.MetaEncodeBuffer->Data(), + // TSInfo.MetaEncodeBuffer->m_FixedSize}; + core::iovec m{nullptr, 0}; + core::iovec a{nullptr, 0}; + if (TSInfo.AttributeEncodeBuffer) + { + a = {TSInfo.AttributeEncodeBuffer->Data(), + TSInfo.AttributeEncodeBuffer->m_FixedSize}; + } + MetaBuffer = m_BP5Serializer.CopyMetadataToContiguous( + TSInfo.NewMetaMetaBlocks, {m}, {a}, {m_ThisTimestepDataSize}, + {m_StartDataPos}); + + if (m_Aggregator->m_Comm.Size() > 1) + { // level 1 + m_Profiler.Start("meta_gather1"); + size_t LocalSize = MetaBuffer.size(); + std::vector RecvCounts = + m_Aggregator->m_Comm.GatherValues(LocalSize, 0); + std::vector RecvBuffer; + if (m_Aggregator->m_Comm.Rank() == 0) + { + uint64_t TotalSize = 0; + for (auto &n : RecvCounts) + TotalSize += n; + RecvBuffer.resize(TotalSize); + /*std::cout << "MD Lvl-1: rank " << m_Comm.Rank() << " gather " + << TotalSize << " bytes from aggregator group" + << std::endl;*/ + } + m_Aggregator->m_Comm.GathervArrays(MetaBuffer.data(), LocalSize, + RecvCounts.data(), RecvCounts.size(), + RecvBuffer.data(), 0); + m_Profiler.Stop("meta_gather1"); + if (m_Aggregator->m_Comm.Rank() == 0) + { + std::vector + UniqueMetaMetaBlocks; + std::vector DataSizes; + std::vector WriterDataPositions; + std::vector AttributeBlocks; + auto Metadata = m_BP5Serializer.BreakoutContiguousMetadata( + RecvBuffer, RecvCounts, UniqueMetaMetaBlocks, AttributeBlocks, + DataSizes, WriterDataPositions); + + MetaBuffer.clear(); + MetaBuffer = m_BP5Serializer.CopyMetadataToContiguous( + UniqueMetaMetaBlocks, Metadata, AttributeBlocks, DataSizes, + WriterDataPositions); + } + } // level 1 + m_Profiler.Stop("meta_lvl1"); + m_Profiler.Start("meta_lvl2"); + // level 2 + if (m_Aggregator->m_Comm.Rank() == 0) + { + std::vector RecvBuffer; + std::vector *buf; + std::vector RecvCounts; + size_t LocalSize = MetaBuffer.size(); + if (m_CommAggregators.Size() > 1) + { + m_Profiler.Start("meta_gather2"); + RecvCounts = m_CommAggregators.GatherValues(LocalSize, 0); + if (m_CommAggregators.Rank() == 0) + { + uint64_t TotalSize = 0; + for (auto &n : RecvCounts) + TotalSize += n; + RecvBuffer.resize(TotalSize); + /*std::cout << "MD Lvl-2: rank " << m_Comm.Rank() << " gather " + << TotalSize << " bytes from aggregator group" + << std::endl;*/ + } + + m_CommAggregators.GathervArrays( + MetaBuffer.data(), LocalSize, RecvCounts.data(), + RecvCounts.size(), RecvBuffer.data(), 0); + buf = &RecvBuffer; + m_Profiler.Stop("meta_gather2"); + } + else + { + buf = &MetaBuffer; + RecvCounts.push_back(LocalSize); + } + + if (m_CommAggregators.Rank() == 0) + { + std::vector + UniqueMetaMetaBlocks; + std::vector DataSizes; + std::vector AttributeBlocks; + m_WriterDataPos.resize(0); + auto Metadata = m_BP5Serializer.BreakoutContiguousMetadata( + *buf, RecvCounts, UniqueMetaMetaBlocks, AttributeBlocks, + DataSizes, m_WriterDataPos); + assert(m_WriterDataPos.size() == + static_cast(m_Comm.Size())); + WriteMetaMetadata(UniqueMetaMetaBlocks); + m_LatestMetaDataPos = m_MetaDataPos; + m_LatestMetaDataSize = WriteMetadata(Metadata, AttributeBlocks); + // m_LatestMetaDataPos = 0; + // m_LatestMetaDataSize = 0; + if (!m_Parameters.AsyncWrite) + { + WriteMetadataFileIndex(m_LatestMetaDataPos, + m_LatestMetaDataSize); + } + } + } // level 2 + m_Profiler.Stop("meta_lvl2"); + + char key[1000]; + int rc; + + sprintf(key, "step%d-rank%d", m_WriterStep, m_Comm.Rank()); + std::cout << __FILE__ << "::" << __func__ << "(), step: " << m_WriterStep + << std::endl; + std::cout << "Rank = " << m_Comm.Rank() + << ", Metadata size = " << TSInfo.MetaEncodeBuffer->m_FixedSize + << std::endl; + std::cout << "key = " << key << std::endl; + std::cout << "Printing the first 10 bytes of Metadata" << std::endl; + char *data = reinterpret_cast(TSInfo.MetaEncodeBuffer->Data()); + for (int i = 0; i < 10; i++) + { + // std::cout << std::hex << std::setw(2) << std::setfill('0') << + // static_cast(data[i]) << " "; + std::cout << static_cast(data[i]) << " "; + } + std::cout << std::endl; + rc = daos_kv_put(oh, DAOS_TX_NONE, 0, key, + TSInfo.MetaEncodeBuffer->m_FixedSize, + TSInfo.MetaEncodeBuffer->Data(), NULL); + ASSERT(rc == 0, "daos_kv_put() failed with %d", rc); + + if (m_Parameters.AsyncWrite) + { + /* Start counting computation blocks between EndStep and next BeginStep + * each time */ + { + m_AsyncWriteLock.lock(); + m_ComputationBlockTimes.clear(); + m_ComputationBlocksLength = 0.0; + m_ComputationBlockID = 0; + m_AsyncWriteLock.unlock(); + } + } + + m_Profiler.Stop("endstep"); + m_WriterStep++; + m_EndStepEnd = Now(); + /* Seconds ts2 = Now() - m_EngineStart; + std::cout << "END STEP ended at: " << ts2.count() << std::endl;*/ +} + +// PRIVATE +void DaosWriter::Init() +{ + m_BP5Serializer.m_Engine = this; + m_RankMPI = m_Comm.Rank(); + InitParameters(); + InitAggregator(); + InitTransports(); + InitDAOS(); + InitBPBuffer(); +} + +void DaosWriter::InitParameters() +{ + ParseParams(m_IO, m_Parameters); + m_WriteToBB = !(m_Parameters.BurstBufferPath.empty()); + m_DrainBB = m_WriteToBB && m_Parameters.BurstBufferDrain; + + unsigned int nproc = (unsigned int)m_Comm.Size(); + m_Parameters.NumAggregators = + helper::SetWithinLimit(m_Parameters.NumAggregators, 0U, nproc); + m_Parameters.NumSubFiles = + helper::SetWithinLimit(m_Parameters.NumSubFiles, 0U, nproc); + m_Parameters.AggregatorRatio = + helper::SetWithinLimit(m_Parameters.AggregatorRatio, 0U, nproc); + if (m_Parameters.NumAggregators == 0) + { + if (m_Parameters.AggregatorRatio > 0) + { + m_Parameters.NumAggregators = helper::SetWithinLimit( + nproc / m_Parameters.AggregatorRatio, 0U, nproc); + } + else if (m_Parameters.NumSubFiles > 0) + { + m_Parameters.NumAggregators = + helper::SetWithinLimit(m_Parameters.NumSubFiles, 0U, nproc); + } + } + m_Parameters.NumSubFiles = helper::SetWithinLimit( + m_Parameters.NumSubFiles, 0U, m_Parameters.NumAggregators); + + // Limiting to max 64MB page size + m_Parameters.StripeSize = + helper::SetWithinLimit(m_Parameters.StripeSize, 0U, 67108864U); + if (m_Parameters.StripeSize == 0) + { + m_Parameters.StripeSize = 4096; + } + + if (m_Parameters.DirectIO) + { + if (m_Parameters.DirectIOAlignBuffer == 0) + { + m_Parameters.DirectIOAlignBuffer = m_Parameters.DirectIOAlignOffset; + } + m_BP5Serializer.m_BufferBlockSize = m_Parameters.DirectIOAlignOffset; + m_BP5Serializer.m_BufferAlign = m_Parameters.DirectIOAlignBuffer; + if (m_Parameters.StripeSize % m_Parameters.DirectIOAlignOffset) + { + size_t k = + m_Parameters.StripeSize / m_Parameters.DirectIOAlignOffset + 1; + m_Parameters.StripeSize = k * m_Parameters.DirectIOAlignOffset; + } + if (m_Parameters.BufferChunkSize % m_Parameters.DirectIOAlignOffset) + { + size_t k = m_Parameters.BufferChunkSize / + m_Parameters.DirectIOAlignOffset + + 1; + m_Parameters.BufferChunkSize = k * m_Parameters.DirectIOAlignOffset; + } + } + + m_BP5Serializer.m_StatsLevel = m_Parameters.StatsLevel; +} + +uint64_t DaosWriter::CountStepsInMetadataIndex(format::BufferSTL &bufferSTL) +{ + const auto &buffer = bufferSTL.m_Buffer; + size_t &position = bufferSTL.m_Position; + + if (buffer.size() < m_IndexHeaderSize) + { + m_AppendMetadataPos = 0; + m_AppendMetaMetadataPos = 0; + m_AppendMetadataIndexPos = 0; + m_AppendDataPos.resize(m_Aggregator->m_NumAggregators, + 0ULL); // safe bet + return 0; + } + + // Check endinanness + position = m_EndianFlagPosition; + const uint8_t endianness = helper::ReadValue(buffer, position); + bool IsLittleEndian = (endianness == 0) ? true : false; + if (helper::IsLittleEndian() != IsLittleEndian) + { + std::string m = (IsLittleEndian ? "Little" : "Big"); + + helper::Throw( + "Engine", "DaosWriter", "CountStepsInMetadataIndex", + "ADIOS2 BP5 Engine only supports appending with the same " + "endianness. The existing file is " + + m + "Endian"); + } + + // BP version + position = m_BPVersionPosition; + uint8_t Version = + helper::ReadValue(buffer, position, IsLittleEndian); + if (Version != 5) + { + helper::Throw( + "Engine", "DaosWriter", "CountStepsInMetadataIndex", + "ADIOS2 BP5 Engine only supports bp format " + "version 5, found " + + std::to_string(Version) + " version"); + } + + // BP minor version + position = m_BPMinorVersionPosition; + uint8_t minorVersion = + helper::ReadValue(buffer, position, IsLittleEndian); + if (minorVersion != m_BP5MinorVersion) + { + helper::Throw( + "Engine", "DaosWriter", "CountStepsInMetadataIndex", + "Current ADIOS2 BP5 Engine can only append to bp format 5." + + std::to_string(m_BP5MinorVersion) + " but this file is 5." + + std::to_string(minorVersion) + " version"); + } + + position = m_ColumnMajorFlagPosition; + const uint8_t columnMajor = + helper::ReadValue(buffer, position, IsLittleEndian); + const uint8_t NowColumnMajor = + (m_IO.m_ArrayOrder == ArrayOrdering::ColumnMajor) ? 'y' : 'n'; + if (columnMajor != NowColumnMajor) + { + std::string m = (columnMajor == 'y' ? "column" : "row"); + helper::Throw( + "Engine", "DaosWriter", "CountStepsInMetadataIndex", + "ADIOS2 BP5 Engine only supports appending with the same " + "column/row major settings as it was written." + " Existing file is " + + m + " major"); + } + + position = m_IndexHeaderSize; // after the header + // Just count the steps first + unsigned int availableSteps = 0; + uint64_t nDataFiles = 0; + while (position < buffer.size()) + { + const unsigned char recordID = + helper::ReadValue(buffer, position, IsLittleEndian); + position += sizeof(uint64_t); // recordLength + + switch (recordID) + { + case IndexRecord::WriterMapRecord: + { + m_AppendWriterCount = + helper::ReadValue(buffer, position, IsLittleEndian); + m_AppendAggregatorCount = + helper::ReadValue(buffer, position, IsLittleEndian); + m_AppendSubfileCount = + helper::ReadValue(buffer, position, IsLittleEndian); + if (m_AppendSubfileCount > nDataFiles) + { + nDataFiles = m_AppendSubfileCount; + } + // jump over writermap + position += m_AppendWriterCount * sizeof(uint64_t); + break; + } + case IndexRecord::StepRecord: + { + position += 2 * sizeof(uint64_t); // MetadataPos, MetadataSize + const uint64_t FlushCount = + helper::ReadValue(buffer, position, IsLittleEndian); + // jump over the metadata positions + position += + sizeof(uint64_t) * m_AppendWriterCount * ((2 * FlushCount) + 1); + availableSteps++; + break; + } + } + } + + unsigned int targetStep = 0; + + if (m_Parameters.AppendAfterSteps < 0) + { + // -1 means append after last step + int s = (int)availableSteps + m_Parameters.AppendAfterSteps + 1; + if (s < 0) + { + s = 0; + } + targetStep = static_cast(s); + } + else + { + targetStep = static_cast(m_Parameters.AppendAfterSteps); + } + if (targetStep > availableSteps) + { + targetStep = availableSteps; + } + + m_AppendDataPos.resize(nDataFiles, 0ULL); + + if (!targetStep) + { + // append at 0 is like writing new file + m_AppendMetadataPos = 0; + m_AppendMetaMetadataPos = 0; + m_AppendMetadataIndexPos = 0; + return 0; + } + + m_AppendMetadataPos = MaxSizeT; // size of header + m_AppendMetaMetadataPos = MaxSizeT; + m_AppendMetadataIndexPos = MaxSizeT; + std::fill(m_AppendDataPos.begin(), m_AppendDataPos.end(), MaxSizeT); + + if (targetStep == availableSteps) + { + // append after existing steps + return targetStep; + } + + // append but not at 0 and not after existing steps + // Read each record now completely to get offsets at step+1 + position = m_IndexHeaderSize; + unsigned int currentStep = 0; + std::vector writerToFileMap; + // reading one step beyond target to get correct offsets + while (currentStep <= targetStep && position < buffer.size()) + { + const unsigned char recordID = + helper::ReadValue(buffer, position, IsLittleEndian); + position += sizeof(uint64_t); // recordLength + + switch (recordID) + { + case IndexRecord::WriterMapRecord: + { + m_AppendWriterCount = + helper::ReadValue(buffer, position, IsLittleEndian); + m_AppendAggregatorCount = + helper::ReadValue(buffer, position, IsLittleEndian); + m_AppendSubfileCount = + helper::ReadValue(buffer, position, IsLittleEndian); + + // Get the process -> subfile map + writerToFileMap.clear(); + for (uint64_t i = 0; i < m_AppendWriterCount; i++) + { + const uint64_t subfileIdx = helper::ReadValue( + buffer, position, IsLittleEndian); + writerToFileMap.push_back(subfileIdx); + } + break; + } + case IndexRecord::StepRecord: + { + m_AppendMetadataIndexPos = position - sizeof(unsigned char) - + sizeof(uint64_t); // pos of RecordID + const uint64_t MetadataPos = + helper::ReadValue(buffer, position, IsLittleEndian); + position += sizeof(uint64_t); // MetadataSize + const uint64_t FlushCount = + helper::ReadValue(buffer, position, IsLittleEndian); + + m_AppendMetadataPos = static_cast(MetadataPos); + + if (currentStep == targetStep) + { + // we need the very first (smallest) write position to each + // subfile Offsets and sizes, 2*FlushCount + 1 per writer + for (uint64_t i = 0; i < m_AppendWriterCount; i++) + { + // first flush/write position will do + const size_t FirstDataPos = + static_cast(helper::ReadValue( + buffer, position, IsLittleEndian)); + position += + sizeof(uint64_t) * 2 * FlushCount; // no need to read + /* std::cout << "Writer " << i << " subfile " << + writerToFileMap[i] << " first data loc:" << + FirstDataPos << std::endl; */ + if (FirstDataPos < m_AppendDataPos[writerToFileMap[i]]) + { + m_AppendDataPos[writerToFileMap[i]] = FirstDataPos; + } + } + } + else + { + // jump over all data offsets in this step + position += sizeof(uint64_t) * m_AppendWriterCount * + (1 + 2 * FlushCount); + } + currentStep++; + break; + } + } + } + return targetStep; +} + +void DaosWriter::InitAggregator() +{ + // in BP5, aggregation is "always on", but processes may be alone, so + // m_Aggregator.m_IsActive is always true + // m_Aggregator.m_Comm.Rank() will always succeed (not abort) + // m_Aggregator.m_SubFileIndex is always set + + if (m_Parameters.AggregationType == (int)AggregationType::EveryoneWrites || + m_Parameters.AggregationType == + (int)AggregationType::EveryoneWritesSerial) + { + m_Parameters.NumSubFiles = m_Parameters.NumAggregators; + m_AggregatorEveroneWrites.Init(m_Parameters.NumAggregators, + m_Parameters.NumSubFiles, m_Comm); + m_IAmDraining = m_AggregatorEveroneWrites.m_IsAggregator; + m_IAmWritingData = true; + DataWritingComm = &m_AggregatorEveroneWrites.m_Comm; + m_Aggregator = static_cast( + &m_AggregatorEveroneWrites); + } + else + { + size_t numNodes = m_AggregatorTwoLevelShm.PreInit(m_Comm); + (void)numNodes; + m_AggregatorTwoLevelShm.Init(m_Parameters.NumAggregators, + m_Parameters.NumSubFiles, m_Comm); + + /*std::cout << "Rank " << m_RankMPI << " aggr? " + << m_AggregatorTwoLevelShm.m_IsAggregator << " master? " + << m_AggregatorTwoLevelShm.m_IsMasterAggregator + << " aggr size = " << m_AggregatorTwoLevelShm.m_Size + << " rank = " << m_AggregatorTwoLevelShm.m_Rank + << " subfile = " << m_AggregatorTwoLevelShm.m_SubStreamIndex + << " type = " << m_Parameters.AggregationType << std::endl;*/ + + m_IAmDraining = m_AggregatorTwoLevelShm.m_IsMasterAggregator; + m_IAmWritingData = m_AggregatorTwoLevelShm.m_IsAggregator; + DataWritingComm = &m_AggregatorTwoLevelShm.m_AggregatorChainComm; + m_Aggregator = + static_cast(&m_AggregatorTwoLevelShm); + } + + /* comm for Aggregators only. + * We are only interested in the chain of rank 0s + */ + int color = m_Aggregator->m_Comm.Rank(); + m_CommAggregators = + m_Comm.Split(color, 0, "creating level 2 chain of aggregators at Open"); +} + +void DaosWriter::InitTransports() +{ + if (m_IO.m_TransportsParameters.empty()) + { + Params defaultTransportParameters; + defaultTransportParameters["transport"] = "File"; + m_IO.m_TransportsParameters.push_back(defaultTransportParameters); + } + + if (m_WriteToBB) + { + m_BBName = m_Parameters.BurstBufferPath + PathSeparator + m_Name; + } + else + { + m_BBName = m_Name; + } + /* From this point, engine writes to m_BBName, which points to either + the BB file system if BB is turned on, or to the target file system. + m_Name always points to the target file system, to which the drainer + should write if BB is turned on + */ + + // Names passed to IO AddTransport option with key "Name" + const std::vector transportsNames = + m_FileDataManager.GetFilesBaseNames(m_BBName, + m_IO.m_TransportsParameters); + + // /path/name.bp.dir/name.bp.rank + m_SubStreamNames = + GetBPSubStreamNames(transportsNames, m_Aggregator->m_SubStreamIndex); + + if (m_IAmDraining) + { + // Only (master)aggregators will run draining processes + if (m_DrainBB) + { + const std::vector drainTransportNames = + m_FileDataManager.GetFilesBaseNames( + m_Name, m_IO.m_TransportsParameters); + m_DrainSubStreamNames = GetBPSubStreamNames( + drainTransportNames, m_Aggregator->m_SubStreamIndex); + /* start up BB thread */ + // m_FileDrainer.SetVerbose( + // m_Parameters.BurstBufferVerbose, + // m_Comm.Rank()); + m_FileDrainer.Start(); + } + } + + /* Create the directories either on target or burst buffer if used */ + // m_BP4Serializer.m_Profiler.Start("mkdir"); + + if (m_Comm.Rank() == 0) + { + m_MetadataFileNames = GetBPMetadataFileNames(transportsNames); + m_MetaMetadataFileNames = GetBPMetaMetadataFileNames(transportsNames); + m_MetadataIndexFileNames = GetBPMetadataIndexFileNames(transportsNames); + } + m_FileMetadataManager.MkDirsBarrier(m_MetadataFileNames, + m_IO.m_TransportsParameters, + m_Parameters.NodeLocal || m_WriteToBB); + /* Create the directories on burst buffer if used */ + if (m_DrainBB) + { + /* Create the directories on target anyway by main thread */ + m_FileDataManager.MkDirsBarrier(m_DrainSubStreamNames, + m_IO.m_TransportsParameters, + m_Parameters.NodeLocal); + } + + /* Everyone opens its data file. Each aggregation chain opens + one data file and does so in chain, not everyone at once */ + if (m_Parameters.AsyncOpen) + { + for (size_t i = 0; i < m_IO.m_TransportsParameters.size(); ++i) + { + m_IO.m_TransportsParameters[i]["asyncopen"] = "true"; + } + } + + if (m_Parameters.DirectIO) + { + for (size_t i = 0; i < m_IO.m_TransportsParameters.size(); ++i) + { + m_IO.m_TransportsParameters[i]["DirectIO"] = "true"; + } + } + + bool useProfiler = true; + + if (m_IAmWritingData) + { + m_FileDataManager.OpenFiles(m_SubStreamNames, m_OpenMode, + m_IO.m_TransportsParameters, useProfiler, + *DataWritingComm); + } + + if (m_IAmDraining) + { + if (m_DrainBB) + { + for (const auto &name : m_DrainSubStreamNames) + { + m_FileDrainer.AddOperationOpen(name, m_OpenMode); + } + } + } + + if (m_Comm.Rank() == 0) + { + // force turn off directio to metadata files + for (size_t i = 0; i < m_IO.m_TransportsParameters.size(); ++i) + { + m_IO.m_TransportsParameters[i]["DirectIO"] = "false"; + } + m_FileMetaMetadataManager.OpenFiles(m_MetaMetadataFileNames, m_OpenMode, + m_IO.m_TransportsParameters, + useProfiler); + + m_FileMetadataManager.OpenFiles(m_MetadataFileNames, m_OpenMode, + m_IO.m_TransportsParameters, + useProfiler); + + m_FileMetadataIndexManager.OpenFiles( + m_MetadataIndexFileNames, m_OpenMode, m_IO.m_TransportsParameters, + useProfiler); + + if (m_DrainBB) + { + const std::vector drainTransportNames = + m_FileDataManager.GetFilesBaseNames( + m_Name, m_IO.m_TransportsParameters); + m_DrainMetadataFileNames = + GetBPMetadataFileNames(drainTransportNames); + m_DrainMetadataIndexFileNames = + GetBPMetadataIndexFileNames(drainTransportNames); + + for (const auto &name : m_DrainMetadataFileNames) + { + m_FileDrainer.AddOperationOpen(name, m_OpenMode); + } + for (const auto &name : m_DrainMetadataIndexFileNames) + { + m_FileDrainer.AddOperationOpen(name, m_OpenMode); + } + } + } +} + +void DaosWriter::InitDAOS() +{ + // Rank 0 - Connect to DAOS pool, and open container + int rc; + rc = gethostname(node, sizeof(node)); + ASSERT(rc == 0, "buffer for hostname too small"); + if (m_Comm.Rank() == 0) + { + /** connect to the just created DAOS pool */ + rc = daos_pool_connect(pool_label, DSS_PSETID, + // DAOS_PC_EX , + DAOS_PC_RW /* read write access */, + &poh /* returned pool handle */, + NULL /* returned pool info */, NULL /* event */); + ASSERT(rc == 0, "pool connect failed with %d", rc); + } + + /** share pool handle with peer tasks */ + daos_handle_share(&poh, DaosWriter::HANDLE_POOL); + + if (m_Comm.Rank() == 0) + { + /** open container */ + rc = daos_cont_open(poh, cont_label, DAOS_COO_RW, &coh, NULL, NULL); + ASSERT(rc == 0, "container open failed with %d", rc); + } + + /** share container handle with peer tasks */ + daos_handle_share(&coh, HANDLE_CO); + + if (m_Comm.Rank() == 0) + { + /** Open a DAOS KV object */ + rc = daos_obj_generate_oid(coh, &oid, DAOS_OT_KV_HASHED, OC_SX, 0, 0); + ASSERT(rc == 0, "daos_obj_generate_oid failed with %d", rc); + } + + // Rank 0 will broadcast the DAOS KV OID + MPI_Bcast(&oid.hi, 1, MPI_UNSIGNED_LONG, 0, MPI_COMM_WORLD); + MPI_Bcast(&oid.lo, 1, MPI_UNSIGNED_LONG, 0, MPI_COMM_WORLD); + + // Open KV object + rc = daos_kv_open(coh, oid, 0, &oh, NULL); + ASSERT(rc == 0, "daos_kv_open failed with %d", rc); + FILE *fp = fopen("./share/oid.txt", "w"); + if (fp == NULL) + { + perror("fopen"); + exit(1); + } + fprintf(fp, "%" PRIu64 "\n%" PRIu64 "\n", oid.hi, oid.lo); + fclose(fp); +} + +/*generate the header for the metadata index file*/ +void DaosWriter::MakeHeader(std::vector &buffer, size_t &position, + const std::string fileType, const bool isActive) +{ + auto lf_CopyVersionChar = [](const std::string version, + std::vector &buffer, size_t &position) { + helper::CopyToBuffer(buffer, position, version.c_str()); + }; + + // auto &buffer = b.m_Buffer; + // auto &position = b.m_Position; + // auto &absolutePosition = b.m_AbsolutePosition; + if (position > 0) + { + helper::Throw( + "Engine", "DaosWriter", "MakeHeader", + "BP4Serializer::MakeHeader can only be called for an empty " + "buffer. This one for " + + fileType + " already has content of " + + std::to_string(position) + " bytes."); + } + + if (buffer.size() < m_IndexHeaderSize) + { + buffer.resize(m_IndexHeaderSize); + } + + const std::string majorVersion(std::to_string(ADIOS2_VERSION_MAJOR)); + const std::string minorVersion(std::to_string(ADIOS2_VERSION_MINOR)); + const std::string patchVersion(std::to_string(ADIOS2_VERSION_PATCH)); + + // byte 0-31: Readable tag + if (position != m_VersionTagPosition) + { + helper::Throw( + "Engine", "DaosWriter", "MakeHeader", + "ADIOS Coding ERROR in BP4Serializer::MakeHeader. Version Tag " + "position mismatch"); + } + std::string versionLongTag("ADIOS-BP v" + majorVersion + "." + + minorVersion + "." + patchVersion + " "); + size_t maxTypeLen = m_VersionTagLength - versionLongTag.size(); + const std::string fileTypeStr = fileType.substr(0, maxTypeLen); + versionLongTag += fileTypeStr; + const size_t versionLongTagSize = versionLongTag.size(); + if (versionLongTagSize < m_VersionTagLength) + { + helper::CopyToBuffer(buffer, position, versionLongTag.c_str(), + versionLongTagSize); + position += m_VersionTagLength - versionLongTagSize; + } + else if (versionLongTagSize > m_VersionTagLength) + { + helper::CopyToBuffer(buffer, position, versionLongTag.c_str(), + m_VersionTagLength); + } + else + { + helper::CopyToBuffer(buffer, position, versionLongTag.c_str(), + m_VersionTagLength); + } + + // byte 32-35: MAJOR MINOR PATCH Unused + + lf_CopyVersionChar(majorVersion, buffer, position); + lf_CopyVersionChar(minorVersion, buffer, position); + lf_CopyVersionChar(patchVersion, buffer, position); + ++position; + + // Note: Reader does process and use bytes 36-38 in + // BP4Deserialize.cpp::ParseMetadataIndex(). + // Order and position must match there. + + // byte 36: endianness + if (position != m_EndianFlagPosition) + { + helper::Throw( + "Engine", "DaosWriter", "MakeHeader", + "ADIOS Coding ERROR in DaosWriter::MakeHeader. Endian Flag " + "position mismatch"); + } + const uint8_t endianness = helper::IsLittleEndian() ? 0 : 1; + helper::CopyToBuffer(buffer, position, &endianness); + + // byte 37: BP Version 5 + if (position != m_BPVersionPosition) + { + helper::Throw( + "Engine", "DaosWriter", "MakeHeader", + "ADIOS Coding ERROR in DaosWriter::MakeHeader. BP Version " + "position mismatch"); + } + const uint8_t version = 5; + helper::CopyToBuffer(buffer, position, &version); + + // byte 38: BP Minor version 1 + if (position != m_BPMinorVersionPosition) + { + helper::Throw( + "Engine", "DaosWriter", "MakeHeader", + "ADIOS Coding ERROR in DaosWriter::MakeHeader. BP Minor version " + "position mismatch"); + } + const uint8_t minorversion = m_BP5MinorVersion; + helper::CopyToBuffer(buffer, position, &minorversion); + + // byte 39: Active flag (used in Index Table only) + if (position != m_ActiveFlagPosition) + { + helper::Throw( + "Engine", "DaosWriter", "MakeHeader", + "ADIOS Coding ERROR in DaosWriter::MakeHeader. Active Flag " + "position mismatch"); + } + const uint8_t activeFlag = (isActive ? 1 : 0); + helper::CopyToBuffer(buffer, position, &activeFlag); + + // byte 40 columnMajor + // write if data is column major in metadata and data + const uint8_t columnMajor = + (m_IO.m_ArrayOrder == ArrayOrdering::ColumnMajor) ? 'y' : 'n'; + helper::CopyToBuffer(buffer, position, &columnMajor); + + // byte 41-63: unused + position += 23; + // absolutePosition = position; +} + +void DaosWriter::UpdateActiveFlag(const bool active) +{ + const char activeChar = (active ? '\1' : '\0'); + m_FileMetadataIndexManager.WriteFileAt(&activeChar, 1, + m_ActiveFlagPosition); + m_FileMetadataIndexManager.FlushFiles(); + m_FileMetadataIndexManager.SeekToFileEnd(); + if (m_DrainBB) + { + for (size_t i = 0; i < m_MetadataIndexFileNames.size(); ++i) + { + m_FileDrainer.AddOperationWriteAt(m_DrainMetadataIndexFileNames[i], + m_ActiveFlagPosition, 1, + &activeChar); + m_FileDrainer.AddOperationSeekEnd(m_DrainMetadataIndexFileNames[i]); + } + } +} + +void DaosWriter::InitBPBuffer() +{ + if (m_OpenMode == Mode::Append) + { + format::BufferSTL preMetadataIndex; + size_t preMetadataIndexFileSize; + + if (m_Comm.Rank() == 0) + { + preMetadataIndexFileSize = + m_FileMetadataIndexManager.GetFileSize(0); + preMetadataIndex.m_Buffer.resize(preMetadataIndexFileSize); + preMetadataIndex.m_Buffer.assign(preMetadataIndex.m_Buffer.size(), + '\0'); + preMetadataIndex.m_Position = 0; + m_FileMetadataIndexManager.ReadFile( + preMetadataIndex.m_Buffer.data(), preMetadataIndexFileSize); + } + m_Comm.BroadcastVector(preMetadataIndex.m_Buffer); + m_WriterStep = CountStepsInMetadataIndex(preMetadataIndex); + + // truncate and seek + if (m_Aggregator->m_IsAggregator) + { + const size_t off = m_AppendDataPos[m_Aggregator->m_SubStreamIndex]; + if (off < MaxSizeT) + { + m_FileDataManager.Truncate(off); + // Seek is needed since truncate does not seek. + // SeekTo instead of SeetToFileEnd in case a transport + // does not support actual truncate. + m_FileDataManager.SeekTo(off); + m_DataPos = off; + } + else + { + m_DataPos = m_FileDataManager.GetFileSize(0); + } + } + + if (m_Comm.Rank() == 0) + { + // Truncate existing metadata file + if (m_AppendMetadataPos < MaxSizeT) + { + m_MetaDataPos = m_AppendMetadataPos; + m_FileMetadataManager.Truncate(m_MetaDataPos); + m_FileMetadataManager.SeekTo(m_MetaDataPos); + } + else + { + m_MetaDataPos = m_FileMetadataManager.GetFileSize(0); + m_FileMetadataManager.SeekToFileEnd(); + } + + // Truncate existing meta-meta file + if (m_AppendMetaMetadataPos < MaxSizeT) + { + m_FileMetaMetadataManager.Truncate(m_AppendMetaMetadataPos); + m_FileMetaMetadataManager.SeekTo(m_AppendMetaMetadataPos); + } + else + { + m_FileMetadataIndexManager.SeekToFileEnd(); + } + + // Set the flag in the header of metadata index table to 1 again + // to indicate a new run begins + UpdateActiveFlag(true); + + // Truncate existing index file + if (m_AppendMetadataIndexPos < MaxSizeT) + { + m_FileMetadataIndexManager.Truncate(m_AppendMetadataIndexPos); + m_FileMetadataIndexManager.SeekTo(m_AppendMetadataIndexPos); + } + else + { + m_FileMetadataIndexManager.SeekToFileEnd(); + } + } + m_AppendDataPos.clear(); + } + + if (!m_WriterStep) + { + /* This is a new file or append at 0 + * Make headers in data buffer and metadata buffer (but do not write + * them yet so that Open() can stay free of writing to disk) + */ + if (m_Comm.Rank() == 0) + { + m_FileMetadataIndexManager.SeekToFileBegin(); + m_FileMetadataManager.SeekToFileBegin(); + m_FileMetaMetadataManager.SeekToFileBegin(); + } + // last attempt to clean up datafile if called with append mode, + // data existed but index was missing + if (m_Aggregator->m_IsAggregator) + { + m_FileDataManager.SeekTo(0); + } + } + + if (m_Comm.Rank() == 0) + { + m_WriterDataPos.resize(m_Comm.Size()); + } + + if (!m_WriterStep || + m_AppendWriterCount != static_cast(m_Comm.Size()) || + m_AppendAggregatorCount != + static_cast(m_Aggregator->m_NumAggregators) || + m_AppendSubfileCount != + static_cast(m_Aggregator->m_SubStreams)) + { + // new Writer Map is needed, generate now, write later + const uint64_t a = + static_cast(m_Aggregator->m_SubStreamIndex); + m_WriterSubfileMap = m_Comm.GatherValues(a, 0); + } +} + +void DaosWriter::EnterComputationBlock() noexcept +{ + if (m_Parameters.AsyncWrite && !m_BetweenStepPairs) + { + m_ComputationBlockStart = Now(); + { + m_AsyncWriteLock.lock(); + m_InComputationBlock = true; + m_AsyncWriteLock.unlock(); + } + } +} + +void DaosWriter::ExitComputationBlock() noexcept +{ + if (m_Parameters.AsyncWrite && m_InComputationBlock) + { + double t = Seconds(Now() - m_ComputationBlockStart).count(); + { + m_AsyncWriteLock.lock(); + if (t > 0.1) // only register long enough intervals + { + m_ComputationBlockTimes.emplace_back(m_ComputationBlockID, t); + m_ComputationBlocksLength += t; + } + m_InComputationBlock = false; + ++m_ComputationBlockID; + m_AsyncWriteLock.unlock(); + } + } +} + +void DaosWriter::FlushData(const bool isFinal) +{ + BufferV *DataBuf; + if (m_Parameters.BufferVType == (int)BufferVType::MallocVType) + { + DataBuf = m_BP5Serializer.ReinitStepData( + new MallocV("DaosWriter", false, m_BP5Serializer.m_BufferAlign, + m_BP5Serializer.m_BufferBlockSize, + m_Parameters.InitialBufferSize, + m_Parameters.GrowthFactor), + m_Parameters.AsyncWrite || m_Parameters.DirectIO); + } + else + { + DataBuf = m_BP5Serializer.ReinitStepData( + new ChunkV("DaosWriter", false, m_BP5Serializer.m_BufferAlign, + m_BP5Serializer.m_BufferBlockSize, + m_Parameters.BufferChunkSize), + m_Parameters.AsyncWrite || m_Parameters.DirectIO); + } + + auto databufsize = DataBuf->Size(); + WriteData(DataBuf); + /* DataBuf is deleted in WriteData() */ + DataBuf = nullptr; + + m_ThisTimestepDataSize += databufsize; + + if (!isFinal) + { + size_t tmp[2]; + // aggregate start pos and data size to rank 0 + tmp[0] = m_StartDataPos; + tmp[1] = databufsize; + + std::vector RecvBuffer; + if (m_Comm.Rank() == 0) + { + RecvBuffer.resize(m_Comm.Size() * 2); + } + m_Comm.GatherArrays(tmp, 2, RecvBuffer.data(), 0); + if (m_Comm.Rank() == 0) + { + FlushPosSizeInfo.push_back(RecvBuffer); + } + } +} + +void DaosWriter::Flush(const int transportIndex) {} + +void DaosWriter::PerformDataWrite() { FlushData(false); } + +void DaosWriter::DestructorClose(bool Verbose) noexcept +{ + if (Verbose) + { + std::cerr << "BP5 Writer \"" << m_Name + << "\" Destroyed without a prior Close()." << std::endl; + std::cerr << "This may result in corrupt output." << std::endl; + } + // close metadata index file + UpdateActiveFlag(false); + m_IsOpen = false; +} + +DaosWriter::~DaosWriter() +{ + if (m_IsOpen) + { + DestructorClose(m_FailVerbose); + } + m_IsOpen = false; +} + +void DaosWriter::DoClose(const int transportIndex) +{ + PERFSTUBS_SCOPED_TIMER("DaosWriter::Close"); + + if ((m_WriterStep == 0) && !m_BetweenStepPairs) + { + /* never did begin step, do one now */ + BeginStep(StepMode::Update); + } + if (m_BetweenStepPairs) + { + EndStep(); + } + + TimePoint wait_start = Now(); + Seconds wait(0.0); + if (m_WriteFuture.valid()) + { + m_Profiler.Start("WaitOnAsync"); + m_AsyncWriteLock.lock(); + m_flagRush = true; + m_AsyncWriteLock.unlock(); + m_WriteFuture.get(); + wait += Now() - wait_start; + m_Profiler.Stop("WaitOnAsync"); + } + + m_FileDataManager.CloseFiles(transportIndex); + // Delete files from temporary storage if draining was on + + if (m_Comm.Rank() == 0) + { + // close metadata file + m_FileMetadataManager.CloseFiles(); + + // close metametadata file + m_FileMetaMetadataManager.CloseFiles(); + } + + if (m_Parameters.AsyncWrite) + { + // wait until all process' writing thread completes + m_Profiler.Start("WaitOnAsync"); + wait_start = Now(); + m_Comm.Barrier(); + AsyncWriteDataCleanup(); + wait += Now() - wait_start; + if (m_Comm.Rank() == 0 && m_Parameters.verbose > 0) + { + std::cout << "Close waited " << wait.count() + << " seconds on async threads" << std::endl; + } + m_Profiler.Stop("WaitOnAsync"); + } + + if (m_Comm.Rank() == 0) + { + if (m_Parameters.AsyncWrite) + { + WriteMetadataFileIndex(m_LatestMetaDataPos, m_LatestMetaDataSize); + } + // close metadata index file + UpdateActiveFlag(false); + m_FileMetadataIndexManager.CloseFiles(); + } + + FlushProfiler(); +} + +void DaosWriter::FlushProfiler() +{ + auto transportTypes = m_FileDataManager.GetTransportsTypes(); + + // find first File type output, where we can write the profile + int fileTransportIdx = -1; + for (size_t i = 0; i < transportTypes.size(); ++i) + { + if (transportTypes[i].compare(0, 4, "File") == 0) + { + fileTransportIdx = static_cast(i); + } + } + + auto transportProfilers = m_FileDataManager.GetTransportsProfilers(); + + auto transportTypesMD = m_FileMetadataManager.GetTransportsTypes(); + auto transportProfilersMD = m_FileMetadataManager.GetTransportsProfilers(); + + transportTypes.insert(transportTypes.end(), transportTypesMD.begin(), + transportTypesMD.end()); + + transportProfilers.insert(transportProfilers.end(), + transportProfilersMD.begin(), + transportProfilersMD.end()); + + // m_Profiler.WriteOut(transportTypes, transportProfilers); + + const std::string lineJSON( + m_Profiler.GetRankProfilingJSON(transportTypes, transportProfilers) + + ",\n"); + + const std::vector profilingJSON( + m_Profiler.AggregateProfilingJSON(lineJSON)); + + if (m_RankMPI == 0) + { + // std::cout << "write profiling file!" << std::endl; + std::string profileFileName; + if (m_DrainBB) + { + // auto bpTargetNames = + // m_BP4Serializer.GetBPBaseNames({m_Name}); + std::vector bpTargetNames = {m_Name}; + if (fileTransportIdx > -1) + { + profileFileName = + bpTargetNames[fileTransportIdx] + "/profiling.json"; + } + else + { + profileFileName = bpTargetNames[0] + "_profiling.json"; + } + m_FileDrainer.AddOperationWrite( + profileFileName, profilingJSON.size(), profilingJSON.data()); + } + else + { + transport::FileFStream profilingJSONStream(m_Comm); + // auto bpBaseNames = + // m_BP4Serializer.GetBPBaseNames({m_BBName}); + std::vector bpBaseNames = {m_Name}; + if (fileTransportIdx > -1) + { + profileFileName = + bpBaseNames[fileTransportIdx] + "/profiling.json"; + } + else + { + profileFileName = bpBaseNames[0] + "_profiling.json"; + } + profilingJSONStream.Open(profileFileName, Mode::Write); + profilingJSONStream.Write(profilingJSON.data(), + profilingJSON.size()); + profilingJSONStream.Close(); + } + } +} + +size_t DaosWriter::DebugGetDataBufferSize() const +{ + return m_BP5Serializer.DebugGetDataBufferSize(); +} + +void DaosWriter::PutCommon(VariableBase &variable, const void *values, + bool sync) +{ + if (!m_BetweenStepPairs) + { + BeginStep(StepMode::Update); + } + + // if the user buffer is allocated on the GPU always use sync mode + if (variable.GetMemorySpace(values) != MemorySpace::Host) + sync = true; + + size_t *Shape = NULL; + size_t *Start = NULL; + size_t *Count = NULL; + size_t DimCount = variable.m_Count.size(); + + if (variable.m_ShapeID == ShapeID::GlobalArray) + { + Shape = variable.m_Shape.data(); + Count = variable.m_Count.data(); + Start = variable.m_Start.data(); + } + else if (variable.m_ShapeID == ShapeID::LocalArray) + { + Count = variable.m_Count.data(); + } + else if (variable.m_ShapeID == ShapeID::JoinedArray) + { + Count = variable.m_Count.data(); + Shape = variable.m_Shape.data(); + } + + size_t ObjSize; + if (variable.m_Type == DataType::Struct) + { + ObjSize = variable.m_ElementSize; + } + else + { + ObjSize = helper::GetDataTypeSize(variable.m_Type); + } + + if (!sync) + { + /* If arrays is small, force copying to internal buffer to aggregate + * small writes */ + size_t n = helper::GetTotalSize(variable.m_Count) * ObjSize; + if (n < m_Parameters.MinDeferredSize) + { + sync = true; + } + } + + if (!variable.m_MemoryCount.empty()) + { + int DimCount = variable.m_Count.size(); + std::vector ZeroDims(DimCount); + // get a temporary span then fill with memselection now + format::BufferV::BufferPos bp5span(0, 0, 0); + m_BP5Serializer.Marshal((void *)&variable, variable.m_Name.c_str(), + variable.m_Type, variable.m_ElementSize, + DimCount, Shape, Count, Start, nullptr, false, + &bp5span); + void *ptr = + m_BP5Serializer.GetPtr(bp5span.bufferIdx, bp5span.posInBuffer); + + const bool sourceRowMajor = helper::IsRowMajor(m_IO.m_HostLanguage); + + helper::NdCopy( + (const char *)values, helper::CoreDims(ZeroDims), + variable.m_MemoryCount, sourceRowMajor, false, (char *)ptr, + variable.m_MemoryStart, variable.m_Count, sourceRowMajor, false, + ObjSize, helper::CoreDims(), helper::CoreDims(), helper::CoreDims(), + helper::CoreDims(), false /* safemode */, variable.m_MemSpace); + } + else + { + if (variable.m_Type == DataType::String) + { + std::string &source = *(std::string *)values; + void *p = &(source[0]); + m_BP5Serializer.Marshal((void *)&variable, variable.m_Name.c_str(), + variable.m_Type, variable.m_ElementSize, + DimCount, Shape, Count, Start, &p, sync, + nullptr); + } + else + m_BP5Serializer.Marshal((void *)&variable, variable.m_Name.c_str(), + variable.m_Type, variable.m_ElementSize, + DimCount, Shape, Count, Start, values, sync, + nullptr); + } +} + +#define declare_type(T) \ + void DaosWriter::DoPut(Variable &variable, \ + typename Variable::Span &span, \ + const bool initialize, const T &value) \ + { \ + PERFSTUBS_SCOPED_TIMER("DaosWriter::Put"); \ + PutCommonSpan(variable, span, initialize, value); \ + } + +ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type) +#undef declare_type + +#define declare_type(T) \ + void DaosWriter::DoPutSync(Variable &variable, const T *data) \ + { \ + PutCommon(variable, data, true); \ + } \ + void DaosWriter::DoPutDeferred(Variable &variable, const T *data) \ + { \ + PutCommon(variable, data, false); \ + } + +ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + +#define declare_type(T, L) \ + T *DaosWriter::DoBufferData_##L(const int bufferIdx, \ + const size_t payloadPosition, \ + const size_t bufferID) noexcept \ + { \ + return reinterpret_cast( \ + m_BP5Serializer.GetPtr(bufferIdx, payloadPosition)); \ + } + +ADIOS2_FOREACH_PRIMITVE_STDTYPE_2ARGS(declare_type) +#undef declare_type + +void DaosWriter::DoPutStructSync(VariableStruct &variable, const void *data) +{ + PutCommon(variable, data, true); +} + +void DaosWriter::DoPutStructDeferred(VariableStruct &variable, const void *data) +{ + PutCommon(variable, data, false); +} + +void DaosWriter::daos_handle_share(daos_handle_t *hdl, int type) +{ + d_iov_t ghdl = {NULL, 0, 0}; + int rc; + + if (m_Comm.Rank() == 0) + { + /** fetch size of global handle */ + if (type == DaosWriter::HANDLE_POOL) + rc = daos_pool_local2global(*hdl, &ghdl); + else + rc = daos_cont_local2global(*hdl, &ghdl); + ASSERT(rc == 0, "local2global failed with %d", rc); + } + + /** broadcast size of global handle to all peers */ + MPI_Bcast(&ghdl.iov_buf_len, 1, MPI_UINT64_T, 0, MPI_COMM_WORLD); + + /** allocate buffer for global pool handle */ + ghdl.iov_buf = malloc(ghdl.iov_buf_len); + ghdl.iov_len = ghdl.iov_buf_len; + + if (m_Comm.Rank() == 0) + { + /** generate actual global handle to share with peer tasks */ + if (type == DaosWriter::HANDLE_POOL) + rc = daos_pool_local2global(*hdl, &ghdl); + else + rc = daos_cont_local2global(*hdl, &ghdl); + ASSERT(rc == 0, "local2global failed with %d", rc); + } + + /** broadcast global handle to all peers */ + MPI_Bcast(ghdl.iov_buf, ghdl.iov_len, MPI_BYTE, 0, MPI_COMM_WORLD); + + if (m_Comm.Rank() != 0) + { + /** unpack global handle */ + if (type == DaosWriter::HANDLE_POOL) + { + /* NB: Only pool_global2local are different */ + rc = daos_pool_global2local(ghdl, hdl); + } + else + { + rc = daos_cont_global2local(poh, ghdl, hdl); + } + ASSERT(rc == 0, "global2local failed with %d", rc); + } + + free(ghdl.iov_buf); + + MPI_Barrier(MPI_COMM_WORLD); +} + +} // end namespace engine +} // end namespace core +} // end namespace adios2 diff --git a/source/adios2/engine/daos/DaosWriter.h b/source/adios2/engine/daos/DaosWriter.h new file mode 100644 index 0000000000..3b1acbf2eb --- /dev/null +++ b/source/adios2/engine/daos/DaosWriter.h @@ -0,0 +1,403 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * DaosWriter.h + * + */ + +#ifndef ADIOS2_ENGINE_DAOS_DAOSWRITER_H_ +#define ADIOS2_ENGINE_DAOS_DAOSWRITER_H_ +#define DSS_PSETID "daos_server" + +#include "adios2/common/ADIOSConfig.h" +#include "adios2/core/CoreTypes.h" +#include "adios2/core/Engine.h" +#include "adios2/engine/daos/DaosEngine.h" +#include "adios2/helper/adiosComm.h" +#include "adios2/helper/adiosMemory.h" // PaddingToAlignOffset +#include "adios2/toolkit/aggregator/mpi/MPIChain.h" +#include "adios2/toolkit/aggregator/mpi/MPIShmChain.h" +#include "adios2/toolkit/burstbuffer/FileDrainerSingleThread.h" +#include "adios2/toolkit/format/bp5/BP5Serializer.h" +#include "adios2/toolkit/format/buffer/BufferV.h" +#include "adios2/toolkit/shm/Spinlock.h" +#include "adios2/toolkit/shm/TokenChain.h" +#include "adios2/toolkit/transportman/TransportMan.h" +#include +#include +#include + +#define FAIL(fmt, ...) \ + do \ + { \ + fprintf(stderr, "Process %d(%s): " fmt " aborting\n", m_Comm.Rank(), \ + node, ##__VA_ARGS__); \ + MPI_Abort(MPI_COMM_WORLD, 1); \ + } while (0) +#define ASSERT(cond, ...) \ + do \ + { \ + if (!(cond)) \ + FAIL(__VA_ARGS__); \ + } while (0) + +namespace adios2 +{ +namespace core +{ +namespace engine +{ + +class DaosWriter : public DaosEngine, public core::Engine +{ + +public: + /** + * Constructor for file Writer in Daos format + * @param name unique name given to the engine + * @param openMode w (supported), r, a from OpenMode in ADIOSTypes.h + * @param comm multi-process communicator + */ + DaosWriter(IO &io, const std::string &name, const Mode mode, + helper::Comm comm); + + ~DaosWriter(); + + StepStatus BeginStep(StepMode mode, + const float timeoutSeconds = -1.0) final; + size_t CurrentStep() const final; + void PerformPuts() final; + void PerformDataWrite() final; + void EndStep() final; + void Flush(const int transportIndex = -1) final; + + size_t DebugGetDataBufferSize() const final; + +private: + /** Single object controlling BP buffering */ + format::BP5Serializer m_BP5Serializer; + + /** Manage BP data files Transports from IO AddTransport */ + transportman::TransportMan m_FileDataManager; + + /** Manages the optional collective metadata files */ + transportman::TransportMan m_FileMetadataManager; + + /* transport manager for managing the metadata index file */ + transportman::TransportMan m_FileMetadataIndexManager; + + transportman::TransportMan m_FileMetaMetadataManager; + + /* DAOS declarations */ + + uuid_t pool_uuid, cont_uuid; + char *pool_label = "pool_ranjansv"; + char *cont_label = "adios-daos-engine-cont"; + + /* Declare variables for pool and container handles */ + daos_handle_t poh, coh; + + enum DAOS_handleType + { + HANDLE_POOL, + HANDLE_CO, + }; + + /* Declare variables for the KV object */ + daos_handle_t oh; + daos_obj_id_t oid; + + char node[128] = "unknown"; + + int64_t m_WriterStep = 0; + /* + * Burst buffer variables + */ + /** true if burst buffer is used to write */ + bool m_WriteToBB = false; + /** true if burst buffer is drained to disk */ + bool m_DrainBB = true; + /** File drainer thread if burst buffer is used */ + burstbuffer::FileDrainerSingleThread m_FileDrainer; + /** m_Name modified with burst buffer path if BB is used, + * == m_Name otherwise. + * m_Name is a constant of Engine and is the user provided target path + */ + std::string m_BBName; + /* Name of subfiles to directly write to (for all transports) + * This is either original target or burst buffer if used */ + std::vector m_SubStreamNames; + /* Name of subfiles on target if burst buffer is used (for all transports) + */ + std::vector m_DrainSubStreamNames; + std::vector m_MetadataFileNames; + std::vector m_DrainMetadataFileNames; + std::vector m_MetaMetadataFileNames; + std::vector m_MetadataIndexFileNames; + std::vector m_DrainMetadataIndexFileNames; + std::vector m_ActiveFlagFileNames; + + bool m_BetweenStepPairs = false; + + void Init() final; + + /** Parses parameters from IO SetParameters */ + void InitParameters() final; + /** Set up the aggregator */ + void InitAggregator(); + /** Complete opening/createing metadata and data files */ + void InitTransports() final; + /** DAOS pool connection and container opening */ + void InitDAOS(); + /** Allocates memory and starts a PG group */ + void InitBPBuffer(); + void NotifyEngineAttribute(std::string name, DataType type) noexcept; + /** Notify the engine when a new attribute is defined or modified. Called + * from IO.tcc + */ + void NotifyEngineAttribute(std::string name, AttributeBase *Attr, + void *data) noexcept; + + void EnterComputationBlock() noexcept; + /** Inform about computation block through User->ADIOS->IO */ + void ExitComputationBlock() noexcept; + +#define declare_type(T) \ + void DoPut(Variable &variable, typename Variable::Span &span, \ + const bool initialize, const T &value) final; + + ADIOS2_FOREACH_PRIMITIVE_STDTYPE_1ARG(declare_type) +#undef declare_type + + template + void PutCommonSpan(Variable &variable, typename Variable::Span &span, + const bool initialize, const T &value); + +#define declare_type(T) \ + void DoPutSync(Variable &, const T *) final; \ + void DoPutDeferred(Variable &, const T *) final; + + ADIOS2_FOREACH_STDTYPE_1ARG(declare_type) +#undef declare_type + + void PutCommon(VariableBase &variable, const void *data, bool sync); + +#define declare_type(T, L) \ + T *DoBufferData_##L(const int bufferIdx, const size_t payloadPosition, \ + const size_t bufferID = 0) noexcept final; + + ADIOS2_FOREACH_PRIMITVE_STDTYPE_2ARGS(declare_type) +#undef declare_type + + void DoPutStructSync(VariableStruct &, const void *) final; + void DoPutStructDeferred(VariableStruct &, const void *) final; + + void PutStruct(VariableStruct &, const void *, bool); + + void FlushData(const bool isFinal = false); + + void DoClose(const int transportIndex = -1) final; + + /** Write a profiling.json file from m_BP1Writer and m_TransportsManager + * profilers*/ + void WriteProfilingJSONFile(); + + void WriteMetaMetadata( + const std::vector MetaMetaBlocks); + + void WriteMetadataFileIndex(uint64_t MetaDataPos, uint64_t MetaDataSize); + + uint64_t WriteMetadata(const std::vector &MetaDataBlocks, + const std::vector &AttributeBlocks); + + /** Write Data to disk, in an aggregator chain */ + void WriteData(format::BufferV *Data); + void WriteData_EveryoneWrites(format::BufferV *Data, + bool SerializedWriters); + void WriteData_EveryoneWrites_Async(format::BufferV *Data, + bool SerializedWriters); + void WriteData_TwoLevelShm(format::BufferV *Data); + void WriteData_TwoLevelShm_Async(format::BufferV *Data); + + void UpdateActiveFlag(const bool active); + + void WriteCollectiveMetadataFile(const bool isFinal = false); + + void MarshalAttributes(); + + /* Two-level-shm aggregator functions */ + void WriteMyOwnData(format::BufferV *Data); + void SendDataToAggregator(format::BufferV *Data); + void WriteOthersData(const size_t TotalSize); + + template + void PerformPutCommon(Variable &variable); + + void FlushProfiler(); + + /** manages all communication tasks in aggregation */ + aggregator::MPIAggregator *m_Aggregator; // points to one of these below + aggregator::MPIShmChain m_AggregatorTwoLevelShm; + aggregator::MPIChain m_AggregatorEveroneWrites; + bool m_IAmDraining = false; + bool m_IAmWritingData = false; + helper::Comm *DataWritingComm; // processes that write the same data file + // aggregators only (valid if m_Aggregator->m_Comm.Rank() == 0) + helper::Comm m_CommAggregators; + adios2::profiling::JSONProfiler m_Profiler; + +protected: + virtual void DestructorClose(bool Verbose) noexcept; + +private: + // updated during WriteMetaData + uint64_t m_MetaDataPos = 0; + + /** On every process, at the end of writing, this holds the offset + * where they started writing (needed for global metadata) + */ + uint64_t m_StartDataPos = 0; + /** On aggregators, at the end of writing, this holds the starting offset + * to the next step's writing; otherwise used as temporary offset variable + * during writing on every process and points to the end of the process' + * data block in the file (not used for anything) + */ + uint64_t m_DataPos = 0; + + /* + * Total data written this timestep + */ + uint64_t m_ThisTimestepDataSize = 0; + + /** rank 0 collects m_StartDataPos in this vector for writing it + * to the index file + */ + std::vector m_WriterDataPos; + + bool m_MarshalAttributesNecessary = true; + + std::vector> FlushPosSizeInfo; + + void MakeHeader(std::vector &buffer, size_t &position, + const std::string fileType, const bool isActive); + + std::vector m_WriterSubfileMap; // rank => subfile index + + // Append helper data + std::vector m_AppendDataPos; // each subfile append pos + size_t m_AppendMetadataPos; // metadata file append pos + size_t m_AppendMetaMetadataPos; // meta-metadata file append pos + size_t m_AppendMetadataIndexPos; // index file append pos + uint32_t m_AppendWriterCount; // last active number of writers + unsigned int m_AppendAggregatorCount; // last active number of aggr + unsigned int m_AppendSubfileCount; // last active number of subfiles + /* Process existing index, fill in append variables, + * and return the actual step we land after appending. + * Uses parameter AppendAfterStep + * It resets m_Aggregator->m_NumAggregators so init aggregators later + */ + uint64_t CountStepsInMetadataIndex(format::BufferSTL &bufferSTL); + + /* Async write's future */ + std::future m_WriteFuture; + // variables to delay writing to index file + uint64_t m_LatestMetaDataPos; + uint64_t m_LatestMetaDataSize; + Seconds m_LastTimeBetweenSteps = Seconds(0.0); + Seconds m_TotalTimeBetweenSteps = Seconds(0.0); + Seconds m_AvgTimeBetweenSteps = Seconds(0.0); + Seconds m_ExpectedTimeBetweenSteps = Seconds(0.0); + TimePoint m_EndStepEnd; + TimePoint m_EngineStart; + TimePoint m_BeginStepStart; + bool m_flagRush; // main thread flips this in Close, async thread watches it + bool m_InComputationBlock = false; // main thread flips this in Clos + TimePoint m_ComputationBlockStart; + /* block counter and length in seconds */ + size_t m_ComputationBlockID = 0; + + struct ComputationBlockInfo + { + size_t blockID; + double length; // seconds + ComputationBlockInfo(const size_t id, const double len) + : blockID(id), length(len){}; + }; + + std::vector m_ComputationBlockTimes; + /* sum of computationBlockTimes at start of async IO; */ + double m_ComputationBlocksLength = 0.0; + + /* struct of data passed from main thread to async write thread at launch */ + struct AsyncWriteInfo + { + adios2::aggregator::MPIAggregator *aggregator; + int rank_global; + helper::Comm comm_chain; + int rank_chain; + int nproc_chain; + TimePoint tstart; + adios2::shm::TokenChain *tokenChain; + transportman::TransportMan *tm; + adios2::format::BufferV *Data; + uint64_t startPos; + uint64_t totalSize; + double deadline; // wall-clock time available in seconds + bool *flagRush; // flipped from false to true by main thread + bool *inComputationBlock; // flipped back and forth by main thread + // comm-free time within deadline in seconds + double computationBlocksLength; + std::vector expectedComputationBlocks; // a copy + std::vector + *currentComputationBlocks; // extended by main thread + size_t *currentComputationBlockID; // increased by main thread + shm::Spinlock *lock; // race condition over currentComp* variables + }; + + AsyncWriteInfo *m_AsyncWriteInfo; + /* lock to handle race condition over the following currentComp* variables + m_InComputationBlock / AsyncWriteInfo::inComputationBlock + m_ComputationBlockID / AsyncWriteInfo::currentComputationBlockID + m_flagRush / AsyncWriteInfo::flagRush + Currently not used + m_ComputationBlockTimes / AsyncWriteInfo::currentComputationBlocks + Note: The rush flag does not need protection but CI TSAN sanitizer + screams data race if not protected. + */ + shm::Spinlock m_AsyncWriteLock; + + /* Static functions that will run in another thread */ + static int AsyncWriteThread_EveryoneWrites(AsyncWriteInfo *info); + static int AsyncWriteThread_TwoLevelShm(AsyncWriteInfo *info); + static void AsyncWriteThread_TwoLevelShm_Aggregator(AsyncWriteInfo *info); + static void AsyncWriteThread_TwoLevelShm_SendDataToAggregator( + aggregator::MPIShmChain *a, format::BufferV *Data); + + /* write own data used by both + EveryoneWrites and TwoLevelShm async threads */ + static void AsyncWriteOwnData(AsyncWriteInfo *info, + std::vector &DataVec, + const size_t totalsize, + const bool seekOnFirstWrite); + enum class ComputationStatus + { + InComp, + NotInComp_ExpectMore, + NoMoreComp + }; + static ComputationStatus IsInComputationBlock(AsyncWriteInfo *info, + size_t &compBlockIdx); + + void AsyncWriteDataCleanup(); + void AsyncWriteDataCleanup_EveryoneWrites(); + void AsyncWriteDataCleanup_TwoLevelShm(); + + void daos_handle_share(daos_handle_t *, int); +}; + +} // end namespace engine +} // end namespace core +} // end namespace adios2 + +#endif /* ADIOS2_ENGINE_DAOS_DAOSWRITER_H_ */ diff --git a/source/adios2/engine/daos/DaosWriter.tcc b/source/adios2/engine/daos/DaosWriter.tcc new file mode 100644 index 0000000000..9b72a65f5b --- /dev/null +++ b/source/adios2/engine/daos/DaosWriter.tcc @@ -0,0 +1,97 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * DaosWriter.tcc implementation of template functions with known type + * + */ +#ifndef ADIOS2_ENGINE_DAOS_DAOSWRITER_TCC_ +#define ADIOS2_ENGINE_DAOS_DAOSWRITER_TCC_ + +#include "DaosWriter.h" +#include "adios2/helper/adiosMath.h" + +namespace adios2 +{ +namespace core +{ +namespace engine +{ + +template +void DaosWriter::PutCommonSpan(Variable &variable, + typename Variable::Span &span, + const bool initialize, const T &value) +{ + format::BufferV::BufferPos bp5span(0, 0, 0); + + size_t *Shape = NULL; + size_t *Start = NULL; + size_t *Count = NULL; + size_t DimCount = 0; + + if (!m_BetweenStepPairs) + { + BeginStep(StepMode::Update); + } + if (variable.m_ShapeID == ShapeID::GlobalArray) + { + DimCount = variable.m_Shape.size(); + Shape = variable.m_Shape.data(); + Start = variable.m_Start.data(); + Count = variable.m_Count.data(); + } + else if (variable.m_ShapeID == ShapeID::JoinedArray) + { + Shape = variable.m_Shape.data(); + DimCount = variable.m_Count.size(); + Count = variable.m_Count.data(); + } + else if (variable.m_ShapeID == ShapeID::LocalArray) + { + DimCount = variable.m_Count.size(); + Count = variable.m_Count.data(); + } + + if (std::is_same::value) + { + m_BP5Serializer.Marshal((void *)&variable, variable.m_Name.c_str(), + variable.m_Type, variable.m_ElementSize, + DimCount, Shape, Count, Start, nullptr, false, + &bp5span); + } + else + m_BP5Serializer.Marshal((void *)&variable, variable.m_Name.c_str(), + variable.m_Type, variable.m_ElementSize, + DimCount, Shape, Count, Start, nullptr, false, + &bp5span); + + span.m_PayloadPosition = bp5span.posInBuffer; + span.m_BufferIdx = bp5span.bufferIdx; + span.m_Value = value; + + /* initialize buffer if needed */ + if (initialize) + { + const size_t ElemCount = m_BP5Serializer.CalcSize(DimCount, Count); + T *itBegin = reinterpret_cast( + m_BP5Serializer.GetPtr(span.m_BufferIdx, span.m_PayloadPosition)); + + // TODO from BP4: does std::fill_n have a bug in gcc or due to + // optimizations this is impossible due to memory alignment? This seg + // faults in Release mode only . Even RelWithDebInfo works, replacing + // with explicit loop below using access operator [] + // std::fill_n(itBegin, blockSize, span->m_Value); + + for (size_t i = 0; i < ElemCount; ++i) + { + itBegin[i] = value; + } + } +} + +} // end namespace engine +} // end namespace core +} // end namespace adios2 + +#endif /* ADIOS2_ENGINE_DAOS_DAOSWRITER_TCC_ */ diff --git a/source/adios2/engine/daos/DaosWriter_EveryoneWrites_Async.cpp b/source/adios2/engine/daos/DaosWriter_EveryoneWrites_Async.cpp new file mode 100644 index 0000000000..4549ed9dc8 --- /dev/null +++ b/source/adios2/engine/daos/DaosWriter_EveryoneWrites_Async.cpp @@ -0,0 +1,357 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * BP5Writer.cpp + * + */ + +#include "DaosWriter.h" +#include "DaosWriter.tcc" + +#include "adios2/common/ADIOSMacros.h" +#include "adios2/core/IO.h" +#include "adios2/helper/adiosFunctions.h" //CheckIndexRange +#include "adios2/toolkit/format/buffer/chunk/ChunkV.h" +#include "adios2/toolkit/format/buffer/malloc/MallocV.h" +#include "adios2/toolkit/transport/file/FileFStream.h" +#include + +#include // max +#include +#include + +namespace adios2 +{ +namespace core +{ +namespace engine +{ + +using namespace adios2::format; + +DaosWriter::ComputationStatus +DaosWriter::IsInComputationBlock(AsyncWriteInfo *info, size_t &compBlockIdx) +{ + ComputationStatus compStatus = ComputationStatus::NotInComp_ExpectMore; + size_t nExpectedBlocks = info->expectedComputationBlocks.size(); + + if (compBlockIdx >= nExpectedBlocks) + { + compStatus = ComputationStatus::NoMoreComp; + } + else + { + bool inComp = false; + size_t compBlockID = 0; + // access variables modified by main thread to avoid data race + info->lock->lock(); + compBlockID = *info->currentComputationBlockID; + inComp = *info->inComputationBlock; + info->lock->unlock(); + + /* Track which computation block we are in */ + if (inComp) + { + while (compBlockIdx < nExpectedBlocks && + info->expectedComputationBlocks[compBlockIdx].blockID < + compBlockID) + { + ++compBlockIdx; + } + if (info->expectedComputationBlocks[compBlockIdx].blockID > + compBlockID) + { + // the current computation block is a short one that was not + // recorded + compStatus = ComputationStatus::NotInComp_ExpectMore; + } + else + { + compStatus = ComputationStatus::InComp; + } + } + } + return compStatus; +} + +void DaosWriter::AsyncWriteOwnData(AsyncWriteInfo *info, + std::vector &DataVec, + const size_t totalsize, + const bool seekOnFirstWrite) +{ + /* local variables to track variables modified by main thread */ + size_t compBlockIdx = 0; /* position in vector to get length */ + + /* In a loop, write the data in smaller blocks */ + size_t nBlocks = DataVec.size(); + size_t wrote = 0; + size_t block = 0; + size_t temp_offset = 0; + size_t max_size = std::max(1024 * 1024UL, totalsize / 100UL); + + bool firstWrite = seekOnFirstWrite; + while (block < nBlocks) + { + bool doRush = false; + bool doSleep = false; + + info->lock->lock(); + doRush = *info->flagRush; + info->lock->unlock(); + + if (!doRush) + { + ComputationStatus compStatus = + IsInComputationBlock(info, compBlockIdx); + + /* Scheduling decisions: + Cases: + 1. Not in a computation block AND we still expect more + computation blocks down the line ==> Sleep + 2. In computation block ==> Write + 3. We are at the end of a computation block (how close??) AND we + still expect more computation blocks down the line 3. ==> Sleep + 4. We are at the end of the LAST computation block ==> Write + 5. No more computation blocks expected ==> Write all at once + 6. Main thread set flagRush ==> Write all at once + -- case 3 not handled yet properly + */ + + switch (compStatus) + { + case ComputationStatus::NotInComp_ExpectMore: + // case 1 + doSleep = true; + break; + case ComputationStatus::NoMoreComp: + // case 5 + doRush = true; + break; + default: + // cases 2, 3, 4 + break; + } + } + + if (doRush) + { + auto vec = std::vector(DataVec.begin() + block, + DataVec.end()); + vec[0].iov_base = + (const char *)DataVec[block].iov_base + temp_offset; + vec[0].iov_len = DataVec[block].iov_len - temp_offset; + size_t pos = MaxSizeT; // <==> no seek inside WriteFileAt + if (firstWrite) + { + pos = info->startPos + wrote; // seek to pos + } + /*std::cout << "Async write on Rank " << info->rank_global + << " write the rest of " << totalsize - wrote + << " bytes at pos " << pos << std::endl;*/ + + info->tm->WriteFileAt(vec.data(), vec.size(), pos); + + break; /* Exit loop after this final write */ + } + + if (doSleep) + { + std::this_thread::sleep_for(core::Seconds(0.01)); + continue; + } + + /* Write next batch of data */ + + /* Get the next n bytes from the current block, current offset */ + size_t n = DataVec[block].iov_len - temp_offset; + if (n > max_size) + { + n = max_size; + } + + if (firstWrite) + { + info->tm->WriteFileAt((const char *)DataVec[block].iov_base + + temp_offset, + n, info->startPos); + firstWrite = false; + } + else + { + info->tm->WriteFiles( + (const char *)DataVec[block].iov_base + temp_offset, n); + } + + /* Have we processed the entire block or staying with it? */ + if (n + temp_offset < DataVec[block].iov_len) + { + temp_offset += n; + } + else + { + temp_offset = 0; + ++block; + } + wrote += n; + } +}; + +int DaosWriter::AsyncWriteThread_EveryoneWrites(AsyncWriteInfo *info) +{ + if (info->tokenChain) + { + if (info->rank_chain > 0) + { + info->tokenChain->RecvToken(); + } + } + + std::vector DataVec = info->Data->DataVec(); + const uint64_t mysize = info->Data->Size(); + AsyncWriteOwnData(info, DataVec, mysize, true); + + if (info->tokenChain) + { + uint64_t t = 1; + info->tokenChain->SendToken(t); + if (!info->rank_chain) + { + info->tokenChain->RecvToken(); + } + } + delete info->Data; + return 1; +}; + +void DaosWriter::WriteData_EveryoneWrites_Async(format::BufferV *Data, + bool SerializedWriters) +{ + + const aggregator::MPIChain *a = + dynamic_cast(m_Aggregator); + + // new step writing starts at offset m_DataPos on aggregator + // others will wait for the position to arrive from the rank below + + if (a->m_Comm.Rank() > 0) + { + a->m_Comm.Recv( + &m_DataPos, 1, a->m_Comm.Rank() - 1, 0, + "Chain token in DaosWriter::WriteData_EveryoneWrites_Async"); + } + + // align to PAGE_SIZE + m_DataPos += + helper::PaddingToAlignOffset(m_DataPos, m_Parameters.StripeSize); + m_StartDataPos = m_DataPos; + + if (a->m_Comm.Rank() < a->m_Comm.Size() - 1) + { + uint64_t nextWriterPos = m_DataPos + Data->Size(); + a->m_Comm.Isend( + &nextWriterPos, 1, a->m_Comm.Rank() + 1, 0, + "Chain token in DaosWriter::WriteData_EveryoneWrites_Async"); + } + + m_DataPos += Data->Size(); + + /* a->comm can span multiple nodes but we need comm inside a node + when doing serialized aggregation */ + m_AsyncWriteInfo = new AsyncWriteInfo(); + m_AsyncWriteInfo->aggregator = nullptr; + m_AsyncWriteInfo->rank_global = m_Comm.Rank(); + if (SerializedWriters) + { + m_AsyncWriteInfo->comm_chain = a->m_Comm.GroupByShm(); + m_AsyncWriteInfo->rank_chain = m_AsyncWriteInfo->comm_chain.Rank(); + m_AsyncWriteInfo->nproc_chain = m_AsyncWriteInfo->comm_chain.Size(); + m_AsyncWriteInfo->tokenChain = + new shm::TokenChain(&m_AsyncWriteInfo->comm_chain); + } + else + { + m_AsyncWriteInfo->comm_chain = helper::Comm(); // not needed + m_AsyncWriteInfo->rank_chain = a->m_Comm.Rank(); + m_AsyncWriteInfo->nproc_chain = a->m_Comm.Size(); + m_AsyncWriteInfo->tokenChain = nullptr; + } + m_AsyncWriteInfo->tstart = m_EngineStart; + m_AsyncWriteInfo->tm = &m_FileDataManager; + m_AsyncWriteInfo->Data = Data; + m_AsyncWriteInfo->startPos = m_StartDataPos; + m_AsyncWriteInfo->totalSize = Data->Size(); + m_AsyncWriteInfo->deadline = m_ExpectedTimeBetweenSteps.count(); + m_AsyncWriteInfo->flagRush = &m_flagRush; + m_AsyncWriteInfo->lock = &m_AsyncWriteLock; + + if (m_ComputationBlocksLength > 0.0 && + m_Parameters.AsyncWrite == (int)AsyncWrite::Guided) + { + m_AsyncWriteInfo->inComputationBlock = &m_InComputationBlock; + m_AsyncWriteInfo->computationBlocksLength = m_ComputationBlocksLength; + if (m_AsyncWriteInfo->deadline < m_ComputationBlocksLength) + { + m_AsyncWriteInfo->deadline = m_ComputationBlocksLength; + } + m_AsyncWriteInfo->expectedComputationBlocks = + m_ComputationBlockTimes; // copy! + m_AsyncWriteInfo->currentComputationBlocks = + &m_ComputationBlockTimes; // ptr! + m_AsyncWriteInfo->currentComputationBlockID = &m_ComputationBlockID; + + /* Clear current block tracker now so that async thread does not get + confused with the past info */ + m_ComputationBlockTimes.clear(); + m_ComputationBlocksLength = 0.0; + m_ComputationBlockID = 0; + } + else + { + if (m_Parameters.AsyncWrite == (int)AsyncWrite::Naive) + { + m_AsyncWriteInfo->deadline = 0; + } + m_AsyncWriteInfo->inComputationBlock = nullptr; + m_AsyncWriteInfo->computationBlocksLength = 0.0; + m_AsyncWriteInfo->currentComputationBlocks = nullptr; + m_AsyncWriteInfo->currentComputationBlockID = nullptr; + } + + m_WriteFuture = std::async( + std::launch::async, AsyncWriteThread_EveryoneWrites, m_AsyncWriteInfo); + + // At this point modifying Data in main thread is prohibited !!! + + if (a->m_Comm.Size() > 1) + { + // at the end, last rank sends back the final data pos to first rank + // so it can update its data pos + if (a->m_Comm.Rank() == a->m_Comm.Size() - 1) + { + a->m_Comm.Isend(&m_DataPos, 1, 0, 0, + "Final chain token in " + "DaosWriter::WriteData_EveryoneWrites_Async"); + } + if (a->m_Comm.Rank() == 0) + { + a->m_Comm.Recv( + &m_DataPos, 1, a->m_Comm.Size() - 1, 0, + "Chain token in DaosWriter::WriteData_EveryoneWrites_Async"); + } + } +} + +void DaosWriter::AsyncWriteDataCleanup_EveryoneWrites() +{ + if (m_AsyncWriteInfo->tokenChain) + { + delete m_AsyncWriteInfo->tokenChain; + } + delete m_AsyncWriteInfo; + m_AsyncWriteInfo = nullptr; +} + +} // end namespace engine +} // end namespace core +} // end namespace adios2 diff --git a/source/adios2/engine/daos/DaosWriter_TwoLevelShm.cpp b/source/adios2/engine/daos/DaosWriter_TwoLevelShm.cpp new file mode 100644 index 0000000000..9ef11dd74b --- /dev/null +++ b/source/adios2/engine/daos/DaosWriter_TwoLevelShm.cpp @@ -0,0 +1,298 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * DaosWriter.cpp + * + */ + +#include "DaosWriter.h" + +#include "adios2/common/ADIOSMacros.h" +#include "adios2/core/IO.h" +#include "adios2/helper/adiosFunctions.h" //CheckIndexRange, PaddingToAlignOffset +#include "adios2/toolkit/format/buffer/chunk/ChunkV.h" +#include "adios2/toolkit/format/buffer/malloc/MallocV.h" +#include "adios2/toolkit/shm/TokenChain.h" +#include "adios2/toolkit/transport/file/FileFStream.h" +#include + +#include +#include +#include + +namespace adios2 +{ +namespace core +{ +namespace engine +{ + +using namespace adios2::format; + +void DaosWriter::WriteData_TwoLevelShm(format::BufferV *Data) +{ + aggregator::MPIShmChain *a = + dynamic_cast(m_Aggregator); + + // new step writing starts at offset m_DataPos on master aggregator + // other aggregators to the same file will need to wait for the position + // to arrive from the rank below + + // align to PAGE_SIZE (only valid on master aggregator at this point) + m_DataPos += + helper::PaddingToAlignOffset(m_DataPos, m_Parameters.StripeSize); + + // Each aggregator needs to know the total size they write + // This calculation is valid on aggregators only + std::vector mySizes = a->m_Comm.GatherValues(Data->Size()); + uint64_t myTotalSize = 0; + uint64_t maxSize = 0; + for (auto s : mySizes) + { + myTotalSize += s; + if (s > maxSize) + { + maxSize = s; + } + } + + if (a->m_Comm.Size() > 1) + { + size_t alignment_size = sizeof(max_align_t); + if (m_Parameters.DirectIO) + { + alignment_size = m_Parameters.DirectIOAlignOffset; + } + a->CreateShm(static_cast(maxSize), m_Parameters.MaxShmSize, + alignment_size); + } + + shm::TokenChain tokenChain(&a->m_Comm); + + if (a->m_IsAggregator) + { + // In each aggregator chain, send from master down the line + // these total sizes, so every aggregator knows where to start + if (a->m_AggregatorChainComm.Rank() > 0) + { + a->m_AggregatorChainComm.Recv( + &m_DataPos, 1, a->m_AggregatorChainComm.Rank() - 1, 0, + "AggregatorChain token in DaosWriter::WriteData_TwoLevelShm"); + // align to PAGE_SIZE + m_DataPos += helper::PaddingToAlignOffset(m_DataPos, + m_Parameters.StripeSize); + } + m_StartDataPos = m_DataPos; // metadata needs this info + if (a->m_AggregatorChainComm.Rank() < + a->m_AggregatorChainComm.Size() - 1) + { + uint64_t nextWriterPos = m_DataPos + myTotalSize; + a->m_AggregatorChainComm.Isend( + &nextWriterPos, 1, a->m_AggregatorChainComm.Rank() + 1, 0, + "Chain token in DaosWriter::WriteData"); + } + else if (a->m_AggregatorChainComm.Size() > 1) + { + // send back final position from last aggregator in file to master + // aggregator + uint64_t nextWriterPos = m_DataPos + myTotalSize; + a->m_AggregatorChainComm.Isend( + &nextWriterPos, 1, 0, 0, + "Chain token in DaosWriter::WriteData"); + } + + /*std::cout << "Rank " << m_Comm.Rank() + << " aggregator start writing step " << m_WriterStep + << " to subfile " << a->m_SubStreamIndex << " at pos " + << m_DataPos << " totalsize " << myTotalSize << std::endl;*/ + + // Send token to first non-aggregator to start filling shm + // Also informs next process its starting offset (for correct metadata) + uint64_t nextWriterPos = m_DataPos + Data->Size(); + tokenChain.SendToken(nextWriterPos); + + WriteMyOwnData(Data); + + /* Write from shm until every non-aggr sent all data */ + if (a->m_Comm.Size() > 1) + { + WriteOthersData(myTotalSize - Data->Size()); + } + + // Master aggregator needs to know where the last writing ended by the + // last aggregator in the chain, so that it can start from the correct + // position at the next output step + if (a->m_AggregatorChainComm.Size() > 1 && + !a->m_AggregatorChainComm.Rank()) + { + a->m_AggregatorChainComm.Recv( + &m_DataPos, 1, a->m_AggregatorChainComm.Size() - 1, 0, + "Chain token in DaosWriter::WriteData"); + } + } + else + { + // non-aggregators fill shared buffer in marching order + // they also receive their starting offset this way + m_StartDataPos = tokenChain.RecvToken(); + + /*std::cout << "Rank " << m_Comm.Rank() + << " non-aggregator recv token to fill shm = " + << m_StartDataPos << std::endl;*/ + + SendDataToAggregator(Data); + + uint64_t nextWriterPos = m_StartDataPos + Data->Size(); + tokenChain.SendToken(nextWriterPos); + } + + if (a->m_Comm.Size() > 1) + { + a->DestroyShm(); + } +} + +void DaosWriter::WriteMyOwnData(format::BufferV *Data) +{ + std::vector DataVec = Data->DataVec(); + m_StartDataPos = m_DataPos; + m_FileDataManager.WriteFileAt(DataVec.data(), DataVec.size(), + m_StartDataPos); + m_DataPos += Data->Size(); +} + +/*std::string DoubleBufferToString(const double *b, int n) +{ + std::ostringstream out; + out.precision(1); + out << std::fixed << "["; + char s[32]; + + for (int i = 0; i < n; ++i) + { + snprintf(s, sizeof(s), "%g", b[i]); + out << s; + if (i < n - 1) + { + out << ", "; + } + } + out << "]"; + return out.str(); +}*/ + +void DaosWriter::SendDataToAggregator(format::BufferV *Data) +{ + /* Only one process is running this function at once + See shmFillerToken in the caller function + + In a loop, copy the local data into the shared memory, alternating + between the two segments. + */ + + aggregator::MPIShmChain *a = + dynamic_cast(m_Aggregator); + + std::vector DataVec = Data->DataVec(); + size_t nBlocks = DataVec.size(); + + // size_t sent = 0; + size_t block = 0; + size_t temp_offset = 0; + while (block < nBlocks) + { + // potentially blocking call waiting on Aggregator + aggregator::MPIShmChain::ShmDataBuffer *b = a->LockProducerBuffer(); + // b->max_size: how much we can copy + // b->actual_size: how much we actually copy + b->actual_size = 0; + while (true) + { + /* Copy n bytes from the current block, current offset to shm + making sure to use up to shm_size bytes + */ + size_t n = DataVec[block].iov_len - temp_offset; + if (n > (b->max_size - b->actual_size)) + { + n = b->max_size - b->actual_size; + } + std::memcpy(&b->buf[b->actual_size], + (const char *)DataVec[block].iov_base + temp_offset, n); + b->actual_size += n; + + /* Have we processed the entire block or staying with it? */ + if (n + temp_offset < DataVec[block].iov_len) + { + temp_offset += n; + } + else + { + temp_offset = 0; + ++block; + } + + /* Have we reached the max allowed shm size ?*/ + if (b->actual_size >= b->max_size) + { + break; + } + if (block >= nBlocks) + { + break; + } + } + // sent += b->actual_size; + + /*if (m_RankMPI >= 42) + { + std::cout << "Rank " << m_Comm.Rank() + << " filled shm, data_size = " << b->actual_size + << " block = " << block + << " temp offset = " << temp_offset << " sent = " << sent + << " buf = " << static_cast(b->buf) << " = " + << DoubleBufferToString((double *)b->buf, + b->actual_size / sizeof(double)) + << std::endl; + }*/ + + a->UnlockProducerBuffer(); + } +} +void DaosWriter::WriteOthersData(size_t TotalSize) +{ + /* Only an Aggregator calls this function */ + aggregator::MPIShmChain *a = + dynamic_cast(m_Aggregator); + + size_t wrote = 0; + while (wrote < TotalSize) + { + // potentially blocking call waiting on some non-aggr process + aggregator::MPIShmChain::ShmDataBuffer *b = a->LockConsumerBuffer(); + + /*std::cout << "Rank " << m_Comm.Rank() + << " write from shm, data_size = " << b->actual_size + << " total so far = " << wrote + << " buf = " << static_cast(b->buf) << " = " + << DoubleBufferToString((double *)b->buf, + b->actual_size / sizeof(double)) + << std::endl;*/ + /*<< " buf = " << static_cast(b->buf) << " = [" + << (int)b->buf[0] << (int)b->buf[1] << "..." + << (int)b->buf[b->actual_size - 2] + << (int)b->buf[b->actual_size - 1] << "]" << std::endl;*/ + + // b->actual_size: how much we need to write + m_FileDataManager.WriteFiles(b->buf, b->actual_size); + + wrote += b->actual_size; + + a->UnlockConsumerBuffer(); + } + m_DataPos += TotalSize; +} + +} // end namespace engine +} // end namespace core +} // end namespace adios2 diff --git a/source/adios2/engine/daos/DaosWriter_TwoLevelShm_Async.cpp b/source/adios2/engine/daos/DaosWriter_TwoLevelShm_Async.cpp new file mode 100644 index 0000000000..4c632ff3ff --- /dev/null +++ b/source/adios2/engine/daos/DaosWriter_TwoLevelShm_Async.cpp @@ -0,0 +1,357 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + * + * DaosWriter.cpp + * + */ + +#include "DaosWriter.h" + +#include "adios2/common/ADIOSMacros.h" +#include "adios2/core/CoreTypes.h" +#include "adios2/core/IO.h" +#include "adios2/helper/adiosFunctions.h" //CheckIndexRange, PaddingToAlignOffset +#include "adios2/toolkit/format/buffer/chunk/ChunkV.h" +#include "adios2/toolkit/format/buffer/malloc/MallocV.h" +#include "adios2/toolkit/transport/file/FileFStream.h" +#include + +#include +#include +#include +#include + +namespace adios2 +{ +namespace core +{ +namespace engine +{ + +using namespace adios2::format; + +/* Aggregator part of the async two level aggregation Guided version + This process is the one writing to disk +*/ +void DaosWriter::AsyncWriteThread_TwoLevelShm_Aggregator(AsyncWriteInfo *info) +{ + aggregator::MPIShmChain *a = + dynamic_cast(info->aggregator); + uint64_t totalSize = info->totalSize; + + /* Write own data first */ + { + std::vector DataVec = info->Data->DataVec(); + const uint64_t mysize = info->Data->Size(); + info->tm->SeekTo(info->startPos); + AsyncWriteOwnData(info, DataVec, mysize, false); + totalSize -= mysize; + } + + /* Write from shm until every non-aggr sent all data */ + std::vector DataVec(1); + size_t wrote = 0; + while (wrote < totalSize) + { + /* Write the next shm block now */ + // potentially blocking call waiting on some non-aggr process + aggregator::MPIShmChain::ShmDataBuffer *b = a->LockConsumerBuffer(); + // b->actual_size: how much we need to write + DataVec[0].iov_base = b->buf; + DataVec[0].iov_len = b->actual_size; + AsyncWriteOwnData(info, DataVec, b->actual_size, false); + wrote += b->actual_size; + a->UnlockConsumerBuffer(); + } +} + +/* Non-aggregator part of the async two level aggregation. + This process passes data to Aggregator through SHM segment. + tokenChain in caller ensures only one process (per aggregator chain) + is running this function at a time +*/ +void DaosWriter::AsyncWriteThread_TwoLevelShm_SendDataToAggregator( + aggregator::MPIShmChain *a, format::BufferV *Data) +{ + /* In a loop, copy the local data into the shared memory, alternating + between the two segments. + */ + + std::vector DataVec = Data->DataVec(); + size_t nBlocks = DataVec.size(); + + // size_t sent = 0; + size_t block = 0; + size_t temp_offset = 0; + while (block < nBlocks) + { + // potentially blocking call waiting on Aggregator + aggregator::MPIShmChain::ShmDataBuffer *b = a->LockProducerBuffer(); + // b->max_size: how much we can copy + // b->actual_size: how much we actually copy + b->actual_size = 0; + while (true) + { + /* Copy n bytes from the current block, current offset to shm + making sure to use up to shm_size bytes + */ + size_t n = DataVec[block].iov_len - temp_offset; + if (n > (b->max_size - b->actual_size)) + { + n = b->max_size - b->actual_size; + } + std::memcpy(&b->buf[b->actual_size], + (const char *)DataVec[block].iov_base + temp_offset, n); + b->actual_size += n; + + /* Have we processed the entire block or staying with it? */ + if (n + temp_offset < DataVec[block].iov_len) + { + temp_offset += n; + } + else + { + temp_offset = 0; + ++block; + } + + /* Have we reached the max allowed shm size ?*/ + if (b->actual_size >= b->max_size) + { + break; + } + if (block >= nBlocks) + { + break; + } + } + // sent += b->actual_size; + a->UnlockProducerBuffer(); + } +} + +int DaosWriter::AsyncWriteThread_TwoLevelShm(AsyncWriteInfo *info) +{ + /* DO NOT use MPI in this separate thread, including destroying + shm segments explicitely (a->DestroyShm) or implicitely (tokenChain) */ + Seconds ts = Now() - info->tstart; + // std::cout << "ASYNC rank " << info->rank_global + // << " starts at: " << ts.count() << std::endl; + aggregator::MPIShmChain *a = + dynamic_cast(info->aggregator); + if (a->m_IsAggregator) + { + // Send token to first non-aggregator to start filling shm + // Also informs next process its starting offset (for correct + // metadata) + uint64_t nextWriterPos = info->startPos + info->Data->Size(); + info->tokenChain->SendToken(nextWriterPos); + AsyncWriteThread_TwoLevelShm_Aggregator(info); + info->tokenChain->RecvToken(); + } + else + { + // non-aggregators fill shared buffer in marching order + // they also receive their starting offset this way + uint64_t startPos = info->tokenChain->RecvToken(); + AsyncWriteThread_TwoLevelShm_SendDataToAggregator(a, info->Data); + uint64_t nextWriterPos = startPos + info->Data->Size(); + info->tokenChain->SendToken(nextWriterPos); + } + delete info->Data; + + ts = Now() - info->tstart; + /*std::cout << "ASYNC " << info->rank_global << " ended at: " << ts.count() + << std::endl;*/ + return 1; +}; + +void DaosWriter::WriteData_TwoLevelShm_Async(format::BufferV *Data) +{ + aggregator::MPIShmChain *a = + dynamic_cast(m_Aggregator); + + // new step writing starts at offset m_DataPos on master aggregator + // other aggregators to the same file will need to wait for the position + // to arrive from the rank below + + // align to PAGE_SIZE (only valid on master aggregator at this point) + m_DataPos += + helper::PaddingToAlignOffset(m_DataPos, m_Parameters.StripeSize); + + // Each aggregator needs to know the total size they write + // This calculation is valid on aggregators only + std::vector mySizes = a->m_Comm.GatherValues(Data->Size()); + uint64_t myTotalSize = 0; + uint64_t maxSize = 0; + for (auto s : mySizes) + { + myTotalSize += s; + if (s > maxSize) + { + maxSize = s; + } + } + + if (a->m_Comm.Size() > 1) + { + size_t alignment_size = sizeof(max_align_t); + if (m_Parameters.DirectIO) + { + alignment_size = m_Parameters.DirectIOAlignOffset; + } + a->CreateShm(static_cast(maxSize), m_Parameters.MaxShmSize, + alignment_size); + } + + if (a->m_IsAggregator) + { + // In each aggregator chain, send from master down the line + // these total sizes, so every aggregator knows where to start + if (a->m_AggregatorChainComm.Rank() > 0) + { + a->m_AggregatorChainComm.Recv( + &m_DataPos, 1, a->m_AggregatorChainComm.Rank() - 1, 0, + "AggregatorChain token in DaosWriter::WriteData_TwoLevelShm"); + // align to PAGE_SIZE + m_DataPos += helper::PaddingToAlignOffset(m_DataPos, + m_Parameters.StripeSize); + } + m_StartDataPos = m_DataPos; // metadata needs this info + if (a->m_AggregatorChainComm.Rank() < + a->m_AggregatorChainComm.Size() - 1) + { + uint64_t nextWriterPos = m_DataPos + myTotalSize; + a->m_AggregatorChainComm.Isend( + &nextWriterPos, 1, a->m_AggregatorChainComm.Rank() + 1, 0, + "Chain token in DaosWriter::WriteData"); + } + else if (a->m_AggregatorChainComm.Size() > 1) + { + // send back final position from last aggregator in file to master + // aggregator + uint64_t nextWriterPos = m_DataPos + myTotalSize; + a->m_AggregatorChainComm.Isend( + &nextWriterPos, 1, 0, 0, + "Chain token in DaosWriter::WriteData"); + } + + // Master aggregator needs to know where the last writing ended by the + // last aggregator in the chain, so that it can start from the correct + // position at the next output step + if (!a->m_AggregatorChainComm.Rank()) + { + if (a->m_AggregatorChainComm.Size() > 1) + { + a->m_AggregatorChainComm.Recv( + &m_DataPos, 1, a->m_AggregatorChainComm.Size() - 1, 0, + "Chain token in DaosWriter::WriteData"); + } + else + { + m_DataPos = m_StartDataPos + myTotalSize; + } + } + } + + /*std::cout << "Rank " << m_Comm.Rank() << " start data async " + << " to subfile " << a->m_SubStreamIndex << " at pos " + << m_StartDataPos << std::endl;*/ + + m_AsyncWriteInfo = new AsyncWriteInfo(); + m_AsyncWriteInfo->aggregator = m_Aggregator; + m_AsyncWriteInfo->rank_global = m_Comm.Rank(); + m_AsyncWriteInfo->rank_chain = a->m_Comm.Rank(); + m_AsyncWriteInfo->nproc_chain = a->m_Comm.Size(); + m_AsyncWriteInfo->comm_chain = helper::Comm(); // unused in this aggregation + m_AsyncWriteInfo->tstart = m_EngineStart; + m_AsyncWriteInfo->tokenChain = new shm::TokenChain(&a->m_Comm); + m_AsyncWriteInfo->tm = &m_FileDataManager; + m_AsyncWriteInfo->Data = Data; + m_AsyncWriteInfo->flagRush = &m_flagRush; + m_AsyncWriteInfo->lock = &m_AsyncWriteLock; + + // Metadata collection needs m_StartDataPos correctly set on + // every process before we call the async writing thread + if (a->m_IsAggregator) + { + // Informs next process its starting offset (for correct metadata) + uint64_t nextWriterPos = m_StartDataPos + Data->Size(); + m_AsyncWriteInfo->tokenChain->SendToken(nextWriterPos); + m_AsyncWriteInfo->tokenChain->RecvToken(); + } + else + { + // non-aggregators fill shared buffer in marching order + // they also receive their starting offset this way + m_StartDataPos = m_AsyncWriteInfo->tokenChain->RecvToken(); + uint64_t nextWriterPos = m_StartDataPos + Data->Size(); + m_AsyncWriteInfo->tokenChain->SendToken(nextWriterPos); + } + + // Launch data writing thread, m_StartDataPos is valid + // m_DataPos is already pointing to the end of the write, do not use here. + m_AsyncWriteInfo->startPos = m_StartDataPos; + m_AsyncWriteInfo->totalSize = myTotalSize; + m_AsyncWriteInfo->deadline = m_ExpectedTimeBetweenSteps.count(); + + if (m_ComputationBlocksLength > 0.0 && + m_Parameters.AsyncWrite == (int)AsyncWrite::Guided) + { + m_AsyncWriteInfo->inComputationBlock = &m_InComputationBlock; + m_AsyncWriteInfo->computationBlocksLength = m_ComputationBlocksLength; + if (m_AsyncWriteInfo->deadline < m_ComputationBlocksLength) + { + m_AsyncWriteInfo->deadline = m_ComputationBlocksLength; + } + m_AsyncWriteInfo->expectedComputationBlocks = + m_ComputationBlockTimes; // copy! + m_AsyncWriteInfo->currentComputationBlocks = + &m_ComputationBlockTimes; // ptr! + m_AsyncWriteInfo->currentComputationBlockID = &m_ComputationBlockID; + + /* Clear current block tracker now so that async thread does not get + confused with the past info */ + m_ComputationBlockTimes.clear(); + m_ComputationBlocksLength = 0.0; + m_ComputationBlockID = 0; + } + else + { + if (m_Parameters.AsyncWrite == (int)AsyncWrite::Naive) + { + m_AsyncWriteInfo->deadline = 0; + } + m_AsyncWriteInfo->inComputationBlock = nullptr; + m_AsyncWriteInfo->computationBlocksLength = 0.0; + m_AsyncWriteInfo->currentComputationBlocks = nullptr; + m_AsyncWriteInfo->currentComputationBlockID = nullptr; + } + + m_WriteFuture = std::async(std::launch::async, AsyncWriteThread_TwoLevelShm, + m_AsyncWriteInfo); + + /* At this point it is prohibited in the main thread + - to modify Data, which will be deleted in the async thread any tiume + - to use m_FileDataManager until next BeginStep, which is being used + in the async thread to write data + */ +} + +void DaosWriter::AsyncWriteDataCleanup_TwoLevelShm() +{ + aggregator::MPIShmChain *a = + dynamic_cast(m_AsyncWriteInfo->aggregator); + if (a->m_Comm.Size() > 1) + { + a->DestroyShm(); + } + delete m_AsyncWriteInfo->tokenChain; + delete m_AsyncWriteInfo; + m_AsyncWriteInfo = nullptr; +} + +} // end namespace engine +} // end namespace core +} // end namespace adios2 diff --git a/source/adios2/helper/adiosGPUFunctions.h b/source/adios2/helper/adiosGPUFunctions.h index 8069a41b1e..78a2f323a8 100644 --- a/source/adios2/helper/adiosGPUFunctions.h +++ b/source/adios2/helper/adiosGPUFunctions.h @@ -6,7 +6,7 @@ #endif #ifdef ADIOS2_HAVE_KOKKOS -#include "adios2/helper/adiosKokkos.h" +#include "adios2/helper/kokkos/adiosKokkos.h" #endif #endif /* ADIOS2_HELPER_ADIOSGPUFUNCTIONS_H_ */ diff --git a/source/adios2/helper/kokkos/CMakeLists.txt b/source/adios2/helper/kokkos/CMakeLists.txt new file mode 100644 index 0000000000..c37a3c9417 --- /dev/null +++ b/source/adios2/helper/kokkos/CMakeLists.txt @@ -0,0 +1,25 @@ +#------------------------------------------------------------------------------# +# Distributed under the OSI-approved Apache License, Version 2.0. See +# accompanying file Copyright.txt for details. +#------------------------------------------------------------------------------# + +if (NOT DEFINED Kokkos_CXX_COMPILER) + message(FATAL_ERROR "ADIOS: Kokkos module requires the Kokkos_CXX_COMPILER variable") +endif() + +# CXX Compiler settings only in for this subdir +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_COMPILER "${Kokkos_CXX_COMPILER}") + +add_library(adios2_core_kokkos adiosKokkos.h adiosKokkos.cpp) + +set_target_properties(adios2_core_kokkos PROPERTIES + VISIBILITY_INLINES_HIDDEN ON + INCLUDE_DIRECTORIES "$;$" + EXPORT_NAME core_kokkos + OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}_core_kokkos + ) + +kokkos_compilation(SOURCE adiosKokkos.cpp) +target_link_libraries(adios2_core_kokkos PRIVATE Kokkos::kokkos) diff --git a/source/adios2/helper/adiosKokkos.cpp b/source/adios2/helper/kokkos/adiosKokkos.cpp similarity index 83% rename from source/adios2/helper/adiosKokkos.cpp rename to source/adios2/helper/kokkos/adiosKokkos.cpp index 7a9f53c66b..dffc18e29e 100644 --- a/source/adios2/helper/adiosKokkos.cpp +++ b/source/adios2/helper/kokkos/adiosKokkos.cpp @@ -13,10 +13,10 @@ namespace { -template void KokkosDeepCopy(const char *src, char *dst, size_t byteCount) { - Kokkos::View> srcView(src, byteCount); Kokkos::View * /*values*/, const size_t /*size*/, std::complex & /*min*/, std::complex & /*max*/) @@ -62,22 +66,12 @@ namespace helper { void MemcpyGPUToBuffer(char *dst, const char *GPUbuffer, size_t byteCount) { -#ifdef ADIOS2_HAVE_KOKKOS_CUDA - KokkosDeepCopy(GPUbuffer, dst, byteCount); -#endif -#ifdef ADIOS2_HAVE_KOKKOS_HIP - KokkosDeepCopy(GPUbuffer, dst, byteCount); -#endif + KokkosDeepCopy(GPUbuffer, dst, byteCount); } void MemcpyBufferToGPU(char *GPUbuffer, const char *src, size_t byteCount) { -#ifdef ADIOS2_HAVE_KOKKOS_CUDA - KokkosDeepCopy(src, GPUbuffer, byteCount); -#endif -#ifdef ADIOS2_HAVE_KOKKOS_HIP - KokkosDeepCopy(src, GPUbuffer, byteCount); -#endif + KokkosDeepCopy(src, GPUbuffer, byteCount); } bool IsGPUbuffer(const void *ptr) @@ -98,6 +92,15 @@ bool IsGPUbuffer(const void *ptr) { return true; } +#endif +#ifdef ADIOS2_HAVE_KOKKOS_SYCL + auto ret = + sycl::address_space_cast(ptr); + if (ret != nullptr) + { + return true; + } #endif return false; } @@ -121,6 +124,7 @@ void KokkosInit() settings.set_device_id(device_id); } #endif + // GetDevice not supported for SYCL, use the default device Kokkos::initialize(settings); } diff --git a/source/adios2/helper/adiosKokkos.h b/source/adios2/helper/kokkos/adiosKokkos.h similarity index 100% rename from source/adios2/helper/adiosKokkos.h rename to source/adios2/helper/kokkos/adiosKokkos.h diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp index 06d2176856..af4c780cf3 100644 --- a/source/adios2/toolkit/format/bp5/BP5Serializer.cpp +++ b/source/adios2/toolkit/format/bp5/BP5Serializer.cpp @@ -1333,19 +1333,27 @@ BP5Serializer::TimestepInfo BP5Serializer::CloseTimestep(int timestep, MBase->BitField = tmp; NewAttribute = false; - struct TimestepInfo Ret - { - Formats, Metadata, AttrData, CurDataBuffer - }; + struct TimestepInfo Ret; + Ret.NewMetaMetaBlocks = Formats; + Ret.MetaEncodeBuffer.reset(Metadata); + Ret.AttributeEncodeBuffer.reset(AttrData); + Ret.DataBuffer = CurDataBuffer; CurDataBuffer = NULL; + if (Info.AttributeFields) + { free_FMfield_list(Info.AttributeFields); - Info.AttributeFields = NULL; + Info.AttributeFields = NULL; + } Info.AttributeFieldCount = 0; + if (Info.AttributeData) + { free(Info.AttributeData); - Info.AttributeData = NULL; + Info.AttributeData = NULL; + } Info.AttributeSize = 0; + return Ret; } diff --git a/source/adios2/toolkit/format/bp5/BP5Serializer.h b/source/adios2/toolkit/format/bp5/BP5Serializer.h index 5ceeb54dad..b0f225b0df 100644 --- a/source/adios2/toolkit/format/bp5/BP5Serializer.h +++ b/source/adios2/toolkit/format/bp5/BP5Serializer.h @@ -36,17 +36,9 @@ class BP5Serializer : virtual public BP5Base struct TimestepInfo { std::vector NewMetaMetaBlocks; - Buffer *MetaEncodeBuffer; - Buffer *AttributeEncodeBuffer; - BufferV *DataBuffer; - - ~TimestepInfo() - { - delete MetaEncodeBuffer; - if (AttributeEncodeBuffer) - delete AttributeEncodeBuffer; - delete DataBuffer; - } + std::shared_ptr MetaEncodeBuffer; + std::shared_ptr AttributeEncodeBuffer; + BufferV* DataBuffer; }; typedef struct _MetadataInfo diff --git a/source/adios2/toolkit/sst/CMakeLists.txt b/source/adios2/toolkit/sst/CMakeLists.txt index 5f6a1875e2..070c8f41a1 100644 --- a/source/adios2/toolkit/sst/CMakeLists.txt +++ b/source/adios2/toolkit/sst/CMakeLists.txt @@ -54,7 +54,7 @@ endif() set_target_properties(sst PROPERTIES OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}_sst VERSION ${ADIOS2_LIBRARY_VERSION} - SOVERSION ${ADIOS2_VERSION_MAJOR} + SOVERSION ${ADIOS2_LIBRARY_SOVERSION} ) #------------------------------------------------------------------------------# diff --git a/source/adios2/toolkit/transport/Transport.h b/source/adios2/toolkit/transport/Transport.h index 4511155355..524ebfe216 100644 --- a/source/adios2/toolkit/transport/Transport.h +++ b/source/adios2/toolkit/transport/Transport.h @@ -150,6 +150,8 @@ class Transport virtual void Seek(const size_t start = MaxSizeT) = 0; + virtual size_t CurrentPos() = 0; + virtual void Truncate(const size_t length) = 0; virtual void MkDir(const std::string &fileName) = 0; diff --git a/source/adios2/toolkit/transport/file/FileAWSSDK.h b/source/adios2/toolkit/transport/file/FileAWSSDK.h index 552251738d..96f38be8c3 100644 --- a/source/adios2/toolkit/transport/file/FileAWSSDK.h +++ b/source/adios2/toolkit/transport/file/FileAWSSDK.h @@ -76,6 +76,8 @@ class FileAWSSDK : public Transport void Seek(const size_t start = MaxSizeT) final; + size_t CurrentPos() final { return m_SeekPos; }; + void Truncate(const size_t length) final; void MkDir(const std::string &fileName) final; diff --git a/source/adios2/toolkit/transport/file/FileDaos.h b/source/adios2/toolkit/transport/file/FileDaos.h index 411fb55def..f78592a6c1 100644 --- a/source/adios2/toolkit/transport/file/FileDaos.h +++ b/source/adios2/toolkit/transport/file/FileDaos.h @@ -58,6 +58,8 @@ class FileDaos : public Transport void Seek(const size_t start = MaxSizeT) final; + size_t CurrentPos() final { return m_GlobalOffset; }; + void Truncate(const size_t length) final; void MkDir(const std::string &fileName) final; diff --git a/source/adios2/toolkit/transport/file/FileFStream.cpp b/source/adios2/toolkit/transport/file/FileFStream.cpp index aa64cf1115..ad342ec032 100644 --- a/source/adios2/toolkit/transport/file/FileFStream.cpp +++ b/source/adios2/toolkit/transport/file/FileFStream.cpp @@ -361,6 +361,18 @@ void FileFStream::Seek(const size_t start) } } +size_t FileFStream::CurrentPos() +{ + if (m_OpenMode == Mode::Write || m_OpenMode == Mode::Append) + { + return static_cast(m_FileStream.tellp()); + } + else + { + return static_cast(m_FileStream.tellg()); + } +} + void FileFStream::Truncate(const size_t length) { #if __cplusplus >= 201703L diff --git a/source/adios2/toolkit/transport/file/FileFStream.h b/source/adios2/toolkit/transport/file/FileFStream.h index 3f2e022474..22009c8136 100644 --- a/source/adios2/toolkit/transport/file/FileFStream.h +++ b/source/adios2/toolkit/transport/file/FileFStream.h @@ -60,6 +60,8 @@ class FileFStream : public Transport void Seek(const size_t start = MaxSizeT) final; + size_t CurrentPos() final; + void Truncate(const size_t length) final; void MkDir(const std::string &fileName) final; diff --git a/source/adios2/toolkit/transport/file/FileIME.cpp b/source/adios2/toolkit/transport/file/FileIME.cpp index 5d91b67eca..976bf2a681 100644 --- a/source/adios2/toolkit/transport/file/FileIME.cpp +++ b/source/adios2/toolkit/transport/file/FileIME.cpp @@ -354,6 +354,12 @@ void FileIME::Seek(const size_t start) } } +size_t FileIME::CurrentPos() +{ + return static_cast( + ime_client_native2_lseek(m_FileDescriptor, 0, SEEK_CUR)); +} + void FileIME::Truncate(const size_t length) { helper::Throw( diff --git a/source/adios2/toolkit/transport/file/FileIME.h b/source/adios2/toolkit/transport/file/FileIME.h index 92532714c7..0ebc98f13a 100644 --- a/source/adios2/toolkit/transport/file/FileIME.h +++ b/source/adios2/toolkit/transport/file/FileIME.h @@ -59,6 +59,8 @@ class FileIME : public Transport void Seek(const size_t start = MaxSizeT) final; + size_t CurrentPos() final; + void Truncate(const size_t length) final; void MkDir(const std::string &fileName) final; diff --git a/source/adios2/toolkit/transport/file/FilePOSIX.cpp b/source/adios2/toolkit/transport/file/FilePOSIX.cpp index bf41198508..a845e07df3 100644 --- a/source/adios2/toolkit/transport/file/FilePOSIX.cpp +++ b/source/adios2/toolkit/transport/file/FilePOSIX.cpp @@ -600,6 +600,11 @@ void FilePOSIX::Seek(const size_t start) } } +size_t FilePOSIX::CurrentPos() +{ + return static_cast(lseek(m_FileDescriptor, 0, SEEK_CUR)); +} + void FilePOSIX::Truncate(const size_t length) { WaitForOpen(); diff --git a/source/adios2/toolkit/transport/file/FilePOSIX.h b/source/adios2/toolkit/transport/file/FilePOSIX.h index 4f352b833c..774e699277 100644 --- a/source/adios2/toolkit/transport/file/FilePOSIX.h +++ b/source/adios2/toolkit/transport/file/FilePOSIX.h @@ -66,6 +66,8 @@ class FilePOSIX : public Transport void Seek(const size_t start = MaxSizeT) final; + size_t CurrentPos() final; + void Truncate(const size_t length) final; void MkDir(const std::string &fileName) final; diff --git a/source/adios2/toolkit/transport/file/FileStdio.cpp b/source/adios2/toolkit/transport/file/FileStdio.cpp index 24fab067fa..b7ee29593e 100644 --- a/source/adios2/toolkit/transport/file/FileStdio.cpp +++ b/source/adios2/toolkit/transport/file/FileStdio.cpp @@ -452,6 +452,11 @@ void FileStdio::Seek(const size_t start) } } +size_t FileStdio::CurrentPos() +{ + return static_cast(std::ftell(m_File)); +} + #ifdef _WIN32 void FileStdio::Truncate(const size_t length) { diff --git a/source/adios2/toolkit/transport/file/FileStdio.h b/source/adios2/toolkit/transport/file/FileStdio.h index 7882004626..13e71de161 100644 --- a/source/adios2/toolkit/transport/file/FileStdio.h +++ b/source/adios2/toolkit/transport/file/FileStdio.h @@ -62,6 +62,8 @@ class FileStdio : public Transport void Seek(const size_t start) final; + size_t CurrentPos() final; + void Truncate(const size_t length) final; void MkDir(const std::string &fileName) final; diff --git a/source/adios2/toolkit/transport/null/NullTransport.h b/source/adios2/toolkit/transport/null/NullTransport.h index 1595ce793a..fa85b2710e 100644 --- a/source/adios2/toolkit/transport/null/NullTransport.h +++ b/source/adios2/toolkit/transport/null/NullTransport.h @@ -58,6 +58,8 @@ class NullTransport : public Transport void Seek(const size_t start = MaxSizeT) override; + size_t CurrentPos() override { return 0; }; + void Truncate(const size_t length) override; protected: diff --git a/source/adios2/toolkit/transport/shm/ShmSystemV.h b/source/adios2/toolkit/transport/shm/ShmSystemV.h index fb8838448e..eb4c1d5020 100644 --- a/source/adios2/toolkit/transport/shm/ShmSystemV.h +++ b/source/adios2/toolkit/transport/shm/ShmSystemV.h @@ -53,6 +53,8 @@ class ShmSystemV : public Transport void Seek(const size_t start = MaxSizeT) final; + size_t CurrentPos() final { return 0; }; + void MkDir(const std::string &fileName) final; private: diff --git a/source/adios2/toolkit/transportman/TransportMan.cpp b/source/adios2/toolkit/transportman/TransportMan.cpp index c2e21b6aec..c2f41001ea 100644 --- a/source/adios2/toolkit/transportman/TransportMan.cpp +++ b/source/adios2/toolkit/transportman/TransportMan.cpp @@ -384,6 +384,14 @@ void TransportMan::SeekTo(const size_t start, const int transportIndex) } } +size_t TransportMan::CurrentPos(const int transportIndex) +{ + auto itTransport = m_Transports.find(transportIndex); + CheckFile(itTransport, ", in call to CurrentPos with index " + + std::to_string(transportIndex)); + return itTransport->second->CurrentPos(); +} + void TransportMan::Truncate(const size_t length, const int transportIndex) { if (transportIndex == -1) diff --git a/source/adios2/toolkit/transportman/TransportMan.h b/source/adios2/toolkit/transportman/TransportMan.h index 32ae4b1343..4b46b98381 100644 --- a/source/adios2/toolkit/transportman/TransportMan.h +++ b/source/adios2/toolkit/transportman/TransportMan.h @@ -207,6 +207,8 @@ class TransportMan void SeekTo(const size_t start, const int transportIndex = -1); + size_t CurrentPos(const int transportIndex); + void Truncate(const size_t length, const int transportIndex = -1); /** diff --git a/source/h5vol/H5VolUtil.c b/source/h5vol/H5VolUtil.c index 6f44a18c70..80f463eaaf 100644 --- a/source/h5vol/H5VolUtil.c +++ b/source/h5vol/H5VolUtil.c @@ -56,16 +56,16 @@ void *safe_ralloc(void *ptr, size_t newsize, unsigned long line) return p; } -void gUtilConvert(hsize_t *fromH5, size_t *to, uint ndims) +void gUtilConvert(hsize_t *fromH5, size_t *to, size_t ndims) { - uint i = 0; + size_t i = 0; for (i = 0; i < ndims; i++) { to[i] = fromH5[i]; } } -int gUtilADIOS2GetShape(hid_t space_id, size_t *shape, uint ndims) +int gUtilADIOS2GetShape(hid_t space_id, size_t *shape, size_t ndims) { if (gUtilADIOS2IsScalar(space_id)) { diff --git a/source/h5vol/H5VolUtil.h b/source/h5vol/H5VolUtil.h index 305e6ab695..4797bd36f9 100644 --- a/source/h5vol/H5VolUtil.h +++ b/source/h5vol/H5VolUtil.h @@ -37,9 +37,9 @@ int gUtilADIOS2GetDim(hid_t space_id); // h5 uses hsize_t for dimensions (unsigned long long) // adios uses size_t // -void gUtilConvert(hsize_t *fromH5, size_t *to, uint ndims); +void gUtilConvert(hsize_t *fromH5, size_t *to, size_t ndims); -int gUtilADIOS2GetShape(hid_t space_id, size_t *shape, uint ndims); +int gUtilADIOS2GetShape(hid_t space_id, size_t *shape, size_t ndims); int gUtilADIOS2GetBlockInfo(hid_t hyperSlab_id, size_t *start, size_t *count, hsize_t ndims); diff --git a/testing/adios2/engine/bp/CMakeLists.txt b/testing/adios2/engine/bp/CMakeLists.txt index a5d95afecb..18ac338fb1 100644 --- a/testing/adios2/engine/bp/CMakeLists.txt +++ b/testing/adios2/engine/bp/CMakeLists.txt @@ -130,6 +130,7 @@ bp_gtest_add_tests_helper(WriteReadLocalVariables MPI_ALLOW) bp_gtest_add_tests_helper(WriteReadLocalVariablesSel MPI_ALLOW) bp_gtest_add_tests_helper(WriteReadLocalVariablesSelHighLevel MPI_ALLOW) bp_gtest_add_tests_helper(ChangingShape MPI_ALLOW) +bp_gtest_add_tests_helper(ChangingShapeWithinStep MPI_ALLOW) bp_gtest_add_tests_helper(WriteReadBlockInfo MPI_ALLOW) bp_gtest_add_tests_helper(WriteReadVariableSpan MPI_ALLOW) bp3_bp4_gtest_add_tests_helper(TimeAggregation MPI_ALLOW) @@ -179,6 +180,7 @@ gtest_add_tests_helper(WriteNull MPI_ALLOW BP Engine.BP. .BP3 # BP4 and BP5 but NOT BP3 bp4_bp5_gtest_add_tests_helper(WriteAppendReadADIOS2 MPI_ALLOW) +bp4_bp5_gtest_add_tests_helper(OpenWithMetadata MPI_NONE) # BP4 only for now # gtest_add_tests_helper(WriteAppendReadADIOS2 MPI_ALLOW BP Engine.BP. .BP4 diff --git a/testing/adios2/engine/bp/TestBPChangingShape.cpp b/testing/adios2/engine/bp/TestBPChangingShape.cpp index 3942ecd528..9546f094b7 100644 --- a/testing/adios2/engine/bp/TestBPChangingShape.cpp +++ b/testing/adios2/engine/bp/TestBPChangingShape.cpp @@ -18,16 +18,12 @@ #include -#include "../SmallTestData.h" - std::string engineName; // comes from command line class BPChangingShape : public ::testing::Test { public: BPChangingShape() = default; - - SmallTestData m_TestData; }; TEST_F(BPChangingShape, BPWriteReadShape2D) @@ -180,187 +176,6 @@ TEST_F(BPChangingShape, BPWriteReadShape2D) } } -TEST_F(BPChangingShape, MultiBlock) -{ - // Write multiple blocks and change shape in between - // At read, the last shape should be used not the first one - // This test guarantees that one can change the variable shape - // until EndStep() - - const std::string fname("BPChangingShapeMultiblock.bp"); - const int nsteps = 2; - const std::vector nblocks = {2, 3}; - EXPECT_EQ(nsteps, nblocks.size()); - int rank = 0, nproc = 1; - -#if ADIOS2_USE_MPI - MPI_Comm_rank(MPI_COMM_WORLD, &rank); - MPI_Comm_size(MPI_COMM_WORLD, &nproc); - adios2::ADIOS adios(MPI_COMM_WORLD); -#else - adios2::ADIOS adios; -#endif - - // Writer - { - adios2::IO outIO = adios.DeclareIO("Output"); - - if (!engineName.empty()) - { - outIO.SetEngine(engineName); - } - - adios2::Engine writer = outIO.Open(fname, adios2::Mode::Write); - - const size_t dim0 = static_cast(nproc); - const size_t off0 = static_cast(rank); - auto var = - outIO.DefineVariable("v", {dim0, 1}, {off0, 0}, {1, 1}); - - if (!rank) - { - std::cout << "Writing:" << std::endl; - } - for (size_t i = 0; i < nsteps; i++) - { - writer.BeginStep(); - - double value = - static_cast(rank) + static_cast(i + 1) / 10.0; - - for (size_t j = 0; j < static_cast(nblocks[i]); j++) - { - var.SetShape({dim0, j + 1}); - var.SetSelection({{off0, j}, {1, 1}}); - - if (!rank) - { - std::cout << "Step " << i << " block " << j << " shape (" - << var.Shape()[0] << ", " << var.Shape()[1] << ")" - << " value = " << value << std::endl; - } - - writer.Put(var, &value, adios2::Mode::Sync); - value += 0.01; - } - writer.EndStep(); - } - writer.Close(); - } - - // Reader with streaming - { - adios2::IO inIO = adios.DeclareIO("Input"); - - if (!engineName.empty()) - { - inIO.SetEngine(engineName); - } - adios2::Engine reader = inIO.Open(fname, adios2::Mode::Read); - - if (!rank) - { - std::cout << "Reading as stream with BeginStep/EndStep:" - << std::endl; - } - - int step = 0; - while (true) - { - adios2::StepStatus status = - reader.BeginStep(adios2::StepMode::Read); - - if (status != adios2::StepStatus::OK) - { - break; - } - - size_t expected_shape = nblocks[step]; - - auto var = inIO.InquireVariable("v"); - EXPECT_TRUE(var); - - if (!rank) - { - - std::cout << "Step " << step << " shape (" << var.Shape()[0] - << ", " << var.Shape()[1] << ")" << std::endl; - } - - EXPECT_EQ(var.Shape()[0], nproc); - EXPECT_EQ(var.Shape()[1], expected_shape); - - var.SetSelection( - {{0, 0}, {static_cast(nproc), expected_shape}}); - - // Check data on rank 0 - if (!rank) - { - std::vector data(nproc * expected_shape); - reader.Get(var, data.data()); - - reader.PerformGets(); - - for (int i = 0; i < nproc; i++) - { - double value = static_cast(i) + - static_cast(step + 1) / 10.0; - - for (int j = 0; j < nblocks[step]; j++) - { - EXPECT_EQ(data[i * nblocks[step] + j], value); - value += 0.01; - } - } - } - - reader.EndStep(); - ++step; - } - reader.Close(); - } - - // Reader with file reading - { - adios2::IO inIO = adios.DeclareIO("InputFile"); - - if (!engineName.empty()) - { - inIO.SetEngine(engineName); - } - adios2::Engine reader = - inIO.Open(fname, adios2::Mode::ReadRandomAccess); - - if (!rank) - { - std::cout << "Reading as file with SetStepSelection:" << std::endl; - } - - auto var = inIO.InquireVariable("v"); - EXPECT_TRUE(var); - for (int step = 0; step < nsteps; step++) - { - var.SetStepSelection({step, 1}); - if (!rank) - { - std::cout << "Step " << step << " shape (" << var.Shape()[0] - << ", " << var.Shape()[1] << ")" << std::endl; - } - size_t expected_shape = nblocks[step]; - EXPECT_EQ(var.Shape()[0], nproc); - EXPECT_EQ(var.Shape()[1], expected_shape); - - var.SetSelection( - {{0, 0}, {static_cast(nproc), expected_shape}}); - - std::vector data(nproc * expected_shape); - reader.Get(var, data.data()); - - EXPECT_THROW(reader.EndStep(), std::logic_error); - } - } -} - int main(int argc, char **argv) { #if ADIOS2_USE_MPI diff --git a/testing/adios2/engine/bp/TestBPChangingShapeWithinStep.cpp b/testing/adios2/engine/bp/TestBPChangingShapeWithinStep.cpp new file mode 100644 index 0000000000..ba1388fe69 --- /dev/null +++ b/testing/adios2/engine/bp/TestBPChangingShapeWithinStep.cpp @@ -0,0 +1,293 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + */ +#include +#include + +#include +#include //std::iota +#include +#include + +#include +#include + +#include + +std::string engineName; // comes from command line + +using ParamType = std::tuple; + +class BPChangingShapeWithinStep : public ::testing::TestWithParam +{ +public: + BPChangingShapeWithinStep() = default; + virtual void SetUp(){}; + virtual void TearDown(){}; +}; + +TEST_P(BPChangingShapeWithinStep, MultiBlock) +{ + // Write multiple blocks and change shape in between + // At read, the last shape should be used not the first one + // This test guarantees that one can change the variable shape + // until EndStep() + + auto operatorName = std::get<0>(GetParam()); + auto params = std::get<1>(GetParam()); + double epsilon = std::get<2>(GetParam()); + + const std::string fname("BPChangingShapeMultiblock_" + operatorName + + ".bp"); + const int nsteps = 2; + const std::vector nblocks = {2, 3}; + const int N = 16384; // size of one block (should be big enough to compress) + EXPECT_EQ(nsteps, nblocks.size()); + int rank = 0, nproc = 1; + +#if ADIOS2_USE_MPI + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &nproc); + adios2::ADIOS adios(MPI_COMM_WORLD); +#else + adios2::ADIOS adios; +#endif + + // Writer + { + adios2::IO outIO = adios.DeclareIO("Output"); + + if (!engineName.empty()) + { + outIO.SetEngine(engineName); + } + + adios2::Engine writer = outIO.Open(fname, adios2::Mode::Write); + + const size_t dim0 = static_cast(nproc); + const size_t off0 = static_cast(rank); + auto var = + outIO.DefineVariable("v", {dim0, 1}, {off0, 0}, {1, 1}); + + if (operatorName != "none") + { + auto op = adios.DefineOperator("compressor", operatorName, params); + var.AddOperation(op); + } + + if (!rank) + { + std::cout << "Writing to :" << fname; + if (operatorName != "none") + { + std::cout << " with operator " << operatorName; + } + std::cout << std::endl; + } + for (size_t i = 0; i < nsteps; i++) + { + writer.BeginStep(); + + double value = + static_cast(rank) + static_cast(i + 1) / 10.0; + + for (size_t j = 0; j < static_cast(nblocks[i]); j++) + { + std::vector data(N, value); + var.SetShape({dim0, (j + 1) * N}); + var.SetSelection({{off0, j * N}, {1, N}}); + + if (!rank) + { + std::cout << "Step " << i << " block " << j << " shape (" + << var.Shape()[0] << ", " << var.Shape()[1] << ")" + << " value = " << value << " data[] = " << data[0] + << " .. " << data[N - 1] << std::endl; + } + + writer.Put(var, data.data(), adios2::Mode::Sync); + value += 0.01; + } + writer.EndStep(); + } + writer.Close(); + } + + // Reader with streaming + { + adios2::IO inIO = adios.DeclareIO("Input"); + + if (!engineName.empty()) + { + inIO.SetEngine(engineName); + } + adios2::Engine reader = inIO.Open(fname, adios2::Mode::Read); + + if (!rank) + { + std::cout << "Reading as stream with BeginStep/EndStep:" + << std::endl; + } + + int step = 0; + while (true) + { + adios2::StepStatus status = + reader.BeginStep(adios2::StepMode::Read); + + if (status != adios2::StepStatus::OK) + { + break; + } + + size_t expected_shape = N * nblocks[step]; + + auto var = inIO.InquireVariable("v"); + EXPECT_TRUE(var); + + if (!rank) + { + + std::cout << "Step " << step << " shape (" << var.Shape()[0] + << ", " << var.Shape()[1] << ")" << std::endl; + } + + EXPECT_EQ(var.Shape()[0], nproc); + EXPECT_EQ(var.Shape()[1], expected_shape); + + var.SetSelection( + {{0, 0}, {static_cast(nproc), expected_shape}}); + + // Check data on rank 0 + if (!rank) + { + std::vector data(nproc * expected_shape); + reader.Get(var, data.data()); + + reader.PerformGets(); + + for (int i = 0; i < nproc; i++) + { + double value = static_cast(i) + + static_cast(step + 1) / 10.0; + + for (int j = 0; j < nblocks[step]; j++) + { + EXPECT_LE( + fabs(data[(i * nblocks[step] + j) * N] - value), + epsilon); + value += 0.01; + } + } + } + + reader.EndStep(); + ++step; + } + reader.Close(); + } + + // Reader with file reading + { + adios2::IO inIO = adios.DeclareIO("InputFile"); + + if (!engineName.empty()) + { + inIO.SetEngine(engineName); + } + adios2::Engine reader = + inIO.Open(fname, adios2::Mode::ReadRandomAccess); + + if (!rank) + { + std::cout << "Reading as file with SetStepSelection:" << std::endl; + } + + auto var = inIO.InquireVariable("v"); + EXPECT_TRUE(var); + for (int step = 0; step < nsteps; step++) + { + var.SetStepSelection({step, 1}); + if (!rank) + { + std::cout << "Step " << step << " shape (" << var.Shape()[0] + << ", " << var.Shape()[1] << ")" << std::endl; + } + size_t expected_shape = N * nblocks[step]; + EXPECT_EQ(var.Shape()[0], nproc); + EXPECT_EQ(var.Shape()[1], expected_shape); + + var.SetSelection( + {{0, 0}, {static_cast(nproc), expected_shape}}); + + std::vector data(nproc * expected_shape); + reader.Get(var, data.data()); + reader.PerformGets(); + + for (int i = 0; i < nproc; i++) + { + double value = static_cast(i) + + static_cast(step + 1) / 10.0; + + for (int j = 0; j < nblocks[step]; j++) + { + EXPECT_LE(fabs(data[(i * nblocks[step] + j) * N] - value), + epsilon); + value += 0.01; + } + } + + EXPECT_THROW(reader.EndStep(), std::logic_error); + } + } +} + +adios2::Params paccuracy = {{"accuracy", "0.001"}}; +adios2::Params pblosc = {{"clevel", "9"}}; + +std::vector p = {{"none", paccuracy, 0.0} +#ifdef ADIOS2_HAVE_BLOSC2 + , + {"blosc", pblosc, 0.0} +#endif +#ifdef ADIOS2_HAVE_MGARD + , + {"mgard", paccuracy, 0.001} +#endif +#ifdef ADIOS2_HAVE_ZFP +#ifndef ADIOS2_HAVE_ZFP_CUDA // only test on CPU + , + {"zfp", paccuracy, 0.001} +#endif +#endif + +}; + +INSTANTIATE_TEST_SUITE_P(Multiblock, BPChangingShapeWithinStep, + ::testing::ValuesIn(p)); + +int main(int argc, char **argv) +{ +#if ADIOS2_USE_MPI + int provided; + + // MPI_THREAD_MULTIPLE is only required if you enable the SST MPI_DP + MPI_Init_thread(nullptr, nullptr, MPI_THREAD_MULTIPLE, &provided); +#endif + + int result; + ::testing::InitGoogleTest(&argc, argv); + + if (argc > 1) + { + engineName = std::string(argv[1]); + } + result = RUN_ALL_TESTS(); + +#if ADIOS2_USE_MPI + MPI_Finalize(); +#endif + + return result; +} diff --git a/testing/adios2/engine/bp/TestBPOpenWithMetadata.cpp b/testing/adios2/engine/bp/TestBPOpenWithMetadata.cpp new file mode 100644 index 0000000000..31f227b80b --- /dev/null +++ b/testing/adios2/engine/bp/TestBPOpenWithMetadata.cpp @@ -0,0 +1,203 @@ +/* + * Distributed under the OSI-approved Apache License, Version 2.0. See + * accompanying file Copyright.txt for details. + */ +#include +#include + +#include +#include //std::iota +#include + +#include + +#include + +#include "../SmallTestData.h" + +std::string engineName; // comes from command line +std::string engineParameters; // comes from command line + +class BPOpenWithMetadata : public ::testing::Test +{ +public: + BPOpenWithMetadata() = default; + + SmallTestData m_TestData; +}; + +//****************************************************************************** +// Create an output +// Open normally +// Get metadata +// Open again with metadata +//****************************************************************************** + +// ADIOS2 BP write and read 1D arrays +TEST_F(BPOpenWithMetadata, ADIOS2BPOpenWithMetadata) +{ + const std::string fname("ADIOS2BPOpenWithMetadata.bp"); + const size_t Nx = 6; + const size_t NSteps = 3; + + adios2::ADIOS adios; + { + adios2::IO io = adios.DeclareIO("TestIO"); + const adios2::Dims shape{Nx}; + const adios2::Dims start{0}; + const adios2::Dims count{Nx}; + auto v = io.DefineVariable("r64", shape, start, count); + + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + else + { + // Create the BP Engine + io.SetEngine("BPFile"); + } + if (!engineParameters.empty()) + { + io.SetParameters(engineParameters); + } + + adios2::Engine bpWriter = io.Open(fname, adios2::Mode::Write); + EXPECT_EQ(bpWriter.OpenMode(), adios2::Mode::Write); + for (size_t step = 0; step < NSteps; ++step) + { + // Generate test data for each process uniquely + SmallTestData currentTestData = generateNewSmallTestData( + m_TestData, static_cast(step), 0, 1); + + bpWriter.BeginStep(); + bpWriter.Put(v, currentTestData.R64.data()); + bpWriter.EndStep(); + } + bpWriter.Close(); + } + + char *md; + size_t mdsize; + + { + adios2::IO io = adios.DeclareIO("ReadIO"); + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + if (!engineParameters.empty()) + { + io.SetParameters(engineParameters); + } + + adios2::Engine bpReader = + io.Open(fname, adios2::Mode::ReadRandomAccess); + + bpReader.GetMetadata(&md, &mdsize); + + auto var_r64 = io.InquireVariable("r64"); + EXPECT_TRUE(var_r64); + ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64.Steps(), NSteps); + ASSERT_EQ(var_r64.Shape()[0], Nx); + + SmallTestData testData; + std::array R64; + + const adios2::Dims start{0}; + const adios2::Dims count{Nx}; + const adios2::Box sel(start, count); + + var_r64.SetSelection(sel); + + for (size_t t = 0; t < NSteps; ++t) + { + var_r64.SetStepSelection({t, 1}); + + // Generate test data for each rank uniquely + SmallTestData currentTestData = + generateNewSmallTestData(m_TestData, static_cast(t), 0, 1); + + bpReader.Get(var_r64, R64.data(), adios2::Mode::Sync); + for (size_t i = 0; i < Nx; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i; + std::string msg = ss.str(); + EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg; + } + } + bpReader.Close(); + } + + /* Open again with metadata */ + { + adios2::IO io = adios.DeclareIO("ReadIOMD"); + if (!engineName.empty()) + { + io.SetEngine(engineName); + } + if (!engineParameters.empty()) + { + io.SetParameters(engineParameters); + } + + adios2::Engine bpReader = io.Open(fname, md, mdsize); + + auto var_r64 = io.InquireVariable("r64"); + EXPECT_TRUE(var_r64); + ASSERT_EQ(var_r64.ShapeID(), adios2::ShapeID::GlobalArray); + ASSERT_EQ(var_r64.Steps(), NSteps); + ASSERT_EQ(var_r64.Shape()[0], Nx); + + SmallTestData testData; + std::array R64; + + const adios2::Dims start{0}; + const adios2::Dims count{Nx}; + const adios2::Box sel(start, count); + + var_r64.SetSelection(sel); + + for (size_t t = 0; t < NSteps; ++t) + { + var_r64.SetStepSelection({t, 1}); + + // Generate test data for each rank uniquely + SmallTestData currentTestData = + generateNewSmallTestData(m_TestData, static_cast(t), 0, 1); + + bpReader.Get(var_r64, R64.data(), adios2::Mode::Sync); + for (size_t i = 0; i < Nx; ++i) + { + std::stringstream ss; + ss << "t=" << t << " i=" << i; + std::string msg = ss.str(); + EXPECT_EQ(R64[i], currentTestData.R64[i]) << msg; + } + } + bpReader.Close(); + } +} + +//****************************************************************************** +// main +//****************************************************************************** + +int main(int argc, char **argv) +{ + int result; + ::testing::InitGoogleTest(&argc, argv); + + if (argc > 1) + { + engineName = std::string(argv[1]); + } + if (argc > 2) + { + engineParameters = std::string(argv[2]); + } + result = RUN_ALL_TESTS(); + return result; +} diff --git a/thirdparty/EVPath/CMakeLists.txt b/thirdparty/EVPath/CMakeLists.txt index fbf8369ced..27fb73c742 100644 --- a/thirdparty/EVPath/CMakeLists.txt +++ b/thirdparty/EVPath/CMakeLists.txt @@ -18,5 +18,6 @@ set(EVPATH_HEADER_COMPONENT adios2_evpath-development) add_subdirectory(EVPath) set(EVPath_DIR ${CMAKE_CURRENT_BINARY_DIR}/EVPath CACHE INTERNAL "") +setup_libversion_dir(EVPath) message_end_thirdparty() diff --git a/thirdparty/atl/CMakeLists.txt b/thirdparty/atl/CMakeLists.txt index b84d29550c..e15be092dd 100644 --- a/thirdparty/atl/CMakeLists.txt +++ b/thirdparty/atl/CMakeLists.txt @@ -15,5 +15,6 @@ set(ATL_HEADER_COMPONENT adios2_atl-development) add_subdirectory(atl) set(atl_DIR ${CMAKE_CURRENT_BINARY_DIR}/atl CACHE INTERNAL "") +setup_libversion_dir(atl) message_end_thirdparty() diff --git a/thirdparty/dill/CMakeLists.txt b/thirdparty/dill/CMakeLists.txt index 7f649a0af8..dd89f46bb5 100644 --- a/thirdparty/dill/CMakeLists.txt +++ b/thirdparty/dill/CMakeLists.txt @@ -14,5 +14,6 @@ set(DILL_HEADER_COMPONENT adios2_dill-development) add_subdirectory(dill) set(dill_DIR ${CMAKE_CURRENT_BINARY_DIR}/dill CACHE INTERNAL "") +setup_libversion_dir(dill) message_end_thirdparty() diff --git a/thirdparty/enet/CMakeLists.txt b/thirdparty/enet/CMakeLists.txt index 8b6a04cb18..5034e2e53a 100644 --- a/thirdparty/enet/CMakeLists.txt +++ b/thirdparty/enet/CMakeLists.txt @@ -13,5 +13,6 @@ set(BUILD_DOCS OFF) add_subdirectory(enet) set(enet_DIR ${CMAKE_CURRENT_BINARY_DIR}/enet CACHE INTERNAL "") +setup_libversion_dir(enet) message_end_thirdparty() diff --git a/thirdparty/ffs/CMakeLists.txt b/thirdparty/ffs/CMakeLists.txt index 0a36a8d8b8..e47512f581 100644 --- a/thirdparty/ffs/CMakeLists.txt +++ b/thirdparty/ffs/CMakeLists.txt @@ -18,5 +18,6 @@ set(FFS_HEADER_COMPONENT adios2_ffs-development) add_subdirectory(ffs) set(ffs_DIR ${CMAKE_CURRENT_BINARY_DIR}/ffs CACHE INTERNAL "") +setup_libversion_dir(ffs) message_end_thirdparty() diff --git a/thirdparty/perfstubs/perfstubs/CMakeLists.txt b/thirdparty/perfstubs/perfstubs/CMakeLists.txt index 734a8c89d8..3276a7c067 100644 --- a/thirdparty/perfstubs/perfstubs/CMakeLists.txt +++ b/thirdparty/perfstubs/perfstubs/CMakeLists.txt @@ -5,7 +5,11 @@ configure_file( add_library(perfstubs perfstubs_api/timer.c) -set_property(TARGET perfstubs PROPERTY OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}_perfstubs) +set_target_properties(perfstubs PROPERTIES + OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}_perfstubs + VERSION ${ADIOS2_LIBRARY_VERSION} + SOVERSION ${ADIOS2_LIBRARY_SOVERSION} +) target_link_libraries(perfstubs INTERFACE dl m)