From 1fd7a1cb2995573610974554c92e066866ccff4b Mon Sep 17 00:00:00 2001 From: David Galiffi Date: Tue, 5 Nov 2024 10:14:25 -0500 Subject: [PATCH] Fix for proto files not being viewable in Perfetto UI (#16) - Fix for proto files not being viewable in Perfetto UI - Ported from https://github.com/ROCm/omnitrace/pull/411 - Update Workflows - Use V47 trace_processor_shell for certain OS releases. - RedHat 8, SUSE 15.5, and Ubuntu 20.04 are no longer compatible with the latest trace_processor_shell. - Incompatible version of GLIBC. - Remove notes about Perfetto workaround in documentation. --------- Signed-off-by: David Galiffi --- .github/workflows/opensuse.yml | 7 +- .github/workflows/redhat.yml | 8 ++- .github/workflows/ubuntu-focal.yml | 16 ++--- .github/workflows/ubuntu-jammy.yml | 4 +- README.md | 4 -- .../understanding-rocprof-sys-output.rst | 5 -- docs/install/install.rst | 5 -- docs/what-is-rocprof-sys.rst | 5 -- source/lib/core/config.cpp | 70 +++++++++++++++---- source/lib/core/config.hpp | 7 ++ source/lib/core/perfetto.cpp | 38 +++++----- 11 files changed, 105 insertions(+), 64 deletions(-) diff --git a/.github/workflows/opensuse.yml b/.github/workflows/opensuse.yml index b2c58ebb..f6de06e6 100644 --- a/.github/workflows/opensuse.yml +++ b/.github/workflows/opensuse.yml @@ -58,8 +58,11 @@ jobs: timeout_minutes: 25 max_attempts: 5 command: | - wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin && - chmod +x /opt/trace_processor/bin/trace_processor_shell + if [ "${{ matrix.os-release }}" == "15.5" ]; then + wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v47.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin && + chmod +x /opt/trace_processor/bin/trace_processor_shell + fi + python3 -m pip install --upgrade pip && python3 -m pip install --upgrade numpy perfetto dataclasses && python3 -m pip install 'cmake==3.18.4' && for i in 6 7 8 9 10 11; do /opt/conda/envs/py3.${i}/bin/python -m pip install --upgrade numpy perfetto dataclasses; done diff --git a/.github/workflows/redhat.yml b/.github/workflows/redhat.yml index ebdfbe03..d48ab579 100644 --- a/.github/workflows/redhat.yml +++ b/.github/workflows/redhat.yml @@ -56,13 +56,16 @@ jobs: run: echo "CC=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')" >> $GITHUB_ENV && echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV && + echo "OS_VERSION_MAJOR=$(cat /etc/os-release | grep 'VERSION_ID' | sed 's/=/ /1' | awk '{print $NF}' | sed 's/"//g' | sed 's/\./ /g' | awk '{print $1}')" >> $GITHUB_ENV && env - name: Install Packages shell: bash run: | - wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin && - chmod +x /opt/trace_processor/bin/trace_processor_shell + if [ $OS_VERSION_MAJOR -eq 8 ]; then + wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v47.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin && + chmod +x /opt/trace_processor/bin/trace_processor_shell + fi python3 -m pip install --upgrade pip && python3 -m pip install --upgrade numpy perfetto dataclasses && python3 -m pip install 'cmake==3.18.4' && @@ -73,7 +76,6 @@ jobs: timeout-minutes: 30 shell: bash run: | - OS_VERSION_MAJOR=$(cat /etc/os-release | grep 'VERSION_ID' | sed 's/=/ /1' | awk '{print $NF}' | sed 's/"//g' | sed 's/\./ /g' | awk '{print $1}') RPM_TAG=".el${OS_VERSION_MAJOR}" ROCM_VERSION=${{ matrix.rocm-version }} ROCM_MAJOR=$(echo ${ROCM_VERSION} | sed 's/\./ /g' | awk '{print $1}') diff --git a/.github/workflows/ubuntu-focal.yml b/.github/workflows/ubuntu-focal.yml index 1371fc52..2b226758 100644 --- a/.github/workflows/ubuntu-focal.yml +++ b/.github/workflows/ubuntu-focal.yml @@ -94,12 +94,12 @@ jobs: apt-get update && apt-get upgrade -y && apt-get install -y autoconf bison build-essential clang environment-modules gettext libiberty-dev libmpich-dev libtool m4 mpich python3-pip texinfo ${{ matrix.compiler }} && - wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin && + wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v47.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin && chmod +x /opt/trace_processor/bin/trace_processor_shell && python3 -m pip install --upgrade pip && python3 -m pip install --upgrade numpy perfetto dataclasses && python3 -m pip install 'cmake==3.18.4' && - for i in 6 7 8 9 10 11; do /opt/conda/envs/py3.${i}/bin/python -m pip install --upgrade numpy perfetto dataclasses; done + for i in 6 7 8 9 10 11; do /opt/conda/envs/py3.${i}/bin/python -m pip install --upgrade numpy perfetto dataclasses; done && apt-get -y --purge autoremove && apt-get -y clean && /opt/conda/bin/conda clean -y -a @@ -271,12 +271,12 @@ jobs: apt-get update && apt-get install -y autoconf bison build-essential clang curl gettext libfabric-dev libnuma1 libomp-dev libopenmpi-dev libpapi-dev libtool libudev1 m4 openmpi-bin python3-pip texinfo && apt-get install -y rocm-dev && - wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin && + wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v47.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin && chmod +x /opt/trace_processor/bin/trace_processor_shell && python3 -m pip install --upgrade pip && python3 -m pip install --upgrade numpy perfetto dataclasses && python3 -m pip install 'cmake==3.18.4' && - for i in 6 7 8 9 10 11; do /opt/conda/envs/py3.${i}/bin/python -m pip install --upgrade numpy perfetto dataclasses; done + for i in 6 7 8 9 10 11; do /opt/conda/envs/py3.${i}/bin/python -m pip install --upgrade numpy perfetto dataclasses; done && apt-get -y --purge autoremove && apt-get -y clean && /opt/conda/bin/conda clean -y -a @@ -421,10 +421,10 @@ jobs: max_attempts: 5 command: | sudo apt-get update && - sudo apt-get install -y autoconf bison build-essential clang environment-modules gettext libomp-dev libtool m4 python3-pip texinfo ${{ matrix.compiler }} ${{ matrix.deps }} && + sudo apt-get install -y autoconf bison build-essential clang environment-modules gettext libomp-dev libtool m4 python3-pip texinfo ${{ matrix.compiler }} ${{ matrix.deps }} && if [ "${{ matrix.mpi }}" = "mpich" ]; then sudo apt-get install -y libmpich-dev mpich; fi && if [ "${{ matrix.mpi }}" = "openmpi" ]; then sudo apt-get install -y libopenmpi-dev openmpi-bin libfabric-dev; fi && - wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin && + wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v47.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin && chmod +x /opt/trace_processor/bin/trace_processor_shell && python3 -m pip install --upgrade pip && python3 -m pip install --upgrade numpy perfetto dataclasses && @@ -577,12 +577,12 @@ jobs: command: | apt-get update && apt-get install -y autoconf bison build-essential clang environment-modules gcc g++ libmpich-dev libomp-dev libtool m4 mpich python3-pip texinfo && - wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin && + wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v47.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin && chmod +x /opt/trace_processor/bin/trace_processor_shell && python3 -m pip install --upgrade pip && python3 -m pip install --upgrade numpy perfetto dataclasses && python3 -m pip install 'cmake==3.18.4' && - for i in 6 7 8 9 10 11; do /opt/conda/envs/py3.${i}/bin/python -m pip install --upgrade numpy perfetto dataclasses; done + for i in 6 7 8 9 10 11; do /opt/conda/envs/py3.${i}/bin/python -m pip install --upgrade numpy perfetto dataclasses; done && apt-get -y --purge autoremove && apt-get -y clean && /opt/conda/bin/conda clean -y -a diff --git a/.github/workflows/ubuntu-jammy.yml b/.github/workflows/ubuntu-jammy.yml index a23cf360..b69b0634 100644 --- a/.github/workflows/ubuntu-jammy.yml +++ b/.github/workflows/ubuntu-jammy.yml @@ -109,9 +109,7 @@ jobs: apt-get update && apt-get install -y software-properties-common && apt-get upgrade -y && - apt-get install -y build-essential m4 autoconf libtool python3-pip libiberty-dev clang libomp-dev libopenmpi-dev libfabric-dev openmpi-bin environment-modules ${{ matrix.compiler }} && - wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin && - chmod +x /opt/trace_processor/bin/trace_processor_shell && + apt-get install -y autoconf bison build-essential clang environment-modules gettext libfabric-dev libiberty-dev libomp-dev libopenmpi-dev libtool m4 openmpi-bin python3-pip texinfo ${{ matrix.compiler }} && python3 -m pip install --upgrade pip && python3 -m pip install --upgrade numpy perfetto dataclasses && python3 -m pip install 'cmake==3.18.4' && diff --git a/README.md b/README.md index e9c2ec47..c4853014 100755 --- a/README.md +++ b/README.md @@ -7,10 +7,6 @@ [![Installer Packaging (CPack)](https://github.com/ROCm/rocprofiler-systems/actions/workflows/cpack.yml/badge.svg)](https://github.com/ROCm/rocprofiler-systems/actions/workflows/cpack.yml) [![Documentation](https://github.com/ROCm/rocprofiler-systems/actions/workflows/docs.yml/badge.svg)](https://github.com/ROCm/rocprofiler-systems/actions/workflows/docs.yml) -> [!NOTE] -> Perfetto validation is done with trace_processor v46.0, as there is a known issue with v47.0. -If you are experiencing problems viewing your trace in the latest version of [Perfetto](http://ui.perfetto.dev), then try using [Perfetto UI v46.0](https://ui.perfetto.dev/v46.0-35b3d9845/#!/). - ## Overview ROCm Systems Profiler (rocprofiler-systems), formerly Omnitrace, is a comprehensive profiling and tracing tool for parallel applications written in C, C++, Fortran, HIP, OpenCL, and Python which execute on the CPU or CPU+GPU. diff --git a/docs/how-to/understanding-rocprof-sys-output.rst b/docs/how-to/understanding-rocprof-sys-output.rst index cdd75cd9..22549e24 100644 --- a/docs/how-to/understanding-rocprof-sys-output.rst +++ b/docs/how-to/understanding-rocprof-sys-output.rst @@ -323,11 +323,6 @@ absolute path, then all ``ROCPROFSYS_OUTPUT_PATH`` and similar settings are ignored. Visit `ui.perfetto.dev `_ and open this file. -.. important:: - Perfetto validation is done with trace_processor v46.0 as there is a known issue with v47.0. - If you are experiencing problems viewing your trace in the latest version of `Perfetto `_, - then try using `Perfetto UI v46.0 `_. - .. image:: ../data/rocprof-sys-perfetto.png :alt: Visualization of a performance graph in Perfetto diff --git a/docs/install/install.rst b/docs/install/install.rst index 5467a9d0..285f635b 100644 --- a/docs/install/install.rst +++ b/docs/install/install.rst @@ -253,11 +253,6 @@ into Perfetto support for ROCm Systems Profiler, for example, ``ROCPROFSYS_USE_P is passed along to Perfetto and is displayed when the ``.proto`` file is visualized in `the Perfetto UI `_. -.. important:: - Perfetto validation is done with trace_processor v46.0 as there is a known issue with v47.0. - If you are experiencing problems viewing your trace in the latest version of `Perfetto `_, - then try using `Perfetto UI v46.0 `_. - .. code-block:: shell git clone https://github.com/ROCm/rocprofiler-systems.git rocprof-sys-source diff --git a/docs/what-is-rocprof-sys.rst b/docs/what-is-rocprof-sys.rst index 952a0197..09ec88a6 100644 --- a/docs/what-is-rocprof-sys.rst +++ b/docs/what-is-rocprof-sys.rst @@ -15,11 +15,6 @@ A visualization of the comprehensive ROCm Systems Profiler results can be observ web browser. Upload the Perfetto (``.proto``) output files produced by ROCm Systems Profiler at `ui.perfetto.dev `_ to see the details. -.. important:: - Perfetto validation is done with trace_processor v46.0 as there is a known issue with v47.0. - If you are experiencing problems viewing your trace in the latest version of `Perfetto `_, - then try using `Perfetto UI v46.0 `_. - Aggregated high-level results are available as human-readable text files and JSON files for programmatic analysis. The JSON output files are compatible with the `hatchet `_ Python package. Hatchet converts diff --git a/source/lib/core/config.cpp b/source/lib/core/config.cpp index c519dfd7..38fbc4e5 100644 --- a/source/lib/core/config.cpp +++ b/source/lib/core/config.cpp @@ -2454,18 +2454,37 @@ tmp_file::~tmp_file() remove(); } -bool -tmp_file::open(std::ios::openmode _mode) +void +tmp_file::touch() const { - ROCPROFSYS_BASIC_VERBOSE(2, "Opening temporary file '%s'...\n", filename.c_str()); - if(!filepath::exists(filename)) { // if the filepath does not exist, open in out mode to create it - std::ofstream _ofs{}; + auto _ofs = std::ofstream{}; filepath::open(_ofs, filename); } +} + +bool +tmp_file::open(int _mode, int _perms) +{ + ROCPROFSYS_BASIC_VERBOSE(2, "Opening temporary file '%s'...\n", filename.c_str()); + + touch(); + m_pid = getpid(); + fd = ::open(filename.c_str(), _mode, _perms); + return (fd > 0); +} + +bool +tmp_file::open(std::ios::openmode _mode) +{ + ROCPROFSYS_BASIC_VERBOSE(2, "Opening temporary file '%s'...\n", filename.c_str()); + + touch(); + + m_pid = getpid(); stream.open(filename, _mode); return (stream.is_open() && stream.good()); @@ -2476,14 +2495,10 @@ tmp_file::fopen(const char* _mode) { ROCPROFSYS_BASIC_VERBOSE(2, "Opening temporary file '%s'...\n", filename.c_str()); - if(!filepath::exists(filename)) - { - // if the filepath does not exist, open in out mode to create it - std::ofstream _ofs{}; - filepath::open(_ofs, filename); - } + touch(); - file = filepath::fopen(filename, _mode); + m_pid = getpid(); + file = filepath::fopen(filename, _mode); if(file) fd = ::fileno(file); return (file != nullptr && fd > 0); @@ -2492,6 +2507,8 @@ tmp_file::fopen(const char* _mode) bool tmp_file::flush() { + if(m_pid != getpid()) return false; + if(stream.is_open()) { stream.flush(); @@ -2508,6 +2525,18 @@ tmp_file::flush() } return (_ret == 0); } + else if(fd > 0) + { + int _ret = ::fsync(fd); + int _cnt = 0; + while(_ret == EAGAIN || _ret == EINTR) + { + std::this_thread::sleep_for(std::chrono::milliseconds{ 100 }); + _ret = ::fsync(fd); + if(++_cnt > 10) break; + } + return (_ret == 0); + } return true; } @@ -2517,6 +2546,8 @@ tmp_file::close() { flush(); + if(m_pid != getpid()) return false; + if(stream.is_open()) { stream.close(); @@ -2532,6 +2563,15 @@ tmp_file::close() } return (_ret == 0); } + else if(fd > 0) + { + auto _ret = ::close(fd); + if(_ret == 0) + { + fd = -1; + } + return (_ret == 0); + } return true; } @@ -2539,6 +2579,8 @@ tmp_file::close() bool tmp_file::remove() { + if(m_pid != getpid()) return false; + close(); if(filepath::exists(filename)) { @@ -2553,7 +2595,9 @@ tmp_file::remove() tmp_file::operator bool() const { - return (stream.is_open() && stream.good()) || (file != nullptr && fd > 0); + return (m_pid == getpid()) && + ((stream.is_open() && stream.good()) || (file != nullptr && fd > 0) || + (file == nullptr && fd > 0)); } std::shared_ptr diff --git a/source/lib/core/config.hpp b/source/lib/core/config.hpp index 2b9847bc..2ccbce23 100644 --- a/source/lib/core/config.hpp +++ b/source/lib/core/config.hpp @@ -374,6 +374,7 @@ struct tmp_file tmp_file(std::string); ~tmp_file(); + bool open(int, int); bool open(std::ios::openmode = std::ios::binary | std::ios::in | std::ios::out); bool fopen(const char* = "r+"); bool flush(); @@ -386,6 +387,12 @@ struct tmp_file std::fstream stream = {}; FILE* file = nullptr; int fd = -1; + +private: + void touch() const; + +private: + pid_t m_pid = getpid(); }; std::shared_ptr diff --git a/source/lib/core/perfetto.cpp b/source/lib/core/perfetto.cpp index ac2c2e50..427ca6e6 100644 --- a/source/lib/core/perfetto.cpp +++ b/source/lib/core/perfetto.cpp @@ -22,10 +22,13 @@ #include "perfetto.hpp" #include "config.hpp" +#include "debug.hpp" #include "library/runtime.hpp" #include "perfetto_fwd.hpp" #include "utility.hpp" +#include + namespace rocprofsys { namespace perfetto @@ -120,18 +123,18 @@ start() if(!_tmp_file) { _tmp_file = config::get_tmp_file("perfetto-trace", "proto"); - _tmp_file->fopen("w+"); - } - else - { - ROCPROFSYS_VERBOSE(2, "Resuming perfetto...\n"); - _tmp_file->fopen("a+"); + _tmp_file->open(O_RDWR | O_CREAT | O_TRUNC, 0600); } } ROCPROFSYS_VERBOSE(2, "Setup perfetto...\n"); int _fd = (_tmp_file) ? _tmp_file->fd : -1; auto& cfg = get_config(); + tracing_session->SetOnErrorCallback([](::perfetto::TracingError _err) { + if(_err.code == ::perfetto::TracingError::kTracingFailed) + ROCPROFSYS_WARNING(0, "perfetto encountered a tracing error: %s\n", + _err.message.c_str()); + }); tracing_session->Setup(cfg, _fd); tracing_session->StartBlocking(); } @@ -174,7 +177,7 @@ post_process(tim::manager* _timemory_manager, bool& _perfetto_output_error) if(_tmp_file && *_tmp_file) { _tmp_file->close(); - FILE* _fdata = fopen(_tmp_file->filename.c_str(), "rb"); + FILE* _fdata = ::fopen(_tmp_file->filename.c_str(), "rb"); if(!_fdata) { @@ -184,22 +187,25 @@ post_process(tim::manager* _timemory_manager, bool& _perfetto_output_error) return char_vec_t{ tracing_session->ReadTraceBlocking() }; } - fseek(_fdata, 0, SEEK_END); - size_t _fnum_elem = ftell(_fdata); - fseek(_fdata, 0, SEEK_SET); // same as rewind(f); + ::fseek(_fdata, 0, SEEK_END); + size_t _fnum_elem = ::ftell(_fdata); + ::fseek(_fdata, 0, SEEK_SET); // same as rewind(f); - _data.resize(_fnum_elem + 1); - auto _fnum_read = fread(_data.data(), sizeof(char), _fnum_elem, _fdata); - fclose(_fdata); + _data.resize(_fnum_elem, '\0'); + auto _fnum_read = ::fread(_data.data(), sizeof(char), _fnum_elem, _fdata); + ::fclose(_fdata); ROCPROFSYS_CI_THROW( _fnum_read != _fnum_elem, "Error! read %zu elements from perfetto trace file '%s'. Expected %zu\n", _fnum_read, _tmp_file->filename.c_str(), _fnum_elem); } + else + { + _data = char_vec_t{ tracing_session->ReadTraceBlocking() }; + } - return utility::combine(_data, - char_vec_t{ tracing_session->ReadTraceBlocking() }); + return _data; }; auto trace_data = char_vec_t{}; @@ -251,7 +257,7 @@ post_process(tim::manager* _timemory_manager, bool& _perfetto_output_error) else { // Write the trace into a file. - ofs.write(&trace_data[0], trace_data.size()); + ofs.write(trace_data.data(), trace_data.size()); if(config::get_verbose() >= 0) _fom.append("%s", "Done"); // NOLINT if(_timemory_manager) _timemory_manager->add_file_output("protobuf", "perfetto", _filename);