Fix for proto files not being viewable in Perfetto UI (#16)

- Fix for proto files not being viewable in Perfetto UI
  - Ported from ROCm/omnitrace#411

- Update Workflows

- Use V47 trace_processor_shell for certain OS releases.
  - RedHat 8, SUSE 15.5, and Ubuntu 20.04 are no longer compatible with the latest trace_processor_shell.
  - Incompatible version of GLIBC.

- Remove notes about Perfetto workaround in documentation.

---------

Signed-off-by: David Galiffi <David.Galiffi@amd.com>

.github/workflows/opensuse.yml

@@ -58,8 +58,11 @@ jobs:
         timeout_minutes: 25
         max_attempts: 5
         command: |
-          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
-          chmod +x /opt/trace_processor/bin/trace_processor_shell
+          if [ "${{ matrix.os-release }}" == "15.5" ]; then
+            wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v47.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+            chmod +x /opt/trace_processor/bin/trace_processor_shell
+          fi
+          python3 -m pip install --upgrade pip &&
           python3 -m pip install --upgrade numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.18.4' &&
           for i in 6 7 8 9 10 11; do /opt/conda/envs/py3.${i}/bin/python -m pip install --upgrade numpy perfetto dataclasses; done

.github/workflows/redhat.yml

@@ -56,13 +56,16 @@ jobs:
       run:
         echo "CC=$(echo '${{ matrix.compiler }}' | sed 's/+/c/g')" >> $GITHUB_ENV &&
         echo "CXX=${{ matrix.compiler }}" >> $GITHUB_ENV &&
+        echo "OS_VERSION_MAJOR=$(cat /etc/os-release | grep 'VERSION_ID' | sed 's/=/ /1' | awk '{print $NF}' | sed 's/"//g' | sed 's/\./ /g' | awk '{print $1}')" >> $GITHUB_ENV &&
         env
 
     - name: Install Packages
       shell: bash
       run: |
-          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
-          chmod +x /opt/trace_processor/bin/trace_processor_shell
+          if [ $OS_VERSION_MAJOR -eq 8 ]; then
+            wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v47.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+            chmod +x /opt/trace_processor/bin/trace_processor_shell
+          fi
           python3 -m pip install --upgrade pip &&
           python3 -m pip install --upgrade numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.18.4' &&
@@ -73,7 +76,6 @@ jobs:
       timeout-minutes: 30
       shell: bash
       run: |
-        OS_VERSION_MAJOR=$(cat /etc/os-release | grep 'VERSION_ID' | sed 's/=/ /1' | awk '{print $NF}' | sed 's/"//g' | sed 's/\./ /g' | awk '{print $1}')
         RPM_TAG=".el${OS_VERSION_MAJOR}"
         ROCM_VERSION=${{ matrix.rocm-version }}
         ROCM_MAJOR=$(echo ${ROCM_VERSION} | sed 's/\./ /g' | awk '{print $1}')

.github/workflows/ubuntu-focal.yml

@@ -94,12 +94,12 @@ jobs:
           apt-get update &&
           apt-get upgrade -y &&
           apt-get install -y autoconf bison build-essential clang environment-modules gettext libiberty-dev libmpich-dev libtool m4 mpich python3-pip texinfo ${{ matrix.compiler }} &&
-          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v47.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
           chmod +x /opt/trace_processor/bin/trace_processor_shell &&
           python3 -m pip install --upgrade pip &&
           python3 -m pip install --upgrade numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.18.4' &&
-          for i in 6 7 8 9 10 11; do /opt/conda/envs/py3.${i}/bin/python -m pip install --upgrade numpy perfetto dataclasses; done
+          for i in 6 7 8 9 10 11; do /opt/conda/envs/py3.${i}/bin/python -m pip install --upgrade numpy perfetto dataclasses; done &&
           apt-get -y --purge autoremove &&
           apt-get -y clean &&
           /opt/conda/bin/conda clean -y -a
@@ -271,12 +271,12 @@ jobs:
           apt-get update &&
           apt-get install -y autoconf bison build-essential clang curl gettext libfabric-dev libnuma1 libomp-dev libopenmpi-dev libpapi-dev libtool libudev1 m4 openmpi-bin python3-pip texinfo &&
           apt-get install -y rocm-dev &&
-          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v47.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
           chmod +x /opt/trace_processor/bin/trace_processor_shell &&
           python3 -m pip install --upgrade pip &&
           python3 -m pip install --upgrade numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.18.4' &&
-          for i in 6 7 8 9 10 11; do /opt/conda/envs/py3.${i}/bin/python -m pip install --upgrade numpy perfetto dataclasses; done
+          for i in 6 7 8 9 10 11; do /opt/conda/envs/py3.${i}/bin/python -m pip install --upgrade numpy perfetto dataclasses; done &&
           apt-get -y --purge autoremove &&
           apt-get -y clean &&
           /opt/conda/bin/conda clean -y -a
@@ -421,10 +421,10 @@ jobs:
         max_attempts: 5
         command: |
           sudo apt-get update &&
-          sudo apt-get install -y autoconf bison build-essential clang environment-modules gettext libomp-dev libtool m4 python3-pip texinfo ${{ matrix.compiler }} ${{ matrix.deps }}  &&
+          sudo apt-get install -y autoconf bison build-essential clang environment-modules gettext libomp-dev libtool m4 python3-pip texinfo ${{ matrix.compiler }} ${{ matrix.deps }} &&
           if [ "${{ matrix.mpi }}" = "mpich" ]; then sudo apt-get install -y libmpich-dev mpich; fi &&
           if [ "${{ matrix.mpi }}" = "openmpi" ]; then sudo apt-get install -y libopenmpi-dev openmpi-bin libfabric-dev; fi &&
-          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v47.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
           chmod +x /opt/trace_processor/bin/trace_processor_shell &&
           python3 -m pip install --upgrade pip &&
           python3 -m pip install --upgrade numpy perfetto dataclasses &&
@@ -577,12 +577,12 @@ jobs:
         command: |
           apt-get update &&
           apt-get install -y autoconf bison build-essential clang environment-modules gcc g++ libmpich-dev libomp-dev libtool m4 mpich python3-pip texinfo &&
-          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v47.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
           chmod +x /opt/trace_processor/bin/trace_processor_shell &&
           python3 -m pip install --upgrade pip &&
           python3 -m pip install --upgrade numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.18.4' &&
-          for i in 6 7 8 9 10 11; do /opt/conda/envs/py3.${i}/bin/python -m pip install --upgrade numpy perfetto dataclasses; done
+          for i in 6 7 8 9 10 11; do /opt/conda/envs/py3.${i}/bin/python -m pip install --upgrade numpy perfetto dataclasses; done &&
           apt-get -y --purge autoremove &&
           apt-get -y clean &&
           /opt/conda/bin/conda clean -y -a

.github/workflows/ubuntu-jammy.yml

@@ -109,9 +109,7 @@ jobs:
           apt-get update &&
           apt-get install -y software-properties-common &&
           apt-get upgrade -y &&
-          apt-get install -y build-essential m4 autoconf libtool python3-pip libiberty-dev clang libomp-dev libopenmpi-dev libfabric-dev openmpi-bin environment-modules ${{ matrix.compiler }} &&
-          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
-          chmod +x /opt/trace_processor/bin/trace_processor_shell &&
+          apt-get install -y autoconf bison build-essential clang environment-modules gettext libfabric-dev libiberty-dev libomp-dev libopenmpi-dev libtool m4 openmpi-bin python3-pip texinfo ${{ matrix.compiler }} &&
           python3 -m pip install --upgrade pip &&
           python3 -m pip install --upgrade numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.18.4' &&

README.md

@@ -7,10 +7,6 @@
 [![Installer Packaging (CPack)](https://github.com/ROCm/rocprofiler-systems/actions/workflows/cpack.yml/badge.svg)](https://github.com/ROCm/rocprofiler-systems/actions/workflows/cpack.yml)
 [![Documentation](https://github.com/ROCm/rocprofiler-systems/actions/workflows/docs.yml/badge.svg)](https://github.com/ROCm/rocprofiler-systems/actions/workflows/docs.yml)
 
-> [!NOTE]
-> Perfetto validation is done with trace_processor v46.0, as there is a known issue with v47.0.
-If you are experiencing problems viewing your trace in the latest version of [Perfetto](http://ui.perfetto.dev), then try using [Perfetto UI v46.0](https://ui.perfetto.dev/v46.0-35b3d9845/#!/).
-
 ## Overview
 
 ROCm Systems Profiler (rocprofiler-systems), formerly Omnitrace, is a comprehensive profiling and tracing tool for parallel applications written in C, C++, Fortran, HIP, OpenCL, and Python which execute on the CPU or CPU+GPU.

docs/how-to/understanding-rocprof-sys-output.rst

@@ -323,11 +323,6 @@ absolute path, then all ``ROCPROFSYS_OUTPUT_PATH`` and similar
 settings are ignored. Visit `ui.perfetto.dev <https://ui.perfetto.dev>`_ and open
 this file.
 
-.. important::
-   Perfetto validation is done with trace_processor v46.0 as there is a known issue with v47.0.
-   If you are experiencing problems viewing your trace in the latest version of `Perfetto <http://ui.perfetto.dev>`_,
-   then try using `Perfetto UI v46.0 <https://ui.perfetto.dev/v46.0-35b3d9845/#!/>`_.
-
 .. image:: ../data/rocprof-sys-perfetto.png
    :alt: Visualization of a performance graph in Perfetto
 

docs/install/install.rst

@@ -253,11 +253,6 @@ into Perfetto support for ROCm Systems Profiler, for example, ``ROCPROFSYS_USE_P
 is passed along to Perfetto and is displayed when the ``.proto`` file is visualized
 in `the Perfetto UI <https://ui.perfetto.dev>`_.
 
-.. important::
-   Perfetto validation is done with trace_processor v46.0 as there is a known issue with v47.0.
-   If you are experiencing problems viewing your trace in the latest version of `Perfetto <http://ui.perfetto.dev>`_,
-   then try using `Perfetto UI v46.0 <https://ui.perfetto.dev/v46.0-35b3d9845/#!/>`_.
-
 .. code-block:: shell
 
    git clone https://github.com/ROCm/rocprofiler-systems.git rocprof-sys-source

docs/what-is-rocprof-sys.rst

@@ -15,11 +15,6 @@ A visualization of the comprehensive ROCm Systems Profiler results can be observ
 web browser. Upload the Perfetto (``.proto``) output files produced by ROCm Systems Profiler at
 `ui.perfetto.dev <https://ui.perfetto.dev/>`_ to see the details.
 
-.. important::
-   Perfetto validation is done with trace_processor v46.0 as there is a known issue with v47.0.
-   If you are experiencing problems viewing your trace in the latest version of `Perfetto <http://ui.perfetto.dev>`_,
-   then try using `Perfetto UI v46.0 <https://ui.perfetto.dev/v46.0-35b3d9845/#!/>`_.
-
 Aggregated high-level results are available as human-readable text files and
 JSON files for programmatic analysis. The JSON output files are compatible with the
 `hatchet <https://github.com/hatchet/hatchet>`_ Python package. Hatchet converts

source/lib/core/config.cpp

@@ -2454,18 +2454,37 @@ tmp_file::~tmp_file()
     remove();
 }
 
-bool
-tmp_file::open(std::ios::openmode _mode)
+void
+tmp_file::touch() const
 {
-    ROCPROFSYS_BASIC_VERBOSE(2, "Opening temporary file '%s'...\n", filename.c_str());
-
     if(!filepath::exists(filename))
     {
         // if the filepath does not exist, open in out mode to create it
-        std::ofstream _ofs{};
+        auto _ofs = std::ofstream{};
         filepath::open(_ofs, filename);
     }
+}
+
+bool
+tmp_file::open(int _mode, int _perms)
+{
+    ROCPROFSYS_BASIC_VERBOSE(2, "Opening temporary file '%s'...\n", filename.c_str());
+
+    touch();
+    m_pid = getpid();
+    fd    = ::open(filename.c_str(), _mode, _perms);
 
+    return (fd > 0);
+}
+
+bool
+tmp_file::open(std::ios::openmode _mode)
+{
+    ROCPROFSYS_BASIC_VERBOSE(2, "Opening temporary file '%s'...\n", filename.c_str());
+
+    touch();
+
+    m_pid = getpid();
     stream.open(filename, _mode);
 
     return (stream.is_open() && stream.good());
@@ -2476,14 +2495,10 @@ tmp_file::fopen(const char* _mode)
 {
     ROCPROFSYS_BASIC_VERBOSE(2, "Opening temporary file '%s'...\n", filename.c_str());
 
-    if(!filepath::exists(filename))
-    {
-        // if the filepath does not exist, open in out mode to create it
-        std::ofstream _ofs{};
-        filepath::open(_ofs, filename);
-    }
+    touch();
 
-    file = filepath::fopen(filename, _mode);
+    m_pid = getpid();
+    file  = filepath::fopen(filename, _mode);
     if(file) fd = ::fileno(file);
 
     return (file != nullptr && fd > 0);
@@ -2492,6 +2507,8 @@ tmp_file::fopen(const char* _mode)
 bool
 tmp_file::flush()
 {
+    if(m_pid != getpid()) return false;
+
     if(stream.is_open())
     {
         stream.flush();
@@ -2508,6 +2525,18 @@ tmp_file::flush()
         }
         return (_ret == 0);
     }
+    else if(fd > 0)
+    {
+        int _ret = ::fsync(fd);
+        int _cnt = 0;
+        while(_ret == EAGAIN || _ret == EINTR)
+        {
+            std::this_thread::sleep_for(std::chrono::milliseconds{ 100 });
+            _ret = ::fsync(fd);
+            if(++_cnt > 10) break;
+        }
+        return (_ret == 0);
+    }
 
     return true;
 }
@@ -2517,6 +2546,8 @@ tmp_file::close()
 {
     flush();
 
+    if(m_pid != getpid()) return false;
+
     if(stream.is_open())
     {
         stream.close();
@@ -2532,13 +2563,24 @@ tmp_file::close()
         }
         return (_ret == 0);
     }
+    else if(fd > 0)
+    {
+        auto _ret = ::close(fd);
+        if(_ret == 0)
+        {
+            fd = -1;
+        }
+        return (_ret == 0);
+    }
 
     return true;
 }
 
 bool
 tmp_file::remove()
 {
+    if(m_pid != getpid()) return false;
+
     close();
     if(filepath::exists(filename))
     {
@@ -2553,7 +2595,9 @@ tmp_file::remove()
 
 tmp_file::operator bool() const
 {
-    return (stream.is_open() && stream.good()) || (file != nullptr && fd > 0);
+    return (m_pid == getpid()) &&
+           ((stream.is_open() && stream.good()) || (file != nullptr && fd > 0) ||
+            (file == nullptr && fd > 0));
 }
 
 std::shared_ptr<tmp_file>

source/lib/core/config.hpp

@@ -374,6 +374,7 @@ struct tmp_file
     tmp_file(std::string);
     ~tmp_file();
 
+    bool open(int, int);
     bool open(std::ios::openmode = std::ios::binary | std::ios::in | std::ios::out);
     bool fopen(const char* = "r+");
     bool flush();
@@ -386,6 +387,12 @@ struct tmp_file
     std::fstream stream   = {};
     FILE*        file     = nullptr;
     int          fd       = -1;
+
+private:
+    void touch() const;
+
+private:
+    pid_t m_pid = getpid();
 };
 
 std::shared_ptr<tmp_file>