drakcore: fix drak-gen-ptxed, unregister it from postprocessing (#517)

docs/ipt.rst

@@ -2,35 +2,92 @@
 Using Intel Processor Trace Features (Experimental)
 ===================================================
 
+Enable IPT plugin in drakrun
+----------------------------
+
+1. In ``/etc/drakrun/config.ini``, add ``ipt`` plugin under ``[drakvuf_plugins]`` section ``__all__`` in order to enable IPT tracing.
+2. In ``/etc/drakrun/scripts/cfg.template`` add a new entry: ``vmtrace_buf_kb = 8192``
+3. Execute ``systemctl restart drakrun@1`` (repeat for each drakrun instance if you have scaled them up).
+
+
 Install required extra dependencies
 -----------------------------------
 
-In order to analyze IPT data streams, you need to install ``libipt``, ``xed``, ``ptdump`` (modified) and ``ptxed``.
+In order to analyze IPT data streams, you need to install ``libipt``, ``xed``, ``ptdump`` (modified), ``ptxed`` and ``drak-ipt-blocks`` tools.
 
 .. code-block :: console
 
-  $ rm -rf /tmp/iptbuild
-  $ mkdir /tmp/iptbuild
-  $ cd /tmp/iptbuild
+  rm -rf /tmp/iptbuild
+  mkdir /tmp/iptbuild
+  cd /tmp/iptbuild
+
+  git clone https://github.com/icedevml/libipt.git
+  git clone https://github.com/intelxed/xed.git
+  git clone https://github.com/intelxed/mbuild.git
+  git clone https://github.com/gabime/spdlog.git
+  git clone https://github.com/CERT-Polska/drakvuf-sandbox.git
 
-  $ git clone https://github.com/icedevml/libipt.git
-  $ git clone https://github.com/intelxed/xed.git
-  $ git clone https://github.com/intelxed/mbuild.git
+  cd xed
+  ./mfile.py --share
+  ./mfile.py --prefix=/usr/local install
+  ldconfig
 
-  $ cd xed
-  $ ./mfile.py --share
-  $ ./mfile.py --prefix=/usr/local install
-  $ ldconfig
+  cd ../libipt
+  git checkout
+  cmake -D PTDUMP=On -D PTXED=On .
+  make install
+  
+  cd ../spdlog
+  cmake .
+  make -j$(nproc) install
 
-  $ cd ../libipt
-  $ git checkout
-  $ cmake -D PTDUMP=On -D PTXED=On .
-  $ make install
+  cd ../drakvuf-sandbox/drakcore/drakcore/tools/ipt
+  cmake .
+  make install
 
 
 Generate trace disassembly
 --------------------------
 
-1. Download the completed analysis from MinIO to your local hard drive
-2. Execute ``drak-gen-ptxed --analysis . --cr3 <target_process_cr3> --vcpu 0``
-3. After few minutes it should start printing full trace disassembly of the targeted process
+1. Perform an analysis with IPT plugin enabled
+2. Download the completed analysis from MinIO to your local hard drive
+3. Find CR3 of the target process you want to disassemble (hint: `syscall.log` will contain CR3 values)
+4. Execute ``drak-ipt-disasm --analysis . --cr3 <target_process_cr3> --vcpu 0``
+5. After few minutes it should start printing full trace disassembly of the targeted process
+6. You can also try `--blocks` switch for `drak-ipt-disasm` to get a list of executed basic blocks for this process
+
+**Example (executed basic blocks):**
+
+.. code-block :: console
+
+  # drak-ipt-disasm --analysis . --cr3 0x735bb000 --vcpu 0 --blocks
+  [2021-04-19 23:47:41.717] [console] [info] Decoding
+  { "event": "block_executed", "data": "0x7feff565088" }
+  { "event": "block_executed", "data": "0x7feff75450f" }
+  { "event": "block_executed", "data": "0x7feff754505" }
+  { "event": "block_executed", "data": "0x7feff75450d" }
+  { "event": "block_executed", "data": "0x7feff5656ac" }
+  { "event": "block_executed", "data": "0x7feff5656dc" }
+  { "event": "block_executed", "data": "0x7feff5656fb" }
+  { "event": "block_executed", "data": "0x7feff565068" }
+  { "event": "block_executed", "data": "0x7feff751530" }
+  { "event": "block_executed", "data": "0x7feff751552" }
+  ...
+
+
+**Example (full usermode disassembly):**
+
+.. code-block :: console
+
+  # drak-ipt-disasm --analysis . --cr3 0x735bb000 --vcpu 0 | grep -v ptwrite | grep -v cbr
+  [enabled]
+  [exec mode: 64-bit]
+  000007feff565088  movdqu xmmword ptr [rip+0x1b2b80], xmm0
+  000007feff565090  ret
+  000007feff75450f  add rbx, 0x8
+  000007feff754513  cmp rbx, rdi
+  000007feff754516  jb 0x7feff754505
+  000007feff754505  mov rax, qword ptr [rbx]
+  000007feff754508  test rax, rax
+  000007feff75450b  jz 0x7feff75450f
+  ...

drakcore/debian/drakcore.links

@@ -4,5 +4,5 @@ opt/venvs/drakcore/bin/drak-postprocess usr/bin/drak-postprocess
 opt/venvs/drakcore/bin/drak-healthcheck usr/bin/drak-healthcheck
 opt/venvs/drakcore/bin/drak-vncpasswd usr/bin/drak-vncpasswd
 opt/venvs/drakcore/bin/drak-upgrade-db usr/bin/drak-upgrade-db
-opt/venvs/drakcore/bin/drak-gen-ptxed usr/bin/drak-gen-ptxed
+opt/venvs/drakcore/bin/drak-ipt-disasm usr/bin/drak-ipt-disasm
 opt/venvs/drakcore/bin/drak-ipt-filter usr/bin/drak-ipt-filter

drakcore/drakcore/bin/drak-ipt-disasm

@@ -0,0 +1,4 @@
+#!/usr/bin/env python
+
+from drakcore.ipt_disasm import cmdline_main
+cmdline_main()

drakcore/drakcore/postprocess/ipt_disasm.py → drakcore/drakcore/ipt_disasm.py

@@ -12,7 +12,7 @@
 
 from karton.core import Task, RemoteResource
 from typing import Dict
-from drakcore.postprocess.ipt_utils import log, load_drakvuf_output, get_fault_va, get_fault_pa, get_trap_pa, get_frame_va, page_align, is_page_aligned, select_cr3, hexint
+from drakcore.ipt_utils import log, load_drakvuf_output, get_fault_va, get_fault_pa, get_trap_pa, get_frame_va, page_align, is_page_aligned, select_cr3, hexint
 from zipfile import ZipFile
 
 
@@ -76,26 +76,27 @@ def match_frames(page_faults, frames, foreign_frames):
                 unresolved += 1
             else:
                 foreign_resolved += 1
-        log.info("%#016x -> %s", va_page, frame['FrameFile'] if frame else "?")
+        log.info("%#016x -> %s", va_page, frame['DumpFile'] if frame else "?")
         if frame:
-            results.append((va_page, frame['FrameFile']))
+            results.append((va_page, frame['DumpFile']))
 
     log.info("Failed to resolve %d faults. Let's hope they're not related to code", unresolved)
     log.info("Resolved %d from external CR3", foreign_resolved)
 
     return results
 
 
-def main(analysis_dir, cr3_value, vcpu):
+def get_ptxed_cmdline(analysis_dir, cr3_value, vcpu, use_blocks=False):
     log.debug("Analysis directory: %s", analysis_dir)
     log.debug("CR3: %#x", cr3_value)
 
     if not is_page_aligned(cr3_value):
         log.critical("CR3 must be aligned to page! Got %#x", cr3_value)
         return
 
-    page_faults = load_drakvuf_output(analysis_dir / "pagefault.log")
-    executed_frames = load_drakvuf_output(analysis_dir / "execframe.log")
+    codemon_out = load_drakvuf_output(analysis_dir / "codemon.log")
+    page_faults = [obj for obj in codemon_out if obj['EventType'] == 'pagefault']
+    executed_frames = [obj for obj in codemon_out if obj['EventType'] == 'execframe']
 
     faults_in_process = list(select_cr3(lambda cr3: cr3 == cr3_value, page_faults))
     frames_in_process = list(select_cr3(lambda cr3: cr3 == cr3_value, executed_frames))
@@ -112,58 +113,26 @@ def main(analysis_dir, cr3_value, vcpu):
     pages = []
     for addr, fname in mappings:
         name = Path(fname).name
-        fpath = analysis_dir / "ipt" / "frames" / name
+        fpath = analysis_dir / "ipt" / "dumps" / name
         if fpath.stat().st_size == 0x1000:
             pages.append("--raw")
             pages.append(f"{fpath}:0x{addr:x}")
-    ptxed_cmdline = [
-        "/opt/libipt/bin/ptxed",
-        "--block-decoder",
-        "--pt",
-        os.path.join(analysis_dir, "ipt", f"ipt_stream_vcpu{vcpu}"),
-        *pages
-    ]
 
-    log.info("IPT: Succesfully generated ptxed command line")
-    return ptxed_cmdline
-
-
-def generate_ipt_disasm(task: Task, resources: Dict[str, RemoteResource], minio):
-    with tempfile.TemporaryDirectory() as tmpdir:
-        tmpdir = Path(tmpdir)
-        resources["inject.log"].download_to_file(str(tmpdir / "inject.log"))
-
-        with open(str(tmpdir / "inject.log"), "r") as f:
-            inject_log = json.loads(f.read().split('\n')[0].strip())
-
-        injected_pid = inject_log["InjectedPid"]
-        resources["execframe.log"].download_to_file(str(tmpdir / "execframe.log"))
+    binary = ["ptxed", "--block-decoder"]
 
-        with open(str(tmpdir / "execframe.log"), "r") as f:
-            for line in f:
-                obj = json.loads(line)
+    if use_blocks:
+        binary = ["drak-ipt-blocks", "--cr3", hex(cr3_value)]
 
-                if obj.get("PID") == injected_pid:
-                    injected_cr3 = hexint(obj["CR3"])
-                    break
-            else:
-                log.error("Failed to find injected process' CR3, not doing IPT disasm")
-                return
-
-        resources["pagefault.log"].download_to_file(str(tmpdir / "pagefault.log"))
-        resources["execframe.log"].download_to_file(str(tmpdir / "execframe.log"))
-
-        with resources["ipt.zip"].download_temporary_file() as ipt_zip_tmp:
-            with ZipFile(ipt_zip_tmp) as ipt_zip:
-                ipt_zip.extractall(tmpdir)
-
-        main(tmpdir, injected_cr3)
+    ptxed_cmdline = binary + pages
+    log.info("IPT: Succesfully generated ptxed command line")
+    return ptxed_cmdline
 
 
 def cmdline_main():
     parser = argparse.ArgumentParser()
     parser.add_argument("--dry-run", action="store_true", default=False, help="Print generated ptxed command, don't run it")
     parser.add_argument("--verbose", action="store_true", default=False, help="Print additional debug messages")
+    parser.add_argument("--blocks", action="store_true", default=False, help="Use drak-ipt-blocks instead of ptxed")
     parser.add_argument("--analysis", help="Analysis directory (as downloaded from MinIO)")
     parser.add_argument("--cr3", type=hexint, help="CR3 of process of interest")
     parser.add_argument("--vcpu", type=int, help="Number of vCPU to disassemble")
@@ -175,10 +144,10 @@ def cmdline_main():
     analysis_dir = Path(args.analysis)
     cr3_value = args.cr3
 
-    ptxed_cmdline = main(analysis_dir, cr3_value, args.vcpu)
+    ptxed_cmdline = get_ptxed_cmdline(analysis_dir, cr3_value, args.vcpu, args.blocks)
 
     if args.dry_run:
-        print(subprocess.list2cmdline(ptxed_cmdline))
+        print(subprocess.list2cmdline(ptxed_cmdline + ["--pt", "FILTERED_PT_FILE"]))
         sys.exit(0)
 
     with tempfile.NamedTemporaryFile() as f:
@@ -193,4 +162,5 @@ def cmdline_main():
         subprocess.run(' | '.join(filter_cmdline), shell=True)
 
         logging.info("Generating trace disassembly")
+        ptxed_cmdline = ptxed_cmdline + ["--pt", f.name]
         subprocess.run(ptxed_cmdline)

drakcore/drakcore/postprocess/ipt_utils.py → drakcore/drakcore/ipt_utils.py

@@ -66,7 +66,7 @@ def get_trap_pa(execframe):
 
 
 def get_frame_va(execframe):
-    return hexint(execframe['FrameVA'])
+    return hexint(execframe['PageVA'])
 
 
 def page_align(addr):

drakcore/drakcore/postprocess/__init__.py

@@ -6,7 +6,6 @@
 from drakcore.postprocess.log_index import generate_log_index
 from drakcore.postprocess.pstree import build_process_tree
 from drakcore.postprocess.slice_logs import slice_drakmon_logs
-from drakcore.postprocess.ipt_disasm import generate_ipt_disasm
 from drakcore.postprocess.wireshark_key_file_gen import generate_wireshark_key_file
 
 PostprocessPlugin = namedtuple("PostprocessPlugin", ('handler', 'required'))
@@ -26,8 +25,6 @@
     # yields wireshark_key_file.txt
     PostprocessPlugin(generate_wireshark_key_file, required=['tlsmon.log']),
 
-    PostprocessPlugin(generate_ipt_disasm, required=['execframe.log', 'pagefault.log', 'ipt.zip']),
-
     # yields index/{name}
     PostprocessPlugin(generate_log_index, required=[]),
     # this should be the final step

drakcore/drakcore/tools/ipt/CMakeLists.txt

@@ -1,6 +1,7 @@
 cmake_minimum_required(VERSION 3.2)
-project(ipt-tools CXX)
+project(drak-ipt-tools CXX)
 
-add_executable(ipt-blocks ipt-blocks.cpp)
-target_link_libraries(ipt-blocks ipt json-c spdlog fmt)
-set_property(TARGET ipt-blocks PROPERTY CXX_STANDARD 17)
+add_executable(drak-ipt-blocks drak-ipt-blocks.cpp)
+target_link_libraries(drak-ipt-blocks ipt json-c spdlog fmt)
+set_property(TARGET drak-ipt-blocks PROPERTY CXX_STANDARD 17)
+install(TARGETS drak-ipt-blocks RUNTIME DESTINATION bin)

drakcore/setup.py

@@ -23,7 +23,7 @@
              'drakcore/bin/drak-healthcheck',
              'drakcore/bin/drak-vncpasswd',
              'drakcore/bin/drak-upgrade-db',
-             'drakcore/bin/drak-gen-ptxed',
+             'drakcore/bin/drak-ipt-disasm',
              'drakcore/bin/drak-ipt-filter'],
     classifiers=[
         "Programming Language :: Python",