[ESI][Runtime] Generate C++ header files for constants

integration_test/Dialect/ESI/runtime/callback.mlir

@@ -1,9 +1,13 @@
 // REQUIRES: esi-cosim, esi-runtime, rtl-sim
+
+// Generate SV files
 // RUN: rm -rf %t6 && mkdir %t6 && cd %t6
-// RUN: circt-opt %s --esi-connect-services --esi-appid-hier=top=top --esi-build-manifest=top=top --esi-clean-metadata > %t4.mlir
-// RUN: circt-opt %t4.mlir --lower-esi-to-physical --lower-esi-bundles --lower-esi-ports --lower-esi-to-hw=platform=cosim --lower-seq-to-sv --lower-hwarith-to-hw --canonicalize --export-split-verilog -o %t3.mlir
+// RUN: mkdir hw && cd hw
+// RUN: circt-opt %s --esi-connect-services --esi-appid-hier=top=top --esi-build-manifest=top=top --esi-clean-metadata --lower-esi-to-physical --lower-esi-bundles --lower-esi-ports --lower-esi-to-hw=platform=cosim --lower-seq-to-sv --lower-hwarith-to-hw --canonicalize --export-split-verilog
 // RUN: cd ..
-// RUN: esi-cosim.py --source %t6 --top top -- esitester cosim env wait | FileCheck %s
+
+// Test cosimulation
+// RUN: esi-cosim.py --source %t6/hw --top top -- esitester cosim env wait | FileCheck %s
 
 hw.module @top(in %clk : !seq.clock, in %rst : i1) {
   hw.instance "PrintfExample" sym @PrintfExample @PrintfExample(clk: %clk: !seq.clock, rst: %rst: i1) -> ()

integration_test/Dialect/ESI/runtime/loopback.mlir

@@ -1,12 +1,27 @@
 // REQUIRES: esi-cosim, esi-runtime, rtl-sim
 // RUN: rm -rf %t6 && mkdir %t6 && cd %t6
-// RUN: circt-opt %s --esi-connect-services --esi-appid-hier=top=top --esi-build-manifest=top=top --esi-clean-metadata > %t4.mlir
-// RUN: circt-opt %t4.mlir --lower-esi-to-physical --lower-esi-bundles --lower-esi-ports --lower-esi-to-hw=platform=cosim --lower-seq-to-sv --lower-hwarith-to-hw --canonicalize --export-split-verilog -o %t3.mlir
+
+// Generate SV files
+// RUN: mkdir hw && cd hw
+// RUN: circt-opt %s --esi-connect-services --esi-appid-hier=top=top --esi-build-manifest=top=top --esi-clean-metadata --lower-esi-to-physical --lower-esi-bundles --lower-esi-ports --lower-esi-to-hw=platform=cosim --lower-seq-to-sv --lower-hwarith-to-hw --canonicalize --export-split-verilog -o %t3.mlir
 // RUN: cd ..
-// RUN: esiquery trace w:%t6/esi_system_manifest.json info | FileCheck %s --check-prefix=QUERY-INFO
-// RUN: esiquery trace w:%t6/esi_system_manifest.json hier | FileCheck %s --check-prefix=QUERY-HIER
-// RUN: %python %s.py trace w:%t6/esi_system_manifest.json
-// RUN: esi-cosim.py --source %t6 --top top -- %python %s.py cosim env
+
+// Test ESI utils
+// RUN: esiquery trace w:%t6/hw/esi_system_manifest.json info | FileCheck %s --check-prefix=QUERY-INFO
+// RUN: esiquery trace w:%t6/hw/esi_system_manifest.json hier | FileCheck %s --check-prefix=QUERY-HIER
+
+// Test cosimulation
+// RUN: esi-cosim.py --source %t6/hw --top top -- %python %s.py cosim env
+
+// Test C++ header generation against the manifest file
+// RUN: %python -m esiaccel.cppgen --file %t6/hw/esi_system_manifest.json --output-dir %t6/include
+// RUN: %host_cxx -I %t6/include %s.cpp -o %t6/test
+// RUN: %t6/test | FileCheck %s --check-prefix=CPP-TEST
+
+// Test C++ header generation against a live accelerator
+// RUN: esi-cosim.py --source %t6 --top top -- %python -m esiaccel.cppgen --platform cosim --connection env --output-dir %t6/include
+// RUN: %host_cxx -I %t6/include %s.cpp -o %t6/test
+// RUN: %t6/test | FileCheck %s --check-prefix=CPP-TEST
 
 !sendI8 = !esi.bundle<[!esi.channel<i8> from "send"]>
 !recvI8 = !esi.bundle<[!esi.channel<i8> to "recv"]>
@@ -109,6 +124,8 @@ hw.module @top(in %clk: !seq.clock, in %rst: i1) {
   hw.instance "loopback_array" @LoopbackArray() -> ()
 }
 
+// CPP-TEST: depth: 0x5
+
 // QUERY-INFO: API version: 0
 // QUERY-INFO: ********************************
 // QUERY-INFO: * Module information

integration_test/Dialect/ESI/runtime/loopback.mlir.cpp

@@ -0,0 +1,5 @@
+#include "LoopbackIP.h"
+
+#include <stdio.h>
+
+int main() { printf("depth: 0x%x\n", esi_system::LoopbackIP::depth); }

integration_test/lit.cfg.py

@@ -187,8 +187,9 @@
 if config.bindings_tcl_enabled:
   config.available_features.add('bindings_tcl')
 
-# Add host c compiler.
+# Add host c/c++ compiler.
 config.substitutions.append(("%host_cc", config.host_cc))
+config.substitutions.append(("%host_cxx", config.host_cxx))
 
 # Enable clang-tidy if it has been detected.
 if config.clang_tidy_path != "":

lib/Dialect/ESI/runtime/CMakeLists.txt

@@ -122,6 +122,7 @@ set(ESICppRuntimeBackendHeaders
 set(ESIPythonRuntimeSources
   python/esiaccel/__init__.py
   python/esiaccel/accelerator.py
+  python/esiaccel/cppgen.py
   python/esiaccel/types.py
   python/esiaccel/utils.py
 )

lib/Dialect/ESI/runtime/cpp/include/esi/backends/Trace.h

@@ -43,6 +43,9 @@ class TraceAccelerator : public esi::AcceleratorConnection {
     // Data read from the accelerator is read from the trace file.
     // TODO: Full trace mode not yet supported.
     // Read
+
+    // Discard all data sent to the accelerator. Disable trace file generation.
+    Discard,
   };
 
   /// Create a trace-based accelerator backend.

lib/Dialect/ESI/runtime/cpp/lib/backends/Trace.cpp

@@ -49,6 +49,8 @@ struct esi::backends::trace::TraceAccelerator::Impl {
       if (!traceWrite->is_open())
         throw std::runtime_error("failed to open trace file '" +
                                  traceFile.string() + "'");
+    } else if (mode == Discard) {
+      traceWrite = nullptr;
     } else {
       assert(false && "not implemented");
     }
@@ -76,9 +78,11 @@ struct esi::backends::trace::TraceAccelerator::Impl {
   void write(const AppIDPath &id, const std::string &portName, const void *data,
              size_t size);
   std::ostream &write(std::string service) {
+    assert(traceWrite && "traceWrite is null");
     *traceWrite << "[" << service << "] ";
     return *traceWrite;
   }
+  bool isWriteable() { return traceWrite; }
 
 private:
   std::ofstream *traceWrite;
@@ -90,6 +94,8 @@ struct esi::backends::trace::TraceAccelerator::Impl {
 void TraceAccelerator::Impl::write(const AppIDPath &id,
                                    const std::string &portName,
                                    const void *data, size_t size) {
+  if (!isWriteable())
+    return;
   std::string b64data;
   utils::encodeBase64(data, size, b64data);
 
@@ -105,7 +111,7 @@ TraceAccelerator::connect(Context &ctxt, std::string connectionString) {
 
   // Parse the connection std::string.
   // <mode>:<manifest path>[:<traceFile>]
-  std::regex connPattern("(\\w):([^:]+)(:(\\w+))?");
+  std::regex connPattern("([\\w-]):([^:]+)(:(\\w+))?");
   std::smatch match;
   if (regex_search(connectionString, match, connPattern)) {
     modeStr = match[1];
@@ -121,6 +127,8 @@ TraceAccelerator::connect(Context &ctxt, std::string connectionString) {
   Mode mode;
   if (modeStr == "w")
     mode = Write;
+  else if (modeStr == "-")
+    mode = Discard;
   else
     throw std::runtime_error("unknown mode '" + modeStr + "'");
 
@@ -260,7 +268,8 @@ class TraceHostMem : public HostMem {
       this->size = size;
     }
     virtual ~TraceHostMemRegion() {
-      impl.write("HostMem") << "free " << ptr << std::endl;
+      if (impl.isWriteable())
+        impl.write("HostMem") << "free " << ptr << std::endl;
       free(ptr);
     }
     virtual void *getPtr() const override { return ptr; }
@@ -276,22 +285,26 @@ class TraceHostMem : public HostMem {
   allocate(std::size_t size, HostMem::Options opts) const override {
     auto ret =
         std::unique_ptr<HostMemRegion>(new TraceHostMemRegion(size, impl));
-    impl.write("HostMem 0x")
-        << ret->getPtr() << " allocate " << size
-        << " bytes. Writeable: " << opts.writeable
-        << ", useLargePages: " << opts.useLargePages << std::endl;
+    if (impl.isWriteable())
+      impl.write("HostMem 0x")
+          << ret->getPtr() << " allocate " << size
+          << " bytes. Writeable: " << opts.writeable
+          << ", useLargePages: " << opts.useLargePages << std::endl;
     return ret;
   }
   virtual bool mapMemory(void *ptr, std::size_t size,
                          HostMem::Options opts) const override {
-    impl.write("HostMem") << "map 0x" << ptr << " size " << size
-                          << " bytes. Writeable: " << opts.writeable
-                          << ", useLargePages: " << opts.useLargePages
-                          << std::endl;
+
+    if (impl.isWriteable())
+      impl.write("HostMem")
+          << "map 0x" << ptr << " size " << size
+          << " bytes. Writeable: " << opts.writeable
+          << ", useLargePages: " << opts.useLargePages << std::endl;
     return true;
   }
   virtual void unmapMemory(void *ptr) const override {
-    impl.write("HostMem") << "unmap 0x" << ptr << std::endl;
+    if (impl.isWriteable())
+      impl.write("HostMem") << "unmap 0x" << ptr << std::endl;
   }
 
 private:

lib/Dialect/ESI/runtime/pyproject.toml

@@ -44,3 +44,4 @@ classifiers = [
 [project.scripts]
 esiquery = "esiaccel.utils:run_esiquery"
 esi-cosim = "esiaccel.utils:run_esi_cosim"
+esi-cppgen = "esiaccel.cppgen:run"

lib/Dialect/ESI/runtime/python/esiaccel/cppgen.py

@@ -0,0 +1,193 @@
+#  Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+#  See https://llvm.org/LICENSE.txt for license information.
+#  SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+# Code generation from ESI manifests to C++ headers. Intended to be extensible
+# for other languages.
+
+from typing import List, Type, Optional
+from .accelerator import AcceleratorConnection
+from .esiCppAccel import ModuleInfo
+from . import types
+
+import argparse
+from pathlib import Path
+import textwrap
+import sys
+
+_thisdir = Path(__file__).absolute().resolve().parent
+
+
+class Generator:
+  """Base class for all generators."""
+
+  language: Optional[str] = None
+
+  def __init__(self, conn: AcceleratorConnection):
+    self.manifest = conn.manifest()
+
+  def generate(self, output_dir: Path, system_name: str):
+    raise NotImplementedError("Generator.generate() must be overridden")
+
+
+class CppGenerator(Generator):
+  """Generate C++ headers from an ESI manifest."""
+
+  language = "C++"
+
+  # Supported bit widths for lone integer types.
+  int_width_support = set([8, 16, 32, 64])
+
+  def get_type_str(self, type: types.ESIType) -> str:
+    """Get the textual code for the storage class of a type.
+
+    Examples: uint32_t, int64_t, CustomStruct."""
+
+    if isinstance(type, (types.BitsType, types.IntType)):
+      if type.bit_width not in self.int_width_support:
+        raise ValueError(f"Unsupported integer width: {type.bit_width}")
+      if isinstance(type, (types.BitsType, types.UIntType)):
+        return f"uint{type.bit_width}_t"
+      return f"int{type.bit_width}_t"
+    raise NotImplementedError(f"Type '{type}' not supported for C++ generation")
+
+  def get_consts_str(self, module_info: ModuleInfo) -> str:
+    """Get the C++ code for a constant in a module."""
+    const_strs: List[str] = [
+        f"static constexpr {self.get_type_str(const.type)} "
+        f"{name} = 0x{const.value:x};"
+        for name, const in module_info.constants.items()
+    ]
+    return "\n".join(const_strs)
+
+  def write_modules(self, output_dir: Path, system_name: str):
+    """Write the C++ one header for each module in the manifest."""
+
+    for module_info in self.manifest.module_infos:
+      s = f"""
+      /// Generated header for {system_name} module {module_info.name}.
+      #pragma once
+      #include "types.h"
+
+      namespace {system_name} {{
+        class {module_info.name} {{
+        public:
+          {self.get_consts_str(module_info)}
+        }};
+      }} // namespace {system_name}
+      """
+
+      hdr_file = output_dir / f"{module_info.name}.h"
+      with open(hdr_file, "w") as hdr:
+        hdr.write(textwrap.dedent(s))
+
+  def write_type(self, hdr, type: types.ESIType):
+    if isinstance(type, (types.BitsType, types.IntType)):
+      # Bit vector types use standard C++ types.
+      return
+    raise NotImplementedError(f"Type '{type}' not supported for C++ generation")
+
+  def write_types(self, output_dir: Path, system_name: str):
+    hdr_file = output_dir / "types.h"
+    with open(hdr_file, "w") as hdr:
+      hdr.write(
+          textwrap.dedent(f"""
+      // Generated header for {system_name} types.
+      #pragma once
+
+      #include <cstdint>
+
+      namespace {system_name} {{
+      """))
+
+      for type in self.manifest.type_table:
+        try:
+          self.write_type(hdr, type)
+        except NotImplementedError:
+          sys.stderr.write(
+              f"Warning: type '{type}' not supported for C++ generation\n")
+
+      hdr.write(
+          textwrap.dedent(f"""
+      }} // namespace {system_name}
+      """))
+
+  def generate(self, output_dir: Path, system_name: str):
+    self.write_types(output_dir, system_name)
+    self.write_modules(output_dir, system_name)
+
+
+def run(generator: Type[Generator] = CppGenerator,
+        cmdline_args=sys.argv) -> int:
+  """Create and run a generator reading options from the command line."""
+
+  argparser = argparse.ArgumentParser(
+      description=f"Generate {generator.language} headers from an ESI manifest",
+      formatter_class=argparse.RawDescriptionHelpFormatter,
+      epilog=textwrap.dedent("""
+        Can read the manifest from either a file OR a running accelerator.
+
+        Usage examples:
+          # To read the manifest from a file:
+          esi-cppgen --file /path/to/manifest.json
+
+          # To read the manifest from a running accelerator:
+          esi-cppgen --platform cosim --connection localhost:1234
+      """))
+
+  argparser.add_argument("--file",
+                         type=str,
+                         default=None,
+                         help="Path to the manifest file.")
+  argparser.add_argument(
+      "--platform",
+      type=str,
+      help="Name of backend for live accelerator connection.")
+  argparser.add_argument(
+      "--connection",
+      type=str,
+      help="Connection string for live accelerator connection.")
+  argparser.add_argument("--output-dir",
+                         type=str,
+                         default=".",
+                         help="Output directory for generated files.")
+  argparser.add_argument("--system-name",
+                         type=str,
+                         default="esi_system",
+                         help="Name of the ESI system.")
+
+  if (len(cmdline_args) <= 1):
+    argparser.print_help()
+    return 1
+  args = argparser.parse_args(cmdline_args[1:])
+
+  if args.file is not None and args.platform is not None:
+    print("Cannot specify both --file and --platform")
+    return 1
+
+  conn: AcceleratorConnection
+  if args.file is not None:
+    conn = AcceleratorConnection("trace", f"-:{args.file}")
+  elif args.platform is not None:
+    if args.connection is None:
+      print("Must specify --connection with --platform")
+      return 1
+    conn = AcceleratorConnection(args.platform, args.connection)
+  else:
+    print("Must specify either --file or --platform")
+    return 1
+
+  output_dir = Path(args.output_dir)
+  if output_dir.exists() and not output_dir.is_dir():
+    print(f"Output directory {output_dir} is not a directory")
+    return 1
+  if not output_dir.exists():
+    output_dir.mkdir(parents=True)
+
+  gen = generator(conn)
+  gen.generate(output_dir, args.system_name)
+  return 0
+
+
+if __name__ == '__main__':
+  sys.exit(run())