MVP Batch renderer WIP

src/esp/bindings/GfxReplayBindings.cpp

@@ -58,6 +58,19 @@ void initGfxReplayBindings(py::module& m) {
           },
           R"(Save a render keyframe; a render keyframe can be loaded later and used to draw observations.)")
 
+      .def(
+          "extract_keyframe",
+          [](ReplayManager& self) {
+            if (!self.getRecorder()) {
+              throw std::runtime_error(
+                  "replay save not enabled. See "
+                  "SimulatorConfiguration.enable_gfx_replay_save.");
+            }
+            return esp::gfx::replay::Recorder::keyframeToString(
+                self.getRecorder()->extractKeyframe());
+          },
+          R"(Extract a render keyframe as a JSON-formatted string)")
+
       .def(
           "add_user_transform_to_keyframe",
           [](ReplayManager& self, const std::string& name,

src/esp/bindings/SimBindings.cpp

@@ -15,6 +15,7 @@
 #include "esp/gfx/Renderer.h"
 #include "esp/gfx/replay/ReplayManager.h"
 #include "esp/scene/SemanticScene.h"
+#include "esp/sim/ReplayBatchRenderer.h"
 #include "esp/sim/Simulator.h"
 #include "esp/sim/SimulatorConfiguration.h"
 
@@ -355,6 +356,50 @@ void initSimBindings(py::module& m) {
       .def("get_debug_line_render", &Simulator::getDebugLineRender,
            pybind11::return_value_policy::reference,
            R"(Get visualization helper for rendering lines.)");
+
+  // ==== ReplayBatchRendererConfiguration ====
+  py::class_<ReplayBatchRendererConfiguration,
+             ReplayBatchRendererConfiguration::ptr>(
+      m, "ReplayBatchRendererConfiguration")
+      .def(py::init(&ReplayBatchRendererConfiguration::create<>))
+      .def_readwrite("num_environments",
+                     &ReplayBatchRendererConfiguration::numEnvironments,
+                     R"(todo)")
+      .def_readwrite("sensor_specifications",
+                     &ReplayBatchRendererConfiguration::sensorSpecifications,
+                     R"(todo)")
+      .def_readwrite("gpu_device_id",
+                     &ReplayBatchRendererConfiguration::gpuDeviceId, R"(todo)")
+      .def_readwrite(
+          "force_separate_semantic_scene_graph",
+          &ReplayBatchRendererConfiguration::forceSeparateSemanticSceneGraph,
+          R"(todo)")
+      .def_readwrite(
+          "leave_context_with_background_renderer",
+          &ReplayBatchRendererConfiguration::leaveContextWithBackgroundRenderer,
+          R"(todo)");
+
+  // ==== ReplayBatchRenderer ====
+  py::class_<ReplayBatchRenderer, ReplayBatchRenderer::ptr>(
+      m, "ReplayBatchRenderer")
+      // modify constructor to pass MetadataMediator
+      .def(py::init<const ReplayBatchRendererConfiguration&>())
+      .def_property_readonly("renderer", &ReplayBatchRenderer::getRenderer)
+      .def("get_scene_graph", &ReplayBatchRenderer::getSceneGraph,
+           R"(PYTHON DOES NOT GET OWNERSHIP)",
+           py::return_value_policy::reference)
+      .def("get_semantic_scene_graph",
+           &ReplayBatchRenderer::getSemanticSceneGraph,
+           R"(PYTHON DOES NOT GET OWNERSHIP)",
+           py::return_value_policy::reference)
+      .def("get_environment_sensors",
+           &ReplayBatchRenderer::getEnvironmentSensors)
+      .def("set_sensor_transforms_from_keyframe",
+           &ReplayBatchRenderer::setSensorTransformsFromKeyframe)
+      .def("get_environment_sensor_parent_node",
+           &ReplayBatchRenderer::getEnvironmentSensorParentNode)
+      .def("set_environment_keyframe",
+           &ReplayBatchRenderer::setEnvironmentKeyframe);
 }
 
 }  // namespace sim

src/esp/gfx/replay/Player.cpp

@@ -176,6 +176,13 @@ void Player::appendJSONKeyframe(const std::string& keyframe) {
   appendKeyframe(keyframeFromString(keyframe));
 }
 
+void Player::setSingleKeyframe(Keyframe&& keyframe) {
+  keyframes_.clear();
+  frameIndex_ = -1;
+  keyframes_.emplace_back(std::move(keyframe));
+  setKeyframeIndex(0);
+}
+
 void Player::setSemanticIdForSubtree(esp::scene::SceneNode* rootNode,
                                      int semanticId) {
   if (rootNode->getSemanticId() == semanticId) {

src/esp/gfx/replay/Player.h

@@ -64,7 +64,7 @@ class Player {
    * @brief Given a JSON string encoding a keyframe, returns the keyframe
    * itself.
    */
-  Keyframe keyframeFromString(const std::string& keyframe);
+  static Keyframe keyframeFromString(const std::string& keyframe);
 
   /**
    * @brief Get the currently-set keyframe, or -1 if no keyframe is set.
@@ -112,6 +112,8 @@ class Player {
    */
   void appendKeyframe(Keyframe&& keyframe);
 
+  void setSingleKeyframe(Keyframe&& keyframe);
+
   /**
    * @brief Appends a JSON keyframe to the keyframe list.
    */

src/esp/gfx/replay/Recorder.cpp

@@ -70,6 +70,13 @@ void Recorder::saveKeyframe() {
   advanceKeyframe();
 }
 
+Keyframe Recorder::extractKeyframe() {
+  updateInstanceStates();
+  const auto retVal = std::move(currKeyframe_);
+  currKeyframe_ = Keyframe{};
+  return retVal;
+}
+
 const Keyframe& Recorder::getLatestKeyframe() {
   CORRADE_ASSERT(!savedKeyframes_.empty(),
                  "Recorder::getLatestKeyframe() : Trying to access latest "

src/esp/gfx/replay/Recorder.h

@@ -66,6 +66,8 @@ class Recorder {
    */
   void saveKeyframe();
 
+  Keyframe extractKeyframe();
+
   /**
    * @brief Returns the last saved keyframe.
    */
@@ -108,7 +110,7 @@ class Recorder {
   /**
    * @brief returns JSONized version of given keyframe.
    */
-  std::string keyframeToString(const Keyframe& keyframe);
+  static std::string keyframeToString(const Keyframe& keyframe);
 
   /**
    * @brief Reserved for unit-testing.

src/esp/io/JsonBuiltinTypes.h

@@ -39,6 +39,7 @@ void addMember(JsonGenericValue& value,
 template <typename T>
 bool readMember(const JsonGenericValue& value, const char* name, T& x);
 
+#if 0  // Reference code
 /**
  * @brief Fallback implementation for fromJsonValue to produce a runtime error
  * for types that haven't implemented fromJsonValue.
@@ -70,6 +71,7 @@ JsonGenericValue toJsonValue(const T&, JsonAllocator&) {
       << typeid(T).name() << ".";
   return JsonGenericValue(rapidjson::kObjectType);
 }
+#endif
 
 // toJsonValue wrappers for the 7 rapidjson builtin types. A JsonGenericValue
 // can be directly constructed from the builtin types.

src/esp/io/JsonEspTypes.cpp

@@ -101,5 +101,32 @@ bool fromJsonValue(const JsonGenericValue& obj,
   return true;
 }
 
+JsonGenericValue toJsonValue(
+    const metadata::attributes::ObjectInstanceShaderType& x,
+    JsonAllocator& allocator) {
+  return toJsonValue(metadata::attributes::getShaderTypeName(x), allocator);
+}
+
+bool fromJsonValue(const JsonGenericValue& obj,
+                   metadata::attributes::ObjectInstanceShaderType& x) {
+  std::string shaderTypeToUseString;
+  // read as string
+  bool shaderTypeSucceess = fromJsonValue(obj, shaderTypeToUseString);
+  // convert to enum
+  if (shaderTypeSucceess) {
+    const std::string shaderTypeLC =
+        Cr::Utility::String::lowercase(shaderTypeToUseString);
+    auto mapIter = metadata::attributes::ShaderTypeNamesMap.find(shaderTypeLC);
+    ESP_CHECK(
+        mapIter != metadata::attributes::ShaderTypeNamesMap.end(),
+        "Illegal shader_type value"
+            << shaderTypeToUseString
+            << "specified in JSON to be used to set AssetInfo.shaderTypeToUse. "
+               "Aborting.");
+    x = mapIter->second;
+  }
+  return shaderTypeSucceess;
+}
+
 }  // namespace io
 }  // namespace esp

src/esp/io/JsonEspTypes.h

@@ -111,32 +111,12 @@ inline bool fromJsonValue(const JsonGenericValue& obj,
   return true;
 }
 
-inline JsonGenericValue toJsonValue(
+JsonGenericValue toJsonValue(
     const metadata::attributes::ObjectInstanceShaderType& x,
-    JsonAllocator& allocator) {
-  return toJsonValue(metadata::attributes::getShaderTypeName(x), allocator);
-}
+    JsonAllocator& allocator);
 
-inline bool fromJsonValue(const JsonGenericValue& obj,
-                          metadata::attributes::ObjectInstanceShaderType& x) {
-  std::string shaderTypeToUseString;
-  // read as string
-  bool shaderTypeSucceess = fromJsonValue(obj, shaderTypeToUseString);
-  // convert to enum
-  if (shaderTypeSucceess) {
-    const std::string shaderTypeLC =
-        Cr::Utility::String::lowercase(shaderTypeToUseString);
-    auto mapIter = metadata::attributes::ShaderTypeNamesMap.find(shaderTypeLC);
-    ESP_CHECK(
-        mapIter != metadata::attributes::ShaderTypeNamesMap.end(),
-        "Illegal shader_type value"
-            << shaderTypeToUseString
-            << "specified in JSON to be used to set AssetInfo.shaderTypeToUse. "
-               "Aborting.");
-    x = mapIter->second;
-  }
-  return shaderTypeSucceess;
-}
+bool fromJsonValue(const JsonGenericValue& obj,
+                   metadata::attributes::ObjectInstanceShaderType& x);
 
 inline JsonGenericValue toJsonValue(
     const esp::assets::RenderAssetInstanceCreationInfo& x,

src/esp/sim/CMakeLists.txt

@@ -1,6 +1,11 @@
 add_library(
   sim STATIC
-  Simulator.cpp Simulator.h SimulatorConfiguration.cpp SimulatorConfiguration.h
+  ReplayBatchRenderer.cpp
+  ReplayBatchRenderer.h
+  Simulator.cpp
+  Simulator.h
+  SimulatorConfiguration.cpp
+  SimulatorConfiguration.h
 )
 
 target_link_libraries(