Add light serialization to the gfx-replay format.

src/esp/assets/ResourceManager.cpp

@@ -2975,6 +2975,20 @@ void ResourceManager::joinSemanticHierarchy(
   }
 }
 
+void ResourceManager::setLightSetup(gfx::LightSetup setup,
+                                    const Mn::ResourceKey& key) {
+  // add lights to recorder keyframe
+  if (gfxReplayRecorder_) {
+    gfxReplayRecorder_->clearLightsFromKeyframe();
+    for (const auto& light : setup) {
+      gfxReplayRecorder_->addLightToKeyframe(light);
+    }
+  }
+
+  shaderManager_.set(key, std::move(setup), Mn::ResourceDataState::Mutable,
+                     Mn::ResourcePolicy::Manual);
+}
+
 std::unique_ptr<MeshData> ResourceManager::createJoinedCollisionMesh(
     const std::string& filename) const {
   std::unique_ptr<MeshData> mesh = std::make_unique<MeshData>();

src/esp/assets/ResourceManager.h

@@ -427,10 +427,7 @@ class ResourceManager {
    */
   void setLightSetup(gfx::LightSetup setup,
                      const Mn::ResourceKey& key = Mn::ResourceKey{
-                         DEFAULT_LIGHTING_KEY}) {
-    shaderManager_.set(key, std::move(setup), Mn::ResourceDataState::Mutable,
-                       Mn::ResourcePolicy::Manual);
-  }
+                         DEFAULT_LIGHTING_KEY});
 
   /**
    * @brief Construct a unified @ref MeshData from a loaded asset's collision

src/esp/gfx/replay/Keyframe.h

@@ -55,6 +55,7 @@ struct Keyframe {
   std::vector<std::pair<RenderAssetInstanceKey, RenderAssetInstanceState>>
       stateUpdates;
   std::unordered_map<std::string, Transform> userTransforms;
+  std::vector<LightInfo> lights;
 
   ESP_SMART_POINTERS(Keyframe)
 };

src/esp/gfx/replay/Player.cpp

@@ -27,8 +27,10 @@ Keyframe Player::keyframeFromString(const std::string& keyframe) {
   return res;
 }
 
-Player::Player(const LoadAndCreateRenderAssetInstanceCallback& callback)
-    : loadAndCreateRenderAssetInstanceCallback(callback) {}
+Player::Player(const LoadAndCreateRenderAssetInstanceCallback& callback,
+               const SetLightsCallback& lightsCallback)
+    : loadAndCreateRenderAssetInstanceCallback(callback),
+      setLightsCallback(lightsCallback) {}
 
 void Player::readKeyframesFromFile(const std::string& filepath) {
   close();
@@ -166,6 +168,8 @@ void Player::applyKeyframe(const Keyframe& keyframe) {
     node->setRotation(state.absTransform.rotation);
     setSemanticIdForSubtree(node, state.semanticId);
   }
+
+  setLightsCallback(keyframe.lights);
 }
 
 void Player::appendKeyframe(Keyframe&& keyframe) {

src/esp/gfx/replay/Player.h

@@ -44,12 +44,14 @@ class Player {
       std::function<esp::scene::SceneNode*(
           const esp::assets::AssetInfo&,
           const esp::assets::RenderAssetInstanceCreationInfo&)>;
+  using SetLightsCallback = std::function<void(const gfx::LightSetup& lights)>;
 
   /**
    * @brief Construct a Player.
    * @param callback A function to load and create a render asset instance.
    */
-  explicit Player(const LoadAndCreateRenderAssetInstanceCallback& callback);
+  explicit Player(const LoadAndCreateRenderAssetInstanceCallback& callback,
+                  const SetLightsCallback& lightsCallback);
 
   ~Player();
 
@@ -126,6 +128,8 @@ class Player {
 
   LoadAndCreateRenderAssetInstanceCallback
       loadAndCreateRenderAssetInstanceCallback;
+  SetLightsCallback setLightsCallback;
+
   int frameIndex_ = -1;
   std::vector<Keyframe> keyframes_;
   std::map<std::string, esp::assets::AssetInfo> assetInfos_;

src/esp/gfx/replay/Recorder.cpp

@@ -96,6 +96,14 @@ void Recorder::addUserTransformToKeyframe(const std::string& name,
   getKeyframe().userTransforms[name] = Transform{translation, rotation};
 }
 
+void Recorder::addLightToKeyframe(const LightInfo& lightInfo) {
+  getKeyframe().lights.emplace_back(lightInfo);
+}
+
+void Recorder::clearLightsFromKeyframe() {
+  getKeyframe().lights.clear();
+}
+
 void Recorder::addLoadsCreationsDeletions(KeyframeIterator begin,
                                           KeyframeIterator end,
                                           Keyframe* dest) {
@@ -180,6 +188,8 @@ void Recorder::updateInstanceStates() {
 void Recorder::advanceKeyframe() {
   savedKeyframes_.emplace_back(std::move(currKeyframe_));
   currKeyframe_ = Keyframe{};
+  // TODO: record light deltas rather than absolute states
+  currKeyframe_.lights = savedKeyframes_.back().lights;
 }
 
 void Recorder::writeSavedKeyframesToFile(const std::string& filepath,

src/esp/gfx/replay/Recorder.h

@@ -90,6 +90,18 @@ class Recorder {
                                   const Magnum::Vector3& translation,
                                   const Magnum::Quaternion& rotation);
 
+  /**
+   * @brief Add a light to the current keyframe.
+   *
+   * @param lightInfo Parameters of the light to be added to the keyframe.
+   */
+  void addLightToKeyframe(const LightInfo& lightInfo);
+
+  /**
+   * @brief Delete all lights from the current keyframe.
+   */
+  void clearLightsFromKeyframe();
+
   /**
    * @brief write saved keyframes to file.
    * @param filepath

src/esp/gfx/replay/ReplayManager.cpp

@@ -10,7 +10,8 @@ namespace replay {
 
 std::shared_ptr<Player> ReplayManager::readKeyframesFromFile(
     const std::string& filepath) {
-  auto player = std::make_shared<Player>(playerCallback_);
+  auto player =
+      std::make_shared<Player>(playerCallback_, playerLightsCallback_);
   player->readKeyframesFromFile(filepath);
   if (player->getNumKeyframes() == 0) {
     ESP_ERROR(Mn::Debug::Flag::NoSpace)
@@ -21,7 +22,8 @@ std::shared_ptr<Player> ReplayManager::readKeyframesFromFile(
 }
 
 std::shared_ptr<Player> ReplayManager::createEmptyPlayer() {
-  auto player = std::make_shared<Player>(playerCallback_);
+  auto player =
+      std::make_shared<Player>(playerCallback_, playerLightsCallback_);
   return player;
 }
 

src/esp/gfx/replay/ReplayManager.h

@@ -41,6 +41,14 @@ class ReplayManager {
     playerCallback_ = callback;
   }
 
+  /**
+   * @brief Set a Player light creation callback; this is needed to construct
+   * Player instances.
+   */
+  void setPlayerLightsCallback(const Player::SetLightsCallback& callback) {
+    playerLightsCallback_ = callback;
+  }
+
   /**
    * @brief Read keyframes from a file and construct a Player. Returns nullptr
    * if no keyframes could be read.
@@ -56,6 +64,7 @@ class ReplayManager {
  private:
   std::shared_ptr<Recorder> recorder_;
   Player::LoadAndCreateRenderAssetInstanceCallback playerCallback_;
+  Player::SetLightsCallback playerLightsCallback_;
 
   ESP_SMART_POINTERS(ReplayManager)
 };

src/esp/io/JsonEspTypes.cpp

@@ -49,6 +49,16 @@ JsonGenericValue toJsonValue(const gfx::replay::Keyframe& keyframe,
     io::addMember(obj, "userTransforms", userTransformsArray, allocator);
   }
 
+  if (!keyframe.lights.empty()) {
+    JsonGenericValue lightsArray(rapidjson::kArrayType);
+    for (const auto& light : keyframe.lights) {
+      JsonGenericValue lightObj(rapidjson::kObjectType);
+      io::addMember(lightObj, "light", light, allocator);
+      lightsArray.PushBack(lightObj, allocator);
+    }
+    io::addMember(obj, "lights", lightsArray, allocator);
+  }
+
   return obj;
 }
 
@@ -98,6 +108,16 @@ bool fromJsonValue(const JsonGenericValue& obj,
     }
   }
 
+  itr = obj.FindMember("lights");
+  if (itr != obj.MemberEnd()) {
+    const JsonGenericValue& lightsArray = itr->value;
+    for (const auto& lightObj : lightsArray.GetArray()) {
+      gfx::LightInfo light;
+      io::readMember(lightObj, "light", light);
+      keyframe.lights.emplace_back(std::move(light));
+    }
+  }
+
   return true;
 }
 

src/esp/io/JsonEspTypes.h

@@ -169,6 +169,32 @@ inline bool fromJsonValue(const JsonGenericValue& obj,
   return true;
 }
 
+inline JsonGenericValue toJsonValue(const esp::gfx::LightInfo& x,
+                                    JsonAllocator& allocator) {
+  JsonGenericValue obj(rapidjson::kObjectType);
+  addMember(obj, "vector", x.vector, allocator);
+  addMember(obj, "color", x.color, allocator);
+  addMember(obj, "model", x.model, allocator);
+  return obj;
+}
+
+inline bool fromJsonValue(const JsonGenericValue& obj, esp::gfx::LightInfo& x) {
+  readMember(obj, "vector", x.vector);
+  readMember(obj, "color", x.color);
+  readMember(obj, "model", x.model);
+  return true;
+}
+
+inline JsonGenericValue toJsonValue(const esp::gfx::LightPositionModel& x,
+                                    JsonAllocator& allocator) {
+  return toJsonValue(static_cast<int>(x), allocator);
+}
+
+inline bool fromJsonValue(const JsonGenericValue& obj,
+                          esp::gfx::LightPositionModel& x) {
+  return fromJsonValue(obj, (int&)x);  // TODO: unsafe cast of external data.
+}
+
 JsonGenericValue toJsonValue(const esp::gfx::replay::Keyframe& x,
                              JsonAllocator& allocator);
 

src/esp/io/JsonMagnumTypes.h

@@ -65,6 +65,68 @@ inline bool fromJsonValue(const JsonGenericValue& obj, Magnum::Vector3& val) {
   return false;
 }
 
+inline JsonGenericValue toJsonValue(const Magnum::Vector4& vec,
+                                    JsonAllocator& allocator) {
+  return toJsonArrayHelper(vec.data(), 4, allocator);
+}
+
+/**
+ * @brief Specialization to handle Magnum::Vector4 values. Populate passed @p
+ * val with value. Returns whether successfully populated, or not. Logs an error
+ * if inappropriate type.
+ *
+ * @param obj json value to parse
+ * @param val destination value to be populated
+ * @return whether successful or not
+ */
+inline bool fromJsonValue(const JsonGenericValue& obj, Magnum::Vector4& val) {
+  if (obj.IsArray() && obj.Size() == 4) {
+    for (rapidjson::SizeType i = 0; i < 4; ++i) {
+      if (obj[i].IsNumber()) {
+        val[i] = obj[i].GetDouble();
+      } else {
+        ESP_ERROR() << "Invalid numeric value specified in JSON Vec4, index :"
+                    << i;
+        return false;
+      }
+    }
+    return true;
+  }
+  return false;
+}
+
+inline JsonGenericValue toJsonValue(const Magnum::Color3& color,
+                                    JsonAllocator& allocator) {
+  return toJsonArrayHelper(color.data(), 3, allocator);
+}
+
+/**
+ * @brief Specialization to handle Magnum::Color3 values. Populate passed @p
+ * val with value. Returns whether successfully populated, or not. Logs an error
+ * if inappropriate type.
+ *
+ * @param obj json value to parse
+ * @param val destination value to be populated
+ * @return whether successful or not
+ */
+inline bool fromJsonValue(const JsonGenericValue& obj, Magnum::Color3& val) {
+  if (obj.IsArray() && obj.Size() == 3) {
+    Magnum::Vector3 vec3;
+    for (rapidjson::SizeType i = 0; i < 3; ++i) {
+      if (obj[i].IsNumber()) {
+        vec3[i] = obj[i].GetDouble();
+      } else {
+        ESP_ERROR() << "Invalid numeric value specified in JSON Color3, index :"
+                    << i;
+        return false;
+      }
+    }
+    val = Magnum::Color3(vec3);
+    return true;
+  }
+  return false;
+}
+
 inline JsonGenericValue toJsonValue(const Magnum::Color4& color,
                                     JsonAllocator& allocator) {
   return toJsonArrayHelper(color.data(), 4, allocator);

src/esp/sim/Simulator.cpp

@@ -633,6 +633,11 @@ void Simulator::reconfigureReplayManager(bool enableGfxReplaySave) {
           -> scene::SceneNode* {
         return loadAndCreateRenderAssetInstance(assetInfo, creation);
       });
+  // provide Player light creation callback
+  gfxReplayMgr_->setPlayerLightsCallback(
+      [this](const gfx::LightSetup& lights) -> void {
+        this->setLightSetup(lights);
+      });
 }
 
 void Simulator::updateShadowMapDrawableGroup() {