Dynamically set object textures from existing UE material or texture PNG

AirLib/include/api/RpcLibClientBase.hpp

@@ -130,6 +130,8 @@ namespace airlib
         msr::airlib::Kinematics::State simGetGroundTruthKinematics(const std::string& vehicle_name = "") const;
         msr::airlib::Environment::State simGetGroundTruthEnvironment(const std::string& vehicle_name = "") const;
         std::vector<std::string> simSwapTextures(const std::string& tags, int tex_id = 0, int component_id = 0, int material_id = 0);
+        bool simSetObjectMaterial(const std::string& object_name, const std::string& material_name);
+        bool simSetObjectMaterialFromTexture(const std::string& object_name, const std::string& texture_path);
 
         // Recording APIs
         void startRecording();

AirLib/include/api/WorldSimApiBase.hpp

@@ -72,6 +72,8 @@ namespace airlib
         virtual bool runConsoleCommand(const std::string& command) = 0;
         virtual bool setObjectScale(const std::string& object_name, const Vector3r& scale) = 0;
         virtual std::unique_ptr<std::vector<std::string>> swapTextures(const std::string& tag, int tex_id = 0, int component_id = 0, int material_id = 0) = 0;
+        virtual bool setObjectMaterial(const std::string& object_name, const std::string& material_name) = 0;
+        virtual bool setObjectMaterialFromTexture(const std::string& object_name, const std::string& texture_path) = 0;
         virtual vector<MeshPositionVertexBuffersResponse> getMeshPositionVertexBuffers() const = 0;
 
         virtual bool createVoxelGrid(const Vector3r& position, const int& x_size, const int& y_size, const int& z_size, const float& res, const std::string& output_file) = 0;

AirLib/src/api/RpcLibClientBase.cpp

@@ -411,6 +411,16 @@ __pragma(warning(disable : 4239))
             return pimpl_->client.call("simSwapTextures", tags, tex_id, component_id, material_id).as<vector<string>>();
         }
 
+        bool RpcLibClientBase::simSetObjectMaterial(const std::string& object_name, const std::string& material_name)
+        {
+            return pimpl_->client.call("simSetObjectMaterial", object_name, material_name).as<bool>();
+        }
+
+        bool RpcLibClientBase::simSetObjectMaterialFromTexture(const std::string& object_name, const std::string& texture_path)
+        {
+            return pimpl_->client.call("simSetObjectMaterialFromTexture", object_name, texture_path).as<bool>();
+        }
+
         bool RpcLibClientBase::simLoadLevel(const string& level_name)
         {
             return pimpl_->client.call("simLoadLevel", level_name).as<bool>();

AirLib/src/api/RpcLibServerBase.cpp

@@ -397,6 +397,14 @@ namespace airlib
             return *getWorldSimApi()->swapTextures(tag, tex_id, component_id, material_id);
         });
 
+        pimpl_->server.bind("simSetObjectMaterial", [&](const std::string& object_name, const std::string& material_name) -> bool {
+            return getWorldSimApi()->setObjectMaterial(object_name, material_name);
+        });
+
+        pimpl_->server.bind("simSetObjectMaterialFromTexture", [&](const std::string& object_name, const std::string& texture_path) -> bool {
+            return getWorldSimApi()->setObjectMaterialFromTexture(object_name, texture_path);
+        });
+
         pimpl_->server.bind("startRecording", [&]() -> void {
             getWorldSimApi()->startRecording();
         });

PythonClient/airsim/client.py

@@ -177,6 +177,33 @@ def simSwapTextures(self, tags, tex_id = 0, component_id = 0, material_id = 0):
         """
         return self.client.call("simSwapTextures", tags, tex_id, component_id, material_id)
 
+    def simSetObjectMaterial(self, object_name, material_name):
+        """
+        Runtime Swap Texture API
+        See https://microsoft.github.io/AirSim/retexturing/ for details
+        Args:
+            object_name (str): name of object to set material for
+            material_name (str): name of material to set for object
+
+        Returns:
+            bool: True if material was set
+        """
+        return self.client.call("simSetObjectMaterial", object_name, material_name)
+
+    def simSetObjectMaterialFromTexture(self, object_name, texture_path):
+        """
+        Runtime Swap Texture API
+        See https://microsoft.github.io/AirSim/retexturing/ for details
+        Args:
+            object_name (str): name of object to set material for
+            texture_path (str): path to texture to set for object
+
+        Returns:
+            bool: True if material was set
+        """
+        return self.client.call("simSetObjectMaterialFromTexture", object_name, texture_path)
+
+
     # time-of-day control
     def simSetTimeOfDay(self, is_enabled, start_datetime = "", is_start_datetime_dst = False, celestial_clock_speed = 1, update_interval_secs = 60, move_sun = True):
         """

PythonClient/environment/change_texture_example.py

@@ -0,0 +1,7 @@
+import airsim
+
+c = airsim.MultirotorClient()
+c.confirmConnection()
+
+c.simSetObjectMaterialFromTexture("OrangeBall", "sample_texture.jpg")
+

Unity/AirLibWrapper/AirsimWrapper/Source/WorldSimApi.cpp

@@ -67,6 +67,22 @@ std::unique_ptr<std::vector<std::string>> WorldSimApi::swapTextures(const std::s
     return result;
 }
 
+bool WorldSimApi::setObjectMaterialFromTexture(const std::string& object_name, const std::string& texture_path)
+{
+    throw std::invalid_argument(common_utils::Utils::stringf(
+                                    "setObjectMaterialFromTexture is not supported on unity")
+                                    .c_str());
+    return false;
+}
+
+bool WorldSimApi::setObjectMaterial(const std::string& object_name, const std::string& material_name)
+{
+    throw std::invalid_argument(common_utils::Utils::stringf(
+                                    "setObjectMaterial is not supported on unity")
+                                    .c_str());
+    return false;
+}
+
 std::vector<std::string> WorldSimApi::listSceneObjects(const std::string& name_regex) const
 {
     std::vector<std::string> result;

Unity/AirLibWrapper/AirsimWrapper/Source/WorldSimApi.h

@@ -38,6 +38,8 @@ class WorldSimApi : public msr::airlib::WorldSimApiBase
                                  const std::string& message_param = "", unsigned char severity = 0) override;
 
     virtual std::unique_ptr<std::vector<std::string>> swapTextures(const std::string& tag, int tex_id = 0, int component_id = 0, int material_id = 0) override;
+    virtual bool setObjectMaterial(const std::string& object_name, const std::string& material_name) override;
+    virtual bool setObjectMaterialFromTexture(const std::string& object_name, const std::string& texture_path) override;
     virtual std::vector<std::string> listSceneObjects(const std::string& name_regex) const override;
     virtual Pose getObjectPose(const std::string& object_name) const override;
 

Unreal/Plugins/AirSim/Source/SimMode/SimModeBase.cpp

@@ -64,6 +64,10 @@ ASimModeBase::ASimModeBase()
     }
     else
         loading_screen_widget_ = nullptr;
+    static ConstructorHelpers::FObjectFinder<UMaterial> domain_rand_mat_finder(TEXT("Material'/AirSim/HUDAssets/DomainRandomizationMaterial.DomainRandomizationMaterial'"));
+    if (domain_rand_mat_finder.Succeeded()) {
+        domain_rand_material_ = domain_rand_mat_finder.Object;
+    }
 }
 
 void ASimModeBase::toggleLoadingScreen(bool is_visible)

Unreal/Plugins/AirSim/Source/SimMode/SimModeBase.h

@@ -120,6 +120,7 @@ class AIRSIM_API ASimModeBase : public AActor
 
     TMap<FString, FAssetData> asset_map;
     TMap<FString, AActor*> scene_object_map;
+    UMaterial* domain_rand_material_;
 
 protected: //must overrides
     typedef msr::airlib::AirSimSettings AirSimSettings;

Unreal/Plugins/AirSim/Source/WorldSimApi.cpp

@@ -7,6 +7,7 @@
 #include "DrawDebugHelpers.h"
 #include "Runtime/Engine/Classes/Components/LineBatchComponent.h"
 #include "Runtime/Engine/Classes/Engine/Engine.h"
+#include "ImageUtils.h"
 #include <cstdlib>
 #include <ctime>
 
@@ -431,6 +432,74 @@ std::unique_ptr<std::vector<std::string>> WorldSimApi::swapTextures(const std::s
     return swappedObjectNames;
 }
 
+bool WorldSimApi::setObjectMaterialFromTexture(const std::string& object_name, const std::string& texture_path)
+{
+    bool success = false;
+    UAirBlueprintLib::RunCommandOnGameThread([this, &object_name, &texture_path, &success]() {
+        if (!IsValid(simmode_->domain_rand_material_)) {
+            UAirBlueprintLib::LogMessageString("Cannot find material for domain randomization",
+                                               "",
+                                               LogDebugLevel::Failure);
+        }
+        else {
+            UTexture2D* texture_desired = FImageUtils::ImportFileAsTexture2D(FString(texture_path.c_str()));
+            AActor* actor = UAirBlueprintLib::FindActor<AActor>(simmode_, FString(object_name.c_str()));
+
+            if (IsValid(actor)) {
+                TArray<UStaticMeshComponent*> components;
+                actor->GetComponents<UStaticMeshComponent>(components);
+                for (UStaticMeshComponent* staticMeshComponent : components) {
+                    UMaterialInstanceDynamic* dynamic_material = UMaterialInstanceDynamic::Create(simmode_->domain_rand_material_, staticMeshComponent);
+                    dynamic_material->SetTextureParameterValue("TextureParameter", texture_desired);
+                    staticMeshComponent->SetMaterial(0, dynamic_material);
+                }
+                success = true;
+            }
+            else {
+                UAirBlueprintLib::LogMessageString("Cannot find specified actor for domain randomization",
+                                                   "",
+                                                   LogDebugLevel::Failure);
+            }
+        }
+    },
+                                             true);
+
+    return success;
+}
+
+bool WorldSimApi::setObjectMaterial(const std::string& object_name, const std::string& material_name)
+{
+    bool success = false;
+    UAirBlueprintLib::RunCommandOnGameThread([this, &object_name, &material_name, &success]() {
+        AActor* actor = UAirBlueprintLib::FindActor<AActor>(simmode_, FString(object_name.c_str()));
+        UMaterial* material = static_cast<UMaterial*>(StaticLoadObject(UMaterial::StaticClass(), nullptr, *FString(material_name.c_str())));
+
+        if (!IsValid(material)) {
+            UAirBlueprintLib::LogMessageString("Cannot find specified material for domain randomization",
+                                               "",
+                                               LogDebugLevel::Failure);
+        }
+        else {
+            if (IsValid(actor)) {
+                TArray<UStaticMeshComponent*> components;
+                actor->GetComponents<UStaticMeshComponent>(components);
+                for (UStaticMeshComponent* staticMeshComponent : components) {
+                    staticMeshComponent->SetMaterial(0, material);
+                }
+                success = true;
+            }
+            else {
+                UAirBlueprintLib::LogMessageString("Cannot find specified actor for domain randomization",
+                                                   "",
+                                                   LogDebugLevel::Failure);
+            }
+        }
+    },
+                                             true);
+
+    return success;
+}
+
 //----------- Plotting APIs ----------/
 void WorldSimApi::simFlushPersistentMarkers()
 {

Unreal/Plugins/AirSim/Source/WorldSimApi.h

@@ -48,6 +48,8 @@ class WorldSimApi : public msr::airlib::WorldSimApiBase
                                  const std::string& message_param = "", unsigned char severity = 0) override;
 
     virtual std::unique_ptr<std::vector<std::string>> swapTextures(const std::string& tag, int tex_id = 0, int component_id = 0, int material_id = 0) override;
+    virtual bool setObjectMaterial(const std::string& object_name, const std::string& material_name) override;
+    virtual bool setObjectMaterialFromTexture(const std::string& object_name, const std::string& texture_path) override;
     virtual std::vector<std::string> listSceneObjects(const std::string& name_regex) const override;
     virtual Pose getObjectPose(const std::string& object_name) const override;
     virtual bool setObjectPose(const std::string& object_name, const Pose& pose, bool teleport) override;