Skip to content

Commit

Permalink
asset import optimization - compile multiple assets at the same time
Browse files Browse the repository at this point in the history
  • Loading branch information
@nem0
nem0 committed Sep 12, 2023
1 parent 9fc327c commit 9a76171
Show file tree
Hide file tree
Showing 2 changed files with 48 additions and 89 deletions.
108 changes: 33 additions & 75 deletions src/editor/asset_compiler.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,20 +45,6 @@ struct HashFunc<Path>
};


struct AssetCompilerTask : Thread
{
AssetCompilerTask(AssetCompilerImpl& compiler, IAllocator& allocator)
: Thread(allocator)
, m_compiler(compiler)
{}

int task() override;

AssetCompilerImpl& m_compiler;
volatile bool m_finished = false;
};


void AssetCompiler::IPlugin::addSubresources(AssetCompiler& compiler, const Path& path)
{
const ResourceType type = compiler.getResourceType(path);
Expand Down Expand Up @@ -86,10 +72,8 @@ struct AssetCompilerImpl : AssetCompiler {
, m_load_hook(*this)
, m_allocator(app.getAllocator(), "asset compiler")
, m_plugins(m_allocator)
, m_task(*this, m_allocator)
, m_to_compile(m_allocator)
, m_compiled(m_allocator)
, m_semaphore(0, 0x7fFFffFF)
, m_registered_extensions(m_allocator)
, m_resources(m_allocator)
, m_generations(m_allocator)
Expand All @@ -105,7 +89,6 @@ struct AssetCompilerImpl : AssetCompiler {
const char* base_path = fs.getBasePath();
m_watcher = FileSystemWatcher::create(base_path, m_allocator);
m_watcher->getCallback().bind<&AssetCompilerImpl::onFileChanged>(this);
m_task.create("Asset compiler", true);
Path path(base_path, ".lumix/resources");
if (!os::dirExists(path)) {
if (!os::makePath(path.c_str())) logError("Could not create ", path);
Expand Down Expand Up @@ -193,10 +176,6 @@ struct AssetCompilerImpl : AssetCompiler {
}

ASSERT(m_plugins.empty());
m_task.m_finished = true;
m_to_compile.emplace();
m_semaphore.signal();
m_task.destroy();
ResourceManagerHub& rm = m_app.getEngine().getResourceManager();
rm.setLoadHook(nullptr);
}
Expand Down Expand Up @@ -234,6 +213,7 @@ struct AssetCompilerImpl : AssetCompiler {

bool writeCompiledResource(const Path& path, Span<const u8> data) override {
PROFILE_FUNCTION();
jobs::enter(&m_lz4_mutex);
constexpr u32 COMPRESSION_SIZE_LIMIT = 4096;
OutputMemoryStream compressed(m_allocator);
i32 compressed_size = 0;
Expand All @@ -250,6 +230,7 @@ struct AssetCompilerImpl : AssetCompiler {
}
compressed.resize(compressed_size);
}
jobs::exit(&m_lz4_mutex);

FileSystem& fs = m_app.getEngine().getFileSystem();
const Path out_path(".lumix/resources/", path.getHash().getHashValue(), ".res");
Expand Down Expand Up @@ -583,7 +564,6 @@ struct AssetCompilerImpl : AssetCompiler {
}

void pushToCompileQueue(const Path& path) {
MutexGuard lock(m_to_compile_mutex);
auto iter = m_generations.find(path);
if (!iter.isValid()) {
iter = m_generations.insert(path, 0);
Expand All @@ -599,7 +579,6 @@ struct AssetCompilerImpl : AssetCompiler {
m_to_compile.push(job);
++m_compile_batch_count;
++m_batch_remaining_count;
m_semaphore.signal();
}

CompileJob popCompiledResource()
Expand Down Expand Up @@ -632,12 +611,7 @@ struct AssetCompilerImpl : AssetCompiler {
if (ImGui::Begin("Resource compilation", nullptr, flags)) {
ImGui::TextUnformatted("Compiling resources...");
ImGui::ProgressBar(((float)m_compile_batch_count - m_batch_remaining_count) / m_compile_batch_count);
Path path;
{
MutexGuard lock(m_to_compile_mutex);
path = m_res_in_progress;
}
ImGui::TextWrapped("%s", path.c_str());
ImGui::TextWrapped("%s", m_res_in_progress.c_str());
}
ImGui::End();
ImGui::PopStyleVar();
Expand All @@ -652,24 +626,44 @@ struct AssetCompilerImpl : AssetCompiler {
return nullptr;
}

void update() override
{
void runOneJob() {
if (m_to_compile.empty()) return;

AssetCompilerImpl::CompileJob p = m_to_compile.back();
m_to_compile.pop();
auto iter = m_generations.find(p.path);
const bool is_most_recent = p.generation == iter.value();
if (!is_most_recent) {
--m_batch_remaining_count;
return;
}

m_res_in_progress = p.path.c_str();

jobs::runLambda([p, this]() mutable {
PROFILE_BLOCK("compile asset");
profiler::pushString(p.path.c_str());
p.compiled = compile(p.path);
if (!p.compiled) logError("Failed to compile resource ", p.path);
MutexGuard lock(m_compiled_mutex);
m_compiled.push(p);
}, nullptr);
}

void update() override {
for(;;) {
runOneJob();
CompileJob job = popCompiledResource();
if (job.path.isEmpty()) break;

// this can take some time, mutex is probably not the best option

const u32 generation = [&](){
MutexGuard lock(m_to_compile_mutex);
return m_generations[job.path];
}();
const u32 generation = m_generations[job.path];
if (job.generation != generation) continue;

// this can take some time, mutex is probably not the best option
MutexGuard lock(m_compiled_mutex);
// reload/continue loading resource and its subresources
for (const ResourceItem& ri : m_resources) {
if (!endsWithInsensitive(ri.path, job.path)) continue;;
if (!endsWithInsensitive(ri.path, job.path)) continue;

Resource* r = getResource(ri.path);
if (r) {
Expand Down Expand Up @@ -797,8 +791,6 @@ struct AssetCompilerImpl : AssetCompiler {
}

TagAllocator m_allocator;
Semaphore m_semaphore;
Mutex m_to_compile_mutex;
Mutex m_compiled_mutex;
Mutex m_changed_mutex;
Mutex m_plugin_mutex;
Expand All @@ -812,7 +804,6 @@ struct AssetCompilerImpl : AssetCompiler {
StudioApp& m_app;
LoadHook m_load_hook;
HashMap<RuntimeHash, IPlugin*> m_plugins;
AssetCompilerTask m_task;
UniquePtr<FileSystemWatcher> m_watcher;
HashMap<FilePathHash, ResourceItem> m_resources;
HashMap<u32, ResourceType, HashFuncDirect<u32>> m_registered_extensions;
Expand All @@ -821,47 +812,14 @@ struct AssetCompilerImpl : AssetCompiler {
bool m_init_finished = false;
Array<Resource*> m_on_init_load;
u8* m_lz4_state = nullptr;
jobs::Mutex m_lz4_mutex;

u32 m_compile_batch_count = 0;
u32 m_batch_remaining_count = 0;
Path m_res_in_progress;
};


int AssetCompilerTask::task()
{
while (!m_finished) {
m_compiler.m_semaphore.wait();
AssetCompilerImpl::CompileJob p = [&]{
MutexGuard lock(m_compiler.m_to_compile_mutex);
AssetCompilerImpl::CompileJob p = m_compiler.m_to_compile.back();
if (p.path.isEmpty()) return p;

auto iter = m_compiler.m_generations.find(p.path);
const bool is_most_recent = p.generation == iter.value();
if (is_most_recent) {
m_compiler.m_res_in_progress = p.path.c_str();
}
else {
p.path = Path();
--m_compiler.m_batch_remaining_count;
}
m_compiler.m_to_compile.pop();
return p;
}();
if (!p.path.isEmpty()) {
PROFILE_BLOCK("compile asset");
profiler::pushString(p.path.c_str());
p.compiled = m_compiler.compile(p.path);
if (!p.compiled) logError("Failed to compile resource ", p.path);
MutexGuard lock(m_compiler.m_compiled_mutex);
m_compiler.m_compiled.push(p);
}
}
return 0;
}


UniquePtr<AssetCompiler> AssetCompiler::create(StudioApp& app) {
return UniquePtr<AssetCompilerImpl>::create(app.getAllocator(), app);
}
Expand Down
29 changes: 15 additions & 14 deletions src/renderer/editor/render_plugins.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2424,7 +2424,6 @@ struct ModelPlugin final : AssetBrowser::IPlugin, AssetCompiler::IPlugin {
explicit ModelPlugin(StudioApp& app)
: m_app(app)
, m_tile(app.getAllocator())
, m_fbx_importer(app)
{
app.getAssetCompiler().registerExtension("fbx", Model::TYPE);
}
Expand All @@ -2447,7 +2446,6 @@ struct ModelPlugin final : AssetBrowser::IPlugin, AssetCompiler::IPlugin {
void init() {
Engine& engine = m_app.getEngine();
m_renderer = static_cast<Renderer*>(engine.getSystemManager().getSystem("renderer"));
m_fbx_importer.init();
}

void addSubresources(AssetCompiler& compiler, const Path& _path) override {
Expand Down Expand Up @@ -2522,14 +2520,15 @@ struct ModelPlugin final : AssetBrowser::IPlugin, AssetCompiler::IPlugin {
cfg.anim_translation_error = meta.anim_translation_error;
cfg.skeleton = meta.skeleton;
cfg.root_motion_bone = BoneNameHash(meta.root_motion_bone.c_str());
if (!m_fbx_importer.setSource(filepath, false, meta.force_skin)) return false;
if (m_fbx_importer.getBoneCount() == 0 && m_fbx_importer.getMeshes().empty() && m_fbx_importer.getAnimations().empty()) {
if (m_fbx_importer.getOFBXScene()) {
if (m_fbx_importer.getOFBXScene()->getMeshCount() > 0) {
FBXImporter importer(m_app);
if (!importer.setSource(filepath, false, meta.force_skin)) return false;
if (importer.getBoneCount() == 0 && importer.getMeshes().empty() && importer.getAnimations().empty()) {
if (importer.getOFBXScene()) {
if (importer.getOFBXScene()->getMeshCount() > 0) {
logError("No meshes with materials found in ", src);
}
else {
if (m_fbx_importer.getBoneCount() == 0) {
if (importer.getBoneCount() == 0) {
logError("No meshes or animations found in ", src);
}
}
Expand All @@ -2539,16 +2538,16 @@ struct ModelPlugin final : AssetBrowser::IPlugin, AssetCompiler::IPlugin {
bool any_written = false;
if (meta.split) {
cfg.origin = FBXImporter::ImportConfig::Origin::CENTER;
any_written = m_fbx_importer.writeSubmodels(filepath, cfg) || any_written;
any_written = m_fbx_importer.writePrefab(filepath, cfg) || any_written;
any_written = importer.writeSubmodels(filepath, cfg) || any_written;
any_written = importer.writePrefab(filepath, cfg) || any_written;
}
cfg.origin = FBXImporter::ImportConfig::Origin::SOURCE;
any_written = m_fbx_importer.writeModel(src, cfg) || any_written;
any_written = m_fbx_importer.writeMaterials(filepath, cfg) || any_written;
any_written = importer.writeModel(src, cfg) || any_written;
any_written = importer.writeMaterials(filepath, cfg) || any_written;
if (!meta.ignore_animations) {
any_written = m_fbx_importer.writeAnimations(filepath, cfg) || any_written;
any_written = importer.writeAnimations(filepath, cfg) || any_written;
}
any_written = m_fbx_importer.writePhysics(filepath, cfg) || any_written;
any_written = importer.writePhysics(filepath, cfg) || any_written;
return any_written;
}

Expand Down Expand Up @@ -3124,7 +3123,6 @@ struct ModelPlugin final : AssetBrowser::IPlugin, AssetCompiler::IPlugin {
StudioApp& m_app;
Renderer* m_renderer = nullptr;
TexturePlugin* m_texture_plugin;
FBXImporter m_fbx_importer;
jobs::Signal m_subres_signal;
gpu::ProgramHandle m_downscale_program = gpu::INVALID_PROGRAM;
};
Expand Down Expand Up @@ -5076,6 +5074,7 @@ struct StudioAppPlugin : StudioApp::IPlugin
, m_terrain_plugin(app)
, m_instanced_model_plugin(app)
, m_model_plugin(app)
, m_fbx_importer(app)
, m_procedural_geom_plugin(app)
{}

Expand All @@ -5096,6 +5095,7 @@ struct StudioAppPlugin : StudioApp::IPlugin
void init() override
{
PROFILE_FUNCTION();
m_fbx_importer.init();
m_renderdoc_capture_action.init("Capture RenderDoc", "Capture with RenderDoc", "capture_renderdoc", "", Action::GLOBAL);
m_renderdoc_capture_action.func.bind<&StudioAppPlugin::captureRenderDoc>(this);

Expand Down Expand Up @@ -5379,6 +5379,7 @@ struct StudioAppPlugin : StudioApp::IPlugin
}

StudioApp& m_app;
FBXImporter m_fbx_importer; // only for preloading impostor shadow shader // TODO do this in a better way
Action m_renderdoc_capture_action;
UniquePtr<ParticleEditor> m_particle_editor;
EditorUIRenderPlugin m_editor_ui_render_plugin;
Expand Down

0 comments on commit 9a76171

Please sign in to comment.