From c9edce72d3bc73b566ba1c10761ae0333411248c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20=C4=90=E1=BB=A9c=20Long?= Date: Sat, 19 Jun 2021 09:57:35 +0700 Subject: [PATCH] Implement mesh skinning --- Cargo.toml | 11 + assets/models/SimpleSkin/SimpleSkin.gltf | 1 + crates/bevy_animation_rig/Cargo.toml | 24 ++ crates/bevy_animation_rig/src/lib.rs | 39 ++ crates/bevy_animation_rig/src/skinned_mesh.rs | 369 ++++++++++++++++++ .../bevy_animation_rig/src/skinned_mesh.vert | 44 +++ crates/bevy_gltf/Cargo.toml | 1 + crates/bevy_gltf/src/loader.rs | 104 ++++- crates/bevy_internal/Cargo.toml | 1 + crates/bevy_internal/src/default_plugins.rs | 3 + crates/bevy_internal/src/lib.rs | 6 + crates/bevy_internal/src/prelude.rs | 4 + docs/cargo_features.md | 1 + examples/README.md | 8 + examples/animation/custom_skinned_mesh.rs | 161 ++++++++ examples/animation/gltf_skinned_mesh.rs | 72 ++++ 16 files changed, 843 insertions(+), 6 deletions(-) create mode 100644 assets/models/SimpleSkin/SimpleSkin.gltf create mode 100644 crates/bevy_animation_rig/Cargo.toml create mode 100644 crates/bevy_animation_rig/src/lib.rs create mode 100644 crates/bevy_animation_rig/src/skinned_mesh.rs create mode 100644 crates/bevy_animation_rig/src/skinned_mesh.vert create mode 100644 examples/animation/custom_skinned_mesh.rs create mode 100644 examples/animation/gltf_skinned_mesh.rs diff --git a/Cargo.toml b/Cargo.toml index 9e8551ffc3dafc..bc75993980525a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ members = ["crates/*", "examples/ios", "tools/ci"] [features] default = [ + "bevy_animation_rig", "bevy_audio", "bevy_dynamic_plugin", "bevy_gilrs", @@ -41,6 +42,7 @@ dynamic = ["bevy_dylib"] render = ["bevy_internal/bevy_pbr", "bevy_internal/bevy_render", "bevy_internal/bevy_sprite", "bevy_internal/bevy_text", "bevy_internal/bevy_ui"] # Optional bevy crates +bevy_animation_rig = ["bevy_internal/bevy_animation_rig"] bevy_audio = ["bevy_internal/bevy_audio"] bevy_dynamic_plugin = ["bevy_internal/bevy_dynamic_plugin"] bevy_gilrs = ["bevy_internal/bevy_gilrs"] @@ -176,6 +178,15 @@ path = "examples/3d/wireframe.rs" name = "z_sort_debug" path = "examples/3d/z_sort_debug.rs" +# Animation +[[example]] +name = "custom_skinned_mesh" +path = "examples/animation/custom_skinned_mesh.rs" + +[[example]] +name = "gltf_skinned_mesh" +path = "examples/animation/gltf_skinned_mesh.rs" + # Application [[example]] name = "custom_loop" diff --git a/assets/models/SimpleSkin/SimpleSkin.gltf b/assets/models/SimpleSkin/SimpleSkin.gltf new file mode 100644 index 00000000000000..6e68616c727629 --- /dev/null +++ b/assets/models/SimpleSkin/SimpleSkin.gltf @@ -0,0 +1 @@ +{"scenes":[{"nodes":[0]}],"nodes":[{"skin":0,"mesh":0,"children":[1]},{"children":[2],"translation":[0,1,0]},{"rotation":[0,0,0,1]}],"meshes":[{"primitives":[{"attributes":{"POSITION":1,"JOINTS_0":2,"WEIGHTS_0":3},"indices":0}]}],"skins":[{"inverseBindMatrices":4,"joints":[1,2]}],"animations":[{"channels":[{"sampler":0,"target":{"node":2,"path":"rotation"}}],"samplers":[{"input":5,"interpolation":"LINEAR","output":6}]}],"buffers":[{"uri":"data:application/gltf-buffer;base64,AAABAAMAAAADAAIAAgADAAUAAgAFAAQABAAFAAcABAAHAAYABgAHAAkABgAJAAgAAAAAAAAAAAAAAAAAAACAPwAAAAAAAAAAAAAAAAAAAD8AAAAAAACAPwAAAD8AAAAAAAAAAAAAgD8AAAAAAACAPwAAgD8AAAAAAAAAAAAAwD8AAAAAAACAPwAAwD8AAAAAAAAAAAAAAEAAAAAAAACAPwAAAEAAAAAA","byteLength":168},{"uri":"data:application/gltf-buffer;base64,AAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAgD8AAAAAAAAAAAAAAAAAAIA/AAAAAAAAAAAAAAAAAABAPwAAgD4AAAAAAAAAAAAAQD8AAIA+AAAAAAAAAAAAAAA/AAAAPwAAAAAAAAAAAAAAPwAAAD8AAAAAAAAAAAAAgD4AAEA/AAAAAAAAAAAAAIA+AABAPwAAAAAAAAAAAAAAAAAAgD8AAAAAAAAAAAAAAAAAAIA/AAAAAAAAAAA=","byteLength":320},{"uri":"data:application/gltf-buffer;base64,AACAPwAAAAAAAAAAAAAAAAAAAAAAAIA/AAAAAAAAAAAAAAAAAAAAAAAAgD8AAAAAAAAAvwAAgL8AAAAAAACAPwAAgD8AAAAAAAAAAAAAAAAAAAAAAACAPwAAAAAAAAAAAAAAAAAAAAAAAIA/AAAAAAAAAL8AAIC/AAAAAAAAgD8=","byteLength":128},{"uri":"data:application/gltf-buffer;base64,AAAAAAAAAD8AAIA/AADAPwAAAEAAACBAAABAQAAAYEAAAIBAAACQQAAAoEAAALBAAAAAAAAAAAAAAAAAAACAPwAAAAAAAAAAkxjEPkSLbD8AAAAAAAAAAPT9ND/0/TQ/AAAAAAAAAAD0/TQ/9P00PwAAAAAAAAAAkxjEPkSLbD8AAAAAAAAAAAAAAAAAAIA/AAAAAAAAAAAAAAAAAACAPwAAAAAAAAAAkxjEvkSLbD8AAAAAAAAAAPT9NL/0/TQ/AAAAAAAAAAD0/TS/9P00PwAAAAAAAAAAkxjEvkSLbD8AAAAAAAAAAAAAAAAAAIA/","byteLength":240}],"bufferViews":[{"buffer":0,"byteOffset":0,"byteLength":48,"target":34963},{"buffer":0,"byteOffset":48,"byteLength":120,"target":34962},{"buffer":1,"byteOffset":0,"byteLength":320,"byteStride":16},{"buffer":2,"byteOffset":0,"byteLength":128},{"buffer":3,"byteOffset":0,"byteLength":240}],"accessors":[{"bufferView":0,"byteOffset":0,"componentType":5123,"count":24,"type":"SCALAR","max":[9],"min":[0]},{"bufferView":1,"byteOffset":0,"componentType":5126,"count":10,"type":"VEC3","max":[1,2,0],"min":[0,0,0]},{"bufferView":2,"byteOffset":0,"componentType":5123,"count":10,"type":"VEC4","max":[0,1,0,0],"min":[0,1,0,0]},{"bufferView":2,"byteOffset":160,"componentType":5126,"count":10,"type":"VEC4","max":[1,1,0,0],"min":[0,0,0,0]},{"bufferView":3,"byteOffset":0,"componentType":5126,"count":2,"type":"MAT4","max":[1,0,0,0,0,1,0,0,0,0,1,0,-0.5,-1,0,1],"min":[1,0,0,0,0,1,0,0,0,0,1,0,-0.5,-1,0,1]},{"bufferView":4,"byteOffset":0,"componentType":5126,"count":12,"type":"SCALAR","max":[5.5],"min":[0]},{"bufferView":4,"byteOffset":48,"componentType":5126,"count":12,"type":"VEC4","max":[0,0,0.707,1],"min":[0,0,-0.707,0.707]}],"asset":{"version":"2.0"}} \ No newline at end of file diff --git a/crates/bevy_animation_rig/Cargo.toml b/crates/bevy_animation_rig/Cargo.toml new file mode 100644 index 00000000000000..fd543b25232ad8 --- /dev/null +++ b/crates/bevy_animation_rig/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "bevy_animation_rig" +version = "0.5.0" +edition = "2018" +authors = [ + "Bevy Contributors ", + "Carter Anderson ", +] +description = "Bevy Engine Animation Rigging System" +homepage = "https://bevyengine.org" +repository = "https://github.com/bevyengine/bevy" +license = "MIT" +keywords = ["bevy", "animation", "rig", "skeleton"] + +[dependencies] +# bevy +bevy_app = { path = "../bevy_app", version = "0.5.0" } +bevy_asset = { path = "../bevy_asset", version = "0.5.0" } +bevy_ecs = { path = "../bevy_ecs", version = "0.5.0" } +bevy_math = { path = "../bevy_math", version = "0.5.0" } +bevy_pbr = { path = "../bevy_pbr", version = "0.5.0" } +bevy_reflect = { path = "../bevy_reflect", version = "0.5.0", features = ["bevy"] } +bevy_render = { path = "../bevy_render", version = "0.5.0" } +bevy_transform = { path = "../bevy_transform", version = "0.5.0" } diff --git a/crates/bevy_animation_rig/src/lib.rs b/crates/bevy_animation_rig/src/lib.rs new file mode 100644 index 00000000000000..a4d4c9e576b5df --- /dev/null +++ b/crates/bevy_animation_rig/src/lib.rs @@ -0,0 +1,39 @@ +use bevy_app::{AppBuilder, CoreStage, Plugin, StartupStage}; +use bevy_asset::AddAsset; +use bevy_ecs::{ + schedule::{ParallelSystemDescriptorCoercion, SystemLabel}, + system::IntoSystem, +}; +use bevy_transform::TransformSystem; + +mod skinned_mesh; +pub use skinned_mesh::*; + +#[derive(Debug, Hash, PartialEq, Eq, Clone, SystemLabel)] +pub enum AnimationRigSystem { + SkinnedMeshSetup, + SkinnedMeshUpdate, +} + +#[derive(Default)] +pub struct AnimationRigPlugin; + +impl Plugin for AnimationRigPlugin { + fn build(&self, app: &mut AppBuilder) { + app.register_type::() + .add_asset::() + .add_startup_system_to_stage( + StartupStage::PreStartup, + skinned_mesh_setup + .system() + .label(AnimationRigSystem::SkinnedMeshSetup), + ) + .add_system_to_stage( + CoreStage::PostUpdate, + skinned_mesh_update + .system() + .label(AnimationRigSystem::SkinnedMeshUpdate) + .after(TransformSystem::TransformPropagate), + ); + } +} diff --git a/crates/bevy_animation_rig/src/skinned_mesh.rs b/crates/bevy_animation_rig/src/skinned_mesh.rs new file mode 100644 index 00000000000000..eba28914d1459a --- /dev/null +++ b/crates/bevy_animation_rig/src/skinned_mesh.rs @@ -0,0 +1,369 @@ +use bevy_asset::{Assets, Handle, HandleUntyped}; +use bevy_ecs::{ + entity::{Entity, EntityMap, MapEntities, MapEntitiesError}, + reflect::{ReflectComponent, ReflectMapEntities}, + system::{Query, Res, ResMut}, +}; +use bevy_math::Mat4; +use bevy_pbr::render_graph; +use bevy_reflect::{ + serde, DynamicStruct, FieldIter, Reflect, ReflectMut, ReflectRef, Struct, TypeUuid, +}; +use bevy_render::{ + pipeline::PipelineDescriptor, + render_graph::{RenderGraph, RenderResourcesNode}, + renderer::{ + RenderResource, RenderResourceHints, RenderResourceIterator, RenderResourceType, + RenderResources, + }, + shader::{Shader, ShaderStage}, + texture::Texture, +}; +use bevy_transform::components::GlobalTransform; + +/// The name of skinned mesh node +pub mod node { + pub const SKINNED_MESH: &str = "skinned_mesh"; +} + +/// The name of skinned mesh buffer +pub mod buffer { + pub const JOINT_TRANSFORMS: &str = "JointTransforms"; +} + +/// Specify RenderPipelines with this handle to render the skinned mesh. +pub const SKINNED_MESH_PIPELINE_HANDLE: HandleUntyped = + HandleUntyped::weak_from_u64(PipelineDescriptor::TYPE_UUID, 0x14db1922328e7fcc); + +/// Used to update and bind joint transforms to the skinned mesh render pipeline specified with [`SKINNED_MESH_PIPELINE_HANDLE`]. +/// +/// The length of entities vector passed to [`SkinnedMesh::new()`] should equal to the number of matrices inside [`SkinnedMeshInverseBindposes`]. +/// +/// The content of `joint_transforms` can be modified manually if [`skinned_mesh_update`] system is disabled. +/// +/// # Example +/// ``` +/// use bevy_animation_rig::{SkinnedMesh, SKINNED_MESH_PIPELINE_HANDLE}; +/// use bevy_ecs::{entity::Entity, system::Commands}; +/// use bevy_pbr::prelude::PbrBundle; +/// use bevy_render::pipeline::{RenderPipeline, RenderPipelines}; +/// +/// fn example_system(mut commands: Commands) { +/// commands.spawn_bundle(PbrBundle { +/// render_pipelines: RenderPipelines::from_pipelines( +/// vec![RenderPipeline::new(SKINNED_MESH_PIPELINE_HANDLE.typed())] +/// ), +/// ..Default::default() +/// }).insert(SkinnedMesh::new( +/// // Refer to [`SkinnedMeshInverseBindposes`] example on how to create inverse bindposes data. +/// Default::default(), +/// // Specify joint entities here. +/// vec![Entity::new(0)] +/// )); +/// } +/// ``` +#[derive(Debug, Default, Clone, Reflect)] +#[reflect(Component, MapEntities)] +pub struct SkinnedMesh { + pub inverse_bindposes: Handle, + pub joints: Vec, +} + +impl SkinnedMesh { + pub fn new( + inverse_bindposes: Handle, + joint_entities: Vec, + ) -> Self { + Self { + inverse_bindposes, + joints: joint_entities + .iter() + .map(|&entity| SkinnedMeshJoint { + entity, + transform: Mat4::IDENTITY, + }) + .collect(), + } + } + + pub fn update_joint_transforms( + &mut self, + inverse_bindposes_assets: &Res>, + global_transform_query: &Query<&GlobalTransform>, + ) { + let inverse_bindposes = inverse_bindposes_assets + .get(self.inverse_bindposes.clone()) + .unwrap(); + + for (joint, &inverse_bindpose) in self.joints.iter_mut().zip(inverse_bindposes.0.iter()) { + let global_transform = global_transform_query.get(joint.entity).unwrap(); + joint.transform = global_transform.compute_matrix() * inverse_bindpose; + } + } +} + +impl MapEntities for SkinnedMesh { + fn map_entities(&mut self, entity_map: &EntityMap) -> Result<(), MapEntitiesError> { + for joint in &mut self.joints { + joint.entity = entity_map.get(joint.entity)?; + } + + Ok(()) + } +} + +impl RenderResource for SkinnedMesh { + fn resource_type(&self) -> Option { + Some(RenderResourceType::Buffer) + } + + fn write_buffer_bytes(&self, buffer: &mut [u8]) { + let transform_size = std::mem::size_of::<[f32; 16]>(); + + for (index, joint) in self.joints.iter().enumerate() { + joint.transform.write_buffer_bytes( + &mut buffer[index * transform_size..(index + 1) * transform_size], + ); + } + } + + fn buffer_byte_len(&self) -> Option { + Some(self.joints.len() * std::mem::size_of::<[f32; 16]>()) + } + + fn texture(&self) -> Option<&Handle> { + None + } +} + +impl RenderResources for SkinnedMesh { + fn render_resources_len(&self) -> usize { + 1 + } + + fn get_render_resource(&self, index: usize) -> Option<&dyn RenderResource> { + (index == 0).then(|| self as &dyn RenderResource) + } + + fn get_render_resource_name(&self, index: usize) -> Option<&str> { + (index == 0).then(|| buffer::JOINT_TRANSFORMS) + } + + // Used to tell GLSL to use storage buffer instead of uniform buffer + fn get_render_resource_hints(&self, index: usize) -> Option { + (index == 0).then(|| RenderResourceHints::BUFFER) + } + + fn iter(&self) -> RenderResourceIterator { + RenderResourceIterator::new(self) + } +} + +/// Store data for each joint belongs to the [`SkinnedMesh`] +#[derive(Debug, Clone)] +pub struct SkinnedMeshJoint { + pub entity: Entity, + pub transform: Mat4, +} + +/// Manually implement [`bevy_reflect::Reflect`] for [`SkinnedMeshJoint`] to work around an issue, +/// where spawning a scene with a component containings a vector of structs would result in runtime panic. +unsafe impl Reflect for SkinnedMeshJoint { + #[inline] + fn type_name(&self) -> &str { + std::any::type_name::() + } + + #[inline] + fn any(&self) -> &dyn std::any::Any { + self + } + + #[inline] + fn any_mut(&mut self) -> &mut dyn std::any::Any { + self + } + + /// Workaround + #[inline] + fn clone_value(&self) -> Box { + Box::new(self.clone()) + } + + #[inline] + fn set(&mut self, value: Box) -> Result<(), Box> { + *self = value.take()?; + Ok(()) + } + + #[inline] + fn apply(&mut self, value: &dyn Reflect) { + if let ReflectRef::Struct(struct_value) = value.reflect_ref() { + for (i, value) in struct_value.iter_fields().enumerate() { + let name = struct_value.name_at(i).unwrap(); + self.field_mut(name).map(|v| v.apply(value)); + } + } else { + panic!("Attempted to apply non-struct type to struct type."); + } + } + + fn reflect_ref(&self) -> ReflectRef { + ReflectRef::Struct(self) + } + + fn reflect_mut(&mut self) -> ReflectMut { + ReflectMut::Struct(self) + } + + fn serializable(&self) -> Option { + None + } + + fn reflect_hash(&self) -> Option { + None + } + + fn reflect_partial_eq(&self, _value: &dyn Reflect) -> Option { + None + } +} + +/// Manually implement [`bevy_reflect::Struct`] for [`SkinnedMeshJoint`] because it is required by [`bevy_reflect::Reflect`] trait. +impl Struct for SkinnedMeshJoint { + fn field(&self, name: &str) -> Option<&dyn Reflect> { + match name { + "entity" => Some(&self.entity), + "transform" => Some(&self.transform), + _ => None, + } + } + + fn field_mut(&mut self, name: &str) -> Option<&mut dyn Reflect> { + match name { + "entity" => Some(&mut self.entity), + "transform" => Some(&mut self.transform), + _ => None, + } + } + + fn field_at(&self, index: usize) -> Option<&dyn Reflect> { + match index { + 0 => Some(&self.entity), + 1 => Some(&self.transform), + _ => None, + } + } + + fn field_at_mut(&mut self, index: usize) -> Option<&mut dyn Reflect> { + match index { + 0 => Some(&mut self.entity), + 1 => Some(&mut self.transform), + _ => None, + } + } + + fn name_at(&self, index: usize) -> Option<&str> { + match index { + 0 => Some("entity"), + 1 => Some("transform"), + _ => None, + } + } + + fn field_len(&self) -> usize { + 2 + } + + fn iter_fields(&self) -> FieldIter { + FieldIter::new(self) + } + + fn clone_dynamic(&self) -> DynamicStruct { + let mut dynamic = DynamicStruct::default(); + dynamic.set_name(self.type_name().to_string()); + dynamic.insert_boxed("entity", self.entity.clone_value()); + dynamic.insert_boxed("transform", self.transform.clone_value()); + dynamic + } +} + +/// Store joint inverse bindpose matrices. It can be shared between SkinnedMesh instances using assets. +/// +/// The matrices can be loaded automatically from glTF or can be defined manually. +/// +/// # Example +/// ``` +/// use bevy_asset::Assets; +/// use bevy_animation_rig::{SkinnedMesh, SkinnedMeshInverseBindposes, SKINNED_MESH_PIPELINE_HANDLE}; +/// use bevy_ecs::{entity::Entity, system::{Commands, ResMut}}; +/// use bevy_math::Mat4; +/// use bevy_pbr::prelude::PbrBundle; +/// use bevy_render::pipeline::{RenderPipeline, RenderPipelines}; +/// +/// fn example_system(mut commands: Commands, mut skinned_mesh_inverse_bindposes_assets: ResMut>) { +/// // A skeleton with only 2 joints +/// let skinned_mesh_inverse_bindposes = skinned_mesh_inverse_bindposes_assets.add(SkinnedMeshInverseBindposes(vec![ +/// Mat4::IDENTITY, +/// Mat4::IDENTITY, +/// ])); +/// +/// // The inverse bindposes then can be shared between multiple skinned mesh instances +/// for _ in 0..3 { +/// commands.spawn_bundle(PbrBundle { +/// render_pipelines: RenderPipelines::from_pipelines( +/// vec![RenderPipeline::new(SKINNED_MESH_PIPELINE_HANDLE.typed())] +/// ), +/// ..Default::default() +/// }).insert(SkinnedMesh::new( +/// skinned_mesh_inverse_bindposes.clone(), +/// // Remember to assign joint entity here! +/// vec![Entity::new(0); 2], +/// )); +/// } +/// } +/// ``` +#[derive(Debug, TypeUuid)] +#[uuid = "b9f155a9-54ec-4026-988f-e0a03e99a76f"] +pub struct SkinnedMeshInverseBindposes(pub Vec); + +pub fn skinned_mesh_setup( + mut pipelines: ResMut>, + mut shaders: ResMut>, + mut render_graph: ResMut, +) { + let mut skinned_mesh_pipeline = pipelines + .get(render_graph::PBR_PIPELINE_HANDLE) + .unwrap() + .clone(); + skinned_mesh_pipeline.name = Some("Skinned Mesh Pipeline".into()); + skinned_mesh_pipeline.shader_stages.vertex = shaders.add(Shader::from_glsl( + ShaderStage::Vertex, + include_str!("skinned_mesh.vert"), + )); + pipelines.set_untracked(SKINNED_MESH_PIPELINE_HANDLE, skinned_mesh_pipeline); + + render_graph.add_system_node( + node::SKINNED_MESH, + RenderResourcesNode::::new(false), + ); + render_graph + .add_node_edge( + node::SKINNED_MESH, + bevy_render::render_graph::base::node::MAIN_PASS, + ) + .unwrap(); +} + +pub fn skinned_mesh_update( + skinned_mesh_inverse_bindposes_assets: Res>, + global_transform_query: Query<&GlobalTransform>, + mut skinned_mesh_query: Query<&mut SkinnedMesh>, +) { + skinned_mesh_query.for_each_mut(|mut skinned_mesh| { + skinned_mesh.update_joint_transforms( + &skinned_mesh_inverse_bindposes_assets, + &global_transform_query, + ); + }); +} diff --git a/crates/bevy_animation_rig/src/skinned_mesh.vert b/crates/bevy_animation_rig/src/skinned_mesh.vert new file mode 100644 index 00000000000000..a899804396a038 --- /dev/null +++ b/crates/bevy_animation_rig/src/skinned_mesh.vert @@ -0,0 +1,44 @@ +#version 450 + +layout(location = 0) in vec3 Vertex_Position; +layout(location = 1) in vec3 Vertex_Normal; +layout(location = 2) in vec2 Vertex_Uv; +layout(location = 3) in vec4 Vertex_JointWeight; +layout(location = 4) in uvec4 Vertex_JointIndex; + +#ifdef STANDARDMATERIAL_NORMAL_MAP +layout(location = 5) in vec4 Vertex_Tangent; +#endif + +layout(location = 0) out vec3 v_WorldPosition; +layout(location = 1) out vec3 v_WorldNormal; +layout(location = 2) out vec2 v_Uv; + +layout(set = 0, binding = 0) uniform CameraViewProj { + mat4 ViewProj; +}; + +#ifdef STANDARDMATERIAL_NORMAL_MAP +layout(location = 3) out vec4 v_WorldTangent; +#endif + +layout(set = 2, binding = 0) buffer JointTransforms { + mat4[] Joints; +}; + +void main() { + mat4 Model = + Vertex_JointWeight.x * Joints[Vertex_JointIndex.x] + + Vertex_JointWeight.y * Joints[Vertex_JointIndex.y] + + Vertex_JointWeight.z * Joints[Vertex_JointIndex.z] + + Vertex_JointWeight.w * Joints[Vertex_JointIndex.w]; + + vec4 world_position = Model * vec4(Vertex_Position, 1.0); + v_WorldPosition = world_position.xyz; + v_WorldNormal = mat3(Model) * Vertex_Normal; + v_Uv = Vertex_Uv; +#ifdef STANDARDMATERIAL_NORMAL_MAP + v_WorldTangent = vec4(mat3(Model) * Vertex_Tangent.xyz, Vertex_Tangent.w); +#endif + gl_Position = ViewProj * world_position; +} diff --git a/crates/bevy_gltf/Cargo.toml b/crates/bevy_gltf/Cargo.toml index 1fa7db10f97bfc..00788cb58c09f8 100644 --- a/crates/bevy_gltf/Cargo.toml +++ b/crates/bevy_gltf/Cargo.toml @@ -14,6 +14,7 @@ keywords = ["bevy"] [dependencies] # bevy +bevy_animation_rig = { path = "../bevy_animation_rig", version = "0.5.0" } bevy_app = { path = "../bevy_app", version = "0.5.0" } bevy_asset = { path = "../bevy_asset", version = "0.5.0" } bevy_core = { path = "../bevy_core", version = "0.5.0" } diff --git a/crates/bevy_gltf/src/loader.rs b/crates/bevy_gltf/src/loader.rs index efc41b6378333a..6577e4297e6aa1 100644 --- a/crates/bevy_gltf/src/loader.rs +++ b/crates/bevy_gltf/src/loader.rs @@ -1,18 +1,22 @@ use anyhow::Result; +use bevy_animation_rig::{SkinnedMesh, SkinnedMeshInverseBindposes, SKINNED_MESH_PIPELINE_HANDLE}; use bevy_asset::{ AssetIoError, AssetLoader, AssetPath, BoxedFuture, Handle, LoadContext, LoadedAsset, }; use bevy_core::Name; -use bevy_ecs::world::World; +use bevy_ecs::{entity::Entity, world::World}; use bevy_log::warn; use bevy_math::Mat4; -use bevy_pbr::prelude::{PbrBundle, StandardMaterial}; +use bevy_pbr::{ + prelude::{PbrBundle, StandardMaterial}, + render_graph::PBR_PIPELINE_HANDLE, +}; use bevy_render::{ camera::{ Camera, CameraProjection, OrthographicProjection, PerspectiveProjection, VisibleEntities, }, mesh::{Indices, Mesh, VertexAttributeValues}, - pipeline::PrimitiveTopology, + pipeline::{PrimitiveTopology, RenderPipeline, RenderPipelines}, prelude::{Color, Texture}, render_graph::base, texture::{AddressMode, FilterMode, ImageType, SamplerDescriptor, TextureError, TextureFormat}, @@ -155,6 +159,20 @@ async fn load_gltf<'a, 'b>( mesh.set_attribute(Mesh::ATTRIBUTE_COLOR, vertex_attribute); } + if let Some(vertex_attribute) = reader + .read_joints(0) + .map(|v| VertexAttributeValues::Uint16x4(v.into_u16().collect())) + { + mesh.set_attribute(Mesh::ATTRIBUTE_JOINT_INDEX, vertex_attribute); + } + + if let Some(vertex_attribute) = reader + .read_weights(0) + .map(|v| VertexAttributeValues::Float32x4(v.into_f32().collect())) + { + mesh.set_attribute(Mesh::ATTRIBUTE_JOINT_WEIGHT, vertex_attribute); + } + if let Some(indices) = reader.read_indices() { mesh.set_indices(Some(Indices::U32(indices.into_u32().collect()))); }; @@ -273,17 +291,44 @@ async fn load_gltf<'a, 'b>( load_context.set_labeled_asset(&label, LoadedAsset::new(texture)); }); + let skinned_mesh_inverse_bindposes: Vec<_> = gltf + .skins() + .map(|gltf_skin| { + let reader = gltf_skin.reader(|buffer| Some(&buffer_data[buffer.index()])); + let inverse_bindposes = reader + .read_inverse_bind_matrices() + .unwrap() + .map(|mat| Mat4::from_cols_array_2d(&mat)) + .collect(); + + load_context.set_labeled_asset( + &skin_label(&gltf_skin), + LoadedAsset::new(SkinnedMeshInverseBindposes(inverse_bindposes)), + ) + }) + .collect(); + let mut scenes = vec![]; let mut named_scenes = HashMap::new(); for scene in gltf.scenes() { let mut err = None; let mut world = World::default(); + let mut node_index_to_entity_map = HashMap::new(); + let mut entity_to_skin_index_map = HashMap::new(); + world .spawn() .insert_bundle((Transform::identity(), GlobalTransform::identity())) .with_children(|parent| { for node in scene.nodes() { - let result = load_node(&node, parent, load_context, &buffer_data); + let result = load_node( + &node, + parent, + load_context, + &buffer_data, + &mut node_index_to_entity_map, + &mut entity_to_skin_index_map, + ); if result.is_err() { err = Some(result); return; @@ -293,6 +338,21 @@ async fn load_gltf<'a, 'b>( if let Some(Err(err)) = err { return Err(err); } + + for (&entity, &skin_index) in &entity_to_skin_index_map { + let mut entity = world.entity_mut(entity); + let skin = gltf.skins().nth(skin_index).unwrap(); + let joint_entities: Vec<_> = skin + .joints() + .map(|node| node_index_to_entity_map[&node.index()]) + .collect(); + + entity.insert(SkinnedMesh::new( + skinned_mesh_inverse_bindposes[skin_index].clone(), + joint_entities, + )); + } + let scene_handle = load_context .set_labeled_asset(&scene_label(&scene), LoadedAsset::new(Scene::new(world))); @@ -447,6 +507,8 @@ fn load_node( world_builder: &mut WorldChildBuilder, load_context: &mut LoadContext, buffer_data: &[Vec], + node_index_to_entity_map: &mut HashMap, + entity_to_skin_index_map: &mut HashMap, ) -> Result<(), GltfError> { let transform = gltf_node.transform(); let mut gltf_error = None; @@ -508,6 +570,9 @@ fn load_node( } } + // Map node index to entity + node_index_to_entity_map.insert(gltf_node.index(), node.id()); + node.with_children(|parent| { if let Some(mesh) = gltf_node.mesh() { // append primitives @@ -522,23 +587,44 @@ fn load_node( load_material(&material, load_context); } + let mut node = parent.spawn(); + + let mut pipeline = PBR_PIPELINE_HANDLE.typed(); + + // Mark for adding skinned mesh + if let Some(skin) = gltf_node.skin() { + entity_to_skin_index_map.insert(node.id(), skin.index()); + pipeline = SKINNED_MESH_PIPELINE_HANDLE.typed(); + } + let primitive_label = primitive_label(&mesh, &primitive); let mesh_asset_path = AssetPath::new_ref(load_context.path(), Some(&primitive_label)); let material_asset_path = AssetPath::new_ref(load_context.path(), Some(&material_label)); - parent.spawn_bundle(PbrBundle { + node.insert_bundle(PbrBundle { mesh: load_context.get_handle(mesh_asset_path), material: load_context.get_handle(material_asset_path), + render_pipelines: RenderPipelines::from_pipelines(vec![RenderPipeline::new( + pipeline, + )]), ..Default::default() }); + node.insert(Name::new("PBR Renderer")); } } // append other nodes for child in gltf_node.children() { - if let Err(err) = load_node(&child, parent, load_context, buffer_data) { + if let Err(err) = load_node( + &child, + parent, + load_context, + buffer_data, + node_index_to_entity_map, + entity_to_skin_index_map, + ) { gltf_error = Some(err); return; } @@ -579,6 +665,10 @@ fn scene_label(scene: &gltf::Scene) -> String { format!("Scene{}", scene.index()) } +fn skin_label(skin: &gltf::Skin) -> String { + format!("Skin{}", skin.index()) +} + fn texture_sampler(texture: &gltf::Texture) -> SamplerDescriptor { let gltf_sampler = texture.sampler(); @@ -647,6 +737,7 @@ async fn load_buffers( load_context: &LoadContext<'_>, asset_path: &Path, ) -> Result>, GltfError> { + const GLTF_BUFFER_URI: &str = "application/gltf-buffer"; const OCTET_STREAM_URI: &str = "application/octet-stream"; let mut buffer_data = Vec::new(); @@ -658,6 +749,7 @@ async fn load_buffers( .unwrap(); let uri = uri.as_ref(); let buffer_bytes = match DataUri::parse(uri) { + Ok(data_uri) if data_uri.mime_type == GLTF_BUFFER_URI => data_uri.decode()?, Ok(data_uri) if data_uri.mime_type == OCTET_STREAM_URI => data_uri.decode()?, Ok(_) => return Err(GltfError::BufferFormatUnsupported), Err(()) => { diff --git a/crates/bevy_internal/Cargo.toml b/crates/bevy_internal/Cargo.toml index 9d557aff5d1675..295a42c6c4e0c1 100644 --- a/crates/bevy_internal/Cargo.toml +++ b/crates/bevy_internal/Cargo.toml @@ -62,6 +62,7 @@ bevy_utils = { path = "../bevy_utils", version = "0.5.0" } bevy_window = { path = "../bevy_window", version = "0.5.0" } bevy_tasks = { path = "../bevy_tasks", version = "0.5.0" } # bevy (optional) +bevy_animation_rig = { path = "../bevy_animation_rig", optional = true, version = "0.5.0" } bevy_audio = { path = "../bevy_audio", optional = true, version = "0.5.0" } bevy_gltf = { path = "../bevy_gltf", optional = true, version = "0.5.0" } bevy_pbr = { path = "../bevy_pbr", optional = true, version = "0.5.0" } diff --git a/crates/bevy_internal/src/default_plugins.rs b/crates/bevy_internal/src/default_plugins.rs index f42f45c477a144..543a377784151d 100644 --- a/crates/bevy_internal/src/default_plugins.rs +++ b/crates/bevy_internal/src/default_plugins.rs @@ -62,6 +62,9 @@ impl PluginGroup for DefaultPlugins { group.add(AssetPlugin::default()); group.add(ScenePlugin::default()); + #[cfg(feature = "bevy_animation_rig")] + group.add(bevy_animation_rig::AnimationRigPlugin::default()); + #[cfg(feature = "bevy_render")] group.add(RenderPlugin::default()); diff --git a/crates/bevy_internal/src/lib.rs b/crates/bevy_internal/src/lib.rs index 62fb7cf270b565..9b082f148831e7 100644 --- a/crates/bevy_internal/src/lib.rs +++ b/crates/bevy_internal/src/lib.rs @@ -76,6 +76,12 @@ pub mod window { pub use bevy_window::*; } +#[cfg(feature = "bevy_animation_rig")] +pub mod animation_rig { + //! Skinned mesh rendering. + pub use bevy_animation_rig::*; +} + #[cfg(feature = "bevy_audio")] pub mod audio { //! Provides types and plugins for audio playback. diff --git a/crates/bevy_internal/src/prelude.rs b/crates/bevy_internal/src/prelude.rs index 9362aea02f108a..77ae3664893dd4 100644 --- a/crates/bevy_internal/src/prelude.rs +++ b/crates/bevy_internal/src/prelude.rs @@ -7,6 +7,10 @@ pub use crate::{ pub use bevy_derive::bevy_main; +#[doc(hidden)] +#[cfg(feature = "bevy_animation_rig")] +pub use crate::animation_rig::*; + #[doc(hidden)] #[cfg(feature = "bevy_audio")] pub use crate::audio::prelude::*; diff --git a/docs/cargo_features.md b/docs/cargo_features.md index 7dc65b95761671..2eeb8872399015 100644 --- a/docs/cargo_features.md +++ b/docs/cargo_features.md @@ -5,6 +5,7 @@ |feature name|description| |-|-| |bevy_audio|Audio support. Support for all audio formats depends on this.| +|bevy_animation_rig|Skinned mesh support.| |bevy_dynamic_plugins|Plugins for dynamic loading (libloading).| |bevy_gilrs|Adds gamepad support.| |bevy_gltf|[glTF](https://www.khronos.org/gltf/) support.| diff --git a/examples/README.md b/examples/README.md index e3b8878c136f0b..47cda66967e224 100644 --- a/examples/README.md +++ b/examples/README.md @@ -39,6 +39,7 @@ git checkout v0.4.0 - [Cross-Platform Examples](#cross-platform-examples) - [2D Rendering](#2d-rendering) - [3D Rendering](#3d-rendering) + - [Animation](#animation) - [Application](#application) - [Assets](#assets) - [Async Tasks](#async-tasks) @@ -108,6 +109,13 @@ Example | File | Description `wireframe` | [`3d/wireframe.rs`](./3d/wireframe.rs) | Showcases wireframe rendering `z_sort_debug` | [`3d/z_sort_debug.rs`](./3d/z_sort_debug.rs) | Visualizes camera Z-ordering +## Animation + +Example | File | Description +--- | --- | --- +`custom_skinned_mesh` | [`animation/custom_skinned_mesh.rs`](./animation/custom_skinned_mesh.rs) | Skinned mesh example with mesh and joints data defined in code. +`gltf_skinned_mesh` | [`animation/gltf_skinned_mesh.rs`](./animation/gltf_skinned_mesh.rs) | Skinned mesh example with mesh and joints data loaded from a glTF file. + ## Application Example | File | Description diff --git a/examples/animation/custom_skinned_mesh.rs b/examples/animation/custom_skinned_mesh.rs new file mode 100644 index 00000000000000..3621a27c074498 --- /dev/null +++ b/examples/animation/custom_skinned_mesh.rs @@ -0,0 +1,161 @@ +use std::f32::consts::PI; + +use bevy::{ + pbr::AmbientLight, + prelude::*, + render::{ + mesh::Indices, + pipeline::{PrimitiveTopology, RenderPipeline}, + }, +}; + +/// Skinned mesh example with mesh and joints data defined in code. +/// Example taken from https://github.com/KhronosGroup/glTF-Tutorials/blob/master/gltfTutorial/gltfTutorial_019_SimpleSkin.md +fn main() { + App::build() + .add_plugins(DefaultPlugins) + .insert_resource(AmbientLight { + brightness: 1.0, + ..Default::default() + }) + .add_startup_system(setup.system()) + .add_system(joint_animation.system()) + .run(); +} + +/// Used to mark a joint to be animated in the [`joint_animation`] system. +struct AnimatedJoint; + +/// Construct a mesh and a skeleton with 2 joints for that mesh, +/// and mark the second joint to be animated. +/// It is similar to the scene defined in `models/SimpleSkin/SimpleSkin.gltf` +fn setup( + mut commands: Commands, + mut meshes: ResMut>, + mut materials: ResMut>, + mut skinned_mesh_inverse_bindposes_assets: ResMut>, +) { + // Create a camera + let mut camera = OrthographicCameraBundle::new_2d(); + camera.orthographic_projection.near = -1.0; + camera.orthographic_projection.far = 1.0; + camera.orthographic_projection.scale = 0.005; + camera.transform = Transform::from_xyz(0.0, 1.0, 0.0); + commands.spawn_bundle(camera); + + // Create inverse bindpose matrices for a skeleton consists of 2 joints + let inverse_bindposes = + skinned_mesh_inverse_bindposes_assets.add(SkinnedMeshInverseBindposes(vec![ + Mat4::from_translation(Vec3::new(-0.5, -1.0, 0.0)), + Mat4::from_translation(Vec3::new(-0.5, -1.0, 0.0)), + ])); + + // Create a mesh + let mut mesh = Mesh::new(PrimitiveTopology::TriangleList); + // Set mesh vertex positions + mesh.set_attribute( + Mesh::ATTRIBUTE_POSITION, + vec![ + [0.0, 0.0, 0.0], + [1.0, 0.0, 0.0], + [0.0, 0.5, 0.0], + [1.0, 0.5, 0.0], + [0.0, 1.0, 0.0], + [1.0, 1.0, 0.0], + [0.0, 1.5, 0.0], + [1.0, 1.5, 0.0], + [0.0, 2.0, 0.0], + [1.0, 2.0, 0.0], + ], + ); + // Set mesh vertex normals + mesh.set_attribute(Mesh::ATTRIBUTE_NORMAL, vec![[0.0, 0.0, 1.0]; 10]); + // Set mesh vertex UVs. Although the mesh doesn't have any texture applied, + // UVs are still required by the render pipeline. So these UVs are zeroed out. + mesh.set_attribute(Mesh::ATTRIBUTE_UV_0, vec![[0.0, 0.0]; 10]); + // Set mesh vertex joint indices for mesh skinning. + // Each vertex gets 4 indices used to address the `JointTransforms` array in the vertex shader + // as well as `SkinnedMeshJoint` array in the `SkinnedMesh` component. + // This means that a maximum of 4 joints can affect a single vertex. + mesh.set_attribute( + Mesh::ATTRIBUTE_JOINT_INDEX, + vec![ + [0, 0, 0, 0], + [0, 0, 0, 0], + [0, 1, 0, 0], + [0, 1, 0, 0], + [0, 1, 0, 0], + [0, 1, 0, 0], + [0, 1, 0, 0], + [0, 1, 0, 0], + [0, 1, 0, 0], + [0, 1, 0, 0], + ], + ); + // Set mesh vertex joint weights for mesh skinning. + // Each vertex gets 4 joint weights corresponding to the 4 joint indices assigned to it. + // The sum of these weights should equal to 1. + mesh.set_attribute( + Mesh::ATTRIBUTE_JOINT_WEIGHT, + vec![ + [1.00, 0.00, 0.0, 0.0], + [1.00, 0.00, 0.0, 0.0], + [0.75, 0.25, 0.0, 0.0], + [0.75, 0.25, 0.0, 0.0], + [0.50, 0.50, 0.0, 0.0], + [0.50, 0.50, 0.0, 0.0], + [0.25, 0.75, 0.0, 0.0], + [0.25, 0.75, 0.0, 0.0], + [0.00, 1.00, 0.0, 0.0], + [0.00, 1.00, 0.0, 0.0], + ], + ); + // Tell bevy to construct triangles from a list of vertex indices, + // where each 3 vertex indices form an triangle. + mesh.set_indices(Some(Indices::U16(vec![ + 0, 1, 3, 0, 3, 2, 2, 3, 5, 2, 5, 4, 4, 5, 7, 4, 7, 6, 6, 7, 9, 6, 9, 8, + ]))); + + // Create joint entities + let joint_0 = commands + .spawn_bundle(( + Transform::from_xyz(0.0, 1.0, 0.0), + GlobalTransform::identity(), + )) + .id(); + let joint_1 = commands + .spawn_bundle(( + AnimatedJoint, + Transform::identity(), + GlobalTransform::identity(), + )) + .id(); + + // Set joint_1 as a child of joint_0. + commands.entity(joint_0).push_children(&[joint_1]); + + // Each joint in this vector corresponds to each inverse bindpose matrix in `SkinnedMeshInverseBindposes`. + let joint_entities = vec![joint_0, joint_1]; + + // Create skinned mesh renderer. Note that its transform doesn't affect the position of the mesh. + commands + .spawn_bundle(PbrBundle { + mesh: meshes.add(mesh), + material: materials.add(Color::WHITE.into()), + render_pipelines: RenderPipelines::from_pipelines(vec![RenderPipeline::new( + SKINNED_MESH_PIPELINE_HANDLE.typed(), + )]), + ..Default::default() + }) + .insert(SkinnedMesh::new(inverse_bindposes, joint_entities)); +} + +/// Animate the joint marked with [`AnimatedJoint`] component. +fn joint_animation(time: Res