WIP: OpenXR integration

assets/shaders/hot.vert

@@ -4,6 +4,7 @@ layout(location = 0) in vec3 Vertex_Position;
 
 layout(set = 0, binding = 0) uniform CameraViewProj {
     mat4 ViewProj;
+    mat4 ViewProj2;
 };
 
 layout(set = 1, binding = 0) uniform Transform {

crates/bevy_gltf/src/loader.rs

@@ -475,7 +475,7 @@ fn load_node(
 
                 node.insert(Camera {
                     name: Some(base::camera::CAMERA_2D.to_owned()),
-                    projection_matrix: orthographic_projection.get_projection_matrix(),
+                    projection_matrices: vec![orthographic_projection.get_projection_matrix()],
                     ..Default::default()
                 });
                 node.insert(orthographic_projection);
@@ -494,7 +494,7 @@ fn load_node(
                 }
                 node.insert(Camera {
                     name: Some(base::camera::CAMERA_3D.to_owned()),
-                    projection_matrix: perspective_projection.get_projection_matrix(),
+                    projection_matrices: vec![perspective_projection.get_projection_matrix()],
                     ..Default::default()
                 });
                 node.insert(perspective_projection);

crates/bevy_pbr/src/render_graph/pbr_pipeline/pbr.frag

@@ -43,7 +43,6 @@ struct PointLight {
     vec4 color;
     vec4 lightParams;
 };
-
 struct DirectionalLight {
     vec4 direction;
     vec4 color;
@@ -61,6 +60,7 @@ layout(location = 0) out vec4 o_Target;
 
 layout(set = 0, binding = 0) uniform CameraViewProj {
     mat4 ViewProj;
+    mat4 ViewProj2;
 };
 layout(std140, set = 0, binding = 1) uniform CameraPosition {
     vec4 CameraPos;

crates/bevy_pbr/src/render_graph/pbr_pipeline/pbr.vert

@@ -1,4 +1,5 @@
 #version 450
+#extension GL_EXT_multiview : enable
 
 layout(location = 0) in vec3 Vertex_Position;
 layout(location = 1) in vec3 Vertex_Normal;
@@ -14,6 +15,7 @@ layout(location = 2) out vec2 v_Uv;
 
 layout(set = 0, binding = 0) uniform CameraViewProj {
     mat4 ViewProj;
+    mat4 ViewProj2;
 };
 
 #ifdef STANDARDMATERIAL_NORMAL_MAP
@@ -32,5 +34,13 @@ void main() {
 #ifdef STANDARDMATERIAL_NORMAL_MAP
     v_WorldTangent = vec4(mat3(Model) * Vertex_Tangent.xyz, Vertex_Tangent.w);
 #endif
-    gl_Position = ViewProj * world_position;
+
+    mat4 eyeViewProj;
+    if (gl_ViewIndex == 0) {
+        eyeViewProj = ViewProj;
+    } else {
+        eyeViewProj = ViewProj2;
+    }
+
+    gl_Position = eyeViewProj * world_position;
 }

crates/bevy_render/src/camera/camera.rs

@@ -9,14 +9,16 @@ use bevy_ecs::{
 };
 use bevy_math::{Mat4, Vec2, Vec3};
 use bevy_reflect::{Reflect, ReflectDeserialize};
-use bevy_transform::components::GlobalTransform;
+use bevy_transform::components::{GlobalTransform, Transform};
 use bevy_window::{WindowCreated, WindowId, WindowResized, Windows};
 use serde::{Deserialize, Serialize};
 
 #[derive(Default, Debug, Reflect)]
 #[reflect(Component)]
 pub struct Camera {
-    pub projection_matrix: Mat4,
+    pub projection_matrices: Vec<Mat4>,
+    pub position_matrices: Vec<Mat4>,
+
     pub name: Option<String>,
     #[reflect(ignore)]
     pub window: WindowId,
@@ -51,7 +53,7 @@ impl Camera {
         let window_size = Vec2::new(window.width(), window.height());
         // Build a transform to convert from world to NDC using camera data
         let world_to_ndc: Mat4 =
-            self.projection_matrix * camera_transform.compute_matrix().inverse();
+            self.projection_matrices[0] * camera_transform.compute_matrix().inverse();
         let ndc_space_coords: Vec3 = world_to_ndc.project_point3(world_position);
         // NDC z-values outside of 0 < z < 1 are behind the camera and are thus not in screen space
         if ndc_space_coords.z < 0.0 || ndc_space_coords.z > 1.0 {
@@ -61,6 +63,13 @@ impl Camera {
         let screen_space_coords = (ndc_space_coords.truncate() + Vec2::ONE) / 2.0 * window_size;
         Some(screen_space_coords)
     }
+
+    pub fn get_view_projection(&self, view_idx: usize) -> Option<Mat4> {
+        let projection_matrix = self.projection_matrices.get(view_idx)?;
+        let position = self.position_matrices.get(view_idx)?;
+
+        Some(*projection_matrix * position.inverse())
+    }
 }
 
 #[allow(clippy::type_complexity)]
@@ -105,9 +114,11 @@ pub fn camera_system<T: CameraProjection + Component>(
                 || camera_projection.is_changed()
             {
                 camera_projection.update(window.width(), window.height());
-                camera.projection_matrix = camera_projection.get_projection_matrix();
+                camera.projection_matrices = vec![camera_projection.get_projection_matrix()];
                 camera.depth_calculation = camera_projection.depth_calculation();
             }
         }
     }
 }
+
+// FIXME: add a system to update Camera.position_matrices based on GlobalTransform?

crates/bevy_render/src/pipeline/pipeline_compiler.rs

@@ -228,6 +228,12 @@ impl PipelineCompiler {
                     .attributes
                     .push(compiled_vertex_attribute);
             } else {
+                if shader_vertex_attribute.name == "gl_ViewIndex" {
+                    // gl_ViewIndex will be supplied by graphics runtime (multiview)
+                    // not require to be submitted by mesh
+                    continue;
+                }
+
                 panic!(
                     "Attribute {} is required by shader, but not supplied by mesh. Either remove the attribute from the shader or supply the attribute ({}) to the mesh.",
                     shader_vertex_attribute.name,

crates/bevy_render/src/render_graph/graph.rs

@@ -74,6 +74,25 @@ impl RenderGraph {
             .ok_or(RenderGraphError::InvalidNode(label))
     }
 
+    pub fn replace_node<T>(
+        &mut self,
+        label: impl Into<NodeLabel>,
+        node: T,
+    ) -> Result<(), RenderGraphError>
+    where
+        T: Node,
+    {
+        let label = label.into();
+        let node_id = self.get_node_id(&label)?;
+        let node_state = self
+            .nodes
+            .get_mut(&node_id)
+            .ok_or(RenderGraphError::InvalidNode(label))?;
+
+        node_state.replace_node(node);
+        Ok(())
+    }
+
     pub fn get_node_id(&self, label: impl Into<NodeLabel>) -> Result<NodeId, RenderGraphError> {
         let label = label.into();
         match label {

crates/bevy_render/src/render_graph/node.rs

@@ -183,6 +183,13 @@ impl NodeState {
 
         Ok(())
     }
+
+    pub fn replace_node<T>(&mut self, node: T)
+    where
+        T: Node,
+    {
+        self.node = Box::new(node);
+    }
 }
 
 #[derive(Debug, Clone, Eq, PartialEq)]

crates/bevy_render/src/render_graph/nodes/camera_node.rs

@@ -68,6 +68,8 @@ pub struct CameraNodeState {
     staging_buffer: Option<BufferId>,
 }
 
+const VIEW_PROJ_COUNT: usize = 2;
+const VIEW_PROJS_SIZE: usize = std::mem::size_of::<[[[f32; 4]; 4]; VIEW_PROJ_COUNT]>();
 const MATRIX_SIZE: usize = std::mem::size_of::<[[f32; 4]; 4]>();
 const VEC4_SIZE: usize = std::mem::size_of::<[f32; 4]>();
 
@@ -97,7 +99,7 @@ pub fn camera_node_system(
         let staging_buffer = render_resource_context.create_buffer(BufferInfo {
             size:
                 // ViewProj
-                MATRIX_SIZE +
+                VIEW_PROJS_SIZE +
                 // View
                 MATRIX_SIZE +
                 // Position
@@ -112,15 +114,16 @@ pub fn camera_node_system(
 
     if bindings.get(CAMERA_VIEW_PROJ).is_none() {
         let buffer = render_resource_context.create_buffer(BufferInfo {
-            size: MATRIX_SIZE,
+            size: VIEW_PROJS_SIZE,
             buffer_usage: BufferUsage::COPY_DST | BufferUsage::UNIFORM,
             ..Default::default()
         });
+
         bindings.set(
             CAMERA_VIEW_PROJ,
             RenderResourceBinding::Buffer {
                 buffer,
-                range: 0..MATRIX_SIZE as u64,
+                range: 0..VIEW_PROJS_SIZE as u64,
                 dynamic_index: None,
             },
         );
@@ -180,22 +183,30 @@ pub fn camera_node_system(
     }
 
     if let Some(RenderResourceBinding::Buffer { buffer, .. }) = bindings.get(CAMERA_VIEW_PROJ) {
-        let view_proj = camera.projection_matrix * view.inverse();
+        let range = offset..(offset + VIEW_PROJS_SIZE as u64);
+
         render_resource_context.write_mapped_buffer(
             staging_buffer,
-            offset..(offset + MATRIX_SIZE as u64),
+            range,
             &mut |data, _renderer| {
-                data[0..MATRIX_SIZE].copy_from_slice(view_proj.to_cols_array_2d().as_bytes());
+                for i in 0..VIEW_PROJ_COUNT {
+                    if let Some(view_proj) = camera.get_view_projection(i) {
+                        data[i * (VIEW_PROJS_SIZE / VIEW_PROJ_COUNT)
+                            ..(i + 1) * VIEW_PROJS_SIZE / VIEW_PROJ_COUNT]
+                            .copy_from_slice(view_proj.to_cols_array_2d().as_bytes());
+                    }
+                }
             },
         );
+
         state.command_queue.copy_buffer_to_buffer(
             staging_buffer,
             offset,
             *buffer,
             0,
-            MATRIX_SIZE as u64,
+            VIEW_PROJS_SIZE as u64,
         );
-        offset += MATRIX_SIZE as u64;
+        offset += VIEW_PROJS_SIZE as u64;
     }
 
     if let Some(RenderResourceBinding::Buffer { buffer, .. }) = bindings.get(CAMERA_POSITION) {

crates/bevy_render/src/render_graph/nodes/window_texture_node.rs

@@ -26,6 +26,10 @@ impl WindowTextureNode {
             window_resized_event_reader: Default::default(),
         }
     }
+
+    pub fn descriptor(&self) -> &TextureDescriptor {
+        &self.descriptor
+    }
 }
 
 impl Node for WindowTextureNode {

crates/bevy_render/src/shader/shader_reflect.rs

@@ -322,6 +322,7 @@ mod tests {
             layout(location = 0) out vec4 v_Position;
             layout(set = 0, binding = 0) uniform CameraViewProj {
                 mat4 ViewProj;
+                mat4 ViewProj2;
             };
             layout(set = 1, binding = 0) uniform texture2D Texture;
 
@@ -379,7 +380,7 @@ mod tests {
                             name: "CameraViewProj".into(),
                             bind_type: BindType::Uniform {
                                 has_dynamic_offset: false,
-                                property: UniformProperty::Struct(vec![UniformProperty::Mat4]),
+                                property: UniformProperty::Struct(vec![UniformProperty::Mat4, UniformProperty::Mat4]),
                             },
                             shader_stage: BindingShaderStage::VERTEX,
                         }]

crates/bevy_sprite/src/render/sprite.vert

@@ -8,6 +8,7 @@ layout(location = 0) out vec2 v_Uv;
 
 layout(set = 0, binding = 0) uniform CameraViewProj {
     mat4 ViewProj;
+    mat4 ViewProj2;
 };
 
 layout(set = 2, binding = 0) uniform Transform {

crates/bevy_sprite/src/render/sprite_sheet.vert

@@ -9,6 +9,7 @@ layout(location = 1) out vec4 v_Color;
 
 layout(set = 0, binding = 0) uniform CameraViewProj {
     mat4 ViewProj;
+    mat4 ViewProj2;
 };
 
 // TODO: merge dimensions into "sprites" buffer when that is supported in the Uniforms derive abstraction

crates/bevy_ui/src/render/ui.vert

@@ -8,6 +8,7 @@ layout(location = 0) out vec2 v_Uv;
 
 layout(set = 0, binding = 0) uniform CameraViewProj {
     mat4 ViewProj;
+    mat4 ViewProj2;
 };
 
 layout(set = 1, binding = 0) uniform Transform {

crates/bevy_wgpu/src/lib.rs

@@ -14,13 +14,12 @@ use bevy_ecs::{
     system::{IntoExclusiveSystem, IntoSystem},
     world::World,
 };
-use bevy_render::{
-    renderer::{shared_buffers_update_system, RenderResourceContext, SharedBuffers},
-    RenderStage,
-};
+use bevy_render::renderer::{RenderResourceContext, SharedBuffers, TextureId, shared_buffers_update_system};
 use futures_lite::future;
 use renderer::WgpuRenderResourceContext;
-use std::borrow::Cow;
+use std::{borrow::Cow, sync::Arc};
+
+pub use bevy_render::RenderStage;
 
 #[derive(Clone, Copy)]
 pub enum WgpuFeature {
@@ -124,11 +123,28 @@ pub fn get_wgpu_render_system(world: &mut World) -> impl FnMut(&mut World) {
     let resource_context = WgpuRenderResourceContext::new(wgpu_renderer.device.clone());
     world.insert_resource::<Box<dyn RenderResourceContext>>(Box::new(resource_context));
     world.insert_resource(SharedBuffers::new(4096));
+    world.insert_resource(WgpuRendererHandles {
+        device: wgpu_renderer.device.clone(),
+    });
+    world.insert_resource(WgpuRenderState {
+        should_render: true,
+        add_textures: Vec::new(),
+    });
+
     move |world| {
         wgpu_renderer.update(world);
     }
 }
 
+pub struct WgpuRendererHandles {
+    pub device: Arc<wgpu::Device>,
+}
+
+pub struct WgpuRenderState {
+    pub should_render: bool,
+    pub add_textures: Vec<TextureView>,
+}
+
 #[derive(Default, Clone)]
 pub struct WgpuOptions {
     pub device_label: Option<Cow<'static, str>>,
@@ -185,3 +201,7 @@ impl Default for WgpuPowerOptions {
         WgpuPowerOptions::HighPerformance
     }
 }
+pub struct TextureView {
+    pub id: TextureId,
+    pub texture_view: wgpu::TextureView,
+}

crates/bevy_wgpu/src/renderer/wgpu_render_graph_executor.rs

@@ -69,7 +69,7 @@ impl WgpuRenderGraphExecutor {
                                     outputs.get(*output_index).expect("Output should be set.");
                                 input_slot.resource = Some(output_resource);
                             } else {
-                                panic!("No edge connected to input.")
+                                panic!("No edge connected to input (input node {:?}). Did you call `output.set(...)` in the node where the input originates", node_state.name);
                             }
                         }
                         node_state.node.update(

crates/bevy_wgpu/src/renderer/wgpu_render_resource_context.rs

@@ -22,7 +22,7 @@ use std::{
     ops::Range,
     sync::Arc,
 };
-use wgpu::util::DeviceExt;
+use wgpu::{util::DeviceExt, TextureView};
 
 #[derive(Clone, Debug)]
 pub struct WgpuRenderResourceContext {
@@ -240,6 +240,11 @@ impl WgpuRenderResourceContext {
         swap_chain_outputs.insert(id, next_texture);
         Some(id)
     }
+
+    pub fn add_wgpu_texture_view(&self, id: TextureId, view: TextureView) {
+        let mut texture_views = self.resources.texture_views.write();
+        texture_views.insert(id, view);
+    }
 }
 
 impl RenderResourceContext for WgpuRenderResourceContext {