feat(webgpu): Ability to override render targets on RenderPipeline (#…

…2245)

docs/api-reference/core/texture-formats.mdx

@@ -56,7 +56,7 @@ Note that even though a GPU supports creating and sampling textures of a certain
 | `rgb9e5ufloat`                        | <Ft f="rgb9e5ufloat" />             | <L f="rgb9e5ufloat" />             | <R f="rgb9e5ufloat" />             | `GL.RGB9_E5`                                                                                                            |
 | `rg11b10ufloat`                       | <Ft f="rg11b10ufloat" />            | <L f="rg11b10ufloat" />            | <R f="rg11b10ufloat" />            | `GL.R11F_G11F_B10F`                                                                                                     |
 | `rgb10a2unorm`                        | <Ft f="rgb10a2unorm" />             | <L f="rgb10a2unorm" />             | <R f="rgb10a2unorm" />             | `GL.RGB10_A2`                                                                                                           |
-| `rgb10a2uint-webgl`                   | <Ft f="rgb10a2uint-webgl" />        | <L f="rgb10a2uint-webgl" />        | <R f="rgb10a2uint-webgl" />        | `GL.RGB10_A2UI`                                                                                                         |
+| `rgb10a2uint`                   | <Ft f="rgb10a2uint" />        | <L f="rgb10a2uint" />        | <R f="rgb10a2uint" />        | `GL.RGB10_A2UI`                                                                                                         |
 | **6 bytes per pixel formats**         |                                     |                                    |                                    |
 | `rgb16unorm-webgl`                    | <Ft f="rgb16unorm-webgl" />         | <L f="rgb16unorm-webgl" />         | <R f="rgb16unorm-webgl" />         |                                                                                                                         |
 | `rgb16snorm-webgl`                    | <Ft f="rgb16snorm-webgl" />         | <L f="rgb16snorm-webgl" />         | <R f="rgb16snorm-webgl" />         |                                                                                                                         |

modules/core/src/adapter/canvas-context.ts

@@ -7,7 +7,7 @@ import type {Device} from './device';
 import type {Framebuffer} from './resources/framebuffer';
 import {log} from '../utils/log';
 import {uid} from '../utils/uid';
-import type {TextureFormat, DepthStencilTextureFormat} from '../gpu-type-utils/texture-formats';
+import type {DepthStencilTextureFormat} from '../gpu-type-utils/texture-formats';
 
 /** Properties for a CanvasContext */
 export type CanvasContextProps = {
@@ -44,7 +44,7 @@ export abstract class CanvasContext {
   static defaultProps: Required<CanvasContextProps> = {
     id: undefined!,
     canvas: null,
-    width: 800, // width are height are only used by headless gl
+    width: 800,
     height: 600,
     useDevicePixels: true,
     autoResize: true,
@@ -55,18 +55,15 @@ export abstract class CanvasContext {
   };
 
   abstract readonly device: Device;
+  abstract readonly handle: unknown;
   readonly id: string;
+
   readonly props: Required<CanvasContextProps>;
   readonly canvas: HTMLCanvasElement | OffscreenCanvas;
   readonly htmlCanvas?: HTMLCanvasElement;
   readonly offscreenCanvas?: OffscreenCanvas;
   readonly type: 'html-canvas' | 'offscreen-canvas' | 'node';
 
-  /** Format of returned textures: "bgra8unorm", "rgba8unorm" */
-  abstract readonly format: TextureFormat;
-  /** Default stencil format for depth textures */
-  abstract readonly depthStencilFormat: TextureFormat;
-
   protected _initializedResolvers = withResolvers<void>();
 
   /** Promise that resolved once the resize observer has updated the pixel size */

modules/core/src/adapter/device.ts

@@ -314,8 +314,8 @@ export abstract class Device {
     // Callbacks
     onError: (error: Error) => log.error(error.message)(),
     onResize: (context: CanvasContext, info: {oldPixelSize: [number, number]}) => {
-      const [prevWidth, prevHeight] = info.oldPixelSize;
       const [width, height] = context.getPixelSize();
+      const [prevWidth, prevHeight] = info.oldPixelSize;
       log.log(1, `${context} Resized ${prevWidth}x${prevHeight} => ${width}x${height}px`)();
     },
     onVisibilityChange: (context: CanvasContext) =>
@@ -361,6 +361,8 @@ export abstract class Device {
   readonly id: string;
   /** type of this device */
   abstract readonly type: 'webgl' | 'webgpu' | 'unknown';
+  abstract readonly handle: unknown;
+
   /** A copy of the device props  */
   readonly props: Required<DeviceProps>;
   /** Available for the application to store data on the device */
@@ -384,6 +386,11 @@ export abstract class Device {
   /** WebGPU style device limits */
   abstract get limits(): DeviceLimits;
 
+  /** Optimal TextureFormat for displaying 8-bit depth, standard dynamic range content on this system. */
+  abstract preferredColorFormat: 'rgba8unorm' | 'bgra8unorm';
+  /** Default depth format used on this system */
+  abstract preferredDepthFormat: 'depth16' | 'depth24plus' | 'depth32float';
+
   /** Determines what operations are supported on a texture format, checking against supported device features */
   getTextureFormatCapabilities(format: TextureFormat): DeviceTextureFormatCapabilities {
     const genericCapabilities = getTextureFormatCapabilities(format);

modules/core/src/adapter/resources/render-pipeline.ts

@@ -7,10 +7,13 @@ import type {UniformValue} from '../types/uniforms';
 import type {PrimitiveTopology, RenderPipelineParameters} from '../types/parameters';
 import type {ShaderLayout, Binding} from '../types/shader-layout';
 import type {BufferLayout} from '../types/buffer-layout';
-// import {normalizeAttributeMap} from '../helpers/attribute-bindings';
-import {Resource, ResourceProps} from './resource';
+import type {
+  ColorTextureFormat,
+  DepthStencilTextureFormat
+} from '@luma.gl/core/gpu-type-utils/texture-formats';
 import type {Shader} from './shader';
 import type {RenderPass} from './render-pass';
+import {Resource, ResourceProps} from './resource';
 import {VertexArray} from './vertex-array';
 import {TransformFeedback} from './transform-feedback';
 
@@ -37,15 +40,18 @@ export type RenderPipelineProps = ResourceProps & {
 
   /** Determines how vertices are read from the 'vertex' attributes */
   topology?: PrimitiveTopology;
+
+  // color attachment information (needed on WebGPU)
+
+  /** Color attachments expected by this pipeline. Defaults to [device.preferredColorFormat]. Array needs not be contiguous. */
+  colorAttachmentFormats?: (ColorTextureFormat | null)[];
+  /** Depth attachment expected by this pipeline (if depth parameters are specified). Defaults to device.preferredDepthFormat */
+  depthStencilAttachmentFormat?: DepthStencilTextureFormat;
+
   /** Parameters that are controlled by pipeline */
   parameters?: RenderPipelineParameters;
 
-  // /** Use instanced rendering? */
-  // isInstanced?: boolean;
-  // /** Number of instances */
-  // instanceCount?: number;
-  // /** Number of vertices */
-  // vertexCount?: number;
+  // Dynamic bindings (TODO - pipelines should be immutable, move to RenderPass)
 
   /** Buffers, Textures, Samplers for the shader bindings */
   bindings?: Record<string, Binding>;
@@ -71,11 +77,11 @@ export abstract class RenderPipeline extends Resource<RenderPipelineProps> {
     shaderLayout: null,
     bufferLayout: [],
     topology: 'triangle-list',
-    parameters: {},
 
-    // isInstanced: false,
-    // instanceCount: 0,
-    // vertexCount: 0,
+    colorAttachmentFormats: undefined!,
+    depthStencilAttachmentFormat: undefined!,
+
+    parameters: {},
 
     bindings: {},
     uniforms: {}

modules/core/src/gpu-type-utils/texture-format-table.ts

@@ -117,7 +117,7 @@ const TEXTURE_FORMAT_TABLE: Readonly<Record<TextureFormat, TextureFormatDefiniti
   'rgb9e5ufloat': {channels: 'rgb', packed: true, render: rgb9e5ufloat_renderable}, // , filter: true},
   'rg11b10ufloat': {channels: 'rgb', bitsPerChannel: [11, 11, 10, 0], packed: true, p: 1,render: float32_renderable},
   'rgb10a2unorm': {channels: 'rgba',  bitsPerChannel: [10, 10, 10, 2], packed: true, p: 1},
-  'rgb10a2uint-webgl': {channels: 'rgba', bitsPerChannel: [10, 10, 10, 2], packed: true, p: 1, wgpu: false},
+  'rgb10a2uint': {channels: 'rgba',  bitsPerChannel: [10, 10, 10, 2], packed: true, p: 1},
 
   // 48-bit formats
   'rgb16unorm-webgl': {f: norm16_renderable}, // rgb not renderable

modules/core/src/gpu-type-utils/texture-formats.ts

@@ -54,6 +54,7 @@ export type WebGPUColorTextureFormat =
   // Packed 32-bit formats
   | 'rgb9e5ufloat'
   | 'rgb10a2unorm'
+  | 'rgb10a2uint'
   | 'rg11b10ufloat'
 
   // 64-bit formats
@@ -143,7 +144,7 @@ export type WebGL2ColorTextureFormat =
   | 'rgb8snorm-webgl'
   | 'rg16unorm-webgl'
   | 'rg16snorm-webgl'
-  | 'rgb10a2uint-webgl'
+  | 'rgb10a2uint'
   | 'rgb16unorm-webgl'
   | 'rgb16snorm-webgl'
   | 'rgba16unorm-webgl'

modules/test-utils/src/null-device/null-canvas-context.ts

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import type {CanvasContextProps, TextureFormat} from '@luma.gl/core';
+import type {CanvasContextProps} from '@luma.gl/core';
 import {CanvasContext} from '@luma.gl/core';
 import type {NullDevice} from './null-device';
 import {NullFramebuffer} from './resources/null-framebuffer';
@@ -12,8 +12,7 @@ import {NullFramebuffer} from './resources/null-framebuffer';
  */
 export class NullCanvasContext extends CanvasContext {
   readonly device: NullDevice;
-  readonly format: TextureFormat = 'rgba8unorm';
-  readonly depthStencilFormat: TextureFormat = 'depth24plus';
+  readonly handle = null;
 
   presentationSize: [number, number];
   private _framebuffer: NullFramebuffer | null = null;

modules/test-utils/src/null-device/null-device.ts

@@ -47,6 +47,11 @@ export class NullDevice extends Device {
     return true;
   }
   readonly type = 'unknown';
+  readonly handle = null;
+
+  readonly preferredColorFormat = 'rgba8unorm';
+  readonly preferredDepthFormat = 'depth24plus';
+
   features: DeviceFeatures = new DeviceFeatures([], this.props._disabledFeatures);
   limits: NullDeviceLimits = new NullDeviceLimits();
   readonly info = NullDeviceInfo;

modules/webgl/src/adapter/converters/webgl-texture-table.ts

@@ -148,8 +148,8 @@ export const WEBGL_TEXTURE_FORMATS: Record<TextureFormat, WebGLFormatInfo> = {
   'rgb9e5ufloat': {gl: GL.RGB9_E5}, // , filter: true},
   'rg11b10ufloat': {gl: GL.R11F_G11F_B10F, rb: true},
   'rgb10a2unorm': {gl: GL.RGB10_A2, rb: true},
-  'rgb10a2uint-webgl': {gl: GL.RGB10_A2UI, rb: true},
-
+  'rgb10a2uint': {gl: GL.RGB10_A2UI, rb: true},
+  
   // 48-bit formats
   'rgb16unorm-webgl': {gl: GL.RGB16_EXT}, // rgb not renderable
   'rgb16snorm-webgl': {gl: GL.RGB16_SNORM_EXT}, // rgb not renderable

modules/webgl/src/adapter/webgl-canvas-context.ts

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import type {CanvasContextProps, TextureFormat} from '@luma.gl/core';
+import type {CanvasContextProps} from '@luma.gl/core';
 import {CanvasContext} from '@luma.gl/core';
 import {WebGLDevice} from './webgl-device';
 import {WEBGLFramebuffer} from './resources/webgl-framebuffer';
@@ -12,10 +12,8 @@ import {WEBGLFramebuffer} from './resources/webgl-framebuffer';
  */
 export class WebGLCanvasContext extends CanvasContext {
   readonly device: WebGLDevice;
-  readonly format: TextureFormat = 'rgba8unorm';
-  readonly depthStencilFormat: TextureFormat = 'depth24plus';
+  readonly handle: unknown = null;
 
-  presentationSize: [number, number];
   private _framebuffer: WEBGLFramebuffer | null = null;
 
   get [Symbol.toStringTag](): string {

modules/webgl/src/adapter/webgl-device.ts

@@ -72,9 +72,7 @@ import {getWebGLExtension} from '../context/helpers/webgl-extensions';
 
 /** WebGPU style Device API for a WebGL context */
 export class WebGLDevice extends Device {
-  //
   // Public `Device` API
-  //
 
   /** type of this device */
   readonly type = 'webgl';
@@ -83,10 +81,12 @@ export class WebGLDevice extends Device {
   readonly handle: WebGL2RenderingContext;
   features: WebGLDeviceFeatures;
   limits: WebGLDeviceLimits;
-
   readonly info: DeviceInfo;
   readonly canvasContext: WebGLCanvasContext;
 
+  readonly preferredColorFormat = 'rgba8unorm';
+  readonly preferredDepthFormat = 'depth24plus';
+
   readonly lost: Promise<{reason: 'destroyed'; message: string}>;
 
   private _resolveContextLost?: (value: {reason: 'destroyed'; message: string}) => void;

modules/webgpu/src/adapter/resources/webgpu-render-pipeline.ts

@@ -166,34 +166,43 @@ export class WebGPURenderPipeline extends RenderPipeline {
       buffers: getVertexBufferLayout(this.shaderLayout, this.props.bufferLayout)
     };
 
+    // Populate color targets
+    // TODO - at the moment blend and write mask are only set on the first target
+    const targets: (GPUColorTargetState | null)[] = [];
+    if (this.props.colorAttachmentFormats) {
+      for (const format of this.props.colorAttachmentFormats) {
+        targets.push(format ? {format: getWebGPUTextureFormat(format)} : null);
+      }
+    } else {
+      targets.push({format: getWebGPUTextureFormat(this.device.preferredColorFormat)});
+    }
+
     // Set up the fragment stage
     const fragment: GPUFragmentState = {
       module: (this.props.fs as WebGPUShader).handle,
       entryPoint: this.props.fragmentEntryPoint || 'main',
-      targets: [
-        {
-          format: getWebGPUTextureFormat(this.device.getDefaultCanvasContext().format)
-        }
-      ]
+      targets
     };
 
-    const depthStencil: GPUDepthStencilState | undefined = this.props.parameters.depthWriteEnabled
-      ? {
-          format: getWebGPUTextureFormat(this.device.getDefaultCanvasContext().depthStencilFormat)
-        }
-      : undefined;
-
     // Create a partially populated descriptor
     const descriptor: GPURenderPipelineDescriptor = {
       vertex,
       fragment,
-      depthStencil,
       primitive: {
         topology: this.props.topology
       },
       layout: 'auto'
     };
 
+    // Set depth format if required, defaulting to the preferred depth format
+    const depthFormat = this.props.depthStencilAttachmentFormat || this.device.preferredDepthFormat;
+
+    if (this.props.parameters.depthWriteEnabled) {
+      descriptor.depthStencil = {
+        format: getWebGPUTextureFormat(depthFormat)
+      };
+    }
+
     // Set parameters on the descriptor
     applyParametersToRenderPipelineDescriptor(descriptor, this.props.parameters);
 

modules/webgpu/src/adapter/webgpu-canvas-context.ts

@@ -4,9 +4,8 @@
 
 // / <reference types="@webgpu/types" />
 
-import type {TextureFormat, DepthStencilTextureFormat, CanvasContextProps} from '@luma.gl/core';
+import type {DepthStencilTextureFormat, CanvasContextProps} from '@luma.gl/core';
 import {CanvasContext, Texture, log} from '@luma.gl/core';
-import {getWebGPUTextureFormat} from './helpers/convert-texture-format';
 import {WebGPUDevice} from './webgpu-device';
 import {WebGPUFramebuffer} from './resources/webgpu-framebuffer';
 import {WebGPUTexture} from './resources/webgpu-texture';
@@ -18,11 +17,7 @@ import {WebGPUTexture} from './resources/webgpu-texture';
  */
 export class WebGPUCanvasContext extends CanvasContext {
   readonly device: WebGPUDevice;
-  readonly gpuCanvasContext: GPUCanvasContext;
-  /** Format of returned textures: "bgra8unorm", "rgba8unorm", "rgba16float". */
-  readonly format: TextureFormat = navigator.gpu.getPreferredCanvasFormat() as TextureFormat;
-  /** Default stencil format for depth textures */
-  readonly depthStencilFormat: TextureFormat = 'depth24plus';
+  readonly handle: GPUCanvasContext;
 
   private depthStencilAttachment: WebGPUTexture | null = null;
 
@@ -34,20 +29,14 @@ export class WebGPUCanvasContext extends CanvasContext {
     super(props);
     this.device = device;
 
-    // @ts-ignore TODO - we don't handle OffscreenRenderingContext.
-    this.gpuCanvasContext = this.canvas.getContext('webgpu');
-    // TODO this has been replaced
-    // this.format = this.gpuCanvasContext.getPreferredFormat(adapter);
-    this.format = 'bgra8unorm';
-
     // Base class constructor cannot access derived methods/fields, so we need to call these functions in the subclass constructor
     this._setAutoCreatedCanvasId(`${this.device.id}-canvas`);
     this.updateSize([this.drawingBufferWidth, this.drawingBufferHeight]);
   }
 
   /** Destroy any textures produced while configured and remove the context configuration. */
   destroy(): void {
-    this.gpuCanvasContext.unconfigure();
+    this.handle.unconfigure();
   }
 
   /** Update framebuffer with properly resized "swap chain" texture views */
@@ -92,9 +81,9 @@ export class WebGPUCanvasContext extends CanvasContext {
 
     // Reconfigure the canvas size.
     // https://www.w3.org/TR/webgpu/#canvas-configuration
-    this.gpuCanvasContext.configure({
+    this.handle.configure({
       device: this.device.handle,
-      format: getWebGPUTextureFormat(this.format),
+      format: this.device.preferredColorFormat,
       // Can be used to define e.g. -srgb views
       // viewFormats: [...]
       colorSpace: this.props.colorSpace,
@@ -121,8 +110,8 @@ export class WebGPUCanvasContext extends CanvasContext {
   getCurrentTexture(): WebGPUTexture {
     return this.device.createTexture({
       id: `${this.id}#color-texture`,
-      handle: this.gpuCanvasContext.getCurrentTexture(),
-      format: this.format
+      handle: this.handle.getCurrentTexture(),
+      format: this.device.preferredColorFormat
     });
   }