Improve xrui textures

packages/editor/src/functions/takeScreenshot.ts

@@ -25,7 +25,7 @@ import {
   TransformComponent
 } from '@etherealengine/engine/src/transform/components/TransformComponent'
 import { getState } from '@etherealengine/hyperflux'
-import { KTX2Encoder } from '@etherealengine/xrui/core/textures/KTX2Encoder.bundle'
+import { KTX2Encoder } from '@etherealengine/xrui/core/textures/KTX2Encoder'
 
 import { EditorState } from '../services/EditorServices'
 
@@ -111,7 +111,12 @@ export async function takeScreenshot(
   scenePreviewCamera.aspect = prevAspect
   scenePreviewCamera.updateProjectionMatrix()
 
-  const ktx2texture = (await ktx2Encoder.encode(imageData, false, 10, false, false)) as ArrayBuffer
+  const ktx2texture = (await ktx2Encoder.encode(imageData, {
+    srgb: true,
+    uastc: true,
+    uastcZstandard: true
+  })) as ArrayBuffer
+
   return new Blob([ktx2texture])
 }
 

packages/engine/src/assets/exporters/gltf/extensions/BasisuExporterExtension.ts

@@ -1,4 +1,3 @@
-import { ImageDataType } from '@loaders.gl/images'
 import { CompressedTexture, LinearEncoding, Texture } from 'three'
 
 import { dispatchAction } from '@etherealengine/hyperflux'
@@ -74,7 +73,7 @@ export default class BasisuExporterExtension extends ExporterExtension implement
           if (!writer.json.images) writer.json.images = []
           const imgIdx = writer.json.images.push(imageDef) - 1
 
-          const imgData: ImageDataType = image.getContext('2d')!.getImageData(0, 0, image.width, image.height) as any
+          const imgData = image.getContext('2d')!.getImageData(0, 0, image.width, image.height)
           const imgId = getImageHash(image)
 
           writer.extensionsUsed[this.name] = true
@@ -86,46 +85,54 @@ export default class BasisuExporterExtension extends ExporterExtension implement
           const ktx2Encoder = new KTX2Encoder()
           similarImages.length && ((imageDef.uri = this.imageIdCache[similarImages[0]]) || resolve(null))
           !similarImages.length &&
-            ktx2Encoder.encode(imgData, false, 2, false, false).then(async (arrayBuffer) => {
-              if (writer.options.embedImages) {
-                const bufferIdx = writer.processBuffer(arrayBuffer)
-
-                const bufferViewDef = {
-                  buffer: bufferIdx,
-                  byteOffset: writer.byteOffset,
-                  byteLength: arrayBuffer.byteLength
+            ktx2Encoder
+              .encode(imgData, {
+                qualityLevel: 256,
+                compressionLevel: 5,
+                srgb: true, // TODO: set false if normal map
+                normalMap: false, // TODO: set true if normal map
+                mipmaps: true
+              })
+              .then(async (arrayBuffer) => {
+                if (writer.options.embedImages) {
+                  const bufferIdx = writer.processBuffer(arrayBuffer)
+
+                  const bufferViewDef = {
+                    buffer: bufferIdx,
+                    byteOffset: writer.byteOffset,
+                    byteLength: arrayBuffer.byteLength
+                  }
+                  writer.byteOffset += arrayBuffer.byteLength
+                  const bufferViewIdx = writer.json.bufferViews.push(bufferViewDef) - 1
+
+                  imageDef.bufferView = bufferViewIdx
+                } else {
+                  const [_, projectName, basePath] = /projects\/([^/]+)\/assets\/(.*)$/.exec(writer.options.path!)!
+                  const baseURI = basePath.includes('/') ? basePath.slice(0, basePath.lastIndexOf('/')) : '.'
+                  const relativeURI = `${writer.options.resourceURI ?? baseURI}/images/${imgId}.ktx2`
+                  const projectSpaceURI = `${baseURI}/${
+                    writer.options.resourceURI ? `${writer.options.resourceURI}/` : ''
+                  }images`
+                  imageDef.uri = relativeURI
+                  this.imageIdCache[stagedHash] = relativeURI
+                  this.lshIndex.add(stagedHash, stagedHash)
+                  const saveParms = {
+                    name: `${imgId}`,
+                    byteLength: arrayBuffer.byteLength,
+                    uri: `${projectSpaceURI}/${imgId}.ktx2`,
+                    buffer: arrayBuffer
+                  }
+                  dispatchAction(
+                    BufferHandlerExtension.saveBuffer({
+                      saveParms,
+                      projectName,
+                      modelName: `${imgId}`
+                    })
+                  )
                 }
-                writer.byteOffset += arrayBuffer.byteLength
-                const bufferViewIdx = writer.json.bufferViews.push(bufferViewDef) - 1
-
-                imageDef.bufferView = bufferViewIdx
-              } else {
-                const [_, projectName, basePath] = /projects\/([^/]+)\/assets\/(.*)$/.exec(writer.options.path!)!
-                const baseURI = basePath.includes('/') ? basePath.slice(0, basePath.lastIndexOf('/')) : '.'
-                const relativeURI = `${writer.options.resourceURI ?? baseURI}/images/${imgId}.ktx2`
-                const projectSpaceURI = `${baseURI}/${
-                  writer.options.resourceURI ? `${writer.options.resourceURI}/` : ''
-                }images`
-                imageDef.uri = relativeURI
-                this.imageIdCache[stagedHash] = relativeURI
-                this.lshIndex.add(stagedHash, stagedHash)
-                const saveParms = {
-                  name: `${imgId}`,
-                  byteLength: arrayBuffer.byteLength,
-                  uri: `${projectSpaceURI}/${imgId}.ktx2`,
-                  buffer: arrayBuffer
-                }
-                dispatchAction(
-                  BufferHandlerExtension.saveBuffer({
-                    saveParms,
-                    projectName,
-                    modelName: `${imgId}`
-                  })
-                )
-              }
-              ktx2Encoder.pool.dispose()
-              resolve(null)
-            })
+                ktx2Encoder.pool.dispose()
+                resolve(null)
+              })
         })
       })
     )

packages/engine/src/scene/classes/ImageUtils.ts

@@ -24,7 +24,7 @@ import {
 } from 'three'
 import { WebGLRenderer } from 'three/src/renderers/WebGLRenderer'
 
-import { KTX2Encoder } from '@etherealengine/xrui/core/textures/KTX2Encoder.bundle'
+import { KTX2Encoder } from '@etherealengine/xrui/core/textures/KTX2Encoder'
 
 import BasisuExporterExtension from '../../assets/exporters/gltf/extensions/BasisuExporterExtension'
 import { GLTFWriter } from '../../assets/exporters/gltf/GLTFExporter'
@@ -161,7 +161,11 @@ export const convertCubemapToKTX2 = async (
   const imageData = new ImageData(new Uint8ClampedArray(pixels), width, height)
   renderer.setRenderTarget(null) // pass `null` to set canvas as render target
 
-  const ktx2texture = (await ktx2write.encode(imageData, false, -1, true, false)) as ArrayBuffer
+  const ktx2texture = (await ktx2write.encode(imageData, {
+    srgb: true,
+    qualityLevel: 256,
+    compressionLevel: 5
+  })) as ArrayBuffer
 
   if (returnAsBlob) {
     return new Blob([ktx2texture])

packages/xrui/core/WebLayer.ts

@@ -52,7 +52,7 @@ export class WebLayer {
   }
 
   get computedPixelRatio(): number {
-    return this.pixelRatio ?? this.parentLayer?.computedPixelRatio ?? 1
+    return this.pixelRatio ?? this.parentLayer?.computedPixelRatio ?? 1.5
   }
 
   allStateHashes = new Set<string>()

packages/xrui/core/WebLayerManagerBase.ts

@@ -17,9 +17,7 @@ import { bufferToHex } from './hex-utils'
 import { serializeToString } from './serialization-utils'
 import { getParentsHTML } from './serialization-utils'
 import { getAllEmbeddedStyles } from './serialization/getAllEmbeddedStyles'
-import type { KTX2Encoder as KTX2EncoderType } from './textures/KTX2Encoder'
-// @ts-ignore
-import { KTX2Encoder } from './textures/KTX2Encoder.bundle.js'
+import { KTX2Encoder, UASTCFlags } from './textures/KTX2Encoder'
 import { WebLayer } from './WebLayer'
 import { WebRenderer } from './WebRenderer'
 
@@ -134,7 +132,7 @@ export class WebLayerManagerBase {
   rasterizeQueue = [] as { hash: StateHash; svgUrl: string; resolve: (val: any) => void; promise: any }[]
   optimizeQueue = [] as { textureHash: TextureHash; resolve: (val: any) => void; promise: any }[]
 
-  ktx2Encoder = new KTX2Encoder() as any as KTX2EncoderType
+  ktx2Encoder = new KTX2Encoder()
 
   private _unsavedTextureData = new Map<TextureHash, TextureStoreData>()
   private _stateData = new Map<StateHash | HTMLMediaElement, StateData>()
@@ -331,10 +329,16 @@ export class WebLayerManagerBase {
     if (!canvas) throw new Error('Missing texture canvas')
     if (this.ktx2Encoder.pool.limit === 0) return
 
-    const imageData = this.getImageData(canvas)
+    const image = this.getImageData(canvas)
     let ktx2Texture: ArrayBuffer
     try {
-      ktx2Texture = await this.ktx2Encoder.encode(imageData as any)
+      ktx2Texture = await this.ktx2Encoder.encode(image, {
+        srgb: true,
+        // compressionLevel: 0,
+        // qualityLevel: 256
+        uastc: true,
+        uastcFlags: UASTCFlags.UASTCLevelFastest
+      })
     } catch (error: any) {
       console.error(`KTX2 encoding failed for image (${canvas.width}, ${canvas.height}) `, error)
       this.ktx2Encoder.pool.dispose()

packages/xrui/core/textures/KTX2Encoder.ts

@@ -1,20 +1,113 @@
-import { ImageDataType } from '@loaders.gl/images'
-
 import { WorkerPool } from '../WorkerPool'
 import KTX2WorkerBody from './KTX2Worker.bundle.txt?raw'
 
 const workerBlob = new Blob([KTX2WorkerBody], { type: 'text/javascript' })
 const workerURL = URL.createObjectURL(workerBlob)
 
-export type EncodeRequest = {
-  image: ImageDataType
-  useSRGB?: boolean
+export enum UASTCFlags {
+  /** Fastest is the lowest quality, although it's stil substantially higher quality vs. BC1/ETC1. It supports 5 modes.
+		/* The output may be somewhat blocky because this setting doesn't support 2/3-subset UASTC modes, but it should be less blocky vs. BC1/ETC1.
+		/* This setting doesn't write BC1 hints, so BC1 transcoding will be slower. 
+		/* Transcoded ETC1 quality will be lower because it only considers 2 hints out of 32.
+		/* Avg. 43.45 dB
+     */
+  UASTCLevelFastest = 0,
+
+  /** Faster is ~3x slower than fastest. It supports 9 modes.
+		/* Avg. 46.49 dB
+		*/
+  UASTCLevelFaster = 1,
+
+  /** Default is ~5.5x slower than fastest. It supports 14 modes.
+		/* Avg. 47.47 dB
+		*/
+  UASTCLevelDefault = 2,
+
+  /** Slower is ~14.5x slower than fastest. It supports all 18 modes.
+		/* Avg. 48.01 dB
+		*/
+  UASTCLevelSlower = 3,
+
+  /** VerySlow is ~200x slower than fastest. 
+		/* The best quality the codec is capable of, but you'll need to be patient or have a lot of cores.
+		/* Avg. 48.24 dB
+		*/
+  UASTCLevelVerySlow = 4,
+
+  UASTCLevelMask = 0xf,
+
+  /** By default the encoder tries to strike a balance between UASTC and transcoded BC7 quality.
+		/** These flags allow you to favor only optimizing for lowest UASTC error, or lowest BC7 error.
+		*/
+  UASTCFavorUASTCError = 8,
+  UASTCFavorBC7Error = 16,
+
+  UASTCETC1FasterHints = 64,
+  UASTCETC1FastestHints = 128,
+  UASTCETC1DisableFlipAndIndividual = 256,
+
+  /**
+   * Favor UASTC modes 0 and 10 more than the others (this is experimental, it's useful for RDO compression)
+   */
+  UASTCFavorSimplerModes = 512
+}
+export interface KTX2EncodeOptions {
+  /**
+   * If true, the input is assumed to be in sRGB space. Be sure to set this correctly! (Examples: True on photos, albedo/spec maps, and false on normal maps.)
+   * @default false
+   */
+  srgb?: boolean
+  /**
+   * Sets the ETC1S encoder's quality level, which controls the file size vs. quality tradeoff
+   * Range is [1,256]
+   * @default 128
+   */
   qualityLevel?: number
-  encodeUASTC?: boolean
+  /**
+   * The compression_level parameter controls the encoder perf vs. file size tradeoff for ETC1S files
+   * It does not directly control file size vs. quality - see qualityLevel
+   * Range is [0,6]
+   * @default 2
+   */
+  compressionLevel?: number
+  /**
+   * If true, the encoder will output a UASTC texture, otherwise a ETC1S texture.
+   * @default false
+   */
+  uastc?: boolean
+  /**
+   * Use UASTC Zstandard supercompression. Use with uastc = true
+   * @default false
+   */
+  uastcZstandard?: boolean
+  /**
+   * Sets the UASTC encoding performance vs. quality tradeoff, and other lesser used UASTC encoder flags.
+   * This is a combination of flags. See UASTCFlags
+   */
+  uastcFlags?: number
+  /**
+   * Tunes several codec parameters so compression works better on normal maps.
+   * @default false
+   */
+  normalMap?: boolean
+  /**
+   * If true, the encoder will generate mipmaps.
+   * @default false
+   */
   mipmaps?: boolean
+  /**
+   * If true the source images will be Y flipped before compression
+   * @default false
+   */
+  yFlip?: boolean
+}
+
+export type KTX2EncodeRequestData = {
+  image: ImageData
+  options: KTX2EncodeOptions
 }
 
-export type EncodeResponse = {
+export type KTX2EncodeResponseData = {
   texture: ArrayBuffer
   error?: string
 }
@@ -30,25 +123,14 @@ export class KTX2Encoder {
     this.pool.setWorkerLimit(limit)
   }
 
-  async encode(
-    imageData: ImageDataType,
-    useSRGB = false,
-    qualityLevel = 10,
-    encodeUASTC = false,
-    mipmaps = true
-  ): Promise<ArrayBuffer> {
-    const responseMessage = await this.pool.postMessage<EncodeResponse>(
-      {
-        image: imageData,
-        useSRGB,
-        qualityLevel,
-        encodeUASTC,
-        mipmaps
-      } as EncodeRequest,
-      [imageData.data.buffer]
+  async encode(image: ImageData, options: KTX2EncodeOptions): Promise<ArrayBuffer> {
+    const responseMessage = await this.pool.postMessage<KTX2EncodeResponseData>(
+      { image, options } as KTX2EncodeRequestData,
+      [image.data.buffer]
     )
     if (responseMessage.data.error) throw new Error(responseMessage.data.error)
-    if (!responseMessage.data.texture) throw new Error('Encoding failed')
+    if (!responseMessage.data.texture || responseMessage.data.texture.byteLength === 0)
+      throw new Error('Encoding failed')
     return responseMessage.data.texture
   }
 }