KTX2 Compression for Thumbnails, Envmaps

packages/client-core/src/world/utils.ts

@@ -31,7 +31,7 @@ export const loadSceneJsonOffline = async (projectName, sceneName) => {
   getMutableState(SceneState).sceneData.set({
     scene: parseSceneDataCacheURLsLocal(projectName, sceneData),
     name: sceneName,
-    thumbnailUrl: `${fileServer}/projects/${locationName}.thumbnail.jpeg`,
+    thumbnailUrl: `${fileServer}/projects/${locationName}.thumbnail.ktx2`,
     project: projectName
   })
 }

packages/editor/src/components/assets/ScenesPanel.tsx

@@ -1,9 +1,12 @@
 import React, { useEffect, useRef, useState } from 'react'
 import { useTranslation } from 'react-i18next'
+import { CompressedTexture, LinearEncoding, LuminanceFormat, RGBAFormat, sRGBEncoding, Texture } from 'three'
 
 import { useRouter } from '@etherealengine/client-core/src/common/services/RouterService'
 import { SceneData } from '@etherealengine/common/src/interfaces/SceneInterface'
 import multiLogger from '@etherealengine/common/src/logger'
+import { AssetLoader } from '@etherealengine/engine/src/assets/classes/AssetLoader'
+import createReadableTexture from '@etherealengine/engine/src/assets/functions/createReadableTexture'
 import { EngineActions } from '@etherealengine/engine/src/ecs/classes/EngineState'
 import { dispatchAction } from '@etherealengine/hyperflux'
 
@@ -37,11 +40,18 @@ export default function ScenesPanel({ loadScene, newScene, toggleRefetchScenes }
   const route = useRouter()
   const editorState = useEditorState()
   const [DialogComponent, setDialogComponent] = useDialog()
+  const [fetched, setFetch] = useState(false)
 
+  const [thumbnails, setThumbnails] = useState<Map<string, string>>(new Map<string, string>())
   const fetchItems = async () => {
     try {
       const data = await getScenes(editorState.projectName.value!)
+      for (let i = 0; i < data.length; i++) {
+        const ktx2url = await getSceneURL(data[i].thumbnailUrl)
+        thumbnails.set(data[i].name, ktx2url)
+      }
       setScenes(data ?? [])
+      console.log(data)
     } catch (error) {
       logger.error(error, 'Error fetching scenes')
     }
@@ -127,6 +137,11 @@ export default function ScenesPanel({ loadScene, newScene, toggleRefetchScenes }
     if (e.key == 'Enter' && activeScene) finishRenaming()
   }
 
+  const getSceneURL = async (url) => {
+    const texture = (await AssetLoader.loadAsync(url)) as CompressedTexture
+    return (await createReadableTexture(texture, { url: true })) as string
+  }
+
   return (
     <>
       <div id="file-browser-panel" className={styles.panelContainer}>
@@ -141,7 +156,7 @@ export default function ScenesPanel({ loadScene, newScene, toggleRefetchScenes }
               <div className={styles.sceneContainer} key={i}>
                 <a onClick={(e) => onClickExisting(e, scene)}>
                   <div className={styles.thumbnailContainer}>
-                    <img src={scene.thumbnailUrl} alt="" crossOrigin="anonymous" />
+                    <img src={thumbnails.get(scene.name)} alt="" crossOrigin="anonymous" />
                   </div>
                   <div className={styles.detailBlock}>
                     {activeScene === scene && isRenaming ? (

packages/editor/src/functions/takeScreenshot.ts

@@ -1,4 +1,15 @@
-import { Camera, PerspectiveCamera } from 'three'
+import { blob } from 'stream/consumers'
+import {
+  _SRGBFormat,
+  Camera,
+  ClampToEdgeWrapping,
+  LinearFilter,
+  PerspectiveCamera,
+  RGBAFormat,
+  sRGBEncoding,
+  UnsignedByteType,
+  WebGLRenderTarget
+} from 'three'
 
 import { Engine } from '@etherealengine/engine/src/ecs/classes/Engine'
 import { SceneState } from '@etherealengine/engine/src/ecs/classes/Scene'
@@ -14,9 +25,9 @@ import {
   TransformComponent
 } from '@etherealengine/engine/src/transform/components/TransformComponent'
 import { getState } from '@etherealengine/hyperflux'
+import { KTX2Encoder } from '@etherealengine/xrui/core/textures/KTX2Encoder.bundle'
 
 import { EditorState } from '../services/EditorServices'
-import { getCanvasBlob } from './thumbnails'
 
 function getResizedCanvas(canvas: HTMLCanvasElement, width: number, height: number) {
   const tmpCanvas = document.createElement('canvas')
@@ -29,6 +40,8 @@ function getResizedCanvas(canvas: HTMLCanvasElement, width: number, height: numb
 
 const query = defineQuery([ScenePreviewCameraComponent])
 
+const ktx2Encoder = new KTX2Encoder()
+
 /**
  * Function takeScreenshot used for taking screenshots.
  *
@@ -71,47 +84,35 @@ export async function takeScreenshot(
   const originalWidth = EngineRenderer.instance.renderer.domElement.width
   const originalHeight = EngineRenderer.instance.renderer.domElement.height
 
-  // Rendering the scene to the new canvas with given size
-  await new Promise<void>((resolve, reject) => {
-    const interval = setInterval(() => {
-      const viewport = EngineRenderer.instance.renderContext.getParameter(
-        EngineRenderer.instance.renderContext.VIEWPORT
-      )
-      const pixelRatio = EngineRenderer.instance.renderer.getPixelRatio()
-      if (viewport[2] === width * pixelRatio && viewport[3] === height * pixelRatio) {
-        clearTimeout(timeout)
-        clearInterval(interval)
-        resolve()
-      }
-    }, 10)
-    const timeout = setTimeout(() => {
-      console.warn('Could not resize viewport in time')
-      clearTimeout(timeout)
-      clearInterval(interval)
-      reject()
-    }, 10000)
-
-    // set up effect composer
-    EngineRenderer.instance.effectComposer.setMainCamera(scenePreviewCamera as Camera)
-    EngineRenderer.instance.effectComposer.setSize(width, height, true)
-  })
-
   EngineRenderer.instance.effectComposer.render()
   EngineRenderer.instance.effectComposer.setMainCamera(Engine.instance.camera)
 
-  const blob = await getCanvasBlob(
-    getResizedCanvas(EngineRenderer.instance.renderer.domElement, width, height),
-    compressed ? 'image/jpeg' : 'image/png',
-    compressed ? 0.9 : 1
-  )
+  const renderer = EngineRenderer.instance.renderer
+  renderer.outputEncoding = sRGBEncoding
+  const renderTarget = new WebGLRenderTarget(width, height, {
+    minFilter: LinearFilter,
+    magFilter: LinearFilter,
+    wrapS: ClampToEdgeWrapping,
+    wrapT: ClampToEdgeWrapping,
+    encoding: sRGBEncoding,
+    format: RGBAFormat,
+    type: UnsignedByteType
+  })
 
+  renderer.setRenderTarget(renderTarget)
+  renderer.render(Engine.instance.scene, scenePreviewCamera)
+  const pixels = new Uint8Array(4 * width * height)
+  renderer.readRenderTargetPixels(renderTarget, 0, 0, width, height, pixels)
+  const imageData = new ImageData(new Uint8ClampedArray(pixels), width, height)
+  renderer.setRenderTarget(null) // pass `null` to set canvas as render target
   EngineRenderer.instance.effectComposer.setSize(originalWidth, originalHeight, true)
 
   // Restoring previous state
   scenePreviewCamera.aspect = prevAspect
   scenePreviewCamera.updateProjectionMatrix()
 
-  return blob
+  const ktx2texture = (await ktx2Encoder.encode(imageData, false, 10, false, false)) as ArrayBuffer
+  return new Blob([ktx2texture])
 }
 
 /** @todo make size configurable */
@@ -126,7 +127,7 @@ export const downloadScreenshot = () => {
     const editorState = getState(EditorState)
 
     link.href = blobUrl
-    link.download = editorState.projectName + '_' + editorState.sceneName + '_thumbnail.png'
+    link.download = editorState.projectName + '_' + editorState.sceneName + '_thumbnail.ktx2'
 
     document.body.appendChild(link)
 

packages/editor/src/functions/uploadEnvMapBake.ts

@@ -14,7 +14,10 @@ import {
 import { EngineRenderer } from '@etherealengine/engine/src/renderer/WebGLRendererSystem'
 import { beforeMaterialCompile } from '@etherealengine/engine/src/scene/classes/BPCEMShader'
 import CubemapCapturer from '@etherealengine/engine/src/scene/classes/CubemapCapturer'
-import { convertCubemapToEquiImageData } from '@etherealengine/engine/src/scene/classes/ImageUtils'
+import {
+  convertCubemapToEquiImageData,
+  convertCubemapToKTX2
+} from '@etherealengine/engine/src/scene/classes/ImageUtils'
 import { EnvMapBakeComponent } from '@etherealengine/engine/src/scene/components/EnvMapBakeComponent'
 import { NameComponent } from '@etherealengine/engine/src/scene/components/NameComponent'
 import { ScenePreviewCameraComponent } from '@etherealengine/engine/src/scene/components/ScenePreviewCamera'
@@ -92,20 +95,20 @@ export const uploadBPCEMBakeToServer = async (entity: Entity) => {
 
   if (isSceneEntity) Engine.instance.scene.environment = renderTarget.texture
 
-  const blob = (await convertCubemapToEquiImageData(
+  const blob = await convertCubemapToKTX2(
     EngineRenderer.instance.renderer,
     renderTarget.texture,
     bakeComponent.resolution,
     bakeComponent.resolution,
     true
-  )) as Blob
+  )
 
   if (!blob) return null!
 
   const nameComponent = getComponent(entity, NameComponent)
   const sceneName = accessEditorState().sceneName.value!
   const projectName = accessEditorState().projectName.value!
-  const filename = isSceneEntity ? `${sceneName}.envmap.png` : `${sceneName}-${nameComponent.replace(' ', '-')}.png`
+  const filename = isSceneEntity ? `${sceneName}.envmap.ktx2` : `${sceneName}-${nameComponent.replace(' ', '-')}.ktx2`
 
   const url = (await uploadProjectFiles(projectName, [new File([blob], filename)]).promises[0])[0]
 
@@ -127,7 +130,7 @@ export const uploadCubemapBakeToServer = async (name: string, position: Vector3)
   const cubemapCapturer = new CubemapCapturer(EngineRenderer.instance.renderer, Engine.instance.scene, resolution)
   const renderTarget = cubemapCapturer.update(position)
 
-  const blob = (await convertCubemapToEquiImageData(
+  const blob = (await convertCubemapToKTX2(
     EngineRenderer.instance.renderer,
     renderTarget.texture,
     resolution,
@@ -139,7 +142,7 @@ export const uploadCubemapBakeToServer = async (name: string, position: Vector3)
 
   const sceneName = accessEditorState().sceneName.value!
   const projectName = accessEditorState().projectName.value!
-  const filename = `${sceneName}-${name.replace(' ', '-')}.png`
+  const filename = `${sceneName}-${name.replace(' ', '-')}.ktx2`
 
   const url = (await uploadProjectFiles(projectName, [new File([blob], filename)])[0])[0]
 

packages/engine/src/assets/functions/createReadableTexture.ts

@@ -7,6 +7,7 @@ import {
   PlaneGeometry,
   Scene,
   ShaderMaterial,
+  sRGBEncoding,
   Texture,
   Uniform,
   Vector4,

packages/engine/src/scene/classes/ImageUtils.ts

@@ -1,3 +1,4 @@
+import { compress } from 'fflate'
 import {
   BackSide,
   ClampToEdgeWrapping,
@@ -8,6 +9,7 @@ import {
   LinearFilter,
   Mesh,
   MeshBasicMaterial,
+  Object3D,
   OrthographicCamera,
   PlaneGeometry,
   PMREMGenerator,
@@ -22,6 +24,11 @@ import {
 } from 'three'
 import { WebGLRenderer } from 'three/src/renderers/WebGLRenderer'
 
+import { KTX2Encoder } from '@etherealengine/xrui/core/textures/KTX2Encoder.bundle'
+
+import BasisuExporterExtension from '../../assets/exporters/gltf/extensions/BasisuExporterExtension'
+import { GLTFWriter } from '../../assets/exporters/gltf/GLTFExporter'
+
 export const ImageProjection = {
   Flat: 'Flat',
   Equirectangular360: 'Equirectangular360'
@@ -108,6 +115,61 @@ export const downloadImage = (imageData: ImageData, imageName = 'Image', width:
   }, 'image/png')
 }
 
+const ktx2write = new KTX2Encoder()
+
+export const convertCubemapToKTX2 = async (
+  renderer: WebGLRenderer,
+  source: CubeTexture,
+  width: number,
+  height: number,
+  returnAsBlob: boolean
+) => {
+  const scene = new Scene()
+  const material = new RawShaderMaterial({
+    uniforms: {
+      map: new Uniform(new CubeTexture())
+    },
+    vertexShader: vertexShader,
+    fragmentShader: fragmentShader,
+    side: DoubleSide,
+    transparent: true
+  })
+  const quad = new Mesh(new PlaneGeometry(1, 1), material)
+  scene.add(quad)
+  const camera = new OrthographicCamera(1 / -2, 1 / 2, 1 / 2, 1 / -2, -10000, 10000)
+
+  quad.scale.set(width, height, 1)
+  camera.left = width / 2
+  camera.right = width / -2
+  camera.top = height / -2
+  camera.bottom = height / 2
+  camera.updateProjectionMatrix()
+  const renderTarget = new WebGLRenderTarget(width, height, {
+    minFilter: LinearFilter,
+    magFilter: LinearFilter,
+    wrapS: ClampToEdgeWrapping,
+    wrapT: ClampToEdgeWrapping,
+    format: RGBAFormat,
+    type: UnsignedByteType
+  })
+
+  renderer.setRenderTarget(renderTarget)
+  quad.material.uniforms.map.value = source
+  renderer.render(scene, camera)
+  const pixels = new Uint8Array(4 * width * height)
+  renderer.readRenderTargetPixels(renderTarget, 0, 0, width, height, pixels)
+  const imageData = new ImageData(new Uint8ClampedArray(pixels), width, height)
+  renderer.setRenderTarget(null) // pass `null` to set canvas as render target
+
+  const ktx2texture = (await ktx2write.encode(imageData, false, -1, true, false)) as ArrayBuffer
+
+  if (returnAsBlob) {
+    return new Blob([ktx2texture])
+  }
+
+  return ktx2texture
+}
+
 //convert Cubemap To Equirectangular map
 export const convertCubemapToEquiImageData = (
   renderer: WebGLRenderer,
@@ -161,6 +223,14 @@ export const convertCubemapToEquiImageData = (
     ctx.putImageData(imageData, 0, 0)
     return new Promise<Blob | null>((resolve) => canvas.toBlob(resolve))
   }
+
+  /* const writer = new GLTFWriter()
+  writer.processSampler(source)
+
+  const g = new GLTF
+  writer.write(new Object3D())
+*/
+
   return imageData
 }
 

packages/projects/default-project/apartment.scene.json

@@ -327,7 +327,9 @@
           "name": "gltf-model",
           "props": {
             "src": "__$project$__/default-project/assets/apartment.glb",
-            "resource": null,
+            "resource": {
+              "src": "__$project$__/default-project/assets/apartment.glb"
+            },
             "generateBVH": true,
             "avoidCameraOcclusion": false
           }
@@ -342,10 +344,10 @@
         {
           "name": "envmap",
           "props": {
-            "type": "Skybox",
-            "envMapTextureType": "Cubemap",
+            "type": "Texture",
+            "envMapTextureType": "Equirectangular",
             "envMapSourceColor": 1193046,
-            "envMapSourceURL": "/hdr/cubemap/skyboxsun25deg/",
+            "envMapSourceURL": "__$project$__/default-project/apartment.envmap.ktx2",
             "envMapIntensity": 1
           }
         }
@@ -1071,7 +1073,9 @@
           "name": "gltf-model",
           "props": {
             "src": "__$project$__/default-project/assets/keycard.glb",
-            "resource": null,
+            "resource": {
+              "src": "__$project$__/default-project/assets/keycard.glb"
+            },
             "generateBVH": true,
             "avoidCameraOcclusion": false
           }