Add Realism Effects

packages/client-core/src/components/InstanceChat/InstanceChat.skiptest.tsx

@@ -12,8 +12,6 @@ import { createMockAPI } from '../../../tests/createMockAPI'
 import { API } from '../../API'
 import { accessChatState } from '../../social/services/ChatService'
 
-import '@etherealengine/engine/src/patchEngineNode'
-
 describe('Instance Chat Component', () => {
   let rootContainer: HTMLDivElement
 

packages/client-core/src/components/InstanceChat/index.module.scss

@@ -1,9 +1,6 @@
 .backdrop {
   position: fixed;
-  top: 0;
-  right: 0;
-  left: 0;
-  bottom: 0;
+  inset: 0;
   width: 100vw;
   background: #00000091;
   pointer-events: all;

packages/editor/src/components/properties/PostProcessingSettingsEditor.tsx

@@ -153,7 +153,50 @@ const EffectsOptions: EffectOptionsType = {
   ColorDepthEffect: {
     bits: { propertyType: PropertyTypes.Number, name: 'Bits', min: -1, max: 1, step: 0.01 }
   },
-  LinearTosRGBEffect: {}
+  LinearTosRGBEffect: {},
+  SSGIEffect: {
+    distance: { propertyType: PropertyTypes.Number, name: 'Distance', min: 0.001, max: 10, step: 0.01 },
+    thickness: { propertyType: PropertyTypes.Number, name: 'Thickness', min: 0, max: 5, step: 0.01 },
+    autoThickness: { propertyType: PropertyTypes.Boolean, name: 'Auto Thickness' },
+    maxRoughness: { propertyType: PropertyTypes.Number, name: 'Max Roughness', min: 0, max: 1, step: 0.01 },
+    blend: { propertyType: PropertyTypes.Number, name: 'Blend', min: 0, max: 1, step: 0.001 },
+    denoiseIterations: { propertyType: PropertyTypes.Number, name: 'Denoise Iterations', min: 0, max: 5, step: 1 },
+    denoiseKernel: { propertyType: PropertyTypes.Number, name: 'Denoise Kernel', min: 1, max: 5, step: 1 },
+    denoiseDiffuse: { propertyType: PropertyTypes.Number, name: 'Denoise Diffuse', min: 0, max: 50, step: 0.01 },
+    denoiseSpecular: { propertyType: PropertyTypes.Number, name: 'Denoise Specular', min: 0, max: 50, step: 0.01 },
+    depthPhi: { propertyType: PropertyTypes.Number, name: 'Depth Phi', min: 0, max: 15, step: 0.001 },
+    normalPhi: { propertyType: PropertyTypes.Number, name: 'Normal Phi', min: 0, max: 50, step: 0.001 },
+    roughnessPhi: { propertyType: PropertyTypes.Number, name: 'Roughness Phi', min: 0, max: 100, step: 0.001 },
+    envBlur: { propertyType: PropertyTypes.Number, name: 'Environment Blur', min: 0, max: 1, step: 0.01 },
+    importanceSampling: { propertyType: PropertyTypes.Boolean, name: 'Importance Sampling' },
+    directLightMultiplier: {
+      propertyType: PropertyTypes.Number,
+      name: 'Direct Light Multiplier',
+      min: 0.001,
+      max: 10,
+      step: 0.01
+    },
+    steps: { propertyType: PropertyTypes.Number, name: 'Steps', min: 0, max: 256, step: 1 },
+    refineSteps: { propertyType: PropertyTypes.Number, name: 'Refine Steps', min: 0, max: 16, step: 1 },
+    spp: { propertyType: PropertyTypes.Number, name: 'SPP', min: 1, max: 32, step: 1 },
+    resolutionScale: { propertyType: PropertyTypes.Number, name: 'Resolution Scale', min: 0.25, max: 1, step: 0.25 },
+    missedRays: { propertyType: PropertyTypes.Boolean, name: 'Missed Rays' }
+  },
+  TRAAEffect: {
+    blend: { propertyType: PropertyTypes.Number, name: 'Blend', min: 0, max: 1, step: 0.001 },
+    constantBlend: { propertyType: PropertyTypes.Boolean, name: 'Constant Blend' },
+    dilation: { propertyType: PropertyTypes.Boolean, name: 'Dilation' },
+    blockySampling: { propertyType: PropertyTypes.Boolean, name: 'Blocky Sampling' },
+    logTransform: { propertyType: PropertyTypes.Boolean, name: 'Log Transform' },
+    depthDistance: { propertyType: PropertyTypes.Number, name: 'Depth Distance', min: 0.01, max: 100, step: 0.01 },
+    worldDistance: { propertyType: PropertyTypes.Number, name: 'World Distance', min: 0.01, max: 100, step: 0.01 },
+    neighborhoodClamping: { propertyType: PropertyTypes.Boolean, name: 'Neighborhood Clamping' }
+  },
+  MotionBlurEffect: {
+    intensity: { propertyType: PropertyTypes.Number, name: 'Intensity', min: 0, max: 10, step: 0.01 },
+    jitter: { propertyType: PropertyTypes.Number, name: 'Jitter', min: 0, max: 10, step: 0.01 },
+    samples: { propertyType: PropertyTypes.Number, name: 'Samples', min: 1, max: 64, step: 1 }
+  }
 }
 
 const BlendFunctionSelect = Object.entries(BlendFunction).map(([label, value]) => {

packages/editor/src/functions/EditorControlFunctions.test.ts

@@ -2,6 +2,8 @@ import assert from 'assert'
 import { Vector3 } from 'three'
 
 import { destroyEngine, Engine } from '@etherealengine/engine/src/ecs/classes/Engine'
+import { Entity } from '@etherealengine/engine/src/ecs/classes/Entity'
+import { SceneState } from '@etherealengine/engine/src/ecs/classes/Scene'
 import {
   addComponent,
   defineComponent,
@@ -25,11 +27,6 @@ import { applyIncomingActions, getState } from '@etherealengine/hyperflux'
 import { registerEditorReceptors } from '../services/EditorServicesReceptor'
 import { EditorControlFunctions } from './EditorControlFunctions'
 
-import '@etherealengine/engine/src/patchEngineNode'
-
-import { Entity } from '@etherealengine/engine/src/ecs/classes/Entity'
-import { SceneState } from '@etherealengine/engine/src/ecs/classes/Scene'
-
 class TempProp {
   data: number
   constructor(data: number) {

packages/engine/package.json

@@ -51,6 +51,7 @@
     "lodash": "4.17.21",
     "postprocessing": "^6.30.1",
     "react": "18.2.0",
+    "realism-effects": "1.0.17",
     "screen-space-reflections": "^2.5.0",
     "sharp": "^0.31.3",
     "simplex-noise": "^4.0.1",
@@ -59,7 +60,6 @@
     "three.quarks": "0.8.6",
     "ts-matches": "5.3.0",
     "typescript": "5.0.2",
-    "url-polyfill": "1.1.12",
     "vite": "4.2.1",
     "web-worker": "1.2.0"
   },

packages/engine/src/ecs/classes/Engine.ts

@@ -8,6 +8,7 @@ import { HyperStore } from '@etherealengine/hyperflux/functions/StoreFunctions'
 import { NetworkTopics } from '../../networking/classes/Network'
 
 import '../utils/threejsPatches'
+import '../../patchEngineNode'
 
 import { EventQueue } from '@dimforge/rapier3d-compat'
 import type { FeathersApplication } from '@feathersjs/feathers'

packages/engine/src/patchEngineNode.ts

@@ -1,75 +1,77 @@
-import { Blob } from 'buffer'
-import fetch, { Headers, Request, Response } from 'node-fetch'
+import { isNode } from './common/functions/getEnvironment'
 
-// needed for URL.createObjectURL
-import 'url-polyfill'
+if (isNode) {
+  const { Blob } = require('buffer')
+  const fetch = require('node-fetch')
 
-;(globalThis as any).fetch = fetch
-;(globalThis as any).Request = Request
-;(globalThis as any).Response = Response
-;(globalThis as any).Headers = Headers
-;(globalThis as any).self = globalThis
+  if (!globalThis.fetch) globalThis.fetch = fetch as any
+  if (!globalThis.Request) globalThis.Request = fetch.Request as any
+  if (!globalThis.Response) globalThis.Response = fetch.Response as any
+  if (!globalThis.Headers) globalThis.Headers = fetch.Headers as any
+  if (!globalThis.self) globalThis.self = globalThis as any
 
-// import URL from 'url'
-;(globalThis as any).self.URL = URL
-;(globalThis as any).Blob = Blob
+  // this will be added in node 19
+  if (!globalThis.URL.createObjectURL) globalThis.URL.createObjectURL = (blob) => null!
+  if (!globalThis.Blob) globalThis.Blob = Blob as any
 
-const _localStorage = {} as any
-;(globalThis as any).localStorage = {
-  setItem: (key, val) => {
-    _localStorage[key] = val
-  },
-  getItem: (key) => {
-    return _localStorage[key] ?? null
-  }
-}
-
-// patches for headless-gl - currently unused
-
-/*
-// patch navigator
-if (!globalThis.navigator)
-  (globalThis as any).navigator = {
-    product: 'NativeScript', // patch axios so it doesnt complain,
-    userAgent: 'node'
-  }
-
-// todo: move this out of module scope
-function addEventListener(event, func, bind_) {}
-
-// patch window prop for three
-if (!globalThis.window) (globalThis as any).window = {}
-Object.assign((globalThis as any).window, {
-  innerWidth: 1920,
-  innerHeight: 1080,
-  addEventListener,
-  URL
-})
+  const _localStorage = {} as any
+  if (!globalThis.localStorage)
+    globalThis.localStorage = {
+      setItem: (key, val) => {
+        _localStorage[key] = val
+      },
+      getItem: (key) => {
+        return _localStorage[key] ?? null
+      }
+    } as any
 
-class Image {}
+  // patches for headless-gl - currently unused
 
-// patch three ImageLoader
-if (!globalThis.document) (globalThis as any).document = {}
-Object.assign((globalThis as any).document, {
-  createElement: (type, ...args) => {
-    switch (type) {
-      case 'div': // patch for sinon
-      default:
-        return
+  /*
+  // patch navigator
+  if (!globalThis.navigator)
+    (globalThis as any).navigator = {
+      product: 'NativeScript', // patch axios so it doesnt complain,
+      userAgent: 'node'
     }
-  },
-  URL,
-  createElementNS: (ns, type) => {
-    if (type === 'img') {
-      const img = new Image() as any
-      img.addEventListener = (type, handler) => {
-        img['on' + type] = handler.bind(img)
+
+  // todo: move this out of module scope
+  function addEventListener(event, func, bind_) {}
+
+  // patch window prop for three
+  if (!globalThis.window) (globalThis as any).window = {}
+  Object.assign((globalThis as any).window, {
+    innerWidth: 1920,
+    innerHeight: 1080,
+    addEventListener,
+    URL
+  })
+
+  class Image {}
+
+  // patch three ImageLoader
+  if (!globalThis.document) (globalThis as any).document = {}
+  Object.assign((globalThis as any).document, {
+    createElement: (type, ...args) => {
+      switch (type) {
+        case 'div': // patch for sinon
+        default:
+          return
       }
-      img.removeEventListener = (type) => {
-        img['on' + type] = null
+    },
+    URL,
+    createElementNS: (ns, type) => {
+      if (type === 'img') {
+        const img = new Image() as any
+        img.addEventListener = (type, handler) => {
+          img['on' + type] = handler.bind(img)
+        }
+        img.removeEventListener = (type) => {
+          img['on' + type] = null
+        }
+        return img
       }
-      return img
     }
-  }
-})
-*/
+  })
+  */
+}

packages/engine/src/renderer/functions/configureEffectComposer.ts

@@ -1,4 +1,5 @@
 import { BlendFunction, DepthDownsamplingPass, EffectPass, NormalPass, RenderPass, TextureEffect } from 'postprocessing'
+import { VelocityDepthNormalPass } from 'realism-effects'
 import { NearestFilter, PerspectiveCamera, RGBAFormat, WebGLRenderTarget } from 'three'
 
 import { getMutableState, NO_PROXY } from '@etherealengine/hyperflux'
@@ -12,9 +13,11 @@ import { changeRenderMode } from './changeRenderMode'
 export const configureEffectComposer = (remove?: boolean, camera: PerspectiveCamera = Engine.instance.camera): void => {
   if (!EngineRenderer.instance) return
 
+  const scene = Engine.instance.scene
+
   if (!EngineRenderer.instance.renderPass) {
     // we always want to have at least the render pass enabled
-    const renderPass = new RenderPass(Engine.instance.scene, camera)
+    const renderPass = new RenderPass(scene, camera)
     EngineRenderer.instance.effectComposer.addPass(renderPass)
     EngineRenderer.instance.renderPass = renderPass
   }
@@ -35,7 +38,9 @@ export const configureEffectComposer = (remove?: boolean, camera: PerspectiveCam
   const effects: any[] = []
   const effectKeys = EffectMap.keys()
 
-  const normalPass = new NormalPass(Engine.instance.scene, camera, {
+  const composer = EngineRenderer.instance.effectComposer
+
+  const normalPass = new NormalPass(scene, camera, {
     renderTarget: new WebGLRenderTarget(1, 1, {
       minFilter: NearestFilter,
       magFilter: NearestFilter,
@@ -49,6 +54,9 @@ export const configureEffectComposer = (remove?: boolean, camera: PerspectiveCam
     resolutionScale: 0.5
   })
 
+  const velocityDepthNormalPass = new VelocityDepthNormalPass(scene, camera)
+  let useVelocityDepthNormalPass = false
+
   for (let key of effectKeys) {
     const effect = postProcessingEffects[key]
 
@@ -62,24 +70,41 @@ export const configureEffectComposer = (remove?: boolean, camera: PerspectiveCam
         ...effect,
         normalDepthBuffer: depthDownsamplingPass.texture
       })
-      EngineRenderer.instance.effectComposer[key] = eff
+      composer[key] = eff
       effects.push(eff)
     } else if (key === Effects.SSREffect) {
-      const eff = new effectClass(Engine.instance.scene, camera, effect)
-      EngineRenderer.instance.effectComposer[key] = eff
+      const eff = new effectClass(scene, camera, effect)
+      composer[key] = eff
       effects.push(eff)
     } else if (key === Effects.DepthOfFieldEffect) {
       const eff = new effectClass(camera, effect)
-      EngineRenderer.instance.effectComposer[key] = eff
+      composer[key] = eff
       effects.push(eff)
     } else if (key === Effects.OutlineEffect) {
-      const eff = new effectClass(Engine.instance.scene, camera, effect)
-      EngineRenderer.instance.effectComposer[key] = eff
+      const eff = new effectClass(scene, camera, effect)
+      composer[key] = eff
+      effects.push(eff)
+    } else if (key === Effects.SSGIEffect) {
+      const eff = new effectClass(scene, camera, velocityDepthNormalPass, effect)
+      useVelocityDepthNormalPass = true
+      composer[key] = eff
+      effects.push(eff)
+    } else if (key === Effects.TRAAEffect) {
+      // todo support more than 1 texture
+      const textureCount = 1
+      const eff = new effectClass(scene, camera, velocityDepthNormalPass, textureCount, effect)
+      useVelocityDepthNormalPass = true
+      composer[key] = eff
+      effects.push(eff)
+    } else if (key === Effects.MotionBlurEffect) {
+      const eff = new effectClass(velocityDepthNormalPass, effect)
+      useVelocityDepthNormalPass = true
+      composer[key] = eff
       effects.push(eff)
     } else {
       if (effectClass) {
         const eff = new effectClass(effect)
-        EngineRenderer.instance.effectComposer[key] = eff
+        composer[key] = eff
         effects.push(eff)
       }
     }
@@ -91,8 +116,10 @@ export const configureEffectComposer = (remove?: boolean, camera: PerspectiveCam
       texture: depthDownsamplingPass.texture
     })
 
-    EngineRenderer.instance.effectComposer.addPass(depthDownsamplingPass)
-    EngineRenderer.instance.effectComposer.addPass(new EffectPass(camera, ...effects, textureEffect))
+    if (useVelocityDepthNormalPass) composer.addPass(velocityDepthNormalPass)
+
+    composer.addPass(depthDownsamplingPass)
+    composer.addPass(new EffectPass(camera, ...effects, textureEffect))
   }
 
   if (getMutableState(EngineState).isEditor.value) changeRenderMode()