Correct clipping of children in render-to-texture nodes (#458)

This PR resolves an issue where children nodes rendered as part of a
render-to-texture were not properly clipped according to the parent
node's bounds. The fix ensures that clipping is correctly applied within
the render-to-texture pipeline, maintaining visual consistency and
expected behavior for nested scenes

examples/tests/rtt-nested-clipping.ts

@@ -0,0 +1,76 @@
+import type { ExampleSettings } from '../common/ExampleSettings.js';
+import rocko from '../assets/rocko.png';
+
+export default async function test({ renderer, testRoot }: ExampleSettings) {
+  const node = renderer.createNode({
+    x: 0,
+    y: 0,
+    width: 1920,
+    height: 1080,
+    color: 0x000000ff,
+    parent: testRoot,
+  });
+
+  // RTT Node 1
+  const rttNode = renderer.createNode({
+    x: 100,
+    y: 200,
+    width: 400,
+    height: 400,
+    parent: node,
+    rtt: true,
+    zIndex: 5,
+    colorTop: 0xfff00fff,
+    colorBottom: 0x00ffffff,
+  });
+
+  const rect = renderer.createNode({
+    x: 0,
+    y: 0,
+    width: 300,
+    height: 300,
+    parent: rttNode,
+    clipping: true,
+    color: 0xff0000ff,
+  });
+
+  renderer.createTextNode({
+    x: 0,
+    y: 0,
+    text: 'Render to texture',
+    parent: rttNode,
+    fontSize: 48,
+    color: 0xffffffff,
+    fontFamily: 'Ubuntu',
+  });
+
+  renderer.createNode({
+    x: 50,
+    y: 100,
+    width: 600,
+    height: 600,
+    parent: rttNode,
+    src: rocko,
+  });
+
+  window.addEventListener('keydown', (e) => {
+    if (e.key === 's') {
+      // rect.clipping = !rect.clipping;
+      rect
+        .animate(
+          {
+            x: 100, //going to render out of bounds as well
+          },
+          {
+            duration: 3000,
+            easing: 'ease-out',
+            loop: true,
+            stopMethod: 'reverse',
+          },
+        )
+        .start();
+    }
+  });
+
+  // Define the page function to configure different test scenarios
+}

src/core/renderers/webgl/WebGlCoreRenderOp.ts

@@ -23,11 +23,9 @@ import type { WebGlCoreCtxTexture } from './WebGlCoreCtxTexture.js';
 import type { WebGlCoreRendererOptions } from './WebGlCoreRenderer.js';
 import type { BufferCollection } from './internal/BufferCollection.js';
 import type { Dimensions } from '../../../common/CommonTypes.js';
-import type { Rect, RectWithValid } from '../../lib/utils.js';
+import type { RectWithValid } from '../../lib/utils.js';
 import type { WebGlContextWrapper } from '../../lib/WebGlContextWrapper.js';
 
-const MAX_TEXTURES = 8; // TODO: get from gl
-
 /**
  * Can render multiple quads with multiple textures (up to vertex shader texture limit)
  *
@@ -95,13 +93,22 @@ export class WebGlCoreRenderOp extends CoreRenderOp {
     // Clipping
     if (this.clippingRect.valid) {
       const { x, y, width, height } = this.clippingRect;
-      const pixelRatio = options.pixelRatio;
+      const pixelRatio = this.parentHasRenderTexture ? 1 : options.pixelRatio;
       const canvasHeight = options.canvas.height;
 
       const clipX = Math.round(x * pixelRatio);
       const clipWidth = Math.round(width * pixelRatio);
       const clipHeight = Math.round(height * pixelRatio);
-      const clipY = Math.round(canvasHeight - clipHeight - y * pixelRatio);
+      let clipY = Math.round(canvasHeight - clipHeight - y * pixelRatio);
+
+      // if parent has render texture, we need to adjust the scissor rect
+      // to be relative to the parent's framebuffer
+      if (this.parentHasRenderTexture) {
+        clipY = this.framebufferDimensions
+          ? this.framebufferDimensions.height - this.dimensions.height
+          : 0;
+      }
+
       glw.setScissorTest(true);
       glw.scissor(clipX, clipY, clipWidth, clipHeight);
     } else {