Fix ShaderLab variable list declaration & CRLF break & macro #if bug (

#2465) * fix: supprot global declaration list * fix: `#if` bug when macro #define no body * fix: compatible with CRLF line break
galacean · Jan 8, 2025 · b9b2e1c · b9b2e1c
1 parent 0c340a2
commit b9b2e1c
Show file tree

Hide file tree

Showing 10 changed files with 124 additions and 25 deletions.
diff --git a/packages/shader-lab/src/common/BaseScanner.ts b/packages/shader-lab/src/common/BaseScanner.ts
@@ -11,7 +11,7 @@ export type OnToken = (token: BaseToken, scanner: BaseScanner) => void;
  * @internal
  */
 export default class BaseScanner {
-  private static _spaceCharsWithBreak = [" ", "\t", "\n"];
+  private static _spaceCharsWithBreak = [" ", "\t", "\n", "\r"];
   private static _spaceChars = [" ", "\t"];
   private static _checkIsIn(checked: string, chars: string[]): boolean {
     for (let i = 0; i < chars.length; i++) {
@@ -117,7 +117,11 @@ export default class BaseScanner {
       const start = this.getCurPosition();
       this.advance(2);
       // single line comments
-      while (this.getCurChar() !== "\n") this._advance();
+      let curChar = this.getCurChar();
+      while (curChar !== "\n" && curChar !== "\r" && !this.isEnd()) {
+        this._advance();
+        curChar = this.getCurChar();
+      }
       this.skipCommentsAndSpace();
       return ShaderLab.createRange(start, this.getCurPosition());
     } else if (this.peek(2) === "/*") {

diff --git a/packages/shader-lab/src/common/types.ts b/packages/shader-lab/src/common/types.ts
@@ -134,6 +134,7 @@ export enum ETokenType {
   COLON,
   /** = */
   EQUAL,
+  /** ; */
   SEMICOLON,
   /** ! */
   BANG,

diff --git a/packages/shader-lab/src/lalr/CFG.ts b/packages/shader-lab/src/lalr/CFG.ts
@@ -16,21 +16,34 @@ const productionAndRules: [GrammarSymbol[], TranslationRule | undefined][] = [
 
   ...GrammarUtils.createProductionWithOptions(NoneTerminal.global_declaration, [
     [NoneTerminal.precision_specifier],
-    [NoneTerminal.variable_declaration],
+    [NoneTerminal.variable_declaration_statement],
     [NoneTerminal.struct_specifier],
     [NoneTerminal.function_definition]
   ]),
 
   ...GrammarUtils.createProductionWithOptions(
     NoneTerminal.variable_declaration,
     [
-      [EKeyword.GS_RenderQueueType, ETokenType.ID, ETokenType.SEMICOLON],
-      [NoneTerminal.fully_specified_type, ETokenType.ID, ETokenType.SEMICOLON],
-      [NoneTerminal.fully_specified_type, ETokenType.ID, NoneTerminal.array_specifier, ETokenType.SEMICOLON]
+      [NoneTerminal.fully_specified_type, ETokenType.ID],
+      [NoneTerminal.fully_specified_type, ETokenType.ID, NoneTerminal.array_specifier]
     ],
     ASTNode.VariableDeclaration.pool
   ),
 
+  ...GrammarUtils.createProductionWithOptions(
+    NoneTerminal.variable_declaration_list,
+    [
+      [NoneTerminal.variable_declaration],
+      [NoneTerminal.variable_declaration_list, ETokenType.COMMA, ETokenType.ID],
+      [NoneTerminal.variable_declaration_list, ETokenType.COMMA, ETokenType.ID, NoneTerminal.array_specifier]
+    ],
+    ASTNode.VariableDeclarationList.pool
+  ),
+
+  ...GrammarUtils.createProductionWithOptions(NoneTerminal.variable_declaration_statement, [
+    [NoneTerminal.variable_declaration_list, ETokenType.SEMICOLON]
+  ]),
+
   ...GrammarUtils.createProductionWithOptions(
     NoneTerminal.ext_builtin_type_specifier_nonarray,
     [

diff --git a/packages/shader-lab/src/lexer/Utils.ts b/packages/shader-lab/src/lexer/Utils.ts
@@ -34,7 +34,8 @@ export default class LexerUtils {
   static isSpace(charCode: number) {
     return (
       charCode === 9 || // Tab
-      charCode === 10 || // Line break
+      charCode === 10 || // Line break - /n
+      charCode === 13 || // Carriage return -/r
       charCode === 32 // Space
     );
   }

diff --git a/packages/shader-lab/src/parser/AST.ts b/packages/shader-lab/src/parser/AST.ts
@@ -51,6 +51,9 @@ export abstract class TreeNode implements IPoolElement {
     return visitor.defaultCodeGen(this.children);
   }
 
+  /**
+   * Do semantic analyze right after the ast node is generated.
+   */
   semanticAnalyze(sa: SematicAnalyzer) {}
 }
 
@@ -178,21 +181,31 @@ export namespace ASTNode {
     }
 
     override semanticAnalyze(sa: SematicAnalyzer): void {
-      const fullyType = this.children[0] as FullySpecifiedType;
-      const id = this.children[1] as Token;
-      this.typeSpecifier = fullyType.typeSpecifier;
+      const children = this.children;
+      const childrenLen = children.length;
+      const fullyType = children[0] as FullySpecifiedType;
+      const typeSpecifier = fullyType.typeSpecifier;
+      this.typeSpecifier = typeSpecifier;
+      this.arraySpecifier = typeSpecifier.arraySpecifier;
+
+      const id = children[1] as Token;
 
       let sm: VarSymbol;
-      if (this.children.length === 2 || this.children.length === 4) {
-        const symbolType = new SymbolType(fullyType.type, fullyType.typeSpecifier.lexeme);
-        const initializer = this.children[3] as Initializer;
+      if (childrenLen === 2 || childrenLen === 4) {
+        const symbolType = new SymbolType(fullyType.type, typeSpecifier.lexeme, this.arraySpecifier);
+        const initializer = children[3] as Initializer;
 
         sm = new VarSymbol(id.lexeme, symbolType, false, initializer);
       } else {
-        const arraySpecifier = this.children[2] as ArraySpecifier;
+        const arraySpecifier = children[2] as ArraySpecifier;
+        // #if _VERBOSE
+        if (arraySpecifier && this.arraySpecifier) {
+          sa.reportError(arraySpecifier.location, "Array of array is not supported.");
+        }
+        // #endif
         this.arraySpecifier = arraySpecifier;
-        const symbolType = new SymbolType(fullyType.type, fullyType.typeSpecifier.lexeme, arraySpecifier);
-        const initializer = this.children[4] as Initializer;
+        const symbolType = new SymbolType(fullyType.type, typeSpecifier.lexeme, this.arraySpecifier);
+        const initializer = children[4] as Initializer;
 
         sm = new VarSymbol(id.lexeme, symbolType, false, initializer);
       }
@@ -288,6 +301,9 @@ export namespace ASTNode {
     get arraySize(): number {
       return (this.children?.[1] as ArraySpecifier)?.size;
     }
+    get arraySpecifier(): ArraySpecifier {
+      return this.children[1] as ArraySpecifier;
+    }
 
     get isCustom() {
       return typeof this.type === "string";
@@ -1110,17 +1126,49 @@ export namespace ASTNode {
 
   @ASTNodeDecorator(NoneTerminal.variable_declaration)
   export class VariableDeclaration extends TreeNode {
+    type: FullySpecifiedType;
+
     override semanticAnalyze(sa: SematicAnalyzer): void {
-      const type = this.children[0] as FullySpecifiedType;
-      const ident = this.children[1] as Token;
-      let sm: VarSymbol;
-      sm = new VarSymbol(ident.lexeme, new SymbolType(type.type, type.typeSpecifier.lexeme), true, this);
+      const children = this.children;
+      const type = children[0] as FullySpecifiedType;
+      const ident = children[1] as Token;
+      this.type = type;
+      const sm = new VarSymbol(ident.lexeme, new SymbolType(type.type, type.typeSpecifier.lexeme), true, this);
 
       sa.symbolTableStack.insert(sm);
     }
 
     override codeGen(visitor: CodeGenVisitor): string {
-      return visitor.visitGlobalVariableDeclaration(this);
+      return visitor.visitGlobalVariableDeclaration(this) + ";";
+    }
+  }
+
+  @ASTNodeDecorator(NoneTerminal.variable_declaration_list)
+  export class VariableDeclarationList extends TreeNode {
+    type: FullySpecifiedType;
+
+    override semanticAnalyze(sa: SematicAnalyzer): void {
+      const { children } = this;
+      const length = children.length;
+      const variableDeclaration = children[0] as VariableDeclaration;
+      const type = variableDeclaration.type;
+      this.type = type;
+
+      if (length === 1) {
+        return;
+      }
+
+      const ident = children[2] as Token;
+
+      const newVariable = VariableDeclaration.pool.get();
+      if (length === 3) {
+        // variable_declaration_list ',' id
+        newVariable.set(ident.location, [type, ident]);
+      } else {
+        // variable_declaration_list ',' id array_specifier
+        newVariable.set(ident.location, [type, ident, children[3] as ArraySpecifier]);
+      }
+      newVariable.semanticAnalyze(sa);
     }
   }
 

diff --git a/packages/shader-lab/src/parser/GrammarSymbol.ts b/packages/shader-lab/src/parser/GrammarSymbol.ts
@@ -10,6 +10,8 @@ export enum NoneTerminal {
   // glsl
   global_declaration,
   variable_declaration,
+  variable_declaration_list,
+  variable_declaration_statement,
   array_specifier_list,
   array_specifier,
   ext_builtin_type_specifier_nonarray,

diff --git a/packages/shader-lab/src/parser/TargetParser.y b/packages/shader-lab/src/parser/TargetParser.y
@@ -66,16 +66,25 @@ gs_shader_program:
 
 global_declaration:
     precision_specifier
-    | variable_declaration
+    | variable_declaration_statement
     | struct_specifier
     | function_definition
     ;
 
 variable_declaration:
-    fully_specified_type id ';'
-    | fully_specified_type id array_specifier ';'
+    fully_specified_type id
+    | fully_specified_type id array_specifier
     ;
 
+variable_declaration_list:
+    variable_declaration
+    | variable_declaration_list ',' id
+    | variable_declaration_list ',' id array_specifier
+    ;
+
+variable_declaration_statement:
+    variable_declaration_list ';'
+
 variable_identifier:
     id
     ;

diff --git a/packages/shader-lab/src/parser/builtin/functions.ts b/packages/shader-lab/src/parser/builtin/functions.ts
@@ -286,10 +286,13 @@ BuiltinFunction._create("textureLod", EGenType.GVec4, EGenType.GSampler3D, EKeyw
 BuiltinFunction._create("textureLod", EGenType.GVec4, EGenType.GSamplerCube, EKeyword.VEC3, EKeyword.FLOAT);
 BuiltinFunction._create("textureLod", EKeyword.FLOAT, EKeyword.SAMPLER2D_SHADOW, EKeyword.VEC3, EKeyword.FLOAT);
 BuiltinFunction._create("textureLod", EGenType.GVec4, EGenType.GSampler2DArray, EKeyword.VEC3, EKeyword.FLOAT);
+BuiltinFunction._create("texture2DLodEXT", EGenType.GVec4, EGenType.GSampler2D, EKeyword.VEC2, EKeyword.FLOAT);
+BuiltinFunction._create("texture2DLodEXT", EGenType.GVec4, EGenType.GSampler3D, EKeyword.VEC3, EKeyword.FLOAT);
 
 BuiltinFunction._create("textureCube", EKeyword.SAMPLER_CUBE, EKeyword.VEC3);
 BuiltinFunction._create("textureCube", EKeyword.SAMPLER_CUBE, EKeyword.VEC3, EKeyword.FLOAT);
 BuiltinFunction._create("textureCubeLod", EKeyword.SAMPLER_CUBE, EKeyword.VEC3, EKeyword.FLOAT);
+BuiltinFunction._create("textureCubeLodEXT", EGenType.GVec4, EGenType.GSamplerCube, EKeyword.VEC3, EKeyword.FLOAT);
 
 BuiltinFunction._create(
   "textureOffset",

diff --git a/packages/shader-lab/src/preprocessor/PpParser.ts b/packages/shader-lab/src/preprocessor/PpParser.ts
@@ -439,7 +439,7 @@ export class PpParser {
         return !!this._definedMacros.get(macro.lexeme);
       } else {
         const macro = this._definedMacros.get(id.lexeme);
-        if (!macro) {
+        if (!macro || !macro.body) {
           return false;
         }
         if (macro.isFunction) {

diff --git a/tests/src/shader-lab/shaders/demo.shader b/tests/src/shader-lab/shaders/demo.shader
@@ -84,9 +84,20 @@ Shader "Water" {
 
     /*Comment without leading space*/
 
+    // test global declaration list.
+    vec2 v1, v2[2], v3[3];
+
       v2f vert(a2v v) {
         v2f o;
 
+        vec2 weights[2], offsets[2];
+        weights[0] = vec2(.1);
+        offsets[1] = vec2(.1);
+
+        float[2] c;
+        c[0] = 1.0;
+        c[1] = .4;
+
         o.v_uv = v.TEXCOORD_0;
         vec4 tmp = renderer_MVMat * v.POSITION;
         o.v_position = tmp.xyz;
@@ -109,6 +120,13 @@ Shader "Water" {
           gl_FragColor = linearToGamma(gl_FragColor);
         #endif
 
+
+#define REFRACTION_MODE
+
+#if REFRACTION_MODE == 1
+
+#endif
+
         // For testing only (macro)
         #if SCENE_SHADOW_TYPE == 2 || defined(XX_Macro)
           gl_FragColor = linearToGamma(gl_FragColor);