Fix ShaderLab variable list declaration & CRLF break & macro #if bug

packages/shader-lab/src/codeGen/CodeGenVisitor.ts

@@ -125,7 +125,7 @@ export abstract class CodeGenVisitor {
     return this.defaultCodeGen(node.children);
   }
 
-  visitGlobalVariableDeclaration(node: ASTNode.VariableDeclaration): string {
+  visitVariableDeclaration(node: ASTNode.VariableDeclaration): string {
     const fullType = node.children[0];
     if (fullType instanceof ASTNode.FullySpecifiedType && fullType.typeSpecifier.isCustom) {
       VisitorContext.context.referenceGlobal(<string>fullType.type, ESymbolType.STRUCT);

packages/shader-lab/src/codeGen/GLES300.ts

@@ -1,7 +1,7 @@
 import { ASTNode } from "../parser/AST";
 import { SymbolType } from "../parser/types";
 import { BaseToken as Token } from "../common/BaseToken";
-import { EKeyword, ETokenType, ShaderPosition } from "../common";
+import { EKeyword, ETokenType } from "../common";
 import { GLESVisitor } from "./GLESVisitor";
 import { EShaderStage } from "../common/Enums";
 import { ICodeSegment } from "./types";

packages/shader-lab/src/common/BaseScanner.ts

@@ -117,7 +117,7 @@ export default class BaseScanner {
       const start = this.getCurPosition();
       this.advance(2);
       // single line comments
-      while (this.getCurChar() !== "\n") this._advance();
+      while (this.getCurChar() !== "\n" && !this.isEnd()) this._advance();
       this.skipCommentsAndSpace();
       return ShaderLab.createRange(start, this.getCurPosition());
     } else if (this.peek(2) === "/*") {

packages/shader-lab/src/common/types.ts

@@ -132,6 +132,7 @@ export enum ETokenType {
   COLON,
   /** = */
   EQUAL,
+  /** ; */
   SEMICOLON,
   /** ! */
   BANG,

packages/shader-lab/src/lalr/CFG.ts

@@ -16,21 +16,34 @@ const productionAndRules: [GrammarSymbol[], TranslationRule | undefined][] = [
 
   ...GrammarUtils.createProductionWithOptions(ENonTerminal.global_declaration, [
     [ENonTerminal.precision_specifier],
-    [ENonTerminal.variable_declaration],
+    [ENonTerminal.variable_declaration_statement],
     [ENonTerminal.struct_specifier],
     [ENonTerminal.function_definition]
   ]),
 
   ...GrammarUtils.createProductionWithOptions(
     ENonTerminal.variable_declaration,
     [
-      [EKeyword.GS_RenderQueueType, ETokenType.ID, ETokenType.SEMICOLON],
-      [ENonTerminal.fully_specified_type, ETokenType.ID, ETokenType.SEMICOLON],
-      [ENonTerminal.fully_specified_type, ETokenType.ID, ENonTerminal.array_specifier, ETokenType.SEMICOLON]
+      [ENonTerminal.fully_specified_type, ETokenType.ID],
+      [ENonTerminal.fully_specified_type, ETokenType.ID, ENonTerminal.array_specifier]
     ],
     ASTNode.VariableDeclaration.pool
   ),
 
+  ...GrammarUtils.createProductionWithOptions(
+    ENonTerminal.variable_declaration_list,
+    [
+      [ENonTerminal.variable_declaration],
+      [ENonTerminal.variable_declaration_list, ETokenType.COMMA, ETokenType.ID],
+      [ENonTerminal.variable_declaration_list, ETokenType.COMMA, ETokenType.ID, ENonTerminal.array_specifier]
+    ],
+    ASTNode.VariableDeclarationList.pool
+  ),
+
+  ...GrammarUtils.createProductionWithOptions(ENonTerminal.variable_declaration_statement, [
+    [ENonTerminal.variable_declaration_list, ETokenType.SEMICOLON]
+  ]),
+
   ...GrammarUtils.createProductionWithOptions(
     ENonTerminal.ext_builtin_type_specifier_nonarray,
     [

packages/shader-lab/src/lalr/Utils.ts

@@ -44,17 +44,6 @@ export default class GrammarUtils {
     return ret;
   }
 
-  static createProductionOptions(common: GrammarSymbol[], position: number, opts: GrammarSymbol[][]) {
-    const ret: GrammarSymbol[][] = [];
-    for (const opt of opts) {
-      const list = common.slice(0, position);
-      list.push(...opt);
-      list.push(...common.slice(position));
-      ret.push(list);
-    }
-    return ret;
-  }
-
   static addMapSetItem<K, T>(map: Map<K, Set<T>>, k: K, v: T) {
     const set = map.get(k) ?? new Set();
     set.add(v);

packages/shader-lab/src/parser/AST.ts

@@ -234,17 +234,23 @@
       const fullyType = this.children[0] as FullySpecifiedType;
       const id = this.children[1] as Token;
       this.typeSpecifier = fullyType.typeSpecifier;
+      this.arraySpecifier = fullyType.typeSpecifier.arraySpecifier;
 
       let sm: VarSymbol;
       if (this.children.length === 2 || this.children.length === 4) {
-        const symbolType = new SymbolType(fullyType.type, fullyType.typeSpecifier.lexeme);
+        const symbolType = new SymbolType(fullyType.type, fullyType.typeSpecifier.lexeme, this.arraySpecifier);
         const initializer = this.children[3] as Initializer;
 
         sm = new VarSymbol(id.lexeme, symbolType, false, initializer);
       } else {
         const arraySpecifier = this.children[2] as ArraySpecifier;
+        // #if _VERBOSE
+        if (arraySpecifier && this.arraySpecifier) {
+          sa.error(arraySpecifier.location, "Array of array is not supported.");
+        }
+        // #endif
         this.arraySpecifier = arraySpecifier;
-        const symbolType = new SymbolType(fullyType.type, fullyType.typeSpecifier.lexeme, arraySpecifier);
+        const symbolType = new SymbolType(fullyType.type, fullyType.typeSpecifier.lexeme, this.arraySpecifier);
         const initializer = this.children[4] as Initializer;
 
         sm = new VarSymbol(id.lexeme, symbolType, false, initializer);
@@ -381,6 +387,9 @@
     get arraySize(): number {
       return (this.children?.[1] as ArraySpecifier)?.size;
     }
+    get arraySpecifier(): ArraySpecifier {
+      return this.children[1] as ArraySpecifier;
+    }
 
     get isCustom() {
       return typeof this.type === "string";
@@ -459,7 +468,7 @@
         else {
           const id = child as VariableIdentifier;
           if (!id.symbolInfo) {
-            sa.error(id.location, "Undeclared symbol:", id.lexeme);
+            sa.error(id.location, `Undeclared symbol: ${id.lexeme}`);
           }
           if (!ParserUtils.typeCompatible(EKeyword.INT, id.typeInfo)) {
             sa.error(id.location, "Invalid integer.");
@@ -533,11 +542,6 @@
       } else if (this.children.length === 4 || this.children.length === 6) {
         const typeInfo = this.typeInfo;
         const arraySpecifier = this.children[3] as ArraySpecifier;
-        // #if _VERBOSE
-        if (typeInfo.arraySpecifier && arraySpecifier) {
-          sa.error(arraySpecifier.location, "Array of array is not supported.");
-        }
-        // #endif
         typeInfo.arraySpecifier = arraySpecifier;
         const id = this.children[2] as Token;
         sm = new VarSymbol(id.lexeme, typeInfo, false, this);
@@ -872,7 +876,7 @@
         const fnSymbol = sa.symbolTable.lookup({ ident: fnIdent, symbolType: ESymbolType.FN, signature: paramSig });
         if (!fnSymbol) {
           // #if _VERBOSE
-          sa.error(this.location, "No overload function type found: ", functionIdentifier.ident);
+          sa.error(this.location, `No overload function type found: ${functionIdentifier.ident}`);
           // #endif
           return;
         }
@@ -1398,21 +1402,57 @@
   export class VariableDeclaration extends TreeNode {
     static pool = ShaderLabUtils.createObjectPool(VariableDeclaration);
 
+    type: FullySpecifiedType;
+
     override set(loc: ShaderRange, children: NodeChild[]) {
       super.set(loc, children, ENonTerminal.variable_declaration);
     }
 
     override semanticAnalyze(sa: SematicAnalyzer): void {
       const type = this.children[0] as FullySpecifiedType;
       const ident = this.children[1] as Token;
-      let sm: VarSymbol;
-      sm = new VarSymbol(ident.lexeme, new SymbolType(type.type, type.typeSpecifier.lexeme), true, this);
+      this.type = type;
+      const sm = new VarSymbol(ident.lexeme, new SymbolType(type.type, type.typeSpecifier.lexeme), true, this);
 
       sa.symbolTable.insert(sm);
     }
 
     override codeGen(visitor: CodeGenVisitor): string {
-      return visitor.visitGlobalVariableDeclaration(this);
+      return visitor.visitVariableDeclaration(this) + ";";
+    }
+  }
+
+  export class VariableDeclarationList extends TreeNode {
+    static pool = ShaderLabUtils.createObjectPool(VariableDeclarationList);
+
+    type: FullySpecifiedType;
+
+    override set(loc: ShaderRange, children: NodeChild[]): void {
+      super.set(loc, children, ENonTerminal.variable_declaration_list);
+    }
+
+    override semanticAnalyze(sa: SematicAnalyzer): void {
+      const { children } = this;
+      const length = children.length;
+      const variableDeclation = children[0] as VariableDeclaration;
+      const type = variableDeclation.type;
+      this.type = type;
+
+      if (length === 1) {
+        return;
+      }
+
+      const ident = children[2] as Token;
+
+      const newVariable = VariableDeclaration.pool.get();
+      if (length === 3) {
+        // variable_declaration_list ',' id
+        newVariable.set(ident.location, [type, ident]);
+      } else {
+        // variable_declaration_list ',' id array_specifier
+        newVariable.set(ident.location, [type, ident, children[3] as ArraySpecifier]);
+      }
+      newVariable.semanticAnalyze(sa);
     }
   }
 
@@ -1453,7 +1493,7 @@
       this.symbolInfo = sa.symbolTable.lookup({ ident: token.lexeme, symbolType: ESymbolType.VAR }) as VarSymbol;
       // #if _VERBOSE
       if (!this.symbolInfo) {
-        sa.error(this.location, "undeclared identifier:", token.lexeme);
+        sa.error(this.location, `undeclared identifier: ${token.lexeme}`);
       }
       // #endif
     }

packages/shader-lab/src/parser/GrammarSymbol.ts

@@ -10,6 +10,8 @@ export enum ENonTerminal {
   // glsl
   global_declaration,
   variable_declaration,
+  variable_declaration_list,
+  variable_declaration_statement,
   array_specifier_list,
   array_specifier,
   ext_builtin_type_specifier_nonarray,

packages/shader-lab/src/parser/SemanticAnalyzer.ts

@@ -9,6 +9,7 @@ import { ShaderLab } from "../ShaderLab";
 // #if _VERBOSE
 import { GSError } from "../GSError";
 // #endif
+import { Logger } from "@galacean/engine";
 
 export type TranslationRule<T = any> = (sa: SematicAnalyzer, ...tokens: NodeChild[]) => T;
 
@@ -24,7 +25,7 @@ export default class SematicAnalyzer {
   private _shaderData = new ShaderData();
 
   // #if _VERBOSE
-  readonly errors: GSError[] = [];
+  readonly errors: Error[] = [];
   // #endif
 
   get shaderData() {
@@ -64,13 +65,13 @@ export default class SematicAnalyzer {
     return this._translationRuleTable.get(pid);
   }
 
-  error(loc: ShaderRange, ...param: any[]) {
+  error(loc: ShaderRange, message: string) {
     // #if _VERBOSE
-    const err = new GSError(GSErrorName.CompilationError, param.join(""), loc, ShaderLab._processingPassText);
+    const err = new GSError(GSErrorName.CompilationError, message, loc, ShaderLab._processingPassText);
     this.errors.push(err);
     return err;
     // #else
-    throw new Error(param.join(""));
+    Logger.error(message);
     // #endif
   }
 }

packages/shader-lab/src/parser/TargetParser.y

@@ -64,16 +64,25 @@ gs_shader_program:
 
 global_declaration:
     precision_specifier
-    | variable_declaration
+    | variable_declaration_statement
     | struct_specifier
     | function_definition
     ;
 
 variable_declaration:
-    fully_specified_type id ';'
-    | fully_specified_type id array_specifier ';'
+    fully_specified_type id
+    | fully_specified_type id array_specifier
     ;
 
+variable_declaration_list:
+    variable_declaration
+    | variable_declaration_list ',' id
+    | variable_declaration_list ',' id array_specifier
+    ;
+
+variable_declaration_statement:
+    variable_declaration_list ';'
+
 variable_identifier:
     id
     ;

packages/shader-lab/src/parser/builtin/functions.ts

@@ -286,10 +286,13 @@ BuiltinFunction._create("textureLod", EGenType.GVec4, EGenType.GSampler3D, EKeyw
 BuiltinFunction._create("textureLod", EGenType.GVec4, EGenType.GSamplerCube, EKeyword.VEC3, EKeyword.FLOAT);
 BuiltinFunction._create("textureLod", EKeyword.FLOAT, EKeyword.SAMPLER2D_SHADOW, EKeyword.VEC3, EKeyword.FLOAT);
 BuiltinFunction._create("textureLod", EGenType.GVec4, EGenType.GSampler2DArray, EKeyword.VEC3, EKeyword.FLOAT);
+BuiltinFunction._create("texture2DLodEXT", EGenType.GVec4, EGenType.GSampler2D, EKeyword.VEC2, EKeyword.FLOAT);
+BuiltinFunction._create("texture2DLodEXT", EGenType.GVec4, EGenType.GSampler3D, EKeyword.VEC3, EKeyword.FLOAT);
 
 BuiltinFunction._create("textureCube", EKeyword.SAMPLER_CUBE, EKeyword.VEC3);
 BuiltinFunction._create("textureCube", EKeyword.SAMPLER_CUBE, EKeyword.VEC3, EKeyword.FLOAT);
 BuiltinFunction._create("textureCubeLod", EKeyword.SAMPLER_CUBE, EKeyword.VEC3, EKeyword.FLOAT);
+BuiltinFunction._create("textureCubeLodEXT", EGenType.GVec4, EGenType.GSamplerCube, EKeyword.VEC3, EKeyword.FLOAT);
 
 BuiltinFunction._create(
   "textureOffset",

tests/src/shader-lab/shaders/demo.shader

@@ -84,9 +84,20 @@ Shader "Water" {
 
     /*Comment without leading space*/
 
+    // test global declaration list.
+    vec2 v1, v2[2], v3[3];
+
       v2f vert(a2v v) {
         v2f o;
 
+        vec2 weights[2], offsets[2];
+        weights[0] = vec2(.1);
+        offsets[1] = vec2(.1);
+
+        float[2] c;
+        c[0] = 1.0;
+        c[1] = .4;
+
         o.v_uv = v.TEXCOORD_0;
         vec4 tmp = renderer_MVMat * v.POSITION;
         o.v_position = tmp.xyz;
@@ -109,6 +120,14 @@ Shader "Water" {
           gl_FragColor = linearToGamma(gl_FragColor);
         #endif
 
+
+#define MATERIAL_ENABLE_SS_REFRACTION
+
+        #ifdef MATERIAL_ENABLE_SS_REFRACTION 
+
+    // last lint comment
+#endif
+
         // For testing only (macro)
         #if SCENE_SHADOW_TYPE == 2 || defined(XX_Macro)
           gl_FragColor = linearToGamma(gl_FragColor);