Skip to content

Commit

Permalink
Fold constants for Python and JavaScript runtimes, fixes #3698, fixes #…
Browse files Browse the repository at this point in the history
…3699

Signed-off-by: Ivan Kochurkin <kvanttt@gmail.com>
  • Loading branch information
KvanTTT committed May 7, 2022
1 parent a9e0ccf commit b32214b
Show file tree
Hide file tree
Showing 6 changed files with 79 additions and 18 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -371,6 +371,9 @@ public static RuntimeTestDescriptor[] getRuntimeTestDescriptors(String group, St
descriptors.add(GeneratedLexerDescriptors.getLargeLexerDescriptor(targetName));
descriptors.add(GeneratedLexerDescriptors.getAtnStatesSizeMoreThan65535Descriptor(targetName));
}
else if (group.equals("ParserExec")) {
descriptors.add(GeneratedLexerDescriptors.getMultiTokenAlternativeDescriptor(targetName));
}

return descriptors.toArray(new RuntimeTestDescriptor[0]);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -127,4 +127,42 @@ static RuntimeTestDescriptor getAtnStatesSizeMoreThan65535Descriptor(String targ
// result.skipTargets.add("Go"); // syntax error
return result;
}

static RuntimeTestDescriptor getMultiTokenAlternativeDescriptor(String targetName) {
UniversalRuntimeTestDescriptor result = new UniversalRuntimeTestDescriptor();
result.name = "MultiTokenAlternative";
result.notes = "";
result.targetName = targetName;
result.testType = "Parser";

final int tokensCount = 64;

StringBuilder rule = new StringBuilder("t: ");
StringBuilder tokens = new StringBuilder();
StringBuilder input = new StringBuilder();
StringBuilder output = new StringBuilder();
for (int i = 0; i < tokensCount; i++) {
String currentToken = "T" + i;
rule.append(currentToken);
if (i < tokensCount - 1) {
rule.append(" | ");
} else {
rule.append(";");
}
tokens.append(currentToken).append(": '").append(currentToken).append("';\n");
input.append(currentToken).append(" ");
output.append(currentToken);
}

result.grammar = "grammar P;\n" +
"r: t+ EOF {<writeln(\"$text\")>};\n" +
rule + "\n" +
tokens + "\n" +
"WS: [ ]+ -> skip;";
result.grammarName = "P";
result.startRule = "r";
result.input = input.toString();
result.output = output + "\n";
return result;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -454,12 +454,12 @@ TestSetInline(s) ::= <<

// Javascript language spec - shift operators are 32 bits long max
testShiftInRange(shiftAmount) ::= <<
((<shiftAmount>) & ~0x1f) == 0
(<shiftAmount> & ~0x1f) == 0
>>

// produces smaller bytecode only when bits.ttypes contains more than two items
bitsetBitfieldComparison(s, bits) ::= <%
(<testShiftInRange({<offsetShiftVar(s.varName, bits.shift)>})> && ((1 \<\< <offsetShiftVar(s.varName, bits.shift)>) & (<bits.ttypes:{ttype | (1 \<\< <offsetShiftType(ttype, bits.shift)>)}; separator=" | ">)) !== 0)
(<testShiftInRange({<offsetShiftVar(s.varName, bits.shift)>})> && ((1 \<\< <offsetShiftVar(s.varName, bits.shift)>) & <bits.calculated>) !== 0)
%>

isZero ::= [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -448,12 +448,12 @@ TestSetInline(s) ::= <<

// Java language spec 15.19 - shift operators mask operands rather than overflow to 0... need range test
testShiftInRange(shiftAmount) ::= <<
((<shiftAmount>) & ~0x3f) == 0
(<shiftAmount> & ~0x3f) == 0
>>

// produces smaller bytecode only when bits.ttypes contains more than two items
bitsetBitfieldComparison(s, bits) ::= <%
(<testShiftInRange({<offsetShiftVar(s.varName, bits.shift)>})> and ((1 \<\< <offsetShiftVar(s.varName, bits.shift)>) & (<bits.ttypes:{ttype | (1 \<\< <offsetShiftType(ttype, bits.shift)>)}; separator=" | ">)) != 0)
(<testShiftInRange({<offsetShiftVar(s.varName, bits.shift)>})> and ((1 \<\< <offsetShiftVar(s.varName, bits.shift)>) & <bits.calculated>) != 0)
%>

isZero ::= [
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -461,12 +461,12 @@ TestSetInline(s) ::= <<

// Java language spec 15.19 - shift operators mask operands rather than overflow to 0... need range test
testShiftInRange(shiftAmount) ::= <<
((<shiftAmount>) & ~0x3f) == 0
(<shiftAmount> & ~0x3f) == 0
>>

// produces smaller bytecode only when bits.ttypes contains more than two items
bitsetBitfieldComparison(s, bits) ::= <%
(<testShiftInRange({<offsetShiftVar(s.varName, bits.shift)>})> and ((1 \<\< <offsetShiftVar(s.varName, bits.shift)>) & (<bits.ttypes:{ttype | (1 \<\< <offsetShiftType(ttype, bits.shift)>)}; separator=" | ">)) != 0)
(<testShiftInRange({<offsetShiftVar(s.varName, bits.shift)>})> and (1 \<\< <offsetShiftVar(s.varName, bits.shift)>) & <bits.calculated>) != 0
%>

isZero ::= [
Expand Down
44 changes: 32 additions & 12 deletions tool/src/org/antlr/v4/codegen/model/TestSetInline.java
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
package org.antlr.v4.codegen.model;

import org.antlr.v4.codegen.OutputModelFactory;
import org.antlr.v4.codegen.Target;
import org.antlr.v4.runtime.misc.IntervalSet;
import org.antlr.v4.tool.ast.GrammarAST;

Expand All @@ -15,9 +16,9 @@

/** */
public class TestSetInline extends SrcOp {
public int bitsetWordSize;
public String varName;
public Bitset[] bitsets;
public final int bitsetWordSize;
public final String varName;
public final Bitset[] bitsets;

public TestSetInline(OutputModelFactory factory, GrammarAST ast, IntervalSet set, int wordSize) {
super(factory, ast);
Expand All @@ -32,29 +33,48 @@ private static Bitset[] createBitsets(OutputModelFactory factory,
IntervalSet set,
int wordSize,
boolean useZeroOffset) {
List<Bitset> bitsetList = new ArrayList<Bitset>();
List<Bitset> bitsetList = new ArrayList<>();
Target target = factory.getGenerator().getTarget();
Bitset current = null;
for (int ttype : set.toArray()) {
Bitset current = !bitsetList.isEmpty() ? bitsetList.get(bitsetList.size() - 1) : null;
if (current == null || ttype > (current.shift + wordSize-1)) {
current = new Bitset();
int shift;
if (useZeroOffset && ttype >= 0 && ttype < wordSize-1) {
current.shift = 0;
shift = 0;
}
else {
current.shift = ttype;
shift = ttype;
}

current = new Bitset(shift);
bitsetList.add(current);
}

current.ttypes.add(factory.getGenerator().getTarget().getTokenTypeAsTargetLabel(factory.getGrammar(), ttype));
current.addToken(ttype, target.getTokenTypeAsTargetLabel(factory.getGrammar(), ttype));
}

return bitsetList.toArray(new Bitset[0]);
}

public static final class Bitset {
public int shift;
public final List<String> ttypes = new ArrayList<String>();
public final int shift;
private final List<String> ttypes = new ArrayList<>();
private long calculated;

public Bitset(int shift) {
this.shift = shift;
}

public void addToken(int type, String name) {
ttypes.add(name);
calculated |= 1L << (type - shift);
}

public List<String> getTtypes() {
return ttypes;
}

public long getCalculated() {
return calculated;
}
}
}

0 comments on commit b32214b

Please sign in to comment.