Changed the tokenizer so it is bug-for-bug compatible with the CPytho…

…n tokenizer in versions 3.10 and newer in the case where a backslash (continuation character) is located by itself on a line. This addresses #7799.
microsoft · Apr 29, 2024 · fcf5b37 · fcf5b37
1 parent 1aed0e3
commit fcf5b37
Show file tree

Hide file tree

Showing 4 changed files with 32 additions and 3 deletions.
diff --git a/packages/pyright-internal/src/parser/tokenizer.ts b/packages/pyright-internal/src/parser/tokenizer.ts
@@ -494,13 +494,25 @@ export class Tokenizer {
                     } else {
                         this._cs.advance(2);
                     }
+
                     this._addLineRange();
+
+                    if (this._tokens.length > 0 && this._tokens[this._tokens.length - 1].type === TokenType.NewLine) {
+                        this._readIndentationAfterNewLine();
+                    }
                     return true;
-                } else if (this._cs.nextChar === Char.LineFeed) {
+                }
+
+                if (this._cs.nextChar === Char.LineFeed) {
                     this._cs.advance(2);
                     this._addLineRange();
+
+                    if (this._tokens.length > 0 && this._tokens[this._tokens.length - 1].type === TokenType.NewLine) {
+                        this._readIndentationAfterNewLine();
+                    }
                     return true;
                 }
+
                 return this._handleInvalid();
             }
 

diff --git a/packages/pyright-internal/src/tests/parser.test.ts b/packages/pyright-internal/src/tests/parser.test.ts
@@ -25,14 +25,20 @@ test('Empty', () => {
     assert.equal(parserOutput.parseTree.statements.length, 0);
 });
 
-test('Sample1', () => {
+test('Parser1', () => {
     const diagSink = new DiagnosticSink();
-    const parserOutput = TestUtils.parseSampleFile('sample1.py', diagSink).parserOutput;
+    const parserOutput = TestUtils.parseSampleFile('parser1.py', diagSink).parserOutput;
 
     assert.equal(diagSink.fetchAndClear().length, 0);
     assert.equal(parserOutput.parseTree.statements.length, 4);
 });
 
+test('Parser2', () => {
+    const diagSink = new DiagnosticSink();
+    TestUtils.parseSampleFile('parser2.py', diagSink);
+    assert.strictEqual(diagSink.getErrors().length, 0);
+});
+
 test('FStringEmptyTuple', () => {
     assert.doesNotThrow(() => {
         const diagSink = new DiagnosticSink();

diff --git a/...ght-internal/src/tests/samples/sample1.py → ...ght-internal/src/tests/samples/parser1.py b/...ght-internal/src/tests/samples/sample1.py → ...ght-internal/src/tests/samples/parser1.py
diff --git a/packages/pyright-internal/src/tests/samples/parser2.py b/packages/pyright-internal/src/tests/samples/parser2.py
@@ -0,0 +1,11 @@
+class A:
+\
+    pass
+
+class B:
+ \
+    pass
+
+class C:
+ \
+pass