From fcf5b372be2c537808ca4654dc3895f756a1c6bc Mon Sep 17 00:00:00 2001
From: Eric Traut <eric@traut.com>
Date: Mon, 29 Apr 2024 16:00:54 -0700
Subject: [PATCH] Changed the tokenizer so it is bug-for-bug compatible with
 the CPython tokenizer in versions 3.10 and newer in the case where a
 backslash (continuation character) is located by itself on a line. This
 addresses #7799.

---
 packages/pyright-internal/src/parser/tokenizer.ts  | 14 +++++++++++++-
 packages/pyright-internal/src/tests/parser.test.ts | 10 ++++++++--
 .../src/tests/samples/{sample1.py => parser1.py}   |  0
 .../pyright-internal/src/tests/samples/parser2.py  | 11 +++++++++++
 4 files changed, 32 insertions(+), 3 deletions(-)
 rename packages/pyright-internal/src/tests/samples/{sample1.py => parser1.py} (100%)
 create mode 100644 packages/pyright-internal/src/tests/samples/parser2.py

diff --git a/packages/pyright-internal/src/parser/tokenizer.ts b/packages/pyright-internal/src/parser/tokenizer.ts
index 323df0ed578f..0fd13951c831 100644
--- a/packages/pyright-internal/src/parser/tokenizer.ts
+++ b/packages/pyright-internal/src/parser/tokenizer.ts
@@ -494,13 +494,25 @@ export class Tokenizer {
                     } else {
                         this._cs.advance(2);
                     }
+
                     this._addLineRange();
+
+                    if (this._tokens.length > 0 && this._tokens[this._tokens.length - 1].type === TokenType.NewLine) {
+                        this._readIndentationAfterNewLine();
+                    }
                     return true;
-                } else if (this._cs.nextChar === Char.LineFeed) {
+                }
+
+                if (this._cs.nextChar === Char.LineFeed) {
                     this._cs.advance(2);
                     this._addLineRange();
+
+                    if (this._tokens.length > 0 && this._tokens[this._tokens.length - 1].type === TokenType.NewLine) {
+                        this._readIndentationAfterNewLine();
+                    }
                     return true;
                 }
+
                 return this._handleInvalid();
             }
 
diff --git a/packages/pyright-internal/src/tests/parser.test.ts b/packages/pyright-internal/src/tests/parser.test.ts
index 18edd3e4d730..892f770c9ec7 100644
--- a/packages/pyright-internal/src/tests/parser.test.ts
+++ b/packages/pyright-internal/src/tests/parser.test.ts
@@ -25,14 +25,20 @@ test('Empty', () => {
     assert.equal(parserOutput.parseTree.statements.length, 0);
 });
 
-test('Sample1', () => {
+test('Parser1', () => {
     const diagSink = new DiagnosticSink();
-    const parserOutput = TestUtils.parseSampleFile('sample1.py', diagSink).parserOutput;
+    const parserOutput = TestUtils.parseSampleFile('parser1.py', diagSink).parserOutput;
 
     assert.equal(diagSink.fetchAndClear().length, 0);
     assert.equal(parserOutput.parseTree.statements.length, 4);
 });
 
+test('Parser2', () => {
+    const diagSink = new DiagnosticSink();
+    TestUtils.parseSampleFile('parser2.py', diagSink);
+    assert.strictEqual(diagSink.getErrors().length, 0);
+});
+
 test('FStringEmptyTuple', () => {
     assert.doesNotThrow(() => {
         const diagSink = new DiagnosticSink();
diff --git a/packages/pyright-internal/src/tests/samples/sample1.py b/packages/pyright-internal/src/tests/samples/parser1.py
similarity index 100%
rename from packages/pyright-internal/src/tests/samples/sample1.py
rename to packages/pyright-internal/src/tests/samples/parser1.py
diff --git a/packages/pyright-internal/src/tests/samples/parser2.py b/packages/pyright-internal/src/tests/samples/parser2.py
new file mode 100644
index 000000000000..7467c6aefe56
--- /dev/null
+++ b/packages/pyright-internal/src/tests/samples/parser2.py
@@ -0,0 +1,11 @@
+class A:
+\
+    pass
+
+class B:
+ \
+    pass
+
+class C:
+ \
+pass