test(csv): add grapheme length test (#5304)

denoland · Jul 4, 2024 · ae95651 · ae95651
1 parent 9d5d887
commit ae95651
Show file tree

Hide file tree

Showing 4 changed files with 45 additions and 8 deletions.
diff --git a/csv/_io.ts b/csv/_io.ts
@@ -4,6 +4,8 @@
 // https://github.com/golang/go/blob/master/LICENSE
 // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
 
+import { graphemeLength } from "./_shared.ts";
+
 /** Options for {@linkcode parseRecord}. */
 export interface ReadOptions {
   /** Character which separates values.
@@ -94,7 +96,9 @@ export async function parseRecord(
       if (!opt.lazyQuotes) {
         const j = field.indexOf(quote);
         if (j >= 0) {
-          const col = fullLine.length + j - line.length;
+          const col = graphemeLength(
+            fullLine.slice(0, fullLine.length - line.slice(j).length),
+          );
           throw new ParseError(startLine + 1, lineIndex, col, ERR_BARE_QUOTE);
         }
       }
@@ -132,7 +136,9 @@ export async function parseRecord(
             recordBuffer += quote;
           } else {
             // `"*` sequence (invalid non-escaped quote).
-            const col = fullLine.length - line.length - quoteLen;
+            const col = graphemeLength(
+              fullLine.slice(0, fullLine.length - line.length - quoteLen),
+            );
             throw new ParseError(startLine + 1, lineIndex, col, ERR_QUOTE);
           }
         } else if (line.length > 0 || !reader.isEOF()) {
@@ -145,7 +151,7 @@ export async function parseRecord(
           if (r === null) {
             // Abrupt end of file (EOF or error).
             if (!opt.lazyQuotes) {
-              const col = fullLine.length;
+              const col = graphemeLength(fullLine);
               throw new ParseError(startLine + 1, lineIndex, col, ERR_QUOTE);
             }
             fieldIndexes.push(recordBuffer.length);
@@ -155,7 +161,7 @@ export async function parseRecord(
         } else {
           // Abrupt end of file (EOF on error).
           if (!opt.lazyQuotes) {
-            const col = fullLine.length;
+            const col = graphemeLength(fullLine);
             throw new ParseError(startLine + 1, lineIndex, col, ERR_QUOTE);
           }
           fieldIndexes.push(recordBuffer.length);

diff --git a/csv/_shared.ts b/csv/_shared.ts
@@ -0,0 +1,15 @@
+// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
+
+/**
+ * returns length of a string considering surrogate pairs
+ * ```ts
+ * function graphemeLength(s: string) {
+ *   return Array.from(s).length;
+ * }
+ * graphemeLength("🐱") // 1
+ * "🐱".length // 2
+ * ```
+ */
+export function graphemeLength(s: string) {
+  return Array.from(s).length;
+}
diff --git a/csv/parse.ts b/csv/parse.ts
@@ -12,6 +12,7 @@ import {
   type ReadOptions,
   type RecordWithColumn,
 } from "./_io.ts";
+import { graphemeLength } from "./_shared.ts";
 
 export { ParseError, type ParseResult, type RecordWithColumn };
 
@@ -108,7 +109,9 @@ class Parser {
         if (!this.#options.lazyQuotes) {
           const j = field.indexOf(quote);
           if (j >= 0) {
-            const col = fullLine.length + j - line.length;
+            const col = graphemeLength(
+              fullLine.slice(0, fullLine.length - line.slice(j).length),
+            );
             throw new ParseError(startLine + 1, lineIndex, col, ERR_BARE_QUOTE);
           }
         }
@@ -146,7 +149,9 @@ class Parser {
               recordBuffer += quote;
             } else {
               // `"*` sequence (invalid non-escaped quote).
-              const col = fullLine.length - line.length - quoteLen;
+              const col = graphemeLength(
+                fullLine.slice(0, fullLine.length - line.length - quoteLen),
+              );
               throw new ParseError(startLine + 1, lineIndex, col, ERR_QUOTE);
             }
           } else if (line.length > 0 || !(this.#isEOF())) {
@@ -159,7 +164,7 @@ class Parser {
             if (r === null) {
               // Abrupt end of file (EOF or error).
               if (!this.#options.lazyQuotes) {
-                const col = fullLine.length;
+                const col = graphemeLength(fullLine);
                 throw new ParseError(startLine + 1, lineIndex, col, ERR_QUOTE);
               }
               fieldIndexes.push(recordBuffer.length);
@@ -169,7 +174,7 @@ class Parser {
           } else {
             // Abrupt end of file (EOF on error).
             if (!this.#options.lazyQuotes) {
-              const col = fullLine.length;
+              const col = graphemeLength(fullLine);
               throw new ParseError(startLine + 1, lineIndex, col, ERR_QUOTE);
             }
             fieldIndexes.push(recordBuffer.length);

diff --git a/csv/parse_test.ts b/csv/parse_test.ts
@@ -198,6 +198,17 @@ Deno.test({
         );
       },
     });
+    await t.step({
+      name: "error column grapheme number",
+      fn() {
+        const input = `a,b,🐱"`;
+        assertThrows(
+          () => parse(input),
+          ParseError,
+          'parse error on line 1, column 5: bare " in non-quoted-field',
+        );
+      },
+    });
     await t.step({
       name: "TrimQuote",
       fn() {