Merge pull request #175 from blackflux/dev

[Gally]: master <- dev

README.md

@@ -85,6 +85,10 @@ contains({ a: [1, 2], b: 'c' }, { a: [1] });
 // => false
 ```
 
+### jsonSmartParse(str: String)
+
+Deals with parsing invalid json outputs that are commonly generated by llms
+
 ### Merge(logic: Object = {})(...obj: Object[])
 
 Allows merging of objects. The logic defines paths that map to a field, or a function, to merge by.

package.json

@@ -26,7 +26,8 @@
     "js-gardener": "5.0.4",
     "lodash.clonedeep": "4.5.0",
     "lodash.samplesize": "4.2.0",
-    "node-tdd": "5.2.4"
+    "node-tdd": "5.2.4",
+    "smart-fs": "4.0.1"
   },
   "licenses": [
     {

src/core/json-smart-parse.js

@@ -0,0 +1,29 @@
+export default (str) => {
+  let result = str;
+  if (result.includes('```')) {
+    result = result.split(/```(?:json)?/)[1];
+  }
+
+  const unescaped = [];
+  let escaped = false;
+  let quoted = false;
+  for (let i = 0; i < result.length; i += 1) {
+    const char = result[i];
+    if (['\n', '\r', '\t'].includes(char) && quoted) {
+      unescaped.push(i);
+    }
+    if (!escaped && char === '"') {
+      quoted = !quoted;
+    }
+    escaped = char === '\\' ? !escaped : false;
+  }
+
+  result = result.replace(/\n/g, (m, idx) => {
+    if (unescaped.includes(idx)) {
+      return JSON.stringify(m).slice(1, -1);
+    }
+    return m;
+  });
+
+  return JSON.parse(result);
+};

src/index.js

@@ -2,3 +2,4 @@ export { default as align } from './core/align.js';
 export { default as clone } from './core/clone.js';
 export { default as contains } from './core/contains.js';
 export { default as Merge } from './core/merge.js';
+export { default as jsonSmartParse } from './core/json-smart-parse.js';

test/core/json-smart-parse.spec.js

@@ -0,0 +1,18 @@
+import fs from 'smart-fs';
+import { expect } from 'chai';
+import { describe } from 'node-tdd';
+import jsonSmartParse from '../../src/core/json-smart-parse.js';
+
+const filename = fs.filename(import.meta.url);
+const fixtureDir = `${filename}__fixtures`;
+
+describe('Testing jsonSmartParse', () => {
+  // eslint-disable-next-line mocha/no-setup-in-describe
+  fs.walkDir(fixtureDir).forEach((file) => {
+    it(`Testing ${file}`, async ({ fixture }) => {
+      const { input, expected } = fixture(file);
+      const result = jsonSmartParse(input);
+      expect(result).to.deep.equal(expected);
+    });
+  });
+});

test/core/json-smart-parse.spec.js__fixtures/basic-line-break-escaped-quote.json

@@ -0,0 +1,6 @@
+{
+  "input": "{ \"x\": \"\nhello\\\"\nyou\n\" }",
+  "expected": {
+    "x": "\nhello\"\nyou\n"
+  }
+}

test/core/json-smart-parse.spec.js__fixtures/basic-line-break-escaped-quote2.json

@@ -0,0 +1,6 @@
+{
+  "input": "{ \"x\\\"y\": \"\nhello\\\"\nyou\n\" }",
+  "expected": {
+    "x\"y": "\nhello\"\nyou\n"
+  }
+}

test/core/json-smart-parse.spec.js__fixtures/basic-line-break-everywhere.json

@@ -0,0 +1,6 @@
+{
+  "input": "\n{\n \n\"x\":\n\"\nhello\n\nyou\n\"\n \n}\n",
+  "expected": {
+    "x": "\nhello\n\nyou\n"
+  }
+}

test/core/json-smart-parse.spec.js__fixtures/basic-line-break-multi-escape.json

@@ -0,0 +1,6 @@
+{
+  "input": "{ \"x\": \"hello\\\nyou\" }",
+  "expected": {
+    "x": "hello\\nyou"
+  }
+}

test/core/json-smart-parse.spec.js__fixtures/basic-line-break.json

@@ -0,0 +1,6 @@
+{
+  "input": "{ \"x\": \"hello\nyou\" }",
+  "expected": {
+    "x": "hello\nyou"
+  }
+}

test/core/json-smart-parse.spec.js__fixtures/json-tick-wrap.json

@@ -0,0 +1,6 @@
+{
+  "input": "```json{ \"x\": \"hello\nyou\" }```",
+  "expected": {
+    "x": "hello\nyou"
+  }
+}

test/core/json-smart-parse.spec.js__fixtures/json-tick-wrap2.json

@@ -0,0 +1,6 @@
+{
+  "input": "```{ \"x\": \"hello\nyou\" }```",
+  "expected": {
+    "x": "hello\nyou"
+  }
+}

test/index.spec.js

@@ -7,7 +7,8 @@ describe('Testing index.js', () => {
       'Merge',
       'align',
       'clone',
-      'contains'
+      'contains',
+      'jsonSmartParse'
     ]);
   });
 });