feat: Separate markdown into headings and paragraphs

.changeset/config.json

@@ -15,6 +15,7 @@
     "@growi/app",
     "@growi/slackbot-proxy",
     "@growi/custom-icons",
+    "@growi/markdown-splitter",
     "@growi/editor",
     "@growi/presentation",
     "@growi/preset-*",

apps/app/next.config.js

@@ -65,6 +65,7 @@ const getTranspilePackages = () => {
 const optimizePackageImports = [
   '@growi/core',
   '@growi/editor',
+  '@growi/markdown-splitter',
   '@growi/pluginkit',
   '@growi/presentation',
   '@growi/preset-themes',

apps/app/package.json

@@ -222,6 +222,7 @@
     "@growi/core-styles": "link:../../packages/core-styles",
     "@growi/custom-icons": "link:../../packages/custom-icons",
     "@growi/editor": "link:../../packages/editor",
+    "@growi/markdown-splitter": "link:../../packages/markdown-splitter",
     "@growi/ui": "link:../../packages/ui",
     "@handsontable/react": "=2.1.0",
     "@next/bundle-analyzer": "^14.1.3",

packages/markdown-splitter/.eslintignore

@@ -0,0 +1,2 @@
+/dist/**
+/types/**

packages/markdown-splitter/.eslintrc.cjs

@@ -0,0 +1,5 @@
+module.exports = {
+  extends: [
+    'weseek/react',
+  ],
+};

packages/markdown-splitter/.gitignore

@@ -0,0 +1 @@
+/dist

packages/markdown-splitter/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@growi/markdown-splitter",
+  "version": "1.0.0",
+  "license": "MIT",
+  "private": "true",
+  "type": "module",
+  "module": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist"
+  ],
+  "main": "dist/index.cjs",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "require": "./dist/index.cjs"
+    }
+  },
+  "scripts": {
+    "build": "vite build",
+    "clean": "shx rm -rf dist",
+    "dev": "vite build --mode dev",
+    "watch": "yarn dev -w --emptyOutDir=false",
+    "lint:js": "yarn eslint **/*.{js,ts}",
+    "lint:typecheck": "tsc",
+    "lint": "npm-run-all -p lint:*",
+    "test": "vitest run --coverage"
+  },
+  "devDependencies": {
+    "eslint-plugin-regex": "^1.8.0",
+    "hast-util-sanitize": "^4.1.0",
+    "pako": "^2.1.0",
+    "throttle-debounce": "^5.0.0",
+    "unified": "^10.1.2",
+    "unist-util-visit": "^4.0.0"
+  },
+  "peerDependencies": {
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0"
+  },
+  "dependencies": {
+  }
+}

packages/markdown-splitter/src/index.ts

@@ -0,0 +1 @@
+export * from './services/markdown-splitter';

packages/markdown-splitter/src/services/markdown-splitter.ts

@@ -0,0 +1,106 @@
+export type Chunk = {
+  label: string;
+  text: string;
+};
+
+/**
+ * Processes and adds a new chunk to the chunks array if content is not empty.
+ * Clears the contentLines array after processing.
+ * @param chunks - The array to store chunks.
+ * @param contentLines - The array of content lines.
+ * @param label - The label for the content chunk.
+ */
+function processPendingContent(chunks: Chunk[], contentLines: string[], label: string) {
+  const text = contentLines.join('\n').trimEnd();
+  if (text !== '') {
+    chunks.push({ label, text });
+  }
+  contentLines.length = 0; // Clear the contentLines array
+}
+
+/**
+ * Updates the section numbers based on the heading depth and returns the updated section label.
+ * Handles non-consecutive heading levels by initializing missing levels with 1.
+ * @param sectionNumbers - The current section numbers.
+ * @param depth - The depth of the heading (e.g., # is depth 1).
+ * @returns The updated section label.
+ */
+function updateSectionNumbers(sectionNumbers: number[], depth: number): string {
+  if (depth > sectionNumbers.length) {
+    // If depth increases, initialize missing levels with 1
+    while (sectionNumbers.length < depth) {
+      sectionNumbers.push(1);
+    }
+  }
+  else if (depth === sectionNumbers.length) {
+    // Same level, increment the last number
+    sectionNumbers[depth - 1]++;
+  }
+  else {
+    // Depth decreases, remove deeper levels and increment current level
+    sectionNumbers.splice(depth);
+    sectionNumbers[depth - 1]++;
+  }
+  return sectionNumbers.join('-');
+}
+
+/**
+ * Splits Markdown text into labeled chunks, considering content that may start before any headers
+ * and handling non-consecutive heading levels. Preserves list indentation and leading spaces while
+ * reducing unnecessary line breaks. Ensures that no empty line is added between sections.
+ * @param markdown - The input Markdown string.
+ * @returns An array of labeled chunks.
+ */
+export function splitMarkdownIntoChunks(markdown: string): Chunk[] {
+  const chunks: Chunk[] = [];
+  const sectionNumbers: number[] = [];
+
+  if (typeof markdown !== 'string' || markdown.trim() === '') {
+    return chunks;
+  }
+
+  const lines = markdown.split('\n');
+  const contentLines: string[] = [];
+  let currentLabel = '';
+  let previousLineEmpty = false;
+
+  for (const line of lines) {
+    const trimmedLine = line.trim();
+
+    if (trimmedLine.startsWith('#')) {
+      // Process any pending content before starting a new section
+      if (contentLines.length > 0) {
+        const contentLabel = currentLabel !== '' ? `${currentLabel}-content` : '0-content';
+        processPendingContent(chunks, contentLines, contentLabel);
+      }
+
+      // Match heading level and text
+      const headerMatch = trimmedLine.match(/^(#+)\s+(.*)/);
+      if (headerMatch) {
+        const headingDepth = headerMatch[1].length;
+        currentLabel = updateSectionNumbers(sectionNumbers, headingDepth);
+        chunks.push({ label: `${currentLabel}-heading`, text: line });
+      }
+    }
+    else if (trimmedLine === '') {
+      // Handle empty lines to avoid multiple consecutive empty lines
+      if (!previousLineEmpty && contentLines.length > 0) {
+        contentLines.push('');
+        previousLineEmpty = true;
+      }
+    }
+    else {
+      // Add non-empty lines to the current content
+      contentLines.push(line);
+      previousLineEmpty = false;
+    }
+  }
+
+  // Process any remaining content after the last line
+  if (contentLines.length > 0) {
+    const contentLabel = currentLabel !== '' ? `${currentLabel}-content` : '0-content';
+    processPendingContent(chunks, contentLines, contentLabel);
+  }
+
+  return chunks;
+}