[#40, #54] Configuration file with computed fields and include/exclude file options

markdowndb.config.js

@@ -0,0 +1,3 @@
+export default {
+  customFields: []
+};

package.json

@@ -47,6 +47,7 @@
     "@portaljs/remark-wiki-link": "^1.0.4",
     "gray-matter": "^4.0.3",
     "knex": "^2.4.2",
+    "micromatch": "^4.0.5",
     "react-markdown": "^9.0.1",
     "remark-gfm": "^3.0.1",
     "remark-parse": "^10.0.1",
@@ -57,6 +58,7 @@
     "@changesets/changelog-github": "^0.4.8",
     "@changesets/cli": "^2.26.1",
     "@types/jest": "^29.5.1",
+    "@types/micromatch": "^4.0.6",
     "@typescript-eslint/eslint-plugin": "^5.59.5",
     "@typescript-eslint/parser": "^5.59.5",
     "eslint": "^8.40.0",
@@ -69,4 +71,4 @@
     "ts-node": "^10.9.1",
     "typescript": "^5.0.4"
   }
-}
+}

src/bin/index.js

@@ -1,21 +1,41 @@
 #!/usr/bin/env node
-
+import * as path from 'path';
 import { MarkdownDB } from "../lib/markdowndb.js";
 
+async function loadConfig(configFilePath) {
+  const normalizedPath = path.resolve(configFilePath || 'markdowndb.config.js');
+  const fileUrl = new URL(`file://${normalizedPath}`);
+
+  try {
+    // Import the module using the file URL 
+    const configModule = await import(fileUrl.href);
+    return configModule.default;
+  } catch (error) {
+    if (configFilePath) {
+      throw new Error(`Error loading configuration file from ${normalizedPath}: ${error.message}`);
+    }
+  }
+}
+
 // TODO get these from markdowndb.config.js or something
 const dbPath = "markdown.db";
 const ignorePatterns = [/Excalidraw/, /\.obsidian/, /DS_Store/];
-const [contentPath] = process.argv.slice(2);
+const [contentPath, configFilePath] = process.argv.slice(2);
 
 if (!contentPath) {
   throw new Error("Invalid/Missing path to markdown content folder");
 }
 
+// Load the configuration file dynamically
+const config = await loadConfig(configFilePath);
+
 const client = new MarkdownDB({
   client: "sqlite3",
   connection: {
     filename: dbPath,
   },
+}, {
+  customConfig: config
 });
 
 await client.init();

src/lib/CustomConfig.ts

@@ -0,0 +1,8 @@
+import { FileInfo } from "./process.js";
+import { Root } from "remark-parse/lib/index.js";
+
+export interface CustomConfig {
+  customFields?: ((fileInfo: FileInfo, ast: Root) => any)[];
+  include?: string[];
+  exclude?: string[];
+}

src/lib/indexFolder.ts

@@ -1,33 +1,76 @@
 import { FileInfo, processFile } from "./process.js";
 import { recursiveWalkDir } from "./recursiveWalkDir.js";
+import { CustomConfig } from "./CustomConfig.js";
+import micromatch from "micromatch";
 
 export function indexFolder(
   folderPath: string,
   pathToUrlResolver: (filePath: string) => string,
+  config: CustomConfig,
   ignorePatterns?: RegExp[]
 ) {
   const filePathsToIndex = recursiveWalkDir(folderPath);
   const filteredFilePathsToIndex = filePathsToIndex.filter((filePath) =>
-    shouldIncludeFile(filePath, ignorePatterns)
+    shouldIncludeFile({
+      filePath,
+      ignorePatterns,
+      includeGlob: config.include,
+      excludeGlob: config.exclude,
+    })
   );
   const files: FileInfo[] = [];
+
+  const customFields = config.customFields || [];
   for (const filePath of filteredFilePathsToIndex) {
-    const fileObject = processFile(
-      folderPath,
-      filePath,
+    const fileObject = processFile(folderPath, filePath, {
       pathToUrlResolver,
-      filePathsToIndex
-    );
+      filePathsToIndex,
+      customFields,
+    });
     files.push(fileObject);
   }
   return files;
 }
 
-function shouldIncludeFile(
-  filePath: string,
-  ignorePatterns?: RegExp[]
-): boolean {
-  return !(
-    ignorePatterns && ignorePatterns.some((pattern) => pattern.test(filePath))
-  );
+function shouldIncludeFile({
+  filePath,
+  ignorePatterns,
+  includeGlob,
+  excludeGlob,
+}: {
+  filePath: string;
+  ignorePatterns?: RegExp[];
+  includeGlob?: string[];
+  excludeGlob?: string[];
+}): boolean {
+  const normalizedFilePath = filePath.replace(/\\/g, "/");
+
+  if (
+    ignorePatterns &&
+    ignorePatterns.some((pattern) => pattern.test(normalizedFilePath))
+  ) {
+    return false;
+  }
+
+  // Check if the file should be included based on includeGlob
+  if (
+    includeGlob &&
+    !includeGlob.some((pattern) =>
+      micromatch.isMatch(normalizedFilePath, pattern)
+    )
+  ) {
+    return false;
+  }
+
+  // Check if the file should be excluded based on excludeGlob
+  if (
+    excludeGlob &&
+    excludeGlob.some((pattern) =>
+      micromatch.isMatch(normalizedFilePath, pattern)
+    )
+  ) {
+    return false;
+  }
+
+  return true;
 }

src/lib/markdowndb.ts

@@ -12,6 +12,7 @@
   getUniqueValues,
 } from "./databaseUtils.js";
 import fs from "fs";
+import { CustomConfig } from "./CustomConfig.js";
 
 const defaultFilePathToUrl = (filePath: string) => {
   let url = filePath
@@ -22,7 +23,7 @@
   return encodeURI(url);
 };
 
 const resolveLinkToUrlPath = (link: string, sourceFilePath?: string) => {
   if (!sourceFilePath) {
     return link;
   }
@@ -40,14 +41,42 @@
  */
 export class MarkdownDB {
   config: Knex.Config;
+  customConfig: CustomConfig = {};
+  //@ts-ignore
   db: Knex;
 
   /**
    * Constructs a new MarkdownDB instance.
-   * @param {Knex.Config} config - Knex configuration object.
+   * @param {Knex.Config} knexConfig - Knex configuration object.
+   * @param {CustomConfig} [customConfig] - Custom configuration object.
    */
-  constructor(config: Knex.Config) {
-    this.config = config;
+  constructor(
+    knexConfig: Knex.Config,
+    options?: { customConfig?: CustomConfig; configFilePath?: string }
+  ) {
+    this.config = knexConfig;
+    if (options?.customConfig) {
+      this.customConfig = options.customConfig;
+    } else {
+      this.loadConfiguration(options?.configFilePath);
+    }
+  }
+
+  private async loadConfiguration(configFilePath?: string) {
+    const normalizedPath = path.resolve(configFilePath || "markdowndb.config.js");
+    const fileUrl = new URL(`file://${normalizedPath}`);
+
+    try {
+      // Import the module using the file URL
+      const configModule = await import(fileUrl.href);
+      this.customConfig = configModule.default;
+    } catch (error: any) {
+      if (configFilePath) {
+        throw new Error(
+          `Error loading configuration file from ${normalizedPath}: ${error.message}`
+        );
+      }
+    }
   }
 
   /**
@@ -82,6 +111,7 @@
     const fileObjects = indexFolder(
       folderPath,
       pathToUrlResolver,
+      this.customConfig,
       ignorePatterns
     );
     const filesToInsert = fileObjects.map(mapFileToInsert);
@@ -248,7 +278,7 @@
   }
 }
 
 function writeJsonToFile(filePath: string, jsonData: any[]) {
   try {
     const directory = path.dirname(filePath);
     if (!fs.existsSync(directory)) {
@@ -257,7 +287,7 @@
 
     const jsonString = JSON.stringify(jsonData, null, 2);
     fs.writeFileSync(filePath, jsonString);
   } catch (error: any) {
     console.error(`Error writing data to ${filePath}: ${error}`);
   }
 }

src/lib/parseFile.ts

@@ -28,6 +28,7 @@ export function parseFile(source: string, options?: ParsingOptions) {
   metadata.tasks = tasks;
 
   return {
+    ast,
     metadata,
     links,
   };

src/lib/process.ts

@@ -4,6 +4,7 @@
 
 import { File } from "./schema.js";
 import { WikiLink, parseFile } from "./parseFile.js";
+import { Root } from "remark-parse/lib/index.js";
 
 export interface FileInfo extends File {
   tags: string[];
@@ -15,8 +16,15 @@
 export function processFile(
   rootFolder: string,
   filePath: string,
-  pathToUrlResolver: (filePath: string) => string,
-  filePathsToIndex: string[]
+  {
+    pathToUrlResolver = (filePath) => filePath,
+    filePathsToIndex = [],
+    customFields = [(fileInfo: FileInfo, ast: Root) => {}],
+  }: {
+    pathToUrlResolver: (filePath: string) => string;
+    filePathsToIndex: string[];
+    customFields: ((fileInfo: FileInfo, ast: Root) => any)[];
+  }
 ) {
   // Remove rootFolder from filePath
   const relativePath = path.relative(rootFolder, filePath);
@@ -52,7 +60,7 @@
     flag: "r",
   });
 
-  const { metadata, links } = parseFile(source, {
+  const { ast, metadata, links } = parseFile(source, {
     from: relativePath,
     permalinks: filePathsToIndex,
   });
@@ -66,6 +74,10 @@
 
   const tags = metadata?.tags || [];
   fileInfo.tags = tags;
+  for (let index = 0; index < customFields.length; index++) {
+    const customFieldFunction = customFields[index];
+    customFieldFunction(fileInfo, ast);
+  }
 
   return fileInfo;
 }

src/tests/extractWikiLinks.spec.ts

@@ -1,4 +1,4 @@
-import { extractWikiLinks, processAST } from "../lib/parseFile";
+import { ParsingOptions, extractWikiLinks, processAST } from "../lib/parseFile";
 
 // TODO test for links with headings and aliases ?
 // TODO test pdf embeds
@@ -7,7 +7,7 @@ import { extractWikiLinks, processAST } from "../lib/parseFile";
 // TODO test custom extractors
 // TODO test with other remark plugins e.g. original wiki links
 
-const getLinksFromSource = (source: string, options?) => {
+const getLinksFromSource = (source: string, options?: ParsingOptions) => {
   const from = "abc/foobar.md";
   const ast = processAST(source, options);
   const links = extractWikiLinks(ast, { from: from, ...options });