From fb1c39f3b6126e4b4061b949aa2e626b072745c4 Mon Sep 17 00:00:00 2001
From: Katerina Skroumpelou <mandarini@users.noreply.github.com>
Date: Thu, 14 Sep 2023 20:36:42 +0300
Subject: [PATCH] feat(nx-dev): add embeddings for community plugins (#19168)

---
 nx-dev/util-ai/src/lib/chat-utils.ts          | 13 ++++---
 .../create-embeddings/src/main.mts            | 36 ++++++++++++++++---
 2 files changed, 40 insertions(+), 9 deletions(-)

diff --git a/nx-dev/util-ai/src/lib/chat-utils.ts b/nx-dev/util-ai/src/lib/chat-utils.ts
index ce9b7fb597448..c16dff73c3174 100644
--- a/nx-dev/util-ai/src/lib/chat-utils.ts
+++ b/nx-dev/util-ai/src/lib/chat-utils.ts
@@ -65,10 +65,15 @@ export function getListOfSources(
       return false;
     })
     .map((section) => {
-      const url = new URL('https://nx.dev');
-      url.pathname = section.url_partial as string;
-      if (section.slug) {
-        url.hash = section.slug;
+      let url: URL;
+      if (section.url_partial?.startsWith('https://')) {
+        url = new URL(section.url_partial);
+      } else {
+        url = new URL('https://nx.dev');
+        url.pathname = section.url_partial as string;
+        if (section.slug) {
+          url.hash = section.slug;
+        }
       }
       return {
         heading: section.heading,
diff --git a/tools/documentation/create-embeddings/src/main.mts b/tools/documentation/create-embeddings/src/main.mts
index 456382982d83c..d0c1c66d8b535 100644
--- a/tools/documentation/create-embeddings/src/main.mts
+++ b/tools/documentation/create-embeddings/src/main.mts
@@ -20,6 +20,7 @@ import manifestsExtending from '../../../../docs/generated/manifests/extending-n
 import manifestsNx from '../../../../docs/generated/manifests/nx.json' assert { type: 'json' };
 import manifestsPackages from '../../../../docs/generated/manifests/packages.json' assert { type: 'json' };
 import manifestsTags from '../../../../docs/generated/manifests/tags.json' assert { type: 'json' };
+import communityPlugins from '../../../../community/approved-plugins.json' assert { type: 'json' };
 
 let identityMap = {};
 
@@ -125,14 +126,16 @@ class MarkdownEmbeddingSource extends BaseEmbeddingSource {
   constructor(
     source: string,
     public filePath: string,
-    public url_partial: string
+    public url_partial: string,
+    public fileContent?: string
   ) {
     const path = filePath.replace(/^docs/, '').replace(/\.md?$/, '');
     super(source, path, url_partial);
   }
 
   async load() {
-    const contents = await readFile(this.filePath, 'utf8');
+    const contents =
+      this.fileContent ?? (await readFile(this.filePath, 'utf8'));
 
     const { checksum, sections } = processMdxForSearch(contents);
 
@@ -209,6 +212,14 @@ async function generateEmbeddings() {
         entry.url_partial
       );
     }),
+    ...createMarkdownForCommunityPlugins().map((content, index) => {
+      return new MarkdownEmbeddingSource(
+        'community-plugins',
+        '/community/approved-plugins.json#' + index,
+        content.url,
+        content.text
+      );
+    }),
   ];
 
   console.log(`Discovered ${embeddingSources.length} pages`);
@@ -307,7 +318,10 @@ async function generateEmbeddings() {
 
           const [responseData] = embeddingResponse.data;
 
-          const longer_heading = createLongerHeading(heading, url_partial);
+          const longer_heading =
+            source !== 'community-plugins'
+              ? createLongerHeading(heading, url_partial)
+              : heading;
 
           const { error: insertPageSectionError, data: pageSection } =
             await supabaseClient
@@ -446,7 +460,7 @@ function createLongerHeading(
   if (url_partial?.length) {
     if (heading?.length && heading !== null && heading !== 'null') {
       return `${heading}${` - ${
-        url_partial.split('/')?.[1]?.[0].toUpperCase() +
+        url_partial.split('/')?.[1]?.[0]?.toUpperCase() +
         url_partial.split('/')?.[1]?.slice(1)
       }`}`;
     } else {
@@ -454,7 +468,7 @@ function createLongerHeading(
         .split('#')[0]
         .split('/')
         .map((part) =>
-          part?.length ? part[0].toUpperCase() + part.slice(1) + ' - ' : ''
+          part?.length ? part[0]?.toUpperCase() + part.slice(1) + ' - ' : ''
         )
         .join('')
         .slice(0, -3);
@@ -462,6 +476,18 @@ function createLongerHeading(
   }
 }
 
+function createMarkdownForCommunityPlugins(): {
+  text: string;
+  url: string;
+}[] {
+  return communityPlugins.map((plugin) => {
+    return {
+      text: `## ${plugin.name} plugin\n\nThere is a ${plugin.name} community plugin.\n\nHere is the description for it: ${plugin.description}\n\nHere is the link to it: [${plugin.url}](${plugin.url})\n\nHere is the list of all the plugins that exist for Nx: https://nx.dev/extending-nx/registry`,
+      url: plugin.url,
+    };
+  });
+}
+
 async function main() {
   await generateEmbeddings();
 }