Skip to content

Commit

Permalink
soft switch to core's content nodes and add the mime-type to icon map…
Browse files Browse the repository at this point in the history 
…ping
  • Loading branch information
@aubin-tchoi
aubin-tchoi committed Jan 17, 2025
1 parent 0c70d79 commit 9c67dd6
Show file tree
Hide file tree
Showing 3 changed files with 75 additions and 80 deletions.
85 changes: 9 additions & 76 deletions front/lib/api/content_nodes.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import type {
DataSourceViewContentNode,
DataSourceViewType,
} from "@dust-tt/types";
import { assertNever, MIME_TYPES } from "@dust-tt/types";
import { assertNever } from "@dust-tt/types";

import type { DataSourceViewResource } from "@app/lib/resources/data_source_view_resource";
import type logger from "@app/logger/logger";
Expand Down Expand Up @@ -115,86 +115,19 @@ export function computeNodesDiff({
}
}

export function getContentNodeMetadata(
node: CoreAPIContentNode,
viewType: "tables" | "documents"
): {
export function getContentNodeMetadata(node: CoreAPIContentNode): {
type: ContentNodeType;
} {
switch (node.mime_type) {
case MIME_TYPES.CONFLUENCE.PAGE:
return { type: "file" };
case MIME_TYPES.CONFLUENCE.SPACE:
return { type: "folder" };
case MIME_TYPES.GITHUB.REPOSITORY:
return { type: "folder" };
case MIME_TYPES.GITHUB.CODE_ROOT:
return { type: "folder" };
case MIME_TYPES.GITHUB.CODE_DIRECTORY:
return { type: "folder" };
case MIME_TYPES.GITHUB.CODE_FILE:
return { type: "file" };
case MIME_TYPES.GITHUB.ISSUES:
// this is approximate and will be cleaned up when we turn ContentNodeType into the same nodeType as in core
// the main point is that it correctly identifies documents as files as this is used in ContentNodeTree
switch (node.node_type) {
case "Table":
return { type: "database" };
case MIME_TYPES.GITHUB.ISSUE:
return { type: "file" };
case MIME_TYPES.GITHUB.DISCUSSIONS:
return { type: "channel" };
case MIME_TYPES.GITHUB.DISCUSSION:
return { type: "file" };
case MIME_TYPES.GOOGLE_DRIVE.FOLDER:
case "Folder":
return { type: "folder" };
case MIME_TYPES.INTERCOM.COLLECTION:
return { type: "folder" };
case MIME_TYPES.INTERCOM.TEAMS_FOLDER:
return { type: "channel" };
case MIME_TYPES.INTERCOM.CONVERSATION:
return { type: "file" };
case MIME_TYPES.INTERCOM.TEAM:
return { type: "folder" };
case MIME_TYPES.INTERCOM.HELP_CENTER:
return { type: "database" };
case MIME_TYPES.INTERCOM.ARTICLE:
return { type: "file" };
case MIME_TYPES.MICROSOFT.FOLDER:
return { type: "folder" };
case MIME_TYPES.NOTION.UNKNOWN_FOLDER:
return { type: "folder" };
case MIME_TYPES.NOTION.DATABASE:
return { type: "database" };
case MIME_TYPES.NOTION.PAGE:
return { type: "file" };
case MIME_TYPES.SLACK.CHANNEL:
return { type: "channel" };
case MIME_TYPES.SLACK.THREAD:
return { type: "file" };
case MIME_TYPES.SLACK.MESSAGES:
return { type: "file" };
case MIME_TYPES.SNOWFLAKE.DATABASE:
return { type: "folder" };
case MIME_TYPES.SNOWFLAKE.SCHEMA:
return { type: "folder" };
case MIME_TYPES.SNOWFLAKE.TABLE:
return { type: "database" };
case MIME_TYPES.WEBCRAWLER.FOLDER:
return { type: "folder" };
case MIME_TYPES.ZENDESK.BRAND:
return { type: "folder" };
case MIME_TYPES.ZENDESK.HELP_CENTER:
return { type: "folder" };
case MIME_TYPES.ZENDESK.CATEGORY:
return { type: "folder" };
case MIME_TYPES.ZENDESK.ARTICLE:
return { type: "file" };
case MIME_TYPES.ZENDESK.TICKETS:
return { type: "folder" };
case MIME_TYPES.ZENDESK.TICKET:
case "Document":
return { type: "file" };
default:
let type: ContentNodeType = "file";
if (node.mime_type === "text/csv") {
type = viewType === "tables" ? "database" : "file";
}
return { type };
assertNever(node.node_type);
}
}
4 changes: 2 additions & 2 deletions front/lib/api/data_source_view.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,7 +167,7 @@ function makeCoreDataSourceViewFilter(

async function getContentNodesForDataSourceViewFromCore(
dataSourceView: DataSourceViewResource | DataSourceViewType,
{ internalIds, parentId, viewType }: GetContentNodesForDataSourceViewParams
{ internalIds, parentId }: GetContentNodesForDataSourceViewParams
): Promise<Result<GetContentNodesForDataSourceViewResult, Error>> {
// There's an early return possible on !dataSourceView.dataSource.connectorId && internalIds?.length === 0,
// won't include it for now as we are shadow-reading.
Expand All @@ -189,7 +189,7 @@ async function getContentNodesForDataSourceViewFromCore(

return new Ok({
nodes: coreRes.value.nodes.map((node) => {
const { type } = getContentNodeMetadata(node, viewType);
const { type } = getContentNodeMetadata(node);
return {
internalId: node.node_id,
parentInternalId: node.parent_id ?? null,
Expand Down
66 changes: 64 additions & 2 deletions front/lib/content_nodes.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,35 @@ import {
Square3Stack3DIcon,
} from "@dust-tt/sparkle";
import type { ContentNode } from "@dust-tt/types";
import { assertNever } from "@dust-tt/types";
import { assertNever, MIME_TYPES } from "@dust-tt/types";

// Mime types that should be represented with a Channel icon.
const CHANNEL_MIME_TYPES = [
MIME_TYPES.GITHUB.DISCUSSIONS,
MIME_TYPES.SLACK.CHANNEL,
] as readonly string[];

// Mime types that should be represented with a Database icon.
const DATABASE_MIME_TYPES = [
MIME_TYPES.GITHUB.ISSUES,
MIME_TYPES.SNOWFLAKE.TABLE,
] as readonly string[];

// Mime types that should be represented with a Folder icon.
const FOLDER_MIME_TYPES = [
MIME_TYPES.CONFLUENCE.SPACE,
MIME_TYPES.GOOGLE_DRIVE.FOLDER,
MIME_TYPES.INTERCOM.TEAM,
MIME_TYPES.MICROSOFT.FOLDER,
MIME_TYPES.NOTION.UNKNOWN_FOLDER,
MIME_TYPES.SNOWFLAKE.DATABASE,
MIME_TYPES.SNOWFLAKE.SCHEMA,
MIME_TYPES.WEBCRAWLER.FOLDER,
MIME_TYPES.ZENDESK.BRAND,
MIME_TYPES.ZENDESK.HELP_CENTER,
MIME_TYPES.ZENDESK.CATEGORY,
MIME_TYPES.ZENDESK.TICKETS,
] as readonly string[];

function getVisualForFileContentNode(node: ContentNode & { type: "file" }) {
if (node.expandable) {
Expand All @@ -17,7 +45,18 @@ function getVisualForFileContentNode(node: ContentNode & { type: "file" }) {
return DocumentIcon;
}

export function getVisualForContentNode(node: ContentNode) {
export function getVisualForContentNode(
node: ContentNode,
useMimeType = false
) {
if (useMimeType) {
return getVisualForContentNodeBasedOnMimeType(node);
} else {
return getVisualForContentNodeBaseOnType(node);
}
}

function getVisualForContentNodeBaseOnType(node: ContentNode) {
switch (node.type) {
case "channel":
if (node.providerVisibility === "private") {
Expand All @@ -40,3 +79,26 @@ export function getVisualForContentNode(node: ContentNode) {
assertNever(node.type);
}
}

function getVisualForContentNodeBasedOnMimeType(node: ContentNode) {
if (!node.mimeType) {
// putting a default value here to please TS,
// but mimeType should always be defined in a world where we get it from core (currently kept for retro-compatibility with connectors' /content_nodes endpoints)
return getVisualForFileContentNode(node as ContentNode & { type: "file" });
}

if (CHANNEL_MIME_TYPES.includes(node.mimeType)) {
if (node.providerVisibility === "private") {
return LockIcon;
}
return ChatBubbleLeftRightIcon;
}
if (DATABASE_MIME_TYPES.includes(node.mimeType)) {
return Square3Stack3DIcon;
}
if (FOLDER_MIME_TYPES.includes(node.mimeType)) {
return FolderIcon;
}
// TODO: we have an issue here in that depending on the view type we sometimes want to render "text/csv" as databases or as files
return getVisualForFileContentNode(node as ContentNode & { type: "file" });
}

0 comments on commit 9c67dd6

Please sign in to comment.