Centralize mimetypes for nodes used internally (#9802)

* add mime types for github

* use the generic mime types for github

* add mime types for Confluence

* add mime types for Google drive

* 📝

* add Intercom mime types

* add Microsoft mime types

* 📝

* add Notion mime types

* add Slack mime types

* add Snowflake mime types

* add webcrawler mime types

* add Zendesk mime types

* fix imports

* keep existing mime type for slack threads

* add intercom conversation type

* add intercom article type

* add mime types for snowflake databases and schemas (just rebased)

connectors/migrations/20241219_backfill_intercom_data_source_folders.ts

@@ -1,7 +1,7 @@
+import { INTERCOM_MIME_TYPES } from "@dust-tt/types";
 import { makeScript } from "scripts/helpers";
 
 import {
-  getDataSourceNodeMimeType,
   getHelpCenterCollectionInternalId,
   getHelpCenterInternalId,
   getParentIdsForCollection,
@@ -36,7 +36,7 @@ async function createFolderNodes(execute: boolean) {
         parents: [getTeamsInternalId(connector.id)],
         parentId: null,
         title: "Conversations",
-        mimeType: getDataSourceNodeMimeType("CONVERSATIONS_FOLDER"),
+        mimeType: INTERCOM_MIME_TYPES.CONVERSATIONS,
       });
     }
 
@@ -60,7 +60,7 @@ async function createFolderNodes(execute: boolean) {
             parents: [teamInternalId, getTeamsInternalId(connector.id)],
             parentId: getTeamsInternalId(connector.id),
             title: team.name,
-            mimeType: getDataSourceNodeMimeType("TEAM"),
+            mimeType: INTERCOM_MIME_TYPES.TEAM,
           });
         }
       },
@@ -99,7 +99,7 @@ async function createFolderNodes(execute: boolean) {
             parents: [helpCenterInternalId],
             parentId: null,
             title: helpCenter.name,
-            mimeType: getDataSourceNodeMimeType("HELP_CENTER"),
+            mimeType: INTERCOM_MIME_TYPES.HELP_CENTER,
           });
         }
 
@@ -133,7 +133,7 @@ async function createFolderNodes(execute: boolean) {
                 parents: collectionParents,
                 parentId: collectionParents[1] || null,
                 title: collection.name,
-                mimeType: getDataSourceNodeMimeType("COLLECTION"),
+                mimeType: INTERCOM_MIME_TYPES.COLLECTION,
               });
             }
           },

connectors/src/connectors/confluence/temporal/activities.ts

@@ -1,5 +1,6 @@
 import type { ModelId } from "@dust-tt/types";
 import {
+  CONFLUENCE_MIME_TYPES,
   ConfluenceClientError,
   isConfluenceNotFoundError,
 } from "@dust-tt/types";
@@ -221,7 +222,7 @@ export async function confluenceUpsertSpaceFolderActivity({
     parents: [makeSpaceInternalId(spaceId)],
     parentId: null,
     title: spaceName,
-    mimeType: "application/vnd.dust.confluence.space",
+    mimeType: CONFLUENCE_MIME_TYPES.SPACE,
   });
 }
 
@@ -328,7 +329,7 @@ async function upsertConfluencePageToDataSource({
       timestampMs: lastPageVersionCreatedAt.getTime(),
       upsertContext: { sync_type: syncType },
       title: page.title,
-      mimeType: "application/vnd.dust.confluence.page",
+      mimeType: CONFLUENCE_MIME_TYPES.PAGE,
       async: true,
     });
   }

connectors/src/connectors/github/temporal/activities.ts

@@ -1,5 +1,5 @@
 import type { CoreAPIDataSourceDocumentSection, ModelId } from "@dust-tt/types";
-import { assertNever } from "@dust-tt/types";
+import { assertNever, GITHUB_MIME_TYPES } from "@dust-tt/types";
 import { Context } from "@temporalio/activity";
 import { hash as blake3 } from "blake3";
 import { promises as fs } from "fs";
@@ -308,7 +308,7 @@ export async function githubUpsertIssueActivity(
       sync_type: isBatchSync ? "batch" : "incremental",
     },
     title: issue.title,
-    mimeType: "application/vnd.dust.github.issue",
+    mimeType: GITHUB_MIME_TYPES.ISSUE,
     async: true,
   });
 
@@ -494,7 +494,7 @@ export async function githubUpsertDiscussionActivity(
       sync_type: isBatchSync ? "batch" : "incremental",
     },
     title: discussion.title,
-    mimeType: "application/vnd.dust.github.discussion",
+    mimeType: GITHUB_MIME_TYPES.DISCUSSION,
     async: true,
   });
 
@@ -960,7 +960,7 @@ export async function githubCodeSyncActivity({
     title: "Code",
     parents: [getCodeRootInternalId(repoId), getRepositoryInternalId(repoId)],
     parentId: getRepositoryInternalId(repoId),
-    mimeType: "application/vnd.dust.github.code.root",
+    mimeType: GITHUB_MIME_TYPES.CODE_ROOT,
   });
 
   let githubCodeRepository = await GithubCodeRepository.findOne({
@@ -1172,7 +1172,7 @@ export async function githubCodeSyncActivity({
               sync_type: isBatchSync ? "batch" : "incremental",
             },
             title: f.fileName,
-            mimeType: "application/vnd.dust.github.code.file",
+            mimeType: GITHUB_MIME_TYPES.CODE_FILE,
             async: true,
           });
 
@@ -1216,7 +1216,7 @@ export async function githubCodeSyncActivity({
           parents,
           parentId: parents[1],
           title: d.dirName,
-          mimeType: "application/vnd.dust.github.code.directory",
+          mimeType: GITHUB_MIME_TYPES.CODE_DIRECTORY,
         });
 
         // Find directory or create it.
@@ -1356,7 +1356,7 @@ export async function githubUpsertRepositoryFolderActivity({
     title: repoName,
     parents: [getRepositoryInternalId(repoId)],
     parentId: null,
-    mimeType: "application/vnd.dust.github.repository",
+    mimeType: GITHUB_MIME_TYPES.REPOSITORY,
   });
 }
 
@@ -1377,7 +1377,7 @@ export async function githubUpsertIssuesFolderActivity({
     title: "Issues",
     parents: [getIssuesInternalId(repoId), getRepositoryInternalId(repoId)],
     parentId: getRepositoryInternalId(repoId),
-    mimeType: "application/vnd.dust.github.issues",
+    mimeType: GITHUB_MIME_TYPES.ISSUES,
   });
 }
 
@@ -1401,7 +1401,7 @@ export async function githubUpsertDiscussionsFolderActivity({
       getRepositoryInternalId(repoId),
     ],
     parentId: getRepositoryInternalId(repoId),
-    mimeType: "application/vnd.dust.github.discussions",
+    mimeType: GITHUB_MIME_TYPES.DISCUSSIONS,
   });
 }
 
@@ -1422,6 +1422,6 @@ export async function githubUpsertCodeRootFolderActivity({
     title: "Code",
     parents: [getCodeRootInternalId(repoId), getRepositoryInternalId(repoId)],
     parentId: getRepositoryInternalId(repoId),
-    mimeType: "application/vnd.dust.github.code.root",
+    mimeType: GITHUB_MIME_TYPES.CODE_ROOT,
   });
 }

connectors/src/connectors/google_drive/temporal/activities.ts

@@ -1,4 +1,5 @@
 import type { ModelId } from "@dust-tt/types";
+import { GOOGLE_DRIVE_MIME_TYPES } from "@dust-tt/types";
 import { uuid4 } from "@temporalio/workflow";
 import type { drive_v3 } from "googleapis";
 import type { GaxiosResponse, OAuth2Client } from "googleapis-common";
@@ -73,7 +74,7 @@ export async function upsertSharedWithMeFolder(connectorId: ModelId) {
     parents: [folderId],
     parentId: null,
     title: "Shared with me",
-    mimeType: "application/vnd.dust.googledrive.folder",
+    mimeType: GOOGLE_DRIVE_MIME_TYPES.FOLDER,
   });
 }
 
@@ -514,7 +515,7 @@ export async function incrementalSync(
           parents,
           parentId: parents[1] || null,
           title: driveFile.name ?? "",
-          mimeType: "application/vnd.dust.googledrive.folder",
+          mimeType: GOOGLE_DRIVE_MIME_TYPES.FOLDER,
         });
 
         await GoogleDriveFiles.upsert({
@@ -859,7 +860,7 @@ export async function markFolderAsVisited(
     parents,
     parentId: parents[1] || null,
     title: file.name ?? "",
-    mimeType: "application/vnd.dust.googledrive.folder",
+    mimeType: GOOGLE_DRIVE_MIME_TYPES.FOLDER,
   });
 
   await GoogleDriveFiles.upsert({

connectors/src/connectors/intercom/lib/utils.ts

@@ -6,28 +6,6 @@ import type {
 } from "@connectors/connectors/intercom/lib/types";
 import { IntercomCollection } from "@connectors/lib/models/intercom";
 
-/**
- * Mimetypes
- */
-export function getDataSourceNodeMimeType(
-  intercomNodeType:
-    | "COLLECTION"
-    | "TEAM"
-    | "CONVERSATIONS_FOLDER"
-    | "HELP_CENTER"
-): string {
-  switch (intercomNodeType) {
-    case "COLLECTION":
-      return "application/vnd.dust.intercom.collection";
-    case "CONVERSATIONS_FOLDER":
-      return "application/vnd.dust.intercom.teams-folder";
-    case "TEAM":
-      return "application/vnd.dust.intercom.team";
-    case "HELP_CENTER":
-      return "application/vnd.dust.intercom.help-center";
-  }
-}
-
 /**
  * From id to internalId
  */

connectors/src/connectors/intercom/temporal/activities.ts

@@ -1,4 +1,5 @@
 import type { ModelId } from "@dust-tt/types";
+import { INTERCOM_MIME_TYPES } from "@dust-tt/types";
 import { Op } from "sequelize";
 
 import { getIntercomAccessToken } from "@connectors/connectors/intercom/lib/intercom_access_token";
@@ -11,7 +12,6 @@ import {
 } from "@connectors/connectors/intercom/lib/intercom_api";
 import type { IntercomSyncAllConversationsStatus } from "@connectors/connectors/intercom/lib/types";
 import {
-  getDataSourceNodeMimeType,
   getHelpCenterInternalId,
   getTeamInternalId,
   getTeamsInternalId,
@@ -176,7 +176,7 @@ export async function syncHelpCenterOnlyActivity({
     title: helpCenterOnIntercom.display_name || "Help Center",
     parents: [helpCenterInternalId],
     parentId: null,
-    mimeType: getDataSourceNodeMimeType("HELP_CENTER"),
+    mimeType: INTERCOM_MIME_TYPES.HELP_CENTER,
   });
 
   // If all children collections are not allowed anymore we delete the Help Center data
@@ -509,7 +509,7 @@ export async function syncTeamOnlyActivity({
     title: teamOnIntercom.name,
     parents: [teamInternalId, getTeamsInternalId(connectorId)],
     parentId: getTeamsInternalId(connectorId),
-    mimeType: getDataSourceNodeMimeType("TEAM"),
+    mimeType: INTERCOM_MIME_TYPES.TEAM,
   });
 
   return true;
@@ -744,6 +744,6 @@ export async function upsertIntercomTeamsFolderActivity({
     title: "Conversations",
     parents: [getTeamsInternalId(connectorId)],
     parentId: null,
-    mimeType: getDataSourceNodeMimeType("CONVERSATIONS_FOLDER"),
+    mimeType: INTERCOM_MIME_TYPES.CONVERSATIONS,
   });
 }

connectors/src/connectors/intercom/temporal/sync_conversation.ts

@@ -1,4 +1,5 @@
 import type { ModelId } from "@dust-tt/types";
+import { INTERCOM_MIME_TYPES } from "@dust-tt/types";
 import TurndownService from "turndown";
 
 import { getIntercomAccessToken } from "@connectors/connectors/intercom/lib/intercom_access_token";
@@ -331,7 +332,7 @@ export async function syncConversation({
       sync_type: syncType,
     },
     title: convoTitle,
-    mimeType: "application/vnd.dust.intercom.conversation",
+    mimeType: INTERCOM_MIME_TYPES.CONVERSATION,
     async: true,
   });
 }

connectors/src/connectors/intercom/temporal/sync_help_center.ts

@@ -1,4 +1,5 @@
 import type { ModelId } from "@dust-tt/types";
+import { INTERCOM_MIME_TYPES } from "@dust-tt/types";
 import TurndownService from "turndown";
 
 import { getIntercomAccessToken } from "@connectors/connectors/intercom/lib/intercom_access_token";
@@ -10,7 +11,6 @@ import type {
 import {
   getArticleInAppUrl,
   getCollectionInAppUrl,
-  getDataSourceNodeMimeType,
   getHelpCenterArticleInternalId,
   getHelpCenterCollectionInternalId,
   getHelpCenterInternalId,
@@ -229,7 +229,7 @@ export async function upsertCollectionWithChildren({
     title: collection.name,
     parents: collectionParents,
     parentId: collectionParents[1],
-    mimeType: getDataSourceNodeMimeType("COLLECTION"),
+    mimeType: INTERCOM_MIME_TYPES.COLLECTION,
   });
 
   // Then we call ourself recursively on the children collections
@@ -429,7 +429,7 @@ export async function upsertArticle({
         sync_type: "batch",
       },
       title: article.title,
-      mimeType: "application/vnd.dust.intercom.article",
+      mimeType: INTERCOM_MIME_TYPES.ARTICLE,
       async: true,
     });
     await articleOnDb.update({

connectors/src/connectors/microsoft/temporal/activities.ts

@@ -1,5 +1,9 @@
 import type { ModelId } from "@dust-tt/types";
-import { cacheWithRedis, removeNulls } from "@dust-tt/types";
+import {
+  cacheWithRedis,
+  MICROSOFT_MIME_TYPES,
+  removeNulls,
+} from "@dust-tt/types";
 import type { Client } from "@microsoft/microsoft-graph-client";
 import { GraphError } from "@microsoft/microsoft-graph-client";
 import type { DriveItem } from "@microsoft/microsoft-graph-types";
@@ -209,7 +213,7 @@ export async function getRootNodesToSyncFromResources(
         parents: [createdOrUpdatedResource.internalId],
         parentId: null,
         title: createdOrUpdatedResource.name ?? "",
-        mimeType: "application/vnd.dust.microsoft.folder",
+        mimeType: MICROSOFT_MIME_TYPES.FOLDER,
       }),
     { concurrency: 5 }
   );
@@ -481,7 +485,7 @@ export async function syncFiles({
         parents: [createdOrUpdatedResource.internalId, ...parentsOfParent],
         parentId: parentsOfParent[0],
         title: createdOrUpdatedResource.name ?? "",
-        mimeType: "application/vnd.dust.microsoft.folder",
+        mimeType: MICROSOFT_MIME_TYPES.FOLDER,
       }),
     { concurrency: 5 }
   );
@@ -655,7 +659,7 @@ export async function syncDeltaForRootNodesInDrive({
           parents: [blob.internalId],
           parentId: null,
           title: blob.name ?? "",
-          mimeType: "application/vnd.dust.microsoft.folder",
+          mimeType: MICROSOFT_MIME_TYPES.FOLDER,
         });
 
         // add parent information to new node resource. for the toplevel folder,
@@ -871,7 +875,7 @@ async function updateDescendantsParentsInCore({
     parents,
     parentId: parents[1] || null,
     title: folder.name ?? "",
-    mimeType: "application/vnd.dust.microsoft.folder",
+    mimeType: MICROSOFT_MIME_TYPES.FOLDER,
   });
 
   await concurrentExecutor(

connectors/src/connectors/notion/index.ts

@@ -3,6 +3,7 @@ import {
   Err,
   getNotionDatabaseTableId,
   getOAuthConnectionAccessToken,
+  NOTION_MIME_TYPES,
   Ok,
 } from "@dust-tt/types";
 import _ from "lodash";
@@ -104,7 +105,7 @@ export class NotionConnectorManager extends BaseConnectorManager<null> {
       parents: [folderId],
       parentId: null,
       title: "Orphaned Resources",
-      mimeType: "application/vnd.dust.notion.unknown-folder",
+      mimeType: NOTION_MIME_TYPES.UNKNOWN_FOLDER,
     });
 
     try {