diff --git a/connectors/migrations/20241219_backfill_intercom_data_source_folders.ts b/connectors/migrations/20241219_backfill_intercom_data_source_folders.ts index d0ed53d635dc..41ac9919eacd 100644 --- a/connectors/migrations/20241219_backfill_intercom_data_source_folders.ts +++ b/connectors/migrations/20241219_backfill_intercom_data_source_folders.ts @@ -1,7 +1,7 @@ +import { INTERCOM_MIME_TYPES } from "@dust-tt/types"; import { makeScript } from "scripts/helpers"; import { - getDataSourceNodeMimeType, getHelpCenterCollectionInternalId, getHelpCenterInternalId, getParentIdsForCollection, @@ -36,7 +36,7 @@ async function createFolderNodes(execute: boolean) { parents: [getTeamsInternalId(connector.id)], parentId: null, title: "Conversations", - mimeType: getDataSourceNodeMimeType("CONVERSATIONS_FOLDER"), + mimeType: INTERCOM_MIME_TYPES.CONVERSATIONS, }); } @@ -60,7 +60,7 @@ async function createFolderNodes(execute: boolean) { parents: [teamInternalId, getTeamsInternalId(connector.id)], parentId: getTeamsInternalId(connector.id), title: team.name, - mimeType: getDataSourceNodeMimeType("TEAM"), + mimeType: INTERCOM_MIME_TYPES.TEAM, }); } }, @@ -99,7 +99,7 @@ async function createFolderNodes(execute: boolean) { parents: [helpCenterInternalId], parentId: null, title: helpCenter.name, - mimeType: getDataSourceNodeMimeType("HELP_CENTER"), + mimeType: INTERCOM_MIME_TYPES.HELP_CENTER, }); } @@ -133,7 +133,7 @@ async function createFolderNodes(execute: boolean) { parents: collectionParents, parentId: collectionParents[1] || null, title: collection.name, - mimeType: getDataSourceNodeMimeType("COLLECTION"), + mimeType: INTERCOM_MIME_TYPES.COLLECTION, }); } }, diff --git a/connectors/src/connectors/confluence/temporal/activities.ts b/connectors/src/connectors/confluence/temporal/activities.ts index dfbcb88955f2..c42bda84ed5d 100644 --- a/connectors/src/connectors/confluence/temporal/activities.ts +++ b/connectors/src/connectors/confluence/temporal/activities.ts @@ -1,5 +1,6 @@ import type { ModelId } from "@dust-tt/types"; import { + CONFLUENCE_MIME_TYPES, ConfluenceClientError, isConfluenceNotFoundError, } from "@dust-tt/types"; @@ -221,7 +222,7 @@ export async function confluenceUpsertSpaceFolderActivity({ parents: [makeSpaceInternalId(spaceId)], parentId: null, title: spaceName, - mimeType: "application/vnd.dust.confluence.space", + mimeType: CONFLUENCE_MIME_TYPES.SPACE, }); } @@ -328,7 +329,7 @@ async function upsertConfluencePageToDataSource({ timestampMs: lastPageVersionCreatedAt.getTime(), upsertContext: { sync_type: syncType }, title: page.title, - mimeType: "application/vnd.dust.confluence.page", + mimeType: CONFLUENCE_MIME_TYPES.PAGE, async: true, }); } diff --git a/connectors/src/connectors/github/temporal/activities.ts b/connectors/src/connectors/github/temporal/activities.ts index 487cb08a7eb8..57910729485c 100644 --- a/connectors/src/connectors/github/temporal/activities.ts +++ b/connectors/src/connectors/github/temporal/activities.ts @@ -1,5 +1,5 @@ import type { CoreAPIDataSourceDocumentSection, ModelId } from "@dust-tt/types"; -import { assertNever } from "@dust-tt/types"; +import { assertNever, GITHUB_MIME_TYPES } from "@dust-tt/types"; import { Context } from "@temporalio/activity"; import { hash as blake3 } from "blake3"; import { promises as fs } from "fs"; @@ -308,7 +308,7 @@ export async function githubUpsertIssueActivity( sync_type: isBatchSync ? "batch" : "incremental", }, title: issue.title, - mimeType: "application/vnd.dust.github.issue", + mimeType: GITHUB_MIME_TYPES.ISSUE, async: true, }); @@ -494,7 +494,7 @@ export async function githubUpsertDiscussionActivity( sync_type: isBatchSync ? "batch" : "incremental", }, title: discussion.title, - mimeType: "application/vnd.dust.github.discussion", + mimeType: GITHUB_MIME_TYPES.DISCUSSION, async: true, }); @@ -960,7 +960,7 @@ export async function githubCodeSyncActivity({ title: "Code", parents: [getCodeRootInternalId(repoId), getRepositoryInternalId(repoId)], parentId: getRepositoryInternalId(repoId), - mimeType: "application/vnd.dust.github.code.root", + mimeType: GITHUB_MIME_TYPES.CODE_ROOT, }); let githubCodeRepository = await GithubCodeRepository.findOne({ @@ -1172,7 +1172,7 @@ export async function githubCodeSyncActivity({ sync_type: isBatchSync ? "batch" : "incremental", }, title: f.fileName, - mimeType: "application/vnd.dust.github.code.file", + mimeType: GITHUB_MIME_TYPES.CODE_FILE, async: true, }); @@ -1216,7 +1216,7 @@ export async function githubCodeSyncActivity({ parents, parentId: parents[1], title: d.dirName, - mimeType: "application/vnd.dust.github.code.directory", + mimeType: GITHUB_MIME_TYPES.CODE_DIRECTORY, }); // Find directory or create it. @@ -1356,7 +1356,7 @@ export async function githubUpsertRepositoryFolderActivity({ title: repoName, parents: [getRepositoryInternalId(repoId)], parentId: null, - mimeType: "application/vnd.dust.github.repository", + mimeType: GITHUB_MIME_TYPES.REPOSITORY, }); } @@ -1377,7 +1377,7 @@ export async function githubUpsertIssuesFolderActivity({ title: "Issues", parents: [getIssuesInternalId(repoId), getRepositoryInternalId(repoId)], parentId: getRepositoryInternalId(repoId), - mimeType: "application/vnd.dust.github.issues", + mimeType: GITHUB_MIME_TYPES.ISSUES, }); } @@ -1401,7 +1401,7 @@ export async function githubUpsertDiscussionsFolderActivity({ getRepositoryInternalId(repoId), ], parentId: getRepositoryInternalId(repoId), - mimeType: "application/vnd.dust.github.discussions", + mimeType: GITHUB_MIME_TYPES.DISCUSSIONS, }); } @@ -1422,6 +1422,6 @@ export async function githubUpsertCodeRootFolderActivity({ title: "Code", parents: [getCodeRootInternalId(repoId), getRepositoryInternalId(repoId)], parentId: getRepositoryInternalId(repoId), - mimeType: "application/vnd.dust.github.code.root", + mimeType: GITHUB_MIME_TYPES.CODE_ROOT, }); } diff --git a/connectors/src/connectors/google_drive/temporal/activities.ts b/connectors/src/connectors/google_drive/temporal/activities.ts index 65e1a2859835..d9ee917aa548 100644 --- a/connectors/src/connectors/google_drive/temporal/activities.ts +++ b/connectors/src/connectors/google_drive/temporal/activities.ts @@ -1,4 +1,5 @@ import type { ModelId } from "@dust-tt/types"; +import { GOOGLE_DRIVE_MIME_TYPES } from "@dust-tt/types"; import { uuid4 } from "@temporalio/workflow"; import type { drive_v3 } from "googleapis"; import type { GaxiosResponse, OAuth2Client } from "googleapis-common"; @@ -73,7 +74,7 @@ export async function upsertSharedWithMeFolder(connectorId: ModelId) { parents: [folderId], parentId: null, title: "Shared with me", - mimeType: "application/vnd.dust.googledrive.folder", + mimeType: GOOGLE_DRIVE_MIME_TYPES.FOLDER, }); } @@ -514,7 +515,7 @@ export async function incrementalSync( parents, parentId: parents[1] || null, title: driveFile.name ?? "", - mimeType: "application/vnd.dust.googledrive.folder", + mimeType: GOOGLE_DRIVE_MIME_TYPES.FOLDER, }); await GoogleDriveFiles.upsert({ @@ -859,7 +860,7 @@ export async function markFolderAsVisited( parents, parentId: parents[1] || null, title: file.name ?? "", - mimeType: "application/vnd.dust.googledrive.folder", + mimeType: GOOGLE_DRIVE_MIME_TYPES.FOLDER, }); await GoogleDriveFiles.upsert({ diff --git a/connectors/src/connectors/intercom/lib/utils.ts b/connectors/src/connectors/intercom/lib/utils.ts index a763145be6e5..fdc81d0f7470 100644 --- a/connectors/src/connectors/intercom/lib/utils.ts +++ b/connectors/src/connectors/intercom/lib/utils.ts @@ -6,28 +6,6 @@ import type { } from "@connectors/connectors/intercom/lib/types"; import { IntercomCollection } from "@connectors/lib/models/intercom"; -/** - * Mimetypes - */ -export function getDataSourceNodeMimeType( - intercomNodeType: - | "COLLECTION" - | "TEAM" - | "CONVERSATIONS_FOLDER" - | "HELP_CENTER" -): string { - switch (intercomNodeType) { - case "COLLECTION": - return "application/vnd.dust.intercom.collection"; - case "CONVERSATIONS_FOLDER": - return "application/vnd.dust.intercom.teams-folder"; - case "TEAM": - return "application/vnd.dust.intercom.team"; - case "HELP_CENTER": - return "application/vnd.dust.intercom.help-center"; - } -} - /** * From id to internalId */ diff --git a/connectors/src/connectors/intercom/temporal/activities.ts b/connectors/src/connectors/intercom/temporal/activities.ts index de1b06a85ed0..fd7196176193 100644 --- a/connectors/src/connectors/intercom/temporal/activities.ts +++ b/connectors/src/connectors/intercom/temporal/activities.ts @@ -1,4 +1,5 @@ import type { ModelId } from "@dust-tt/types"; +import { INTERCOM_MIME_TYPES } from "@dust-tt/types"; import { Op } from "sequelize"; import { getIntercomAccessToken } from "@connectors/connectors/intercom/lib/intercom_access_token"; @@ -11,7 +12,6 @@ import { } from "@connectors/connectors/intercom/lib/intercom_api"; import type { IntercomSyncAllConversationsStatus } from "@connectors/connectors/intercom/lib/types"; import { - getDataSourceNodeMimeType, getHelpCenterInternalId, getTeamInternalId, getTeamsInternalId, @@ -176,7 +176,7 @@ export async function syncHelpCenterOnlyActivity({ title: helpCenterOnIntercom.display_name || "Help Center", parents: [helpCenterInternalId], parentId: null, - mimeType: getDataSourceNodeMimeType("HELP_CENTER"), + mimeType: INTERCOM_MIME_TYPES.HELP_CENTER, }); // If all children collections are not allowed anymore we delete the Help Center data @@ -509,7 +509,7 @@ export async function syncTeamOnlyActivity({ title: teamOnIntercom.name, parents: [teamInternalId, getTeamsInternalId(connectorId)], parentId: getTeamsInternalId(connectorId), - mimeType: getDataSourceNodeMimeType("TEAM"), + mimeType: INTERCOM_MIME_TYPES.TEAM, }); return true; @@ -744,6 +744,6 @@ export async function upsertIntercomTeamsFolderActivity({ title: "Conversations", parents: [getTeamsInternalId(connectorId)], parentId: null, - mimeType: getDataSourceNodeMimeType("CONVERSATIONS_FOLDER"), + mimeType: INTERCOM_MIME_TYPES.CONVERSATIONS, }); } diff --git a/connectors/src/connectors/intercom/temporal/sync_conversation.ts b/connectors/src/connectors/intercom/temporal/sync_conversation.ts index 56b00b70447a..93c245ed4ae9 100644 --- a/connectors/src/connectors/intercom/temporal/sync_conversation.ts +++ b/connectors/src/connectors/intercom/temporal/sync_conversation.ts @@ -1,4 +1,5 @@ import type { ModelId } from "@dust-tt/types"; +import { INTERCOM_MIME_TYPES } from "@dust-tt/types"; import TurndownService from "turndown"; import { getIntercomAccessToken } from "@connectors/connectors/intercom/lib/intercom_access_token"; @@ -331,7 +332,7 @@ export async function syncConversation({ sync_type: syncType, }, title: convoTitle, - mimeType: "application/vnd.dust.intercom.conversation", + mimeType: INTERCOM_MIME_TYPES.CONVERSATION, async: true, }); } diff --git a/connectors/src/connectors/intercom/temporal/sync_help_center.ts b/connectors/src/connectors/intercom/temporal/sync_help_center.ts index 8aa9d21dd94c..8868f7163376 100644 --- a/connectors/src/connectors/intercom/temporal/sync_help_center.ts +++ b/connectors/src/connectors/intercom/temporal/sync_help_center.ts @@ -1,4 +1,5 @@ import type { ModelId } from "@dust-tt/types"; +import { INTERCOM_MIME_TYPES } from "@dust-tt/types"; import TurndownService from "turndown"; import { getIntercomAccessToken } from "@connectors/connectors/intercom/lib/intercom_access_token"; @@ -10,7 +11,6 @@ import type { import { getArticleInAppUrl, getCollectionInAppUrl, - getDataSourceNodeMimeType, getHelpCenterArticleInternalId, getHelpCenterCollectionInternalId, getHelpCenterInternalId, @@ -229,7 +229,7 @@ export async function upsertCollectionWithChildren({ title: collection.name, parents: collectionParents, parentId: collectionParents[1], - mimeType: getDataSourceNodeMimeType("COLLECTION"), + mimeType: INTERCOM_MIME_TYPES.COLLECTION, }); // Then we call ourself recursively on the children collections @@ -429,7 +429,7 @@ export async function upsertArticle({ sync_type: "batch", }, title: article.title, - mimeType: "application/vnd.dust.intercom.article", + mimeType: INTERCOM_MIME_TYPES.ARTICLE, async: true, }); await articleOnDb.update({ diff --git a/connectors/src/connectors/microsoft/temporal/activities.ts b/connectors/src/connectors/microsoft/temporal/activities.ts index 1815014be96b..a64f72b3ac59 100644 --- a/connectors/src/connectors/microsoft/temporal/activities.ts +++ b/connectors/src/connectors/microsoft/temporal/activities.ts @@ -1,5 +1,9 @@ import type { ModelId } from "@dust-tt/types"; -import { cacheWithRedis, removeNulls } from "@dust-tt/types"; +import { + cacheWithRedis, + MICROSOFT_MIME_TYPES, + removeNulls, +} from "@dust-tt/types"; import type { Client } from "@microsoft/microsoft-graph-client"; import { GraphError } from "@microsoft/microsoft-graph-client"; import type { DriveItem } from "@microsoft/microsoft-graph-types"; @@ -209,7 +213,7 @@ export async function getRootNodesToSyncFromResources( parents: [createdOrUpdatedResource.internalId], parentId: null, title: createdOrUpdatedResource.name ?? "", - mimeType: "application/vnd.dust.microsoft.folder", + mimeType: MICROSOFT_MIME_TYPES.FOLDER, }), { concurrency: 5 } ); @@ -481,7 +485,7 @@ export async function syncFiles({ parents: [createdOrUpdatedResource.internalId, ...parentsOfParent], parentId: parentsOfParent[0], title: createdOrUpdatedResource.name ?? "", - mimeType: "application/vnd.dust.microsoft.folder", + mimeType: MICROSOFT_MIME_TYPES.FOLDER, }), { concurrency: 5 } ); @@ -655,7 +659,7 @@ export async function syncDeltaForRootNodesInDrive({ parents: [blob.internalId], parentId: null, title: blob.name ?? "", - mimeType: "application/vnd.dust.microsoft.folder", + mimeType: MICROSOFT_MIME_TYPES.FOLDER, }); // add parent information to new node resource. for the toplevel folder, @@ -871,7 +875,7 @@ async function updateDescendantsParentsInCore({ parents, parentId: parents[1] || null, title: folder.name ?? "", - mimeType: "application/vnd.dust.microsoft.folder", + mimeType: MICROSOFT_MIME_TYPES.FOLDER, }); await concurrentExecutor( diff --git a/connectors/src/connectors/notion/index.ts b/connectors/src/connectors/notion/index.ts index 39bc0a4efe37..fea202ad3086 100644 --- a/connectors/src/connectors/notion/index.ts +++ b/connectors/src/connectors/notion/index.ts @@ -3,6 +3,7 @@ import { Err, getNotionDatabaseTableId, getOAuthConnectionAccessToken, + NOTION_MIME_TYPES, Ok, } from "@dust-tt/types"; import _ from "lodash"; @@ -104,7 +105,7 @@ export class NotionConnectorManager extends BaseConnectorManager { parents: [folderId], parentId: null, title: "Orphaned Resources", - mimeType: "application/vnd.dust.notion.unknown-folder", + mimeType: NOTION_MIME_TYPES.UNKNOWN_FOLDER, }); try { diff --git a/connectors/src/connectors/notion/temporal/activities.ts b/connectors/src/connectors/notion/temporal/activities.ts index 77c0a1714e79..864fcd25842f 100644 --- a/connectors/src/connectors/notion/temporal/activities.ts +++ b/connectors/src/connectors/notion/temporal/activities.ts @@ -4,7 +4,12 @@ import type { PageObjectProperties, ParsedNotionBlock, } from "@dust-tt/types"; -import { assertNever, getNotionDatabaseTableId, slugify } from "@dust-tt/types"; +import { + assertNever, + getNotionDatabaseTableId, + NOTION_MIME_TYPES, + slugify, +} from "@dust-tt/types"; import { isFullBlock, isFullPage, isNotionClientError } from "@notionhq/client"; import type { PageObjectResponse } from "@notionhq/client/build/src/api-endpoints"; import { Context } from "@temporalio/activity"; @@ -1824,7 +1829,7 @@ export async function renderAndUpsertPageFromCache({ parents: parents, parentId: parents[1] || null, title: parentDb.title ?? "Untitled Notion Database", - mimeType: "application/vnd.dust.notion.database", + mimeType: NOTION_MIME_TYPES.DATABASE, }), localLogger ); @@ -2048,7 +2053,7 @@ export async function renderAndUpsertPageFromCache({ sync_type: isFullSync ? "batch" : "incremental", }, title: title ?? "", - mimeType: "application/vnd.dust.notion.page", + mimeType: NOTION_MIME_TYPES.PAGE, async: true, }); } @@ -2545,7 +2550,7 @@ export async function upsertDatabaseStructuredDataFromCache({ parents: parentIds, parentId: parentIds[1] || null, title: dbModel.title ?? "Untitled Notion Database", - mimeType: "application/vnd.dust.notion.database", + mimeType: NOTION_MIME_TYPES.DATABASE, }), localLogger ); @@ -2606,7 +2611,7 @@ export async function upsertDatabaseStructuredDataFromCache({ sync_type: "batch", }, title: databaseName, - mimeType: "application/vnd.dust.notion.database", + mimeType: NOTION_MIME_TYPES.DATABASE, async: true, }); } else { diff --git a/connectors/src/connectors/slack/temporal/activities.ts b/connectors/src/connectors/slack/temporal/activities.ts index d9a2a1c0ab28..5cfa0691bfa5 100644 --- a/connectors/src/connectors/slack/temporal/activities.ts +++ b/connectors/src/connectors/slack/temporal/activities.ts @@ -1,5 +1,9 @@ import type { CoreAPIDataSourceDocumentSection, ModelId } from "@dust-tt/types"; -import { cacheWithRedis, safeSubstring } from "@dust-tt/types"; +import { + cacheWithRedis, + safeSubstring, + SLACK_MIME_TYPES, +} from "@dust-tt/types"; import type { CodedError, WebAPIPlatformError, @@ -254,7 +258,7 @@ export async function syncChannel( title: `#${channel.name}`, parentId: null, parents: [slackChannelInternalIdFromSlackChannelId(channelId)], - mimeType: "application/vnd.dust.slack.channel", + mimeType: SLACK_MIME_TYPES.CHANNEL, }); } @@ -632,7 +636,7 @@ export async function syncNonThreaded( sync_type: isBatchSync ? "batch" : "incremental", }, title: tags.find((t) => t.startsWith("title:"))?.split(":")[1] ?? "", - mimeType: "text/vnd.dust.slack.thread", + mimeType: SLACK_MIME_TYPES.THREAD, async: true, }); } @@ -846,7 +850,7 @@ export async function syncThread( sync_type: isBatchSync ? "batch" : "incremental", }, title: tags.find((t) => t.startsWith("title:"))?.split(":")[1] ?? "", - mimeType: "text/vnd.dust.slack.thread", + mimeType: SLACK_MIME_TYPES.THREAD, async: true, }); } diff --git a/connectors/src/connectors/snowflake/temporal/activities.ts b/connectors/src/connectors/snowflake/temporal/activities.ts index 719686202372..d7947bc4527e 100644 --- a/connectors/src/connectors/snowflake/temporal/activities.ts +++ b/connectors/src/connectors/snowflake/temporal/activities.ts @@ -1,5 +1,5 @@ import type { ModelId } from "@dust-tt/types"; -import { isSnowflakeCredentials } from "@dust-tt/types"; +import { isSnowflakeCredentials, SNOWFLAKE_MIME_TYPES } from "@dust-tt/types"; import { connectToSnowflake, @@ -168,7 +168,7 @@ export async function syncSnowflakeConnection(connectorId: ModelId) { title: table.databaseName, parents: [table.databaseName], parentId: null, - mimeType: "application/vnd.snowflake.database", + mimeType: SNOWFLAKE_MIME_TYPES.DATABASE, }); // upsert a folder for the schema (child of the database) @@ -179,7 +179,7 @@ export async function syncSnowflakeConnection(connectorId: ModelId) { title: table.schemaName, parents: [schemaId, table.databaseName], parentId: table.databaseName, - mimeType: "application/vnd.snowflake.schema", + mimeType: SNOWFLAKE_MIME_TYPES.SCHEMA, }); await upsertDataSourceRemoteTable({ @@ -192,7 +192,7 @@ export async function syncSnowflakeConnection(connectorId: ModelId) { parents: [table.internalId, schemaId, table.databaseName], parentId: schemaId, title: table.name, - mimeType: "application/vnd.snowflake.table", + mimeType: SNOWFLAKE_MIME_TYPES.TABLE, }); await table.update({ lastUpsertedAt: new Date(), diff --git a/connectors/src/connectors/webcrawler/temporal/activities.ts b/connectors/src/connectors/webcrawler/temporal/activities.ts index 49e208c376c1..70a441d0aec7 100644 --- a/connectors/src/connectors/webcrawler/temporal/activities.ts +++ b/connectors/src/connectors/webcrawler/temporal/activities.ts @@ -3,6 +3,7 @@ import { stripNullBytes, WEBCRAWLER_MAX_DEPTH, WEBCRAWLER_MAX_PAGES, + WEBCRAWLER_MIME_TYPES, } from "@dust-tt/types"; import { validateUrl } from "@dust-tt/types/src/shared/utils/url_utils"; import { Context } from "@temporalio/activity"; @@ -291,7 +292,7 @@ export async function crawlWebsiteByConnectorId(connectorId: ModelId) { parents, parentId: parents[1] || null, title: folder, - mimeType: "application/vnd.dust.webcrawler.folder", + mimeType: WEBCRAWLER_MIME_TYPES.FOLDER, }); createdFolders.add(folder); diff --git a/connectors/src/connectors/zendesk/lib/sync_article.ts b/connectors/src/connectors/zendesk/lib/sync_article.ts index 9e048321126f..1c31c0dd3919 100644 --- a/connectors/src/connectors/zendesk/lib/sync_article.ts +++ b/connectors/src/connectors/zendesk/lib/sync_article.ts @@ -1,4 +1,5 @@ import type { ModelId } from "@dust-tt/types"; +import { ZENDESK_MIME_TYPES } from "@dust-tt/types"; import TurndownService from "turndown"; import type { @@ -175,7 +176,7 @@ export async function syncArticle({ loggerArgs: { ...loggerArgs, articleId: article.id }, upsertContext: { sync_type: "batch" }, title: article.title, - mimeType: "application/vnd.dust.zendesk.article", + mimeType: ZENDESK_MIME_TYPES.ARTICLE, async: true, }); await articleInDb.update({ lastUpsertedTs: new Date(currentSyncDateMs) }); diff --git a/connectors/src/connectors/zendesk/lib/sync_category.ts b/connectors/src/connectors/zendesk/lib/sync_category.ts index d68e3010065f..1ae683817183 100644 --- a/connectors/src/connectors/zendesk/lib/sync_category.ts +++ b/connectors/src/connectors/zendesk/lib/sync_category.ts @@ -1,4 +1,5 @@ import type { ModelId } from "@dust-tt/types"; +import { ZENDESK_MIME_TYPES } from "@dust-tt/types"; import type { ZendeskFetchedCategory } from "@connectors/@types/node-zendesk"; import { @@ -117,6 +118,6 @@ export async function syncCategory({ parents, parentId: parents[1], title: categoryInDb.name, - mimeType: "application/vnd.dust.zendesk.category", + mimeType: ZENDESK_MIME_TYPES.CATEGORY, }); } diff --git a/connectors/src/connectors/zendesk/lib/sync_ticket.ts b/connectors/src/connectors/zendesk/lib/sync_ticket.ts index 61752b1cfd58..7c0388459202 100644 --- a/connectors/src/connectors/zendesk/lib/sync_ticket.ts +++ b/connectors/src/connectors/zendesk/lib/sync_ticket.ts @@ -1,4 +1,5 @@ import type { ModelId } from "@dust-tt/types"; +import { ZENDESK_MIME_TYPES } from "@dust-tt/types"; import TurndownService from "turndown"; import type { @@ -236,7 +237,7 @@ ${comments loggerArgs: { ...loggerArgs, ticketId: ticket.id }, upsertContext: { sync_type: "batch" }, title: ticket.subject, - mimeType: "application/vnd.dust.zendesk.ticket", + mimeType: ZENDESK_MIME_TYPES.TICKET, async: true, }); await ticketInDb.update({ lastUpsertedTs: new Date(currentSyncDateMs) }); diff --git a/connectors/src/connectors/zendesk/temporal/activities.ts b/connectors/src/connectors/zendesk/temporal/activities.ts index ad8e151fae70..f36d4915183d 100644 --- a/connectors/src/connectors/zendesk/temporal/activities.ts +++ b/connectors/src/connectors/zendesk/temporal/activities.ts @@ -1,4 +1,5 @@ import type { ModelId } from "@dust-tt/types"; +import { ZENDESK_MIME_TYPES } from "@dust-tt/types"; import _ from "lodash"; import { getBrandInternalId } from "@connectors/connectors/zendesk/lib/id_conversions"; @@ -136,7 +137,7 @@ export async function syncZendeskBrandActivity({ parents: [brandInternalId], parentId: null, title: brandInDb.name, - mimeType: "application/vnd.dust.zendesk.brand", + mimeType: ZENDESK_MIME_TYPES.BRAND, }); // using the content node to get one source of truth regarding the parent relationship @@ -147,7 +148,7 @@ export async function syncZendeskBrandActivity({ parents: [helpCenterNode.internalId, helpCenterNode.parentInternalId], parentId: helpCenterNode.parentInternalId, title: helpCenterNode.title, - mimeType: "application/vnd.dust.zendesk.helpcenter", + mimeType: ZENDESK_MIME_TYPES.HELP_CENTER, }); // using the content node to get one source of truth regarding the parent relationship @@ -158,7 +159,7 @@ export async function syncZendeskBrandActivity({ parents: [ticketsNode.internalId, ticketsNode.parentInternalId], parentId: ticketsNode.parentInternalId, title: ticketsNode.title, - mimeType: "application/vnd.dust.zendesk.tickets", + mimeType: ZENDESK_MIME_TYPES.TICKETS, }); // updating the entry in db @@ -334,7 +335,7 @@ export async function syncZendeskCategoryActivity({ parents, parentId: parents[1], title: categoryInDb.name, - mimeType: "application/vnd.dust.zendesk.category", + mimeType: ZENDESK_MIME_TYPES.CATEGORY, }); // otherwise, we update the category name and lastUpsertedTs diff --git a/connectors/src/connectors/zendesk/temporal/incremental_activities.ts b/connectors/src/connectors/zendesk/temporal/incremental_activities.ts index 28550badaa78..6990dce9189f 100644 --- a/connectors/src/connectors/zendesk/temporal/incremental_activities.ts +++ b/connectors/src/connectors/zendesk/temporal/incremental_activities.ts @@ -1,4 +1,5 @@ import type { ModelId } from "@dust-tt/types"; +import { ZENDESK_MIME_TYPES } from "@dust-tt/types"; import { syncArticle } from "@connectors/connectors/zendesk/lib/sync_article"; import { @@ -153,7 +154,7 @@ export async function syncZendeskArticleUpdateBatchActivity({ parents, parentId: parents[1], title: category.name, - mimeType: "application/vnd.dust.zendesk.category", + mimeType: ZENDESK_MIME_TYPES.CATEGORY, }); } else { /// ignoring these to proceed with the other articles, but these might have to be checked at some point diff --git a/types/src/index.ts b/types/src/index.ts index 2a3cb231f704..ccad866df7e3 100644 --- a/types/src/index.ts +++ b/types/src/index.ts @@ -78,6 +78,7 @@ export * from "./shared/cache"; export * from "./shared/deployment"; export * from "./shared/env"; export * from "./shared/feature_flags"; +export * from "./shared/internal_mime_types"; export * from "./shared/message_classification"; export * from "./shared/model_id"; export * from "./shared/rate_limiter"; diff --git a/types/src/shared/internal_mime_types.ts b/types/src/shared/internal_mime_types.ts new file mode 100644 index 000000000000..959afb2dd03a --- /dev/null +++ b/types/src/shared/internal_mime_types.ts @@ -0,0 +1,107 @@ +export const CONFLUENCE_MIME_TYPES = { + SPACE: "application/vnd.dust.confluence.space", + PAGE: "application/vnd.dust.confluence.page", +}; + +export type ConfluenceMimeType = + (typeof CONFLUENCE_MIME_TYPES)[keyof typeof CONFLUENCE_MIME_TYPES]; + +export const GITHUB_MIME_TYPES = { + REPOSITORY: "application/vnd.dust.github.repository", + CODE_ROOT: "application/vnd.dust.github.code.root", + CODE_DIRECTORY: "application/vnd.dust.github.code.directory", + CODE_FILE: "application/vnd.dust.github.code.file", + ISSUES: "application/vnd.dust.github.issues", + ISSUE: "application/vnd.dust.github.issue", + DISCUSSIONS: "application/vnd.dust.github.discussions", + DISCUSSION: "application/vnd.dust.github.discussion", +}; + +export type GithubMimeType = + (typeof GITHUB_MIME_TYPES)[keyof typeof GITHUB_MIME_TYPES]; + +export const GOOGLE_DRIVE_MIME_TYPES = { + FOLDER: "application/vnd.dust.googledrive.folder", + // for files and spreadsheets, we keep Google's mime types +}; + +export type GoogleDriveMimeType = + (typeof GOOGLE_DRIVE_MIME_TYPES)[keyof typeof GOOGLE_DRIVE_MIME_TYPES]; + +export const INTERCOM_MIME_TYPES = { + COLLECTION: "application/vnd.dust.intercom.collection", + CONVERSATIONS: "application/vnd.dust.intercom.teams-folder", + CONVERSATION: "application/vnd.dust.intercom.conversation", + TEAM: "application/vnd.dust.intercom.team", + HELP_CENTER: "application/vnd.dust.intercom.help-center", + ARTICLE: "application/vnd.dust.intercom.article", +}; + +export type IntercomMimeType = + (typeof INTERCOM_MIME_TYPES)[keyof typeof INTERCOM_MIME_TYPES]; + +export const MICROSOFT_MIME_TYPES = { + FOLDER: "application/vnd.dust.microsoft.folder", + // for files and spreadsheets, we keep Microsoft's mime types +}; + +export type MicrosoftMimeType = + (typeof MICROSOFT_MIME_TYPES)[keyof typeof MICROSOFT_MIME_TYPES]; + +export const NOTION_MIME_TYPES = { + UNKNOWN_FOLDER: "application/vnd.dust.notion.unknown-folder", + DATABASE: "application/vnd.dust.notion.database", + PAGE: "application/vnd.dust.notion.page", +}; + +export type NotionMimeType = + (typeof NOTION_MIME_TYPES)[keyof typeof NOTION_MIME_TYPES]; + +export const SLACK_MIME_TYPES = { + CHANNEL: "application/vnd.dust.slack.channel", + THREAD: "text/vnd.dust.slack.thread", +}; + +export type SlackMimeType = + (typeof SLACK_MIME_TYPES)[keyof typeof SLACK_MIME_TYPES]; + +export const SNOWFLAKE_MIME_TYPES = { + DATABASE: "application/vnd.snowflake.database", + SCHEMA: "application/vnd.snowflake.schema", + TABLE: "application/vnd.snowflake.table", +}; + +export type SnowflakeMimeType = + (typeof SNOWFLAKE_MIME_TYPES)[keyof typeof SNOWFLAKE_MIME_TYPES]; + +export const WEBCRAWLER_MIME_TYPES = { + FOLDER: "application/vnd.dust.webcrawler.folder", + // pages are upserted as text/html, not an internal mime type +}; + +export type WebcrawlerMimeType = + (typeof WEBCRAWLER_MIME_TYPES)[keyof typeof WEBCRAWLER_MIME_TYPES]; + +export const ZENDESK_MIME_TYPES = { + BRAND: "application/vnd.dust.zendesk.brand", + HELP_CENTER: "application/vnd.dust.zendesk.helpcenter", + CATEGORY: "application/vnd.dust.zendesk.category", + ARTICLE: "application/vnd.dust.zendesk.article", + TICKETS: "application/vnd.dust.zendesk.tickets", + TICKET: "application/vnd.dust.zendesk.ticket", +}; + +export type ZendeskMimeType = + (typeof ZENDESK_MIME_TYPES)[keyof typeof ZENDESK_MIME_TYPES]; + +export type DustMimeType = + | ConfluenceMimeType + | GithubMimeType + | GoogleDriveMimeType + | IntercomMimeType + | MicrosoftMimeType + | NotionMimeType + | SlackMimeType + | SnowflakeMimeType + | WebcrawlerMimeType + | ZendeskMimeType;