diff --git a/.env.example b/.env.example index 7f63481fd..0c4fe65b2 100644 --- a/.env.example +++ b/.env.example @@ -109,7 +109,24 @@ LINEAR_TICKETING_CLOUD_CLIENT_SECRET= # Box BOX_FILESTORAGE_CLOUD_CLIENT_ID= BOX_FILESTORAGE_CLOUD_CLIENT_SECRET= +# Onedrive +ONEDRIVE_FILESTORAGE_CLOUD_CLIENT_ID= +ONEDRIVE_FILESTORAGE_CLOUD_CLIENT_SECRET= +# dropbox +DROPBOX_FILESTORAGE_CLOUD_CLIENT_ID= +DROPBOX_FILESTORAGE_CLOUD_CLIENT_SECRET= +# Google Drive +GOOGLEDRIVE_FILESTORAGE_CLOUD_CLIENT_ID= +GOOGLEDRIVE_FILESTORAGE_CLOUD_CLIENT_SECRET= + +# Google Drive +SHAREPOINT_FILESTORAGE_CLOUD_CLIENT_ID= +SHAREPOINT_FILESTORAGE_CLOUD_CLIENT_SECRET= + +# Google Drive +DROPBOX_FILESTORAGE_CLOUD_CLIENT_ID= +DROPBOX_FILESTORAGE_CLOUD_CLIENT_SECRET= # ================================================ # HRIS @@ -150,8 +167,42 @@ SQUARESPACE_ECOMMERCE_CLOUD_CLIENT_SECRET= # Webapp settings # Must be set in the perspective of the end user browser -NEXT_PUBLIC_BACKEND_DOMAIN=http://localhost:3000 # https://api.panora.dev/ +NEXT_PUBLIC_BACKEND_DOMAIN=http://localhost:3000 NEXT_PUBLIC_MAGIC_LINK_DOMAIN=http://localhost:81 -NEXT_PUBLIC_WEBAPP_DOMAIN="http://localhost" -NEXT_PUBLIC_DISTRIBUTION="selfhost" # selfhost or managed +NEXT_PUBLIC_WEBAPP_DOMAIN=http://localhost +NEXT_PUBLIC_DISTRIBUTION=selfhost # selfhost or managed + + + +# VEC DBs +## pinecone +PINECONE_API_KEY= +PINECONE_INDEX_NAME= + +## qdrant +QDRANT_BASE_URL= +QDRANT_API_KEY= +## chroma +CHROMADB_URL= +## weaviate +WEAVIATE_URL= +WEAVIATE_API_KEY= +# turbopuffer +TURBOPUFFER_API_KEY= +# milvus +MILVUS_ADDRESS= + +# EMBEDDINGS +JINA_API_KEY= +COHERE_API_KEY= +OPENAI_API_KEY= + +# ================================================ +# Minio (s3 file storage for documents) +# ================================================ +MINIO_ROOT_USER=myaccesskey13 +MINIO_ROOT_PASSWORD=mysecretkey12 + +UNSTRUCTURED_API_KEY= +UNSTRUCTURED_API_URL= diff --git a/README.md b/README.md index 60f91bc15..004567e27 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,7 @@ Panora supports integration with the following objects across multiple platforms | Box | | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | | | Dropbox | | ✔️ | ✔️ | ✔️ | ✔️ | ✔️ | | | OneDrive | ✔️ | ✔️ | ✔️| ✔️ | ✔️ | | | +| Sharepoint | ✔️ | ✔️ | ✔️| ✔️ | ✔️ | | | ### Ecommerce Unified API @@ -117,8 +118,8 @@ Your favourite software is missing? [Ask the community to build a connector!](ht ## 🧠 Retrieval Engine for RAG -- [ ] Access and manage data from any source, including documents, chunk & vectors -- [ ] Semantic, keyword and hybrid search against a vector database +- [x] Access and manage data from any source, including documents, chunk & vectors +- [x] Semantic, keyword and hybrid search against a vector database ## 🪄 Integrations Coming Soon @@ -129,7 +130,7 @@ Your favourite software is missing? [Ask the community to build a connector!](ht - [x] Redtail CRM - [x] Wealthbox - [x] Leadsquared -- [ ] Salesforce +- [x] Salesforce - [ ] Affinity CRM - [ ] Odoo - [ ] Intelliflow @@ -145,7 +146,6 @@ Your favourite software is missing? [Ask the community to build a connector!](ht - [ ] Service Now - [ ] Wrike - [ ] Dixa -- [ ] Service Now - [ ] Asana - [ ] Aha - [ ] Clickup @@ -158,10 +158,10 @@ Your favourite software is missing? [Ask the community to build a connector!](ht #### File Storage -- [ ] Google Drive -- [ ] Dropbox -- [ ] Sharepoint -- [ ] One Drive +- [x] Google Drive +- [x] Dropbox +- [x] Sharepoint +- [x] One Drive #### Productivity diff --git a/apps/magic-link/src/hooks/useOAuth.ts b/apps/magic-link/src/hooks/useOAuth.ts index b54dd1926..8af512305 100644 --- a/apps/magic-link/src/hooks/useOAuth.ts +++ b/apps/magic-link/src/hooks/useOAuth.ts @@ -14,12 +14,10 @@ type UseOAuthProps = { value: string | null; }, onSuccess: () => void; - additionalParams?: { - end_user_domain: string; - } + additionalParams?: {[key: string]: any} }; -const useOAuth = ({ providerName, vertical, returnUrl, projectId, linkedUserId,additionalParams, redirectIngressUri, onSuccess }: UseOAuthProps) => { +const useOAuth = ({ providerName, vertical, returnUrl, projectId, linkedUserId, additionalParams, redirectIngressUri, onSuccess }: UseOAuthProps) => { const [isReady, setIsReady] = useState(false); const intervalRef = useRef | null>(null); const authWindowRef = useRef(null); @@ -51,7 +49,7 @@ const useOAuth = ({ providerName, vertical, returnUrl, projectId, linkedUserId,a const openModal = async (onWindowClose: () => void) => { const apiUrl = config.API_URL!; const authUrl = await constructAuthUrl({ - projectId, linkedUserId, providerName, returnUrl, apiUrl , vertical,additionalParams, redirectUriIngress: redirectIngressUri + projectId, linkedUserId, providerName, returnUrl, apiUrl , vertical, additionalParams, redirectUriIngress: redirectIngressUri }); if (!authUrl) { diff --git a/apps/magic-link/src/lib/ProviderModal.tsx b/apps/magic-link/src/lib/ProviderModal.tsx index fc97b9a3e..d2406c082 100644 --- a/apps/magic-link/src/lib/ProviderModal.tsx +++ b/apps/magic-link/src/lib/ProviderModal.tsx @@ -22,8 +22,10 @@ interface IBasicAuthFormData { } const domainFormats: { [key: string]: string } = { - microsoftdynamicssales: '{YOUR_DOMAIN}.api.crm3.dynamics.com', - bigcommerce: 'If your api domain is https://api.bigcommerce.com/stores/eubckcvkzg/v3 then store_hash is eubckcvkzg', + salesforce: 'If your Salesforce site URL is https://acme-dev.lightning.force.com, acme-dev is your domain', + sharepoint: 'If the SharePoint site URL is https://joedoe.sharepoint.com/sites/acme-dev, joedoe is the tenant and acme-dev is the site name.', + microsoftdynamicssales: 'If your Microsoft Dynamics URL is acme-dev.api.crm3.dynamics.com then acme-dev is the organization name.', + bigcommerce: 'If your api domain is https://api.bigcommerce.com/stores/joehash123/v3 then store_hash is joehash123.', }; const ProviderModal = () => { @@ -35,18 +37,16 @@ const ProviderModal = () => { const [startFlow, setStartFlow] = useState(false); const [preStartFlow, setPreStartFlow] = useState(false); const [openBasicAuthDialog,setOpenBasicAuthDialog] = useState(false); - const [openDomainDialog, setOpenDomainDialog] = useState(false); const [projectId, setProjectId] = useState(""); const [data, setData] = useState([]); const [isProjectIdReady, setIsProjectIdReady] = useState(false); const [errorResponse,setErrorResponse] = useState<{ errorPresent: boolean; errorMessage : string }>({errorPresent:false,errorMessage:''}) - const [endUserDomain, setEndUserDomain] = useState(''); const [loading, setLoading] = useState<{ status: boolean; provider: string }>({status: false, provider: ''}); - + const [additionalParams, setAdditionalParams] = useState<{[key: string]: string}>({}); const [uniqueMagicLinkId, setUniqueMagicLinkId] = useState(null); const [openSuccessDialog,setOpenSuccessDialog] = useState(false); const [currentProviderLogoURL,setCurrentProviderLogoURL] = useState('') @@ -121,13 +121,10 @@ const ProviderModal = () => { console.log('OAuth successful'); setOpenSuccessDialog(true); }, - additionalParams: { - end_user_domain: endUserDomain - } + additionalParams }); const onWindowClose = () => { - setSelectedProvider({ provider: '', category: '' @@ -163,38 +160,6 @@ const ProviderModal = () => { } }, [startFlow, isReady]); - - - const handleWalletClick = (walletName: string, category: string) => { - setSelectedProvider({provider: walletName.toLowerCase(), category: category.toLowerCase()}); - const logoPath = CONNECTORS_METADATA[category.toLowerCase()][walletName.toLowerCase()].logoPath; - setCurrentProviderLogoURL(logoPath); - setCurrentProvider(walletName.toLowerCase()) - setPreStartFlow(true); - }; - - const handleStartFlow = () => { - const providerMetadata = CONNECTORS_METADATA[selectedProvider.category][selectedProvider.provider]; - if (providerMetadata.authStrategy.strategy === AuthStrategy.api_key || providerMetadata.authStrategy.strategy === AuthStrategy.basic) { - setOpenBasicAuthDialog(true); - } else if (providerMetadata?.options?.end_user_domain) { - setOpenDomainDialog(true); - } else { - // OAUTH2 WITHOUT EXTRA PARAMS - setLoading({ status: true, provider: selectedProvider?.provider! }); - setStartFlow(true); - } - } - - const handleCategoryClick = (category: string) => { - setPreStartFlow(false); - setSelectedProvider({ - provider: '', - category: '' - }); - setSelectedCategory(category); - }; - const CloseSuccessDialog = (close : boolean) => { if(!close) { @@ -228,52 +193,50 @@ const ProviderModal = () => { onCloseBasicAuthDialog(false); setLoading({status: true, provider: selectedProvider?.provider!}); setPreStartFlow(false); - // Creating Basic Auth Connection - createApiKeyConnection({ - query : { - linkedUserId: magicLink?.id_linked_user as string, - projectId: projectId, - providerName: selectedProvider?.provider!, - vertical: selectedProvider?.category! + const providerMetadata = CONNECTORS_METADATA[selectedProvider.category][selectedProvider.provider]; + + if (providerMetadata.authStrategy.strategy === AuthStrategy.oauth2) { + console.log("values are "+ JSON.stringify(values)) + setAdditionalParams(values); + setStartFlow(true); + }else{ + createApiKeyConnection({ + query : { + linkedUserId: magicLink?.id_linked_user as string, + projectId: projectId, + providerName: selectedProvider?.provider!, + vertical: selectedProvider?.category! + }, + data: values }, - data: values - }, - { - onSuccess: () => { - setSelectedProvider({ - provider: '', - category: '' - }); - - setLoading({ + { + onSuccess: () => { + setSelectedProvider({ + provider: '', + category: '' + }); + + setLoading({ + status: false, + provider: '' + }); + setOpenSuccessDialog(true); + }, + onError: (error) => { + setErrorResponse({errorPresent:true,errorMessage: error.message}); + setLoading({ status: false, provider: '' - }); - setOpenSuccessDialog(true); - }, - onError: (error) => { - setErrorResponse({errorPresent:true,errorMessage: error.message}); - setLoading({ - status: false, - provider: '' - }); - setSelectedProvider({ - provider: '', - category: '' - }); - } - }); - } - - const onCloseDomainDialog = (dialogState: boolean) => { - setOpenDomainDialog(dialogState); - } - - const onDomainSubmit = () => { - setOpenDomainDialog(false); - setLoading({ status: true, provider: selectedProvider?.provider! }); - setStartFlow(true); + }); + setSelectedProvider({ + provider: '', + category: '' + }); + } + }); + } + } const filteredProviders = data.filter(provider => @@ -288,10 +251,8 @@ const ProviderModal = () => { setCurrentProvider(provider.name.toLowerCase()) const providerMetadata = CONNECTORS_METADATA[provider.vertical!.toLowerCase()][provider.name.toLowerCase()]; - if (providerMetadata.authStrategy.strategy === AuthStrategy.api_key || providerMetadata.authStrategy.strategy === AuthStrategy.basic) { + if (providerMetadata.authStrategy.strategy === AuthStrategy.api_key || providerMetadata.authStrategy.strategy === AuthStrategy.basic || (providerMetadata.authStrategy.strategy === AuthStrategy.oauth2 && providerMetadata.authStrategy.properties)) { setOpenBasicAuthDialog(true); - } else if (providerMetadata?.options?.end_user_domain) { - setOpenDomainDialog(true); } else { setLoading({ status: true, provider: provider.name.toLowerCase() }); setStartFlow(true); @@ -416,68 +377,12 @@ const ProviderModal = () => { {errors2[fieldName] &&

{errors2[fieldName]?.message}

} ))} - -

- A third-party accountant will be added. -

- - - - - - - - {/* Domain Dialog */} - - -
- -
- -
- {selectedProvider?.category && selectedProvider?.provider && CONNECTORS_METADATA[selectedProvider.category]?.[selectedProvider.provider] && ( - <> -
- {selectedProvider.provider} -
-

- Connect your {selectedProvider.provider.charAt(0).toUpperCase() + selectedProvider.provider.slice(1)} Account -

- - )} - -
{ e.preventDefault(); onDomainSubmit(); }} className="w-full space-y-4"> -
- setEndUserDomain(e.target.value)} - /> - {errors2.end_user_domain &&

{errors2.end_user_domain.message}

} -
- - {domainFormats[selectedProvider?.provider?.toLowerCase()] && ( -

- e.g., {domainFormats[selectedProvider.provider.toLowerCase()]} + {domainFormats[selectedProvider.provider] && ( +

+ {domainFormats[selectedProvider.provider]}

)} -

- A third-party accountant will be added. -

-
+ +
+ + + Pull Frequency Settings + Set the sync frequency for each vertical + + + {VERTICALS.map(vertical => ( +
+ +
+ + +
+
+ ))} + + +
+ +
@@ -226,6 +353,10 @@ export default function Page() {
+ + + + diff --git a/apps/webapp/src/components/Configuration/LinkedUsers/AddLinkedAccount.tsx b/apps/webapp/src/components/Configuration/LinkedUsers/AddLinkedAccount.tsx index e03c12e71..3fc53e9e3 100644 --- a/apps/webapp/src/components/Configuration/LinkedUsers/AddLinkedAccount.tsx +++ b/apps/webapp/src/components/Configuration/LinkedUsers/AddLinkedAccount.tsx @@ -65,7 +65,7 @@ const AddLinkedAccount = () => { import: false }) const [files, setFiles] = useState([]) -const [successImporting, setSuccessImporting]=useState(false) + const [successImporting, setSuccessImporting]=useState(false) const { createLinkedUserPromise } = useCreateLinkedUser(); const { createBatchLinkedUserPromise } = useCreateBatchLinkedUser(); diff --git a/apps/webapp/src/components/Configuration/RAGSettings/RAGSettingsPage.tsx b/apps/webapp/src/components/Configuration/RAGSettings/RAGSettingsPage.tsx new file mode 100644 index 000000000..30b9ed1f0 --- /dev/null +++ b/apps/webapp/src/components/Configuration/RAGSettings/RAGSettingsPage.tsx @@ -0,0 +1,555 @@ +import React, { useEffect, useState } from 'react'; +import { useForm } from "react-hook-form"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { useQueryClient } from "@tanstack/react-query"; +import { usePostHog } from 'posthog-js/react'; +import { toast } from "sonner"; +import * as z from "zod"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Switch } from "@/components/ui/switch"; +import { Label } from "@/components/ui/label"; +import { Button } from "@/components/ui/button"; +import { Form, FormControl, FormField, FormItem, FormLabel, FormMessage } from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import config from "@/lib/config"; +import useProjectStore from "@/state/projectStore"; +import useConnectionStrategies from "@/hooks/get/useConnectionStrategies"; +import useCreateConnectionStrategy from "@/hooks/create/useCreateConnectionStrategy"; +import useUpdateConnectionStrategy from "@/hooks/update/useUpdateConnectionStrategy"; +import useConnectionStrategyAuthCredentials from "@/hooks/get/useConnectionStrategyAuthCredentials"; +import Image from 'next/image'; + +const formSchema = z.object({ + vectorDatabase: z.string(), + embeddingModel: z.string(), + apiKey: z.string().optional(), + baseUrl: z.string().optional(), + url: z.string().optional(), + indexName: z.string().optional(), + embeddingApiKey: z.string().optional(), +}); + +export function RAGSettingsPage() { + const { idProject } = useProjectStore(); + const queryClient = useQueryClient(); + const posthog = usePostHog(); + + const { data: connectionStrategies } = useConnectionStrategies(); + const { createCsPromise } = useCreateConnectionStrategy(); + const { updateCsPromise } = useUpdateConnectionStrategy(); + const { mutateAsync: fetchCredentials } = useConnectionStrategyAuthCredentials(); + + const [ragModeActive, setRagModeActive] = useState(false); + + const form = useForm>({ + resolver: zodResolver(formSchema), + defaultValues: { + vectorDatabase: "", + embeddingModel: "", + apiKey: "", + baseUrl: "", + indexName: "", + url: "", + embeddingApiKey: "", + }, + }); + + const vectorDbConnectionStrategies = connectionStrategies?.filter( + (cs) => cs.type.startsWith('vector_db.') + ) || []; + + const embeddingModelConnectionStrategies = connectionStrategies?.filter( + (cs) => cs.type.startsWith('embedding_model.') + ) || []; + + useEffect(() => { + const currentVectorDb = form.getValues().vectorDatabase; + const currentEmbeddingModel = form.getValues().embeddingModel; + + const currentStrategy = vectorDbConnectionStrategies.find( + cs => cs.type === `vector_db.${currentVectorDb}` + ); + + const embeddingModelStrategy = embeddingModelConnectionStrategies.find( + cs => cs.type === `embedding_model.${currentEmbeddingModel}` + ); + + if (currentStrategy) { + fetchCredentials({ type: currentStrategy.type, attributes: getAttributesForVectorDb(currentVectorDb) }, { + onSuccess(data) { + setFormValuesFromCredentials(currentVectorDb, data); + setRagModeActive(currentStrategy.status === true); + } + }); + } else { + // Reset form values for the fields specific to vector databases + form.setValue("apiKey", ""); + form.setValue("baseUrl", ""); + form.setValue("url", ""); + form.setValue("indexName", ""); + setRagModeActive(false); + } + + if (embeddingModelStrategy) { + fetchCredentials({ type: embeddingModelStrategy.type, attributes: ['api_key'] }, { + onSuccess(data) { + form.setValue("embeddingApiKey", data[0]); + } + }); + } else { + form.setValue("embeddingApiKey", ""); + } + }, [form.watch("vectorDatabase"), form.watch("embeddingModel"), connectionStrategies]); + + const getAttributesForVectorDb = (vectorDb: string): string[] => { + switch (vectorDb) { + case 'turbopuffer': + return ['api_key']; + case 'pinecone': + return ['api_key', 'index_name']; + case 'qdrant': + return ['api_key', 'base_url']; + case 'chromadb': + return ['url']; + case 'weaviate': + return ['api_key', 'url']; + default: + return []; + } + }; + + const setFormValuesFromCredentials = (vectorDb: string, data: string[]) => { + switch (vectorDb) { + case 'turbopuffer': + form.setValue("apiKey", data[0]); + break; + case 'pinecone': + form.setValue("apiKey", data[0]); + form.setValue("indexName", data[1]); + break; + case 'qdrant': + form.setValue("apiKey", data[0]); + form.setValue("baseUrl", data[1]); + break; + case 'chromadb': + form.setValue("url", data[0]); + break; + case 'weaviate': + form.setValue("apiKey", data[0]); + form.setValue("url", data[1]); + break; + } + }; + + function onSubmit(values: z.infer) { + const { vectorDatabase, apiKey, baseUrl, url, indexName, embeddingModel, embeddingApiKey } = values; + const currentStrategy = vectorDbConnectionStrategies.find( + cs => cs.type === `vector_db.${vectorDatabase}` + ); + const currentEmbeddingModelStrategy = embeddingModelConnectionStrategies.find( + cs => cs.type === `embedding_model.${embeddingModel}` + ); + + const performUpdate = !!currentStrategy; + const performEmbeddingModelUpdate = !!currentEmbeddingModelStrategy; + + let attributes: string[] = []; + let attributeValues: string[] = []; + let embeddingModelAttributes: string[] = ['api_key']; + let embeddingModelAttributeValues: string[] = [embeddingApiKey!]; + + switch (vectorDatabase) { + case 'turbopuffer': + attributes = ['api_key']; + attributeValues = [apiKey!]; + break; + case 'pinecone': + attributes = ['api_key', 'index_name']; + attributeValues = [apiKey!, indexName!]; + break; + case 'qdrant': + attributes = ['api_key', 'base_url']; + attributeValues = [apiKey!, baseUrl!]; + break; + case 'chromadb': + attributes = ['url']; + attributeValues = [url!]; + break; + case 'weaviate': + attributes = ['api_key', 'url']; + attributeValues = [apiKey!, url!]; + break; + } + + const promise = performUpdate + ? updateCsPromise({ + id_cs: currentStrategy!.id_connection_strategy, + updateToggle: false, + status: ragModeActive, + attributes, + values: attributeValues, + }) + : createCsPromise({ + type: `vector_db.${vectorDatabase}`, + attributes, + values: attributeValues, + }); + + const embeddingModelPromise = performEmbeddingModelUpdate + ? updateCsPromise({ + id_cs: currentEmbeddingModelStrategy!.id_connection_strategy, + updateToggle: false, + status: true, + attributes: embeddingModelAttributes, + values: embeddingModelAttributeValues, + }) + : createCsPromise({ + type: `embedding_model.${embeddingModel}`, + attributes: embeddingModelAttributes, + values: embeddingModelAttributeValues, + }); + + toast.promise(Promise.all([promise, embeddingModelPromise]), { + loading: 'Saving RAG settings...', + success: () => { + queryClient.invalidateQueries({ queryKey: ['connection-strategies'] }); + return "RAG settings saved successfully"; + }, + error: (err: any) => err.message || 'An error occurred', + }); + + posthog?.capture(`RAG_settings_${performUpdate ? 'updated' : 'created'}`, { + id_project: idProject, + vector_database: vectorDatabase, + embedding_model: values.embeddingModel, + mode: config.DISTRIBUTION + }); + } + + const handleRagModeChange = (checked: boolean) => { + setRagModeActive(checked); + }; + + const renderVectorDbInputs = () => { + const vectorDatabase = form.watch("vectorDatabase"); + switch (vectorDatabase) { + case 'turbopuffer': + return ( + ( + + API Key + + + + + + )} + /> + ); + case 'pinecone': + return ( + <> + ( + + API Key + + + + + + )} + /> + ( + + Index Name + + + + + + )} + /> + + ); + case 'qdrant': + return ( + <> + ( + + API Key + + + + + + )} + /> + ( + + Base URL + + + + + + )} + /> + + ); + case 'chromadb': + return ( + ( + + URL + + + + + + )} + /> + ); + case 'weaviate': + return ( + <> + ( + + API Key + + + + + + )} + /> + ( + + URL + + + + + + )} + /> + + ); + default: + return null; + } + }; + + const renderEmbeddingModelInputs = () => { + const embeddingModel = form.watch("embeddingModel"); + if (embeddingModel === "OPENAI_ADA_SMALL_1536" || embeddingModel === "OPENAI_ADA_LARGE_3072" || embeddingModel === "OPENAI_ADA_002" || embeddingModel === "COHERE_MULTILINGUAL_V3") { + return ( + ( + + API Key + + + + + + )} + /> + ); + } + return null; + }; + + return ( + + + RAG Settings + Configure your Retrieval-Augmented Generation settings + + + + + ( + + Vector Database + + + + )} + /> + {renderVectorDbInputs()} + + + ( + + Embedding Model + + + + )} + /> + + {renderEmbeddingModelInputs()} + +
+ + +
+ + + + + + + ); +} + diff --git a/apps/webapp/src/components/Events/EventsTable.tsx b/apps/webapp/src/components/Events/EventsTable.tsx index 680d91197..288e00d23 100644 --- a/apps/webapp/src/components/Events/EventsTable.tsx +++ b/apps/webapp/src/components/Events/EventsTable.tsx @@ -44,7 +44,7 @@ export default function EventsTable() { return ( <> {transformedEvents && ( - + )} ); diff --git a/apps/webapp/src/hooks/create/useCreateLogin.tsx b/apps/webapp/src/hooks/create/useCreateLogin.tsx index c3d37a488..79731a949 100644 --- a/apps/webapp/src/hooks/create/useCreateLogin.tsx +++ b/apps/webapp/src/hooks/create/useCreateLogin.tsx @@ -14,15 +14,12 @@ interface ILoginInputDto { password_hash:string } -interface ILoginOutputDto { - user: IUserDto, - access_token: string -} - const useCreateLogin = () => { const add = async (userData: ILoginInputDto) => { // Fetch the token - const response = await fetch(`${config.API_URL}/auth/login`, { + console.log("API_URL: ", config.API_URL); // Add this line + const apiUrl = new URL('/auth/login', config.API_URL).toString(); + const response = await fetch(apiUrl, { method: 'POST', body: JSON.stringify(userData), headers: { @@ -31,7 +28,7 @@ const useCreateLogin = () => { }); if (!response.ok) { - throw new Error("Login Failed!!") + throw new Error("Login Failed!! ") } return response.json(); diff --git a/apps/webapp/src/hooks/create/useCreatePullFrequency.tsx b/apps/webapp/src/hooks/create/useCreatePullFrequency.tsx new file mode 100644 index 000000000..ccd6fd968 --- /dev/null +++ b/apps/webapp/src/hooks/create/useCreatePullFrequency.tsx @@ -0,0 +1,45 @@ +import { useMutation, useQueryClient } from '@tanstack/react-query'; +import config from '@/lib/config'; +import Cookies from 'js-cookie'; + +export type UpdatePullFrequencyData = Record; + +export const useUpdatePullFrequency = () => { + const add = async (data: UpdatePullFrequencyData) => { + const response = await fetch(`${config.API_URL}/sync/internal/pull-frequencies`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${Cookies.get('access_token')}`, + }, + body: JSON.stringify(data), + }); + + if (!response.ok) { + const errorData = await response.json(); + throw new Error(errorData.message || "Failed to update pull frequencies"); + } + + return response.json(); + } + + const createPullFrequencyPromise = (data: UpdatePullFrequencyData) => { + return new Promise(async (resolve, reject) => { + try { + const result = await add(data); + resolve(result); + + } catch (error) { + reject(error); + } + }); + }; + return { + mutationFn: useMutation({ + mutationFn: add, + }), + createPullFrequencyPromise + } +}; + +export default useUpdatePullFrequency; diff --git a/apps/webapp/src/hooks/get/useGetPullFrequencies.tsx b/apps/webapp/src/hooks/get/useGetPullFrequencies.tsx new file mode 100644 index 000000000..e7b8d4d45 --- /dev/null +++ b/apps/webapp/src/hooks/get/useGetPullFrequencies.tsx @@ -0,0 +1,27 @@ +import config from '@/lib/config'; +import { useQuery } from '@tanstack/react-query'; +import { projects_pull_frequency as PullFrequency } from 'api'; +import Cookies from 'js-cookie'; + + +const usePullFrequencies = () => { + return useQuery({ + queryKey: ['pull-frequencies'], + queryFn: async (): Promise => { + const response = await fetch(`${config.API_URL}/sync/internal/pull-frequencies`,{ + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${Cookies.get('access_token')}`, + }, + }); + if (!response.ok) { + const errorData = await response.json(); + throw new Error(errorData.message || "Unknown error occurred"); + } + return response.json(); + } + }); +}; + +export default usePullFrequencies; diff --git a/apps/webapp/src/hooks/get/useRagSettings.tsx b/apps/webapp/src/hooks/get/useRagSettings.tsx new file mode 100644 index 000000000..e4aa027cf --- /dev/null +++ b/apps/webapp/src/hooks/get/useRagSettings.tsx @@ -0,0 +1,15 @@ +import config from '@/lib/config'; +import { useQuery } from '@tanstack/react-query'; +import { api_keys as ApiKey } from 'api'; +import Cookies from 'js-cookie'; + +const useRagSettings = () => { + return useQuery({ + queryKey: ['rag-settings'], + queryFn: async (): Promise => { + return [] + } + }); +}; + +export default useRagSettings; diff --git a/apps/webapp/src/hooks/update/useUpdateRagSettings.tsx b/apps/webapp/src/hooks/update/useUpdateRagSettings.tsx new file mode 100644 index 000000000..b8529d8f7 --- /dev/null +++ b/apps/webapp/src/hooks/update/useUpdateRagSettings.tsx @@ -0,0 +1,35 @@ +import config from '@/lib/config'; +import { useMutation } from '@tanstack/react-query'; +import Cookies from 'js-cookie'; + +interface IUpdateProjectConnectorsDto { + column: string; + status: boolean; +} + +const useUpdateRagSettings = () => { + const update = async (data: IUpdateProjectConnectorsDto) => { + return [] + }; + + const updateRagSettingsPromise = (data: IUpdateProjectConnectorsDto) => { + return new Promise(async (resolve, reject) => { + try { + const result = await update(data); + resolve(result); + + } catch (error) { + reject(error); + } + }); + }; + + return { + mutate: useMutation({ + mutationFn: update, + }), + updateRagSettingsPromise, + }; +}; + +export default useUpdateRagSettings; diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 419021b49..70a9045e7 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -191,6 +191,25 @@ services: POSTHOG_HOST: ${POSTHOG_HOST} POSTHOG_KEY: ${POSTHOG_KEY} PH_TELEMETRY: ${PH_TELEMETRY} + SALESFORCE_CRM_CLOUD_CLIENT_ID: ${SALESFORCE_CRM_CLOUD_CLIENT_ID} + SALESFORCE_CRM_CLOUD_CLIENT_SECRET: ${SALESFORCE_CRM_CLOUD_CLIENT_SECRET} + OPENAI_API_KEY: ${OPENAI_API_KEY} + JINA_API_KEY: ${JINA_API_KEY} + COHERE_API_KEY: ${COHERE_API_KEY} + AWS_S3_REGION: ${AWS_S3_REGION} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + UNSTRUCTURED_API_KEY: ${UNSTRUCTURED_API_KEY} + UNSTRUCTURED_API_URL: ${UNSTRUCTURED_API_URL} + PINECONE_API_KEY: ${PINECONE_API_KEY} + PINECONE_INDEX_NAME: ${PINECONE_INDEX_NAME} + QDRANT_BASE_URL: ${QDRANT_BASE_URL} + QDRANT_API_KEY: ${QDRANT_API_KEY} + CHROMADB_URL: ${CHROMADB_URL} + WEAVIATE_URL: ${WEAVIATE_URL} + WEAVIATE_API_KEY: ${WEAVIATE_API_KEY} + TURBOPUFFER_API_KEY: ${TURBOPUFFER_API_KEY} + MILVUS_ADDRESS: ${MILVUS_ADDRESS} restart: unless-stopped ports: @@ -230,12 +249,12 @@ services: dockerfile: ./apps/webapp/Dockerfile.dev context: ./ args: - VITE_BACKEND_DOMAIN: ${NEXT_PUBLIC_BACKEND_DOMAIN} + VITE_BACKEND_DOMAIN: http://localhost:3000 environment: NEXT_PUBLIC_POSTHOG_KEY: ${POSTHOG_KEY} NEXT_PUBLIC_POSTHOG_HOST: ${POSTHOG_HOST} NEXT_PUBLIC_DISTRIBUTION: ${DISTRIBUTION} - NEXT_PUBLIC_BACKEND_DOMAIN: ${NEXT_PUBLIC_BACKEND_DOMAIN} + NEXT_PUBLIC_BACKEND_DOMAIN: http://localhost:3000 NEXT_PUBLIC_MAGIC_LINK_DOMAIN: ${NEXT_PUBLIC_MAGIC_LINK_DOMAIN} NEXT_PUBLIC_WEBAPP_DOMAIN: ${NEXT_PUBLIC_WEBAPP_DOMAIN} NEXT_PUBLIC_REDIRECT_WEBHOOK_INGRESS: ${REDIRECT_TUNNEL_INGRESS} @@ -256,7 +275,7 @@ services: dockerfile: ./apps/magic-link/Dockerfile.dev context: ./ args: - VITE_BACKEND_DOMAIN: ${NEXT_PUBLIC_BACKEND_DOMAIN} + VITE_BACKEND_DOMAIN: http://localhost:3000 restart: always ports: - 81:5173 @@ -312,9 +331,29 @@ services: volumes: - ./docs/:/app + minio: + image: minio/minio + ports: + - "9000:9000" + - "9001:9001" + volumes: + - minio_storage:/data + environment: + MINIO_ROOT_USER: ${MINIO_ROOT_USER} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD} + command: server --console-address ":9001" /data + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + networks: + - backend + volumes: local_pgdata: pgadmin-data: + minio_storage: networks: frontend: diff --git a/docker-compose.source.yml b/docker-compose.source.yml index 98fcd922d..6ebe20104 100644 --- a/docker-compose.source.yml +++ b/docker-compose.source.yml @@ -191,7 +191,26 @@ services: POSTHOG_HOST: "https://us.i.posthog.com" POSTHOG_KEY: "phc_WhWJfNPOHAuWVdyTacGxrPa9JW54scnofA9KVEjFcFw" PH_TELEMETRY: "TRUE" - + SALESFORCE_CRM_CLOUD_CLIENT_ID: ${SALESFORCE_CRM_CLOUD_CLIENT_ID} + SALESFORCE_CRM_CLOUD_CLIENT_SECRET: ${SALESFORCE_CRM_CLOUD_CLIENT_SECRET} + OPENAI_API_KEY: ${OPENAI_API_KEY} + JINA_API_KEY: ${JINA_API_KEY} + COHERE_API_KEY: ${COHERE_API_KEY} + AWS_S3_REGION: ${AWS_S3_REGION} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + UNSTRUCTURED_API_KEY: ${UNSTRUCTURED_API_KEY} + UNSTRUCTURED_API_URL: ${UNSTRUCTURED_API_URL} + PINECONE_API_KEY: ${PINECONE_API_KEY} + PINECONE_INDEX_NAME: ${PINECONE_INDEX_NAME} + QDRANT_BASE_URL: ${QDRANT_BASE_URL} + QDRANT_API_KEY: ${QDRANT_API_KEY} + CHROMADB_URL: ${CHROMADB_URL} + WEAVIATE_URL: ${WEAVIATE_URL} + WEAVIATE_API_KEY: ${WEAVIATE_API_KEY} + TURBOPUFFER_API_KEY: ${TURBOPUFFER_API_KEY} + MILVUS_ADDRESS: ${MILVUS_ADDRESS} + restart: unless-stopped ports: - 3000:3000 diff --git a/docker-compose.yml b/docker-compose.yml index 7c5b299bc..65c57ff5f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -185,6 +185,26 @@ services: POSTHOG_HOST: "https://us.i.posthog.com" POSTHOG_KEY: "phc_WhWJfNPOHAuWVdyTacGxrPa9JW54scnofA9KVEjFcFw" PH_TELEMETRY: "TRUE" + SALESFORCE_CRM_CLOUD_CLIENT_ID: ${SALESFORCE_CRM_CLOUD_CLIENT_ID} + SALESFORCE_CRM_CLOUD_CLIENT_SECRET: ${SALESFORCE_CRM_CLOUD_CLIENT_SECRET} + OPENAI_API_KEY: ${OPENAI_API_KEY} + JINA_API_KEY: ${JINA_API_KEY} + COHERE_API_KEY: ${COHERE_API_KEY} + AWS_S3_REGION: ${AWS_S3_REGION} + AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID} + AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY} + UNSTRUCTURED_API_KEY: ${UNSTRUCTURED_API_KEY} + UNSTRUCTURED_API_URL: ${UNSTRUCTURED_API_URL} + PINECONE_API_KEY: ${PINECONE_API_KEY} + PINECONE_INDEX_NAME: ${PINECONE_INDEX_NAME} + QDRANT_BASE_URL: ${QDRANT_BASE_URL} + QDRANT_API_KEY: ${QDRANT_API_KEY} + CHROMADB_URL: ${CHROMADB_URL} + WEAVIATE_URL: ${WEAVIATE_URL} + WEAVIATE_API_KEY: ${WEAVIATE_API_KEY} + TURBOPUFFER_API_KEY: ${TURBOPUFFER_API_KEY} + MILVUS_ADDRESS: ${MILVUS_ADDRESS} + restart: unless-stopped ports: - 3000:3000 diff --git a/packages/api/package.json b/packages/api/package.json index 76b963879..21b09ada9 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -26,7 +26,16 @@ "prebuild-oauth-connector": "node --experimental-detect-module ./scripts/oauthConnector.js" }, "dependencies": { + "@aws-sdk/client-s3": "^3.649.0", "@axiomhq/pino": "^1.0.0", + "@langchain/cohere": "^0.2.2", + "@langchain/community": "^0.2.32", + "@langchain/core": "^0.2.31", + "@langchain/openai": "^0.2.10", + "@langchain/pinecone": "^0.0.9", + "@langchain/qdrant": "^0.0.5", + "@langchain/textsplitters": "^0.0.3", + "@langchain/weaviate": "^0.0.5", "@nestjs/bull": "^10.0.1", "@nestjs/common": "^10.0.0", "@nestjs/config": "^3.1.1", @@ -40,14 +49,20 @@ "@nestjs/throttler": "^5.1.1", "@ntegral/nestjs-sentry": "^4.0.0", "@panora/shared": "workspace:*", + "@pinecone-database/pinecone": "^3.0.2", "@prisma/client": "^5.4.2", + "@qdrant/js-client-rest": "^1.11.0", "@sentry/node": "8.9.2", "@sentry/profiling-node": "^8.9.2", "@sentry/tracing": "^7.80.0", "@shopify/shopify-api": "^11.1.0", + "@turbopuffer/turbopuffer": "^0.5.10", + "@zilliz/milvus2-sdk-node": "^2.4.8", "axios": "^1.5.1", "bcrypt": "^5.1.1", "bull": "^4.11.5", + "chromadb": "^1.8.1", + "chromadb-default-embed": "^2.13.2", "class-transformer": "^0.5.1", "class-validator": "^0.14.0", "cookie-parser": "^1.4.6", @@ -58,6 +73,8 @@ "install": "^0.13.0", "js-yaml": "^4.1.0", "jwt-decode": "^4.0.0", + "langchain": "^0.2.18", + "mime-types": "^2.1.35", "nestjs-pino": "^3.5.0", "nodemailer": "^6.9.14", "openai": "^4.38.5", @@ -65,6 +82,7 @@ "passport-headerapikey": "^1.2.2", "passport-jwt": "^4.0.1", "passport-local": "^1.0.0", + "pdf-parse": "^1.1.1", "pino-pretty": "^10.2.3", "posthog-node": "^4.2.0", "qs": "^6.12.3", @@ -72,7 +90,18 @@ "rxjs": "^7.8.1", "stytch": "^10.5.0", "uuid": "^9.0.1", - "yargs": "^17.7.2" + "weaviate-client": "^3.1.4", + "yargs": "^17.7.2", + "googleapis": "^144.0.0", + "@aws-sdk/lib-storage": "^3.649.0", + "google-auth-library": "^9.14.1", + "xlsx": "^0.18.5", + "csv": "^6.3.10", + "mammoth": "^1.8.0", + "marked": "^14.1.2", + "csv-parse": "^5.5.6", + "d3-dsv": "^3.0.1", + "csvtojson": "^2.0.10" }, "devDependencies": { "@nestjs-modules/mailer": "^2.0.2", @@ -82,6 +111,7 @@ "@types/cookie-parser": "^1.4.6", "@types/express": "^4.17.17", "@types/jest": "^29.5.2", + "@types/mime-types": "^2.1.4", "@types/node": "^20.3.1", "@types/passport-jwt": "^3.0.12", "@types/passport-local": "^1.0.37", diff --git a/packages/api/prisma/schema.prisma b/packages/api/prisma/schema.prisma index a44828b12..45ad8e9b0 100644 --- a/packages/api/prisma/schema.prisma +++ b/packages/api/prisma/schema.prisma @@ -1639,7 +1639,7 @@ model ecom_product_variants { /// This model or at least one of its fields has comments in the database, and requires an additional setup for migrations: Read more: https://pris.ly/d/database-comments model events { id_event String @id(map: "pk_jobs") @db.Uuid - id_connection String @db.Uuid + id_connection String? @db.Uuid id_project String @db.Uuid type String status String @@ -1648,8 +1648,8 @@ model events { url String provider String timestamp DateTime @default(now()) @db.Timestamptz(6) - id_linked_user String @db.Uuid - linked_users linked_users @relation(fields: [id_linked_user], references: [id_linked_user], onDelete: NoAction, onUpdate: NoAction, map: "fk_12") + id_linked_user String? @db.Uuid + linked_users linked_users? @relation(fields: [id_linked_user], references: [id_linked_user], onDelete: NoAction, onUpdate: NoAction, map: "fk_12") jobs_status_history jobs_status_history[] webhook_delivery_attempts webhook_delivery_attempts[] @@ -1965,18 +1965,19 @@ model linked_users { /// This model or at least one of its fields has comments in the database, and requires an additional setup for migrations: Read more: https://pris.ly/d/database-comments model projects { - id_project String @id(map: "pk_projects") @db.Uuid - name String - sync_mode String - pull_frequency BigInt? - redirect_url String? - id_user String @db.Uuid - id_connector_set String @db.Uuid - api_keys api_keys[] - connections connections[] - linked_users linked_users[] - users users @relation(fields: [id_user], references: [id_user], onDelete: NoAction, onUpdate: NoAction, map: "fk_46_1") - connector_sets connector_sets @relation(fields: [id_connector_set], references: [id_connector_set], onDelete: NoAction, onUpdate: NoAction, map: "fk_project_connectorsetid") + id_project String @id(map: "pk_projects") @db.Uuid + name String + sync_mode String + pull_frequency BigInt? + redirect_url String? + id_user String @db.Uuid + id_connector_set String @db.Uuid + api_keys api_keys[] + connections connections[] + linked_users linked_users[] + users users @relation(fields: [id_user], references: [id_user], onDelete: NoAction, onUpdate: NoAction, map: "fk_46_1") + connector_sets connector_sets @relation(fields: [id_connector_set], references: [id_connector_set], onDelete: NoAction, onUpdate: NoAction, map: "fk_project_connectorsetid") + projects_pull_frequency projects_pull_frequency? @@index([id_connector_set], map: "fk_connectors_sets") } @@ -2128,3 +2129,18 @@ model webhook_delivery_attempts { @@index([id_event], map: "fk_webhook_delivery_attempt_eventid") @@index([id_webhooks_reponse], map: "fk_webhook_delivery_attempt_webhook_responseid") } + +model projects_pull_frequency { + id_projects_pull_frequency String @id(map: "pk_projects_pull_frequency") @db.Uuid + crm BigInt? + ats BigInt? + hris BigInt? + accounting BigInt? + filestorage BigInt? + ecommerce BigInt? + ticketing BigInt? + created_at DateTime @default(now()) @db.Timestamptz(6) + modified_at DateTime @default(now()) @db.Timestamptz(6) + id_project String @unique(map: "uq_projects_pull_frequency_project") @db.Uuid + projects projects @relation(fields: [id_project], references: [id_project], onDelete: NoAction, onUpdate: NoAction, map: "fk_projects_pull_frequency_project") +} diff --git a/packages/api/scripts/connectorUpdate.js b/packages/api/scripts/connectorUpdate.js index b8eb6f582..bb52371d9 100755 --- a/packages/api/scripts/connectorUpdate.js +++ b/packages/api/scripts/connectorUpdate.js @@ -265,7 +265,10 @@ function updateModuleFileForMapper(moduleFile, newServiceDirs, objectType) { // Generate and insert new service imports newServiceDirs.forEach((serviceName) => { const mapperClass = - serviceName.charAt(0).toUpperCase() + serviceName.slice(1) + objectType + 'Mapper'; + serviceName.charAt(0).toUpperCase() + + serviceName.slice(1) + + objectType + + 'Mapper'; const importStatement = `import { ${mapperClass} } from './services/${serviceName}/mappers';\n`; if (!moduleFileContent.includes(importStatement)) { moduleFileContent = importStatement + moduleFileContent; @@ -404,17 +407,20 @@ function updateSeedSQLFile(seedSQLFile, newServiceDirs, vertical) { fileContent = fileContent.replace(lastMatch[1], newColumnsSection); // Update each VALUES section - fileContent = fileContent.replace(/INSERT INTO connector_sets \(([^)]+)\) VALUES(.*?);/gs, (match) => { - return match - .replace(/\),\s*\(/g, '),\n (') // Fix line formatting - .replace(/\([^\)]+\)/g, (values, index) => { - if (values.startsWith('(id_connector_set')) { - return values - } - let newValues = newColumns.map(() => 'TRUE').join(', '); - return values.slice(0, -1) + ', ' + newValues + ')'; - }); - }); + fileContent = fileContent.replace( + /INSERT INTO connector_sets \(([^)]+)\) VALUES(.*?);/gs, + (match) => { + return match + .replace(/\),\s*\(/g, '),\n (') // Fix line formatting + .replace(/\([^\)]+\)/g, (values, index) => { + if (values.startsWith('(id_connector_set')) { + return values; + } + let newValues = newColumns.map(() => 'TRUE').join(', '); + return values.slice(0, -1) + ', ' + newValues + ')'; + }); + }, + ); } // Write the modified content back to the file console.log(fileContent); @@ -427,9 +433,15 @@ function updateSeedSQLFile(seedSQLFile, newServiceDirs, vertical) { function updateObjectTypes(baseDir, objectType, vertical) { const __dirname = path.dirname(fileURLToPath(import.meta.url)); const servicesDir = path.join(__dirname, baseDir); + const targetFileName = + vertical === 'filestorage' + ? 'file-storage' + : vertical === 'marketingautomation' + ? 'marketing-automation' + : vertical; const targetFile = path.join( __dirname, - `../src/@core/utils/types/original/original.${vertical}.ts`, + `../src/@core/utils/types/original/original.${targetFileName}.ts`, ); const newServiceDirs = scanDirectory(servicesDir); @@ -468,7 +480,7 @@ function updateObjectTypes(baseDir, objectType, vertical) { ); updateModuleFileForService(moduleFile, newServiceDirs); - updateModuleFileForMapper(moduleFile, newServiceDirs, objectType) + updateModuleFileForMapper(moduleFile, newServiceDirs, objectType); // Path to the mappings file // const mappingsFile = path.join( @@ -522,4 +534,4 @@ if (import.meta.url === process.argv[1]) { const argv = yargs(hideBin(process.argv)).argv; const baseDir = `../src/${argv.vertical.toLowerCase()}/${argv.objectType.toLowerCase()}/services`; -updateObjectTypes(baseDir, argv.objectType, argv.vertical); \ No newline at end of file +updateObjectTypes(baseDir, argv.objectType, argv.vertical); diff --git a/packages/api/scripts/init.sql b/packages/api/scripts/init.sql index 6a714c7a4..43efca349 100644 --- a/packages/api/scripts/init.sql +++ b/packages/api/scripts/init.sql @@ -552,7 +552,10 @@ CREATE TABLE connector_sets ats_ashby boolean NULL, ecom_webflow boolean NULL, crm_microsoftdynamicssales boolean NULL, - crm_affinity boolean NULL, + fs_dropbox boolean NULL, + fs_googledrive boolean NULL, + fs_sharepoint boolean NULL, + fs_onedrive boolean NULL, CONSTRAINT PK_project_connector PRIMARY KEY ( id_connector_set ) ); @@ -2755,7 +2758,7 @@ CREATE INDEX FKx_hris_employee_payroll_runs_deduction_hris_employee_payroll_Id O CREATE TABLE events ( id_event uuid NOT NULL, - id_connection uuid NOT NULL, + id_connection uuid NULL, id_project uuid NOT NULL, type text NOT NULL, status text NOT NULL, @@ -2764,7 +2767,7 @@ CREATE TABLE events url text NOT NULL, provider text NOT NULL, "timestamp" timestamp with time zone NOT NULL DEFAULT NOW(), - id_linked_user uuid NOT NULL, + id_linked_user uuid NULL, CONSTRAINT PK_jobs PRIMARY KEY ( id_event ), CONSTRAINT FK_12 FOREIGN KEY ( id_linked_user ) REFERENCES linked_users ( id_linked_user ) ); diff --git a/packages/api/scripts/seed.sql b/packages/api/scripts/seed.sql index d727bb268..dc087a70d 100644 --- a/packages/api/scripts/seed.sql +++ b/packages/api/scripts/seed.sql @@ -1,10 +1,11 @@ INSERT INTO users (id_user, identification_strategy, email, password_hash, first_name, last_name) VALUES ('0ce39030-2901-4c56-8db0-5e326182ec6b', 'b2c','local@panora.dev', '$2b$10$Y7Q8TWGyGuc5ecdIASbBsuXMo3q/Rs3/cnY.mLZP4tUgfGUOCUBlG', 'local', 'Panora'); -INSERT INTO connector_sets (id_connector_set, crm_hubspot, crm_zoho, crm_pipedrive, crm_attio, crm_zendesk, crm_close, tcg_zendesk, tcg_gorgias, tcg_front, tcg_jira, tcg_gitlab, fs_box, tcg_github, hris_deel, hris_sage, ats_ashby, crm_microsoftdynamicssales, ecom_webflow, tcg_linear, ecom_shopify, ecom_woocommerce, ecom_amazon, ecom_squarespace, hris_gusto, crm_affinity) VALUES - ('1709da40-17f7-4d3a-93a0-96dc5da6ddd7', TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE), - ('852dfff8-ab63-4530-ae49-e4b2924407f8', TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE), - ('aed0f856-f802-4a79-8640-66d441581a99', TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE); + +INSERT INTO connector_sets (id_connector_set, crm_hubspot, crm_zoho, crm_pipedrive, crm_attio, crm_zendesk, crm_close, tcg_zendesk, tcg_gorgias, tcg_front, tcg_jira, tcg_gitlab, fs_box, tcg_github, hris_deel, hris_sage, ats_ashby, crm_microsoftdynamicssales, ecom_webflow, tcg_linear, ecom_shopify, ecom_woocommerce, ecom_amazon, ecom_squarespace, hris_gusto, fs_googledrive, fs_dropbox, fs_sharepoint, fs_onedrive) VALUES + ('1709da40-17f7-4d3a-93a0-96dc5da6ddd7', TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE), + ('852dfff8-ab63-4530-ae49-e4b2924407f8', TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE), + ('aed0f856-f802-4a79-8640-66d441581a99', TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE); INSERT INTO projects (id_project, name, sync_mode, id_user, id_connector_set) VALUES ('1e468c15-aa57-4448-aa2b-7fed640d1e3d', 'Project 1', 'pull', '0ce39030-2901-4c56-8db0-5e326182ec6b', '1709da40-17f7-4d3a-93a0-96dc5da6ddd7'), diff --git a/packages/api/src/@core/@core-services/environment/environment.service.ts b/packages/api/src/@core/@core-services/environment/environment.service.ts index c8f25f3c0..fb0b19662 100644 --- a/packages/api/src/@core/@core-services/environment/environment.service.ts +++ b/packages/api/src/@core/@core-services/environment/environment.service.ts @@ -11,7 +11,7 @@ export type RateLimit = { limit: string; }; -//for providers secret it is of the form +// for providers secret it is of the form // get{provider_name}{vertical_name}Secret @Injectable() export class EnvironmentService { @@ -64,4 +64,71 @@ export class EnvironmentService { limit: this.configService.get('THROTTLER_LIMIT'), }; } + + getChromaCreds(): string { + return this.configService.get('CHROMADB_URL'); + } + + getMilvusCreds() { + return { + address: this.configService.get('MILVUS_ADDRESS'), + }; + } + getPineconeCreds() { + return { + apiKey: this.configService.get('PINECONE_API_KEY'), + indexName: this.configService.get('PINECONE_INDEX_NAME'), + }; + } + + getWeaviateCreds() { + return { + url: this.configService.get('WEAVIATE_URL'), + apiKey: this.configService.get('WEAVIATE_API_KEY'), + }; + } + + getTurboPufferApiKey(): string { + return this.configService.get('TURBOPUFFER_API_KEY'); + } + + getQdrantCreds() { + return { + baseUrl: this.configService.get('QDRANT_BASE_URL'), + apiKey: this.configService.get('QDRANT_API_KEY'), + }; + } + + getAwsCredentials() { + return { + region: this.configService.get('AWS_REGION'), + accessKeyId: this.configService.get('AWS_ACCESS_KEY_ID'), + secretAccessKey: this.configService.get('AWS_SECRET_ACCESS_KEY'), + }; + } + getMinioCredentials() { + return { + accessKeyId: this.configService.get('MINIO_ROOT_USER'), + secretAccessKey: this.configService.get('MINIO_ROOT_PASSWORD'), + }; + } + + getOpenAIApiKey(): string { + return this.configService.get('OPENAI_API_KEY'); + } + + getCohereApiKey(): string { + return this.configService.get('COHERE_API_KEY'); + } + + getJinaApiKey(): string { + return this.configService.get('JINA_API_KEY'); + } + + getUnstructuredCreds() { + return { + apiKey: this.configService.get('UNSTRUCTURED_API_KEY'), + apiUrl: this.configService.get('UNSTRUCTURED_API_URL'), + }; + } } diff --git a/packages/api/src/@core/@core-services/module.ts b/packages/api/src/@core/@core-services/module.ts index 78e7a51cb..4381224d8 100644 --- a/packages/api/src/@core/@core-services/module.ts +++ b/packages/api/src/@core/@core-services/module.ts @@ -1,20 +1,22 @@ +import { RagModule } from '@@core/rag/rag.module'; import { Global, Module } from '@nestjs/common'; -import { MappersRegistry } from './registries/mappers.registry'; -import { UnificationRegistry } from './registries/unification.registry'; -import { CoreSyncRegistry } from './registries/core-sync.registry'; +import { ConnectionUtils } from '../connections/@utils/index'; +import { FieldMappingService } from './../field-mapping/field-mapping.service'; import { EncryptionService } from './encryption/encryption.service'; -import { CoreUnification } from './unification/core-unification.service'; import { LoggerService } from './logger/logger.service'; -import { ConnectionUtils } from '../connections/@utils/index'; -import { CategoryConnectionRegistry } from './registries/connections-categories.registry'; import { PrismaService } from './prisma/prisma.service'; -import { FieldMappingService } from './../field-mapping/field-mapping.service'; import { BullQueueModule } from './queues/queue.module'; +import { CategoryConnectionRegistry } from './registries/connections-categories.registry'; +import { CoreSyncRegistry } from './registries/core-sync.registry'; +import { MappersRegistry } from './registries/mappers.registry'; +import { UnificationRegistry } from './registries/unification.registry'; import { RetryModule } from './request-retry/module'; +import { CoreUnification } from './unification/core-unification.service'; +import { RagService } from '@@core/rag/rag.service'; @Global() @Module({ - imports: [BullQueueModule, RetryModule], + imports: [BullQueueModule, RetryModule, RagModule], providers: [ PrismaService, MappersRegistry, @@ -26,6 +28,7 @@ import { RetryModule } from './request-retry/module'; LoggerService, ConnectionUtils, FieldMappingService, + RagService, ], exports: [ PrismaService, @@ -40,6 +43,7 @@ import { RetryModule } from './request-retry/module'; FieldMappingService, BullQueueModule, RetryModule, + RagService, ], }) export class CoreSharedModule {} diff --git a/packages/api/src/@core/@core-services/queues/queue.module.ts b/packages/api/src/@core/@core-services/queues/queue.module.ts index b004fc79e..8ce85065e 100644 --- a/packages/api/src/@core/@core-services/queues/queue.module.ts +++ b/packages/api/src/@core/@core-services/queues/queue.module.ts @@ -18,6 +18,9 @@ import { Queues } from './types'; { name: Queues.FAILED_PASSTHROUGH_REQUESTS_HANDLER, }, + { + name: Queues.RAG_DOCUMENT_PROCESSING, + }, ), ], providers: [BullQueueService], diff --git a/packages/api/src/@core/@core-services/queues/shared.service.ts b/packages/api/src/@core/@core-services/queues/shared.service.ts index 3afbff53b..08c38289f 100644 --- a/packages/api/src/@core/@core-services/queues/shared.service.ts +++ b/packages/api/src/@core/@core-services/queues/shared.service.ts @@ -14,6 +14,8 @@ export class BullQueueService { public readonly syncJobsQueue: Queue, @InjectQueue(Queues.FAILED_PASSTHROUGH_REQUESTS_HANDLER) public readonly failedPassthroughRequestsQueue: Queue, + @InjectQueue(Queues.RAG_DOCUMENT_PROCESSING) + private ragDocumentQueue: Queue, ) {} // getters @@ -30,23 +32,24 @@ export class BullQueueService { getFailedPassthroughRequestsQueue() { return this.failedPassthroughRequestsQueue; } + getRagDocumentQueue() { + return this.ragDocumentQueue; + } - // setters - async queueSyncJob(jobName: string, cron: string) { + async queueSyncJob(jobName: string, jobData: any, cron: string) { const jobs = await this.syncJobsQueue.getRepeatableJobs(); for (const job of jobs) { if (job.name === jobName) { await this.syncJobsQueue.removeRepeatableByKey(job.key); } } - // Add new job with the job name - await this.syncJobsQueue.add( - jobName, - {}, - { - repeat: { cron }, - jobId: jobName, // Using jobId to identify repeatable jobs - }, - ); + //await this.syncJobsQueue.add('health-check', {}, { attempts: 1 }); + + // Add new job with the job name and data + const res = await this.syncJobsQueue.add(jobName, jobData, { + repeat: { cron }, + jobId: jobName, // Using jobId to identify repeatable jobs + }); + console.log('job is ' + JSON.stringify(res)); } } diff --git a/packages/api/src/@core/@core-services/queues/types.ts b/packages/api/src/@core/@core-services/queues/types.ts index 5a9874ae2..5cd578e4b 100644 --- a/packages/api/src/@core/@core-services/queues/types.ts +++ b/packages/api/src/@core/@core-services/queues/types.ts @@ -3,4 +3,5 @@ export enum Queues { PANORA_WEBHOOKS_SENDER = 'PANORA_WEBHOOKS_SENDER', // Queue sends Panora webhooks to clients listening for important events SYNC_JOBS_WORKER = 'SYNC_JOBS_WORKER', // Queue which syncs data from remote 3rd parties FAILED_PASSTHROUGH_REQUESTS_HANDLER = 'FAILED_PASSTHROUGH_REQUESTS_HANDLER', // Queue which handles failed passthrough request due to rate limit and retries it with backOff + RAG_DOCUMENT_PROCESSING = 'RAG_DOCUMENT_PROCESSING', } diff --git a/packages/api/src/@core/@core-services/unification/ingest-data.service.ts b/packages/api/src/@core/@core-services/unification/ingest-data.service.ts index a10cbcc85..0d062095b 100644 --- a/packages/api/src/@core/@core-services/unification/ingest-data.service.ts +++ b/packages/api/src/@core/@core-services/unification/ingest-data.service.ts @@ -3,13 +3,20 @@ import { CoreSyncRegistry } from '../registries/core-sync.registry'; import { CoreUnification } from './core-unification.service'; import { v4 as uuidv4 } from 'uuid'; import { PrismaService } from '../prisma/prisma.service'; -import { ApiResponse, TargetObject } from '@@core/utils/types'; +import { + ApiResponse, + getFileExtensionFromMimeType, + TargetObject, +} from '@@core/utils/types'; import { UnifySourceType } from '@@core/utils/types/unify.output'; import { WebhookService } from '../webhooks/panora-webhooks/webhook.service'; import { ConnectionUtils } from '@@core/connections/@utils'; import { IBaseObjectService, SyncParam } from '@@core/utils/types/interface'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { LoggerService } from '../logger/logger.service'; +import { RagService } from '@@core/rag/rag.service'; +import { FileInfo } from '@@core/rag/types'; +import { fs_files as FileStorageFile } from '@prisma/client'; @Injectable() export class IngestDataService { @@ -21,6 +28,7 @@ export class IngestDataService { private connectionUtils: ConnectionUtils, private logger: LoggerService, private fieldMappingService: FieldMappingService, + private ragService: RagService, ) {} async syncForLinkedUser( @@ -169,6 +177,30 @@ export class IngestDataService { ...Object.values(extraParams || {}), ); + // insert the files in our s3 bucket so we can process them for our RAG + if (vertical === 'filestorage' && commonObject === 'file') { + const filesInfo: FileInfo[] = data + .filter((file: FileStorageFile) => file.file_url && file.mime_type) + .map((file: FileStorageFile) => ({ + id: file.id_fs_file, + url: file.file_url, + provider: integrationId, + s3Key: `${projectId}/${linkedUserId}/${ + file.id_fs_file + }.${getFileExtensionFromMimeType(file.mime_type)}`, + fileType: getFileExtensionFromMimeType(file.mime_type), + })); + + if (filesInfo.length > 0) { + console.log('During sync, found files to process for RAG...'); + await this.ragService.queueDocumentProcessing( + filesInfo, + projectId, + linkedUserId, + ); + } + } + const event = await this.prisma.events.create({ data: { id_connection: connectionId, diff --git a/packages/api/src/@core/connections/@token-refresh/refresh.service.ts b/packages/api/src/@core/connections/@token-refresh/refresh.service.ts index c56b2a069..cf3f7db8a 100644 --- a/packages/api/src/@core/connections/@token-refresh/refresh.service.ts +++ b/packages/api/src/@core/connections/@token-refresh/refresh.service.ts @@ -49,7 +49,7 @@ export class OAuthTokenRefreshService implements OnModuleInit { } } catch (error) { this.logger.error( - `Failed to refresh token for connection: ${connection.id_connection}`, + `Failed to refresh token for connection: ${connection.id_connection} ${connection.provider_slug}`, error, ); } diff --git a/packages/api/src/@core/connections/crm/crm.connection.module.ts b/packages/api/src/@core/connections/crm/crm.connection.module.ts index c8e2afb5b..9c0c9c987 100644 --- a/packages/api/src/@core/connections/crm/crm.connection.module.ts +++ b/packages/api/src/@core/connections/crm/crm.connection.module.ts @@ -21,6 +21,7 @@ import { ZohoConnectionService } from './services/zoho/zoho.service'; import { WealthboxConnectionService } from './services/wealthbox/wealthbox.service'; import { AcceloConnectionService } from './services/accelo/accelo.service'; import { MicrosoftDynamicsSalesConnectionService } from './services/microsoftdynamicssales/microsoftdynamicssales.service'; +import { SalesforceConnectionService } from './services/salesforce/salesforce.service'; @Module({ imports: [WebhookModule, BullQueueModule], @@ -46,6 +47,7 @@ import { MicrosoftDynamicsSalesConnectionService } from './services/microsoftdyn WealthboxConnectionService, AcceloConnectionService, MicrosoftDynamicsSalesConnectionService, + SalesforceConnectionService, ], exports: [CrmConnectionsService], }) diff --git a/packages/api/src/@core/connections/crm/services/microsoftdynamicssales/microsoftdynamicssales.service.ts b/packages/api/src/@core/connections/crm/services/microsoftdynamicssales/microsoftdynamicssales.service.ts index 6d2a9cb4a..e20001b67 100644 --- a/packages/api/src/@core/connections/crm/services/microsoftdynamicssales/microsoftdynamicssales.service.ts +++ b/packages/api/src/@core/connections/crm/services/microsoftdynamicssales/microsoftdynamicssales.service.ts @@ -13,16 +13,11 @@ import { } from '@@core/connections/@utils/types'; import { PassthroughResponse } from '@@core/passthrough/types'; import { Injectable } from '@nestjs/common'; -import { - AuthStrategy, - CONNECTORS_METADATA, - OAuth2AuthData, - providerToType, -} from '@panora/shared'; +import { AuthStrategy, OAuth2AuthData, providerToType } from '@panora/shared'; import axios from 'axios'; +import { URLSearchParams } from 'url'; import { v4 as uuidv4 } from 'uuid'; import { ServiceRegistry } from '../registry.service'; -import { URLSearchParams } from 'url'; export type MicrosoftDynamicsSalesOAuthResponse = { access_token: string; @@ -96,7 +91,7 @@ export class MicrosoftDynamicsSalesConnectionService extends AbstractBaseConnect async handleCallback(opts: OAuthCallbackParams) { try { - const { linkedUserId, projectId, code, resource } = opts; + const { linkedUserId, projectId, code, organization_name } = opts; const isNotUnique = await this.prisma.connections.findFirst({ where: { id_linked_user: linkedUserId, @@ -117,7 +112,7 @@ export class MicrosoftDynamicsSalesConnectionService extends AbstractBaseConnect client_id: CREDENTIALS.CLIENT_ID, client_secret: CREDENTIALS.CLIENT_SECRET, code: code, - scope: `https://${resource}/.default offline_access`, + scope: `https://${organization_name}/.default offline_access`, grant_type: 'authorization_code', }); const res = await axios.post( @@ -146,7 +141,7 @@ export class MicrosoftDynamicsSalesConnectionService extends AbstractBaseConnect data: { access_token: this.cryptoService.encrypt(data.access_token), refresh_token: this.cryptoService.encrypt(data.refresh_token), - account_url: `https://${resource}`, + account_url: `https://${organization_name}`, expiration_timestamp: new Date( new Date().getTime() + Number(data.expires_in) * 1000, ), @@ -162,7 +157,7 @@ export class MicrosoftDynamicsSalesConnectionService extends AbstractBaseConnect provider_slug: 'microsoftdynamicssales', vertical: 'crm', token_type: 'oauth2', - account_url: `https://${resource}`, + account_url: `https://${organization_name}`, access_token: this.cryptoService.encrypt(data.access_token), refresh_token: this.cryptoService.encrypt(data.refresh_token), expiration_timestamp: new Date( diff --git a/packages/api/src/@core/connections/crm/services/salesforce/salesforce.service.ts b/packages/api/src/@core/connections/crm/services/salesforce/salesforce.service.ts new file mode 100644 index 000000000..95dbeb57f --- /dev/null +++ b/packages/api/src/@core/connections/crm/services/salesforce/salesforce.service.ts @@ -0,0 +1,238 @@ +import { EncryptionService } from '@@core/@core-services/encryption/encryption.service'; +import { EnvironmentService } from '@@core/@core-services/environment/environment.service'; +import { LoggerService } from '@@core/@core-services/logger/logger.service'; +import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; +import { RetryHandler } from '@@core/@core-services/request-retry/retry.handler'; +import { ConnectionsStrategiesService } from '@@core/connections-strategies/connections-strategies.service'; +import { ConnectionUtils } from '@@core/connections/@utils'; +import { + AbstractBaseConnectionService, + OAuthCallbackParams, + PassthroughInput, + RefreshParams, +} from '@@core/connections/@utils/types'; +import { PassthroughResponse } from '@@core/passthrough/types'; +import { Injectable } from '@nestjs/common'; +import { + AuthStrategy, + CONNECTORS_METADATA, + DynamicApiUrl, + OAuth2AuthData, + providerToType, +} from '@panora/shared'; +import axios from 'axios'; +import { v4 as uuidv4 } from 'uuid'; +import { ServiceRegistry } from '../registry.service'; + +export type SalesforceAuthResponse = { + access_token: string; + signature: string; + scope: string; + id_token: string; + instance_url: string; + id: string; + token_type: string; + issued_at: string; + refresh_token: string; +}; + +@Injectable() +export class SalesforceConnectionService extends AbstractBaseConnectionService { + private readonly type: string; + + constructor( + protected prisma: PrismaService, + private logger: LoggerService, + private env: EnvironmentService, + protected cryptoService: EncryptionService, + private registry: ServiceRegistry, + private cService: ConnectionsStrategiesService, + private connectionUtils: ConnectionUtils, + private retryService: RetryHandler, + ) { + super(prisma, cryptoService); + this.logger.setContext(SalesforceConnectionService.name); + this.registry.registerService('salesforce', this); + this.type = providerToType('salesforce', 'crm', AuthStrategy.oauth2); + } + + async passthrough( + input: PassthroughInput, + connectionId: string, + ): Promise { + try { + const { headers } = input; + const config = await this.constructPassthrough(input, connectionId); + + const connection = await this.prisma.connections.findUnique({ + where: { + id_connection: connectionId, + }, + }); + + config.headers['Authorization'] = `Bearer ${Buffer.from( + `${this.cryptoService.decrypt(connection.access_token)}:`, + ).toString('base64')}`; + + config.headers = { + ...config.headers, + ...headers, + }; + + return await this.retryService.makeRequest( + { + method: config.method, + url: config.url, + data: config.data, + headers: config.headers, + }, + 'crm.salesforce.passthrough', + config.linkedUserId, + ); + } catch (error) { + throw error; + } + } + + async handleCallback(opts: OAuthCallbackParams) { + try { + const { linkedUserId, projectId, code, domain } = opts; + const isNotUnique = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'salesforce', + vertical: 'crm', + }, + }); + //reconstruct the redirect URI that was passed in the frontend it must be the same + const REDIRECT_URI = `${this.env.getPanoraBaseUrl()}/connections/oauth/callback`; + + const CREDENTIALS = (await this.cService.getCredentials( + projectId, + this.type, + )) as OAuth2AuthData; + + const formData = new URLSearchParams({ + grant_type: 'authorization_code', + client_id: CREDENTIALS.CLIENT_ID, + client_secret: CREDENTIALS.CLIENT_SECRET, + redirect_uri: REDIRECT_URI, + code: code, + }); + const res = await axios.post( + `https://${domain}.my.salesforce.com/services/oauth2/token`, + formData.toString(), + { + headers: { + 'Content-Type': 'application/x-www-form-urlencoded;charset=utf-8', + }, + }, + ); + const data: SalesforceAuthResponse = res.data; + // save tokens for this customer inside our db + let db_res; + const connection_token = uuidv4(); + + if (isNotUnique) { + // Update existing connection + db_res = await this.prisma.connections.update({ + where: { + id_connection: isNotUnique.id_connection, + }, + data: { + access_token: this.cryptoService.encrypt(data.access_token), + refresh_token: this.cryptoService.encrypt(data.refresh_token), + expiration_timestamp: new Date( + new Date().getTime() + 90 * 60 * 1000, // 90 minutes in milliseconds + ), + status: 'valid', + created_at: new Date(), + }, + }); + } else { + // Create new connection + db_res = await this.prisma.connections.create({ + data: { + id_connection: uuidv4(), + connection_token: connection_token, + provider_slug: 'salesforce', + vertical: 'crm', + token_type: 'oauth2', + account_url: ( + CONNECTORS_METADATA['crm']['salesforce'].urls + .apiUrl as DynamicApiUrl + )(domain), + access_token: this.cryptoService.encrypt(data.access_token), + refresh_token: this.cryptoService.encrypt(data.refresh_token), + expiration_timestamp: new Date( + new Date().getTime() + 90 * 60 * 1000, // 90 minutes in milliseconds + ), + status: 'valid', + created_at: new Date(), + projects: { + connect: { id_project: projectId }, + }, + linked_users: { + connect: { + id_linked_user: await this.connectionUtils.getLinkedUserId( + projectId, + linkedUserId, + ), + }, + }, + }, + }); + } + this.logger.log('Successfully added tokens inside DB ' + db_res); + return db_res; + } catch (error) { + throw error; + } + } + + async handleTokenRefresh(opts: RefreshParams) { + try { + const { connectionId, refreshToken, projectId } = opts; + const CREDENTIALS = (await this.cService.getCredentials( + projectId, + this.type, + )) as OAuth2AuthData; + + const params = { + grant_type: 'refresh_token', + client_id: CREDENTIALS.CLIENT_ID, + client_secret: CREDENTIALS.CLIENT_SECRET, + refresh_token: this.cryptoService.decrypt(refreshToken), + }; + const connection = await this.prisma.connections.findUnique({ + where: { + id_connection: connectionId, + }, + }); + + const queryString = new URLSearchParams(params).toString(); + const url = `${connection.account_url}/services/oauth2/token`; + const res = await axios.post(url, queryString, { + headers: { + 'Content-Type': 'application/x-www-form-urlencoded;charset=utf-8', + }, + }); + const data: SalesforceAuthResponse = res.data; + const res_ = await this.prisma.connections.update({ + where: { + id_connection: connectionId, + }, + data: { + access_token: this.cryptoService.encrypt(data.access_token), + refresh_token: this.cryptoService.encrypt(data.refresh_token), + expiration_timestamp: new Date( + new Date().getTime() + 90 * 60 * 1000, // 90 minutes in milliseconds + ), + }, + }); + this.logger.log('OAuth credentials updated : salesforce'); + } catch (error) { + throw error; + } + } +} diff --git a/packages/api/src/@core/connections/filestorage/services/google_drive/google_drive.service.ts b/packages/api/src/@core/connections/filestorage/services/google_drive/google_drive.service.ts index c19eef54c..9a9f6c3df 100644 --- a/packages/api/src/@core/connections/filestorage/services/google_drive/google_drive.service.ts +++ b/packages/api/src/@core/connections/filestorage/services/google_drive/google_drive.service.ts @@ -85,7 +85,7 @@ export class GoogleDriveConnectionService extends AbstractBaseConnectionService data: config.data, headers: config.headers, }, - 'filestorage.google_drive.passthrough', + 'filestorage.googledrive.passthrough', config.linkedUserId, ); } catch (error) { @@ -134,7 +134,6 @@ export class GoogleDriveConnectionService extends AbstractBaseConnectionService let db_res; const connection_token = uuidv4(); - if (isNotUnique) { db_res = await this.prisma.connections.update({ where: { @@ -157,7 +156,7 @@ export class GoogleDriveConnectionService extends AbstractBaseConnectionService data: { id_connection: uuidv4(), connection_token: connection_token, - provider_slug: 'google_drive', + provider_slug: 'googledrive', vertical: 'filestorage', token_type: 'oauth2', account_url: CONNECTORS_METADATA['filestorage']['googledrive'].urls @@ -191,19 +190,19 @@ export class GoogleDriveConnectionService extends AbstractBaseConnectionService async handleTokenRefresh(opts: RefreshParams) { try { const { connectionId, refreshToken, projectId } = opts; - + const CREDENTIALS = (await this.cService.getCredentials( projectId, this.type, )) as OAuth2AuthData; - + const formData = new URLSearchParams({ grant_type: 'refresh_token', refresh_token: this.cryptoService.decrypt(refreshToken), client_id: CREDENTIALS.CLIENT_ID, client_secret: CREDENTIALS.CLIENT_SECRET, }); - + const res = await axios.post( `https://oauth2.googleapis.com/token`, formData.toString(), @@ -217,20 +216,29 @@ export class GoogleDriveConnectionService extends AbstractBaseConnectionService }, ); const data: GoogleDriveOAuthResponse = res.data; + + // Prepare the update data + const updateData: any = { + access_token: this.cryptoService.encrypt(data.access_token), + expiration_timestamp: new Date( + new Date().getTime() + Number(data.expires_in) * 1000, + ), + }; + + // Only update the refresh token if a new one is provided + if (data.refresh_token) { + updateData.refresh_token = this.cryptoService.encrypt(data.refresh_token); + } + await this.prisma.connections.update({ where: { id_connection: connectionId, }, - data: { - access_token: this.cryptoService.encrypt(data.access_token), - refresh_token: this.cryptoService.encrypt(data.refresh_token), - expiration_timestamp: new Date( - new Date().getTime() + Number(data.expires_in) * 1000, - ), - }, + data: updateData, }); this.logger.log('OAuth credentials updated : google_drive '); } catch (error) { + this.logger.error('Error refreshing Google Drive token:', error); throw error; } } diff --git a/packages/api/src/@core/connections/filestorage/services/sharepoint/sharepoint.service.ts b/packages/api/src/@core/connections/filestorage/services/sharepoint/sharepoint.service.ts index 3ff6d03c6..2db6e673e 100644 --- a/packages/api/src/@core/connections/filestorage/services/sharepoint/sharepoint.service.ts +++ b/packages/api/src/@core/connections/filestorage/services/sharepoint/sharepoint.service.ts @@ -16,6 +16,7 @@ import { Injectable } from '@nestjs/common'; import { AuthStrategy, CONNECTORS_METADATA, + DynamicApiUrl, OAuth2AuthData, providerToType, } from '@panora/shared'; @@ -96,7 +97,7 @@ export class SharepointConnectionService extends AbstractBaseConnectionService { async handleCallback(opts: OAuthCallbackParams) { try { - const { linkedUserId, projectId, code } = opts; + const { linkedUserId, projectId, code, site, tenant } = opts; const isNotUnique = await this.prisma.connections.findFirst({ where: { id_linked_user: linkedUserId, @@ -122,7 +123,7 @@ export class SharepointConnectionService extends AbstractBaseConnectionService { grant_type: 'authorization_code', }); const res = await axios.post( - `https://app.sharepoint.com/oauth2/tokens`, + `https://login.microsoftonline.com/common/oauth2/v2.0/token`, formData.toString(), { headers: { @@ -138,6 +139,17 @@ export class SharepointConnectionService extends AbstractBaseConnectionService { 'OAuth credentials : sharepoint filestorage ' + JSON.stringify(data), ); + // get site_id from tenant and sitename + const site_details = await axios.get( + `https://graph.microsoft.com/v1.0/sites/${tenant}.sharepoint.com:/sites/${site}`, + { + headers: { + Authorization: `Bearer ${data.access_token}`, + }, + }, + ); + const site_id = site_details.data.id; + let db_res; const connection_token = uuidv4(); @@ -149,8 +161,10 @@ export class SharepointConnectionService extends AbstractBaseConnectionService { data: { access_token: this.cryptoService.encrypt(data.access_token), refresh_token: this.cryptoService.encrypt(data.refresh_token), - account_url: CONNECTORS_METADATA['filestorage']['sharepoint'].urls - .apiUrl as string, + account_url: ( + CONNECTORS_METADATA['filestorage']['sharepoint'].urls + .apiUrl as DynamicApiUrl + )(site_id), expiration_timestamp: new Date( new Date().getTime() + Number(data.expires_in) * 1000, ), @@ -166,8 +180,10 @@ export class SharepointConnectionService extends AbstractBaseConnectionService { provider_slug: 'sharepoint', vertical: 'filestorage', token_type: 'oauth2', - account_url: CONNECTORS_METADATA['filestorage']['sharepoint'].urls - .apiUrl as string, + account_url: ( + CONNECTORS_METADATA['filestorage']['sharepoint'].urls + .apiUrl as DynamicApiUrl + )(site_id), access_token: this.cryptoService.encrypt(data.access_token), refresh_token: this.cryptoService.encrypt(data.refresh_token), expiration_timestamp: new Date( @@ -214,7 +230,7 @@ export class SharepointConnectionService extends AbstractBaseConnectionService { )) as OAuth2AuthData; const res = await axios.post( - `https://app.sharepoint.com/oauth2/tokens`, + `https://login.microsoftonline.com/common/oauth2/v2.0/token`, formData.toString(), { headers: { diff --git a/packages/api/src/@core/core.module.ts b/packages/api/src/@core/core.module.ts index fb78e6842..bb7310259 100644 --- a/packages/api/src/@core/core.module.ts +++ b/packages/api/src/@core/core.module.ts @@ -16,6 +16,7 @@ import { PassthroughModule } from './passthrough/passthrough.module'; import { ProjectConnectorsModule } from './project-connectors/project-connectors.module'; import { ProjectsModule } from './projects/projects.module'; import { SyncModule } from './sync/sync.module'; +import { RagModule } from './rag/rag.module'; @Module({ imports: [ @@ -35,6 +36,7 @@ import { SyncModule } from './sync/sync.module'; SyncModule, ProjectConnectorsModule, BullQueueModule, + RagModule, ], exports: [ AuthModule, @@ -54,6 +56,7 @@ import { SyncModule } from './sync/sync.module'; ProjectConnectorsModule, IngestDataService, BullQueueModule, + RagModule, ], providers: [IngestDataService], }) diff --git a/packages/api/src/@core/events/events.controller.ts b/packages/api/src/@core/events/events.controller.ts index 58a0af971..c36e3be9a 100644 --- a/packages/api/src/@core/events/events.controller.ts +++ b/packages/api/src/@core/events/events.controller.ts @@ -1,26 +1,25 @@ +import { LoggerService } from '@@core/@core-services/logger/logger.service'; +import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; +import { JwtAuthGuard } from '@@core/auth/guards/jwt-auth.guard'; +import { ApiGetArrayCustomResponse } from '@@core/utils/dtos/openapi.respone.dto'; +import { PaginationDto } from '@@core/utils/dtos/webapp.event.pagination.dto'; import { Controller, Get, Query, + Request, UseGuards, UsePipes, ValidationPipe, - Request, } from '@nestjs/common'; -import { EventsService } from './events.service'; -import { LoggerService } from '@@core/@core-services/logger/logger.service'; import { - ApiExcludeController, ApiExcludeEndpoint, ApiOperation, ApiResponse, ApiTags, } from '@nestjs/swagger'; -import { PaginationDto } from '@@core/utils/dtos/webapp.event.pagination.dto'; -import { JwtAuthGuard } from '@@core/auth/guards/jwt-auth.guard'; -import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; -import { ApiGetArrayCustomResponse } from '@@core/utils/dtos/openapi.respone.dto'; import { EventResponse } from './dto/index.dto'; +import { EventsService } from './events.service'; @ApiTags('events') @Controller('events') diff --git a/packages/api/src/@core/rag/chunking/chunking.service.ts b/packages/api/src/@core/rag/chunking/chunking.service.ts new file mode 100644 index 000000000..dc45670bb --- /dev/null +++ b/packages/api/src/@core/rag/chunking/chunking.service.ts @@ -0,0 +1,80 @@ +import { Injectable } from '@nestjs/common'; +import { RecursiveCharacterTextSplitter } from '@langchain/textsplitters'; +import { Document } from '@langchain/core/documents'; + +@Injectable() +export class DocumentSplitterService { + async chunkDocument( + documents: Document[], + fileType: string, + chunkSize = 1000, + chunkOverlap = 200, + ): Promise { + const chunkedDocuments: Document[] = []; + + for (const document of documents) { + let chunks: Document[]; + + if (fileType === 'json' || fileType === 'csv') { + chunks = await this.chunkJSON(document, chunkSize); + } else { + chunks = await this.chunkText(document, chunkSize, chunkOverlap); + } + + chunkedDocuments.push(...chunks); + } + + return chunkedDocuments; + } + + private async chunkText( + document: Document, + chunkSize: number, + chunkOverlap: number, + ): Promise { + const textSplitter = new RecursiveCharacterTextSplitter({ + chunkSize, + chunkOverlap, + }); + return textSplitter.splitDocuments([document]); + } + + private async chunkJSON( + document: Document, + chunkSize: number, + ): Promise { + const jsonContent = JSON.parse(document.pageContent); + const chunks: Document[] = []; + let currentChunk: Record = {}; + let currentSize = 0; + + for (const [key, value] of Object.entries(jsonContent)) { + const entrySize = JSON.stringify({ [key]: value }).length; + + if (currentSize + entrySize > chunkSize && currentSize > 0) { + chunks.push( + new Document({ + pageContent: JSON.stringify(currentChunk), + metadata: { ...document.metadata, chunk: chunks.length + 1 }, + }), + ); + currentChunk = {}; + currentSize = 0; + } + + currentChunk[key] = value; + currentSize += entrySize; + } + + if (Object.keys(currentChunk).length > 0) { + chunks.push( + new Document({ + pageContent: JSON.stringify(currentChunk), + metadata: { ...document.metadata, chunk: chunks.length + 1 }, + }), + ); + } + + return chunks; + } +} \ No newline at end of file diff --git a/packages/api/src/@core/rag/document.processor.ts b/packages/api/src/@core/rag/document.processor.ts new file mode 100644 index 000000000..e7e465b9b --- /dev/null +++ b/packages/api/src/@core/rag/document.processor.ts @@ -0,0 +1,68 @@ +import { Process, Processor } from '@nestjs/bull'; +import { Job } from 'bull'; +import { FileInfo } from './types'; +import { VectorDatabaseService } from './vecdb/vecdb.service'; +import { S3Service } from '@@core/s3/s3.service'; +import { DocumentSplitterService } from './chunking/chunking.service'; +import { EmbeddingService } from './embedding/embedding.service'; +import { DocumentLoaderService } from './loader/loader.service'; + +@Processor('RAG_DOCUMENT_PROCESSING') +export class ProcessDocumentProcessor { + constructor( + private s3Service: S3Service, + private documentLoaderService: DocumentLoaderService, + private documentSplitterService: DocumentSplitterService, + private embeddingService: EmbeddingService, + private vectorDatabaseService: VectorDatabaseService, + ) {} + + @Process('batchDocs') + async processDocuments( + job: Job<{ filesInfo: FileInfo[]; projectId: string }>, + ) { + const { filesInfo, projectId } = job.data; + const results = []; + + for (const fileInfo of filesInfo) { + try { + const readStream = await this.s3Service.getReadStream(fileInfo.s3Key); + const document = + await this.documentLoaderService.loadDocumentFromStream( + readStream, + fileInfo.fileType, + fileInfo.s3Key, + ); + const chunks = await this.documentSplitterService.chunkDocument( + document, + fileInfo.fileType, + ); + // console.log(`chunks for ${fileInfo.id} are ` + JSON.stringify(chunks)); + const embeddings = await this.embeddingService.generateEmbeddings( + chunks, + projectId + ); + // Split embeddings into smaller batches + const batchSize = 100; // Adjust this value as needed + for (let i = 0; i < chunks.length; i += batchSize) { + const batchChunks = chunks.slice(i, i + batchSize); + const batchEmbeddings = embeddings.slice(i, i + batchSize); + await this.vectorDatabaseService.storeEmbeddings( + fileInfo.id, + batchChunks, + batchEmbeddings, + projectId, + ); + } + results.push(`Successfully processed document ${fileInfo.id}`); + } catch (error) { + console.error(`Error processing document ${fileInfo.id}:`, error); + results.push( + `Failed to process document ${fileInfo.id}: ${error.message}`, + ); + } + } + + return results; + } +} diff --git a/packages/api/src/@core/rag/embedding/embedding.credentials.service.ts b/packages/api/src/@core/rag/embedding/embedding.credentials.service.ts new file mode 100644 index 000000000..c15626f09 --- /dev/null +++ b/packages/api/src/@core/rag/embedding/embedding.credentials.service.ts @@ -0,0 +1,57 @@ +import { Injectable } from '@nestjs/common'; +import { EnvironmentService } from '@@core/@core-services/environment/environment.service'; +import { ConnectionsStrategiesService } from '@@core/connections-strategies/connections-strategies.service'; +import { EmbeddingModelType } from './embedding.service'; + +@Injectable() +export class EmbeddingCredentialsService { + constructor( + private envService: EnvironmentService, + private connectionsStrategiesService: ConnectionsStrategiesService, + ) {} + + async getEmbeddingCredentials( + projectId: string, + embeddingModel: EmbeddingModelType, + ): Promise { + const type = `embedding_model.${embeddingModel.toLowerCase()}`; + const isCustom = await this.connectionsStrategiesService.isCustomCredentials( + projectId, + type, + ); + + if (isCustom) { + return this.getCustomCredentials(projectId, type); + } else { + return this.getManagedCredentials(embeddingModel); + } + } + + private async getCustomCredentials( + projectId: string, + type: string, + ): Promise { + return this.connectionsStrategiesService.getConnectionStrategyData( + projectId, + type, + ['api_key'], + ); + } + + private getManagedCredentials(embeddingModel: EmbeddingModelType): string[] { + switch (embeddingModel) { + case 'OPENAI_ADA_SMALL_512': + case 'OPENAI_ADA_SMALL_1536': + case 'OPENAI_ADA_LARGE_256': + case 'OPENAI_ADA_LARGE_1024': + case 'OPENAI_ADA_LARGE_3072': + return [this.envService.getOpenAIApiKey()]; + case 'COHERE_MULTILINGUAL_V3': + return [this.envService.getCohereApiKey()]; + case 'JINA': + return [this.envService.getJinaApiKey()]; + default: + throw new Error(`Unsupported embedding model: ${embeddingModel}`); + } + } +} \ No newline at end of file diff --git a/packages/api/src/@core/rag/embedding/embedding.service.ts b/packages/api/src/@core/rag/embedding/embedding.service.ts new file mode 100644 index 000000000..fbe32d6d9 --- /dev/null +++ b/packages/api/src/@core/rag/embedding/embedding.service.ts @@ -0,0 +1,107 @@ +import { EnvironmentService } from '@@core/@core-services/environment/environment.service'; +import { CohereEmbeddings } from '@langchain/cohere'; +import { Document } from '@langchain/core/documents'; +import { OpenAIEmbeddings } from '@langchain/openai'; +import { JinaEmbeddings } from '@langchain/community/embeddings/jina'; +import { Injectable, OnModuleInit } from '@nestjs/common'; +import { ConnectionsStrategiesService } from '@@core/connections-strategies/connections-strategies.service'; +import { EmbeddingCredentialsService } from './embedding.credentials.service'; + +export type EmbeddingModelType = + | 'OPENAI_ADA_SMALL_512' + | 'OPENAI_ADA_SMALL_1536' + | 'OPENAI_ADA_LARGE_256' + | 'OPENAI_ADA_LARGE_1024' + | 'OPENAI_ADA_002' + | 'OPENAI_ADA_LARGE_3072' + | 'COHERE_MULTILINGUAL_V3' + | 'JINA'; + +@Injectable() +export class EmbeddingService implements OnModuleInit { + private embeddings: OpenAIEmbeddings | CohereEmbeddings | JinaEmbeddings; + + constructor( + private envService: EnvironmentService, + private connectionsStrategiesService: ConnectionsStrategiesService, + private embeddingCredentialsService: EmbeddingCredentialsService, + ) {} + + async onModuleInit() { + // Initialize with default settings + console.log(); + } + + async initializeEmbeddings(projectId: string) { + let embeddingType: EmbeddingModelType; + let apiKey: string; + + if (projectId) { + const activeStrategies = await this.connectionsStrategiesService.getConnectionStrategiesForProject(projectId); + const activeEmbeddingStrategy = activeStrategies.find( + (strategy) => strategy.type.startsWith('embedding_model.') && strategy.status, + ); + + if (activeEmbeddingStrategy) { + embeddingType = activeEmbeddingStrategy.type.split('.')[1].toUpperCase() as EmbeddingModelType; + [apiKey] = await this.embeddingCredentialsService.getEmbeddingCredentials(projectId, embeddingType); + } else { + embeddingType = 'OPENAI_ADA_002'; + apiKey = this.envService.getOpenAIApiKey(); + } + } else { + embeddingType = 'OPENAI_ADA_002'; + apiKey = this.envService.getOpenAIApiKey(); + } + + switch (embeddingType) { + case 'OPENAI_ADA_002': + case 'OPENAI_ADA_SMALL_512': + case 'OPENAI_ADA_SMALL_1536': + case 'OPENAI_ADA_LARGE_256': + case 'OPENAI_ADA_LARGE_1024': + case 'OPENAI_ADA_LARGE_3072': + this.embeddings = new OpenAIEmbeddings({ + openAIApiKey: apiKey, + modelName: this.getOpenAIModelName(embeddingType), + }); + break; + case 'COHERE_MULTILINGUAL_V3': + this.embeddings = new CohereEmbeddings({ + apiKey, + model: 'multilingual-22-12', + }); + break; + case 'JINA': + this.embeddings = new JinaEmbeddings({ + apiKey, + }); + break; + default: + throw new Error(`Unsupported embedding type: ${embeddingType}`); + } + } + + private getOpenAIModelName(type: EmbeddingModelType): string { + const modelMap: { [key: string]: string } = { + OPENAI_ADA_002: 'text-embedding-ada-002', + OPENAI_ADA_SMALL_512: 'text-embedding-3-small', + OPENAI_ADA_SMALL_1536: 'text-embedding-3-small', + OPENAI_ADA_LARGE_256: 'text-embedding-3-large', + OPENAI_ADA_LARGE_1024: 'text-embedding-3-large', + OPENAI_ADA_LARGE_3072: 'text-embedding-3-large', + }; + return modelMap[type] || 'text-embedding-ada-002'; + } + + async generateEmbeddings(chunks: Document[], projectId: string) { + await this.initializeEmbeddings(projectId); + const texts = chunks.map((chunk) => chunk.pageContent); + return this.embeddings.embedDocuments(texts); + } + + async embedQuery(query: string, projectId?: string) { + // await this.initializeEmbeddings(projectId); + return this.embeddings.embedQuery(query); + } +} \ No newline at end of file diff --git a/packages/api/src/@core/rag/loader/loader.service.ts b/packages/api/src/@core/rag/loader/loader.service.ts new file mode 100644 index 000000000..687f5636b --- /dev/null +++ b/packages/api/src/@core/rag/loader/loader.service.ts @@ -0,0 +1,104 @@ +import { Injectable } from '@nestjs/common'; +import { PDFLoader } from '@langchain/community/document_loaders/fs/pdf'; +import { DocxLoader } from '@langchain/community/document_loaders/fs/docx'; +import { TextLoader } from 'langchain/document_loaders/fs/text'; +import { UnstructuredLoader } from '@langchain/community/document_loaders/fs/unstructured'; +import { Readable } from 'stream'; +import { EnvironmentService } from '@@core/@core-services/environment/environment.service'; +import { Document } from 'langchain/document'; +import * as csvtojson from 'csvtojson'; + +type UnstructuredCredsType = { + apiKey: string; + apiUrl: string; +}; +@Injectable() +export class DocumentLoaderService { + private unstructuredCreds: UnstructuredCredsType; + constructor(private envService: EnvironmentService) { + this.unstructuredCreds = this.envService.getUnstructuredCreds(); + } + async loadDocumentFromStream( + stream: Readable, + fileType: string, + fileName?: string, + ) { + const buffer = await this.streamToBuffer(stream); + const loaders = { + pdf: (buf: Buffer) => new PDFLoader(new Blob([buf])), + docx: (buf: Buffer) => new DocxLoader(new Blob([buf])), + csv: (buf: Buffer) => ({ + load: async () => { + const content = buf.toString('utf-8'); + const lines = content.split('\n'); + const separator = this.detectSeparator(lines[0]); + const headers = lines[0] + .split(separator) + .map((header) => header.trim()); + + const jsonData = await csvtojson({ + delimiter: separator, + headers: headers, + output: 'json', + }).fromString(content); + + const formattedData = jsonData.map((item: any) => { + const formattedItem: Record = {}; + for (const key in item) { + const value = item[key]; + formattedItem[key] = isNaN(Number(value)) ? value : Number(value); + } + return formattedItem; + }); + + // console.log('json data from csv is ' + JSON.stringify(formattedData)); + + return formattedData.map( + (item, index) => + new Document({ + pageContent: JSON.stringify(item), + metadata: { + source: fileName || 'csv', + row: index + 2, + columns: headers, + }, + }), + ); + }, + }), + txt: (buf: Buffer) => new TextLoader(new Blob([buf])), + md: (buf: Buffer) => + new UnstructuredLoader( + { buffer: buf, fileName: fileName }, + { + apiKey: this.unstructuredCreds.apiKey, + apiUrl: this.unstructuredCreds.apiUrl, + }, + ), + }; + + const loaderFunction = loaders[fileType as keyof typeof loaders]; + if (!loaderFunction) { + throw new Error(`Unsupported file type: ${fileType}`); + } + + const loader = loaderFunction(buffer); + return loader.load(); + } + + private detectSeparator(headerLine: string): string { + const possibleSeparators = [',', ';', '\t', '|']; + return possibleSeparators.reduce((a, b) => + headerLine.split(a).length > headerLine.split(b).length ? a : b, + ); + } + + private async streamToBuffer(stream: Readable): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + stream.on('data', (chunk) => chunks.push(Buffer.from(chunk))); + stream.on('error', (err) => reject(err)); + stream.on('end', () => resolve(Buffer.concat(chunks))); + }); + } +} diff --git a/packages/api/src/@core/rag/rag.controller.ts b/packages/api/src/@core/rag/rag.controller.ts new file mode 100644 index 000000000..74ee8c29a --- /dev/null +++ b/packages/api/src/@core/rag/rag.controller.ts @@ -0,0 +1,59 @@ +import { Controller, Post, Body, UseGuards } from '@nestjs/common'; +import { RagService } from './rag.service'; +import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; +import { + ApiBody, + ApiOperation, + ApiParam, + ApiQuery, + ApiTags, + ApiHeader, + //ApiKeyAuth, +} from '@nestjs/swagger'; +import { ConnectionUtils } from '@@core/connections/@utils'; + +@Controller('rag') +export class RagController { + constructor( + private documentEmbeddingService: RagService, + private ragService: RagService, + private connectionUtils: ConnectionUtils, + ) {} + + @Post('query') + @UseGuards(ApiKeyAuthGuard) + async queryEmbeddings(@Body() body: { query: string; topK?: number }) { + return this.documentEmbeddingService.queryEmbeddings(body.query, body.topK); + } + + /* + @ApiOperation({ + operationId: 'listFilestorageFile', + summary: 'List Files', + }) + @ApiHeader({ + name: 'x-connection-token', + required: true, + description: 'The connection token', + example: 'b008e199-eda9-4629-bd41-a01b6195864a', + }) + @Post('process') + @UseGuards(ApiKeyAuthGuard) + async processFile(@Headers('x-connection-token') connection_token: string) { + const { linkedUserId, remoteSource, connectionId, projectId } = + await this.connectionUtils.getConnectionMetadataFromConnectionToken( + connection_token, + ); + const body = { + id: 'ID_1', + url: 'https://drive.google.com/file/d/1rrC1idlFpCBdF3DVzNDp1WeRZ4_mKGho/view', + s3Key: `${projectId}/ID_1.pdf`, + fileType: 'pdf', + }; + return await this.ragService.queueDocumentProcessing( + [body], + projectId, + linkedUserId, + ); + }*/ +} diff --git a/packages/api/src/@core/rag/rag.module.ts b/packages/api/src/@core/rag/rag.module.ts new file mode 100644 index 000000000..51f8ab763 --- /dev/null +++ b/packages/api/src/@core/rag/rag.module.ts @@ -0,0 +1,63 @@ +import { EnvironmentService } from '@@core/@core-services/environment/environment.service'; +import { S3Service } from '@@core/s3/s3.service'; +import { Module } from '@nestjs/common'; +import { DocumentSplitterService } from './chunking/chunking.service'; +import { ProcessDocumentProcessor } from './document.processor'; +import { EmbeddingService } from './embedding/embedding.service'; +import { DocumentLoaderService } from './loader/loader.service'; +import { RagController } from './rag.controller'; +import { RagService } from './rag.service'; +import { ChromaDBService } from './vecdb/chromadb/chromadb.service'; +import { MilvusService } from './vecdb/milvus/milvus.service'; +import { PineconeService } from './vecdb/pinecone/pinecone.service'; +import { TurboPufferService } from './vecdb/turbopuffer/turbopuffer.service'; +import { VectorDatabaseService } from './vecdb/vecdb.service'; +import { WeaviateService } from './vecdb/weaviate/weaviate.service'; +import { QdrantDBService } from './vecdb/qdrant/qdrant.service'; +import { FileModule } from '@filestorage/file/file.module'; +import { VectorDbCredentialsService } from './vecdb/vecdb.credentials.service'; +import { ConnectionsStrategiesService } from '@@core/connections-strategies/connections-strategies.service'; +import { EmbeddingCredentialsService } from './embedding/embedding.credentials.service'; + +@Module({ + imports: [FileModule], + controllers: [RagController], + providers: [ + EnvironmentService, + VectorDatabaseService, + PineconeService, + WeaviateService, + TurboPufferService, + ChromaDBService, + QdrantDBService, + MilvusService, + RagService, + ProcessDocumentProcessor, + S3Service, + DocumentLoaderService, + DocumentSplitterService, + EmbeddingService, + VectorDatabaseService, + EmbeddingCredentialsService, + ConnectionsStrategiesService, + VectorDbCredentialsService, + ], + exports: [ + RagService, + VectorDatabaseService, + ProcessDocumentProcessor, + S3Service, + DocumentLoaderService, + DocumentSplitterService, + EmbeddingService, + PineconeService, + WeaviateService, + TurboPufferService, + ChromaDBService, + QdrantDBService, + MilvusService, + ConnectionsStrategiesService, + VectorDbCredentialsService, + ], +}) +export class RagModule {} diff --git a/packages/api/src/@core/rag/rag.service.ts b/packages/api/src/@core/rag/rag.service.ts new file mode 100644 index 000000000..24452ee49 --- /dev/null +++ b/packages/api/src/@core/rag/rag.service.ts @@ -0,0 +1,46 @@ +import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { Injectable } from '@nestjs/common'; +import { EmbeddingService } from './embedding/embedding.service'; +import { FileInfo } from './types'; +import { VectorDatabaseService } from './vecdb/vecdb.service'; +import { S3Service } from '@@core/s3/s3.service'; + +@Injectable() +export class RagService { + constructor( + private readonly queues: BullQueueService, + private embeddingService: EmbeddingService, + private vectorDatabaseService: VectorDatabaseService, + private s3Service: S3Service, + ) {} + + async queryEmbeddings(query: string, topK = 5) { + const queryEmbedding = await this.embeddingService.embedQuery(query); + const results = await this.vectorDatabaseService.queryEmbeddings( + queryEmbedding, + topK, + ); + return results.map((match: any) => ({ + chunk: match.metadata.text, + metadata: match.metadata, + score: match.score, + embedding: match.embedding, + })); + } + + async queueDocumentProcessing( + filesInfo: FileInfo[], + projectId: string, + linkedUserId: string, + ) { + // todo: check if RAG is enabled for the current projectId and for pricing concerns + // paywall before doing s3 + rag + await this.s3Service.uploadFilesFromUrls(filesInfo, linkedUserId); + await this.queues.getRagDocumentQueue().add('batchDocs', { + filesInfo, + projectId, + linkedUserId, + }); + return { message: `Documents queued for processing` }; + } +} diff --git a/packages/api/src/@core/rag/types/index.ts b/packages/api/src/@core/rag/types/index.ts new file mode 100644 index 000000000..b54a4a832 --- /dev/null +++ b/packages/api/src/@core/rag/types/index.ts @@ -0,0 +1,11 @@ +export interface FileInfo { + id: string; + url: string; + s3Key: string; + provider: string; + fileType: string; +} +export interface ProcessedChunk { + text: string; + metadata: Record; +} diff --git a/packages/api/src/@core/rag/vecdb/chromadb/chromadb.service.ts b/packages/api/src/@core/rag/vecdb/chromadb/chromadb.service.ts new file mode 100644 index 000000000..5c088694e --- /dev/null +++ b/packages/api/src/@core/rag/vecdb/chromadb/chromadb.service.ts @@ -0,0 +1,52 @@ +import { EnvironmentService } from '@@core/@core-services/environment/environment.service'; +import { ProcessedChunk } from '@@core/rag/types'; +import { Injectable } from '@nestjs/common'; +import { ChromaClient } from 'chromadb'; + +@Injectable() +export class ChromaDBService { + private client: ChromaClient; + + constructor(private envService: EnvironmentService) { + //this.initialize(); + } + + async initialize() { + this.client = new ChromaClient({ + path: this.envService.getChromaCreds(), + }); + } + + async storeEmbeddings( + fileId: string, + chunks: ProcessedChunk[], + embeddings: number[][], + ) { + const collection = await this.client.createCollection({ name: fileId }); + await collection.add({ + ids: chunks.map((_, i) => `${fileId}_${i}`), + embeddings: embeddings, + metadatas: chunks.map((chunk) => ({ + text: chunk.text, + ...chunk.metadata, + })), + }); + } + + async queryEmbeddings(queryEmbedding: number[], topK: number) { + const collections = await this.client.listCollections(); + const results = await Promise.all( + collections.map(async (collection) => { + const collectionInstance = await this.client.getCollection({ + name: collection.name, + }); + const result = await collectionInstance.query({ + queryEmbeddings: [queryEmbedding], + nResults: topK, + }); + return result.metadatas[0]; + }), + ); + return results.flat().slice(0, topK); + } +} diff --git a/packages/api/src/@core/rag/vecdb/milvus/milvus.service.ts b/packages/api/src/@core/rag/vecdb/milvus/milvus.service.ts new file mode 100644 index 000000000..d35fafd92 --- /dev/null +++ b/packages/api/src/@core/rag/vecdb/milvus/milvus.service.ts @@ -0,0 +1,100 @@ +import { EnvironmentService } from '@@core/@core-services/environment/environment.service'; +import { ProcessedChunk } from '@@core/rag/types'; +import { Injectable } from '@nestjs/common'; +import { DataType, MilvusClient } from '@zilliz/milvus2-sdk-node'; + +@Injectable() +export class MilvusService { + private client: MilvusClient; + + constructor(private envService: EnvironmentService) { + //this.initialize(); + } + + async initialize() { + const milvus_creds = this.envService.getMilvusCreds(); + this.client = new MilvusClient({ + address: milvus_creds.address, + }); + await this.client.connectPromise; + } + + async storeEmbeddings( + fileId: string, + chunks: ProcessedChunk[], + embeddings: number[][], + ) { + const collection_name = fileId; + await this.client.createCollection({ + collection_name, + fields: [ + { + name: 'id', + description: 'ID field', + data_type: DataType.VarChar, + is_primary_key: true, + max_length: 100, + }, + { + name: 'text', + description: 'Text field', + data_type: DataType.VarChar, + max_length: 65535, + }, + { + name: 'embedding', + description: 'Vector field', + data_type: DataType.FloatVector, + dim: embeddings[0].length, + }, + ], + }); + const data = chunks.map((chunk, i) => ({ + id: `${fileId}_${i}`, + text: chunk.text, + embedding: embeddings[i], + })); + + await this.client.insert({ + collection_name, + data, + }); + + await this.client.createIndex({ + collection_name, + field_name: 'embedding', + index_type: 'HNSW', + params: { efConstruction: 10, M: 4 }, + metric_type: 'L2', + }); + + await this.client.loadCollectionSync({ + collection_name, + }); + } + + async queryEmbeddings(queryEmbedding: number[], topK: number) { + const collections = await this.client.listCollections(); + const results = await Promise.all( + collections.data.map(async (collection) => { + const res = await this.client.search({ + collection_name: collection.name, + vector: queryEmbedding, + filter: '', + params: { nprobe: 10 }, + limit: topK, + output_fields: ['text'], + }); + return res.results.map((hit) => ({ + id: hit.id, + text: hit.text, + score: hit.score, + })); + }), + ); + return results + .flat() + .sort((a, b) => b.score - a.score) + .slice(0, topK); + } +} diff --git a/packages/api/src/@core/rag/vecdb/pinecone/pinecone.service.ts b/packages/api/src/@core/rag/vecdb/pinecone/pinecone.service.ts new file mode 100644 index 000000000..76c85a306 --- /dev/null +++ b/packages/api/src/@core/rag/vecdb/pinecone/pinecone.service.ts @@ -0,0 +1,69 @@ +import { ProcessedChunk } from '@@core/rag/types'; +import { Injectable } from '@nestjs/common'; +import { Pinecone } from '@pinecone-database/pinecone'; + +@Injectable() +export class PineconeService { + private client: Pinecone; + private indexName: string; + + async initialize(credentials: string[]) { + this.client = new Pinecone({ + apiKey: credentials[0], + }); + this.indexName = credentials[1]; + } + + async storeEmbeddings( + fileId: string, + chunks: ProcessedChunk[], + embeddings: number[][], + ) { + const index = this.client.Index(this.indexName); + const vectors = chunks.map((chunk, i) => ({ + id: `${fileId}_${i}`, + values: embeddings[i], + metadata: this.sanitizeMetadata({ + text: chunk.text, + ...chunk.metadata, + }), + })); + await index.upsert(vectors); + console.log(`Inserted embeddings on Pinecone for fileId ${fileId}`); + } + private sanitizeMetadata(metadata: Record): Record { + const sanitized: Record = {}; + for (const [key, value] of Object.entries(metadata)) { + if ( + typeof value === 'string' || + typeof value === 'number' || + typeof value === 'boolean' + ) { + sanitized[key] = value; + } else if ( + Array.isArray(value) && + value.every((item) => typeof item === 'string') + ) { + sanitized[key] = value; + } else if (typeof value === 'object' && value !== null) { + sanitized[key] = JSON.stringify(value); + } + // Ignore other types + } + return sanitized; + } + + async queryEmbeddings(queryEmbedding: number[], topK: number) { + const index = this.client.Index(this.indexName); + const queryResponse = await index.query({ + vector: queryEmbedding, + topK, + includeMetadata: true, + includeValues: true, + }); + return (queryResponse.matches || []).map((match) => ({ + ...match, + embedding: match.values, + })); + } +} diff --git a/packages/api/src/@core/rag/vecdb/qdrant/qdrant.service.ts b/packages/api/src/@core/rag/vecdb/qdrant/qdrant.service.ts new file mode 100644 index 000000000..a26714db7 --- /dev/null +++ b/packages/api/src/@core/rag/vecdb/qdrant/qdrant.service.ts @@ -0,0 +1,56 @@ +import { EnvironmentService } from '@@core/@core-services/environment/environment.service'; +import { ProcessedChunk } from '@@core/rag/types'; +import { Injectable } from '@nestjs/common'; +import { QdrantClient } from '@qdrant/js-client-rest'; + +@Injectable() +export class QdrantDBService { + private client: QdrantClient; + + constructor(private envService: EnvironmentService) { + //this.initialize(); + } + + async initialize() { + const creds = this.envService.getQdrantCreds(); + this.client = new QdrantClient({ + url: `https://${creds.baseUrl}.us-east-0-1.aws.cloud.qdrant.io`, + apiKey: creds.apiKey, + }); + } + + async storeEmbeddings( + fileId: string, + chunks: ProcessedChunk[], + embeddings: number[][], + ) { + await this.client.createCollection(fileId, { + vectors: { size: embeddings[0].length, distance: 'Cosine' }, + }); + await this.client.upsert(fileId, { + wait: true, + points: chunks.map((chunk, i) => ({ + id: `${fileId}_${i}`, + vector: embeddings[i], + payload: { + text: chunk.text, + ...chunk.metadata, + }, + })), + }); + } + + async queryEmbeddings(queryEmbedding: number[], topK: number) { + const { collections } = await this.client.getCollections(); + const results = await Promise.all( + collections.map(async (collection) => { + const result = await this.client.search(collection.name, { + vector: queryEmbedding, + limit: topK, + }); + return result.map((item) => item.payload); + }), + ); + return results.flat().slice(0, topK); + } +} diff --git a/packages/api/src/@core/rag/vecdb/turbopuffer/turbopuffer.service.ts b/packages/api/src/@core/rag/vecdb/turbopuffer/turbopuffer.service.ts new file mode 100644 index 000000000..34d3ba0bd --- /dev/null +++ b/packages/api/src/@core/rag/vecdb/turbopuffer/turbopuffer.service.ts @@ -0,0 +1,48 @@ +import { EnvironmentService } from '@@core/@core-services/environment/environment.service'; +import { ProcessedChunk } from '@@core/rag/types'; +import { Injectable } from '@nestjs/common'; +import { Namespace, Turbopuffer } from '@turbopuffer/turbopuffer'; + +@Injectable() +export class TurboPufferService { + private client: Turbopuffer; + private namespace: Namespace; + + constructor(private envService: EnvironmentService) { + //this.initialize(); + } + + async initialize() { + this.client = new Turbopuffer({ + apiKey: this.envService.getTurboPufferApiKey(), + }); + this.namespace = this.client.namespace('panora-namespace'); + } + + async storeEmbeddings( + fileId: string, + chunks: ProcessedChunk[], + embeddings: number[][], + ) { + const vectors = chunks.map((chunk, i) => ({ + id: `${fileId}_${i}`, + vector: embeddings[i], + attributes: { text: chunk.text, ...chunk.metadata }, + })); + await this.namespace.upsert({ + vectors, + distance_metric: 'cosine_distance', + }); + } + + async queryEmbeddings(queryEmbedding: number[], topK: number) { + const results = await this.namespace.query({ + vector: queryEmbedding, + top_k: topK, + distance_metric: 'cosine_distance', + include_attributes: ['text'], + include_vectors: false, + }); + return results; + } +} diff --git a/packages/api/src/@core/rag/vecdb/vecdb.credentials.service.ts b/packages/api/src/@core/rag/vecdb/vecdb.credentials.service.ts new file mode 100644 index 000000000..8e97e7eee --- /dev/null +++ b/packages/api/src/@core/rag/vecdb/vecdb.credentials.service.ts @@ -0,0 +1,81 @@ +import { Injectable } from '@nestjs/common'; +import { EnvironmentService } from '@@core/@core-services/environment/environment.service'; +import { ConnectionsStrategiesService } from '@@core/connections-strategies/connections-strategies.service'; + +@Injectable() +export class VectorDbCredentialsService { + constructor( + private envService: EnvironmentService, + private connectionsStrategiesService: ConnectionsStrategiesService, + ) {} + + async getVectorDbCredentials( + projectId: string, + vectorDb: string, + ): Promise { + const type = `vector_db.${vectorDb}`; + const isCustom = + await this.connectionsStrategiesService.isCustomCredentials( + projectId, + type, + ); + + if (isCustom) { + return this.getCustomCredentials(projectId, type, vectorDb); + } else { + return this.getManagedCredentials(vectorDb); + } + } + + private async getCustomCredentials( + projectId: string, + type: string, + vectorDb: string, + ) { + const attributes = this.getAttributesForVectorDb(vectorDb); + return this.connectionsStrategiesService.getConnectionStrategyData( + projectId, + type, + attributes, + ); + } + + getManagedCredentials(vectorDb: string): string[] { + switch (vectorDb) { + case 'pinecone': + return [ + this.envService.getPineconeCreds().apiKey, + this.envService.getPineconeCreds().indexName, + ]; + case 'chromadb': + return [this.envService.getChromaCreds()]; + case 'weaviate': + const weaviateCreds = this.envService.getWeaviateCreds(); + return [weaviateCreds.apiKey, weaviateCreds.url]; + case 'turbopuffer': + return [this.envService.getTurboPufferApiKey()]; + case 'qdrant': + const qdrantCreds = this.envService.getQdrantCreds(); + return [qdrantCreds.apiKey, qdrantCreds.baseUrl]; + default: + throw new Error(`Unsupported vector database: ${vectorDb}`); + } + } + + private getAttributesForVectorDb(vectorDb: string): string[] { + switch (vectorDb) { + case 'pinecone': + return ['api_key', 'index_name']; + case 'turbopuffer': + return ['api_key']; + case 'qdrant': + return ['api_key', 'base_url']; + case 'chromadb': + return ['url']; + case 'weaviate': + return ['api_key', 'url']; + default: + throw new Error(`Unsupported vector database: ${vectorDb}`); + } + } +} diff --git a/packages/api/src/@core/rag/vecdb/vecdb.service.ts b/packages/api/src/@core/rag/vecdb/vecdb.service.ts new file mode 100644 index 000000000..1d3340e6e --- /dev/null +++ b/packages/api/src/@core/rag/vecdb/vecdb.service.ts @@ -0,0 +1,113 @@ +import { Document } from '@langchain/core/documents'; +import { Injectable, OnModuleInit } from '@nestjs/common'; +import { ProcessedChunk } from '../types'; +import { ChromaDBService } from './chromadb/chromadb.service'; +import { MilvusService } from './milvus/milvus.service'; +import { PineconeService } from './pinecone/pinecone.service'; +import { QdrantDBService } from './qdrant/qdrant.service'; +import { TurboPufferService } from './turbopuffer/turbopuffer.service'; +import { WeaviateService } from './weaviate/weaviate.service'; +import { ConnectionsStrategiesService } from '@@core/connections-strategies/connections-strategies.service'; +import { VectorDbCredentialsService } from './vecdb.credentials.service'; + +export type VectorDbProvider = + | 'CHROMADB' + | 'PINECONE' + | 'QDRANT' + | 'TURBOPUFFER' + | 'MILVUS' + | 'WEAVIATE'; + +@Injectable() +export class VectorDatabaseService implements OnModuleInit { + private vectorDb: any; + + constructor( + private pineconeService: PineconeService, + private weaviateService: WeaviateService, + private turboPufferService: TurboPufferService, + private chromaDBService: ChromaDBService, + private qdrantService: QdrantDBService, + private milvusService: MilvusService, + private connectionsStrategiesService: ConnectionsStrategiesService, + private vectorDbCredentialsService: VectorDbCredentialsService, + ) {} + + onModuleInit() { + console.log(); + } + + async init(projectId: string) { + const activeStrategies = + await this.connectionsStrategiesService.getConnectionStrategiesForProject( + projectId, + ); + const activeVectorDbStrategy = activeStrategies.find( + (strategy) => strategy.type.startsWith('vector_db.') && strategy.status, + ); + + let dbType: string; + let credentials: string[]; + + if (activeVectorDbStrategy) { + dbType = activeVectorDbStrategy.type.split('.')[1].toLowerCase(); + credentials = + await this.vectorDbCredentialsService.getVectorDbCredentials( + projectId, + dbType, + ); + } else { + // Fall back to managed credentials + dbType = 'pinecone'; + credentials = + this.vectorDbCredentialsService.getManagedCredentials(dbType); + } + switch (dbType) { + case 'pinecone': + this.vectorDb = this.pineconeService; + break; + case 'weaviate': + this.vectorDb = this.weaviateService; + break; + case 'turbopuffer': + this.vectorDb = this.turboPufferService; + break; + case 'chromadb': + this.vectorDb = this.chromaDBService; + break; + case 'qdrant': + this.vectorDb = this.qdrantService; + break; + case 'milvus': + this.vectorDb = this.milvusService; + break; + default: + throw new Error(`Unsupported vector database type: ${dbType}`); + } + + await this.vectorDb.initialize(credentials); + } + + async storeEmbeddings( + fileId: string, + chunks: Document>[], + embeddings: number[][], + projectId: string, + ) { + await this.init(projectId); + const processedChunks: ProcessedChunk[] = chunks.map((chunk) => ({ + text: chunk.pageContent, + metadata: chunk.metadata, + })); + return this.vectorDb.storeEmbeddings( + fileId, + processedChunks, + embeddings, + projectId, + ); + } + + async queryEmbeddings(queryEmbedding: number[], topK: number) { + return this.vectorDb.queryEmbeddings(queryEmbedding, topK); + } +} diff --git a/packages/api/src/@core/rag/vecdb/weaviate/weaviate.service.ts b/packages/api/src/@core/rag/vecdb/weaviate/weaviate.service.ts new file mode 100644 index 000000000..ae92bf0e5 --- /dev/null +++ b/packages/api/src/@core/rag/vecdb/weaviate/weaviate.service.ts @@ -0,0 +1,49 @@ +import { EnvironmentService } from '@@core/@core-services/environment/environment.service'; +import { ProcessedChunk } from '@@core/rag/types'; +import { Injectable } from '@nestjs/common'; +import weaviate from 'weaviate-client'; + +@Injectable() +export class WeaviateService { + private client: any; + + constructor(private envService: EnvironmentService) { + //this.initialize(); + } + + async initialize() { + const weaviate_creds = this.envService.getWeaviateCreds(); + this.client = weaviate.connectToWeaviateCloud(weaviate_creds.url, { + authCredentials: new weaviate.ApiKey(weaviate_creds.apiKey), + }); + } + + async storeEmbeddings( + fileId: string, + chunks: ProcessedChunk[], + embeddings: number[][], + ) { + const className = 'Document'; + for (let i = 0; i < chunks.length; i++) { + await this.client.data + .creator() + .withClassName(className) + .withId(`${fileId}_${i}`) + .withProperties({ text: chunks[i].text, ...chunks[i].metadata }) + .withVector(embeddings[i]) + .do(); + } + } + + async queryEmbeddings(queryEmbedding: number[], topK: number) { + const className = 'Document'; + const result = await this.client.graphql + .get() + .withClassName(className) + .withFields('text metadata') + .withNearVector({ vector: queryEmbedding }) + .withLimit(topK) + .do(); + return result.data.Get[className] || []; + } +} diff --git a/packages/api/src/@core/s3/constants.ts b/packages/api/src/@core/s3/constants.ts new file mode 100644 index 000000000..d45d01dd1 --- /dev/null +++ b/packages/api/src/@core/s3/constants.ts @@ -0,0 +1 @@ +export const BUCKET_NAME = 'panora-documents-bucket'; diff --git a/packages/api/src/@core/s3/s3.service.ts b/packages/api/src/@core/s3/s3.service.ts new file mode 100644 index 000000000..333bbc350 --- /dev/null +++ b/packages/api/src/@core/s3/s3.service.ts @@ -0,0 +1,72 @@ +import { EnvironmentService } from '@@core/@core-services/environment/environment.service'; +import { FileInfo } from '@@core/rag/types'; +import { + GetObjectCommand, + S3Client +} from '@aws-sdk/client-s3'; +import { ServiceRegistry } from '@filestorage/file/services/registry.service'; +import { IFileService } from '@filestorage/file/types'; +import { Injectable } from '@nestjs/common'; +import { Readable } from 'stream'; +import { BUCKET_NAME } from './constants'; + +@Injectable() +export class S3Service { + private s3: S3Client; + + constructor(private envService: EnvironmentService, private fileServiceRegistry: ServiceRegistry) { + // const creds = this.envService.getAwsCredentials(); + const creds = this.envService.getMinioCredentials(); + this.s3 = new S3Client({ + endpoint: 'http://minio:9000', + region: 'us-east-1', + forcePathStyle: true, + //region: creds.region, + credentials: { + accessKeyId: creds.accessKeyId, + secretAccessKey: creds.secretAccessKey, + }, + }); + } + + async uploadFilesFromUrls( + urlsWithKeys: FileInfo[], + linkedUserId: string + ): Promise { + const batchSize = 10; + for (let i = 0; i < urlsWithKeys.length; i += batchSize) { + const batch = urlsWithKeys.slice(i, i + batchSize); + await Promise.all( + batch.map(async ({ id, url, s3Key, provider }) => { + try { + const service: IFileService = this.fileServiceRegistry.getService(provider.toLowerCase().trim()); + if (!service) return; + await service.streamFileToS3( + id, + linkedUserId, + this.s3, + s3Key, + ); + console.log(`Successfully uploaded ${s3Key} from ${provider}`); + } catch (error) { + console.error( + `Failed to upload file from ${url} to ${s3Key} (${provider}):`, + error, + ); + throw error; + } + }), + ); + } + } + + async getReadStream(s3Key: string): Promise { + const getObjectParams = { + Bucket: BUCKET_NAME, + Key: s3Key, + }; + const command = new GetObjectCommand(getObjectParams); + const { Body } = await this.s3.send(command); + return Body as Readable; + } +} diff --git a/packages/api/src/@core/sync/sync.controller.ts b/packages/api/src/@core/sync/sync.controller.ts index 53f3cb11c..2b12794b0 100644 --- a/packages/api/src/@core/sync/sync.controller.ts +++ b/packages/api/src/@core/sync/sync.controller.ts @@ -1,16 +1,76 @@ import { JwtAuthGuard } from '@@core/auth/guards/jwt-auth.guard'; -import { Body, Controller, Get, Param, Post, UseGuards } from '@nestjs/common'; +import { + Body, + Controller, + Get, + Param, + Post, + UseGuards, + Request, +} from '@nestjs/common'; import { ApiOperation, ApiParam, ApiProperty, ApiResponse, ApiTags, + ApiExcludeEndpoint, } from '@nestjs/swagger'; import { LoggerService } from '../@core-services/logger/logger.service'; import { CoreSyncService } from './sync.service'; import { ApiPostCustomResponse } from '@@core/utils/dtos/openapi.respone.dto'; +import { ApiKeyAuthGuard } from '@@core/auth/guards/api-key.guard'; + +export class UpdatePullFrequencyDto { + @ApiProperty({ + type: Number, + example: 1800, + description: 'Frequency in seconds', + }) + crm?: number; + + @ApiProperty({ + type: Number, + example: 3600, + description: 'Frequency in seconds', + }) + ats?: number; + + @ApiProperty({ + type: Number, + example: 7200, + description: 'Frequency in seconds', + }) + hris?: number; + + @ApiProperty({ + type: Number, + example: 14400, + description: 'Frequency in seconds', + }) + accounting?: number; + @ApiProperty({ + type: Number, + example: 28800, + description: 'Frequency in seconds', + }) + filestorage?: number; + + @ApiProperty({ + type: Number, + example: 43200, + description: 'Frequency in seconds', + }) + ecommerce?: number; + + @ApiProperty({ + type: Number, + example: 86400, + description: 'Frequency in seconds', + }) + ticketing?: number; +} export class ResyncStatusDto { @ApiProperty({ type: Date, example: '', nullable: true }) timestamp: Date; @@ -43,6 +103,7 @@ export class ResyncStatusDto { }) status: string; } + @ApiTags('sync') @Controller('sync') export class SyncController { @@ -92,4 +153,100 @@ export class SyncController { const { vertical, provider, linkedUserId } = data; return await this.syncService.resync(vertical, provider, linkedUserId); } + + @ApiOperation({ + operationId: 'updatePullFrequency', + summary: 'Update pull frequency for verticals', + }) + @ApiResponse({ + status: 200, + description: 'Pull frequency updated successfully', + }) + @UseGuards(JwtAuthGuard) + @ApiExcludeEndpoint() + @Post('internal/pull-frequencies') + async updateInternalPullFrequency( + @Request() req: any, + @Body() data: UpdatePullFrequencyDto, + ) { + const projectId = req.user.id_project; + const result = await this.syncService.updatePullFrequency(data, projectId); + + // Convert BigInt values to numbers or strings + const serializedResult = JSON.parse( + JSON.stringify(result, (key, value) => + typeof value === 'bigint' ? value.toString() : value, + ), + ); + + return serializedResult; + } + + @ApiOperation({ + operationId: 'updatePullFrequency', + summary: 'Update pull frequency for verticals', + }) + @ApiResponse({ + status: 200, + description: 'Pull frequency updated successfully', + }) + @UseGuards(ApiKeyAuthGuard) + @Post('pull-frequencies') + async updatePullFrequency( + @Request() req: any, + @Body() data: UpdatePullFrequencyDto, + ) { + const projectId = req.user.id_project; + const result = await this.syncService.updatePullFrequency(data, projectId); + + // Convert BigInt values to numbers or strings + const serializedResult = JSON.parse( + JSON.stringify(result, (key, value) => + typeof value === 'bigint' ? value.toString() : value, + ), + ); + + return serializedResult; + } + + @ApiOperation({ + operationId: 'getPullFrequency', + summary: 'Get pull frequency for verticals', + }) + @ApiResponse({ status: 200, type: UpdatePullFrequencyDto }) + @UseGuards(JwtAuthGuard) + @ApiExcludeEndpoint() + @Get('internal/pull-frequencies') + async getInternalPullFrequency(@Request() req: any) { + const projectId = req.user.id_project; + const result = await this.syncService.getPullFrequency(projectId); + // Convert BigInt values to numbers or strings + const serializedResult = JSON.parse( + JSON.stringify(result, (key, value) => + typeof value === 'bigint' ? value.toString() : value, + ), + ); + + return serializedResult; + } + + @ApiOperation({ + operationId: 'getPullFrequency', + summary: 'Get pull frequency for verticals', + }) + @ApiResponse({ status: 200, type: UpdatePullFrequencyDto }) + @UseGuards(ApiKeyAuthGuard) + @Get('pull-frequencies') + async getPullFrequency(@Request() req: any) { + const projectId = req.user.id_project; + const result = await this.syncService.getPullFrequency(projectId); + // Convert BigInt values to numbers or strings + const serializedResult = JSON.parse( + JSON.stringify(result, (key, value) => + typeof value === 'bigint' ? value.toString() : value, + ), + ); + + return serializedResult; + } } diff --git a/packages/api/src/@core/sync/sync.module.ts b/packages/api/src/@core/sync/sync.module.ts index 7261e06d7..43446beee 100644 --- a/packages/api/src/@core/sync/sync.module.ts +++ b/packages/api/src/@core/sync/sync.module.ts @@ -19,6 +19,7 @@ import { UserModule as TUserModule } from '@ticketing/user/user.module'; import { LoggerService } from '../@core-services/logger/logger.service'; import { SyncController } from './sync.controller'; import { CoreSyncService } from './sync.service'; +import { SyncProcessor } from './sync.processor'; @Module({ imports: [ @@ -59,8 +60,9 @@ import { CoreSyncService } from './sync.service'; TicketModule, TUserModule, CoreSyncService, + SyncProcessor, ], - providers: [CoreSyncService, LoggerService], + providers: [CoreSyncService, LoggerService, SyncProcessor], controllers: [SyncController], }) export class SyncModule {} diff --git a/packages/api/src/@core/sync/sync.processor.ts b/packages/api/src/@core/sync/sync.processor.ts new file mode 100644 index 000000000..5e8681a11 --- /dev/null +++ b/packages/api/src/@core/sync/sync.processor.ts @@ -0,0 +1,47 @@ +import { Processor, Process } from '@nestjs/bull'; +import { Job } from 'bull'; +import { Injectable, Logger } from '@nestjs/common'; +import { Queues } from '@@core/@core-services/queues/types'; +import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; + +@Injectable() +@Processor(Queues.SYNC_JOBS_WORKER) +export class SyncProcessor { + private readonly logger = new Logger(SyncProcessor.name); + + constructor(private registry: CoreSyncRegistry) {} + + @Process('*') + async handleSyncJob(job: Job) { + const { projectId, vertical, commonObject } = job.data; + this.logger.log( + `Starting to process job ${job.id} for ${vertical} ${commonObject} (Project: ${projectId})`, + ); + + try { + const service = this.registry.getService(vertical, commonObject); + if (!service) { + throw new Error( + `No service found for vertical ${vertical} and common object ${commonObject}`, + ); + } + + await service.kickstartSync(projectId); + this.logger.log( + `Successfully processed job ${job.id} for ${vertical} ${commonObject} (Project: ${projectId})`, + ); + } catch (error) { + this.logger.error( + `Error processing job ${job.id} for ${vertical} ${commonObject} (Project: ${projectId})`, + error.stack, + ); + throw error; // Re-throw the error to mark the job as failed + } + } + + @Process('health-check') + async healthCheck(job: Job) { + this.logger.log(`Health check job ${job.id} received`); + return { status: 'OK', timestamp: new Date() }; + } +} diff --git a/packages/api/src/@core/sync/sync.service.ts b/packages/api/src/@core/sync/sync.service.ts index f3b870096..3681361f5 100644 --- a/packages/api/src/@core/sync/sync.service.ts +++ b/packages/api/src/@core/sync/sync.service.ts @@ -1,19 +1,161 @@ +import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; +import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { ENGAGEMENTS_TYPE } from '@crm/@lib/@types'; import { Injectable } from '@nestjs/common'; +import { Cron, CronExpression } from '@nestjs/schedule'; +import { ConnectorCategory, getCommonObjectsForVertical } from '@panora/shared'; import { LoggerService } from '../@core-services/logger/logger.service'; -import { ConnectorCategory } from '@panora/shared'; -import { ENGAGEMENTS_TYPE } from '@crm/@lib/@types'; -import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; import { CoreSyncRegistry } from '../@core-services/registries/core-sync.registry'; +import { UpdatePullFrequencyDto } from './sync.controller'; +import { v4 as uuidv4 } from 'uuid'; + @Injectable() export class CoreSyncService { constructor( private logger: LoggerService, private prisma: PrismaService, private registry: CoreSyncRegistry, + private bullQueueService: BullQueueService, ) { this.logger.setContext(CoreSyncService.name); } + @Cron(CronExpression.EVERY_30_SECONDS) + async checkAndKickstartSync(user_id?: string) { + const users = user_id + ? [ + await this.prisma.users.findUnique({ + where: { + id_user: user_id, + }, + }), + ] + : await this.prisma.users.findMany(); + if (users && users.length > 0) { + for (const user of users) { + const projects = await this.prisma.projects.findMany({ + where: { + id_user: user.id_user, + }, + }); + for (const project of projects) { + const projectSyncConfig = + await this.prisma.projects_pull_frequency.findFirst({ + where: { + id_project: project.id_project, + }, + }); + + if (projectSyncConfig) { + const syncIntervals = { + crm: projectSyncConfig.crm, + ats: projectSyncConfig.ats, + hris: projectSyncConfig.hris, + accounting: projectSyncConfig.accounting, + filestorage: projectSyncConfig.filestorage, + ecommerce: projectSyncConfig.ecommerce, + ticketing: projectSyncConfig.ticketing, + }; + + for (const [vertical, interval] of Object.entries(syncIntervals)) { + const now = new Date(); + const lastSyncEvent = await this.prisma.events.findFirst({ + where: { + id_project: project.id_project, + type: `${vertical}.batchSyncStart`, + }, + orderBy: { + timestamp: 'desc', + }, + }); + + const lastSyncTime = lastSyncEvent + ? lastSyncEvent.timestamp + : new Date(0); + + const hoursSinceLastSync = + (now.getTime() - lastSyncTime.getTime()) / (1000 * 60 * 60); + if (interval && hoursSinceLastSync >= interval) { + await this.prisma.events.create({ + data: { + id_project: project.id_project, + id_event: uuidv4(), + status: 'success', + type: `${vertical}.batchSyncStart`, + method: 'GET', + url: '', + provider: '', + direction: '0', + timestamp: new Date(), + }, + }); + const commonObjects = getCommonObjectsForVertical(vertical); + for (const commonObject of commonObjects) { + const service = this.registry.getService( + vertical, + commonObject, + ); + if (service) { + try { + const cronExpression = this.convertIntervalToCron( + Number(interval), + ); + + await this.bullQueueService.queueSyncJob( + `${vertical}-sync-${commonObject}s`, + { + projectId: project.id_project, + vertical, + commonObject, + }, + cronExpression, + ); + this.logger.log( + `Synced ${vertical}.${commonObject} for project ${project.id_project}`, + ); + } catch (error) { + this.logger.error( + `Error syncing ${vertical}.${commonObject} for project ${project.id_project}: ${error.message}`, + error, + ); + } + } else { + this.logger.warn( + `No service found for ${vertical}.${commonObject}`, + ); + } + } + } + } + } + } + } + } + } + + private convertIntervalToCron(intervalSeconds: number): string { + // If the interval is less than 1 minute, we'll set it to run every minute + if (intervalSeconds < 60) { + return '* * * * *'; + } + + // If the interval is less than 1 hour, use minutes + if (intervalSeconds < 3600) { + const minutes = Math.floor(intervalSeconds / 60); + return `*/${minutes} * * * *`; + } + + // If the interval is less than 1 day, use hours + if (intervalSeconds < 86400) { + const hours = Math.floor(intervalSeconds / 3600); + return `0 */${hours} * * *`; + } + + // For intervals of 1 day or more, use days + const days = Math.floor(intervalSeconds / 86400); + return `0 0 */${days} * *`; + } + //Initial sync which will execute when connection is successfully established async initialSync(vertical: string, provider: string, linkedUserId: string) { try { @@ -392,10 +534,19 @@ export class CoreSyncService { linkedUserId: linkedUserId, }), ]; + if (provider == 'googledrive') { + tasks.push(() => + this.registry.getService('filestorage', 'file').syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUserId, + }), + ); + } for (const task of tasks) { try { await task(); } catch (error) { + console.log(error); this.logger.error(`File Storage Task failed: ${error.message}`, error); } } @@ -412,20 +563,23 @@ export class CoreSyncService { id_connection: connection.id_connection, }, }); - const filesTasks = folders.map( - (folder) => async () => - this.registry.getService('filestorage', 'file').syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUserId, - id_folder: folder.id_fs_folder, - }), - ); + if (provider !== 'googledrive') { + const filesTasks = folders.map( + (folder) => async () => + this.registry.getService('filestorage', 'file').syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUserId, + id_folder: folder.id_fs_folder, + }), + ); - for (const task of filesTasks) { - try { - await task(); - } catch (error) { - this.logger.error(`File Task failed: ${error.message}`, error); + for (const task of filesTasks) { + try { + await task(); + } catch (error) { + console.log(error); + this.logger.error(`File Task failed: ${error.message}`, error); + } } } } @@ -599,4 +753,25 @@ export class CoreSyncService { throw error; } } + + async updatePullFrequency(data: UpdatePullFrequencyDto, projectId: string) { + return await this.prisma.projects_pull_frequency.upsert({ + where: { id_project: projectId }, + update: { + ...data, + modified_at: new Date(), + }, + create: { + id_projects_pull_frequency: uuidv4(), + id_project: projectId, + ...data, + }, + }); + } + + async getPullFrequency(projectId: string) { + return await this.prisma.projects_pull_frequency.findFirst({ + where: { id_project: projectId }, + }); + } } diff --git a/packages/api/src/@core/utils/types/index.ts b/packages/api/src/@core/utils/types/index.ts index 2fc84e729..c1ec43c0b 100644 --- a/packages/api/src/@core/utils/types/index.ts +++ b/packages/api/src/@core/utils/types/index.ts @@ -56,6 +56,22 @@ export function getFileExtension(fileName: string): string | null { return null; } +export function getFileExtensionFromMimeType( + mimeType: string, +): string | undefined { + try { + const normalizedMimeType = mimeType.toLowerCase(); + for (const [extension, mime] of Object.entries(MIME_TYPES)) { + if (mime.toLowerCase() === normalizedMimeType) { + return extension.startsWith('.') ? extension.slice(1) : extension; + } + } + return undefined; + } catch (error) { + throw error; + } +} + export const MIME_TYPES: { [key: string]: string } = { '.aac': 'audio/aac', '.abw': 'application/x-abiword', @@ -96,6 +112,7 @@ export const MIME_TYPES: { [key: string]: string } = { '.mp4': 'video/mp4', '.mpeg': 'video/mpeg', '.mpkg': 'application/vnd.apple.installer+xml', + '.md': 'text/markdown', '.odp': 'application/vnd.oasis.opendocument.presentation', '.ods': 'application/vnd.oasis.opendocument.spreadsheet', '.odt': 'application/vnd.oasis.opendocument.text', diff --git a/packages/api/src/@core/utils/types/original/original.crm.ts b/packages/api/src/@core/utils/types/original/original.crm.ts index 5258c1532..5297d2fca 100644 --- a/packages/api/src/@core/utils/types/original/original.crm.ts +++ b/packages/api/src/@core/utils/types/original/original.crm.ts @@ -154,11 +154,13 @@ import { ZendeskUserInput, ZendeskUserOutput, } from '@ticketing/user/services/zendesk/types'; -import { AffinityCompanyInput, AffinityCompanyOutput } from '@crm/company/services/affinity/types'; -import { AffinityDealInput, AffinityDealOutput } from '@crm/deal/services/affinity/types'; -import { AffinityNoteInput, AffinityNoteOutput } from '@crm/note/services/affinity/types'; -import { AffinityUserInput, AffinityUserOutput } from '@crm/user/services/affinity/types'; -import { AffinityContactInput, AffinityContactOutput } from '@crm/contact/services/affinity/types'; + +import { SalesforceContactInput, SalesforceContactOutput } from '@crm/contact/services/salesforce/types'; +import { SalesforceDealInput, SalesforceDealOutput } from '@crm/deal/services/salesforce/types'; +import { SalesforceCompanyInput, SalesforceCompanyOutput } from '@crm/company/services/salesforce/types'; +import { SalesforceNoteInput, SalesforceNoteOutput } from '@crm/note/services/salesforce/types'; +import { SalesforceTaskInput } from '@crm/task/services/salesforce/types'; +import { SalesforceUserInput, SalesforceUserOutput } from '@crm/user/services/salesforce/types'; /* INPUT */ @@ -170,7 +172,9 @@ export type OriginalContactInput = | ZendeskContactInput | PipedriveContactInput | AttioContactInput - | CloseContactInput | MicrosoftdynamicssalesContactInput; + | CloseContactInput + | MicrosoftdynamicssalesContactInput + | SalesforceContactInput; /* deal */ export type OriginalDealInput = @@ -180,7 +184,9 @@ export type OriginalDealInput = | ZendeskDealOutput | PipedriveDealOutput | CloseDealOutput - | AttioDealInput | MicrosoftdynamicssalesDealInput; + | AttioDealInput + | MicrosoftdynamicssalesDealInput + | SalesforceDealInput /* company */ export type OriginalCompanyInput = @@ -190,7 +196,7 @@ export type OriginalCompanyInput = | ZendeskCompanyOutput | PipedriveCompanyOutput | AttioCompanyOutput - | CloseCompanyOutput | MicrosoftdynamicssalesCompanyInput; + | CloseCompanyOutput | MicrosoftdynamicssalesCompanyInput | SalesforceCompanyInput /* engagement */ export type OriginalEngagementInput = @@ -208,7 +214,7 @@ export type OriginalNoteInput = | ZendeskNoteInput | PipedriveNoteInput | CloseNoteInput - | AttioNoteInput | MicrosoftdynamicssalesNoteInput; + | AttioNoteInput | MicrosoftdynamicssalesNoteInput | SalesforceNoteInput; /* task */ export type OriginalTaskInput = @@ -217,7 +223,7 @@ export type OriginalTaskInput = | ZendeskTaskInput | PipedriveTaskInput | CloseTaskInput - | AttioTaskInput | MicrosoftdynamicssalesTaskInput; + | AttioTaskInput | MicrosoftdynamicssalesTaskInput | SalesforceTaskInput; /* stage */ export type OriginalStageInput = @@ -236,7 +242,7 @@ export type OriginalUserInput = | ZohoUserInput | ZendeskUserInput | PipedriveUserInput - | CloseUserOutput | MicrosoftdynamicssalesUserInput; + | CloseUserOutput | MicrosoftdynamicssalesUserInput | SalesforceUserInput export type CrmObjectInput = | OriginalContactInput @@ -257,7 +263,7 @@ export type OriginalContactOutput = | ZendeskContactOutput | PipedriveContactOutput | AttioContactOutput - | CloseContactOutput | MicrosoftdynamicssalesContactOutput; + | CloseContactOutput | MicrosoftdynamicssalesContactOutput | SalesforceContactOutput; /* deal */ export type OriginalDealOutput = @@ -267,7 +273,7 @@ export type OriginalDealOutput = | ZendeskDealOutput | PipedriveDealOutput | CloseDealOutput - | AttioDealOutput | MicrosoftdynamicssalesDealOutput; + | AttioDealOutput | MicrosoftdynamicssalesDealOutput | SalesforceDealOutput; /* company */ export type OriginalCompanyOutput = @@ -277,7 +283,7 @@ export type OriginalCompanyOutput = | ZendeskCompanyOutput | PipedriveCompanyOutput | AttioCompanyOutput - | CloseCompanyOutput | MicrosoftdynamicssalesCompanyOutput; + | CloseCompanyOutput | MicrosoftdynamicssalesCompanyOutput | SalesforceCompanyOutput; /* engagement */ export type OriginalEngagementOutput = @@ -295,7 +301,7 @@ export type OriginalNoteOutput = | ZendeskNoteOutput | PipedriveNoteOutput | CloseNoteOutput - | AttioNoteOutput | MicrosoftdynamicssalesNoteOutput; + | AttioNoteOutput | MicrosoftdynamicssalesNoteOutput | SalesforceNoteOutput; /* task */ export type OriginalTaskOutput = @@ -324,7 +330,7 @@ export type OriginalUserOutput = | ZendeskUserOutput | PipedriveUserOutput | CloseUserInput - | AttioUserOutput | MicrosoftdynamicssalesUserOutput; + | AttioUserOutput | MicrosoftdynamicssalesUserOutput| SalesforceUserOutput; export type CrmObjectOutput = | OriginalContactOutput diff --git a/packages/api/src/@core/utils/types/original/original.file-storage.ts b/packages/api/src/@core/utils/types/original/original.file-storage.ts index 725bf27a2..6a4c200d6 100644 --- a/packages/api/src/@core/utils/types/original/original.file-storage.ts +++ b/packages/api/src/@core/utils/types/original/original.file-storage.ts @@ -1,5 +1,100 @@ +import { + DropboxGroupInput, + DropboxGroupOutput, +} from '@filestorage/group/services/dropbox/types'; + +import { + DropboxUserInput, + DropboxUserOutput, +} from '@filestorage/user/services/dropbox/types'; + +import { + DropboxFileInput, + DropboxFileOutput, +} from '@filestorage/file/services/dropbox/types'; + +import { + DropboxFolderInput, + DropboxFolderOutput, +} from '@filestorage/folder/services/dropbox/types'; + +import { + BoxSharedLinkInput, + BoxSharedLinkOutput, +} from '@filestorage/sharedlink/services/box/types'; + /* INPUT */ +import { + OnedriveSharedLinkInput, + OnedriveSharedLinkOutput, +} from '@filestorage/sharedlink/services/onedrive/types'; + +import { + OnedrivePermissionInput, + OnedrivePermissionOutput, +} from '@filestorage/permission/services/onedrive/types'; + +import { + OnedriveGroupInput, + OnedriveGroupOutput, +} from '@filestorage/group/services/onedrive/types'; + +import { + OnedriveUserInput, + OnedriveUserOutput, +} from '@filestorage/user/services/onedrive/types'; + +import { + OnedriveFileInput, + OnedriveFileOutput, +} from '@filestorage/file/services/onedrive/types'; + +import { + OnedriveFolderInput, + OnedriveFolderOutput, +} from '@filestorage/folder/services/onedrive/types'; + +import { + OnedriveDriveInput, + OnedriveDriveOutput, +} from '@filestorage/drive/services/onedrive/types'; + +import { + SharepointSharedLinkInput, + SharepointSharedLinkOutput, +} from '@filestorage/sharedlink/services/sharepoint/types'; + +import { + SharepointPermissionInput, + SharepointPermissionOutput, +} from '@filestorage/permission/services/sharepoint/types'; + +import { + SharepointGroupInput, + SharepointGroupOutput, +} from '@filestorage/group/services/sharepoint/types'; + +import { + SharepointUserInput, + SharepointUserOutput, +} from '@filestorage/user/services/sharepoint/types'; + +import { + SharepointFolderInput, + SharepointFolderOutput, +} from '@filestorage/folder/services/sharepoint/types'; + +import { + SharepointFileInput, + SharepointFileOutput, +} from '@filestorage/file/services/sharepoint/types'; + +import { + SharepointDriveInput, + SharepointDriveOutput, +} from '@filestorage/drive/services/sharepoint/types'; + import { BoxFileInput, BoxFileOutput, @@ -16,27 +111,68 @@ import { BoxUserInput, BoxUserOutput, } from '@filestorage/user/services/box/types'; - +import { + GoogleDriveDriveInput, + GoogleDriveDriveOutput, +} from '@filestorage/drive/services/googledrive/types'; +import { + GoogleDriveFileInput, + GoogleDriveFileOutput, +} from '@filestorage/file/services/googledrive/types'; +import { + GoogleDriveFolderInput, + GoogleDriveFolderOutput, +} from '@filestorage/folder/services/googledrive/types'; /* file */ -export type OriginalFileInput = BoxFileInput; /* folder */ -export type OriginalFolderInput = BoxFolderInput; +export type OriginalFileInput = + | BoxFileInput + | OnedriveFileInput + | SharepointFileInput + | DropboxFileInput + | SharepointFileInput + | GoogleDriveFileInput; + +/* folder */ +export type OriginalFolderInput = + | BoxFolderInput + | OnedriveFolderInput + | SharepointFolderInput + | DropboxFolderInput + | SharepointFolderInput + | GoogleDriveFolderInput; /* permission */ -export type OriginalPermissionInput = any; +export type OriginalPermissionInput = + | any + | OnedrivePermissionInput + | SharepointPermissionInput; /* shared link */ export type OriginalSharedLinkInput = any; /* drive */ -export type OriginalDriveInput = any; +export type OriginalDriveInput = + | GoogleDriveDriveInput + | OnedriveDriveInput + | SharepointDriveInput; /* group */ -export type OriginalGroupInput = BoxGroupInput; /* user */ -export type OriginalUserInput = BoxUserInput; +export type OriginalGroupInput = + | BoxGroupInput + | OnedriveGroupInput + | SharepointGroupInput + | DropboxGroupInput; + +/* user */ +export type OriginalUserInput = + | BoxUserInput + | OnedriveUserInput + | SharepointUserInput + | DropboxUserInput; export type FileStorageObjectInput = | OriginalFileInput @@ -50,25 +186,55 @@ export type FileStorageObjectInput = /* OUTPUT */ /* file */ -export type OriginalFileOutput = BoxFileOutput; /* folder */ -export type OriginalFolderOutput = BoxFolderOutput; +export type OriginalFileOutput = + | BoxFileOutput + | OnedriveFileOutput + | SharepointFileOutput + | DropboxFileOutput + | SharepointFileOutput + | GoogleDriveFileOutput; + +/* folder */ +export type OriginalFolderOutput = + | BoxFolderOutput + | OnedriveFolderOutput + | SharepointFolderOutput + | DropboxFolderOutput + | SharepointFolderOutput + | GoogleDriveFolderOutput; /* permission */ -export type OriginalPermissionOutput = any; +export type OriginalPermissionOutput = + | any + | OnedrivePermissionOutput + | SharepointPermissionOutput; /* shared link */ export type OriginalSharedLinkOutput = any; /* drive */ -export type OriginalDriveOutput = any; +export type OriginalDriveOutput = + | GoogleDriveDriveOutput + | OnedriveDriveOutput + | SharepointDriveOutput; /* group */ -export type OriginalGroupOutput = BoxGroupOutput; /* user */ -export type OriginalUserOutput = BoxUserOutput; +export type OriginalGroupOutput = + | BoxGroupOutput + | OnedriveGroupOutput + | SharepointGroupOutput + | DropboxGroupOutput; + +/* user */ +export type OriginalUserOutput = + | BoxUserOutput + | OnedriveUserOutput + | SharepointUserOutput + | DropboxUserOutput; export type FileStorageObjectOutput = | OriginalFileOutput @@ -78,3 +244,13 @@ export type FileStorageObjectOutput = | OriginalDriveOutput | OriginalGroupOutput | OriginalUserOutput; + +export type OriginalSharedlinkInput = + | BoxSharedLinkInput + | OnedriveSharedLinkInput + | SharepointSharedLinkInput; + +export type OriginalSharedlinkOutput = + | BoxSharedLinkOutput + | OnedriveSharedLinkOutput + | SharepointSharedLinkOutput; diff --git a/packages/api/src/accounting/account/sync/sync.service.ts b/packages/api/src/accounting/account/sync/sync.service.ts index 1f6584b6b..41ff4eb61 100644 --- a/packages/api/src/accounting/account/sync/sync.service.ts +++ b/packages/api/src/accounting/account/sync/sync.service.ts @@ -32,39 +32,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('accounting', 'account', this); } - - async onModuleInit() { - // Initialization logic if needed + onModuleInit() { + console.log(''); } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing accounting accounts...'); - const users = user_id - ? [await this.prisma.users.findUnique({ where: { id_user: user_id } })] - : await this.prisma.users.findMany(); - - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { id_user: user.id_user }, - }); - for (const project of projects) { - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { id_project: project.id_project }, - }); - for (const linkedUser of linkedUsers) { - for (const provider of ACCOUNTING_PROVIDERS) { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ACCOUNTING_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/accounting/address/sync/sync.service.ts b/packages/api/src/accounting/address/sync/sync.service.ts index 85a590437..c907a90cd 100644 --- a/packages/api/src/accounting/address/sync/sync.service.ts +++ b/packages/api/src/accounting/address/sync/sync.service.ts @@ -32,44 +32,39 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('accounting', 'address', this); } - - async onModuleInit() { - // Initialization logic if needed + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing accounting addresses...'); - const users = user_id - ? [await this.prisma.users.findUnique({ where: { id_user: user_id } })] - : await this.prisma.users.findMany(); - - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { id_user: user.id_user }, - }); - for (const project of projects) { - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { id_project: project.id_project }, - }); - for (const linkedUser of linkedUsers) { - for (const provider of ACCOUNTING_PROVIDERS) { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ACCOUNTING_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } } - async syncForLinkedUser(param: SyncLinkedUserType) { try { const { integrationId, linkedUserId } = param; diff --git a/packages/api/src/accounting/attachment/sync/sync.service.ts b/packages/api/src/accounting/attachment/sync/sync.service.ts index a19b71b99..4dcd0b7ff 100644 --- a/packages/api/src/accounting/attachment/sync/sync.service.ts +++ b/packages/api/src/accounting/attachment/sync/sync.service.ts @@ -32,39 +32,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('accounting', 'attachment', this); } - - async onModuleInit() { - // Initialization logic if needed + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing accounting attachments...'); - const users = user_id - ? [await this.prisma.users.findUnique({ where: { id_user: user_id } })] - : await this.prisma.users.findMany(); - - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { id_user: user.id_user }, - }); - for (const project of projects) { - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { id_project: project.id_project }, - }); - for (const linkedUser of linkedUsers) { - for (const provider of ACCOUNTING_PROVIDERS) { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ACCOUNTING_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/accounting/balancesheet/sync/sync.service.ts b/packages/api/src/accounting/balancesheet/sync/sync.service.ts index 8ed1ba82d..5d094a32f 100644 --- a/packages/api/src/accounting/balancesheet/sync/sync.service.ts +++ b/packages/api/src/accounting/balancesheet/sync/sync.service.ts @@ -33,44 +33,39 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('accounting', 'balancesheet', this); } - - async onModuleInit() { - // Initialization logic if needed + onModuleInit() { +// } - @Cron('0 */12 * * *') // every 12 hours - async kickstartSync(user_id?: string) { + @Cron('0 */8 * * *') // every 8 hours + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing accounting balance sheets...'); - const users = user_id - ? [await this.prisma.users.findUnique({ where: { id_user: user_id } })] - : await this.prisma.users.findMany(); - - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { id_user: user.id_user }, - }); - for (const project of projects) { - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { id_project: project.id_project }, - }); - for (const linkedUser of linkedUsers) { - for (const provider of ACCOUNTING_PROVIDERS) { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ACCOUNTING_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } } - async syncForLinkedUser(param: SyncLinkedUserType) { try { const { integrationId, linkedUserId } = param; diff --git a/packages/api/src/accounting/cashflowstatement/sync/sync.service.ts b/packages/api/src/accounting/cashflowstatement/sync/sync.service.ts index a33b883de..af3df7dfd 100644 --- a/packages/api/src/accounting/cashflowstatement/sync/sync.service.ts +++ b/packages/api/src/accounting/cashflowstatement/sync/sync.service.ts @@ -33,46 +33,41 @@ export class SyncService implements OnModuleInit, IBaseSync { private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); - this.registry.registerService('accounting', 'cashflow_statement', this); + this.registry.registerService('accounting', 'cashflowstatement', this); } - - async onModuleInit() { - // Initialization logic if needed + onModuleInit() { + // } - @Cron('0 */12 * * *') // every 12 hours - async kickstartSync(user_id?: string) { + @Cron('0 */8 * * *') // every 8 hours + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing accounting cash flow statements...'); - const users = user_id - ? [await this.prisma.users.findUnique({ where: { id_user: user_id } })] - : await this.prisma.users.findMany(); - - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { id_user: user.id_user }, - }); - for (const project of projects) { - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { id_project: project.id_project }, - }); - for (const linkedUser of linkedUsers) { - for (const provider of ACCOUNTING_PROVIDERS) { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ACCOUNTING_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } } - async syncForLinkedUser(param: SyncLinkedUserType) { try { const { integrationId, linkedUserId } = param; diff --git a/packages/api/src/accounting/companyinfo/sync/sync.service.ts b/packages/api/src/accounting/companyinfo/sync/sync.service.ts index 5a96908fc..b1dc21902 100644 --- a/packages/api/src/accounting/companyinfo/sync/sync.service.ts +++ b/packages/api/src/accounting/companyinfo/sync/sync.service.ts @@ -30,41 +30,37 @@ export class SyncService implements OnModuleInit, IBaseSync { private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); - this.registry.registerService('accounting', 'company_info', this); + this.registry.registerService('accounting', 'companyinfo', this); } - - async onModuleInit() { - // Initialization logic if needed + onModuleInit() { + // } - @Cron('0 */12 * * *') // every 12 hours - async kickstartSync(user_id?: string) { + @Cron('0 */8 * * *') // every 8 hours + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing accounting company info...'); - const users = user_id - ? [await this.prisma.users.findUnique({ where: { id_user: user_id } })] - : await this.prisma.users.findMany(); - - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { id_user: user.id_user }, - }); - for (const project of projects) { - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { id_project: project.id_project }, - }); - for (const linkedUser of linkedUsers) { - for (const provider of ACCOUNTING_PROVIDERS) { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ACCOUNTING_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/accounting/contact/sync/sync.service.ts b/packages/api/src/accounting/contact/sync/sync.service.ts index f3e6a1919..89a24f8c2 100644 --- a/packages/api/src/accounting/contact/sync/sync.service.ts +++ b/packages/api/src/accounting/contact/sync/sync.service.ts @@ -32,39 +32,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('accounting', 'contact', this); } - - async onModuleInit() { - // Initialization logic if needed + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing accounting contacts...'); - const users = user_id - ? [await this.prisma.users.findUnique({ where: { id_user: user_id } })] - : await this.prisma.users.findMany(); - - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { id_user: user.id_user }, - }); - for (const project of projects) { - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { id_project: project.id_project }, - }); - for (const linkedUser of linkedUsers) { - for (const provider of ACCOUNTING_PROVIDERS) { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ACCOUNTING_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/accounting/creditnote/sync/sync.service.ts b/packages/api/src/accounting/creditnote/sync/sync.service.ts index e26428023..a160c9cdc 100644 --- a/packages/api/src/accounting/creditnote/sync/sync.service.ts +++ b/packages/api/src/accounting/creditnote/sync/sync.service.ts @@ -30,41 +30,37 @@ export class SyncService implements OnModuleInit, IBaseSync { private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); - this.registry.registerService('accounting', 'credit_note', this); + this.registry.registerService('accounting', 'creditnote', this); } - - async onModuleInit() { - // Initialization logic if needed + onModuleInit() { + // } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing accounting credit notes...'); - const users = user_id - ? [await this.prisma.users.findUnique({ where: { id_user: user_id } })] - : await this.prisma.users.findMany(); - - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { id_user: user.id_user }, - }); - for (const project of projects) { - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { id_project: project.id_project }, - }); - for (const linkedUser of linkedUsers) { - for (const provider of ACCOUNTING_PROVIDERS) { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ACCOUNTING_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/accounting/expense/sync/sync.service.ts b/packages/api/src/accounting/expense/sync/sync.service.ts index b302d6eb4..7de268b48 100644 --- a/packages/api/src/accounting/expense/sync/sync.service.ts +++ b/packages/api/src/accounting/expense/sync/sync.service.ts @@ -35,39 +35,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('accounting', 'expense', this); } - - async onModuleInit() { - // Initialization logic if needed + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing accounting expenses...'); - const users = user_id - ? [await this.prisma.users.findUnique({ where: { id_user: user_id } })] - : await this.prisma.users.findMany(); - - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { id_user: user.id_user }, - }); - for (const project of projects) { - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { id_project: project.id_project }, - }); - for (const linkedUser of linkedUsers) { - for (const provider of ACCOUNTING_PROVIDERS) { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ACCOUNTING_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/accounting/incomestatement/sync/sync.service.ts b/packages/api/src/accounting/incomestatement/sync/sync.service.ts index 3e4415bb7..0d8ea0ab8 100644 --- a/packages/api/src/accounting/incomestatement/sync/sync.service.ts +++ b/packages/api/src/accounting/incomestatement/sync/sync.service.ts @@ -30,41 +30,37 @@ export class SyncService implements OnModuleInit, IBaseSync { private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); - this.registry.registerService('accounting', 'income_statement', this); + this.registry.registerService('accounting', 'incomestatement', this); } - - async onModuleInit() { - // Initialization logic if needed + onModuleInit() { + // } - @Cron('0 */12 * * *') // every 12 hours - async kickstartSync(user_id?: string) { + @Cron('0 */8 * * *') // every 8 hours + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing accounting income statements...'); - const users = user_id - ? [await this.prisma.users.findUnique({ where: { id_user: user_id } })] - : await this.prisma.users.findMany(); - - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { id_user: user.id_user }, - }); - for (const project of projects) { - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { id_project: project.id_project }, - }); - for (const linkedUser of linkedUsers) { - for (const provider of ACCOUNTING_PROVIDERS) { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ACCOUNTING_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/accounting/invoice/sync/sync.service.ts b/packages/api/src/accounting/invoice/sync/sync.service.ts index c3fa05443..870f10f73 100644 --- a/packages/api/src/accounting/invoice/sync/sync.service.ts +++ b/packages/api/src/accounting/invoice/sync/sync.service.ts @@ -35,39 +35,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('accounting', 'invoice', this); } - - async onModuleInit() { - // Initialization logic if needed + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing accounting invoices...'); - const users = user_id - ? [await this.prisma.users.findUnique({ where: { id_user: user_id } })] - : await this.prisma.users.findMany(); - - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { id_user: user.id_user }, - }); - for (const project of projects) { - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { id_project: project.id_project }, - }); - for (const linkedUser of linkedUsers) { - for (const provider of ACCOUNTING_PROVIDERS) { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ACCOUNTING_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/accounting/item/sync/sync.service.ts b/packages/api/src/accounting/item/sync/sync.service.ts index a6df008da..adbe12a19 100644 --- a/packages/api/src/accounting/item/sync/sync.service.ts +++ b/packages/api/src/accounting/item/sync/sync.service.ts @@ -32,44 +32,39 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('accounting', 'item', this); } - - async onModuleInit() { - // Initialization logic if needed + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing accounting items...'); - const users = user_id - ? [await this.prisma.users.findUnique({ where: { id_user: user_id } })] - : await this.prisma.users.findMany(); - - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { id_user: user.id_user }, - }); - for (const project of projects) { - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { id_project: project.id_project }, - }); - for (const linkedUser of linkedUsers) { - for (const provider of ACCOUNTING_PROVIDERS) { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ACCOUNTING_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } } - async syncForLinkedUser(param: SyncLinkedUserType) { try { const { integrationId, linkedUserId } = param; diff --git a/packages/api/src/accounting/journalentry/sync/sync.service.ts b/packages/api/src/accounting/journalentry/sync/sync.service.ts index 8c5da919b..939cabd96 100644 --- a/packages/api/src/accounting/journalentry/sync/sync.service.ts +++ b/packages/api/src/accounting/journalentry/sync/sync.service.ts @@ -33,41 +33,37 @@ export class SyncService implements OnModuleInit, IBaseSync { private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); - this.registry.registerService('accounting', 'journal_entry', this); + this.registry.registerService('accounting', 'journalentry', this); } - - async onModuleInit() { - // Initialization logic if needed + onModuleInit() { + // } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing accounting journal entries...'); - const users = user_id - ? [await this.prisma.users.findUnique({ where: { id_user: user_id } })] - : await this.prisma.users.findMany(); - - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { id_user: user.id_user }, - }); - for (const project of projects) { - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { id_project: project.id_project }, - }); - for (const linkedUser of linkedUsers) { - for (const provider of ACCOUNTING_PROVIDERS) { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ACCOUNTING_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/accounting/payment/sync/sync.service.ts b/packages/api/src/accounting/payment/sync/sync.service.ts index cf1da5205..02aaa02ed 100644 --- a/packages/api/src/accounting/payment/sync/sync.service.ts +++ b/packages/api/src/accounting/payment/sync/sync.service.ts @@ -35,39 +35,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('accounting', 'payment', this); } - - async onModuleInit() { - // Initialization logic if needed + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing accounting payments...'); - const users = user_id - ? [await this.prisma.users.findUnique({ where: { id_user: user_id } })] - : await this.prisma.users.findMany(); - - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { id_user: user.id_user }, - }); - for (const project of projects) { - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { id_project: project.id_project }, - }); - for (const linkedUser of linkedUsers) { - for (const provider of ACCOUNTING_PROVIDERS) { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ACCOUNTING_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/accounting/phonenumber/sync/sync.service.ts b/packages/api/src/accounting/phonenumber/sync/sync.service.ts index 2d1e73807..4605cce9b 100644 --- a/packages/api/src/accounting/phonenumber/sync/sync.service.ts +++ b/packages/api/src/accounting/phonenumber/sync/sync.service.ts @@ -30,41 +30,37 @@ export class SyncService implements OnModuleInit, IBaseSync { private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); - this.registry.registerService('accounting', 'phone_number', this); + this.registry.registerService('accounting', 'phonenumber', this); } - - async onModuleInit() { - // Initialization logic if needed + onModuleInit() { + // } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing accounting phone numbers...'); - const users = user_id - ? [await this.prisma.users.findUnique({ where: { id_user: user_id } })] - : await this.prisma.users.findMany(); - - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { id_user: user.id_user }, - }); - for (const project of projects) { - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { id_project: project.id_project }, - }); - for (const linkedUser of linkedUsers) { - for (const provider of ACCOUNTING_PROVIDERS) { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ACCOUNTING_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/accounting/purchaseorder/sync/sync.service.ts b/packages/api/src/accounting/purchaseorder/sync/sync.service.ts index f5e75eeb3..289df1b22 100644 --- a/packages/api/src/accounting/purchaseorder/sync/sync.service.ts +++ b/packages/api/src/accounting/purchaseorder/sync/sync.service.ts @@ -33,41 +33,37 @@ export class SyncService implements OnModuleInit, IBaseSync { private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); - this.registry.registerService('accounting', 'purchase_order', this); + this.registry.registerService('accounting', 'purchaseorder', this); } - - async onModuleInit() { - // Initialization logic if needed + onModuleInit() { + // } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing accounting purchase orders...'); - const users = user_id - ? [await this.prisma.users.findUnique({ where: { id_user: user_id } })] - : await this.prisma.users.findMany(); - - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { id_user: user.id_user }, - }); - for (const project of projects) { - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { id_project: project.id_project }, - }); - for (const linkedUser of linkedUsers) { - for (const provider of ACCOUNTING_PROVIDERS) { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ACCOUNTING_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/accounting/taxrate/sync/sync.service.ts b/packages/api/src/accounting/taxrate/sync/sync.service.ts index 9c737358d..aebb95e01 100644 --- a/packages/api/src/accounting/taxrate/sync/sync.service.ts +++ b/packages/api/src/accounting/taxrate/sync/sync.service.ts @@ -30,41 +30,37 @@ export class SyncService implements OnModuleInit, IBaseSync { private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); - this.registry.registerService('accounting', 'tax_rate', this); + this.registry.registerService('accounting', 'taxrate', this); } - - async onModuleInit() { - // Initialization logic if needed + onModuleInit() { + // } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing accounting tax rates...'); - const users = user_id - ? [await this.prisma.users.findUnique({ where: { id_user: user_id } })] - : await this.prisma.users.findMany(); - - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { id_user: user.id_user }, - }); - for (const project of projects) { - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { id_project: project.id_project }, - }); - for (const linkedUser of linkedUsers) { - for (const provider of ACCOUNTING_PROVIDERS) { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ACCOUNTING_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/accounting/trackingcategory/sync/sync.service.ts b/packages/api/src/accounting/trackingcategory/sync/sync.service.ts index 9ddc21bea..82382253f 100644 --- a/packages/api/src/accounting/trackingcategory/sync/sync.service.ts +++ b/packages/api/src/accounting/trackingcategory/sync/sync.service.ts @@ -30,41 +30,37 @@ export class SyncService implements OnModuleInit, IBaseSync { private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); - this.registry.registerService('accounting', 'tracking_category', this); + this.registry.registerService('accounting', 'trackingcategory', this); } - - async onModuleInit() { - // Initialization logic if needed + onModuleInit() { + // } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing accounting tracking categories...'); - const users = user_id - ? [await this.prisma.users.findUnique({ where: { id_user: user_id } })] - : await this.prisma.users.findMany(); - - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { id_user: user.id_user }, - }); - for (const project of projects) { - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { id_project: project.id_project }, - }); - for (const linkedUser of linkedUsers) { - for (const provider of ACCOUNTING_PROVIDERS) { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ACCOUNTING_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/accounting/transaction/sync/sync.service.ts b/packages/api/src/accounting/transaction/sync/sync.service.ts index 13f6d92a4..763e1b876 100644 --- a/packages/api/src/accounting/transaction/sync/sync.service.ts +++ b/packages/api/src/accounting/transaction/sync/sync.service.ts @@ -35,39 +35,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('accounting', 'transaction', this); } - - async onModuleInit() { - // Initialization logic if needed + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing accounting transactions...'); - const users = user_id - ? [await this.prisma.users.findUnique({ where: { id_user: user_id } })] - : await this.prisma.users.findMany(); - - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { id_user: user.id_user }, - }); - for (const project of projects) { - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { id_project: project.id_project }, - }); - for (const linkedUser of linkedUsers) { - for (const provider of ACCOUNTING_PROVIDERS) { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ACCOUNTING_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/accounting/vendorcredit/sync/sync.service.ts b/packages/api/src/accounting/vendorcredit/sync/sync.service.ts index c55697a27..0007ce1c2 100644 --- a/packages/api/src/accounting/vendorcredit/sync/sync.service.ts +++ b/packages/api/src/accounting/vendorcredit/sync/sync.service.ts @@ -33,41 +33,37 @@ export class SyncService implements OnModuleInit, IBaseSync { private ingestService: IngestDataService, ) { this.logger.setContext(SyncService.name); - this.registry.registerService('accounting', 'vendor_credit', this); + this.registry.registerService('accounting', 'vendorcredit', this); } - - async onModuleInit() { - // Initialization logic if needed + onModuleInit() { + // } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing accounting vendor credits...'); - const users = user_id - ? [await this.prisma.users.findUnique({ where: { id_user: user_id } })] - : await this.prisma.users.findMany(); - - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { id_user: user.id_user }, - }); - for (const project of projects) { - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { id_project: project.id_project }, - }); - for (const linkedUser of linkedUsers) { - for (const provider of ACCOUNTING_PROVIDERS) { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ACCOUNTING_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/ats/activity/sync/sync.processor.ts b/packages/api/src/ats/activity/sync/sync.processor.ts deleted file mode 100644 index f5acd700f..000000000 --- a/packages/api/src/ats/activity/sync/sync.processor.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - @Process('ats-sync-activities') - async handleSyncCompanies(job: Job) { - try { - console.log(`Processing queue -> ats-sync-activities ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing ats activities', error); - } - } -} diff --git a/packages/api/src/ats/activity/sync/sync.service.ts b/packages/api/src/ats/activity/sync/sync.service.ts index 00a77af1e..917345ad3 100644 --- a/packages/api/src/ats/activity/sync/sync.service.ts +++ b/packages/api/src/ats/activity/sync/sync.service.ts @@ -35,84 +35,38 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'activity', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob( - 'ats-sync-activities', - '0 0 * * *', - ); - } catch (error) { - throw error; - } + onModuleInit() { +// } //function used by sync worker which populate our ats_activities table //its role is to fetch all activities from providers 3rd parties and save the info inside our db // @Cron('*/2 * * * *') // every 2 minutes (for testing) @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log(`Syncing activities....`); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = ATS_PROVIDERS; - for (const provider of providers) { - try { - const connection = await this.prisma.connections.findFirst({ - where: { - id_linked_user: linkedUser.id_linked_user, - provider_slug: provider.toLowerCase(), - }, - }); - //call the sync comments for every candidate of the linkedUser (an acitivty is tied to a candidate) - const candidates = - await this.prisma.ats_candidates.findMany({ - where: { - id_connection: connection?.id_connection, - }, - }); - for (const candidate of candidates) { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - id_candidate: candidate.id_ats_candidate, - }); - } - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } @@ -125,7 +79,9 @@ export class SyncService implements OnModuleInit, IBaseSync { const service: IActivityService = this.serviceRegistry.getService(integrationId); if (!service) { - this.logger.log(`No service found in {vertical:ats, commonObject: activity} for integration ID: ${integrationId}`); + this.logger.log( + `No service found in {vertical:ats, commonObject: activity} for integration ID: ${integrationId}`, + ); return; } @@ -167,14 +123,12 @@ export class SyncService implements OnModuleInit, IBaseSync { where: { subject: activity.subject, id_ats_candidate: activity.candidate_id, - }, }); } else { existingActivity = await this.prisma.ats_activities.findFirst({ where: { remote_id: originId, - }, }); } @@ -204,7 +158,6 @@ export class SyncService implements OnModuleInit, IBaseSync { id_ats_activity: uuidv4(), created_at: new Date(), remote_id: originId, - }, }); } diff --git a/packages/api/src/ats/application/sync/sync.processor.ts b/packages/api/src/ats/application/sync/sync.processor.ts deleted file mode 100644 index c9b20e465..000000000 --- a/packages/api/src/ats/application/sync/sync.processor.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - - @Process('ats-sync-applications') - async handleSyncApplications(job: Job) { - try { - console.log(`Processing queue -> ats-sync-applications ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing ats applications', error); - } - } -} diff --git a/packages/api/src/ats/application/sync/sync.service.ts b/packages/api/src/ats/application/sync/sync.service.ts index dfa4436f6..3d538f775 100644 --- a/packages/api/src/ats/application/sync/sync.service.ts +++ b/packages/api/src/ats/application/sync/sync.service.ts @@ -1,15 +1,13 @@ import { LoggerService } from '@@core/@core-services/logger/logger.service'; import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; import { BullQueueService } from '@@core/@core-services/queues/shared.service'; -import { IBaseSync, SyncLinkedUserType } from '@@core/utils/types/interface'; -import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; -import { ApiResponse } from '@@core/utils/types'; +import { IBaseSync, SyncLinkedUserType } from '@@core/utils/types/interface'; import { OriginalApplicationOutput } from '@@core/utils/types/original/original.ats'; -import { AtsObject } from '@ats/@lib/@types'; import { Injectable, OnModuleInit } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; import { ATS_PROVIDERS } from '@panora/shared'; @@ -35,65 +33,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'application', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob( - 'ats-sync-applications', - '0 0 * * *', - ); - } catch (error) { - throw error; - } + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing applications...'); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = ATS_PROVIDERS; - for (const provider of providers) { - try { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } @@ -105,7 +73,9 @@ export class SyncService implements OnModuleInit, IBaseSync { const service: IApplicationService = this.serviceRegistry.getService(integrationId); if (!service) { - this.logger.log(`No service found in {vertical:ats, commonObject: application} for integration ID: ${integrationId}`); + this.logger.log( + `No service found in {vertical:ats, commonObject: application} for integration ID: ${integrationId}`, + ); return; } @@ -137,7 +107,6 @@ export class SyncService implements OnModuleInit, IBaseSync { await this.prisma.ats_applications.findFirst({ where: { remote_id: originId, - }, }); @@ -169,7 +138,6 @@ export class SyncService implements OnModuleInit, IBaseSync { id_ats_application: uuidv4(), created_at: new Date(), remote_id: originId, - }, }); } diff --git a/packages/api/src/ats/attachment/sync/sync.service.ts b/packages/api/src/ats/attachment/sync/sync.service.ts index 70e16f312..078932b4a 100644 --- a/packages/api/src/ats/attachment/sync/sync.service.ts +++ b/packages/api/src/ats/attachment/sync/sync.service.ts @@ -35,16 +35,8 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'attachment', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob( - 'ats-sync-attachments', - '0 0 * * *', - ); - } catch (error) { - throw error; - } + onModuleInit() { +// } // it is synced within candidate sync @@ -71,7 +63,6 @@ export class SyncService implements OnModuleInit, IBaseSync { where: { file_name: attachment.file_name ?? null, file_url: attachment.file_url ?? null, - }, }); } else { @@ -79,7 +70,6 @@ export class SyncService implements OnModuleInit, IBaseSync { await this.prisma.ats_candidate_attachments.findFirst({ where: { remote_id: originId, - }, }); } @@ -110,7 +100,6 @@ export class SyncService implements OnModuleInit, IBaseSync { id_ats_attachment: uuidv4(), created_at: new Date(), remote_id: originId, - }, }); } diff --git a/packages/api/src/ats/candidate/sync/sync.processor.ts b/packages/api/src/ats/candidate/sync/sync.processor.ts deleted file mode 100644 index e52ca93d5..000000000 --- a/packages/api/src/ats/candidate/sync/sync.processor.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - - @Process('ats-sync-candidates') - async handleSyncCandidates(job: Job) { - try { - console.log(`Processing queue -> ats-sync-candidates ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing ats candidates', error); - } - } -} diff --git a/packages/api/src/ats/candidate/sync/sync.service.ts b/packages/api/src/ats/candidate/sync/sync.service.ts index 733f2a156..b708e070d 100644 --- a/packages/api/src/ats/candidate/sync/sync.service.ts +++ b/packages/api/src/ats/candidate/sync/sync.service.ts @@ -39,65 +39,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'candidate', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob( - 'ats-sync-candidates', - '0 0 * * *', - ); - } catch (error) { - throw error; - } + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing candidates...'); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = ATS_PROVIDERS; - for (const provider of providers) { - try { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } @@ -109,7 +79,9 @@ export class SyncService implements OnModuleInit, IBaseSync { const service: ICandidateService = this.serviceRegistry.getService(integrationId); if (!service) { - this.logger.log(`No service found in {vertical:ats, commonObject: candidate} for integration ID: ${integrationId}`); + this.logger.log( + `No service found in {vertical:ats, commonObject: candidate} for integration ID: ${integrationId}`, + ); return; } @@ -249,7 +221,6 @@ export class SyncService implements OnModuleInit, IBaseSync { const existingCandidate = await this.prisma.ats_candidates.findFirst({ where: { remote_id: originId, - }, }); @@ -289,7 +260,6 @@ export class SyncService implements OnModuleInit, IBaseSync { id_ats_candidate: uuidv4(), created_at: new Date(), remote_id: originId, - }, }); } diff --git a/packages/api/src/ats/department/sync/sync.processor.ts b/packages/api/src/ats/department/sync/sync.processor.ts deleted file mode 100644 index 4c01a02f0..000000000 --- a/packages/api/src/ats/department/sync/sync.processor.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - - @Process('ats-sync-departments') - async handleSyncDepartments(job: Job) { - try { - console.log(`Processing queue -> ats-sync-departments ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing ats departments', error); - } - } -} diff --git a/packages/api/src/ats/department/sync/sync.service.ts b/packages/api/src/ats/department/sync/sync.service.ts index 363cb1b2a..3ae858155 100644 --- a/packages/api/src/ats/department/sync/sync.service.ts +++ b/packages/api/src/ats/department/sync/sync.service.ts @@ -35,77 +35,48 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'department', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob( - 'ats-sync-departments', - '0 0 * * *', - ); - } catch (error) { - throw error; - } + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing departments...'); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = ATS_PROVIDERS; - for (const provider of providers) { - try { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } } - async syncForLinkedUser(param: SyncLinkedUserType) { try { const { integrationId, linkedUserId } = param; const service: IDepartmentService = this.serviceRegistry.getService(integrationId); if (!service) { - this.logger.log(`No service found in {vertical:ats, commonObject: department} for integration ID: ${integrationId}`); + this.logger.log( + `No service found in {vertical:ats, commonObject: department} for integration ID: ${integrationId}`, + ); return; } @@ -138,14 +109,12 @@ export class SyncService implements OnModuleInit, IBaseSync { existingDepartment = await this.prisma.ats_departments.findFirst({ where: { name: department.name, - }, }); } else { existingDepartment = await this.prisma.ats_departments.findFirst({ where: { remote_id: originId, - }, }); } @@ -169,7 +138,6 @@ export class SyncService implements OnModuleInit, IBaseSync { id_ats_department: uuidv4(), created_at: new Date(), remote_id: originId, - }, }); } diff --git a/packages/api/src/ats/eeocs/sync/sync.processor.ts b/packages/api/src/ats/eeocs/sync/sync.processor.ts deleted file mode 100644 index 1da3bafe1..000000000 --- a/packages/api/src/ats/eeocs/sync/sync.processor.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - - @Process('ats-sync-eeocs') - async handleSyncEeocs(job: Job) { - try { - console.log(`Processing queue -> ats-sync-eeocs ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing ats eeocs', error); - } - } -} diff --git a/packages/api/src/ats/eeocs/sync/sync.service.ts b/packages/api/src/ats/eeocs/sync/sync.service.ts index 9d41c1665..deb189c3d 100644 --- a/packages/api/src/ats/eeocs/sync/sync.service.ts +++ b/packages/api/src/ats/eeocs/sync/sync.service.ts @@ -35,62 +35,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'eeocs', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob('ats-sync-eeocs', '0 0 * * *'); - } catch (error) { - throw error; - } + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing EEOCs...'); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = ATS_PROVIDERS; - for (const provider of providers) { - try { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } @@ -102,7 +75,9 @@ export class SyncService implements OnModuleInit, IBaseSync { const service: IEeocsService = this.serviceRegistry.getService(integrationId); if (!service) { - this.logger.log(`No service found in {vertical:ats, commonObject: eeocs} for integration ID: ${integrationId}`); + this.logger.log( + `No service found in {vertical:ats, commonObject: eeocs} for integration ID: ${integrationId}`, + ); return; } @@ -133,7 +108,6 @@ export class SyncService implements OnModuleInit, IBaseSync { const existingEeoc = await this.prisma.ats_eeocs.findFirst({ where: { remote_id: originId, - }, }); @@ -161,7 +135,6 @@ export class SyncService implements OnModuleInit, IBaseSync { id_ats_eeoc: uuidv4(), created_at: new Date(), remote_id: originId, - }, }); } diff --git a/packages/api/src/ats/interview/sync/sync.processor.ts b/packages/api/src/ats/interview/sync/sync.processor.ts deleted file mode 100644 index c832b5366..000000000 --- a/packages/api/src/ats/interview/sync/sync.processor.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - - @Process('ats-sync-interviews') - async handleSyncInterviews(job: Job) { - try { - console.log(`Processing queue -> ats-sync-interviews ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing ats interviews', error); - } - } -} diff --git a/packages/api/src/ats/interview/sync/sync.service.ts b/packages/api/src/ats/interview/sync/sync.service.ts index 16fcda0fb..930077f1d 100644 --- a/packages/api/src/ats/interview/sync/sync.service.ts +++ b/packages/api/src/ats/interview/sync/sync.service.ts @@ -34,65 +34,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'interview', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob( - 'ats-sync-interviews', - '0 0 * * *', - ); - } catch (error) { - throw error; - } + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing interviews...'); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = ATS_PROVIDERS; - for (const provider of providers) { - try { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } @@ -104,7 +74,9 @@ export class SyncService implements OnModuleInit, IBaseSync { const service: IInterviewService = this.serviceRegistry.getService(integrationId); if (!service) { - this.logger.log(`No service found in {vertical:ats, commonObject: interview} for integration ID: ${integrationId}`); + this.logger.log( + `No service found in {vertical:ats, commonObject: interview} for integration ID: ${integrationId}`, + ); return; } @@ -135,7 +107,6 @@ export class SyncService implements OnModuleInit, IBaseSync { const existingInterview = await this.prisma.ats_interviews.findFirst({ where: { remote_id: originId, - }, }); @@ -167,7 +138,6 @@ export class SyncService implements OnModuleInit, IBaseSync { id_ats_interview: uuidv4(), created_at: new Date(), remote_id: originId, - }, }); } diff --git a/packages/api/src/ats/job/sync/sync.processor.ts b/packages/api/src/ats/job/sync/sync.processor.ts deleted file mode 100644 index 8813dbf14..000000000 --- a/packages/api/src/ats/job/sync/sync.processor.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - - @Process('ats-sync-jobs') - async handleSyncJobs(job: Job) { - try { - console.log(`Processing queue -> ats-sync-jobs ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing ats jobs', error); - } - } -} diff --git a/packages/api/src/ats/job/sync/sync.service.ts b/packages/api/src/ats/job/sync/sync.service.ts index 6c2b0a5a1..eb5c8859d 100644 --- a/packages/api/src/ats/job/sync/sync.service.ts +++ b/packages/api/src/ats/job/sync/sync.service.ts @@ -30,62 +30,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'job', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob('ats-sync-jobs', '0 0 * * *'); - } catch (error) { - throw error; - } + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing jobs...'); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = ATS_PROVIDERS; - for (const provider of providers) { - try { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/ats/jobinterviewstage/sync/sync.processor.ts b/packages/api/src/ats/jobinterviewstage/sync/sync.processor.ts deleted file mode 100644 index 4d0d8bdb3..000000000 --- a/packages/api/src/ats/jobinterviewstage/sync/sync.processor.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - - @Process('ats-sync-job-interview-stages') - async handleSyncJobInterviewStages(job: Job) { - try { - console.log( - `Processing queue -> ats-sync-job-interview-stages ${job.id}`, - ); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing ats job interview stages', error); - } - } -} diff --git a/packages/api/src/ats/jobinterviewstage/sync/sync.service.ts b/packages/api/src/ats/jobinterviewstage/sync/sync.service.ts index 1f407f18d..77949f2ae 100644 --- a/packages/api/src/ats/jobinterviewstage/sync/sync.service.ts +++ b/packages/api/src/ats/jobinterviewstage/sync/sync.service.ts @@ -35,65 +35,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'jobinterviewstage', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob( - 'ats-sync-jobinterviewstage', - '0 0 * * *', - ); - } catch (error) { - throw error; - } + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing job interview stages...'); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = ATS_PROVIDERS; - for (const provider of providers) { - try { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } @@ -105,7 +75,9 @@ export class SyncService implements OnModuleInit, IBaseSync { const service: IJobInterviewStageService = this.serviceRegistry.getService(integrationId); if (!service) { - this.logger.log(`No service found in {vertical:ats, commonObject: jobinterviewstage} for integration ID: ${integrationId}`); + this.logger.log( + `No service found in {vertical:ats, commonObject: jobinterviewstage} for integration ID: ${integrationId}`, + ); return; } @@ -137,7 +109,6 @@ export class SyncService implements OnModuleInit, IBaseSync { await this.prisma.ats_job_interview_stages.findFirst({ where: { remote_id: originId, - }, }); @@ -163,7 +134,6 @@ export class SyncService implements OnModuleInit, IBaseSync { id_ats_job_interview_stage: uuidv4(), created_at: new Date(), remote_id: originId, - }, }); } diff --git a/packages/api/src/ats/offer/sync/sync.processor.ts b/packages/api/src/ats/offer/sync/sync.processor.ts deleted file mode 100644 index 04053af2a..000000000 --- a/packages/api/src/ats/offer/sync/sync.processor.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - - @Process('ats-sync-offers') - async handleSyncOffers(job: Job) { - try { - console.log(`Processing queue -> ats-sync-offers ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing ats offers', error); - } - } -} diff --git a/packages/api/src/ats/offer/sync/sync.service.ts b/packages/api/src/ats/offer/sync/sync.service.ts index 064319ef8..5cf0ce4c9 100644 --- a/packages/api/src/ats/offer/sync/sync.service.ts +++ b/packages/api/src/ats/offer/sync/sync.service.ts @@ -35,74 +35,48 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'offer', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob('ats-sync-offers', '0 0 * * *'); - } catch (error) { - throw error; - } + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing offers...'); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = ATS_PROVIDERS; - for (const provider of providers) { - try { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } } - async syncForLinkedUser(param: SyncLinkedUserType) { try { const { integrationId, linkedUserId } = param; const service: IOfferService = this.serviceRegistry.getService(integrationId); if (!service) { - this.logger.log(`No service found in {vertical:ats, commonObject: offer} for integration ID: ${integrationId}`); + this.logger.log( + `No service found in {vertical:ats, commonObject: offer} for integration ID: ${integrationId}`, + ); return; } @@ -133,7 +107,6 @@ export class SyncService implements OnModuleInit, IBaseSync { const existingOffer = await this.prisma.ats_offers.findFirst({ where: { remote_id: originId, - }, }); @@ -163,7 +136,6 @@ export class SyncService implements OnModuleInit, IBaseSync { created_at: new Date(), id_linked_user: linkedUserId, remote_id: originId, - }, }); } diff --git a/packages/api/src/ats/office/sync/sync.processor.ts b/packages/api/src/ats/office/sync/sync.processor.ts deleted file mode 100644 index dd46b564b..000000000 --- a/packages/api/src/ats/office/sync/sync.processor.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - - @Process('ats-sync-offices') - async handleSyncOffices(job: Job) { - try { - console.log(`Processing queue -> ats-sync-offices ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing ats offices', error); - } - } -} diff --git a/packages/api/src/ats/office/sync/sync.service.ts b/packages/api/src/ats/office/sync/sync.service.ts index 69d1b3885..e58b6b3fe 100644 --- a/packages/api/src/ats/office/sync/sync.service.ts +++ b/packages/api/src/ats/office/sync/sync.service.ts @@ -35,62 +35,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'office', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob('ats-sync-offices', '0 0 * * *'); - } catch (error) { - throw error; - } + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing offices...'); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = ATS_PROVIDERS; - for (const provider of providers) { - try { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } @@ -102,7 +75,9 @@ export class SyncService implements OnModuleInit, IBaseSync { const service: IOfficeService = this.serviceRegistry.getService(integrationId); if (!service) { - this.logger.log(`No service found in {vertical:ats, commonObject: office} for integration ID: ${integrationId}`); + this.logger.log( + `No service found in {vertical:ats, commonObject: office} for integration ID: ${integrationId}`, + ); return; } @@ -135,14 +110,12 @@ export class SyncService implements OnModuleInit, IBaseSync { existingOffice = await this.prisma.ats_offices.findFirst({ where: { name: office.name, - }, }); } else { existingOffice = await this.prisma.ats_offices.findFirst({ where: { remote_id: originId, - }, }); } @@ -167,7 +140,6 @@ export class SyncService implements OnModuleInit, IBaseSync { id_ats_office: uuidv4(), created_at: new Date(), remote_id: originId, - }, }); } diff --git a/packages/api/src/ats/rejectreason/sync/sync.processor.ts b/packages/api/src/ats/rejectreason/sync/sync.processor.ts deleted file mode 100644 index 006f6c398..000000000 --- a/packages/api/src/ats/rejectreason/sync/sync.processor.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - - @Process('ats-sync-reject-reasons') - async handleSyncRejectReasons(job: Job) { - try { - console.log(`Processing queue -> ats-sync-reject-reasons ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing ats reject reasons', error); - } - } -} diff --git a/packages/api/src/ats/rejectreason/sync/sync.service.ts b/packages/api/src/ats/rejectreason/sync/sync.service.ts index b1089f54b..155550f9f 100644 --- a/packages/api/src/ats/rejectreason/sync/sync.service.ts +++ b/packages/api/src/ats/rejectreason/sync/sync.service.ts @@ -34,65 +34,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'rejectreason', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob( - 'ats-sync-rejectreason', - '0 0 * * *', - ); - } catch (error) { - throw error; - } + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing reject reasons...'); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = ATS_PROVIDERS; - for (const provider of providers) { - try { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } @@ -104,7 +74,9 @@ export class SyncService implements OnModuleInit, IBaseSync { const service: IRejectReasonService = this.serviceRegistry.getService(integrationId); if (!service) { - this.logger.log(`No service found in {vertical:ats, commonObject: rejectreason} for integration ID: ${integrationId}`); + this.logger.log( + `No service found in {vertical:ats, commonObject: rejectreason} for integration ID: ${integrationId}`, + ); return; } @@ -138,7 +110,6 @@ export class SyncService implements OnModuleInit, IBaseSync { { where: { name: rejectReason.name, - }, }, ); @@ -147,7 +118,6 @@ export class SyncService implements OnModuleInit, IBaseSync { { where: { remote_id: originId, - }, }, ); @@ -172,7 +142,6 @@ export class SyncService implements OnModuleInit, IBaseSync { id_ats_reject_reason: uuidv4(), created_at: new Date(), remote_id: originId, - }, }); } diff --git a/packages/api/src/ats/scorecard/sync/sync.processor.ts b/packages/api/src/ats/scorecard/sync/sync.processor.ts deleted file mode 100644 index 1947d0e94..000000000 --- a/packages/api/src/ats/scorecard/sync/sync.processor.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - - @Process('ats-sync-score-cards') - async handleSyncScoreCards(job: Job) { - try { - console.log(`Processing queue -> ats-sync-score-cards ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing ats score cards', error); - } - } -} diff --git a/packages/api/src/ats/scorecard/sync/sync.service.ts b/packages/api/src/ats/scorecard/sync/sync.service.ts index a7a0d83d0..d86af665e 100644 --- a/packages/api/src/ats/scorecard/sync/sync.service.ts +++ b/packages/api/src/ats/scorecard/sync/sync.service.ts @@ -35,65 +35,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'scorecard', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob( - 'ats-sync-scorecards', - '0 0 * * *', - ); - } catch (error) { - throw error; - } + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing score cards...'); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = ATS_PROVIDERS; - for (const provider of providers) { - try { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } @@ -105,7 +75,9 @@ export class SyncService implements OnModuleInit, IBaseSync { const service: IScoreCardService = this.serviceRegistry.getService(integrationId); if (!service) { - this.logger.log(`No service found in {vertical:ats, commonObject: scorecard} for integration ID: ${integrationId}`); + this.logger.log( + `No service found in {vertical:ats, commonObject: scorecard} for integration ID: ${integrationId}`, + ); return; } @@ -139,14 +111,12 @@ export class SyncService implements OnModuleInit, IBaseSync { where: { overall_recommendation: scoreCard.overall_recommendation, id_ats_application: scoreCard.application_id, - }, }); } else { existingScoreCard = await this.prisma.ats_scorecards.findFirst({ where: { remote_id: originId, - }, }); } @@ -174,7 +144,6 @@ export class SyncService implements OnModuleInit, IBaseSync { id_ats_scorecard: uuidv4(), created_at: new Date(), remote_id: originId, - }, }); } diff --git a/packages/api/src/ats/tag/sync/sync.processor.ts b/packages/api/src/ats/tag/sync/sync.processor.ts deleted file mode 100644 index 3f4498d08..000000000 --- a/packages/api/src/ats/tag/sync/sync.processor.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - - @Process('ats-sync-tags') - async handleSyncTags(job: Job) { - try { - console.log(`Processing queue -> ats-sync-tags ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing ats tags', error); - } - } -} diff --git a/packages/api/src/ats/tag/sync/sync.service.ts b/packages/api/src/ats/tag/sync/sync.service.ts index d8d62b1fe..eba9351ff 100644 --- a/packages/api/src/ats/tag/sync/sync.service.ts +++ b/packages/api/src/ats/tag/sync/sync.service.ts @@ -11,7 +11,7 @@ import { IBaseSync, SyncLinkedUserType } from '@@core/utils/types/interface'; import { OriginalTagOutput } from '@@core/utils/types/original/original.ats'; import { Injectable, OnModuleInit } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; -import { FILESTORAGE_PROVIDERS } from '@panora/shared'; +import { ATS_PROVIDERS, FILESTORAGE_PROVIDERS } from '@panora/shared'; import { ats_candidate_tags as AtsTag } from '@prisma/client'; import { v4 as uuidv4 } from 'uuid'; import { ServiceRegistry } from '../services/registry.service'; @@ -34,62 +34,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'tag', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob('ats-sync-tags', '0 0 * * *'); - } catch (error) { - throw error; - } + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing tags...'); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = FILESTORAGE_PROVIDERS; - for (const provider of providers) { - try { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } @@ -101,7 +74,9 @@ export class SyncService implements OnModuleInit, IBaseSync { const service: ITagService = this.serviceRegistry.getService(integrationId); if (!service) { - this.logger.log(`No service found in {vertical:ats, commonObject: tag} for integration ID: ${integrationId}`); + this.logger.log( + `No service found in {vertical:ats, commonObject: tag} for integration ID: ${integrationId}`, + ); return; } @@ -134,14 +109,12 @@ export class SyncService implements OnModuleInit, IBaseSync { existingTag = await this.prisma.ats_candidate_tags.findFirst({ where: { name: tag.name, - }, }); } else { existingTag = await this.prisma.ats_candidate_tags.findFirst({ where: { remote_id: originId, - }, }); } @@ -165,7 +138,6 @@ export class SyncService implements OnModuleInit, IBaseSync { id_ats_candidate_tag: uuidv4(), created_at: new Date(), remote_id: originId, - }, }); } diff --git a/packages/api/src/ats/user/sync/sync.processor.ts b/packages/api/src/ats/user/sync/sync.processor.ts deleted file mode 100644 index caf2a310e..000000000 --- a/packages/api/src/ats/user/sync/sync.processor.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - - @Process('ats-sync-users') - async handleSyncUsers(job: Job) { - try { - console.log(`Processing queue -> ats-sync-users ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing ats users', error); - } - } -} diff --git a/packages/api/src/ats/user/sync/sync.service.ts b/packages/api/src/ats/user/sync/sync.service.ts index 8d2061c85..abd89aabf 100644 --- a/packages/api/src/ats/user/sync/sync.service.ts +++ b/packages/api/src/ats/user/sync/sync.service.ts @@ -35,62 +35,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('ats', 'user', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob('ats-sync-users', '0 0 * * *'); - } catch (error) { - throw error; - } + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing users...'); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = ATS_PROVIDERS; - for (const provider of providers) { - try { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ATS_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } @@ -102,7 +75,9 @@ export class SyncService implements OnModuleInit, IBaseSync { const service: IUserService = this.serviceRegistry.getService(integrationId); if (!service) { - this.logger.log(`No service found in {vertical:ats, commonObject: user} for integration ID: ${integrationId}`); + this.logger.log( + `No service found in {vertical:ats, commonObject: user} for integration ID: ${integrationId}`, + ); return; } @@ -133,7 +108,6 @@ export class SyncService implements OnModuleInit, IBaseSync { const existingUser = await this.prisma.ats_users.findFirst({ where: { remote_id: originId, - }, }); @@ -162,7 +136,6 @@ export class SyncService implements OnModuleInit, IBaseSync { id_ats_user: uuidv4(), created_at: new Date(), remote_id: originId, - }, }); } diff --git a/packages/api/src/crm/company/company.module.ts b/packages/api/src/crm/company/company.module.ts index 568eee7aa..b61a8755d 100644 --- a/packages/api/src/crm/company/company.module.ts +++ b/packages/api/src/crm/company/company.module.ts @@ -1,15 +1,5 @@ -import { AffinityCompanyMapper } from './services/affinity/mappers'; -import { AffinityService } from './services/affinity'; -import { MicrosoftdynamicssalesCompanyMapper } from './services/microsoftdynamicssales/mappers'; -import { MicrosoftdynamicssalesService } from './services/microsoftdynamicssales'; -import { EncryptionService } from '@@core/@core-services/encryption/encryption.service'; -import { LoggerService } from '@@core/@core-services/logger/logger.service'; -import { BullQueueModule } from '@@core/@core-services/queues/queue.module'; -import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; -import { ConnectionUtils } from '@@core/connections/@utils'; -import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { Utils } from '@crm/@lib/@utils'; import { Module } from '@nestjs/common'; import { CompanyController } from './company.controller'; @@ -20,33 +10,34 @@ import { CloseCompanyMapper } from './services/close/mappers'; import { CompanyService } from './services/company.service'; import { HubspotService } from './services/hubspot'; import { HubspotCompanyMapper } from './services/hubspot/mappers'; +import { MicrosoftdynamicssalesService } from './services/microsoftdynamicssales'; +import { MicrosoftdynamicssalesCompanyMapper } from './services/microsoftdynamicssales/mappers'; import { PipedriveService } from './services/pipedrive'; import { PipedriveCompanyMapper } from './services/pipedrive/mappers'; import { ServiceRegistry } from './services/registry.service'; +import { SalesforceService } from './services/salesforce'; import { ZendeskService } from './services/zendesk'; import { ZendeskCompanyMapper } from './services/zendesk/mappers'; import { ZohoService } from './services/zoho'; import { ZohoCompanyMapper } from './services/zoho/mappers'; import { SyncService } from './sync/sync.service'; +import { SalesforceCompanyMapper } from './services/salesforce/mappers'; @Module({ controllers: [CompanyController], providers: [ CompanyService, - SyncService, WebhookService, - ServiceRegistry, - Utils, IngestDataService, - /* PROVIDERS SERVICES */ ZendeskService, ZohoService, PipedriveService, HubspotService, + SalesforceService, AttioService, CloseService, @@ -55,6 +46,7 @@ import { SyncService } from './sync/sync.service'; CloseCompanyMapper, HubspotCompanyMapper, PipedriveCompanyMapper, + SalesforceCompanyMapper, ZendeskCompanyMapper, ZohoCompanyMapper, MicrosoftdynamicssalesService, diff --git a/packages/api/src/crm/company/services/salesforce/index.ts b/packages/api/src/crm/company/services/salesforce/index.ts new file mode 100644 index 000000000..bdfd56f44 --- /dev/null +++ b/packages/api/src/crm/company/services/salesforce/index.ts @@ -0,0 +1,116 @@ +import { Injectable } from '@nestjs/common'; +import { ICompanyService } from '@crm/company/types'; +import { CrmObject } from '@crm/@lib/@types'; +import axios from 'axios'; +import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; +import { LoggerService } from '@@core/@core-services/logger/logger.service'; +import { EncryptionService } from '@@core/@core-services/encryption/encryption.service'; +import { ApiResponse } from '@@core/utils/types'; +import { ServiceRegistry } from '../registry.service'; +import { + commonSalesforceCompanyProperties, + SalesforceCompanyInput, + SalesforceCompanyOutput, + } from './types'; +import { SyncParam } from '@@core/utils/types/interface'; + +@Injectable() +export class SalesforceService implements ICompanyService { + constructor( + private prisma: PrismaService, + private logger: LoggerService, + private cryptoService: EncryptionService, + private registry: ServiceRegistry, + ) { + this.logger.setContext( + CrmObject.company.toUpperCase() + ':' + SalesforceService.name, + ); + this.registry.registerService('salesforce', this); + } + + async addCompany( + companyData: SalesforceCompanyInput, + linkedUserId: string, + ): Promise> { + try { + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'salesforce', + vertical: 'crm', + }, + }); + + const instanceUrl = connection.account_url; + const resp = await axios.post( + `${instanceUrl}/services/data/v56.0/sobjects/Account/`, + JSON.stringify(companyData), + { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }, + ); + + return { + data: resp.data, + message: 'Salesforce company created', + statusCode: 201, + }; + } catch (error) { + throw error; + } + } + + async sync(data: SyncParam): Promise> { + try { + const { linkedUserId, custom_properties, pageSize, cursor } = data; + + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'salesforce', + vertical: 'crm', + }, + }); + + const instanceUrl = connection.account_url; + let pagingString = `${pageSize ? `ORDER BY Id DESC LIMIT ${pageSize} ` : ''}${ + cursor ? `OFFSET ${cursor}` : '' + }`; + if (!pageSize && !cursor) { + pagingString = 'LIMIT 200'; + } + + const commonPropertyNames = Object.keys(commonSalesforceCompanyProperties); + const allProperties = [...commonPropertyNames, ...custom_properties]; + const fields = allProperties.join(','); + + const query = `SELECT ${fields} FROM Account ${pagingString}`; + + const resp = await axios.get( + `${instanceUrl}/services/data/v56.0/query/?q=${encodeURIComponent(query)}`, + { + headers: { + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }, + ); + + this.logger.log(`Synced Salesforce companies!`); + + return { + data: resp.data.records, + message: 'Salesforce companies retrieved', + statusCode: 200, + }; + } catch (error) { + throw error; + } + } +} \ No newline at end of file diff --git a/packages/api/src/crm/company/services/salesforce/mappers.ts b/packages/api/src/crm/company/services/salesforce/mappers.ts new file mode 100644 index 000000000..862008290 --- /dev/null +++ b/packages/api/src/crm/company/services/salesforce/mappers.ts @@ -0,0 +1,149 @@ +import { SalesforceCompanyInput, SalesforceCompanyOutput } from './types'; +import { + UnifiedCrmCompanyInput, + UnifiedCrmCompanyOutput, +} from '@crm/company/types/model.unified'; +import { ICompanyMapper } from '@crm/company/types'; +import { Utils } from '@crm/@lib/@utils'; +import { Injectable } from '@nestjs/common'; +import { MappersRegistry } from '@@core/@core-services/registries/mappers.registry'; + +@Injectable() +export class SalesforceCompanyMapper implements ICompanyMapper { + constructor(private mappersRegistry: MappersRegistry, private utils: Utils) { + this.mappersRegistry.registerService('crm', 'company', 'salesforce', this); + } + + async desunify( + source: UnifiedCrmCompanyInput, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const result: any = { + Name: source.name, + Industry: source.industry || null, + }; + + if (source.number_of_employees) { + result.NumberOfEmployees = source.number_of_employees; + } + + if (source.phone_numbers && source.phone_numbers.length > 0) { + result.Phone = source.phone_numbers[0].phone_number; + } + + if (source.addresses && source.addresses.length > 0) { + const address = source.addresses[0]; + result.BillingCity = address.city; + result.BillingState = address.state; + result.BillingPostalCode = address.postal_code; + result.BillingStreet = address.street_1; + result.BillingCountry = address.country; + } + + if (source.user_id) { + const owner_id = await this.utils.getRemoteIdFromUserUuid(source.user_id); + if (owner_id) { + result.OwnerId = owner_id; + } + } + + if (customFieldMappings && source.field_mappings) { + for (const [k, v] of Object.entries(source.field_mappings)) { + const mapping = customFieldMappings.find( + (mapping) => mapping.slug === k, + ); + if (mapping) { + result[mapping.remote_id] = v; + } + } + } + + return result; + } + + async unify( + source: SalesforceCompanyOutput | SalesforceCompanyOutput[], + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + if (!Array.isArray(source)) { + return this.mapSingleCompanyToUnified( + source, + connectionId, + customFieldMappings, + ); + } + return Promise.all( + source.map((company) => + this.mapSingleCompanyToUnified( + company, + connectionId, + customFieldMappings, + ), + ), + ); + } + + private async mapSingleCompanyToUnified( + company: SalesforceCompanyOutput, + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const field_mappings: { [key: string]: any } = {}; + if (customFieldMappings) { + for (const mapping of customFieldMappings) { + field_mappings[mapping.slug] = company[mapping.remote_id]; + } + } + + let opts: any = {}; + if (company.OwnerId) { + const owner_id = await this.utils.getUserUuidFromRemoteId( + company.OwnerId, + connectionId, + ); + if (owner_id) { + opts = { + user_id: owner_id, + }; + } + } + + return { + remote_id: company.Id, + remote_data: company, + name: company.Name, + industry: company.Industry, + number_of_employees: company.NumberOfEmployees, + addresses: [ + { + street_1: company.BillingStreet, + city: company.BillingCity, + state: company.BillingState, + postal_code: company.BillingPostalCode, + country: company.BillingCountry, + address_type: 'BILLING', + owner_type: 'COMPANY', + }, + ], + phone_numbers: [ + { + phone_number: company.Phone, + phone_type: 'WORK', + owner_type: 'COMPANY', + }, + ], + field_mappings, + ...opts, + }; + } +} \ No newline at end of file diff --git a/packages/api/src/crm/company/services/salesforce/types.ts b/packages/api/src/crm/company/services/salesforce/types.ts new file mode 100644 index 000000000..2dd5261d2 --- /dev/null +++ b/packages/api/src/crm/company/services/salesforce/types.ts @@ -0,0 +1,81 @@ +export interface SalesforceCompanyInput { + Name: string; + AccountNumber?: string; + AccountSource?: string; + AnnualRevenue?: number; + BillingAddress?: { + city?: string; + country?: string; + postalCode?: string; + state?: string; + street?: string; + }; + Description?: string; + Fax?: string; + Industry?: string; + NumberOfEmployees?: number; + OwnerId?: string; + ParentId?: string; + Phone?: string; + Rating?: string; + ShippingAddress?: { + city?: string; + country?: string; + postalCode?: string; + state?: string; + street?: string; + }; + Sic?: string; + SicDesc?: string; + Site?: string; + TickerSymbol?: string; + Type?: string; + Website?: string; + [key: string]: any; +} + +export interface SalesforceCompanyOutput extends SalesforceCompanyInput { + Id: string; + CreatedDate: string; + LastModifiedDate: string; + IsDeleted: boolean; +} + +export const commonSalesforceCompanyProperties = { + Id: '', + Name: '', + AccountNumber: '', + AccountSource: '', + AnnualRevenue: 0, + BillingAddress: { + city: '', + country: '', + postalCode: '', + state: '', + street: '', + }, + Description: '', + Fax: '', + Industry: '', + NumberOfEmployees: 0, + OwnerId: '', + ParentId: '', + Phone: '', + Rating: '', + ShippingAddress: { + city: '', + country: '', + postalCode: '', + state: '', + street: '', + }, + Sic: '', + SicDesc: '', + Site: '', + TickerSymbol: '', + Type: '', + Website: '', + CreatedDate: '', + LastModifiedDate: '', + IsDeleted: false, +}; \ No newline at end of file diff --git a/packages/api/src/crm/company/sync/sync.processor.ts b/packages/api/src/crm/company/sync/sync.processor.ts deleted file mode 100644 index 8fd922767..000000000 --- a/packages/api/src/crm/company/sync/sync.processor.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - @Process('crm-sync-companies') - async handleSyncCompanies(job: Job) { - try { - console.log(`Processing queue -> crm-sync-companies ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing crm companies', error); - } - } -} diff --git a/packages/api/src/crm/company/sync/sync.service.ts b/packages/api/src/crm/company/sync/sync.service.ts index 189e10d6f..05f5a4927 100644 --- a/packages/api/src/crm/company/sync/sync.service.ts +++ b/packages/api/src/crm/company/sync/sync.service.ts @@ -1,21 +1,20 @@ -import { Injectable, OnModuleInit } from '@nestjs/common'; import { LoggerService } from '@@core/@core-services/logger/logger.service'; import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; +import { BullQueueService } from '@@core/@core-services/queues/shared.service'; +import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; +import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; +import { IBaseSync, SyncLinkedUserType } from '@@core/utils/types/interface'; +import { OriginalCompanyOutput } from '@@core/utils/types/original/original.crm'; +import { Utils } from '@crm/@lib/@utils'; +import { Injectable, OnModuleInit } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; -import { ApiResponse } from '@@core/utils/types'; +import { CRM_PROVIDERS } from '@panora/shared'; +import { crm_companies as CrmCompany } from '@prisma/client'; import { v4 as uuidv4 } from 'uuid'; -import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; import { ServiceRegistry } from '../services/registry.service'; -import { UnifiedCrmCompanyOutput } from '../types/model.unified'; import { ICompanyService } from '../types'; -import { OriginalCompanyOutput } from '@@core/utils/types/original/original.crm'; -import { crm_companies as CrmCompany } from '@prisma/client'; -import { CRM_PROVIDERS } from '@panora/shared'; -import { Utils } from '@crm/@lib/@utils'; -import { CoreSyncRegistry } from '@@core/@core-services/registries/core-sync.registry'; -import { BullQueueService } from '@@core/@core-services/queues/shared.service'; -import { IBaseSync, SyncLinkedUserType } from '@@core/utils/types/interface'; -import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; +import { UnifiedCrmCompanyOutput } from '../types/model.unified'; @Injectable() export class SyncService implements OnModuleInit, IBaseSync { @@ -32,70 +31,38 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('crm', 'company', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob( - 'crm-sync-companies', - '0 0 * * *', - ); - } catch (error) { - throw error; - } + onModuleInit() { +// } //function used by sync worker which populate our crm_companies table //its role is to fetch all companies from providers 3rd parties and save the info inside our db // @Cron('*/2 * * * *') // every 2 minutes (for testing) @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log(`Syncing companies....`); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = CRM_PROVIDERS.filter( - (provider) => provider !== 'zoho', - ); - for (const provider of providers) { - try { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = CRM_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/crm/contact/contact.module.ts b/packages/api/src/crm/contact/contact.module.ts index 8717e2cfc..06a7119d3 100644 --- a/packages/api/src/crm/contact/contact.module.ts +++ b/packages/api/src/crm/contact/contact.module.ts @@ -1,9 +1,3 @@ -import { AffinityContactMapper } from './services/affinity/mappers'; -import { AffinityService } from './services/affinity'; - -import { MicrosoftdynamicssalesContactMapper } from './services/microsoftdynamicssales/mappers'; -import { MicrosoftdynamicssalesService } from './services/microsoftdynamicssales'; -import { BullQueueModule } from '@@core/@core-services/queues/queue.module'; import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; @@ -17,14 +11,18 @@ import { CloseContactMapper } from './services/close/mappers'; import { ContactService } from './services/contact.service'; import { HubspotService } from './services/hubspot'; import { HubspotContactMapper } from './services/hubspot/mappers'; +import { MicrosoftdynamicssalesService } from './services/microsoftdynamicssales'; +import { MicrosoftdynamicssalesContactMapper } from './services/microsoftdynamicssales/mappers'; import { PipedriveService } from './services/pipedrive'; import { PipedriveContactMapper } from './services/pipedrive/mappers'; import { ServiceRegistry } from './services/registry.service'; +import { SalesforceService } from './services/salesforce'; import { ZendeskService } from './services/zendesk'; import { ZendeskContactMapper } from './services/zendesk/mappers'; import { ZohoService } from './services/zoho'; import { ZohoContactMapper } from './services/zoho/mappers'; import { SyncService } from './sync/sync.service'; +import { SalesforceContactMapper } from './services/salesforce/mappers'; @Module({ controllers: [ContactController], @@ -39,6 +37,7 @@ import { SyncService } from './sync/sync.service'; /* PROVIDERS SERVICES */ AttioService, ZendeskService, + SalesforceService, ZohoService, PipedriveService, HubspotService, @@ -47,6 +46,7 @@ import { SyncService } from './sync/sync.service'; AttioContactMapper, CloseContactMapper, HubspotContactMapper, + SalesforceContactMapper, PipedriveContactMapper, ZendeskContactMapper, ZohoContactMapper, diff --git a/packages/api/src/crm/contact/services/salesforce/index.ts b/packages/api/src/crm/contact/services/salesforce/index.ts new file mode 100644 index 000000000..89d9c6ebe --- /dev/null +++ b/packages/api/src/crm/contact/services/salesforce/index.ts @@ -0,0 +1,112 @@ +import { EncryptionService } from '@@core/@core-services/encryption/encryption.service'; +import { LoggerService } from '@@core/@core-services/logger/logger.service'; +import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; +import { ApiResponse } from '@@core/utils/types'; +import { SyncParam } from '@@core/utils/types/interface'; +import { CrmObject } from '@crm/@lib/@types'; +import { IContactService } from '@crm/contact/types'; +import { Injectable } from '@nestjs/common'; +import axios from 'axios'; +import { ServiceRegistry } from '../registry.service'; +import { + SalesforceContactInput, + SalesforceContactOutput, + } from './types'; + +@Injectable() +export class SalesforceService implements IContactService { + constructor( + private prisma: PrismaService, + private logger: LoggerService, + private cryptoService: EncryptionService, + private registry: ServiceRegistry, + ) { + this.logger.setContext( + CrmObject.contact.toUpperCase() + ':' + SalesforceService.name, + ); + this.registry.registerService('salesforce', this); + } + + async addContact( + contactData: SalesforceContactInput, + linkedUserId: string, + ): Promise> { + try { + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'salesforce', + vertical: 'crm', + }, + }); + + const instanceUrl = connection.account_url; + const resp = await axios.post( + `${instanceUrl}/services/data/v56.0/sobjects/Contact/`, + JSON.stringify(contactData), + { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }, + ); + + return { + data: resp.data, + message: 'Salesforce contact created', + statusCode: 201, + }; + } catch (error) { + throw error; + } + } + + async sync(data: SyncParam): Promise> { + try { + const { linkedUserId, custom_properties, pageSize, cursor } = data; + + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'salesforce', + vertical: 'crm', + }, + }); + + const instanceUrl = connection.account_url; + let pagingString = `${pageSize ? `ORDER BY Id DESC LIMIT ${pageSize} ` : ''}${ + cursor ? `OFFSET ${cursor}` : '' + }`; + if (!pageSize && !cursor) { + pagingString = 'LIMIT 200'; + } + + const fields = custom_properties ? custom_properties.join(',') : 'Id,FirstName,LastName,Email,Phone'; + const query = `SELECT ${fields} FROM Contact ${pagingString}`; + + const resp = await axios.get( + `${instanceUrl}/services/data/v56.0/query/?q=${encodeURIComponent(query)}`, + { + headers: { + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }, + ); + + this.logger.log(`Synced Salesforce contacts!`); + + return { + data: resp.data.records, + message: 'Salesforce contacts retrieved', + statusCode: 200, + }; + } catch (error) { + throw error; + } + } +} \ No newline at end of file diff --git a/packages/api/src/crm/contact/services/salesforce/mappers.ts b/packages/api/src/crm/contact/services/salesforce/mappers.ts new file mode 100644 index 000000000..d70aa5797 --- /dev/null +++ b/packages/api/src/crm/contact/services/salesforce/mappers.ts @@ -0,0 +1,129 @@ +import { + UnifiedCrmContactInput, + UnifiedCrmContactOutput, + } from '@crm/contact/types/model.unified'; + import { IContactMapper } from '@crm/contact/types'; + import { SalesforceContactInput, SalesforceContactOutput } from './types'; + import { Utils } from '@crm/@lib/@utils'; + import { MappersRegistry } from '@@core/@core-services/registries/mappers.registry'; + import { Injectable } from '@nestjs/common'; + + @Injectable() + export class SalesforceContactMapper implements IContactMapper { + constructor(private mappersRegistry: MappersRegistry, private utils: Utils) { + this.mappersRegistry.registerService('crm', 'contact', 'salesforce', this); + } + + async desunify( + source: UnifiedCrmContactInput, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const result: SalesforceContactInput = { + FirstName: source.first_name, + LastName: source.last_name, + }; + + if (source.email_addresses && source.email_addresses.length > 0) { + result.Email = source.email_addresses[0].email_address; + } + + if (source.phone_numbers && source.phone_numbers.length > 0) { + result.Phone = source.phone_numbers[0].phone_number; + } + + if (source.addresses && source.addresses.length > 0) { + result.MailingStreet = source.addresses[0].street_1; + result.MailingCity = source.addresses[0].city; + result.MailingState = source.addresses[0].state; + result.MailingCountry = source.addresses[0].country; + result.MailingPostalCode = source.addresses[0].postal_code; + } + + if (customFieldMappings && source.field_mappings) { + for (const [k, v] of Object.entries(source.field_mappings)) { + const mapping = customFieldMappings.find( + (mapping) => mapping.slug === k, + ); + if (mapping) { + result[mapping.remote_id] = v; + } + } + } + + return result; + } + + async unify( + source: SalesforceContactOutput | SalesforceContactOutput[], + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + if (!Array.isArray(source)) { + return this.mapSingleContactToUnified( + source, + connectionId, + customFieldMappings, + ); + } + return source.map((contact) => + this.mapSingleContactToUnified( + contact, + connectionId, + customFieldMappings, + ), + ); + } + + private mapSingleContactToUnified( + contact: SalesforceContactOutput, + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): UnifiedCrmContactOutput { + const field_mappings: { [key: string]: any } = {}; + if (customFieldMappings) { + for (const mapping of customFieldMappings) { + field_mappings[mapping.slug] = contact[mapping.remote_id]; + } + } + + return { + remote_id: contact.Id, + remote_data: contact, + first_name: contact.FirstName, + last_name: contact.LastName, + email_addresses: [ + { + email_address: contact.Email, + email_address_type: 'PERSONAL', + owner_type: 'contact', + }, + ], + phone_numbers: [ + { + phone_number: contact.Phone, + phone_type: 'PERSONAL', + owner_type: 'contact', + }, + ], + addresses: [ + { + street_1: contact.MailingStreet, + city: contact.MailingCity, + state: contact.MailingState, + postal_code: contact.MailingPostalCode, + country: contact.MailingCountry, + }, + ], + field_mappings, + }; + } + } \ No newline at end of file diff --git a/packages/api/src/crm/contact/services/salesforce/types.ts b/packages/api/src/crm/contact/services/salesforce/types.ts new file mode 100644 index 000000000..d1a7e704f --- /dev/null +++ b/packages/api/src/crm/contact/services/salesforce/types.ts @@ -0,0 +1,43 @@ +export interface SalesforceContactInput { + FirstName?: string; + LastName: string; // Required in Salesforce + Email?: string; + Phone?: string; + MobilePhone?: string; + Title?: string; + Department?: string; + MailingStreet?: string; + MailingCity?: string; + MailingState?: string; + MailingCountry?: string; + MailingPostalCode?: string; + AccountId?: string; // Reference to the Account (Company) the contact is associated with + [key: string]: any; +} + +export interface SalesforceContactOutput extends SalesforceContactInput { + Id: string; + CreatedDate: string; + LastModifiedDate: string; + IsDeleted: boolean; +} + +export const commonSalesforceContactProperties = { + Id: '', + FirstName: '', + LastName: '', + Email: '', + Phone: '', + MobilePhone: '', + Title: '', + Department: '', + MailingStreet: '', + MailingCity: '', + MailingState: '', + MailingCountry: '', + MailingPostalCode: '', + AccountId: '', + CreatedDate: '', + LastModifiedDate: '', + IsDeleted: false, +}; \ No newline at end of file diff --git a/packages/api/src/crm/contact/sync/sync.processor.ts b/packages/api/src/crm/contact/sync/sync.processor.ts deleted file mode 100644 index 86feb3f50..000000000 --- a/packages/api/src/crm/contact/sync/sync.processor.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - - @Process({ name: 'crm-sync-contacts', concurrency: 5 }) - async handleSyncContacts(job: Job) { - try { - console.log(`Processing queue -> crm-sync-contacts ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing crm contacts', error); - } - } -} diff --git a/packages/api/src/crm/contact/sync/sync.service.ts b/packages/api/src/crm/contact/sync/sync.service.ts index 0f388df28..719958474 100644 --- a/packages/api/src/crm/contact/sync/sync.service.ts +++ b/packages/api/src/crm/contact/sync/sync.service.ts @@ -11,7 +11,7 @@ import { OriginalContactOutput } from '@@core/utils/types/original/original.crm' import { Utils } from '@crm/@lib/@utils'; import { UnifiedCrmContactOutput } from '@crm/contact/types/model.unified'; import { Injectable, OnModuleInit } from '@nestjs/common'; -import { Cron } from '@nestjs/schedule'; +import { Cron, CronExpression } from '@nestjs/schedule'; import { CRM_PROVIDERS } from '@panora/shared'; import { crm_contacts as CrmContact } from '@prisma/client'; import { v4 as uuidv4 } from 'uuid'; @@ -35,70 +35,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('crm', 'contact', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob( - 'crm-sync-contacts', - '0 0 * * *', - ); - } catch (error) { - throw error; - } + onModuleInit() { +// } - //function used by sync worker which populate our crm_contacts table - //its role is to fetch all contacts from providers 3rd parties and save the info inside our db - // @Cron('*/2 * * * *') // every 2 minutes (for testing) + @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log(`Syncing contacts....`); - - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = CRM_PROVIDERS.filter( - (provider) => provider !== 'zoho', - ); - for (const provider of providers) { - try { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = CRM_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/crm/deal/deal.module.ts b/packages/api/src/crm/deal/deal.module.ts index a87bf8c7b..9ee57ddff 100644 --- a/packages/api/src/crm/deal/deal.module.ts +++ b/packages/api/src/crm/deal/deal.module.ts @@ -1,9 +1,3 @@ -import { AffinityDealMapper } from './services/affinity/mappers'; -import { AffinityService } from './services/affinity'; - -import { MicrosoftdynamicssalesDealMapper } from './services/microsoftdynamicssales/mappers'; -import { MicrosoftdynamicssalesService } from './services/microsoftdynamicssales'; -import { BullQueueModule } from '@@core/@core-services/queues/queue.module'; import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; import { Utils } from '@crm/@lib/@utils'; @@ -16,14 +10,18 @@ import { CloseDealMapper } from './services/close/mappers'; import { DealService } from './services/deal.service'; import { HubspotService } from './services/hubspot'; import { HubspotDealMapper } from './services/hubspot/mappers'; +import { MicrosoftdynamicssalesService } from './services/microsoftdynamicssales'; +import { MicrosoftdynamicssalesDealMapper } from './services/microsoftdynamicssales/mappers'; import { PipedriveService } from './services/pipedrive'; import { PipedriveDealMapper } from './services/pipedrive/mappers'; import { ServiceRegistry } from './services/registry.service'; +import { SalesforceService } from './services/salesforce'; import { ZendeskService } from './services/zendesk'; import { ZendeskDealMapper } from './services/zendesk/mappers'; import { ZohoService } from './services/zoho'; import { ZohoDealMapper } from './services/zoho/mappers'; import { SyncService } from './sync/sync.service'; +import { SalesforceDealMapper } from './services/salesforce/mappers'; @Module({ controllers: [DealController], @@ -37,6 +35,7 @@ import { SyncService } from './sync/sync.service'; /* PROVIDERS SERVICES */ ZendeskService, ZohoService, + SalesforceService, PipedriveService, HubspotService, CloseService, @@ -47,6 +46,7 @@ import { SyncService } from './sync/sync.service'; PipedriveDealMapper, HubspotDealMapper, AttioDealMapper, + SalesforceDealMapper, CloseDealMapper, MicrosoftdynamicssalesService, MicrosoftdynamicssalesDealMapper, diff --git a/packages/api/src/crm/deal/services/salesforce/index.ts b/packages/api/src/crm/deal/services/salesforce/index.ts new file mode 100644 index 000000000..9ae3c581f --- /dev/null +++ b/packages/api/src/crm/deal/services/salesforce/index.ts @@ -0,0 +1,118 @@ +import { EncryptionService } from '@@core/@core-services/encryption/encryption.service'; +import { LoggerService } from '@@core/@core-services/logger/logger.service'; +import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; +import { ApiResponse } from '@@core/utils/types'; +import { SyncParam } from '@@core/utils/types/interface'; +import { CrmObject } from '@crm/@lib/@types'; +import { IDealService } from '@crm/deal/types'; +import { Injectable } from '@nestjs/common'; +import axios from 'axios'; +import { ServiceRegistry } from '../registry.service'; +import { + SalesforceDealInput, + SalesforceDealOutput, + commonDealSalesforceProperties, +} from './types'; + +@Injectable() +export class SalesforceService implements IDealService { + constructor( + private prisma: PrismaService, + private logger: LoggerService, + private cryptoService: EncryptionService, + private registry: ServiceRegistry, + ) { + this.logger.setContext( + CrmObject.deal.toUpperCase() + ':' + SalesforceService.name, + ); + this.registry.registerService('salesforce', this); + } + + async addDeal( + dealData: SalesforceDealInput, + linkedUserId: string, + ): Promise> { + try { + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'salesforce', + vertical: 'crm', + }, + }); + + const instanceUrl = connection.account_url; + const resp = await axios.post( + `${instanceUrl}/services/data/v56.0/sobjects/Opportunity/`, + JSON.stringify(dealData), + { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }, + ); + + this.logger.log(`Created Salesforce deal!`); + + return { + data: resp.data, + message: 'Salesforce deal created', + statusCode: 201, + }; + } catch (error) { + throw error; + } + } + + async sync(data: SyncParam): Promise> { + try { + const { linkedUserId, custom_properties, pageSize, cursor } = data; + + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'salesforce', + vertical: 'crm', + }, + }); + + const instanceUrl = connection.account_url; + let pagingString = `${pageSize ? `ORDER BY Id DESC LIMIT ${pageSize} ` : ''}${ + cursor ? `OFFSET ${cursor}` : '' + }`; + if (!pageSize && !cursor) { + pagingString = 'LIMIT 200'; + } + + const commonPropertyNames = Object.keys(commonDealSalesforceProperties); + const allProperties = [...commonPropertyNames, ...custom_properties]; + const fields = allProperties.join(','); + + const query = `SELECT ${fields} FROM Opportunity ${pagingString}`; + + const resp = await axios.get( + `${instanceUrl}/services/data/v56.0/query/?q=${encodeURIComponent(query)}`, + { + headers: { + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }, + ); + + this.logger.log(`Synced Salesforce deals!`); + + return { + data: resp.data.records, + message: 'Salesforce deals retrieved', + statusCode: 200, + }; + } catch (error) { + throw error; + } + } +} \ No newline at end of file diff --git a/packages/api/src/crm/deal/services/salesforce/mappers.ts b/packages/api/src/crm/deal/services/salesforce/mappers.ts new file mode 100644 index 000000000..6fabeb8b3 --- /dev/null +++ b/packages/api/src/crm/deal/services/salesforce/mappers.ts @@ -0,0 +1,132 @@ +import { SalesforceDealInput, SalesforceDealOutput } from './types'; +import { + UnifiedCrmDealInput, + UnifiedCrmDealOutput, +} from '@crm/deal/types/model.unified'; +import { IDealMapper } from '@crm/deal/types'; +import { Utils } from '@crm/@lib/@utils'; +import { MappersRegistry } from '@@core/@core-services/registries/mappers.registry'; +import { Injectable } from '@nestjs/common'; + +@Injectable() +export class SalesforceDealMapper implements IDealMapper { + constructor(private mappersRegistry: MappersRegistry, private utils: Utils) { + this.mappersRegistry.registerService('crm', 'deal', 'salesforce', this); + } + + async desunify( + source: UnifiedCrmDealInput, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const result: SalesforceDealInput = { + Name: source.name, + Amount: Number(source.amount), + StageName: source.stage_id || null, + CloseDate: null, // TODO + }; + + if (source.user_id) { + const owner_id = await this.utils.getRemoteIdFromUserUuid(source.user_id); + if (owner_id) { + result.OwnerId = owner_id; + } + } + + if (customFieldMappings && source.field_mappings) { + for (const [k, v] of Object.entries(source.field_mappings)) { + const mapping = customFieldMappings.find( + (mapping) => mapping.slug === k, + ); + if (mapping) { + result[mapping.remote_id] = v; + } + } + } + return result; + } + + async unify( + source: SalesforceDealOutput | SalesforceDealOutput[], + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + if (!Array.isArray(source)) { + return await this.mapSingleDealToUnified( + source, + connectionId, + customFieldMappings, + ); + } + return Promise.all( + source.map((deal) => + this.mapSingleDealToUnified(deal, connectionId, customFieldMappings), + ), + ); + } + + private async mapSingleDealToUnified( + deal: SalesforceDealOutput, + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const field_mappings: { [key: string]: any } = {}; + if (customFieldMappings) { + for (const mapping of customFieldMappings) { + field_mappings[mapping.slug] = deal[mapping.remote_id]; + } + } + + let opts: any = {}; + if (deal.OwnerId) { + const owner_id = await this.utils.getUserUuidFromRemoteId( + deal.OwnerId, + connectionId, + ); + if (owner_id) { + opts = { + ...opts, + user_id: owner_id, + }; + } + } + + if (deal.Amount) { + opts = { + ...opts, + amount: String(deal.Amount), + }; + } + + if (deal.StageName) { + const stage_id = await this.utils.getStageUuidFromStageName( + deal.StageName, + connectionId, + ); + if (stage_id) { + opts = { + ...opts, + stage_id: stage_id, + }; + } + } + + return { + remote_id: deal.Id, + remote_data: deal, + name: deal.Name, + description: deal.Description || '', + close_date: deal.CloseDate, + field_mappings, + ...opts, + }; + } +} \ No newline at end of file diff --git a/packages/api/src/crm/deal/services/salesforce/types.ts b/packages/api/src/crm/deal/services/salesforce/types.ts new file mode 100644 index 000000000..d84f4f480 --- /dev/null +++ b/packages/api/src/crm/deal/services/salesforce/types.ts @@ -0,0 +1,38 @@ +export interface SalesforceDealInput { + Name: string; + Amount?: number; + StageName: string; // Required in Salesforce + CloseDate: string; // Required in Salesforce + AccountId?: string; + OwnerId?: string; + Type?: string; + Probability?: number; + [key: string]: any; +} + +export interface SalesforceDealOutput extends SalesforceDealInput { + Id: string; + CreatedDate: string; + LastModifiedDate: string; + IsDeleted: boolean; + IsClosed: boolean; + IsWon: boolean; +} + +export const commonDealSalesforceProperties = { + Id: '', + Name: '', + Amount: 0, + StageName: '', + CloseDate: '', + AccountId: '', + OwnerId: '', + Type: '', + Probability: 0, + Description: '', + CreatedDate: '', + LastModifiedDate: '', + IsDeleted: false, + IsClosed: false, + IsWon: false, +}; \ No newline at end of file diff --git a/packages/api/src/crm/deal/sync/sync.processor.ts b/packages/api/src/crm/deal/sync/sync.processor.ts deleted file mode 100644 index b01430568..000000000 --- a/packages/api/src/crm/deal/sync/sync.processor.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - @Process('crm-sync-deals') - async handleSyncDeals(job: Job) { - try { - console.log(`Processing queue -> crm-sync-deals ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing crm deals', error); - } - } -} diff --git a/packages/api/src/crm/deal/sync/sync.service.ts b/packages/api/src/crm/deal/sync/sync.service.ts index 95e80032f..cf5d9dedc 100644 --- a/packages/api/src/crm/deal/sync/sync.service.ts +++ b/packages/api/src/crm/deal/sync/sync.service.ts @@ -29,67 +29,38 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('crm', 'deal', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob('crm-sync-deals', '0 0 * * *'); - } catch (error) { - throw error; - } + onModuleInit() { +// } //function used by sync worker which populate our crm_deals table //its role is to fetch all deals from providers 3rd parties and save the info inside our db //@Cron('*/2 * * * *') // every 2 minutes (for testing) @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log(`Syncing deals....`); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = CRM_PROVIDERS.filter( - (provider) => provider !== 'zoho', - ); - for (const provider of providers) { - try { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = CRM_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/crm/engagement/sync/sync.processor.ts b/packages/api/src/crm/engagement/sync/sync.processor.ts deleted file mode 100644 index 9061c5126..000000000 --- a/packages/api/src/crm/engagement/sync/sync.processor.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - @Process('crm-sync-engagements') - async handleSyncEngagements(job: Job) { - try { - console.log(`Processing queue -> crm-sync-engagements ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing crm engagements', error); - } - } -} diff --git a/packages/api/src/crm/engagement/sync/sync.service.ts b/packages/api/src/crm/engagement/sync/sync.service.ts index e5e67f5c3..48f960080 100644 --- a/packages/api/src/crm/engagement/sync/sync.service.ts +++ b/packages/api/src/crm/engagement/sync/sync.service.ts @@ -35,73 +35,38 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('crm', 'engagement', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob( - 'crm-sync-engagements', - '0 0 * * *', - ); - } catch (error) { - throw error; - } + onModuleInit() { +// } //function used by sync worker which populate our crm_engagements table //its role is to fetch all engagements from providers 3rd parties and save the info inside our db //@Cron('*/2 * * * *') // every 2 minutes (for testing) @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log(`Syncing engagements....`); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = CRM_PROVIDERS.filter( - (provider) => provider !== 'zoho', - ); - for (const provider of providers) { - try { - for (const type of ENGAGEMENTS_TYPE) { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - engagement_type: type, - }); - } - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = CRM_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } @@ -115,7 +80,9 @@ export class SyncService implements OnModuleInit, IBaseSync { const service: IEngagementService = this.serviceRegistry.getService(integrationId); if (!service) { - this.logger.log(`No service found in {vertical:crm, commonObject: engagement} for integration ID: ${integrationId}`); + this.logger.log( + `No service found in {vertical:crm, commonObject: engagement} for integration ID: ${integrationId}`, + ); return; } diff --git a/packages/api/src/crm/note/note.module.ts b/packages/api/src/crm/note/note.module.ts index ae4d50733..87ccc0eaf 100644 --- a/packages/api/src/crm/note/note.module.ts +++ b/packages/api/src/crm/note/note.module.ts @@ -1,29 +1,27 @@ -import { AffinityNoteMapper } from './services/affinity/mappers'; -import { AffinityService } from './services/affinity'; - -import { MicrosoftdynamicssalesNoteMapper } from './services/microsoftdynamicssales/mappers'; -import { MicrosoftdynamicssalesService } from './services/microsoftdynamicssales'; -import { BullQueueModule } from '@@core/@core-services/queues/queue.module'; import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; import { Utils } from '@crm/@lib/@utils'; import { Module } from '@nestjs/common'; import { NoteController } from './note.controller'; +import { AttioService } from './services/attio'; +import { AttioNoteMapper } from './services/attio/mappers'; import { CloseService } from './services/close'; import { CloseNoteMapper } from './services/close/mappers'; import { HubspotService } from './services/hubspot'; -import { AttioService } from './services/attio'; import { HubspotNoteMapper } from './services/hubspot/mappers'; +import { MicrosoftdynamicssalesService } from './services/microsoftdynamicssales'; +import { MicrosoftdynamicssalesNoteMapper } from './services/microsoftdynamicssales/mappers'; import { NoteService } from './services/note.service'; import { PipedriveService } from './services/pipedrive'; import { PipedriveNoteMapper } from './services/pipedrive/mappers'; import { ServiceRegistry } from './services/registry.service'; +import { SalesforceService } from './services/salesforce'; import { ZendeskService } from './services/zendesk'; import { ZendeskNoteMapper } from './services/zendesk/mappers'; import { ZohoService } from './services/zoho'; import { ZohoNoteMapper } from './services/zoho/mappers'; -import { AttioNoteMapper } from './services/attio/mappers'; import { SyncService } from './sync/sync.service'; +import { SalesforceNoteMapper } from './services/salesforce/mappers'; @Module({ controllers: [NoteController], providers: [ @@ -39,6 +37,7 @@ import { SyncService } from './sync/sync.service'; PipedriveService, HubspotService, AttioService, + SalesforceService, CloseService, /* PROVIDERS MAPPERS */ ZendeskNoteMapper, @@ -46,6 +45,7 @@ import { SyncService } from './sync/sync.service'; PipedriveNoteMapper, AttioNoteMapper, HubspotNoteMapper, + SalesforceNoteMapper, CloseNoteMapper, MicrosoftdynamicssalesService, MicrosoftdynamicssalesNoteMapper, diff --git a/packages/api/src/crm/note/services/salesforce/index.ts b/packages/api/src/crm/note/services/salesforce/index.ts new file mode 100644 index 000000000..18d5b6221 --- /dev/null +++ b/packages/api/src/crm/note/services/salesforce/index.ts @@ -0,0 +1,130 @@ +import { EncryptionService } from '@@core/@core-services/encryption/encryption.service'; +import { LoggerService } from '@@core/@core-services/logger/logger.service'; +import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; +import { ApiResponse } from '@@core/utils/types'; +import { SyncParam } from '@@core/utils/types/interface'; +import { CrmObject } from '@crm/@lib/@types'; +import { INoteService } from '@crm/note/types'; +import { Injectable } from '@nestjs/common'; +import axios from 'axios'; +import { ServiceRegistry } from '../registry.service'; +import { + SalesforceNoteInput, + SalesforceNoteOutput, + commonNoteSalesforceProperties, +} from './types'; + +@Injectable() +export class SalesforceService implements INoteService { + constructor( + private prisma: PrismaService, + private logger: LoggerService, + private cryptoService: EncryptionService, + private registry: ServiceRegistry, + ) { + this.logger.setContext( + CrmObject.note.toUpperCase() + ':' + SalesforceService.name, + ); + this.registry.registerService('salesforce', this); + } + + async addNote( + noteData: SalesforceNoteInput, + linkedUserId: string, + ): Promise> { + try { + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'salesforce', + vertical: 'crm', + }, + }); + + const instanceUrl = connection.account_url; + const resp = await axios.post( + `${instanceUrl}/services/data/v56.0/sobjects/Note/`, + JSON.stringify(noteData), + { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }, + ); + + // Fetch the created note to get all details + const noteId = resp.data.id; + const final_resp = await axios.get( + `${instanceUrl}/services/data/v56.0/sobjects/Note/${noteId}`, + { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }, + ); + + return { + data: final_resp.data, + message: 'Salesforce note created', + statusCode: 201, + }; + } catch (error) { + throw error; + } + } + + async sync(data: SyncParam): Promise> { + try { + const { linkedUserId, custom_properties, pageSize, cursor } = data; + + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'salesforce', + vertical: 'crm', + }, + }); + + const instanceUrl = connection.account_url; + let pagingString = `${pageSize ? `ORDER BY Id DESC LIMIT ${pageSize} ` : ''}${ + cursor ? `OFFSET ${cursor}` : '' + }`; + if (!pageSize && !cursor) { + pagingString = 'LIMIT 200'; + } + + const commonPropertyNames = Object.keys(commonNoteSalesforceProperties); + const allProperties = [...commonPropertyNames, ...custom_properties]; + const fields = allProperties.join(','); + + const query = `SELECT ${fields} FROM Note ${pagingString}`; + + const resp = await axios.get( + `${instanceUrl}/services/data/v56.0/query/?q=${encodeURIComponent(query)}`, + { + headers: { + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }, + ); + + this.logger.log(`Synced Salesforce notes!`); + + return { + data: resp.data.records, + message: 'Salesforce notes retrieved', + statusCode: 200, + }; + } catch (error) { + throw error; + } + } +} \ No newline at end of file diff --git a/packages/api/src/crm/note/services/salesforce/mappers.ts b/packages/api/src/crm/note/services/salesforce/mappers.ts new file mode 100644 index 000000000..6ba9e441b --- /dev/null +++ b/packages/api/src/crm/note/services/salesforce/mappers.ts @@ -0,0 +1,132 @@ +import { MappersRegistry } from '@@core/@core-services/registries/mappers.registry'; +import { Utils } from '@crm/@lib/@utils'; +import { INoteMapper } from '@crm/note/types'; +import { + UnifiedCrmNoteInput, + UnifiedCrmNoteOutput, +} from '@crm/note/types/model.unified'; +import { Injectable } from '@nestjs/common'; +import { SalesforceNoteInput, SalesforceNoteOutput } from './types'; + +@Injectable() +export class SalesforceNoteMapper implements INoteMapper { + constructor(private mappersRegistry: MappersRegistry, private utils: Utils) { + this.mappersRegistry.registerService('crm', 'note', 'salesforce', this); + } + + async desunify( + source: UnifiedCrmNoteInput, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const result: SalesforceNoteInput = { + Content: source.content, + Title: source.content, // TODO: source.title || 'Note', + }; + + if (source.user_id) { + const owner_id = await this.utils.getRemoteIdFromUserUuid(source.user_id); + if (owner_id) { + result.OwnerId = owner_id; + } + } + + if (source.deal_id) { + const id = await this.utils.getRemoteIdFromDealUuid(source.deal_id); + result.ParentId = id; + } else if (source.contact_id) { + const id = await this.utils.getRemoteIdFromContactUuid(source.contact_id); + result.ParentId = id; + } else if (source.company_id) { + const id = await this.utils.getRemoteIdFromCompanyUuid(source.company_id); + result.ParentId = id; + } + + if (customFieldMappings && source.field_mappings) { + for (const [k, v] of Object.entries(source.field_mappings)) { + const mapping = customFieldMappings.find( + (mapping) => mapping.slug === k, + ); + if (mapping) { + result[mapping.remote_id] = v; + } + } + } + + return result; + } + + async unify( + source: SalesforceNoteOutput | SalesforceNoteOutput[], + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + if (!Array.isArray(source)) { + return await this.mapSingleNoteToUnified( + source, + connectionId, + customFieldMappings, + ); + } + + return Promise.all( + source.map((note) => + this.mapSingleNoteToUnified(note, connectionId, customFieldMappings), + ), + ); + } + + private async mapSingleNoteToUnified( + note: SalesforceNoteOutput, + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const field_mappings: { [key: string]: any } = {}; + if (customFieldMappings) { + for (const mapping of customFieldMappings) { + field_mappings[mapping.slug] = note[mapping.remote_id]; + } + } + + let opts: any = {}; + if (note.OwnerId) { + const owner_id = await this.utils.getUserUuidFromRemoteId( + note.OwnerId, + connectionId, + ); + if (owner_id) { + opts = { + ...opts, + user_id: owner_id, + }; + } + } + + if (note.ParentId) { + // Determine the type of the parent (deal, contact, or company) + // This might require additional API calls or logic to determine the object type + // For this example, we'll assume it's a deal, but you should implement proper logic here + opts.deal_id = await this.utils.getDealUuidFromRemoteId( + note.ParentId, + connectionId, + ); + } + + return { + remote_id: note.Id, + remote_data: note, + content: note.Body, + title: note.Title, + field_mappings, + ...opts, + }; + } +} \ No newline at end of file diff --git a/packages/api/src/crm/note/services/salesforce/types.ts b/packages/api/src/crm/note/services/salesforce/types.ts new file mode 100644 index 000000000..8a55cfd31 --- /dev/null +++ b/packages/api/src/crm/note/services/salesforce/types.ts @@ -0,0 +1,27 @@ +export interface SalesforceNoteInput { + Title: string; // Required in Salesforce + Content: string; // Required in Salesforce + [key: string]: any; +} + +export interface SalesforceNoteOutput extends SalesforceNoteInput { + Id: string; + OwnerId: string; + CreatedDate: string; + LastModifiedDate: string; + IsDeleted: boolean; + FileExtension: string; + ContentSize: number; +} + +export const commonNoteSalesforceProperties = { + Id: '', + Title: '', + Content: '', + OwnerId: '', + CreatedDate: '', + LastModifiedDate: '', + IsDeleted: false, + FileExtension: '', + ContentSize: 0, +}; \ No newline at end of file diff --git a/packages/api/src/crm/note/sync/sync.processor.ts b/packages/api/src/crm/note/sync/sync.processor.ts deleted file mode 100644 index f1d4187a0..000000000 --- a/packages/api/src/crm/note/sync/sync.processor.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - @Process('crm-sync-notes') - async handleSyncNotes(job: Job) { - try { - console.log(`Processing queue -> crm-sync-notes ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing crm notes', error); - } - } -} diff --git a/packages/api/src/crm/note/sync/sync.service.ts b/packages/api/src/crm/note/sync/sync.service.ts index 5fe1b08aa..ae800f8ee 100644 --- a/packages/api/src/crm/note/sync/sync.service.ts +++ b/packages/api/src/crm/note/sync/sync.service.ts @@ -35,67 +35,38 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('crm', 'note', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob('crm-sync-notes', '0 0 * * *'); - } catch (error) { - throw error; - } + onModuleInit() { +// } //function used by sync worker which populate our crm_notes table //its role is to fetch all notes from providers 3rd parties and save the info inside our db //@Cron('*/2 * * * *') // every 2 minutes (for testing) @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log(`Syncing notes....`); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = CRM_PROVIDERS.filter( - (provider) => provider !== 'zoho', - ); - for (const provider of providers) { - try { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = CRM_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } @@ -108,7 +79,9 @@ export class SyncService implements OnModuleInit, IBaseSync { const service: INoteService = this.serviceRegistry.getService(integrationId); if (!service) { - this.logger.log(`No service found in {vertical:crm, commonObject: note} for integration ID: ${integrationId}`); + this.logger.log( + `No service found in {vertical:crm, commonObject: note} for integration ID: ${integrationId}`, + ); return; } diff --git a/packages/api/src/crm/stage/sync/sync.processor.ts b/packages/api/src/crm/stage/sync/sync.processor.ts deleted file mode 100644 index e89656da8..000000000 --- a/packages/api/src/crm/stage/sync/sync.processor.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - @Process('crm-sync-stages') - async handleSyncStages(job: Job) { - try { - console.log(`Processing queue -> crm-sync-stages ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing crm stages', error); - } - } -} diff --git a/packages/api/src/crm/stage/sync/sync.service.ts b/packages/api/src/crm/stage/sync/sync.service.ts index d59dc256d..73803e869 100644 --- a/packages/api/src/crm/stage/sync/sync.service.ts +++ b/packages/api/src/crm/stage/sync/sync.service.ts @@ -33,87 +33,38 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('crm', 'stage', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob('crm-sync-stages', '0 0 * * *'); - } catch (error) { - throw error; - } + onModuleInit() { +// } //function used by sync worker which populate our crm_stages table //its role is to fetch all stages from providers 3rd parties and save the info inside our db //@Cron('*/2 * * * *') // every 2 minutes (for testing) @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log(`Syncing stages....`); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = CRM_PROVIDERS.filter( - (provider) => provider !== 'zoho', - ); - for (const provider of providers) { - try { - try { - const connection = - await this.prisma.connections.findFirst({ - where: { - id_linked_user: linkedUser.id_linked_user, - provider_slug: provider.toLowerCase(), - }, - }); - //call the sync comments for every ticket of the linkedUser (a comment is tied to a ticket) - const deals = await this.prisma.crm_deals.findMany({ - where: { - id_connection: connection?.id_connection, - }, - }); - for (const deal of deals) { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - deal_id: deal.id_crm_deal, - }); - } - } catch (error) { - throw error; - } - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = CRM_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } @@ -126,7 +77,9 @@ export class SyncService implements OnModuleInit, IBaseSync { const service: IStageService = this.serviceRegistry.getService(integrationId); if (!service) { - this.logger.log(`No service found in {vertical:crm, commonObject: stage} for integration ID: ${integrationId}`); + this.logger.log( + `No service found in {vertical:crm, commonObject: stage} for integration ID: ${integrationId}`, + ); return; } diff --git a/packages/api/src/crm/task/services/salesforce/index.ts b/packages/api/src/crm/task/services/salesforce/index.ts new file mode 100644 index 000000000..4d84bd650 --- /dev/null +++ b/packages/api/src/crm/task/services/salesforce/index.ts @@ -0,0 +1,130 @@ +import { EncryptionService } from '@@core/@core-services/encryption/encryption.service'; +import { LoggerService } from '@@core/@core-services/logger/logger.service'; +import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; +import { ApiResponse } from '@@core/utils/types'; +import { SyncParam } from '@@core/utils/types/interface'; +import { CrmObject } from '@crm/@lib/@types'; +import { ITaskService } from '@crm/task/types'; +import { Injectable } from '@nestjs/common'; +import axios from 'axios'; +import { ServiceRegistry } from '../registry.service'; +import { + SalesforceTaskInput, + SalesforceTaskOutput, + commonTaskSalesforceProperties, +} from './types'; + +@Injectable() +export class SalesforceService implements ITaskService { + constructor( + private prisma: PrismaService, + private logger: LoggerService, + private cryptoService: EncryptionService, + private registry: ServiceRegistry, + ) { + this.logger.setContext( + CrmObject.task.toUpperCase() + ':' + SalesforceService.name, + ); + this.registry.registerService('salesforce', this); + } + + async addTask( + taskData: SalesforceTaskInput, + linkedUserId: string, + ): Promise> { + try { + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'salesforce', + vertical: 'crm', + }, + }); + + const instanceUrl = connection.account_url; + const resp = await axios.post( + `${instanceUrl}/services/data/v56.0/sobjects/Task/`, + JSON.stringify(taskData), + { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }, + ); + + // Fetch the created task to get all details + const taskId = resp.data.id; + const final_resp = await axios.get( + `${instanceUrl}/services/data/v56.0/sobjects/Task/${taskId}`, + { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }, + ); + + return { + data: final_resp.data, + message: 'Salesforce task created', + statusCode: 201, + }; + } catch (error) { + throw error; + } + } + + async sync(data: SyncParam): Promise> { + try { + const { linkedUserId, custom_properties, pageSize, cursor } = data; + + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'salesforce', + vertical: 'crm', + }, + }); + + const instanceUrl = connection.account_url; + let pagingString = `${pageSize ? `ORDER BY Id DESC LIMIT ${pageSize} ` : ''}${ + cursor ? `OFFSET ${cursor}` : '' + }`; + if (!pageSize && !cursor) { + pagingString = 'LIMIT 200'; + } + + const commonPropertyNames = Object.keys(commonTaskSalesforceProperties); + const allProperties = [...commonPropertyNames, ...custom_properties]; + const fields = allProperties.join(','); + + const query = `SELECT ${fields} FROM Task ${pagingString}`; + + const resp = await axios.get( + `${instanceUrl}/services/data/v56.0/query/?q=${encodeURIComponent(query)}`, + { + headers: { + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }, + ); + + this.logger.log(`Synced Salesforce tasks!`); + + return { + data: resp.data.records, + message: 'Salesforce tasks retrieved', + statusCode: 200, + }; + } catch (error) { + throw error; + } + } +} \ No newline at end of file diff --git a/packages/api/src/crm/task/services/salesforce/mappers.ts b/packages/api/src/crm/task/services/salesforce/mappers.ts new file mode 100644 index 000000000..ca9d9de73 --- /dev/null +++ b/packages/api/src/crm/task/services/salesforce/mappers.ts @@ -0,0 +1,158 @@ +import { SalesforceTaskInput, SalesforceTaskOutput } from './types'; +import { + TaskStatus, + UnifiedCrmTaskInput, + UnifiedCrmTaskOutput, +} from '@crm/task/types/model.unified'; +import { ITaskMapper } from '@crm/task/types'; +import { Utils } from '@crm/@lib/@utils'; +import { MappersRegistry } from '@@core/@core-services/registries/mappers.registry'; +import { Injectable } from '@nestjs/common'; + +@Injectable() +export class SalesforceTaskMapper implements ITaskMapper { + constructor(private mappersRegistry: MappersRegistry, private utils: Utils) { + this.mappersRegistry.registerService('crm', 'task', 'salesforce', this); + } + + mapToTaskStatus(data: string): TaskStatus { + switch (data) { + case 'Not Started': + return 'PENDING'; + case 'Completed': + return 'COMPLETED'; + default: + return data; + } + } + + reverseMapToTaskStatus(data: TaskStatus): string { + switch (data) { + case 'COMPLETED': + return 'Completed'; + case 'PENDING': + return 'Not Started'; + default: + return 'Not Started'; + } + } + + async desunify( + source: UnifiedCrmTaskInput, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const result: SalesforceTaskInput = { + Subject: source.subject || '', + Description: source.content || '', + Status: this.reverseMapToTaskStatus(source.status as TaskStatus), + //Priority: source.priority || 'Normal', + }; + + if (source.user_id) { + const owner_id = await this.utils.getRemoteIdFromUserUuid(source.user_id); + if (owner_id) { + result.OwnerId = owner_id; + } + } + + if (source.deal_id) { + const id = await this.utils.getRemoteIdFromDealUuid(source.deal_id); + result.WhatId = id; + } else if (source.company_id) { + const id = await this.utils.getRemoteIdFromCompanyUuid(source.company_id); + result.WhatId = id; + } + + if (customFieldMappings && source.field_mappings) { + for (const [k, v] of Object.entries(source.field_mappings)) { + const mapping = customFieldMappings.find( + (mapping) => mapping.slug === k, + ); + if (mapping) { + result[mapping.remote_id] = v; + } + } + } + + return result; + } + + async unify( + source: SalesforceTaskOutput | SalesforceTaskOutput[], + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + if (!Array.isArray(source)) { + return await this.mapSingleTaskToUnified( + source, + connectionId, + customFieldMappings, + ); + } + + return Promise.all( + source.map((task) => + this.mapSingleTaskToUnified(task, connectionId, customFieldMappings), + ), + ); + } + + private async mapSingleTaskToUnified( + task: SalesforceTaskOutput, + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const field_mappings: { [key: string]: any } = {}; + if (customFieldMappings) { + for (const mapping of customFieldMappings) { + field_mappings[mapping.slug] = task[mapping.remote_id]; + } + } + + let opts: any = {}; + if (task.OwnerId) { + const owner_id = await this.utils.getUserUuidFromRemoteId( + task.OwnerId, + connectionId, + ); + if (owner_id) { + opts = { + ...opts, + user_id: owner_id, + }; + } + } + + opts.status = this.mapToTaskStatus(task.Status); + + if (task.WhatId) { + // Determine if WhatId is a deal or company + // This might require additional API calls or logic + // For this example, we'll assume it's a deal, but you should implement proper logic here + opts.deal_id = await this.utils.getDealUuidFromRemoteId( + task.WhatId, + connectionId, + ); + } + + return { + remote_id: task.Id, + remote_data: task, + subject: task.Subject, + content: task.Description, + priority: task.Priority, + due_date: task.ActivityDate, + field_mappings, + ...opts, + }; + } +} \ No newline at end of file diff --git a/packages/api/src/crm/task/services/salesforce/types.ts b/packages/api/src/crm/task/services/salesforce/types.ts new file mode 100644 index 000000000..9d2c9c00a --- /dev/null +++ b/packages/api/src/crm/task/services/salesforce/types.ts @@ -0,0 +1,35 @@ +export interface SalesforceTaskInput { + Subject: string; + Description?: string; + Status: string; + Priority?: string; + ActivityDate?: string; + OwnerId?: string; + WhatId?: string; // Related To ID (can be Account, Opportunity, etc.) + WhoId?: string; // Related Contact or Lead ID + [key: string]: any; +} + +export interface SalesforceTaskOutput extends SalesforceTaskInput { + Id: string; + CreatedDate: string; + LastModifiedDate: string; + IsDeleted: boolean; + IsClosed: boolean; +} + +export const commonTaskSalesforceProperties = { + Id: '', + Subject: '', + Description: '', + Status: '', + Priority: '', + ActivityDate: '', + OwnerId: '', + WhatId: '', + WhoId: '', + CreatedDate: '', + LastModifiedDate: '', + IsDeleted: false, + IsClosed: false, +}; \ No newline at end of file diff --git a/packages/api/src/crm/task/sync/sync.processor.ts b/packages/api/src/crm/task/sync/sync.processor.ts deleted file mode 100644 index e25a4567f..000000000 --- a/packages/api/src/crm/task/sync/sync.processor.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - @Process('crm-sync-tasks') - async handleSyncTasks(job: Job) { - try { - console.log(`Processing queue -> crm-sync-tasks ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing crm tasks', error); - } - } -} diff --git a/packages/api/src/crm/task/sync/sync.service.ts b/packages/api/src/crm/task/sync/sync.service.ts index 1085c9025..c8a86dc43 100644 --- a/packages/api/src/crm/task/sync/sync.service.ts +++ b/packages/api/src/crm/task/sync/sync.service.ts @@ -36,67 +36,38 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('crm', 'task', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob('crm-sync-tasks', '0 0 * * *'); - } catch (error) { - throw error; - } + onModuleInit() { +// } //function used by sync worker which populate our crm_tasks table //its role is to fetch all tasks from providers 3rd parties and save the info inside our db //@Cron('*/2 * * * *') // every 2 minutes (for testing) @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log(`Syncing tasks....`); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = CRM_PROVIDERS.filter( - (provider) => provider !== 'zoho', - ); - for (const provider of providers) { - try { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = CRM_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } @@ -109,7 +80,9 @@ export class SyncService implements OnModuleInit, IBaseSync { const service: ITaskService = this.serviceRegistry.getService(integrationId); if (!service) { - this.logger.log(`No service found in {vertical:crm, commonObject: task} for integration ID: ${integrationId}`); + this.logger.log( + `No service found in {vertical:crm, commonObject: task} for integration ID: ${integrationId}`, + ); return; } diff --git a/packages/api/src/crm/task/task.module.ts b/packages/api/src/crm/task/task.module.ts index 63800e06a..8b17fd7e2 100644 --- a/packages/api/src/crm/task/task.module.ts +++ b/packages/api/src/crm/task/task.module.ts @@ -1,11 +1,11 @@ -import { MicrosoftdynamicssalesTaskMapper } from './services/microsoftdynamicssales/mappers'; import { MicrosoftdynamicssalesService } from './services/microsoftdynamicssales'; -import { BullQueueModule } from '@@core/@core-services/queues/queue.module'; - +import { MicrosoftdynamicssalesTaskMapper } from './services/microsoftdynamicssales/mappers'; import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; import { Utils } from '@crm/@lib/@utils'; import { Module } from '@nestjs/common'; +import { AttioService } from './services/attio'; +import { AttioTaskMapper } from './services/attio/mappers'; import { CloseService } from './services/close'; import { CloseTaskMapper } from './services/close/mappers'; import { HubspotService } from './services/hubspot'; @@ -13,15 +13,15 @@ import { HubspotTaskMapper } from './services/hubspot/mappers'; import { PipedriveService } from './services/pipedrive'; import { PipedriveTaskMapper } from './services/pipedrive/mappers'; import { ServiceRegistry } from './services/registry.service'; +import { SalesforceService } from './services/salesforce'; import { TaskService } from './services/task.service'; import { ZendeskService } from './services/zendesk'; -import { AttioService } from './services/attio'; import { ZendeskTaskMapper } from './services/zendesk/mappers'; -import { AttioTaskMapper } from './services/attio/mappers'; import { ZohoService } from './services/zoho'; import { ZohoTaskMapper } from './services/zoho/mappers'; import { SyncService } from './sync/sync.service'; import { TaskController } from './task.controller'; +import { SalesforceTaskMapper } from './services/salesforce/mappers'; @Module({ controllers: [TaskController], providers: [ @@ -38,6 +38,7 @@ import { TaskController } from './task.controller'; HubspotService, AttioService, CloseService, + SalesforceService, /* PROVIDERS MAPPERS */ ZendeskTaskMapper, ZohoTaskMapper, @@ -46,6 +47,7 @@ import { TaskController } from './task.controller'; CloseTaskMapper, AttioTaskMapper, MicrosoftdynamicssalesService, + SalesforceTaskMapper, MicrosoftdynamicssalesTaskMapper, ], exports: [SyncService, ServiceRegistry, WebhookService], diff --git a/packages/api/src/crm/task/types/model.unified.ts b/packages/api/src/crm/task/types/model.unified.ts index 57be7f4cf..9db6b0fed 100644 --- a/packages/api/src/crm/task/types/model.unified.ts +++ b/packages/api/src/crm/task/types/model.unified.ts @@ -1,7 +1,7 @@ import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger'; import { IsIn, IsOptional, IsString, IsUUID } from 'class-validator'; -export type TaskStatus = 'PENDING' | 'COMPLETED'; +export type TaskStatus = 'PENDING' | 'COMPLETED' | string; export class UnifiedCrmTaskInput { @ApiProperty({ type: String, diff --git a/packages/api/src/crm/user/services/salesforce/index.ts b/packages/api/src/crm/user/services/salesforce/index.ts new file mode 100644 index 000000000..22badde90 --- /dev/null +++ b/packages/api/src/crm/user/services/salesforce/index.ts @@ -0,0 +1,75 @@ +import { EncryptionService } from '@@core/@core-services/encryption/encryption.service'; +import { LoggerService } from '@@core/@core-services/logger/logger.service'; +import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; +import { ApiResponse } from '@@core/utils/types'; +import { SyncParam } from '@@core/utils/types/interface'; +import { CrmObject } from '@crm/@lib/@types'; +import { IUserService } from '@crm/user/types'; +import { Injectable } from '@nestjs/common'; +import axios from 'axios'; +import { ServiceRegistry } from '../registry.service'; +import { SalesforceUserOutput, commonUserSalesforceProperties } from './types'; + +@Injectable() +export class SalesforceService implements IUserService { + constructor( + private prisma: PrismaService, + private logger: LoggerService, + private cryptoService: EncryptionService, + private registry: ServiceRegistry, + ) { + this.logger.setContext( + CrmObject.user.toUpperCase() + ':' + SalesforceService.name, + ); + this.registry.registerService('salesforce', this); + } + + async sync(data: SyncParam): Promise> { + try { + const { linkedUserId, custom_properties, pageSize, cursor } = data; + + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'salesforce', + vertical: 'crm', + }, + }); + + const instanceUrl = connection.account_url; + let pagingString = `${pageSize ? `ORDER BY Id DESC LIMIT ${pageSize} ` : ''}${ + cursor ? `OFFSET ${cursor}` : '' + }`; + if (!pageSize && !cursor) { + pagingString = 'LIMIT 200'; + } + + const commonPropertyNames = Object.keys(commonUserSalesforceProperties); + const allProperties = [...commonPropertyNames, ...custom_properties]; + const fields = allProperties.join(','); + + const query = `SELECT ${fields} FROM User ${pagingString}`; + + const resp = await axios.get( + `${instanceUrl}/services/data/v56.0/query/?q=${encodeURIComponent(query)}`, + { + headers: { + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }, + ); + + this.logger.log(`Synced Salesforce users!`); + + return { + data: resp.data.records, + message: 'Salesforce users retrieved', + statusCode: 200, + }; + } catch (error) { + throw error; + } + } +} \ No newline at end of file diff --git a/packages/api/src/crm/user/services/salesforce/mappers.ts b/packages/api/src/crm/user/services/salesforce/mappers.ts new file mode 100644 index 000000000..5d53dda62 --- /dev/null +++ b/packages/api/src/crm/user/services/salesforce/mappers.ts @@ -0,0 +1,75 @@ +import { SalesforceUserInput, SalesforceUserOutput } from './types'; +import { + UnifiedCrmUserInput, + UnifiedCrmUserOutput, +} from '@crm/user/types/model.unified'; +import { IUserMapper } from '@crm/user/types'; +import { MappersRegistry } from '@@core/@core-services/registries/mappers.registry'; +import { Injectable } from '@nestjs/common'; +import { Utils } from '@crm/@lib/@utils'; + +@Injectable() +export class SalesforceUserMapper implements IUserMapper { + constructor(private mappersRegistry: MappersRegistry, private utils: Utils) { + this.mappersRegistry.registerService('crm', 'user', 'salesforce', this); + } + + desunify( + source: UnifiedCrmUserInput, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): SalesforceUserInput { + // Salesforce doesn't typically allow creating users via API, + // so this method might not be needed. If it is, implement the logic here. + return {}; + } + + async unify( + source: SalesforceUserOutput | SalesforceUserOutput[], + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + if (!Array.isArray(source)) { + return this.mapSingleUserToUnified( + source, + connectionId, + customFieldMappings, + ); + } + // Handling array of SalesforceUserOutput + return Promise.all( + source.map((user) => + this.mapSingleUserToUnified(user, connectionId, customFieldMappings), + ), + ); + } + + private mapSingleUserToUnified( + user: SalesforceUserOutput, + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): UnifiedCrmUserOutput { + const field_mappings: { [key: string]: any } = {}; + if (customFieldMappings) { + for (const mapping of customFieldMappings) { + field_mappings[mapping.slug] = user[mapping.remote_id]; + } + } + + return { + remote_id: user.Id, + remote_data: user, + name: `${user.FirstName} ${user.LastName}`, + email: user.Email, + field_mappings, + }; + } +} \ No newline at end of file diff --git a/packages/api/src/crm/user/services/salesforce/types.ts b/packages/api/src/crm/user/services/salesforce/types.ts new file mode 100644 index 000000000..e1c46d3db --- /dev/null +++ b/packages/api/src/crm/user/services/salesforce/types.ts @@ -0,0 +1,38 @@ +export interface SalesforceUserInput { + // Salesforce doesn't typically allow creating users via API, + // so this interface might be empty or have limited fields + [key: string]: any; +} + +export interface SalesforceUserOutput { + Id: string; + Username: string; + Email: string; + FirstName: string; + LastName: string; + IsActive: boolean; + UserRoleId?: string; + ProfileId: string; + Alias: string; + TimeZoneSidKey: string; + LocaleSidKey: string; + EmailEncodingKey: string; + LanguageLocaleKey: string; + [key: string]: any; +} + +export const commonUserSalesforceProperties = { + Id: '', + Username: '', + Email: '', + FirstName: '', + LastName: '', + IsActive: false, + UserRoleId: '', + ProfileId: '', + Alias: '', + TimeZoneSidKey: '', + LocaleSidKey: '', + EmailEncodingKey: '', + LanguageLocaleKey: '', +}; \ No newline at end of file diff --git a/packages/api/src/crm/user/sync/sync.processor.ts b/packages/api/src/crm/user/sync/sync.processor.ts deleted file mode 100644 index eb6ebe0f3..000000000 --- a/packages/api/src/crm/user/sync/sync.processor.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - @Process('crm-sync-users') - async handleSyncUsers(job: Job) { - try { - console.log(`Processing queue -> crm-sync-users ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing crm users', error); - } - } -} diff --git a/packages/api/src/crm/user/sync/sync.service.ts b/packages/api/src/crm/user/sync/sync.service.ts index 50a1a9415..3c9e28d6e 100644 --- a/packages/api/src/crm/user/sync/sync.service.ts +++ b/packages/api/src/crm/user/sync/sync.service.ts @@ -35,67 +35,38 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('crm', 'user', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob('crm-sync-users', '0 0 * * *'); - } catch (error) { - throw error; - } + onModuleInit() { +// } //function used by sync worker which populate our crm_users table //its role is to fetch all users from providers 3rd parties and save the info inside our db //@Cron('*/2 * * * *') // every 2 minutes (for testing) @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log(`Syncing users....`); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = CRM_PROVIDERS.filter( - (provider) => provider !== 'zoho', - ); - for (const provider of providers) { - try { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = CRM_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } @@ -108,7 +79,9 @@ export class SyncService implements OnModuleInit, IBaseSync { const service: IUserService = this.serviceRegistry.getService(integrationId); if (!service) { - this.logger.log(`No service found in {vertical:crm, commonObject: user} for integration ID: ${integrationId}`); + this.logger.log( + `No service found in {vertical:crm, commonObject: user} for integration ID: ${integrationId}`, + ); return; } diff --git a/packages/api/src/crm/user/user.module.ts b/packages/api/src/crm/user/user.module.ts index 578c2adaf..56a7af5e4 100644 --- a/packages/api/src/crm/user/user.module.ts +++ b/packages/api/src/crm/user/user.module.ts @@ -24,6 +24,8 @@ import { ZohoService } from './services/zoho'; import { ZohoUserMapper } from './services/zoho/mappers'; import { SyncService } from './sync/sync.service'; import { UserController } from './user.controller'; +import { SalesforceService } from './services/salesforce'; +import { SalesforceUserMapper } from './services/salesforce/mappers'; @Module({ controllers: [UserController], providers: [ @@ -40,6 +42,8 @@ import { UserController } from './user.controller'; HubspotService, AttioService, CloseService, + SalesforceService, + MicrosoftdynamicssalesService, /* PROVIDERS MAPPERS */ ZendeskUserMapper, ZohoUserMapper, @@ -47,7 +51,7 @@ import { UserController } from './user.controller'; HubspotUserMapper, AttioUserMapper, CloseUserMapper, - MicrosoftdynamicssalesService, + SalesforceUserMapper, MicrosoftdynamicssalesUserMapper, AffinityService, AffinityUserMapper, diff --git a/packages/api/src/ecommerce/customer/sync/sync.processor.ts b/packages/api/src/ecommerce/customer/sync/sync.processor.ts deleted file mode 100644 index 1f01a3006..000000000 --- a/packages/api/src/ecommerce/customer/sync/sync.processor.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - - @Process('ecommerce-sync-customers') - async handleSyncCustomers(job: Job) { - try { - console.log(`Processing queue -> ecommerce-sync-customers ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing ecommerce customers', error); - } - } -} diff --git a/packages/api/src/ecommerce/customer/sync/sync.service.ts b/packages/api/src/ecommerce/customer/sync/sync.service.ts index 95b00397c..6a5c82d6f 100644 --- a/packages/api/src/ecommerce/customer/sync/sync.service.ts +++ b/packages/api/src/ecommerce/customer/sync/sync.service.ts @@ -35,65 +35,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('ecommerce', 'customer', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob( - 'ecommerce-sync-customers', - '0 0 * * *', - ); - } catch (error) { - throw error; - } + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing customers...'); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = ECOMMERCE_PROVIDERS; - for (const provider of providers) { - try { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ECOMMERCE_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/ecommerce/fulfillment/sync/sync.processor.ts b/packages/api/src/ecommerce/fulfillment/sync/sync.processor.ts deleted file mode 100644 index 2c0ecfdc7..000000000 --- a/packages/api/src/ecommerce/fulfillment/sync/sync.processor.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - - @Process('ecommerce-sync-fulfillments') - async handleSyncFulfillments(job: Job) { - try { - console.log(`Processing queue -> ecommerce-sync-fulfillments ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing ecommerce fulfillments', error); - } - } -} diff --git a/packages/api/src/ecommerce/fulfillment/sync/sync.service.ts b/packages/api/src/ecommerce/fulfillment/sync/sync.service.ts index 64a9ac89b..c9852b74b 100644 --- a/packages/api/src/ecommerce/fulfillment/sync/sync.service.ts +++ b/packages/api/src/ecommerce/fulfillment/sync/sync.service.ts @@ -34,79 +34,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.registry.registerService('ecommerce', 'fulfillment', this); } - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob( - 'ecommerce-sync-fulfillments', - '0 0 * * *', - ); - } catch (error) { - throw error; - } + onModuleInit() { + // } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing fulfillments...'); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = ECOMMERCE_PROVIDERS; - for (const provider of providers) { - try { - const connection = await this.prisma.connections.findFirst({ - where: { - id_linked_user: linkedUser.id_linked_user, - provider_slug: provider.toLowerCase(), - }, - }); - //call the sync comments for every ticket of the linkedUser (a comment is tied to a ticket) - const orders = await this.prisma.ecom_orders.findMany({ - where: { - id_connection: connection.id_connection, - }, - }); - for (const order of orders) { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - id_order: order.id_ecom_order, - }); - } - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ECOMMERCE_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/ecommerce/fulfillmentorders/sync/sync.processor.ts b/packages/api/src/ecommerce/fulfillmentorders/sync/sync.processor.ts deleted file mode 100644 index 1b454bbb0..000000000 --- a/packages/api/src/ecommerce/fulfillmentorders/sync/sync.processor.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - - @Process('ecommerce-sync-fulfillmentorderss') - async handleSyncFulfillmentOrderss(job: Job) { - try { - console.log( - `Processing queue -> ecommerce-sync-fulfillmentorderss ${job.id}`, - ); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing ecommerce fulfillmentorderss', error); - } - } -} diff --git a/packages/api/src/ecommerce/fulfillmentorders/sync/sync.service.ts b/packages/api/src/ecommerce/fulfillmentorders/sync/sync.service.ts index 716a0fe77..286afc060 100644 --- a/packages/api/src/ecommerce/fulfillmentorders/sync/sync.service.ts +++ b/packages/api/src/ecommerce/fulfillmentorders/sync/sync.service.ts @@ -55,17 +55,6 @@ export class SyncService implements OnModuleInit, IBaseSync { return; } - /*async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob( - 'ecommerce-sync-fulfillmentorderss', - '0 0 * * *', - ); - } catch (error) { - throw error; - } - }*/ - //@Cron('0 */8 * * *') // every 8 hours /*async kickstartSync(user_id?: string) { try { diff --git a/packages/api/src/ecommerce/order/sync/sync.processor.ts b/packages/api/src/ecommerce/order/sync/sync.processor.ts deleted file mode 100644 index c3d981d5c..000000000 --- a/packages/api/src/ecommerce/order/sync/sync.processor.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - - @Process('ecommerce-sync-orders') - async handleSyncOrders(job: Job) { - try { - console.log(`Processing queue -> ecommerce-sync-orders ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing ecommerce orders', error); - } - } -} diff --git a/packages/api/src/ecommerce/order/sync/sync.service.ts b/packages/api/src/ecommerce/order/sync/sync.service.ts index e24fdc2f1..e2d33bdbc 100644 --- a/packages/api/src/ecommerce/order/sync/sync.service.ts +++ b/packages/api/src/ecommerce/order/sync/sync.service.ts @@ -33,65 +33,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('ecommerce', 'order', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob( - 'ecommerce-sync-orders', - '0 0 * * *', - ); - } catch (error) { - throw error; - } + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing orders...'); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = ECOMMERCE_PROVIDERS; - for (const provider of providers) { - try { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ECOMMERCE_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/ecommerce/product/sync/sync.processor.ts b/packages/api/src/ecommerce/product/sync/sync.processor.ts deleted file mode 100644 index 28b6b41de..000000000 --- a/packages/api/src/ecommerce/product/sync/sync.processor.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - - @Process('ecommerce-sync-products') - async handleSyncProducts(job: Job) { - try { - console.log(`Processing queue -> ecommerce-sync-products ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing ecommerce products', error); - } - } -} diff --git a/packages/api/src/ecommerce/product/sync/sync.service.ts b/packages/api/src/ecommerce/product/sync/sync.service.ts index af6d4b654..ce337ad9d 100644 --- a/packages/api/src/ecommerce/product/sync/sync.service.ts +++ b/packages/api/src/ecommerce/product/sync/sync.service.ts @@ -35,65 +35,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('ecommerce', 'product', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob( - 'ecommerce-sync-products', - '0 0 * * *', - ); - } catch (error) { - throw error; - } + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing products...'); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = ECOMMERCE_PROVIDERS; - for (const provider of providers) { - try { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = ECOMMERCE_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/filestorage/drive/drive.module.ts b/packages/api/src/filestorage/drive/drive.module.ts index b0fc715fe..73dd1967a 100644 --- a/packages/api/src/filestorage/drive/drive.module.ts +++ b/packages/api/src/filestorage/drive/drive.module.ts @@ -1,14 +1,18 @@ -import { BullQueueModule } from '@@core/@core-services/queues/queue.module'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; +import { Utils } from '@filestorage/@lib/@utils'; import { Module } from '@nestjs/common'; -import { DriveController } from './drive.controller'; import { DriveService } from './services/drive.service'; +import { GoogleDriveService } from './services/googledrive'; +import { GoogleDriveMapper } from './services/googledrive/mappers'; +import { OnedriveService } from './services/onedrive'; +import { OnedriveDriveMapper } from './services/onedrive/mappers'; import { ServiceRegistry } from './services/registry.service'; +import { SharepointService } from './services/sharepoint'; +import { SharepointDriveMapper } from './services/sharepoint/mappers'; import { SyncService } from './sync/sync.service'; -import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; -import { Utils } from '@filestorage/@lib/@utils'; + @Module({ - controllers: [DriveController], providers: [ DriveService, SyncService, @@ -17,6 +21,12 @@ import { Utils } from '@filestorage/@lib/@utils'; ServiceRegistry, Utils, /* PROVIDERS SERVICES */ + OnedriveService, + GoogleDriveService, + GoogleDriveMapper, + OnedriveDriveMapper, + SharepointService, + SharepointDriveMapper, ], exports: [SyncService], }) diff --git a/packages/api/src/filestorage/drive/services/googledrive/index.ts b/packages/api/src/filestorage/drive/services/googledrive/index.ts new file mode 100644 index 000000000..945037f97 --- /dev/null +++ b/packages/api/src/filestorage/drive/services/googledrive/index.ts @@ -0,0 +1,111 @@ +import { EncryptionService } from '@@core/@core-services/encryption/encryption.service'; +import { LoggerService } from '@@core/@core-services/logger/logger.service'; +import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; +import { ApiResponse } from '@@core/utils/types'; +import { DesunifyReturnType } from '@@core/utils/types/desunify.input'; +import { SyncParam } from '@@core/utils/types/interface'; +import { FileStorageObject } from '@filestorage/@lib/@types'; +import { IDriveService } from '@filestorage/drive/types'; +import { Injectable } from '@nestjs/common'; +import { OAuth2Client } from 'google-auth-library'; +import { google } from 'googleapis'; +import { ServiceRegistry } from '../registry.service'; +import { GoogleDriveDriveOutput } from './types'; + +@Injectable() +export class GoogleDriveService implements IDriveService { + constructor( + private prisma: PrismaService, + private logger: LoggerService, + private cryptoService: EncryptionService, + private registry: ServiceRegistry, + ) { + this.logger.setContext( + `${FileStorageObject.file.toUpperCase()}:${GoogleDriveService.name}`, + ); + this.registry.registerService('googledrive', this); + } + + private async getGoogleClient(linkedUserId: string): Promise { + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'googledrive', + vertical: 'filestorage', + }, + }); + + if (!connection) { + throw new Error('Connection not found'); + } + + const oauth2Client = new google.auth.OAuth2(); + oauth2Client.setCredentials({ + access_token: this.cryptoService.decrypt(connection.access_token), + refresh_token: this.cryptoService.decrypt(connection.refresh_token), + }); + + return oauth2Client; + } + + async addDrive( + driveData: DesunifyReturnType, + linkedUserId: string, + ): Promise> { + try { + const oauth2Client = await this.getGoogleClient(linkedUserId); + const drive = google.drive({ version: 'v3', auth: oauth2Client }); + + const response = await drive.drives.create({ + requestBody: { + name: driveData.name, + }, + }); + + const createdDrive: GoogleDriveDriveOutput = { + id: response.data.id || '', + name: response.data.name || '', + kind: response.data.kind || '', + }; + + return { + data: createdDrive, + message: 'Google Drive created successfully', + statusCode: 201, + }; + } catch (error) { + this.logger.error('Error creating Google Drive', error); + throw error; + } + } + + async sync(data: SyncParam): Promise> { + try { + const { linkedUserId } = data; + const oauth2Client = await this.getGoogleClient(linkedUserId); + const drive = google.drive({ version: 'v3', auth: oauth2Client }); + + const response = await drive.drives.list({ + pageSize: 100, + }); + + const drives: GoogleDriveDriveOutput[] = (response.data.drives || []).map( + (drive) => ({ + id: drive.id || '', + name: drive.name || '', + kind: drive.kind || '', + }), + ); + this.logger.log(`Synced Google Drive drives!`); + + return { + data: drives, + message: 'Google Drive drives retrieved', + statusCode: 200, + }; + } catch (error) { + this.logger.error('Error syncing Google Drive drives', error); + throw error; + } + } +} diff --git a/packages/api/src/filestorage/drive/services/googledrive/mappers.ts b/packages/api/src/filestorage/drive/services/googledrive/mappers.ts new file mode 100644 index 000000000..f157a4483 --- /dev/null +++ b/packages/api/src/filestorage/drive/services/googledrive/mappers.ts @@ -0,0 +1,87 @@ +import { MappersRegistry } from '@@core/@core-services/registries/mappers.registry'; +import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { Utils } from '@filestorage/@lib/@utils'; +import { + UnifiedFilestorageDriveInput, + UnifiedFilestorageDriveOutput, +} from '@filestorage/drive/types/model.unified'; +import { Injectable } from '@nestjs/common'; +import { GoogleDriveDriveInput, GoogleDriveDriveOutput } from './types'; +import { IDriveMapper } from '@filestorage/drive/types'; + +@Injectable() +export class GoogleDriveMapper implements IDriveMapper { + constructor( + private mappersRegistry: MappersRegistry, + private utils: Utils, + private coreUnificationService: CoreUnification, + ) { + this.mappersRegistry.registerService( + 'filestorage', + 'drive', + 'googledrive', + this, + ); + } + + async desunify( + source: UnifiedFilestorageDriveInput, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + return { + name: source.name, + }; + } + + async unify( + source: GoogleDriveDriveOutput | GoogleDriveDriveOutput[], + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + if (!Array.isArray(source)) { + return await this.mapSingleDriveToUnified( + source, + connectionId, + customFieldMappings, + ); + } + return Promise.all( + source.map((drive) => + this.mapSingleDriveToUnified(drive, connectionId, customFieldMappings), + ), + ); + } + + private async mapSingleDriveToUnified( + drive: GoogleDriveDriveOutput, + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const field_mappings: { [key: string]: any } = {}; + if (customFieldMappings) { + for (const mapping of customFieldMappings) { + field_mappings[mapping.slug] = drive[mapping.remote_id]; + } + } + + const result: UnifiedFilestorageDriveOutput = { + remote_id: drive.id, + remote_data: drive, + name: drive.name, + remote_created_at: drive.createdTime, + drive_url: `https://drive.google.com/drive/folders/${drive.id}`, + field_mappings, + }; + + return result; + } +} diff --git a/packages/api/src/filestorage/drive/services/googledrive/types.ts b/packages/api/src/filestorage/drive/services/googledrive/types.ts new file mode 100644 index 000000000..ffe31c4b2 --- /dev/null +++ b/packages/api/src/filestorage/drive/services/googledrive/types.ts @@ -0,0 +1,48 @@ +export interface GoogleDriveDriveOutput { + id: string; + name: string; + colorRgb?: string; + kind: string; + backgroundImageLink?: string; + capabilities?: { + canAddChildren?: boolean; + canComment?: boolean; + canCopy?: boolean; + canDeleteDrive?: boolean; + canDownload?: boolean; + canEdit?: boolean; + canListChildren?: boolean; + canManageMembers?: boolean; + canReadRevisions?: boolean; + canRename?: boolean; + canRenameDrive?: boolean; + canChangeDriveBackground?: boolean; + canShare?: boolean; + canChangeCopyRequiresWriterPermissionRestriction?: boolean; + canChangeDomainUsersOnlyRestriction?: boolean; + canChangeDriveMembersOnlyRestriction?: boolean; + canChangeSharingFoldersRequiresOrganizerPermissionRestriction?: boolean; + canResetDriveRestrictions?: boolean; + canDeleteChildren?: boolean; + canTrashChildren?: boolean; + }; + themeId?: string; + backgroundImageFile?: { + id: string; + xCoordinate: number; + yCoordinate: number; + width: number; + }; + createdTime?: string; + hidden?: boolean; + restrictions?: { + copyRequiresWriterPermission?: boolean; + domainUsersOnly?: boolean; + driveMembersOnly?: boolean; + adminManagedRestrictions?: boolean; + sharingFoldersRequiresOrganizerPermission?: boolean; + }; + orgUnitId?: string; +} + +export type GoogleDriveDriveInput = Partial; diff --git a/packages/api/src/filestorage/drive/services/onedrive/index.ts b/packages/api/src/filestorage/drive/services/onedrive/index.ts new file mode 100644 index 000000000..4d6229441 --- /dev/null +++ b/packages/api/src/filestorage/drive/services/onedrive/index.ts @@ -0,0 +1,70 @@ +import { EncryptionService } from '@@core/@core-services/encryption/encryption.service'; +import { LoggerService } from '@@core/@core-services/logger/logger.service'; +import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; +import { ApiResponse } from '@@core/utils/types'; +import { SyncParam } from '@@core/utils/types/interface'; +import { FileStorageObject } from '@filestorage/@lib/@types'; +import { IDriveService } from '@filestorage/drive/types'; +import { Injectable } from '@nestjs/common'; +import axios from 'axios'; +import { ServiceRegistry } from '../registry.service'; +import { OnedriveDriveOutput } from './types'; +import { DesunifyReturnType } from '@@core/utils/types/desunify.input'; +import { OriginalDriveOutput } from '@@core/utils/types/original/original.file-storage'; + +@Injectable() +export class OnedriveService implements IDriveService { + constructor( + private prisma: PrismaService, + private logger: LoggerService, + private cryptoService: EncryptionService, + private registry: ServiceRegistry, + ) { + this.logger.setContext( + `${FileStorageObject.file.toUpperCase()}:${OnedriveService.name}`, + ); + this.registry.registerService('onedrive', this); + } + + async addDrive( + driveData: DesunifyReturnType, + linkedUserId: string, + ): Promise> { + // No API to add drive in onedrive + return; + } + + async sync(data: SyncParam): Promise> { + try { + const { linkedUserId } = data; + + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'onedrive', + vertical: 'filestorage', + }, + }); + + const resp = await axios.get(`${connection.account_url}/v1.0/drives`, { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }); + + const drives: OnedriveDriveOutput[] = resp.data.value; + this.logger.log(`Synced onedrive drives !`); + + return { + data: drives, + message: 'Onedrive drives retrived', + statusCode: 200, + }; + } catch (error) { + throw error; + } + } +} diff --git a/packages/api/src/filestorage/drive/services/onedrive/mappers.ts b/packages/api/src/filestorage/drive/services/onedrive/mappers.ts new file mode 100644 index 000000000..891150ec8 --- /dev/null +++ b/packages/api/src/filestorage/drive/services/onedrive/mappers.ts @@ -0,0 +1,86 @@ +import { MappersRegistry } from '@@core/@core-services/registries/mappers.registry'; +import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { Utils } from '@filestorage/@lib/@utils'; +import { + UnifiedFilestorageDriveInput, + UnifiedFilestorageDriveOutput, +} from '@filestorage/drive/types/model.unified'; +import { Injectable } from '@nestjs/common'; +import { OnedriveDriveInput, OnedriveDriveOutput } from './types'; +import { IDriveMapper } from '@filestorage/drive/types'; + +@Injectable() +export class OnedriveDriveMapper implements IDriveMapper { + constructor( + private mappersRegistry: MappersRegistry, + private utils: Utils, + private coreUnificationService: CoreUnification, + ) { + this.mappersRegistry.registerService( + 'filestorage', + 'drive', + 'onedrive', + this, + ); + } + + async desunify( + source: UnifiedFilestorageDriveInput, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + return; + } + + async unify( + source: OnedriveDriveOutput | OnedriveDriveOutput[], + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + if (!Array.isArray(source)) { + return await this.mapSingleDriveToUnified( + source, + connectionId, + customFieldMappings, + ); + } + // Handling array of OnedriveDriveOutput + return Promise.all( + source.map((drive) => + this.mapSingleDriveToUnified(drive, connectionId, customFieldMappings), + ), + ); + } + + private async mapSingleDriveToUnified( + drive: OnedriveDriveOutput, + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const field_mappings: { [key: string]: any } = {}; + if (customFieldMappings) { + for (const mapping of customFieldMappings) { + field_mappings[mapping.slug] = drive[mapping.remote_id]; + } + } + + const result: UnifiedFilestorageDriveOutput = { + remote_id: drive.id, + remote_data: drive, + name: drive.name, + remote_created_at: drive.createdDateTime, + drive_url: drive.webUrl, + field_mappings, + }; + + return result; + } +} diff --git a/packages/api/src/filestorage/drive/services/onedrive/types.ts b/packages/api/src/filestorage/drive/services/onedrive/types.ts new file mode 100644 index 000000000..9cc1f591d --- /dev/null +++ b/packages/api/src/filestorage/drive/services/onedrive/types.ts @@ -0,0 +1,130 @@ +/** + * Represents the response from the OneDrive API for a specific drive. + * @see https://learn.microsoft.com/en-us/graph/api/resources/drive?view=graph-rest-1.0 + */ +export interface OnedriveDriveOutput { + /** The date and time when the drive was created. */ + readonly createdDateTime: string; + /** A user-visible description of the drive. */ + description: string; + /** The unique identifier of the drive. */ + readonly id: string; + /** The date and time when the drive was last modified. */ + readonly lastModifiedDateTime: string; + /** The name of the drive. */ + name: string; + /** URL that displays the resource in the browser. */ + readonly webUrl: string; + /** Describes the type of drive represented by this resource. */ + readonly driveType: 'personal' | 'business' | 'documentLibrary'; + /** Identity of the user, device, or application which created the item. Read-only. */ + readonly createdBy: IdentitySet; + /** Identity of the user, device, and application which last modified the item. Read-only. */ + readonly lastModifiedBy: IdentitySet; + /** The user account that owns the drive. */ + readonly owner?: IdentitySet; + /** Information about the drive's storage space quota. */ + readonly quota?: Quota; + /** SharePoint identifiers for REST compatibility. */ + readonly sharepointIds?: SharepointIds; + /** Indicates that this is a system-managed drive. */ + readonly system?: SystemFacet; +} + +/** + * Represents a set of identities, such as user, device, or application identities. + * @see https://learn.microsoft.com/en-us/graph/api/resources/identityset?view=graph-rest-1.0 + */ +export interface IdentitySet { + /** Identity representing an application. */ + readonly application?: Identity; + /** Identity representing an application instance. */ + readonly applicationInstance?: Identity; + /** Identity representing a conversation. */ + readonly conversation?: Identity; + /** Identity representing a conversation identity type. */ + readonly conversationIdentityType?: Identity; + /** Identity representing a device. */ + readonly device?: Identity; + /** Identity representing encrypted identity information. */ + readonly encrypted?: Identity; + /** Identity representing an on-premises identity. */ + readonly onPremises?: Identity; + /** Identity representing a guest user. */ + readonly guest?: Identity; + /** Identity representing a phone identity. */ + readonly phone?: Identity; + /** Identity representing a user. */ + readonly user?: Identity; +} + +/** + * Represents a generic identity used in various identity sets. + * @see https://learn.microsoft.com/en-us/graph/api/resources/identity?view=graph-rest-1.0 + */ +export interface Identity { + /** The display name of the identity. */ + readonly displayName?: string; + /** The ID of the identity. */ + readonly id?: string; + /** The identity type (such as user, application, or device). */ + readonly identityType?: string; + /** The email address of the identity. */ + readonly email?: string; +} + +/** + * Represents the storage quota information of a drive. + */ +export interface Quota { + /** The total number of bytes deleted from the drive. */ + readonly deleted: number; + /** The total number of bytes remaining in the drive's quota. */ + readonly remaining: number; + /** The state of the drive's quota (e.g., normal, nearing, exceeded). */ + readonly state: 'normal' | 'nearing' | 'critical' | 'exceeded'; + /** The total number of bytes in the drive's quota. */ + readonly total: number; + /** The total number of bytes used in the drive. */ + readonly used: number; + /** Information about storage plan upgrades, if available. */ + readonly storagePlanInformation?: StoragePlanInformation; +} + +/** + * Represents storage plan upgrade information. + */ +export interface StoragePlanInformation { + /** Indicates whether an upgrade is available for the storage plan. */ + readonly upgradeAvailable: boolean; +} + +/** + * Represents SharePoint-specific identifiers for an item. + */ +export interface SharepointIds { + /** The unique identifier (GUID) for the item's list in SharePoint. */ + readonly listId: string; + /** An integer identifier for the item within the containing list. */ + readonly listItemId: string; + /** The unique identifier (GUID) for the item within OneDrive for Business or a SharePoint site. */ + readonly listItemUniqueId: string; + /** The unique identifier (GUID) for the item's site collection (SPSite). */ + readonly siteId: string; + /** The SharePoint URL for the site that contains the item. */ + readonly siteUrl: string; + /** The unique identifier (GUID) for the tenancy. */ + readonly tenantId: string; + /** The unique identifier (GUID) for the item's site (SPWeb). */ + readonly webId: string; +} + +/** + * Represents system-related metadata for the drive. + */ +export interface SystemFacet { + // Add properties specific to the system facet if needed. + readonly [key: string]: any; +} + +export type OnedriveDriveInput = Partial; diff --git a/packages/api/src/filestorage/drive/services/sharepoint/index.ts b/packages/api/src/filestorage/drive/services/sharepoint/index.ts new file mode 100644 index 000000000..9fece13b1 --- /dev/null +++ b/packages/api/src/filestorage/drive/services/sharepoint/index.ts @@ -0,0 +1,71 @@ +import { EncryptionService } from '@@core/@core-services/encryption/encryption.service'; +import { LoggerService } from '@@core/@core-services/logger/logger.service'; +import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; +import { ApiResponse } from '@@core/utils/types'; +import { SyncParam } from '@@core/utils/types/interface'; +import { FileStorageObject } from '@filestorage/@lib/@types'; +import { IDriveService } from '@filestorage/drive/types'; +import { Injectable } from '@nestjs/common'; +import axios from 'axios'; +import { ServiceRegistry } from '../registry.service'; +import { SharepointDriveOutput } from './types'; +import { DesunifyReturnType } from '@@core/utils/types/desunify.input'; +import { OriginalDriveOutput } from '@@core/utils/types/original/original.file-storage'; + +@Injectable() +export class SharepointService implements IDriveService { + constructor( + private prisma: PrismaService, + private logger: LoggerService, + private cryptoService: EncryptionService, + private registry: ServiceRegistry, + ) { + this.logger.setContext( + `${FileStorageObject.file.toUpperCase()}:${SharepointService.name}`, + ); + this.registry.registerService('sharepoint', this); + } + + async addDrive( + driveData: DesunifyReturnType, + linkedUserId: string, + ): Promise> { + // No API to add drive in Sharepoint + return; + } + + async sync(data: SyncParam): Promise> { + try { + const { linkedUserId } = data; + + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'sharepoint', + vertical: 'filestorage', + }, + }); + + const resp = await axios.get(`${connection.account_url}/drives`, { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }); + + const drives: SharepointDriveOutput[] = resp.data.value; + this.logger.log(`Synced sharepoint drives !`); + + return { + data: drives, + message: 'Sharepoint drives retrived', + statusCode: 200, + }; + } catch (error) { + console.log(error.response); + throw error; + } + } +} diff --git a/packages/api/src/filestorage/drive/services/sharepoint/mappers.ts b/packages/api/src/filestorage/drive/services/sharepoint/mappers.ts new file mode 100644 index 000000000..4b11e0283 --- /dev/null +++ b/packages/api/src/filestorage/drive/services/sharepoint/mappers.ts @@ -0,0 +1,86 @@ +import { MappersRegistry } from '@@core/@core-services/registries/mappers.registry'; +import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { Utils } from '@filestorage/@lib/@utils'; +import { + UnifiedFilestorageDriveInput, + UnifiedFilestorageDriveOutput, +} from '@filestorage/drive/types/model.unified'; +import { Injectable } from '@nestjs/common'; +import { SharepointDriveInput, SharepointDriveOutput } from './types'; +import { IDriveMapper } from '@filestorage/drive/types'; + +@Injectable() +export class SharepointDriveMapper implements IDriveMapper { + constructor( + private mappersRegistry: MappersRegistry, + private utils: Utils, + private coreUnificationService: CoreUnification, + ) { + this.mappersRegistry.registerService( + 'filestorage', + 'drive', + 'sharepoint', + this, + ); + } + + async desunify( + source: UnifiedFilestorageDriveInput, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + return; + } + + async unify( + source: SharepointDriveOutput | SharepointDriveOutput[], + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + if (!Array.isArray(source)) { + return await this.mapSingleDriveToUnified( + source, + connectionId, + customFieldMappings, + ); + } + // Handling array of SharepointDriveOutput + return Promise.all( + source.map((drive) => + this.mapSingleDriveToUnified(drive, connectionId, customFieldMappings), + ), + ); + } + + private async mapSingleDriveToUnified( + drive: SharepointDriveOutput, + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const field_mappings: { [key: string]: any } = {}; + if (customFieldMappings) { + for (const mapping of customFieldMappings) { + field_mappings[mapping.slug] = drive[mapping.remote_id]; + } + } + + const result: UnifiedFilestorageDriveOutput = { + remote_id: drive.id, + remote_data: drive, + name: drive.name, + remote_created_at: drive.createdDateTime, + drive_url: drive.webUrl, + field_mappings, + }; + + return result; + } +} diff --git a/packages/api/src/filestorage/drive/services/sharepoint/types.ts b/packages/api/src/filestorage/drive/services/sharepoint/types.ts new file mode 100644 index 000000000..8ce603089 --- /dev/null +++ b/packages/api/src/filestorage/drive/services/sharepoint/types.ts @@ -0,0 +1,130 @@ +/** + * Represents the response from the Sharepoint API for a specific drive. + * @see https://learn.microsoft.com/en-us/graph/api/resources/drive?view=graph-rest-1.0 + */ +export interface SharepointDriveOutput { + /** The date and time when the drive was created. */ + readonly createdDateTime: string; + /** A user-visible description of the drive. */ + description: string; + /** The unique identifier of the drive. */ + readonly id: string; + /** The date and time when the drive was last modified. */ + readonly lastModifiedDateTime: string; + /** The name of the drive. */ + name: string; + /** URL that displays the resource in the browser. */ + readonly webUrl: string; + /** Describes the type of drive represented by this resource. */ + readonly driveType: 'personal' | 'business' | 'documentLibrary'; + /** Identity of the user, device, or application which created the item. Read-only. */ + readonly createdBy: IdentitySet; + /** Identity of the user, device, and application which last modified the item. Read-only. */ + readonly lastModifiedBy: IdentitySet; + /** The user account that owns the drive. */ + readonly owner?: IdentitySet; + /** Information about the drive's storage space quota. */ + readonly quota?: Quota; + /** SharePoint identifiers for REST compatibility. */ + readonly sharepointIds?: SharepointIds; + /** Indicates that this is a system-managed drive. */ + readonly system?: SystemFacet; +} + +/** + * Represents a set of identities, such as user, device, or application identities. + * @see https://learn.microsoft.com/en-us/graph/api/resources/identityset?view=graph-rest-1.0 + */ +export interface IdentitySet { + /** Identity representing an application. */ + readonly application?: Identity; + /** Identity representing an application instance. */ + readonly applicationInstance?: Identity; + /** Identity representing a conversation. */ + readonly conversation?: Identity; + /** Identity representing a conversation identity type. */ + readonly conversationIdentityType?: Identity; + /** Identity representing a device. */ + readonly device?: Identity; + /** Identity representing encrypted identity information. */ + readonly encrypted?: Identity; + /** Identity representing an on-premises identity. */ + readonly onPremises?: Identity; + /** Identity representing a guest user. */ + readonly guest?: Identity; + /** Identity representing a phone identity. */ + readonly phone?: Identity; + /** Identity representing a user. */ + readonly user?: Identity; +} + +/** + * Represents a generic identity used in various identity sets. + * @see https://learn.microsoft.com/en-us/graph/api/resources/identity?view=graph-rest-1.0 + */ +export interface Identity { + /** The display name of the identity. */ + readonly displayName?: string; + /** The ID of the identity. */ + readonly id?: string; + /** The identity type (such as user, application, or device). */ + readonly identityType?: string; + /** The email address of the identity. */ + readonly email?: string; +} + +/** + * Represents the storage quota information of a drive. + */ +export interface Quota { + /** The total number of bytes deleted from the drive. */ + readonly deleted: number; + /** The total number of bytes remaining in the drive's quota. */ + readonly remaining: number; + /** The state of the drive's quota (e.g., normal, nearing, exceeded). */ + readonly state: 'normal' | 'nearing' | 'critical' | 'exceeded'; + /** The total number of bytes in the drive's quota. */ + readonly total: number; + /** The total number of bytes used in the drive. */ + readonly used: number; + /** Information about storage plan upgrades, if available. */ + readonly storagePlanInformation?: StoragePlanInformation; +} + +/** + * Represents storage plan upgrade information. + */ +export interface StoragePlanInformation { + /** Indicates whether an upgrade is available for the storage plan. */ + readonly upgradeAvailable: boolean; +} + +/** + * Represents SharePoint-specific identifiers for an item. + */ +export interface SharepointIds { + /** The unique identifier (GUID) for the item's list in SharePoint. */ + readonly listId: string; + /** An integer identifier for the item within the containing list. */ + readonly listItemId: string; + /** The unique identifier (GUID) for the item within OneDrive for Business or a SharePoint site. */ + readonly listItemUniqueId: string; + /** The unique identifier (GUID) for the item's site collection (SPSite). */ + readonly siteId: string; + /** The SharePoint URL for the site that contains the item. */ + readonly siteUrl: string; + /** The unique identifier (GUID) for the tenancy. */ + readonly tenantId: string; + /** The unique identifier (GUID) for the item's site (SPWeb). */ + readonly webId: string; +} + +/** + * Represents system-related metadata for the drive. + */ +export interface SystemFacet { + // Add properties specific to the system facet if needed. + readonly [key: string]: any; +} + +export type SharepointDriveInput = Partial; diff --git a/packages/api/src/filestorage/drive/sync/sync.processor.ts b/packages/api/src/filestorage/drive/sync/sync.processor.ts deleted file mode 100644 index 7efcb5613..000000000 --- a/packages/api/src/filestorage/drive/sync/sync.processor.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - - @Process('filestorage-sync-drives') - async handleSyncDrives(job: Job) { - try { - console.log(`Processing queue -> filestorage-sync-drives ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing filestorage drives', error); - } - } -} diff --git a/packages/api/src/filestorage/drive/sync/sync.service.ts b/packages/api/src/filestorage/drive/sync/sync.service.ts index 8101ed061..7ef892a7b 100644 --- a/packages/api/src/filestorage/drive/sync/sync.service.ts +++ b/packages/api/src/filestorage/drive/sync/sync.service.ts @@ -32,65 +32,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('filestorage', 'drive', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob( - 'filestorage-sync-drives', - '0 0 * * *', - ); - } catch (error) { - throw error; - } + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing drives...'); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = FILESTORAGE_PROVIDERS; - for (const provider of providers) { - try { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = FILESTORAGE_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/filestorage/file/file.module.ts b/packages/api/src/filestorage/file/file.module.ts index 87661dd95..b46829cee 100644 --- a/packages/api/src/filestorage/file/file.module.ts +++ b/packages/api/src/filestorage/file/file.module.ts @@ -1,16 +1,22 @@ -import { BullQueueModule } from '@@core/@core-services/queues/queue.module'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; +import { Utils } from '@filestorage/@lib/@utils'; import { Module } from '@nestjs/common'; import { FileController } from './file.controller'; import { BoxService } from './services/box'; import { BoxFileMapper } from './services/box/mappers'; +import { DropboxService } from './services/dropbox'; +import { DropboxFileMapper } from './services/dropbox/mappers'; import { FileService } from './services/file.service'; +import { GoogleDriveService } from './services/googledrive'; +import { GoogleDriveFileMapper } from './services/googledrive/mappers'; +import { OnedriveService } from './services/onedrive'; +import { OnedriveFileMapper } from './services/onedrive/mappers'; import { ServiceRegistry } from './services/registry.service'; +import { SharepointService } from './services/sharepoint'; +import { SharepointFileMapper } from './services/sharepoint/mappers'; import { SyncService } from './sync/sync.service'; -import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; -import { Utils } from '@filestorage/@lib/@utils'; - @Module({ controllers: [FileController], providers: [ @@ -22,9 +28,18 @@ import { Utils } from '@filestorage/@lib/@utils'; Utils, /* MAPPERS SERVICES */ BoxFileMapper, + OnedriveFileMapper, + GoogleDriveFileMapper, /* PROVIDERS SERVICES */ BoxService, + SharepointService, + SharepointFileMapper, + OnedriveService, + OnedriveFileMapper, + DropboxService, + DropboxFileMapper, + GoogleDriveService, ], - exports: [SyncService], + exports: [SyncService, ServiceRegistry], }) export class FileModule {} diff --git a/packages/api/src/filestorage/file/services/box/mappers.ts b/packages/api/src/filestorage/file/services/box/mappers.ts index 75e979a05..b6eeeab90 100644 --- a/packages/api/src/filestorage/file/services/box/mappers.ts +++ b/packages/api/src/filestorage/file/services/box/mappers.ts @@ -90,12 +90,13 @@ export class BoxFileMapper implements IFileMapper { remote_id: file.id, remote_data: file, name: file.name || null, - type: file.extension || null, + //type: file.extension || null, file_url: file.shared_link?.url || null, mime_type: this.utils.getMimeType(file.name) || null, size: file.size?.toString() || null, permission: null, field_mappings, + folder_id: null, ...opts, //remote_created_at: file.created_at || null, //remote_modified_at: file.modified_at || null, diff --git a/packages/api/src/filestorage/file/services/dropbox/index.ts b/packages/api/src/filestorage/file/services/dropbox/index.ts new file mode 100644 index 000000000..e6da90b7b --- /dev/null +++ b/packages/api/src/filestorage/file/services/dropbox/index.ts @@ -0,0 +1,113 @@ +import { EncryptionService } from '@@core/@core-services/encryption/encryption.service'; +import { LoggerService } from '@@core/@core-services/logger/logger.service'; +import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; +import { ApiResponse } from '@@core/utils/types'; +import { SyncParam } from '@@core/utils/types/interface'; +import { FileStorageObject } from '@filestorage/@lib/@types'; +import { IFileService } from '@filestorage/file/types'; +import { Injectable } from '@nestjs/common'; +import axios from 'axios'; +import { ServiceRegistry } from '../registry.service'; +import { DropboxFileOutput } from './types'; +import { UnifiedFilestorageFolderOutput } from '@filestorage/folder/types/model.unified'; + +@Injectable() +export class DropboxService implements IFileService { + constructor( + private prisma: PrismaService, + private logger: LoggerService, + private cryptoService: EncryptionService, + private registry: ServiceRegistry, + ) { + this.logger.setContext( + FileStorageObject.file.toUpperCase() + ':' + DropboxService.name, + ); + this.registry.registerService('dropbox', this); + } + + async getAllFilesInFolder( + folderPath: string, + connection: any, + ): Promise { + // ref: https://www.dropbox.com/developers/documentation/http/documentation#files-list_folder + const files: DropboxFileOutput[] = []; + let cursor: string | null = null; + let hasMore = true; + + while (hasMore) { + const url = cursor + ? `${connection.account_url}/files/list_folder/continue` + : `${connection.account_url}/files/list_folder`; + + const data = cursor ? { cursor } : { path: folderPath, recursive: false }; + + try { + const response = await axios.post(url, data, { + headers: { + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + 'Content-Type': 'application/json', + }, + }); + + const { entries, has_more, cursor: newCursor } = response.data; + + // Collect all file entries + files.push(...entries.filter((entry: any) => entry['.tag'] === 'file')); + + hasMore = has_more; + cursor = newCursor; + } catch (error) { + console.error('Error listing files in folder:', error); + throw new Error('Failed to list all files in the folder.'); + } + } + + return files; + } + + async sync(data: SyncParam): Promise> { + try { + const { linkedUserId, id_folder } = data; + if (!id_folder) return; + + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'dropbox', + vertical: 'filestorage', + }, + }); + + const folder = await this.prisma.fs_folders.findUnique({ + where: { + id_fs_folder: id_folder as string, + }, + }); + + const remote_data = await this.prisma.remote_data.findFirst({ + where: { + ressource_owner_id: folder.id_fs_folder, + }, + }); + + const folder_remote_data = JSON.parse(remote_data.data); + + const files = await this.getAllFilesInFolder( + folder_remote_data.path_display, + connection, + ); + + this.logger.log(`Synced dropbox files !`); + + return { + data: files, + message: 'Dropbox files retrieved', + statusCode: 200, + }; + } catch (error) { + throw error; + } + } +} diff --git a/packages/api/src/filestorage/file/services/dropbox/mappers.ts b/packages/api/src/filestorage/file/services/dropbox/mappers.ts new file mode 100644 index 000000000..686601082 --- /dev/null +++ b/packages/api/src/filestorage/file/services/dropbox/mappers.ts @@ -0,0 +1,97 @@ +import { MappersRegistry } from '@@core/@core-services/registries/mappers.registry'; +import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { OriginalSharedLinkOutput } from '@@core/utils/types/original/original.file-storage'; +import { FileStorageObject } from '@filestorage/@lib/@types'; +import { Utils } from '@filestorage/@lib/@utils'; +import { IFileMapper } from '@filestorage/file/types'; +import { + UnifiedFilestorageFileInput, + UnifiedFilestorageFileOutput, +} from '@filestorage/file/types/model.unified'; +import { UnifiedFilestorageSharedlinkOutput } from '@filestorage/sharedlink/types/model.unified'; +import { Injectable } from '@nestjs/common'; +import { DropboxFileInput, DropboxFileOutput } from './types'; + +@Injectable() +export class DropboxFileMapper implements IFileMapper { + constructor( + private mappersRegistry: MappersRegistry, + private utils: Utils, + private coreUnificationService: CoreUnification, + ) { + this.mappersRegistry.registerService( + 'filestorage', + 'file', + 'dropbox', + this, + ); + } + + async desunify( + source: UnifiedFilestorageFileInput, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + // todo: do something with customFieldMappings + return { + path: `/${source.name}`, + mode: 'add', + autorename: true, + }; + } + + async unify( + source: DropboxFileOutput | DropboxFileOutput[], + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + if (!Array.isArray(source)) { + return await this.mapSingleFileToUnified( + source, + connectionId, + customFieldMappings, + ); + } + // Handling array of DropboxFileOutput + return Promise.all( + source.map((file) => + this.mapSingleFileToUnified(file, connectionId, customFieldMappings), + ), + ); + } + + private async mapSingleFileToUnified( + file: DropboxFileOutput, + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const result: UnifiedFilestorageFileOutput = { + remote_id: file.id, + remote_data: file, + name: file.name, + file_url: null, + mime_type: null, + size: file.size.toString(), + folder_id: null, + permission: null, + shared_link: null, + field_mappings: {}, + }; + + if (customFieldMappings) { + for (const mapping of customFieldMappings) { + result.field_mappings[mapping.slug] = file[mapping.remote_id]; + } + } + + return result; + } +} diff --git a/packages/api/src/filestorage/file/services/dropbox/types.ts b/packages/api/src/filestorage/file/services/dropbox/types.ts new file mode 100644 index 000000000..9cd5ac9c3 --- /dev/null +++ b/packages/api/src/filestorage/file/services/dropbox/types.ts @@ -0,0 +1,232 @@ +/** + * Represents a file-specific entry in the Dropbox API. + */ +export interface DropboxFileOutput { + /** + * A constant tag indicating the entry is a file. + * Value will always be `'file'`. + */ + '.tag': 'file'; + + /** + * The name of the file. + */ + name: string; + + /** + * The lowercased path of the file, useful for case-insensitive comparisons. + */ + path_lower: string; + + /** + * The display path of the file, with original casing. + */ + path_display: string; + + /** + * The Dropbox unique identifier for the file. + */ + id: string; + + /** + * The size of the file in bytes. + */ + size: number; + + /** + * A hash of the file content, useful for detecting file changes. + */ + content_hash: string; + + /** + * The revision number of the file. + * Useful for file versioning. + */ + rev: string; + + /** + * The timestamp of when the file was last modified on the client. + */ + client_modified: string; + + /** + * The timestamp of when the file was last modified on the Dropbox server. + */ + server_modified: string; + + /** + * Whether the file is downloadable. + */ + is_downloadable: boolean; + + /** + * Information about file sharing, such as if it is read-only or shared. + * This field is present if the file is part of a shared folder. + */ + sharing_info?: SharingInfo; + + /** + * Information about the export of the file, if it's an exportable file (e.g., Google Docs). + */ + export_info?: ExportInfo; + + /** + * The property groups associated with the file. + */ + property_groups?: PropertyGroup[]; + + /** + * Indicates whether the file has any explicit member policy. + */ + has_explicit_shared_members?: boolean; + + /** + * Information about file locking, if applicable. + */ + file_lock_info?: FileLockInfo; +} + +/** + * Represents sharing information for a file. + */ +export interface SharingInfo { + /** + * Whether the file is read-only for the current user. + */ + read_only: boolean; + + /** + * The ID of the parent shared folder, if this file is inside a shared folder. + */ + parent_shared_folder_id?: string; + + /** + * The unique ID of the shared folder. + */ + shared_folder_id?: string; + + /** + * Whether the file can be shared externally. + */ + traverse_only?: boolean; + + /** + * Whether the user has permission to manage sharing. + */ + no_access?: boolean; +} + +/** + * Represents export information for a file, if applicable. + */ +export interface ExportInfo { + /** + * The format to which the file can be exported (e.g., pdf, docx). + */ + export_as: string; +} + +/** + * Represents a property group associated with the file. + */ +export interface PropertyGroup { + /** + * The template ID of the property group. + */ + template_id: string; + + /** + * The list of properties under this group. + */ + fields: PropertyField[]; +} + +/** + * Represents a property field in a property group. + */ +export interface PropertyField { + /** + * The name of the property. + */ + name: string; + + /** + * The value of the property. + */ + value: string; +} +/** + * Represents a file lock information. + */ +export interface FileLockInfo { + /** + * The timestamp when the lock was created. + */ + created: string; + + /** + * Whether the user is the lockholder. + */ + is_lockholder: boolean; + + /** + * The name of the lockholder. + */ + lockholder_name: string; +} + +/** + * Represents the request body for uploading a new file in Dropbox. + */ +export interface DropboxFileInput { + /** + * The path in the user's Dropbox to save the file. + * Must match the pattern `(/(.|[\r\n])*)|(ns:[0-9]+(/.*)?)|(id:.*)?` + * Example: "/new_folder/myfile.txt" + */ + path: string; + + /** + * Selects what to do if the file already exists. + * The default for this union is "add". + * Options: "add", "overwrite", "update" + */ + mode: 'add' | 'overwrite' | 'update'; + + /** + * If true, Dropbox will automatically rename the file in case of a conflict. + * The default is false. + */ + autorename?: boolean; + + /** + * The value to store as the client_modified timestamp. + * Optional field in ISO 8601 format (e.g., "2024-09-12T14:00:00Z"). + */ + client_modified?: string; + + /** + * If true, suppresses user notifications about this file modification. + * The default is false. + */ + mute?: boolean; + + /** + * List of custom properties to add to the file. + * Optional field. + */ + property_groups?: PropertyGroup[]; + + /** + * If true, enforces stricter conflict detection. + * Defaults to false. + */ + strict_conflict?: boolean; + + /** + * A hash of the file content uploaded in this call. + * If provided, the uploaded content must match this hash. + * Optional field with length between 64 characters. + */ + content_hash?: string; +} diff --git a/packages/api/src/filestorage/file/services/googledrive/index.ts b/packages/api/src/filestorage/file/services/googledrive/index.ts new file mode 100644 index 000000000..5b1795eee --- /dev/null +++ b/packages/api/src/filestorage/file/services/googledrive/index.ts @@ -0,0 +1,318 @@ +import { EncryptionService } from '@@core/@core-services/encryption/encryption.service'; +import { LoggerService } from '@@core/@core-services/logger/logger.service'; +import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; +import { BUCKET_NAME } from '@@core/s3/constants'; +import { ApiResponse } from '@@core/utils/types'; +import { SyncParam } from '@@core/utils/types/interface'; +import { S3Client } from '@aws-sdk/client-s3'; +import { Upload } from '@aws-sdk/lib-storage'; +import { FileStorageObject } from '@filestorage/@lib/@types'; +import { IFileService } from '@filestorage/file/types'; +import { Injectable } from '@nestjs/common'; +import { OAuth2Client } from 'google-auth-library'; +import { google } from 'googleapis'; +import * as mammoth from 'mammoth'; +import * as marked from 'marked'; +import { Readable } from 'stream'; +import * as XLSX from 'xlsx'; +import { ServiceRegistry } from '../registry.service'; +import { GoogleDriveFileOutput } from './types'; + +@Injectable() +export class GoogleDriveService implements IFileService { + constructor( + private prisma: PrismaService, + private logger: LoggerService, + private cryptoService: EncryptionService, + private registry: ServiceRegistry, + ) { + this.logger.setContext( + FileStorageObject.file.toUpperCase() + ':' + GoogleDriveService.name, + ); + this.registry.registerService('googledrive', this); + } + + async sync(data: SyncParam): Promise> { + try { + const { linkedUserId, id_folder } = data; + + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'googledrive', + vertical: 'filestorage', + }, + }); + + if (!connection) return; + + const auth = new OAuth2Client(); + auth.setCredentials({ + access_token: this.cryptoService.decrypt(connection.access_token), + }); + const drive = google.drive({ version: 'v3', auth }); + + const response = await drive.files.list({ + q: 'trashed = false', + fields: + 'files(id, name, mimeType, modifiedTime, size, parents, webViewLink)', + pageSize: 1000, // Adjust as needed + }); + + const files: GoogleDriveFileOutput[] = response.data.files.map( + (file) => ({ + id: file.id!, + name: file.name!, + mimeType: file.mimeType!, + modifiedTime: file.modifiedTime!, + size: file.size!, + parents: file.parents, + webViewLink: file.webViewLink, + }), + ); + this.logger.log(`Synced googledrive files !`); + + return { + data: files, + message: 'Google Drive files retrieved', + statusCode: 200, + }; + } catch (error) { + throw error; + } + } + + extractFileId(url: string): string | null { + const match = url.match(/\/d\/([^/]+)/); + return match ? match[1] : null; + } + + async streamFileToS3( + file_id: string, + linkedUserId: string, + s3Client: S3Client, + s3Key: string, + ) { + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'googledrive', + vertical: 'filestorage', + }, + }); + + const file = await this.prisma.fs_files.findUnique({ + where: { + id_fs_file: file_id, + }, + }); + const auth = new OAuth2Client(); + auth.setCredentials({ + access_token: this.cryptoService.decrypt(connection.access_token), + }); + const drive = google.drive({ version: 'v3', auth }); + + const fileUniqueIdentifier = this.extractFileId(file.file_url); + // Get file metadata + const fileMetadata = await drive.files.get({ + fileId: fileUniqueIdentifier, + fields: 'name,mimeType', + }); + + let processedContent: string | Buffer; + const mimeType = fileMetadata.data.mimeType; + let contentType = 'text/plain'; // Default content type + + switch (mimeType) { + case 'application/pdf': + contentType = 'application/pdf'; + processedContent = await this.downloadFile(drive, fileUniqueIdentifier); + break; + case 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': + case 'text/csv': + contentType = 'text/csv'; + processedContent = await this.downloadFile(drive, fileUniqueIdentifier); + break; + case 'text/tab-separated-values': + processedContent = await this.processSpreadsheet( + drive, + fileUniqueIdentifier, + mimeType, + ); + break; + case 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': + case 'text/plain': + case 'text/markdown': + case 'application/rtf': + case 'application/vnd.openxmlformats-officedocument.presentationml.presentation': + case 'text/html': + case 'message/rfc822': + case 'application/vnd.ms-outlook': + processedContent = await this.processTextContent( + drive, + fileUniqueIdentifier, + mimeType, + ); + break; + case 'application/json': + processedContent = await this.processJsonContent( + drive, + fileUniqueIdentifier, + ); + break; + default: + throw new Error(`Unsupported file type: ${mimeType}`); + } + + const upload = new Upload({ + client: s3Client, + params: { + Bucket: BUCKET_NAME, + Key: s3Key, + Body: Readable.from(processedContent), + ContentType: contentType, + ContentDisposition: `attachment; filename="${fileMetadata.data.name}"`, + }, + }); + + try { + await upload.done(); + console.log(`Successfully uploaded ${s3Key} to ${BUCKET_NAME}`); + } catch (error) { + console.error('Error uploading to S3:', error); + throw error; + } + } + private async downloadFile(drive: any, fileId: string): Promise { + const response = await drive.files.get( + { fileId, alt: 'media' }, + { responseType: 'arraybuffer' }, + ); + return Buffer.from(response.data); + } + + private async processSpreadsheet( + drive: any, + fileId: string, + mimeType: string, + ): Promise { + const fileContent = await this.downloadFile(drive, fileId); + let result = ''; + + if (mimeType === 'text/csv') { + const content = fileContent.toString('utf-8'); + const lines = content.split('\n').filter((line) => line.trim() !== ''); + + if (lines.length === 0) { + return 'Empty CSV file'; + } + + // Detect separator + const possibleSeparators = [',', ';', '\t', '|']; + const firstLine = lines[0]; + const separator = + possibleSeparators.find((sep) => firstLine.includes(sep)) || ','; + + // Extract headers and determine the number of columns + const headerMatch = firstLine.match(/^(.*?):(.*)/); + const headers = headerMatch + ? headerMatch[2].split(separator).map((h) => h.trim()) + : firstLine.split(separator).map((h) => h.trim()); + const columnCount = headers.length; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const dataMatch = line.match(/^(.*?):(.*)/); + const values = dataMatch + ? dataMatch[2].split(separator).map((v) => v.trim()) + : line.split(separator).map((v) => v.trim()); + + if (values.length === columnCount) { + for (let j = 0; j < columnCount; j++) { + result += `${headers[j]}: ${values[j]}\n`; + } + result += '\n'; + } + } + } else if ( + mimeType === + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' + ) { + const workbook = XLSX.read(fileContent, { type: 'buffer' }); + workbook.SheetNames.forEach((sheetName) => { + const sheet = workbook.Sheets[sheetName]; + const data = XLSX.utils.sheet_to_json(sheet, { header: 1 }) as any[][]; + const headers = data[0] as string[]; + for (let i = 1; i < data.length; i++) { + const row = data[i] as any[]; + for (let j = 0; j < row.length; j++) { + result += `${headers[j]}: ${row[j]}\n`; + } + result += '\n'; + } + }); + } else { + throw new Error(`Unsupported spreadsheet type: ${mimeType}`); + } + + return result; + } + private async processTextContent( + drive: any, + fileId: string, + mimeType: string, + ): Promise { + const fileContent = await this.downloadFile(drive, fileId); + if ( + mimeType === + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' + ) { + const result = await mammoth.extractRawText({ buffer: fileContent }); + return result.value; + } else if (mimeType === 'text/markdown') { + return marked.parse(fileContent.toString()); + } else { + // For other text-based formats, we'll assume they're already in plain text + return fileContent.toString(); + } + } + private async processJsonContent( + drive: any, + fileId: string, + ): Promise { + const fileContent = await this.downloadFile(drive, fileId); + const jsonContent = JSON.parse(fileContent.toString()); + + function flattenObject(obj: any, prefix = ''): { [key: string]: any } { + return Object.keys(obj).reduce( + (acc: { [key: string]: any }, k: string) => { + const pre = prefix.length ? prefix + '.' : ''; + if ( + typeof obj[k] === 'object' && + obj[k] !== null && + !Array.isArray(obj[k]) + ) { + Object.assign(acc, flattenObject(obj[k], pre + k)); + } else if (Array.isArray(obj[k])) { + obj[k].forEach((item: any, index: number) => { + if (typeof item === 'object' && item !== null) { + Object.assign(acc, flattenObject(item, `${pre}${k}[${index}]`)); + } else { + acc[`${pre}${k}[${index}]`] = item; + } + }); + } else { + acc[pre + k] = obj[k]; + } + return acc; + }, + {}, + ); + } + + const flattened = flattenObject(jsonContent); + return Object.entries(flattened) + .map(([key, value]) => `${key}: ${value}`) + .join('\n'); + } +} diff --git a/packages/api/src/filestorage/file/services/googledrive/mappers.ts b/packages/api/src/filestorage/file/services/googledrive/mappers.ts new file mode 100644 index 000000000..7e1bcbe75 --- /dev/null +++ b/packages/api/src/filestorage/file/services/googledrive/mappers.ts @@ -0,0 +1,101 @@ +import { MappersRegistry } from '@@core/@core-services/registries/mappers.registry'; +import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { Utils } from '@filestorage/@lib/@utils'; +import { IFileMapper } from '@filestorage/file/types'; +import { + UnifiedFilestorageFileInput, + UnifiedFilestorageFileOutput, +} from '@filestorage/file/types/model.unified'; +import { Injectable } from '@nestjs/common'; +import { GoogleDriveFileInput, GoogleDriveFileOutput } from './types'; + +@Injectable() +export class GoogleDriveFileMapper implements IFileMapper { + constructor( + private mappersRegistry: MappersRegistry, + private utils: Utils, + private coreUnificationService: CoreUnification, + ) { + this.mappersRegistry.registerService( + 'filestorage', + 'file', + 'googledrive', + this, + ); + } + + async desunify( + source: UnifiedFilestorageFileInput, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + return { + name: source.name, + mimeType: source.mime_type, + parents: source.folder_id ? [source.folder_id] : undefined, + }; + } + + async unify( + source: GoogleDriveFileOutput | GoogleDriveFileOutput[], + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + if (!Array.isArray(source)) { + return await this.mapSingleFileToUnified( + source, + connectionId, + customFieldMappings, + ); + } + return Promise.all( + source.map((file) => + this.mapSingleFileToUnified(file, connectionId, customFieldMappings), + ), + ); + } + + private async mapSingleFileToUnified( + file: GoogleDriveFileOutput, + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const field_mappings: { [key: string]: any } = {}; + if (customFieldMappings) { + for (const mapping of customFieldMappings) { + field_mappings[mapping.slug] = file[mapping.remote_id]; + } + } + const opts: any = {}; + if (file.parents && file.parents.length > 0) { + const folder_id = await this.utils.getFolderIdFromRemote( + file.parents[0], + connectionId, + ); + opts.folder_id = folder_id; + } + + return { + remote_id: file.id, + remote_data: file, + name: file.name, + file_url: file.webViewLink || file.webContentLink || null, + mime_type: file.mimeType || null, + size: file.size || null, + permission: null, + shared_link: null, + ...opts, + field_mappings, + created_at: file.createdTime ? new Date(file.createdTime) : null, + modified_at: file.modifiedTime ? new Date(file.modifiedTime) : null, + }; + } +} diff --git a/packages/api/src/filestorage/file/services/googledrive/types.ts b/packages/api/src/filestorage/file/services/googledrive/types.ts new file mode 100644 index 000000000..657b0c0ec --- /dev/null +++ b/packages/api/src/filestorage/file/services/googledrive/types.ts @@ -0,0 +1,108 @@ +export type GoogleDriveFileInput = Partial + +export interface GoogleDriveFileOutput { + kind?: string; + id: string; + name: string; + mimeType: string; + description?: string; + starred?: boolean; + trashed?: boolean; + explicitlyTrashed?: boolean; + parents?: string[]; + properties?: { [key: string]: string }; + appProperties?: { [key: string]: string }; + spaces?: string[]; + version?: string; + webContentLink?: string; + webViewLink?: string; + iconLink?: string; + thumbnailLink?: string; + viewedByMe?: boolean; + viewedByMeTime?: string; + createdTime?: string; + modifiedTime?: string; + modifiedByMeTime?: string; + sharedWithMeTime?: string; + sharingUser?: any; + owners?: any[]; + teamDriveId?: string; + driveId?: string; + lastModifyingUser?: any; + shared?: boolean; + ownedByMe?: boolean; + capabilities?: { + canEdit?: boolean; + canComment?: boolean; + canShare?: boolean; + canCopy?: boolean; + canDownload?: boolean; + canListChildren?: boolean; + canAddChildren?: boolean; + canRemoveChildren?: boolean; + canDelete?: boolean; + canRename?: boolean; + canTrash?: boolean; + canUntrash?: boolean; + canMoveItemWithinDrive?: boolean; + canMoveItemOutOfDrive?: boolean; + canAddFolderFromAnotherDrive?: boolean; + canMoveItemIntoTeamDrive?: boolean; + canMoveItemOutOfTeamDrive?: boolean; + canModifyContent?: boolean; + canModifyContentRestriction?: boolean; + canReadRevisions?: boolean; + canChangeCopyRequiresWriterPermission?: boolean; + canModifyLabels?: boolean; + [key: string]: boolean | undefined; + }; + viewersCanCopyContent?: boolean; + writersCanShare?: boolean; + permissions?: any[]; + permissionIds?: string[]; + hasAugmentedPermissions?: boolean; + folderColorRgb?: string; + originalFilename?: string; + fullFileExtension?: string; + fileExtension?: string; + md5Checksum?: string; + size?: string; + quotaBytesUsed?: string; + headRevisionId?: string; + contentHints?: { + thumbnail?: { + image?: string; + mimeType?: string; + }; + indexableText?: string; + }; + imageMediaMetadata?: { + width?: number; + height?: number; + rotation?: number; + // Add other image metadata fields as needed + }; + videoMediaMetadata?: { + width?: number; + height?: number; + durationMillis?: string; + }; + isAppAuthorized?: boolean; + exportLinks?: { [key: string]: string }; + shortcutDetails?: { + targetId?: string; + targetMimeType?: string; + targetResourceKey?: string; + }; + contentRestrictions?: any[]; + resourceKey?: string; + linkShareMetadata?: { + securityUpdateEligible?: boolean; + securityUpdateEnabled?: boolean; + }; + labelInfo?: { + labels?: any[]; + }; + sha1Checksum?: string; + sha256Checksum?: string; +} diff --git a/packages/api/src/filestorage/file/services/onedrive/index.ts b/packages/api/src/filestorage/file/services/onedrive/index.ts new file mode 100644 index 000000000..3895ed570 --- /dev/null +++ b/packages/api/src/filestorage/file/services/onedrive/index.ts @@ -0,0 +1,92 @@ +import { EncryptionService } from '@@core/@core-services/encryption/encryption.service'; +import { LoggerService } from '@@core/@core-services/logger/logger.service'; +import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; +import { ApiResponse } from '@@core/utils/types'; +import { SyncParam } from '@@core/utils/types/interface'; +import { FileStorageObject } from '@filestorage/@lib/@types'; +import { IFileService } from '@filestorage/file/types'; +import { Injectable } from '@nestjs/common'; +import axios from 'axios'; +import { ServiceRegistry } from '../registry.service'; +import { OnedriveFileOutput } from './types'; + +@Injectable() +export class OnedriveService implements IFileService { + constructor( + private prisma: PrismaService, + private logger: LoggerService, + private cryptoService: EncryptionService, + private registry: ServiceRegistry, + ) { + this.logger.setContext( + `${FileStorageObject.file.toUpperCase()}:${OnedriveService.name}`, + ); + this.registry.registerService('onedrive', this); + } + + // todo: add addFile method + + async sync(data: SyncParam): Promise> { + try { + const { linkedUserId, id_folder } = data; + if (!id_folder) return; + + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'onedrive', + vertical: 'filestorage', + }, + }); + + const folder = await this.prisma.fs_folders.findUnique({ + where: { + id_fs_folder: id_folder as string, + }, + }); + + const resp = await axios.get( + `${connection.account_url}/v1.0/drive/items/${folder.remote_id}/children`, + { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }, + ); + + const files: OnedriveFileOutput[] = resp.data.value.filter( + (elem) => !elem.folder, // files don't have a folder property + ); + + // Add permission shared link is also included in permissions in one-drive) + await Promise.all( + files.map(async (driveItem) => { + const resp = await axios.get( + `${connection.account_url}/v1.0/drive/items/${driveItem.id}/permissions`, + { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }, + ); + driveItem.permissions = resp.data.value; + }), + ); + + this.logger.log(`Synced onedrive files !`); + return { + data: files, + message: "One Drive's files retrieved", + statusCode: 200, + }; + } catch (error) { + throw error; + } + } +} diff --git a/packages/api/src/filestorage/file/services/onedrive/mappers.ts b/packages/api/src/filestorage/file/services/onedrive/mappers.ts new file mode 100644 index 000000000..e8dd383f0 --- /dev/null +++ b/packages/api/src/filestorage/file/services/onedrive/mappers.ts @@ -0,0 +1,136 @@ +import { MappersRegistry } from '@@core/@core-services/registries/mappers.registry'; +import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { + OriginalPermissionOutput, + OriginalSharedLinkOutput, +} from '@@core/utils/types/original/original.file-storage'; +import { FileStorageObject } from '@filestorage/@lib/@types'; +import { Utils } from '@filestorage/@lib/@utils'; +import { IFileMapper } from '@filestorage/file/types'; +import { + UnifiedFilestorageFileInput, + UnifiedFilestorageFileOutput, +} from '@filestorage/file/types/model.unified'; +import { UnifiedFilestorageSharedlinkOutput } from '@filestorage/sharedlink/types/model.unified'; +import { Injectable } from '@nestjs/common'; +import { OnedriveFileInput, OnedriveFileOutput } from './types'; + +@Injectable() +export class OnedriveFileMapper implements IFileMapper { + constructor( + private mappersRegistry: MappersRegistry, + private utils: Utils, + private coreUnificationService: CoreUnification, + ) { + this.mappersRegistry.registerService( + 'filestorage', + 'file', + 'onedrive', + this, + ); + } + + async desunify( + source: UnifiedFilestorageFileInput, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + // todo: do something with customFieldMappings + return { + name: source.name, + file: { + mimeType: source.mime_type, + }, + size: parseInt(source.size), + parentReference: { + id: source.folder_id, + }, + }; + } + + async unify( + source: OnedriveFileOutput | OnedriveFileOutput[], + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + if (!Array.isArray(source)) { + return await this.mapSingleFileToUnified( + source, + connectionId, + customFieldMappings, + ); + } + // Handling array of OnedriveFileOutput + return Promise.all( + source.map((file) => + this.mapSingleFileToUnified(file, connectionId, customFieldMappings), + ), + ); + } + + private async mapSingleFileToUnified( + file: OnedriveFileOutput, + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const field_mappings: { [key: string]: any } = {}; + if (customFieldMappings) { + for (const mapping of customFieldMappings) { + field_mappings[mapping.slug] = file[mapping.remote_id]; + } + } + + const opts: any = {}; + if (file.permissions?.length) { + const permissions = await this.coreUnificationService.unify< + OriginalPermissionOutput[] + >({ + sourceObject: file.permissions, + targetType: FileStorageObject.permission, + providerName: 'onedrive', + vertical: 'filestorage', + connectionId, + customFieldMappings: [], + }); + opts.permissions = permissions; + + // shared link + if (file.permissions.some((p) => p.link)) { + const sharedLinks = + await this.coreUnificationService.unify({ + sourceObject: file.permissions.find((p) => p.link), + targetType: FileStorageObject.sharedlink, + providerName: 'onedrive', + vertical: 'filestorage', + connectionId, + customFieldMappings: [], + }); + opts.shared_links = sharedLinks; + } + } + + // todo: handle folder + + return { + remote_id: file.id, + remote_data: file, + name: file.name, + file_url: file.webUrl, + mime_type: file.file.mimeType, + size: file.size.toString(), + folder_id: null, + // permission: opts.permissions?.[0] || null, + permission: null, + shared_link: opts.shared_links?.[0] || null, + field_mappings, + }; + } +} diff --git a/packages/api/src/filestorage/file/services/onedrive/types.ts b/packages/api/src/filestorage/file/services/onedrive/types.ts new file mode 100644 index 000000000..d2b3a12ab --- /dev/null +++ b/packages/api/src/filestorage/file/services/onedrive/types.ts @@ -0,0 +1,210 @@ +import { IdentitySet } from '@filestorage/drive/services/onedrive/types'; +import { + Deleted, + FileSystemInfo, + ItemReference, +} from '@filestorage/folder/services/onedrive/types'; +import { OnedrivePermissionOutput } from '@filestorage/permission/services/onedrive/types'; + +/** + * Represents the input for a folder item in OneDrive. + * @see https://learn.microsoft.com/en-us/graph/api/resources/driveitem?view=graph-rest-1.0 + */ +export interface OnedriveFileOutput { + /** The unique identifier of the item within the Drive. */ + readonly id?: string; + /** The name of the item (filename and extension). */ + name?: string; + /** The URL that displays the resource in the browser. */ + readonly webUrl?: string; + /** File system information on the client. */ + fileSystemInfo?: FileSystemInfo; + /** Parent information, if the item has a parent. */ + parentReference?: ItemReference; + /** The unique identifier of the drive instance that contains the driveItem. */ + readonly driveId?: string; + /** Identifies the type of drive. */ + readonly driveType?: string; + /** Information about the deleted state of the item. */ + deleted?: Deleted; + /** Description of the item. */ + description?: string; + /** Permissions associated with the folder. */ + permissions?: OnedrivePermissionOutput[]; + /** Date and time the item was last modified. Read-only. */ + readonly lastModifiedDateTime?: string; + /** Size of the item in bytes. Read-only. */ + readonly size?: number; + /** Identity of the user, device, and application that created the item. Read-only. */ + readonly createdBy?: IdentitySet; + /** Identity of the user, device, and application that last modified the item. Read-only. */ + readonly lastModifiedBy?: IdentitySet; + /** Date and time of item creation. Read-only. */ + readonly createdDateTime?: string; + /** File metadata Read-only. */ + readonly file: File; + /** Audio metadata, if the item is an audio file. Read-only. Read-only. Only on OneDrive Personal. */ + readonly audio?: Audio; + /** Bundle metadata, if the item is a bundle. Read-only. */ + readonly bundle?: Bundle; + /** The content stream, if the item represents a file. */ + content?: string; + /** Image metadata, if the item is an image. Read-only. */ + readonly image?: Image; + /** Photo metadata, if the item is a photo. Read-only. */ + readonly photo?: Photo; + /** Video metadata, if the item is a video. Read-only. */ + readonly video?: Video; + /** WebDAV compatible URL for the item. */ + readonly webDavUrl?: string; +} + +/** + * Represents the input for a folder item in OneDrive. + * @see https://learn.microsoft.com/en-us/graph/api/resources/file?view=graph-rest-1.0 + */ +export interface File { + /**The MIME type for the file. This is determined by logic on the server and might not be the value provided when the file was uploaded. Read-only. */ + mimeType: string; + /** Hashes of the file's binary content, if available. Read-only. */ + hashes?: Hashes; +} + +/** + * The hashes resource groups available hashes into a single structure for an item. + * @see https://learn.microsoft.com/en-us/graph/api/resources/hashes?view=graph-rest-1.0 + */ +export interface Hashes { + /** The CRC32 value of the file in little endian (if available). Read-only. */ + readonly crc32Hash?: string; + /** A proprietary hash of the file that can be used to determine if the contents of the file have changed (if available). Read-only. */ + readonly quickXorHash?: string; + /** SHA1 hash for the contents of the file (if available). Read-only. */ + readonly sha1Hash?: string; + /** SHA256 hash for the contents of the file (if available). Read-only. */ + readonly sha256Hash?: string; +} + +/** + * Represents metadata for an audio file. + * @see https://learn.microsoft.com/en-us/graph/api/resources/audio?view=graph-rest-1.0 + */ +export interface Audio { + /** The title of the album for this audio file. */ + album?: string; + /** The artist named on the album for the audio file. */ + albumArtist?: string; + /** The performing artist for the audio file. */ + artist?: string; + /** Bitrate expressed in kbps. */ + bitrate?: number; + /** The name of the composer of the audio file. */ + composers?: string; + /** Copyright information for the audio file. */ + copyright?: string; + /** The number of the disc this audio file came from. */ + disc?: number; + /** The total number of discs in this album. */ + discCount?: number; + /** Duration of the audio file, expressed in milliseconds. */ + duration?: number; + /** The genre of this audio file. */ + genre?: string; + /** Indicates if the file is protected with digital rights management. */ + hasDrm?: boolean; + /** Indicates if the file is encoded with a variable bitrate. */ + isVariableBitrate?: boolean; + /** The title of the audio file. */ + title?: string; + /** The number of the track on the original disc for this audio file. */ + track?: number; + /** The total number of tracks on the original disc for this audio file. */ + trackCount?: number; + /** The year the audio file was recorded. */ + year?: number; +} + +/** + * Represents metadata for a bundle. + * @see https://learn.microsoft.com/en-us/graph/api/resources/bundle?view=graph-rest-1.0 + */ +export interface Bundle { + /** If the bundle is an album, then the album property is included. */ + album?: Album; + /** Number of children contained immediately within this container. */ + childCount?: number; +} + +/** + * Represents album-specific metadata for a bundle. + * @see https://learn.microsoft.com/en-us/graph/api/resources/album?view=graph-rest-1.0 + */ +export interface Album { + /** Unique identifier of the driveItem that is the cover of the album. */ + coverImageItemId?: string; +} + +/** + * Represents metadata for an image. + * @see https://learn.microsoft.com/en-us/graph/api/resources/image?view=graph-rest-1.0 + */ +export interface Image { + /** Optional. Height of the image, in pixels. Read-only. */ + readonly height?: number; + /** Optional. Width of the image, in pixels. Read-only. */ + readonly width?: number; +} + +/** + * Represents metadata for a photo. + * @see https://learn.microsoft.com/en-us/graph/api/resources/photo?view=graph-rest-1.0 + */ +export interface Photo { + /** Camera manufacturer. Read-only. */ + readonly cameraMake?: string; + /** Camera model. Read-only. */ + readonly cameraModel?: string; + /** The denominator for the exposure time fraction from the camera. Read-only. */ + readonly exposureDenominator?: number; + /** The numerator for the exposure time fraction from the camera. Read-only. */ + readonly exposureNumerator?: number; + /** The F-stop value from the camera. Read-only. */ + readonly fNumber?: number; + /** The focal length from the camera. Read-only. */ + readonly focalLength?: number; + /** The ISO value from the camera. Read-only. */ + readonly iso?: number; + /** The orientation value from the camera. Writable on OneDrive Personal. */ + orientation?: number; + /** Represents the date and time the photo was taken. Read-only. */ + readonly takenDateTime?: string; +} + +/** + * Represents metadata for a video file. + * @see https://learn.microsoft.com/en-us/graph/api/resources/video?view=graph-rest-1.0 + */ +export interface Video { + /** Number of audio bits per sample. */ + audioBitsPerSample?: number; + /** Number of audio channels. */ + audioChannels?: number; + /** Name of the audio format (AAC, MP3, etc.). */ + audioFormat?: string; + /** Number of audio samples per second. */ + audioSamplesPerSecond?: number; + /** Bit rate of the video in bits per second. */ + bitrate?: number; + /** Duration of the file in milliseconds. */ + duration?: number; + /** "Four character code" name of the video format. */ + fourCC?: string; + /** Frame rate of the video. */ + frameRate?: number; + /** Height of the video, in pixels. */ + height?: number; + /** Width of the video, in pixels. */ + width?: number; +} + +export type OnedriveFileInput = Partial; diff --git a/packages/api/src/filestorage/file/services/sharepoint/index.ts b/packages/api/src/filestorage/file/services/sharepoint/index.ts new file mode 100644 index 000000000..44d5bc8a7 --- /dev/null +++ b/packages/api/src/filestorage/file/services/sharepoint/index.ts @@ -0,0 +1,92 @@ +import { EncryptionService } from '@@core/@core-services/encryption/encryption.service'; +import { LoggerService } from '@@core/@core-services/logger/logger.service'; +import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; +import { ApiResponse } from '@@core/utils/types'; +import { SyncParam } from '@@core/utils/types/interface'; +import { FileStorageObject } from '@filestorage/@lib/@types'; +import { IFileService } from '@filestorage/file/types'; +import { Injectable } from '@nestjs/common'; +import axios from 'axios'; +import { ServiceRegistry } from '../registry.service'; +import { SharepointFileOutput } from './types'; + +@Injectable() +export class SharepointService implements IFileService { + constructor( + private prisma: PrismaService, + private logger: LoggerService, + private cryptoService: EncryptionService, + private registry: ServiceRegistry, + ) { + this.logger.setContext( + `${FileStorageObject.file.toUpperCase()}:${SharepointService.name}`, + ); + this.registry.registerService('sharepoint', this); + } + + // todo: add addFile method + + async sync(data: SyncParam): Promise> { + try { + const { linkedUserId, id_folder } = data; + if (!id_folder) return; + + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'sharepoint', + vertical: 'filestorage', + }, + }); + + const folder = await this.prisma.fs_folders.findUnique({ + where: { + id_fs_folder: id_folder as string, + }, + }); + + const resp = await axios.get( + `${connection.account_url}/drive/items/${folder.remote_id}/children`, + { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }, + ); + + const files: SharepointFileOutput[] = resp.data.value.filter( + (elem) => !elem.folder, // files don't have a folder property + ); + + // Add permission shared link is also included in permissions in one-drive) + await Promise.all( + files.map(async (driveItem) => { + const resp = await axios.get( + `${connection.account_url}/drive/items/${driveItem.id}/permissions`, + { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }, + ); + driveItem.permissions = resp.data.value; + }), + ); + + this.logger.log(`Synced sharepoint files !`); + return { + data: files, + message: "One Drive's files retrieved", + statusCode: 200, + }; + } catch (error) { + throw error; + } + } +} diff --git a/packages/api/src/filestorage/file/services/sharepoint/mappers.ts b/packages/api/src/filestorage/file/services/sharepoint/mappers.ts new file mode 100644 index 000000000..7fd0f227b --- /dev/null +++ b/packages/api/src/filestorage/file/services/sharepoint/mappers.ts @@ -0,0 +1,136 @@ +import { MappersRegistry } from '@@core/@core-services/registries/mappers.registry'; +import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { + OriginalPermissionOutput, + OriginalSharedLinkOutput, +} from '@@core/utils/types/original/original.file-storage'; +import { FileStorageObject } from '@filestorage/@lib/@types'; +import { Utils } from '@filestorage/@lib/@utils'; +import { IFileMapper } from '@filestorage/file/types'; +import { + UnifiedFilestorageFileInput, + UnifiedFilestorageFileOutput, +} from '@filestorage/file/types/model.unified'; +import { UnifiedFilestorageSharedlinkOutput } from '@filestorage/sharedlink/types/model.unified'; +import { Injectable } from '@nestjs/common'; +import { SharepointFileInput, SharepointFileOutput } from './types'; + +@Injectable() +export class SharepointFileMapper implements IFileMapper { + constructor( + private mappersRegistry: MappersRegistry, + private utils: Utils, + private coreUnificationService: CoreUnification, + ) { + this.mappersRegistry.registerService( + 'filestorage', + 'file', + 'sharepoint', + this, + ); + } + + async desunify( + source: UnifiedFilestorageFileInput, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + // todo: do something with customFieldMappings + return { + name: source.name, + file: { + mimeType: source.mime_type, + }, + size: parseInt(source.size), + parentReference: { + id: source.folder_id, + }, + }; + } + + async unify( + source: SharepointFileOutput | SharepointFileOutput[], + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + if (!Array.isArray(source)) { + return await this.mapSingleFileToUnified( + source, + connectionId, + customFieldMappings, + ); + } + // Handling array of SharepointFileOutput + return Promise.all( + source.map((file) => + this.mapSingleFileToUnified(file, connectionId, customFieldMappings), + ), + ); + } + + private async mapSingleFileToUnified( + file: SharepointFileOutput, + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const field_mappings: { [key: string]: any } = {}; + if (customFieldMappings) { + for (const mapping of customFieldMappings) { + field_mappings[mapping.slug] = file[mapping.remote_id]; + } + } + + const opts: any = {}; + if (file.permissions?.length) { + const permissions = await this.coreUnificationService.unify< + OriginalPermissionOutput[] + >({ + sourceObject: file.permissions, + targetType: FileStorageObject.permission, + providerName: 'sharepoint', + vertical: 'filestorage', + connectionId, + customFieldMappings: [], + }); + opts.permissions = permissions; + + // shared link + if (file.permissions.some((p) => p.link)) { + const sharedLinks = + await this.coreUnificationService.unify({ + sourceObject: file.permissions.find((p) => p.link), + targetType: FileStorageObject.sharedlink, + providerName: 'sharepoint', + vertical: 'filestorage', + connectionId, + customFieldMappings: [], + }); + opts.shared_links = sharedLinks; + } + } + + // todo: handle folder + + return { + remote_id: file.id, + remote_data: file, + name: file.name, + file_url: file.webUrl, + mime_type: file.file.mimeType, + size: file.size.toString(), + folder_id: null, + // permission: opts.permissions?.[0] || null, + permission: null, + shared_link: opts.shared_links?.[0] || null, + field_mappings, + }; + } +} diff --git a/packages/api/src/filestorage/file/services/sharepoint/types.ts b/packages/api/src/filestorage/file/services/sharepoint/types.ts new file mode 100644 index 000000000..32c34810a --- /dev/null +++ b/packages/api/src/filestorage/file/services/sharepoint/types.ts @@ -0,0 +1,210 @@ +import { IdentitySet } from '@filestorage/drive/services/sharepoint/types'; +import { + Deleted, + FileSystemInfo, + ItemReference, +} from '@filestorage/folder/services/sharepoint/types'; +import { SharepointPermissionOutput } from '@filestorage/permission/services/sharepoint/types'; + +/** + * Represents the input for a folder item in Sharepoint. + * @see https://learn.microsoft.com/en-us/graph/api/resources/driveitem?view=graph-rest-1.0 + */ +export interface SharepointFileOutput { + /** The unique identifier of the item within the Drive. */ + readonly id?: string; + /** The name of the item (filename and extension). */ + name?: string; + /** The URL that displays the resource in the browser. */ + readonly webUrl?: string; + /** File system information on the client. */ + fileSystemInfo?: FileSystemInfo; + /** Parent information, if the item has a parent. */ + parentReference?: ItemReference; + /** The unique identifier of the drive instance that contains the driveItem. */ + readonly driveId?: string; + /** Identifies the type of drive. */ + readonly driveType?: string; + /** Information about the deleted state of the item. */ + deleted?: Deleted; + /** Description of the item. */ + description?: string; + /** Permissions associated with the folder. */ + permissions?: SharepointPermissionOutput[]; + /** Date and time the item was last modified. Read-only. */ + readonly lastModifiedDateTime?: string; + /** Size of the item in bytes. Read-only. */ + readonly size?: number; + /** Identity of the user, device, and application that created the item. Read-only. */ + readonly createdBy?: IdentitySet; + /** Identity of the user, device, and application that last modified the item. Read-only. */ + readonly lastModifiedBy?: IdentitySet; + /** Date and time of item creation. Read-only. */ + readonly createdDateTime?: string; + /** File metadata Read-only. */ + readonly file: File; + /** Audio metadata, if the item is an audio file. Read-only. Read-only. Only on OneDrive Personal. */ + readonly audio?: Audio; + /** Bundle metadata, if the item is a bundle. Read-only. */ + readonly bundle?: Bundle; + /** The content stream, if the item represents a file. */ + content?: string; + /** Image metadata, if the item is an image. Read-only. */ + readonly image?: Image; + /** Photo metadata, if the item is a photo. Read-only. */ + readonly photo?: Photo; + /** Video metadata, if the item is a video. Read-only. */ + readonly video?: Video; + /** WebDAV compatible URL for the item. */ + readonly webDavUrl?: string; +} + +/** + * Represents the input for a folder item in OneDrive. + * @see https://learn.microsoft.com/en-us/graph/api/resources/file?view=graph-rest-1.0 + */ +export interface File { + /**The MIME type for the file. This is determined by logic on the server and might not be the value provided when the file was uploaded. Read-only. */ + mimeType: string; + /** Hashes of the file's binary content, if available. Read-only. */ + hashes?: Hashes; +} + +/** + * The hashes resource groups available hashes into a single structure for an item. + * @see https://learn.microsoft.com/en-us/graph/api/resources/hashes?view=graph-rest-1.0 + */ +export interface Hashes { + /** The CRC32 value of the file in little endian (if available). Read-only. */ + readonly crc32Hash?: string; + /** A proprietary hash of the file that can be used to determine if the contents of the file have changed (if available). Read-only. */ + readonly quickXorHash?: string; + /** SHA1 hash for the contents of the file (if available). Read-only. */ + readonly sha1Hash?: string; + /** SHA256 hash for the contents of the file (if available). Read-only. */ + readonly sha256Hash?: string; +} + +/** + * Represents metadata for an audio file. + * @see https://learn.microsoft.com/en-us/graph/api/resources/audio?view=graph-rest-1.0 + */ +export interface Audio { + /** The title of the album for this audio file. */ + album?: string; + /** The artist named on the album for the audio file. */ + albumArtist?: string; + /** The performing artist for the audio file. */ + artist?: string; + /** Bitrate expressed in kbps. */ + bitrate?: number; + /** The name of the composer of the audio file. */ + composers?: string; + /** Copyright information for the audio file. */ + copyright?: string; + /** The number of the disc this audio file came from. */ + disc?: number; + /** The total number of discs in this album. */ + discCount?: number; + /** Duration of the audio file, expressed in milliseconds. */ + duration?: number; + /** The genre of this audio file. */ + genre?: string; + /** Indicates if the file is protected with digital rights management. */ + hasDrm?: boolean; + /** Indicates if the file is encoded with a variable bitrate. */ + isVariableBitrate?: boolean; + /** The title of the audio file. */ + title?: string; + /** The number of the track on the original disc for this audio file. */ + track?: number; + /** The total number of tracks on the original disc for this audio file. */ + trackCount?: number; + /** The year the audio file was recorded. */ + year?: number; +} + +/** + * Represents metadata for a bundle. + * @see https://learn.microsoft.com/en-us/graph/api/resources/bundle?view=graph-rest-1.0 + */ +export interface Bundle { + /** If the bundle is an album, then the album property is included. */ + album?: Album; + /** Number of children contained immediately within this container. */ + childCount?: number; +} + +/** + * Represents album-specific metadata for a bundle. + * @see https://learn.microsoft.com/en-us/graph/api/resources/album?view=graph-rest-1.0 + */ +export interface Album { + /** Unique identifier of the driveItem that is the cover of the album. */ + coverImageItemId?: string; +} + +/** + * Represents metadata for an image. + * @see https://learn.microsoft.com/en-us/graph/api/resources/image?view=graph-rest-1.0 + */ +export interface Image { + /** Optional. Height of the image, in pixels. Read-only. */ + readonly height?: number; + /** Optional. Width of the image, in pixels. Read-only. */ + readonly width?: number; +} + +/** + * Represents metadata for a photo. + * @see https://learn.microsoft.com/en-us/graph/api/resources/photo?view=graph-rest-1.0 + */ +export interface Photo { + /** Camera manufacturer. Read-only. */ + readonly cameraMake?: string; + /** Camera model. Read-only. */ + readonly cameraModel?: string; + /** The denominator for the exposure time fraction from the camera. Read-only. */ + readonly exposureDenominator?: number; + /** The numerator for the exposure time fraction from the camera. Read-only. */ + readonly exposureNumerator?: number; + /** The F-stop value from the camera. Read-only. */ + readonly fNumber?: number; + /** The focal length from the camera. Read-only. */ + readonly focalLength?: number; + /** The ISO value from the camera. Read-only. */ + readonly iso?: number; + /** The orientation value from the camera. Writable on OneDrive Personal. */ + orientation?: number; + /** Represents the date and time the photo was taken. Read-only. */ + readonly takenDateTime?: string; +} + +/** + * Represents metadata for a video file. + * @see https://learn.microsoft.com/en-us/graph/api/resources/video?view=graph-rest-1.0 + */ +export interface Video { + /** Number of audio bits per sample. */ + audioBitsPerSample?: number; + /** Number of audio channels. */ + audioChannels?: number; + /** Name of the audio format (AAC, MP3, etc.). */ + audioFormat?: string; + /** Number of audio samples per second. */ + audioSamplesPerSecond?: number; + /** Bit rate of the video in bits per second. */ + bitrate?: number; + /** Duration of the file in milliseconds. */ + duration?: number; + /** "Four character code" name of the video format. */ + fourCC?: string; + /** Frame rate of the video. */ + frameRate?: number; + /** Height of the video, in pixels. */ + height?: number; + /** Width of the video, in pixels. */ + width?: number; +} + +export type SharepointFileInput = Partial; diff --git a/packages/api/src/filestorage/file/sync/sync.processor.ts b/packages/api/src/filestorage/file/sync/sync.processor.ts deleted file mode 100644 index 73f68aa9b..000000000 --- a/packages/api/src/filestorage/file/sync/sync.processor.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - - @Process('filestorage-sync-files') - async handleSyncFiles(job: Job) { - try { - console.log(`Processing queue -> filestorage-sync-files ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing filestorage files', error); - } - } -} diff --git a/packages/api/src/filestorage/file/sync/sync.service.ts b/packages/api/src/filestorage/file/sync/sync.service.ts index fe4df1fec..7c5e21b04 100644 --- a/packages/api/src/filestorage/file/sync/sync.service.ts +++ b/packages/api/src/filestorage/file/sync/sync.service.ts @@ -6,9 +6,10 @@ import { CoreUnification } from '@@core/@core-services/unification/core-unificat import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; import { FieldMappingService } from '@@core/field-mapping/field-mapping.service'; -import { ApiResponse } from '@@core/utils/types'; import { IBaseSync, SyncLinkedUserType } from '@@core/utils/types/interface'; import { OriginalFileOutput } from '@@core/utils/types/original/original.file-storage'; +import { UnifiedFilestoragePermissionOutput } from '@filestorage/permission/types/model.unified'; +import { UnifiedFilestorageSharedlinkOutput } from '@filestorage/sharedlink/types/model.unified'; import { Injectable, OnModuleInit } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; import { FILESTORAGE_PROVIDERS } from '@panora/shared'; @@ -17,8 +18,6 @@ import { v4 as uuidv4 } from 'uuid'; import { ServiceRegistry } from '../services/registry.service'; import { IFileService } from '../types'; import { UnifiedFilestorageFileOutput } from '../types/model.unified'; -import { UnifiedFilestorageSharedlinkOutput } from '@filestorage/sharedlink/types/model.unified'; -import { UnifiedFilestoragePermissionOutput } from '@filestorage/permission/types/model.unified'; @Injectable() export class SyncService implements OnModuleInit, IBaseSync { @@ -36,85 +35,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('filestorage', 'file', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob( - 'filestorage-sync-files', - '0 0 * * *', - ); - } catch (error) { - throw error; - } + onModuleInit() { + // } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing files...'); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = FILESTORAGE_PROVIDERS; - for (const provider of providers) { - try { - const connection = await this.prisma.connections.findFirst({ - where: { - id_linked_user: linkedUser.id_linked_user, - provider_slug: provider.toLowerCase(), - }, - }); - //call the sync files for every folder of the linkedUser (a file might be tied to a folder) - const folders = await this.prisma.fs_folders.findMany({ - where: { - id_connection: connection.id_connection, - }, - }); - for (const folder of folders) { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - folder_id: folder.id_fs_folder, - }); - } - // do a batch sync without folders as some providers might accept it - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = FILESTORAGE_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/filestorage/file/types/index.ts b/packages/api/src/filestorage/file/types/index.ts index 99d8ecc9b..89f35318f 100644 --- a/packages/api/src/filestorage/file/types/index.ts +++ b/packages/api/src/filestorage/file/types/index.ts @@ -1,15 +1,25 @@ import { DesunifyReturnType } from '@@core/utils/types/desunify.input'; -import { UnifiedFilestorageFileInput, UnifiedFilestorageFileOutput } from './model.unified'; +import { + UnifiedFilestorageFileInput, + UnifiedFilestorageFileOutput, +} from './model.unified'; import { OriginalFileOutput } from '@@core/utils/types/original/original.file-storage'; import { ApiResponse } from '@@core/utils/types'; import { IBaseObjectService, SyncParam } from '@@core/utils/types/interface'; - +import { S3Client } from '@aws-sdk/client-s3'; export interface IFileService extends IBaseObjectService { addFile?( fileData: DesunifyReturnType, linkedUserId: string, ): Promise>; + streamFileToS3?( + file_id: string, + linkedUserId: string, + s3Client: S3Client, + s3Key: string, + ): Promise; + sync(data: SyncParam): Promise>; } diff --git a/packages/api/src/filestorage/folder/folder.module.ts b/packages/api/src/filestorage/folder/folder.module.ts index b6cecab33..2b83c1c2c 100644 --- a/packages/api/src/filestorage/folder/folder.module.ts +++ b/packages/api/src/filestorage/folder/folder.module.ts @@ -1,4 +1,3 @@ -import { BullQueueModule } from '@@core/@core-services/queues/queue.module'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; @@ -7,8 +6,16 @@ import { Module } from '@nestjs/common'; import { FolderController } from './folder.controller'; import { BoxService } from './services/box'; import { BoxFolderMapper } from './services/box/mappers'; +import { DropboxService } from './services/dropbox'; +import { DropboxFolderMapper } from './services/dropbox/mappers'; import { FolderService } from './services/folder.service'; +import { GoogleDriveFolderService } from './services/googledrive'; +import { GoogleDriveFolderMapper } from './services/googledrive/mappers'; +import { OnedriveService } from './services/onedrive'; +import { OnedriveFolderMapper } from './services/onedrive/mappers'; import { ServiceRegistry } from './services/registry.service'; +import { SharepointService } from './services/sharepoint'; +import { SharepointFolderMapper } from './services/sharepoint/mappers'; import { SyncService } from './sync/sync.service'; @Module({ @@ -20,10 +27,19 @@ import { SyncService } from './sync/sync.service'; WebhookService, ServiceRegistry, IngestDataService, - BoxFolderMapper, Utils, + BoxFolderMapper, + OnedriveFolderMapper, + GoogleDriveFolderMapper, /* PROVIDERS SERVICES */ BoxService, + SharepointService, + SharepointFolderMapper, + OnedriveService, + OnedriveFolderMapper, + DropboxService, + DropboxFolderMapper, + GoogleDriveFolderService, ], exports: [SyncService], }) diff --git a/packages/api/src/filestorage/folder/services/dropbox/index.ts b/packages/api/src/filestorage/folder/services/dropbox/index.ts new file mode 100644 index 000000000..a3e60dc70 --- /dev/null +++ b/packages/api/src/filestorage/folder/services/dropbox/index.ts @@ -0,0 +1,129 @@ +import { Injectable } from '@nestjs/common'; +import { IFolderService } from '@filestorage/folder/types'; +import { FileStorageObject } from '@filestorage/@lib/@types'; +import axios from 'axios'; +import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; +import { LoggerService } from '@@core/@core-services/logger/logger.service'; +import { ActionType, handle3rdPartyServiceError } from '@@core/utils/errors'; +import { EncryptionService } from '@@core/@core-services/encryption/encryption.service'; +import { ApiResponse } from '@@core/utils/types'; +import { ServiceRegistry } from '../registry.service'; +import { SyncParam } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; +import { UnifiedFilestorageFileOutput } from '@filestorage/file/types/model.unified'; +import { DropboxFolderInput, DropboxFolderOutput } from './types'; +import { BoxFolderOutput } from '../box/types'; +// import { BoxFileOutput } from '@filestorage/file/services/box/types'; + +@Injectable() +export class DropboxService implements IFolderService { + constructor( + private prisma: PrismaService, + private logger: LoggerService, + private cryptoService: EncryptionService, + private registry: ServiceRegistry, + private ingestService: IngestDataService, + ) { + this.logger.setContext( + `${FileStorageObject.folder.toUpperCase()}:${DropboxService.name}`, + ); + this.registry.registerService('dropbox', this); + } + + async addFolder( + folderData: DropboxFolderInput, + linkedUserId: string, + ): Promise> { + try { + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'dropbox', + vertical: 'filestorage', + }, + }); + // ref: https://www.dropbox.com/developers/documentation/http/documentation#files-create_folder + const resp = await axios.post( + `${connection.account_url}/files/create_folder_v2`, + JSON.stringify(folderData), + { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }, + ); + return { + data: resp?.data, + message: 'Dropbox folder created', + statusCode: 201, + }; + } catch (error) { + console.log(error.response); + throw error; + } + } + + async getAllFolders(connection: any): Promise { + // ref: https://www.dropbox.com/developers/documentation/http/documentation#files-list_folder + const folders: DropboxFolderOutput[] = []; + let cursor: string | null = null; + let hasMore = true; + + while (hasMore) { + const url = cursor + ? `${connection.account_url}/files/list_folder/continue` + : `${connection.account_url}/files/list_folder`; + const data = cursor ? { cursor } : { path: '', recursive: true }; + + const response = await axios.post(url, data, { + headers: { + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + 'Content-Type': 'application/json', + }, + }); + + const { entries, has_more, cursor: newCursor } = response.data; + + // Collect all folder entries + folders.push( + ...entries.filter((entry: any) => entry['.tag'] === 'folder'), + ); + + hasMore = has_more; + cursor = newCursor; + } + + return folders; + } + + async sync(data: SyncParam): Promise> { + try { + const { linkedUserId } = data; + + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'dropbox', + vertical: 'filestorage', + }, + }); + + const results = await this.getAllFolders(connection); + this.logger.log(`Synced dropbox folders !`); + + return { + data: results, + message: 'Dropbox folders retrieved', + statusCode: 200, + }; + } catch (error) { + console.log(error.response); + throw error; + } + } +} diff --git a/packages/api/src/filestorage/folder/services/dropbox/mappers.ts b/packages/api/src/filestorage/folder/services/dropbox/mappers.ts new file mode 100644 index 000000000..0928a71a6 --- /dev/null +++ b/packages/api/src/filestorage/folder/services/dropbox/mappers.ts @@ -0,0 +1,110 @@ +import { MappersRegistry } from '@@core/@core-services/registries/mappers.registry'; +import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { OriginalSharedLinkOutput } from '@@core/utils/types/original/original.file-storage'; +import { Utils } from '@filestorage/@lib/@utils'; +import { IFolderMapper } from '@filestorage/folder/types'; +import { + UnifiedFilestorageFolderInput, + UnifiedFilestorageFolderOutput, +} from '@filestorage/folder/types/model.unified'; +import { UnifiedFilestorageSharedlinkOutput } from '@filestorage/sharedlink/types/model.unified'; +import { Injectable } from '@nestjs/common'; +import { FileStorageObject } from '@filestorage/@lib/@types'; +import { DropboxFolderInput, DropboxFolderOutput } from './types'; + +@Injectable() +export class DropboxFolderMapper implements IFolderMapper { + constructor( + private mappersRegistry: MappersRegistry, + private utils: Utils, + private coreUnificationService: CoreUnification, + ) { + this.mappersRegistry.registerService( + 'filestorage', + 'folder', + 'dropbox', + this, + ); + } + + async desunify( + source: UnifiedFilestorageFolderInput, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const result: DropboxFolderInput = { + path: `/${source.name}`, + autorename: true, + }; + + if (customFieldMappings && source.field_mappings) { + for (const [k, v] of Object.entries(source.field_mappings)) { + const mapping = customFieldMappings.find( + (mapping) => mapping.slug === k, + ); + if (mapping) { + result[mapping.remote_id] = v; + } + } + } + + return result; + } + + async unify( + source: DropboxFolderOutput | DropboxFolderOutput[], + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise< + UnifiedFilestorageFolderOutput | UnifiedFilestorageFolderOutput[] + > { + if (!Array.isArray(source)) { + return await this.mapSingleFolderToUnified( + source, + connectionId, + customFieldMappings, + ); + } else { + return await Promise.all( + source.map((s) => + this.mapSingleFolderToUnified(s, connectionId, customFieldMappings), + ), + ); + } + } + + private async mapSingleFolderToUnified( + folder: DropboxFolderOutput, + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const result: UnifiedFilestorageFolderOutput = { + remote_id: folder.id, + remote_data: folder, + name: folder.name, + size: null, + folder_url: null, + description: null, + drive_id: null, + parent_folder_id: null, + shared_link: null, + permission: null, + field_mappings: {}, + }; + + if (customFieldMappings) { + for (const mapping of customFieldMappings) { + result.field_mappings[mapping.slug] = folder[mapping.remote_id]; + } + } + return result; + } +} diff --git a/packages/api/src/filestorage/folder/services/dropbox/types.ts b/packages/api/src/filestorage/folder/services/dropbox/types.ts new file mode 100644 index 000000000..7e6309562 --- /dev/null +++ b/packages/api/src/filestorage/folder/services/dropbox/types.ts @@ -0,0 +1,73 @@ +/** + * Represents a folder-specific entry in the Dropbox API. + */ +export interface DropboxFolderOutput { + /** + * A constant tag indicating the entry is a folder. + * Value will always be `'folder'`. + */ + '.tag': 'folder'; + + /** + * The name of the folder. + */ + name: string; + + /** + * The lowercased path of the folder, useful for case-insensitive comparisons. + */ + path_lower: string; + + /** + * The display path of the folder, with original casing. + */ + path_display: string; + + /** + * The Dropbox unique identifier for the folder. + */ + id: string; + + /** + * Information about folder sharing, such as if it is read-only or shared. + * This field is present if the folder is part of a shared folder. + */ + sharing_info?: SharingInfo; +} + +/** + * Represents sharing information for a folder. + */ +export interface SharingInfo { + /** + * Whether the folder is read-only for the current user. + */ + read_only: boolean; + + /** + * The ID of the parent shared folder, if this folder is inside a shared folder. + */ + parent_shared_folder_id?: string; + + /** + * The unique ID of the shared folder. + */ + shared_folder_id?: string; +} + +/** + * Represents the request body for creating a new folder in Dropbox. + */ +export interface DropboxFolderInput { + /** + * The path to the folder you want to create, including the new folder's name. + * Example: "/new_folder_name" + */ + path: string; + + /** + * If true, the folder will be automatically renamed if a conflict occurs (i.e., if a folder with the same name already exists). + * Defaults to false. + */ + autorename?: boolean; +} diff --git a/packages/api/src/filestorage/folder/services/googledrive/index.ts b/packages/api/src/filestorage/folder/services/googledrive/index.ts new file mode 100644 index 000000000..a029f1047 --- /dev/null +++ b/packages/api/src/filestorage/folder/services/googledrive/index.ts @@ -0,0 +1,142 @@ +import { EncryptionService } from '@@core/@core-services/encryption/encryption.service'; +import { LoggerService } from '@@core/@core-services/logger/logger.service'; +import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; +import { ApiResponse } from '@@core/utils/types'; +import { SyncParam } from '@@core/utils/types/interface'; +import { FileStorageObject } from '@filestorage/@lib/@types'; +import { IFolderService } from '@filestorage/folder/types'; +import { Injectable } from '@nestjs/common'; +import { OAuth2Client } from 'google-auth-library'; +import { google } from 'googleapis'; +import { ServiceRegistry } from '../registry.service'; +import { GoogleDriveFolderInput, GoogleDriveFolderOutput } from './types'; + +@Injectable() +export class GoogleDriveFolderService implements IFolderService { + constructor( + private prisma: PrismaService, + private logger: LoggerService, + private cryptoService: EncryptionService, + private registry: ServiceRegistry, + ) { + this.logger.setContext( + `${FileStorageObject.folder.toUpperCase()}:${ + GoogleDriveFolderService.name + }`, + ); + this.registry.registerService('googledrive', this); + } + + async addFolder( + folderData: GoogleDriveFolderInput, + linkedUserId: string, + ): Promise> { + try { + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'googledrive', + vertical: 'filestorage', + }, + }); + + if (!connection) { + return { + data: null, + message: 'Connection not found', + statusCode: 404, + }; + } + + const auth = new OAuth2Client(); + auth.setCredentials({ + access_token: this.cryptoService.decrypt(connection.access_token), + }); + const drive = google.drive({ version: 'v3', auth }); + + const fileMetadata = { + name: folderData.name, + mimeType: 'application/vnd.google-apps.folder', + parents: folderData.parents, + }; + const response = await drive.files.create({ + requestBody: fileMetadata, + fields: 'id, name, mimeType, createdTime, modifiedTime, parents', + }); + + const createdFolder: GoogleDriveFolderOutput = { + id: response.data.id!, + name: response.data.name!, + mimeType: response.data.mimeType!, + createdTime: response.data.createdTime!, + modifiedTime: response.data.modifiedTime!, + parents: response.data.parents, + }; + + return { + data: createdFolder, + message: 'Google Drive folder created', + statusCode: 201, + }; + } catch (error) { + this.logger.error('Error creating Google Drive folder', error); + throw error; + } + } + + async sync(data: SyncParam): Promise> { + try { + const { linkedUserId } = data; + + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'googledrive', + vertical: 'filestorage', + }, + }); + + if (!connection) { + return { + data: [], + message: 'Connection not found', + statusCode: 404, + }; + } + + const auth = new OAuth2Client(); + auth.setCredentials({ + access_token: this.cryptoService.decrypt(connection.access_token), + }); + const drive = google.drive({ version: 'v3', auth }); + + const response = await drive.files.list({ + q: "mimeType = 'application/vnd.google-apps.folder' and trashed = false", + fields: 'files(id, name, mimeType, createdTime, modifiedTime, parents)', + pageSize: 1000, // Adjust as needed + }); + + const folders: GoogleDriveFolderOutput[] = response.data.files.map( + (folder) => ({ + id: folder.id!, + name: folder.name!, + mimeType: folder.mimeType!, + createdTime: folder.createdTime!, + modifiedTime: folder.modifiedTime!, + parents: folder.parents, + }), + ); + + this.logger.log(`Synced Google Drive folders!`); + + return { + data: folders, + message: 'Google Drive folders retrieved', + statusCode: 200, + }; + } catch (error) { + this.logger.error('Error syncing Google Drive folders', error); + throw error; + } + } +} diff --git a/packages/api/src/filestorage/folder/services/googledrive/mappers.ts b/packages/api/src/filestorage/folder/services/googledrive/mappers.ts new file mode 100644 index 000000000..2d4caa6ff --- /dev/null +++ b/packages/api/src/filestorage/folder/services/googledrive/mappers.ts @@ -0,0 +1,101 @@ +import { MappersRegistry } from '@@core/@core-services/registries/mappers.registry'; +import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { Utils } from '@filestorage/@lib/@utils'; +import { IFolderMapper } from '@filestorage/folder/types'; +import { + UnifiedFilestorageFolderInput, + UnifiedFilestorageFolderOutput, +} from '@filestorage/folder/types/model.unified'; +import { Injectable } from '@nestjs/common'; +import { GoogleDriveFolderInput, GoogleDriveFolderOutput } from './types'; + +@Injectable() +export class GoogleDriveFolderMapper implements IFolderMapper { + constructor( + private mappersRegistry: MappersRegistry, + private utils: Utils, + private coreUnificationService: CoreUnification, + ) { + this.mappersRegistry.registerService( + 'filestorage', + 'folder', + 'googledrive', + this, + ); + } + + async desunify( + source: UnifiedFilestorageFolderInput, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + return { + name: source.name, + mimeType: 'application/vnd.google-apps.folder', + parents: source.parent_folder_id ? [source.parent_folder_id] : undefined, + }; + } + + async unify( + source: GoogleDriveFolderOutput | GoogleDriveFolderOutput[], + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise< + UnifiedFilestorageFolderOutput | UnifiedFilestorageFolderOutput[] + > { + if (!Array.isArray(source)) { + return await this.mapSingleFolderToUnified( + source, + connectionId, + customFieldMappings, + ); + } + return Promise.all( + source.map((folder) => + this.mapSingleFolderToUnified( + folder, + connectionId, + customFieldMappings, + ), + ), + ); + } + + private async mapSingleFolderToUnified( + folder: GoogleDriveFolderOutput, + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const field_mappings: { [key: string]: any } = {}; + if (customFieldMappings) { + for (const mapping of customFieldMappings) { + field_mappings[mapping.slug] = + folder[mapping.remote_id as keyof GoogleDriveFolderOutput]; + } + } + const opts: any = {}; + if (folder.parents && folder.parents.length > 0) { + const folder_id = await this.utils.getFolderIdFromRemote( + folder.parents[0], + connectionId, + ); + opts.folder_id = folder_id; + } + + return { + remote_id: folder.id, + remote_data: folder, + name: folder.name, + ...opts, + field_mappings, + }; + } +} diff --git a/packages/api/src/filestorage/folder/services/googledrive/types.ts b/packages/api/src/filestorage/folder/services/googledrive/types.ts new file mode 100644 index 000000000..37da27973 --- /dev/null +++ b/packages/api/src/filestorage/folder/services/googledrive/types.ts @@ -0,0 +1,37 @@ +export interface GoogleDriveFolderInput { + name: string; + mimeType: string; + parents?: string[]; +} + +export interface GoogleDriveFolderOutput { + id: string; + name: string; + mimeType: string; + createdTime: string; + modifiedTime: string; + parents?: string[]; + webViewLink?: string; + webContentLink?: string; + iconLink?: string; + hasThumbnail?: boolean; + thumbnailLink?: string; + shared?: boolean; + ownedByMe?: boolean; + capabilities?: { + canEdit?: boolean; + canShare?: boolean; + canDelete?: boolean; + canAddChildren?: boolean; + canRemoveChildren?: boolean; + canRename?: boolean; + canMoveItemWithinDrive?: boolean; + canMoveItemOutOfDrive?: boolean; + canTrash?: boolean; + canUntrash?: boolean; + }; + permissions?: any[]; // You can define a more specific type if needed + trashed?: boolean; + explicitlyTrashed?: boolean; + spaces?: string[]; +} diff --git a/packages/api/src/filestorage/folder/services/onedrive/index.ts b/packages/api/src/filestorage/folder/services/onedrive/index.ts new file mode 100644 index 000000000..8ae7b041b --- /dev/null +++ b/packages/api/src/filestorage/folder/services/onedrive/index.ts @@ -0,0 +1,187 @@ +import { Injectable } from '@nestjs/common'; +import { IFolderService } from '@filestorage/folder/types'; +import { FileStorageObject } from '@filestorage/@lib/@types'; +import axios from 'axios'; +import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; +import { LoggerService } from '@@core/@core-services/logger/logger.service'; +import { ActionType, handle3rdPartyServiceError } from '@@core/utils/errors'; +import { EncryptionService } from '@@core/@core-services/encryption/encryption.service'; +import { ApiResponse } from '@@core/utils/types'; +import { ServiceRegistry } from '../registry.service'; +import { OnedriveFolderInput, OnedriveFolderOutput } from './types'; +import { SyncParam } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; +import { UnifiedFilestorageFileOutput } from '@filestorage/file/types/model.unified'; +import { OnedriveFileOutput } from '@filestorage/file/services/onedrive/types'; + +@Injectable() +export class OnedriveService implements IFolderService { + constructor( + private prisma: PrismaService, + private logger: LoggerService, + private cryptoService: EncryptionService, + private registry: ServiceRegistry, + private ingestService: IngestDataService, + ) { + this.logger.setContext( + `${FileStorageObject.folder.toUpperCase()}:${OnedriveService.name}`, + ); + this.registry.registerService('onedrive', this); + } + + async addFolder( + folderData: OnedriveFolderInput, + linkedUserId: string, + ): Promise> { + try { + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'onedrive', + vertical: 'filestorage', + }, + }); + + // Currently adding in root folder, might need to change + const resp = await axios.post( + `${connection.account_url}/v1.0/drive/root/children`, + JSON.stringify({ + name: folderData.name, + folder: {}, + '@microsoft.graph.conflictBehavior': 'rename', // 'rename' | 'fail' | 'replace' + }), + { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }, + ); + + return { + data: resp.data, + message: 'Onedrive folder created', + statusCode: 201, + }; + } catch (error) { + throw error; + } + } + + async iterativeGetOnedriveFolders( + remote_folder_id: string, + linkedUserId: string, + ): Promise { + try { + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'onedrive', + vertical: 'filestorage', + }, + }); + + let result = [], + depth = 0, + batch = [remote_folder_id]; + + while (batch.length > 0) { + if (depth > 5) { + // todo: handle this better + break; + } + + const nestedFolders = await Promise.all( + batch.map(async (folder_id) => { + const resp = await axios.get( + `${connection.account_url}/v1.0/drive/items/${folder_id}/children`, + { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }, + ); + + // Add permissions (shared link is also included in permissions in one-drive) + await Promise.all( + resp.data.value.map(async (driveItem) => { + const resp = await axios.get( + `${connection.account_url}/v1.0/drive/items/${driveItem.id}/permissions`, + { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }, + ); + driveItem.permissions = resp.data.value; + }), + ); + + const folders = resp.data.value.filter( + (driveItem) => driveItem.folder, + ); + + // const files = resp.data.value.filter( + // (driveItem) => !driveItem.folder, + // ); + + // await this.ingestService.ingestData< + // UnifiedFilestorageFileOutput, + // OnedriveFileOutput + // >( + // files, + // 'onedrive', + // connection.id_connection, + // 'filestorage', + // FileStorageObject.file, + // ); + + return folders; + }), + ); + + // nestedFolders = [[subfolder1, subfolder2], [subfolder3, subfolder4]] + result = result.concat(nestedFolders.flat()); + batch = nestedFolders.flat().map((folder) => folder.id); + this.logger.log(`Batch size: ${batch.length} at depth ${depth}`); + depth++; + } + + return result; + } catch (error) { + throw error; + } + } + + async sync(data: SyncParam): Promise> { + try { + this.logger.log('Syncing onedrive folders'); + const { linkedUserId } = data; + + const folders = await this.iterativeGetOnedriveFolders( + 'root', + linkedUserId, + ); + + this.logger.log(`${folders.length} onedrive folders found`); + this.logger.log(`Synced onedrive folders !`); + + return { + data: folders, + message: 'Onedrive folders synced', + statusCode: 200, + }; + } catch (error) { + this.logger.log('Error in onedrive sync '); + throw error; + } + } +} diff --git a/packages/api/src/filestorage/folder/services/onedrive/mappers.ts b/packages/api/src/filestorage/folder/services/onedrive/mappers.ts new file mode 100644 index 000000000..c0e9c39b2 --- /dev/null +++ b/packages/api/src/filestorage/folder/services/onedrive/mappers.ts @@ -0,0 +1,146 @@ +import { MappersRegistry } from '@@core/@core-services/registries/mappers.registry'; +import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { OriginalSharedLinkOutput } from '@@core/utils/types/original/original.file-storage'; +import { Utils } from '@filestorage/@lib/@utils'; +import { IFolderMapper } from '@filestorage/folder/types'; +import { + UnifiedFilestorageFolderInput, + UnifiedFilestorageFolderOutput, +} from '@filestorage/folder/types/model.unified'; +import { UnifiedFilestorageSharedlinkOutput } from '@filestorage/sharedlink/types/model.unified'; +import { Injectable } from '@nestjs/common'; +import { OnedriveFolderInput, OnedriveFolderOutput } from './types'; +import { FileStorageObject } from '@filestorage/@lib/@types'; +import { OriginalPermissionOutput } from '@@core/utils/types/original/original.file-storage'; + +@Injectable() +export class OnedriveFolderMapper implements IFolderMapper { + constructor( + private mappersRegistry: MappersRegistry, + private utils: Utils, + private coreUnificationService: CoreUnification, + ) { + this.mappersRegistry.registerService( + 'filestorage', + 'folder', + 'onedrive', + this, + ); + } + + async desunify( + source: UnifiedFilestorageFolderInput, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const result = { + name: source.name, + folder: {}, + description: source.description, + }; + + if (customFieldMappings && source.field_mappings) { + for (const [k, v] of Object.entries(source.field_mappings)) { + const mapping = customFieldMappings.find( + (mapping) => mapping.slug === k, + ); + if (mapping) { + result[mapping.remote_id] = v; + } + } + } + + return result; + } + + async unify( + source: OnedriveFolderOutput | OnedriveFolderOutput[], + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise< + UnifiedFilestorageFolderOutput | UnifiedFilestorageFolderOutput[] + > { + if (!Array.isArray(source)) { + return await this.mapSingleFolderToUnified( + source, + connectionId, + customFieldMappings, + ); + } + return await Promise.all( + source.map((s) => + this.mapSingleFolderToUnified(s, connectionId, customFieldMappings), + ), + ); + } + + private async mapSingleFolderToUnified( + folder: OnedriveFolderOutput, + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const field_mappings: { [key: string]: any } = {}; + if (customFieldMappings) { + for (const mapping of customFieldMappings) { + field_mappings[mapping.slug] = folder[mapping.remote_id]; + } + } + + const opts: any = {}; + if (folder.permissions?.length) { + const permissions = await this.coreUnificationService.unify< + OriginalPermissionOutput[] + >({ + sourceObject: folder.permissions, + targetType: FileStorageObject.permission, + providerName: 'onedrive', + vertical: 'filestorage', + connectionId, + customFieldMappings: [], + }); + opts.permissions = permissions; + + // shared link + if (folder.permissions.some((p) => p.link)) { + const sharedLinks = + await this.coreUnificationService.unify({ + sourceObject: folder.permissions.find((p) => p.link), + targetType: FileStorageObject.sharedlink, + providerName: 'onedrive', + vertical: 'filestorage', + connectionId, + customFieldMappings: [], + }); + opts.shared_links = sharedLinks; + } + } + + const result = { + remote_id: folder.id, + remote_data: folder, + name: folder.name, + folder_url: folder.webUrl, + description: folder.description, + drive_id: null, + parent_folder_id: await this.utils.getFolderIdFromRemote( + folder.parentReference?.id, + connectionId, + ), + // permission: opts.permissions?.[0] || null, + permission: null, + size: folder.size.toString(), + shared_link: opts.shared_links?.[0] || null, + field_mappings, + }; + + return result; + } +} diff --git a/packages/api/src/filestorage/folder/services/onedrive/types.ts b/packages/api/src/filestorage/folder/services/onedrive/types.ts new file mode 100644 index 000000000..725a92f46 --- /dev/null +++ b/packages/api/src/filestorage/folder/services/onedrive/types.ts @@ -0,0 +1,155 @@ +import { + IdentitySet, + SharepointIds, +} from '@filestorage/drive/services/onedrive/types'; +import { OnedrivePermissionOutput } from '@filestorage/permission/services/onedrive/types'; + +/** + * Represents the input for a folder item in OneDrive. + * @see https://learn.microsoft.com/en-us/graph/api/resources/driveitem?view=graph-rest-1.0 + */ +export interface OnedriveFolderInput { + /** The unique identifier of the item within the Drive. */ + readonly id?: string; + /** The name of the item (filename and extension). */ + name?: string; + /** The URL that displays the resource in the browser. */ + readonly webUrl?: string; + /** Folder metadata. */ + folder?: Folder; + /** File system information on the client. */ + fileSystemInfo?: FileSystemInfo; + /** Parent information, if the item has a parent. */ + parentReference?: ItemReference; + /** The unique identifier of the drive instance that contains the driveItem. */ + readonly driveId?: string; + /** Identifies the type of drive. */ + readonly driveType?: string; + /** Information about the deleted state of the item. */ + deleted?: Deleted; + /** Description of the item. */ + description?: string; + /** Indicates the number of children contained immediately within this folder. */ + readonly childCount?: number; + /** Information about pending operations on the item. */ + pendingOperations?: PendingOperations; + /** View recommendations for the folder. */ + folderView?: FolderView; + /** SharePoint identifiers useful for REST compatibility. */ + readonly sharepointIds?: SharepointIds; + /** Special folder metadata. */ + readonly specialFolder?: SpecialFolder; + /** Identity of the user who created the folder. */ + readonly createdByUser?: IdentitySet; + /** Identity of the user who last modified the folder. */ + readonly lastModifiedByUser?: IdentitySet; + /** Permissions associated with the folder. */ + permissions?: OnedrivePermissionOutput[]; + /** Date and time the item was last modified. Read-only. */ + readonly lastModifiedDateTime?: string; + /** Date and time of item creation. Read-only. */ + readonly createdDateTime?: string; + /** Size of the item in bytes. Read-only. */ + readonly size?: number; + /** Identity of the user, device, and application that created the item. Read-only. */ + readonly createdBy?: IdentitySet; + /** Identity of the user, device, and application that last modified the item. Read-only. */ + readonly lastModifiedBy?: IdentitySet; +} + +/** + * Represents the folder metadata. + */ +export interface Folder { + /** The number of children contained immediately within this container. */ + readonly childCount?: number; + /** A collection of properties defining the recommended view for the folder. */ + view?: FolderView; +} + +/** + * Represents file system information for a client. + */ +export interface FileSystemInfo { + /** The UTC date and time the file was created on a client. */ + readonly createdDateTime?: string; + /** The UTC date and time the file was last accessed. */ + readonly lastAccessedDateTime?: string; + /** The UTC date and time the file was last modified on a client. */ + readonly lastModifiedDateTime?: string; +} + +/** + * Represents folder view recommendations. + */ +export interface FolderView { + /** How items in the folder are sorted. */ + sortBy?: + | 'default' + | 'name' + | 'type' + | 'size' + | 'takenOrCreatedDateTime' + | 'lastModifiedDateTime' + | 'sequence'; + /** The order in which items are sorted. */ + sortOrder?: 'ascending' | 'descending'; + /** The type of view recommended for the folder. */ + viewType?: 'default' | 'icons' | 'details' | 'thumbnails'; +} + +/** + * Represents the reference to an item. + */ +export interface ItemReference { + /** Unique identifier of the drive instance that contains the driveItem. */ + readonly driveId?: string; + /** Identifies the type of drive. */ + readonly driveType?: string; + /** Unique identifier of the driveItem in the drive or listItem in a list. */ + readonly id?: string; + /** The name of the item being referenced. */ + readonly name?: string; + /** Percent-encoded path to navigate to the item. */ + readonly path?: string; + /** Unique identifier for a shared resource. */ + readonly shareId?: string; + /** SharePoint identifiers useful for REST compatibility. */ + readonly sharepointIds?: SharepointIds; + /** ID of the site containing the parent document library or list. */ + readonly siteId?: string; +} + +/** + * Represents information about pending operations on an item. + */ +export interface PendingOperations { + /** Indicates that an operation that might update the binary content of a file is pending completion. */ + readonly pendingContentUpdate?: PendingContentUpdate; +} + +/** + * Represents information about an operation that might affect the binary content of the driveItem. + */ +export interface PendingContentUpdate { + /** Date and time the pending binary operation was queued in UTC time. */ + readonly queuedDateTime?: string; +} + +/** + * Represents special folder metadata. + */ +export interface SpecialFolder { + /** The unique identifier for this item in the /drive/special collection. */ + readonly name?: string; +} + +/** + * Represents information about the deleted state of an item. + */ +export interface Deleted { + /** Represents the state of the deleted item. */ + state?: string; +} + +export type OnedriveFolderOutput = OnedriveFolderInput; diff --git a/packages/api/src/filestorage/folder/services/sharepoint/index.ts b/packages/api/src/filestorage/folder/services/sharepoint/index.ts new file mode 100644 index 000000000..62731fc19 --- /dev/null +++ b/packages/api/src/filestorage/folder/services/sharepoint/index.ts @@ -0,0 +1,188 @@ +import { Injectable } from '@nestjs/common'; +import { IFolderService } from '@filestorage/folder/types'; +import { FileStorageObject } from '@filestorage/@lib/@types'; +import axios from 'axios'; +import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; +import { LoggerService } from '@@core/@core-services/logger/logger.service'; +import { ActionType, handle3rdPartyServiceError } from '@@core/utils/errors'; +import { EncryptionService } from '@@core/@core-services/encryption/encryption.service'; +import { ApiResponse } from '@@core/utils/types'; +import { ServiceRegistry } from '../registry.service'; +import { SharepointFolderInput, SharepointFolderOutput } from './types'; +import { SyncParam } from '@@core/utils/types/interface'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; +import { UnifiedFilestorageFileOutput } from '@filestorage/file/types/model.unified'; +import { SharepointFileOutput } from '@filestorage/file/services/sharepoint/types'; + +@Injectable() +export class SharepointService implements IFolderService { + constructor( + private prisma: PrismaService, + private logger: LoggerService, + private cryptoService: EncryptionService, + private registry: ServiceRegistry, + private ingestService: IngestDataService, + ) { + this.logger.setContext( + `${FileStorageObject.folder.toUpperCase()}:${SharepointService.name}`, + ); + this.registry.registerService('sharepoint', this); + } + + async addFolder( + folderData: SharepointFolderInput, + linkedUserId: string, + ): Promise> { + try { + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'sharepoint', + vertical: 'filestorage', + }, + }); + + // Currently adding in root folder, might need to change + const resp = await axios.post( + `${connection.account_url}/drive/root/children`, + JSON.stringify({ + name: folderData.name, + folder: {}, + '@microsoft.graph.conflictBehavior': 'rename', // 'rename' | 'fail' | 'replace' + }), + { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }, + ); + + return { + data: resp.data, + message: 'Sharepoint folder created', + statusCode: 201, + }; + } catch (error) { + console.log(error.response?.data); + throw error; + } + } + + async iterativeGetSharepointFolders( + remote_folder_id: string, + linkedUserId: string, + ): Promise { + try { + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'sharepoint', + vertical: 'filestorage', + }, + }); + + let result = [], + depth = 0, + batch = [remote_folder_id]; + + while (batch.length > 0) { + if (depth > 5) { + // todo: handle this better + break; + } + + const nestedFolders = await Promise.all( + batch.map(async (folder_id) => { + const resp = await axios.get( + `${connection.account_url}/drive/items/${folder_id}/children`, + { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }, + ); + + // Add permissions (shared link is also included in permissions in one-drive) + // await Promise.all( + // resp.data.value.map(async (driveItem) => { + // const resp = await axios.get( + // `${connection.account_url}/drive/items/${driveItem.id}/permissions`, + // { + // headers: { + // 'Content-Type': 'application/json', + // Authorization: `Bearer ${this.cryptoService.decrypt( + // connection.access_token, + // )}`, + // }, + // }, + // ); + // driveItem.permissions = resp.data.value; + // }), + // ); + + const folders = resp.data.value.filter( + (driveItem) => driveItem.folder, + ); + + // const files = resp.data.value.filter( + // (driveItem) => !driveItem.folder, + // ); + + // await this.ingestService.ingestData< + // UnifiedFilestorageFileOutput, + // SharepointFileOutput + // >( + // files, + // 'sharepoint', + // connection.id_connection, + // 'filestorage', + // FileStorageObject.file, + // ); + + return folders; + }), + ); + + // nestedFolders = [[subfolder1, subfolder2], [subfolder3, subfolder4]] + result = result.concat(nestedFolders.flat()); + batch = nestedFolders.flat().map((folder) => folder.id); + this.logger.log(`Batch size: ${batch.length} at depth ${depth}`); + depth++; + } + + return result; + } catch (error) { + throw error; + } + } + + async sync(data: SyncParam): Promise> { + try { + this.logger.log('Syncing sharepoint folders'); + const { linkedUserId } = data; + + const folders = await this.iterativeGetSharepointFolders( + 'root', + linkedUserId, + ); + + this.logger.log(`${folders.length} sharepoint folders found`); + this.logger.log(`Synced sharepoint folders !`); + + return { + data: folders, + message: 'Sharepoint folders synced', + statusCode: 200, + }; + } catch (error) { + this.logger.log('Error in sharepoint sync '); + throw error; + } + } +} diff --git a/packages/api/src/filestorage/folder/services/sharepoint/mappers.ts b/packages/api/src/filestorage/folder/services/sharepoint/mappers.ts new file mode 100644 index 000000000..b0c7f106b --- /dev/null +++ b/packages/api/src/filestorage/folder/services/sharepoint/mappers.ts @@ -0,0 +1,146 @@ +import { MappersRegistry } from '@@core/@core-services/registries/mappers.registry'; +import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; +import { OriginalSharedLinkOutput } from '@@core/utils/types/original/original.file-storage'; +import { Utils } from '@filestorage/@lib/@utils'; +import { IFolderMapper } from '@filestorage/folder/types'; +import { + UnifiedFilestorageFolderInput, + UnifiedFilestorageFolderOutput, +} from '@filestorage/folder/types/model.unified'; +import { UnifiedFilestorageSharedlinkOutput } from '@filestorage/sharedlink/types/model.unified'; +import { Injectable } from '@nestjs/common'; +import { SharepointFolderInput, SharepointFolderOutput } from './types'; +import { FileStorageObject } from '@filestorage/@lib/@types'; +import { OriginalPermissionOutput } from '@@core/utils/types/original/original.file-storage'; + +@Injectable() +export class SharepointFolderMapper implements IFolderMapper { + constructor( + private mappersRegistry: MappersRegistry, + private utils: Utils, + private coreUnificationService: CoreUnification, + ) { + this.mappersRegistry.registerService( + 'filestorage', + 'folder', + 'sharepoint', + this, + ); + } + + async desunify( + source: UnifiedFilestorageFolderInput, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const result = { + name: source.name, + folder: {}, + description: source.description, + }; + + if (customFieldMappings && source.field_mappings) { + for (const [k, v] of Object.entries(source.field_mappings)) { + const mapping = customFieldMappings.find( + (mapping) => mapping.slug === k, + ); + if (mapping) { + result[mapping.remote_id] = v; + } + } + } + + return result; + } + + async unify( + source: SharepointFolderOutput | SharepointFolderOutput[], + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise< + UnifiedFilestorageFolderOutput | UnifiedFilestorageFolderOutput[] + > { + if (!Array.isArray(source)) { + return await this.mapSingleFolderToUnified( + source, + connectionId, + customFieldMappings, + ); + } + return await Promise.all( + source.map((s) => + this.mapSingleFolderToUnified(s, connectionId, customFieldMappings), + ), + ); + } + + private async mapSingleFolderToUnified( + folder: SharepointFolderOutput, + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const field_mappings: { [key: string]: any } = {}; + if (customFieldMappings) { + for (const mapping of customFieldMappings) { + field_mappings[mapping.slug] = folder[mapping.remote_id]; + } + } + + const opts: any = {}; + if (folder.permissions?.length) { + const permissions = await this.coreUnificationService.unify< + OriginalPermissionOutput[] + >({ + sourceObject: folder.permissions, + targetType: FileStorageObject.permission, + providerName: 'sharepoint', + vertical: 'filestorage', + connectionId, + customFieldMappings: [], + }); + opts.permissions = permissions; + + // shared link + if (folder.permissions.some((p) => p.link)) { + const sharedLinks = + await this.coreUnificationService.unify({ + sourceObject: folder.permissions.find((p) => p.link), + targetType: FileStorageObject.sharedlink, + providerName: 'sharepoint', + vertical: 'filestorage', + connectionId, + customFieldMappings: [], + }); + opts.shared_links = sharedLinks; + } + } + + const result = { + remote_id: folder.id, + remote_data: folder, + name: folder.name, + folder_url: folder.webUrl, + description: folder.description, + drive_id: null, + parent_folder_id: await this.utils.getFolderIdFromRemote( + folder.parentReference?.id, + connectionId, + ), + // permission: opts.permissions?.[0] || null, + permission: null, + size: folder.size.toString(), + shared_link: opts.shared_links?.[0] || null, + field_mappings, + }; + + return result; + } +} diff --git a/packages/api/src/filestorage/folder/services/sharepoint/types.ts b/packages/api/src/filestorage/folder/services/sharepoint/types.ts new file mode 100644 index 000000000..5bf8784a4 --- /dev/null +++ b/packages/api/src/filestorage/folder/services/sharepoint/types.ts @@ -0,0 +1,155 @@ +import { + IdentitySet, + SharepointIds, +} from '@filestorage/drive/services/sharepoint/types'; +import { SharepointPermissionOutput } from '@filestorage/permission/services/sharepoint/types'; + +/** + * Represents the input for a folder item in OneDrive. + * @see https://learn.microsoft.com/en-us/graph/api/resources/driveitem?view=graph-rest-1.0 + */ +export interface SharepointFolderInput { + /** The unique identifier of the item within the Drive. */ + readonly id?: string; + /** The name of the item (filename and extension). */ + name?: string; + /** The URL that displays the resource in the browser. */ + readonly webUrl?: string; + /** Folder metadata. */ + folder?: Folder; + /** File system information on the client. */ + fileSystemInfo?: FileSystemInfo; + /** Parent information, if the item has a parent. */ + parentReference?: ItemReference; + /** The unique identifier of the drive instance that contains the driveItem. */ + readonly driveId?: string; + /** Identifies the type of drive. */ + readonly driveType?: string; + /** Information about the deleted state of the item. */ + deleted?: Deleted; + /** Description of the item. */ + description?: string; + /** Indicates the number of children contained immediately within this folder. */ + readonly childCount?: number; + /** Information about pending operations on the item. */ + pendingOperations?: PendingOperations; + /** View recommendations for the folder. */ + folderView?: FolderView; + /** SharePoint identifiers useful for REST compatibility. */ + readonly sharepointIds?: SharepointIds; + /** Special folder metadata. */ + readonly specialFolder?: SpecialFolder; + /** Identity of the user who created the folder. */ + readonly createdByUser?: IdentitySet; + /** Identity of the user who last modified the folder. */ + readonly lastModifiedByUser?: IdentitySet; + /** Permissions associated with the folder. */ + permissions?: SharepointPermissionOutput[]; + /** Date and time the item was last modified. Read-only. */ + readonly lastModifiedDateTime?: string; + /** Date and time of item creation. Read-only. */ + readonly createdDateTime?: string; + /** Size of the item in bytes. Read-only. */ + readonly size?: number; + /** Identity of the user, device, and application that created the item. Read-only. */ + readonly createdBy?: IdentitySet; + /** Identity of the user, device, and application that last modified the item. Read-only. */ + readonly lastModifiedBy?: IdentitySet; +} + +/** + * Represents the folder metadata. + */ +export interface Folder { + /** The number of children contained immediately within this container. */ + readonly childCount?: number; + /** A collection of properties defining the recommended view for the folder. */ + view?: FolderView; +} + +/** + * Represents file system information for a client. + */ +export interface FileSystemInfo { + /** The UTC date and time the file was created on a client. */ + readonly createdDateTime?: string; + /** The UTC date and time the file was last accessed. */ + readonly lastAccessedDateTime?: string; + /** The UTC date and time the file was last modified on a client. */ + readonly lastModifiedDateTime?: string; +} + +/** + * Represents folder view recommendations. + */ +export interface FolderView { + /** How items in the folder are sorted. */ + sortBy?: + | 'default' + | 'name' + | 'type' + | 'size' + | 'takenOrCreatedDateTime' + | 'lastModifiedDateTime' + | 'sequence'; + /** The order in which items are sorted. */ + sortOrder?: 'ascending' | 'descending'; + /** The type of view recommended for the folder. */ + viewType?: 'default' | 'icons' | 'details' | 'thumbnails'; +} + +/** + * Represents the reference to an item. + */ +export interface ItemReference { + /** Unique identifier of the drive instance that contains the driveItem. */ + readonly driveId?: string; + /** Identifies the type of drive. */ + readonly driveType?: string; + /** Unique identifier of the driveItem in the drive or listItem in a list. */ + readonly id?: string; + /** The name of the item being referenced. */ + readonly name?: string; + /** Percent-encoded path to navigate to the item. */ + readonly path?: string; + /** Unique identifier for a shared resource. */ + readonly shareId?: string; + /** SharePoint identifiers useful for REST compatibility. */ + readonly sharepointIds?: SharepointIds; + /** ID of the site containing the parent document library or list. */ + readonly siteId?: string; +} + +/** + * Represents information about pending operations on an item. + */ +export interface PendingOperations { + /** Indicates that an operation that might update the binary content of a file is pending completion. */ + readonly pendingContentUpdate?: PendingContentUpdate; +} + +/** + * Represents information about an operation that might affect the binary content of the driveItem. + */ +export interface PendingContentUpdate { + /** Date and time the pending binary operation was queued in UTC time. */ + readonly queuedDateTime?: string; +} + +/** + * Represents special folder metadata. + */ +export interface SpecialFolder { + /** The unique identifier for this item in the /drive/special collection. */ + readonly name?: string; +} + +/** + * Represents information about the deleted state of an item. + */ +export interface Deleted { + /** Represents the state of the deleted item. */ + state?: string; +} + +export type SharepointFolderOutput = SharepointFolderInput; diff --git a/packages/api/src/filestorage/folder/sync/sync.processor.ts b/packages/api/src/filestorage/folder/sync/sync.processor.ts deleted file mode 100644 index d347f0241..000000000 --- a/packages/api/src/filestorage/folder/sync/sync.processor.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - - @Process('filestorage-sync-folders') - async handleSyncFolders(job: Job) { - try { - console.log(`Processing queue -> filestorage-sync-folders ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing filestorage folders', error); - } - } -} diff --git a/packages/api/src/filestorage/folder/sync/sync.service.ts b/packages/api/src/filestorage/folder/sync/sync.service.ts index a0c38330d..878f1857d 100644 --- a/packages/api/src/filestorage/folder/sync/sync.service.ts +++ b/packages/api/src/filestorage/folder/sync/sync.service.ts @@ -32,65 +32,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('filestorage', 'folder', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob( - 'filestorage-sync-folders', - '0 0 * * *', - ); - } catch (error) { - throw error; - } + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing folders...'); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = FILESTORAGE_PROVIDERS; - for (const provider of providers) { - try { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = FILESTORAGE_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/filestorage/group/group.module.ts b/packages/api/src/filestorage/group/group.module.ts index f4a5cea75..f911b3d26 100644 --- a/packages/api/src/filestorage/group/group.module.ts +++ b/packages/api/src/filestorage/group/group.module.ts @@ -1,3 +1,9 @@ +import { DropboxGroupMapper } from './services/dropbox/mappers'; +import { DropboxService } from './services/dropbox'; +import { SharepointGroupMapper } from './services/sharepoint/mappers'; +import { SharepointService } from './services/sharepoint'; +import { OnedriveGroupMapper } from './services/onedrive/mappers'; +import { OnedriveService } from './services/onedrive'; import { BullQueueModule } from '@@core/@core-services/queues/queue.module'; import { CoreUnification } from '@@core/@core-services/unification/core-unification.service'; import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; @@ -24,6 +30,12 @@ import { SyncService } from './sync/sync.service'; BoxGroupMapper, /* PROVIDERS SERVICES */ BoxService, + SharepointService, + SharepointGroupMapper, + OnedriveService, + OnedriveGroupMapper, + DropboxService, + DropboxGroupMapper, ], exports: [SyncService], }) diff --git a/packages/api/src/filestorage/group/services/box/mappers.ts b/packages/api/src/filestorage/group/services/box/mappers.ts index 51ffb0b9f..457e2b818 100644 --- a/packages/api/src/filestorage/group/services/box/mappers.ts +++ b/packages/api/src/filestorage/group/services/box/mappers.ts @@ -64,7 +64,7 @@ export class BoxGroupMapper implements IGroupMapper { return { remote_id: group.id, name: group.name || null, - users: null, + users: [], remote_was_deleted: null, //created_at: group.created_at || null, //modified_at: group.modified_at || null, diff --git a/packages/api/src/filestorage/group/services/dropbox/index.ts b/packages/api/src/filestorage/group/services/dropbox/index.ts new file mode 100644 index 000000000..4ebc75247 --- /dev/null +++ b/packages/api/src/filestorage/group/services/dropbox/index.ts @@ -0,0 +1,62 @@ +import { EncryptionService } from '@@core/@core-services/encryption/encryption.service'; +import { LoggerService } from '@@core/@core-services/logger/logger.service'; +import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; +import { ApiResponse } from '@@core/utils/types'; +import { SyncParam } from '@@core/utils/types/interface'; +import { FileStorageObject } from '@filestorage/@lib/@types'; +import { IGroupService } from '@filestorage/group/types'; +import { Injectable } from '@nestjs/common'; +import axios from 'axios'; +import { ServiceRegistry } from '../registry.service'; +import { DropboxGroupOutput } from './types'; + +@Injectable() +export class DropboxService implements IGroupService { + constructor( + private prisma: PrismaService, + private logger: LoggerService, + private cryptoService: EncryptionService, + private registry: ServiceRegistry, + ) { + this.logger.setContext( + `${FileStorageObject.group.toUpperCase()}:${DropboxService.name}`, + ); + this.registry.registerService('dropbox', this); + } + + async sync(data: SyncParam): Promise> { + try { + const { linkedUserId } = data; + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'dropbox', + vertical: 'filestorage', + }, + }); + + // ref: https://www.dropbox.com/developers/documentation/http/teams#team-groups-list + const resp = await axios.post( + `${connection.account_url}/team/groups/list`, + { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }, + ); + + this.logger.log(`Synced dropbox groups !`); + + return { + data: resp.data.groups, + message: 'Dropbox groups retrieved', + statusCode: 200, + }; + } catch (error) { + throw error; + } + } +} diff --git a/packages/api/src/filestorage/group/services/dropbox/mappers.ts b/packages/api/src/filestorage/group/services/dropbox/mappers.ts new file mode 100644 index 000000000..3bbc9b241 --- /dev/null +++ b/packages/api/src/filestorage/group/services/dropbox/mappers.ts @@ -0,0 +1,78 @@ +import { + UnifiedFilestorageGroupInput, + UnifiedFilestorageGroupOutput, +} from '@filestorage/group/types/model.unified'; +import { IGroupMapper } from '@filestorage/group/types'; +import { Utils } from '@filestorage/@lib/@utils'; +import { MappersRegistry } from '@@core/@core-services/registries/mappers.registry'; +import { Injectable } from '@nestjs/common'; +import { DropboxGroupInput, DropboxGroupOutput } from './types'; + +@Injectable() +export class DropboxGroupMapper implements IGroupMapper { + constructor(private mappersRegistry: MappersRegistry, private utils: Utils) { + this.mappersRegistry.registerService( + 'filestorage', + 'group', + 'dropbox', + this, + ); + } + + async desunify( + source: UnifiedFilestorageGroupInput, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + return; + } + + async unify( + source: DropboxGroupOutput | DropboxGroupOutput[], + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + if (!Array.isArray(source)) { + return await this.mapSingleGroupToUnified( + source, + connectionId, + customFieldMappings, + ); + } + // Handling array of DropboxGroupOutput + return Promise.all( + source.map((group) => + this.mapSingleGroupToUnified(group, connectionId, customFieldMappings), + ), + ); + } + + private async mapSingleGroupToUnified( + group: DropboxGroupOutput, + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const field_mappings: { [key: string]: any } = {}; + if (customFieldMappings) { + for (const mapping of customFieldMappings) { + field_mappings[mapping.slug] = group[mapping.remote_id]; + } + } + return { + remote_id: group.group_id, + remote_data: group, + name: group.group_name, + users: [], + field_mappings, + remote_was_deleted: null, + }; + } +} diff --git a/packages/api/src/filestorage/group/services/dropbox/types.ts b/packages/api/src/filestorage/group/services/dropbox/types.ts new file mode 100644 index 000000000..7d5707948 --- /dev/null +++ b/packages/api/src/filestorage/group/services/dropbox/types.ts @@ -0,0 +1,73 @@ +/** + * Represents a group in Dropbox. + */ +export interface DropboxGroupOutput { + /** + * The name of the group. + */ + group_name: string; + + /** + * The unique identifier for the group. + */ + group_id: string; + + /** + * The management type of the group. + * This field indicates who is allowed to manage the group. + */ + group_management_type: { + '.tag': GroupManagementType; + }; + + /** + * An external ID associated with the group. + * This field is optional and allows an admin to attach an arbitrary ID to the group. + */ + group_external_id?: string; + + /** + * The number of members in the group. + * This field is optional. + */ + member_count?: number; +} + +/** + * Represents the type of management for a group in Dropbox. + * This determines who is allowed to manage the group. + */ +type GroupManagementType = + | 'user_managed' + | 'company_managed' + | 'system_managed'; + +/** + * Represents the input data for creating or updating a group in Dropbox. + */ +export interface DropboxGroupInput { + /** + * The name of the group. + */ + group_name: string; + + /** + * Whether to automatically add the creator of the group as an owner. + * The default value is `false`. + */ + add_creator_as_owner?: boolean; + + /** + * An external ID associated with the group. + * This field allows the creator of a team to attach an arbitrary external ID to the group. + * This field is optional. + */ + group_external_id?: string; + + /** + * The management type of the group. + * Determines whether the group can be managed by selected users or only by team admins. + * This field is optional. + */ + group_management_type?: GroupManagementType; +} diff --git a/packages/api/src/filestorage/group/services/onedrive/index.ts b/packages/api/src/filestorage/group/services/onedrive/index.ts new file mode 100644 index 000000000..fbd0e58df --- /dev/null +++ b/packages/api/src/filestorage/group/services/onedrive/index.ts @@ -0,0 +1,57 @@ +import { EncryptionService } from '@@core/@core-services/encryption/encryption.service'; +import { LoggerService } from '@@core/@core-services/logger/logger.service'; +import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; +import { ApiResponse } from '@@core/utils/types'; +import { SyncParam } from '@@core/utils/types/interface'; +import { FileStorageObject } from '@filestorage/@lib/@types'; +import { IGroupService } from '@filestorage/group/types'; +import { Injectable } from '@nestjs/common'; +import axios from 'axios'; +import { ServiceRegistry } from '../registry.service'; +import { OnedriveGroupOutput } from './types'; + +@Injectable() +export class OnedriveService implements IGroupService { + constructor( + private prisma: PrismaService, + private logger: LoggerService, + private cryptoService: EncryptionService, + private registry: ServiceRegistry, + ) { + this.logger.setContext( + FileStorageObject.group.toUpperCase() + ':' + OnedriveService.name, + ); + this.registry.registerService('onedrive', this); + } + + async sync(data: SyncParam): Promise> { + try { + const { linkedUserId } = data; + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'onedrive', + vertical: 'filestorage', + }, + }); + const resp = await axios.get(`${connection.account_url}/v1.0/groups`, { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }); + + this.logger.log(`Synced onedrive groups !`); + + return { + data: resp.data.value, + message: 'Onedrive groups retrieved', + statusCode: 200, + }; + } catch (error) { + throw error; + } + } +} diff --git a/packages/api/src/filestorage/group/services/onedrive/mappers.ts b/packages/api/src/filestorage/group/services/onedrive/mappers.ts new file mode 100644 index 000000000..14588c33a --- /dev/null +++ b/packages/api/src/filestorage/group/services/onedrive/mappers.ts @@ -0,0 +1,82 @@ +import { + UnifiedFilestorageGroupInput, + UnifiedFilestorageGroupOutput, +} from '@filestorage/group/types/model.unified'; +import { IGroupMapper } from '@filestorage/group/types'; +import { Utils } from '@filestorage/@lib/@utils'; +import { MappersRegistry } from '@@core/@core-services/registries/mappers.registry'; +import { Injectable } from '@nestjs/common'; +import { OnedriveGroupInput, OnedriveGroupOutput } from './types'; + +@Injectable() +export class OnedriveGroupMapper implements IGroupMapper { + constructor(private mappersRegistry: MappersRegistry, private utils: Utils) { + this.mappersRegistry.registerService( + 'filestorage', + 'group', + 'onedrive', + this, + ); + } + + async desunify( + source: UnifiedFilestorageGroupInput, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + return; + } + + async unify( + source: OnedriveGroupOutput | OnedriveGroupOutput[], + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + if (!Array.isArray(source)) { + return await this.mapSingleGroupToUnified( + source, + connectionId, + customFieldMappings, + ); + } + // Handling array of OneDriveGroupOutput + return Promise.all( + source.map((group) => + this.mapSingleGroupToUnified(group, connectionId, customFieldMappings), + ), + ); + } + + private async mapSingleGroupToUnified( + group: OnedriveGroupOutput, + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const field_mappings: { [key: string]: any } = {}; + if (customFieldMappings) { + for (const mapping of customFieldMappings) { + field_mappings[mapping.slug] = group[mapping.remote_id]; + } + } + + // todo: do something about users + // https://graph.microsoft.com/v1.0/groups/group-id/members + + return { + remote_id: group.id, + remote_data: group, + name: group.mailNickname, + remote_was_deleted: group.deletedDateTime !== null, + field_mappings, + users: [], + }; + } +} diff --git a/packages/api/src/filestorage/group/services/onedrive/types.ts b/packages/api/src/filestorage/group/services/onedrive/types.ts new file mode 100644 index 000000000..1eec3aa0d --- /dev/null +++ b/packages/api/src/filestorage/group/services/onedrive/types.ts @@ -0,0 +1,145 @@ +export interface OnedriveGroupInput { + /** Unique identifier for the group. */ + id?: string; + /** + * Timestamp of when the group was created. The value can't be modified and is automatically populated when the group is + * created. The Timestamp type represents date and time information using ISO 8601 format and is always in UTC time. For + * example, midnight UTC on January 1, 2014 is 2014-01-01T00:00:00Z. Returned by default. Read-only. + */ + createdDateTime?: string; + /** + * An optional description for the group. Returned by default. Supports $filter (eq, ne, not, ge, le, startsWith) and + * $search. + */ + description?: string; + /** + * Date and time when this object was deleted. Always null when the object hasn't been deleted. + */ + deletedDateTime?: string; + /** + * The display name for the group. This property is required when a group is created and can't be cleared during updates. + * Maximum length is 256 characters. Returned by default. Supports $filter (eq, ne, not, ge, le, in, startsWith, and eq on + * null values), $search, and $orderby. + */ + displayName?: string; + /** + * Timestamp of when the group is set to expire. It's null for security groups, but for Microsoft 365 groups, it + * represents when the group is set to expire as defined in the groupLifecyclePolicy. The Timestamp type represents date + * and time information using ISO 8601 format and is always in UTC. For example, midnight UTC on January 1, 2014 is + * 2014-01-01T00:00:00Z. Returned by default. Supports $filter (eq, ne, not, ge, le, in). Read-only. + */ + expirationDateTime?: string; + /** + * Specifies the group type and its membership. If the collection contains Unified, the group is a Microsoft 365 group; + * otherwise, it's either a security group or a distribution group. For details, see groups overview.If the collection + * includes DynamicMembership, the group has dynamic membership; otherwise, membership is static. Returned by default. + * Supports $filter (eq, not). + */ + groupTypes?: string[]; + /** + * When a group is associated with a team, this property determines whether the team is in read-only mode.To read this + * property, use the /group/{groupId}/team endpoint or the Get team API. To update this property, use the archiveTeam and + * unarchiveTeam APIs. + */ + isArchived?: boolean; + /** + * The SMTP address for the group, for example, 'serviceadmins@contoso.com'. Returned by default. Read-only. Supports + * $filter (eq, ne, not, ge, le, in, startsWith, and eq on null values). + */ + mail?: string; + // Specifies whether the group is mail-enabled. Required. Returned by default. Supports $filter (eq, ne, not). + mailEnabled?: boolean; + /** + * The mail alias for the group, unique for Microsoft 365 groups in the organization. Maximum length is 64 characters. + * This property can contain only characters in the ASCII character set 0 - 127 except the following characters: @ () / [] + * ' ; : &lt;&gt; , SPACE. Required. Returned by default. Supports $filter (eq, ne, not, ge, le, in, startsWith, + * and eq on null values). + */ + mailNickname?: string; + /** + * The preferred data location for the Microsoft 365 group. By default, the group inherits the group creator's preferred + * data location. To set this property, the calling app must be granted the Directory.ReadWrite.All permission and the + * user be assigned at least one of the following Microsoft Entra roles: User Account Administrator Directory Writer + * Exchange Administrator SharePoint Administrator For more information about this property, see OneDrive Online + * Multi-Geo. Nullable. Returned by default. + */ + preferredDataLocation?: string; + /** + * The preferred language for a Microsoft 365 group. Should follow ISO 639-1 Code; for example, en-US. Returned by + * default. Supports $filter (eq, ne, not, ge, le, in, startsWith, and eq on null values). + */ + preferredLanguage?: string; + /** + * Email addresses for the group that direct to the same group mailbox. For example: ['SMTP: bob@contoso.com', 'smtp: + * bob@sales.contoso.com']. The any operator is required to filter expressions on multi-valued properties. Returned by + * default. Read-only. Not nullable. Supports $filter (eq, not, ge, le, startsWith, endsWith, /$count eq 0, /$count ne 0). + */ + proxyAddresses?: string[]; + /** + * Timestamp of when the group was last renewed. This value can't be modified directly and is only updated via the renew + * service action. The Timestamp type represents date and time information using ISO 8601 format and is always in UTC. For + * example, midnight UTC on January 1, 2014 is 2014-01-01T00:00:00Z. Returned by default. Supports $filter (eq, ne, not, + * ge, le, in). Read-only. + */ + + renewedDateTime?: string; + /** + * Specifies whether the group is a security group. Required. Returned by default. Supports $filter (eq, ne, not, in). + */ + securityEnabled?: boolean; + /** + * Security identifier of the group, used in Windows scenarios. Read-only. Returned by default. + */ + securityIdentifier?: string; + // The unique identifier that can be assigned to a group and used as an alternate key. Immutable. Read-only. + uniqueName?: string; + /** + * Specifies the group join policy and group content visibility for groups. Possible values are: Private, Public, or + * HiddenMembership. HiddenMembership can be set only for Microsoft 365 groups when the groups are created. It can't be + * updated later. Other values of visibility can be updated after group creation. If visibility value isn't specified + * during group creation on Microsoft Graph, a security group is created as Private by default, and the Microsoft 365 + * group is Public. Groups assignable to roles are always Private. To learn more, see group visibility options. Returned + * by default. Nullable. + */ + visibility?: string; + /** + * The user (or application) that created the group. NOTE: This property isn't set if the user is an administrator. + * Read-only. + */ + createdOnBehalfOf?: DirectoryObject; + /** + * Groups that this group is a member of. HTTP Methods: GET (supported for all groups). Read-only. Nullable. Supports + * $expand. + */ + memberOf?: DirectoryObject[]; + /** + * The members of this group, who can be users, devices, other groups, or service principals. Supports the List members, + * Add member, and Remove member operations. Nullable. Supports $expand including nested $select. For example, + * /groups?$filter=startsWith(displayName,'Role')&$select=id,displayName&$expand=members($select=id,userPrincipalName,displayName). + */ + members?: DirectoryObject[]; + /** + * The owners of the group. Limited to 100 owners. Nullable. If this property isn't specified when creating a Microsoft + * 365 group, the calling user is automatically assigned as the group owner. Supports $filter (/$count eq 0, /$count ne 0, + * /$count eq 1, /$count ne 1). Supports $expand including nested $select. For example, + * /groups?$filter=startsWith(displayName,'Role')&$select=id,displayName&$expand=owners($select=id,userPrincipalName,displayName). + */ + owners?: DirectoryObject[]; +} + +/** + * Base type for all directory objects. + * @interface + */ +export interface DirectoryObject { + /** + * The unique identifier for an entity. Read-only. + */ + id?: string; + /** + * Date and time when this object was deleted. Always null when the object hasn't been deleted. + */ + deletedDateTime?: string; +} + +export type OnedriveGroupOutput = Partial; diff --git a/packages/api/src/filestorage/group/services/sharepoint/index.ts b/packages/api/src/filestorage/group/services/sharepoint/index.ts new file mode 100644 index 000000000..f839eccdf --- /dev/null +++ b/packages/api/src/filestorage/group/services/sharepoint/index.ts @@ -0,0 +1,61 @@ +import { EncryptionService } from '@@core/@core-services/encryption/encryption.service'; +import { LoggerService } from '@@core/@core-services/logger/logger.service'; +import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; +import { ApiResponse } from '@@core/utils/types'; +import { SyncParam } from '@@core/utils/types/interface'; +import { FileStorageObject } from '@filestorage/@lib/@types'; +import { IGroupService } from '@filestorage/group/types'; +import { Injectable } from '@nestjs/common'; +import axios from 'axios'; +import { ServiceRegistry } from '../registry.service'; +import { SharepointGroupOutput } from './types'; + +@Injectable() +export class SharepointService implements IGroupService { + constructor( + private prisma: PrismaService, + private logger: LoggerService, + private cryptoService: EncryptionService, + private registry: ServiceRegistry, + ) { + this.logger.setContext( + `${FileStorageObject.group.toUpperCase()}:${SharepointService.name}`, + ); + this.registry.registerService('sharepoint', this); + } + + async sync(data: SyncParam): Promise> { + try { + const { linkedUserId } = data; + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'sharepoint', + vertical: 'filestorage', + }, + }); + // remove /sites/site_id from account_url + const url = connection.account_url.replace(/\/sites\/.+$/, ''); + + // ref: https://learn.microsoft.com/en-us/graph/api/user-list?view=graph-rest-1.0&tabs=http + const resp = await axios.get(`${url}/groups`, { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }); + + this.logger.log(`Synced sharepoint groups !`); + + return { + data: resp.data.value, + message: 'Sharepoint groups retrieved', + statusCode: 200, + }; + } catch (error) { + throw error; + } + } +} diff --git a/packages/api/src/filestorage/group/services/sharepoint/mappers.ts b/packages/api/src/filestorage/group/services/sharepoint/mappers.ts new file mode 100644 index 000000000..3cd197384 --- /dev/null +++ b/packages/api/src/filestorage/group/services/sharepoint/mappers.ts @@ -0,0 +1,82 @@ +import { + UnifiedFilestorageGroupInput, + UnifiedFilestorageGroupOutput, +} from '@filestorage/group/types/model.unified'; +import { IGroupMapper } from '@filestorage/group/types'; +import { Utils } from '@filestorage/@lib/@utils'; +import { MappersRegistry } from '@@core/@core-services/registries/mappers.registry'; +import { Injectable } from '@nestjs/common'; +import { SharepointGroupInput, SharepointGroupOutput } from './types'; + +@Injectable() +export class SharepointGroupMapper implements IGroupMapper { + constructor(private mappersRegistry: MappersRegistry, private utils: Utils) { + this.mappersRegistry.registerService( + 'filestorage', + 'group', + 'sharepoint', + this, + ); + } + + async desunify( + source: UnifiedFilestorageGroupInput, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + return; + } + + async unify( + source: SharepointGroupOutput | SharepointGroupOutput[], + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + if (!Array.isArray(source)) { + return await this.mapSingleGroupToUnified( + source, + connectionId, + customFieldMappings, + ); + } + // Handling array of SharepointGroupOutput + return Promise.all( + source.map((group) => + this.mapSingleGroupToUnified(group, connectionId, customFieldMappings), + ), + ); + } + + private async mapSingleGroupToUnified( + group: SharepointGroupOutput, + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const field_mappings: { [key: string]: any } = {}; + if (customFieldMappings) { + for (const mapping of customFieldMappings) { + field_mappings[mapping.slug] = group[mapping.remote_id]; + } + } + + // todo: do something about users + // https://graph.microsoft.com/groups/group-id/members + + return { + remote_id: group.id, + remote_data: group, + name: group.mailNickname, + remote_was_deleted: group.deletedDateTime !== null, + field_mappings, + users: [], + }; + } +} diff --git a/packages/api/src/filestorage/group/services/sharepoint/types.ts b/packages/api/src/filestorage/group/services/sharepoint/types.ts new file mode 100644 index 000000000..a54c0b5e6 --- /dev/null +++ b/packages/api/src/filestorage/group/services/sharepoint/types.ts @@ -0,0 +1,145 @@ +export interface SharepointGroupInput { + /** Unique identifier for the group. */ + id?: string; + /** + * Timestamp of when the group was created. The value can't be modified and is automatically populated when the group is + * created. The Timestamp type represents date and time information using ISO 8601 format and is always in UTC time. For + * example, midnight UTC on January 1, 2014 is 2014-01-01T00:00:00Z. Returned by default. Read-only. + */ + createdDateTime?: string; + /** + * An optional description for the group. Returned by default. Supports $filter (eq, ne, not, ge, le, startsWith) and + * $search. + */ + description?: string; + /** + * Date and time when this object was deleted. Always null when the object hasn't been deleted. + */ + deletedDateTime?: string; + /** + * The display name for the group. This property is required when a group is created and can't be cleared during updates. + * Maximum length is 256 characters. Returned by default. Supports $filter (eq, ne, not, ge, le, in, startsWith, and eq on + * null values), $search, and $orderby. + */ + displayName?: string; + /** + * Timestamp of when the group is set to expire. It's null for security groups, but for Microsoft 365 groups, it + * represents when the group is set to expire as defined in the groupLifecyclePolicy. The Timestamp type represents date + * and time information using ISO 8601 format and is always in UTC. For example, midnight UTC on January 1, 2014 is + * 2014-01-01T00:00:00Z. Returned by default. Supports $filter (eq, ne, not, ge, le, in). Read-only. + */ + expirationDateTime?: string; + /** + * Specifies the group type and its membership. If the collection contains Unified, the group is a Microsoft 365 group; + * otherwise, it's either a security group or a distribution group. For details, see groups overview.If the collection + * includes DynamicMembership, the group has dynamic membership; otherwise, membership is static. Returned by default. + * Supports $filter (eq, not). + */ + groupTypes?: string[]; + /** + * When a group is associated with a team, this property determines whether the team is in read-only mode.To read this + * property, use the /group/{groupId}/team endpoint or the Get team API. To update this property, use the archiveTeam and + * unarchiveTeam APIs. + */ + isArchived?: boolean; + /** + * The SMTP address for the group, for example, 'serviceadmins@contoso.com'. Returned by default. Read-only. Supports + * $filter (eq, ne, not, ge, le, in, startsWith, and eq on null values). + */ + mail?: string; + // Specifies whether the group is mail-enabled. Required. Returned by default. Supports $filter (eq, ne, not). + mailEnabled?: boolean; + /** + * The mail alias for the group, unique for Microsoft 365 groups in the organization. Maximum length is 64 characters. + * This property can contain only characters in the ASCII character set 0 - 127 except the following characters: @ () / [] + * ' ; : &lt;&gt; , SPACE. Required. Returned by default. Supports $filter (eq, ne, not, ge, le, in, startsWith, + * and eq on null values). + */ + mailNickname?: string; + /** + * The preferred data location for the Microsoft 365 group. By default, the group inherits the group creator's preferred + * data location. To set this property, the calling app must be granted the Directory.ReadWrite.All permission and the + * user be assigned at least one of the following Microsoft Entra roles: User Account Administrator Directory Writer + * Exchange Administrator SharePoint Administrator For more information about this property, see OneDrive Online + * Multi-Geo. Nullable. Returned by default. + */ + preferredDataLocation?: string; + /** + * The preferred language for a Microsoft 365 group. Should follow ISO 639-1 Code; for example, en-US. Returned by + * default. Supports $filter (eq, ne, not, ge, le, in, startsWith, and eq on null values). + */ + preferredLanguage?: string; + /** + * Email addresses for the group that direct to the same group mailbox. For example: ['SMTP: bob@contoso.com', 'smtp: + * bob@sales.contoso.com']. The any operator is required to filter expressions on multi-valued properties. Returned by + * default. Read-only. Not nullable. Supports $filter (eq, not, ge, le, startsWith, endsWith, /$count eq 0, /$count ne 0). + */ + proxyAddresses?: string[]; + /** + * Timestamp of when the group was last renewed. This value can't be modified directly and is only updated via the renew + * service action. The Timestamp type represents date and time information using ISO 8601 format and is always in UTC. For + * example, midnight UTC on January 1, 2014 is 2014-01-01T00:00:00Z. Returned by default. Supports $filter (eq, ne, not, + * ge, le, in). Read-only. + */ + + renewedDateTime?: string; + /** + * Specifies whether the group is a security group. Required. Returned by default. Supports $filter (eq, ne, not, in). + */ + securityEnabled?: boolean; + /** + * Security identifier of the group, used in Windows scenarios. Read-only. Returned by default. + */ + securityIdentifier?: string; + // The unique identifier that can be assigned to a group and used as an alternate key. Immutable. Read-only. + uniqueName?: string; + /** + * Specifies the group join policy and group content visibility for groups. Possible values are: Private, Public, or + * HiddenMembership. HiddenMembership can be set only for Microsoft 365 groups when the groups are created. It can't be + * updated later. Other values of visibility can be updated after group creation. If visibility value isn't specified + * during group creation on Microsoft Graph, a security group is created as Private by default, and the Microsoft 365 + * group is Public. Groups assignable to roles are always Private. To learn more, see group visibility options. Returned + * by default. Nullable. + */ + visibility?: string; + /** + * The user (or application) that created the group. NOTE: This property isn't set if the user is an administrator. + * Read-only. + */ + createdOnBehalfOf?: DirectoryObject; + /** + * Groups that this group is a member of. HTTP Methods: GET (supported for all groups). Read-only. Nullable. Supports + * $expand. + */ + memberOf?: DirectoryObject[]; + /** + * The members of this group, who can be users, devices, other groups, or service principals. Supports the List members, + * Add member, and Remove member operations. Nullable. Supports $expand including nested $select. For example, + * /groups?$filter=startsWith(displayName,'Role')&$select=id,displayName&$expand=members($select=id,userPrincipalName,displayName). + */ + members?: DirectoryObject[]; + /** + * The owners of the group. Limited to 100 owners. Nullable. If this property isn't specified when creating a Microsoft + * 365 group, the calling user is automatically assigned as the group owner. Supports $filter (/$count eq 0, /$count ne 0, + * /$count eq 1, /$count ne 1). Supports $expand including nested $select. For example, + * /groups?$filter=startsWith(displayName,'Role')&$select=id,displayName&$expand=owners($select=id,userPrincipalName,displayName). + */ + owners?: DirectoryObject[]; +} + +/** + * Base type for all directory objects. + * @interface + */ +export interface DirectoryObject { + /** + * The unique identifier for an entity. Read-only. + */ + id?: string; + /** + * Date and time when this object was deleted. Always null when the object hasn't been deleted. + */ + deletedDateTime?: string; +} + +export type SharepointGroupOutput = Partial; diff --git a/packages/api/src/filestorage/group/sync/sync.processor.ts b/packages/api/src/filestorage/group/sync/sync.processor.ts deleted file mode 100644 index 0846d9244..000000000 --- a/packages/api/src/filestorage/group/sync/sync.processor.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { Processor, Process } from '@nestjs/bull'; -import { Job } from 'bull'; -import { SyncService } from './sync.service'; -import { Queues } from '@@core/@core-services/queues/types'; - -@Processor(Queues.SYNC_JOBS_WORKER) -export class SyncProcessor { - constructor(private syncService: SyncService) {} - - @Process('filestorage-sync-groups') - async handleSyncGroups(job: Job) { - try { - console.log(`Processing queue -> filestorage-sync-groups ${job.id}`); - await this.syncService.kickstartSync(); - } catch (error) { - console.error('Error syncing filestorage groups', error); - } - } -} diff --git a/packages/api/src/filestorage/group/sync/sync.service.ts b/packages/api/src/filestorage/group/sync/sync.service.ts index 1a988fc14..fc4125103 100644 --- a/packages/api/src/filestorage/group/sync/sync.service.ts +++ b/packages/api/src/filestorage/group/sync/sync.service.ts @@ -34,65 +34,35 @@ export class SyncService implements OnModuleInit, IBaseSync { this.logger.setContext(SyncService.name); this.registry.registerService('filestorage', 'group', this); } - - async onModuleInit() { - try { - await this.bullQueueService.queueSyncJob( - 'filestorage-sync-groups', - '0 0 * * *', - ); - } catch (error) { - throw error; - } + onModuleInit() { +// } @Cron('0 */8 * * *') // every 8 hours - async kickstartSync(user_id?: string) { + async kickstartSync(id_project?: string) { try { - this.logger.log('Syncing groups...'); - const users = user_id - ? [ - await this.prisma.users.findUnique({ - where: { - id_user: user_id, - }, - }), - ] - : await this.prisma.users.findMany(); - if (users && users.length > 0) { - for (const user of users) { - const projects = await this.prisma.projects.findMany({ - where: { - id_user: user.id_user, - }, - }); - for (const project of projects) { - const id_project = project.id_project; - const linkedUsers = await this.prisma.linked_users.findMany({ - where: { - id_project: id_project, - }, - }); - linkedUsers.map(async (linkedUser) => { - try { - const providers = FILESTORAGE_PROVIDERS; - for (const provider of providers) { - try { - await this.syncForLinkedUser({ - integrationId: provider, - linkedUserId: linkedUser.id_linked_user, - }); - } catch (error) { - throw error; - } - } - } catch (error) { - throw error; - } - }); + const linkedUsers = await this.prisma.linked_users.findMany({ + where: { + id_project: id_project, + }, + }); + linkedUsers.map(async (linkedUser) => { + try { + const providers = FILESTORAGE_PROVIDERS; + for (const provider of providers) { + try { + await this.syncForLinkedUser({ + integrationId: provider, + linkedUserId: linkedUser.id_linked_user, + }); + } catch (error) { + throw error; + } } + } catch (error) { + throw error; } - } + }); } catch (error) { throw error; } diff --git a/packages/api/src/filestorage/permission/permission.module.ts b/packages/api/src/filestorage/permission/permission.module.ts index 9eb5a76d5..a15ae2253 100644 --- a/packages/api/src/filestorage/permission/permission.module.ts +++ b/packages/api/src/filestorage/permission/permission.module.ts @@ -1,3 +1,7 @@ +import { SharepointPermissionMapper } from './services/sharepoint/mappers'; +import { SharepointService } from './services/sharepoint'; +import { OnedrivePermissionMapper } from './services/onedrive/mappers'; +import { OnedriveService } from './services/onedrive'; import { EncryptionService } from '@@core/@core-services/encryption/encryption.service'; import { LoggerService } from '@@core/@core-services/logger/logger.service'; import { WebhookService } from '@@core/@core-services/webhooks/panora-webhooks/webhook.service'; @@ -29,6 +33,10 @@ import { Utils } from '@filestorage/@lib/@utils'; IngestDataService, /* PROVIDERS SERVICES */ + SharepointService, + SharepointPermissionMapper, + OnedriveService, + OnedrivePermissionMapper, ], exports: [SyncService], }) diff --git a/packages/api/src/filestorage/permission/services/onedrive/index.ts b/packages/api/src/filestorage/permission/services/onedrive/index.ts new file mode 100644 index 000000000..9372558e7 --- /dev/null +++ b/packages/api/src/filestorage/permission/services/onedrive/index.ts @@ -0,0 +1,80 @@ +import { Injectable } from '@nestjs/common'; +import { IPermissionService } from '@filestorage/permission/types'; +import { FileStorageObject } from '@panora/shared'; +import axios from 'axios'; +import { PrismaService } from '@@core/@core-services/prisma/prisma.service'; +import { LoggerService } from '@@core/@core-services/logger/logger.service'; +import { EncryptionService } from '@@core/@core-services/encryption/encryption.service'; +import { ApiResponse } from '@@core/utils/types'; +import { ServiceRegistry } from '../registry.service'; +import { OnedrivePermissionInput, OnedrivePermissionOutput } from './types'; +import { SyncParam } from '@@core/utils/types/interface'; + +@Injectable() +export class OnedriveService implements IPermissionService { + constructor( + private prisma: PrismaService, + private logger: LoggerService, + private cryptoService: EncryptionService, + private registry: ServiceRegistry, + ) { + this.logger.setContext( + FileStorageObject.permission.toUpperCase() + ':' + OnedriveService.name, + ); + this.registry.registerService('onedrive', this); + } + + async sync( + data: SyncParam, + ): Promise> { + try { + const { linkedUserId, extra } = data; + // TODO: where it comes from ?? extra?: { object_name: 'folder' | 'file'; value: string }, + + const connection = await this.prisma.connections.findFirst({ + where: { + id_linked_user: linkedUserId, + provider_slug: 'onedrive', + vertical: 'filestorage', + }, + }); + let remote_id; + if (extra.object_name == 'folder') { + const a = await this.prisma.fs_folders.findUnique({ + where: { + id_fs_folder: extra.value, + }, + }); + remote_id = a.remote_id; + } + if (extra.object_name == 'file') { + const a = await this.prisma.fs_files.findUnique({ + where: { + id_fs_file: extra.value, + }, + }); + + remote_id = a.remote_id; + } + + const resp = await axios.get( + `${connection.account_url}/v1.0/drive/items/${remote_id}/permissions`, + { + headers: { + Authorization: `Bearer ${this.cryptoService.decrypt( + connection.access_token, + )}`, + }, + }, + ); + + return { + data: resp.data.value as OnedrivePermissionOutput[], + message: 'Synced onedrive permissions !', + statusCode: 200, + }; + } catch (error) { + throw error; + } + } +} diff --git a/packages/api/src/filestorage/permission/services/onedrive/mappers.ts b/packages/api/src/filestorage/permission/services/onedrive/mappers.ts new file mode 100644 index 000000000..a78cedb67 --- /dev/null +++ b/packages/api/src/filestorage/permission/services/onedrive/mappers.ts @@ -0,0 +1,95 @@ +import { + UnifiedFilestoragePermissionInput, + UnifiedFilestoragePermissionOutput, +} from '@filestorage/permission/types/model.unified'; +import { IPermissionMapper } from '@filestorage/permission/types'; +import { MappersRegistry } from '@@core/@core-services/registries/mappers.registry'; +import { Injectable } from '@nestjs/common'; +import { OriginalPermissionOutput } from '@@core/utils/types/original/original.file-storage'; +import { IngestDataService } from '@@core/@core-services/unification/ingest-data.service'; +import { OnedrivePermissionInput, OnedrivePermissionOutput } from './types'; + +@Injectable() +export class OnedrivePermissionMapper implements IPermissionMapper { + constructor( + private mappersRegistry: MappersRegistry, + private ingestService: IngestDataService, + ) { + this.mappersRegistry.registerService( + 'filestorage', + 'permission', + 'onedrive', + this, + ); + } + + async desunify( + source: UnifiedFilestoragePermissionInput, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + return; + } + + async unify( + source: OnedrivePermissionOutput | OnedrivePermissionOutput[], + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise< + UnifiedFilestoragePermissionOutput | UnifiedFilestoragePermissionOutput[] + > { + if (!Array.isArray(source)) { + return await this.mapSinglePermissionToUnified( + source, + connectionId, + customFieldMappings, + ); + } + // Handling array of OnedrivePermissionOutput + return Promise.all( + source.map((permission) => + this.mapSinglePermissionToUnified( + permission, + connectionId, + customFieldMappings, + ), + ), + ); + } + + private async mapSinglePermissionToUnified( + permission: OnedrivePermissionOutput, + connectionId: string, + customFieldMappings?: { + slug: string; + remote_id: string; + }[], + ): Promise { + const field_mappings: { [key: string]: any } = {}; + if (customFieldMappings) { + for (const mapping of customFieldMappings) { + field_mappings[mapping.slug] = permission[mapping.remote_id]; + } + } + + return { + remote_id: permission.id, + remote_data: permission, + roles: permission.roles.map((role) => role.toUpperCase()), + type: + permission.link?.type === 'edit' + ? 'WRITE' + : permission.link?.type === 'view' + ? 'READ' + : permission.link?.type, + user_id: null, + group_id: null, + field_mappings, + }; + } +} diff --git a/packages/api/src/filestorage/permission/services/onedrive/types.ts b/packages/api/src/filestorage/permission/services/onedrive/types.ts new file mode 100644 index 000000000..bda7dfa81 --- /dev/null +++ b/packages/api/src/filestorage/permission/services/onedrive/types.ts @@ -0,0 +1,109 @@ +import { + Identity, + IdentitySet, +} from '@filestorage/drive/services/onedrive/types'; +import { ItemReference } from '@filestorage/folder/services/onedrive/types'; + +/** + * Represents a permission associated with a folder item. + * @see https://learn.microsoft.com/en-us/graph/api/driveitem-invite?view=graph-rest-1.0&tabs=http + */ +export interface OnedrivePermissionOutput { + /** The unique identifier of the permission among all permissions on the item. */ + id?: string; + /** Indicates whether the password is set for this permission. */ + hasPassword?: boolean; + /** For link type permissions, the details of the users to whom permission was granted. */ + grantedToV2?: SharePointIdentitySet; + /** Provides a reference to the ancestor of the current permission, if it's inherited from an ancestor. */ + inheritedFrom?: ItemReference; + /** Details of any associated sharing invitation for this permission. */ + invitation?: SharingInvitation; + /** Provides the link details of the current permission, if it's a link type permission. */ + link?: SharingLink; + /** The type of permission, for example, read. */ + roles?: ('read' | 'write' | 'owner')[]; + /** A unique token that can be used to access this shared item via the shares API. */ + shareId?: string; + /** A format of yyyy-MM-ddTHH:mm:ssZ of DateTimeOffset indicates the expiration time of the permission. DateTime.MinValue indicates there's no expiration set for this permission. Optional. */ + expirationDateTime?: string; +} + +/** + * Represents the sharing invitation details for a permission. + * @see https://learn.microsoft.com/en-us/graph/api/resources/sharinginvitation?view=graph-rest-1.0 + */ +export interface SharingInvitation { + /** The email address of the recipient. */ + readonly email?: string; + /** Provides information about who sent the invitation that created this permission, if that information is available. Read-only. */ + readonly readonlyinvitedBy?: IdentitySet; + /** If true the recipient of the invitation needs to sign in in order to access the shared item. Read-only. */ + readonly signInRequired?: boolean; +} + +/** + * Represents the sharing link details for a permission. + * @see https://learn.microsoft.com/en-us/graph/api/resources/sharinglink?view=graph-rest-1.0 + */ +export interface SharingLink { + /** The URL that opens the item in the browser on the OneDrive website. */ + webUrl?: string; + /** The type of sharing link. */ + type?: 'view' | 'edit' | 'embed'; + /** The scope of the link represented by this permission. */ + scope?: 'anonymous' | 'organization' | 'existingAccess' | 'users'; + /** If true, then the user can only use this link to view the item on the web, and cannot use it to download the contents of the item. */ + preventsDownload?: boolean; + /** For embed links, this property contains the HTML code for an