Skip to content

Commit

Permalink
refactor: Update fs_folders schema and Google Drive sync logic to han…
Browse files Browse the repository at this point in the history 
…dle unresolved folders.
  • Loading branch information
@amuwal
amuwal committed Dec 17, 2024
1 parent 0306091 commit 7ed61a1
Show file tree
Hide file tree
Showing 4 changed files with 71 additions and 31 deletions.
2 changes: 1 addition & 1 deletion packages/api/prisma/schema.prisma
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1297,7 +1297,7 @@ model fs_folders {
size BigInt?
name String?
description String?
parent_folder String? @db.Uuid
parent_folder String?
remote_id String?
created_at DateTime @db.Timestamptz(6)
modified_at DateTime @db.Timestamptz(6)
Expand Down
2 changes: 1 addition & 1 deletion packages/api/scripts/init.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -819,7 +819,7 @@ CREATE TABLE fs_folders
"size" bigint NULL,
name text NULL,
description text NULL,
parent_folder uuid NULL,
parent_folder text NULL,
remote_id text NULL,
created_at timestamp with time zone NOT NULL,
modified_at timestamp with time zone NOT NULL,
Expand Down
96 changes: 68 additions & 28 deletions packages/api/src/filestorage/folder/services/googledrive/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,11 +106,6 @@ export class GoogleDriveFolderService implements IFolderService {
): Promise<GoogleDriveFolderOutput[]> {
const drive = google.drive({ version: 'v3', auth });

const lastSyncTime = await this.getLastSyncTime(connectionId);
if (lastSyncTime) {
console.log(`Last sync time is ${lastSyncTime.toISOString()}`);
}

const rootDriveId = await drive.files
.get({
fileId: 'root',
Expand All @@ -127,10 +122,7 @@ export class GoogleDriveFolderService implements IFolderService {
let pageToken: string | null = null;

const buildQuery = (parentId: string | null, driveId: string): string => {
let baseQuery = `mimeType='application/vnd.google-apps.folder' and trashed=false`;
if (lastSyncTime) {
baseQuery += ` and modifiedTime >= '${lastSyncTime.toISOString()}'`;
}
const baseQuery = `mimeType='application/vnd.google-apps.folder' and trashed=false`;
return parentId
? `${baseQuery} and '${parentId}' in parents`
: `${baseQuery} and '${driveId}' in parents`;
Expand Down Expand Up @@ -228,15 +220,17 @@ export class GoogleDriveFolderService implements IFolderService {

allFolders.push(...currentLevelFolders);

for (const folder of currentLevelFolders) {
await populateFolders(
folder.id,
folder.internal_id,
level + 1,
allFolders,
driveId,
);
}
await Promise.all(
currentLevelFolders.map((folder) =>
populateFolders(
folder.id,
folder.internal_id,
level + 1,
allFolders,
driveId,
),
),
);
}

try {
Expand Down Expand Up @@ -386,9 +380,10 @@ export class GoogleDriveFolderService implements IFolderService {
MAX_RETRIES,
INITIAL_BACKOFF,
);
const unresolvedFolders = await this.getUnresolvedFolders(connectionId);
const folderIdToInternalIdMap = new Map<string, string>();
const foldersToSync: GoogleDriveFolderOutput[] = []; // output
let remainingFolders = modifiedFolders;
let remainingFolders = modifiedFolders.concat(unresolvedFolders);

// Create a cache for parent lookups to minimize DB queries
const parentLookupCache = new Map<string, string | null>();
Expand Down Expand Up @@ -446,13 +441,15 @@ export class GoogleDriveFolderService implements IFolderService {

// Check if we're stuck in a loop (no folders processed in this iteration)
if (foldersStillPending.length === remainingFolders.length) {
this.logger.error(
`Processing stopped with ${foldersStillPending.length} unresolved folders - possible orphans or circular references`,
'foldersStillPending',
`${JSON.stringify(foldersStillPending)}`,
this.logger.warn(
`Marking ${foldersStillPending.length} unresolved folders as 'unresolved'. Will try again in next sync.`,
);
console.log('foldersStillPending', foldersStillPending);
throw new Error('Processing stopped with unresolved folders');

const unresolvedFolders = foldersStillPending.map((folder) => ({
...folder,
internal_parent_folder_id: 'unresolved',
}));
foldersToSync.push(...unresolvedFolders);
}

remainingFolders = foldersStillPending;
Expand All @@ -473,7 +470,7 @@ export class GoogleDriveFolderService implements IFolderService {
lastSyncTime: Date,
maxRetries: number,
initialBackoff: number,
): Promise<any[]> {
): Promise<GoogleDriveFolderOutput[]> {
let attempt = 0;
let backoff = initialBackoff;

Expand Down Expand Up @@ -527,9 +524,9 @@ export class GoogleDriveFolderService implements IFolderService {
private async getModifiedFolders(
drive: any,
lastSyncTime: Date,
): Promise<any[]> {
): Promise<GoogleDriveFolderOutput[]> {
let pageToken: string | null = null;
const folders: any[] = [];
const folders: GoogleDriveFolderOutput[] = [];
const query = `modifiedTime >= '${lastSyncTime.toISOString()}'`;

do {
Expand All @@ -553,6 +550,47 @@ export class GoogleDriveFolderService implements IFolderService {
return folders;
}

async getUnresolvedFolders(
connectionId: string,
): Promise<GoogleDriveFolderOutput[]> {
// Get unresolved folders
const unresolvedFolders = await this.prisma.fs_folders.findMany({
where: {
id_connection: connectionId,
parent_folder: 'unresolved',
},
select: {
id_fs_folder: true,
name: true,
},
});

// Get remote data for all folders in a single query
const folderIds = unresolvedFolders.map((folder) => folder.id_fs_folder);
const remoteDataEntries = await this.prisma.remote_data.findMany({
where: {
ressource_owner_id: {
in: folderIds,
},
},
});

// Create a map for quick lookup
const remoteDataMap = new Map(
remoteDataEntries.map((entry) => [entry.ressource_owner_id, entry.data]),
);

// Map and parse the remote data, filtering out any null values
return unresolvedFolders
.map((folder) => {
const remoteData = remoteDataMap.get(folder.id_fs_folder);
return remoteData
? (JSON.parse(remoteData) as GoogleDriveFolderOutput)
: null;
})
.filter((folder): folder is GoogleDriveFolderOutput => folder !== null);
}

async sync(data: SyncParam): Promise<ApiResponse<GoogleDriveFolderOutput[]>> {
try {
const { linkedUserId } = data;
Expand Down Expand Up @@ -591,6 +629,8 @@ export class GoogleDriveFolderService implements IFolderService {
connection.id_connection,
);

console.log(`Got ${folders.length} folders`);

await this.ingestPermissionsForFolders(folders, connection.id_connection);
this.logger.log(`Synced ${folders.length} Google Drive folders!`);

Expand Down
2 changes: 1 addition & 1 deletion packages/api/src/filestorage/folder/sync/sync.service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ export class SyncService implements OnModuleInit, IBaseSync {
description: folder.description ?? null,
id_fs_drive: drive_id_by_remote_drive_id ?? null,
id_fs_permissions: folder.permissions,
parent_folder: folder.parent_folder_id ?? null,
parent_folder: folder.parent_folder_id ?? uuidv4(),
modified_at: new Date(),
remote_created_at: folder.remote_created_at ?? null,
remote_modified_at: folder.remote_modified_at ?? null,
Expand Down

0 comments on commit 7ed61a1

Please sign in to comment.