Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added parallelism to sync #731

Merged
merged 3 commits into from
Dec 5, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -84,11 +84,12 @@ public async Task RunAsync(
if (_departmentFilter.Length != 0)
departments = departments.Where(d => _departmentFilter.Any(df => d.FullDepartment!.Contains(df)));

logger.LogInformation("Found departments {Departments}", JsonConvert.SerializeObject(departments, Formatting.Indented));

var apiDepartments = new List<ApiResourceOwnerDepartment>();

foreach (var orgUnit in departments)
// Set up parallelism
var threadCount = 2;

await Parallel.ForEachAsync(departments, new ParallelOptions { MaxDegreeOfParallelism = threadCount }, async (orgUnit, cancellationToken) =>
{
var resourceOwners = orgUnit.Management.Persons
.Select(p => Guid.Parse(p.AzureUniqueId))
Expand All @@ -102,38 +103,42 @@ public async Task RunAsync(
.ToArray();

var recipients = resourceOwners.Concat(delegatedResponsibles).ToArray();

if (recipients.Length == 0)
{
logger.LogInformation("Skipping department {Department} as it has no resource owners or delegated responsibles", orgUnit.FullDepartment);
continue;
return;
}

apiDepartments.Add(new ApiResourceOwnerDepartment()
lock (apiDepartments)
{
DepartmentSapId = orgUnit.SapId!,
FullDepartmentName = orgUnit.FullDepartment!,
ResourceOwnersAzureUniqueId = resourceOwners,
DelegateResourceOwnersAzureUniqueId = delegatedResponsibles
});
}
apiDepartments.Add(new ApiResourceOwnerDepartment()
{
DepartmentSapId = orgUnit.SapId!,
FullDepartmentName = orgUnit.FullDepartment!,
ResourceOwnersAzureUniqueId = resourceOwners,
DelegateResourceOwnersAzureUniqueId = delegatedResponsibles
});
}
});

var enqueueTimeForDepartmentMapping = QueueTimeHelper.CalculateEnqueueTime(apiDepartments, _totalBatchTime, logger);

logger.LogInformation("Syncing departments {Departments}", JsonConvert.SerializeObject(enqueueTimeForDepartmentMapping, Formatting.Indented));


foreach (var department in apiDepartments)
await Parallel.ForEachAsync(apiDepartments, new ParallelOptions { MaxDegreeOfParallelism = threadCount }, async (department, cancellationToken) =>
{
try
{
//TODO: Do one batch update instead of individual updates
// TODO: Do one batch update instead of individual updates
// Update the database
await summaryApiClient.PutDepartmentAsync(department, cancellationToken);
}
catch (Exception e)
{
logger.LogCritical(e, "Failed to PUT department {Department}", JsonConvert.SerializeObject(department, Formatting.Indented));
continue;
logger.LogCritical(e, "Failed to PUT department {Department}",
JsonConvert.SerializeObject(department, Formatting.Indented));
return;
}

try
Expand All @@ -143,9 +148,11 @@ public async Task RunAsync(
}
catch (Exception e)
{
logger.LogCritical(e, "Failed to send department to queue {Department}", JsonConvert.SerializeObject(department, Formatting.Indented));
logger.LogCritical(e, "Failed to send department to queue {Department}",
JsonConvert.SerializeObject(department, Formatting.Indented));
}
}
});


logger.LogInformation("weekly-department-recipients-sync completed");
}
Expand Down
Loading