Skip to content

Commit

Permalink
chore(summary): Add more logging and other minor fixes (#701)
Browse files Browse the repository at this point in the history
- [ ] New feature
- [ ] Bug fix
- [ ] High impact

**Description of work:**

Adds more logging and changes log level of some logs.

Added check if the departmentFilter string is emtpy => if empty then do
all departments

Added departmentFilter to az sender

Update if any recipients check in the sync az function

**Testing:**
- [ ] Can be tested
- [x] Automatic tests created / updated
- [x] Local tests are passing



**Checklist:**
- [x] Considered automated tests
- [ ] Considered updating specification / documentation
- [ ] Considered work items 
- [x] Considered security
- [ ] Performed developer testing
- [x] Checklist finalized / ready for review
  • Loading branch information
Jonathanio123 authored Sep 11, 2024
1 parent d334e71 commit bea5ea2
Show file tree
Hide file tree
Showing 9 changed files with 464 additions and 13 deletions.
7 changes: 7 additions & 0 deletions src/Fusion.Resources.sln
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Fusion.Resources.Infrastruc
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Common", "Common", "{3C253096-BCFA-40D7-8C3F-F3F3B3BA4F1C}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Fusion.Summary.Functions.Tests", "tests\Fusion.Summary.Functions.Tests\Fusion.Summary.Functions.Tests.csproj", "{F819CC16-861C-4E8E-97FB-4B0928BD8704}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -163,6 +165,10 @@ Global
{53F9FAF1-EFA4-4951-9693-1E350F6428A8}.Debug|Any CPU.Build.0 = Debug|Any CPU
{53F9FAF1-EFA4-4951-9693-1E350F6428A8}.Release|Any CPU.ActiveCfg = Release|Any CPU
{53F9FAF1-EFA4-4951-9693-1E350F6428A8}.Release|Any CPU.Build.0 = Release|Any CPU
{F819CC16-861C-4E8E-97FB-4B0928BD8704}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F819CC16-861C-4E8E-97FB-4B0928BD8704}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F819CC16-861C-4E8E-97FB-4B0928BD8704}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F819CC16-861C-4E8E-97FB-4B0928BD8704}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down Expand Up @@ -193,6 +199,7 @@ Global
{0F9CC682-2DD6-4422-A321-D7100780A739} = {286E3C76-C860-4661-9AEF-817458D1231B}
{3C253096-BCFA-40D7-8C3F-F3F3B3BA4F1C} = {996B233A-1FC5-4CF5-97AD-B9C09F3E0A53}
{CC1EF38A-2741-4D92-A33A-3970E6F73797} = {3C253096-BCFA-40D7-8C3F-F3F3B3BA4F1C}
{F819CC16-861C-4E8E-97FB-4B0928BD8704} = {D12DC33D-BB83-40E4-9F36-18F59351A21D}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {F47D8FDB-3919-45C8-8C08-486405ECEC01}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ public DepartmentResourceOwnerSync(
{
_totalBatchTime = TimeSpan.FromHours(4.5);

logger.LogWarning("Configuration variable 'total_batch_time_in_minutes' not found, batching messages over {BatchTime}", _totalBatchTime);
logger.LogInformation("Configuration variable 'total_batch_time_in_minutes' not found, batching messages over {BatchTime}", _totalBatchTime);
}
}

Expand All @@ -72,13 +72,17 @@ public async Task RunAsync(
var client = new ServiceBusClient(_serviceBusConnectionString);
var sender = client.CreateSender(_weeklySummaryQueueName);

logger.LogInformation("weekly-department-recipients-sync fetching departments with department filter {DepartmentFilter}", JsonConvert.SerializeObject(_departmentFilter, Formatting.Indented));

// Fetch all departments
var departments = (await lineOrgApiClient.GetOrgUnitDepartmentsAsync())
.DistinctBy(d => d.SapId)
.Where(d => d.FullDepartment != null && d.SapId != null)
.Where(d => _departmentFilter.Any(df => d.FullDepartment.Contains(df)))
.Where(d => d.Management.Persons.Length > 0);
.Where(d => d.FullDepartment != null && d.SapId != null);

if (_departmentFilter.Length != 0)
departments = departments.Where(d => _departmentFilter.Any(df => d.FullDepartment!.Contains(df)));

logger.LogInformation("Found departments {Departments}", JsonConvert.SerializeObject(departments, Formatting.Indented));

var apiDepartments = new List<ApiResourceOwnerDepartment>();

Expand All @@ -95,6 +99,13 @@ public async Task RunAsync(
.Distinct()
.ToArray();

var recipients = resourceOwners.Concat(delegatedResponsibles).ToArray();
if (recipients.Length == 0)
{
logger.LogInformation("Skipping department {Department} as it has no resource owners or delegated responsibles", orgUnit.FullDepartment);
continue;
}

apiDepartments.Add(new ApiResourceOwnerDepartment()
{
DepartmentSapId = orgUnit.SapId!,
Expand All @@ -113,12 +124,13 @@ public async Task RunAsync(
{
try
{
//TODO: Do one batch update instead of individual updates
// Update the database
await summaryApiClient.PutDepartmentAsync(department, cancellationToken);
}
catch (Exception e)
{
logger.LogError(e, "Failed to PUT department {Department}", JsonConvert.SerializeObject(department, Formatting.Indented));
logger.LogCritical(e, "Failed to PUT department {Department}", JsonConvert.SerializeObject(department, Formatting.Indented));
continue;
}

Expand All @@ -129,9 +141,11 @@ public async Task RunAsync(
}
catch (Exception e)
{
logger.LogError(e, "Failed to send department to queue {Department}", JsonConvert.SerializeObject(department, Formatting.Indented));
logger.LogCritical(e, "Failed to send department to queue {Department}", JsonConvert.SerializeObject(department, Formatting.Indented));
}
}

logger.LogInformation("weekly-department-recipients-sync completed");
}


Expand All @@ -156,6 +170,8 @@ private Dictionary<ApiResourceOwnerDepartment, DateTimeOffset> CalculateDepartme
var currentTime = DateTimeOffset.UtcNow;
var minutesPerReportSlice = _totalBatchTime.TotalMinutes / apiDepartments.Count;

logger.LogInformation("Minutes allocated for each worker: {MinutesPerReportSlice}", minutesPerReportSlice);

var departmentDelayMapping = new Dictionary<ApiResourceOwnerDepartment, DateTimeOffset>();
foreach (var department in apiDepartments)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ public class WeeklyDepartmentSummarySender
private readonly IConfiguration configuration;

private int _maxDegreeOfParallelism;
private readonly string[] _departmentFilter;

public WeeklyDepartmentSummarySender(ISummaryApiClient summaryApiClient, INotificationApiClient notificationApiClient,
ILogger<WeeklyDepartmentSummarySender> logger, IConfiguration configuration)
Expand All @@ -32,14 +33,21 @@ public WeeklyDepartmentSummarySender(ISummaryApiClient summaryApiClient, INotifi
this.configuration = configuration;

_maxDegreeOfParallelism = int.TryParse(configuration["weekly-department-summary-sender-parallelism"], out var result) ? result : 2;
_departmentFilter = configuration["departmentFilter"]?.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) ?? ["PRD"];
}

[FunctionName("weekly-department-summary-sender")]
public async Task RunAsync([TimerTrigger("0 0 5 * * MON", RunOnStartup = false)] TimerInfo timerInfo)
{
logger.LogInformation("weekly-department-summary-sender started with department filter {DepartmentFilter}", JsonConvert.SerializeObject(_departmentFilter, Formatting.Indented));

// TODO: Use OData query to filter departments
var departments = await summaryApiClient.GetDepartmentsAsync();

if (departments is null || !departments.Any())
if (_departmentFilter.Length != 0)
departments = departments?.Where(d => _departmentFilter.Any(df => d.FullDepartmentName!.Contains(df))).ToArray();

if (departments is null || departments.Count == 0)
{
logger.LogCritical("No departments found. Exiting");
return;
Expand All @@ -52,6 +60,8 @@ public async Task RunAsync([TimerTrigger("0 0 5 * * MON", RunOnStartup = false)]

// Use Parallel.ForEachAsync to easily limit the number of parallel requests
await Parallel.ForEachAsync(departments, options, async (department, _) => await CreateAndSendNotificationsAsync(department));

logger.LogInformation("weekly-department-summary-sender completed");
}

private async Task CreateAndSendNotificationsAsync(ApiResourceOwnerDepartment department)
Expand All @@ -64,15 +74,15 @@ private async Task CreateAndSendNotificationsAsync(ApiResourceOwnerDepartment de

if (summaryReport is null)
{
logger.LogCritical(
logger.LogWarning(
"No summary report found for department {Department}. Unable to send report notification",
JsonConvert.SerializeObject(department, Formatting.Indented));
return;
}
}
catch (Exception e)
{
logger.LogError(e, "Failed to get summary report for department {Department}", JsonConvert.SerializeObject(department, Formatting.Indented));
logger.LogCritical(e, "Failed to get summary report for department {Department}", JsonConvert.SerializeObject(department, Formatting.Indented));
return;
}

Expand All @@ -83,7 +93,7 @@ private async Task CreateAndSendNotificationsAsync(ApiResourceOwnerDepartment de
}
catch (Exception e)
{
logger.LogError(e, "Failed to create notification for department {DepartmentSapId} | Report {Report}", department.DepartmentSapId, JsonConvert.SerializeObject(summaryReport, Formatting.Indented));
logger.LogCritical(e, "Failed to create notification for department {DepartmentSapId} | Report {Report}", department.DepartmentSapId, JsonConvert.SerializeObject(summaryReport, Formatting.Indented));
return;
}

Expand All @@ -95,11 +105,11 @@ private async Task CreateAndSendNotificationsAsync(ApiResourceOwnerDepartment de
{
var result = await notificationApiClient.SendNotification(notification, azureId);
if (!result)
logger.LogError("Failed to send notification to user with AzureId {AzureId} | Report {Report}", azureId, JsonConvert.SerializeObject(summaryReport, Formatting.Indented));
logger.LogCritical("Failed to send notification to user with AzureId {AzureId} | Report {Report}", azureId, JsonConvert.SerializeObject(summaryReport, Formatting.Indented));
}
catch (Exception e)
{
logger.LogError(e, "Failed to send notification to user with AzureId {AzureId} | Report {Report}", azureId, JsonConvert.SerializeObject(summaryReport, Formatting.Indented));
logger.LogCritical(e, "Failed to send notification to user with AzureId {AzureId} | Report {Report}", azureId, JsonConvert.SerializeObject(summaryReport, Formatting.Indented));
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ public async Task RunAsync(
[ServiceBusTrigger("%department_summary_weekly_queue%", Connection = "AzureWebJobsServiceBus")]
ServiceBusReceivedMessage message, ServiceBusMessageActions messageReceiver)
{
_logger.LogInformation("weekly-department-summary-worker started with message: {MessageBody}", message.Body);
try
{
var dto = await JsonSerializer.DeserializeAsync<ApiResourceOwnerDepartment>(message.Body.ToStream());
Expand All @@ -50,6 +51,7 @@ public async Task RunAsync(
{
// Complete the message regardless of outcome.
await messageReceiver.CompleteMessageAsync(message);
_logger.LogInformation("weekly-department-summary-worker completed");
}
}

Expand All @@ -66,7 +68,7 @@ private async Task CreateAndStoreReportAsync(ApiResourceOwnerDepartment message)
// Check if the department has personnel, abort if not
if (departmentPersonnel.Count == 0)
{
_logger.LogInformation("Department contains no personnel, no need to store report");
_logger.LogInformation("Department {Department} contains no valid personnel, no need to store report", message.FullDepartmentName);
return;
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>net8.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>

<IsPackable>false</IsPackable>
<IsTestProject>true</IsTestProject>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="FluentAssertions" Version="6.12.1"/>
<PackageReference Include="coverlet.collector" Version="6.0.2">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="GitHubActionsTestLogger" Version="2.4.1">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.11.1"/>
<PackageReference Include="xunit" Version="2.9.0"/>
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
</ItemGroup>

<ItemGroup>
<Using Include="Xunit"/>
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\..\Fusion.Summary.Functions\Fusion.Summary.Functions.csproj"/>
</ItemGroup>

<ItemGroup>
<None Remove="nuget.config"/>
</ItemGroup>
<ItemGroup>
<Content Update="nuget.config">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
<ExcludeFromSingleFile>true</ExcludeFromSingleFile>
<CopyToPublishDirectory>PreserveNewest</CopyToPublishDirectory>
</Content>
</ItemGroup>

</Project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
using Fusion.Resources.Functions.Common.ApiClients;

namespace Fusion.Summary.Functions.Tests.Notifications.Mock;

public abstract class NotificationReportApiResponseMock
{
public static List<IResourcesApiClient.InternalPersonnelPerson> GetMockedInternalPersonnel(
double personnelCount,
double workload,
double otherTasks,
double vacationLeave,
double absenceLeave)
{
var personnel = new List<IResourcesApiClient.InternalPersonnelPerson>();
for (var i = 0; i < personnelCount; i++)
{
personnel.Add(new IResourcesApiClient.InternalPersonnelPerson()
{
EmploymentStatuses = new List<IResourcesApiClient.ApiPersonAbsence>
{
new()
{
Type = IResourcesApiClient.ApiAbsenceType.Vacation,
AppliesFrom = DateTime.UtcNow.AddDays(-1 - i),
AppliesTo = DateTime.UtcNow.AddDays(1 + i * 10),
AbsencePercentage = vacationLeave
},
new()
{
Type = IResourcesApiClient.ApiAbsenceType.OtherTasks,
AppliesFrom = DateTime.UtcNow.AddDays(-1 - i),
AppliesTo = DateTime.UtcNow.AddDays(1 + i * 10),
AbsencePercentage = otherTasks
},
new()
{
Type = IResourcesApiClient.ApiAbsenceType.Absence,
AppliesFrom = DateTime.UtcNow.AddDays(-1 - i),
AppliesTo = DateTime.UtcNow.AddDays(1 + i * 10),
AbsencePercentage = absenceLeave
}
},
PositionInstances = new List<IResourcesApiClient.PersonnelPosition>
{
new()
{
AppliesFrom = DateTime.UtcNow.AddDays(-1 - i),
AppliesTo = DateTime.UtcNow.AddDays(1 + i * 10),
Workload = workload
}
}
}
);
;
}

return personnel;
}

public static List<IResourcesApiClient.InternalPersonnelPerson> GetMockedInternalPersonnelWithInstancesWithAndWithoutChanges(double personnelCount)
{
var personnel = new List<IResourcesApiClient.InternalPersonnelPerson>();
for (var i = 0; i < personnelCount; i++)
{
personnel.Add(new IResourcesApiClient.InternalPersonnelPerson()
{
// Should return 4 instances for each person
PositionInstances = new List<IResourcesApiClient.PersonnelPosition>
{
new()
{
// One active instance without any changes
AppliesFrom = DateTime.UtcNow.AddDays(-1 - i),
AppliesTo = DateTime.UtcNow.AddDays(1 + i * 10),
AllocationState = null,
AllocationUpdated = null
},
new()
{
// One active instance that contains changes done within the last week
AppliesFrom = DateTime.UtcNow.AddDays(-1 - i),
AppliesTo = DateTime.UtcNow.AddDays(1 + i * 10),
AllocationState = "ChangeByTaskOwner",
AllocationUpdated = DateTime.UtcNow
},
new()
{
// One active instance that contains changes done more than a week ago
AppliesFrom = DateTime.UtcNow.AddDays(-1 - i),
AppliesTo = DateTime.UtcNow.AddDays(1 + i * 10),
AllocationState = "ChangeByTaskOwner",
AllocationUpdated = DateTime.UtcNow.AddDays(-8)
},
new()
{
// One instance that will become active in more than 3 months that contains changes
AppliesFrom = DateTime.UtcNow.AddMonths(4),
AppliesTo = DateTime.UtcNow.AddMonths(4 + i),
AllocationState = "ChangeByTaskOwner",
AllocationUpdated = DateTime.UtcNow
}
}
}
);
;
}

return personnel;
}
}
Loading

0 comments on commit bea5ea2

Please sign in to comment.