feat: Add endpoint to send newsletter consent email (#645)

* feat: Add endpoint to send newsletter consent email

* feat: Allow for dynamic setting for removing registeredUsers from mail list

* chore: Add maximum number of retries when getting export status
Needed to prevent potential infinite loop

* chore: Make send-newsletter-consent endpoint protected

* chore: Improvements

* chore: Add content value dynamically based on fn call
Not quite sure, what this value is for, but it is better to have it like this for now

* fix: Linter error

* chore: Improve TS definitions

* chore: Code cleanup

apps/api/src/common/mapChunk.ts

@@ -0,0 +1,17 @@
+/**
+ * Create a chunked array of new Map()
+ * @param map map to be chunked
+ * @param chunkSize The size of the chunk
+ * @returns Array chunk of new Map()
+ */
+
+export function mapChunk<T extends Map<any, any>>(map: T, chunkSize: number) {
+  return Array.from(map.entries()).reduce<T[]>((chunk, curr, index) => {
+    const ch = Math.floor(index / chunkSize)
+    if (!chunk[ch]) {
+      chunk[ch] = new Map() as T
+    }
+    chunk[ch].set(curr[0], curr[1])
+    return chunk
+  }, [])
+}

apps/api/src/notifications/dto/massmail.dto.ts

@@ -0,0 +1,35 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { Expose } from 'class-transformer'
+import { IsDateString, IsNumber, IsOptional, IsString } from 'class-validator'
+
+export class MassMailDto {
+  @ApiProperty()
+  @Expose()
+  @IsString()
+  listId: string
+
+  @ApiProperty()
+  @Expose()
+  @IsString()
+  templateId: string
+
+  @ApiProperty()
+  @Expose()
+  @IsString()
+  @IsOptional()
+  subject: string
+
+  //Sendgrid limits sending emails to 1000 at once.
+  @ApiProperty()
+  @Expose()
+  @IsNumber()
+  @IsOptional()
+  chunkSize = 1000
+
+  //Remove users registered after the dateThreshold from mail list
+  @ApiProperty()
+  @Expose()
+  @IsDateString()
+  @IsOptional()
+  dateThreshold: Date = new Date()
+}

apps/api/src/notifications/notifications.controller.ts

@@ -1,6 +1,6 @@
 import { BadRequestException, Body, Controller, Get, Post } from '@nestjs/common'
 import { ApiTags } from '@nestjs/swagger'
-import { AuthenticatedUser, Public } from 'nest-keycloak-connect'
+import { AuthenticatedUser, Public, RoleMatchingMode, Roles } from 'nest-keycloak-connect'
 import {
   SendConfirmationDto,
   SubscribeDto,
@@ -11,6 +11,8 @@ import {
 
 import { MarketingNotificationsService } from './notifications.service'
 import { KeycloakTokenParsed } from '../auth/keycloak'
+import { MassMailDto } from './dto/massmail.dto'
+import { RealmViewSupporters, ViewSupporters } from '@podkrepi-bg/podkrepi-types'
 
 @ApiTags('notifications')
 @Controller('notifications')
@@ -63,4 +65,13 @@ export class MarketingNotificationsController {
       user.email || '',
     )
   }
+
+  @Post('/send-newsletter-consent')
+  @Roles({
+    roles: [RealmViewSupporters.role, ViewSupporters.role],
+    mode: RoleMatchingMode.ANY,
+  })
+  async sendMassMail(@Body() data: MassMailDto) {
+    return await this.marketingNotificationsService.sendConsentMail(data)
+  }
 }

apps/api/src/notifications/notifications.service.ts

@@ -27,11 +27,28 @@ import {
   UnregisteredNotificationConsent,
 } from '@prisma/client'
 import { NotificationsProviderInterface } from './providers/notifications.interface.providers'
-import { SendGridParams } from './providers/notifications.sendgrid.types'
+import { ContactsResponse, SendGridParams } from './providers/notifications.sendgrid.types'
 import { DateTime } from 'luxon'
 import * as crypto from 'crypto'
 import { CampaignService } from '../campaign/campaign.service'
 import { KeycloakTokenParsed } from '../auth/keycloak'
+import { MassMailDto } from './dto/massmail.dto'
+import { randomUUID } from 'crypto'
+import { mapChunk } from '../common/mapChunk'
+
+type UnregisteredInsert = {
+  id: string
+  email: string
+  consent: boolean
+}
+
+type MailList = {
+  id: string
+  hash: string
+  registered: boolean
+}
+
+export type ContactsMap = Map<string, MailList>
 
 @Injectable()
 export class MarketingNotificationsService {
@@ -492,4 +509,88 @@ export class MarketingNotificationsService {
 
     return minutesPassed <= period
   }
+
+  private generateMapFromMailList(emailList: string[], contacts: ContactsMap): void {
+    for (const email of emailList) {
+      const id = randomUUID()
+
+      contacts.set(email, {
+        id: id,
+        hash: this.generateHash(id),
+        registered: false,
+      })
+    }
+  }
+
+  private updateMailListMap(
+    regUser: Person[],
+    contacts: ContactsMap,
+    skipAfterDate: Date,
+    unregisteredConsent: UnregisteredNotificationConsent[],
+  ) {
+    for (const registeredUser of regUser) {
+      const createdAt = new Date(registeredUser.createdAt)
+
+      // Remove email if it belongs to user created after the change has been deployed, as they had already decided
+      // whether to give consent or not.
+      if (contacts.get(registeredUser.email as string) && createdAt > skipAfterDate) {
+        Logger.debug(`Removing email ${registeredUser.email} from list`)
+        contacts.delete(registeredUser.email as string)
+        continue
+      }
+      //Update the value of this mail
+      contacts.set(registeredUser.email as string, {
+        id: registeredUser.id,
+        hash: this.generateHash(registeredUser.id),
+        registered: true,
+      })
+    }
+
+    Logger.debug('Removing emails in unregistered consent emails')
+    for (const consent of unregisteredConsent) {
+      if (contacts.has(consent.email)) {
+        Logger.debug(`Removing email ${consent.email}`)
+        contacts.delete(consent.email)
+        continue
+      }
+    }
+  }
+
+  private async insertUnregisteredConsentFromContacts(contacts: ContactsMap) {
+    const emailsToAdd: UnregisteredInsert[] = []
+    for (const [key, value] of contacts) {
+      if (value.registered) continue
+      emailsToAdd.push({ id: value.id, email: key, consent: false })
+    }
+
+    await this.prisma.unregisteredNotificationConsent.createMany({
+      data: emailsToAdd,
+    })
+  }
+  async sendConsentMail(data: MassMailDto) {
+    const contacts = await this.marketingNotificationsProvider.getContactsFromList(data)
+
+    const sendList: ContactsMap = new Map()
+    const emailList = contacts.map((contact: ContactsResponse) => contact.email)
+    this.generateMapFromMailList(emailList, sendList)
+    const registeredMails = await this.prisma.person.findMany({
+      where: { email: { in: emailList } },
+    })
+
+    const unregisteredUsers = await this.prisma.unregisteredNotificationConsent.findMany()
+
+    const skipUsersAfterDate = new Date(data.dateThreshold)
+    this.updateMailListMap(registeredMails, sendList, skipUsersAfterDate, unregisteredUsers)
+
+    await this.insertUnregisteredConsentFromContacts(sendList)
+
+    const contactsChunked = mapChunk<ContactsMap>(sendList, data.chunkSize)
+    Logger.debug(`Splitted email list into ${contactsChunked.length} chunk`)
+    await this.marketingNotificationsProvider.sendBulkEmail(
+      data,
+      contactsChunked,
+      'Podkrepi.BG Newsletter Subscription Consent',
+    )
+    return { contactCount: sendList.size }
+  }
 }

apps/api/src/notifications/providers/notifications.interface.providers.ts

@@ -1,3 +1,7 @@
+import { MassMailDto } from '../dto/massmail.dto'
+import { ContactsMap } from '../notifications.service'
+import { PersonalizationData } from '@sendgrid/helpers/classes/personalization'
+
 type NotificationsInterfaceParams = {
   CreateListParams: unknown
   UpdateListParams: unknown
@@ -8,6 +12,7 @@ type NotificationsInterfaceParams = {
   RemoveFromUnsubscribedParams: unknown
   AddToUnsubscribedParams: unknown
   SendNotificationParams: unknown
+  GetContactsFromListParam: unknown
 
   // Responses
   CreateListRes: unknown
@@ -19,6 +24,7 @@ type NotificationsInterfaceParams = {
   RemoveFromUnsubscribedRes: unknown
   AddToUnsubscribedRes: unknown
   SendNotificationRes: unknown
+  GetContactsFromListRes: unknown
   contactListsRes: unknown
 }
 
@@ -36,5 +42,20 @@ export abstract class NotificationsProviderInterface<
     data: T['RemoveFromUnsubscribedParams'],
   ): Promise<T['RemoveFromUnsubscribedRes']>
   abstract sendNotification(data: T['SendNotificationParams']): Promise<T['SendNotificationRes']>
+  abstract getContactsFromList(
+    data: T['GetContactsFromListParam'],
+  ): Promise<T['GetContactsFromListRes']>
+  abstract prepareTemplatePersonalizations(
+    data: MassMailDto,
+    contacts: ContactsMap,
+    date?: Date,
+  ): PersonalizationData[]
+
+  abstract sendBulkEmail(
+    data: MassMailDto,
+    contactsMap: ContactsMap[],
+    value: string,
+    timeout?: number,
+  ): Promise<void>
   abstract getContactLists(): Promise<T['contactListsRes']>
 }