Skip to content

Commit

Permalink
Add validation and do not save all contacts into memory when importing (
Browse files Browse the repository at this point in the history 
#102)

COM-1062

Empty File:


https://github.com/user-attachments/assets/62c57e29-4e06-4e52-8e0e-5bb4ba69c786


File with only wrong rows (4 wrong and 1 empty --> empty is removed
automatically):
(in the video there is column(s). I changed it later to row(s) see last
video)

https://github.com/user-attachments/assets/5931a792-b393-4bf2-b1e9-3e44efa36b07


5 correct ones:


https://github.com/user-attachments/assets/200cf026-a58a-4a63-9597-9c2f61dace6f


2 invalid, 3 correct:



https://github.com/user-attachments/assets/54d7491a-11ae-4e37-974b-a6ce4cda43e7
  • Loading branch information
@raphaelblum
raphaelblum authored Oct 7, 2024
1 parent a804b03 commit f675cd0
Show file tree
Hide file tree
Showing 7 changed files with 317 additions and 153 deletions.
12 changes: 12 additions & 0 deletions .changeset/six-mails-join.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
---
"@comet/brevo-admin": patch
"@comet/brevo-api": patch
---

CSV Import Validation Improvements and Bug Fix

Add better validation for csv imports.

Add better feedback after a csv import when something goes wrong. User can download a file with failing rows.

Fix a bug when importing via csv in a targetgroup. The contact was only added to the manually assigned contacts and not to the actual target group.
141 changes: 109 additions & 32 deletions packages/admin/src/common/contactImport/useContactImportFromCsv.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,12 @@ import { useApolloClient } from "@apollo/client";
import { RefetchQueriesInclude } from "@apollo/client/core/types";
import { Alert, Loading, messages, useErrorDialog } from "@comet/admin";
import { Upload } from "@comet/admin-icons";
import { Box, Button, Dialog, DialogActions, DialogContent, DialogTitle } from "@mui/material";
import { Box, Dialog, DialogActions, DialogContent, DialogTitle, styled } from "@mui/material";
import Button from "@mui/material/Button";
import saveAs from "file-saver";
import * as React from "react";
import { useDropzone } from "react-dropzone";
import { FormattedMessage } from "react-intl";
import { FormattedMessage, useIntl } from "react-intl";

import { GQLEmailCampaignContentScopeInput } from "../../graphql.generated";
import { CrudMoreActionsItem } from "../../temp/CrudMoreActionsMenu";
Expand Down Expand Up @@ -49,13 +51,22 @@ interface ComponentProps extends UseContactImportProps {
fileInputRef: React.RefObject<HTMLInputElement>;
}

interface ImportInformation {
failed: number;
created: number;
updated: number;
failedColumns: Record<string, string>[];
errorMessage?: string;
}

const ContactImportComponent = ({ scope, targetGroupId, fileInputRef, refetchQueries }: ComponentProps) => {
const apolloClient = useApolloClient();
const [importingCsv, setImportingCsv] = React.useState(false);
const [importSuccessful, setImportSuccessful] = React.useState(false);
const dialogOpen = importingCsv || importSuccessful;
const [importInformation, setImportInformation] = React.useState<ImportInformation | null>(null);
const dialogOpen = importingCsv || !!importInformation;
const errorDialog = useErrorDialog();
const config = useBrevoConfig();
const intl = useIntl();

function upload(file: File, scope: GQLEmailCampaignContentScopeInput, listIds?: string[]): Promise<Response> {
const formData = new FormData();
Expand All @@ -70,6 +81,34 @@ const ContactImportComponent = ({ scope, targetGroupId, fileInputRef, refetchQue
});
}

const saveErrorFile = () => {
const failedColumns = importInformation?.failedColumns;
if (!failedColumns || failedColumns.length === 0) {
throw new Error(intl.formatMessage({ id: "export", defaultMessage: "No failed columns to save" }));
}

let errorData = "";

// Add headers to the file without trailing semicolon
const headers = Object.keys(failedColumns[0]);
const headerStr = headers.join(";");
errorData = `${headerStr.replace(/;+$/, "")}\n`; // Remove trailing semicolon from the header

// Add each row of failed columns data
for (const column of failedColumns) {
// Use Object.values to get the values of each column
const row = Object.values(column); // No need to check for undefined/null

// Join row values and remove the trailing semicolon if not needed
const rowStr = row.join(";");
errorData += `${rowStr.replace(/;+$/, "")}\n`;
}

// Create and download the file
const file = new Blob([errorData], { type: "text/csv;charset=utf-8" });
saveAs(file, `error-log-${new Date().toISOString()}.csv`);
};

const { getInputProps } = useDropzone({
accept: { "text/csv": [] },
multiple: false,
Expand All @@ -81,30 +120,32 @@ const ContactImportComponent = ({ scope, targetGroupId, fileInputRef, refetchQue
const response = await upload(file, scope, targetGroupId ? [targetGroupId] : []);
apolloClient.refetchQueries({ include: refetchQueries });

const data = (await response.json()) as ImportInformation;

if (response.ok) {
setImportingCsv(false);
setImportSuccessful(true);

if (data.errorMessage) {
errorDialog?.showError({
title: <FormattedMessage {...messages.error} />,
userMessage: data.errorMessage,
error: data.errorMessage,
});
} else {
setImportInformation(data);
}
} else {
const errorResponse = await response.json();
throw new Error(JSON.stringify(errorResponse));
throw new Error(JSON.stringify(data));
}
} catch (e) {
setImportingCsv(false);

let userMessage = (
const userMessage = (
<FormattedMessage
id="cometBrevoModule.useContactImport.error.defaultMessage"
defaultMessage="An error occured during the import. Please try again in a while or contact your administrator if the error persists."
/>
);
if (e?.message && typeof e.message === "string" && e.message.includes("Too many contacts")) {
userMessage = (
<FormattedMessage
id="cometBrevoModule.useContactImport.error.tooManyContactsMessage"
defaultMessage="Too many contacts in file. Currently we only support 100 contacts at once."
/>
);
}

errorDialog?.showError({
title: <FormattedMessage {...messages.error} />,
Expand All @@ -123,32 +164,62 @@ const ContactImportComponent = ({ scope, targetGroupId, fileInputRef, refetchQue
{importingCsv && (
<FormattedMessage id="cometBrevoModule.useContactImport.importing.title" defaultMessage="Importing contacts from CSV..." />
)}
{importSuccessful && (
{importInformation && (
<FormattedMessage id="cometBrevoModule.useContactImport.importSuccessful.title" defaultMessage="Import successful" />
)}
</DialogTitle>
<DialogContent>
{importingCsv && <Loading />}
{importSuccessful && (
{importInformation && (
<>
<FormattedMessage
id="cometBrevoModule.useContactImport.importSuccessful.message"
defaultMessage="The contacts have been imported successfully"
/>
<Box mt={2}>
<Alert severity="warning">
<FormattedMessage
id="cometBrevoModule.useContactImport.importSuccessful.doiNotice"
defaultMessage="Contacts who have not yet confirmed their subscription will receive a double opt-in email to complete the process. These contacts will not appear in this list until they confirm their subscription. Once confirmed, they will automatically be added to the appropriate target group(s)."
/>
</Alert>
</Box>
{importInformation.created > 0 && (
<FormattedMessage
id="cometBrevoModule.useContactImport.importSuccessful.contactsImported"
defaultMessage="{amount} contact(s) have been created successfully."
values={{ amount: importInformation.created }}
/>
)}
{importInformation.updated > 0 && (
<FormattedMessage
id="cometBrevoModule.useContactImport.importSuccessful.contactsUpdated"
defaultMessage="{amount} contact(s) have been updated."
values={{ amount: importInformation.updated }}
/>
)}

{importInformation.failed > 0 && (
<Box mt={2}>
<Alert severity="error">
<FormattedMessage
id="cometBrevoModule.useContactImport.error.contactsCouldNotBeImported"
defaultMessage="{amount} contact(s) could not be imported. <link>Download this file</link> to get the failing row(s)."
values={{
amount: importInformation.failed,
link: (chunks: React.ReactNode) => (
<CsvDownloadLink onClick={saveErrorFile}>{chunks}</CsvDownloadLink>
),
}}
/>
</Alert>
</Box>
)}

{(importInformation.created > 0 || importInformation.updated > 0) && (
<Box mt={2}>
<Alert severity="warning">
<FormattedMessage
id="cometBrevoModule.useContactImport.importSuccessful.doiNotice"
defaultMessage="Contacts who have not yet confirmed their subscription will receive a double opt-in email to complete the process. These contacts will not appear in this list until they confirm their subscription. Once confirmed, they will automatically be added to the appropriate target group(s)."
/>
</Alert>
</Box>
)}
</>
)}
</DialogContent>
<DialogActions>
{importSuccessful && (
<Button onClick={() => setImportSuccessful(false)} variant="contained">
{importInformation && (
<Button onClick={() => setImportInformation(null)} variant="contained">
<FormattedMessage {...messages.ok} />
</Button>
)}
Expand All @@ -157,3 +228,9 @@ const ContactImportComponent = ({ scope, targetGroupId, fileInputRef, refetchQue
</>
);
};

const CsvDownloadLink = styled("span")`
color: ${({ theme }) => theme.palette.info.main};
text-decoration: underline;
cursor: pointer;
`;
2 changes: 2 additions & 0 deletions packages/api/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@
"@nestjs/graphql": "^10.0.0",
"@nestjs/platform-express": "^9.0.0",
"@types/jest": "^29.5.0",
"@types/lodash.isequal": "^4.0.0",
"@types/mime-db": "^1.43.5",
"@types/node-fetch": "^2.5.12",
"@types/rimraf": "^3.0.0",
Expand All @@ -64,6 +65,7 @@
"graphql": "^15.5.0",
"jest": "^29.5.0",
"jest-junit": "^15.0.0",
"lodash.isequal": "^4.5.0",
"nestjs-console": "^8.0.0",
"pg-error-constants": "^1.0.0",
"prettier": "^2.0.0",
Expand Down
20 changes: 9 additions & 11 deletions packages/api/src/brevo-contact/brevo-contact-import.console.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,21 +81,19 @@ export function createBrevoContactImportConsole({ Scope }: { Scope: Type<EmailCa
],
})
@CreateRequestContext()
async execute(options: CommandOptions): Promise<void> {
const redirectUrl = this.config.brevo.resolveConfig(options.scope).redirectUrlForImport;
const content = fs.readFileSync(options.path);
if (!this.validateRedirectUrl(redirectUrl, options.scope)) {
async execute({ scope, path, targetGroupIds }: CommandOptions): Promise<void> {
const redirectUrl = this.config.brevo.resolveConfig(scope).redirectUrlForImport;
const fileStream = fs.createReadStream(path);
if (!this.validateRedirectUrl(redirectUrl, scope)) {
throw new InvalidOptionArgumentError("Invalid scope. Scope is not allowed");
}

const targetGroups = await this.targetGroupRepository.find({ id: { $in: options.targetGroupIds } });

const result = await this.brevoContactImportService.importContactsFromCsv(
content.toString("utf8"),
options.scope,
const result = await this.brevoContactImportService.importContactsFromCsv({
fileStream,
scope,
redirectUrl,
targetGroups,
);
targetGroupIds,
});

this.logger.log(result);
}
Expand Down
36 changes: 18 additions & 18 deletions packages/api/src/brevo-contact/brevo-contact-import.controller.ts
View file
Original file line number Diff line number Diff line change
@@ -1,22 +1,20 @@
import { CometValidationException, RequiredPermission } from "@comet/cms-api";
import { EntityRepository } from "@mikro-orm/core";
import { InjectRepository } from "@mikro-orm/nestjs";
import { Body, Controller, Inject, Post, Type, UploadedFile, UseInterceptors } from "@nestjs/common";
import { FileInterceptor } from "@nestjs/platform-express";
import { TargetGroupInterface } from "src/target-group/entity/target-group-entity.factory";
import { Readable } from "stream";

import { BrevoContactImportService } from "../brevo-contact/brevo-contact-import.service";
import { BrevoModuleConfig } from "../config/brevo-module.config";
import { BREVO_MODULE_CONFIG } from "../config/brevo-module.constants";
import { EmailCampaignScopeInterface } from "../types";
import { CsvImportInformation } from "./brevo-contact-import.service";

export function createBrevoContactImportController({ Scope }: { Scope: Type<EmailCampaignScopeInterface> }): Type<unknown> {
@Controller("brevo-contacts-csv")
class BrevoContactImportController {
constructor(
@Inject(BREVO_MODULE_CONFIG) private readonly config: BrevoModuleConfig,
@Inject(BrevoContactImportService) private readonly brevoContactImportService: BrevoContactImportService,
@InjectRepository("TargetGroup") private readonly targetGroupRepository: EntityRepository<TargetGroupInterface>,
) {}

@Post("upload")
Expand All @@ -33,23 +31,25 @@ export function createBrevoContactImportController({ Scope }: { Scope: Type<Emai
}),
)
@RequiredPermission(["brevo-newsletter"], { skipScopeCheck: true })
async upload(@UploadedFile() file: Express.Multer.File, @Body("scope") scope: string, @Body("listIds") listIds?: string): Promise<void> {
const content = file.buffer.toString("utf8");
async upload(
@UploadedFile() file: Express.Multer.File,
@Body("scope") scope: string,
@Body("listIds") listIds?: string,
): Promise<CsvImportInformation> {
const parsedScope = JSON.parse(scope) as EmailCampaignScopeInterface;

const redirectUrl = this.config.brevo.resolveConfig(parsedScope).redirectUrlForImport;
const contacts = await this.brevoContactImportService.parseCsvToBrevoContacts(content, redirectUrl);

if (contacts.length > 100) {
throw new CometValidationException("Too many contacts in file. Currently we only support 100 contacts at once.");
}

let parsedListIds = undefined;
if (listIds) parsedListIds = JSON.parse(listIds) as string[];

const targetGroups = await this.targetGroupRepository.find({ id: { $in: parsedListIds } });

await this.brevoContactImportService.importContactsFromCsv(content, parsedScope, redirectUrl, targetGroups);
let targetGroupIds = undefined;
if (listIds) targetGroupIds = JSON.parse(listIds) as string[];

const stream = Readable.from(file.buffer);
return this.brevoContactImportService.importContactsFromCsv({
fileStream: stream,
scope: parsedScope,
redirectUrl,
targetGroupIds,
isAdminImport: true,
});
}
}

Expand Down
Loading

0 comments on commit f675cd0

Please sign in to comment.