From 9a015b57989d1e52ff2a61d678e16886c2427754 Mon Sep 17 00:00:00 2001 From: Pablo Costa Date: Mon, 28 Oct 2024 18:09:34 -0300 Subject: [PATCH 1/5] Remove report attribution (#3184) --- ...0.25T21.09.57.remove-report-attribution.js | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 site/gatsby-site/migrations/2024.10.25T21.09.57.remove-report-attribution.js diff --git a/site/gatsby-site/migrations/2024.10.25T21.09.57.remove-report-attribution.js b/site/gatsby-site/migrations/2024.10.25T21.09.57.remove-report-attribution.js new file mode 100644 index 0000000000..4d2aa82e76 --- /dev/null +++ b/site/gatsby-site/migrations/2024.10.25T21.09.57.remove-report-attribution.js @@ -0,0 +1,24 @@ +const config = require('../config'); + +/** @type {import('umzug').MigrationFn} */ +exports.up = async ({ context: { client } }) => { + const reportsCollection = client.db(config.realm.production_db.db_name).collection('reports'); + + // Change "AIAAIC" to "Anonymous" if it is the only element in the `submitters` array + const result1 = await reportsCollection.updateMany( + { submitters: ['AIAAIC'] }, + { $set: { submitters: ['Anonymous'] } } + ); + + console.log(`Modified ${result1.modifiedCount} documents where "AIAAIC" was the only submitter`); + + // Remove "AIAAIC" from the `submitters` array if there are multiple elements + const result2 = await reportsCollection.updateMany( + { submitters: 'AIAAIC' }, + { $pull: { submitters: 'AIAAIC' } } + ); + + console.log( + `Modified ${result2.modifiedCount} documents where "AIAAIC" was one of the submitters` + ); +}; From 50a9c6efbdea978192489f8494b933a97a0d23ec Mon Sep 17 00:00:00 2001 From: Cesar Varela Date: Tue, 29 Oct 2024 19:16:49 -0300 Subject: [PATCH 2/5] Make workflow fail on response error even on status code 200 (#3191) --- .github/workflows/process-notifications.yml | 25 +++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/.github/workflows/process-notifications.yml b/.github/workflows/process-notifications.yml index e2e8618c68..97ea7b9268 100644 --- a/.github/workflows/process-notifications.yml +++ b/.github/workflows/process-notifications.yml @@ -19,7 +19,24 @@ jobs: SITE_URL: ${{ vars.SITE_URL }} PROCESS_NOTIFICATIONS_SECRET: ${{ secrets.PROCESS_NOTIFICATIONS_SECRET }} run: | - curl -X POST "$SITE_URL/api/graphql" \ - -H "Content-Type: application/json" \ - -H "PROCESS_NOTIFICATIONS_SECRET: $PROCESS_NOTIFICATIONS_SECRET" \ - -d '{"query":"mutation { processNotifications }"}' + RESPONSE=$(curl -s -o response.json -w "%{http_code}" -X POST "$SITE_URL/api/graphql" \ + -H "Content-Type: application/json" \ + -H "PROCESS_NOTIFICATIONS_SECRET: $PROCESS_NOTIFICATIONS_SECRET" \ + -d '{"query":"mutation { processNotifications }"}') + + HTTP_STATUS=$RESPONSE + + if [ "$HTTP_STATUS" -ne 200 ]; then + echo "GraphQL mutation failed with HTTP status: $HTTP_STATUS" + cat response.json + exit 1 + fi + + if jq -e '.errors' response.json > /dev/null; then + echo "GraphQL mutation failed with errors:" + jq '.errors' response.json + exit 1 + fi + + echo "GraphQL mutation succeeded!" + jq '.' response.json \ No newline at end of file From 8a4d21468b35ab4575fb1e5bc67cdfaf63786355 Mon Sep 17 00:00:00 2001 From: Pablo Costa Date: Tue, 29 Oct 2024 20:11:56 -0300 Subject: [PATCH 3/5] Transform JSON CSV values to readable text (#3142) --- site/db-backup/bin/taxonomy_csv_export.py | 39 ++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/site/db-backup/bin/taxonomy_csv_export.py b/site/db-backup/bin/taxonomy_csv_export.py index aecb1cfdf1..895b64a39e 100644 --- a/site/db-backup/bin/taxonomy_csv_export.py +++ b/site/db-backup/bin/taxonomy_csv_export.py @@ -3,6 +3,28 @@ import csv import os +def extract_entities(json_data): + entities = [] + for item in json_data['attributes']: + if item['short_name'] == 'Entity': + entities.append(item['value_json']) + break + return entities + +def extract_snippets(json_data): + snippets = '' + for item in json_data['attributes']: + # convert value_json to string + escapedText = json.loads(item['value_json']) + + if(isinstance(escapedText, list)): + escapedText = ', '.join(escapedText) + else: + escapedText = str(escapedText) + + snippets = snippets + item['short_name'] + ': ' + escapedText + '\n' + return snippets + def taxonomy_csv_export(namespace, taxa_file, classification_file, target_folder): print(f"Processing namespace '{namespace}'...") @@ -52,8 +74,23 @@ def taxonomy_csv_export(namespace, taxa_file, classification_file, target_folder attribute_dict = {} for attr in attributes: value = json.loads(attr['value_json']) if 'value_json' in attr else '' + if isinstance(value, list): - value = ', '.join(map(str, value)) + flattened_list = [] + for item in value: + # Transform JSON entities data into a list of entities + if isinstance(item, dict) and attr['short_name'] == 'Entities': + flattened_list.append(extract_entities(item)) + else: + # Transform JSON snippets data into text + if isinstance(item, dict) and 'Snippets' in attr['short_name']: + flattened_list.append(extract_snippets(item)) + else: + flattened_list.append(item) + # Flatten the list and remove extra quotes + flattened_list = [str(item).strip('"') for sublist in flattened_list for item in (sublist if isinstance(sublist, list) else [sublist])] + value = ', '.join(flattened_list) + attribute_dict[attr['short_name']] = value # Get the corresponding values for each header From c1c25e7e090091892cba59208bfbf84316aa68eb Mon Sep 17 00:00:00 2001 From: Cesar Varela Date: Thu, 31 Oct 2024 15:17:06 -0300 Subject: [PATCH 4/5] Get atlas admin data in one call (#3193) * Get atlas admin data in one call * Update workflow to fail with Netlify errors --- .github/workflows/process-notifications.yml | 9 +++---- site/gatsby-site/server/fields/common.ts | 26 +++++++++++++++++++ .../server/fields/notifications.ts | 16 ++++-------- .../server/tests/notifications.spec.ts | 10 +++---- 4 files changed, 40 insertions(+), 21 deletions(-) diff --git a/.github/workflows/process-notifications.yml b/.github/workflows/process-notifications.yml index 97ea7b9268..265cba6100 100644 --- a/.github/workflows/process-notifications.yml +++ b/.github/workflows/process-notifications.yml @@ -11,7 +11,6 @@ on: jobs: execute-mutation: runs-on: ubuntu-latest - environment: ${{ inputs.environment }} steps: - name: Call GraphQL API @@ -32,11 +31,11 @@ jobs: exit 1 fi - if jq -e '.errors' response.json > /dev/null; then - echo "GraphQL mutation failed with errors:" - jq '.errors' response.json + if jq -e 'has("errors") or has("errorType")' response.json > /dev/null; then + echo "GraphQL mutation failed with the following response:" + jq '.' response.json exit 1 fi echo "GraphQL mutation succeeded!" - jq '.' response.json \ No newline at end of file + jq '.' response.json diff --git a/site/gatsby-site/server/fields/common.ts b/site/gatsby-site/server/fields/common.ts index 48248acda1..f853fe02e5 100644 --- a/site/gatsby-site/server/fields/common.ts +++ b/site/gatsby-site/server/fields/common.ts @@ -87,6 +87,7 @@ export interface UserAdminData { creationDate?: Date; lastAuthenticationDate?: Date; disabled?: boolean; + userId?: string; } export const getUserAdminData = async (userId: string) => { @@ -106,6 +107,31 @@ export const getUserAdminData = async (userId: string) => { return response; } +export const getUsersAdminData = async () => { + + const response = await apiRequest({ path: `/users` }); + + const result: UserAdminData[] = []; + + for (const userData of response) { + + if (userData.data?.email) { + + const user: UserAdminData = {}; + + user.email = userData.data.email; + user.creationDate = new Date(userData.creation_date * 1000); + user.lastAuthenticationDate = new Date(userData.last_authentication_date * 1000); + user.disabled = userData.disabled; + user.userId = userData._id; + + result.push(user); + } + } + + return result; +} + interface SendEmailParams { recipients: { email: string; diff --git a/site/gatsby-site/server/fields/notifications.ts b/site/gatsby-site/server/fields/notifications.ts index 4013b3b6e7..b271acdc6f 100644 --- a/site/gatsby-site/server/fields/notifications.ts +++ b/site/gatsby-site/server/fields/notifications.ts @@ -2,7 +2,7 @@ import { GraphQLFieldConfigMap, GraphQLInt } from 'graphql'; import { generateQueryFields } from '../utils'; import { Context, DBEntity, DBIncident, DBReport, DBSubscription, DBNotification } from '../interfaces'; import { NotificationType } from '../types/notification'; -import { getUserAdminData, sendEmail } from './common'; +import { getUserAdminData, getUsersAdminData, sendEmail } from './common'; import * as reporter from '../reporter'; import { hasHeaderSecret, isRole } from '../rules'; import config from '../config'; @@ -14,18 +14,12 @@ export const queryFields: GraphQLFieldConfigMap = { const getRecipients = async (userIds: string[]) => { - const recipients = []; + const users = await getUsersAdminData(); - for (const userId of userIds) { - const userResponse = await getUserAdminData(userId); + const recipients = users + .filter((user) => userIds.includes(user.userId!)) + .map((user) => ({ email: user.email!, userId: user.userId! })); - if (userResponse?.email) { - recipients.push({ - email: userResponse.email, - userId, - }); - } - } return recipients; } diff --git a/site/gatsby-site/server/tests/notifications.spec.ts b/site/gatsby-site/server/tests/notifications.spec.ts index 81fcf3298e..3a5367aaa4 100644 --- a/site/gatsby-site/server/tests/notifications.spec.ts +++ b/site/gatsby-site/server/tests/notifications.spec.ts @@ -160,7 +160,7 @@ describe(`Notifications`, () => { }; jest.spyOn(context, 'verifyToken').mockResolvedValue({ sub: "123" }) - jest.spyOn(common, 'getUserAdminData').mockResolvedValue({ email: 'test@test.com' }); + jest.spyOn(common, 'getUsersAdminData').mockResolvedValue([{ userId: '123', email: 'test@test.com' }]); const sendEmailMock = jest.spyOn(common, 'sendEmail').mockResolvedValue(); const response = await makeRequest(url, mutationData, { ['PROCESS_NOTIFICATIONS_SECRET']: config.PROCESS_NOTIFICATIONS_SECRET });; @@ -290,7 +290,7 @@ describe(`Notifications`, () => { }; jest.spyOn(context, 'verifyToken').mockResolvedValue({ sub: "123" }) - jest.spyOn(common, 'getUserAdminData').mockResolvedValue({ email: 'test@test.com' }); + jest.spyOn(common, 'getUsersAdminData').mockResolvedValue([{ userId: '123', email: 'test@test.com' }]); const sendEmailMock = jest.spyOn(common, 'sendEmail').mockResolvedValue(); const response = await makeRequest(url, mutationData, { ['PROCESS_NOTIFICATIONS_SECRET']: config.PROCESS_NOTIFICATIONS_SECRET });; @@ -422,7 +422,7 @@ describe(`Notifications`, () => { }; jest.spyOn(context, 'verifyToken').mockResolvedValue({ sub: "123" }) - jest.spyOn(common, 'getUserAdminData').mockResolvedValue({ email: 'test@test.com' }); + jest.spyOn(common, 'getUsersAdminData').mockResolvedValue([{ userId: '123', email: 'test@test.com' }]); const sendEmailMock = jest.spyOn(common, 'sendEmail').mockResolvedValue(); const response = await makeRequest(url, mutationData, { ['PROCESS_NOTIFICATIONS_SECRET']: config.PROCESS_NOTIFICATIONS_SECRET });; @@ -548,7 +548,7 @@ describe(`Notifications`, () => { }; jest.spyOn(context, 'verifyToken').mockResolvedValue({ sub: "123" }) - jest.spyOn(common, 'getUserAdminData').mockResolvedValue({ email: 'test@test.com' }); + jest.spyOn(common, 'getUsersAdminData').mockResolvedValue([{ userId: '123', email: 'test@test.com' }]); const sendEmailMock = jest.spyOn(common, 'sendEmail').mockResolvedValue(); const response = await makeRequest(url, mutationData, { ['PROCESS_NOTIFICATIONS_SECRET']: config.PROCESS_NOTIFICATIONS_SECRET });; @@ -675,7 +675,7 @@ describe(`Notifications`, () => { }; jest.spyOn(context, 'verifyToken').mockResolvedValue({ sub: "123" }) - jest.spyOn(common, 'getUserAdminData').mockResolvedValue({ email: 'test@test.com' }); + jest.spyOn(common, 'getUsersAdminData').mockResolvedValue([{ userId: '123', email: 'test@test.com' }]); const sendEmailMock = jest.spyOn(common, 'sendEmail').mockResolvedValue(); const response = await makeRequest(url, mutationData, { ['PROCESS_NOTIFICATIONS_SECRET']: config.PROCESS_NOTIFICATIONS_SECRET });; From 74515a4b60b3edca97c81dbc28e976cda172b404 Mon Sep 17 00:00:00 2001 From: Cesar Varela Date: Thu, 31 Oct 2024 17:05:02 -0300 Subject: [PATCH 5/5] Add missing environment setting (#3196) --- .github/workflows/process-notifications.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/process-notifications.yml b/.github/workflows/process-notifications.yml index 265cba6100..2c4509059b 100644 --- a/.github/workflows/process-notifications.yml +++ b/.github/workflows/process-notifications.yml @@ -11,7 +11,7 @@ on: jobs: execute-mutation: runs-on: ubuntu-latest - + environment: ${{ inputs.environment }} steps: - name: Call GraphQL API env: