diff --git a/.github/workflows/db-backup.yml b/.github/workflows/db-backup.yml index e81e0c2e92..f2e8af8589 100644 --- a/.github/workflows/db-backup.yml +++ b/.github/workflows/db-backup.yml @@ -7,7 +7,7 @@ on: inputs: environment: description: The Github environment to load secrets from - type: string + type: string required: true defaults: @@ -25,6 +25,11 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + - name: Install dependencies run: | sudo apt-get update @@ -32,19 +37,18 @@ jobs: coreutils \ bash \ tzdata \ - python3-pip \ curl \ npm wget -qO - https://www.mongodb.org/static/pgp/server-6.0.asc | gpg --dearmor | sudo tee /usr/share/keyrings/mongodb.gpg > /dev/null echo "deb [ arch=amd64 signed-by=/usr/share/keyrings/mongodb.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/6.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-6.0.list sudo apt update sudo apt install -y mongodb-database-tools - - - name: Install boto3 - run: pip install boto3 - name: Generate public backup run: | + python3 -m venv venv + source venv/bin/activate + pip install boto3 ./bin/backup.sh ./bin/prune.sh ./bin/list.sh diff --git a/.github/workflows/test-api.yml b/.github/workflows/test-api.yml index 3ebcc65620..0fd81cddaf 100644 --- a/.github/workflows/test-api.yml +++ b/.github/workflows/test-api.yml @@ -68,6 +68,10 @@ jobs: E2E_ADMIN_USERNAME: "" E2E_ADMIN_PASSWORD: "" ROLLBAR_POST_SERVER_ITEM_ACCESS_TOKEN: ${{ secrets.GATSBY_ROLLBAR_TOKEN }} + SENDGRID_API_KEY: something + SENDGRID_SENDER_NAME: Test Preview + SENDGRID_SENDER: test@test.com + PROCESS_NOTIFICATIONS_SECRET: sarasa - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v4 diff --git a/.github/workflows/test-playwright-full.yml b/.github/workflows/test-playwright-full.yml index e718d3ff33..8488845831 100644 --- a/.github/workflows/test-playwright-full.yml +++ b/.github/workflows/test-playwright-full.yml @@ -134,6 +134,10 @@ jobs: SHARD_INDEX: ${{ matrix.shardIndex }} SHARD_TOTAL: ${{ matrix.shardTotal }} TEST_FOLDER: playwright/e2e-full/ + SENDGRID_API_KEY: something + SENDGRID_SENDER_NAME: Test Preview + SENDGRID_SENDER: test@test.com + PROCESS_NOTIFICATIONS_SECRET: sarasa - name: Upload Playwright traces if: failure() diff --git a/.github/workflows/test-playwright.yml b/.github/workflows/test-playwright.yml index 5f08ad7b95..563a29bcf6 100644 --- a/.github/workflows/test-playwright.yml +++ b/.github/workflows/test-playwright.yml @@ -89,6 +89,10 @@ jobs: SHARD_INDEX: ${{ matrix.shardIndex }} SHARD_TOTAL: ${{ matrix.shardTotal }} TEST_FOLDER: playwright/e2e/ + SENDGRID_API_KEY: something + SENDGRID_SENDER_NAME: Test Preview + SENDGRID_SENDER: test@test.com + PROCESS_NOTIFICATIONS_SECRET: sarasa - uses: actions/upload-artifact@v4 if: ${{ !cancelled() }} diff --git a/site/gatsby-site/migrations/2024.10.01T23.20.28.remove-null-epoch_date_modified-values.js b/site/gatsby-site/migrations/2024.10.01T23.20.28.remove-null-epoch_date_modified-values.js new file mode 100644 index 0000000000..651d97faf3 --- /dev/null +++ b/site/gatsby-site/migrations/2024.10.01T23.20.28.remove-null-epoch_date_modified-values.js @@ -0,0 +1,30 @@ +const config = require('../config'); + +/** @type {import('umzug').MigrationFn} */ +exports.up = async ({ context: { client } }) => { + // If the incident has "epoch_date_modified" field and it is null, unset it + const incidents_collection = client + .db(config.realm.production_db.db_name) + .collection('incidents'); + + const incidents_epoch_date_modified_results = await incidents_collection.updateMany( + { epoch_date_modified: null }, + { $unset: { epoch_date_modified: '' } } + ); + + console.log( + `Updated ${incidents_epoch_date_modified_results.modifiedCount} incidents with null epoch_date_modified field` + ); + + // If the report has "epoch_date_modified" field and it is null, unset it + const reports_collection = client.db(config.realm.production_db.db_name).collection('reports'); + + const reports_epoch_date_modified_results = await reports_collection.updateMany( + { epoch_date_modified: null }, + { $unset: { epoch_date_modified: '' } } + ); + + console.log( + `Updated ${reports_epoch_date_modified_results.modifiedCount} reports with null epoch_date_modified field` + ); +}; diff --git a/site/gatsby-site/server/fields/submissions.ts b/site/gatsby-site/server/fields/submissions.ts index b2db5b2eee..19c32b6d8b 100644 --- a/site/gatsby-site/server/fields/submissions.ts +++ b/site/gatsby-site/server/fields/submissions.ts @@ -97,7 +97,6 @@ export const mutationFields: GraphQLFieldConfigMap = { reports: [], editors, date: submission.incident_date, - epoch_date_modified: submission.epoch_date_modified, "Alleged deployer of AI system": submission.deployers || [], "Alleged developer of AI system": submission.developers || [], "Alleged harmed or nearly harmed parties": submission.harmed_parties || [], @@ -113,6 +112,9 @@ export const mutationFields: GraphQLFieldConfigMap = { from_reports: [report_number] } } + if(submission.epoch_date_modified) { + newIncident.epoch_date_modified = submission.epoch_date_modified; + } await incidents.insertOne({ ...newIncident, incident_id: newIncident.incident_id }); @@ -214,7 +216,6 @@ export const mutationFields: GraphQLFieldConfigMap = { date_published: new Date(submission.date_published), date_submitted: new Date(submission.date_submitted), epoch_date_downloaded: getUnixTime(submission.date_downloaded), - epoch_date_modified: submission.epoch_date_modified, epoch_date_published: getUnixTime(submission.date_published), epoch_date_submitted: getUnixTime(submission.date_submitted), image_url: submission.image_url, @@ -237,6 +238,10 @@ export const mutationFields: GraphQLFieldConfigMap = { newReport.user = submission.user; } + if(submission.epoch_date_modified) { + newReport.epoch_date_modified = submission.epoch_date_modified; + } + await reports.insertOne({ ...newReport, report_number: newReport.report_number }); const incident_ids: number[] = parentIncidents.map(incident => incident.incident_id!);