From ac5bef35449a80e32b8ae2d98dfbc542a82fc687 Mon Sep 17 00:00:00 2001 From: Luna McNulty Date: Wed, 1 Mar 2023 19:00:55 -0500 Subject: [PATCH 01/16] Add stub of migration to add political bias labels --- ...3.01T23.01.01.add-political-bias-labels.js | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 site/gatsby-site/migrations/2023.03.01T23.01.01.add-political-bias-labels.js diff --git a/site/gatsby-site/migrations/2023.03.01T23.01.01.add-political-bias-labels.js b/site/gatsby-site/migrations/2023.03.01T23.01.01.add-political-bias-labels.js new file mode 100644 index 0000000000..440fe082ed --- /dev/null +++ b/site/gatsby-site/migrations/2023.03.01T23.01.01.add-political-bias-labels.js @@ -0,0 +1,51 @@ +//const config = require('../config'); +const axios = require('axios'); + +const jsdom = require('jsdom'); + +exports.up = async (/*{ context: { client } }*/) => { + const { JSDOM } = jsdom; + + for (const alignment of ['left', 'leftcenter', 'center', 'right-center', 'right']) { + console.log(`alignment`, alignment); + const mbfcResponse = await axios.get('https://mediabiasfactcheck.com/' + alignment); + + if (mbfcResponse.status == 200) { + const mbfcPage = new JSDOM(mbfcResponse.data); + + const sources = [...mbfcPage.window.document.querySelectorAll('#mbfc-table tr')] + .map((tr) => { + const deepestChild = getDeepestChild(tr); + + let tokens = deepestChild.textContent.split(' '); + + let lastToken = tokens.pop(); + + let domain; + + if (lastToken[0] == '(') { + domain = lastToken.slice(1, lastToken.length - 1); // Remove parentheses + } else { + tokens.push(lastToken); + } + const title = tokens.join(' '); + + return { title, domain }; + }) + .filter((source) => source?.title?.length > 0); + + console.log(`sources`, sources); + } + } + throw 'ok'; +}; + +exports.down = async () => {}; + +var getDeepestChild = (htmlNode) => { + if (htmlNode.children.length == 0) { + return htmlNode; + } else { + return getDeepestChild(htmlNode.children[0]); + } +}; From b759b1ff816166172a0e9f9cd2900204f89e4ce4 Mon Sep 17 00:00:00 2001 From: Luna McNulty Date: Thu, 2 Mar 2023 15:45:18 -0500 Subject: [PATCH 02/16] Add db integration to migration --- ...3.01T23.01.01.add-political-bias-labels.js | 95 +++++++++++++++---- 1 file changed, 74 insertions(+), 21 deletions(-) diff --git a/site/gatsby-site/migrations/2023.03.01T23.01.01.add-political-bias-labels.js b/site/gatsby-site/migrations/2023.03.01T23.01.01.add-political-bias-labels.js index 440fe082ed..6339114fff 100644 --- a/site/gatsby-site/migrations/2023.03.01T23.01.01.add-political-bias-labels.js +++ b/site/gatsby-site/migrations/2023.03.01T23.01.01.add-political-bias-labels.js @@ -1,46 +1,99 @@ -//const config = require('../config'); +const config = require('../config'); + const axios = require('axios'); const jsdom = require('jsdom'); -exports.up = async (/*{ context: { client } }*/) => { +exports.up = async ({ context: { client } }) => { const { JSDOM } = jsdom; + const publications = []; + + const addBiasLabel = (publicationTitle, publicationDomain, labeler, label) => { + const existingPublication = publications.find((p) => p.domain == publicationDomain); + + if (existingPublication) { + existingPublication.biasLabels.push({ label, labeler }); + } else { + publications.push({ + title: publicationTitle, + domain: publicationDomain, + biasLabels: [{ label, labeler }], + }); + } + }; + for (const alignment of ['left', 'leftcenter', 'center', 'right-center', 'right']) { - console.log(`alignment`, alignment); const mbfcResponse = await axios.get('https://mediabiasfactcheck.com/' + alignment); if (mbfcResponse.status == 200) { const mbfcPage = new JSDOM(mbfcResponse.data); - const sources = [...mbfcPage.window.document.querySelectorAll('#mbfc-table tr')] - .map((tr) => { - const deepestChild = getDeepestChild(tr); + for (const tr of [...mbfcPage.window.document.querySelectorAll('#mbfc-table tr')]) { + const deepestChild = getDeepestChild(tr); + + let tokens = deepestChild.textContent.split(' '); - let tokens = deepestChild.textContent.split(' '); + let lastToken = tokens.pop(); - let lastToken = tokens.pop(); + let domain; - let domain; + if (lastToken[0] == '(') { + domain = lastToken.slice(1, lastToken.length - 1); // Remove parentheses + } else { + tokens.push(lastToken); + } + const title = tokens.join(' '); - if (lastToken[0] == '(') { - domain = lastToken.slice(1, lastToken.length - 1); // Remove parentheses - } else { - tokens.push(lastToken); - } - const title = tokens.join(' '); + if (domain) { + addBiasLabel( + title, + domain, + 'mediabiasfactcheck.com', + { + left: 'left', + right: 'right', - return { title, domain }; - }) - .filter((source) => source?.title?.length > 0); + // These occur in most sources, and it's best to normalize them + // to a specific spelling / wording across sources + // so we can check if different sources agree. + leftcenter: 'center-left', + 'right-center': 'center-right', - console.log(`sources`, sources); + // They use "center" in the url but "least biased" in the site text. + // There's a difference between being unbiased + // and being biased towards the center. + // They seem to be trying to capture the former: + // + // > These sources have minimal bias and use very few loaded words + // > (wording that attempts to influence an audience + // > by using appeal to emotion or stereotypes). + // > The reporting is factual and usually sourced. + // > These are the most credible media sources. + // + center: 'least biased', + }[alignment] + ); + } + } } } - throw 'ok'; + await client.connect(); + + const publicationsCollection = await client + .db(config.realm.production_db.db_name) + .createCollection('publications'); + + for (const publication of publications) { + publicationsCollection.insertOne(publication); + } }; -exports.down = async () => {}; +exports.down = async ({ context: { client } }) => { + await client.connect(); + + await client.db(config.realm.production_db.db_name).dropCollection('publications'); +}; var getDeepestChild = (htmlNode) => { if (htmlNode.children.length == 0) { From 3fcc72361e19bc257d93b413dc8939ca934578b1 Mon Sep 17 00:00:00 2001 From: Luna McNulty Date: Fri, 3 Mar 2023 16:40:36 -0500 Subject: [PATCH 03/16] Make bias icons display on cite page --- site/gatsby-site/gatsby-config.js | 1 + site/gatsby-site/gatsby-node.js | 11 +++ ...3.01T23.01.01.add-political-bias-labels.js | 24 ++++- .../page-creators/createCitationPages.js | 22 ++++- .../src/components/reports/ReportCard.js | 92 +++++++++++++++++-- site/gatsby-site/src/templates/cite.js | 7 +- .../aiidprod/publications/relationships.json | 1 + .../aiidprod/publications/rules.json | 28 ++++++ .../aiidprod/publications/schema.json | 18 ++++ 9 files changed, 191 insertions(+), 13 deletions(-) create mode 100644 site/realm/data_sources/mongodb-atlas/aiidprod/publications/relationships.json create mode 100644 site/realm/data_sources/mongodb-atlas/aiidprod/publications/rules.json create mode 100644 site/realm/data_sources/mongodb-atlas/aiidprod/publications/schema.json diff --git a/site/gatsby-site/gatsby-config.js b/site/gatsby-site/gatsby-config.js index 38d845015b..3372b3eb9c 100755 --- a/site/gatsby-site/gatsby-config.js +++ b/site/gatsby-site/gatsby-config.js @@ -91,6 +91,7 @@ const plugins = [ 'classifications', 'reports', 'entities', + 'publications', ], connectionString: config.mongodb.connectionString, extraParams: { diff --git a/site/gatsby-site/gatsby-node.js b/site/gatsby-site/gatsby-node.js index 7a3a137e0b..924267d259 100644 --- a/site/gatsby-site/gatsby-node.js +++ b/site/gatsby-site/gatsby-node.js @@ -284,6 +284,17 @@ exports.createSchemaCustomization = ({ actions }) => { public: Boolean complete_from: completeFrom } + + + type mongodbAiidprodPublicationsHarm_labels { + label: String + labeler: String + } + type mongodbAiidprodPublications implements Node { + domain: String + title: String + harm_labels: [mongodbAiidprodPublicationsHarm_labels] + } `; createTypes(typeDefs); diff --git a/site/gatsby-site/migrations/2023.03.01T23.01.01.add-political-bias-labels.js b/site/gatsby-site/migrations/2023.03.01T23.01.01.add-political-bias-labels.js index 6339114fff..1aac5262a0 100644 --- a/site/gatsby-site/migrations/2023.03.01T23.01.01.add-political-bias-labels.js +++ b/site/gatsby-site/migrations/2023.03.01T23.01.01.add-political-bias-labels.js @@ -13,17 +13,25 @@ exports.up = async ({ context: { client } }) => { const existingPublication = publications.find((p) => p.domain == publicationDomain); if (existingPublication) { - existingPublication.biasLabels.push({ label, labeler }); + existingPublication.bias_labels.push({ label, labeler }); } else { publications.push({ title: publicationTitle, domain: publicationDomain, - biasLabels: [{ label, labeler }], + bias_labels: [{ label, labeler }], }); } }; - for (const alignment of ['left', 'leftcenter', 'center', 'right-center', 'right']) { + for (const alignment of [ + 'left', + 'leftcenter', + 'center', + 'right-center', + 'right', + 'fake-news', + 'conspiracy', + ]) { const mbfcResponse = await axios.get('https://mediabiasfactcheck.com/' + alignment); if (mbfcResponse.status == 200) { @@ -39,7 +47,12 @@ exports.up = async ({ context: { client } }) => { let domain; if (lastToken[0] == '(') { - domain = lastToken.slice(1, lastToken.length - 1); // Remove parentheses + domain = new URL( + 'http://' + + lastToken + .slice(1, lastToken.length - 1) // Remove parentheses + .replace(/^(www|m)\./, '') + ).hostname; } else { tokens.push(lastToken); } @@ -71,7 +84,10 @@ exports.up = async ({ context: { client } }) => { // > The reporting is factual and usually sourced. // > These are the most credible media sources. // + // Same with "fake-news" and "conspiracy". center: 'least biased', + 'fake-news': 'questionable', + conspiracy: 'conspiracy/pseudoscience', }[alignment] ); } diff --git a/site/gatsby-site/page-creators/createCitationPages.js b/site/gatsby-site/page-creators/createCitationPages.js index d7e899bc29..7cb0961b98 100644 --- a/site/gatsby-site/page-creators/createCitationPages.js +++ b/site/gatsby-site/page-creators/createCitationPages.js @@ -5,7 +5,7 @@ const { switchLocalizedPath } = require('../i18n'); const createCitationPages = async (graphql, createPage, { languages }) => { const result = await graphql( ` - query IncidentIDs { + query ContextData { allMongodbAiidprodIncidents { nodes { incident_id @@ -30,13 +30,26 @@ const createCitationPages = async (graphql, createPage, { languages }) => { language image_url cloudinary_id + source_domain + } + } + + allMongodbAiidprodPublications { + nodes { + title + domain + bias_labels { + label + labeler + } } } } ` ); - const { allMongodbAiidprodIncidents, allMongodbAiidprodReports } = result.data; + const { allMongodbAiidprodIncidents, allMongodbAiidprodReports, allMongodbAiidprodPublications } = + result.data; // Incident reports list const incidentReportsMap = {}; @@ -78,6 +91,10 @@ const createCitationPages = async (graphql, createPage, { languages }) => { reports: incidentReportsMap[incident_id], })); + const publications = allMongodbAiidprodPublications.nodes.filter((publication) => + incidentReportsMap[incident_id].some((report) => report.source_domain == publication.domain) + ); + pageContexts.push({ incident, incident_id, @@ -87,6 +104,7 @@ const createCitationPages = async (graphql, createPage, { languages }) => { nlp_similar_incidents, editor_similar_incidents, editor_dissimilar_incidents, + publications, }); } diff --git a/site/gatsby-site/src/components/reports/ReportCard.js b/site/gatsby-site/src/components/reports/ReportCard.js index 088a4fc63c..25feb4b9d4 100644 --- a/site/gatsby-site/src/components/reports/ReportCard.js +++ b/site/gatsby-site/src/components/reports/ReportCard.js @@ -13,10 +13,18 @@ import TranslationBadge from 'components/i18n/TranslationBadge'; import { Badge } from 'flowbite-react'; import { RESPONSE_TAG } from 'utils/entities'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { faQuestionCircle } from '@fortawesome/free-solid-svg-icons'; +import { + faQuestionCircle, + faArrowCircleLeft, + faArrowCircleRight, + faChevronCircleLeft, + faChevronCircleRight, + faCheckCircle, + faExclamationCircle, +} from '@fortawesome/free-solid-svg-icons'; import { hasVariantData } from 'utils/variants'; -const ReportCard = ({ item, className = '', incidentId }) => { +const ReportCard = ({ item, className = '', incidentId, publications }) => { const { isRole, loading } = useUserContext(); const { t } = useTranslation(); @@ -36,6 +44,32 @@ const ReportCard = ({ item, className = '', incidentId }) => { } }; + const publication = (publications || []).find((p) => p.domain == item.source_domain); + + const centerLeftBias = + publication && publication.bias_labels.some((biasLabel) => biasLabel.label == 'center-left'); + + const centerRightBias = + publication && publication.bias_labels.some((biasLabel) => biasLabel.label == 'center-right'); + + const leftBias = + publication && publication.bias_labels.some((biasLabel) => biasLabel.label == 'left'); + + const rightBias = + publication && publication.bias_labels.some((biasLabel) => biasLabel.label == 'right'); + + const leastBiased = + publication && publication.bias_labels.some((biasLabel) => biasLabel.label == 'least biased'); + + const questionable = + publication && publication.bias_labels.some((biasLabel) => biasLabel.label == 'questionable'); + + const biasTitle = + 'The bias of this source was assessed as follows:\n\n' + + (publication?.bias_labels || []) + .map((biasLabel) => `- "${biasLabel.label}" by ${biasLabel.labeler}`) + .join('\n'); + const toggleReadMoreKeyDown = (e) => { if (e.key === 'Enter') { toggleReadMore(); @@ -82,10 +116,56 @@ const ReportCard = ({ item, className = '', incidentId }) => {
- - {item.source_domain} ·{' '} - {item.date_published ? item.date_published.substring(0, 4) : 'Needs publish date'} - + + {leastBiased && ( + + )} + {questionable && ( +
+ +
+ )} + {centerLeftBias && ( + + )} + {centerRightBias && ( + + )} + {leftBias && ( + + )} + {rightBias && ( + + )} + + {item.source_domain} ·{' '} + {item.date_published ? item.date_published.substring(0, 4) : 'Needs publish date'} + + {!loading && isRole('incident_editor') && ( + setModalVisible(false)}> + {publicationName} + +

The bias of this source was assessed as follows:

+
    + {bias_labels.map((biasLabel) => ( +
  • + “{biasLabel.label}” by {biasLabel.labeler} +
    • + ))} +
+ + + + ); +} diff --git a/site/gatsby-site/src/components/discover/hitTypes/shared.js b/site/gatsby-site/src/components/discover/hitTypes/shared.js index c75b24ac61..b7bd1e2f05 100644 --- a/site/gatsby-site/src/components/discover/hitTypes/shared.js +++ b/site/gatsby-site/src/components/discover/hitTypes/shared.js @@ -6,15 +6,7 @@ import { Highlight } from 'react-instantsearch-dom'; import styled from 'styled-components'; import { VIEW_TYPES } from 'utils/discover'; import WebArchiveLink from '../../../components/ui/WebArchiveLink'; -import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { - faArrowCircleLeft, - faArrowCircleRight, - faChevronCircleLeft, - faChevronCircleRight, - faCheckCircle, - faExclamationCircle, -} from '@fortawesome/free-solid-svg-icons'; +import { BiasIcon } from 'components/BiasLabels'; const linkHoverHighlight = ` a:not(:hover) { @@ -69,79 +61,10 @@ export function HeaderTitle({ item, ...props }) { } export function SourceDomainSubtitle({ item, ...props }) { - const centerLeftBias = (item?.bias_labels || []).some( - (biasLabel) => biasLabel.label == 'center-left' - ); - - const centerRightBias = (item?.bias_labels || []).some( - (biasLabel) => biasLabel.label == 'center-right' - ); - - const leftBias = (item?.bias_labels || []).some((biasLabel) => biasLabel.label == 'left'); - - const rightBias = (item?.bias_labels || []).some((biasLabel) => biasLabel.label == 'right'); - - const leastBiased = (item?.bias_labels || []).some( - (biasLabel) => biasLabel.label == 'least biased' - ); - - const questionable = (item?.bias_labels || []).some( - (biasLabel) => biasLabel.label == 'questionable' - ); - - const biasTitle = - 'The bias of this source was assessed as follows:\n\n' + - (item?.bias_labels || []) - .map((biasLabel) => `- "${biasLabel.label}" by ${biasLabel.labeler}`) - .join('\n'); - return (
- {leastBiased && ( - - )} - {questionable && ( - - - - )} - {centerLeftBias && ( - - )} - {centerRightBias && ( - - )} - {leftBias && ( - - )} - {rightBias && ( - - )} + {item.source_domain} · {format(fromUnixTime(item.epoch_date_published), 'yyyy')} diff --git a/site/gatsby-site/src/components/reports/ReportCard.js b/site/gatsby-site/src/components/reports/ReportCard.js index 25feb4b9d4..3af858644c 100644 --- a/site/gatsby-site/src/components/reports/ReportCard.js +++ b/site/gatsby-site/src/components/reports/ReportCard.js @@ -13,16 +13,9 @@ import TranslationBadge from 'components/i18n/TranslationBadge'; import { Badge } from 'flowbite-react'; import { RESPONSE_TAG } from 'utils/entities'; import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { - faQuestionCircle, - faArrowCircleLeft, - faArrowCircleRight, - faChevronCircleLeft, - faChevronCircleRight, - faCheckCircle, - faExclamationCircle, -} from '@fortawesome/free-solid-svg-icons'; +import { faQuestionCircle } from '@fortawesome/free-solid-svg-icons'; import { hasVariantData } from 'utils/variants'; +import { BiasIcon } from 'components/BiasLabels'; const ReportCard = ({ item, className = '', incidentId, publications }) => { const { isRole, loading } = useUserContext(); @@ -46,30 +39,6 @@ const ReportCard = ({ item, className = '', incidentId, publications }) => { const publication = (publications || []).find((p) => p.domain == item.source_domain); - const centerLeftBias = - publication && publication.bias_labels.some((biasLabel) => biasLabel.label == 'center-left'); - - const centerRightBias = - publication && publication.bias_labels.some((biasLabel) => biasLabel.label == 'center-right'); - - const leftBias = - publication && publication.bias_labels.some((biasLabel) => biasLabel.label == 'left'); - - const rightBias = - publication && publication.bias_labels.some((biasLabel) => biasLabel.label == 'right'); - - const leastBiased = - publication && publication.bias_labels.some((biasLabel) => biasLabel.label == 'least biased'); - - const questionable = - publication && publication.bias_labels.some((biasLabel) => biasLabel.label == 'questionable'); - - const biasTitle = - 'The bias of this source was assessed as follows:\n\n' + - (publication?.bias_labels || []) - .map((biasLabel) => `- "${biasLabel.label}" by ${biasLabel.labeler}`) - .join('\n'); - const toggleReadMoreKeyDown = (e) => { if (e.key === 'Enter') { toggleReadMore(); @@ -117,50 +86,10 @@ const ReportCard = ({ item, className = '', incidentId, publications }) => {
- {leastBiased && ( - - )} - {questionable && ( -
- -
- )} - {centerLeftBias && ( - - )} - {centerRightBias && ( - - )} - {leftBias && ( - - )} - {rightBias && ( - - )} + {item.source_domain} ·{' '} {item.date_published ? item.date_published.substring(0, 4) : 'Needs publish date'} From 165cf97d2e39d0795c68531f29a682a8d5c4df5a Mon Sep 17 00:00:00 2001 From: Luna McNulty Date: Thu, 9 Mar 2023 14:14:33 -0500 Subject: [PATCH 08/16] Disable modal during SSR --- site/gatsby-site/src/components/BiasLabels.js | 32 +++++++++++-------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/site/gatsby-site/src/components/BiasLabels.js b/site/gatsby-site/src/components/BiasLabels.js index 08f0e10c1e..45d244e486 100644 --- a/site/gatsby-site/src/components/BiasLabels.js +++ b/site/gatsby-site/src/components/BiasLabels.js @@ -60,19 +60,25 @@ export function BiasIcon({ bias_labels, publicationName, className, style }) { ) )} - setModalVisible(false)}> - {publicationName} - -

The bias of this source was assessed as follows:

-
    - {bias_labels.map((biasLabel) => ( -
  • - “{biasLabel.label}” by {biasLabel.labeler} -
    • - ))} -
- - + { + // The modal causes server-side-rendering problems, + // so we need to disable it in that context. + window && ( + setModalVisible(false)}> + {publicationName} + +

The bias of this source was assessed as follows:

+
    + {bias_labels.map((biasLabel) => ( +
  • + “{biasLabel.label}” by {biasLabel.labeler} +
    • + ))} +
+ + + ) + } ); } From 8df57fea71e8a41c541cc7597fc7ea5566a32f14 Mon Sep 17 00:00:00 2001 From: Luna McNulty Date: Thu, 9 Mar 2023 15:24:33 -0500 Subject: [PATCH 09/16] Check type explicitly --- site/gatsby-site/src/components/BiasLabels.js | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/site/gatsby-site/src/components/BiasLabels.js b/site/gatsby-site/src/components/BiasLabels.js index 45d244e486..66cb09f094 100644 --- a/site/gatsby-site/src/components/BiasLabels.js +++ b/site/gatsby-site/src/components/BiasLabels.js @@ -50,11 +50,8 @@ export function BiasIcon({ bias_labels, publicationName, className, style }) { }[labelName] } className={`mr-1 mt-px ${ - labelName == 'least biased' - ? 'text-green-500' - : labelName == 'questionable' - ? 'text-red-500' - : 'text-inherit' + { 'least biased': 'text-green-500', questionable: 'text-red-500' }[labelName] || + 'text-inherit' }`} /> ) @@ -63,7 +60,7 @@ export function BiasIcon({ bias_labels, publicationName, className, style }) { { // The modal causes server-side-rendering problems, // so we need to disable it in that context. - window && ( + typeof window === 'undefined' && ( setModalVisible(false)}> {publicationName} From 822c3c43fdce3f889dd2a90bfb96b4155ac2da65 Mon Sep 17 00:00:00 2001 From: Luna McNulty Date: Thu, 9 Mar 2023 16:24:12 -0500 Subject: [PATCH 10/16] Add missing negator --- site/gatsby-site/src/components/BiasLabels.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/gatsby-site/src/components/BiasLabels.js b/site/gatsby-site/src/components/BiasLabels.js index 66cb09f094..30e3f1c713 100644 --- a/site/gatsby-site/src/components/BiasLabels.js +++ b/site/gatsby-site/src/components/BiasLabels.js @@ -60,7 +60,7 @@ export function BiasIcon({ bias_labels, publicationName, className, style }) { { // The modal causes server-side-rendering problems, // so we need to disable it in that context. - typeof window === 'undefined' && ( + typeof window !== 'undefined' && ( setModalVisible(false)}> {publicationName} From 78bbef0b4f9fb79fade2f7332456fa26e2e7f7b1 Mon Sep 17 00:00:00 2001 From: Luna McNulty Date: Fri, 17 Mar 2023 19:08:21 -0400 Subject: [PATCH 11/16] Render modal conditionally, deduplicate labels, remove green badge --- site/gatsby-site/src/components/BiasLabels.js | 50 ++++++++++--------- .../src/components/reports/ReportCard.js | 4 +- 2 files changed, 30 insertions(+), 24 deletions(-) diff --git a/site/gatsby-site/src/components/BiasLabels.js b/site/gatsby-site/src/components/BiasLabels.js index 30e3f1c713..8c6b611abf 100644 --- a/site/gatsby-site/src/components/BiasLabels.js +++ b/site/gatsby-site/src/components/BiasLabels.js @@ -13,6 +13,11 @@ import { export function BiasIcon({ bias_labels, publicationName, className, style }) { bias_labels ||= []; + // The modal causes server-side-rendering problems, + // so we need to disable rendering it + // until an interaction has occurred on the client-side. + const [modalRendered, setModalRendered] = useState(false); + const [modalVisible, setModalVisible] = useState(false); const hasLabelName = (labelName) => bias_labels.some((biasLabel) => biasLabel.label == labelName); @@ -31,7 +36,11 @@ export function BiasIcon({ bias_labels, publicationName, className, style }) { - { - // The modal causes server-side-rendering problems, - // so we need to disable it in that context. - typeof window !== 'undefined' && ( - setModalVisible(false)}> - {publicationName} - -

The bias of this source was assessed as follows:

-
    - {bias_labels.map((biasLabel) => ( -
  • - “{biasLabel.label}” by {biasLabel.labeler} -
    • - ))} -
- - - ) - } + {modalRendered && ( + setModalVisible(false)}> + {publicationName} + +

The bias of this source was assessed as follows:

+
    + {bias_labels.map((biasLabel) => ( +
  • + “{biasLabel.label}” by {biasLabel.labeler} +
    • + ))} +
+ + + )} ); } diff --git a/site/gatsby-site/src/components/reports/ReportCard.js b/site/gatsby-site/src/components/reports/ReportCard.js index 83f19b331d..ca72772367 100644 --- a/site/gatsby-site/src/components/reports/ReportCard.js +++ b/site/gatsby-site/src/components/reports/ReportCard.js @@ -89,7 +89,9 @@ const ReportCard = ({ item, className = '', incidentId, publications }) => {
JSON.stringify(biasLabel))) + ).map((json) => JSON.parse(json))} publicationName={publication?.title} /> From d13df962d30e1ebe44309fec518b8e7a0020aedb Mon Sep 17 00:00:00 2001 From: Luna McNulty Date: Thu, 23 Mar 2023 15:19:26 -0400 Subject: [PATCH 12/16] Add check for reports --- .../src/components/cite/SimilarIncidents.js | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/site/gatsby-site/src/components/cite/SimilarIncidents.js b/site/gatsby-site/src/components/cite/SimilarIncidents.js index ab0177146a..2190bfeb02 100644 --- a/site/gatsby-site/src/components/cite/SimilarIncidents.js +++ b/site/gatsby-site/src/components/cite/SimilarIncidents.js @@ -35,18 +35,20 @@ const SimilarIncidentCard = ({ incident, flaggable = true, flagged, parentIncide return ( - {(incident.reports[0].cloudinary_id || incident.reports[0]?.image_url) && ( - - )} + {incident?.reports?.length > 0 && + (incident.reports[0].cloudinary_id || incident.reports[0]?.image_url) && ( + + )}

{locale == 'en' && incident.title ? incident.title : incident.reports[0].title} From bffa62e029491fc02028c64c3c62ce9d00babb78 Mon Sep 17 00:00:00 2001 From: Luna McNulty Date: Mon, 27 Mar 2023 12:47:04 -0400 Subject: [PATCH 13/16] Add subsetting for Algolia --- site/gatsby-site/src/utils/AlgoliaUpdater.js | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/site/gatsby-site/src/utils/AlgoliaUpdater.js b/site/gatsby-site/src/utils/AlgoliaUpdater.js index ec3c5ef145..4a20d24021 100644 --- a/site/gatsby-site/src/utils/AlgoliaUpdater.js +++ b/site/gatsby-site/src/utils/AlgoliaUpdater.js @@ -10,7 +10,7 @@ const { isCompleteReport } = require('./variants'); // to fit within the Algolia tier limits. // // TODO: Put this configuration in a more convenient place. -const LIMIT = 200; //Number.MAX_SAFE_INTEGER; +const LIMIT = 50; //Number.MAX_SAFE_INTEGER; const truncate = (doc) => { for (const [key, value] of Object.entries(doc)) { @@ -185,9 +185,23 @@ class AlgoliaUpdater { const truncatedData = downloadData.map(truncate); - const smallData = truncatedData.slice(0, LIMIT); + const ids = Array.from(new Set(truncatedData.map((entry) => entry.incident_id))); + + const smallData = ids.reduce( + (entries, id) => + entries.concat( + truncatedData[ + [10, 34, 186, 477, 443, 23] + .concat(Object.keys(config.header.search.featured)) + .includes(id) + ? 'filter' + : 'find' + ]((entry) => entry.incident_id == id) + ), + [] + ); - return smallData; + return LIMIT < Number.MAX_SAFE_INTEGER ? smallData : truncatedData; }; getClassifications = async () => { From afb48ab831e54280711a8e1f3dbaf8820c53d03f Mon Sep 17 00:00:00 2001 From: Luna McNulty Date: Wed, 26 Apr 2023 16:20:53 -0400 Subject: [PATCH 14/16] Fix alignment --- site/gatsby-site/src/components/discover/hitTypes/shared.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/site/gatsby-site/src/components/discover/hitTypes/shared.js b/site/gatsby-site/src/components/discover/hitTypes/shared.js index 73c5b98a83..fcc343c368 100644 --- a/site/gatsby-site/src/components/discover/hitTypes/shared.js +++ b/site/gatsby-site/src/components/discover/hitTypes/shared.js @@ -43,7 +43,7 @@ export function HeaderTitle({ item, ...props }) { export function SourceDomainSubtitle({ item, ...props }) { return ( -
+
{item.source_domain} · {format(fromUnixTime(item.epoch_date_published), 'yyyy')} From 0d11c770f7a99f1cbbc8633fb8cbb3dad3b28251 Mon Sep 17 00:00:00 2001 From: Luna McNulty Date: Fri, 12 May 2023 18:34:01 -0400 Subject: [PATCH 15/16] Remove algolia updates --- .../components/discover/hitTypes/shared.js | 4 +-- site/gatsby-site/src/utils/AlgoliaUpdater.js | 36 ++++--------------- 2 files changed, 7 insertions(+), 33 deletions(-) diff --git a/site/gatsby-site/src/components/discover/hitTypes/shared.js b/site/gatsby-site/src/components/discover/hitTypes/shared.js index fcc343c368..7c179baf2d 100644 --- a/site/gatsby-site/src/components/discover/hitTypes/shared.js +++ b/site/gatsby-site/src/components/discover/hitTypes/shared.js @@ -4,7 +4,6 @@ import React from 'react'; import { Highlight } from 'react-instantsearch-dom'; import { VIEW_TYPES } from 'utils/discover'; import WebArchiveLink from '../../../components/ui/WebArchiveLink'; -import { BiasIcon } from 'components/BiasLabels'; export function citationReportUrl(item, viewType) { let path = null; @@ -43,8 +42,7 @@ export function HeaderTitle({ item, ...props }) { export function SourceDomainSubtitle({ item, ...props }) { return ( -
- +
{item.source_domain} · {format(fromUnixTime(item.epoch_date_published), 'yyyy')} diff --git a/site/gatsby-site/src/utils/AlgoliaUpdater.js b/site/gatsby-site/src/utils/AlgoliaUpdater.js index 4a20d24021..ae3c46cd8d 100644 --- a/site/gatsby-site/src/utils/AlgoliaUpdater.js +++ b/site/gatsby-site/src/utils/AlgoliaUpdater.js @@ -10,7 +10,7 @@ const { isCompleteReport } = require('./variants'); // to fit within the Algolia tier limits. // // TODO: Put this configuration in a more convenient place. -const LIMIT = 50; //Number.MAX_SAFE_INTEGER; +const LIMIT = Number.MAX_SAFE_INTEGER; const truncate = (doc) => { for (const [key, value] of Object.entries(doc)) { @@ -77,7 +77,7 @@ const getClassificationArray = ({ classification, taxonomy }) => { return result; }; -const reportToEntry = ({ incident = null, report, publication = null }) => { +const reportToEntry = ({ incident = null, report }) => { let featuredValue = 0; if (config?.header?.search?.featured) { @@ -94,7 +94,6 @@ const reportToEntry = ({ incident = null, report, publication = null }) => { const entry = { authors: report.authors, - bias_labels: publication?.bias_labels, description: report.description, epoch_date_downloaded: report.epoch_date_downloaded, epoch_date_modified: report.epoch_date_modified, @@ -145,7 +144,7 @@ class AlgoliaUpdater { this.algoliaClient = algoliaClient; } - generateIndexEntries = async ({ reports, incidents, classifications, taxa, publications }) => { + generateIndexEntries = async ({ reports, incidents, classifications, taxa }) => { let classificationsHash = {}; classifications.forEach((classification) => { @@ -164,9 +163,7 @@ class AlgoliaUpdater { if (reports.some((r) => r.report_number == report_number)) { const report = reports.find((r) => r.report_number == report_number) || {}; - const publication = publications.find((p) => p.domain == report.source_domain); - - const entry = reportToEntry({ incident, report, publication }); + const entry = reportToEntry({ incident, report }); if (classificationsHash[entry.incident_id]) { entry.classifications = classificationsHash[entry.incident_id]; @@ -185,23 +182,9 @@ class AlgoliaUpdater { const truncatedData = downloadData.map(truncate); - const ids = Array.from(new Set(truncatedData.map((entry) => entry.incident_id))); - - const smallData = ids.reduce( - (entries, id) => - entries.concat( - truncatedData[ - [10, 34, 186, 477, 443, 23] - .concat(Object.keys(config.header.search.featured)) - .includes(id) - ? 'filter' - : 'find' - ]((entry) => entry.incident_id == id) - ), - [] - ); + const smallData = truncatedData.slice(0, LIMIT); - return LIMIT < Number.MAX_SAFE_INTEGER ? smallData : truncatedData; + return smallData; }; getClassifications = async () => { @@ -294,10 +277,6 @@ class AlgoliaUpdater { return fullReports; }; - getPublications = async () => { - return await this.mongoClient.db('aiidprod').collection(`publications`).find({}).toArray(); - }; - uploadToAlgolia = async ({ language, entries }) => { const indexName = `instant_search-${language}`; @@ -423,14 +402,11 @@ class AlgoliaUpdater { const reports = await this.getReports({ language }); - const publications = await this.getPublications(); - const entries = await this.generateIndexEntries({ reports, incidents, classifications, taxa, - publications, }); await this.mongoClient.close(); From 80b4fc54687c4672634eb85dad1fc6e136291239 Mon Sep 17 00:00:00 2001 From: Luna McNulty Date: Fri, 12 May 2023 21:02:28 -0400 Subject: [PATCH 16/16] Add missing publications arguments --- site/gatsby-site/src/templates/citeDynamicTemplate.js | 2 ++ site/gatsby-site/src/templates/citeTemplate.js | 7 ++++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/site/gatsby-site/src/templates/citeDynamicTemplate.js b/site/gatsby-site/src/templates/citeDynamicTemplate.js index 5ca8e984d0..f4e15b4f45 100644 --- a/site/gatsby-site/src/templates/citeDynamicTemplate.js +++ b/site/gatsby-site/src/templates/citeDynamicTemplate.js @@ -20,6 +20,7 @@ function CiteDynamicTemplate({ editor_dissimilar_incidents, locationPathName, setIsLiveData, + publications, }) { const { locale } = useLocalization(); @@ -154,6 +155,7 @@ function CiteDynamicTemplate({ editor_dissimilar_incidents={editor_dissimilar_incidents} liveVersion={true} setIsLiveData={setIsLiveData} + publications={publications} /> ) )} diff --git a/site/gatsby-site/src/templates/citeTemplate.js b/site/gatsby-site/src/templates/citeTemplate.js index 6cee7dfd60..e8bcaffa6a 100644 --- a/site/gatsby-site/src/templates/citeTemplate.js +++ b/site/gatsby-site/src/templates/citeTemplate.js @@ -45,6 +45,7 @@ function CiteTemplate({ editor_dissimilar_incidents, liveVersion = false, setIsLiveData, + publications, }) { const { isRole, user } = useUserContext(); @@ -337,7 +338,11 @@ function CiteTemplate({ {sortedReports.map((report) => ( - + ))}