From 1ef65f381c10a886b9226bc67309ad2942e6d160 Mon Sep 17 00:00:00 2001 From: Luna McNulty Date: Tue, 11 Apr 2023 14:16:02 -0400 Subject: [PATCH 01/67] Add empty endpoint --- site/gatsby-site/src/api/riskManagement/v1/risks.js | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 site/gatsby-site/src/api/riskManagement/v1/risks.js diff --git a/site/gatsby-site/src/api/riskManagement/v1/risks.js b/site/gatsby-site/src/api/riskManagement/v1/risks.js new file mode 100644 index 0000000000..b7a5be9ce7 --- /dev/null +++ b/site/gatsby-site/src/api/riskManagement/v1/risks.js @@ -0,0 +1,3 @@ +export default async function handler(req, res) { + res.status(200).json(req); +} From 0885d0a4eba710e50de3c261dc765981253e6785 Mon Sep 17 00:00:00 2001 From: Luna McNulty Date: Wed, 12 Apr 2023 11:33:34 -0400 Subject: [PATCH 02/67] Add risks endpoint --- .../src/api/riskManagement/v1/risks.js | 202 +++++++++++++++++- 1 file changed, 201 insertions(+), 1 deletion(-) diff --git a/site/gatsby-site/src/api/riskManagement/v1/risks.js b/site/gatsby-site/src/api/riskManagement/v1/risks.js index b7a5be9ce7..d7982acfb0 100644 --- a/site/gatsby-site/src/api/riskManagement/v1/risks.js +++ b/site/gatsby-site/src/api/riskManagement/v1/risks.js @@ -1,3 +1,203 @@ +import { MongoClient } from 'mongodb'; + +import config from '../../../../config'; + + +const groupable = (array) => { + array.groupBy = (keyFunction, valueFunction) => { + const groups = {}; + for (const element of array) { + const key = keyFunction(element); + groups[key] ||= []; + groups[key].push( + valueFunction ? valueFunction(element) : element + ); + } + return groups; + } + array.groupByMultiple = (keyFunction, valueFunction) => { + const groups = {}; + for (const element of array) { + const keys = keyFunction(element); + for (const key of keys) { + groups[key] ||= new Set(); + groups[key].add( + valueFunction ? valueFunction(element) : element + ); + } + } + for (const group in groups) { + groups[group] = Array.from(groups[group]); + } + return groups; + } + return array; +} + export default async function handler(req, res) { - res.status(200).json(req); + + + const mongoClient = new MongoClient(config.mongodb.translationsConnectionString); + + const incidentsCollection = mongoClient.db('aiidprod').collection('incidents'); + + const classificationsCollection = mongoClient.db('aiidprod').collection('classifications'); + + const selectors = [ +// { +// short_name: "Known AI Technical Failure", +// value_json: { $regex: '"Tuning Issues"' } +// }, +// { +// short_name: "Known AI Technology", +// value_json: { $regex: '"Content-based Filtering"' } +// }, + ] + + const tagStrings = req.query.tags.split('___'); + + const tagsByNamespace = {}; + + for (const tagString of tagStrings) { + const parts = tagString.split(":"); + const namespace = parts[0]; + tagsByNamespace[namespace] ||= []; + const tag = {}; + tag.short_name = parts[1]; + if (parts.length > 2) { + tag.value_json = {$regex: `"${parts[2]}"`}; + } + tagsByNamespace[namespace].push(tag); + } + + const matchingClassifications = await classificationsCollection.find( + { + $or: Object.keys(tagsByNamespace).map( + namespace => ({ + namespace, + attributes: { + $elemMatch: { + $or: tagsByNamespace[namespace] + } + } + }) + ) + }, + //{ projection: { incident_id: 1 }} + ).toArray(); + + const matchingIncidentIds = matchingClassifications.map(classification => classification.incident_id); + + const classificationsByIncident = groupable(matchingClassifications).groupBy( + classification => classification.incident_id + ); + + const failureAttributeQuery = { + attributes: { + $elemMatch: { + short_name: { + $in: [ + "Known AI Technical Failure", + "Potential AI Technical Failure" + ] + } + } + } + }; + + const matchingIncidents = await incidentsCollection.find( + { + incident_id: { + $in: matchingIncidentIds + }, + }, + { projection: { incident_id: 1, title: 1 }} + ).toArray(); + + const matchingIncidentsByIncidentId = groupable(matchingIncidents).groupBy( + incident => incident.incident_id + ); + + const matchingFailureClassifications = await classificationsCollection.find( + { + incident_id: { + $in: matchingIncidentIds + }, + ...failureAttributeQuery + }, + { + projection: { + namespace: 1, + incident_id: 1, + ...failureAttributeQuery + } + } + ).toArray(); + + const classificationsByFailure = + groupable(matchingFailureClassifications) + .groupByMultiple( + classification => classification.attributes.reduce( + (tags, attribute) => tags.concat( + JSON.parse(attribute.value_json) +// .map( +// tag => [ +// classification.namespace, +// attribute.short_name, +// tag +// ].join(':') +// ) + ), [] + ) + ); + + const risks = Object.keys(classificationsByFailure).map( + failure => { + const classifications = classificationsByFailure[failure]; + return { + tag: failure, + precedents: classifications.map(classification => ({ + ...classification, + title: matchingIncidentsByIncidentId[classification.incident_id][0].title + })) /*.map(classification => + matchingIncidentsByIncidentId[classifications.incident_id] + )*/ + } + } + ).sort((a, b) => b.precedents.length - a.precedents.length); + +/* + const result = await classificationsCollection.find( + { + namespace: "GMF", + attributes: { + $elemMatch: { + $or: [ + ...selectors + ] + } + } + }, + { projection: { + _id: 0, + incident_id: 1, + attributes: 1 + } + } + ).toArray(); + + + const result = await incidentsCollection.find( + { incident_id: 1 + + }, + { projection: { + incident_id: 1, + title: 1 + } + } + ).toArray(); +*/ + + res.status(200).json(risks); } From 4a7318d947f1fb4c860e2094a76de699721bacbd Mon Sep 17 00:00:00 2001 From: Luna McNulty Date: Fri, 5 May 2023 20:14:36 -0400 Subject: [PATCH 03/67] Convert attributes to tags correctly --- .../src/api/riskManagement/v1/risks.js | 259 ++++++++---------- 1 file changed, 117 insertions(+), 142 deletions(-) diff --git a/site/gatsby-site/src/api/riskManagement/v1/risks.js b/site/gatsby-site/src/api/riskManagement/v1/risks.js index d7982acfb0..31f1a25cc3 100644 --- a/site/gatsby-site/src/api/riskManagement/v1/risks.js +++ b/site/gatsby-site/src/api/riskManagement/v1/risks.js @@ -2,96 +2,29 @@ import { MongoClient } from 'mongodb'; import config from '../../../../config'; - -const groupable = (array) => { - array.groupBy = (keyFunction, valueFunction) => { - const groups = {}; - for (const element of array) { - const key = keyFunction(element); - groups[key] ||= []; - groups[key].push( - valueFunction ? valueFunction(element) : element - ); - } - return groups; - } - array.groupByMultiple = (keyFunction, valueFunction) => { - const groups = {}; - for (const element of array) { - const keys = keyFunction(element); - for (const key of keys) { - groups[key] ||= new Set(); - groups[key].add( - valueFunction ? valueFunction(element) : element - ); - } - } - for (const group in groups) { - groups[group] = Array.from(groups[group]); - } - return groups; - } - return array; -} - export default async function handler(req, res) { - const mongoClient = new MongoClient(config.mongodb.translationsConnectionString); const incidentsCollection = mongoClient.db('aiidprod').collection('incidents'); const classificationsCollection = mongoClient.db('aiidprod').collection('classifications'); - const selectors = [ -// { -// short_name: "Known AI Technical Failure", -// value_json: { $regex: '"Tuning Issues"' } -// }, -// { -// short_name: "Known AI Technology", -// value_json: { $regex: '"Content-based Filtering"' } -// }, - ] - - const tagStrings = req.query.tags.split('___'); + const classificationsMatchingSearchTags = await classificationsCollection.find( + getRiskClassificationsMongoQuery(req.query), + ).toArray(); - const tagsByNamespace = {}; + const incidentIdsMatchingSearchTags = classificationsMatchingSearchTags.map(classification => classification.incident_id); - for (const tagString of tagStrings) { - const parts = tagString.split(":"); - const namespace = parts[0]; - tagsByNamespace[namespace] ||= []; - const tag = {}; - tag.short_name = parts[1]; - if (parts.length > 2) { - tag.value_json = {$regex: `"${parts[2]}"`}; - } - tagsByNamespace[namespace].push(tag); - } - - const matchingClassifications = await classificationsCollection.find( + const incidentsMatchingSearchTags = await incidentsCollection.find( { - $or: Object.keys(tagsByNamespace).map( - namespace => ({ - namespace, - attributes: { - $elemMatch: { - $or: tagsByNamespace[namespace] - } - } - }) - ) + incident_id: { + $in: incidentIdsMatchingSearchTags + }, }, - //{ projection: { incident_id: 1 }} + { projection: { incident_id: 1, title: 1 }} ).toArray(); - const matchingIncidentIds = matchingClassifications.map(classification => classification.incident_id); - - const classificationsByIncident = groupable(matchingClassifications).groupBy( - classification => classification.incident_id - ); - const failureAttributeQuery = { attributes: { $elemMatch: { @@ -104,24 +37,10 @@ export default async function handler(req, res) { } } }; - - const matchingIncidents = await incidentsCollection.find( + const failureClassificationsMatchingIncidentIdsMatchingSearchTags = await classificationsCollection.find( { incident_id: { - $in: matchingIncidentIds - }, - }, - { projection: { incident_id: 1, title: 1 }} - ).toArray(); - - const matchingIncidentsByIncidentId = groupable(matchingIncidents).groupBy( - incident => incident.incident_id - ); - - const matchingFailureClassifications = await classificationsCollection.find( - { - incident_id: { - $in: matchingIncidentIds + $in: incidentIdsMatchingSearchTags }, ...failureAttributeQuery }, @@ -133,71 +52,127 @@ export default async function handler(req, res) { } } ).toArray(); + + // TODO: Use a shorter name for this + const failureClassificationsMatchingIncidentIdsMatchingSearchTags_ByFailure = ( + groupable(failureClassificationsMatchingIncidentIdsMatchingSearchTags).groupByMultiple( + classification => tagsFromClassification(classification) + ) + ); - const classificationsByFailure = - groupable(matchingFailureClassifications) - .groupByMultiple( - classification => classification.attributes.reduce( - (tags, attribute) => tags.concat( - JSON.parse(attribute.value_json) -// .map( -// tag => [ -// classification.namespace, -// attribute.short_name, -// tag -// ].join(':') -// ) - ), [] - ) - ); - - const risks = Object.keys(classificationsByFailure).map( + const risks = Object.keys(failureClassificationsMatchingIncidentIdsMatchingSearchTags_ByFailure).map( failure => { - const classifications = classificationsByFailure[failure]; + const failureClassifications = failureClassificationsMatchingIncidentIdsMatchingSearchTags_ByFailure[failure]; return { tag: failure, - precedents: classifications.map(classification => ({ - ...classification, - title: matchingIncidentsByIncidentId[classification.incident_id][0].title - })) /*.map(classification => - matchingIncidentsByIncidentId[classifications.incident_id] - )*/ + precedents: failureClassifications.map(failureClassification => { + const classificationsMatchingIncidentIdOfFailureClassification = classificationsMatchingSearchTags.filter( + c => c.incident_id == failureClassification.incident_id + ); + return { + incident_id: failureClassification.incident_id, + title: incidentsMatchingSearchTags.find( + incident => incident.incident_id == failureClassification.incident_id + )?.title, + tags: classificationsMatchingIncidentIdOfFailureClassification.map(c => tagsFromClassification(c)) + } + }) } } ).sort((a, b) => b.precedents.length - a.precedents.length); -/* - const result = await classificationsCollection.find( - { - namespace: "GMF", + res.status(200).json(risks); + +} + +function getRiskClassificationsMongoQuery(queryParams) { + const tagStrings = queryParams.tags.split('___'); + + const tagSearch = {}; + + for (const tagString of tagStrings) { + const parts = tagString.split(":"); + const namespace = parts[0]; + tagSearch[namespace] ||= []; + const tag = {}; + tag.short_name = parts[1]; + if (parts.length > 2) { + tag.value_json = {$regex: `"${parts[2]}"`}; + } + tagSearch[namespace].push(tag); + } + console.log(`tagSearch`, tagSearch); + + return { + $or: Object.keys(tagSearch).map( + namespace => ({ + namespace, attributes: { $elemMatch: { - $or: [ - ...selectors - ] + $or: tagSearch[namespace] } } - }, - { projection: { - _id: 0, - incident_id: 1, - attributes: 1 - } - } - ).toArray(); + }) + ) + } +} +var tagsFromClassification = (classification) => ( + // classification: + // { + // attributes: [ + // { short_name: "Known AI Goal"}, + // value_json: '["Content Recommendation", "Something"]' } + // ... + // ] + // } + joinArrays( + classification.attributes.map( + attribute => ( + [].concat(JSON.parse(attribute.value_json)) + .filter(value => Array.isArray(value) || typeof value !== 'object') + .map( + value => [ + classification.namespace, + attribute.short_name, + value + ].join(':') + ) + ) + ) + ) +); - const result = await incidentsCollection.find( - { incident_id: 1 +var joinArrays = (arrays) => arrays.reduce((result, array) => result.concat(array), []); - }, - { projection: { - incident_id: 1, - title: 1 +var groupable = (array) => { + array.groupBy = (keyFunction, valueFunction) => { + const groups = {}; + for (const element of array) { + const key = keyFunction(element); + groups[key] ||= []; + groups[key].push( + valueFunction ? valueFunction(element) : element + ); + } + return groups; + } + array.groupByMultiple = (keyFunction, valueFunction) => { + const groups = {}; + for (const element of array) { + const keys = keyFunction(element); + console.log(`keys`, keys); + for (const key of keys) { + groups[key] ||= new Set(); + groups[key].add( + valueFunction ? valueFunction(element) : element + ); } } - ).toArray(); -*/ - - res.status(200).json(risks); + for (const group in groups) { + groups[group] = Array.from(groups[group]); + } + return groups; + } + return array; } From c915ceb074076208284fcf07a9c326584cba98af Mon Sep 17 00:00:00 2001 From: Luna McNulty Date: Thu, 11 May 2023 11:51:29 -0400 Subject: [PATCH 04/67] wip --- site/gatsby-site/src/pages/apps/checklists.js | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 site/gatsby-site/src/pages/apps/checklists.js diff --git a/site/gatsby-site/src/pages/apps/checklists.js b/site/gatsby-site/src/pages/apps/checklists.js new file mode 100644 index 0000000000..6039c717c1 --- /dev/null +++ b/site/gatsby-site/src/pages/apps/checklists.js @@ -0,0 +1,76 @@ +import React, { useState } from 'react'; +import Layout from 'components/Layout'; +import { useTranslation } from 'react-i18next'; +import { Button, TextInput, Textarea } from 'flowbite-react'; +import { debounce } from 'debounce'; +import { + useQueryParams, + StringParam, + createEnumParam, + withDefault, + NumberParam, + BooleanParam +} from 'use-query-params'; + +import AiidHelmet from 'components/AiidHelmet'; +import Tags from 'components/forms/Tags.js'; +import { Formik } from 'formik'; + +export default function ChecklistsPage(props) { + const { + location: { pathname }, + } = props; + + const [query, setQuery] = useQueryParams({ + id: StringParam, + }); + + const { t } = useTranslation(); + + const updateQuery = () => {}; + + return ( + + + {t('Risk Checklists')} + + {query.id ? ( + + {({ values, handleChange, handleSubmit, setFieldTouched, setFieldValue, isSubmitting }) => ( + <> +
+

Risk Checklist for ""

+ + +