Split ga response (#434)

* add time metrics * map analysis through redis * move hash to utils and make db method write to redis optionally * lint * Added: Front end support for weaker links calls * Added: DB seperation for smaller inital response and cache extras * Added: DB side of part split response logic * Refactor and tests --------- Co-authored-by: Spyros <[email protected]>
OWASP · Oct 23, 2023 · efa968c · efa968c
1 parent f508bca
commit efa968c
Show file tree

Hide file tree

Showing 8 changed files with 491 additions and 79 deletions.
diff --git a/.gitignore b/.gitignore
@@ -34,4 +34,5 @@ coverage/
 standards_cache.sqlite
 
 ### Neo4j
-neo4j/
+neo4j/
+.neo4j/
diff --git a/application/database/db.py b/application/database/db.py
@@ -494,15 +494,12 @@ def format_path_record(rec):
 
     @classmethod
     def standards(self) -> List[str]:
-        tools = []
+        results = []
         for x in db.cypher_query("""MATCH (n:NeoTool) RETURN DISTINCT n.name""")[0]:
-            tools.extend(x)
-        standards = []
-        for x in db.cypher_query("""MATCH (n:NeoStandard) RETURN DISTINCT n.name""")[
-            0
-        ]:  # 0 is the results, 1 is the "n.name" param
-            standards.extend(x)
-        return list(set([x for x in tools] + [x for x in standards]))
+            results.extend(x)
+        for x in db.cypher_query("""MATCH (n:NeoStandard) RETURN DISTINCT n.name""")[0]:
+            results.extend(x)
+        return list(set(results))
 
     @staticmethod
     def parse_node(node: NeoDocument) -> cre_defs.Document:
@@ -1774,21 +1771,38 @@ def gap_analysis(
     if base_standard is None:
         return None
     grouped_paths = {}
+    extra_paths_dict = {}
+    GA_STRONG_UPPER_LIMIT = 2
+
     for node in base_standard:
         key = node.id
         if key not in grouped_paths:
-            grouped_paths[key] = {"start": node, "paths": {}}
+            grouped_paths[key] = {"start": node, "paths": {}, "extra": 0}
+            extra_paths_dict[key] = {"paths": {}}
 
     for path in paths:
         key = path["start"].id
         end_key = path["end"].id
         path["score"] = get_path_score(path)
         del path["start"]
-        if end_key in grouped_paths[key]["paths"]:
-            if grouped_paths[key]["paths"][end_key]["score"] > path["score"]:
+        if path["score"] <= GA_STRONG_UPPER_LIMIT:
+            if end_key in extra_paths_dict[key]["paths"]:
+                del extra_paths_dict[key]["paths"][end_key]
+                grouped_paths[key]["extra"] -= 1
+            if end_key in grouped_paths[key]["paths"]:
+                if grouped_paths[key]["paths"][end_key]["score"] > path["score"]:
+                    grouped_paths[key]["paths"][end_key] = path
+            else:
                 grouped_paths[key]["paths"][end_key] = path
         else:
-            grouped_paths[key]["paths"][end_key] = path
+            if end_key in grouped_paths[key]["paths"]:
+                continue
+            if end_key in extra_paths_dict[key]:
+                if extra_paths_dict[key]["paths"][end_key]["score"] > path["score"]:
+                    extra_paths_dict[key]["paths"][end_key] = path
+            else:
+                extra_paths_dict[key]["paths"][end_key] = path
+                grouped_paths[key]["extra"] += 1
 
     if (
         store_in_cache
@@ -1799,6 +1813,11 @@ def gap_analysis(
             cache_key = make_array_hash(node_names)
 
         conn.set(cache_key, flask_json.dumps({"result": grouped_paths}))
-        return (node_names, {})
+        for key in extra_paths_dict:
+            conn.set(
+                cache_key + "->" + key,
+                flask_json.dumps({"result": extra_paths_dict[key]}),
+            )
+        return (node_names, {}, {})
 
-    return (node_names, grouped_paths)
+    return (node_names, grouped_paths, extra_paths_dict)
diff --git a/application/frontend/src/const.ts b/application/frontend/src/const.ts
@@ -37,3 +37,5 @@ export const GRAPH = '/graph';
 export const DEEPLINK = '/deeplink';
 export const BROWSEROOT = '/root_cres';
 export const GAP_ANALYSIS = '/map_analysis';
+
+export const GA_STRONG_UPPER_LIMIT = 2; // remember to change this in the Python code too
diff --git a/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx b/application/frontend/src/pages/GapAnalysis/GapAnalysis.tsx
@@ -1,20 +1,10 @@
 import axios from 'axios';
-import React, { useEffect, useRef, useState } from 'react';
+import React, { useCallback, useEffect, useRef, useState } from 'react';
 import { useLocation } from 'react-router-dom';
-import {
-  Accordion,
-  Button,
-  Container,
-  Dropdown,
-  DropdownItemProps,
-  Grid,
-  Icon,
-  Label,
-  Popup,
-  Table,
-} from 'semantic-ui-react';
+import { Button, Dropdown, DropdownItemProps, Icon, Popup, Table } from 'semantic-ui-react';
 
 import { LoadingAndErrorIndicator } from '../../components/LoadingAndErrorIndicator';
+import { GA_STRONG_UPPER_LIMIT } from '../../const';
 import { useEnvironment } from '../../hooks';
 import { GapAnalysisPathStart } from '../../types';
 import { getDocumentDisplayName } from '../../utils';
@@ -51,14 +41,14 @@ function useQuery() {
 
 const GetStrength = (score) => {
   if (score == 0) return 'Direct';
-  if (score <= 2) return 'Strong';
+  if (score <= GA_STRONG_UPPER_LIMIT) return 'Strong';
   if (score >= 20) return 'Weak';
   return 'Average';
 };
 
 const GetStrengthColor = (score) => {
   if (score === 0) return 'darkgreen';
-  if (score <= 2) return '#93C54B';
+  if (score <= GA_STRONG_UPPER_LIMIT) return '#93C54B';
   if (score >= 20) return 'Red';
   return 'Orange';
 };
@@ -100,8 +90,10 @@ const GetResultLine = (path, gapAnalysis, key) => {
             <br />
             <b style={{ color: GetStrengthColor(0) }}>{GetStrength(0)}</b>: Directly Linked
             <br />
-            <b style={{ color: GetStrengthColor(2) }}>{GetStrength(2)}</b>: Closely connected likely to have
-            majority overlap
+            <b style={{ color: GetStrengthColor(GA_STRONG_UPPER_LIMIT) }}>
+              {GetStrength(GA_STRONG_UPPER_LIMIT)}
+            </b>
+            : Closely connected likely to have majority overlap
             <br />
             <b style={{ color: GetStrengthColor(6) }}>{GetStrength(6)}</b>: Connected likely to have partial
             overlap
@@ -127,21 +119,12 @@ export const GapAnalysis = () => {
   );
   const [gaJob, setgaJob] = useState<string>('');
   const [gapAnalysis, setGapAnalysis] = useState<Record<string, GapAnalysisPathStart>>();
-  const [activeIndex, SetActiveIndex] = useState<string>();
   const [loadingStandards, setLoadingStandards] = useState<boolean>(false);
   const [loadingGA, setLoadingGA] = useState<boolean>(false);
   const [error, setError] = useState<string | null | object>(null);
   const { apiUrl } = useEnvironment();
   const timerIdRef = useRef<NodeJS.Timer>();
 
-  const GetStrongPathsCount = (paths) =>
-    Math.max(
-      Object.values<any>(paths).filter(
-        (x) => GetStrength(x.score) === 'Strong' || GetStrength(x.score) === 'Direct'
-      ).length,
-      3
-    );
-
   useEffect(() => {
     const fetchData = async () => {
       const result = await axios.get(`${apiUrl}/standards`);
@@ -225,11 +208,20 @@ export const GapAnalysis = () => {
     });
   }, [BaseStandard, CompareStandard, setGapAnalysis, setLoadingGA, setError]);
 
-  const handleAccordionClick = (e, titleProps) => {
-    const { index } = titleProps;
-    const newIndex = activeIndex === index ? -1 : index;
-    SetActiveIndex(newIndex);
-  };
+  const getWeakLinks = useCallback(
+    async (key) => {
+      if (!gapAnalysis) return;
+      const result = await axios.get(
+        `${apiUrl}/map_analysis_weak_links?standard=${BaseStandard}&standard=${CompareStandard}&key=${key}`
+      );
+      if (result.data.result) {
+        gapAnalysis[key].weakLinks = result.data.result.paths;
+        setGapAnalysis(undefined); //THIS HAS TO BE THE WRONG WAY OF DOING THIS
+        setGapAnalysis(gapAnalysis);
+      }
+    },
+    [gapAnalysis, setGapAnalysis]
+  );
 
   return (
     <div style={{ margin: '0 auto', maxWidth: '95vw' }}>
@@ -299,29 +291,19 @@ export const GapAnalysis = () => {
                     <Table.Cell style={{ minWidth: '35vw' }}>
                       {Object.values<any>(gapAnalysis[key].paths)
                         .sort((a, b) => a.score - b.score)
-                        .slice(0, GetStrongPathsCount(gapAnalysis[key].paths))
                         .map((path) => GetResultLine(path, gapAnalysis, key))}
-                      {Object.keys(gapAnalysis[key].paths).length > 3 && (
-                        <Accordion>
-                          <Accordion.Title
-                            active={activeIndex === key}
-                            index={key}
-                            onClick={handleAccordionClick}
-                          >
-                            <Button>More Links (Total: {Object.keys(gapAnalysis[key].paths).length})</Button>
-                          </Accordion.Title>
-                          <Accordion.Content active={activeIndex === key}>
-                            {Object.values<any>(gapAnalysis[key].paths)
-                              .sort((a, b) => a.score - b.score)
-                              .slice(
-                                GetStrongPathsCount(gapAnalysis[key].paths),
-                                Object.keys(gapAnalysis[key].paths).length
-                              )
-                              .map((path) => GetResultLine(path, gapAnalysis, key))}
-                          </Accordion.Content>
-                        </Accordion>
+                      {gapAnalysis[key].weakLinks &&
+                        Object.values<any>(gapAnalysis[key].weakLinks)
+                          .sort((a, b) => a.score - b.score)
+                          .map((path) => GetResultLine(path, gapAnalysis, key))}
+                      {gapAnalysis[key].extra > 0 && !gapAnalysis[key].weakLinks && (
+                        <Button onClick={async () => await getWeakLinks(key)}>
+                          See Weak Links ({gapAnalysis[key].extra})
+                        </Button>
+                      )}
+                      {Object.keys(gapAnalysis[key].paths).length === 0 && gapAnalysis[key].extra === 0 && (
+                        <i>No links Found</i>
                       )}
-                      {Object.keys(gapAnalysis[key].paths).length === 0 && <i>No links Found</i>}
                     </Table.Cell>
                   </Table.Row>
                 ))}

diff --git a/application/frontend/src/types.ts b/application/frontend/src/types.ts
@@ -35,4 +35,6 @@ interface GapAnalysisPath {
 export interface GapAnalysisPathStart {
   start: Document;
   paths: Record<string, GapAnalysisPath>;
+  extra: number;
+  weakLinks: Record<string, GapAnalysisPath>;
 }
-Original file line number
+Diff line change
@@ Expand Up / @@ -34,4 +34,5 @@ coverage/ @@
     standards_cache.sqlite
     ### Neo4j
-    neo4j/
+    neo4j/
+    .neo4j/