Precompute map analysis

.github/workflows/e2e.yml

@@ -7,6 +7,13 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v2
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11.4'
+          cache: 'pip'
+      - uses: actions/setup-node@v3
+        with:
+          cache: 'yarn'
       - name: Install python dependencies
         run: sudo apt-get update && sudo apt-get install -y python3-setuptools python3-pip chromium-browser libgbm1 && make install-deps 
       - name: Test-e2e

.github/workflows/test.yml

@@ -7,9 +7,14 @@ jobs:
     steps:
       - name: Check out code
         uses: actions/checkout@v2
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11.4' 
+          cache: 'pip'
+      - uses: actions/setup-node@v3
+        with:
+          cache: 'yarn'
       - name: Install python dependencies
         run: sudo apt-get update && sudo apt-get install -y python3-setuptools python3-pip && make install-deps
       - name: Test
         run: make test
-      - name: Test-e2e
-        run: make e2e

.gitignore

@@ -16,6 +16,7 @@ Vagrantfile
 
 ## act secrets
 .secrets/
+.env
 
 ### conventions ###
 venv/
@@ -30,4 +31,7 @@ yarn-error.log
 coverage/
 
 ### Dev db
-standards_cache.sqlite
+standards_cache.sqlite
+
+### Neo4j
+neo4j/

Makefile

@@ -1,23 +1,38 @@
 
 .ONESHELL:
 
-.PHONY: dev-run run test covers install-deps dev docker lint frontend clean all
+.PHONY: run test covers install-deps dev docker lint frontend clean all
 
 prod-run:
 	cp cres/db.sqlite standards_cache.sqlite; gunicorn cre:app --log-file=-
 
-dev-run:
-	. ./venv/bin/activate && FLASK_APP=cre.py FLASK_CONFIG=development flask run
+docker-neo4j:
+	docker start cre-neo4j 2>/dev/null   || docker run -d --name cre-neo4j --env NEO4J_PLUGINS='["apoc"]'  --env NEO4J_AUTH=neo4j/password --volume=`pwd`/.neo4j/data:/data --volume=`pwd`/.neo4j/logs:/logs --workdir=/var/lib/neo4j -p 7474:7474 -p 7687:7687 neo4j
+
+docker-redis:
+	docker start redis-stack 2>/dev/null || docker run -d --name redis-stack -p 6379:6379 -p 8001:8001 redis/redis-stack:latest
+
+start-containers: docker-neo4j docker-redis
+
+start-worker:
+	. ./venv/bin/activate
+	FLASK_APP=`pwd`/cre.py python cre.py --start_worker
+
+dev-flask:
+	. ./venv/bin/activate
+	FLASK_APP=`pwd`/cre.py  FLASK_CONFIG=development flask run
+
 e2e:
 	yarn build
 	[ -d "./venv" ] && . ./venv/bin/activate
 	export FLASK_APP=$(CURDIR)/cre.py
 	export FLASK_CONFIG=development
-	fFLASK_CONFIG=development flask run&
-	
+	flask run&
+
 	yarn test:e2e
 	killall yarn
 	killall flask
+
 test:
 	[ -d "./venv" ] && . ./venv/bin/activate
 	export FLASK_APP=$(CURDIR)/cre.py
@@ -79,4 +94,8 @@ import-all:
 	[ -d "./venv" ] && . ./venv/bin/activate
 	rm -rf standards_cache.sqlite && make migrate-upgrade && export FLASK_APP=$(CURDIR)/cre.py && python cre.py --add --from_spreadsheet https://docs.google.com/spreadsheets/d/1eZOEYgts7d_-Dr-1oAbogPfzBLh6511b58pX3b59kvg && python cre.py --generate_embeddings && python cre.py --zap_in --cheatsheets_in --github_tools_in  --capec_in --owasp_secure_headers_in --pci_dss_4_in --juiceshop_in &&	python cre.py --generate_embeddings
 
+import-neo4j:
+	[ -d "./venv" ] && . ./venv/bin/activate
+	export FLASK_APP=$(CURDIR)/cre.py && python cre.py --populate_neo4j_db
+
 all: clean lint test dev dev-run

Procfile

@@ -1 +1,2 @@
-web: gunicorn cre:app --log-file=-
+web: gunicorn cre:app --log-file=-g
+worker: FLASK_APP=`pwd`/cre.py python cre.py --start_worker

README.md

@@ -60,11 +60,22 @@ To add a remote spreadsheet to your local database you can run
 <pre>python cre.py --add --from_spreadsheet < google sheets url></pre>
 
 To run the web application for development you can run
-<pre>make dev-run</pre>
+<pre>
+$ make start-containers
+$ make start-worker 
+
+# in a seperate shell
+$ make dev-flask
+</pre>
 
 Alternatively, you can use the dockerfile with
 <pre>make docker && make docker-run</pre>
 
+Some features like Gap Analysis require a neo4j DB running, you can start this with
+<pre>make docker-neo4j</pre>
+enviroment varaibles for app to connect to neo4jDB (default):
+- NEO4J_URL (neo4j//neo4j:password@localhost:7687)
+
 To run the web application for production you need gunicorn and you can run from within the cre_sync dir
 <pre>make prod-run</pre>
 
@@ -84,4 +95,4 @@ Please see [Contributing](CONTRIBUTING.md) for contributing instructions
 
 Roadmap
 ---
-For a roadmap of what we would like to be done please see the [issues](https://github.com/OWASP/common-requirement-enumeration/issues).
+For a roadmap of what we would like to be done please see the [issues](https://github.com/OWASP/common-requirement-enumeration/issues).

application/cmd/cre_main.py

@@ -17,7 +17,6 @@
 from application.utils.external_project_parsers import (
     capec_parser,
     cwe,
-    ccmv3,
     ccmv4,
     cheatsheets_parser,
     misc_tools_parser,
@@ -375,14 +374,6 @@ def run(args: argparse.Namespace) -> None:  # pragma: no cover
     if args.export:
         cache = db_connect(args.cache_file)
         cache.export(args.export)
-    if args.csa_ccm_v3_in:
-        ccmv3.parse_ccm(
-            ccmFile=sheet_utils.readSpreadsheet(
-                alias="",
-                url="https://docs.google.com/spreadsheets/d/1b5i8OV919aiqW2KcYWOQvkLorL1bRPqjthJxLH0QpD8",
-            ),
-            cache=db_connect(args.cache_file),
-        )
     if args.csa_ccm_v4_in:
         ccmv4.parse_ccm(
             ccmFile=sheet_utils.readSpreadsheet(
@@ -426,6 +417,12 @@ def run(args: argparse.Namespace) -> None:  # pragma: no cover
         generate_embeddings(args.cache_file)
     if args.owasp_proj_meta:
         owasp_metadata_to_cre(args.owasp_proj_meta)
+    if args.populate_neo4j_db:
+        populate_neo4j_db(args.cache_file)
+    if args.start_worker:
+        from application.worker import start_worker
+
+        start_worker(args.cache_file)
 
 
 def db_connect(path: str):
@@ -530,3 +527,11 @@ def owasp_metadata_to_cre(meta_file: str):
     },
     """
     raise NotImplementedError("someone needs to work on this")
+
+
+def populate_neo4j_db(cache: str):
+    logger.info(f"Populating neo4j DB: Connecting to SQL DB")
+    database = db_connect(path=cache)
+    logger.info(f"Populating neo4j DB: Populating")
+    database.neo_db.populate_DB(database.session)
+    logger.info(f"Populating neo4j DB: Complete")