From 42167636cf42f85da02474a176b33ea4e7cafb21 Mon Sep 17 00:00:00 2001 From: MaKyOtOx Date: Tue, 14 Mar 2023 16:28:03 +0100 Subject: [PATCH 1/3] fix #166 APIvoid error --- VERSION | 2 +- engines/apivoid/Dockerfile | 2 +- engines/apivoid/README.md | 4 +- engines/apivoid/VERSION | 2 +- engines/apivoid/apivoid.json.sample | 4 +- engines/apivoid/engine-apivoid.py | 123 ++++++++++++++++++++++------ 6 files changed, 105 insertions(+), 32 deletions(-) diff --git a/VERSION b/VERSION index f0ed3796..df808199 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.5.10 +1.5.11 diff --git a/engines/apivoid/Dockerfile b/engines/apivoid/Dockerfile index fd3e346d..91bba8eb 100644 --- a/engines/apivoid/Dockerfile +++ b/engines/apivoid/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:3.16.3 -LABEL Name="APIVoid\ \(Patrowl engine\)" Version="1.4.31" +LABEL Name="APIVoid\ \(Patrowl engine\)" Version="1.4.32" # Create the target repo RUN mkdir -p /opt/patrowl-engines/apivoid diff --git a/engines/apivoid/README.md b/engines/apivoid/README.md index 73dd38ae..14af75f4 100755 --- a/engines/apivoid/README.md +++ b/engines/apivoid/README.md @@ -19,8 +19,8 @@ set your APIKey in APIVOID_APIKEY environment variable ## Start with Docker ``` -docker build . -t engine-apivoid -docker run -p5022:5022 -e APIVOID_APIKEY=XXXXX engine-apivoid +docker build . -t patrowl/engine-apivoid +docker run -p5022:5022 -e APIVOID_APIKEY=XXXXX patrowl/engine-apivoid ``` ## Testing URLs diff --git a/engines/apivoid/VERSION b/engines/apivoid/VERSION index d156665a..00bbe72a 100644 --- a/engines/apivoid/VERSION +++ b/engines/apivoid/VERSION @@ -1 +1 @@ -1.4.31 +1.4.32 diff --git a/engines/apivoid/apivoid.json.sample b/engines/apivoid/apivoid.json.sample index ab990a0e..b7a7b4c7 100755 --- a/engines/apivoid/apivoid.json.sample +++ b/engines/apivoid/apivoid.json.sample @@ -1,8 +1,8 @@ { "name": "APIVOID", - "version": "1.4.28", + "version": "1.4.32", "description": "APIVoid reputation API", - "allowed_asset_types": ["domain", "ip", "url", "ip-subnet"], + "allowed_asset_types": ["domain", "fqdn", "ip", "url", "ip-subnet"], "apikeys": [ "xx", "yy" diff --git a/engines/apivoid/engine-apivoid.py b/engines/apivoid/engine-apivoid.py index 1bb06bc7..dff35a53 100755 --- a/engines/apivoid/engine-apivoid.py +++ b/engines/apivoid/engine-apivoid.py @@ -7,12 +7,13 @@ import json import time import requests +import datetime import re from urllib.parse import urlparse -from flask import Flask, request, jsonify +from flask import Flask, request, jsonify, send_from_directory from concurrent.futures import ThreadPoolExecutor from ratelimit import limits, sleep_and_retry -from netaddr import IPNetwork +from netaddr import IPNetwork, IPAddress from netaddr.core import AddrFormatError from PatrowlEnginesUtils.PatrowlEngine import PatrowlEngine @@ -25,7 +26,7 @@ APP_MAXSCANS = int(os.environ.get('APP_MAXSCANS', 25)) APP_ENGINE_NAME = "apivoid" APP_BASE_DIR = os.path.dirname(os.path.realpath(__file__)) -VERSION = "1.4.28" +VERSION = "1.4.32" engine = PatrowlEngine( app=app, @@ -173,8 +174,14 @@ def stop_scan(scan_id): @app.route('/engines/apivoid/getreport/') def getreport(scan_id): - """Get report on finished scans.""" - return engine.getreport(scan_id) + if not scan_id.isdecimal(): + return jsonify({"status": "error", "reason": "scan_id must be numeric digits only"}) + filepath = f"{APP_BASE_DIR}/results/apivoid_{scan_id}.json" + + if not os.path.exists(filepath): + return jsonify({"status": "error", "reason": f"report file for scan_id '{scan_id}' not found"}) + + return send_from_directory(f"{APP_BASE_DIR}/results/", "apivoid_{scan_id}.json") def _loadconfig(): @@ -213,7 +220,7 @@ def start_scan(): if len(engine.scans) == APP_MAXSCANS: res.update({ "status": "error", - "reason": "Scan refused: max concurrent active scans reached ({})".format(APP_MAXSCANS) + "reason": f"Scan refused: max concurrent active scans reached ({APP_MAXSCANS})" }) return jsonify(res) @@ -223,7 +230,7 @@ def start_scan(): res.update({ "status": "refused", "details": { - "reason": "bad scanner status {}".format(engine.scanner['status']) + "reason": f"Bad scanner status {engine.scanner['status']}" }}) return jsonify(res) @@ -259,7 +266,7 @@ def start_scan(): if asset["datatype"] not in engine.scanner["allowed_asset_types"]: res.update({ "status": "error", - "reason": "asset '{}' datatype '{}' not supported".format(asset["value"],asset["datatype"]) + "reason": "asset '{}' has unsupported datatype '{}'".format(asset["value"], asset["datatype"]) }) return jsonify(res) @@ -271,6 +278,12 @@ def start_scan(): if asset["datatype"] == "url": parsed_uri = urlparse(asset["value"]) asset["value"] = parsed_uri.netloc + + # Check the netloc type + if is_valid_ip(asset["value"]): + asset["datatype"] == "ip" + else: + asset["datatype"] == "domain" assets.append(asset["value"]) @@ -280,7 +293,7 @@ def start_scan(): res.update({ "status": "refused", "details": { - "reason": "scan '{}' already launched".format(data['scan_id']), + "reason": f"scan '{data['scan_id']}' already launched", } }) return jsonify(res) @@ -310,7 +323,7 @@ def start_scan(): if 'domain_reputation' in scan['options'].keys() and data['options']['domain_reputation']: for asset in data["assets"]: - if asset["datatype"] == "domain": + if asset["datatype"] in ["domain", "fqdn"]: th = this.pool.submit(_scan_domain_reputation, scan_id, asset["value"]) engine.scans[scan_id]['futures'].append(th) @@ -330,7 +343,7 @@ def _scan_ip_reputation(scan_id, asset): try: engine.scans[scan_id]["findings"][asset]['ip_reputation'] = get_report_ip_reputation(scan_id, asset, apikey) except Exception as ex: - app.logger.error("_scan_ip_reputation failed {}".format(re.sub(r'/' + apikey + '/', r'/***/', ex.__str__()))) + app.logger.error("_scan_ip_reputation failed: {}".format(re.sub(r'/' + apikey + '/', r'/***/', ex.__str__()))) return False return True @@ -343,7 +356,7 @@ def _scan_domain_reputation(scan_id, asset): try: engine.scans[scan_id]["findings"][asset]['domain_reputation'] = get_report_domain_reputation(scan_id, asset, apikey) except Exception as ex: - app.logger.error("_scan_domain_reputation failed {}".format(re.sub(r'/' + apikey + '/', r'/***/', ex.__str__()))) + app.logger.error("_scan_domain_reputation failed: {}".format(re.sub(r'/' + apikey + '/', r'/***/', ex.__str__()))) return False return True @@ -365,7 +378,7 @@ def get_report_ip_reputation(scan_id, asset, apikey): response = requests.get(scan_url) # print(response.content) except Exception as ex: - app.logger.error("get_report_ip_reputation failed {}".format(re.sub(r'/' + apikey + '/', r'/***/', ex.__str__()))) + app.logger.error("get_report_ip_reputation failed: {}".format(re.sub(r'/' + apikey + '/', r'/***/', ex.__str__()))) return [] return response.content @@ -380,7 +393,7 @@ def get_report_domain_reputation(scan_id, asset, apikey): response = requests.get(scan_url) # print(response.content) except Exception as ex: - app.logger.error("get_report_domain_reputation failed {}".format(re.sub(r'/' + apikey + '/', r'/***/', ex.__str__()))) + app.logger.error("get_report_domain_reputation failed: {}".format(re.sub(r'/' + apikey + '/', r'/***/', ex.__str__()))) return [] return response.content @@ -400,18 +413,34 @@ def _parse_results(scan_id): ts = int(time.time() * 1000) for asset in engine.scans[scan_id]["findings"]: + if 'ip_reputation' in engine.scans[scan_id]["findings"][asset].keys(): res = json.loads(engine.scans[scan_id]["findings"][asset]['ip_reputation']) + if 'data' in res: + severity = "info" + report_summary = "" + try: + detections = res["data"]["report"]["blacklists"]["detections"] + risk_score = res["data"]["report"]["risk_score"]["result"] + if risk_score == 100: + severity = "high" + elif risk_score >= 70: + severity = "medium" + + report_summary = f" (detect:{detections}, risk:{risk_score})" + except Exception: + pass + nb_vulns['info'] += 1 issues.append({ "issue_id": len(issues) + 1, - "severity": "info", "confidence": "certain", + "severity": severity, "confidence": "certain", "target": { "addr": [asset], "protocol": "domain" }, - "title": "IP Reputation Check", + "title": "IP Reputation Check"+report_summary, "description": f"IP Reputation Check for '{asset}'\n\nSee raw_data", "solution": "n/a", "metadata": { @@ -421,18 +450,34 @@ def _parse_results(scan_id): "raw": res['data'], "timestamp": ts }) + if 'domain_reputation' in engine.scans[scan_id]["findings"][asset].keys(): res = json.loads(engine.scans[scan_id]["findings"][asset]['domain_reputation']) + if 'data' in res: + severity = "info" + report_summary = "" + try: + detections = res["data"]["report"]["blacklists"]["detections"] + risk_score = res["data"]["report"]["risk_score"]["result"] + if risk_score == 100: + severity = "high" + elif risk_score >= 70: + severity = "medium" + + report_summary = f" (detect:{detections}, risk:{risk_score})" + except Exception: + pass + nb_vulns['info'] += 1 issues.append({ "issue_id": len(issues) + 1, - "severity": "info", "confidence": "certain", + "severity": severity, "confidence": "certain", "target": { "addr": [asset], "protocol": "domain" }, - "title": "Domain Reputation Check", + "title": "Domain Reputation Check"+report_summary, "description": f"Domain Reputation Check for '{asset}'\n\nSee raw_data", "solution": "n/a", "metadata": { @@ -469,8 +514,10 @@ def getfindings(scan_id): # check if the scan is finished status_scan(scan_id) if engine.scans[scan_id]['status'] != "FINISHED": - res.update({"status": "error", - "reason": f"scan_id '{scan_id}' not finished (status={engine.scans[scan_id]['status']})"}) + res.update({ + "status": "error", + "reason": f"scan_id '{scan_id}' not finished (status={engine.scans[scan_id]['status']})" + }) return jsonify(res) status, issues, summary = _parse_results(scan_id) @@ -484,15 +531,30 @@ def getfindings(scan_id): } scan.update(status) - - # remove the scan from the active scan list - clean_scan(scan_id) - - res.update({"scan": scan, "summary": summary, "issues": issues}) + + res_data = {"scan": scan, "summary": summary, "issues": issues} + + # Store the findings in a file + with open(f"{APP_BASE_DIR}/results/apivoid_{scan_id}.json", 'w') as report_file: + json.dump(res_data, report_file, default=_json_serial) + + # # Remove the scan from the active scan list + # clean_scan(scan_id) + + # Prepare response + res.update(res_data) res.update(status) return jsonify(res) +def is_valid_ip(ip): + try: + IPAddress(ip) + except (TypeError, ValueError, AddrFormatError): + return False + return True + + def is_valid_subnet(subnet): try: IPNetwork(subnet) @@ -509,6 +571,17 @@ def get_ips_from_subnet(subnet): return [str(ip) for ip in IPNetwork(subnet)] +def _json_serial(obj): + """ + JSON serializer for objects not serializable by default json code + Used for datetime serialization when the results are written in file + """ + if isinstance(obj, datetime.datetime) or isinstance(obj, datetime.date): + serial = obj.isoformat() + return serial + raise TypeError("Type not serializable") + + @app.before_first_request def main(): """First function called.""" From f501fc4f2ed87752202370800d43254ff0d3693f Mon Sep 17 00:00:00 2001 From: MaKyOtOx Date: Tue, 14 Mar 2023 16:42:56 +0100 Subject: [PATCH 2/3] update version --- engines/apivoid/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engines/apivoid/__init__.py b/engines/apivoid/__init__.py index e40804ce..dc091835 100644 --- a/engines/apivoid/__init__.py +++ b/engines/apivoid/__init__.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- __title__ = 'patrowl_engine_apivoid' -__version__ = '1.4.27' +__version__ = '1.4.32' __author__ = 'Nicolas MATTIOCCO' __license__ = 'AGPLv3' -__copyright__ = 'Copyright (C) 2020-2022 Nicolas Mattiocco - @MaKyOtOx' +__copyright__ = 'Copyright (C) 2020-2023 Nicolas Mattiocco - @MaKyOtOx' From 9db8e0ec943b589116a0e92d6a8933a5bd2c5249 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20COLL?= Date: Mon, 17 Apr 2023 14:56:22 +0200 Subject: [PATCH 3/3] 1.4.32-rc1 apivoid --- engines/apivoid/Dockerfile | 2 +- engines/apivoid/VERSION | 2 +- engines/apivoid/apivoid.json.sample | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/engines/apivoid/Dockerfile b/engines/apivoid/Dockerfile index 91bba8eb..db5ccf2d 100644 --- a/engines/apivoid/Dockerfile +++ b/engines/apivoid/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:3.16.3 -LABEL Name="APIVoid\ \(Patrowl engine\)" Version="1.4.32" +LABEL Name="APIVoid\ \(Patrowl engine\)" Version="1.4.32-rc1" # Create the target repo RUN mkdir -p /opt/patrowl-engines/apivoid diff --git a/engines/apivoid/VERSION b/engines/apivoid/VERSION index 00bbe72a..26affb5d 100644 --- a/engines/apivoid/VERSION +++ b/engines/apivoid/VERSION @@ -1 +1 @@ -1.4.32 +1.4.32-rc1 diff --git a/engines/apivoid/apivoid.json.sample b/engines/apivoid/apivoid.json.sample index b7a7b4c7..2c42cd29 100755 --- a/engines/apivoid/apivoid.json.sample +++ b/engines/apivoid/apivoid.json.sample @@ -1,6 +1,6 @@ { "name": "APIVOID", - "version": "1.4.32", + "version": "1.4.32-rc1", "description": "APIVoid reputation API", "allowed_asset_types": ["domain", "fqdn", "ip", "url", "ip-subnet"], "apikeys": [