From 42167636cf42f85da02474a176b33ea4e7cafb21 Mon Sep 17 00:00:00 2001 From: MaKyOtOx Date: Tue, 14 Mar 2023 16:28:03 +0100 Subject: [PATCH 1/9] fix #166 APIvoid error --- VERSION | 2 +- engines/apivoid/Dockerfile | 2 +- engines/apivoid/README.md | 4 +- engines/apivoid/VERSION | 2 +- engines/apivoid/apivoid.json.sample | 4 +- engines/apivoid/engine-apivoid.py | 123 ++++++++++++++++++++++------ 6 files changed, 105 insertions(+), 32 deletions(-) diff --git a/VERSION b/VERSION index f0ed3796..df808199 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.5.10 +1.5.11 diff --git a/engines/apivoid/Dockerfile b/engines/apivoid/Dockerfile index fd3e346d..91bba8eb 100644 --- a/engines/apivoid/Dockerfile +++ b/engines/apivoid/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:3.16.3 -LABEL Name="APIVoid\ \(Patrowl engine\)" Version="1.4.31" +LABEL Name="APIVoid\ \(Patrowl engine\)" Version="1.4.32" # Create the target repo RUN mkdir -p /opt/patrowl-engines/apivoid diff --git a/engines/apivoid/README.md b/engines/apivoid/README.md index 73dd38ae..14af75f4 100755 --- a/engines/apivoid/README.md +++ b/engines/apivoid/README.md @@ -19,8 +19,8 @@ set your APIKey in APIVOID_APIKEY environment variable ## Start with Docker ``` -docker build . -t engine-apivoid -docker run -p5022:5022 -e APIVOID_APIKEY=XXXXX engine-apivoid +docker build . -t patrowl/engine-apivoid +docker run -p5022:5022 -e APIVOID_APIKEY=XXXXX patrowl/engine-apivoid ``` ## Testing URLs diff --git a/engines/apivoid/VERSION b/engines/apivoid/VERSION index d156665a..00bbe72a 100644 --- a/engines/apivoid/VERSION +++ b/engines/apivoid/VERSION @@ -1 +1 @@ -1.4.31 +1.4.32 diff --git a/engines/apivoid/apivoid.json.sample b/engines/apivoid/apivoid.json.sample index ab990a0e..b7a7b4c7 100755 --- a/engines/apivoid/apivoid.json.sample +++ b/engines/apivoid/apivoid.json.sample @@ -1,8 +1,8 @@ { "name": "APIVOID", - "version": "1.4.28", + "version": "1.4.32", "description": "APIVoid reputation API", - "allowed_asset_types": ["domain", "ip", "url", "ip-subnet"], + "allowed_asset_types": ["domain", "fqdn", "ip", "url", "ip-subnet"], "apikeys": [ "xx", "yy" diff --git a/engines/apivoid/engine-apivoid.py b/engines/apivoid/engine-apivoid.py index 1bb06bc7..dff35a53 100755 --- a/engines/apivoid/engine-apivoid.py +++ b/engines/apivoid/engine-apivoid.py @@ -7,12 +7,13 @@ import json import time import requests +import datetime import re from urllib.parse import urlparse -from flask import Flask, request, jsonify +from flask import Flask, request, jsonify, send_from_directory from concurrent.futures import ThreadPoolExecutor from ratelimit import limits, sleep_and_retry -from netaddr import IPNetwork +from netaddr import IPNetwork, IPAddress from netaddr.core import AddrFormatError from PatrowlEnginesUtils.PatrowlEngine import PatrowlEngine @@ -25,7 +26,7 @@ APP_MAXSCANS = int(os.environ.get('APP_MAXSCANS', 25)) APP_ENGINE_NAME = "apivoid" APP_BASE_DIR = os.path.dirname(os.path.realpath(__file__)) -VERSION = "1.4.28" +VERSION = "1.4.32" engine = PatrowlEngine( app=app, @@ -173,8 +174,14 @@ def stop_scan(scan_id): @app.route('/engines/apivoid/getreport/') def getreport(scan_id): - """Get report on finished scans.""" - return engine.getreport(scan_id) + if not scan_id.isdecimal(): + return jsonify({"status": "error", "reason": "scan_id must be numeric digits only"}) + filepath = f"{APP_BASE_DIR}/results/apivoid_{scan_id}.json" + + if not os.path.exists(filepath): + return jsonify({"status": "error", "reason": f"report file for scan_id '{scan_id}' not found"}) + + return send_from_directory(f"{APP_BASE_DIR}/results/", "apivoid_{scan_id}.json") def _loadconfig(): @@ -213,7 +220,7 @@ def start_scan(): if len(engine.scans) == APP_MAXSCANS: res.update({ "status": "error", - "reason": "Scan refused: max concurrent active scans reached ({})".format(APP_MAXSCANS) + "reason": f"Scan refused: max concurrent active scans reached ({APP_MAXSCANS})" }) return jsonify(res) @@ -223,7 +230,7 @@ def start_scan(): res.update({ "status": "refused", "details": { - "reason": "bad scanner status {}".format(engine.scanner['status']) + "reason": f"Bad scanner status {engine.scanner['status']}" }}) return jsonify(res) @@ -259,7 +266,7 @@ def start_scan(): if asset["datatype"] not in engine.scanner["allowed_asset_types"]: res.update({ "status": "error", - "reason": "asset '{}' datatype '{}' not supported".format(asset["value"],asset["datatype"]) + "reason": "asset '{}' has unsupported datatype '{}'".format(asset["value"], asset["datatype"]) }) return jsonify(res) @@ -271,6 +278,12 @@ def start_scan(): if asset["datatype"] == "url": parsed_uri = urlparse(asset["value"]) asset["value"] = parsed_uri.netloc + + # Check the netloc type + if is_valid_ip(asset["value"]): + asset["datatype"] == "ip" + else: + asset["datatype"] == "domain" assets.append(asset["value"]) @@ -280,7 +293,7 @@ def start_scan(): res.update({ "status": "refused", "details": { - "reason": "scan '{}' already launched".format(data['scan_id']), + "reason": f"scan '{data['scan_id']}' already launched", } }) return jsonify(res) @@ -310,7 +323,7 @@ def start_scan(): if 'domain_reputation' in scan['options'].keys() and data['options']['domain_reputation']: for asset in data["assets"]: - if asset["datatype"] == "domain": + if asset["datatype"] in ["domain", "fqdn"]: th = this.pool.submit(_scan_domain_reputation, scan_id, asset["value"]) engine.scans[scan_id]['futures'].append(th) @@ -330,7 +343,7 @@ def _scan_ip_reputation(scan_id, asset): try: engine.scans[scan_id]["findings"][asset]['ip_reputation'] = get_report_ip_reputation(scan_id, asset, apikey) except Exception as ex: - app.logger.error("_scan_ip_reputation failed {}".format(re.sub(r'/' + apikey + '/', r'/***/', ex.__str__()))) + app.logger.error("_scan_ip_reputation failed: {}".format(re.sub(r'/' + apikey + '/', r'/***/', ex.__str__()))) return False return True @@ -343,7 +356,7 @@ def _scan_domain_reputation(scan_id, asset): try: engine.scans[scan_id]["findings"][asset]['domain_reputation'] = get_report_domain_reputation(scan_id, asset, apikey) except Exception as ex: - app.logger.error("_scan_domain_reputation failed {}".format(re.sub(r'/' + apikey + '/', r'/***/', ex.__str__()))) + app.logger.error("_scan_domain_reputation failed: {}".format(re.sub(r'/' + apikey + '/', r'/***/', ex.__str__()))) return False return True @@ -365,7 +378,7 @@ def get_report_ip_reputation(scan_id, asset, apikey): response = requests.get(scan_url) # print(response.content) except Exception as ex: - app.logger.error("get_report_ip_reputation failed {}".format(re.sub(r'/' + apikey + '/', r'/***/', ex.__str__()))) + app.logger.error("get_report_ip_reputation failed: {}".format(re.sub(r'/' + apikey + '/', r'/***/', ex.__str__()))) return [] return response.content @@ -380,7 +393,7 @@ def get_report_domain_reputation(scan_id, asset, apikey): response = requests.get(scan_url) # print(response.content) except Exception as ex: - app.logger.error("get_report_domain_reputation failed {}".format(re.sub(r'/' + apikey + '/', r'/***/', ex.__str__()))) + app.logger.error("get_report_domain_reputation failed: {}".format(re.sub(r'/' + apikey + '/', r'/***/', ex.__str__()))) return [] return response.content @@ -400,18 +413,34 @@ def _parse_results(scan_id): ts = int(time.time() * 1000) for asset in engine.scans[scan_id]["findings"]: + if 'ip_reputation' in engine.scans[scan_id]["findings"][asset].keys(): res = json.loads(engine.scans[scan_id]["findings"][asset]['ip_reputation']) + if 'data' in res: + severity = "info" + report_summary = "" + try: + detections = res["data"]["report"]["blacklists"]["detections"] + risk_score = res["data"]["report"]["risk_score"]["result"] + if risk_score == 100: + severity = "high" + elif risk_score >= 70: + severity = "medium" + + report_summary = f" (detect:{detections}, risk:{risk_score})" + except Exception: + pass + nb_vulns['info'] += 1 issues.append({ "issue_id": len(issues) + 1, - "severity": "info", "confidence": "certain", + "severity": severity, "confidence": "certain", "target": { "addr": [asset], "protocol": "domain" }, - "title": "IP Reputation Check", + "title": "IP Reputation Check"+report_summary, "description": f"IP Reputation Check for '{asset}'\n\nSee raw_data", "solution": "n/a", "metadata": { @@ -421,18 +450,34 @@ def _parse_results(scan_id): "raw": res['data'], "timestamp": ts }) + if 'domain_reputation' in engine.scans[scan_id]["findings"][asset].keys(): res = json.loads(engine.scans[scan_id]["findings"][asset]['domain_reputation']) + if 'data' in res: + severity = "info" + report_summary = "" + try: + detections = res["data"]["report"]["blacklists"]["detections"] + risk_score = res["data"]["report"]["risk_score"]["result"] + if risk_score == 100: + severity = "high" + elif risk_score >= 70: + severity = "medium" + + report_summary = f" (detect:{detections}, risk:{risk_score})" + except Exception: + pass + nb_vulns['info'] += 1 issues.append({ "issue_id": len(issues) + 1, - "severity": "info", "confidence": "certain", + "severity": severity, "confidence": "certain", "target": { "addr": [asset], "protocol": "domain" }, - "title": "Domain Reputation Check", + "title": "Domain Reputation Check"+report_summary, "description": f"Domain Reputation Check for '{asset}'\n\nSee raw_data", "solution": "n/a", "metadata": { @@ -469,8 +514,10 @@ def getfindings(scan_id): # check if the scan is finished status_scan(scan_id) if engine.scans[scan_id]['status'] != "FINISHED": - res.update({"status": "error", - "reason": f"scan_id '{scan_id}' not finished (status={engine.scans[scan_id]['status']})"}) + res.update({ + "status": "error", + "reason": f"scan_id '{scan_id}' not finished (status={engine.scans[scan_id]['status']})" + }) return jsonify(res) status, issues, summary = _parse_results(scan_id) @@ -484,15 +531,30 @@ def getfindings(scan_id): } scan.update(status) - - # remove the scan from the active scan list - clean_scan(scan_id) - - res.update({"scan": scan, "summary": summary, "issues": issues}) + + res_data = {"scan": scan, "summary": summary, "issues": issues} + + # Store the findings in a file + with open(f"{APP_BASE_DIR}/results/apivoid_{scan_id}.json", 'w') as report_file: + json.dump(res_data, report_file, default=_json_serial) + + # # Remove the scan from the active scan list + # clean_scan(scan_id) + + # Prepare response + res.update(res_data) res.update(status) return jsonify(res) +def is_valid_ip(ip): + try: + IPAddress(ip) + except (TypeError, ValueError, AddrFormatError): + return False + return True + + def is_valid_subnet(subnet): try: IPNetwork(subnet) @@ -509,6 +571,17 @@ def get_ips_from_subnet(subnet): return [str(ip) for ip in IPNetwork(subnet)] +def _json_serial(obj): + """ + JSON serializer for objects not serializable by default json code + Used for datetime serialization when the results are written in file + """ + if isinstance(obj, datetime.datetime) or isinstance(obj, datetime.date): + serial = obj.isoformat() + return serial + raise TypeError("Type not serializable") + + @app.before_first_request def main(): """First function called.""" From f501fc4f2ed87752202370800d43254ff0d3693f Mon Sep 17 00:00:00 2001 From: MaKyOtOx Date: Tue, 14 Mar 2023 16:42:56 +0100 Subject: [PATCH 2/9] update version --- engines/apivoid/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/engines/apivoid/__init__.py b/engines/apivoid/__init__.py index e40804ce..dc091835 100644 --- a/engines/apivoid/__init__.py +++ b/engines/apivoid/__init__.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- __title__ = 'patrowl_engine_apivoid' -__version__ = '1.4.27' +__version__ = '1.4.32' __author__ = 'Nicolas MATTIOCCO' __license__ = 'AGPLv3' -__copyright__ = 'Copyright (C) 2020-2022 Nicolas Mattiocco - @MaKyOtOx' +__copyright__ = 'Copyright (C) 2020-2023 Nicolas Mattiocco - @MaKyOtOx' From 5f7c9d0237da707062d8fb767a93fd3375ff3727 Mon Sep 17 00:00:00 2001 From: Florent Date: Thu, 30 Mar 2023 15:06:42 +0200 Subject: [PATCH 3/9] Quick Fix on docker / send_from_directorty https://stackoverflow.com/questions/73276384/getting-an-error-attachment-filename-does-not-exist-in-my-docker-environment --- engines/nmap/engine-nmap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engines/nmap/engine-nmap.py b/engines/nmap/engine-nmap.py index dfd19e85..875f339c 100644 --- a/engines/nmap/engine-nmap.py +++ b/engines/nmap/engine-nmap.py @@ -957,7 +957,7 @@ def getreport(scan_id): f"{BASE_DIR}/results", f"nmap_{scan_id}.json", mimetype='application/json', - attachment_filename=f"nmap_{scan_id}.json", + download_name=f"nmap_{scan_id}.json", as_attachment=True ) From d64ae378f18786dbb255ab63bdca5bc51276b5d9 Mon Sep 17 00:00:00 2001 From: MaKyOtOx Date: Wed, 12 Apr 2023 15:41:24 +0200 Subject: [PATCH 4/9] Fix #302 OWL_DNS supports SaaS detection --- VERSION | 2 +- engines/owl_dns/Dockerfile | 2 +- engines/owl_dns/VERSION | 2 +- engines/owl_dns/engine-owl_dns.py | 69 ++++++++- engines/owl_dns/etc/ip-ranges.json | 220 ++++++++++++++++------------- 5 files changed, 189 insertions(+), 106 deletions(-) diff --git a/VERSION b/VERSION index f0ed3796..df808199 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.5.10 +1.5.11 diff --git a/engines/owl_dns/Dockerfile b/engines/owl_dns/Dockerfile index b6ec1ffd..a8a20dab 100644 --- a/engines/owl_dns/Dockerfile +++ b/engines/owl_dns/Dockerfile @@ -1,5 +1,5 @@ FROM ubuntu:20.04 -LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.5.0" +LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.5.1" # Install dependencies RUN apt-get update && \ diff --git a/engines/owl_dns/VERSION b/engines/owl_dns/VERSION index bc80560f..26ca5946 100644 --- a/engines/owl_dns/VERSION +++ b/engines/owl_dns/VERSION @@ -1 +1 @@ -1.5.0 +1.5.1 diff --git a/engines/owl_dns/engine-owl_dns.py b/engines/owl_dns/engine-owl_dns.py index 664655a9..5da3ca30 100644 --- a/engines/owl_dns/engine-owl_dns.py +++ b/engines/owl_dns/engine-owl_dns.py @@ -268,6 +268,12 @@ def start_scan(): if asset["datatype"] == "ip": th = this.pool.submit(_cloud_check, scan_id, asset["value"], asset["datatype"]) this.scans[scan_id]['futures'].append(th) + + if 'do_saas_check' in scan['options'].keys() and data['options']['do_saas_check']: + for asset in data["assets"]: + if asset["datatype"] == "ip": + th = this.pool.submit(_saas_check, scan_id, asset["value"], asset["datatype"]) + this.scans[scan_id]['futures'].append(th) res.update({ "status": "accepted", @@ -426,6 +432,7 @@ def _reverse_whois(scan_id, asset, datatype): return res + def is_ipaddr_in_subnet(ip: str, subnet: str) -> bool: """Check if the IP address is part of the subnet""" try: @@ -437,25 +444,26 @@ def is_ipaddr_in_subnet(ip: str, subnet: str) -> bool: def _check_ip(ip: str, record_types: list = []) -> dict: - """Check IP from CDN, WAF, Cloud providers public records.""" + """Check IP from CDN, WAF, Cloud, SaaS providers public records.""" with open(this.scanner['external_ip_ranges_path']) as all_data_file: all_data = json.loads(all_data_file.read()) - all_data_types = all_data.keys() # ["cdn", "waf", "cloud", "parking"] + all_data_types = all_data.keys() # ["cdn", "waf", "cloud", "parking", "saas"] data_types = [] ip_provider = "" if len(record_types) > 0: for record_type in record_types: - if record_type not in all_data.keys(): - all_data_types = [record_type] - - for data_type in all_data_types: + if record_type in all_data.keys(): + data_types.append(record_type) + else: + data_types = all_data_types + + for data_type in data_types: for provider in all_data[data_type].keys(): for subnet in all_data[data_type][provider]["ipv4"]: if is_ipaddr_in_subnet(ip, subnet): - data_types.append(data_type) ip_provider = provider break @@ -463,6 +471,7 @@ def _check_ip(ip: str, record_types: list = []) -> dict: return {} return {"attributes": data_types, "provider": ip_provider} + def __get_ip_targets(asset : str, datatype: str) -> list: targets = [] if datatype == "ip": @@ -532,6 +541,26 @@ def _cloud_check(scan_id: str, asset: str, datatype: str) -> dict: return res + +def _saas_check(scan_id: str, asset: str, datatype: str) -> dict: + targets = __get_ip_targets(asset, datatype) + + if len(targets) == 0: + return {} + + for target in targets: + res = _check_ip(target, ["saas"]) + + scan_lock = threading.RLock() + with scan_lock: + if 'saas_check' not in this.scans[scan_id]['findings'].keys(): + this.scans[scan_id]['findings']['saas_check'] = {} + if bool(res): + this.scans[scan_id]['findings']['saas_check'].update({asset: res}) + + return res + + def _recursive_spf_lookups(spf_line): spf_lookups = 0 for word in spf_line.split(" "): @@ -545,6 +574,7 @@ def _recursive_spf_lookups(spf_line): spf_lookups += _recursive_spf_lookups(value) return spf_lookups + def _do_dmarc_check(scan_id,asset_value): dmarc_dict = {"no_dmarc_record": "info"} dns_records = __dns_resolve_asset(asset_value, "TXT") @@ -566,6 +596,7 @@ def _do_dmarc_check(scan_id,asset_value): this.scans[scan_id]["findings"]["dmarc_dict"] = {asset_value: dmarc_dict} this.scans[scan_id]["findings"]["dmarc_dict_dns_records"] = {asset_value: dns_records} + def _do_dkim_check(scan_id, asset_value): dkim_dict = {} found_dkim = False @@ -587,6 +618,7 @@ def _do_dkim_check(scan_id, asset_value): this.scans[scan_id]["findings"]["dkim_dict"] = {asset_value: dkim_dict} this.scans[scan_id]["findings"]["dkim_dict_dns_records"] = {asset_value: dns_records} + def _perform_spf_check(scan_id,asset_value): dns_records = __dns_resolve_asset(asset_value, "TXT") spf_dict = { @@ -1286,6 +1318,29 @@ def _parse_results(scan_id): "timestamp": ts }) + # is IP supported by a SaaS service ? + if 'saas_check' in scan['findings'].keys(): + for asset in scan['findings']['saas_check'].keys(): + nb_vulns['info'] += 1 + provider = scan['findings']['saas_check'][asset]["provider"] + issues.append({ + "issue_id": len(issues) + 1, + "severity": "info", "confidence": "certain", + "target": { + "addr": [asset], + "protocol": "domain" + }, + "title": f"Behind SaaS Provider: '{provider}'", + "description": f"Behind SaaS Provider: '{provider}'", + "solution": "n/a", + "metadata": { + "tags": ["saas", provider] + }, + "type": "saas_check", + "raw": scan['findings']['saas_check'][asset], + "timestamp": ts + }) + # subdomain list # bad messages replied by Sublist3r diff --git a/engines/owl_dns/etc/ip-ranges.json b/engines/owl_dns/etc/ip-ranges.json index 177d1d48..f90843ab 100644 --- a/engines/owl_dns/etc/ip-ranges.json +++ b/engines/owl_dns/etc/ip-ranges.json @@ -710,6 +710,17 @@ "2a06:98c0::/29", "2c0f:f248::/32" ] + }, + "sucuri": { + "ipv4": [ + "192.88.134.0/23", + "185.93.228.0/22", + "66.248.200.0/22", + "208.109.0.0/22" + ], + "ipv6": [ + "2a02:fe80::/29" + ] } }, "cloud": { @@ -5714,102 +5725,6 @@ ] }, - "salesforce": { - "ipv4": [ - "13.108.0.0/14", - "66.231.80.0/20", - "68.232.192.0/20", - "96.43.144.0/20", - "128.17.0.0/16", - "128.245.0.0/16", - "136.146.0.0/15", - "198.245.80.0/20", - "199.122.120.0/21", - "204.14.232.0/21", - "34.226.36.48/28", - "34.211.108.32/28", - "13.58.135.64/28", - "13.56.32.176/28", - "35.182.14.32/28", - "52.60.248.0/22", - "52.60.252.0/22", - "3.98.2.135/32", - "3.98.8.160/32", - "3.97.226.192/32", - "13.210.4.0/22", - "13.210.8.0/22", - "13.210.180.120/32", - "13.238.98.67/32", - "54.252.37.181/32", - "3.6.203.25/32", - "13.127.212.138/32", - "15.206.226.165/32", - "15.207.181.18/32", - "15.207.182.186/32", - "65.0.79.252/32", - "3.225.240.254/32", - "18.204.28.162/32", - "18.214.12.209/32", - "34.202.86.120/32", - "34.204.111.166/32", - "52.44.156.44/32", - "44.233.69.21/32", - "44.237.79.66/32", - "52.36.20.11/32", - "35.80.213.208/32", - "35.161.141.162/32", - "44.234.249.148/32", - "85.222.128.0/19", - "159.92.128.0/17", - "160.8.0.0/16", - "161.71.0.0/17", - "163.76.128.0/17", - "163.79.128.0/17", - "185.79.140.0/22", - "34.253.190.64/28", - "35.158.127.48/28", - "35.176.92.16/28", - "13.36.84.96/28", - "13.37.59.29/32", - "15.236.110.244/32", - "15.236.160.173/32", - "13.50.12.176/28", - "101.53.160.0/19", - "104.161.128.0/17", - "161.32.64.0/18", - "161.32.128.0/17", - "161.71.128.0/17", - "182.50.76.0/22", - "202.129.242.0/23", - "13.113.196.48/28", - "13.228.64.80/28", - "13.124.145.0/28", - "13.126.23.64/28", - "13.210.3.208/28", - "13.215.171.240/28", - "35.73.89.117/32", - "18.181.43.11/32", - "54.95.206.252/32", - "54.254.118.123/32", - "13.251.9.241/32", - "13.250.175.171/32", - "43.201.151.176/28", - "54.233.205.0/28", - "177.71.229.247/32", - "18.228.66.156/32", - "18.228.207.180/32", - "13.109.128.0/19", - "13.109.160.0/21", - "13.109.192.0/19", - "101.53.176.0/20", - "160.8.0.0/21", - "161.71.16.0/20", - "161.71.40.0/22", - "161.71.176.0/21", - "182.50.76.0/22" - ], - "ipv6": [] - }, "ibm": { "ipv4": [ "12.96.160.0/21", @@ -6216,5 +6131,118 @@ ], "ipv6": [] } + }, + "saas": { + "salesforce": { + "ipv4": [ + "13.108.0.0/14", + "66.231.80.0/20", + "68.232.192.0/20", + "96.43.144.0/20", + "128.17.0.0/16", + "128.245.0.0/16", + "136.146.0.0/15", + "198.245.80.0/20", + "199.122.120.0/21", + "204.14.232.0/21", + "34.226.36.48/28", + "34.211.108.32/28", + "13.58.135.64/28", + "13.56.32.176/28", + "35.182.14.32/28", + "52.60.248.0/22", + "52.60.252.0/22", + "3.98.2.135/32", + "3.98.8.160/32", + "3.97.226.192/32", + "13.210.4.0/22", + "13.210.8.0/22", + "13.210.180.120/32", + "13.238.98.67/32", + "54.252.37.181/32", + "3.6.203.25/32", + "13.127.212.138/32", + "15.206.226.165/32", + "15.207.181.18/32", + "15.207.182.186/32", + "65.0.79.252/32", + "3.225.240.254/32", + "18.204.28.162/32", + "18.214.12.209/32", + "34.202.86.120/32", + "34.204.111.166/32", + "52.44.156.44/32", + "44.233.69.21/32", + "44.237.79.66/32", + "52.36.20.11/32", + "35.80.213.208/32", + "35.161.141.162/32", + "44.234.249.148/32", + "85.222.128.0/19", + "159.92.128.0/17", + "160.8.0.0/16", + "161.71.0.0/17", + "163.76.128.0/17", + "163.79.128.0/17", + "185.79.140.0/22", + "34.253.190.64/28", + "35.158.127.48/28", + "35.176.92.16/28", + "13.36.84.96/28", + "13.37.59.29/32", + "15.236.110.244/32", + "15.236.160.173/32", + "13.50.12.176/28", + "101.53.160.0/19", + "104.161.128.0/17", + "161.32.64.0/18", + "161.32.128.0/17", + "161.71.128.0/17", + "182.50.76.0/22", + "202.129.242.0/23", + "13.113.196.48/28", + "13.228.64.80/28", + "13.124.145.0/28", + "13.126.23.64/28", + "13.210.3.208/28", + "13.215.171.240/28", + "35.73.89.117/32", + "18.181.43.11/32", + "54.95.206.252/32", + "54.254.118.123/32", + "13.251.9.241/32", + "13.250.175.171/32", + "43.201.151.176/28", + "54.233.205.0/28", + "177.71.229.247/32", + "18.228.66.156/32", + "18.228.207.180/32", + "13.109.128.0/19", + "13.109.160.0/21", + "13.109.192.0/19", + "101.53.176.0/20", + "160.8.0.0/21", + "161.71.16.0/20", + "161.71.40.0/22", + "161.71.176.0/21", + "182.50.76.0/22" + ], + "ipv6": [] + }, + "wix": { + "ipv4": [ + "3.104.90.0/24", + "18.228.249.0/24", + "3.122.181.0/24", + "3.112.80.0/24", + "3.1.77.0/24", + "54.244.51.0/24", + "185.230.60.0/22", + "199.15.160.0/24", + "199.15.163.0/24", + "204.2.207.0/24" + ], + "ipv6": [] + } } } From b04e98104ac3bf4ec27d0f84c0fee97909435b1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20COLL?= Date: Mon, 17 Apr 2023 14:48:06 +0200 Subject: [PATCH 5/9] 1.5.11-rc1 owl_dns SaaS support --- VERSION | 2 +- engines/owl_dns/Dockerfile | 2 +- engines/owl_dns/VERSION | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/VERSION b/VERSION index df808199..7359307b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.5.11 +1.5.11-rc1 diff --git a/engines/owl_dns/Dockerfile b/engines/owl_dns/Dockerfile index a8a20dab..dcfe0e29 100644 --- a/engines/owl_dns/Dockerfile +++ b/engines/owl_dns/Dockerfile @@ -1,5 +1,5 @@ FROM ubuntu:20.04 -LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.5.1" +LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.5.1-rc1" # Install dependencies RUN apt-get update && \ diff --git a/engines/owl_dns/VERSION b/engines/owl_dns/VERSION index 26ca5946..d5598e00 100644 --- a/engines/owl_dns/VERSION +++ b/engines/owl_dns/VERSION @@ -1 +1 @@ -1.5.1 +1.5.1-rc1 From 9db8e0ec943b589116a0e92d6a8933a5bd2c5249 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20COLL?= Date: Mon, 17 Apr 2023 14:56:22 +0200 Subject: [PATCH 6/9] 1.4.32-rc1 apivoid --- engines/apivoid/Dockerfile | 2 +- engines/apivoid/VERSION | 2 +- engines/apivoid/apivoid.json.sample | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/engines/apivoid/Dockerfile b/engines/apivoid/Dockerfile index 91bba8eb..db5ccf2d 100644 --- a/engines/apivoid/Dockerfile +++ b/engines/apivoid/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:3.16.3 -LABEL Name="APIVoid\ \(Patrowl engine\)" Version="1.4.32" +LABEL Name="APIVoid\ \(Patrowl engine\)" Version="1.4.32-rc1" # Create the target repo RUN mkdir -p /opt/patrowl-engines/apivoid diff --git a/engines/apivoid/VERSION b/engines/apivoid/VERSION index 00bbe72a..26affb5d 100644 --- a/engines/apivoid/VERSION +++ b/engines/apivoid/VERSION @@ -1 +1 @@ -1.4.32 +1.4.32-rc1 diff --git a/engines/apivoid/apivoid.json.sample b/engines/apivoid/apivoid.json.sample index b7a7b4c7..2c42cd29 100755 --- a/engines/apivoid/apivoid.json.sample +++ b/engines/apivoid/apivoid.json.sample @@ -1,6 +1,6 @@ { "name": "APIVOID", - "version": "1.4.32", + "version": "1.4.32-rc1", "description": "APIVoid reputation API", "allowed_asset_types": ["domain", "fqdn", "ip", "url", "ip-subnet"], "apikeys": [ From 422bdbebb782204b8ac77bc033dfcf343b50ee16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20COLL?= Date: Mon, 17 Apr 2023 16:09:05 +0200 Subject: [PATCH 7/9] 1.4.42-rc1 nmap --- engines/nmap/Dockerfile | 2 +- engines/nmap/VERSION | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/engines/nmap/Dockerfile b/engines/nmap/Dockerfile index e8d8d8e2..d5c1f0f4 100644 --- a/engines/nmap/Dockerfile +++ b/engines/nmap/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:3.16.3 -LABEL Name="Nmap\ \(Patrowl engine\)" Version="1.4.41" +LABEL Name="Nmap\ \(Patrowl engine\)" Version="1.4.42-rc1" # Set the working directory RUN mkdir -p /opt/patrowl-engines/nmap diff --git a/engines/nmap/VERSION b/engines/nmap/VERSION index d65e023d..472c27aa 100644 --- a/engines/nmap/VERSION +++ b/engines/nmap/VERSION @@ -1 +1 @@ -1.4.41 +1.4.42-rc1 From 90951141ab206c04dd14ab21f99550780d430d48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20COLL?= Date: Wed, 19 Apr 2023 16:26:31 +0200 Subject: [PATCH 8/9] Alpine by default for OWL_DNS --- engines/owl_dns/Dockerfile | 27 +++++++---------- .../{Dockerfile.alpine => Dockerfile.ubuntu} | 29 ++++++++++++------- engines/owl_dns/VERSION | 2 +- 3 files changed, 29 insertions(+), 29 deletions(-) rename engines/owl_dns/{Dockerfile.alpine => Dockerfile.ubuntu} (67%) diff --git a/engines/owl_dns/Dockerfile b/engines/owl_dns/Dockerfile index dcfe0e29..a1dcc7fc 100644 --- a/engines/owl_dns/Dockerfile +++ b/engines/owl_dns/Dockerfile @@ -1,21 +1,19 @@ -FROM ubuntu:20.04 -LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.5.1-rc1" +FROM alpine:3.16.3 +LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.0.2" # Install dependencies -RUN apt-get update && \ - apt-get install -y --no-install-recommends python3-dnspython \ - python3-geoip python3-whois python3-requests python3-ssdeep \ - python3-pip python3-setuptools python3-dev git wget locales && \ - locale-gen en_US.UTF-8 -ENV LC_ALL=en_US.UTF-8 -ENV LANG=en_US.UTF-8 -ENV LANGUAGE=en_US.UTF-8 +RUN apk add --update --no-cache \ + python3 python3-dev py3-pip \ + git \ + && rm -rf /var/cache/apk/* # Create the target repo RUN mkdir -p /opt/patrowl-engines/owl_dns RUN mkdir -p /opt/patrowl-engines/owl_dns/results RUN mkdir -p /opt/patrowl-engines/owl_dns/external-libs -WORKDIR /opt/patrowl-engines/owl_dns/ + +# Set the working directory to /opt/ +WORKDIR /opt/patrowl-engines/owl_dns # Copy the current directory contents into the container at / COPY __init__.py . @@ -33,17 +31,12 @@ WORKDIR /opt/patrowl-engines/owl_dns/external-libs/Sublist3r RUN pip3 install --trusted-host pypi.python.org -r requirements.txt WORKDIR /opt/patrowl-engines/owl_dns/external-libs RUN git clone https://github.com/elceef/dnstwist -WORKDIR /opt/patrowl-engines/owl_dns/external-libs/dnstwist -RUN pip3 install --trusted-host pypi.python.org -r requirements.txt - -# Set the working directory to /opt/ -WORKDIR /opt/patrowl-engines/owl_dns # Install python modules for engine +WORKDIR /opt/patrowl-engines/owl_dns/ RUN pip3 install --upgrade pip RUN pip3 install --trusted-host pypi.python.org -r requirements.txt - # TCP port exposed by the container (NAT) EXPOSE 5006 diff --git a/engines/owl_dns/Dockerfile.alpine b/engines/owl_dns/Dockerfile.ubuntu similarity index 67% rename from engines/owl_dns/Dockerfile.alpine rename to engines/owl_dns/Dockerfile.ubuntu index ef283223..dcfe0e29 100644 --- a/engines/owl_dns/Dockerfile.alpine +++ b/engines/owl_dns/Dockerfile.ubuntu @@ -1,19 +1,21 @@ -FROM alpine:3.16.3 -LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.0.2" +FROM ubuntu:20.04 +LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.5.1-rc1" # Install dependencies -RUN apk add --update --no-cache \ - python3 python3-dev py3-pip \ - git \ - && rm -rf /var/cache/apk/* +RUN apt-get update && \ + apt-get install -y --no-install-recommends python3-dnspython \ + python3-geoip python3-whois python3-requests python3-ssdeep \ + python3-pip python3-setuptools python3-dev git wget locales && \ + locale-gen en_US.UTF-8 +ENV LC_ALL=en_US.UTF-8 +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US.UTF-8 # Create the target repo RUN mkdir -p /opt/patrowl-engines/owl_dns RUN mkdir -p /opt/patrowl-engines/owl_dns/results RUN mkdir -p /opt/patrowl-engines/owl_dns/external-libs - -# Set the working directory to /opt/ -WORKDIR /opt/patrowl-engines/owl_dns +WORKDIR /opt/patrowl-engines/owl_dns/ # Copy the current directory contents into the container at / COPY __init__.py . @@ -31,14 +33,19 @@ WORKDIR /opt/patrowl-engines/owl_dns/external-libs/Sublist3r RUN pip3 install --trusted-host pypi.python.org -r requirements.txt WORKDIR /opt/patrowl-engines/owl_dns/external-libs RUN git clone https://github.com/elceef/dnstwist +WORKDIR /opt/patrowl-engines/owl_dns/external-libs/dnstwist +RUN pip3 install --trusted-host pypi.python.org -r requirements.txt + +# Set the working directory to /opt/ +WORKDIR /opt/patrowl-engines/owl_dns # Install python modules for engine -WORKDIR /opt/patrowl-engines/owl_dns/ RUN pip3 install --upgrade pip RUN pip3 install --trusted-host pypi.python.org -r requirements.txt + # TCP port exposed by the container (NAT) EXPOSE 5006 # Run app.py when the container launches -CMD ["gunicorn", "engine-owl_dns:app", "-b", "0.0.0.0:5006", "--access-logfile", "-"] +CMD ["gunicorn", "engine-owl_dns:app", "-b", "0.0.0.0:5006", "--access-logfile", "-", "--threads", "10"] diff --git a/engines/owl_dns/VERSION b/engines/owl_dns/VERSION index d5598e00..3feb9a92 100644 --- a/engines/owl_dns/VERSION +++ b/engines/owl_dns/VERSION @@ -1 +1 @@ -1.5.1-rc1 +1.5.1-rc2 From 65b27d6dd7f83f8ae40a4cd3a96a940c688f3e32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20COLL?= Date: Thu, 27 Apr 2023 17:38:14 +0200 Subject: [PATCH 9/9] 1.5.11 update --- VERSION | 2 +- engines/apivoid/Dockerfile | 2 +- engines/apivoid/VERSION | 2 +- engines/apivoid/apivoid.json.sample | 2 +- engines/nmap/Dockerfile | 2 +- engines/nmap/VERSION | 2 +- engines/owl_dns/Dockerfile | 2 +- engines/owl_dns/VERSION | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/VERSION b/VERSION index 7359307b..df808199 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.5.11-rc1 +1.5.11 diff --git a/engines/apivoid/Dockerfile b/engines/apivoid/Dockerfile index db5ccf2d..91bba8eb 100644 --- a/engines/apivoid/Dockerfile +++ b/engines/apivoid/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:3.16.3 -LABEL Name="APIVoid\ \(Patrowl engine\)" Version="1.4.32-rc1" +LABEL Name="APIVoid\ \(Patrowl engine\)" Version="1.4.32" # Create the target repo RUN mkdir -p /opt/patrowl-engines/apivoid diff --git a/engines/apivoid/VERSION b/engines/apivoid/VERSION index 26affb5d..00bbe72a 100644 --- a/engines/apivoid/VERSION +++ b/engines/apivoid/VERSION @@ -1 +1 @@ -1.4.32-rc1 +1.4.32 diff --git a/engines/apivoid/apivoid.json.sample b/engines/apivoid/apivoid.json.sample index 2c42cd29..b7a7b4c7 100755 --- a/engines/apivoid/apivoid.json.sample +++ b/engines/apivoid/apivoid.json.sample @@ -1,6 +1,6 @@ { "name": "APIVOID", - "version": "1.4.32-rc1", + "version": "1.4.32", "description": "APIVoid reputation API", "allowed_asset_types": ["domain", "fqdn", "ip", "url", "ip-subnet"], "apikeys": [ diff --git a/engines/nmap/Dockerfile b/engines/nmap/Dockerfile index d5c1f0f4..b5677d8f 100644 --- a/engines/nmap/Dockerfile +++ b/engines/nmap/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:3.16.3 -LABEL Name="Nmap\ \(Patrowl engine\)" Version="1.4.42-rc1" +LABEL Name="Nmap\ \(Patrowl engine\)" Version="1.4.42" # Set the working directory RUN mkdir -p /opt/patrowl-engines/nmap diff --git a/engines/nmap/VERSION b/engines/nmap/VERSION index 472c27aa..3fd1ba6b 100644 --- a/engines/nmap/VERSION +++ b/engines/nmap/VERSION @@ -1 +1 @@ -1.4.42-rc1 +1.4.42 diff --git a/engines/owl_dns/Dockerfile b/engines/owl_dns/Dockerfile index a1dcc7fc..6f0bf90e 100644 --- a/engines/owl_dns/Dockerfile +++ b/engines/owl_dns/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:3.16.3 -LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.0.2" +LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.5.1" # Install dependencies RUN apk add --update --no-cache \ diff --git a/engines/owl_dns/VERSION b/engines/owl_dns/VERSION index 3feb9a92..26ca5946 100644 --- a/engines/owl_dns/VERSION +++ b/engines/owl_dns/VERSION @@ -1 +1 @@ -1.5.1-rc2 +1.5.1