From a552336beabb4ebb83bbaed15b7d9b894ebfcfa3 Mon Sep 17 00:00:00 2001 From: MaKyOtOx Date: Fri, 29 Sep 2023 16:26:57 +0200 Subject: [PATCH 1/2] fix ARS-135 fix whois --- VERSION | 2 +- engines/owl_dns/Dockerfile | 2 +- engines/owl_dns/VERSION | 2 +- engines/owl_dns/engine-owl_dns.py | 21 ++++++++++++++------- engines/owl_dns/install.sh | 0 engines/owl_dns/requirements.txt | 4 ++-- 6 files changed, 19 insertions(+), 12 deletions(-) mode change 100644 => 100755 engines/owl_dns/install.sh diff --git a/VERSION b/VERSION index df808199..41336a1c 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.5.11 +1.5.12 diff --git a/engines/owl_dns/Dockerfile b/engines/owl_dns/Dockerfile index 6f0bf90e..19b2c965 100644 --- a/engines/owl_dns/Dockerfile +++ b/engines/owl_dns/Dockerfile @@ -1,5 +1,5 @@ FROM alpine:3.16.3 -LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.5.1" +LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.5.2" # Install dependencies RUN apk add --update --no-cache \ diff --git a/engines/owl_dns/VERSION b/engines/owl_dns/VERSION index 26ca5946..4cda8f19 100644 --- a/engines/owl_dns/VERSION +++ b/engines/owl_dns/VERSION @@ -1 +1 @@ -1.5.1 +1.5.2 diff --git a/engines/owl_dns/engine-owl_dns.py b/engines/owl_dns/engine-owl_dns.py index 5da3ca30..3ef530f7 100644 --- a/engines/owl_dns/engine-owl_dns.py +++ b/engines/owl_dns/engine-owl_dns.py @@ -316,6 +316,7 @@ def _get_wf_whois(apikey:str, value:str): return json.loads(resp.text) + def _get_wf_domains(wf_url:str, max_pages:int): wf_domains = [] page = 1 @@ -352,16 +353,16 @@ def _reverse_whois(scan_id, asset, datatype): if len(this.wf_apitokens) == 0: # No whoisfreak API Token available return res - + # Select an API KEY apikey = this.wf_apitokens[random.randint(0, len(this.wf_apitokens)-1)] - + # Check the asset is a valid domain name or IP Address if datatype in ["domain", "fqdn"]: if not __is_domain(asset): return res - # w = whois.whois(asset) - w = whois.query(asset, force=True) + w = whois.whois(asset) + # w = whois.query(asset, force=True) # print(w.name, w.registrant, w.owner) # if w.domain_name is None: @@ -718,13 +719,15 @@ def _get_whois(scan_id, asset): res = {} is_domain = __is_domain(asset) is_ip = __is_ip_addr(asset) - + + print(asset, is_domain, is_ip) + # Check the asset is a valid domain name or IP Address if not is_domain and not is_ip: return res if is_domain: - # w = whois.whois(asset) + w = whois.whois(asset) # if w.domain_name is None: # res.update({ # asset: {"errors": w} @@ -733,12 +736,15 @@ def _get_whois(scan_id, asset): # res.update({ # asset: {"raw": {'dict': w, 'text': w.text}, "text": w.text, "type": "domain"} # }) - w = whois.query(asset, force=True, include_raw_whois_text=True) + # w = whois.query(asset, force=True) + print("w.name:", w.name) if w.name is None: res.update({ asset: {"errors": w.__dict__} }) else: + print("w.__dict__:", w.__dict__) + print("w.__dict__:", w.__dict__) res.update({ asset: {"raw": {'dict': w.__dict__, 'text': w.text}, "text": w.text, "type": "domain"} }) @@ -748,6 +754,7 @@ def _get_whois(scan_id, asset): asset: {"raw": {'dict': w, 'text': "see raw"}, "text": "see raw", "type": "ip"} }) + print(res) scan_lock = threading.RLock() with scan_lock: if 'whois' not in this.scans[scan_id]['findings'].keys(): diff --git a/engines/owl_dns/install.sh b/engines/owl_dns/install.sh old mode 100644 new mode 100755 diff --git a/engines/owl_dns/requirements.txt b/engines/owl_dns/requirements.txt index dab865f4..34c8786f 100644 --- a/engines/owl_dns/requirements.txt +++ b/engines/owl_dns/requirements.txt @@ -14,7 +14,7 @@ itsdangerous==2.1.2 Jinja2==3.1.2 MarkupSafe==2.1.1 PatrowlEnginesUtils>=1.0.2 -#python-whois #==0.8.0 +python-whois==0.8.0 netaddr==0.8.0 requests==2.28.1 setuptools #==65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability @@ -22,4 +22,4 @@ six==1.16.0 urllib3==1.26.13 validators==0.20.0 werkzeug==2.2.3 -whois==0.8 +#whois==0.9.27 From 5b95973f3490f2efb8fabb46a4c534f7debc5c9f Mon Sep 17 00:00:00 2001 From: MaKyOtOx Date: Fri, 29 Sep 2023 16:37:52 +0200 Subject: [PATCH 2/2] remove log --- engines/owl_dns/engine-owl_dns.py | 111 +++++++++++++----------------- 1 file changed, 49 insertions(+), 62 deletions(-) diff --git a/engines/owl_dns/engine-owl_dns.py b/engines/owl_dns/engine-owl_dns.py index 3ef530f7..21a4fea5 100644 --- a/engines/owl_dns/engine-owl_dns.py +++ b/engines/owl_dns/engine-owl_dns.py @@ -303,11 +303,11 @@ def __is_domain(host): return res -def _get_wf_reverse_url(apikey:str, type:str, value:str): +def _get_wf_reverse_url(apikey: str, type: str, value: str): return f"https://api.whoisfreaks.com/v1.0/whois?apiKey={apikey}&whois=reverse&{type}={value}&mode=mini" -def _get_wf_whois(apikey:str, value:str): +def _get_wf_whois(apikey: str, value: str): w = None resp = requests.get(f"https://api.whoisfreaks.com/v1.0/whois?apiKey={apikey}&whois=live&domainName={value}") if resp.status_code != 200: @@ -317,14 +317,14 @@ def _get_wf_whois(apikey:str, value:str): return json.loads(resp.text) -def _get_wf_domains(wf_url:str, max_pages:int): +def _get_wf_domains(wf_url: str, max_pages: int): wf_domains = [] page = 1 resp = requests.get(wf_url+f"&page={page}") if resp.status_code != 200: app.logger.debug("_get_wf_domains", wf_url, resp.status_code) return wf_domains - + # get results from page 1 results = json.loads(resp.text) if 'whois_domains_historical' not in results.keys(): @@ -413,7 +413,7 @@ def _reverse_whois(scan_id, asset, datatype): wf_url = _get_wf_reverse_url(apikey, wf_type, wf_value) wf_domains = _get_wf_domains(wf_url, max_pages) domains.extend(wf_domains) - except Exception as e: + except Exception: if len(domains) == 0: return res @@ -473,7 +473,7 @@ def _check_ip(ip: str, record_types: list = []) -> dict: return {"attributes": data_types, "provider": ip_provider} -def __get_ip_targets(asset : str, datatype: str) -> list: +def __get_ip_targets(asset: str, datatype: str) -> list: targets = [] if datatype == "ip": targets = [asset] @@ -482,9 +482,9 @@ def __get_ip_targets(asset : str, datatype: str) -> list: targets = socket.gethostbyname_ex(asset)[2] except Exception: return [] - + return targets - + def _cdn_check(scan_id: str, asset: str, datatype: str) -> dict: targets = __get_ip_targets(asset, datatype) @@ -566,7 +566,7 @@ def _recursive_spf_lookups(spf_line): spf_lookups = 0 for word in spf_line.split(" "): if "include:" in word: - url = word.replace("include:","") + url = word.replace("include:", "") spf_lookups += 1 dns_resolve = __dns_resolve_asset(url, "TXT") for record in dns_resolve: @@ -576,7 +576,7 @@ def _recursive_spf_lookups(spf_line): return spf_lookups -def _do_dmarc_check(scan_id,asset_value): +def _do_dmarc_check(scan_id, asset_value): dmarc_dict = {"no_dmarc_record": "info"} dns_records = __dns_resolve_asset(asset_value, "TXT") for record in dns_records: @@ -620,7 +620,7 @@ def _do_dkim_check(scan_id, asset_value): this.scans[scan_id]["findings"]["dkim_dict_dns_records"] = {asset_value: dns_records} -def _perform_spf_check(scan_id,asset_value): +def _perform_spf_check(scan_id, asset_value): dns_records = __dns_resolve_asset(asset_value, "TXT") spf_dict = { "no_spf_found": "high", @@ -728,25 +728,15 @@ def _get_whois(scan_id, asset): if is_domain: w = whois.whois(asset) - # if w.domain_name is None: - # res.update({ - # asset: {"errors": w} - # }) - # else: - # res.update({ - # asset: {"raw": {'dict': w, 'text': w.text}, "text": w.text, "type": "domain"} - # }) # w = whois.query(asset, force=True) - print("w.name:", w.name) + # print(w.__dict_) if w.name is None: res.update({ - asset: {"errors": w.__dict__} + asset: {"errors": w} }) else: - print("w.__dict__:", w.__dict__) - print("w.__dict__:", w.__dict__) res.update({ - asset: {"raw": {'dict': w.__dict__, 'text': w.text}, "text": w.text, "type": "domain"} + asset: {"raw": {'dict': w, 'text': w.text}, "text": w.text, "type": "domain"} }) if is_ip: w = IPWhois(str(asset).strip()).lookup_rdap() @@ -754,7 +744,6 @@ def _get_whois(scan_id, asset): asset: {"raw": {'dict': w, 'text': "see raw"}, "text": "see raw", "type": "ip"} }) - print(res) scan_lock = threading.RLock() with scan_lock: if 'whois' not in this.scans[scan_id]['findings'].keys(): @@ -1059,14 +1048,13 @@ def _parse_results(scan_id): # for record in value: # entry = "Record type '{}': {}".format(record['record_type'], ", ".join(record['values'])) # dns_resolve_str = "".join((dns_resolve_str, entry+"\n")) - + for record in dns_records: entry = "Record type '{}': {}".format(record['record_type'], ", ".join(record['values'])) dns_resolve_str = "".join((dns_resolve_str, entry+"\n")) dns_resolve_hash = hashlib.sha1(dns_resolve_str.encode("utf-8")).hexdigest()[:6] - nb_vulns['info'] += 1 issues.append({ "issue_id": len(issues) + 1, @@ -1097,23 +1085,23 @@ def _parse_results(scan_id): h = str(c) + str(spf_check_dns_records) spf_hash = hashlib.sha1(h.encode('utf-8')).hexdigest()[:6] issues.append({ - "issue_id": len(issues) + 1, - "severity": spf_check[c], "confidence": "certain", - "target": { - "addr": [asset], - "protocol": "domain" - }, - "title": "SPF found for '{}' (HASH: {})".format( - asset, spf_hash), - "description": "{}\n".format(c), - "solution": "n/a", - "metadata": { - "tags": ["domains", "spf"] - }, - "type": "spf_check", - "raw": scan['findings']['spf_dict'][asset], - "timestamp": ts - }) + "issue_id": len(issues) + 1, + "severity": spf_check[c], "confidence": "certain", + "target": { + "addr": [asset], + "protocol": "domain" + }, + "title": "SPF found for '{}' (HASH: {})".format( + asset, spf_hash), + "description": "{}\n".format(c), + "solution": "n/a", + "metadata": { + "tags": ["domains", "spf"] + }, + "type": "spf_check", + "raw": scan['findings']['spf_dict'][asset], + "timestamp": ts + }) if 'dkim_dict' in scan['findings'].keys(): for asset in scan['findings']['dkim_dict'].keys(): @@ -1147,24 +1135,23 @@ def _parse_results(scan_id): h = str(c) + str(dmarc_check_dns_records) dmarc_hash = hashlib.sha1(h.encode('utf-8')).hexdigest()[:6] issues.append({ - "issue_id": len(issues) + 1, - "severity": dmarc_check[c], "confidence": "certain", - "target": { - "addr": [asset], - "protocol": "domain" - }, - "title": "DMARC for '{}' (HASH: {})".format( - asset, dmarc_hash), - "description": "{}\n".format(c), - "solution": "n/a", - "metadata": { - "tags": ["domains", "dmarc"] - }, - "type": "dmarc_check", - "raw": scan['findings']['dmarc_dict'][asset], - "timestamp": ts - }) - + "issue_id": len(issues) + 1, + "severity": dmarc_check[c], "confidence": "certain", + "target": { + "addr": [asset], + "protocol": "domain" + }, + "title": "DMARC for '{}' (HASH: {})".format( + asset, dmarc_hash), + "description": "{}\n".format(c), + "solution": "n/a", + "metadata": { + "tags": ["domains", "dmarc"] + }, + "type": "dmarc_check", + "raw": scan['findings']['dmarc_dict'][asset], + "timestamp": ts + }) # subdomain resolve if 'subdomains_resolve' in scan['findings'].keys():