Merge pull request #298 from Patrowl/develop

Update 1.5.10

.gitlab-ci.yml

@@ -5,8 +5,8 @@ variables:
 
 stages:
   - build
-  - test
-  - run
+#   - test
+#   - run
 
 .build-push:
   stage: build
@@ -25,36 +25,36 @@ stages:
       changes:
         - $FOLDER_PATH/VERSION
 
-.test-build:
-  stage: test
-  image:
-    name: gcr.io/kaniko-project/executor:debug
-    entrypoint: [""]
-  script:
-    - echo "Building Dockerfile $STACK on branch $CI_COMMIT_BRANCH"
-    - /kaniko/executor --insecure --cleanup --context $CI_PROJECT_DIR/$FOLDER_PATH --dockerfile $CI_PROJECT_DIR/$FOLDER_PATH/Dockerfile --no-push
-  rules:
-    - if: $CI_COMMIT_BRANCH != "master"
-      changes:
-        - $FOLDER_PATH/*
+# .test-build:
+#   stage: test
+#   image:
+#     name: gcr.io/kaniko-project/executor:debug
+#     entrypoint: [""]
+#   script:
+#     - echo "Building Dockerfile $STACK on branch $CI_COMMIT_BRANCH"
+#     - /kaniko/executor --insecure --cleanup --context $CI_PROJECT_DIR/$FOLDER_PATH --dockerfile $CI_PROJECT_DIR/$FOLDER_PATH/Dockerfile --no-push
+#   rules:
+#     - if: $CI_COMMIT_BRANCH != "master"
+#       changes:
+#         - $FOLDER_PATH/*
 
-.run-job:
-  stage: run
-  image:
-    name: $REGISTRY_URL/$STACK:$VERSION
-  script:
-    - echo "Running engine $STACK on branch $CI_COMMIT_BRANCH"
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH
-      changes:
-        - $FOLDER_PATH/VERSION
-  after_script:
-    - docker images | grep $STACK
-    - docker stop $(docker ps -a -q --filter ancestor=$REGISTRY_URL/$STACK:$VERSION)
+# .run-job:
+#   stage: run
+#   image:
+#     name: $REGISTRY_URL/$STACK:$VERSION
+#   script:
+#     - echo "Running engine $STACK on branch $CI_COMMIT_BRANCH"
+#   rules:
+#     - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_BRANCH
+#       changes:
+#         - $FOLDER_PATH/VERSION
+#   after_script:
+#     - docker images | grep $STACK
+#     - docker stop $(docker ps -a -q --filter ancestor=$REGISTRY_URL/$STACK:$VERSION)
 
 include:
   - local: '/engines/apivoid/.apivoid.gitlab-ci.yml'
-  - local: '/engines/arachni/.arachni.gitlab-ci.yml'
+#   - local: '/engines/arachni/.arachni.gitlab-ci.yml'
 #   - local: '/engines/burp/.burp.gitlab-ci.yml'
 #   - local: '/engines/censys/.censys.gitlab-ci.yml'
   - local: '/engines/certstream/.certstream.gitlab-ci.yml'

VERSION

@@ -1 +1 @@
-1.5.9-2 // PatrowlEngines
+1.5.10

engines/apivoid/Dockerfile

@@ -1,5 +1,5 @@
 FROM alpine:3.16.3
-LABEL Name="APIVoid\ \(Patrowl engine\)" Version="1.4.30"
+LABEL Name="APIVoid\ \(Patrowl engine\)" Version="1.4.31"
 
 # Create the target repo
 RUN mkdir -p /opt/patrowl-engines/apivoid

engines/apivoid/VERSION

@@ -1 +1 @@
-1.4.30
+1.4.31

engines/apivoid/requirements.txt

@@ -1,15 +1,15 @@
-certifi==2022.9.24
+certifi==2022.12.7
 chardet==3.0.4
 click==8.0
-flask==1.1.2
+Flask==2.2.3
 gunicorn==20.1.0
 idna==2.5
 itsdangerous==2.0
-Jinja2==2.11.3
-MarkupSafe==1.1.1
-requests==2.28.1
-urllib3==1.24.3
-werkzeug==0.15.6
+Jinja2==3.1.2
+MarkupSafe==2.1.1
+requests==2.28.2
+urllib3==1.26.14
+werkzeug==2.2.3
 PatrowlEnginesUtils>=1.0.1
 ratelimit==2.2.1
 netaddr==0.8.0

engines/arachni/Dockerfile

@@ -1,5 +1,5 @@
 FROM ubuntu:20.04
-LABEL Name="Arachni\ \(Patrowl engine\)" Version="1.4.28"
+LABEL Name="Arachni\ \(Patrowl engine\)" Version="1.4.29"
 
 ENV VERSION_FRAMEWORK 1.5.1
 ENV VERSION_ARACHNI $VERSION_FRAMEWORK-0.5.12

engines/arachni/VERSION

@@ -1 +1 @@
-1.4.28
+1.4.29

engines/arachni/requirements.txt

@@ -3,25 +3,25 @@ attrs==18.2.0
 certifi
 chardet==3.0.4
 click
-flask==2.0.1
+Flask==2.2.3
 funcsigs==1.0.2
 gevent==1.4.0
 greenlet==0.4.15
 gunicorn==20.0.4
 idna==2.5
 itsdangerous
-Jinja2
-MarkupSafe
+Jinja2==3.1.2
+MarkupSafe==2.1.1
 more-itertools==4.3.0
 pathlib2==2.3.2
 PatrowlEnginesUtils>=0.0.12
 pluggy==0.7.1
 psutil==5.6.7
 py==1.10.0
 pytest==3.8.1
-requests
+requests==2.28.2
 scandir==1.9.0
 six==1.11.0
-urllib3==1.25
-werkzeug==0.15.6
+urllib3==1.26.5
+werkzeug==2.2.3
 setuptools==65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability

engines/censys/VERSION

@@ -1 +1 @@
-1.4.26
+1.4.27

engines/censys/requirements.txt

@@ -4,18 +4,18 @@ certifi==2022.12.7
 cffi==1.10.0
 chardet==3.0.4
 click==6.7
-cryptography==3.3.2
+cryptography==39.0.1
 enum34==1.1.6
-flask==1.1.2
+Flask==2.2.3
 idna==2.5
 ipaddress==1.0.18
 itsdangerous==0.24
-Jinja2==2.10.1
-MarkupSafe==1.1.1
+Jinja2==3.1.2
+MarkupSafe==2.1.1
 netaddr==0.7.19
 pycparser==2.18
 pyOpenSSL==17.5.0
 requests==2.25.0
 six==1.10.0
-urllib3==1.25
-werkzeug==0.15.6
+urllib3==1.26.5
+werkzeug==2.2.3

engines/nmap/Dockerfile

@@ -1,5 +1,5 @@
 FROM alpine:3.16.3
-LABEL Name="Nmap\ \(Patrowl engine\)" Version="1.4.40"
+LABEL Name="Nmap\ \(Patrowl engine\)" Version="1.4.41"
 
 # Set the working directory
 RUN mkdir -p /opt/patrowl-engines/nmap

engines/nmap/VERSION

@@ -1 +1 @@
-1.4.40
+1.4.41

engines/nmap/banner.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env -S python3 -OO
 # coding:utf8
 
-# Copyright (c) 2021-2022, Patrowl and contributors
+# Copyright (c) 2021-2023, Patrowl and contributors
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without

engines/nmap/engine-nmap.py

@@ -10,6 +10,7 @@
 import urllib
 import time
 import datetime
+from collections import defaultdict
 from shlex import split
 from urllib.parse import urlparse
 from copy import deepcopy
@@ -224,6 +225,8 @@ def _scan_thread(scan_id):
             cmd += " --max-parallelism {}".format(options.get(opt_key))
         if opt_key == "min-hostgroup":  # /!\ @todo / Security issue: Sanitize parameters here
             cmd += " --min-hostgroup {}".format(options.get(opt_key))
+        if opt_key == "timing-template":
+            cmd += " -T{}".format(options.get(opt_key))
 
     cmd += " -iL " + hosts_filename
 
@@ -617,11 +620,17 @@ def _parse_report(filename, scan_id):
         if host.find('os') is not None:
             osinfo = host.find('os').find('osmatch')
             if osinfo is not None:
+                os_data = defaultdict(list)
+                os_data['name'] = osinfo.get('name')
+                os_data['accuracy'] = osinfo.get('accuracy')
+                for osclass in osinfo.findall('osclass'):
+                    os_data['cpe'].append(osclass.find('cpe').text)
                 res.append(deepcopy(_add_issue(scan_id, target, ts,
                     "OS: {}".format(osinfo.get('name')),
                     "The scan detected that the host run in OS '{}' (accuracy={}%)"
                         .format(osinfo.get('name'), osinfo.get('accuracy')),
                     type="host_osinfo",
+                    raw=os_data,
                     confidence="undefined")))
 
         openports = False
@@ -656,14 +665,20 @@ def _parse_report(filename, scan_id):
 
                     # Check if a CPE has been identified
                     cpe_info = ""
-                    cpe_link = None
+                    cpe_links = []
                     cpe_refs = {}
-                    if port.find('service').find("cpe") is not None:
-                        cpe_vector = port.find('service').find("cpe").text
-                        cpe_link = _get_cpe_link(cpe_vector)
-                        cpe_info = f"\n The following CPE vector has been identified: {cpe_vector}"
-                        cpe_refs = {"CPE": [cpe_vector]}
-                        port_data.update({"cpe": [cpe_vector]})
+                    cpe_vectors = []
+                    for cpe in port.find('service').findall("cpe"):
+                        if cpe is not None:
+                            cpe_vector = cpe.text
+                            cpe_link = _get_cpe_link(cpe_vector)
+                            cpe_info += f"\n The following CPE vector has been identified: {cpe_vector}"
+                            cpe_refs = {"CPE": [cpe_vector]}
+                            cpe_vectors.append(cpe_vector)
+                            cpe_links.append(cpe_link)
+                    if cpe_vectors:
+                        cpe_refs = {"CPE": cpe_vectors}
+                        port_data.update({"cpe": cpe_vectors})
 
                     # <service name="http" product="Pulse Secure VPN gateway http config" devicetype="security-misc" tunnel="ssl" method="probed" conf="10"/>
                     # Detection method
@@ -708,7 +723,7 @@ def _parse_report(filename, scan_id):
                             .format(svc_name, proto, portid, cpe_info, product),
                         type="port_info",
                         raw=port_data,
-                        links=[cpe_link],
+                        links=cpe_links,
                         vuln_refs=cpe_refs)))
 
                 if port_state not in ["filtered", "closed"]:

engines/nmap/requirements.txt

@@ -1,17 +1,17 @@
 certifi==2022.12.7
 chardet==3.0.4
 click==8.1.3
-Flask==1.1.2
+Flask==2.2.3
 gevent==22.10.2
 greenlet==2.0.1
 gunicorn==20.1.0
 idna==2.7
 itsdangerous==2.0.0
-Jinja2==2.11.3
-MarkupSafe==1.1.1
+Jinja2==3.1.2
+MarkupSafe==2.1.1
 PatrowlEnginesUtils>=1.0.2
 psutil==5.9.4
-requests>=2.20.0
+requests==2.28.2
 setuptools==65.5.1 # not directly required, pinned by Snyk to avoid a vulnerability
-urllib3>=1.26.5
-Werkzeug==2.0.2
+urllib3==1.26.14
+werkzeug==2.2.3

engines/owl_dns/Dockerfile

@@ -1,5 +1,5 @@
 FROM ubuntu:20.04
-LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.4.41"
+LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.5.0"
 
 # Install dependencies
 RUN apt-get update && \

engines/owl_dns/Dockerfile.alpine

@@ -1,5 +1,5 @@
 FROM alpine:3.16.3
-LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.0.1"
+LABEL Name="Patrowl\ DNS\ \(Patrowl engine\)" Version="1.0.2"
 
 # Install dependencies
 RUN apk add --update --no-cache \

engines/owl_dns/VERSION

@@ -1 +1 @@
-1.4.41
+1.5.0