attempt upgrade of pywps and owslib dependencies

.github/ISSUE_TEMPLATE/bug-report.md

@@ -11,7 +11,7 @@ assignees: fmigneault
 
 <!-- A clear and concise description of what the bug is. -->
 
-## To Reproduce
+## How to Reproduce
 
 <!-- 
 Steps to reproduce the behavior:

.github/ISSUE_TEMPLATE/security.md

@@ -0,0 +1,41 @@
+---
+
+name: Security and Vulnerability Report
+about: Report a problem regarding security or identified vulnerabilities in code or dependencies.
+labels: triage/security
+assignees: fmigneault
+
+---
+
+## Describe the security issue
+
+<!-- A clear and concise description of what issue. -->
+
+## How to Reproduce
+
+<!-- 
+Steps to reproduce the issue:
+1. '...'
+2. '...'
+3. '...'
+-->
+
+## Known Solutions
+
+<!-- A clear and concise description of what you expected to happen. -->
+
+## Additional References
+
+<!-- URL to related issues or CVE reports -->
+
+## Context
+
+<!-- 
+    If applicable, add screenshots or drag-drop files to help explain the problem.
+    Also, please provide any of the following information if relevant.
+
+- OS: \[e.g. Linux|Windows] (if running locally)
+- Browser \[e.g. chrome, safari] (if running as a service)
+- Instance: \[ADES|EMS|Hybrid] and URL
+- Version \["1.2.3", see `/versions` endpoint]
+-->

CHANGES.rst

@@ -92,6 +92,13 @@ Fixes:
   ``ExtendedInt`` and ``ExtendedBool`` schema types to guarantee original data type explicitly defined are preserved.
 - Fix ``runningSeconds`` field reporting to be of ``float`` type although implicit ``int`` type conversion could occur.
 - Fix validation of ``Execute`` inputs schemas to adequately distinguish between optional inputs and incorrect formats.
+- Fix resolution of ``Accept-Language`` negotiation forwarded to local or remote WPS process execution.
+- Fix XML security issue flagged within dependencies to ``PyWPS`` and ``OWSLib`` by pinning requirements to
+  versions ``pywps==4.5.0`` and ``owslib==0.25.0``, and apply the same fix in `Weaver` code (see following for details:
+  `geopython/pywps#616 <https://github.com/geopython/pywps/pull/616>`_,
+  `geopython/pywps#618 <https://github.com/geopython/pywps/pull/618>`_,
+  `geopython/pywps#624 <https://github.com/geopython/pywps/issues/624>`_,
+  `CVE-2021-39371 <https://nvd.nist.gov/vuln/detail/CVE-2021-39371>`_).
 
 `3.5.0 <https://github.com/crim-ca/weaver/tree/3.5.0>`_ (2021-08-19)
 ========================================================================

SECURITY.md

@@ -0,0 +1,16 @@
+# Weaver Security Policy
+
+## Supported Versions
+
+Security and vulnerability reports should be submitted as an issue of type [Security and Vulnerability Report][svr].
+
+[svr]: https://github.com/crim-ca/weaver/issues/new?assignees=fmigneault&labels=triage%2Fsecurity&template=security.md
+
+## Supported Versions
+
+Patches for security vulnerabilities will be released for the following versions:
+
+| Version | Supported          |
+| ------- | ------------------ |
+| 4.x     | :white_check_mark: |
+| < 4.x   | :x:                |

requirements.txt

@@ -37,7 +37,7 @@ lxml
 mako
 # esgf-compute-api (cwt) needs oauthlib but doesn't add it in their requirements
 oauthlib
-owslib>=0.19.2
+owslib==0.25.0
 pymongo
 # pyproj>=2 employed by OWSLib, but make requirements stricter
 pyproj>=3
@@ -48,7 +48,7 @@ pyramid_mako
 python-dateutil
 pyramid_rewrite
 pytz
-pywps==4.4.3
+pywps==4.5.0
 pyyaml>=5.2
 # required by: cwltool => schema_salad, make requirement stricter because it has longer wheel build time
 rdflib-jsonld==0.5.0

tests/functional/test_docker_app.py

@@ -2,12 +2,12 @@
 import os
 import tempfile
 
-import lxml.etree
 import pytest
 from owslib.wps import ComplexDataInput, WPSExecution
 
 from tests.functional.utils import WpsPackageConfigBase
 from tests.utils import mocked_execute_process, mocked_sub_requests
+from weaver import xml_util
 from weaver.execute import EXECUTE_MODE_ASYNC, EXECUTE_RESPONSE_DOCUMENT, EXECUTE_TRANSMISSION_MODE_REFERENCE
 from weaver.formats import CONTENT_TYPE_ANY_XML, CONTENT_TYPE_APP_JSON, CONTENT_TYPE_APP_XML, CONTENT_TYPE_TEXT_PLAIN
 from weaver.processes.wps_package import CWL_REQUIREMENT_APP_DOCKER
@@ -208,7 +208,7 @@ def wps_execute(self, version, accept):
                 wps_outputs = [(self.out_key, True)]  # as reference
                 wps_exec = WPSExecution(version=version, url=wps_url)
                 wps_req = wps_exec.buildRequest(self.process_id, wps_inputs, wps_outputs)
-                wps_data = lxml.etree.tostring(wps_req)
+                wps_data = xml_util.tostring(wps_req)
                 wps_headers = {"Accept": accept, "Content-Type": CONTENT_TYPE_APP_XML}
                 wps_params = None
             resp = mocked_sub_requests(self.app, wps_method, wps_url,
@@ -219,8 +219,8 @@ def wps_execute(self, version, accept):
             # parse response status
             if accept == CONTENT_TYPE_APP_XML:
                 assert resp.content_type in CONTENT_TYPE_ANY_XML, test_content
-                xml = lxml.etree.fromstring(str2bytes(resp.text))
-                status_url = xml.get("statusLocation")
+                xml_body = xml_util.fromstring(str2bytes(resp.text))
+                status_url = xml_body.get("statusLocation")
                 job_id = status_url.split("/")[-1]
             elif accept == CONTENT_TYPE_APP_JSON:
                 assert resp.content_type == CONTENT_TYPE_APP_JSON, test_content

tests/functional/test_wps_app.py

@@ -12,7 +12,6 @@
 import pyramid.testing
 import pytest
 import xmltodict
-from lxml import etree
 
 from tests.utils import (
     get_test_weaver_app,
@@ -23,6 +22,7 @@
     setup_config_with_pywps,
     setup_mongodb_processstore
 )
+from weaver import xml_util
 from weaver.formats import CONTENT_TYPE_ANY_XML, CONTENT_TYPE_APP_XML
 from weaver.processes.wps_default import HelloWPS
 from weaver.processes.wps_testing import WpsTestProcess
@@ -85,8 +85,8 @@ def test_getcaps_filtered_processes_by_visibility(self):
         assert resp.status_code == 200
         assert resp.content_type in CONTENT_TYPE_ANY_XML
         resp.mustcontain("<wps:ProcessOfferings>")
-        root = etree.fromstring(str2bytes(resp.text))  # test response has no 'content'
-        process_offerings = list(filter(lambda e: "ProcessOfferings" in e.tag, root.iter(etree.Element)))
+        root = xml_util.fromstring(str2bytes(resp.text))  # test response has no 'content'
+        process_offerings = list(filter(lambda e: "ProcessOfferings" in e.tag, root.iter(xml_util.Element)))
         assert len(process_offerings) == 1
         processes = [p for p in process_offerings[0]]
         ids = [pi.text for pi in [list(filter(lambda e: e.tag.endswith("Identifier"), p))[0] for p in processes]]

tests/functional/test_wps_package.py

@@ -12,6 +12,7 @@
 import logging
 import os
 import tempfile
+from copy import deepcopy
 from inspect import cleandoc
 from typing import TYPE_CHECKING
 
@@ -33,6 +34,7 @@
 )
 from weaver.execute import EXECUTE_MODE_ASYNC, EXECUTE_RESPONSE_DOCUMENT, EXECUTE_TRANSMISSION_MODE_REFERENCE
 from weaver.formats import (
+    ACCEPT_LANGUAGES,
     CONTENT_TYPE_APP_JSON,
     CONTENT_TYPE_APP_NETCDF,
     CONTENT_TYPE_APP_TAR,
@@ -1160,6 +1162,58 @@ def test_valid_io_min_max_occurs_as_str_or_int(self):
                     "Field '{}' of input '{}'({}) is expected to be '{}' but was '{}'" \
                     .format(field, process_input, i, proc_in_exp, proc_in_res)
 
+    def test_execute_job_with_accept_languages(self):
+        """
+        Test that different accept language matching supported languages all successfully execute and apply them.
+
+        Invalid accept languages must be correctly reported as not supported.
+        """
+        cwl = {
+            "cwlVersion": "v1.0",
+            "class": "CommandLineTool",
+            "baseCommand": "echo",
+            "inputs": {"message": {"type": "string", "inputBinding": {"position": 1}}},
+            "outputs": {"output": {"type": "File", "outputBinding": {"glob": "stdout.log"}}}
+        }
+        body = {
+            "processDescription": {"process": {"id": self._testMethodName}},
+            "deploymentProfileName": "http://www.opengis.net/profiles/eoc/wpsApplication",
+            "executionUnit": [{"unit": cwl}],
+        }
+        self.deploy_process(body)
+        exec_body = {
+            "mode": EXECUTE_MODE_ASYNC,
+            "response": EXECUTE_RESPONSE_DOCUMENT,
+            "inputs": [{"id": "message", "value": "test"}],
+            "outputs": [{"id": "output", "transmissionMode": EXECUTE_TRANSMISSION_MODE_REFERENCE}]
+        }
+        headers = deepcopy(self.json_headers)
+
+        with contextlib.ExitStack() as stack_exec:
+            for mock_exec in mocked_execute_process():
+                stack_exec.enter_context(mock_exec)
+            proc_url = "/processes/{}/jobs".format(self._testMethodName)
+
+            valid_languages = [(lang, True) for lang in ACCEPT_LANGUAGES]
+            wrong_languages = [(lang, False) for lang in ["ru", "fr-CH"]]
+            for lang, accept in valid_languages + wrong_languages:
+                headers["Accept-Language"] = lang
+                resp = mocked_sub_requests(self.app, "post_json", proc_url, timeout=5, expect_errors=not accept,
+                                           data=exec_body, headers=headers, only_local=True)
+                code = resp.status_code
+                if accept:  # must execute until completion with success
+                    assert code in [200, 201], "Failed with: [{}]\nReason:\n{}".format(code, resp.json)
+                    status_url = resp.json.get("location")
+                    self.monitor_job(status_url, timeout=5, return_status=True)  # wait until success
+                    job_id = resp.json.get("jobID")
+                    job = self.job_store.fetch_by_id(job_id)
+                    assert job.accept_language == lang
+                else:
+                    # job not even created
+                    assert code == 406, "Error code should indicate not acceptable header"
+                    desc = resp.json.get("description")
+                    assert "language" in desc and lang in desc, "Expected error description to indicate bad language"
+
     @mocked_aws_credentials
     @mocked_aws_s3
     @mocked_http_file

tests/functional/utils.py

@@ -90,12 +90,16 @@ def _try_get_logs(self, status_url):
             return "Error logs:\n{}".format("\n".join(_resp.json))
         return ""
 
-    def monitor_job(self, status_url, timeout=None, delta=None):
-        # type: (str, Optional[int], Optional[int]) -> JSON
+    def monitor_job(self, status_url, timeout=None, delta=None, return_status=False):
+        # type: (str, Optional[int], Optional[int], bool) -> JSON
         """
         Job polling of status URL until completion or timeout.
 
-        :return: result of the successful job
+        :param status_url: URL with job ID where to monitor execution.
+        :param timeout: timeout of monitoring until completion or abort.
+        :param delta: interval (seconds) between polling monitor requests.
+        :param return_status: return final status body instead of results once job completed.
+        :return: result of the successful job, or the status body if requested.
         :raises AssertionError: when job fails or took too long to complete.
         """
 
@@ -120,6 +124,8 @@ def check_job_status(_resp, running=False):
             once = False
             left -= delta
         check_job_status(resp)
+        if return_status:
+            return resp.json
         resp = self.app.get("{}/results".format(status_url), headers=self.json_headers)
         assert resp.status_code == 200, "Error job info:\n{}".format(resp.json)
         return resp.json

tests/processes/test_wps3_process.py

@@ -65,8 +65,9 @@ def mock_update_status(*_, **__):
             wps_params = {"service": "wps", "request": "execute", "identifier": test_process, "version": "1.0.0"}
             req = Request(method="GET", params=wps_params)
             setattr(req, "args", wps_params)
+            setattr(req, "path", "/wps")
             req = WPSRequest(req)
-            wps = Wps3Process({}, {}, test_process, req, mock_update_status)
+            wps = Wps3Process({}, {}, test_process, req, mock_update_status)  # noqa
             try:
                 wps.execute(test_cwl_inputs, "", {})
             except TestDoneEarlyExit:

tests/test_utils.py

@@ -11,7 +11,6 @@
 
 import mock
 import pytest
-from lxml import etree
 from pyramid.httpexceptions import (
     HTTPConflict,
     HTTPCreated,
@@ -26,7 +25,7 @@
 from requests.exceptions import HTTPError as RequestsHTTPError
 
 from tests.utils import mocked_aws_credentials, mocked_aws_s3, mocked_aws_s3_bucket_test_file, mocked_file_response
-from weaver import status
+from weaver import status, xml_util
 from weaver.utils import (
     NullType,
     assert_sane_name,
@@ -116,7 +115,7 @@ def test_xml_strip_ns():
 version="1.0.0"
 xsi:schemaLocation="http://www.opengis.net/wps/1.0.0 http://schemas.opengis.net/wps/1.0.0/wpsExecute_request.xsd"/>"""
 
-    doc = etree.fromstring(wps_xml)
+    doc = xml_util.fromstring(wps_xml)
     assert doc.tag == "{http://www.opengis.net/wps/1.0.0}Execute"
     xml_strip_ns(doc)
     assert doc.tag == "Execute"

weaver/datatype.py

@@ -11,13 +11,13 @@
 from typing import TYPE_CHECKING
 from urllib.parse import urljoin, urlparse
 
-import lxml.etree
 import pyramid.httpexceptions
 import requests.exceptions
 from dateutil.parser import parse as dt_parse
 from owslib.wps import Process as ProcessOWS, WPSException
 from pywps import Process as ProcessWPS
 
+from weaver import xml_util
 from weaver.exceptions import ProcessInstanceError
 from weaver.execute import (
     EXECUTE_CONTROL_OPTION_ASYNC,
@@ -44,7 +44,6 @@
     STATUS_UNKNOWN,
     map_status
 )
-from weaver.typedefs import XML
 from weaver.utils import localize_datetime  # for backward compatibility of previously saved jobs not time-locale-aware
 from weaver.utils import (
     fully_qualified_name,
@@ -874,8 +873,8 @@ def request(self, request):
         """
         XML request for WPS execution submission as string (binary).
         """
-        if isinstance(request, XML):
-            request = lxml.etree.tostring(request)
+        if isinstance(request, xml_util.XML):
+            request = xml_util.tostring(request)
         self["request"] = request
 
     @property
@@ -892,8 +891,8 @@ def response(self, response):
         """
         XML status response from WPS execution submission as string (binary).
         """
-        if isinstance(response, XML):
-            response = lxml.etree.tostring(response)
+        if isinstance(response, xml_util.XML):
+            response = xml_util.tostring(response)
         self["response"] = response
 
     def _job_url(self, base_url=None):

weaver/formats.py

@@ -20,9 +20,9 @@
 ACCEPT_LANGUAGE_EN_US = "en-US"
 
 ACCEPT_LANGUAGES = frozenset([
+    ACCEPT_LANGUAGE_EN_US,  # place first to match default of PyWPS and most existing remote servers
     ACCEPT_LANGUAGE_EN_CA,
     ACCEPT_LANGUAGE_FR_CA,
-    ACCEPT_LANGUAGE_EN_US,
 ])
 
 # Content-Types

weaver/processes/builtin/metalink2netcdf.py

@@ -9,15 +9,14 @@
 import sys
 from tempfile import TemporaryDirectory
 
-from lxml import etree
-
 CUR_DIR = os.path.abspath(os.path.dirname(__file__))
 sys.path.insert(0, CUR_DIR)
 # root to allow 'from weaver import <...>'
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(CUR_DIR))))
 
 # place weaver specific imports after sys path fixing to ensure they are found from external call
 # pylint: disable=C0413,wrong-import-order
+from weaver import xml_util  # isort:skip # noqa: E402
 from weaver.utils import fetch_file  # isort:skip # noqa: E402
 
 PACKAGE_NAME = os.path.split(os.path.splitext(__file__)[0])[-1]
@@ -47,8 +46,7 @@ def m2n(metalink_reference, index, output_dir):
             LOGGER.debug("Fetching Metalink file: [%s]", metalink_reference)
             metalink_path = fetch_file(metalink_reference, tmp_dir, timeout=10, retry=3)
             LOGGER.debug("Reading Metalink file: [%s]", metalink_path)
-            parser = etree.HTMLParser()
-            xml_data = etree.parse(metalink_path, parser)
+            xml_data = xml_util.parse(metalink_path)
             LOGGER.debug("Parsing Metalink file references.")
             nc_file_url = xml_data.xpath("string(//metalink/file[" + str(index) + "]/metaurl)")
             LOGGER.debug("Fetching NetCDF reference from Metalink file: [%s]", metalink_reference)