Merge branch 'master' of github.com:ckan/ckanext-spatial

.github/workflows/test-postgis.yml

@@ -17,7 +17,11 @@ jobs:
     needs: lint
     strategy:
       matrix:
-        ckan-version: [2.9, 2.9-py2]
+        include:
+          - ckan-version: 2.9
+            harvester-version: 'master'
+          - ckan-version: 2.9-py2
+            harvester-version: 'v1.4.2'
       fail-fast: false
 
     name: CKAN ${{ matrix.ckan-version }}
@@ -53,9 +57,10 @@ jobs:
         psql --host=postgres --username=postgres --command="CREATE USER datastore_write WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
         psql --host=postgres --username=postgres --command="CREATE USER datastore_read WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;"
         createdb --encoding=utf-8 --host=postgres --username=postgres --owner=datastore_write datastore_test
+
     - name: Install harvester
       run: |
-        git clone https://github.com/ckan/ckanext-harvest
+        git clone --branch ${{ matrix.harvester-version }} https://github.com/ckan/ckanext-harvest
         cd ckanext-harvest
         pip install -r pip-requirements.txt
         pip install -r dev-requirements.txt
@@ -72,6 +77,19 @@ jobs:
           gcc \
           libxml2-dev \
           libxslt-dev
+
+    - name: Patch to test pyproj
+      if: ${{ matrix.ckan-version == '2.9-py2' || matrix.ckan-version == '2.8' || matrix.ckan-version == '2.7' }}
+      run: |
+        pip install cython==0.29.36
+        pip install --no-use-pep517 pyproj==2.2.2
+
+    - name: Patch to test pyproj
+      if: ${{ matrix.ckan-version == '2.9' }}
+      run: |
+        pip install cython==0.29.36
+        pip install --no-use-pep517 pyproj==2.6.1
+
     - name: Install dependencies (python2)
       if: ${{ matrix.ckan-version == '2.9-py2' || matrix.ckan-version == '2.8' || matrix.ckan-version == '2.7' }}
       run: |
@@ -86,6 +104,7 @@ jobs:
           python3-dev
         pip install -r requirements.txt
         pip install -r requirements-postgis.txt
+
     - name: Install requirements
       run: |
         pip install -e .

.github/workflows/test.yml

@@ -23,12 +23,15 @@ jobs:
           - ckan-version: "2.10"
             solr-image: "2.10-spatial"
             requirements-file: 'requirements.txt'
+            harvester-version: 'master'
           - ckan-version: 2.9
             solr-image: 2.9-solr8-spatial
             requirements-file: 'requirements.txt'
+            harvester-version: 'master'
           - ckan-version: 2.9-py2
             solr-image: 2.9-py2-solr8-spatial
             requirements-file: 'requirements-py2.txt'
+            harvester-version: 'v1.4.2'
       fail-fast: false
 
     name: CKAN ${{ matrix.ckan-version }}, Solr ${{ matrix.solr-image }}
@@ -89,6 +92,18 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-spatial-ckan-${{ matrix.ckan-version }}-${{ hashFiles('*requirements*.txt') }}
 
+    - name: Patch to test pyproj
+      if: ${{ matrix.ckan-version == '2.9' }}
+      run: |
+        pip install cython==0.29.36
+        pip install --no-use-pep517 pyproj==2.6.1
+
+    - name: Patch to test pyproj
+      if: ${{ matrix.ckan-version == '2.9-py2' }}
+      run: |
+        pip install cython==0.28.4
+        pip install --no-use-pep517 pyproj==2.2.2
+
     - name: Install dependencies from ${{ matrix.requirements-file }}
       if: steps.cache.outputs.cache-hit != 'true'
       run: |
@@ -98,7 +113,7 @@ jobs:
       if: steps.cache.outputs.cache-hit != 'true'
       run: |
         echo "Installing harvester"
-        git clone --depth 1 https://github.com/ckan/ckanext-harvest
+        git clone --depth 1 --branch ${{ matrix.harvester-version }} https://github.com/ckan/ckanext-harvest
         cd ckanext-harvest
         echo "upgrade pip"
         pip install --upgrade pip

ckanext/spatial/harvesters/base.py

@@ -23,7 +23,7 @@
 from ckan import model
 from ckan.lib.helpers import json
 from ckan import logic
-from ckan.lib.navl.validators import not_empty
+from ckan.lib.navl.validators import not_empty, unicode_safe
 from ckan.lib.search.index import PackageSearchIndex
 from ckanext.harvest.harvesters.base import munge_tag
 
@@ -33,7 +33,7 @@
 from ckanext.spatial.validation import Validators, all_validators
 from ckanext.spatial.harvested_metadata import ISODocument
 from ckanext.spatial.interfaces import ISpatialHarvester
-from ckantoolkit import config
+from ckantoolkit import config, unicode_safe
 
 log = logging.getLogger(__name__)
 
@@ -646,7 +646,7 @@ def import_stage(self, harvest_object):
 
         # The default package schema does not like Upper case tags
         tag_schema = logic.schema.default_tags_schema()
-        tag_schema['name'] = [not_empty, six.text_type]
+        tag_schema['name'] = [not_empty, unicode_safe]
 
         # Flag this object as the current one
         harvest_object.current = True
@@ -660,7 +660,7 @@ def import_stage(self, harvest_object):
             # We need to explicitly provide a package ID, otherwise ckanext-spatial
             # won't be be able to link the extent to the package.
             package_dict['id'] = six.text_type(uuid.uuid4())
-            package_schema['id'] = [six.text_type]
+            package_schema['id'] = [unicode_safe]
 
             # Save reference to the package on the object
             harvest_object.package_id = package_dict['id']

ckanext/spatial/harvesters/waf.py

@@ -271,7 +271,7 @@ def _get_scraper(server):
         return 'apache'
     if 'nginx' in server.lower():
         return 'nginx'
-    if server == 'Microsoft-IIS/7.5':
+    if 'Microsoft-IIS' in server:
         return 'iis'
     else:
         return 'other'

ckanext/spatial/lib/csw_client.py

@@ -65,7 +65,7 @@ class CswService(OwsService):
     """
     Perform various operations on a CSW service
     """
-    from owslib.csw import CatalogueServiceWeb as _Implementation
+    from owslib.catalogue.csw2 import CatalogueServiceWeb as _Implementation
 
     def __init__(self, endpoint=None):
         super(CswService, self).__init__(endpoint)
@@ -74,7 +74,7 @@ def __init__(self, endpoint=None):
     def getrecords(self, qtype=None, keywords=[],
                    typenames="csw:Record", esn="brief",
                    skip=0, count=10, outputschema="gmd", **kw):
-        from owslib.csw import namespaces
+        from owslib.catalogue.csw2 import namespaces
         constraints = []
         csw = self._ows(**kw)
 
@@ -102,7 +102,7 @@ def getrecords(self, qtype=None, keywords=[],
     def getidentifiers(self, qtype=None, typenames="csw:Record", esn="brief",
                        keywords=[], limit=None, page=10, outputschema="gmd",
                        startposition=0, cql=None, **kw):
-        from owslib.csw import namespaces
+        from owslib.catalogue.csw2 import namespaces
         constraints = []
         csw = self._ows(**kw)
 
@@ -154,7 +154,7 @@ def getidentifiers(self, qtype=None, typenames="csw:Record", esn="brief",
             kwa["startposition"] = startposition
 
     def getrecordbyid(self, ids=[], esn="full", outputschema="gmd", **kw):
-        from owslib.csw import namespaces
+        from owslib.catalogue.csw2 import namespaces
         csw = self._ows(**kw)
         kwa = {
             "esn": esn,

doc/install.rst

@@ -9,7 +9,7 @@ Check the Troubleshooting_ section if you get errors at any stage.
           installation, so you can skip all the following steps if you just want the previews.
 
 .. note:: Starting from ckanext-spatial 2.0.0 **PostGIS is no longer required** to use the extension,
-          and its use has been deprected. If for some reason you still need to use the old PostGIS backend
+          and its use has been deprecated. If for some reason you still need to use the old PostGIS backend
           see :ref:`legacy_postgis`.
 
 All commands assume an existing CKAN database named ``ckan_default``.
@@ -62,9 +62,9 @@ Legacy PostGIS support
 ----------------------
 
 Starting from ckanext-spatial 2.0.0 PostGIS is **no longer required** to use the extension,
-and its use has been deprected. If for some reason you are using the deprecated ``postgis``
+and its use has been deprecated. If for some reason you are using the deprecated ``postgis``
 search backend or you still need to store the dataset extents in the PostGIS
-enabled ``package_extent`` table you can use the following steps, but note that this support
+enabled ``package_extent`` table, you can use the following steps, but note that this support
 will removed in the future so update your workflows accordingly.
 
 #. Add the ``ckan.spatial.use_postgis=true`` config option to your ini file.
@@ -78,8 +78,8 @@ On CKAN 2.8 and below use::
 
     (pyenv) $ paster --plugin=ckanext-spatial spatial initdb [srid] --config=mysite.ini
 
-You can define the SRID of the geometry column. Default is 4326. If you are not
-familiar with projections, we recommend to use the default value.
+You can define the SRID of the geometry column. The default is 4326, and if you are not
+familiar with projections, we recommend using the default value.
 
 When enabling the spatial metadata, you can define the projection in which
 extents are stored in the database with the following option. Use the EPSG code

requirements-py2.txt

@@ -1,4 +1,5 @@
 ckantoolkit
+cython==0.29.36
 Shapely>=1.2.13,<2.0.0  
 pyproj==2.2.2
 OWSLib==0.18.0

requirements.txt

@@ -4,9 +4,9 @@ argparse
 pyparsing>=2.1.10
 requests>=1.1.0
 six
-
+cython==0.29.36; python_version < '3.9'
 pyproj==2.6.1; python_version < '3.9'
-pyproj==3.4.1; python_version >= '3.9'
+pyproj @ git+https://github.com/pyproj4/pyproj.git@main; python_version >= '3.9'
 
 Shapely==2.0.1
 OWSLib==0.28.1