diff --git a/.github/workflows/test-postgis.yml b/.github/workflows/test-postgis.yml index fc357549..51562526 100644 --- a/.github/workflows/test-postgis.yml +++ b/.github/workflows/test-postgis.yml @@ -17,7 +17,11 @@ jobs: needs: lint strategy: matrix: - ckan-version: [2.9, 2.9-py2] + include: + - ckan-version: 2.9 + harvester-version: 'master' + - ckan-version: 2.9-py2 + harvester-version: 'v1.4.2' fail-fast: false name: CKAN ${{ matrix.ckan-version }} @@ -53,9 +57,10 @@ jobs: psql --host=postgres --username=postgres --command="CREATE USER datastore_write WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;" psql --host=postgres --username=postgres --command="CREATE USER datastore_read WITH PASSWORD 'pass' NOSUPERUSER NOCREATEDB NOCREATEROLE;" createdb --encoding=utf-8 --host=postgres --username=postgres --owner=datastore_write datastore_test + - name: Install harvester run: | - git clone https://github.com/ckan/ckanext-harvest + git clone --branch ${{ matrix.harvester-version }} https://github.com/ckan/ckanext-harvest cd ckanext-harvest pip install -r pip-requirements.txt pip install -r dev-requirements.txt @@ -72,6 +77,19 @@ jobs: gcc \ libxml2-dev \ libxslt-dev + + - name: Patch to test pyproj + if: ${{ matrix.ckan-version == '2.9-py2' || matrix.ckan-version == '2.8' || matrix.ckan-version == '2.7' }} + run: | + pip install cython==0.29.36 + pip install --no-use-pep517 pyproj==2.2.2 + + - name: Patch to test pyproj + if: ${{ matrix.ckan-version == '2.9' }} + run: | + pip install cython==0.29.36 + pip install --no-use-pep517 pyproj==2.6.1 + - name: Install dependencies (python2) if: ${{ matrix.ckan-version == '2.9-py2' || matrix.ckan-version == '2.8' || matrix.ckan-version == '2.7' }} run: | @@ -86,6 +104,7 @@ jobs: python3-dev pip install -r requirements.txt pip install -r requirements-postgis.txt + - name: Install requirements run: | pip install -e . diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ca970f89..abc2340e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,12 +23,15 @@ jobs: - ckan-version: "2.10" solr-image: "2.10-spatial" requirements-file: 'requirements.txt' + harvester-version: 'master' - ckan-version: 2.9 solr-image: 2.9-solr8-spatial requirements-file: 'requirements.txt' + harvester-version: 'master' - ckan-version: 2.9-py2 solr-image: 2.9-py2-solr8-spatial requirements-file: 'requirements-py2.txt' + harvester-version: 'v1.4.2' fail-fast: false name: CKAN ${{ matrix.ckan-version }}, Solr ${{ matrix.solr-image }} @@ -89,6 +92,18 @@ jobs: restore-keys: | ${{ runner.os }}-spatial-ckan-${{ matrix.ckan-version }}-${{ hashFiles('*requirements*.txt') }} + - name: Patch to test pyproj + if: ${{ matrix.ckan-version == '2.9' }} + run: | + pip install cython==0.29.36 + pip install --no-use-pep517 pyproj==2.6.1 + + - name: Patch to test pyproj + if: ${{ matrix.ckan-version == '2.9-py2' }} + run: | + pip install cython==0.28.4 + pip install --no-use-pep517 pyproj==2.2.2 + - name: Install dependencies from ${{ matrix.requirements-file }} if: steps.cache.outputs.cache-hit != 'true' run: | @@ -98,7 +113,7 @@ jobs: if: steps.cache.outputs.cache-hit != 'true' run: | echo "Installing harvester" - git clone --depth 1 https://github.com/ckan/ckanext-harvest + git clone --depth 1 --branch ${{ matrix.harvester-version }} https://github.com/ckan/ckanext-harvest cd ckanext-harvest echo "upgrade pip" pip install --upgrade pip diff --git a/ckanext/spatial/harvesters/base.py b/ckanext/spatial/harvesters/base.py index 947defc4..9613a7ee 100644 --- a/ckanext/spatial/harvesters/base.py +++ b/ckanext/spatial/harvesters/base.py @@ -23,7 +23,7 @@ from ckan import model from ckan.lib.helpers import json from ckan import logic -from ckan.lib.navl.validators import not_empty +from ckan.lib.navl.validators import not_empty, unicode_safe from ckan.lib.search.index import PackageSearchIndex from ckanext.harvest.harvesters.base import munge_tag @@ -33,7 +33,7 @@ from ckanext.spatial.validation import Validators, all_validators from ckanext.spatial.harvested_metadata import ISODocument from ckanext.spatial.interfaces import ISpatialHarvester -from ckantoolkit import config +from ckantoolkit import config, unicode_safe log = logging.getLogger(__name__) @@ -646,7 +646,7 @@ def import_stage(self, harvest_object): # The default package schema does not like Upper case tags tag_schema = logic.schema.default_tags_schema() - tag_schema['name'] = [not_empty, six.text_type] + tag_schema['name'] = [not_empty, unicode_safe] # Flag this object as the current one harvest_object.current = True @@ -660,7 +660,7 @@ def import_stage(self, harvest_object): # We need to explicitly provide a package ID, otherwise ckanext-spatial # won't be be able to link the extent to the package. package_dict['id'] = six.text_type(uuid.uuid4()) - package_schema['id'] = [six.text_type] + package_schema['id'] = [unicode_safe] # Save reference to the package on the object harvest_object.package_id = package_dict['id'] diff --git a/ckanext/spatial/harvesters/waf.py b/ckanext/spatial/harvesters/waf.py index 4ea3247f..1dd96dee 100644 --- a/ckanext/spatial/harvesters/waf.py +++ b/ckanext/spatial/harvesters/waf.py @@ -271,7 +271,7 @@ def _get_scraper(server): return 'apache' if 'nginx' in server.lower(): return 'nginx' - if server == 'Microsoft-IIS/7.5': + if 'Microsoft-IIS' in server: return 'iis' else: return 'other' diff --git a/ckanext/spatial/lib/csw_client.py b/ckanext/spatial/lib/csw_client.py index 0ac075e5..b553344f 100644 --- a/ckanext/spatial/lib/csw_client.py +++ b/ckanext/spatial/lib/csw_client.py @@ -65,7 +65,7 @@ class CswService(OwsService): """ Perform various operations on a CSW service """ - from owslib.csw import CatalogueServiceWeb as _Implementation + from owslib.catalogue.csw2 import CatalogueServiceWeb as _Implementation def __init__(self, endpoint=None): super(CswService, self).__init__(endpoint) @@ -74,7 +74,7 @@ def __init__(self, endpoint=None): def getrecords(self, qtype=None, keywords=[], typenames="csw:Record", esn="brief", skip=0, count=10, outputschema="gmd", **kw): - from owslib.csw import namespaces + from owslib.catalogue.csw2 import namespaces constraints = [] csw = self._ows(**kw) @@ -102,7 +102,7 @@ def getrecords(self, qtype=None, keywords=[], def getidentifiers(self, qtype=None, typenames="csw:Record", esn="brief", keywords=[], limit=None, page=10, outputschema="gmd", startposition=0, cql=None, **kw): - from owslib.csw import namespaces + from owslib.catalogue.csw2 import namespaces constraints = [] csw = self._ows(**kw) @@ -154,7 +154,7 @@ def getidentifiers(self, qtype=None, typenames="csw:Record", esn="brief", kwa["startposition"] = startposition def getrecordbyid(self, ids=[], esn="full", outputschema="gmd", **kw): - from owslib.csw import namespaces + from owslib.catalogue.csw2 import namespaces csw = self._ows(**kw) kwa = { "esn": esn, diff --git a/doc/install.rst b/doc/install.rst index ca264832..04895472 100644 --- a/doc/install.rst +++ b/doc/install.rst @@ -9,7 +9,7 @@ Check the Troubleshooting_ section if you get errors at any stage. installation, so you can skip all the following steps if you just want the previews. .. note:: Starting from ckanext-spatial 2.0.0 **PostGIS is no longer required** to use the extension, - and its use has been deprected. If for some reason you still need to use the old PostGIS backend + and its use has been deprecated. If for some reason you still need to use the old PostGIS backend see :ref:`legacy_postgis`. All commands assume an existing CKAN database named ``ckan_default``. @@ -62,9 +62,9 @@ Legacy PostGIS support ---------------------- Starting from ckanext-spatial 2.0.0 PostGIS is **no longer required** to use the extension, -and its use has been deprected. If for some reason you are using the deprecated ``postgis`` +and its use has been deprecated. If for some reason you are using the deprecated ``postgis`` search backend or you still need to store the dataset extents in the PostGIS -enabled ``package_extent`` table you can use the following steps, but note that this support +enabled ``package_extent`` table, you can use the following steps, but note that this support will removed in the future so update your workflows accordingly. #. Add the ``ckan.spatial.use_postgis=true`` config option to your ini file. @@ -78,8 +78,8 @@ On CKAN 2.8 and below use:: (pyenv) $ paster --plugin=ckanext-spatial spatial initdb [srid] --config=mysite.ini -You can define the SRID of the geometry column. Default is 4326. If you are not -familiar with projections, we recommend to use the default value. +You can define the SRID of the geometry column. The default is 4326, and if you are not +familiar with projections, we recommend using the default value. When enabling the spatial metadata, you can define the projection in which extents are stored in the database with the following option. Use the EPSG code diff --git a/requirements-py2.txt b/requirements-py2.txt index cdd3536c..8eff5829 100644 --- a/requirements-py2.txt +++ b/requirements-py2.txt @@ -1,4 +1,5 @@ ckantoolkit +cython==0.29.36 Shapely>=1.2.13,<2.0.0 pyproj==2.2.2 OWSLib==0.18.0 diff --git a/requirements.txt b/requirements.txt index fa3e6989..c0c58d1a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,9 +4,9 @@ argparse pyparsing>=2.1.10 requests>=1.1.0 six - +cython==0.29.36; python_version < '3.9' pyproj==2.6.1; python_version < '3.9' -pyproj==3.4.1; python_version >= '3.9' +pyproj @ git+https://github.com/pyproj4/pyproj.git@main; python_version >= '3.9' Shapely==2.0.1 OWSLib==0.28.1