#9320 - Support reading/loading of cloud-optimized geotiff (COG) (#9394…

…) (#9484)

* #9320 - Support reading/loading of cloud-optimized geotiff (COG)

* Url validation modified

* Update cog layer model

* unit test

(cherry picked from commit 920ff39)

docs/developer-guide/maps-configuration.md

@@ -208,6 +208,7 @@ In the case of the background the `thumbURL` is used to show a preview of the la
 - `empty`: special type for empty background
 - `3dtiles`: 3d tiles layers
 - `terrain`: layers that define the elevation profile of the terrain
+- `cog`: Cloud Optimized GeoTIFF layers
 
 #### WMS
 
@@ -1493,6 +1494,24 @@ In order to use these layers they need to be added to the `additionalLayers` in
 }
 ```
 
+#### Cloud Optimized GeoTIFF (COG)
+
+i.e.
+
+```javascript
+{
+    "type": "cog",
+    "title": "Title",
+    "group": "background",
+    "visibility": false,
+    "name": "Name",
+    "sources": [
+        { "url": "https://host-sample/cog1.tif" }, 
+        { "url": "https://host-sample/cog2.tif" }
+    ]
+}
+```
+
 ## Layer groups
 
 Inside the map configuration, near the `layers` entry, you can find also the `groups` entry. This array contains information about the groups in the TOC.

docs/user-guide/catalog.md

@@ -346,3 +346,8 @@ In **general settings of** 3D Tiles service, the user can specify the title to a
     Since the Google Photorealistic 3D Tiles are not ‘survey-grade’ at this time, the use of certain MapStore tools could be considered derivative and, for this reason, prohibited. Please, make sure you have read the [Google conditions of use](https://developers.google.com/maps/documentation/tile/policies)
     (some [FAQs](https://cloud.google.com/blog/products/maps-platform/commonly-asked-questions-about-our-recently-launched-photorealistic-3d-tiles) are also available online for this purpose) before providing Google Photorealistic 3D Tile in your MapStore maps in order to enable only allowed tools (e.g. *Measurement* and *Identify* tools should be probably disabled).
     For this purpose it is possible to appropriately set the [configuration of MapStore plugins](../../developer-guide/maps-configuration/#map-options)  to exclude tools that could conflict with Google policies. Alternatively, it is possible to use a dedicated [application context](application-context.md#configure-plugins) to show Photorealistic 3D Tiles by including only the permitted tools within it.
+
+### Cloud Optimized GeoTIFF
+
+A Cloud Optimized GeoTIFF (COG) is a regular GeoTIFF file, aimed at being hosted on a HTTP file server, with an internal organization that enables more efficient workflows on the cloud. It does this by leveraging the ability of clients issuing ​HTTP GET range requests to ask for just the parts of a file they need.
+MapStore allows to add COG as layers and backgrounds. Through the Catalog tool, a multiple url sources of COG are obtained and converted to layers as each url corresponds to a layer

web/client/api/catalog/COG.js

@@ -0,0 +1,114 @@
+/*
+ * Copyright 2023, GeoSolutions Sas.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import get from 'lodash/get';
+import { Observable } from 'rxjs';
+import { isValidURL } from '../../utils/URLUtils';
+
+export const COG_LAYER_TYPE = 'cog';
+const searchAndPaginate = (layers, startPosition, maxRecords, text) => {
+
+    const filteredLayers = layers
+        .filter(({ title = "" } = {}) => !text
+            || title.toLowerCase().indexOf(text.toLowerCase()) !== -1
+        );
+    const records = filteredLayers
+        .filter((layer, index) => index >= startPosition - 1 && index < startPosition - 1 + maxRecords);
+    return {
+        numberOfRecordsMatched: filteredLayers.length,
+        numberOfRecordsReturned: records.length,
+        nextRecord: startPosition + Math.min(maxRecords, filteredLayers.length) + 1,
+        records
+    };
+};
+export const getRecords = (url, startPosition, maxRecords, text, info = {}) => {
+    const service = get(info, 'options.service');
+    let layers = [];
+    if (service.url) {
+        const urls = service.url?.split(',')?.map(_url => _url?.trim());
+        // each url corresponds to a layer
+        layers = urls.map((_url, index) => {
+            const title = _url.split('/')?.pop()?.replace('.tif', '') || `COG_${index}`;
+            return {
+                ...service,
+                title,
+                type: COG_LAYER_TYPE,
+                sources: [{url: _url}],
+                options: service.options || {}
+            };
+        });
+    }
+    // fake request with generated layers
+    return new Promise((resolve) => {
+        resolve(searchAndPaginate(layers, startPosition, maxRecords, text));
+    });
+
+
+};
+
+export const textSearch = (url, startPosition, maxRecords, text, info = {}) => {
+    return getRecords(url, startPosition, maxRecords, text, info);
+};
+
+const validateCog = (service) => {
+    const urls = service.url?.split(',');
+    const isValid = urls.every(url => isValidURL(url?.trim()));
+    if (service.title && isValid) {
+        return Observable.of(service);
+    }
+    const error = new Error("catalog.config.notValidURLTemplate");
+    // insert valid URL;
+    throw error;
+};
+export const validate = service => {
+    return validateCog(service);
+};
+export const testService = service => {
+    return Observable.of(service);
+};
+
+export const getCatalogRecords = (data) => {
+    if (data && data.records) {
+        return data.records.map(record => {
+            return {
+                serviceType: COG_LAYER_TYPE,
+                isValid: record.sources?.every(source => isValidURL(source.url)),
+                title: record.title || record.provider,
+                sources: record.sources,
+                options: record.options,
+                references: []
+            };
+        });
+    }
+    return null;
+};
+
+/**
+ * Converts a record into a layer
+ */
+export const cogToLayer = (record) => {
+    return {
+        type: COG_LAYER_TYPE,
+        visibility: true,
+        sources: record.sources,
+        title: record.title,
+        options: record.options,
+        name: record.title
+    };
+};
+
+const recordToLayer = (record, options) => {
+    return cogToLayer(record, options);
+};
+
+export const getLayerFromRecord = (record, options, asPromise) => {
+    if (asPromise) {
+        return Promise.resolve(recordToLayer(record, options));
+    }
+    return recordToLayer(record, options);
+};

web/client/api/catalog/__tests__/COG-test.js

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2023, GeoSolutions Sas.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+import { getLayerFromRecord, getCatalogRecords, validate, COG_LAYER_TYPE} from '../COG';
+import expect from 'expect';
+
+
+const record = {sources: [{url: "some.tif"}], title: "some", options: []};
+describe('COG (Abstraction) API', () => {
+    beforeEach(done => {
+        setTimeout(done);
+    });
+
+    afterEach(done => {
+        setTimeout(done);
+    });
+    it('test getLayerFromRecord', () => {
+        const layer = getLayerFromRecord(record, null);
+        expect(layer.title).toBe(record.title);
+        expect(layer.visibility).toBeTruthy();
+        expect(layer.type).toBe(COG_LAYER_TYPE);
+        expect(layer.sources).toEqual(record.sources);
+        expect(layer.name).toBe(record.title);
+    });
+    it('test getLayerFromRecord as promise', () => {
+        getLayerFromRecord(record, null, true).then((layer) => {
+            expect(layer.title).toBe(record.title);
+            expect(layer.visibility).toBeTruthy();
+            expect(layer.type).toBe(COG_LAYER_TYPE);
+            expect(layer.sources).toEqual(record.sources);
+            expect(layer.name).toBe(record.title);
+        });
+    });
+    it('test getCatalogRecords - empty records', () => {
+        const catalogRecords = getCatalogRecords();
+        expect(catalogRecords).toBeFalsy();
+    });
+    it('test getCatalogRecords', () => {
+        const records = getCatalogRecords({records: [record]});
+        const [{serviceType, isValid, title, sources, options }] = records;
+        expect(serviceType).toBe(COG_LAYER_TYPE);
+        expect(isValid).toBeFalsy();
+        expect(title).toBe(record.title);
+        expect(sources).toEqual(record.sources);
+        expect(options).toEqual(record.options);
+    });
+    it('test validate with invalid url', () => {
+        const service = {title: "some", url: "some.tif"};
+        const error = new Error("catalog.config.notValidURLTemplate");
+        try {
+            validate(service);
+        } catch (e) {
+            expect(e).toEqual(error);
+        }
+    });
+    it('test validate with valid url', () => {
+        const service = {title: "some", url: "https://some.tif"};
+        expect(validate(service)).toBeTruthy();
+    });
+});

web/client/api/catalog/index.js

@@ -15,6 +15,7 @@ import * as wfs from './WFS';
 import * as geojson from './GeoJSON';
 import * as backgrounds from './backgrounds';
 import * as threeDTiles from './ThreeDTiles';
+import * as cog from './COG';
 
 /**
  * APIs collection for catalog.
@@ -49,5 +50,6 @@ export default {
     'wmts': wmts,
     'geojson': geojson,
     'backgrounds': backgrounds,
-    '3dtiles': threeDTiles
+    '3dtiles': threeDTiles,
+    'cog': cog
 };

web/client/components/TOC/DefaultLayer.jsx

@@ -97,7 +97,9 @@ class DefaultLayer extends React.Component {
     };
 
     getVisibilityMessage = () => {
-        if (this.props.node.exclusiveMapType) return this.props.node?.type === '3dtiles' && 'toc.notVisibleSwitchTo3D';
+        if (this.props.node.exclusiveMapType) {
+            return this.props.node?.type === '3dtiles' ? 'toc.notVisibleSwitchTo3D' : this.props.node?.type === 'cog' ? 'toc.notVisibleSwitchTo2D' : '';
+        }
         const maxResolution = this.props.node.maxResolution || Infinity;
         return this.props.resolution >=  maxResolution
             ? 'toc.notVisibleZoomIn'

web/client/components/background/BackgroundSelector.jsx

@@ -126,7 +126,7 @@ class BackgroundSelector extends React.Component {
                                 }}
                                 tooltipId="backgroundSelector.deleteTooltip"
                             />}
-                            {this.props.mapIsEditable && !this.props.enabledCatalog && !!(layer.type === 'wms' || layer.type === 'wmts' || layer.type === 'tms' || layer.type === 'tileprovider') &&
+                            {this.props.mapIsEditable && !this.props.enabledCatalog && ['wms', 'wmts', 'tms', 'tileprovider', 'cog'].includes(layer.type) &&
                                 <ToolbarButton
                                     glyph="wrench"
                                     className="square-button-md background-tool-button edit-button"

web/client/components/catalog/editor/MainForm.jsx

@@ -111,6 +111,25 @@ const TmsURLEditor = ({ serviceTypes = [], onChangeServiceProperty, service = {}
     </FormGroup>);
 };
 
+const COGEditor = ({ service = {}, onChangeUrl = () => { } }) => {
+    return (
+        <FormGroup controlId="URL">
+            <Col xs={12}>
+                <ControlLabel><Message msgId="catalog.urls"/>&nbsp;&nbsp;<InfoPopover text={<HTML msgId="catalog.cog.urlTemplateHint" />} /></ControlLabel>
+                <FormControl
+                    type="text"
+                    style={{
+                        textOverflow: "ellipsis"
+                    }}
+                    placeholder={defaultPlaceholder(service)}
+                    value={service && service.url}
+                    onChange={(e) => onChangeUrl(e.target.value)}/>
+            </Col>
+        </FormGroup>
+
+    );
+};
+
 
 /**
  * Main Form for editing a catalog entry
@@ -137,7 +156,7 @@ export default ({
     useEffect(() => {
         !isEmpty(service.url) && handleProtocolValidity(service.url);
     }, [service?.allowUnsecureLayers]);
-    const URLEditor = service.type === "tms" ? TmsURLEditor : DefaultURLEditor;
+    const URLEditor = service.type === "tms" ? TmsURLEditor : service.type === "cog" ? COGEditor : DefaultURLEditor;
     return (
         <Form horizontal >
             <FormGroup controlId="title" key="type-title-row">

web/client/components/catalog/editor/MainFormUtils.js

@@ -7,7 +7,8 @@ export const defaultPlaceholder = (service) => {
         "wms": "e.g. https://mydomain.com/geoserver/wms",
         "csw": "e.g. https://mydomain.com/geoserver/csw",
         "tms": "e.g. https://mydomain.com/geoserver/gwc/service/tms/1.0.0",
-        "3dtiles": "e.g. https://mydomain.com/tileset.json"
+        "3dtiles": "e.g. https://mydomain.com/tileset.json",
+        "cog": "e.g. https://mydomain.com/cog.tif"
     };
     for ( const [key, value] of Object.entries(urlPlaceholder)) {
         if ( key === service.type) {

web/client/components/map/openlayers/plugins/COGLayer.js

@@ -0,0 +1,45 @@
+/**
+ * Copyright 2015, GeoSolutions Sas.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import Layers from '../../../../utils/openlayers/Layers';
+
+import GeoTIFF from 'ol/source/GeoTIFF.js';
+import TileLayer from 'ol/layer/WebGLTile.js';
+
+function create(options) {
+    return new TileLayer({
+        msId: options.id,
+        style: options.style, // TODO style needs to be improved. Currently renders only predefined band and ranges when specified in config
+        opacity: options.opacity !== undefined ? options.opacity : 1,
+        visible: options.visibility,
+        source: new GeoTIFF({
+            convertToRGB: 'auto', // CMYK, YCbCr, CIELab, and ICCLab images will automatically be converted to RGB
+            sources: options.sources,
+            wrapX: true
+        }),
+        zIndex: options.zIndex,
+        minResolution: options.minResolution,
+        maxResolution: options.maxResolution
+    });
+}
+
+Layers.registerType('cog', {
+    create,
+    update(layer, newOptions, oldOptions, map) {
+        if (newOptions.srs !== oldOptions.srs) {
+            return create(newOptions, map);
+        }
+        if (oldOptions.minResolution !== newOptions.minResolution) {
+            layer.setMinResolution(newOptions.minResolution === undefined ? 0 : newOptions.minResolution);
+        }
+        if (oldOptions.maxResolution !== newOptions.maxResolution) {
+            layer.setMaxResolution(newOptions.maxResolution === undefined ? Infinity : newOptions.maxResolution);
+        }
+        return null;
+    }
+});

web/client/components/map/openlayers/plugins/index.js

@@ -19,5 +19,6 @@ export default {
     WFSLayer: require('./WFSLayer').default,
     WFS3Layer: require('./WFS3Layer').default,
     WMSLayer: require('./WMSLayer').default,
-    WMTSLayer: require('./WMTSLayer').default
+    WMTSLayer: require('./WMTSLayer').default,
+    COGLayer: require('./COGLayer').default
 };

web/client/plugins/MetadataExplorer.jsx

@@ -177,7 +177,7 @@ class MetadataExplorerComponent extends React.Component {
 
     static defaultProps = {
         id: "mapstore-metadata-explorer",
-        serviceTypes: [{ name: "csw", label: "CSW" }, { name: "wms", label: "WMS" }, { name: "wmts", label: "WMTS" }, { name: "tms", label: "TMS", allowedProviders: DEFAULT_ALLOWED_PROVIDERS }, { name: "wfs", label: "WFS" }, { name: "3dtiles", label: "3D Tiles" }],
+        serviceTypes: [{ name: "csw", label: "CSW" }, { name: "wms", label: "WMS" }, { name: "wmts", label: "WMTS" }, { name: "tms", label: "TMS", allowedProviders: DEFAULT_ALLOWED_PROVIDERS }, { name: "wfs", label: "WFS" }, { name: "3dtiles", label: "3D Tiles" }, { name: "cog", label: "COG" }],
         active: false,
         wrap: false,
         modal: true,
@@ -277,15 +277,15 @@ const MetadataExplorerPlugin = connect(metadataExplorerSelector, {
 })(MetadataExplorerComponent);
 
 /**
- * MetadataExplorer (Catalog) plugin. Shows the catalogs results (CSW, WMS, WMTS, TMS and WFS).
+ * MetadataExplorer (Catalog) plugin. Shows the catalogs results (CSW, WMS, WMTS, TMS, WFS and COG).
  * Some useful flags in `localConfig.json`:
  * - `noCreditsFromCatalog`: avoid add credits (attribution) from catalog
  *
  * @class
  * @name MetadataExplorer
  * @memberof plugins
  * @prop {string} cfg.hideThumbnail shows/hides thumbnail
- * @prop {object[]} cfg.serviceTypes Service types available to add a new catalog. default: `[{ name: "csw", label: "CSW" }, { name: "wms", label: "WMS" }, { name: "wmts", label: "WMTS" }, { name: "tms", label: "TMS", allowedProviders },{ name: "wfs", label: "WFS" }]`.
+ * @prop {object[]} cfg.serviceTypes Service types available to add a new catalog. default: `[{ name: "csw", label: "CSW" }, { name: "wms", label: "WMS" }, { name: "wmts", label: "WMTS" }, { name: "tms", label: "TMS", allowedProviders },{ name: "wfs", label: "WFS" }, { name: "cog", label: "COG" }]`.
  * `allowedProviders` is a whitelist of tileProviders from ConfigProvider.js. you can set a global variable allowedProviders in localConfig.json to set it up globally. You can configure it to "ALL" to get all the list (at your own risk, some services could change or not be available anymore)
  * @prop {object} cfg.hideIdentifier shows/hides identifier
  * @prop {boolean} cfg.hideExpand shows/hides full description button