From a56f4ed76eafc8f4954a879dc415600ae7eade8a Mon Sep 17 00:00:00 2001 From: Felix Sommer Date: Wed, 18 Dec 2024 11:03:34 +0100 Subject: [PATCH] PB-326: add check provider script get map tile --- package-lock.json | 460 ++++++++++++++++ package.json | 1 + scripts/check-external-layers-providers.js | 495 ++++++++++-------- src/api/layers/layers-external.api.js | 40 +- .../advancedTools/ImportCatalogue/utils.js | 13 +- 5 files changed, 766 insertions(+), 243 deletions(-) diff --git a/package-lock.json b/package-lock.json index 310e45115..f249b3651 100644 --- a/package-lock.json +++ b/package-lock.json @@ -102,6 +102,7 @@ "prettier-plugin-jsdoc": "^1.3.0", "rimraf": "^6.0.1", "sass": "1.77.6", + "sharp": "^0.33.5", "start-server-and-test": "^2.0.9", "typescript": "~5.6.3", "vite": "^5.4.11", @@ -341,6 +342,17 @@ "ms": "^2.1.1" } }, + "node_modules/@emnapi/runtime": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.3.1.tgz", + "integrity": "sha512-kEBmG8KyqtxJZv+ygbEim+KCGtIq1fC22Ms3S4ziXmYKm8uyoLX0MHONVKwp+9opg390VaKRNt4a7A9NwmpNhw==", + "dev": true, + "license": "MIT", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@esbuild/linux-x64": { "version": "0.21.5", "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.21.5.tgz", @@ -613,6 +625,367 @@ "deprecated": "Use @eslint/object-schema instead", "dev": true }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.5.tgz", + "integrity": "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.5.tgz", + "integrity": "sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.4.tgz", + "integrity": "sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.4.tgz", + "integrity": "sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.5.tgz", + "integrity": "sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.4.tgz", + "integrity": "sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.4.tgz", + "integrity": "sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.4.tgz", + "integrity": "sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.4.tgz", + "integrity": "sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.4.tgz", + "integrity": "sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.5.tgz", + "integrity": "sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==", + "cpu": [ + "arm" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.0.5" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.5.tgz", + "integrity": "sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.5.tgz", + "integrity": "sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==", + "cpu": [ + "s390x" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.0.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.5.tgz", + "integrity": "sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.5.tgz", + "integrity": "sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.5.tgz", + "integrity": "sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.5.tgz", + "integrity": "sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==", + "cpu": [ + "wasm32" + ], + "dev": true, + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.2.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.5.tgz", + "integrity": "sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.5.tgz", + "integrity": "sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@intlify/core-base": { "version": "10.0.5", "resolved": "https://registry.npmjs.org/@intlify/core-base/-/core-base-10.0.5.tgz", @@ -3188,6 +3561,20 @@ "node": ">= 0.12.0" } }, + "node_modules/color": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", + "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", + "dev": true, + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1", + "color-string": "^1.9.0" + }, + "engines": { + "node": ">=12.5.0" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -3236,6 +3623,16 @@ "resolved": "https://registry.npmjs.org/color-space/-/color-space-2.0.1.tgz", "integrity": "sha512-nKqUYlo0vZATVOFHY810BSYjmCARrG7e5R3UE3CQlyjJTvv5kSSmPG1kzm/oDyyqjehM+lW1RnEt9It9GNa5JA==" }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "dev": true, + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, "node_modules/colorette": { "version": "2.0.20", "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", @@ -3675,6 +4072,15 @@ "node": ">=6" } }, + "node_modules/detect-libc": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.3.tgz", + "integrity": "sha512-bwy0MGW55bG41VqxxypOsdSdGqLwXPI/focwgTYCFMbdUiBAxLg9CFzG08sz2aqzknwiX7Hkl0bQENjg8iLByw==", + "dev": true, + "engines": { + "node": ">=8" + } + }, "node_modules/devlop": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", @@ -5751,6 +6157,12 @@ "node": ">=10" } }, + "node_modules/is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==", + "dev": true + }, "node_modules/is-binary-path": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", @@ -8594,6 +9006,45 @@ "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==" }, + "node_modules/sharp": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.5.tgz", + "integrity": "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==", + "dev": true, + "hasInstallScript": true, + "dependencies": { + "color": "^4.2.3", + "detect-libc": "^2.0.3", + "semver": "^7.6.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.33.5", + "@img/sharp-darwin-x64": "0.33.5", + "@img/sharp-libvips-darwin-arm64": "1.0.4", + "@img/sharp-libvips-darwin-x64": "1.0.4", + "@img/sharp-libvips-linux-arm": "1.0.5", + "@img/sharp-libvips-linux-arm64": "1.0.4", + "@img/sharp-libvips-linux-s390x": "1.0.4", + "@img/sharp-libvips-linux-x64": "1.0.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.0.4", + "@img/sharp-libvips-linuxmusl-x64": "1.0.4", + "@img/sharp-linux-arm": "0.33.5", + "@img/sharp-linux-arm64": "0.33.5", + "@img/sharp-linux-s390x": "0.33.5", + "@img/sharp-linux-x64": "0.33.5", + "@img/sharp-linuxmusl-arm64": "0.33.5", + "@img/sharp-linuxmusl-x64": "0.33.5", + "@img/sharp-wasm32": "0.33.5", + "@img/sharp-win32-ia32": "0.33.5", + "@img/sharp-win32-x64": "0.33.5" + } + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -8645,6 +9096,15 @@ "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", "dev": true }, + "node_modules/simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", + "dev": true, + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, "node_modules/slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", diff --git a/package.json b/package.json index 4812e9256..4ecffaee0 100644 --- a/package.json +++ b/package.json @@ -135,6 +135,7 @@ "prettier-plugin-jsdoc": "^1.3.0", "rimraf": "^6.0.1", "sass": "1.77.6", + "sharp": "^0.33.5", "start-server-and-test": "^2.0.9", "typescript": "~5.6.3", "vite": "^5.4.11", diff --git a/scripts/check-external-layers-providers.js b/scripts/check-external-layers-providers.js index bae750960..7f0751720 100755 --- a/scripts/check-external-layers-providers.js +++ b/scripts/check-external-layers-providers.js @@ -11,6 +11,7 @@ import axios, { AxiosError } from 'axios' import axiosRetry from 'axios-retry' import { promises as fs } from 'fs' import { exit } from 'process' +import sharp from 'sharp' import writeYamlFile from 'write-yaml-file' import yargs from 'yargs' import { hideBin } from 'yargs/helpers' @@ -19,15 +20,16 @@ import { EXTERNAL_SERVER_TIMEOUT, parseWmsCapabilities, parseWmtsCapabilities, + setWmsGetMapParams, } from '@/api/layers/layers-external.api' import { - guessExternalGetMapUrl, guessExternalLayerUrl, isWmsGetCap, + isWmsUrl, isWmtsGetCap, + isWmtsUrl, } from '@/modules/menu/components/advancedTools/ImportCatalogue/utils' import { LV95 } from '@/utils/coordinates/coordinateSystems' - const SIZE_OF_CONTENT_DISPLAY = 150 const options = yargs(hideBin(process.argv)) @@ -79,96 +81,263 @@ function compareCaseInsensitive(a, b) { } return 0 } +const requestHeaders = { + Origin: 'https://map.geo.admin.ch', + Referer: 'https://map.geo.admin.ch', + 'Sec-Fetch-Site': 'cross-site', +} -async function checkProviderGetMap (provider, response, parser, result) { - // console.log('Checking provider:', response) - - const xmlDoc = parser.parseFromString(response.data, 'text/xml') - let layer - let tileMatrixSetDetails - let tileMatrixSet - let tileMatrix - let tileMatrixRow - let tileMatrixCol - if (xmlDoc.querySelector('WMS_Capabilities')) { - const layerElements = xmlDoc.querySelector('Layer > Name') - // console.log('layerElements:', layerElements.textContent) - - layer = layerElements.textContent - } else if (xmlDoc.querySelector('Capabilities')) { - // const layerElements = xmlDoc.querySelectorAll('Layer > Identifier') - const layerElements = xmlDoc.querySelector('Layer > ows\\:Identifier, Layer > Identifier') - const tileMatrixSetElements = xmlDoc.querySelector('Layer > TileMatrixSetLink > TileMatrixSet'); - tileMatrix = xmlDoc.querySelector('Layer > TileMatrixSetLink > TileMatrixSetLimits > TileMatrixLimits > TileMatrix')?.textContent - tileMatrixRow = xmlDoc.querySelector('Layer > TileMatrixSetLink > TileMatrixSetLimits > TileMatrixLimits > MinTileRow')?.textContent; - tileMatrixCol = xmlDoc.querySelector('Layer > TileMatrixSetLink > TileMatrixSetLimits > TileMatrixLimits > MinTileCol')?.textContent; - tileMatrixSet = tileMatrixSetElements?.textContent || ''; - // console.log('TileMatrixSet:', tileMatrixSet); - // console.log('tileMatrix:', tileMatrix); - // console.log('tileMatrixRow:', tileMatrixRow); - // console.log('tileMatrixCol:', tileMatrixCol); - - tileMatrixSetDetails = Array.from( - xmlDoc.querySelectorAll(`TileMatrixSet > ows\\:Identifier, TileMatrixSet > Identifier`) - ).map((el) => el.textContent); - // console.log('TileMatrixSet Details:', tileMatrixSetDetails); - layer = layerElements.textContent +async function checkProviderGetMapTile(provider, capabilitiesResponse, result) { + const content = capabilitiesResponse.data + let isProviderMapTileValid = true + if (isWmsGetCap(content)) { + isProviderMapTileValid = await handleWms(provider, content, result) + } else if (isWmtsGetCap(content)) { + isProviderMapTileValid = await handleWmts(provider, content, result) } - // console.log('Layers:', layer) - - const getMapUrl = guessExternalGetMapUrl(provider, layer, tileMatrixSet, tileMatrix, tileMatrixRow, tileMatrixCol).toString() - // console.log('getMapUrl', getMapUrl) - const responseGetMap = await axios.get(getMapUrl, { - headers: { - Origin: 'https://map.geo.admin.ch', - Referer: 'https://map.geo.admin.ch', - 'Sec-Fetch-Site': 'cross-site', - }, - timeout: EXTERNAL_SERVER_TIMEOUT, - // }).catch((error) => { - // console.log('Error:', error) - // }) -}) - // console.log('Checking getMap:', responseGetMap) - const isProviderMapValid = checkProviderMapResponse(provider, getMapUrl, responseGetMap, result) + + return isProviderMapTileValid +} + +async function handleWms(provider, content, result) { + let isProviderMapValid = true + const capabilities = parseWmsCapabilities(content, provider) + const layers = capabilities.getAllExternalLayerObjects( + LV95, + 1, // opacity + true, // visible + false // throw error in case of an error + ) + + const firstLeaf = findFirstLeaf(layers) + const finding = capabilities.findLayer(firstLeaf.id) + const crs = capabilities.Capability.Layer.CRS[0] + const style = finding.layer?.Style ? finding.layer?.Style[0]?.Name : 'default' + const getCapabilitiesUrl = + capabilities.Capability.Request.GetCapabilities.DCPType[0].HTTP.Get.OnlineResource + + // If the GetMap URL is the same as the GetCapabilities URL, we skip it because it's already checked + const getMapUrls = capabilities.Capability.Request.GetMap.DCPType.map( + (d) => d.HTTP.Get.OnlineResource + ) + .filter(Boolean) + .filter((url) => getCapabilitiesUrl !== url) + + for (const getMapUrl of getMapUrls) { + const url = setWmsGetMapParams(new URL(getMapUrl), firstLeaf.id, crs, style).toString() + + try { + const { responseGetMap, redirectHeaders } = await fetchMapTile(url, provider) + isProviderMapValid = + isProviderMapValid && + (await checkProviderResponse( + provider, + url, + responseGetMap, + result, + redirectHeaders, + checkProviderResponseContentGetMap + )) + } catch (error) { + isProviderMapValid = false + result.invalid_wms.push(createErrorEntry(provider, url, error, content)) + } + } + return isProviderMapValid } -async function checkProvider(provider, result, parser) { +async function handleWmts(provider, content, result) { + let isProviderGetTileValid = true + const capabilities = parseWmtsCapabilities(content, provider) + const layers = capabilities.getAllExternalLayerObjects( + LV95, + 1, // opacity + true, // visible + false // throw error in case of an error + ) + + const exampleLayer = findFirstLeaf(layers) + const getTileUrlWithPlaceholders = exampleLayer.urlTemplate + const params = { + tileMatrixSet: exampleLayer.tileMatrixSets[0].id, + tileMatrix: exampleLayer.tileMatrixSets[0].tileMatrix[0].Identifier, + tileRow: 0, + tileCol: 0, + style: exampleLayer.style, + } + + // The key for the dimensions is not always the same + const dimensionKey = Object.keys(exampleLayer).find( + (key) => key.toLowerCase() === 'dimensions' || key.toLowerCase() === 'dimension' + ) + + // Find the default value for each dimension + const placeHolderParams = + exampleLayer[dimensionKey]?.reduce((acc, curr) => { + // Find the key for the default value + const defaultKey = Object.keys(curr).find((key) => key.toLowerCase() === 'default') + // Find the key for the identifier + const identifierKey = Object.keys(curr).find( + (key) => key.toLowerCase() === 'identifier' || key.toLowerCase() === 'id' + ) + if (curr[defaultKey]) { + acc[curr[identifierKey]] = curr[defaultKey] + } + return acc + }, params) || params + + let getTileUrl + try { + getTileUrl = replaceUrlPlaceholders(getTileUrlWithPlaceholders, placeHolderParams) + const { response, redirectHeaders } = await fetchMapTile(getTileUrl, provider) + isProviderGetTileValid = await checkProviderResponse( + provider, + getTileUrl, + response, + result, + redirectHeaders, + checkProviderResponseContentGetMap + ) + } catch (error) { + isProviderGetTileValid = false + result.invalid_wmts.push(createErrorEntry(provider, getTileUrl, error, content)) + } + + return isProviderGetTileValid +} + +async function fetchMapTile(url, provider) { + let redirectHeaders = [] + const response = await axios.get(url, { + headers: requestHeaders, + timeout: EXTERNAL_SERVER_TIMEOUT, + responseType: 'arraybuffer', + beforeRedirect: (options_, response) => { + redirectHeaders.push(response.headers) + }, + }) + + if (!response.headers['content-type'].includes('image')) { + throw new Error( + `Invalid provider ${provider}, parsing failed: ${response.headers['content-type']}` + ) + } + + return { response, redirectHeaders } +} + +function createErrorEntry(provider, url, error, content) { + return { + provider, + url, + error: `${error}`, + content: content.slice(0, SIZE_OF_CONTENT_DISPLAY), + } +} + +/** + * Replace placeholders in a URL template with values from a params object + * + * @param {string} urlTemplate URL template with placeholders + * @param {Object} params Object with placeholder values + * @returns {string} URL with placeholders replaced + * @throws {Error} If a placeholder is missing in the params object + */ +function replaceUrlPlaceholders(urlTemplate, params) { + const normalizedParams = Object.entries(params).reduce((acc, [key, _]) => { + acc[key.toLowerCase()] = params[key] + return acc + }, {}) + return urlTemplate.replace(/{(\w+)}/g, (_, key) => { + const lowerKey = key.toLowerCase() + // eslint-disable-next-line no-prototype-builtins + if (normalizedParams.hasOwnProperty(lowerKey)) { + return normalizedParams[lowerKey] + } else { + throw new Error(`Missing value for placeholder: ${key}`) + } + }) +} + +/** + * Find the first leaf layer in a layer tree + * + * @param {Array} layers Array of layers + * @returns {Object} First leaf layer + */ +function findFirstLeaf(layers) { + if (!layers || layers.length === 0) { + return null + } + + for (const layer of layers) { + if (!layer.layers || layer.layers.length === 0) { + return layer + } + const leaf = findFirstLeaf(layer.layers) + if (leaf) { + return leaf + } + } + + return null +} + +async function checkProvider(provider, result) { const url = guessExternalLayerUrl(provider, 'en').toString() + let capabilitiesResponse + const redirectHeaders = [] try { - const response = await axios.get(url, { - headers: { - Origin: 'https://map.geo.admin.ch', - Referer: 'https://map.geo.admin.ch', - 'Sec-Fetch-Site': 'cross-site', + capabilitiesResponse = await axios.get(url, { + headers: requestHeaders, + beforeRedirect: (options_, response) => { + redirectHeaders.push(response.headers) }, timeout: EXTERNAL_SERVER_TIMEOUT, }) - const isProviderValid = true - // const isProviderValid = checkProviderResponse(provider, url, response, result) - const isProviderMapValid = await checkProviderGetMap(provider, response, parser, result) - // const isProviderMapValid = true - // const isProviderMapValid = checkProviderMapResponse(provider, getMapUrl, responseGetMap, result) - if (isProviderValid && isProviderMapValid) { - result.valid_providers.push(provider) - } } catch (error) { if (error instanceof AxiosError) { console.error(`Provider ${provider} is not accessible: ${error}`) result.invalid_providers.push({ - provider: provider, - url: url, + provider, + url, status: error.response?.status, error: error.message, }) } else { throw error } + return + } + + // Validate provider response + const isProviderValid = await checkProviderResponse( + provider, + url, + capabilitiesResponse, + result, + redirectHeaders, + checkProviderResponseContent + ) + + // If provider is invalid or map check fails, skip adding to valid_providers + if ( + isProviderValid && + (await checkProviderGetMapTile(provider, capabilitiesResponse, result)) + ) { + result.valid_providers.push(provider) } } -function checkProviderResponse(provider, url, response, result) { +async function checkProviderResponse( + provider, + url, + response, + result, + redirectHeaders, + checkContentFnc +) { if (![200, 201].includes(response.status)) { console.error(`Provider ${provider} is not valid: status=${response.status}`) result.invalid_providers.push({ @@ -187,49 +356,17 @@ function checkProviderResponse(provider, url, response, result) { status: response.status, headers: response.headers.toJSON(), }) - } else if ( - !['*', 'https://map.geo.admin.ch'].includes( - response.headers.get('access-control-allow-origin').toString()?.trim() - ) - ) { - console.error( - `Provider ${provider} does not have geoadmin in its CORS: status=${ - response.status - }, Access-Control-Allow-Origin=${response.headers.get('access-control-allow-origin')}` - ) - result.invalid_cors.push({ - provider: provider, - url: url, - status: response.status, - headers: response.headers.toJSON(), - }) - } else if (checkProviderResponseContent(provider, url, response, result)) { - console.log(`Provider ${provider} is OK`) - // result.valid_providers.push(provider) - return true - } - - return false -} - -function checkProviderMapResponse(provider, url, response, result) { - if (![200, 201].includes(response.status)) { - console.error(`Provider ${provider} is not valid: status=${response.status}`) - result.invalid_providers.push({ - provider: provider, - url: url, - status: response.status, - }) - } else if (!response.headers.has('access-control-allow-origin')) { + } else if (redirectHeaders.some((header) => !header['access-control-allow-origin'])) { console.error( - `Provider ${provider} does not support CORS: status=${response.status}, ` + + `Provider ${provider} redirects ${redirectHeaders.length} times to a different URL which does not support CORS: status=${response.status}, ` + `missing access-control-allow-origin header` ) + const errorHeader = redirectHeaders.find((header) => !header['access-control-allow-origin']) result.invalid_cors.push({ provider: provider, url: url, status: response.status, - headers: response.headers.toJSON(), + headers: errorHeader, }) } else if ( !['*', 'https://map.geo.admin.ch'].includes( @@ -247,141 +384,94 @@ function checkProviderMapResponse(provider, url, response, result) { status: response.status, headers: response.headers.toJSON(), }) - } else { + } else if (await checkContentFnc(provider, url, response, result)) { console.log(`Provider ${provider} is OK`) - // result.valid_providers.push(provider) return true } - return false - // } else if (checkProviderResponseContent(provider, url, response, result)) { - // console.log(`Provider ${provider} is OK`) - // result.valid_providers.push(provider) - // } } function checkProviderResponseContent(provider, url, response, result) { const content = response.data - let isValid = true + const parseCapabilities = (isCapFn, parseFn, type, result, resultArray) => { + if (!isCapFn(content)) { + return false + } - if (isWmsGetCap(content)) { try { - const capabilities = parseWmsCapabilities(content, url) + const capabilities = parseFn(content, url) const layers = capabilities.getAllExternalLayerObjects( LV95, - 1 /* opacity */, - true /* visible */, - false /* throw Error in case of error */ + 1, // opacity + true, // visible + false // throw Error in case of error ) + if (layers.length === 0) { - throw new Error(`No valid WMS layers found`) + throw new Error(`No valid ${type} layers found`) } } catch (error) { - isValid = false - console.error(`Invalid provider ${provider}, WMS get Cap parsing failed: ${error}`) - result.invalid_wms.push({ - provider: provider, - url: url, + console.error(`Invalid provider ${provider}, ${type} get Cap parsing failed: ${error}`) + result[resultArray].push({ + provider, + url, error: `${error}`, content: content.slice(0, SIZE_OF_CONTENT_DISPLAY), }) + return false } - } else if (isWmtsGetCap(content)) { - try { - const capabilities = parseWmtsCapabilities(content, url) - const layers = capabilities.getAllExternalLayerObjects( - LV95, - 1 /* opacity */, - true /* visible */, - false /* throw Error in case of error */ - ) - if (layers.length === 0) { - throw new Error(`No valid WMTS layers found`) - } - } catch (error) { - isValid = false - console.error(`Invalid provider ${provider}, WMTS get Cap parsing failed: ${error}`) - result.invalid_wmts.push({ - provider: provider, - url: url, - error: `${error}`, - content: content.slice(0, SIZE_OF_CONTENT_DISPLAY), - }) - } - } else { - isValid = false - console.error(`Invalid provider ${url}; file type not recognized`) - result.invalid_content.push({ - provider: provider, - url: url, - content: content.slice(0, SIZE_OF_CONTENT_DISPLAY), - }) + return true + } + if ( + parseCapabilities(isWmsGetCap, parseWmsCapabilities, 'WMS', result, 'invalid_wms') || + parseCapabilities(isWmtsGetCap, parseWmtsCapabilities, 'WMTS', result, 'invalid_wmts') + ) { + return true } - return isValid -} -function checkProviderResponseContentGetMap(provider, url, response, result) { - const content = response.data - let isValid = true + console.error(`Invalid provider ${url}; file type not recognized`) + result.invalid_content.push({ + provider, + url, + content: content.slice(0, SIZE_OF_CONTENT_DISPLAY), + }) + return false +} - if (isWmsGetCap(content)) { - try { - const capabilities = parseWmsCapabilities(content, url) - const layers = capabilities.getAllExternalLayerObjects( - LV95, - 1 /* opacity */, - true /* visible */, - false /* throw Error in case of error */ +async function checkProviderResponseContentGetMap(provider, url, response, result) { + const content = Buffer.from(response.data) + let isValid = false + try { + // Check if the content is a valid image + await sharp(content).metadata() + isValid = true + } catch (error) { + if (isWmsUrl(url)) { + console.error( + `Invalid provider ${provider}, WMS get Map content parsing failed: ${error}` ) - if (layers.length === 0) { - throw new Error(`No valid WMS layers found`) - } - } catch (error) { - isValid = false - console.error(`Invalid provider ${provider}, WMS get Cap parsing failed: ${error}`) result.invalid_wms.push({ provider: provider, url: url, error: `${error}`, - content: content.slice(0, SIZE_OF_CONTENT_DISPLAY), }) - } - } else if (isWmtsGetCap(content)) { - try { - const capabilities = parseWmtsCapabilities(content, url) - const layers = capabilities.getAllExternalLayerObjects( - LV95, - 1 /* opacity */, - true /* visible */, - false /* throw Error in case of error */ + } else if (isWmtsUrl(url)) { + console.error( + `Invalid provider ${provider}, WMTS get Tiles content parsing failed: ${error}` ) - if (layers.length === 0) { - throw new Error(`No valid WMTS layers found`) - } - } catch (error) { - isValid = false - console.error(`Invalid provider ${provider}, WMTS get Cap parsing failed: ${error}`) result.invalid_wmts.push({ provider: provider, url: url, error: `${error}`, - content: content.slice(0, SIZE_OF_CONTENT_DISPLAY), }) } - } else { - isValid = false - console.error(`Invalid provider ${url}; file type not recognized`) - result.invalid_content.push({ - provider: provider, - url: url, - content: content.slice(0, SIZE_OF_CONTENT_DISPLAY), - }) } + return isValid } -async function checkProviders(providers, result, parser) { - return Promise.all(providers.map(async (provider) => checkProvider(provider, result, parser))) +async function checkProviders(providers, result) { + return Promise.all(providers.map(async (provider) => checkProvider(provider, result))) } async function writeResult(result) { @@ -439,11 +529,8 @@ async function main() { } else { providers = JSON.parse(await fs.readFile(options.input, { encoding: 'utf-8' })) } - const parser = new DOMParser() - await checkProviders(providers, result, parser) - // await checkProviders([providers[0]], result, parser) - // await checkProviders([providers[96]], result, parser) + await checkProviders(providers, result) console.log(`Done checking all ${providers.length} providers, writing results...`) try { await writeResult(result) diff --git a/src/api/layers/layers-external.api.js b/src/api/layers/layers-external.api.js index 03ee072f6..b9dde2b5c 100644 --- a/src/api/layers/layers-external.api.js +++ b/src/api/layers/layers-external.api.js @@ -44,18 +44,24 @@ export function setWmsGetCapParams(url, language) { } return url } -export function setWmsGetMapParams(url, layer) { + +/** + * Sets the WMS GetMap url parameters + * + * @param {URL} url Url to set + * @param {string} layer Layer to use + * @param {string} crs CRS/SRS to use + * @param {string} style Style to use + * @returns {URL} Url with wms parameter + */ +export function setWmsGetMapParams(url, layer, crs, style) { // Mandatory params url.searchParams.set('SERVICE', 'WMS') url.searchParams.set('REQUEST', 'GetMap') - // Currently openlayers only supports version 1.3.0 ! url.searchParams.set('VERSION', '1.1.0') url.searchParams.set('LAYERS', layer) - // url.searchParams.set('LAYERS', '1') - url.searchParams.set('STYLES', 'default') - url.searchParams.set('SRS', 'EPSG:4326') - //4326 - // url.searchParams.set('CRS', 'EPSG:21781') + url.searchParams.set('STYLES', style) + url.searchParams.set('SRS', crs) url.searchParams.set('BBOX', '10.0,10.0,10.0001,10.0001') url.searchParams.set('WIDTH', '1') url.searchParams.set('HEIGHT', '1') @@ -64,26 +70,6 @@ export function setWmsGetMapParams(url, layer) { return url } -export function setWmtsTileParams(url, layer, tileMatrixSet, tileMatrix, tileMatrixRow, tileMatrixCol) { - // Mandatory params for WMTS - url.searchParams.set('SERVICE', 'WMTS'); - url.searchParams.set('REQUEST', 'GetTile'); - url.searchParams.set('VERSION', '1.1.0'); - url.searchParams.set('LAYER', layer); - // url.searchParams.set('STYLE', 'default'); - url.searchParams.set('TILEMATRIXSET', tileMatrixSet); - // url.searchParams.set('TILEMATRIXSET', 'defaultMatrixSet'); - url.searchParams.set('TILEMATRIX', tileMatrix); - // url.searchParams.set('TILEMATRIX', 'EPSG:3857:0'); - url.searchParams.set('TILEROW', tileMatrixRow); - url.searchParams.set('TILECOL', tileMatrixCol); - - // Optional parameters - url.searchParams.set('FORMAT', 'image/png') - - return url -} - /** * Read and parse WMS GetCapabilities * diff --git a/src/modules/menu/components/advancedTools/ImportCatalogue/utils.js b/src/modules/menu/components/advancedTools/ImportCatalogue/utils.js index c5844c9ce..fbaf1e8df 100644 --- a/src/modules/menu/components/advancedTools/ImportCatalogue/utils.js +++ b/src/modules/menu/components/advancedTools/ImportCatalogue/utils.js @@ -1,4 +1,4 @@ -import { setWmsGetCapParams, setWmsGetMapParams, setWmtsGetCapParams, setWmtsTileParams } from '@/api/layers/layers-external.api' +import { setWmsGetCapParams, setWmtsGetCapParams } from '@/api/layers/layers-external.api' /** * Checks if file has WMS Capabilities XML content @@ -57,14 +57,3 @@ export function guessExternalLayerUrl(provider, language) { // By default if the URL service type cannot be guessed we use WMS return setWmsGetCapParams(new URL(provider), language) } -export function guessExternalGetMapUrl(provider, layer, tileMatrixSet, tileMatrix, tileMatrixRow, tileMatrixCol) { - if (isWmtsUrl(provider)) { - return setWmtsTileParams(new URL(provider), layer, tileMatrixSet, tileMatrix, tileMatrixRow, tileMatrixCol) - } - if (isWmsUrl(provider)) { - return setWmsGetMapParams(new URL(provider), layer) - } - // // By default if the URL service type cannot be guessed we use WMS - // return setWmsGetCapParams(new URL(provider), layer) - return setWmsGetMapParams(new URL(provider), layer) -}