From 6e0a52e1b2739333eb0921cd553ef05d9783c078 Mon Sep 17 00:00:00 2001 From: Marc Monnerat Date: Tue, 16 Nov 2021 09:43:05 +0100 Subject: [PATCH 1/2] updating code for the new service-shortlink service changing backend hostname, variables for service kml, updating form/multipart, tests and comment changing adminId to kmlId --- mk/config.mk | 5 ++- mk/debug.mk | 2 ++ mk/help.mk | 1 + mk/last.mk | 3 ++ rc_dev | 1 + rc_int | 1 + rc_prod | 1 + src/components/FileStorageService.js | 10 +++--- src/components/PublicStorageService.js | 48 +++++++++++++++++++------- src/components/draw/DrawDirective.js | 7 ++-- src/index.mako.html | 3 ++ src/js/GaModule.js | 3 +- test/specs/FileStorageService.spec.js | 25 +++++++------- test/specs/Loader.spec.js | 2 ++ 14 files changed, 78 insertions(+), 34 deletions(-) diff --git a/mk/config.mk b/mk/config.mk index 4089be8da3..2e2a41d786 100644 --- a/mk/config.mk +++ b/mk/config.mk @@ -79,9 +79,12 @@ QRCODE_PATH ?= /api/qrcode/generate LAST_QRCODE_PATH := $(call lastvalue,qrcode-path) PROXY_URL ?= //service-proxy.prod.bgdi.ch LAST_PROXY_URL := $(call lastvalue,proxy-url) +STORAGE_URL ?= //sys-public.prod.bgdi.ch +STORAGE_TECH_URL ?= //sys-public.prod.bgdi.ch +STORAGE_LAST_URL := $(call lastvalue,storage-url) PYPI_URL ?= https://pypi.org/simple/ LAST_PYPI_URL := $(call lastvalue,pypi-url) -PUBLIC_URL_REGEXP ?= ^https?:\/\/public\..*\.(bgdi|admin)\.ch\/.* +PUBLIC_URL_REGEXP ?= ^https?:\/\/(sys-)?public\..*\.(bgdi|admin)\.ch\/.* ADMIN_URL_REGEXP ?= ^(ftp|http|https):\/\/(.*(\.bgdi\.ch|\.geo\.admin\.ch|\.swisstopo\.cloud)) HREF_REGEXP ?= ^\s*(https?|whatsapp|file|s?ftp|blob|mailto): diff --git a/mk/debug.mk b/mk/debug.mk index bad105d746..e89845854d 100644 --- a/mk/debug.mk +++ b/mk/debug.mk @@ -48,6 +48,8 @@ define buildpage --var "qrcode_path=$(QRCODE_PATH)" \ --var "shop_url=$(SHOP_URL)" \ --var "shop_tech_url=$(SHOP_TECH_URL)" \ + --var "storage_url=$(STORAGE_URL)" \ + --var "storage_tech_url=$(STORAGE_TECH_URL)" \ --var "wms_url=$(WMS_URL)" \ --var "wms_tech_url=$(WMS_TECH_URL)" \ --var "wmts_url=$(WMTS_URL)" \ diff --git a/mk/help.mk b/mk/help.mk index 62e2b53266..8ff6c7888e 100644 --- a/mk/help.mk +++ b/mk/help.mk @@ -101,6 +101,7 @@ help: @echo "- QRCODE_URL qrcode URL (build with: $(LAST_QRCODE_URL), current value: $(QRCODE_URL))" @echo "- QRCODE_PATH qrcode path (build with: $(LAST_QRCODE_PATH), current value: $(QRCODE_PATH))" @echo "- SHOP_URL Service URL (build with: $(LAST_SHOP_URL), current value: $(SHOP_URL))" + @echo "- STORAGE_URL Storage/KML URL (build with: $(LAST_STORAGE_URL), current value: $(STORAGE_URL))" @echo "- WMS_URL Service URL (build with $(LAST_WMS_URL), current value: $(WMS_URL))" @echo "- WMTS_URL Service URL (build with $(LAST_WMTS_URL), current value: $(WMTS_URL))" @echo "- TERRAIN_URL Service URL (build with: $(LAST_TERRAIN_URL), current value: $(TERRAIN_URL))" diff --git a/mk/last.mk b/mk/last.mk index 9a1b3f81c3..84d07b1328 100644 --- a/mk/last.mk +++ b/mk/last.mk @@ -56,6 +56,9 @@ ${PYTHON_VENV}: .build-artefacts/last-pypi-url .build-artefacts/last-qrcode-path:: $(call cachelastvariable,$@,$(QRCODE_PATH),$(LAST_QRCODE_PATH),qrcode-path) +.build-artefacts/last-storage-url:: + $(call cachelastvariable,$@,$(STORAGE_URL),$(LAST_STORAGE_URL),storage-url) + .build-artefacts/last-apache-base-path:: $(call cachelastvariable,$@,$(APACHE_BASE_PATH),$(LAST_APACHE_BASE_PATH),apache-base-path) diff --git a/rc_dev b/rc_dev index a8ec5cc16e..fb23aba8df 100644 --- a/rc_dev +++ b/rc_dev @@ -12,6 +12,7 @@ export PRINT_URL=//service-print.dev.bgdi.ch export PROXY_URL=//service-proxy.bgdi-dev.swisstopo.cloud export QRCODE_URL=//sys-map.dev.bgdi.ch export QRCODE_PATH=/api/qrcode/generate +export STORAGE_URL=//sys-public.dev.bgdi.ch export ADMIN_URL_REGEXP="^(ftp|http|https):\/\/(.*(\.bgdi\.ch|\.geo\.admin\.ch|\.swisstopo\.cloud)|localhost:[0-9]{1,5})" export WMS_URL=//wms-bgdi.dev.bgdi.ch export WMTS_URL=//tod.dev.bgdi.ch diff --git a/rc_int b/rc_int index aa12e4a4a0..0ede8ebf47 100644 --- a/rc_int +++ b/rc_int @@ -11,6 +11,7 @@ export PUBLIC_URL=//public.int.bgdi.ch export PRINT_URL=//service-print.int.bgdi.ch export PROXY_URL=//service-proxy.int.bgdi.ch export QRCODE_URL=//sys-map.int.bgdi.ch +export STORAGE_URL=//sys-public.int.bgdi.ch export WMS_URL=//wms-bgdi-cdn-{s}.int.bgdi.ch export WMTS_URL=//wmts{s}.bgdi.ch export TERRAIN_URL=//terrain100.bgdi.ch diff --git a/rc_prod b/rc_prod index a3659f119e..b5caa0feb7 100644 --- a/rc_prod +++ b/rc_prod @@ -11,6 +11,7 @@ export PUBLIC_URL=//public.geo.admin.ch export PRINT_URL=//print.geo.admin.ch export PROXY_URL=//service-proxy.prod.bgdi.ch export QRCODE_URL=//map.geo.admin.ch +export STORAGE_URL=//public.geo.admin.ch export WMS_URL=//wms{s}.geo.admin.ch export WMTS_URL=//wmts{s}.geo.admin.ch export WMTS_TECH_URL=//wmts. diff --git a/src/components/FileStorageService.js b/src/components/FileStorageService.js index 5ab2849070..7621ae640b 100644 --- a/src/components/FileStorageService.js +++ b/src/components/FileStorageService.js @@ -13,8 +13,8 @@ goog.require('ga_publicstorage_service'); */ module.provider('gaFileStorage', function() { this.$get = function($http, $q, gaPublicStorage) { - var endPoint = '/files'; - var publicEndPoint = ''; + var endPoint = '/api/kml/admin'; + var publicEndPoint = '/kml/files'; var contentType = 'application/vnd.google-earth.kml+xml'; var FileStorage = function() { @@ -35,9 +35,9 @@ goog.require('ga_publicstorage_service'); adminId); }; - this.save = function(id, content) { - return gaPublicStorage.save(endPoint, publicEndPoint, id, content, - contentType); + this.save = function(fileId, adminId, content) { + return gaPublicStorage.save(endPoint, publicEndPoint, fileId, + adminId, content, contentType); }; // Delete the file in s3. Only if an adminId is specified diff --git a/src/components/PublicStorageService.js b/src/components/PublicStorageService.js index ebc23cb3b6..08b9609859 100644 --- a/src/components/PublicStorageService.js +++ b/src/components/PublicStorageService.js @@ -11,14 +11,19 @@ goog.provide('ga_publicstorage_service'); this.$get = function($http, $q, gaGlobalOptions) { var getServiceUrl = function(endPoint, id) { - return gaGlobalOptions.apiUrl + endPoint + ((id) ? '/' + id : ''); + return gaGlobalOptions.storageUrl + endPoint + ((id) ? '/' + id : ''); }; var getPublicUrl = function(endPoint, fileId) { - return gaGlobalOptions.publicUrl + endPoint + + return gaGlobalOptions.storageUrl + endPoint + ((fileId) ? '/' + fileId : ''); }; + var getMetadataUrl = function(endPoint, adminId) { + return gaGlobalOptions.storageUrl + endPoint + + '?admin_id=' + adminId + } + var PublicStorage = function() { // Get the file from a fileId @@ -35,13 +40,14 @@ goog.provide('ga_publicstorage_service'); }; // Get the accessible url of the file from an adminId + // using the metadata request this.getFileUrlFromAdminId = function(endPoint, publicEndPoint, adminId) { var deferred = $q.defer(); - $http.get(getServiceUrl(endPoint, adminId)).then(function(response) { + $http.get(getMetadataUrl(endPoint, adminId)).then(function(response) { var data = response.data; - if (data && data.fileId) { - var url = getPublicUrl(publicEndPoint, data.fileId); + if (data && data.links && data.links.kml) { + var url = data.links.kml; deferred.resolve(url); } else { deferred.reject(); @@ -59,18 +65,36 @@ goog.provide('ga_publicstorage_service'); // returns the same adminId and the same file url // if id is an fileId --> fork the file // returns new adminId and new file url - this.save = function(endPoint, publicEndPoint, id, content, - contentType) { - return $http.post(getServiceUrl(endPoint, id), content, { + this.save = function(endPoint, publicEndPoint, fileId, + adminId, content, contentType) { + + var method = 'PUT' + const formData = new FormData(); + const blob = new Blob([content], { name: 'kml', + filename: 'blob', + type: contentType}) + formData.append('kml', blob); + formData.append('author', 'mf-geoadmin3'); + if (fileId === undefined && adminId === undefined) { + method = 'POST' + } else { + formData.append('admin_id', adminId) + } + return $http({ + method: method, + url: getServiceUrl(endPoint, fileId), + data: formData, + // Use 'undefined' to send 'multipart/form-data' + // See https://stackoverflow.com/a/44726531/996693 headers: { - 'Content-Type': contentType + 'Content-Type': undefined } }).then(function(response) { var data = response.data; return { - adminId: data.adminId, - fileId: data.fileId, - fileUrl: getPublicUrl(publicEndPoint, data.fileId) + adminId: data.admin_id, + fileId: data.file_id, + fileUrl: ((data || {}).links || {}).kml }; }); }; diff --git a/src/components/draw/DrawDirective.js b/src/components/draw/DrawDirective.js index 2854455d01..c3a2d91c77 100644 --- a/src/components/draw/DrawDirective.js +++ b/src/components/draw/DrawDirective.js @@ -757,9 +757,10 @@ goog.require('ga_styles_service'); scope.statusMsgId = 'draw_file_saving'; var kmlString = gaExportKml.create(layer, map.getView().getProjection()); - var id = layer.adminId || - gaFileStorage.getFileIdFromFileUrl(layer.url); - gaFileStorage.save(id, kmlString || '').then( + var adminId = layer.adminId; + var kmlId = (adminId !== undefined) ? + gaFileStorage.getFileIdFromFileUrl(layer.url) : undefined; + gaFileStorage.save(kmlId, adminId, kmlString || '').then( function(data) { scope.statusMsgId = 'draw_file_saved'; diff --git a/src/index.mako.html b/src/index.mako.html index 09287ca9fe..191171392b 100644 --- a/src/index.mako.html +++ b/src/index.mako.html @@ -770,6 +770,7 @@ var shopUrl = setBackend(prtl + cfg['shop_url'], prtl + cfg['shop_tech_url'], 'shop_url'); var publicUrl = setBackend(prtl + cfg['public_url'], prtl + cfg['public_tech_url'], 'public_url'); var printUrl = setBackend(prtl + cfg['print_url'], prtl + cfg['print_tech_url'], 'print_url'); + var storageUrl = setBackend(prtl + cfg['storage_url'], prtl + cfg['storage_tech_url'], 'storage_url'); var proxyUrl = prtl + cfg['proxy_url'] + '/'; var qrcodeUrl = setBackend(prtl + cfg['qrcode_url'], prtl + cfg['qrcode_tech_url'], 'qrcode_url'); var qrcodePath = cfg['qrcode_path']; @@ -800,6 +801,8 @@ printUrl: printUrl, proxyUrl: proxyUrl, shopUrl: shopUrl, + storageUrl: storageUrl, + storagePath: '/api/kml/admin', publicUrl: publicUrl, publicUrlRegexp: new RegExp(cfg['public_url_regexp']), qrcodeUrl: qrcodeUrl, diff --git a/src/js/GaModule.js b/src/js/GaModule.js index 6b53db7757..54eb0a44ad 100644 --- a/src/js/GaModule.js +++ b/src/js/GaModule.js @@ -196,7 +196,8 @@ goog.require('ga_waitcursor_service'); }); module.config(function(gaFileStorageProvider, gaGlobalOptions) { - gaFileStorageProvider.fileStorageUrl = gaGlobalOptions.apiUrl + '/files'; + gaFileStorageProvider.fileStorageUrl = gaGlobalOptions.storageUrl + + gaGlobalOptions.storagePath; gaFileStorageProvider.publicUrl = gaGlobalOptions.publicUrl; }); diff --git a/test/specs/FileStorageService.spec.js b/test/specs/FileStorageService.spec.js index ef5a039a40..bccc943f8a 100644 --- a/test/specs/FileStorageService.spec.js +++ b/test/specs/FileStorageService.spec.js @@ -6,6 +6,8 @@ describe('ga_filestorage_service', function() { adminId = 'aaaaaaaaaaaaaaaaaaaaa', // length must > 20 fileId = 'fffffffffffffffffffff', // length must > 20 fileContent = '', + blob = new Blob([fileContent]), + form = new FormData(), fileInfo = { adminId: adminId, fileId: fileId, @@ -16,14 +18,15 @@ describe('ga_filestorage_service', function() { fileId: fileId, fileUrl: 'https://public.geo.admin.ch/' + fileId }, - serviceUrl, publicUrl; + serviceUrl; + form.append('kml', blob) + form.append('author', 'toto') beforeEach(function() { inject(function($injector, gaGlobalOptions) { gaFileStorage = $injector.get('gaFileStorage'); $httpBackend = $injector.get('$httpBackend'); - serviceUrl = gaGlobalOptions.apiUrl + '/files'; - publicUrl = gaGlobalOptions.publicUrl; + serviceUrl = gaGlobalOptions.storageUrl; }); }); @@ -43,7 +46,7 @@ describe('ga_filestorage_service', function() { describe('#get()', function() { it('gets a file', function() { - var expectedUrl = publicUrl + '/' + fileId; + var expectedUrl = serviceUrl + '/kml/files/' + fileId; $httpBackend.expectGET(expectedUrl).respond(fileContent); gaFileStorage.get(fileId); $httpBackend.flush(); @@ -52,24 +55,22 @@ describe('ga_filestorage_service', function() { describe('#save()', function() { it('creates a file', inject(function($timeout, gaGlobalOptions) { - $httpBackend.expectPOST(serviceUrl, fileContent).respond(fileInfo); - gaFileStorage.save(null, fileContent); + $httpBackend.expectPOST(serviceUrl + '/api/kml/admin', form).respond(fileInfo); + gaFileStorage.save(undefined, undefined, fileContent); $httpBackend.flush(); })); it('updates a file', inject(function($timeout) { - var expectedUrl = serviceUrl + '/' + adminId; - $httpBackend.expectPOST(expectedUrl, fileContent, function(headers) { - return headers['Content-Type'] === 'application/vnd.google-earth.kml+xml'; - }).respond(fileInfo); - gaFileStorage.save(adminId, fileContent); + var expectedUrl = serviceUrl + '/api/kml/admin/' + fileId; + $httpBackend.expectPUT(expectedUrl, form).respond(fileInfo); + gaFileStorage.save(fileId, adminId, fileContent); $httpBackend.flush(); })); }); describe('#del()', function() { it('deletes a file', inject(function($timeout) { - var expectedUrl = serviceUrl + '/' + adminId; + var expectedUrl = serviceUrl + '/api/kml/admin/' + adminId; $httpBackend.expectDELETE(expectedUrl).respond({success: true}); gaFileStorage.del(adminId); $httpBackend.flush(); diff --git a/test/specs/Loader.spec.js b/test/specs/Loader.spec.js index f39de107d6..48e94ae313 100644 --- a/test/specs/Loader.spec.js +++ b/test/specs/Loader.spec.js @@ -16,6 +16,7 @@ beforeEach(function() { var qrcodeUrl = '//sys-map.prod.bgdi.ch'; var qrcodePath = '/api/qrcode/generate'; var shopUrl = '//shop.bgdi.ch'; + var storageUrl = '//sys-public.prod.bgdi.ch'; var wmsUrl = '//wms.geo.admin.ch'; var wmtsUrl = '//tod{s}.bgdi.ch'; var terrainUrl = '//terrain100.geo.admin.ch'; @@ -46,6 +47,7 @@ beforeEach(function() { qrcodeUrl: location.protocol + qrcodeUrl, qrcodePath: qrcodePath, shopUrl: location.protocol + shopUrl, + storageUrl: location.protocol + storageUrl, publicUrl: location.protocol + publicUrl, publicUrlRegexp: /^https?:\/\/public\..*\.(bgdi|admin)\.ch\/.*/, adminUrlRegexp: /^(ftp|http|https):\/\/(.*(\.bgdi|\.geo\.admin)\.ch)/, From 96ca7e271d97bf4afe4323f136e04d531c3d8bd0 Mon Sep 17 00:00:00 2001 From: Marc Monnerat Date: Wed, 30 Mar 2022 13:16:12 +0200 Subject: [PATCH 2/2] temporary reverting to sys-public hostname should be reverted to public.geo.admin.ch once the DNS switch is enforced. --- mk/config.mk | 4 ++-- rc_prod | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mk/config.mk b/mk/config.mk index 2e2a41d786..78ea37045e 100644 --- a/mk/config.mk +++ b/mk/config.mk @@ -66,8 +66,8 @@ LAST_SHOP_URL := $(call lastvalue,shop-url) FEEDBACK_URL ?= //map.geo.admin.ch FEEDBACK_TECH_URL ?= //sys-map.prod.bgdi.ch LAST_FEEDBACK_URL := $(call lastvalue, feedback-url) -PUBLIC_URL ?= //public.geo.admin.ch -PUBLIC_TECH_URL ?= //public. +PUBLIC_URL ?= //sys-public.prod.bgdi.ch +PUBLIC_TECH_URL ?= //sys-public. LAST_PUBLIC_URL := $(call lastvalue,public-url) PRINT_URL ?= //print.geo.admin.ch PRINT_TECH_URL ?= //service-print. diff --git a/rc_prod b/rc_prod index b5caa0feb7..1c3b91daf0 100644 --- a/rc_prod +++ b/rc_prod @@ -11,7 +11,7 @@ export PUBLIC_URL=//public.geo.admin.ch export PRINT_URL=//print.geo.admin.ch export PROXY_URL=//service-proxy.prod.bgdi.ch export QRCODE_URL=//map.geo.admin.ch -export STORAGE_URL=//public.geo.admin.ch +export STORAGE_URL=//sys-public.prod.bgdi.ch export WMS_URL=//wms{s}.geo.admin.ch export WMTS_URL=//wmts{s}.geo.admin.ch export WMTS_TECH_URL=//wmts.