From 2a40b23775904b70d131bf35bdea7d403abd8e80 Mon Sep 17 00:00:00 2001 From: Yingting Chen Date: Thu, 17 Oct 2024 16:28:31 +1300 Subject: [PATCH] feat: function to check changeset status for all dataset --- buildings/gui/reference_data.py | 64 +++++++++++++++++++++++++- buildings/gui/reference_data.ui | 28 ++++++++++- buildings/reference_data/admin_bdys.py | 50 +++++++++++++++++++- buildings/reference_data/topo50.py | 61 ++++++++++++++++++++++++ 4 files changed, 199 insertions(+), 4 deletions(-) diff --git a/buildings/gui/reference_data.py b/buildings/gui/reference_data.py index 174fcf96..f121a9cc 100644 --- a/buildings/gui/reference_data.py +++ b/buildings/gui/reference_data.py @@ -3,9 +3,19 @@ import os.path from functools import partial +from qgis.core import QgsVectorLayer from qgis.PyQt import uic from qgis.PyQt.QtCore import pyqtSlot, Qt -from qgis.PyQt.QtWidgets import QFrame, QLineEdit, QMessageBox, QApplication, QCheckBox +from qgis.PyQt.QtWidgets import ( + QDialog, + QFrame, + QMessageBox, + QApplication, + QCheckBox, + QTableWidget, + QTableWidgetItem, + QVBoxLayout, +) from qgis.PyQt.QtGui import QIcon from buildings.gui.error_dialog import ErrorDialog @@ -45,6 +55,21 @@ ] DATASET_STATSNZ = ["territorial_authority"] +DATASET_TOPO50 = [ + "canal_polygons", + "lagoon_polygons", + "lake_polygons", + "pond_polygons", + "river_polygons", + "swamp_polygons", + "hut_points", + "shelter_points", + "bivouac_points", + "protected_areas_polygons", + "coastlines_and_islands", +] +DATASET_ADMIN_BDYS = ["suburb_locality", "territorial_authority"] + class UpdateReferenceData(QFrame, FORM_CLASS): def __init__(self, dockwidget, parent=None): @@ -79,6 +104,7 @@ def __init__(self, dockwidget, parent=None): self.btn_update.clicked.connect( partial(self.update_clicked, commit_status=True) ) + self.btn_status.clicked.connect(self.btn_status_clicked) def close_cursor(self): self.db.close_cursor() @@ -130,6 +156,42 @@ def disable_checkboxes(self): "\nNOTE: You can't update reference data with\n a dataset in progress \n" ) + @pyqtSlot() + def btn_status_clicked(self): + QApplication.setOverrideCursor(Qt.WaitCursor) + dialog = QDialog() + tbl = QTableWidget() + tbl.setColumnCount(6) + tbl.setHorizontalHeaderLabels( + ["Dataset", "Last Updated", "New Updates", "Insert", "Update", "Delete"] + ) + for dataset in DATASET_LINZ + DATASET_STATSNZ: + api_key = self.check_api_key(dataset) + if api_key is None: + return + if dataset in DATASET_TOPO50: + status = topo50.check_status_topo50(api_key, dataset) + else: + status = admin_bdys.check_status_admin_bdys(api_key, dataset) + row_tbl = tbl.rowCount() + tbl.setRowCount(row_tbl + 1) + tbl.setItem(row_tbl, 0, QTableWidgetItem(status["dataset"])) + tbl.setItem(row_tbl, 1, QTableWidgetItem(status["last_updated"])) + tbl.setItem(row_tbl, 2, QTableWidgetItem(status["new_updates"])) + tbl.setItem(row_tbl, 3, QTableWidgetItem(status["insert"])) + tbl.setItem(row_tbl, 4, QTableWidgetItem(status["update"])) + tbl.setItem(row_tbl, 5, QTableWidgetItem(status["delete"])) + tbl.resizeRowsToContents() + tbl.resizeColumnsToContents() + dialogWidth = tbl.horizontalHeader().length() + 100 + dialogHeight = tbl.verticalHeader().length() + 100 + dialog.setFixedSize(dialogWidth, dialogHeight) + layout = QVBoxLayout() + dialog.setLayout(layout) + layout.addWidget(tbl) + QApplication.restoreOverrideCursor() + dialog.exec() + @pyqtSlot() def update_clicked(self, commit_status=True): """Called when update btn clicked""" diff --git a/buildings/gui/reference_data.ui b/buildings/gui/reference_data.ui index 3307e011..e427666a 100644 --- a/buildings/gui/reference_data.ui +++ b/buildings/gui/reference_data.ui @@ -54,6 +54,32 @@ + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + + 0 + 0 + + + + Status + + + @@ -143,7 +169,7 @@ - Coastlines and Islands + Coastlines and Islands (Overwrite existing) diff --git a/buildings/reference_data/admin_bdys.py b/buildings/reference_data/admin_bdys.py index b275d3ca..fa48a12d 100644 --- a/buildings/reference_data/admin_bdys.py +++ b/buildings/reference_data/admin_bdys.py @@ -1,4 +1,5 @@ from builtins import str +from collections import Counter # script to update canal data @@ -53,8 +54,53 @@ def current_date(): return to_var -# todo: add kx_api_key in config -# todo: combine suburb_locality- and town city +def check_status_admin_bdys(kx_api_key, dataset): + # get last update of layer date from log + from_var = last_update(dataset) + + # current date + to_var = current_date() + + layer = QgsVectorLayer( + URI.format( + kx_api_key, + LAYERS[dataset]["url_base"], + LAYERS[dataset]["layer_id"], + from_var, + to_var, + LAYERS[dataset]["cql_filter"], + ) + ) + if not layer.isValid(): + return { + "dataset": dataset, + "last_updated": from_var, + "new_updates": "error", + "insert": "error", + "update": "error", + "delete": "error", + } + + if layer.featureCount() == 0: + return { + "dataset": dataset, + "last_updated": from_var, + "new_updates": "", + "insert": "0", + "update": "0", + "delete": "0", + } + counts = Counter([feat["__change__"] for feat in layer.getFeatures()]) + return { + "dataset": dataset, + "last_updated": from_var, + "new_updates": "Available", + "insert": str(counts["INSERT"]), + "update": str(counts["UPDATE"]), + "delete": str(counts["DELETE"]), + } + + def update_admin_bdys(kx_api_key, dataset, dbconn: db): # get last update of layer date from log from_var = last_update(dataset) diff --git a/buildings/reference_data/topo50.py b/buildings/reference_data/topo50.py index 84fd3691..926d5abe 100644 --- a/buildings/reference_data/topo50.py +++ b/buildings/reference_data/topo50.py @@ -1,4 +1,6 @@ from builtins import str +from collections import Counter + # script to update canal data @@ -57,6 +59,65 @@ def current_date(): return to_var +def check_status_topo50(kx_api_key, dataset): + if "polygon" in dataset: + column_name = dataset.replace("_polygons", "") + elif "point" in dataset: + column_name = dataset.replace("_points", "") + else: + column_name = dataset + + # get last update of layer date from log + from_var = last_update(column_name) + + # current date + to_var = current_date() + + cql_filter = "" + if dataset == "hut_points": + cql_filter = "&cql_filter=bldg_use='hut'" + elif dataset == "shelter_points": + cql_filter = "&cql_filter=bldg_use='shelter'" + elif dataset == "protected_areas_polygons": + cql_filter = "&cql_filter=type = 'Conservation Area' OR type = 'National Park' OR type ='Wildlife Area'" + + external_id = "t50_fid" + if dataset == "protected_areas_polygons": + external_id = "napalis_id" + + layer = QgsVectorLayer( + URI.format(LDS_LAYER_IDS[dataset], kx_api_key, from_var, to_var, cql_filter) + ) + if not layer.isValid(): + return { + "dataset": dataset, + "last_updated": from_var, + "new_updates": "error", + "insert": "error", + "update": "error", + "delete": "error", + } + + if layer.featureCount() == 0: + return { + "dataset": dataset, + "last_updated": from_var, + "new_updates": "", + "insert": "0", + "update": "0", + "delete": "0", + } + counts = Counter([feat["__change__"] for feat in layer.getFeatures()]) + return { + "dataset": dataset, + "last_updated": from_var, + "new_updates": "Available", + "insert": str(counts["INSERT"]), + "update": str(counts["UPDATE"]), + "delete": str(counts["DELETE"]), + } + + def update_topo50(kx_api_key, dataset, dbconn): # Get name of column in reference log table