From 1a25e405387c82b8f1140696d1c3d6c2031490d0 Mon Sep 17 00:00:00 2001 From: Morg42 <43153739+Morg42@users.noreply.github.com> Date: Sat, 12 Oct 2024 22:16:32 +0200 Subject: [PATCH 1/7] database: add orphan reassignment --- database/__init__.py | 19 +++++++++++++++ database/webif/__init__.py | 5 ++-- database/webif/templates/base_database.html | 4 ++- database/webif/templates/index.html | 27 ++++++++++++++++++++- 4 files changed, 51 insertions(+), 4 deletions(-) diff --git a/database/__init__.py b/database/__init__.py index 65eb46b76..086251a90 100755 --- a/database/__init__.py +++ b/database/__init__.py @@ -967,6 +967,25 @@ def _count_orphanlogentries(self): return + def reassign_orphaned_id(self, orphan_id, to): + """ + Reassign values from orphaned item ID to given item ID + + :param orphan_id: item id of the orphaned item + :param to: item id of the target item + :type orphan_id: int + :type to: int + """ + self.logger.error(f'reassign called: {orphan_id} -> {to}') + cur = self._db_maint.cursor() + self._execute(self._prepare("UPDATE {log} SET item_id = :newid WHERE item_id = :orphanid;"), {'newid': to, 'orphanid': orphan_id}, cur=cur) + self._execute(self._prepare("DELETE FROM {item} WHERE id = :orphanid LIMIT 1;"), {'orphanid': orphan_id}, cur=cur) + self.logger.info(f'reassigned orphaned id {orphan_id} to new id {to}') + cur.close() + self._db_maint.commit() + self.logger.debug('rebuilding orphan list') + self.build_orphanlist() + def _delete_orphan(self, item_path): """ Delete orphan item or logentries it diff --git a/database/webif/__init__.py b/database/webif/__init__.py index 9c9178391..d31a4b270 100755 --- a/database/webif/__init__.py +++ b/database/webif/__init__.py @@ -63,7 +63,7 @@ def __init__(self, webif_dir, plugin): @cherrypy.expose def index(self, reload=None, action=None, item_id=None, item_path=None, time_end=None, day=None, month=None, year=None, - time_orig=None, changed_orig=None): + time_orig=None, changed_orig=None, orphanID=None, newID=None): """ Build index.html for cherrypy @@ -76,6 +76,8 @@ def index(self, reload=None, action=None, item_id=None, item_path=None, time_end if item_path is not None: item = self.plugin.items.return_item(item_path) delete_triggered = False + if orphanID is not None and newID is not None and orphanID != newID: + self.plugin.reassign_orphaned_id(orphanID, to=newID) if action is not None: if action == "delete_log" and item_id is not None: if time_orig is not None and changed_orig is not None: @@ -271,7 +273,6 @@ def db_sqldump(self): return - @cherrypy.expose def cleanup(self): self.plugin.cleanup() diff --git a/database/webif/templates/base_database.html b/database/webif/templates/base_database.html index 8a305091a..0bbea1793 100755 --- a/database/webif/templates/base_database.html +++ b/database/webif/templates/base_database.html @@ -31,7 +31,8 @@ { className: "time", targets: 2 }, { className: "type", targets: 3 }, { className: "id", targets: 4, render: $.fn.dataTable.render.number('.', ',', 0, '') }, - { className: "logcount", targets: 5, render: $.fn.dataTable.render.number('.', ',', 0, '') }, + { className: "reassign", targets: 5 }, + { className: "logcount", targets: 6, render: $.fn.dataTable.render.number('.', ',', 0, '') }, ].concat($.fn.dataTable.defaults.columnDefs)}); {% else %} orphantable = $('#orphantable').DataTable( { @@ -42,6 +43,7 @@ { className: "time", targets: 2 }, { className: "type", targets: 3 }, { className: "id", targets: 4, render: $.fn.dataTable.render.number('.', ',', 0, '') }, + { className: "reassign", targets: 5 }, ].concat($.fn.dataTable.defaults.columnDefs)}); {% endif %} diff --git a/database/webif/templates/index.html b/database/webif/templates/index.html index 6a52ed64b..9939883f1 100755 --- a/database/webif/templates/index.html +++ b/database/webif/templates/index.html @@ -158,6 +158,17 @@ {% set tab3title = _('Verwaiste Items') %} {% block bodytab3 %} + + +
{% if p.remove_orphan or len(p.orphanlist) == 0 %} @@ -174,6 +185,7 @@ {{ _('Letzte Änderung') }} {{ _('Typ') }} {{ _('DB-ID') }} + {{ _('Neuzuweisung') }} {% if p.count_logentries %} {{ _('Anzahl Einträge') }} {% endif %} @@ -181,6 +193,7 @@ {% for item in p.orphanlist %} + {% set itemid = p.id(item, create=False) %} {{ item }} @@ -199,7 +212,19 @@ {% endif %} {% endif %} {{ _(p.db_itemtype(item)) }} - {{ p.id(item, create=False) }} + {{ itemid }} + + + + {% if p.count_logentries %} {{ p._orphan_logcount[p.id(item, create=False)] }} {% endif %} From 6d7e38dba65de6325cc499f163ec32422835cf46 Mon Sep 17 00:00:00 2001 From: Morg42 <43153739+Morg42@users.noreply.github.com> Date: Mon, 14 Oct 2024 09:27:22 +0200 Subject: [PATCH 2/7] database: change selection to modal dialogue --- database/__init__.py | 21 ++++++---- database/webif/__init__.py | 8 ++-- database/webif/static/style.css | 37 +++++++++++++++++ database/webif/templates/index.html | 64 +++++++++++++++++++++-------- 4 files changed, 102 insertions(+), 28 deletions(-) create mode 100644 database/webif/static/style.css diff --git a/database/__init__.py b/database/__init__.py index 086251a90..4128c5dc1 100755 --- a/database/__init__.py +++ b/database/__init__.py @@ -976,15 +976,18 @@ def reassign_orphaned_id(self, orphan_id, to): :type orphan_id: int :type to: int """ - self.logger.error(f'reassign called: {orphan_id} -> {to}') - cur = self._db_maint.cursor() - self._execute(self._prepare("UPDATE {log} SET item_id = :newid WHERE item_id = :orphanid;"), {'newid': to, 'orphanid': orphan_id}, cur=cur) - self._execute(self._prepare("DELETE FROM {item} WHERE id = :orphanid LIMIT 1;"), {'orphanid': orphan_id}, cur=cur) - self.logger.info(f'reassigned orphaned id {orphan_id} to new id {to}') - cur.close() - self._db_maint.commit() - self.logger.debug('rebuilding orphan list') - self.build_orphanlist() + try: + self.logger.warning(f'reassigning orphaned data from (old) id {orphan_id} to (new) id {to}') + cur = self._db_maint.cursor() + self._execute(self._prepare("UPDATE {log} SET item_id = :newid WHERE item_id = :orphanid;"), {'newid': to, 'orphanid': orphan_id}, cur=cur) + self._execute(self._prepare("DELETE FROM {item} WHERE id = :orphanid LIMIT 1;"), {'orphanid': orphan_id}, cur=cur) + self.logger.warning(f'reassigned orphaned id {orphan_id} to new id {to}') + cur.close() + self._db_maint.commit() + self.logger.warning('rebuilding orphan list') + self.build_orphanlist() + except Exception as e: + self.logger.error(f'error on reassigning id {orphan_id} to {to}: {e}') def _delete_orphan(self, item_path): """ diff --git a/database/webif/__init__.py b/database/webif/__init__.py index d31a4b270..79b10c6dd 100755 --- a/database/webif/__init__.py +++ b/database/webif/__init__.py @@ -63,7 +63,7 @@ def __init__(self, webif_dir, plugin): @cherrypy.expose def index(self, reload=None, action=None, item_id=None, item_path=None, time_end=None, day=None, month=None, year=None, - time_orig=None, changed_orig=None, orphanID=None, newID=None): + time_orig=None, changed_orig=None, orphan_id=None, new_id=None): """ Build index.html for cherrypy @@ -76,8 +76,10 @@ def index(self, reload=None, action=None, item_id=None, item_path=None, time_end if item_path is not None: item = self.plugin.items.return_item(item_path) delete_triggered = False - if orphanID is not None and newID is not None and orphanID != newID: - self.plugin.reassign_orphaned_id(orphanID, to=newID) + self.logger.error(f'index called with oid {orphan_id} and nid {new_id}') + if orphan_id is not None and new_id is not None and orphan_id != new_id: + self.logger.error(f'calling reassign for {orphan_id} and {new_id}') + self.plugin.reassign_orphaned_id(orphan_id, to=new_id) if action is not None: if action == "delete_log" and item_id is not None: if time_orig is not None and changed_orig is not None: diff --git a/database/webif/static/style.css b/database/webif/static/style.css new file mode 100644 index 000000000..94a8b7bf9 --- /dev/null +++ b/database/webif/static/style.css @@ -0,0 +1,37 @@ +/* The Modal (background) */ +.or-modal { + display: none; /* Hidden by default */ + position: fixed; /* Stay in place */ + z-index: 999; /* Sit on top */ + left: 0; + top: 0; + width: 100%; /* Full width */ + height: 100%; /* Full height */ + overflow: auto; /* Enable scroll if needed */ + background-color: rgb(0,0,0); /* Fallback color */ + background-color: rgba(0,0,0,0.4); /* Black w/ opacity */ +} + +/* Modal Content/Box */ +.or-modal-content { + background-color: #fefefe; + margin: 15% auto; /* 15% from the top and centered */ + padding: 20px; + border: 1px solid #888; + width: 80%; /* Could be more or less, depending on screen size */ +} + +/* The Close Button */ +.or-close { + color: #aaa; + float: right; + font-size: 28px; + font-weight: bold; +} + +.or-close:hover, +.or-close:focus { + color: black; + text-decoration: none; + cursor: pointer; +} \ No newline at end of file diff --git a/database/webif/templates/index.html b/database/webif/templates/index.html index 9939883f1..c65e8a91a 100755 --- a/database/webif/templates/index.html +++ b/database/webif/templates/index.html @@ -4,6 +4,10 @@ {% set dataSet = 'overview' %} {% set tab1title = _('Database Items') %} +{% block pluginstyles %} + +{% endblock pluginstyles %} + {%- block pluginscripts %} {{ super() }} +
+
+ × +
+ Neuzuweisen von Itemreihe (ID ) +
+
+ Bitte wählen Sie die Itemreihe aus, der die verwaisten Daten zugewiesen werden sollen: + + + +
+
+
+
{% if p.remove_orphan or len(p.orphanlist) == 0 %} @@ -214,16 +255,7 @@ {{ _(p.db_itemtype(item)) }} {{ itemid }} - - + {% if p.count_logentries %} {{ p._orphan_logcount[p.id(item, create=False)] }} From 8fac7513eee18cbd2ff3f92fd136f761ce8578e4 Mon Sep 17 00:00:00 2001 From: Morg42 <43153739+Morg42@users.noreply.github.com> Date: Mon, 14 Oct 2024 19:17:53 +0200 Subject: [PATCH 3/7] database: remove debug code --- database/__init__.py | 6 +++--- database/webif/__init__.py | 2 -- database/webif/templates/index.html | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/database/__init__.py b/database/__init__.py index 4128c5dc1..357714091 100755 --- a/database/__init__.py +++ b/database/__init__.py @@ -977,14 +977,14 @@ def reassign_orphaned_id(self, orphan_id, to): :type to: int """ try: - self.logger.warning(f'reassigning orphaned data from (old) id {orphan_id} to (new) id {to}') + self.logger.debug(f'reassigning orphaned data from (old) id {orphan_id} to (new) id {to}') cur = self._db_maint.cursor() self._execute(self._prepare("UPDATE {log} SET item_id = :newid WHERE item_id = :orphanid;"), {'newid': to, 'orphanid': orphan_id}, cur=cur) self._execute(self._prepare("DELETE FROM {item} WHERE id = :orphanid LIMIT 1;"), {'orphanid': orphan_id}, cur=cur) - self.logger.warning(f'reassigned orphaned id {orphan_id} to new id {to}') + self.logger.info(f'reassigned orphaned id {orphan_id} to new id {to}') cur.close() self._db_maint.commit() - self.logger.warning('rebuilding orphan list') + self.logger.debug('rebuilding orphan list') self.build_orphanlist() except Exception as e: self.logger.error(f'error on reassigning id {orphan_id} to {to}: {e}') diff --git a/database/webif/__init__.py b/database/webif/__init__.py index 79b10c6dd..e5425b6a8 100755 --- a/database/webif/__init__.py +++ b/database/webif/__init__.py @@ -76,9 +76,7 @@ def index(self, reload=None, action=None, item_id=None, item_path=None, time_end if item_path is not None: item = self.plugin.items.return_item(item_path) delete_triggered = False - self.logger.error(f'index called with oid {orphan_id} and nid {new_id}') if orphan_id is not None and new_id is not None and orphan_id != new_id: - self.logger.error(f'calling reassign for {orphan_id} and {new_id}') self.plugin.reassign_orphaned_id(orphan_id, to=new_id) if action is not None: if action == "delete_log" and item_id is not None: diff --git a/database/webif/templates/index.html b/database/webif/templates/index.html index c65e8a91a..c7c7a3605 100755 --- a/database/webif/templates/index.html +++ b/database/webif/templates/index.html @@ -177,7 +177,7 @@ document.getElementById('orphanSelect').selectedIndex = 0; // debug: show call parameters - alert("shngPost('', {orphan_id: " + orphanID + ", new_id: " + newID + "});"); + // alert("shngPost('', {orphan_id: " + orphanID + ", new_id: " + newID + "});"); // call index page with arguments shngPost('', {orphan_id: orphanID, new_id: newID}); From 8df822856180368005ceab1f8384bbd721b651e3 Mon Sep 17 00:00:00 2001 From: Morg42 <43153739+Morg42@users.noreply.github.com> Date: Mon, 14 Oct 2024 19:59:27 +0200 Subject: [PATCH 4/7] database: move to REST communication --- database/__init__.py | 1 + database/webif/__init__.py | 26 +++++++++++++++++++++++--- database/webif/templates/index.html | 22 +++++++++++++++++----- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/database/__init__.py b/database/__init__.py index 357714091..b66ae81b1 100755 --- a/database/__init__.py +++ b/database/__init__.py @@ -988,6 +988,7 @@ def reassign_orphaned_id(self, orphan_id, to): self.build_orphanlist() except Exception as e: self.logger.error(f'error on reassigning id {orphan_id} to {to}: {e}') + return e def _delete_orphan(self, item_path): """ diff --git a/database/webif/__init__.py b/database/webif/__init__.py index e5425b6a8..1fef30513 100755 --- a/database/webif/__init__.py +++ b/database/webif/__init__.py @@ -63,7 +63,7 @@ def __init__(self, webif_dir, plugin): @cherrypy.expose def index(self, reload=None, action=None, item_id=None, item_path=None, time_end=None, day=None, month=None, year=None, - time_orig=None, changed_orig=None, orphan_id=None, new_id=None): + time_orig=None, changed_orig=None): """ Build index.html for cherrypy @@ -76,8 +76,6 @@ def index(self, reload=None, action=None, item_id=None, item_path=None, time_end if item_path is not None: item = self.plugin.items.return_item(item_path) delete_triggered = False - if orphan_id is not None and new_id is not None and orphan_id != new_id: - self.plugin.reassign_orphaned_id(orphan_id, to=new_id) if action is not None: if action == "delete_log" and item_id is not None: if time_orig is not None and changed_orig is not None: @@ -132,6 +130,28 @@ def index(self, reload=None, action=None, item_id=None, item_path=None, time_end tabcount=2, action=action, item_id=item_id, delete_triggered=delete_triggered, language=self.plugin.get_sh().get_defaultlanguage()) + @cherrypy.expose + def reassign(self): + cl = cherrypy.request.headers['Content-Length'] + if not cl: + return + try: + rawbody = cherrypy.request.body.read(int(cl)) + data = json.loads(rawbody) + except Exception: + return + orphan_id = data.get("orphan_id") + new_id = data.get("new_id") + result = {"operation": "request", "result": "success"} + if orphan_id is not None and new_id is not None and orphan_id != new_id: + self.logger.info(f'reassigning orphaned id {orphan_id} to new id {new_id}') + err = self.plugin.reassign_orphaned_id(orphan_id, to=new_id) + if err: + return + return json.dumps(result) + else: + self.logger.warning(f'reassigning orphaned id {orphan_id} to new id {new_id} failed') + @cherrypy.expose def get_data_html(self, dataSet=None, params=None): """ diff --git a/database/webif/templates/index.html b/database/webif/templates/index.html index c7c7a3605..d19ecb524 100755 --- a/database/webif/templates/index.html +++ b/database/webif/templates/index.html @@ -179,11 +179,23 @@ // debug: show call parameters // alert("shngPost('', {orphan_id: " + orphanID + ", new_id: " + newID + "});"); - // call index page with arguments - shngPost('', {orphan_id: orphanID, new_id: newID}); - - // reload page to reflect recalculated orphans - setTimeout(window.location.reload(), 3000); + var mydata = {"orphan_id": orphanID, "new_id": newID}; + $.ajax({ + type: "POST", + url: "reassign", + data: JSON.stringify(mydata), + contentType: 'application/json', + dataType: 'json', + error: function() { + alert("Fehler beim Übermitteln der Daten. Bitte shng-Log prüfen!"); + document.getElementById('orphanModal').style.display = 'none'; + }, + success: function() { + document.getElementById('orphanModal').style.display = 'none'; + // reload page to reflect recalculated orphans + setTimeout(window.location.reload(), 3000); + } + }) } From 3da5851a919db17823e0d77fb68898cf864196e6 Mon Sep 17 00:00:00 2001 From: Morg42 <43153739+Morg42@users.noreply.github.com> Date: Wed, 16 Oct 2024 17:10:44 +0200 Subject: [PATCH 5/7] database: remove debug info from index.html --- database/webif/templates/index.html | 3 --- 1 file changed, 3 deletions(-) diff --git a/database/webif/templates/index.html b/database/webif/templates/index.html index d19ecb524..173072106 100755 --- a/database/webif/templates/index.html +++ b/database/webif/templates/index.html @@ -176,9 +176,6 @@ document.getElementById('orphan-item-name').textContent = ""; document.getElementById('orphanSelect').selectedIndex = 0; - // debug: show call parameters - // alert("shngPost('', {orphan_id: " + orphanID + ", new_id: " + newID + "});"); - var mydata = {"orphan_id": orphanID, "new_id": newID}; $.ajax({ type: "POST", From 368b372db240bab98466b948d29bb7212bc350ca Mon Sep 17 00:00:00 2001 From: Morg42 <43153739+Morg42@users.noreply.github.com> Date: Wed, 16 Oct 2024 17:11:23 +0200 Subject: [PATCH 6/7] database: raw string for regex --- database/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database/__init__.py b/database/__init__.py index b66ae81b1..6b06f3223 100755 --- a/database/__init__.py +++ b/database/__init__.py @@ -1207,7 +1207,7 @@ def _expression(self, func): expression['finalizer'] = func[:func.index(":")] func = func[func.index(":") + 1:] if func == 'count' or func.startswith('count'): - parts = re.match('(count)((<>|!=|<|=|>)(\d+))?', func) + parts = re.match(r'(count)((<>|!=|<|=|>)(\d+))?', func) func = 'count' if parts and parts.group(3) is not None: expression['params']['op'] = parts.group(3) From ca3c8e81d7c296d14a2abdd8666fda9974423394 Mon Sep 17 00:00:00 2001 From: Morg42 <43153739+Morg42@users.noreply.github.com> Date: Sat, 19 Oct 2024 21:59:41 +0200 Subject: [PATCH 7/7] database: add max_reassign_logentries parameter --- database/__init__.py | 22 ++++++++++++++++------ database/plugin.yaml | 10 +++++++++- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/database/__init__.py b/database/__init__.py index b66ae81b1..e2d2ef0c3 100755 --- a/database/__init__.py +++ b/database/__init__.py @@ -51,7 +51,7 @@ class Database(SmartPlugin): """ ALLOW_MULTIINSTANCE = True - PLUGIN_VERSION = '1.6.12' + PLUGIN_VERSION = '1.6.13' # SQL queries: {item} = item table name, {log} = log table name # time, item_id, val_str, val_num, val_bool, changed @@ -104,6 +104,7 @@ def __init__(self, sh, *args, **kwargs): self._precision = self.get_parameter_value('precision') self.count_logentries = self.get_parameter_value('count_logentries') self.max_delete_logentries = self.get_parameter_value('max_delete_logentries') + self.max_reassign_logentries = self.get_parameter_value('max_reassign_logentries') self._default_maxage = float(self.get_parameter_value('default_maxage')) self._copy_database = self.get_parameter_value('copy_database') @@ -976,15 +977,24 @@ def reassign_orphaned_id(self, orphan_id, to): :type orphan_id: int :type to: int """ + log_info = self.logger.warning # info + log_debug = self.logger.error # debug try: - self.logger.debug(f'reassigning orphaned data from (old) id {orphan_id} to (new) id {to}') + log_info(f'reassigning orphaned data from (old) id {orphan_id} to (new) id {to}') cur = self._db_maint.cursor() - self._execute(self._prepare("UPDATE {log} SET item_id = :newid WHERE item_id = :orphanid;"), {'newid': to, 'orphanid': orphan_id}, cur=cur) + count = self.readLogCount(orphan_id, cur=cur) + log_debug(f'found {count} entries to reassign, reassigning {self.max_reassign_logentries} at once') + + while count > 0: + log_debug(f'reassigning {min(count, self.max_reassign_logentries)} log entries') + self._execute(self._prepare("UPDATE {log} SET item_id = :newid WHERE item_id = :orphanid LIMIT :limit;"), {'newid': to, 'orphanid': orphan_id, 'limit': self.max_reassign_logentries}, cur=cur) + count -= self.max_reassign_logentries + self._execute(self._prepare("DELETE FROM {item} WHERE id = :orphanid LIMIT 1;"), {'orphanid': orphan_id}, cur=cur) - self.logger.info(f'reassigned orphaned id {orphan_id} to new id {to}') + log_info(f'reassigned orphaned id {orphan_id} to new id {to}') cur.close() self._db_maint.commit() - self.logger.debug('rebuilding orphan list') + log_debug('rebuilding orphan list') self.build_orphanlist() except Exception as e: self.logger.error(f'error on reassigning id {orphan_id} to {to}: {e}') @@ -1207,7 +1217,7 @@ def _expression(self, func): expression['finalizer'] = func[:func.index(":")] func = func[func.index(":") + 1:] if func == 'count' or func.startswith('count'): - parts = re.match('(count)((<>|!=|<|=|>)(\d+))?', func) + parts = re.match(r'(count)((<>|!=|<|=|>)(\d+))?', func) func = 'count' if parts and parts.group(3) is not None: expression['params']['op'] = parts.group(3) diff --git a/database/plugin.yaml b/database/plugin.yaml index 3f4cbbafe..50f3e59fb 100755 --- a/database/plugin.yaml +++ b/database/plugin.yaml @@ -11,7 +11,7 @@ plugin: keywords: database support: https://knx-user-forum.de/forum/supportforen/smarthome-py/1021844-neues-database-plugin - version: 1.6.12 # Plugin version + version: 1.6.13 # Plugin version sh_minversion: '1.9.3.2' # minimum shNG version to use this plugin # sh_maxversion: # maximum shNG version to use this plugin (leave empty if latest) multi_instance: True # plugin supports multi instance @@ -72,6 +72,14 @@ parameters: de: "Maximal auf einmal zu löschende Anzahl an Log Einträgen mit dem database_maxage Attribut, reduziert die Belastung der Datenbank bei alten Datenbeständen" en: "Maximum number of Logentries to delete at once with database_maxage attribute, reduces load on database with old datasets" + max_reassign_logentries: + type: int + default: 20 # 000 + valid_min: 10 # 00 + description: + de: "Maximal auf einmal neu zuzuweisende Anzahl an Log Einträgen, reduziert die Belastung der Datenbank bei großen Datenbeständen" + en: "Maximum number of Logentries to reassign at once, reduces load on database with large datasets" + default_maxage: type: int default: 0