add scripts to backfill undelegated fields in db

MANIFEST

@@ -56,9 +56,11 @@ share/patch_db_README.txt
 share/patch_mysql_db_zonemaster_backend_ver_1.0.3.pl
 share/patch_mysql_db_zonemaster_backend_ver_5.0.0.pl
 share/patch_mysql_db_zonemaster_backend_ver_5.0.2.pl
+share/patch_mysql_db_zonemaster_backend_ver_7.0.0.pl
 share/patch_postgresql_db_zonemaster_backend_ver_1.0.3.pl
 share/patch_postgresql_db_zonemaster_backend_ver_5.0.0.pl
 share/patch_postgresql_db_zonemaster_backend_ver_7.0.0.pl
+share/patch_sqlite_db_zonemaster_backend_ver_7.0.0.pl
 share/tmpfiles.conf
 share/travis_mysql_backend_config.ini
 share/travis_postgresql_backend_config.ini

README.md

@@ -25,7 +25,7 @@ Zonemaster Engine installed. Please see the
 instructions](https://github.com/zonemaster/zonemaster-engine/blob/master/docs/Installation.md).
 
 
-### Upgrade 
+### Upgrade
 
 If you upgrade Zonemaster-Backend and want to keep the content of the database
 (SQLite, MySQL/MariaDB or PostgreSQL) then you should not reset the database when
@@ -40,7 +40,7 @@ Older than 1.0.3                      | [Upgrade to 1.0.3]    |
 At least 1.0.3 but older than 1.1.0   | [Upgrade to 1.1.0]    |
 At least 1.1.0 but older than 5.0.0   | [Upgrade to 5.0.0]    |
 At least 5.0.0 but older than 5.0.2   | [Upgrade to 5.0.2]    | For MySQL/MariaDB only
-At least 5.0.2 but older than 7.0.0   | [Upgrade to 7.0.0]    | For PostgreSQL only
+At least 5.0.2 but older than 7.0.0   | [Upgrade to 7.0.0]    | For all database types
 
 If the database was created before Zonemaster-Backend version 5.0.2, then you
 have to upgrade it in several steps.

docs/upgrade_db_zonemaster_backend_ver_7.0.0.md

@@ -12,14 +12,20 @@ export ZONEMASTER_BACKEND_CONFIG_FILE="/usr/local/etc/zonemaster/backend_config.
 
 ### SQLite
 
-No patching (upgrading) is needed on zonemaster database on SQLite for this
-version of Zonemaster-Backend.
+Run
+```sh
+cd $(perl -MFile::ShareDir -le 'print File::ShareDir::dist_dir("Zonemaster-Backend")')
+perl patch_sqlite_db_zonemaster_backend_ver_7.0.0.pl
+```
 
 
 ### MySQL (or MariaDB)
 
-No patching (upgrading) is needed on zonemaster database on MySQL (or MariaDB)
-for this version of Zonemaster-Backend.
+Run
+```sh
+cd $(perl -MFile::ShareDir -le 'print File::ShareDir::dist_dir("Zonemaster-Backend")')
+perl patch_mysql_db_zonemaster_backend_ver_7.0.0.pl
+```
 
 
 ### PostgreSQL
@@ -29,4 +35,3 @@ Run
 cd $(perl -MFile::ShareDir -le 'print File::ShareDir::dist_dir("Zonemaster-Backend")')
 perl patch_postgresql_db_zonemaster_backend_ver_7.0.0.pl
 ```
-

share/patch_mysql_db_zonemaster_backend_ver_7.0.0.pl

@@ -0,0 +1,34 @@
+use strict;
+use warnings;
+use utf8;
+use Data::Dumper;
+use Encode;
+use JSON::PP;
+
+use DBI qw(:utils);
+
+use Zonemaster::Backend::Config;
+use Zonemaster::Backend::DB::MySQL;
+
+my $config = Zonemaster::Backend::Config->load_config();
+if ( $config->DB_engine ne 'MySQL' ) {
+    die "The configuration file does not contain the MySQL backend";
+}
+my $db = Zonemaster::Backend::DB::MySQL->from_config( $config );
+my $dbh = $db->dbh;
+
+
+sub patch_db {
+    my @arefs = $dbh->selectall_array('SELECT id, params from test_results', undef);
+    foreach my $row (@arefs) {
+        my $id = @$row[0];
+        my $raw_params = decode_json(@$row[1]);
+        my $ds_info_values = scalar( map { grep (!/^$/, values( %$_ ) ) } @{$raw_params->{ds_info}});
+        my $nameservers_values = scalar( map { grep (!/^$/, values( %$_ ) ) } @{$raw_params->{nameservers}});
+        my $undelegated = $ds_info_values > 0 || $nameservers_values > 0 || 0;
+
+        $dbh->do('UPDATE test_results SET undelegated = ? where id = ?', undef, $undelegated, $id);
+    }
+}
+
+patch_db();

share/patch_postgresql_db_zonemaster_backend_ver_7.0.0.pl

@@ -20,8 +20,60 @@ sub patch_db {
     ####################################################################
     # TEST RESULTS
     ####################################################################
-    $dbh->do( 'ALTER TABLE test_results ADD COLUMN undelegated integer NOT NULL DEFAULT 0' );
+    eval {
+        $dbh->do( 'ALTER TABLE test_results ADD COLUMN undelegated integer NOT NULL DEFAULT 0' );
+    };
+    print "Error while changing DB schema:  " . $@;
+
+    $dbh->do( qq[
+update test_results set undelegated = test_results_undelegated.undelegated_bool::int
+from (
+    select
+        test_results.id,
+        (
+            case when ds_filled.ds_filled is null then false else ds_filled.ds_filled end
+            or
+            case when ns_filled.ns_filled is null then false else ns_filled.ns_filled end
+        ) as undelegated_bool
+    from test_results
+    left join (
+        select
+            count(*) > 0 as ds_filled,
+            id
+        from (
+            select
+                jd.value,
+                id
+            from
+            (
+                select json_array_elements(params->'ds_info') as ja, id
+                from test_results
+            )  as s1,
+            json_each_text(ja) as jd
+        ) as s2
+        where value is not null and value::text != ''::text
+        group by id
+    ) as ds_filled on ds_filled.id = test_results.id
+    left join (
+            select
+            count(*) > 0 as ns_filled,
+            id
+        from (
+            select
+                jd.value,
+                id
+            from
+            (
+                select json_array_elements(params->'nameservers') as ja, id
+                from test_results
+            )  as s1,
+            json_each_text(ja) as jd
+        ) as s2
+        where value is not null and value::text != ''::text
+        group by id
+    ) as ns_filled on ns_filled.id = test_results.id
+) as test_results_undelegated where test_results.id = test_results_undelegated.id;
+    ] );
 }
 
 patch_db();
-

share/patch_sqlite_db_zonemaster_backend_ver_7.0.0.pl

@@ -0,0 +1,34 @@
+use strict;
+use warnings;
+use utf8;
+use Data::Dumper;
+use Encode;
+use JSON::PP;
+
+use DBI qw(:utils);
+
+use Zonemaster::Backend::Config;
+use Zonemaster::Backend::DB::SQLite;
+
+my $config = Zonemaster::Backend::Config->load_config();
+if ( $config->DB_engine ne 'SQLite' ) {
+    die "The configuration file does not contain the SQLite backend";
+}
+my $db = Zonemaster::Backend::DB::SQLite->from_config( $config );
+my $dbh = $db->dbh;
+
+
+sub patch_db {
+    my @arefs = $dbh->selectall_array('SELECT id, params from test_results', undef);
+    foreach my $row (@arefs) {
+        my $id = @$row[0];
+        my $raw_params = decode_json(@$row[1]);
+        my $ds_info_values = scalar( map { grep (!/^$/, values( %$_ ) ) } @{$raw_params->{ds_info}});
+        my $nameservers_values = scalar( map { grep (!/^$/, values( %$_ ) ) } @{$raw_params->{nameservers}});
+        my $undelegated = $ds_info_values > 0 || $nameservers_values > 0 || 0;
+
+        $dbh->do('UPDATE test_results SET undelegated = ? where id = ?', undef, $undelegated, $id);
+    }
+}
+
+patch_db();