Merge pull request #5550 from nextcloud/enh/5451/exclude-previews

allow to exclude previews from restore upon instance restore

Containers/borgbackup/backupscript.sh

@@ -320,6 +320,18 @@ if [ "$BORG_MODE" = restore ]; then
     fi
     echo "Restoring '$SELECTED_ARCHIVE'..."
 
+    # Exclude previews from restore if selected to speed up process
+    ADDITIONAL_RSYNC_EXCLUDES=()
+    ADDITIONAL_BORG_EXCLUDES=()
+    ADDITIONAL_FIND_EXCLUDES=()
+    if [ -n "$RESTORE_EXCLUDE_PREVIEWS" ]; then
+        # Keep these 3 in sync. Beware, the pattern syntax and the paths differ
+        ADDITIONAL_RSYNC_EXCLUDES=(--exclude "nextcloud_aio_nextcloud_data/appdata_*/preview/**")
+        ADDITIONAL_BORG_EXCLUDES=(--exclude "sh:nextcloud_aio_volumes/nextcloud_aio_nextcloud_data/appdata_*/preview/**")
+        ADDITIONAL_FIND_EXCLUDES=(-o -regex 'nextcloud_aio_volumes/nextcloud_aio_nextcloud_data/appdata_[^/]*/preview\(/.*\)?')
+        echo "Excluding previews from restore"
+    fi
+
     # Save Additional Backup dirs
     if [ -f "/nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/additional_backup_directories" ]; then
         ADDITIONAL_BACKUP_DIRECTORIES="$(cat /nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/additional_backup_directories)"
@@ -365,6 +377,7 @@ if [ "$BORG_MODE" = restore ]; then
         --exclude "nextcloud_aio_mastercontainer/data/daily_backup_running" \
         --exclude "nextcloud_aio_mastercontainer/data/session_date_file" \
         --exclude "nextcloud_aio_mastercontainer/session/**" \
+        "${ADDITIONAL_RSYNC_EXCLUDES[@]}" \
         /tmp/borg/nextcloud_aio_volumes/ /nextcloud_aio_volumes/; then
             RESTORE_FAILED=1
             echo "Something failed while restoring from backup."
@@ -391,17 +404,18 @@ if [ "$BORG_MODE" = restore ]; then
         #
         # Older backups may still contain files we've since excluded, so we have to exclude on extract as well.
         cd /  # borg extract has no destination arg and extracts to CWD
-        if ! borg extract "::$SELECTED_ARCHIVE" --progress --exclude-from /borg_excludes --pattern '+nextcloud_aio_volumes/**'
+        if ! borg extract "::$SELECTED_ARCHIVE" --progress --exclude-from /borg_excludes "${ADDITIONAL_BORG_EXCLUDES[@]}" --pattern '+nextcloud_aio_volumes/**'
         then
             RESTORE_FAILED=1
             echo "Failed to extract backup archive."
         else
             # Delete files/dirs present locally, but not in the backup archive, excluding conf files
             # https://unix.stackexchange.com/a/759341
             # This comm does not support -z, but I doubt any file names would have \n in them
-            echo "Deleting local files which do not exist in the backup"
+            #
             # These find patterns need to be kept in sync with the borg_excludes file and the rsync excludes in this
             # file, which use a different syntax (patterns appear in 3 places in total)
+            echo "Deleting local files which do not exist in the backup"
             if ! find nextcloud_aio_volumes \
                     -not \( \
                         -path nextcloud_aio_volumes/nextcloud_aio_apache/caddy \
@@ -417,6 +431,7 @@ if [ "$BORG_MODE" = restore ]; then
                         -o -path nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/daily_backup_running \
                         -o -path nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/session_date_file \
                         -o -path "nextcloud_aio_volumes/nextcloud_aio_mastercontainer/data/id_borg*" \
+                        "${ADDITIONAL_FIND_EXCLUDES[@]}" \
                     \) \
                     | LC_ALL=C sort \
                     | LC_ALL=C comm -23 - \
@@ -488,6 +503,12 @@ if [ "$BORG_MODE" = restore ]; then
     touch "/nextcloud_aio_volumes/nextcloud_aio_nextcloud_data/fingerprint.update"
     chmod 777 "/nextcloud_aio_volumes/nextcloud_aio_nextcloud_data/fingerprint.update"
 
+    # Add file to Netcloud container to trigger a preview scan the next time it starts
+    if [ -n "$RESTORE_EXCLUDE_PREVIEWS" ]; then
+        touch "/nextcloud_aio_volumes/nextcloud_aio_nextcloud_data/trigger-preview.scan"
+        chmod 777 "/nextcloud_aio_volumes/nextcloud_aio_nextcloud_data/trigger-preview.scan"
+    fi
+
     # Delete redis cache
     rm -f "/mnt/redis/dump.rdb"
 fi

Containers/nextcloud/entrypoint.sh

@@ -495,6 +495,12 @@ if [ -f "$NEXTCLOUD_DATA_DIR/fingerprint.update" ]; then
     rm "$NEXTCLOUD_DATA_DIR/fingerprint.update"
 fi
 
+# Perform preview scan if previews were excluded from restore
+if [ -f "$NEXTCLOUD_DATA_DIR/trigger-preview.scan" ]; then
+    php /var/www/html/occ files:scan-app-data preview -vvv
+    rm "$NEXTCLOUD_DATA_DIR/trigger-preview.scan"
+fi
+
 # AIO one-click settings start # Do not remove or change this line!
 # Apply one-click-instance settings
 echo "Applying one-click-instance settings..."

php/containers.json

@@ -477,6 +477,7 @@
         "BORG_PASSWORD=%BORGBACKUP_PASSWORD%",
         "BORG_MODE=%BORGBACKUP_MODE%",
         "SELECTED_RESTORE_TIME=%SELECTED_RESTORE_TIME%",
+        "RESTORE_EXCLUDE_PREVIEWS=%RESTORE_EXCLUDE_PREVIEWS%",
         "BACKUP_RESTORE_PASSWORD=%BACKUP_RESTORE_PASSWORD%",
         "ADDITIONAL_DIRECTORIES_BACKUP=%ADDITIONAL_DIRECTORIES_BACKUP%",
         "BORGBACKUP_HOST_LOCATION=%BORGBACKUP_HOST_LOCATION%",

php/src/Controller/DockerController.php

@@ -113,6 +113,11 @@ public function StartBackupContainerRestore(Request $request, Response $response
         $config = $this->configurationManager->GetConfig();
         $config['backup-mode'] = 'restore';
         $config['selected-restore-time'] = $request->getParsedBody()['selected_restore_time'] ?? '';
+        if (isset($request->getParsedBody()['restore-exclude-previews'])) {
+            $config['restore-exclude-previews'] = 1;
+        } else {
+            $config['restore-exclude-previews'] = '';
+        }
         $this->configurationManager->WriteConfig($config);
 
         $id = self::TOP_CONTAINER;

php/src/Data/ConfigurationManager.php

@@ -427,6 +427,15 @@ public function GetSelectedRestoreTime() : string {
         return $config['selected-restore-time'];
     }
 
+    public function GetRestoreExcludePreviews() : string {
+        $config = $this->GetConfig();
+        if(!isset($config['restore-exclude-previews'])) {
+            $config['restore-exclude-previews'] = '';
+        }
+
+        return $config['restore-exclude-previews'];
+    }
+
     public function GetAIOURL() : string {
         $config = $this->GetConfig();
         if(!isset($config['AIO_URL'])) {

php/src/Docker/DockerActionManager.php

@@ -273,6 +273,8 @@ public function CreateContainer(Container $container) : void {
                     $replacements[1] = $this->configurationManager->GetAIOURL();
                 } elseif ($out[1] === 'SELECTED_RESTORE_TIME') {
                     $replacements[1] = $this->configurationManager->GetSelectedRestoreTime();
+                } elseif ($out[1] === 'RESTORE_EXCLUDE_PREVIEWS') {
+                    $replacements[1] = $this->configurationManager->GetRestoreExcludePreviews();
                 } elseif ($out[1] === 'APACHE_PORT') {
                     $replacements[1] = $this->configurationManager->GetApachePort();
                 } elseif ($out[1] === 'TALK_PORT') {

php/templates/containers.twig

@@ -168,7 +168,8 @@
                                                 {% for restore_time in backup_times %}
                                                     <option value="{{ restore_time }}">{{ restore_time }} UTC</option>
                                                 {% endfor %}
-                                            </select>
+                                            </select><br>
+                                            <input type="checkbox" id="restore-exclude-previews" name="restore-exclude-previews"><label for="restore-exclude-previews">Exclude previews from restore which will speed up the restore process but will trigger a scan of the preview folder as soon as the Nextcloud container starts the next time</label><br>
                                             <input type="submit" value="Restore selected backup"/>
                                         </form>
                                     {% endif %}