Merge pull request #2010 from phillxnet/1722_pool_resize_disk_removal…

…_unknown_internal_error_and_no_UI_counterpart

pool resize disk removal unknown internal error and no UI counterpart. Fixes #1722

src/rockstor/scripts/flash_optimize.py

@@ -73,7 +73,7 @@ def trim_support(disk):
         run_command([YUM, 'install', '-y', 'hdparm'])
         logging.info('Installed hdparm successfully')
 
-    o, e, rc = run_command(['hdparm', '-I', '/dev/%s' % disk])
+    o, e, rc = run_command(['hdparm', '-I', '{}'.format(disk)])
     for l in o:
         if (re.search('Data Set Management TRIM supported', l) is not None):
             logging.debug('TRIM supported. info: %s' % l)
@@ -84,7 +84,7 @@ def trim_support(disk):
 
 def is_flash(disk):
     flash = False
-    o, e, rc = run_command(['udevadm', 'info', '--path=/sys/block/%s' % disk])
+    o, e, rc = run_command(['udevadm', 'info', '--name', disk])
     for l in o:
         if (re.search('ID_BUS=', l) is not None):
             if (l.strip().split()[1].split('=')[1] != 'usb'):
@@ -98,6 +98,8 @@ def is_flash(disk):
     # /sys/block/disk/queue/rotational is not reliable, but if [deadline] is in
     # /sys/block/disk/queue/scheduler, it's fair to assume flash
     logging.debug('Checking if scheduler is set to [deadline] for %s' % disk)
+    disk = disk.split('/')[-1]  # strip off the path
+    # Note that the following may fail for sys on luks dev.
     with open('/sys/block/%s/queue/scheduler' % disk) as sfo:
         for l in sfo.readlines():
             if (re.search('\[deadline\]', l) is not None):

src/rockstor/storageadmin/migrations/0008_auto_20190115_1637.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('storageadmin', '0007_auto_20181210_0740'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='disk',
+            name='allocated',
+            field=models.BigIntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name='disk',
+            name='devid',
+            field=models.PositiveSmallIntegerField(default=0),
+        ),
+        migrations.AddField(
+            model_name='poolbalance',
+            name='internal',
+            field=models.BooleanField(default=False),
+        ),
+    ]

src/rockstor/storageadmin/models/disk.py

@@ -44,8 +44,14 @@ class Disk(models.Model):
     mostly derived from model and serial number.
     """
     name = models.CharField(max_length=128, unique=True)
-    """total size in KB"""
+    """btrfs devid 0 is place holder as real devids start from 1"""
+    devid = models.PositiveSmallIntegerField(default=0)  # 0 to 32767
+    """total size in KB. Zero if btrfs device detached/last stage of delete."""
     size = models.BigIntegerField(default=0)
+    """allocated in KB: ie per device 'used' in 'btrfs fi show' and total
+    listed per device in 'btrfs fi usage /mnt_pt'.
+    """
+    allocated = models.BigIntegerField(default=0)
     """true if disk went offline"""
     offline = models.BooleanField(default=False)
     """whether the disk is partitioned at the moment. relevent for root disks

src/rockstor/storageadmin/models/pool_balance.py

@@ -30,6 +30,8 @@ class PoolBalance(models.Model):
     start_time = models.DateTimeField(auto_now=True)
     end_time = models.DateTimeField(null=True)
     percent_done = models.IntegerField(default=0)
+    # Flag to denote internal auto initiated balance ie during dev delete.
+    internal = models.BooleanField(default=False)
 
     class Meta:
         app_label = 'storageadmin'

src/rockstor/storageadmin/static/storageadmin/js/templates/pool/pool_info_module.jst

@@ -30,6 +30,7 @@
   {{else}}
     <span style="color:red">{{model.mount_status}}</span>
   {{/if}}
-  </strong>
+  </strong><br/>
+  UUID: <strong>{{model.uuid}}</strong>
 </div> <!-- module-content -->
 

src/rockstor/storageadmin/static/storageadmin/js/templates/pool/poolrebalance_table_template.jst

@@ -26,9 +26,11 @@
     <thead>
       <tr>
         <th scope="col" abbr="Id">Id</th>
-        <th scope="col" abbr="Status">Status</th>
+        <th scope="col" abbr="Status">Status <i class="fa fa-info-circle" title="Both balance types: unknown, running, finished, failed.&#013Regular only: cancelling, cancelled, pausing, paused." /> </th>
+        <th scope="col" abbr="Type">Type <i class="fa fa-info-circle" title="Regular or Disk Removal" /></th>
         <th scope="col" abbr="STime">Start Time</th>
-        <th scope="col" abbr="Updates">Percent finished</th>
+        <!-- <th scope="col" abbr="ETime">End Time</th> -->
+        <th scope="col" abbr="Updates">Percent finished <i class="fa fa-info-circle" title="Progress unavailable for Disk Removal." /></th>
         <th scope="col" abbr="Infos">Errors or Notes</th>
       </tr>
     </thead>

src/rockstor/storageadmin/static/storageadmin/js/templates/pool/resize/remove_disks_complete.jst

@@ -6,6 +6,17 @@
 </div>
 <br>
 <div class="alert alert-success">
-  <h4>Resize initiated - disk addition or raid change entails a subsequent Rockstor visible balance which may take several hours. Check status in the Balances tab. Disk delete progress is currently unmonitored.</h4>
+  <h4>Resize initiated - the associated balance can take several hours to complete and usually negatively impacts system performance.
+    <br>Check the Balances tab for status. A page refresh will be required.
+  </h4>
+</div>
+<div class="alert alert-warning">
+  <h4>
+    <strong>Expect reduced Web-UI responsiveness until this balance has finished.</strong>
+    <br>Removed disks will have progressively smaller Allocated GB until they are finally removed.
+    <br>Please note: a detached disk removal can fail with status ending:
+    <br><i>"... no missing devices found to remove"</i>
+    <br> If you encounter this error see the <u>Maintenance required</u> section for guidance then try again after a reboot.
+  </h4>
 </div>
 

src/rockstor/storageadmin/static/storageadmin/js/templates/pool/resize_pool_info.jst

@@ -42,7 +42,9 @@
     <tr>
         <th scope="col" abbr="Name">Name</th>
         <th scope="col" abbr="Temp Name">Temp Name</th>
+        <th scope="col" abbr="Btrfs DevID">Btrfs DevID</th>
         <th scope="col" abbr="Capacity">Capacity</th>
+        <th scope="col" abbr="Allocated (%)">Allocated (%)</th>
         <th scope="col" abbr="write_io_errs">Write I/O errors</th>
         <th scope="col" abbr="read_io_errs">Read I/O errors</th>
         <th scope="col" abbr="flush_io_errs">Flush I/O errors</th>
@@ -66,7 +68,11 @@
         <td>
             {{this.temp_name}}
         </td>
+        <td>
+            {{btrfsDevID this.devid}}
+        </td>
         <td>{{humanReadableSize this.size}}</td>
+        <td>{{humanReadableAllocatedPercent this.allocated this.size}}</td>
         {{ioErrorStatsTableData this.io_error_stats}}
     </tr>
     {{/each}}
@@ -91,7 +97,11 @@
         <td>
             {{this.temp_name}}
         </td>
+        <td>
+            {{btrfsDevID this.devid}}
+        </td>
         <td>{{humanReadableSize this.size}}</td>
+        <td>{{humanReadableAllocatedPercent this.allocated this.size}}</td>
         {{ioErrorStatsTableData this.io_error_stats}}
     </tr>
     {{/each}}

src/rockstor/storageadmin/static/storageadmin/js/views/pool_details_layout_view.js

@@ -263,7 +263,7 @@ PoolDetailsLayoutView = RockstorLayoutView.extend({
         if (confirm('If any detached members are listed use the Resize/ReRaid button - "Remove disks" option instead. Click OK only if "(Some Missing)" and no "detached-..." appear in the Pool page Disks sub-section?')) {
             var raid_level = _this.pool.get('raid');
             var disk_names = ['missing'];
-            var delete_missing_msg = ('Delete missing is initiated (can take several hours), a progress report is currently unavailable. Balance attempts are blocked for this period.');
+            var delete_missing_msg = ('Delete missing initiated - associated balance can take several hours and negatively impact system performance. Check Balances tab for status.');
             $.ajax({
                 url: url,
                 type: 'PUT',
@@ -487,6 +487,29 @@ PoolDetailsLayoutView = RockstorLayoutView.extend({
             return humanize.filesize(size * 1024);
         });
 
+        Handlebars.registerHelper('humanReadableAllocatedPercent', function(allocated, size) {
+            var html = '';
+            html += humanize.filesize(allocated * 1024);
+            // One decimal place of % = 1 GB per TB = normal allocation unit.
+            if (size == 0) {
+                // we likely have a disk delete/removal in operation or a
+                // missing / detached device so flag.
+                html += '<strong><span style="color:darkred"> Missing or removal in progress </span></strong>'
+            } else {
+                html += ' <strong>(' + ((allocated / size) * 100).toFixed(1) + '%)</strong>'
+
+            }
+            return new Handlebars.SafeString(html);
+        });
+
+        Handlebars.registerHelper('btrfsDevID', function(devid){
+            if (devid !== 0) {
+                return devid
+            }
+            var html = '<strong><span style="color:darkred"> Page refresh required </span></strong>';
+            return new Handlebars.SafeString(html)
+        });
+
         Handlebars.registerHelper('isRoot', function(role){
             if (role == 'root') {
                 return true;

src/rockstor/storageadmin/static/storageadmin/js/views/pool_rebalance_table.js

@@ -115,11 +115,31 @@ PoolRebalanceTableModule = RockstorModuleView.extend({
                 html += '<td>' + poolrebalance.get('id') + '</td>';
                 html += '<td>' + poolrebalance.get('status') + '</td>';
                 html += '<td>';
+                internal_balance = poolrebalance.get('internal');
+                percent_done = poolrebalance.get('percent_done')
+                if (internal_balance) {
+                    html += 'Disk Removal'
+                } else {
+                    html += 'Regular'
+                }
+                html += '</td>';
+                html += '<td>';
                 if (poolrebalance.get('start_time')) {
                     html += moment(poolrebalance.get('start_time')).format(RS_DATE_FORMAT);
                 }
                 html += '</td>';
-                html += '<td>' + poolrebalance.get('percent_done') + '</td>';
+                // html += '<td>';
+                // if (poolrebalance.get('end_time')) {
+                //     html += moment(poolrebalance.get('end_time')).format(RS_DATE_FORMAT);
+                // }
+                // html += '</td>';
+                html += '<td>';
+                if (percent_done != 100 && internal_balance) {
+                    html += 'unavailable';
+                } else {
+                    html += percent_done;
+                }
+                html + '</td>';
                 html += '<td>';
                 if (poolrebalance.get('message') != null) {
                     html += poolrebalance.get('message');

src/rockstor/storageadmin/views/command.py

@@ -23,9 +23,8 @@
 from storageadmin.auth import DigestAuthentication
 from rest_framework.permissions import IsAuthenticated
 from storageadmin.views import DiskMixin
-from system.osi import (uptime, kernel_info, get_dev_byid_name,
-                        get_device_path)
-from fs.btrfs import (mount_share, mount_root, get_pool_info,
+from system.osi import (uptime, kernel_info, get_device_mapper_map)
+from fs.btrfs import (mount_share, mount_root, get_dev_pool_info,
                       pool_raid, mount_snap)
 from system.ssh import (sftp_mount_map, sftp_mount)
 from system.services import systemctl
@@ -56,6 +55,10 @@ class CommandView(DiskMixin, NFSExportMixin, APIView):
     @staticmethod
     @transaction.atomic
     def _refresh_pool_state():
+        # Get map of dm-0 to /dev/mapper members ie luks-.. devices.
+        mapped_devs = get_device_mapper_map()
+        # Get temp_names (kernel names) to btrfs pool info for attached devs.
+        dev_pool_info = get_dev_pool_info()
         for p in Pool.objects.all():
             # If our pool has no disks, detached included, then delete it.
             # We leave pools with all detached members in place intentionally.
@@ -70,22 +73,22 @@ def _refresh_pool_state():
                 continue
             try:
                 # Get and save what info we can prior to mount.
-                first_attached_dev = p.disk_set.attached().first()
-                is_root_pool = (p.role == 'root')
-                # Observe any redirect role by using target_name.
-                byid_disk_name, is_byid = get_dev_byid_name(
-                    get_device_path(first_attached_dev.target_name))
-                if is_byid:
-                    pool_info = get_pool_info(first_attached_dev.target_name,
-                                              is_root_pool)
-                    pool_name = pool_info['label']
-                else:
+                first_dev = p.disk_set.attached().first()
+                # Use target_name to account for redirect role.
+                if first_dev.target_name == first_dev.temp_name:
                     logger.error('Skipping pool ({}) mount as attached disk '
                                  '({}) has no by-id name (no serial # ?)'.
-                                 format(p.name,
-                                        first_attached_dev.target_name))
+                                 format(p.name, first_dev.target_name))
                     continue
-                p.name = pool_name
+                if first_dev.temp_name in mapped_devs:
+                    dev_tmp_name = '/dev/mapper/{}'.format(
+                        mapped_devs[first_dev.temp_name])
+                else:
+                    dev_tmp_name = '/dev/{}'.format(first_dev.temp_name)
+                # For now we call get_dev_pool_info() once for each pool.
+                pool_info = dev_pool_info[dev_tmp_name]
+                p.name = pool_info.label
+                p.uuid = pool_info.uuid
                 p.save()
                 mount_root(p)
                 p.raid = pool_raid('%s%s' % (settings.MNT_PT, p.name))['data']