From cee4cdfdfc0bd4a15e66ae6e512ebff1b17be117 Mon Sep 17 00:00:00 2001
From: Philip Guyton <philip@yewtreeapps.com>
Date: Mon, 29 Oct 2018 20:08:08 +0000
Subject: [PATCH] regression - source build fails to fully initialise db #1983

Re-order db migration by early --fake-initial migration of contenttypes.

Refactor initial database initialisation to be only within initrock to
centralised this mechanism, removes previous db migration redundancy
within buildout and some associated legacy db operations and in turn
avoids major duplication of db initialisation on initial clean (no db or
no .initrock file) builds. Move to only invoking the initrock script
(during source build) via systemd dependency execution. This gives
cleaner error reporting re failed dependency on rockstor-pre (which
normally invokes initrock) and follows / tests normal execution flow.

Consequently avoided runtime error:
"Error creating new content types. Please make sure contenttypes
is migrated before trying to migrate apps individually."
on recent first attempt migrations during clean (no db or no .initrock)
builds.

Summary:

- Migrate --fake-initial contenttypes before individual apps.
- Remove buildout db prep to centralise within initrock.
- Use systemd execution chain to invoke initrock / rockstor-pre
- Add systemd services setup step in buildout: required by prior item.
- Add logging for db delete within initrock.
- Add debug logging to existing db --fake migrations
- Remove currently unused gulp-install from buildout, this entry breaks
non legacy builds.
---
 buildout.cfg                     | 30 ++++++++++++++++++++++--------
 src/rockstor/scripts/initrock.py | 11 ++++++++++-
 2 files changed, 32 insertions(+), 9 deletions(-)

diff --git a/buildout.cfg b/buildout.cfg
index d11a9c5a7..077fc5487 100644
--- a/buildout.cfg
+++ b/buildout.cfg
@@ -20,8 +20,8 @@ parts =
       stop-rockstor
       django
       scripts
-      postgres-setup
-      postgres-conf
+#      postgres-setup
+#      postgres-conf
       gunicorn
       nginx-conf
       shellinabox-conf
@@ -35,7 +35,6 @@ parts =
       js-libraries
       js-sync
       collectstatic
-      gulp-install
       supervisor
       supervisord-conf
       create-cert
@@ -43,6 +42,8 @@ parts =
       rockstor-pre-systemd-conf
       rockstor-systemd-conf
       bootstrap-systemd-conf
+# Note the following systemd links are checked / updated in turn by '.initrock'
+      setup-systemd-links
       def-kernel
       start-rockstor
 
@@ -73,13 +74,26 @@ recipe = plone.recipe.command
 command = systemctl stop rockstor
 update-command = ${stop-rockstor:command}
 
+[setup-systemd-links]
+recipe = plone.recipe.command
+command =
+    cp -f ${buildout:directory}/conf/rockstor-pre.service /etc/systemd/system/rockstor-pre.service &&
+    systemctl enable rockstor-pre.service &&
+    cp -f ${buildout:directory}/conf/rockstor.service /etc/systemd/system/rockstor.service &&
+    systemctl enable rockstor.service &&
+    cp -f ${buildout:directory}/conf/rockstor-bootstrap.service /etc/systemd/system/rockstor-bootstrap.service &&
+    systemctl enable rockstor-bootstrap.service &&
+    systemctl daemon-reload
+update-command = ${setup-systemd-links:command}
+
 [start-rockstor]
 recipe = plone.recipe.command
-command = cp -f ${buildout:directory}/conf/rockstor-pre.service /etc/systemd/system/rockstor-pre.service &&
-	  systemctl enable rockstor-pre.service &&
-	  systemctl daemon-reload &&
-	  systemctl restart rockstor-pre
-	  systemctl start rockstor
+# We stop rockstor-pre, just in case, to invoke a re-run via start of rockstor.
+command =
+    systemctl stop rockstor-pre &&
+# Following rockstor start will in turn start rockstor-pre if found stopped.
+# We then follow / test the usual systemd execution chain.
+    systemctl start rockstor
 update-command = ${start-rockstor:command}
 
 
diff --git a/src/rockstor/scripts/initrock.py b/src/rockstor/scripts/initrock.py
index 88bfc4950..e155a0a4a 100644
--- a/src/rockstor/scripts/initrock.py
+++ b/src/rockstor/scripts/initrock.py
@@ -344,6 +344,7 @@ def main():
         logging.debug('Progresql enabled')
         pg_data = '/var/lib/pgsql/data'
         if (os.path.isdir(pg_data)):
+            logger.debug('Deleting /var/lib/pgsql/data')
             shutil.rmtree('/var/lib/pgsql/data')
         logging.info('initializing Postgresql...')
         run_command(['/usr/bin/postgresql-setup', 'initdb'])
@@ -382,8 +383,14 @@ def main():
     logging.info('Running app database migrations...')
     migration_cmd = [DJANGO, 'migrate', '--noinput', ]
     fake_migration_cmd = migration_cmd + ['--fake']
+    fake_initial_migration_cmd = migration_cmd + ['--fake-initial']
     smartdb_opts = ['--database=smart_manager', 'smart_manager']
 
+    # Migrate Content types before individual apps
+    logger.debug('migrate (--fake-initial) contenttypes')
+    run_command(
+        fake_initial_migration_cmd + ['--database=default', 'contenttypes'])
+
     for app in ('storageadmin', 'smart_manager'):
         db = 'default'
         if app == 'smart_manager':
@@ -397,11 +404,13 @@ def main():
                 break
         if not initial_faked:
             db_arg = '--database=%s' % db
+            logger.debug('migrate (--fake) db=({}) app=({}) 0001_initial'
+                         .format(db, app))
             run_command(fake_migration_cmd + [db_arg, app, '0001_initial'])
 
+    run_command(migration_cmd + ['auth'])
     run_command(migration_cmd + ['storageadmin'])
     run_command(migration_cmd + smartdb_opts)
-    run_command(migration_cmd + ['auth'])
     run_command(migration_cmd + ['django_ztask'])
     logging.info('Done')
     logging.info('Running prepdb...')