Merge pull request #6 from ambitioninc/develop

Dynamic initial data supports deletions

README.md

@@ -3,29 +3,29 @@
 
 Django Dynamic Initial Data is a `django>=1.6` app that helps solve the problem of initializing data for apps with
 dependencies and other conditional data. Rather than having static fixtures for each app, the initial data
-can be created and updated dynamically. This can be done more easily by making use of the `django-manager-utils`
-upsert functionality. Currently the data initialization process does not handle deleted records, but this is
-a planned feature.
+can be created and updated dynamically. Furthermore, Django Dynamic Initial Data also handles when objects are
+deleted from initial data, a feature that Django's initial data fixture system lacks.
 
 ## Table of Contents
 
 1. [Installation] (#installation)
-1. [A Brief Overview] (#a-brief-overview)
-1. [Example] (#example)
+2. [A Brief Overview] (#a-brief-overview)
+3. [Example] (#example)
+4. [Handling Deletions](#handling-deletions)
 
 ## Installation
 To install Django Dynamic Initial Data:
 
 ```shell
-pip install git+https://github.com/ambitioninc/django-dynamic-initial-data.git
+pip install django-dynamic-initial-data
 ```
 
 Add Django Dynamic Initial Data to your `INSTALLED_APPS` to get started:
 
 settings.py
 ```python
 INSTALLED_APPS = (
-    'django-dynamic-initial-data',
+    'dynamic_initial_data',
 )
 ```
 
@@ -80,3 +80,38 @@ python manage.py update_initial_data --app 'app_path'
 Documentation on using `upsert` and `bulk_upsert` can be found below:
 - https://github.com/ambitioninc/django-manager-utils#upsert
 - https://github.com/ambitioninc/django-manager-utils#bulk_upsert
+
+## Handling Deletions
+One difficulty when specifying initial data in Django apps is the inability to deploy initial data to your project and then subsequently remove any initial data fixtures. If one removes an object in an initial_data.json file, Django does not handle its deletion next time it is deployed, which can cause headaches with lingering objects.
+
+Django Dynamic Initial Data fixes this problem by allowing the user to either:
+
+1. Return all managed initial data objects as an array from the update_initial_data function.
+2. Explicitly register objects for deletion with the register_for_deletion(*model_objs) method.
+
+Note that it is up to the user to be responsible for always registering every object every time, regardless if the object was updated or created by the initial data process. Doing this allows Django Dynamic Initial Data to remove any objects that were previosly managed. For example, assume you have an InitialData class that manages two users with the user names "hello" and "world".
+
+```python
+from dynamic_initial_data.base import BaseInitialData
+
+class InitialData(BaseInitialData):
+    def update_initial_data(self):
+        hello = Account.objects.get_or_create(name='hello')
+        world = Account.objects.get_or_create(name='world')
+        # register the accounts for deletion
+        self.register_for_deletion(hello, world)
+```
+
+After this code is created, the initial data process now owns the "hello" and "world" account objects. If these objects are not registered for deletion in subsequent versions of the code, they will be deleted when the initial data process executes. For example, assume the first piece of code executed and then the user executed this piece of code:
+
+```python
+from dynamic_initial_data.base import BaseInitialData
+
+class InitialData(BaseInitialData):
+    def update_initial_data(self):
+        world = Account.objects.get_or_create(name='world')
+        # register the accounts for deletion
+        self.register_for_deletion(world)
+```
+
+When this piece of code executes, the previous "hello" account would then be deleted since the initial data process no longer owns it. And don't worry, if it was already deleted by another process, the deletion will not throw an error.

dynamic_initial_data/__init__.py

@@ -1,2 +1,3 @@
 # flake8: noqa
 from .version import __version__
+from .base import BaseInitialData

dynamic_initial_data/base.py

@@ -1,6 +1,11 @@
+from datetime import datetime
+
 from django.conf import settings
+from django.contrib.contenttypes.models import ContentType
+from django.db.transaction import atomic
 
 from dynamic_initial_data.exceptions import InitialDataCircularDependency, InitialDataMissingApp
+from dynamic_initial_data.models import RegisteredForDeletionReceipt
 from dynamic_initial_data.utils.import_string import import_string
 
 
@@ -15,6 +20,20 @@ class BaseInitialData(object):
     """
     dependencies = []
 
+    def __init__(self):
+        # Keep track of any model objects that have been registered for deletion
+        self.model_objs_registered_for_deletion = []
+
+    def get_model_objs_registered_for_deletion(self):
+        return self.model_objs_registered_for_deletion
+
+    def register_for_deletion(self, *model_objs):
+        """
+        Registers model objects for deletion. This means the model object will be deleted from the system when it is
+        no longer being managed by the initial data process.
+        """
+        self.model_objs_registered_for_deletion.extend(model_objs)
+
     def update_initial_data(self, *args, **kwargs):
         """
         Raises an error if the subclass does not implement this
@@ -29,11 +48,20 @@ class InitialDataUpdater(object):
     and running of initialization classes.
     """
     def __init__(self, options=None):
+        # Various options that can be passed to the initial data updater
         options = options or {}
         self.verbose = options.get('verbose', False)
+
+        # Apps that have been updated so far. This allows us to process dependencies on other app
+        # inits easier without performing redundant work
         self.updated_apps = set()
+
+        # A cache of the apps that have been imported for data initialization
         self.loaded_apps = {}
 
+        # A list of all models that have been registered for deletion
+        self.model_objs_registered_for_deletion = []
+
     def get_class_path(self, app):
         """
         Builds the full path to the initial data class based on the specified app.
@@ -66,6 +94,7 @@ def load_app(self, app):
             self.loaded_apps[app] = initial_data_class
         return self.loaded_apps[app]
 
+    @atomic
     def update_app(self, app):
         """
         Loads and runs `update_initial_data` of the specified app. Any dependencies contained within the
@@ -93,18 +122,58 @@ def update_app(self, app):
 
             self.log('Updating app {0}'.format(app))
 
-            initial_data_class().update_initial_data()
+            # Update the initial data of the app and gather any objects returned for deletion. Objects registered for
+            # deletion can either be returned from the update_initial_data function or programmatically added with the
+            # register_for_deletion function in the BaseInitialData class.
+            initial_data_instance = initial_data_class()
+            model_objs_registered_for_deletion = initial_data_instance.update_initial_data() or []
+            model_objs_registered_for_deletion.extend(initial_data_instance.get_model_objs_registered_for_deletion())
+
+            # Add the objects to be deleted from the app to the global list of objects to be deleted.
+            self.model_objs_registered_for_deletion.extend(model_objs_registered_for_deletion)
+
             # keep track that this app has been updated
             self.updated_apps.add(app)
 
+    def handle_deletions(self):
+        """
+        Manages handling deletions of objects that were previously managed by the initial data process but no longer
+        managed. It does so by mantaining a list of receipts for model objects that are registered for deletion on
+        each round of initial data processing. Any receipts that are from previous rounds and not the current
+        round will be deleted.
+        """
+        # Create receipts for every object registered for deletion
+        now = datetime.utcnow()
+        registered_for_deletion_receipts = [
+            RegisteredForDeletionReceipt(
+                model_obj_type=ContentType.objects.get_for_model(model_obj), model_obj_id=model_obj.id,
+                register_time=now)
+            for model_obj in self.model_objs_registered_for_deletion
+        ]
+
+        # Do a bulk upsert on all of the receipts, updating their registration time.
+        RegisteredForDeletionReceipt.objects.bulk_upsert(
+            registered_for_deletion_receipts, ['model_obj_type_id', 'model_obj_id'], update_fields=['register_time'])
+
+        # Delete all receipts and their associated model objects that weren't updated
+        for receipt in RegisteredForDeletionReceipt.objects.exclude(register_time=now).prefetch_related('model_obj'):
+            if receipt.model_obj:
+                receipt.model_obj.delete()
+            receipt.delete()
+
+    @atomic
     def update_all_apps(self):
         """
         Loops through all app names contained in settings.INSTALLED_APPS and calls `update_app`
-        on each one.
+        on each one. Handles any object deletions that happened after all apps have been initialized.
         """
         for app in settings.INSTALLED_APPS:
             self.update_app(app)
 
+        # During update_app, all apps added model objects that were registered for deletion.
+        # Delete all objects that were previously managed by the initial data process
+        self.handle_deletions()
+
     def get_dependency_call_list(self, app, call_list=None):
         """
         Recursively detects any dependency cycles based on the specific app. A running list of

dynamic_initial_data/migrations/0001_initial.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+import datetime
+from south.db import db
+from south.v2 import SchemaMigration
+from django.db import models
+
+
+class Migration(SchemaMigration):
+
+    def forwards(self, orm):
+        # Adding model 'RegisteredForDeletionReceipt'
+        db.create_table(u'dynamic_initial_data_registeredfordeletionreceipt', (
+            (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)),
+            ('model_obj_type', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['contenttypes.ContentType'])),
+            ('model_obj_id', self.gf('django.db.models.fields.PositiveIntegerField')()),
+            ('register_time', self.gf('django.db.models.fields.DateTimeField')()),
+        ))
+        db.send_create_signal(u'dynamic_initial_data', ['RegisteredForDeletionReceipt'])
+
+
+    def backwards(self, orm):
+        # Deleting model 'RegisteredForDeletionReceipt'
+        db.delete_table(u'dynamic_initial_data_registeredfordeletionreceipt')
+
+
+    models = {
+        u'contenttypes.contenttype': {
+            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"},
+            'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}),
+            'name': ('django.db.models.fields.CharField', [], {'max_length': '100'})
+        },
+        u'dynamic_initial_data.registeredfordeletionreceipt': {
+            'Meta': {'object_name': 'RegisteredForDeletionReceipt'},
+            u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}),
+            'model_obj_id': ('django.db.models.fields.PositiveIntegerField', [], {}),
+            'model_obj_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}),
+            'register_time': ('django.db.models.fields.DateTimeField', [], {})
+        }
+    }
+
+    complete_apps = ['dynamic_initial_data']

dynamic_initial_data/models.py

@@ -0,0 +1,21 @@
+from django.contrib.contenttypes import generic
+from django.contrib.contenttypes.models import ContentType
+from django.db import models
+from manager_utils import ManagerUtilsManager
+
+
+class RegisteredForDeletionReceipt(models.Model):
+    """
+    Specifies a receipt of a model object that was registered for deletion by the dynamic
+    initial data process.
+    """
+    # The model object that was registered
+    model_obj_type = models.ForeignKey(ContentType)
+    model_obj_id = models.PositiveIntegerField()
+    model_obj = generic.GenericForeignKey('model_obj_type', 'model_obj_id')
+
+    # The time at which it was registered for deletion
+    register_time = models.DateTimeField()
+
+    # Use manager utils for bulk updating capabilities
+    objects = ManagerUtilsManager()