From 8c20189ec337f5d5515f4aebb4cd719fcc1d619c Mon Sep 17 00:00:00 2001 From: Wes Kendall Date: Sun, 27 Apr 2014 15:42:35 -0400 Subject: [PATCH 01/10] Delete tests.py --- dynamic_initial_data/tests/tests.py | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 dynamic_initial_data/tests/tests.py diff --git a/dynamic_initial_data/tests/tests.py b/dynamic_initial_data/tests/tests.py deleted file mode 100644 index 81d0684..0000000 --- a/dynamic_initial_data/tests/tests.py +++ /dev/null @@ -1,6 +0,0 @@ -from django.test import TestCase - - -class SampleTest(TestCase): - def test_1_equals_1(self): - self.assertEquals(1, 1) From f84f738be929cfc9c5120899be362b5bcb8e0a3b Mon Sep 17 00:00:00 2001 From: Wes Kendall Date: Sun, 27 Apr 2014 22:31:14 -0400 Subject: [PATCH 02/10] Update setup.py --- setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index b061b8e..eb24131 100644 --- a/setup.py +++ b/setup.py @@ -20,12 +20,12 @@ def get_version(): setup( name='django-dynamic-initial-data', version=get_version(), - description='', + description='Dynamic initial data fixtures for Django apps', long_description=open('README.md').read(), - url='', + url='https://github.com/ambitioninc/django-dynamic-initial-data', author='', author_email='opensource@ambition.com', - keywords='', + keywords='Django fixtures', packages=find_packages(), classifiers=[ 'Programming Language :: Python', From 95ff71d803649139c83290f3094ee650aa408446 Mon Sep 17 00:00:00 2001 From: Wes Kendall Date: Sat, 10 May 2014 20:05:04 -0400 Subject: [PATCH 03/10] django dynamic initial data supports deletions --- dynamic_initial_data/__init__.py | 1 + dynamic_initial_data/base.py | 70 ++++++- .../migrations/0001_initial.py | 43 ++++ dynamic_initial_data/migrations/__init__.py | 0 dynamic_initial_data/models.py | 21 ++ dynamic_initial_data/tests/base_tests.py | 129 ++++++++++++ .../tests/integration_tests.py | 195 ++++++++++++++++++ dynamic_initial_data/tests/models.py | 8 + dynamic_initial_data/utils/import_string.py | 2 +- manage.py | 12 ++ run_tests.py | 47 +---- settings.py | 50 +++++ setup.py | 3 +- 13 files changed, 535 insertions(+), 46 deletions(-) create mode 100644 dynamic_initial_data/migrations/0001_initial.py create mode 100644 dynamic_initial_data/migrations/__init__.py create mode 100644 dynamic_initial_data/models.py create mode 100644 dynamic_initial_data/tests/integration_tests.py create mode 100644 dynamic_initial_data/tests/models.py create mode 100644 manage.py create mode 100644 settings.py diff --git a/dynamic_initial_data/__init__.py b/dynamic_initial_data/__init__.py index ed53594..2928e00 100644 --- a/dynamic_initial_data/__init__.py +++ b/dynamic_initial_data/__init__.py @@ -1,2 +1,3 @@ # flake8: noqa from .version import __version__ +from .base import BaseInitialData diff --git a/dynamic_initial_data/base.py b/dynamic_initial_data/base.py index 6f6bffd..92fa861 100644 --- a/dynamic_initial_data/base.py +++ b/dynamic_initial_data/base.py @@ -1,6 +1,10 @@ +from datetime import datetime + from django.conf import settings +from django.contrib.contenttypes.models import ContentType from dynamic_initial_data.exceptions import InitialDataCircularDependency, InitialDataMissingApp +from dynamic_initial_data.models import RegisteredForDeletionReceipt from dynamic_initial_data.utils.import_string import import_string @@ -15,6 +19,20 @@ class BaseInitialData(object): """ dependencies = [] + def __init__(self): + # Keep track of any model objects that have been registered for deletion + self.model_objs_registered_for_deletion = [] + + def get_model_objs_registered_for_deletion(self): + return self.model_objs_registered_for_deletion + + def register_for_deletion(self, *model_objs): + """ + Registers model objects for deletion. This means the model object will be deleted from the system when it is + no longer being managed by the initial data process. + """ + self.model_objs_registered_for_deletion.extend(model_objs) + def update_initial_data(self, *args, **kwargs): """ Raises an error if the subclass does not implement this @@ -29,11 +47,20 @@ class InitialDataUpdater(object): and running of initialization classes. """ def __init__(self, options=None): + # Various options that can be passed to the initial data updater options = options or {} self.verbose = options.get('verbose', False) + + # Apps that have been updated so far. This allows us to process dependencies on other app + # inits easier without performing redundant work self.updated_apps = set() + + # A cache of the apps that have been imported for data initialization self.loaded_apps = {} + # A list of all models that have been registered for deletion + self.model_objs_registered_for_deletion = [] + def get_class_path(self, app): """ Builds the full path to the initial data class based on the specified app. @@ -93,18 +120,57 @@ def update_app(self, app): self.log('Updating app {0}'.format(app)) - initial_data_class().update_initial_data() + # Update the initial data of the app and gather any objects returned for deletion. Objects registered for + # deletion can either be returned from the update_initial_data function or programmatically added with the + # regsiter_for_deletion function in the BaseInitialData class. + initial_data_instance = initial_data_class() + model_objs_registered_for_deletion = initial_data_instance.update_initial_data() or [] + model_objs_registered_for_deletion.extend(initial_data_instance.get_model_objs_registered_for_deletion()) + + # Add the objects to be deleted from the app to the global list of objects to be deleted. + self.model_objs_registered_for_deletion.extend(model_objs_registered_for_deletion) + # keep track that this app has been updated self.updated_apps.add(app) + def handle_deletions(self): + """ + Manages handling deletions of objects that were previously managed by the initial data process but no longer + managed. It does so by mantaining a list of receipts for model objects that are registered for deletion on + each round of initial data processing. Any receipts that are from previous rounds and not the current + round will be deleted. + """ + # Create receipts for every object regsitered for deletion + now = datetime.utcnow() + registered_for_deletion_receipts = [ + RegisteredForDeletionReceipt( + model_obj_type=ContentType.objects.get_for_model(model_obj), model_obj_id=model_obj.id, + register_time=now) + for model_obj in self.model_objs_registered_for_deletion + ] + + # Do a bulk upsert on all of the receipts, updating their registration time. + RegisteredForDeletionReceipt.objects.bulk_upsert( + registered_for_deletion_receipts, ['model_obj_type_id', 'model_obj_id'], update_fields=['register_time']) + + # Delete all receipts and their associated model objects that weren't updated + for receipt in RegisteredForDeletionReceipt.objects.exclude(register_time=now).prefetch_related('model_obj'): + if receipt.model_obj: + receipt.model_obj.delete() + receipt.delete() + def update_all_apps(self): """ Loops through all app names contained in settings.INSTALLED_APPS and calls `update_app` - on each one. + on each one. Handles any object deletions that happened after all apps have been initialized. """ for app in settings.INSTALLED_APPS: self.update_app(app) + # During update_app, all apps added model objects that were registered for deletion. + # Delete all objects that were previously managed by the initial data process + self.handle_deletions() + def get_dependency_call_list(self, app, call_list=None): """ Recursively detects any dependency cycles based on the specific app. A running list of diff --git a/dynamic_initial_data/migrations/0001_initial.py b/dynamic_initial_data/migrations/0001_initial.py new file mode 100644 index 0000000..52ec532 --- /dev/null +++ b/dynamic_initial_data/migrations/0001_initial.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +import datetime +from south.db import db +from south.v2 import SchemaMigration +from django.db import models + + +class Migration(SchemaMigration): + + def forwards(self, orm): + # Adding model 'RegisteredForDeletionReceipt' + db.create_table(u'dynamic_initial_data_registeredfordeletionreceipt', ( + (u'id', self.gf('django.db.models.fields.AutoField')(primary_key=True)), + ('model_obj_type', self.gf('django.db.models.fields.related.ForeignKey')(to=orm['contenttypes.ContentType'])), + ('model_obj_id', self.gf('django.db.models.fields.PositiveIntegerField')()), + ('register_time', self.gf('django.db.models.fields.DateTimeField')()), + )) + db.send_create_signal(u'dynamic_initial_data', ['RegisteredForDeletionReceipt']) + + + def backwards(self, orm): + # Deleting model 'RegisteredForDeletionReceipt' + db.delete_table(u'dynamic_initial_data_registeredfordeletionreceipt') + + + models = { + u'contenttypes.contenttype': { + 'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label', 'model'),)", 'object_name': 'ContentType', 'db_table': "'django_content_type'"}, + 'app_label': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model': ('django.db.models.fields.CharField', [], {'max_length': '100'}), + 'name': ('django.db.models.fields.CharField', [], {'max_length': '100'}) + }, + u'dynamic_initial_data.registeredfordeletionreceipt': { + 'Meta': {'object_name': 'RegisteredForDeletionReceipt'}, + u'id': ('django.db.models.fields.AutoField', [], {'primary_key': 'True'}), + 'model_obj_id': ('django.db.models.fields.PositiveIntegerField', [], {}), + 'model_obj_type': ('django.db.models.fields.related.ForeignKey', [], {'to': u"orm['contenttypes.ContentType']"}), + 'register_time': ('django.db.models.fields.DateTimeField', [], {}) + } + } + + complete_apps = ['dynamic_initial_data'] \ No newline at end of file diff --git a/dynamic_initial_data/migrations/__init__.py b/dynamic_initial_data/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dynamic_initial_data/models.py b/dynamic_initial_data/models.py new file mode 100644 index 0000000..13f9056 --- /dev/null +++ b/dynamic_initial_data/models.py @@ -0,0 +1,21 @@ +from django.contrib.contenttypes import generic +from django.contrib.contenttypes.models import ContentType +from django.db import models +from manager_utils import ManagerUtilsManager + + +class RegisteredForDeletionReceipt(models.Model): + """ + Specifies a receipt of a model object that was registered for deletion by the dynamic + initial data process. + """ + # The model object that was registered + model_obj_type = models.ForeignKey(ContentType) + model_obj_id = models.PositiveIntegerField() + model_obj = generic.GenericForeignKey('model_obj_type', 'model_obj_id') + + # The time at which it was registered for deletion + register_time = models.DateTimeField() + + # Use manager utils for bulk updating capabilities + objects = ManagerUtilsManager() diff --git a/dynamic_initial_data/tests/base_tests.py b/dynamic_initial_data/tests/base_tests.py index a18f017..ffd2da1 100644 --- a/dynamic_initial_data/tests/base_tests.py +++ b/dynamic_initial_data/tests/base_tests.py @@ -1,10 +1,17 @@ +from datetime import datetime + from django.conf import settings +from django.contrib.contenttypes.models import ContentType from django.test import TestCase +from django_dynamic_fixture import G +from freezegun import freeze_time from mock import patch from dynamic_initial_data.base import BaseInitialData, InitialDataUpdater from dynamic_initial_data.exceptions import InitialDataMissingApp, InitialDataCircularDependency +from dynamic_initial_data.models import RegisteredForDeletionReceipt from dynamic_initial_data.tests.mocks import MockInitialData, MockClass, MockOne, MockTwo, MockThree +from dynamic_initial_data.tests.models import Account class BaseInitialDataTest(TestCase): @@ -16,6 +23,128 @@ def test_base_initial_data(self): with self.assertRaises(NotImplementedError): initial_data.update_initial_data() + def test_register_for_deletion_one_arg(self): + """ + Tests the register_for_deletion_function with one argument. + """ + initial_data = BaseInitialData() + account = G(Account) + initial_data.register_for_deletion(account) + self.assertEquals(initial_data.get_model_objs_registered_for_deletion(), [account]) + + def test_register_for_deletion_multiple_args(self): + """ + Tests the register_for_deletion_function with multiple arguments. + """ + initial_data = BaseInitialData() + account1 = G(Account) + account2 = G(Account) + initial_data.register_for_deletion(account1, account2) + self.assertEquals(initial_data.get_model_objs_registered_for_deletion(), [account1, account2]) + + +class TestHandleDeletions(TestCase): + """ + Tests the handle_deletions functionality in the InitialDataUpater class. + """ + def setUp(self): + super(TestHandleDeletions, self).setUp() + self.initial_data_updater = InitialDataUpdater() + + def test_handle_deletions_no_objs(self): + """ + Tests when there are no objs to handle. The function should not raise any exceptions. + """ + self.initial_data_updater.handle_deletions() + + def test_create_one_obj(self): + """ + Tests creating one object to handle for deletion. + """ + account = G(Account) + self.initial_data_updater.model_objs_registered_for_deletion = [account] + + self.assertEquals(RegisteredForDeletionReceipt.objects.count(), 0) + with freeze_time('2013-04-12'): + self.initial_data_updater.handle_deletions() + receipt = RegisteredForDeletionReceipt.objects.get() + self.assertEquals(receipt.model_obj_type, ContentType.objects.get_for_model(Account)) + self.assertEquals(receipt.model_obj_id, account.id) + self.assertEquals(receipt.register_time, datetime(2013, 4, 12)) + + def test_create_delete_one_obj(self): + """ + Tests creating one object to handle for deletion and then deleting it. + """ + account = G(Account) + self.initial_data_updater.model_objs_registered_for_deletion = [account] + + self.assertEquals(RegisteredForDeletionReceipt.objects.count(), 0) + with freeze_time('2013-04-12'): + self.initial_data_updater.handle_deletions() + receipt = RegisteredForDeletionReceipt.objects.get() + self.assertEquals(receipt.model_obj_type, ContentType.objects.get_for_model(Account)) + self.assertEquals(receipt.model_obj_id, account.id) + self.assertEquals(receipt.register_time, datetime(2013, 4, 12)) + + # Now, don't register the object for deletion and run it again at a different time + self.initial_data_updater.model_objs_registered_for_deletion = [] + with freeze_time('2013-04-12 05:00:00'): + self.initial_data_updater.handle_deletions() + # The object should be deleted, along with its receipt + self.assertEquals(Account.objects.count(), 0) + self.assertEquals(RegisteredForDeletionReceipt.objects.count(), 0) + + def test_create_update_one_obj(self): + """ + Tests creating one object to handle for deletion and then updating it. + """ + account = G(Account) + self.initial_data_updater.model_objs_registered_for_deletion = [account] + + self.assertEquals(RegisteredForDeletionReceipt.objects.count(), 0) + with freeze_time('2013-04-12'): + self.initial_data_updater.handle_deletions() + receipt = RegisteredForDeletionReceipt.objects.get() + self.assertEquals(receipt.model_obj_type, ContentType.objects.get_for_model(Account)) + self.assertEquals(receipt.model_obj_id, account.id) + self.assertEquals(receipt.register_time, datetime(2013, 4, 12)) + + # Run the deletion handler again at a different time. It should not delete the object + with freeze_time('2013-04-12 05:00:00'): + self.initial_data_updater.handle_deletions() + # The object should not be deleted, along with its receipt + self.assertEquals(Account.objects.count(), 1) + self.assertEquals(RegisteredForDeletionReceipt.objects.count(), 1) + self.assertEquals(RegisteredForDeletionReceipt.objects.get().register_time, datetime(2013, 4, 12, 5)) + + def test_delete_already_deleted_obj(self): + """ + Tests the case when an object that was registered for deletion has already been deleted. + """ + account = G(Account) + self.initial_data_updater.model_objs_registered_for_deletion = [account] + + self.assertEquals(RegisteredForDeletionReceipt.objects.count(), 0) + with freeze_time('2013-04-12'): + self.initial_data_updater.handle_deletions() + receipt = RegisteredForDeletionReceipt.objects.get() + self.assertEquals(receipt.model_obj_type, ContentType.objects.get_for_model(Account)) + self.assertEquals(receipt.model_obj_id, account.id) + self.assertEquals(receipt.register_time, datetime(2013, 4, 12)) + + # Delete the model object. The receipt should still exist + account.delete() + self.assertEquals(RegisteredForDeletionReceipt.objects.count(), 1) + + # Now, don't register the object for deletion and run it again at a different time + self.initial_data_updater.model_objs_registered_for_deletion = [] + with freeze_time('2013-04-12 05:00:00'): + self.initial_data_updater.handle_deletions() + # The object should be deleted, along with its receipt + self.assertEquals(Account.objects.count(), 0) + self.assertEquals(RegisteredForDeletionReceipt.objects.count(), 0) + class InitialDataUpdaterTest(TestCase): """ diff --git a/dynamic_initial_data/tests/integration_tests.py b/dynamic_initial_data/tests/integration_tests.py new file mode 100644 index 0000000..645f911 --- /dev/null +++ b/dynamic_initial_data/tests/integration_tests.py @@ -0,0 +1,195 @@ +from django.test import TestCase +from django.test.utils import override_settings +from mock import patch + +from dynamic_initial_data import BaseInitialData +from dynamic_initial_data.base import InitialDataUpdater +from dynamic_initial_data.models import RegisteredForDeletionReceipt +from dynamic_initial_data.tests.models import Account + + +class IntegrationTest(TestCase): + """ + Tests the full initial data process. + """ + def test_create_account(self): + """ + Tests creating a test account in the initial data process. + """ + class AccountInitialData(BaseInitialData): + def update_initial_data(self): + Account.objects.get_or_create() + + # Verify no account objects exist + self.assertEquals(Account.objects.count(), 0) + + with patch.object(InitialDataUpdater, 'load_app', return_value=AccountInitialData) as load_app_mock: + InitialDataUpdater().update_app('test_app') + # It should be called twice - once for initial loading, and twice for dependency testing + self.assertEquals(load_app_mock.call_count, 2) + + # Verify an account object was created + self.assertEquals(Account.objects.count(), 1) + + @override_settings(INSTALLED_APPS=('one_installed_test_app',)) + def test_handle_deletions_returned_from_update_initial_data(self): + """ + Tests handling of deletions when they are returned from the update_initial_data function. + """ + class AccountInitialData1(BaseInitialData): + """ + The initial data code the first time it is called. It registers an account for deletion + by returning it from the update_initial_data function. + """ + def update_initial_data(self): + # Return the object from update_initial_data, thus registering it for deletion + return [Account.objects.get_or_create()[0]] + + class AccountInitialData2(BaseInitialData): + """ + The initial data code the second time it is called. It no longer creates the account object, so the + previously created account object should be deleted. + """ + def update_initial_data(self): + pass + + # Verify no account objects exist + self.assertEquals(Account.objects.count(), 0) + + with patch.object(InitialDataUpdater, 'load_app', return_value=AccountInitialData1): + InitialDataUpdater().update_all_apps() + + # Verify an account object was created and is managed by a deletion receipt + self.assertEquals(Account.objects.count(), 1) + self.assertEquals(RegisteredForDeletionReceipt.objects.count(), 1) + + # Run the initial data process again, this time not registering the account for + # deletion. It should be deleted. + with patch.object(InitialDataUpdater, 'load_app', return_value=AccountInitialData2): + InitialDataUpdater().update_all_apps() + + # Verify there are no accounts or receipts + self.assertEquals(Account.objects.count(), 0) + self.assertEquals(RegisteredForDeletionReceipt.objects.count(), 0) + + @override_settings(INSTALLED_APPS=('one_installed_test_app',)) + def test_handle_deletions_updates_returned_from_update_initial_data(self): + """ + Tests handling of deletions and updates when they are returned from the update_initial_data function. + """ + class AccountInitialData1(BaseInitialData): + """ + The initial data code the first time it is called. It registers two accounts for deletion + by returning it from the update_initial_data function. + """ + def update_initial_data(self): + # Return the object from update_initial_data, thus registering it for deletion + return [Account.objects.get_or_create(name='hi')[0], Account.objects.get_or_create(name='hi2')[0]] + + class AccountInitialData2(BaseInitialData): + """ + The initial data code the second time it is called. It only manages one of the previous accounts + """ + def update_initial_data(self): + return [Account.objects.get_or_create(name='hi')[0]] + + # Verify no account objects exist + self.assertEquals(Account.objects.count(), 0) + + with patch.object(InitialDataUpdater, 'load_app', return_value=AccountInitialData1): + InitialDataUpdater().update_all_apps() + + # Verify two account objects were created and are managed by deletion receipts + self.assertEquals(Account.objects.count(), 2) + self.assertEquals(RegisteredForDeletionReceipt.objects.count(), 2) + + # Run the initial data process again, this time deleting the account named 'hi2' + with patch.object(InitialDataUpdater, 'load_app', return_value=AccountInitialData2): + InitialDataUpdater().update_all_apps() + + # Verify only the 'hi' account exists + self.assertEquals(Account.objects.count(), 1) + self.assertEquals(RegisteredForDeletionReceipt.objects.count(), 1) + self.assertEquals(RegisteredForDeletionReceipt.objects.get().model_obj.name, 'hi') + + @override_settings(INSTALLED_APPS=('one_installed_test_app',)) + def test_handle_deletions_registered_from_update_initial_data(self): + """ + Tests handling of deletions when they are programmatically registered from the update_initial_data function. + """ + class AccountInitialData1(BaseInitialData): + """ + The initial data code the first time it is called. It registers an account for deletion + by returning it from the update_initial_data function. + """ + def update_initial_data(self): + # Register the object for deletion + self.register_for_deletion(Account.objects.get_or_create()[0]) + + class AccountInitialData2(BaseInitialData): + """ + The initial data code the second time it is called. It no longer creates the account object, so the + previously created account object should be deleted. + """ + def update_initial_data(self): + pass + + # Verify no account objects exist + self.assertEquals(Account.objects.count(), 0) + + with patch.object(InitialDataUpdater, 'load_app', return_value=AccountInitialData1): + InitialDataUpdater().update_all_apps() + + # Verify an account object was created and is managed by a deletion receipt + self.assertEquals(Account.objects.count(), 1) + self.assertEquals(RegisteredForDeletionReceipt.objects.count(), 1) + + # Run the initial data process again, this time not registering the account for + # deletion. It should be deleted. + with patch.object(InitialDataUpdater, 'load_app', return_value=AccountInitialData2): + InitialDataUpdater().update_all_apps() + + # Verify there are no accounts or receipts + self.assertEquals(Account.objects.count(), 0) + self.assertEquals(RegisteredForDeletionReceipt.objects.count(), 0) + + @override_settings(INSTALLED_APPS=('one_installed_test_app',)) + def test_handle_deletions_updates_registered_from_update_initial_data(self): + """ + Tests handling of deletions and updates when they are registered from the update_initial_data function. + """ + class AccountInitialData1(BaseInitialData): + """ + The initial data code the first time it is called. It registers two accounts for deletion + by returning it from the update_initial_data function. + """ + def update_initial_data(self): + # Register two account objects for deletion + self.register_for_deletion( + Account.objects.get_or_create(name='hi')[0], Account.objects.get_or_create(name='hi2')[0]) + + class AccountInitialData2(BaseInitialData): + """ + The initial data code the second time it is called. It only manages one of the previous accounts + """ + def update_initial_data(self): + self.register_for_deletion(Account.objects.get_or_create(name='hi')[0]) + + # Verify no account objects exist + self.assertEquals(Account.objects.count(), 0) + + with patch.object(InitialDataUpdater, 'load_app', return_value=AccountInitialData1): + InitialDataUpdater().update_all_apps() + + # Verify two account objects were created and are managed by deletion receipts + self.assertEquals(Account.objects.count(), 2) + self.assertEquals(RegisteredForDeletionReceipt.objects.count(), 2) + + # Run the initial data process again, this time deleting the account named 'hi2' + with patch.object(InitialDataUpdater, 'load_app', return_value=AccountInitialData2): + InitialDataUpdater().update_all_apps() + + # Verify only the 'hi' account exists + self.assertEquals(Account.objects.count(), 1) + self.assertEquals(RegisteredForDeletionReceipt.objects.count(), 1) + self.assertEquals(RegisteredForDeletionReceipt.objects.get().model_obj.name, 'hi') diff --git a/dynamic_initial_data/tests/models.py b/dynamic_initial_data/tests/models.py new file mode 100644 index 0000000..19ef6c4 --- /dev/null +++ b/dynamic_initial_data/tests/models.py @@ -0,0 +1,8 @@ +from django.db import models + + +class Account(models.Model): + """ + A test account model. + """ + name = models.CharField(max_length=64) diff --git a/dynamic_initial_data/utils/import_string.py b/dynamic_initial_data/utils/import_string.py index 76e8243..da73493 100644 --- a/dynamic_initial_data/utils/import_string.py +++ b/dynamic_initial_data/utils/import_string.py @@ -11,7 +11,7 @@ def import_string(module_string): file_name = parts[-2] try: - module_path = __import__(path, globals(), locals(), [file_name]) + module_path = __import__(path, globals(), locals(), [file_name]) except ImportError: return None diff --git a/manage.py b/manage.py new file mode 100644 index 0000000..e374933 --- /dev/null +++ b/manage.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python +import sys + +from settings import configure_settings + + +if __name__ == '__main__': + configure_settings() + + from django.core.management import execute_from_command_line + + execute_from_command_line(sys.argv) diff --git a/run_tests.py b/run_tests.py index 03eecbf..9a5164b 100644 --- a/run_tests.py +++ b/run_tests.py @@ -1,53 +1,16 @@ """ Provides the ability to run test on a standalone Django app. """ -import os import sys -from django.conf import settings from optparse import OptionParser +from django.conf import settings + +from settings import configure_settings -if not settings.configured: - # Determine the database settings depending on if a test_db var is set in CI mode or not - test_db = os.environ.get('DB', None) - if test_db is None: - db_config = { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': 'ambition_dev', - 'USER': 'ambition_dev', - 'PASSWORD': 'ambition_dev', - 'HOST': 'localhost' - } - elif test_db == 'postgres': - db_config = { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'USER': 'postgres', - 'NAME': 'dynamic_initial_data', - } - elif test_db == 'sqlite': - db_config = { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': 'dynamic_initial_data', - } - else: - raise RuntimeError('Unsupported test DB {0}'.format(test_db)) - settings.configure( - DATABASES={ - 'default': db_config, - }, - INSTALLED_APPS=( - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.admin', - 'south', - 'dynamic_initial_data', - 'dynamic_initial_data.tests', - ), - ROOT_URLCONF='dynamic_initial_data.urls', - DEBUG=False, - ) +# Configure the default settings +configure_settings() # Django nose must be imported here since it depends on the settings being configured from django_nose import NoseTestSuiteRunner diff --git a/settings.py b/settings.py new file mode 100644 index 0000000..ca045df --- /dev/null +++ b/settings.py @@ -0,0 +1,50 @@ +import os + +from django.conf import settings +import djcelery + + +def configure_settings(): + """ + Configures settings for manage.py and for run_tests.py. + """ + # Set up celery + djcelery.setup_loader() + + if not settings.configured: + # Determine the database settings depending on if a test_db var is set in CI mode or Nonet + circle_ci = os.environ.get('CIRCLECI', None) + if circle_ci is None: + db_config = { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': 'ambition_dev', + 'USER': 'ambition_dev', + 'PASSWORD': 'ambition_dev', + 'HOST': 'localhost' + } + else: + db_config = { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': 'circle_test', + 'USER': 'ubuntu', + 'PASSWORD': '' + } + + settings.configure( + DATABASES={ + 'default': db_config, + }, + INSTALLED_APPS=( + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.admin', + 'south', + 'dynamic_initial_data', + 'dynamic_initial_data.tests', + ), + ROOT_URLCONF='dynamic_initial_data.urls', + DEBUG=False, + DDF_FILL_NULLABLE_FIELDS=False, + TEST_RUNNER='django_nose.NoseTestSuiteRunner', + ) diff --git a/setup.py b/setup.py index b061b8e..6f6e8b4 100644 --- a/setup.py +++ b/setup.py @@ -37,11 +37,12 @@ def get_version(): license='MIT', install_requires=[ 'django>=1.6', - 'django-manager-utils>=0.3.6', + 'django-manager-utils>=0.3.9', ], tests_require=[ 'psycopg2', 'django-nose', + 'freezegun', 'south', 'mock', ], From a9bb9c1cca1d575afdeee20b4f2be924563072c8 Mon Sep 17 00:00:00 2001 From: Wes Kendall Date: Sat, 10 May 2014 20:17:09 -0400 Subject: [PATCH 04/10] readme --- README.md | 4 ++-- dynamic_initial_data/version.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9112aca..ae2c600 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ a planned feature. To install Django Dynamic Initial Data: ```shell -pip install git+https://github.com/ambitioninc/django-dynamic-initial-data.git +pip install django-dynamic-initial-data ``` Add Django Dynamic Initial Data to your `INSTALLED_APPS` to get started: @@ -25,7 +25,7 @@ Add Django Dynamic Initial Data to your `INSTALLED_APPS` to get started: settings.py ```python INSTALLED_APPS = ( - 'django-dynamic-initial-data', + 'dynamic_initial_data', ) ``` diff --git a/dynamic_initial_data/version.py b/dynamic_initial_data/version.py index df9144c..7fd229a 100644 --- a/dynamic_initial_data/version.py +++ b/dynamic_initial_data/version.py @@ -1 +1 @@ -__version__ = '0.1.1' +__version__ = '0.2.0' From 1a6e66b2a87048ec5171eedd43582de0a8baab35 Mon Sep 17 00:00:00 2001 From: Wes Kendall Date: Sat, 10 May 2014 20:18:21 -0400 Subject: [PATCH 05/10] readme --- README.md | 52 +++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index ae2c600..1d98d58 100644 --- a/README.md +++ b/README.md @@ -3,15 +3,15 @@ Django Dynamic Initial Data is a `django>=1.6` app that helps solve the problem of initializing data for apps with dependencies and other conditional data. Rather than having static fixtures for each app, the initial data -can be created and updated dynamically. This can be done more easily by making use of the `django-manager-utils` -upsert functionality. Currently the data initialization process does not handle deleted records, but this is -a planned feature. +can be created and updated dynamically. Furthermore, Django Dynamic Initial Data also handles when objects are +deleted from initial data, a feature that Django's initial data fixture system lacks. ## Table of Contents 1. [Installation] (#installation) -1. [A Brief Overview] (#a-brief-overview) -1. [Example] (#example) +2. [A Brief Overview] (#a-brief-overview) +3. [Example] (#example) +4. [Handling Deletions](#handling-deletions) ## Installation To install Django Dynamic Initial Data: @@ -80,3 +80,45 @@ python manage.py update_initial_data --app 'app_path' Documentation on using `upsert` and `bulk_upsert` can be found below: - https://github.com/ambitioninc/django-manager-utils#upsert - https://github.com/ambitioninc/django-manager-utils#bulk_upsert + +## Handling Deletions +One difficulty when specifying initial data in Django apps is the inability to deploy initial data to your project and then subsequently +remove any initial data fixtures. If one removes an object in an initial_data.json file, Django does not handle its deletion next time +it is deployed, which can cause headaches with lingering objects. + +Django Dynamic Initial Data fixes this problem by allowing the user to either: + +1. Return all managed initial data objects as an array from the update_initial_data function. +2. Explicitly register objects for deletion with the register_for_deletion(*model_objs) method. + +Note that it is up to the user to be responsible for always registering every object every time, regardless if the object was updated or +created by the initial data process. Doing this allows Django Dynamic Initial Data to remove any objects that were previosly managed. +For example, assume you have an InitialData class that manages two users with the user names "hello" and "world". + +```python +from dynamic_initial_data.base import BaseInitialData + +class InitialData(BaseInitialData): + def update_initial_data(self): + hello = Account.objects.get_or_create(name='hello') + world = Account.objects.get_or_create(name='world') + # register the accounts for deletion + self.register_for_deletion(hello, world) +``` + +After this code is created, the initial data process now owns the "hello" and "world" account objects. If these objects are not registered +for deletion in subsequent versions of the code, they will be deleted when the initial data process executes. For example, assume the first +piece of code executed and then the user executed this piece of code: + +```python +from dynamic_initial_data.base import BaseInitialData + +class InitialData(BaseInitialData): + def update_initial_data(self): + world = Account.objects.get_or_create(name='world') + # register the accounts for deletion + self.register_for_deletion(world) +``` + +When this piece of code executes, the previous "hello" account would then be deleted since the initial data process no longer owns it. And +don't worry, if it was already deleted by another process, the deletion will not throw an error. \ No newline at end of file From 3b2bb6f230672a61d775f262895356d688ba0994 Mon Sep 17 00:00:00 2001 From: Wes Kendall Date: Sat, 10 May 2014 20:21:32 -0400 Subject: [PATCH 06/10] fixed settings --- README.md | 15 ++++----------- settings.py | 4 ---- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 1d98d58..a863184 100644 --- a/README.md +++ b/README.md @@ -82,18 +82,14 @@ Documentation on using `upsert` and `bulk_upsert` can be found below: - https://github.com/ambitioninc/django-manager-utils#bulk_upsert ## Handling Deletions -One difficulty when specifying initial data in Django apps is the inability to deploy initial data to your project and then subsequently -remove any initial data fixtures. If one removes an object in an initial_data.json file, Django does not handle its deletion next time -it is deployed, which can cause headaches with lingering objects. +One difficulty when specifying initial data in Django apps is the inability to deploy initial data to your project and then subsequently remove any initial data fixtures. If one removes an object in an initial_data.json file, Django does not handle its deletion next time it is deployed, which can cause headaches with lingering objects. Django Dynamic Initial Data fixes this problem by allowing the user to either: 1. Return all managed initial data objects as an array from the update_initial_data function. 2. Explicitly register objects for deletion with the register_for_deletion(*model_objs) method. -Note that it is up to the user to be responsible for always registering every object every time, regardless if the object was updated or -created by the initial data process. Doing this allows Django Dynamic Initial Data to remove any objects that were previosly managed. -For example, assume you have an InitialData class that manages two users with the user names "hello" and "world". +Note that it is up to the user to be responsible for always registering every object every time, regardless if the object was updated or created by the initial data process. Doing this allows Django Dynamic Initial Data to remove any objects that were previosly managed. For example, assume you have an InitialData class that manages two users with the user names "hello" and "world". ```python from dynamic_initial_data.base import BaseInitialData @@ -106,9 +102,7 @@ class InitialData(BaseInitialData): self.register_for_deletion(hello, world) ``` -After this code is created, the initial data process now owns the "hello" and "world" account objects. If these objects are not registered -for deletion in subsequent versions of the code, they will be deleted when the initial data process executes. For example, assume the first -piece of code executed and then the user executed this piece of code: +After this code is created, the initial data process now owns the "hello" and "world" account objects. If these objects are not registered for deletion in subsequent versions of the code, they will be deleted when the initial data process executes. For example, assume the first piece of code executed and then the user executed this piece of code: ```python from dynamic_initial_data.base import BaseInitialData @@ -120,5 +114,4 @@ class InitialData(BaseInitialData): self.register_for_deletion(world) ``` -When this piece of code executes, the previous "hello" account would then be deleted since the initial data process no longer owns it. And -don't worry, if it was already deleted by another process, the deletion will not throw an error. \ No newline at end of file +When this piece of code executes, the previous "hello" account would then be deleted since the initial data process no longer owns it. And don't worry, if it was already deleted by another process, the deletion will not throw an error. \ No newline at end of file diff --git a/settings.py b/settings.py index ca045df..d395afb 100644 --- a/settings.py +++ b/settings.py @@ -1,16 +1,12 @@ import os from django.conf import settings -import djcelery def configure_settings(): """ Configures settings for manage.py and for run_tests.py. """ - # Set up celery - djcelery.setup_loader() - if not settings.configured: # Determine the database settings depending on if a test_db var is set in CI mode or Nonet circle_ci = os.environ.get('CIRCLECI', None) From 9a97f92369e4825199aa7f04dfafef9c07d34770 Mon Sep 17 00:00:00 2001 From: Wes Kendall Date: Sat, 10 May 2014 20:34:38 -0400 Subject: [PATCH 07/10] updated travis.yml --- dynamic_initial_data/base.py | 4 ++-- settings.py | 15 ++++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/dynamic_initial_data/base.py b/dynamic_initial_data/base.py index 92fa861..32b1767 100644 --- a/dynamic_initial_data/base.py +++ b/dynamic_initial_data/base.py @@ -122,7 +122,7 @@ def update_app(self, app): # Update the initial data of the app and gather any objects returned for deletion. Objects registered for # deletion can either be returned from the update_initial_data function or programmatically added with the - # regsiter_for_deletion function in the BaseInitialData class. + # register_for_deletion function in the BaseInitialData class. initial_data_instance = initial_data_class() model_objs_registered_for_deletion = initial_data_instance.update_initial_data() or [] model_objs_registered_for_deletion.extend(initial_data_instance.get_model_objs_registered_for_deletion()) @@ -140,7 +140,7 @@ def handle_deletions(self): each round of initial data processing. Any receipts that are from previous rounds and not the current round will be deleted. """ - # Create receipts for every object regsitered for deletion + # Create receipts for every object registered for deletion now = datetime.utcnow() registered_for_deletion_receipts = [ RegisteredForDeletionReceipt( diff --git a/settings.py b/settings.py index d395afb..a3e5fc9 100644 --- a/settings.py +++ b/settings.py @@ -8,9 +8,9 @@ def configure_settings(): Configures settings for manage.py and for run_tests.py. """ if not settings.configured: - # Determine the database settings depending on if a test_db var is set in CI mode or Nonet - circle_ci = os.environ.get('CIRCLECI', None) - if circle_ci is None: + # Determine the database settings depending on if a test_db var is set in CI mode or not + test_db = os.environ.get('DB', None) + if test_db is None: db_config = { 'ENGINE': 'django.db.backends.postgresql_psycopg2', 'NAME': 'ambition_dev', @@ -18,13 +18,14 @@ def configure_settings(): 'PASSWORD': 'ambition_dev', 'HOST': 'localhost' } - else: + elif test_db == 'postgres': db_config = { 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': 'circle_test', - 'USER': 'ubuntu', - 'PASSWORD': '' + 'USER': 'postgres', + 'NAME': 'dynamic_initial_data', } + else: + raise RuntimeError('Unsupported test DB {0}'.format(test_db)) settings.configure( DATABASES={ From 7ad794a6b30e05021d71038d2298faacbabd42a7 Mon Sep 17 00:00:00 2001 From: Wes Kendall Date: Sat, 10 May 2014 20:44:24 -0400 Subject: [PATCH 08/10] updated testing requirements --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 6f6e8b4..1191b82 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,7 @@ def get_version(): ], tests_require=[ 'psycopg2', + 'django-dynamic-fixture', 'django-nose', 'freezegun', 'south', From 18870d39074b9ec1efedd4898e14f90432199c95 Mon Sep 17 00:00:00 2001 From: Wes Kendall Date: Mon, 12 May 2014 10:09:22 -0400 Subject: [PATCH 09/10] added atomic transactions to app updates --- dynamic_initial_data/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dynamic_initial_data/base.py b/dynamic_initial_data/base.py index 32b1767..3170e5a 100644 --- a/dynamic_initial_data/base.py +++ b/dynamic_initial_data/base.py @@ -93,6 +93,7 @@ def load_app(self, app): self.loaded_apps[app] = initial_data_class return self.loaded_apps[app] + @atomic def update_app(self, app): """ Loads and runs `update_initial_data` of the specified app. Any dependencies contained within the @@ -159,6 +160,7 @@ def handle_deletions(self): receipt.model_obj.delete() receipt.delete() + @atomic def update_all_apps(self): """ Loops through all app names contained in settings.INSTALLED_APPS and calls `update_app` From 3e32db4af33e622e48a03865f1dec086638edb03 Mon Sep 17 00:00:00 2001 From: Wes Kendall Date: Mon, 12 May 2014 10:20:42 -0400 Subject: [PATCH 10/10] fixed error --- dynamic_initial_data/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dynamic_initial_data/base.py b/dynamic_initial_data/base.py index 3170e5a..bce1494 100644 --- a/dynamic_initial_data/base.py +++ b/dynamic_initial_data/base.py @@ -2,6 +2,7 @@ from django.conf import settings from django.contrib.contenttypes.models import ContentType +from django.db.transaction import atomic from dynamic_initial_data.exceptions import InitialDataCircularDependency, InitialDataMissingApp from dynamic_initial_data.models import RegisteredForDeletionReceipt