From 56f7cc8a768568e9baaa910558150abb6c138b7d Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Thu, 11 Apr 2013 17:06:46 -0700 Subject: [PATCH 01/53] Refactor and rely on Distribute. - Use the second namespacing technique from PEP382 - `find_packages()` replaces what we were doing manually - exclude `test` directories - add `zip_safe` --- armstrong/__init__.py | 4 +-- armstrong/dev/__init__.py | 4 +-- armstrong/dev/tasks/__init__.py | 10 +++--- armstrong/dev/tests/__init__.py | 4 +-- armstrong/dev/tests/utils/__init__.py | 7 ++-- armstrong/dev/virtualdjango/__init__.py | 4 +-- setup.py | 46 ++++--------------------- 7 files changed, 24 insertions(+), 55 deletions(-) diff --git a/armstrong/__init__.py b/armstrong/__init__.py index 3ad9513..ece379c 100644 --- a/armstrong/__init__.py +++ b/armstrong/__init__.py @@ -1,2 +1,2 @@ -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) +import pkg_resources +pkg_resources.declare_namespace(__name__) diff --git a/armstrong/dev/__init__.py b/armstrong/dev/__init__.py index 3ad9513..ece379c 100644 --- a/armstrong/dev/__init__.py +++ b/armstrong/dev/__init__.py @@ -1,2 +1,2 @@ -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) +import pkg_resources +pkg_resources.declare_namespace(__name__) diff --git a/armstrong/dev/tasks/__init__.py b/armstrong/dev/tasks/__init__.py index d955ae4..b5b607d 100644 --- a/armstrong/dev/tasks/__init__.py +++ b/armstrong/dev/tasks/__init__.py @@ -1,6 +1,5 @@ - -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) +import pkg_resources +pkg_resources.declare_namespace(__name__) from contextlib import contextmanager @@ -9,7 +8,7 @@ except ImportError: coverage = False import os -from os.path import basename, dirname +from os.path import dirname import sys from functools import wraps import unittest @@ -36,6 +35,7 @@ __all__ = ["clean", "command", "create_migration", "docs", "pep8", "test", "reinstall", "runserver", "shell", "spec", "syncdb", ] + def pip_install(func): @wraps(func) def inner(*args, **kwargs): @@ -47,6 +47,7 @@ def inner(*args, **kwargs): func(*args, **kwargs) return inner + @contextmanager def html_coverage_report(directory="./coverage"): # This relies on this being run from within a directory named the same as @@ -182,6 +183,7 @@ def spec(verbosity=4): v.call_command("harvest", apps=fabfile.full_name, verbosity=verbosity) + def get_full_name(): if not hasattr(fabfile, "full_name"): try: diff --git a/armstrong/dev/tests/__init__.py b/armstrong/dev/tests/__init__.py index 3ad9513..ece379c 100644 --- a/armstrong/dev/tests/__init__.py +++ b/armstrong/dev/tests/__init__.py @@ -1,2 +1,2 @@ -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) +import pkg_resources +pkg_resources.declare_namespace(__name__) diff --git a/armstrong/dev/tests/utils/__init__.py b/armstrong/dev/tests/utils/__init__.py index f7ea4a4..1e73219 100644 --- a/armstrong/dev/tests/utils/__init__.py +++ b/armstrong/dev/tests/utils/__init__.py @@ -1,4 +1,5 @@ -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) +import pkg_resources +pkg_resources.declare_namespace(__name__) -from armstrong.dev.tests.utils.base import ArmstrongTestCase, override_settings \ No newline at end of file + +from armstrong.dev.tests.utils.base import ArmstrongTestCase, override_settings diff --git a/armstrong/dev/virtualdjango/__init__.py b/armstrong/dev/virtualdjango/__init__.py index 3ad9513..ece379c 100644 --- a/armstrong/dev/virtualdjango/__init__.py +++ b/armstrong/dev/virtualdjango/__init__.py @@ -1,2 +1,2 @@ -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) +import pkg_resources +pkg_resources.declare_namespace(__name__) diff --git a/setup.py b/setup.py index e34dfcb..bd2ee12 100644 --- a/setup.py +++ b/setup.py @@ -5,11 +5,12 @@ package.json file if you need to adjust metadata about this package. """ -from distutils.core import setup import json -import os +from setuptools import setup, find_packages + info = json.load(open("./package.json")) +NAMESPACE_PACKAGES = [] def convert_to_str(d): @@ -31,7 +32,6 @@ def convert_to_str(d): return d2 info = convert_to_str(info) -NAMESPACE_PACKAGES = [] # TODO: simplify this process @@ -43,48 +43,14 @@ def generate_namespaces(package): generate_namespaces(info["name"]) -if os.path.exists("MANIFEST"): - os.unlink("MANIFEST") - -# Borrowed and modified from django-registration -# Compile the list of packages available, because distutils doesn't have -# an easy way to do this. -packages, data_files = [], [] -root_dir = os.path.dirname(__file__) -if root_dir: - os.chdir(root_dir) - - -def build_package(dirpath, dirnames, filenames): - # Ignore dirnames that start with '.' - for i, dirname in enumerate(dirnames): - if dirname.startswith('.'): - del dirnames[i] - if '__init__.py' in filenames and 'steps.py' not in filenames: - pkg = dirpath.replace(os.path.sep, '.') - if os.path.altsep: - pkg = pkg.replace(os.path.altsep, '.') - packages.append(pkg) - elif filenames: - # Strip off the length of the package name plus the trailing slash - prefix = dirpath[len(info["name"]) + 1:] - for f in filenames: - # Ignore all dot files and any compiled - if f.startswith(".") or f.endswith(".pyc"): - continue - data_files.append(os.path.join(prefix, f)) - - -[build_package(dirpath, dirnames, filenames) for dirpath, dirnames, filenames - in os.walk(info["name"].replace(".", "/"))] - setup_kwargs = { "author": "Bay Citizen & Texas Tribune", "author_email": "dev@armstrongcms.org", "url": "http://github.com/armstrong/%s/" % info["name"], - "packages": packages, - "package_data": {info["name"]: data_files, }, + "packages": find_packages(exclude=["*.tests", "*.tests.*"]), "namespace_packages": NAMESPACE_PACKAGES, + "include_package_data": True, + "zip_safe": False, "classifiers": [ 'Development Status :: 3 - Alpha', 'Environment :: Web Environment', From 8a4fa208ed51602efff19a399e501c696955fb47 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Thu, 11 Apr 2013 17:16:01 -0700 Subject: [PATCH 02/53] Remove custom string conversion in `setup.py`. Distribute from at least 0.6.26 can handle unicode characters in the package json `description` field (unicode in the `name` breaks). Our custom function however couldn't handle any unicode. It broke with: ``` File "setup.py", line 30, in convert_to_str d2[k] = str(v) UnicodeEncodeError: 'ascii' codec can't encode characters ... ``` --- setup.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/setup.py b/setup.py index bd2ee12..7660c79 100644 --- a/setup.py +++ b/setup.py @@ -13,27 +13,6 @@ NAMESPACE_PACKAGES = [] -def convert_to_str(d): - """ - Recursively convert all values in a dictionary to strings - - This is required because setup() does not like unicode in - the values it is supplied. - """ - d2 = {} - for k, v in d.items(): - k = str(k) - if type(v) in [list, tuple]: - d2[k] = [str(a) for a in v] - elif type(v) is dict: - d2[k] = convert_to_str(v) - else: - d2[k] = str(v) - return d2 - -info = convert_to_str(info) - - # TODO: simplify this process def generate_namespaces(package): new_package = ".".join(package.split(".")[0:-1]) From f1bf55790bd3367c8501d802aa6e20f9d779c341 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Thu, 11 Apr 2013 18:30:29 -0700 Subject: [PATCH 03/53] WIP - create a common test runner that sets up the Django environment from an expected `env_settings.py` file. --- armstrong/dev/default_settings.py | 34 +++++++++++++++++++++++++++++++ armstrong/dev/runtests.py | 27 ++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 armstrong/dev/default_settings.py create mode 100644 armstrong/dev/runtests.py diff --git a/armstrong/dev/default_settings.py b/armstrong/dev/default_settings.py new file mode 100644 index 0000000..286355c --- /dev/null +++ b/armstrong/dev/default_settings.py @@ -0,0 +1,34 @@ +# Since we are using configure() we need to manually load the defaults +from django.conf.global_settings import * + +# Grab our package information +import json +package = json.load(open("./package.json")) + + +# +# Default settings +# +DEBUG = True +TESTED_APPS = ['tests'] +DATABASES = { + "default": { + "ENGINE": "django.db.backends.sqlite3" + } +} + + +# +# Component specific settings +# +# Each component needs to create an `env_settings.py` file in its +# root directory that imports this file. Then the component can define +# whatever it needs for its Django environment. example: +# +# from armstrong.dev.default_settings import * +# +# INSTALLED_APPS = [ +# package["name"], +# '%s.tests' % package["name"], +# ] +# diff --git a/armstrong/dev/runtests.py b/armstrong/dev/runtests.py new file mode 100644 index 0000000..40ae4a2 --- /dev/null +++ b/armstrong/dev/runtests.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python + +import sys +from django.conf import settings + +try: + import env_settings as package_settings +except ImportError as e: + sys.stderr.write("ImportError: %s. Running a Django environment " + "for this component requires an `env_settings.py` file.\n" % e) + + +if not settings.configured: + settings.configure(default_settings=package_settings) + + +def runtests(): + from django.test.simple import DjangoTestSuiteRunner + failures = DjangoTestSuiteRunner( + verbosity=2, + interactive=True, + failfast=False).run_tests(settings.TESTED_APPS) + sys.exit(failures) + + +if __name__ == '__main__': + runtests() From dbc518b61a5e5b95f266db73e7f081935ff2706c Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Fri, 19 Apr 2013 16:02:10 -0700 Subject: [PATCH 04/53] Iterate the last commit. `dev_django.py` is now the hub for running a Django environment with the component's settings; it's not just for tests. - `run_django_cmd()` can run any Django command - command line access proxies `manage.py` (so it too can run any Django command) - add loading error messages - `test` is a special case where a) we want to allow specific tests (https://docs.djangoproject.com/en/1.5/topics/testing/overview/#running-tests) b) but default to `TESTED_APPS` --- armstrong/dev/default_settings.py | 17 ++------ armstrong/dev/dev_django.py | 67 +++++++++++++++++++++++++++++++ armstrong/dev/runtests.py | 27 ------------- 3 files changed, 71 insertions(+), 40 deletions(-) create mode 100644 armstrong/dev/dev_django.py delete mode 100644 armstrong/dev/runtests.py diff --git a/armstrong/dev/default_settings.py b/armstrong/dev/default_settings.py index 286355c..acb9390 100644 --- a/armstrong/dev/default_settings.py +++ b/armstrong/dev/default_settings.py @@ -9,26 +9,17 @@ # # Default settings # -DEBUG = True TESTED_APPS = ['tests'] +INSTALLED_APPS = [package["name"], 'tests'] DATABASES = { "default": { "ENGINE": "django.db.backends.sqlite3" } } - -# -# Component specific settings -# -# Each component needs to create an `env_settings.py` file in its -# root directory that imports this file. Then the component can define -# whatever it needs for its Django environment. example: # -# from armstrong.dev.default_settings import * +# A component may override settings by creating an `env_settings.py` +# file in its root directory that imports from this file. # -# INSTALLED_APPS = [ -# package["name"], -# '%s.tests' % package["name"], -# ] +# from armstrong.dev.default_settings import * # diff --git a/armstrong/dev/dev_django.py b/armstrong/dev/dev_django.py new file mode 100644 index 0000000..2f30d99 --- /dev/null +++ b/armstrong/dev/dev_django.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +import os +import sys + +try: + from django.conf import settings +except ImportError as e: + raise ImportError("%s. Check to see if Django " + "is installed in your virtualenv." % e) + + +__all__ = ['run_django_cmd'] + + +# Add the component's directory to the path so we can import from it +sys.path.append(os.getcwd()) + +try: + import env_settings as package_settings +except ImportError as e: + print ("Could not find component specific settings file." + "Using armstrong.dev defaults...") + try: + import armstrong.dev.default_settings as package_settings + except ImportError as e: + raise ImportError("%s. Running a Django environment for this " + "component requires either an `env_settings.py` file or the " + "armstrong.dev `default_settings.py` file." % e) + + +# Setup the Django environment +if not settings.configured: + settings.configure(default_settings=package_settings) + + +def determine_test_args(test_labels): + """ + Limit testing to settings.TESTED_APPS if available while behaving + exactly like `manage.py test` and retaining Django's ability to + explicitly provide test apps/cases/methods on the commandline. + + """ + return test_labels or getattr(settings, 'TESTED_APPS', []) + + +# Import usage +def run_django_cmd(cmd, *args, **kwargs): + if cmd == "test": + args = determine_test_args(args) + + from django.core.management import call_command + return call_command(cmd, *args, **kwargs) + + +# Commandline access +if __name__ == "__main__": + args = sys.argv[2:] + + if len(sys.argv) > 1 and sys.argv[1] == "test": + # anything not a flag (e.g. -v, --version) is treated as a named test + # (it'll be a short list so iterating it twice is okay) + args = [arg for arg in sys.argv[2:] if arg.startswith('-')] + test_labels = [arg for arg in sys.argv[2:] if not arg.startswith('-')] + args = determine_test_args(test_labels) + args + + from django.core.management import execute_from_command_line + execute_from_command_line(sys.argv[:2] + args) diff --git a/armstrong/dev/runtests.py b/armstrong/dev/runtests.py deleted file mode 100644 index 40ae4a2..0000000 --- a/armstrong/dev/runtests.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python - -import sys -from django.conf import settings - -try: - import env_settings as package_settings -except ImportError as e: - sys.stderr.write("ImportError: %s. Running a Django environment " - "for this component requires an `env_settings.py` file.\n" % e) - - -if not settings.configured: - settings.configure(default_settings=package_settings) - - -def runtests(): - from django.test.simple import DjangoTestSuiteRunner - failures = DjangoTestSuiteRunner( - verbosity=2, - interactive=True, - failfast=False).run_tests(settings.TESTED_APPS) - sys.exit(failures) - - -if __name__ == '__main__': - runtests() From 87adf34b48bdbd595d6dd6cd64eb4b1c3459972a Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Wed, 24 Apr 2013 13:01:37 -0700 Subject: [PATCH 05/53] Tidying and change of test location. I'm falling back to the current scheme of `tests/` inside the application directory. (Instead of a root level tests directory/app called "tests" initially introduced in this PR.) The downside is that tests will be distributed in the final code (due to packaging in `setup.py`). The upside is tests are contained. The problem I was having is that a root level "tests" app can conflict when testing multiple Armstrong components in the same virtualenv. If the python path has a path for say armstrong.core.arm_content and you switch to arm_sections, the "tests" app might still be pulled from the earlier path entry. Maybe this is a fridge case; maybe it has to do with my `pip install --editable`; it's easier to keep the current expectations and change them later when I'm moving fewer pieces. --- .gitignore | 7 ++++--- armstrong/dev/default_settings.py | 8 +++++--- armstrong/dev/dev_django.py | 10 +++++++--- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index ba4d11c..a14d7e6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,10 @@ *.pyc *~ *.swp -coverage/* -build/* +*.egg-info +coverage*/ +build/ mydatabase -docs/_build/* +docs/_build/ MANIFEST dist/ diff --git a/armstrong/dev/default_settings.py b/armstrong/dev/default_settings.py index acb9390..d8be608 100644 --- a/armstrong/dev/default_settings.py +++ b/armstrong/dev/default_settings.py @@ -4,16 +4,18 @@ # Grab our package information import json package = json.load(open("./package.json")) +app_name = package['name'].rsplit('.', 1)[1] # # Default settings # -TESTED_APPS = ['tests'] -INSTALLED_APPS = [package["name"], 'tests'] +TESTED_APPS = [app_name] +INSTALLED_APPS = [package['name']] DATABASES = { "default": { - "ENGINE": "django.db.backends.sqlite3" + "ENGINE": "django.db.backends.sqlite3", + "NAME": 'database' } } diff --git a/armstrong/dev/dev_django.py b/armstrong/dev/dev_django.py index 2f30d99..4d09ea0 100644 --- a/armstrong/dev/dev_django.py +++ b/armstrong/dev/dev_django.py @@ -18,7 +18,7 @@ try: import env_settings as package_settings except ImportError as e: - print ("Could not find component specific settings file." + print("Could not find component specific settings file. " "Using armstrong.dev defaults...") try: import armstrong.dev.default_settings as package_settings @@ -43,7 +43,7 @@ def determine_test_args(test_labels): return test_labels or getattr(settings, 'TESTED_APPS', []) -# Import usage +# Import access def run_django_cmd(cmd, *args, **kwargs): if cmd == "test": args = determine_test_args(args) @@ -53,7 +53,7 @@ def run_django_cmd(cmd, *args, **kwargs): # Commandline access -if __name__ == "__main__": +def run_django_cli(): args = sys.argv[2:] if len(sys.argv) > 1 and sys.argv[1] == "test": @@ -65,3 +65,7 @@ def run_django_cmd(cmd, *args, **kwargs): from django.core.management import execute_from_command_line execute_from_command_line(sys.argv[:2] + args) + + +if __name__ == "__main__": + run_django_cli() From 22707fe83119f62a4cc3e8946bbac3b7895bb1b0 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Wed, 24 Apr 2013 13:59:01 -0700 Subject: [PATCH 06/53] WIP remove VirtualDjango in favor of new `run_django_cmd` and move Fabric tasks into a fabfile.py instead of __init__.py --- .../dev/{tasks/__init__.py => fabfile.py} | 68 +++++-------------- 1 file changed, 17 insertions(+), 51 deletions(-) rename armstrong/dev/{tasks/__init__.py => fabfile.py} (75%) diff --git a/armstrong/dev/tasks/__init__.py b/armstrong/dev/fabfile.py similarity index 75% rename from armstrong/dev/tasks/__init__.py rename to armstrong/dev/fabfile.py index d955ae4..b47e9e1 100644 --- a/armstrong/dev/tasks/__init__.py +++ b/armstrong/dev/fabfile.py @@ -1,40 +1,20 @@ - -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) - - -from contextlib import contextmanager -try: - import coverage as coverage -except ImportError: - coverage = False -import os -from os.path import basename, dirname import sys +from os.path import dirname from functools import wraps -import unittest - -import json +from contextlib import contextmanager -from fabric.api import * -from fabric.colors import red +from fabric.api import local, settings +from fabric.colors import yellow, red from fabric.decorators import task -from armstrong.dev.virtualdjango.test_runner import run_tests as run_django_tests -from armstrong.dev.virtualdjango.base import VirtualDjango -from django.core.exceptions import ImproperlyConfigured - -if not "fabfile" in sys.modules: - sys.stderr.write("This expects to have a 'fabfile' module\n") - sys.stderr.write(-1) -fabfile = sys.modules["fabfile"] +from armstrong.dev.dev_django import run_django_cmd FABRIC_TASK_MODULE = True -__all__ = ["clean", "command", "create_migration", "docs", "pep8", "test", - "reinstall", "runserver", "shell", "spec", "syncdb", ] +__all__ = ["clean", "create_migration", "docs", "pep8", "test", + "reinstall", "spec", "proxy"] def pip_install(func): @wraps(func) @@ -47,6 +27,8 @@ def inner(*args, **kwargs): func(*args, **kwargs) return inner + + @contextmanager def html_coverage_report(directory="./coverage"): # This relies on this being run from within a directory named the same as @@ -88,18 +70,6 @@ def create_migration(name, initial=False, auto=True): })) -@task -def command(*cmds): - """Run and arbitrary set of Django commands""" - runner = VirtualDjango() - runner.run(fabfile.settings) - for cmd in cmds: - if type(cmd) is tuple: - args, kwargs = cmd - else: - args = (cmd, ) - kwargs = {} - runner.call_command(*args, **kwargs) @task @@ -139,21 +109,17 @@ def test(): @task -def runserver(): - """Create a Django development server""" - command("runserver") - - -@task -def shell(): - """Launch shell with same settings as test and runserver""" - command("shell") @task -def syncdb(): - """Call syncdb and migrate on project""" - command("syncdb", "migrate") +def proxy(cmd=None, *args, **kwargs): + """Run manage.py using this component's specific Django settings""" + + if cmd is None: + sys.stderr.write(red("Usage: fab proxy:,arg1,kwarg=1\n") + + "which translates to: manage.py command arg1 --kwarg=1\n") + sys.exit(1) + run_django_cmd(cmd, *args, **kwargs) @task From d5c42d7767bd502dbe2a152387d410ac0b9e339d Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Wed, 24 Apr 2013 14:03:48 -0700 Subject: [PATCH 07/53] Replace `reinstall` command with a more flexible `install` --- armstrong/dev/fabfile.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/armstrong/dev/fabfile.py b/armstrong/dev/fabfile.py index b47e9e1..2066ce6 100644 --- a/armstrong/dev/fabfile.py +++ b/armstrong/dev/fabfile.py @@ -1,5 +1,6 @@ import sys from os.path import dirname +from ast import literal_eval from functools import wraps from contextlib import contextmanager @@ -14,7 +15,7 @@ __all__ = ["clean", "create_migration", "docs", "pep8", "test", - "reinstall", "spec", "proxy"] + "install", "spec", "proxy"] def pip_install(func): @wraps(func) @@ -165,9 +166,22 @@ def get_full_name(): sys.stderr.flush() sys.exit(1) return fabfile.full_name +def install(editable=True): + """Install this component (or remove and reinstall)""" + + try: + __import__(package['name']) + except ImportError: + pass + else: + with settings(warn_only=True): + local("pip uninstall --quiet -y %s" % package['name'], capture=False) + + cmd = "pip install --quiet " + cmd += "-e ." if literal_eval(editable) else "." + + with settings(warn_only=True): + local(cmd, capture=False) @task -def reinstall(): - """Install the current component""" - local("pip uninstall -y `basename \\`pwd\\``; pip install .") From e53d8af51c68538960ec942f9c7862a549fbe94a Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Wed, 24 Apr 2013 14:19:46 -0700 Subject: [PATCH 08/53] Remove the `pip_install` decorator that automatically uninstalls/installs the component. Replace in favor of an *explicit* install procedure and a decorator that simply halts if the component isn't installed. I'm in favor of explicit installation and clear messaging. Also I don't believe `pip_install` would have ever worked on a fresh virtualenv. It's hard to test with the multiple avenues of package requirement declaration (between setup.py and the requirements files), but I found that if the package path doesn't exist initially that even after the `pip install`, the component couldn't be imported. Run the exact command again, the path would exist and the command would run fine. I didn't want to add complication of `sys.path.append()` or similar methods. Coupled with the new `install` task, this is more explicit. It also allows pip installing this component in `editable` mode so that it doesn't have to be reinstalled into the environment every time you make a change and run a Fabric task. (This is similar to `setup.py develop` if we used Distribute.) --- armstrong/dev/fabfile.py | 38 ++++++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/armstrong/dev/fabfile.py b/armstrong/dev/fabfile.py index 2066ce6..557b85f 100644 --- a/armstrong/dev/fabfile.py +++ b/armstrong/dev/fabfile.py @@ -13,20 +13,33 @@ FABRIC_TASK_MODULE = True +# Grab our package information +import json +package = json.load(open("./package.json")) + + +def require_self(func=None): + """Decorator to require that this component be installed""" + + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + __import__(package['name']) + except ImportError: + sys.stderr.write( + red("This component needs to be installed first. Run ") + + yellow("`fab install`\n")) + sys.exit(1) + return func(*args, **kwargs) + return wrapper + return decorator if not func else decorator(func) + + __all__ = ["clean", "create_migration", "docs", "pep8", "test", "install", "spec", "proxy"] -def pip_install(func): - @wraps(func) - def inner(*args, **kwargs): - if getattr(fabfile, "pip_install_first", True): - with settings(warn_only=True): - if not os.environ.get("SKIP_INSTALL", False): - local("pip uninstall -y %s" % get_full_name(), capture=False) - local("pip install .", capture=False) - func(*args, **kwargs) - return inner @@ -63,6 +76,7 @@ def clean(): @task +@require_self def create_migration(name, initial=False, auto=True): """Create a South migration for app""" command((("schemamigration", fabfile.main_app, name), { @@ -80,7 +94,7 @@ def pep8(): @task -@pip_install +@require_self def test(): """Run tests against `tested_apps`""" from types import FunctionType @@ -130,7 +144,7 @@ def docs(): @task -@pip_install +@require_self def spec(verbosity=4): """Run harvest to run all of the Lettuce specs""" defaults = {"DATABASES": { From 1c9f9daa584eb6d9c92d42e0fdaccd92cf7b5e26 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Wed, 24 Apr 2013 14:29:43 -0700 Subject: [PATCH 09/53] Remove package requirements in favor of an "a la carte" model of install when you need it. This will make environment building faster (and this will be helpful in automatic virtualenv creation during testing). This also gets us away from pinning versions which seems weird especially for a dev tool. I've loosen the Fabric version requirement but left `fudge` the same because it hasn't been updated in a while so if/when it does, it may be something we should control. Also by way of oversight, Sphinx wasn't required and is necessary for the `docs` command. --- armstrong/dev/fabfile.py | 25 +++++++++++++++++++++++-- package.json | 7 ++----- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/armstrong/dev/fabfile.py b/armstrong/dev/fabfile.py index 557b85f..8ba161c 100644 --- a/armstrong/dev/fabfile.py +++ b/armstrong/dev/fabfile.py @@ -13,6 +13,11 @@ FABRIC_TASK_MODULE = True +__all__ = ["clean", "create_migration", "docs", "pep8", "test", + "install", "spec", "proxy"] + + + # Grab our package information import json package = json.load(open("./package.json")) @@ -36,9 +41,22 @@ def wrapper(*args, **kwargs): return decorator if not func else decorator(func) +def require_pip_module(module): + """Decorator to check for a module and helpfully exit if it's not found""" -__all__ = ["clean", "create_migration", "docs", "pep8", "test", - "install", "spec", "proxy"] + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + __import__(module) + except ImportError: + sys.stderr.write( + yellow("`pip install %s` to enable this feature\n" % module)) + sys.exit(1) + else: + return func(*args, **kwargs) + return wrapper + return decorator @@ -77,6 +95,7 @@ def clean(): @task @require_self +@require_pip_module('south') def create_migration(name, initial=False, auto=True): """Create a South migration for app""" command((("schemamigration", fabfile.main_app, name), { @@ -88,6 +107,7 @@ def create_migration(name, initial=False, auto=True): @task +@require_pip_module('pep8') def pep8(): """Run pep8 on all .py files in ./armstrong""" local('find ./armstrong -name "*.py" | xargs pep8 --repeat', capture=False) @@ -138,6 +158,7 @@ def proxy(cmd=None, *args, **kwargs): @task +@require_pip_module('sphinx') def docs(): """Generate the Sphinx docs for this project""" local("cd docs && make html") diff --git a/package.json b/package.json index c39c8e0..5e61043 100644 --- a/package.json +++ b/package.json @@ -3,10 +3,7 @@ "version": "1.14.0alpha.0", "description": "Tools needed for development and testing of Armstrong", "install_requires": [ - "South==0.7.3", - "Fabric==1.3.3", - "pep8 == 0.6.1", - "coverage == 3.5.1", - "fudge == 1.0.3" + "Fabric<2.0", + "fudge==1.0.3" ] } From f3505b8f7b5cc4114277c4f68924ded9a8243c04 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Wed, 24 Apr 2013 14:32:19 -0700 Subject: [PATCH 10/53] Drop Lettuce (for now at least) and the no-longer-necessary `get_full_name()`. Refactor `create_migration` to use new `run_django_cmd`. --- armstrong/dev/fabfile.py | 52 +++++++--------------------------------- 1 file changed, 9 insertions(+), 43 deletions(-) diff --git a/armstrong/dev/fabfile.py b/armstrong/dev/fabfile.py index 8ba161c..604c293 100644 --- a/armstrong/dev/fabfile.py +++ b/armstrong/dev/fabfile.py @@ -14,7 +14,7 @@ FABRIC_TASK_MODULE = True __all__ = ["clean", "create_migration", "docs", "pep8", "test", - "install", "spec", "proxy"] + "install", "proxy"] @@ -96,14 +96,16 @@ def clean(): @task @require_self @require_pip_module('south') -def create_migration(name, initial=False, auto=True): - """Create a South migration for app""" - command((("schemamigration", fabfile.main_app, name), { - "initial": bool(int(initial)), - "auto": bool(int(auto)), - })) +def create_migration(initial=False): + """Create a South migration for this project""" + from django.conf import settings as django_settings + if 'south' not in (name.lower() for name in django_settings.INSTALLED_APPS): + print("Temporarily adding 'south' into INSTALLED_APPS.") + django_settings.INSTALLED_APPS.append('south') + kwargs = dict(initial=True) if literal_eval(initial) else dict(auto=True) + run_django_cmd('schemamigration', package['name'], **kwargs) @task @@ -165,42 +167,6 @@ def docs(): @task -@require_self -def spec(verbosity=4): - """Run harvest to run all of the Lettuce specs""" - defaults = {"DATABASES": { - "default": { - "ENGINE": "django.db.backends.sqlite3", - "NAME": ":memory:", - }, - }} - - get_full_name() - defaults.update(fabfile.settings) - v = VirtualDjango() - v.run(defaults) - v.call_command("syncdb", interactive=False) - v.call_command("migrate") - v.call_command("harvest", apps=fabfile.full_name, - verbosity=verbosity) - -def get_full_name(): - if not hasattr(fabfile, "full_name"): - try: - package_string = local("cat ./package.json", capture=True) - package_obj = json.loads(package_string) - fabfile.full_name = package_obj['name'] - return fabfile.full_name - except: - sys.stderr.write("\n".join([ - red("No `full_name` variable detected in your fabfile!"), - red("Please set `full_name` to the app's full module"), - red("Additionally, we couldn't read name from package.json"), - "" - ])) - sys.stderr.flush() - sys.exit(1) - return fabfile.full_name def install(editable=True): """Install this component (or remove and reinstall)""" From aaca58463c922c6395104753f575e9e4046306f7 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Wed, 24 Apr 2013 14:37:04 -0700 Subject: [PATCH 11/53] Refactor `coverage` and `test`. This actually separates the two commands allowing normal `manage.py test` running as well as `coverage`. Change coverage directory spec a bit to make it more flexible. (This will be useful for running coverage across multiple test virtualenvs and keeping separate results.) --- armstrong/dev/fabfile.py | 84 ++++++++++++++++------------------------ 1 file changed, 33 insertions(+), 51 deletions(-) diff --git a/armstrong/dev/fabfile.py b/armstrong/dev/fabfile.py index 604c293..efbf710 100644 --- a/armstrong/dev/fabfile.py +++ b/armstrong/dev/fabfile.py @@ -13,9 +13,8 @@ FABRIC_TASK_MODULE = True -__all__ = ["clean", "create_migration", "docs", "pep8", "test", - "install", "proxy"] - +__all__ = ["clean", "create_migration", "docs", "pep8", "proxy", + "coverage", "test", "install"] # Grab our package information @@ -59,32 +58,25 @@ def wrapper(*args, **kwargs): return decorator +@contextmanager +def html_coverage_report(report_directory=None): + package_parent = str(package['name'].rsplit('.', 1)[0]) # fromlist can't handle unicode + module = __import__(package['name'], fromlist=[package_parent]) + base_path = dirname(module.__file__) + import coverage as coverage_api + print("Coverage is covering: %s" % base_path) + cov = coverage_api.coverage(branch=True, source=[base_path]) -@contextmanager -def html_coverage_report(directory="./coverage"): - # This relies on this being run from within a directory named the same as - # the repository on GitHub. It's fragile, but for our purposes, it works. - run_coverage = coverage - if run_coverage and os.environ.get("SKIP_COVERAGE", False): - run_coverage = False - - if run_coverage: - local('rm -rf ' + directory) - package = __import__('site') - base_path = dirname(package.__file__) + '/site-packages/' + get_full_name().replace('.', '/') - print "Coverage is covering: " + base_path - cov = coverage.coverage(branch=True, - source=(base_path,), - omit=('*/migrations/*',)) - cov.start() + cov.start() yield + cov.stop() - if run_coverage: - cov.stop() - cov.html_report(directory=directory) - else: - print "Install coverage.py to measure test coverage" + # Write results + report_directory = report_directory or "coverage" + local('rm -rf ' + report_directory) + cov.html_report(directory=report_directory) + print("Coverage reports available in: %s " % report_directory) @task @@ -117,35 +109,25 @@ def pep8(): @task @require_self -def test(): - """Run tests against `tested_apps`""" - from types import FunctionType - if hasattr(fabfile, 'settings') and type(fabfile.settings) is not FunctionType: - with html_coverage_report(): - run_django_tests(fabfile.settings, *fabfile.tested_apps) - return - else: - test_module = "%s.tests" % get_full_name() - try: - __import__(test_module) - tests = sys.modules[test_module] - except ImportError: - tests = False - pass - - if tests: - test_suite = getattr(tests, "suite", False) - if test_suite: - with html_coverage_report(): - unittest.TextTestRunner().run(test_suite) - return - - raise ImproperlyConfigured( - "Unable to find tests to run. Please see armstrong.dev README." - ) +def test(*args, **kwargs): + """Test this component via `manage.py test`""" + run_django_cmd('test', *args, **kwargs) @task +@require_self +@require_pip_module('coverage') +def coverage(*args, **kwargs): + """Test this project with coverage reports""" + + # Option to pass in the coverage report directory + coverage_dir = kwargs.pop('coverage_dir', None) + + try: + with html_coverage_report(coverage_dir): + run_django_cmd('test', *args, **kwargs) + except (ImportError, EnvironmentError): + sys.exit(1) @task From e4a5d0383c4cd9bc57dd48158ef6cc2087f2eeb0 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Wed, 24 Apr 2013 14:39:05 -0700 Subject: [PATCH 12/53] New feature to remove all Armstrong components (except for armstrong.dev) from the virtualenv. Helpful if you are like me and tend to use the same virtualenv when working with different Armstrong components (perhaps not a great practice). --- armstrong/dev/fabfile.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/armstrong/dev/fabfile.py b/armstrong/dev/fabfile.py index efbf710..b65443b 100644 --- a/armstrong/dev/fabfile.py +++ b/armstrong/dev/fabfile.py @@ -14,7 +14,7 @@ FABRIC_TASK_MODULE = True __all__ = ["clean", "create_migration", "docs", "pep8", "proxy", - "coverage", "test", "install"] + "coverage", "test", "install", "remove_armstrong"] # Grab our package information @@ -168,3 +168,19 @@ def install(editable=True): @task +def remove_armstrong(): + """Remove all armstrong components (except for dev) from this environment""" + + from pip.util import get_installed_distributions + pkgs = get_installed_distributions(local_only=True, include_editables=True) + apps = [pkg for pkg in pkgs + if pkg.key.startswith('armstrong') and pkg.key != 'armstrong.dev'] + + for app in apps: + local("pip uninstall -y %s" % app.key) + + if apps: + print("Note: this hasn't removed other dependencies installed by " + "these components. There's no substitute for a fresh virtualenv.") + else: + print("No armstrong components to remove.") From f0cbf23f72b1d3a56b28a41c3c0258ddb26abe9a Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Wed, 24 Apr 2013 14:39:39 -0700 Subject: [PATCH 13/53] Remove VirtualDjango! --- armstrong/dev/virtualdjango/__init__.py | 2 - armstrong/dev/virtualdjango/base.py | 57 ---------------------- armstrong/dev/virtualdjango/test_runner.py | 12 ----- 3 files changed, 71 deletions(-) delete mode 100644 armstrong/dev/virtualdjango/__init__.py delete mode 100644 armstrong/dev/virtualdjango/base.py delete mode 100644 armstrong/dev/virtualdjango/test_runner.py diff --git a/armstrong/dev/virtualdjango/__init__.py b/armstrong/dev/virtualdjango/__init__.py deleted file mode 100644 index 3ad9513..0000000 --- a/armstrong/dev/virtualdjango/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) diff --git a/armstrong/dev/virtualdjango/base.py b/armstrong/dev/virtualdjango/base.py deleted file mode 100644 index b155a76..0000000 --- a/armstrong/dev/virtualdjango/base.py +++ /dev/null @@ -1,57 +0,0 @@ -import django -import os, sys - -DEFAULT_SETTINGS = { - 'DATABASE_ENGINE': 'sqlite3', - 'DATABASES': { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': 'mydatabase' - } - }, -} - -class VirtualDjango(object): - def __init__(self, - caller=sys.modules['__main__'], - default_settings=DEFAULT_SETTINGS): - self.caller = caller - self.default_settings = default_settings - - - def configure_settings(self, customizations, reset=True): - # Django expects a `DATABASE_ENGINE` value - custom_settings = self.default_settings - custom_settings.update(customizations) - - settings = self.settings - if reset: - self.reset_settings(settings) - settings.configure(**custom_settings) - - def reset_settings(self, settings): - if django.VERSION[:2] == (1, 3): - settings._wrapped = None - return - - # This is the way to reset settings going forward - from django.utils.functional import empty - settings._wrapped = empty - - @property - def settings(self): - from django.conf import settings - return settings - - @property - def call_command(self): - from django.core.management import call_command - return call_command - - def run(self, my_settings): - if hasattr(self.caller, 'setUp'): - self.caller.setUp() - - self.configure_settings(my_settings) - return self.call_command - diff --git a/armstrong/dev/virtualdjango/test_runner.py b/armstrong/dev/virtualdjango/test_runner.py deleted file mode 100644 index 1b18604..0000000 --- a/armstrong/dev/virtualdjango/test_runner.py +++ /dev/null @@ -1,12 +0,0 @@ -from armstrong.dev.virtualdjango.base import VirtualDjango - -class VirtualDjangoTestRunner(VirtualDjango): - def run(self, my_settings, *apps_to_test): - super(VirtualDjangoTestRunner, self).run(my_settings) - self.call_command('test', *apps_to_test) - - def __call__(self, *args, **kwargs): - self.run(*args, **kwargs) - -run_tests = VirtualDjangoTestRunner() - From 8df29a89d470c40ecae952530e77717df5f9ee29 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Fri, 24 May 2013 12:25:39 -0700 Subject: [PATCH 14/53] Fix - literal_eval() needs a string --- armstrong/dev/fabfile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/armstrong/dev/fabfile.py b/armstrong/dev/fabfile.py index b65443b..02b9fa6 100644 --- a/armstrong/dev/fabfile.py +++ b/armstrong/dev/fabfile.py @@ -96,7 +96,7 @@ def create_migration(initial=False): print("Temporarily adding 'south' into INSTALLED_APPS.") django_settings.INSTALLED_APPS.append('south') - kwargs = dict(initial=True) if literal_eval(initial) else dict(auto=True) + kwargs = dict(initial=True) if literal_eval(str(initial)) else dict(auto=True) run_django_cmd('schemamigration', package['name'], **kwargs) @@ -161,7 +161,7 @@ def install(editable=True): local("pip uninstall --quiet -y %s" % package['name'], capture=False) cmd = "pip install --quiet " - cmd += "-e ." if literal_eval(editable) else "." + cmd += "-e ." if literal_eval(str(editable)) else "." with settings(warn_only=True): local(cmd, capture=False) From 923605a941051ce03d580b2505476b15eef5fe3d Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Tue, 25 Jun 2013 17:25:43 -0700 Subject: [PATCH 15/53] Revert "Add `assertInContext` method." This reverts commit 4cda378425c57b874cc08f678fcd3e1b262212a3. --- armstrong/dev/tests/utils/base.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/armstrong/dev/tests/utils/base.py b/armstrong/dev/tests/utils/base.py index f0dde57..33f36fc 100644 --- a/armstrong/dev/tests/utils/base.py +++ b/armstrong/dev/tests/utils/base.py @@ -8,7 +8,6 @@ except ImportError: from .backports import override_settings - class ArmstrongTestCase(DjangoTestCase): def setUp(self): fudge.clear_expectations() @@ -47,13 +46,6 @@ def assertModelHasField(self, model, field_name, field_class=None): field_class.__class__.__name__) self.assertTrue(isinstance(field, field_class), msg=msg) - def assertInContext(self, var_name, other, template_or_context): - # TODO: support passing in a straight "context" (i.e., dict) - context = template_or_context.context_data - self.assertTrue(var_name in context, - msg="`%s` not in provided context" % var_name) - self.assertEqual(context[var_name], other) - def assertNone(self, obj, **kwargs): self.assertTrue(obj is None, **kwargs) From 55788468804b2f4ec17d7a99a656930a3f14c32f Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Tue, 25 Jun 2013 17:27:45 -0700 Subject: [PATCH 16/53] Use the native `assertIsNone()` http://docs.python.org/2.7/library/unittest.html#unittest.TestCase.assertIsNone --- armstrong/dev/tests/utils/base.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/armstrong/dev/tests/utils/base.py b/armstrong/dev/tests/utils/base.py index 33f36fc..c9d6b00 100644 --- a/armstrong/dev/tests/utils/base.py +++ b/armstrong/dev/tests/utils/base.py @@ -46,9 +46,6 @@ def assertModelHasField(self, model, field_name, field_class=None): field_class.__class__.__name__) self.assertTrue(isinstance(field, field_class), msg=msg) - def assertNone(self, obj, **kwargs): - self.assertTrue(obj is None, **kwargs) - def assertIsA(self, obj, cls, **kwargs): self.assertTrue(isinstance(obj, cls), **kwargs) From 16ecd6623267be0524ed9b04ba54a9eedf636517 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Tue, 25 Jun 2013 17:28:34 -0700 Subject: [PATCH 17/53] Use the native `assertIsInstance()` http://docs.python.org/2.7/library/unittest.html#unittest.TestCase.assertIsInstance --- armstrong/dev/tests/utils/base.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/armstrong/dev/tests/utils/base.py b/armstrong/dev/tests/utils/base.py index c9d6b00..a9ae2e2 100644 --- a/armstrong/dev/tests/utils/base.py +++ b/armstrong/dev/tests/utils/base.py @@ -46,8 +46,5 @@ def assertModelHasField(self, model, field_name, field_class=None): field_class.__class__.__name__) self.assertTrue(isinstance(field, field_class), msg=msg) - def assertIsA(self, obj, cls, **kwargs): - self.assertTrue(isinstance(obj, cls), **kwargs) - def assertDoesNotHave(self, obj, attr, **kwargs): self.assertFalse(hasattr(obj, attr), **kwargs) From d7bb6ba46abd09937e797c6bcd167a965d01a726 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Tue, 25 Jun 2013 17:31:36 -0700 Subject: [PATCH 18/53] Drop this one liner test that can be confused with a native test. Its confusion causing is greater than its benefit. --- armstrong/dev/tests/utils/base.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/armstrong/dev/tests/utils/base.py b/armstrong/dev/tests/utils/base.py index a9ae2e2..97f48aa 100644 --- a/armstrong/dev/tests/utils/base.py +++ b/armstrong/dev/tests/utils/base.py @@ -45,6 +45,3 @@ def assertModelHasField(self, model, field_name, field_class=None): msg = "%s.%s is not a %s" % (model.__class__.__name__, field_name, field_class.__class__.__name__) self.assertTrue(isinstance(field, field_class), msg=msg) - - def assertDoesNotHave(self, obj, attr, **kwargs): - self.assertFalse(hasattr(obj, attr), **kwargs) From c69bdee73063006e84f237a17c670e9004c1a13f Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Tue, 25 Jun 2013 17:33:46 -0700 Subject: [PATCH 19/53] Add a note to remove this backport code when we drop Django 1.3 support. --- armstrong/dev/tests/utils/backports.py | 4 +--- armstrong/dev/tests/utils/base.py | 9 ++++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/armstrong/dev/tests/utils/backports.py b/armstrong/dev/tests/utils/backports.py index 6f401b8..47a064e 100644 --- a/armstrong/dev/tests/utils/backports.py +++ b/armstrong/dev/tests/utils/backports.py @@ -1,8 +1,8 @@ - from django.conf import settings, UserSettingsHolder from django.utils.functional import wraps +# DEPRECATED remove when we drop Django 1.3 support class override_settings(object): """ Acts as either a decorator, or a context manager. If it's a decorator it @@ -45,5 +45,3 @@ def enable(self): def disable(self): settings._wrapped = self.wrapped - - diff --git a/armstrong/dev/tests/utils/base.py b/armstrong/dev/tests/utils/base.py index 97f48aa..1cbca9e 100644 --- a/armstrong/dev/tests/utils/base.py +++ b/armstrong/dev/tests/utils/base.py @@ -2,17 +2,20 @@ import fudge from django.db import models -# Backport override_settings from Django 1.4 +# DEPRECATED remove when we drop Django 1.3 support +# Backport override_settings from Django 1.4 try: from django.test.utils import override_settings except ImportError: from .backports import override_settings + class ArmstrongTestCase(DjangoTestCase): def setUp(self): fudge.clear_expectations() fudge.clear_calls() - + + # DEPRECATED remove when we drop Django 1.3 support if not hasattr(DjangoTestCase, 'settings'): # backported from Django 1.4 def settings(self, **kwargs): @@ -23,7 +26,7 @@ def settings(self, **kwargs): .. seealso: https://github.com/django/django/blob/0d670682952fae585ce5c5ec5dc335bd61d66bb2/django/test/testcases.py#L349-354 """ return override_settings(**kwargs) - + def assertRelatedTo(self, model, field_name, related_model, many=False): if many is False: through = models.ForeignKey From 2abf130afdbdf191d5e7b785c1526497961a1b22 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Tue, 25 Jun 2013 17:38:39 -0700 Subject: [PATCH 20/53] Remove `fudge` requirement but keep potentially handy fudge reset behavior. Individual component's should determine if they require fudge (and what specific version they need). --- armstrong/dev/tests/utils/base.py | 15 +++++++++++---- package.json | 3 +-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/armstrong/dev/tests/utils/base.py b/armstrong/dev/tests/utils/base.py index 1cbca9e..c2669ed 100644 --- a/armstrong/dev/tests/utils/base.py +++ b/armstrong/dev/tests/utils/base.py @@ -1,5 +1,4 @@ from django.test import TestCase as DjangoTestCase -import fudge from django.db import models # DEPRECATED remove when we drop Django 1.3 support @@ -9,11 +8,19 @@ except ImportError: from .backports import override_settings +# If the component uses fudge, provide useful shared behavior +try: + import fudge + hasFudge = True +except ImportError: + hasFudge = False + class ArmstrongTestCase(DjangoTestCase): - def setUp(self): - fudge.clear_expectations() - fudge.clear_calls() + if hasFudge: + def setUp(self): + fudge.clear_expectations() + fudge.clear_calls() # DEPRECATED remove when we drop Django 1.3 support if not hasattr(DjangoTestCase, 'settings'): diff --git a/package.json b/package.json index c39c8e0..85be057 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,6 @@ "South==0.7.3", "Fabric==1.3.3", "pep8 == 0.6.1", - "coverage == 3.5.1", - "fudge == 1.0.3" + "coverage == 3.5.1" ] } From 44d8b4f15a723bc3258f853848899cdb41c8e8cd Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Fri, 28 Jun 2013 13:31:16 -0700 Subject: [PATCH 21/53] Documentation --- README.rst | 112 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 73 insertions(+), 39 deletions(-) diff --git a/README.rst b/README.rst index 492ce0b..879b446 100644 --- a/README.rst +++ b/README.rst @@ -6,47 +6,78 @@ This package contains some of the various helpers needed to do development work on the Armstrong packages. If you're not actively developing, or working with development versions of Armstrong, you probably don't need this package. +Installation +------------ +1. ``pip install armstrong.dev`` + + Usage ----- +Most Armstrong components already have the necessary configuration to use these +Dev tools. Type ``fab -l`` to see a list of all of the commands. -Create a `fabfile` (either `fabfile/__init__.py` or simply `fabfile.py`) in -your project and add the following:: +If you are creating a new component (or perhaps updating one that uses +the older, pre 2.0 Dev tools), you'll need these next two steps. - from armstrong.dev.tasks import * +1. Create a ``fabfile.py`` and add the following:: + from armstrong.dev.fabfile import * - settings = { - "DEBUG": True, - # And so on with the keys being the name of the setting and the values - # the appropriate value. - } + # any additional Fabric commands + # ... - main_app = "name.of.your.app" - tested_apps ("another_app", main_app, ) +2. Create an ``env_settings.py`` and add the following:: + from armstrong.dev.default_settings import * -Now your fabfile will expose the various commands for setting up and running -your reusable app inside a virtualenv for testing, interacting with via the -shell, and even running a simple server. + # any additional settings + # ... -Type ``fab -l`` to see a list of all of the commands. +Notable changes in 2.0 +---------------------- +This version offers an easier and more standard way to run a Django +environment with a component's specific settings, either from the +commandline or via import. -Installation ------------- +It provides an "a la carte" requirements approach. Meaning that if you run a +Fabric command that needs a package that isn't installed, it will prompt you +to install it instead of requiring everything up-front. This allows for much +faster virtualenv creation (which saves considerable time in testing) and +doesn't pollute your virtualenv with packages for features you don't use. -:: +``test`` and ``coverage`` will work better with automated test tools like +TravisCI and Tox. These commands also now work like Django's native test +command so that you can pass arguments for running selective tests, i.e:: - name="armstrong.dev" - pip install -e git://github.com/armstrong/$name#egg=$name + fab test [[.[.]]] -**Note**: This currently relies on a development version of Fabric. This -requirement is set to be dropped once Fabric 1.1 is released. To ensure this -runs as expected, install the ``tswicegood/fabric`` fork of Fabric: +Settings are now defined in the normal Django style in an ``env_settings.py`` +file instead of as a dict within the Fabric config. It's not called +"settings.py" to make it clearer that these are settings for the development +and testing of this component, not necessarily values to copy/paste for +incorporating the component into other projects. -:: - pip install -e git://github.com/tswicegood/fabric.git#egg=fabric +Backward incompatible changes in 2.0 +------------------------------------ +* ``env_settings.py`` is necessary and contains the settings that + ``fabfile.py`` used to have. + +* ``fabfile.py`` imports from a different place and no longer defines the + settings configurations. + +Not required but as long as you are reviewing the general state of things, +take care of these things too! + +* Review the ``requirements`` files. +* Add a ``tox.ini`` file. +* Review or add a TravisCI configuration. +* Review ``.gitignore``. You might want to ignore these:: + + .tox/ + coverage*/ + *.egg-info Contributing @@ -57,24 +88,27 @@ Contributing * `Fork it`_ * Create a topic branch to house your changes * Get all of your commits in the new topic branch -* Submit a `pull request`_ +* Submit a `Pull Request`_ + +.. _Pull Request: https://help.github.com/articles/using-pull-requests +.. _Fork it: https://help.github.com/articles/fork-a-repo -License -------- -Copyright 2011 Bay Citizen and Texas Tribune +State of Project +---------------- +Armstrong is an open-source news platform that is freely available to any +organization. It is the result of a collaboration between the `Texas Tribune`_ +and `The Center for Investigative Reporting`_ and a grant from the +`John S. and James L. Knight Foundation`_. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +To follow development, be sure to join the `Google Group`_. - http://www.apache.org/licenses/LICENSE-2.0 +``armstrong.dev`` is part of the `Armstrong`_ project. You're +probably looking for that. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -.. _pull request: http://help.github.com/pull-requests/ -.. _Fork it: http://help.github.com/forking/ +.. _Armstrong: http://www.armstrongcms.org/ +.. _The Center for Investigative Reporting: http://cironline.org/ +.. _John S. and James L. Knight Foundation: http://www.knightfoundation.org/ +.. _Texas Tribune: http://www.texastribune.org/ +.. _Google Group: http://groups.google.com/group/armstrongcms From 820dc9b378ecaee7145f3dc9daa223a91e5d9f35 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Thu, 23 Jan 2014 12:20:46 -0800 Subject: [PATCH 22/53] Remove the anemic Sphinx command --- armstrong/dev/fabfile.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/armstrong/dev/fabfile.py b/armstrong/dev/fabfile.py index 02b9fa6..f63369c 100644 --- a/armstrong/dev/fabfile.py +++ b/armstrong/dev/fabfile.py @@ -13,7 +13,7 @@ FABRIC_TASK_MODULE = True -__all__ = ["clean", "create_migration", "docs", "pep8", "proxy", +__all__ = ["clean", "create_migration", "pep8", "proxy", "coverage", "test", "install", "remove_armstrong"] @@ -141,13 +141,6 @@ def proxy(cmd=None, *args, **kwargs): run_django_cmd(cmd, *args, **kwargs) -@task -@require_pip_module('sphinx') -def docs(): - """Generate the Sphinx docs for this project""" - local("cd docs && make html") - - @task def install(editable=True): """Install this component (or remove and reinstall)""" From fa4d1e4468260fba82e24c14eaa22a8a4ffe9d7f Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Thu, 23 Jan 2014 12:26:47 -0800 Subject: [PATCH 23/53] Default settings - use old test runner for Django 1.6. "mydatabase" is already in many `.gitignore` files so use that name. --- armstrong/dev/default_settings.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/armstrong/dev/default_settings.py b/armstrong/dev/default_settings.py index d8be608..94f5cd5 100644 --- a/armstrong/dev/default_settings.py +++ b/armstrong/dev/default_settings.py @@ -15,10 +15,14 @@ DATABASES = { "default": { "ENGINE": "django.db.backends.sqlite3", - "NAME": 'database' + "NAME": 'mydatabase' } } +import django +if django.VERSION >= (1, 6): # use the old test runner for now + TEST_RUNNER = 'django.test.simple.DjangoTestSuiteRunner' + # # A component may override settings by creating an `env_settings.py` # file in its root directory that imports from this file. From ad0efe64baf68fa1d8ef95ce56c13f407c5f0d0e Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Thu, 23 Jan 2014 11:45:28 -0800 Subject: [PATCH 24/53] slight improvement on `fudge` detection --- armstrong/dev/tests/utils/base.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/armstrong/dev/tests/utils/base.py b/armstrong/dev/tests/utils/base.py index c2669ed..9ecba2d 100644 --- a/armstrong/dev/tests/utils/base.py +++ b/armstrong/dev/tests/utils/base.py @@ -2,22 +2,19 @@ from django.db import models # DEPRECATED remove when we drop Django 1.3 support -# Backport override_settings from Django 1.4 try: from django.test.utils import override_settings except ImportError: from .backports import override_settings -# If the component uses fudge, provide useful shared behavior -try: +try: # If the component uses fudge, provide useful shared behavior import fudge - hasFudge = True except ImportError: - hasFudge = False + fudge = False class ArmstrongTestCase(DjangoTestCase): - if hasFudge: + if fudge: def setUp(self): fudge.clear_expectations() fudge.clear_calls() From d71e62c92ccb2549cd1233bea562cda6416e8419 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Sat, 25 Jan 2014 15:54:19 -0800 Subject: [PATCH 25/53] Call `super()` on ArmstrongTestCase. A good idea, but not strictly necessary as DjangoTestCase doesn't have its own `setUp()` method. --- armstrong/dev/tests/utils/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/armstrong/dev/tests/utils/base.py b/armstrong/dev/tests/utils/base.py index 9ecba2d..235fb85 100644 --- a/armstrong/dev/tests/utils/base.py +++ b/armstrong/dev/tests/utils/base.py @@ -16,6 +16,7 @@ class ArmstrongTestCase(DjangoTestCase): if fudge: def setUp(self): + super(ArmstrongTestCase, self).setUp() fudge.clear_expectations() fudge.clear_calls() From 8b900a8e37b83d89b4228762fe9602e388864377 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Sat, 25 Jan 2014 15:59:45 -0800 Subject: [PATCH 26/53] Fix and simplify `generate_random_users()`. Randomization wasn't guaranteed and I've seen it break with `IntegrityError: column username is not unique`. The new function is more generic and it's a generator. Basically the same fix as armstrong/armstrong.core.arm_layout#14. This is only used in ArmContent, ArmAccess and AppsDonations. The most extensive use is in ArmContent, which is being updated concurrently. Swapping in the new method will be simple. --- armstrong/dev/tests/utils/users.py | 53 ++++++++++++------------------ 1 file changed, 21 insertions(+), 32 deletions(-) diff --git a/armstrong/dev/tests/utils/users.py b/armstrong/dev/tests/utils/users.py index 82cc141..1b44b7e 100644 --- a/armstrong/dev/tests/utils/users.py +++ b/armstrong/dev/tests/utils/users.py @@ -1,33 +1,22 @@ -from armstrong.dev.tests.utils.base import ArmstrongTestCase -from django.contrib.auth.models import User import random - -def generate_random_user(): - r = random.randint(10000, 20000) - return User.objects.create(username="random-user-%d" % r, - first_name="Some", last_name="Random User %d" % r) - - -def generate_random_staff_users(n=2): - orig_users = generate_random_users(n) - users = User.objects.filter(pk__in=[a.id for a in orig_users]) - users.update(is_staff=True) - return [a for a in users] - - -class generate_random_staff_usersTestCase(ArmstrongTestCase): - def test_returns_2_users_by_default(self): - self.assertEqual(len(generate_random_staff_users()), 2) - - def test_returns_n_users(self): - r = random.randint(1, 5) - self.assertEqual(len(generate_random_staff_users(r)), r) - - def test_all_users_are_staff(self): - users = generate_random_staff_users() - for user in users: - self.assertTrue(user.is_staff) - - -def generate_random_users(n=2): - return [generate_random_user() for i in range(n)] \ No newline at end of file +try: + from django.contrib.auth import get_user_model +except ImportError: # Django < 1.5 + from django.contrib.auth.models import User +else: + User = get_user_model() + + +def generate_random_users(count, **extra_fields): + """Generator to create ``count`` number of unique random users""" + + num = random.randint(1000, 2000) + while count > 0: + fields = dict( + username="random-user-%d" % num, + first_name="Some", + last_name="Random User %d" % num, + **extra_fields) + yield User.objects.create(**fields) + num += random.randint(2, 20) + count -= 1 From 99d5316502201aad365ec6974cdc8f4d8347669d Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Sat, 25 Jan 2014 16:50:25 -0800 Subject: [PATCH 27/53] Remove Django magic that makes concrete models. Only used in ArmContent testing and that is being refactored concurrently. --- armstrong/dev/tests/utils/concrete.py | 55 --------------------------- 1 file changed, 55 deletions(-) delete mode 100644 armstrong/dev/tests/utils/concrete.py diff --git a/armstrong/dev/tests/utils/concrete.py b/armstrong/dev/tests/utils/concrete.py deleted file mode 100644 index 620606d..0000000 --- a/armstrong/dev/tests/utils/concrete.py +++ /dev/null @@ -1,55 +0,0 @@ -from django.db import connection -from django.core.management.color import no_style -import random - -def create_concrete_table(func=None, model=None): - style = no_style() - seen_models = connection.introspection.installed_models( - connection.introspection.table_names()) - - def actual_create(model): - sql, _references = connection.creation.sql_create_model(model, style, - seen_models) - cursor = connection.cursor() - for statement in sql: - cursor.execute(statement) - - if func: - def inner(self, *args, **kwargs): - func(self, *args, **kwargs) - actual_create(self.model) - return inner - elif model: - actual_create(model) - - -def destroy_concrete_table(func=None, model=None): - style = no_style() - # Assume that there are no references to destroy, these are supposed to be - # simple models - references = {} - - def actual_destroy(model): - sql = connection.creation.sql_destroy_model(model, references, style) - cursor = connection.cursor() - for statement in sql: - cursor.execute(statement) - - if func: - def inner(self, *args, **kwargs): - func(self, *args, **kwargs) - actual_destroy(self.model) - return inner - elif model: - actual_destroy(model) - - -# TODO: pull into a common dev package so all armstrong code can use it -def concrete(klass): - attrs = {'__module__': concrete.__module__, } - while True: - num = random.randint(1, 10000) - if num not in concrete.already_used: - break - return type("Concrete%s%d" % (klass.__name__, num), (klass, ), attrs) -concrete.already_used = [] From 6a328cb7f9604957fbe398feb379f62f8631c129 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Sun, 26 Jan 2014 11:19:52 -0800 Subject: [PATCH 28/53] Bump to 2.0 alpha version. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c39c8e0..f7e7448 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "armstrong.dev", - "version": "1.14.0alpha.0", + "version": "2.0.0alpha.0", "description": "Tools needed for development and testing of Armstrong", "install_requires": [ "South==0.7.3", From 23c923ac21b23611835a0d697b7a380ffb9a59b2 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Wed, 5 Feb 2014 17:15:50 -0800 Subject: [PATCH 29/53] Isolate loading of Django settings because we don't always need them. I don't want unnecessary errors firing off when the module is imported. I always want to make sure our `env_settings` is loaded before any other access. --- armstrong/dev/dev_django.py | 92 +++++++++++++++++++++++++++---------- armstrong/dev/fabfile.py | 8 ++-- 2 files changed, 73 insertions(+), 27 deletions(-) diff --git a/armstrong/dev/dev_django.py b/armstrong/dev/dev_django.py index 4d09ea0..eeb4deb 100644 --- a/armstrong/dev/dev_django.py +++ b/armstrong/dev/dev_django.py @@ -1,36 +1,79 @@ #!/usr/bin/env python import os import sys +import threading +from functools import wraps -try: - from django.conf import settings -except ImportError as e: - raise ImportError("%s. Check to see if Django " - "is installed in your virtualenv." % e) +__all__ = ['run_django_cmd', 'DjangoSettings'] -__all__ = ['run_django_cmd'] +class DjangoSettings(object): + """ + Isolate settings import so it doesn't happen on module import. -# Add the component's directory to the path so we can import from it -sys.path.append(os.getcwd()) - -try: - import env_settings as package_settings -except ImportError as e: - print("Could not find component specific settings file. " - "Using armstrong.dev defaults...") - try: - import armstrong.dev.default_settings as package_settings - except ImportError as e: - raise ImportError("%s. Running a Django environment for this " - "component requires either an `env_settings.py` file or the " - "armstrong.dev `default_settings.py` file." % e) + Not all of our tasks need Django so we only want to build up + our component environment's settings if and when they are needed. + This approach avoids unnecessary warnings if the settings aren't + available or Django isn't installed. + Do this as a singleton to avoid trying our imports and running + configure() over and over, but note that the object returned is + still the typical ``django.conf.LazySettings`` and so settings + remain mutable. -# Setup the Django environment -if not settings.configured: - settings.configure(default_settings=package_settings) + """ + _singleton_lock = threading.Lock() + _instance = None + + def __new__(cls, *args, **kwargs): + """Threadsafe singleton loading of settings""" + + if not cls._instance: + with cls._singleton_lock: + if not cls._instance: + cls._instance = cls.load_settings() + return cls._instance + + @staticmethod + def load_settings(): + try: + from django.conf import settings + except ImportError as e: + raise ImportError( + "%s. Check to see if Django is installed in your " + "virtualenv." % e) + + # Add the component's directory to the path so we can import from it + sys.path.append(os.getcwd()) + + try: + import env_settings as package_settings + except ImportError as e: + print( + "Could not find component specific settings file. " + "Using armstrong.dev defaults...") + try: + from . import default_settings as package_settings + except ImportError as e: + raise ImportError( + "%s. Running a Django environment for this component " + "requires either an `env_settings.py` file or " + "`armstrong.dev.default_settings.py`." % e) + + # Setup the Django environment + if not settings.configured: + settings.configure(default_settings=package_settings) + + return settings + + +def load_django_settings(func): + @wraps(func) + def wrapper(*args, **kwargs): + DjangoSettings() + return func(*args, **kwargs) + return wrapper def determine_test_args(test_labels): @@ -40,10 +83,12 @@ def determine_test_args(test_labels): explicitly provide test apps/cases/methods on the commandline. """ + settings = DjangoSettings() return test_labels or getattr(settings, 'TESTED_APPS', []) # Import access +@load_django_settings def run_django_cmd(cmd, *args, **kwargs): if cmd == "test": args = determine_test_args(args) @@ -53,6 +98,7 @@ def run_django_cmd(cmd, *args, **kwargs): # Commandline access +@load_django_settings def run_django_cli(): args = sys.argv[2:] diff --git a/armstrong/dev/fabfile.py b/armstrong/dev/fabfile.py index f63369c..4e42e9e 100644 --- a/armstrong/dev/fabfile.py +++ b/armstrong/dev/fabfile.py @@ -8,8 +8,8 @@ from fabric.colors import yellow, red from fabric.decorators import task -from armstrong.dev.dev_django import run_django_cmd +from armstrong.dev.dev_django import run_django_cmd, DjangoSettings FABRIC_TASK_MODULE = True @@ -91,10 +91,10 @@ def clean(): def create_migration(initial=False): """Create a South migration for this project""" - from django.conf import settings as django_settings - if 'south' not in (name.lower() for name in django_settings.INSTALLED_APPS): + settings = DjangoSettings() + if 'south' not in (name.lower() for name in settings.INSTALLED_APPS): print("Temporarily adding 'south' into INSTALLED_APPS.") - django_settings.INSTALLED_APPS.append('south') + settings.INSTALLED_APPS.append('south') kwargs = dict(initial=True) if literal_eval(str(initial)) else dict(auto=True) run_django_cmd('schemamigration', package['name'], **kwargs) From ef357f13ac59339b653a60491d3d5dbe6d25a3b6 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Thu, 6 Feb 2014 16:05:01 -0800 Subject: [PATCH 30/53] Capitalize "Armstrong" and change command name to something easier and more obvious. --- armstrong/dev/fabfile.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/armstrong/dev/fabfile.py b/armstrong/dev/fabfile.py index 4e42e9e..5923900 100644 --- a/armstrong/dev/fabfile.py +++ b/armstrong/dev/fabfile.py @@ -13,7 +13,8 @@ FABRIC_TASK_MODULE = True -__all__ = ["clean", "create_migration", "pep8", "proxy", +__all__ = [ + "clean", "create_migration", "pep8", "managepy", "coverage", "test", "install", "remove_armstrong"] @@ -131,11 +132,12 @@ def coverage(*args, **kwargs): @task -def proxy(cmd=None, *args, **kwargs): +def managepy(cmd=None, *args, **kwargs): """Run manage.py using this component's specific Django settings""" if cmd is None: - sys.stderr.write(red("Usage: fab proxy:,arg1,kwarg=1\n") + + sys.stderr.write( + red("Usage: fab managepy:,arg1,kwarg=1\n") + "which translates to: manage.py command arg1 --kwarg=1\n") sys.exit(1) run_django_cmd(cmd, *args, **kwargs) @@ -162,7 +164,7 @@ def install(editable=True): @task def remove_armstrong(): - """Remove all armstrong components (except for dev) from this environment""" + """Remove all Armstrong components (except for dev) from this environment""" from pip.util import get_installed_distributions pkgs = get_installed_distributions(local_only=True, include_editables=True) @@ -176,4 +178,4 @@ def remove_armstrong(): print("Note: this hasn't removed other dependencies installed by " "these components. There's no substitute for a fresh virtualenv.") else: - print("No armstrong components to remove.") + print("No Armstrong components to remove.") From e9ad4b102e74eee7e9df42c8bc9df57c78e398b5 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Sat, 8 Feb 2014 21:59:45 -0800 Subject: [PATCH 31/53] Make the CLI entry point a little more flexible. --- armstrong/dev/dev_django.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/armstrong/dev/dev_django.py b/armstrong/dev/dev_django.py index eeb4deb..d09549d 100644 --- a/armstrong/dev/dev_django.py +++ b/armstrong/dev/dev_django.py @@ -99,18 +99,19 @@ def run_django_cmd(cmd, *args, **kwargs): # Commandline access @load_django_settings -def run_django_cli(): - args = sys.argv[2:] +def run_django_cli(argv=None): + argv = argv or sys.argv + args = argv[2:] - if len(sys.argv) > 1 and sys.argv[1] == "test": + if len(argv) > 1 and argv[1] == "test": # anything not a flag (e.g. -v, --version) is treated as a named test # (it'll be a short list so iterating it twice is okay) - args = [arg for arg in sys.argv[2:] if arg.startswith('-')] - test_labels = [arg for arg in sys.argv[2:] if not arg.startswith('-')] + args = [arg for arg in argv[2:] if arg.startswith('-')] + test_labels = [arg for arg in argv[2:] if not arg.startswith('-')] args = determine_test_args(test_labels) + args from django.core.management import execute_from_command_line - execute_from_command_line(sys.argv[:2] + args) + execute_from_command_line(argv[:2] + args) if __name__ == "__main__": From 04ad98bf9bc316faa8594e15870667520b351b09 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Thu, 6 Feb 2014 16:18:39 -0800 Subject: [PATCH 32/53] WIP switching from Fabric to Invoke. Task arguments are currently broken as Invoke parses them differently and cannot handle arbitrary args/kwargs. --- README.rst | 29 ++++++++++----------- armstrong/dev/{fabfile.py => tasks.py} | 36 +++++++++++--------------- package.json | 2 +- 3 files changed, 30 insertions(+), 37 deletions(-) rename armstrong/dev/{fabfile.py => tasks.py} (79%) diff --git a/README.rst b/README.rst index 879b446..d0945be 100644 --- a/README.rst +++ b/README.rst @@ -14,16 +14,16 @@ Installation Usage ----- Most Armstrong components already have the necessary configuration to use these -Dev tools. Type ``fab -l`` to see a list of all of the commands. +Dev tools. Type ``invoke --list`` to see a list of all of the commands. If you are creating a new component (or perhaps updating one that uses the older, pre 2.0 Dev tools), you'll need these next two steps. -1. Create a ``fabfile.py`` and add the following:: +1. Create a ``tasks.py`` and add the following:: - from armstrong.dev.fabfile import * + from armstrong.dev.tasks import * - # any additional Fabric commands + # any additional Invoke commands # ... 2. Create an ``env_settings.py`` and add the following:: @@ -40,31 +40,30 @@ This version offers an easier and more standard way to run a Django environment with a component's specific settings, either from the commandline or via import. -It provides an "a la carte" requirements approach. Meaning that if you run a -Fabric command that needs a package that isn't installed, it will prompt you +It provides an "a la carte" requirements approach. Meaning that if you run an +Invoke command that needs a package that isn't installed, it will prompt you to install it instead of requiring everything up-front. This allows for much faster virtualenv creation (which saves considerable time in testing) and doesn't pollute your virtualenv with packages for features you don't use. ``test`` and ``coverage`` will work better with automated test tools like -TravisCI and Tox. These commands also now work like Django's native test -command so that you can pass arguments for running selective tests, i.e:: +TravisCI and Tox.:: - fab test [[.[.]]] + invoke test Settings are now defined in the normal Django style in an ``env_settings.py`` -file instead of as a dict within the Fabric config. It's not called -"settings.py" to make it clearer that these are settings for the development -and testing of this component, not necessarily values to copy/paste for -incorporating the component into other projects. +file instead of as a dict within the tasks file. It's not called "settings.py" +to make it clearer that these are settings for the development and testing +of this component, not necessarily values to copy/paste for incorporating +the component into other projects. Backward incompatible changes in 2.0 ------------------------------------ * ``env_settings.py`` is necessary and contains the settings that - ``fabfile.py`` used to have. + ``tasks.py`` used to have. -* ``fabfile.py`` imports from a different place and no longer defines the +* ``tasks.py`` imports from a different place and no longer defines the settings configurations. Not required but as long as you are reviewing the general state of things, diff --git a/armstrong/dev/fabfile.py b/armstrong/dev/tasks.py similarity index 79% rename from armstrong/dev/fabfile.py rename to armstrong/dev/tasks.py index 5923900..1d42234 100644 --- a/armstrong/dev/fabfile.py +++ b/armstrong/dev/tasks.py @@ -1,17 +1,12 @@ import sys from os.path import dirname -from ast import literal_eval from functools import wraps from contextlib import contextmanager -from fabric.api import local, settings -from fabric.colors import yellow, red -from fabric.decorators import task - +from invoke import task, run from armstrong.dev.dev_django import run_django_cmd, DjangoSettings -FABRIC_TASK_MODULE = True __all__ = [ "clean", "create_migration", "pep8", "managepy", @@ -33,8 +28,8 @@ def wrapper(*args, **kwargs): __import__(package['name']) except ImportError: sys.stderr.write( - red("This component needs to be installed first. Run ") + - yellow("`fab install`\n")) + "This component needs to be installed first. Run " + + "`invoke install`\n") sys.exit(1) return func(*args, **kwargs) return wrapper @@ -51,7 +46,7 @@ def wrapper(*args, **kwargs): __import__(module) except ImportError: sys.stderr.write( - yellow("`pip install %s` to enable this feature\n" % module)) + "`pip install %s` to enable this feature\n" % module) sys.exit(1) else: return func(*args, **kwargs) @@ -75,7 +70,7 @@ def html_coverage_report(report_directory=None): # Write results report_directory = report_directory or "coverage" - local('rm -rf ' + report_directory) + run('rm -rf ' + report_directory) cov.html_report(directory=report_directory) print("Coverage reports available in: %s " % report_directory) @@ -83,7 +78,7 @@ def html_coverage_report(report_directory=None): @task def clean(): """Find and remove all .pyc and .pyo files""" - local('find . -name "*.py[co]" -exec rm {} \;') + run('find . -name "*.py[co]" -exec rm {} \;') @task @@ -97,7 +92,7 @@ def create_migration(initial=False): print("Temporarily adding 'south' into INSTALLED_APPS.") settings.INSTALLED_APPS.append('south') - kwargs = dict(initial=True) if literal_eval(str(initial)) else dict(auto=True) + kwargs = dict(initial=True) if initial else dict(auto=True) run_django_cmd('schemamigration', package['name'], **kwargs) @@ -105,7 +100,7 @@ def create_migration(initial=False): @require_pip_module('pep8') def pep8(): """Run pep8 on all .py files in ./armstrong""" - local('find ./armstrong -name "*.py" | xargs pep8 --repeat', capture=False) + run('find ./armstrong -name "*.py" | xargs pep8 --repeat') @task @@ -137,7 +132,7 @@ def managepy(cmd=None, *args, **kwargs): if cmd is None: sys.stderr.write( - red("Usage: fab managepy:,arg1,kwarg=1\n") + + "Usage: fab managepy:,arg1,kwarg=1\n" + "which translates to: manage.py command arg1 --kwarg=1\n") sys.exit(1) run_django_cmd(cmd, *args, **kwargs) @@ -152,14 +147,12 @@ def install(editable=True): except ImportError: pass else: - with settings(warn_only=True): - local("pip uninstall --quiet -y %s" % package['name'], capture=False) + run("pip uninstall --quiet -y %s" % package['name'], warn=True) cmd = "pip install --quiet " - cmd += "-e ." if literal_eval(str(editable)) else "." + cmd += "-e ." if editable else "." - with settings(warn_only=True): - local(cmd, capture=False) + run(cmd, warn=True) @task @@ -172,10 +165,11 @@ def remove_armstrong(): if pkg.key.startswith('armstrong') and pkg.key != 'armstrong.dev'] for app in apps: - local("pip uninstall -y %s" % app.key) + run("pip uninstall -y %s" % app.key) if apps: - print("Note: this hasn't removed other dependencies installed by " + print( + "Note: this hasn't removed other dependencies installed by " "these components. There's no substitute for a fresh virtualenv.") else: print("No Armstrong components to remove.") diff --git a/package.json b/package.json index 487762e..9417a51 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,6 @@ "version": "2.0.0alpha.0", "description": "Tools needed for development and testing of Armstrong", "install_requires": [ - "Fabric<2.0", + "invoke<1.0" ] } From ac18e67a36c0a012827746b25441da24718ae88c Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Thu, 6 Feb 2014 16:54:17 -0800 Subject: [PATCH 33/53] Use Decorator to create our own decorators to allow the "real" function signature and argspec to bubble up to @task. Invoke needs this "real" information to create the CLI arg options. Invoke cannot parse arbitrary args/kwargs so we can't do it the easy way like Fabric 1.x allows. --- armstrong/dev/tasks.py | 52 +++++++++++++++++++----------------------- package.json | 3 ++- 2 files changed, 26 insertions(+), 29 deletions(-) diff --git a/armstrong/dev/tasks.py b/armstrong/dev/tasks.py index 1d42234..5db6e15 100644 --- a/armstrong/dev/tasks.py +++ b/armstrong/dev/tasks.py @@ -1,10 +1,13 @@ import sys from os.path import dirname -from functools import wraps from contextlib import contextmanager from invoke import task, run +# Decorator keeps the function signature and argspec intact, which we +# need so @task can build out CLI arguments properly +from decorator import decorator + from armstrong.dev.dev_django import run_django_cmd, DjangoSettings @@ -18,40 +21,33 @@ package = json.load(open("./package.json")) -def require_self(func=None): +@decorator +def require_self(func, *args, **kwargs): """Decorator to require that this component be installed""" - def decorator(func): - @wraps(func) - def wrapper(*args, **kwargs): - try: - __import__(package['name']) - except ImportError: - sys.stderr.write( - "This component needs to be installed first. Run " + - "`invoke install`\n") - sys.exit(1) - return func(*args, **kwargs) - return wrapper - return decorator if not func else decorator(func) + try: + __import__(package['name']) + except ImportError: + sys.stderr.write( + "This component needs to be installed first. Run " + + "`invoke install`\n") + sys.exit(1) + return func(*args, **kwargs) def require_pip_module(module): """Decorator to check for a module and helpfully exit if it's not found""" - def decorator(func): - @wraps(func) - def wrapper(*args, **kwargs): - try: - __import__(module) - except ImportError: - sys.stderr.write( - "`pip install %s` to enable this feature\n" % module) - sys.exit(1) - else: - return func(*args, **kwargs) - return wrapper - return decorator + def wrapper(func, *args, **kwargs): + try: + __import__(module) + except ImportError: + sys.stderr.write( + "`pip install %s` to enable this feature\n" % module) + sys.exit(1) + else: + return func(*args, **kwargs) + return decorator(wrapper) @contextmanager diff --git a/package.json b/package.json index 9417a51..fa9e5ba 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,7 @@ "version": "2.0.0alpha.0", "description": "Tools needed for development and testing of Armstrong", "install_requires": [ - "invoke<1.0" + "invoke<1.0", + "decorator<4.0" ] } From c4f40f7ab690de21732b51767a5773709f7dc0b2 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Sat, 8 Feb 2014 22:13:15 -0800 Subject: [PATCH 34/53] Handle Invoke task args. Fabric could handle arbitrary args/kwargs, but Invoke can't. --- README.rst | 11 +++++++++-- armstrong/dev/tasks.py | 21 +++++++-------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/README.rst b/README.rst index d0945be..a37909e 100644 --- a/README.rst +++ b/README.rst @@ -47,9 +47,16 @@ faster virtualenv creation (which saves considerable time in testing) and doesn't pollute your virtualenv with packages for features you don't use. ``test`` and ``coverage`` will work better with automated test tools like -TravisCI and Tox.:: +TravisCI and Tox. These commands also now work like Django's native test +command so that you can pass arguments for running selective tests. Due to +Invoke's argument passing, we need to include any optional args via an +``--extra`` param, i.e.:: - invoke test + invoke test --extra [[.[.]]] + + # enclose multiple args in quotes + # kwargs need to use "=" with no spaces (our limitation, not Invoke's) + invoke test --extra "--verbosity=2 [[.[.]]]" Settings are now defined in the normal Django style in an ``env_settings.py`` file instead of as a dict within the tasks file. It's not called "settings.py" diff --git a/armstrong/dev/tasks.py b/armstrong/dev/tasks.py index 5db6e15..23bb69e 100644 --- a/armstrong/dev/tasks.py +++ b/armstrong/dev/tasks.py @@ -101,37 +101,30 @@ def pep8(): @task @require_self -def test(*args, **kwargs): +def test(extra=None): """Test this component via `manage.py test`""" - run_django_cmd('test', *args, **kwargs) + return managepy('test', extra) @task @require_self @require_pip_module('coverage') -def coverage(*args, **kwargs): +def coverage(coverage_dir=None, extra=None): """Test this project with coverage reports""" - # Option to pass in the coverage report directory - coverage_dir = kwargs.pop('coverage_dir', None) - try: with html_coverage_report(coverage_dir): - run_django_cmd('test', *args, **kwargs) + return test(extra) except (ImportError, EnvironmentError): sys.exit(1) @task -def managepy(cmd=None, *args, **kwargs): +def managepy(cmd, extra=None): """Run manage.py using this component's specific Django settings""" - if cmd is None: - sys.stderr.write( - "Usage: fab managepy:,arg1,kwarg=1\n" + - "which translates to: manage.py command arg1 --kwarg=1\n") - sys.exit(1) - run_django_cmd(cmd, *args, **kwargs) + extra = extra.split() if extra else [] + run_django_cmd(cmd, *extra) @task From c9936b7c605c29b32c8f5684a6a30202840eb334 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Sat, 8 Feb 2014 23:37:35 -0800 Subject: [PATCH 35/53] Fix `--extra` handling for kwargs. Now things like `--extra "test1 test2 --verbosity=0"` work. --- armstrong/dev/dev_django.py | 2 +- armstrong/dev/tasks.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/armstrong/dev/dev_django.py b/armstrong/dev/dev_django.py index d09549d..ae6219b 100644 --- a/armstrong/dev/dev_django.py +++ b/armstrong/dev/dev_django.py @@ -5,7 +5,7 @@ from functools import wraps -__all__ = ['run_django_cmd', 'DjangoSettings'] +__all__ = ['run_django_cmd', 'run_django_cli', 'DjangoSettings'] class DjangoSettings(object): diff --git a/armstrong/dev/tasks.py b/armstrong/dev/tasks.py index 23bb69e..eb907ae 100644 --- a/armstrong/dev/tasks.py +++ b/armstrong/dev/tasks.py @@ -8,7 +8,7 @@ # need so @task can build out CLI arguments properly from decorator import decorator -from armstrong.dev.dev_django import run_django_cmd, DjangoSettings +from armstrong.dev.dev_django import run_django_cmd, run_django_cli, DjangoSettings __all__ = [ @@ -124,7 +124,7 @@ def managepy(cmd, extra=None): """Run manage.py using this component's specific Django settings""" extra = extra.split() if extra else [] - run_django_cmd(cmd, *extra) + run_django_cli(['invoke', cmd] + extra) @task From c4d1cf22fa43ef0faa74b1e35d2d6b38e95a5a8a Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Sun, 9 Feb 2014 14:42:57 -0800 Subject: [PATCH 36/53] help test for Invoke tasks --- armstrong/dev/tasks.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/armstrong/dev/tasks.py b/armstrong/dev/tasks.py index eb907ae..033a812 100644 --- a/armstrong/dev/tasks.py +++ b/armstrong/dev/tasks.py @@ -21,6 +21,12 @@ package = json.load(open("./package.json")) +HELP_TEXT_MANAGEPY = 'any command that `manage.py` normally takes, including "help"' +HELP_TEXT_EXTRA = 'include any arguments this method can normally take. ' \ + 'multiple args need quotes, e.g. --extra "test1 test2 --verbosity=2"' +HELP_TEXT_REPORTS = 'directory to store coverage reports, default: "coverage"' + + @decorator def require_self(func, *args, **kwargs): """Decorator to require that this component be installed""" @@ -99,27 +105,27 @@ def pep8(): run('find ./armstrong -name "*.py" | xargs pep8 --repeat') -@task +@task(help=dict(extra=HELP_TEXT_EXTRA)) @require_self def test(extra=None): """Test this component via `manage.py test`""" return managepy('test', extra) -@task +@task(help=dict(reportdir=HELP_TEXT_REPORTS, extra=HELP_TEXT_EXTRA)) @require_self @require_pip_module('coverage') -def coverage(coverage_dir=None, extra=None): +def coverage(reportdir=None, extra=None): """Test this project with coverage reports""" try: - with html_coverage_report(coverage_dir): + with html_coverage_report(reportdir): return test(extra) except (ImportError, EnvironmentError): sys.exit(1) -@task +@task(help=dict(cmd=HELP_TEXT_MANAGEPY, extra=HELP_TEXT_EXTRA)) def managepy(cmd, extra=None): """Run manage.py using this component's specific Django settings""" @@ -146,7 +152,7 @@ def install(editable=True): @task def remove_armstrong(): - """Remove all Armstrong components (except for dev) from this environment""" + """Remove all Armstrong components (except for Dev) from this environment""" from pip.util import get_installed_distributions pkgs = get_installed_distributions(local_only=True, include_editables=True) From 64abcf2d537be1fe6a2f674c10ead16e2c365697 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Sun, 9 Feb 2014 18:07:53 -0800 Subject: [PATCH 37/53] Django 1.6 uses the native 1.6 test runner. --- README.rst | 3 +++ armstrong/dev/default_settings.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index a37909e..609d054 100644 --- a/README.rst +++ b/README.rst @@ -54,6 +54,9 @@ Invoke's argument passing, we need to include any optional args via an invoke test --extra [[.[.]]] + # or in Django 1.6 + invoke test --extra [[..[.]]] + # enclose multiple args in quotes # kwargs need to use "=" with no spaces (our limitation, not Invoke's) invoke test --extra "--verbosity=2 [[.[.]]]" diff --git a/armstrong/dev/default_settings.py b/armstrong/dev/default_settings.py index 94f5cd5..4045573 100644 --- a/armstrong/dev/default_settings.py +++ b/armstrong/dev/default_settings.py @@ -21,7 +21,7 @@ import django if django.VERSION >= (1, 6): # use the old test runner for now - TEST_RUNNER = 'django.test.simple.DjangoTestSuiteRunner' + TESTED_APPS = ["%s.tests" % package['name']] # # A component may override settings by creating an `env_settings.py` From e8670e5b373f370c660a5f1b95d730540b2deec1 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Sun, 9 Feb 2014 18:27:55 -0800 Subject: [PATCH 38/53] Override and use the Django 1.6 test runner and backport for older Djangos. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new Django test runner "will discover tests in any file named “test*.py” under the current working directory". Previously the "test runner only discovered tests in tests.py and models.py files within a Python package listed in INSTALLED_APPS". This means we can drop `TESTED_APPS` because the runner no longer tests everything in INSTALLED_APPS. That's much easier and worth backporting for just to simplify our code. It also standardizes on the new way so you don't have to remember which Django version is installed when testing your Armstrong component. Finally this opens the door to moving the tests directory and not shipping it. (Well, it's one easy way to make that change.) But we don't use "test*.py" files, we use a `tests/` package. Overriding the runner with this one line change seemed easiest. Easier than passing in a "pattern" kwarg or injecting "--pattern" into our pseudo CLI command launcher. --- README.rst | 5 +- armstrong/dev/default_settings.py | 6 +- armstrong/dev/dev_django.py | 24 +-- armstrong/dev/tests/runner.py | 12 ++ armstrong/dev/tests/utils/runner.py | 293 ++++++++++++++++++++++++++++ 5 files changed, 308 insertions(+), 32 deletions(-) create mode 100644 armstrong/dev/tests/runner.py create mode 100644 armstrong/dev/tests/utils/runner.py diff --git a/README.rst b/README.rst index 609d054..dfea268 100644 --- a/README.rst +++ b/README.rst @@ -52,14 +52,11 @@ command so that you can pass arguments for running selective tests. Due to Invoke's argument passing, we need to include any optional args via an ``--extra`` param, i.e.:: - invoke test --extra [[.[.]]] - - # or in Django 1.6 invoke test --extra [[..[.]]] # enclose multiple args in quotes # kwargs need to use "=" with no spaces (our limitation, not Invoke's) - invoke test --extra "--verbosity=2 [[.[.]]]" + invoke test --extra "--verbosity=2 " Settings are now defined in the normal Django style in an ``env_settings.py`` file instead of as a dict within the tasks file. It's not called "settings.py" diff --git a/armstrong/dev/default_settings.py b/armstrong/dev/default_settings.py index 4045573..35b30f0 100644 --- a/armstrong/dev/default_settings.py +++ b/armstrong/dev/default_settings.py @@ -10,7 +10,6 @@ # # Default settings # -TESTED_APPS = [app_name] INSTALLED_APPS = [package['name']] DATABASES = { "default": { @@ -18,10 +17,7 @@ "NAME": 'mydatabase' } } - -import django -if django.VERSION >= (1, 6): # use the old test runner for now - TESTED_APPS = ["%s.tests" % package['name']] +TEST_RUNNER = "armstrong.dev.tests.runner.ArmstrongDiscoverRunner" # # A component may override settings by creating an `env_settings.py` diff --git a/armstrong/dev/dev_django.py b/armstrong/dev/dev_django.py index ae6219b..526f1e1 100644 --- a/armstrong/dev/dev_django.py +++ b/armstrong/dev/dev_django.py @@ -76,23 +76,9 @@ def wrapper(*args, **kwargs): return wrapper -def determine_test_args(test_labels): - """ - Limit testing to settings.TESTED_APPS if available while behaving - exactly like `manage.py test` and retaining Django's ability to - explicitly provide test apps/cases/methods on the commandline. - - """ - settings = DjangoSettings() - return test_labels or getattr(settings, 'TESTED_APPS', []) - - # Import access @load_django_settings def run_django_cmd(cmd, *args, **kwargs): - if cmd == "test": - args = determine_test_args(args) - from django.core.management import call_command return call_command(cmd, *args, **kwargs) @@ -101,17 +87,9 @@ def run_django_cmd(cmd, *args, **kwargs): @load_django_settings def run_django_cli(argv=None): argv = argv or sys.argv - args = argv[2:] - - if len(argv) > 1 and argv[1] == "test": - # anything not a flag (e.g. -v, --version) is treated as a named test - # (it'll be a short list so iterating it twice is okay) - args = [arg for arg in argv[2:] if arg.startswith('-')] - test_labels = [arg for arg in argv[2:] if not arg.startswith('-')] - args = determine_test_args(test_labels) + args from django.core.management import execute_from_command_line - execute_from_command_line(argv[:2] + args) + execute_from_command_line(argv) if __name__ == "__main__": diff --git a/armstrong/dev/tests/runner.py b/armstrong/dev/tests/runner.py new file mode 100644 index 0000000..046d25d --- /dev/null +++ b/armstrong/dev/tests/runner.py @@ -0,0 +1,12 @@ +try: + from django.test.runner import DiscoverRunner +except ImportError: # < Django 1.6 + from .utils.runner import DiscoverRunner + + +class ArmstrongDiscoverRunner(DiscoverRunner): + def __init__(self, *args, **kwargs): + """Find our "tests" package, not just "test*.py" files""" + + super(ArmstrongDiscoverRunner, self).__init__(*args, **kwargs) + self.pattern = "test*" diff --git a/armstrong/dev/tests/utils/runner.py b/armstrong/dev/tests/utils/runner.py new file mode 100644 index 0000000..4821e1a --- /dev/null +++ b/armstrong/dev/tests/utils/runner.py @@ -0,0 +1,293 @@ +# DEPRECATED backport for Django < 1.6 + +import os +from optparse import make_option + +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured +from django.test import TestCase +from django.test.utils import setup_test_environment, teardown_test_environment +from django.utils import unittest +from django.utils.unittest import TestSuite, defaultTestLoader + + +class DiscoverRunner(object): + """ + A Django test runner that uses unittest2 test discovery. + """ + + test_loader = defaultTestLoader + reorder_by = (TestCase, ) + option_list = ( + make_option('-t', '--top-level-directory', + action='store', dest='top_level', default=None, + help='Top level of project for unittest discovery.'), + make_option('-p', '--pattern', action='store', dest='pattern', + default="test*.py", + help='The test matching pattern. Defaults to test*.py.'), + ) + + def __init__(self, pattern=None, top_level=None, + verbosity=1, interactive=True, failfast=False, + **kwargs): + + self.pattern = pattern + self.top_level = top_level + + self.verbosity = verbosity + self.interactive = interactive + self.failfast = failfast + + def setup_test_environment(self, **kwargs): + setup_test_environment() + settings.DEBUG = False + unittest.installHandler() + + def build_suite(self, test_labels=None, extra_tests=None, **kwargs): + suite = TestSuite() + test_labels = test_labels or ['.'] + extra_tests = extra_tests or [] + + discover_kwargs = {} + if self.pattern is not None: + discover_kwargs['pattern'] = self.pattern + if self.top_level is not None: + discover_kwargs['top_level_dir'] = self.top_level + + for label in test_labels: + kwargs = discover_kwargs.copy() + tests = None + + label_as_path = os.path.abspath(label) + + # if a module, or "module.ClassName[.method_name]", just run those + if not os.path.exists(label_as_path): + tests = self.test_loader.loadTestsFromName(label) + elif os.path.isdir(label_as_path) and not self.top_level: + # Try to be a bit smarter than unittest about finding the + # default top-level for a given directory path, to avoid + # breaking relative imports. (Unittest's default is to set + # top-level equal to the path, which means relative imports + # will result in "Attempted relative import in non-package."). + + # We'd be happy to skip this and require dotted module paths + # (which don't cause this problem) instead of file paths (which + # do), but in the case of a directory in the cwd, which would + # be equally valid if considered as a top-level module or as a + # directory path, unittest unfortunately prefers the latter. + + top_level = label_as_path + while True: + init_py = os.path.join(top_level, '__init__.py') + if os.path.exists(init_py): + try_next = os.path.dirname(top_level) + if try_next == top_level: + # __init__.py all the way down? give up. + break + top_level = try_next + continue + break + kwargs['top_level_dir'] = top_level + + + if not (tests and tests.countTestCases()): + # if no tests found, it's probably a package; try discovery + tests = self.test_loader.discover(start_dir=label, **kwargs) + + # make unittest forget the top-level dir it calculated from this + # run, to support running tests from two different top-levels. + self.test_loader._top_level_dir = None + + suite.addTests(tests) + + for test in extra_tests: + suite.addTest(test) + + return reorder_suite(suite, self.reorder_by) + + def setup_databases(self, **kwargs): + return setup_databases(self.verbosity, self.interactive, **kwargs) + + def run_suite(self, suite, **kwargs): + return unittest.TextTestRunner( + verbosity=self.verbosity, + failfast=self.failfast, + ).run(suite) + + def teardown_databases(self, old_config, **kwargs): + """ + Destroys all the non-mirror databases. + """ + old_names, mirrors = old_config + for connection, old_name, destroy in old_names: + if destroy: + connection.creation.destroy_test_db(old_name, self.verbosity) + + def teardown_test_environment(self, **kwargs): + unittest.removeHandler() + teardown_test_environment() + + def suite_result(self, suite, result, **kwargs): + return len(result.failures) + len(result.errors) + + def run_tests(self, test_labels, extra_tests=None, **kwargs): + """ + Run the unit tests for all the test labels in the provided list. + + Test labels should be dotted Python paths to test modules, test + classes, or test methods. + + A list of 'extra' tests may also be provided; these tests + will be added to the test suite. + + Returns the number of tests that failed. + """ + self.setup_test_environment() + suite = self.build_suite(test_labels, extra_tests) + old_config = self.setup_databases() + result = self.run_suite(suite) + self.teardown_databases(old_config) + self.teardown_test_environment() + return self.suite_result(suite, result) + + +def dependency_ordered(test_databases, dependencies): + """ + Reorder test_databases into an order that honors the dependencies + described in TEST_DEPENDENCIES. + """ + ordered_test_databases = [] + resolved_databases = set() + + # Maps db signature to dependencies of all it's aliases + dependencies_map = {} + + # sanity check - no DB can depend on it's own alias + for sig, (_, aliases) in test_databases: + all_deps = set() + for alias in aliases: + all_deps.update(dependencies.get(alias, [])) + if not all_deps.isdisjoint(aliases): + raise ImproperlyConfigured( + "Circular dependency: databases %r depend on each other, " + "but are aliases." % aliases) + dependencies_map[sig] = all_deps + + while test_databases: + changed = False + deferred = [] + + # Try to find a DB that has all it's dependencies met + for signature, (db_name, aliases) in test_databases: + if dependencies_map[signature].issubset(resolved_databases): + resolved_databases.update(aliases) + ordered_test_databases.append((signature, (db_name, aliases))) + changed = True + else: + deferred.append((signature, (db_name, aliases))) + + if not changed: + raise ImproperlyConfigured( + "Circular dependency in TEST_DEPENDENCIES") + test_databases = deferred + return ordered_test_databases + + +def reorder_suite(suite, classes): + """ + Reorders a test suite by test type. + + `classes` is a sequence of types + + All tests of type classes[0] are placed first, then tests of type + classes[1], etc. Tests with no match in classes are placed last. + """ + class_count = len(classes) + bins = [unittest.TestSuite() for i in range(class_count+1)] + partition_suite(suite, classes, bins) + for i in range(class_count): + bins[0].addTests(bins[i+1]) + return bins[0] + + +def partition_suite(suite, classes, bins): + """ + Partitions a test suite by test type. + + classes is a sequence of types + bins is a sequence of TestSuites, one more than classes + + Tests of type classes[i] are added to bins[i], + tests with no match found in classes are place in bins[-1] + """ + for test in suite: + if isinstance(test, unittest.TestSuite): + partition_suite(test, classes, bins) + else: + for i in range(len(classes)): + if isinstance(test, classes[i]): + bins[i].addTest(test) + break + else: + bins[-1].addTest(test) + + +def setup_databases(verbosity, interactive, **kwargs): + from django.db import connections, DEFAULT_DB_ALIAS + + # First pass -- work out which databases actually need to be created, + # and which ones are test mirrors or duplicate entries in DATABASES + mirrored_aliases = {} + test_databases = {} + dependencies = {} + default_sig = connections[DEFAULT_DB_ALIAS].creation.test_db_signature() + for alias in connections: + connection = connections[alias] + if connection.settings_dict['TEST_MIRROR']: + # If the database is marked as a test mirror, save + # the alias. + mirrored_aliases[alias] = ( + connection.settings_dict['TEST_MIRROR']) + else: + # Store a tuple with DB parameters that uniquely identify it. + # If we have two aliases with the same values for that tuple, + # we only need to create the test database once. + item = test_databases.setdefault( + connection.creation.test_db_signature(), + (connection.settings_dict['NAME'], set()) + ) + item[1].add(alias) + + if 'TEST_DEPENDENCIES' in connection.settings_dict: + dependencies[alias] = ( + connection.settings_dict['TEST_DEPENDENCIES']) + else: + if alias != DEFAULT_DB_ALIAS and connection.creation.test_db_signature() != default_sig: + dependencies[alias] = connection.settings_dict.get( + 'TEST_DEPENDENCIES', [DEFAULT_DB_ALIAS]) + + # Second pass -- actually create the databases. + old_names = [] + mirrors = [] + + for signature, (db_name, aliases) in dependency_ordered( + test_databases.items(), dependencies): + test_db_name = None + # Actually create the database for the first connection + for alias in aliases: + connection = connections[alias] + if test_db_name is None: + test_db_name = connection.creation.create_test_db( + verbosity, autoclobber=not interactive) + destroy = True + else: + connection.settings_dict['NAME'] = test_db_name + destroy = False + old_names.append((connection, db_name, destroy)) + + for alias, mirror_alias in mirrored_aliases.items(): + mirrors.append((alias, connections[alias].settings_dict['NAME'])) + connections[alias].settings_dict['NAME'] = ( + connections[mirror_alias].settings_dict['NAME']) + + return old_names, mirrors From 2d53bb6a0e76b38349f610b25f3859ecef73030f Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Mon, 10 Feb 2014 16:04:02 -0800 Subject: [PATCH 39/53] Relative package import. --- armstrong/dev/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/armstrong/dev/tasks.py b/armstrong/dev/tasks.py index 033a812..0be8e94 100644 --- a/armstrong/dev/tasks.py +++ b/armstrong/dev/tasks.py @@ -8,7 +8,7 @@ # need so @task can build out CLI arguments properly from decorator import decorator -from armstrong.dev.dev_django import run_django_cmd, run_django_cli, DjangoSettings +from .dev_django import run_django_cmd, run_django_cli, DjangoSettings __all__ = [ From a414835e4c4f08e323873470749f1ed8fadd5b32 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Mon, 10 Feb 2014 16:14:28 -0800 Subject: [PATCH 40/53] Packages - make Invoke optional. Invoke isn't *really* optional for development but it *is* for testing. So in favor of even faster test virtualenv building, don't install Invoke. Tests can be run with `python -m armstrong.dev.dev_django test` (instead of `invoke test`). The readme will be updated in a separate commit with a few different things, but in essence a dev environment will need `pip install tox invoke -r requirements\dev.txt`. --- armstrong/dev/tasks.py | 6 +++++- package.json | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/armstrong/dev/tasks.py b/armstrong/dev/tasks.py index 0be8e94..2d4458e 100644 --- a/armstrong/dev/tasks.py +++ b/armstrong/dev/tasks.py @@ -2,7 +2,11 @@ from os.path import dirname from contextlib import contextmanager -from invoke import task, run +try: + from invoke import task, run +except ImportError: + sys.stderr.write("Tasks require Invoke: `pip install invoke`\n") + sys.exit(1) # Decorator keeps the function signature and argspec intact, which we # need so @task can build out CLI arguments properly diff --git a/package.json b/package.json index fa9e5ba..40e3974 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,6 @@ "version": "2.0.0alpha.0", "description": "Tools needed for development and testing of Armstrong", "install_requires": [ - "invoke<1.0", "decorator<4.0" ] } From 4df0c5321282c582edfe5675796120a21b58dd7c Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Tue, 18 Feb 2014 18:13:03 -0800 Subject: [PATCH 41/53] Using pkgutil is fine. Remove improper namespacing. Only `armstrong` is a namespace package; `dev` is a normal package. Also, it's not correct to declare other names in an `__init__.py` file that declares namespacing. --- armstrong/__init__.py | 4 ++-- armstrong/dev/__init__.py | 2 -- armstrong/dev/tasks/__init__.py | 4 ---- armstrong/dev/tests/__init__.py | 2 -- armstrong/dev/tests/utils/__init__.py | 6 +----- armstrong/dev/virtualdjango/__init__.py | 2 -- 6 files changed, 3 insertions(+), 17 deletions(-) diff --git a/armstrong/__init__.py b/armstrong/__init__.py index ece379c..3ad9513 100644 --- a/armstrong/__init__.py +++ b/armstrong/__init__.py @@ -1,2 +1,2 @@ -import pkg_resources -pkg_resources.declare_namespace(__name__) +from pkgutil import extend_path +__path__ = extend_path(__path__, __name__) diff --git a/armstrong/dev/__init__.py b/armstrong/dev/__init__.py index ece379c..e69de29 100644 --- a/armstrong/dev/__init__.py +++ b/armstrong/dev/__init__.py @@ -1,2 +0,0 @@ -import pkg_resources -pkg_resources.declare_namespace(__name__) diff --git a/armstrong/dev/tasks/__init__.py b/armstrong/dev/tasks/__init__.py index b5b607d..1c05156 100644 --- a/armstrong/dev/tasks/__init__.py +++ b/armstrong/dev/tasks/__init__.py @@ -1,7 +1,3 @@ -import pkg_resources -pkg_resources.declare_namespace(__name__) - - from contextlib import contextmanager try: import coverage as coverage diff --git a/armstrong/dev/tests/__init__.py b/armstrong/dev/tests/__init__.py index ece379c..e69de29 100644 --- a/armstrong/dev/tests/__init__.py +++ b/armstrong/dev/tests/__init__.py @@ -1,2 +0,0 @@ -import pkg_resources -pkg_resources.declare_namespace(__name__) diff --git a/armstrong/dev/tests/utils/__init__.py b/armstrong/dev/tests/utils/__init__.py index 1e73219..aebf5bc 100644 --- a/armstrong/dev/tests/utils/__init__.py +++ b/armstrong/dev/tests/utils/__init__.py @@ -1,5 +1 @@ -import pkg_resources -pkg_resources.declare_namespace(__name__) - - -from armstrong.dev.tests.utils.base import ArmstrongTestCase, override_settings +from .base import ArmstrongTestCase, override_settings diff --git a/armstrong/dev/virtualdjango/__init__.py b/armstrong/dev/virtualdjango/__init__.py index ece379c..e69de29 100644 --- a/armstrong/dev/virtualdjango/__init__.py +++ b/armstrong/dev/virtualdjango/__init__.py @@ -1,2 +0,0 @@ -import pkg_resources -pkg_resources.declare_namespace(__name__) From 499d010991af842c7d5078584afa0515c28aa365 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Tue, 18 Feb 2014 18:50:54 -0800 Subject: [PATCH 42/53] Put this back. Partial revert of 56f7cc8a768568e9baaa910558150abb6c138b7d. --- setup.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7660c79..2c63da3 100644 --- a/setup.py +++ b/setup.py @@ -3,8 +3,9 @@ Nothing in this file should need to be edited, please see accompanying package.json file if you need to adjust metadata about this package. -""" +""" +import os import json from setuptools import setup, find_packages @@ -21,6 +22,8 @@ def generate_namespaces(package): NAMESPACE_PACKAGES.append(new_package) generate_namespaces(info["name"]) +if os.path.exists("MANIFEST"): + os.unlink("MANIFEST") setup_kwargs = { "author": "Bay Citizen & Texas Tribune", From 473ab0ad0829455efd7b0fa186993ea3fb684f18 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Tue, 18 Feb 2014 19:06:47 -0800 Subject: [PATCH 43/53] Simplify `namespace_packages` creation and use `pkg_resources`. I was wrong in 4df0c5321282c582edfe5675796120a21b58dd7c, setuptools wants its own method and there's no point fighting against it. Combining namespace_packages and the pkgutil method causes this error: `WARNING: armstrong is a namespace package, but its __init__.py does not declare_namespace(); setuptools 0.7 will REQUIRE this!` --- armstrong/__init__.py | 3 +-- setup.py | 12 +++++------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/armstrong/__init__.py b/armstrong/__init__.py index 3ad9513..de40ea7 100644 --- a/armstrong/__init__.py +++ b/armstrong/__init__.py @@ -1,2 +1 @@ -from pkgutil import extend_path -__path__ = extend_path(__path__, __name__) +__import__('pkg_resources').declare_namespace(__name__) diff --git a/setup.py b/setup.py index 2c63da3..d5dc0c7 100644 --- a/setup.py +++ b/setup.py @@ -11,16 +11,14 @@ info = json.load(open("./package.json")) -NAMESPACE_PACKAGES = [] -# TODO: simplify this process def generate_namespaces(package): - new_package = ".".join(package.split(".")[0:-1]) - if new_package.count(".") > 0: - generate_namespaces(new_package) - NAMESPACE_PACKAGES.append(new_package) -generate_namespaces(info["name"]) + i = package.count(".") + while i: + yield package.rsplit(".", i)[0] + i -= 1 +NAMESPACE_PACKAGES = list(generate_namespaces(info['name'])) if os.path.exists("MANIFEST"): os.unlink("MANIFEST") From a43c5efd281d4938a9bbadc09067a50958849204 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Wed, 19 Feb 2014 10:31:16 -0800 Subject: [PATCH 44/53] It's probably okay to `zip_safe=True` for all of our packages but I'll punt on this for now. I think we only ever do source distributions anyway. Also, packages can set it in their package.json if necessary. --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index d5dc0c7..cac32d5 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,6 @@ def generate_namespaces(package): "packages": find_packages(exclude=["*.tests", "*.tests.*"]), "namespace_packages": NAMESPACE_PACKAGES, "include_package_data": True, - "zip_safe": False, "classifiers": [ 'Development Status :: 3 - Alpha', 'Environment :: Web Environment', From bdf32d6bd87a1bc60cea3b5a5bfd0a665c4e3e82 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Wed, 19 Feb 2014 10:34:01 -0800 Subject: [PATCH 45/53] Use `find_packages()` and since we aren't building `package_data` anymore, we need to use `MANIFEST.in`. That's what it's there for and does a more obvious job. "Explicit is better than implicit." Using MANIFEST requires `include_package_data=True`. --- setup.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index cac32d5..c8b0c63 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,6 @@ -""" -setup.py file for building armstrong components. - -Nothing in this file should need to be edited, please see accompanying -package.json file if you need to adjust metadata about this package. - -""" +# Nothing in this file should need to be edited. +# Use package.json to adjust metadata about this package. +# Use MANIFEST.in to include package-specific data files. import os import json from setuptools import setup, find_packages @@ -27,7 +23,7 @@ def generate_namespaces(package): "author": "Bay Citizen & Texas Tribune", "author_email": "dev@armstrongcms.org", "url": "http://github.com/armstrong/%s/" % info["name"], - "packages": find_packages(exclude=["*.tests", "*.tests.*"]), + "packages": find_packages(), "namespace_packages": NAMESPACE_PACKAGES, "include_package_data": True, "classifiers": [ From 4fa0dbf61cecb4d79c4ba0b8903ff523be372092 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Tue, 18 Feb 2014 16:22:32 -0800 Subject: [PATCH 46/53] Remove unnecessary path modification. --- armstrong/dev/dev_django.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/armstrong/dev/dev_django.py b/armstrong/dev/dev_django.py index 526f1e1..9822938 100644 --- a/armstrong/dev/dev_django.py +++ b/armstrong/dev/dev_django.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -import os import sys import threading from functools import wraps @@ -44,9 +43,6 @@ def load_settings(): "%s. Check to see if Django is installed in your " "virtualenv." % e) - # Add the component's directory to the path so we can import from it - sys.path.append(os.getcwd()) - try: import env_settings as package_settings except ImportError as e: From 67bf687ab33b5fca2d7ecd3060609ba1fb83e378 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Tue, 18 Feb 2014 16:22:32 -0800 Subject: [PATCH 47/53] Update README and add a CHANGES file. --- CHANGES.rst | 66 +++++++++++++++++++++++ MANIFEST.in | 2 +- README.rst | 148 +++++++++++++++++++++++++++++++++++----------------- 3 files changed, 168 insertions(+), 48 deletions(-) create mode 100644 CHANGES.rst diff --git a/CHANGES.rst b/CHANGES.rst new file mode 100644 index 0000000..b05810f --- /dev/null +++ b/CHANGES.rst @@ -0,0 +1,66 @@ +CHANGES +======= + +2.0 (unreleased) +------------------ + +- **Backward incompatible:** ``env_settings.py`` is required and contains + the settings that ``fabfile.py`` used to have. It is written in the normal + Django settings.py style of variable assignments. + +- **Backward incompatible:** ``tasks.py`` is required and replaces the + previously required ``fabfile.py``. It no longer defines Django settings + (now handled in ``env_settings.py``). + +- **Backward incompatible:** switched from Fabric to Invoke, so the ``fab`` + command is replaced with ``invoke``. + +- **Backward incompatible:** Lettuce testing support is gone. + +- **Backward incompatible:** the Sphinx documentation command ``docs`` is gone. + +- **Backward incompatible:** remove ``assertInContext``, ``assertNone``, + ``assertIsA`` and ``assertDoesNotHave``. Some of these duplicated native + Python methods and the potential confusion was greater than the benefit. + +- **Backward incompatible:** individual components need to specify the + ``fudge`` requirement if they need it. + +- **Backward incompatible:** Fix ``generate_random_users()`` and turn it + into a generator. + +- **Backward incompatible:** remove ``concrete`` decorators. The Armstrong + standard practice is to use "support" models when necessary in testing, + which are much easier to use and understand. + +- Setuptools is explictly used. This is not a backwards incompatible change + because anything installed with Pip was automatically and transparently + using Setuptools/Distribute *anyway*. We rely on setup kwargs that Distutils + doesn't support and that only worked because of Pip's behind the scenes swap. + This allowed us to remove boilerplate and better prepares us for Python 3 + and perhaps even more simplifying refactors. Functionally though, this + doesn't change anything. + +- Drop the atypical VirtualDjango in favor of the ``settings.configure()`` + Django bootstrapping method. + +- Bare minimum package requirements for as-fast-as-possible virtualenv + creation. Even Invoke is optional when running tests. Individual tasks + can specify package requirements and will nicely message their needs if + the package is not installed. + +- Run any Django ``manage.py`` command from import or the CLI with + component-specific settings bootstrapped in. + +- Run tests with arguments. Use any args that ``manage.py test`` accepts + to run only specific test cases, change output verbosity, etc. + +- Coverage testing is ready for multiple environments at once (like with Tox). + +- Use (and backport) the Django 1.6 test runner. This standardizes testing + in favor of the newest method so we don't need to be cognisant of the current + Django version as we test across multiple versions. Bonus: because the new + runner is explicit about test discovery, drop the ``TESTED_APP`` code. + +- New ``remove_armstrong`` task command to uninstall every Armstrong component + (except for ArmDev). \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in index 1ab1acc..6b5f581 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ include README.rst +include CHANGES.rst include package.json -include armstrong/cli/templates/standard/requirements/*.txt prune build/* diff --git a/README.rst b/README.rst index dfea268..39279cb 100644 --- a/README.rst +++ b/README.rst @@ -3,21 +3,81 @@ armstrong.dev Tools and such for handling development of Armstrong applications This package contains some of the various helpers needed to do development work -on the Armstrong packages. If you're not actively developing, or working with +on the Armstrong packages. If you're not actively developing, or working with development versions of Armstrong, you probably don't need this package. + Installation ------------ -1. ``pip install armstrong.dev`` + +1. ``pip install armstrong.dev invoke`` **OR**, if all you are doing +is testing, ``pip install tox`` + +`Invoke`_ is not strictly required. ArmDev is as lean as possible to support +fast virtualenv creation both so you can get on with your work and so +multi-environment testing tools like TravisCI and Tox will complete ASAP. +If testing is all you are doing, you can use ``tox`` or +``python -m armstrong.dev.dev_django test``. + +Many of the Invoke tasks have their own package requirements and they will +nicely notify you if something they require needs to be installed. + +.. _Invoke: http://docs.pyinvoke.org/en/latest/index.html Usage ----- Most Armstrong components already have the necessary configuration to use these -Dev tools. Type ``invoke --list`` to see a list of all of the commands. +Dev tools. Specifically, components need ``tasks.py`` and ``env_settings.py`` +files. Assuming these are present: + +``invoke --list`` + to see a list of all available commands + +``invoke --help `` + for help on a specific command + +Several of the tasks take an optional ``--extra`` argument that is used as a +catch-all way of passing arbitrary arguments to the underlying command. Invoke +cannot handle arbitrary args (like Fabric 1.x could) so this is our workaround. +Two general rules: 1) enclose multiple args in quotes 2) kwargs need to use +"=" with no spaces (our limitation, not Invoke's). Example: +``invoke test --extra "--verbosity=2 "`` + +``invoke install [--editable]`` + to "pip install" the component, by default as an `editable`_ install. For + a regular install, use ``--no-editable`` or ``--editable=False``. + +``invoke test [--extra ...]`` + to run tests where --extra handles anything the normal Django + "manage.py test" command accepts. + +``invoke coverage [--reportdir=] [--extra ...]`` + for running test coverage. --extra works the same as in "invoke test" passing + arbitrary args to the underlying test command. --reportdir is where the HTML + report will be created; by default this directory is named "coverage". -If you are creating a new component (or perhaps updating one that uses -the older, pre 2.0 Dev tools), you'll need these next two steps. +``invoke managepy [--extra ...]`` + to run any Django "manage.py" command where --extra handles any arbitrary + args. Example: ``invoke managepy shell`` or + ``invoke managepy runserver --extra 9001`` + +``invoke create_migration [--initial]`` + to create a South migration for the component. An "auto" migration is + default if the --initial flag is not used. + +There are other commands as well, but these are the most useful. Remember +that individual components may provide additional Invoke tasks as well. So +run ``invoke --list`` to discover them all. + + +.. _editable: http://pip.readthedocs.org/en/latest/reference/pip_install.html#editable-installs + + +Component Setup +--------------- +If you are creating a new Armstrong component or updating one that uses the +pre-2.0 ArmDev, you'll need to create (or port to) these two files: 1. Create a ``tasks.py`` and add the following:: @@ -31,11 +91,31 @@ the older, pre 2.0 Dev tools), you'll need these next two steps. from armstrong.dev.default_settings import * # any additional settings + # it's likely you'll need to extend the list of INSTALLED_APPS # ... +Not required but as long as you are reviewing the general state of things, +take care of these too! + +- Review the ``requirements`` files. +- Add a ``tox.ini`` file. +- Drop Lettuce tests and requirements. +- Review the TravisCI configuration. +- Review ``.gitignore``. You might want to ignore these:: + + .tox/ + coverage*/ + *.egg-info + Notable changes in 2.0 ---------------------- +Setuptools is now explicitly used/required instead of Distutils. + +Invoke replaces Fabric for a leaner install without the SSH and crypto +stuff. Invoke is still pre-1.0 release so we might have some adjustment +to do later. + This version offers an easier and more standard way to run a Django environment with a component's specific settings, either from the commandline or via import. @@ -48,15 +128,8 @@ doesn't pollute your virtualenv with packages for features you don't use. ``test`` and ``coverage`` will work better with automated test tools like TravisCI and Tox. These commands also now work like Django's native test -command so that you can pass arguments for running selective tests. Due to -Invoke's argument passing, we need to include any optional args via an -``--extra`` param, i.e.:: - - invoke test --extra [[..[.]]] - - # enclose multiple args in quotes - # kwargs need to use "=" with no spaces (our limitation, not Invoke's) - invoke test --extra "--verbosity=2 " +command so that you can pass arguments for running selective tests or +changing the output verbosity. Settings are now defined in the normal Django style in an ``env_settings.py`` file instead of as a dict within the tasks file. It's not called "settings.py" @@ -64,40 +137,24 @@ to make it clearer that these are settings for the development and testing of this component, not necessarily values to copy/paste for incorporating the component into other projects. - -Backward incompatible changes in 2.0 ------------------------------------- -* ``env_settings.py`` is necessary and contains the settings that - ``tasks.py`` used to have. - -* ``tasks.py`` imports from a different place and no longer defines the - settings configurations. - -Not required but as long as you are reviewing the general state of things, -take care of these things too! - -* Review the ``requirements`` files. -* Add a ``tox.ini`` file. -* Review or add a TravisCI configuration. -* Review ``.gitignore``. You might want to ignore these:: - - .tox/ - coverage*/ - *.egg-info +The full list of changes and backward incompatibilties is available +in **CHANGES.rst**. Contributing ------------ +Development occurs on Github. Participation is welcome! -* Create something awesome -- make the code better, add some functionality, - whatever (this is the hardest part). -* `Fork it`_ -* Create a topic branch to house your changes -* Get all of your commits in the new topic branch -* Submit a `Pull Request`_ +* Found a bug? File it on `Github Issues`_. Include as much detail as you + can and make sure to list the specific component since we use a centralized, + project-wide issue tracker. +* Have code to submit? Fork the repo, consolidate your changes on a topic + branch and create a `pull request`_. +* Questions, need help, discussion? Use our `Google Group`_ mailing list. -.. _Pull Request: https://help.github.com/articles/using-pull-requests -.. _Fork it: https://help.github.com/articles/fork-a-repo +.. _Github Issues: https://github.com/armstrong/armstrong/issues +.. _pull request: http://help.github.com/pull-requests/ +.. _Google Group: http://groups.google.com/group/armstrongcms State of Project @@ -107,14 +164,11 @@ organization. It is the result of a collaboration between the `Texas Tribune`_ and `The Center for Investigative Reporting`_ and a grant from the `John S. and James L. Knight Foundation`_. -To follow development, be sure to join the `Google Group`_. - ``armstrong.dev`` is part of the `Armstrong`_ project. You're probably looking for that. -.. _Armstrong: http://www.armstrongcms.org/ +.. _Texas Tribune: http://www.texastribune.org/ .. _The Center for Investigative Reporting: http://cironline.org/ .. _John S. and James L. Knight Foundation: http://www.knightfoundation.org/ -.. _Texas Tribune: http://www.texastribune.org/ -.. _Google Group: http://groups.google.com/group/armstrongcms +.. _Armstrong: http://www.armstrongcms.org/ \ No newline at end of file From e7ecdabb9e76ded279040bb6e89a2e4ed648d58e Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Mon, 3 Mar 2014 20:06:41 -0800 Subject: [PATCH 48/53] Setup.py metadata. Include the license in the manifest. --- MANIFEST.in | 1 + setup.py | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 6b5f581..a47dcd7 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ +include LICENSE include README.rst include CHANGES.rst include package.json diff --git a/setup.py b/setup.py index c8b0c63..8cc11d2 100644 --- a/setup.py +++ b/setup.py @@ -20,20 +20,21 @@ def generate_namespaces(package): os.unlink("MANIFEST") setup_kwargs = { - "author": "Bay Citizen & Texas Tribune", + "author": "Texas Tribune & The Center for Investigative Reporting", "author_email": "dev@armstrongcms.org", "url": "http://github.com/armstrong/%s/" % info["name"], "packages": find_packages(), "namespace_packages": NAMESPACE_PACKAGES, "include_package_data": True, "classifiers": [ - 'Development Status :: 3 - Alpha', - 'Environment :: Web Environment', + 'Development Status :: 5 - Production/Stable', + 'Environment :: Console', 'Framework :: Django', 'Intended Audience :: Developers', 'License :: OSI Approved :: Apache Software License', 'Operating System :: OS Independent', 'Programming Language :: Python', + 'Topic :: Software Development :: Testing', ], } From eb2e0b1f7d02c6ea67e70c9ff010005b8ea7a9ff Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Tue, 4 Mar 2014 16:35:28 -0800 Subject: [PATCH 49/53] New feature - add a hook to exclude files during Coverage testing. By default, exclude South migrations. --- README.rst | 10 ++++++++-- armstrong/dev/default_settings.py | 2 ++ armstrong/dev/tasks.py | 5 ++++- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/README.rst b/README.rst index 39279cb..f18c231 100644 --- a/README.rst +++ b/README.rst @@ -7,8 +7,8 @@ on the Armstrong packages. If you're not actively developing, or working with development versions of Armstrong, you probably don't need this package. -Installation ------------- +Installation & Configuration +---------------------------- 1. ``pip install armstrong.dev invoke`` **OR**, if all you are doing is testing, ``pip install tox`` @@ -22,6 +22,12 @@ If testing is all you are doing, you can use ``tox`` or Many of the Invoke tasks have their own package requirements and they will nicely notify you if something they require needs to be installed. +**Optional Settings:** (Used in ``env_settings.py``) + +``COVERAGE_EXCLUDE_FILES = ['*/migrations/*']`` + A list of filename patterns for files to exclude during coverage testing. + Individual components are free to extend or replace this setting. + .. _Invoke: http://docs.pyinvoke.org/en/latest/index.html diff --git a/armstrong/dev/default_settings.py b/armstrong/dev/default_settings.py index 35b30f0..ea3767a 100644 --- a/armstrong/dev/default_settings.py +++ b/armstrong/dev/default_settings.py @@ -19,6 +19,8 @@ } TEST_RUNNER = "armstrong.dev.tests.runner.ArmstrongDiscoverRunner" +COVERAGE_EXCLUDE_FILES = ['*/migrations/*'] + # # A component may override settings by creating an `env_settings.py` # file in its root directory that imports from this file. diff --git a/armstrong/dev/tasks.py b/armstrong/dev/tasks.py index 2d4458e..6396f4e 100644 --- a/armstrong/dev/tasks.py +++ b/armstrong/dev/tasks.py @@ -66,9 +66,12 @@ def html_coverage_report(report_directory=None): module = __import__(package['name'], fromlist=[package_parent]) base_path = dirname(module.__file__) + settings = DjangoSettings() + omit = getattr(settings, 'COVERAGE_EXCLUDE_FILES', None) + import coverage as coverage_api print("Coverage is covering: %s" % base_path) - cov = coverage_api.coverage(branch=True, source=[base_path]) + cov = coverage_api.coverage(branch=True, source=[base_path], omit=omit) cov.start() yield From 2696e2147162dee6563f916fb1beecebaf2798be Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Thu, 6 Mar 2014 17:19:57 -0800 Subject: [PATCH 50/53] Readme changes and removing some line-ending whitespace. --- CHANGES.rst | 6 +++--- README.rst | 42 +++++++++++++++++++++++------------------- 2 files changed, 26 insertions(+), 22 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index b05810f..4f7eb73 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -26,7 +26,7 @@ CHANGES - **Backward incompatible:** individual components need to specify the ``fudge`` requirement if they need it. -- **Backward incompatible:** Fix ``generate_random_users()`` and turn it +- **Backward incompatible:** Fix ``generate_random_users()`` and turn it into a generator. - **Backward incompatible:** remove ``concrete`` decorators. The Armstrong @@ -58,9 +58,9 @@ CHANGES - Coverage testing is ready for multiple environments at once (like with Tox). - Use (and backport) the Django 1.6 test runner. This standardizes testing - in favor of the newest method so we don't need to be cognisant of the current + in favor of the newest method so we don't need to be cognizant of the current Django version as we test across multiple versions. Bonus: because the new runner is explicit about test discovery, drop the ``TESTED_APP`` code. - New ``remove_armstrong`` task command to uninstall every Armstrong component - (except for ArmDev). \ No newline at end of file + (except for ArmDev). diff --git a/README.rst b/README.rst index f18c231..4482194 100644 --- a/README.rst +++ b/README.rst @@ -9,15 +9,18 @@ development versions of Armstrong, you probably don't need this package. Installation & Configuration ---------------------------- +If you are just running tests for a component, Tox will grab everything it +needs including ArmDev. -1. ``pip install armstrong.dev invoke`` **OR**, if all you are doing -is testing, ``pip install tox`` +- ``pip install tox`` and run ``tox`` + +Otherwise: + +- ``pip install armstrong.dev invoke`` `Invoke`_ is not strictly required. ArmDev is as lean as possible to support -fast virtualenv creation both so you can get on with your work and so -multi-environment testing tools like TravisCI and Tox will complete ASAP. -If testing is all you are doing, you can use ``tox`` or -``python -m armstrong.dev.dev_django test``. +fast virtualenv creation so multi-environment testing tools like TravisCI +and Tox will complete ASAP. Many of the Invoke tasks have their own package requirements and they will nicely notify you if something they require needs to be installed. @@ -65,7 +68,7 @@ Two general rules: 1) enclose multiple args in quotes 2) kwargs need to use ``invoke managepy [--extra ...]`` to run any Django "manage.py" command where --extra handles any arbitrary - args. Example: ``invoke managepy shell`` or + args. Example: ``invoke managepy shell`` or ``invoke managepy runserver --extra 9001`` ``invoke create_migration [--initial]`` @@ -103,10 +106,14 @@ pre-2.0 ArmDev, you'll need to create (or port to) these two files: Not required but as long as you are reviewing the general state of things, take care of these too! -- Review the ``requirements`` files. -- Add a ``tox.ini`` file. -- Drop Lettuce tests and requirements. -- Review the TravisCI configuration. +- Review the ``requirements`` files +- Review the TravisCI configuration +- Drop Lettuce tests and requirements +- Add a ``tox.ini`` file +- Review the README text and setup.py metadata +- Use Setuptools and fix any improper namespacing +- Stop shipping tests by moving tests/ to the root directory +- Add a ``CHANGES.rst`` file and include it in the MANIFEST - Review ``.gitignore``. You might want to ignore these:: .tox/ @@ -165,16 +172,13 @@ Development occurs on Github. Participation is welcome! State of Project ---------------- -Armstrong is an open-source news platform that is freely available to any -organization. It is the result of a collaboration between the `Texas Tribune`_ +`Armstrong`_ is an open-source news platform that is freely available to any +organization. It is the result of a collaboration between the `Texas Tribune`_ and `The Center for Investigative Reporting`_ and a grant from the -`John S. and James L. Knight Foundation`_. - -``armstrong.dev`` is part of the `Armstrong`_ project. You're -probably looking for that. - +`John S. and James L. Knight Foundation`_. Armstrong is available as a +complete bundle and as individual, stand-alone components. +.. _Armstrong: http://www.armstrongcms.org/ .. _Texas Tribune: http://www.texastribune.org/ .. _The Center for Investigative Reporting: http://cironline.org/ .. _John S. and James L. Knight Foundation: http://www.knightfoundation.org/ -.. _Armstrong: http://www.armstrongcms.org/ \ No newline at end of file From ff2bdc7cdcddf23b5ac113363985a6fca8b58fae Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Tue, 11 Mar 2014 16:19:22 -0700 Subject: [PATCH 51/53] Logging setting - add a default setting to catch "armstrong" loggers (which we don't use, but it's a good idea so add it to the Readme). --- README.rst | 2 ++ armstrong/dev/default_settings.py | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/README.rst b/README.rst index 4482194..6841b8b 100644 --- a/README.rst +++ b/README.rst @@ -113,6 +113,8 @@ take care of these too! - Review the README text and setup.py metadata - Use Setuptools and fix any improper namespacing - Stop shipping tests by moving tests/ to the root directory +- If the component uses logging, consider namespacing it with + ``logger = logging.getLogger(__name__)``. - Add a ``CHANGES.rst`` file and include it in the MANIFEST - Review ``.gitignore``. You might want to ignore these:: diff --git a/armstrong/dev/default_settings.py b/armstrong/dev/default_settings.py index ea3767a..d386bc1 100644 --- a/armstrong/dev/default_settings.py +++ b/armstrong/dev/default_settings.py @@ -21,6 +21,27 @@ COVERAGE_EXCLUDE_FILES = ['*/migrations/*'] +# Add a DEBUG console "armstrong" logger +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'basic': {'format': '%(levelname)s %(module)s--%(message)s'} + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'basic' + } + }, + 'loggers': { + 'armstrong': { + 'level': 'DEBUG', + 'handlers': ['console'] + } + } +} + # # A component may override settings by creating an `env_settings.py` # file in its root directory that imports from this file. From 2d9b2c37fa6956e4d8971ec928803bd7741f8078 Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Tue, 11 Mar 2014 16:20:41 -0700 Subject: [PATCH 52/53] Use a docstring for `default_settings.py` and set `DEBUG=True`, which is the current case for each individual component. --- armstrong/dev/default_settings.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/armstrong/dev/default_settings.py b/armstrong/dev/default_settings.py index d386bc1..abae269 100644 --- a/armstrong/dev/default_settings.py +++ b/armstrong/dev/default_settings.py @@ -1,3 +1,13 @@ +""" +Default settings for Armstrong components running in a dev/test environment + +A component may (and might have to) override or supply additional settings +by creating an `env_settings.py` file in its root directory that imports +from this file. + + from armstrong.dev.default_settings import * + +""" # Since we are using configure() we need to manually load the defaults from django.conf.global_settings import * @@ -6,10 +16,10 @@ package = json.load(open("./package.json")) app_name = package['name'].rsplit('.', 1)[1] - # -# Default settings +# Armstrong default settings # +DEBUG = True INSTALLED_APPS = [package['name']] DATABASES = { "default": { @@ -41,10 +51,3 @@ } } } - -# -# A component may override settings by creating an `env_settings.py` -# file in its root directory that imports from this file. -# -# from armstrong.dev.default_settings import * -# From e94c59485021553c60edc1fe9909339a1c82b53a Mon Sep 17 00:00:00 2001 From: Jon Cotton Date: Tue, 18 Mar 2014 17:22:34 -0700 Subject: [PATCH 53/53] Adjust logger output and fix whitespace. example: "ERROR armstrong.core.arm_layout--backend error text" is more conclusive than "ERROR basic--backend error text". Since anything could be logging, it's better to know the logger name. --- armstrong/dev/default_settings.py | 2 +- armstrong/dev/tests/runner.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/armstrong/dev/default_settings.py b/armstrong/dev/default_settings.py index abae269..19ead69 100644 --- a/armstrong/dev/default_settings.py +++ b/armstrong/dev/default_settings.py @@ -36,7 +36,7 @@ 'version': 1, 'disable_existing_loggers': False, 'formatters': { - 'basic': {'format': '%(levelname)s %(module)s--%(message)s'} + 'basic': {'format': '%(levelname)s %(name)s--%(message)s'} }, 'handlers': { 'console': { diff --git a/armstrong/dev/tests/runner.py b/armstrong/dev/tests/runner.py index 046d25d..5580a56 100644 --- a/armstrong/dev/tests/runner.py +++ b/armstrong/dev/tests/runner.py @@ -1,12 +1,12 @@ try: - from django.test.runner import DiscoverRunner + from django.test.runner import DiscoverRunner except ImportError: # < Django 1.6 - from .utils.runner import DiscoverRunner + from .utils.runner import DiscoverRunner class ArmstrongDiscoverRunner(DiscoverRunner): def __init__(self, *args, **kwargs): - """Find our "tests" package, not just "test*.py" files""" + """Find our "tests" package, not just "test*.py" files""" super(ArmstrongDiscoverRunner, self).__init__(*args, **kwargs) self.pattern = "test*"