diff --git a/.gitignore b/.gitignore index 94fc654..a8df91c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,12 @@ *.py[cod] .DS_Store + share +bin +lib +include .Python + *.db .coverage coverage.xml diff --git a/Vagrantfile b/Vagrantfile index 2c13a71..85c7e6c 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -8,7 +8,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.vm.box = "precise32" config.vm.network :private_network, ip: "192.168.33.101" - config.vm.network "forwarded_port", guest: 9200, host: 9200 + config.vm.network "forwarded_port", guest: 9200, host: 9220 config.vm.provider :virtualbox do |v, override| override.vm.box_url = "http://files.vagrantup.com/precise32.box" diff --git a/docs/quickstart.rst b/docs/quickstart.rst index 5452d0b..8efd38d 100644 --- a/docs/quickstart.rst +++ b/docs/quickstart.rst @@ -35,6 +35,6 @@ Once you have models that inherit from PolymorphicIndexable, you'll need to crea python manage.py synces Migrating Indexes -================ +================= Lorem ipsum \ No newline at end of file diff --git a/elastimorphic/__init__.py b/elastimorphic/__init__.py index 812a593..9869aae 100644 --- a/elastimorphic/__init__.py +++ b/elastimorphic/__init__.py @@ -1,5 +1,4 @@ -from .base import PolymorphicIndexable, SearchManager +from .base import Indexable, PolymorphicIndexable, SearchManager # noqa - -__version__ = "0.1.0-dev" +__version__ = "0.1.0" __all__ = [PolymorphicIndexable, SearchManager] diff --git a/elastimorphic/base.py b/elastimorphic/base.py index f4ba68e..48d3af6 100644 --- a/elastimorphic/base.py +++ b/elastimorphic/base.py @@ -5,6 +5,7 @@ from .conf import settings from .models import polymorphic_indexable_registry +from .mappings.doctype import DocumentType, search_field_factory class ModelSearchResults(SearchResults): @@ -105,12 +106,12 @@ def s(self): @property def es(self): - """Returns a pyelasticsearch object, using the ES URL from the Django settings""" + """Returns an elasticsearch object, using the ES URL from the Django settings""" return get_es(urls=settings.ES_URLS) def refresh(self): """Refreshes the index for this object""" - return self.es.refresh(index=self.model.get_index_name()) + return self.es.indices.refresh(index=self.model.get_index_name()) def query(self, **kwargs): """Just a simple bridge to elasticutils' S().query(), prepopulating the URL @@ -123,86 +124,84 @@ def filter(self, **kwargs): return self.s().filter(**kwargs) -class PolymorphicIndexable(object): - """Mixin for PolymorphicModel, allowing easy indexing and querying. +class Indexable(object): + """A mixing for Django's Model, allowing easy indexing and querying.""" - This class is a mixin, intended to be used on PolymorphicModel classes. To use it, - you just mix it in, and implement a few methods. For example: + @classmethod + def get_es(cls): + return get_es(urls=settings.ES_URLS) - .. code-block:: python + @classmethod + def get_mapping_type_name(cls): + """By default, we'll be using the app_label and module_name properties to get the ES doctype for this object""" + return "%s_%s" % (cls._meta.app_label, cls._meta.module_name) - from django.db import models - from elastimorphic import PolymorphicIndexable, SearchManager - from polymorphic import PolymorphicModel + @classmethod + def get_doctype_class(cls): + doctype_class = type("{}_Mapping".format(cls.__name__), (DocumentType,), {}) + if hasattr(cls, "Mapping"): + doctype_class = cls.Mapping - class ParentIndexable(PolymorphicIndexable, PolymorphicModel): - foo = models.CharField(max_length=255) + exclude = getattr(doctype_class, "exclude", []) - search_objects = SearchManager() + for field_pair in doctype_class.fields: + exclude.append(field_pair[0]) - def extract_document(self): - doc = super(ParentIndexable, self).extract_document() - doc["foo"] = self.foo - return doc + for field in cls._meta.fields: + if field.name in exclude: + continue - @classmethod - def get_mapping_properties(cls): - properties = super(ParentIndexable, cls).get_mapping_properties() - properties.update({ - "foo": {"type": "string"} - }) - return properties + field_tuple = search_field_factory(field) + if field_tuple: + doctype_class.fields.append(field_tuple) - class ChildIndexable(ParentIndexable): - bar = models.IntegerField() + return doctype_class - def extract_document(self): - doc = super(ChildIndexable, self).extract_document() - doc["bar"] = self.bar - return doc + @classmethod + def get_mapping(cls): + doctype_class = cls.get_doctype_class() - @classmethod - def get_mapping_properties(cls): - properties = super(ChildIndexable, cls).get_mapping_properties() - properties.update({ - "bar": {"type": "integer"} - }) - return properties + mapping = doctype_class().get_mapping() + mapping["dynamic"] = "strict" + mapping["_id"] = {"path": cls._meta.pk.get_attname()} + return {cls.get_mapping_type_name(): mapping} - With this example code, after syncdb a new Elasticsearch index named `example_parentindexable` would - be created, with two mappings: `example_parentindexable` and `example_childindexable`. At minimum, you - should implement the :func:`extract_document` instance method, and the :func:`get_mapping_properties` classmethod. - """ + @classmethod + def get_index_name(cls): + index_prefix = slugify(settings.DATABASES[DEFAULT_DB_ALIAS].get("NAME", "bulbs")) + return "%s_%s" % (index_prefix, cls._meta.db_table) def extract_document(self): - """Returns a python dictionary, representing the Elasticseach document for this model instance. + doctype_class = self.get_doctype_class() - By default, this just includes the `polymorphic_ctype id`_, and the primary key, e.g.:: + doctype = doctype_class() + document = {} + for name, field in doctype.fields: + value = getattr(self, name, None) + document[name] = field.to_es(value) + return document - { - "polymorphic_ctype": 1, - "id": 1 - } - - If when you override this method, be sure to at least return the default fields. This is best - done by simply updating the parent's data. For example:: + def index(self, refresh=False): + es = self.get_es() + doc = self.extract_document() + es.update( + index=self.get_index_name(), + doc_type=self.get_mapping_type_name(), + id=self.pk, + body=dict(doc=doc, doc_as_upsert=True) + ) - def extract_document(self): - doc = super(ParentModel, self).extract_document() - doc.update({ - "bar": self.bar - }) - return doc + def save(self, index=True, refresh=False, *args, **kwargs): + result = super(Indexable, self).save(*args, **kwargs) + if index: + self.index(refresh=refresh) + self._index = index + return result - It's also wise to be sure that your data is properly modeled (by overriding :func:`get_mapping`), so that - you're not letting Elasticseach decide your mappings for you. - .. _polymorphic_ctype id: https://github.com/chrisglass/django_polymorphic/blob/master/polymorphic/query.py#L190 - """ - return { - "polymorphic_ctype": self.polymorphic_ctype_id, - self.polymorphic_primary_key_name: self.id - } +class PolymorphicIndexable(Indexable): + """Mixin for PolymorphicModel, allowing easy indexing and querying. + """ @classmethod def get_base_class(cls): @@ -216,23 +215,18 @@ def get_index_name(cls): return "%s_%s" % (index_prefix, cls.get_base_class()._meta.db_table) @classmethod - def get_es(cls): - return get_es(urls=settings.ES_URLS) + def get_mapping_type_names(cls, exclude_base=False): + """Returns the mapping type name of this class and all of its descendants.""" + names = [] + if not exclude_base: + names.append(cls.get_mapping_type_name()) + for subclass in cls.__subclasses__(): + names.extend(subclass.get_mapping_type_names()) + return names @classmethod - def get_mapping(cls): - return { - cls.get_mapping_type_name(): { - "_id": { - "path": cls.polymorphic_primary_key_name - }, - "properties": cls.get_mapping_properties(), - "dynamic": "strict", - "_all": { - "analyzer": "html" - } - } - } + def get_doctypes(cls): + return polymorphic_indexable_registry.get_doctypes(cls) @classmethod def get_mapping_properties(cls): @@ -265,41 +259,46 @@ def get_mapping_properties(cls): } @classmethod - def get_mapping_type_name(cls): - """By default, we'll be using the app_label and module_name properties to get the ES doctype for this object""" - return "%s_%s" % (cls._meta.app_label, cls._meta.module_name) + def get_mapping(cls): + return { + cls.get_mapping_type_name(): { + "_id": { + "path": cls.polymorphic_primary_key_name + }, + "properties": cls.get_mapping_properties(), + "dynamic": "strict", + "_all": { + "analyzer": "html" + } + } + } - @classmethod - def get_mapping_type_names(cls, exclude_base=False): - """Returns the mapping type name of this class and all of its descendants.""" - names = [] - if not exclude_base: - names.append(cls.get_mapping_type_name()) - for subclass in cls.__subclasses__(): - names.extend(subclass.get_mapping_type_names()) - return names + def extract_document(self): + """Returns a python dictionary, representing the Elasticseach document for this model instance. - @classmethod - def get_doctypes(cls): - return polymorphic_indexable_registry.get_doctypes(cls) + By default, this just includes the `polymorphic_ctype id`_, and the primary key, e.g.:: - def index(self, refresh=False): - es = self.get_es() - doc = self.extract_document() - # NOTE: this could be made more efficient with the `doc_as_upsert` - # param when the following pull request is merged into pyelasticsearch: - # https://github.com/rhec/pyelasticsearch/pull/132 - es.update( - self.get_index_name(), - self.get_mapping_type_name(), - self.id, - doc=doc, - upsert=doc - ) + { + "polymorphic_ctype": 1, + "id": 1 + } - def save(self, index=True, refresh=False, *args, **kwargs): - result = super(PolymorphicIndexable, self).save(*args, **kwargs) - if index: - self.index(refresh=refresh) - self._index = index - return result + If when you override this method, be sure to at least return the default fields. This is best + done by simply updating the parent's data. For example:: + + def extract_document(self): + doc = super(ParentModel, self).extract_document() + doc.update({ + "bar": self.bar + }) + return doc + + It's also wise to be sure that your data is properly modeled (by overriding :func:`get_mapping`), so that + you're not letting Elasticseach decide your mappings for you. + + .. _polymorphic_ctype id: https://github.com/chrisglass/django_polymorphic/blob/master/polymorphic/query.py#L190 + """ + return { + "polymorphic_ctype": self.polymorphic_ctype_id, + self.polymorphic_primary_key_name: self.id + } \ No newline at end of file diff --git a/elastimorphic/management/commands/bulk_index.py b/elastimorphic/management/commands/bulk_index.py index 492e86a..92a7de3 100644 --- a/elastimorphic/management/commands/bulk_index.py +++ b/elastimorphic/management/commands/bulk_index.py @@ -5,7 +5,7 @@ from django.core.management.base import BaseCommand from django.db import models from elasticutils import get_es -from pyelasticsearch.client import JsonEncoder +# from pyelasticsearch.client import JsonEncoder from elastimorphic import PolymorphicIndexable from elastimorphic.conf import settings @@ -29,7 +29,6 @@ class Command(BaseCommand): def handle(self, *args, **options): self.es = get_es(urls=settings.ES_URLS) - bulk_endpoint = "%s/_bulk" % settings.ES_URLS[0] chunk_size = options.get("chunk") index_suffix = options.get("index_suffix") @@ -73,31 +72,22 @@ def handle(self, *args, **options): "_id": instance.pk } } - payload.append(json.dumps(meta, cls=JsonEncoder, use_decimal=True)) + payload.append(meta) doc = instance.extract_document() - payload.append(json.dumps(doc, cls=JsonEncoder, use_decimal=True)) + payload.append(doc) if len(payload) / 2 == chunk_size: - r = requests.post(bulk_endpoint, data="\n".join(payload) + "\n") - if r.status_code != 200: - print(payload) - print(r.json()) - else: - # make sure it indexed everything: - result = r.json() - good_items = [item for item in result["items"] if item["index"].get("ok", False)] - if len(good_items) != len(payload) // 2: - self.stdout.write("Bulk indexing error! Item count mismatch.") - bad_items = [item for item in result["items"] if not item["index"].get("ok", False)] - self.stdout.write("These were rejected: %s" % str(bad_items)) - return "Bulk indexing failed." + response = self.es.bulk(body=payload) + good_items = [item for item in response["items"] if item["index"].get("ok", False)] + if len(good_items) != len(payload) // 2: + self.stdout.write("Bulk indexing error! Item count mismatch.") + bad_items = [item for item in response["items"] if not item["index"].get("ok", False)] + self.stdout.write("These were rejected: %s" % str(bad_items)) + return "Bulk indexing failed." num_processed += (len(payload) / 2) self.stdout.write("Indexed %d items" % num_processed) payload = [] if payload: - r = requests.post(bulk_endpoint, data="\n".join(payload) + "\n") - if r.status_code != 200: - print(payload) - print(r.json()) + response = self.es.bulk(body=payload) num_processed += (len(payload) / 2) self.stdout.write("Indexed %d items" % num_processed) diff --git a/elastimorphic/management/commands/es_swap_aliases.py b/elastimorphic/management/commands/es_swap_aliases.py index 372af8e..179861c 100644 --- a/elastimorphic/management/commands/es_swap_aliases.py +++ b/elastimorphic/management/commands/es_swap_aliases.py @@ -20,7 +20,7 @@ def handle(self, index_suffix, **options): es = get_es() alias_actions = [] # remove existing indexes using the aliases we want - existing_aliases = es.aliases() + existing_aliases = es.indices.get_aliases() for index, aliases in existing_aliases.items(): for alias, new_index in indexes.items(): if alias in aliases['aliases']: @@ -38,4 +38,4 @@ def handle(self, index_suffix, **options): "index": index } }) - es.update_aliases(dict(actions=alias_actions)) + es.indices.update_aliases(body=dict(actions=alias_actions)) diff --git a/elastimorphic/management/commands/synces.py b/elastimorphic/management/commands/synces.py index c1ca561..be7b427 100644 --- a/elastimorphic/management/commands/synces.py +++ b/elastimorphic/management/commands/synces.py @@ -1,8 +1,9 @@ from optparse import make_option from django.core.management.base import BaseCommand + +import elasticsearch from elasticutils import get_es -from pyelasticsearch.exceptions import IndexAlreadyExistsError, ElasticHttpError from elastimorphic.conf import settings from elastimorphic.models import polymorphic_indexable_registry @@ -36,7 +37,7 @@ def handle(self, *args, **options): # No suffix supplied. Let's create a map of existing aliases -> indexes # and try to update those instead of creating new indexes. index_suffix = "" - aliases = es.aliases() + aliases = es.get_aliases() for index_name in aliases: index_aliases = aliases[index_name]["aliases"] if index_aliases: @@ -56,32 +57,23 @@ def handle(self, *args, **options): for index, mappings in indexes.items(): if options.get("drop_existing_indexes", False) and index_suffix: - try: - es.delete_index(index) - except ElasticHttpError: - pass - + es.indices.delete(index=index, ignore=[404]) try: - es.create_index(index, settings={ - "settings": settings.ES_SETTINGS - }) - except IndexAlreadyExistsError: + es.indices.create(index=index, body=dict(settings=settings.ES_SETTINGS)) + except elasticsearch.RequestError: try: # TODO: Actually compare the settings. - es.update_settings(index, settings.ES_SETTINGS) - except ElasticHttpError as e: - if e.status_code == 400: - if options.get("force", False): - es.close_index(index) - es.update_settings(index, settings.ES_SETTINGS) - es.open_index(index) - else: - self.stderr.write("Index '%s' already exists, and you're trying to update non-dynamic settings. You will need to use a new suffix, or use the --force option" % index) + es.indices.put_settings(index=index, body=dict(settings=settings.ES_SETTINGS)) + except elasticsearch.RequestError: + if options.get("force", False): + es.indices.close(index=index) + es.indices.put_settings(index=index, body=dict(settings=settings.ES_SETTINGS)) + es.indices.open(index=index) else: - self.stderr.write("ElasticSearch Error: %s" % e) + self.stderr.write("Index '%s' already exists, and you're trying to update non-dynamic settings. You will need to use a new suffix, or use the --force option" % index) - for doctype, mapping in mappings.items(): + for doc_type, mapping in mappings.items(): try: - es.put_mapping(index, doctype, dict(doctype=mapping)) - except ElasticHttpError as e: - self.stderr.write("ES Error: %s" % e.error) + es.indices.put_mapping(index=index, doc_type=doc_type, body=mapping) + except elasticsearch.RequestError as e: + self.stderr.write("ES Error: %s" % e) diff --git a/elastimorphic/mappings/__init__.py b/elastimorphic/mappings/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/elastimorphic/mappings/doctype.py b/elastimorphic/mappings/doctype.py new file mode 100644 index 0000000..77c28ce --- /dev/null +++ b/elastimorphic/mappings/doctype.py @@ -0,0 +1,62 @@ +from .fields import * # noqa + + +class DeclarativeMappingMeta(type): + + def __new__(cls, name, bases, attrs): + fields = [(name_, attrs.pop(name_)) for name_, column in attrs.items() if hasattr(column, "get_definition")] + attrs['fields'] = fields + return super(DeclarativeMappingMeta, cls).__new__(cls, name, bases, attrs) + + +class DocumentType(SearchField): + + __metaclass__ = DeclarativeMappingMeta + field_type = "object" + + def get_mapping(self): + fields = {} + for name, field in self.fields: + defn = field.get_definition() + fields[name] = defn + + mapping = {"properties": fields} + return mapping + + def get_definition(self): + definition = super(DocumentType, self).get_definition() + definition.update(self.get_mapping()) + return definition + + def to_es(self, value): + document = {} + for name, field in self.fields: + value = getattr(self, name, None) + document[name] = field.to_es(value) + return document + + def to_python(self, value): + return None + + +SIMPLE_FIELD_MAPPINGS = { + "CharField": StringField, + "IntegerField": IntegerField, + "FloatField": FloatField, + "DateTimeField": DateField, + "OneToOneField": IntegerField, + "ForeignKey": IntegerField, + "AutoField": IntegerField +} + + +def search_field_factory(field): + """Returns a tuple (name, field) representing the Django model field as a SearchField + """ + + internal_type = field.get_internal_type() + klass = SIMPLE_FIELD_MAPPINGS.get(internal_type) + + if klass: + return (field.get_attname(), klass()) + return None diff --git a/elastimorphic/mappings/fields.py b/elastimorphic/mappings/fields.py new file mode 100644 index 0000000..302141e --- /dev/null +++ b/elastimorphic/mappings/fields.py @@ -0,0 +1,90 @@ +import datetime + +from django.utils import timezone + + +class SearchField(object): + + field_type = None + attrs = [] + + def __init__(self, *args, **kwargs): + # Set all kwargs on self for later access. + for attr in kwargs.keys(): + self.attrs.append(attr) + setattr(self, attr, kwargs.pop(attr, None)) + + def to_es(self, value): + return value + + def to_python(self, value): + return value + + def get_definition(self): + f = {'type': self.field_type} + for attr in self.attrs: + val = getattr(self, attr, None) + if val is not None: + f[attr] = val + return f + + +class StringField(SearchField): + + field_type = "string" + + def to_es(self, value): + if value is None: + return None + return unicode(value) + + def to_python(self, value): + if value is None: + return None + + +class IntegerField(SearchField): + + field_type = "integer" + + def to_es(self, value): + if value is None: + return None + return int(value) + + def to_python(self, value): + if value is None: + return None + return int(value) + + +class FloatField(SearchField): + field_type = "float" + + def to_es(self, value): + if value is None: + return None + return float(value) + + def to_python(self, value): + if value is None: + return None + return float(value) + + +class DateField(SearchField): + field_type = "date" + + def to_es(self, value): + if isinstance(value, (datetime.date, datetime.datetime)): + return value.isoformat() + return value + + def to_python(self, value): + if value is None: + return None + + if isinstance(value, basestring): + datetime.datetime.strptime(value, "%Y-%m-%dT%H:%M:%S.%f+00:00").replace(tzinfo=timezone.utc) + + return value diff --git a/elastimorphic/tests/base.py b/elastimorphic/tests/base.py index 6bee818..5ef6152 100644 --- a/elastimorphic/tests/base.py +++ b/elastimorphic/tests/base.py @@ -1,7 +1,6 @@ from django.core.management import call_command from django.test import TestCase from elasticutils import get_es -from pyelasticsearch.exceptions import ElasticHttpNotFoundError from elastimorphic.conf import settings from elastimorphic.models import polymorphic_indexable_registry @@ -20,7 +19,4 @@ def tearDown(self): def delete_indexes_with_suffix(self, suffix): for base_class in polymorphic_indexable_registry.families.keys(): - try: - self.es.delete_index(base_class.get_index_name() + "_" + suffix) - except ElasticHttpNotFoundError: - pass + self.es.indices.delete(index=base_class.get_index_name() + "_" + suffix, ignore=404) diff --git a/elastimorphic/tests/testapp/models.py b/elastimorphic/tests/testapp/models.py index 074aba2..c7a94b3 100644 --- a/elastimorphic/tests/testapp/models.py +++ b/elastimorphic/tests/testapp/models.py @@ -93,4 +93,4 @@ def get_mapping_properties(cls): @classmethod def get_serializer_class(cls): from .serializers import GrandchildIndexableSerializer - return GrandchildIndexableSerializer + return GrandchildIndexableSerializer \ No newline at end of file diff --git a/elastimorphic/tests/testdeclarative/__init__.py b/elastimorphic/tests/testdeclarative/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/elastimorphic/tests/testdeclarative/models.py b/elastimorphic/tests/testdeclarative/models.py new file mode 100644 index 0000000..3e0bbc0 --- /dev/null +++ b/elastimorphic/tests/testdeclarative/models.py @@ -0,0 +1,43 @@ +from django.db import models +from polymorphic import PolymorphicModel + +from elastimorphic import Indexable, PolymorphicIndexable, SearchManager +from elastimorphic.mappings import fields +from elastimorphic.mappings import doctype + + +class SeparateIndexable(PolymorphicIndexable, PolymorphicModel): + junk = models.CharField(max_length=255) + + search_objects = SearchManager() + + +class ParentIndexable(PolymorphicIndexable, PolymorphicModel): + foo = models.CharField(max_length=255) + + search_objects = SearchManager() + + +class ChildIndexable(ParentIndexable): + bar = models.IntegerField() + + +class RelatedModel(Indexable, models.Model): + qux = models.CharField(max_length=255, null=True, blank=True) + + class Mapping(doctype.DocumentType): + class Meta: + exclude = ("id",) + + qux = fields.StringField() + + +class GrandchildIndexable(ChildIndexable): + baz = models.DateField() + related = models.ForeignKey(RelatedModel, null=True, blank=True) + + class Mapping(doctype.DocumentType): + foo = fields.StringField() + bar = fields.IntegerField(store="yes") + baz = fields.DateField() + related = RelatedModel.Mapping() diff --git a/requirements.txt b/requirements.txt index f9ec21a..df441f6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,3 @@ -Django +Django>=1.5 django_polymorphic==0.5.3 -elasticutils==0.8.2 -pyelasticsearch==0.6.1 -requests==2.2.1 -simplejson==3.3.3 -six==1.5.2 -wsgiref==0.1.2 \ No newline at end of file +elasticutils==0.9.1 \ No newline at end of file diff --git a/setup.py b/setup.py index 2b801dc..25a2714 100755 --- a/setup.py +++ b/setup.py @@ -17,14 +17,9 @@ author_email = "tech@theonion.com" license = "MIT" requires = [ - "Django", + "Django>=1.5", "django_polymorphic==0.5.3", - "elasticutils==0.8.2", - "pyelasticsearch==0.6.1", - "requests==2.2.1", - "simplejson==3.3.3", - "six==1.5.2", - "wsgiref==0.1.2", + "elasticutils==0.9.1", ] @@ -73,7 +68,7 @@ def get_package_data(package): class PyTest(TestCommand): def finalize_options(self): TestCommand.finalize_options(self) - self.test_args = ["elastimorphic"] + self.test_args = ["tests/"] self.test_suite = True def run_tests(self): diff --git a/test_provision.sh b/test_provision.sh index 1c7f958..0909e37 100644 --- a/test_provision.sh +++ b/test_provision.sh @@ -1,7 +1,7 @@ # installs and fires up elasticsearch sudo sh -c "apt-get update -y" sudo sh -c "wget -O - http://packages.elasticsearch.org/GPG-KEY-elasticsearch | apt-key add -" -sudo sh -c "echo deb http://packages.elasticsearch.org/elasticsearch/0.90/debian stable main \ >> /etc/apt/sources.list" +sudo sh -c "echo deb http://packages.elasticsearch.org/elasticsearch/1.0/debian stable main \ >> /etc/apt/sources.list" sudo sh -c "apt-get update -y" sudo sh -c "apt-get install -y openjdk-7-jre elasticsearch" sudo sh -c "update-rc.d elasticsearch defaults 95 10" diff --git a/tests/do_not_test_mappings_yet.py b/tests/do_not_test_mappings_yet.py new file mode 100644 index 0000000..44c197d --- /dev/null +++ b/tests/do_not_test_mappings_yet.py @@ -0,0 +1,99 @@ +import datetime + +from django.test import TestCase +from django.utils import timezone + +from elastimorphic.tests.testapp.models import GrandchildIndexable, ChildIndexable, RelatedModel + + +class MappingTestCase(TestCase): + + maxDiff = 8000 + + def test_manual_mapping(self): + reference_mapping = { + "testapp_grandchildindexable": { + "_id": {"path": "childindexable_ptr_id"}, + "dynamic": "strict", + "properties": { + "id": {"type": "integer"}, + "polymorphic_ctype_id": {"type": "integer"}, + "parentindexable_ptr_id": {"type": "integer"}, + "childindexable_ptr_id": {"type": "integer"}, + + "foo": {"type": "string"}, + "bar": {"type": "integer", "store": "yes"}, + "baz": {"type": "date"}, + + "related": { + "type": "object", + "properties": { + "qux": { + "type": "string" + } + } + }, + } + } + } + + self.assertEqual(reference_mapping, GrandchildIndexable.get_mapping()) + + def test_automatic_mapping(self): + reference_mapping = { + "testapp_childindexable": { + "_id": {"path": "parentindexable_ptr_id"}, + "dynamic": "strict", + "properties": { + "id": {"type": "integer"}, + "polymorphic_ctype_id": {"type": "integer"}, + "parentindexable_ptr_id": {"type": "integer"}, + "foo": {"type": "string"}, + "bar": {"type": "integer"} + } + } + } + + self.assertDictEqual(reference_mapping, ChildIndexable.get_mapping()) + + def test_extract_document(self): + + related = RelatedModel.objects.create(qux="qux") + test_obj = GrandchildIndexable( + foo="Testing", + bar=7, + baz=datetime.datetime(year=2014, month=4, day=23, hour=9).replace(tzinfo=timezone.utc), + related=related + ) + test_obj.save(index=False) + reference_document = { + "id": test_obj.pk, + "parentindexable_ptr_id": test_obj.pk, + "childindexable_ptr_id": test_obj.pk, + "polymorphic_ctype_id": test_obj.polymorphic_ctype_id, + + "foo": "Testing", + "bar": 7, + "baz": "2014-04-23T09:00:00+00:00", + + "related_id": related.id + } + self.assertEqual(reference_document, test_obj.extract_document()) + + # def test_load_document(self): + # mapping = Mapping(model=GrandchildIndexable) + # related = RelatedModel.objects.create(qux="qux") + # test_obj = GrandchildIndexable( + # foo="Testing", + # bar=7, + # baz=datetime.datetime(year=2014, month=4, day=23, hour=9).replace(tzinfo=utc), + # related=related + # ) + # test_obj.save(index=False) + # document = mapping.extract_document(test_obj) + + # loaded_object = mapping.load_document(document) + # self.assertEqual(loaded_object.pk, test_obj.pk) + # self.assertEqual(loaded_object.foo, test_obj.foo) + # self.assertEqual(loaded_object, test_obj) + # self.assertEqual(loaded_object.related.qux, test_obj.related.qux) diff --git a/elastimorphic/tests/test_all.py b/tests/test_all.py similarity index 86% rename from elastimorphic/tests/test_all.py rename to tests/test_all.py index bd182d9..4bda56c 100644 --- a/elastimorphic/tests/test_all.py +++ b/tests/test_all.py @@ -6,15 +6,19 @@ from django.core.management import call_command from django.test import TestCase +import elasticsearch + from elasticutils import get_es -from pyelasticsearch.exceptions import ElasticHttpError from elastimorphic.conf import settings from elastimorphic.models import polymorphic_indexable_registry -from .base import BaseIndexableTestCase -from .testapp.models import ( - ChildIndexable, GrandchildIndexable, ParentIndexable, SeparateIndexable) +from elastimorphic.tests.base import BaseIndexableTestCase +from elastimorphic.tests.testapp.models import ( + ChildIndexable, + GrandchildIndexable, + ParentIndexable, + SeparateIndexable) class IndexableTestCase(BaseIndexableTestCase): @@ -52,13 +56,6 @@ def test_mapping_type_names(self): def test_get_index_mappings(self): pass - def test_primary_key_name_is_correct(self): - a, b, c = [klass.get_mapping().values()[0]["_id"]["path"] for klass in ( - ParentIndexable, ChildIndexable, GrandchildIndexable - )] - self.assertEqual(a, b) - self.assertEqual(b, c) - def test_search(self): self.assertEqual(ParentIndexable.search_objects.s().count(), 3) self.assertEqual(ParentIndexable.search_objects.query(bar=69).count(), 2) @@ -99,23 +96,24 @@ class ManagementTestCase(BaseIndexableTestCase): def test_synces(self): backup_settings = copy.copy(settings.ES_SETTINGS) + test_tokenizer = { + "type": "edgeNGram", + "min_gram": "3", + "max_gram": "4" + } settings.ES_SETTINGS.update({ "index": { "analysis": { "tokenizer": { - "edge_ngram_test_tokenizer": { - "type": "edgeNGram", - "min_gram": "3", - "max_gram": "4" - } + "edge_ngram_test_tokenizer": test_tokenizer } } } }) call_command("synces", self.index_suffix, force=True) - es_settings = self.es.get_settings(ParentIndexable.get_index_name()) + es_settings = self.es.indices.get_settings(index=ParentIndexable.get_index_name()) index_settings = es_settings[es_settings.keys()[0]]["settings"] - self.assertTrue("index.analysis.tokenizer.edge_ngram_test_tokenizer.type" in index_settings) + self.assertEqual(index_settings["index"]["analysis"]["tokenizer"]["edge_ngram_test_tokenizer"], test_tokenizer) settings.ES_SETTINGS = backup_settings @@ -156,7 +154,12 @@ def test_bulk_index(self): es = get_es(urls=settings.ES_URLS) doc = obj.extract_document() doc["foo"] = "DATA LOVERS" - es.update(obj.get_index_name(), obj.get_mapping_type_name(), obj.id, doc=doc, upsert=doc, refresh=True) + es.update( + index=obj.get_index_name(), + doc_type=obj.get_mapping_type_name(), + id=obj.id, + body=dict(doc=doc, doc_as_upsert=True), + refresh=True) # Make sure the bad data works self.assertEqual(ParentIndexable.search_objects.query(foo__match="DATA LOVERS").count(), 1) @@ -211,9 +214,13 @@ def test_index_upgrade(self): class TestDynamicMappings(BaseIndexableTestCase): + maxDiff = 2000 + def test_bad_index(self): """Check to make sure that the mappings are strict""" - mapping = self.es.get_mapping(ParentIndexable.get_index_name(), ParentIndexable.get_mapping_type_name()) + index_mapping = self.es.indices.get_mapping(index=ParentIndexable.get_index_name(), doc_type=ParentIndexable.get_mapping_type_name()) + alias_name = index_mapping.keys()[0] + mapping = index_mapping[alias_name]["mappings"] self.assertDictEqual(mapping, ParentIndexable.get_mapping()) obj = ParentIndexable.objects.create(foo="Fighters") @@ -221,16 +228,17 @@ def test_bad_index(self): doc = obj.extract_document() doc["extra"] = "Just an additional string" - with self.assertRaises(ElasticHttpError): + with self.assertRaises(elasticsearch.RequestError): self.es.update( obj.get_index_name(), obj.get_mapping_type_name(), obj.id, - doc=doc, - upsert=doc + body=dict(doc=doc, doc_as_upsert=True) ) - mapping = self.es.get_mapping(ParentIndexable.get_index_name(), ParentIndexable.get_mapping_type_name()) + index_mapping = self.es.indices.get_mapping(index=ParentIndexable.get_index_name(), doc_type=ParentIndexable.get_mapping_type_name()) + alias_name = index_mapping.keys()[0] + mapping = index_mapping[alias_name]["mappings"] self.assertDictEqual(mapping, ParentIndexable.get_mapping())