Skip to content

Commit

Permalink
Port to opensearch-py library
Browse files Browse the repository at this point in the history
  • Loading branch information
yakky committed Mar 12, 2024
1 parent a51c13f commit 5933723
Show file tree
Hide file tree
Showing 40 changed files with 203 additions and 175 deletions.
14 changes: 7 additions & 7 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,18 @@ jobs:
matrix:
python-version: ["3.8", "3.9", "3.10", "3.11"]
django-version: ["3.2", "4.1", "4.2"]
es-dsl-version: ["6.4", "7.4"]
es-version: ["8.10.2"]
open-dsl-version: ["2.1.1", "2.4.2"]

exclude:
- python-version: "3.11"
django-version: "3.2"

steps:
- name: Install and Run Elasticsearch
uses: elastic/elastic-github-actions/elasticsearch@master
- name: Install and Run Opensearch
uses: esmarkowski/opensearch-github-[email protected]
with:
stack-version: ${{ matrix.es-version }}
version: 2.12.0
security-disabled: true

- uses: actions/checkout@v4

Expand All @@ -47,10 +47,10 @@ jobs:
run: |
python -m pip install --upgrade pip
python -m pip install "Django==${{ matrix.django-version }}"
python -m pip install "elasticsearch-dsl==${{ matrix.es-dsl-version }}"
python -m pip install "opensearchpy==${{ matrix.es-dsl-version }}"
python -m pip install -r requirements_test.txt
- name: Run tests with Python ${{ matrix.python-version }} and Django ${{ matrix.django-version }} and elasticsearch-dsl-py ${{ matrix.es-dsl-version }}
- name: Run tests with Python ${{ matrix.python-version }} and Django ${{ matrix.django-version }} and opensearchpy ${{ matrix.es-dsl-version }}
run: |
TOX_ENV=$(echo "py${{ matrix.python-version }}-django-${{ matrix.django-version }}-es${{ matrix.es-dsl-version }}" | tr -d .)
python -m tox -e $TOX_ENV -- --elasticsearch
Expand Down
2 changes: 1 addition & 1 deletion HISTORY.rst
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ History
~~~~~~~~~~~~~~~~~~
* Support for Django `DecimalField` #141
* Indexing speedup by using `parallel` indexing. #213.
Now you can pass `--parallel` or set `ELASTICSEARCH_DSL_PARALLEL`
Now you can pass `--parallel` or set `OPENSEARCH_DSL_PARALLEL`
in your settings to get indexing speed boost while indexing
through management command.
* Fixing name resolution in management command #206
Expand Down
2 changes: 1 addition & 1 deletion MANIFEST.in
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@ include CONTRIBUTING.rst
include HISTORY.rst
include LICENSE
include README.rst
recursive-include django_elasticsearch_dsl *.html *.png *.gif *js *.css *jpg *jpeg *svg *py
recursive-include django_opensearch_dsl *.html *.png *.gif *js *.css *jpg *jpeg *svg *py
File renamed without changes.
19 changes: 9 additions & 10 deletions django_elasticsearch_dsl/apps.py → django_opensearch_dsl/apps.py
Original file line number Diff line number Diff line change
@@ -1,36 +1,35 @@
from django.apps import AppConfig
from django.conf import settings
from django.utils.module_loading import import_string

from elasticsearch_dsl.connections import connections
from opensearchpy.connection import connections


class DEDConfig(AppConfig):
name = 'django_elasticsearch_dsl'
verbose_name = "Django elasticsearch-dsl"
name = 'django_opensearch_dsl'
verbose_name = "Django opensearch-dsl"
signal_processor = None

def ready(self):
self.module.autodiscover()
connections.configure(**settings.ELASTICSEARCH_DSL)
connections.configure(**settings.OPENSEARCH_DSL)
# Setup the signal processor.
if not self.signal_processor:
signal_processor_path = getattr(
settings,
'ELASTICSEARCH_DSL_SIGNAL_PROCESSOR',
'django_elasticsearch_dsl.signals.RealTimeSignalProcessor'
'OPENSEARCH_DSL_SIGNAL_PROCESSOR',
'django_opensearch_dsl.signals.RealTimeSignalProcessor'
)
signal_processor_class = import_string(signal_processor_path)
self.signal_processor = signal_processor_class(connections)

@classmethod
def autosync_enabled(cls):
return getattr(settings, 'ELASTICSEARCH_DSL_AUTOSYNC', True)
return getattr(settings, 'OPENSEARCH_DSL_AUTOSYNC', True)

@classmethod
def default_index_settings(cls):
return getattr(settings, 'ELASTICSEARCH_DSL_INDEX_SETTINGS', {})
return getattr(settings, 'OPENSEARCH_DSL_INDEX_SETTINGS', {})

@classmethod
def auto_refresh_enabled(cls):
return getattr(settings, 'ELASTICSEARCH_DSL_AUTO_REFRESH', True)
return getattr(settings, 'OPENSEARCH_DSL_AUTO_REFRESH', True)
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@

from django import VERSION as DJANGO_VERSION
from django.db import models
from elasticsearch.helpers import bulk, parallel_bulk
from elasticsearch_dsl import Document as DSLDocument
from opensearchpy.helpers import bulk, parallel_bulk
from opensearchpy import Document as DSLDocument
from six import iteritems

from .exceptions import ModelFieldNotMappedError
from .exceptions import ModelFieldNotMappedError, RedeclaredFieldError
from .fields import (
BooleanField,
DateField,
Expand Down Expand Up @@ -150,7 +150,7 @@ def prepare(self, instance):
@classmethod
def get_model_field_class_to_field_class(cls):
"""
Returns dict of relationship from model field class to elasticsearch
Returns dict of relationship from model field class to opensearch
field class
You may want to override this if you have model field class not included
Expand All @@ -161,7 +161,7 @@ def get_model_field_class_to_field_class(cls):
@classmethod
def to_field(cls, field_name, model_field):
"""
Returns the elasticsearch field instance appropriate for the model
Returns the opensearch field instance appropriate for the model
field class. This is a good place to hook into if you have more complex
model field to ES field logic
"""
Expand Down Expand Up @@ -200,7 +200,7 @@ def parallel_bulk(self, actions, **kwargs):
def generate_id(cls, object_instance):
"""
The default behavior is to use the Django object's pk (id) as the
elasticseach index id (_id). If needed, this method can be overloaded
opensearch index id (_id). If needed, this method can be overloaded
to change this default behavior.
"""
return object_instance.pk
Expand All @@ -219,10 +219,10 @@ def _get_actions(self, object_list, action):
for object_instance in object_list:
if action == 'delete' or self.should_index_object(object_instance):
yield self._prepare_action(object_instance, action)

def get_actions(self, object_list, action):
"""
Generate the elasticsearch payload.
Generate the opensearch payload.
"""
return self._get_actions(object_list, action)

Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,16 +1,12 @@
from types import MethodType

import django
from django.core.exceptions import ObjectDoesNotExist
from django.db import models
from django.db.models.fields.files import FieldFile

if django.VERSION < (4, 0):
from django.utils.encoding import force_text as force_str
else:
from django.utils.encoding import force_str
from django.utils.encoding import force_str
from django.utils.functional import Promise
from elasticsearch_dsl.field import (
from opensearchpy.helpers.field import (
Boolean,
Byte,
Completion,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from copy import deepcopy

from elasticsearch_dsl import Index as DSLIndex
from opensearchpy import Index as DSLIndex
from six import python_2_unicode_compatible

from .apps import DEDConfig
Expand Down
Original file line number Diff line number Diff line change
@@ -1,15 +1,15 @@
from __future__ import unicode_literals, absolute_import
from datetime import datetime

from elasticsearch_dsl import connections
from opensearchpy.connection import connections
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from six.moves import input
from ...registries import registry


class Command(BaseCommand):
help = 'Manage elasticsearch index.'
help = 'Manage opensearch index.'

def __init__(self, *args, **kwargs):
super(Command, self).__init__(*args, **kwargs)
Expand All @@ -21,28 +21,28 @@ def add_arguments(self, parser):
metavar='app[.model]',
type=str,
nargs='*',
help="Specify the model or app to be updated in elasticsearch"
help="Specify the model or app to be updated in opensearch"
)
parser.add_argument(
'--create',
action='store_const',
dest='action',
const='create',
help="Create the indices in elasticsearch"
help="Create the indices in opensearch"
)
parser.add_argument(
'--populate',
action='store_const',
dest='action',
const='populate',
help="Populate elasticsearch indices with models data"
help="Populate opensearch indices with models data"
)
parser.add_argument(
'--delete',
action='store_const',
dest='action',
const='delete',
help="Delete the indices in elasticsearch"
help="Delete the indices in opensearch"
)
parser.add_argument(
'--rebuild',
Expand Down Expand Up @@ -84,7 +84,7 @@ def add_arguments(self, parser):
'--use-alias' args
"""
)
parser.set_defaults(parallel=getattr(settings, 'ELASTICSEARCH_DSL_PARALLEL', False))
parser.set_defaults(parallel=getattr(settings, 'OPENSEARCH_DSL_PARALLEL', False))
parser.add_argument(
'--refresh',
action='store_true',
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@

from django.core.exceptions import ObjectDoesNotExist
from django.core.exceptions import ImproperlyConfigured
from elasticsearch_dsl import AttrDict
from opensearchpy import AttrDict
from six import itervalues, iterkeys, iteritems

from django_elasticsearch_dsl.exceptions import RedeclaredFieldError
from .apps import DEDConfig
from .exceptions import RedeclaredFieldError


class DocumentRegistry(object):
Expand Down Expand Up @@ -129,7 +129,7 @@ def delete_related(self, instance, **kwargs):

def update(self, instance, **kwargs):
"""
Update all the elasticsearch documents attached to this model (if their
Update all the opensearch documents attached to this model (if their
ignore_signals flag allows it)
"""
if not DEDConfig.autosync_enabled():
Expand All @@ -142,7 +142,7 @@ def update(self, instance, **kwargs):

def delete(self, instance, **kwargs):
"""
Delete all the elasticsearch documents attached to this model (if their
Delete all the opensearch documents attached to this model (if their
ignore_signals flag allows it)
"""
self.update(instance, action="delete", **kwargs)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from django.db.models import Case, When
from django.db.models.fields import IntegerField

from elasticsearch_dsl import Search as DSLSearch
from opensearchpy import Search as DSLSearch


class Search(DSLSearch):
Expand All @@ -16,7 +16,7 @@ def _clone(self):

def filter_queryset(self, queryset, keep_search_order=True):
"""
Filter an existing django queryset using the elasticsearch result.
Filter an existing django queryset using the opensearch result.
It costs a query to the sql db.
"""
s = self
Expand Down Expand Up @@ -52,7 +52,7 @@ def _get_queryset(self):

def to_queryset(self, keep_order=True):
"""
Return a django queryset from the elasticsearch result.
Return a django queryset from the opensearch result.
It costs a query to the sql db.
"""
qs = self._get_queryset()
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# encoding: utf-8
"""
A convenient way to attach django-elasticsearch-dsl to Django's signals and
A convenient way to attach django-opensearch-dsl to Django's signals and
cause things to index.
"""

Expand Down Expand Up @@ -169,8 +169,8 @@ def prepare_registry_delete_related_task(self, instance):
def registry_delete_task(doc_label, data):
"""
Handle the bulk delete data on the registry as a Celery task.
The different implementations used are due to the difference between delete and update operations.
The update operation can re-read the updated data from the database to ensure eventual consistency,
The different implementations used are due to the difference between delete and update operations.
The update operation can re-read the updated data from the database to ensure eventual consistency,
but the delete needs to be processed before the database record is deleted to obtain the associated data.
"""
doc_instance = import_module(doc_label)
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import re

from django.test.utils import captured_stderr
from elasticsearch_dsl.connections import connections
from opensearchpy.connection.connections import connections

from ..registries import registry

Expand Down
46 changes: 46 additions & 0 deletions docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
---
version: '3'
services:
opensearch-node1:
image: opensearchproject/opensearch:2.11.1
container_name: opensearch-node1
environment:
- cluster.name=opensearch-cluster
- node.name=opensearch-node1
- discovery.type=single-node
- bootstrap.memory_lock=true # along with the memlock settings below, disables swapping
- OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
- DISABLE_INSTALL_DEMO_CONFIG=true
- DISABLE_SECURITY_PLUGIN=true
ulimits:
memlock:
soft: -1
hard: -1
nofile:
soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems
hard: 65536
volumes:
- opensearch-data1:/usr/share/opensearch/data
ports:
- 9200:9200
- 9600:9600 # required for Performance Analyzer
networks:
- opensearch-net
opensearch-dashboards:
image: opensearchproject/opensearch-dashboards:2.11.1
container_name: opensearch-dashboards
ports:
- 5601:5601
expose:
- '5601'
environment:
OPENSEARCH_HOSTS: '["http://opensearch-node1:9200"]'
networks:
- opensearch-net

volumes:
opensearch-data1:
opensearch-data2:

networks:
opensearch-net:
Loading

0 comments on commit 5933723

Please sign in to comment.