Jd 2023 10 search result title

README.md

@@ -17,7 +17,7 @@ Start local server with elastic search:
 Open two terminal windows.
 1. Start elastic search in the first one:
 ```
-$ sudo docker run -p 127.0.0.1:9200:9200 -p 127.0.0.1:9300:9300 -e "discovery.type=single-node" -e "logger.level=DEBUG" docker.elastic.co/elasticsearch/elasticsearch:7.17.10
+$ sudo docker run -p 127.0.0.1:9200:9200 -p 127.0.0.1:9300:9300 -e "discovery.type=single-node" -e "logger.level=DEBUG" -e 'xpack.security.enabled=false' -e 'xpack.security.enrollment.enabled=false' docker.elastic.co/elasticsearch/elasticsearch:8.10.2
 ```
 
 2. Run the server in the second one. But before you do that, update the search index.

apps/contrib/elasticsearch.py

@@ -1,22 +1,9 @@
-from wagtail.search.backends.elasticsearch7 import \
-    Elasticsearch7AutocompleteQueryCompiler
-from wagtail.search.backends.elasticsearch7 import Elasticsearch7SearchBackend
-from wagtail.search.backends.elasticsearch7 import Elasticsearch7SearchResults
+from wagtail.search.backends.elasticsearch8 import Elasticsearch8SearchBackend
+from wagtail.search.backends.elasticsearch8 import Elasticsearch8SearchResults
 
 
 # Code below taken and modified from wagtails elasticsearch backend
-class ElasticsearchCustomQueryCompiler(
-    Elasticsearch7AutocompleteQueryCompiler
-):
-    # we override this to remove the check if the field exists on the model
-    # as we use a glob pattern (e.g. *page_title_en)
-    def check(self):
-        self._get_filters_from_where_node(self.queryset.query.where,
-                                          check_only=True)
-        list(self._get_order_by())
-
-
-class ElasticsearchResults(Elasticsearch7SearchResults):
+class ElasticsearchResults(Elasticsearch8SearchResults):
     def _get_es_body(self, for_count=False):
         body = {
             'query': self.query_compiler.get_query()
@@ -64,9 +51,8 @@ def _get_results_from_hits(self, hits):
                 yield result
 
 
-class ElasticsearchCustomSearchBackend(Elasticsearch7SearchBackend):
+class ElasticsearchCustomSearchBackend(Elasticsearch8SearchBackend):
     results_class = ElasticsearchResults
-    autocomplete_query_compiler_class = ElasticsearchCustomQueryCompiler
 
 
 SearchBackend = ElasticsearchCustomSearchBackend

apps/contrib/translations.py

@@ -35,19 +35,13 @@ def __get__(self, instance, owner):
             return de
 
 
-def get_search_fields() -> List[str]:
+def get_search_fields(fields: List[str]) -> List[str]:
     """Create a list of fields to search in the current language of the user.
 
     Adds _edgengrams as otherwise autocomplete() won't work.
 
     Returns:
         List of fields with the correct language code set
     """
-    fields = [
-        '*page_title_',
-        '*page_intro_'
-        '*subtitle_',
-        '*body_',
-    ]
     lang = translation.get_language()
-    return [f + lang + '_edgengrams' for f in fields]
+    return [f + "_" + lang for f in fields]

apps/contrib/views.py

@@ -1,15 +1,77 @@
+from itertools import chain
+from typing import Any
+from typing import List
+
 from django.views.generic import ListView
-from wagtail.models import Page
 from wagtail.search.backends import get_search_backend
+from wagtail.search.query import Fuzzy
 
 from apps.contrib.translations import get_search_fields
+from apps.gruenbuch.models import GruenbuchDetailPage
+from apps.gruenbuch.models import GruenbuchIndexPage
+from apps.gruenbuch.models import GruenbuchOverviewPage
+from apps.home.models import DetailPage
+from apps.home.models import HomePage
+from apps.home.models import MicrositeDetailPage
+from apps.home.models import MicrositeOverviewPage
+from apps.home.models import OverviewPage
+from apps.home.models import SimplePage
+from apps.measures.models import MeasuresDetailPage
+from apps.measures.models import MeasuresOverviewPage
 
 
 class SearchResultsView(ListView):
     template_name = 'search_results.html'
 
-    def get_queryset(self):
+    def get_queryset(self) -> List[Any]:
+        """Return list with all search results from the different pages.
+
+        This is an inefficient workaround for
+        `sb.search(query, Page, fields=get_search_fields())` not
+        returning the title in the correct language as the wagtail.models.Page
+        doesn't know about it. The proper way would be to subclass Page with
+        the translated title field and make all custom models inherit from that
+        subclass.
+
+         Returns:
+             List of all search results
+
+        """
         sb = get_search_backend()
         query = self.request.GET.get('q', '')
-        res = sb.autocomplete(query, Page, fields=get_search_fields())
+        # FIXME: Should use one query over a common page model inherited by all
+        # models below
+        res = list(chain(
+            sb.search(Fuzzy(query), HomePage, fields=get_search_fields(
+                ['page_title', 'page_subtitle', 'body'])),
+            sb.search(Fuzzy(query), OverviewPage, fields=get_search_fields(
+                ['page_title', 'page_intro', 'body'])),
+            sb.search(Fuzzy(query), DetailPage, fields=get_search_fields(
+                ['page_title', 'body'])),
+            sb.search(Fuzzy(query), SimplePage, fields=get_search_fields(
+                ['page_title', 'body'])),
+            sb.search(Fuzzy(query), MicrositeOverviewPage,
+                      fields=get_search_fields(
+                      ['page_title', 'page_intro', 'body'])),
+            sb.search(Fuzzy(query), MicrositeDetailPage,
+                      fields=get_search_fields(
+                      ['page_title', 'body'])),
+            sb.search(Fuzzy(query), GruenbuchOverviewPage,
+                      fields=get_search_fields(
+                      ['page_title', 'page_intro', 'body'])),
+            sb.search(Fuzzy(query), GruenbuchIndexPage,
+                      fields=get_search_fields(
+                      ['page_title', 'page_intro', 'body'])),
+            sb.search(Fuzzy(query), GruenbuchDetailPage,
+                      fields=get_search_fields(
+                      ['page_title', 'subtitle', 'body'])),
+            sb.search(Fuzzy(query), MeasuresOverviewPage,
+                      fields=get_search_fields(
+                      ['page_title', 'page_intro', 'swiper', 'teaser'])),
+            sb.search(Fuzzy(query), MeasuresDetailPage,
+                      fields=get_search_fields(
+                      ['page_title', 'body', 'body_participation',
+                       'body_effect', 'contact_organisation_name']) +
+                      ['contact_name']),
+        ))
         return res

apps/gruenbuch/models.py

@@ -127,12 +127,18 @@ class GruenbuchOverviewPage(TranslatedMetadataPageMixin, Page):
         index.AutocompleteField('page_intro_de'),
         index.AutocompleteField('page_intro_en'),
         index.AutocompleteField('page_intro_de_ls'),
+        index.AutocompleteField('body_de'),
+        index.AutocompleteField('body_en'),
+        index.AutocompleteField('body_de_ls'),
         index.SearchField('page_title_de'),
         index.SearchField('page_title_en'),
         index.SearchField('page_title_de_ls'),
         index.SearchField('page_intro_de'),
         index.SearchField('page_intro_en'),
         index.SearchField('page_intro_de_ls'),
+        index.SearchField('body_de'),
+        index.SearchField('body_en'),
+        index.SearchField('body_de_ls'),
     ]
 
 

apps/home/models.py

@@ -269,12 +269,19 @@ class OverviewPage(TranslatedMetadataPageMixin, Page):
         index.AutocompleteField('page_intro_de'),
         index.AutocompleteField('page_intro_en'),
         index.AutocompleteField('page_intro_de_ls'),
+        index.AutocompleteField('body_de'),
+        index.AutocompleteField('body_en'),
+        index.AutocompleteField('body_de_ls'),
         index.SearchField('page_title_de'),
         index.SearchField('page_title_en'),
         index.SearchField('page_title_de_ls'),
         index.SearchField('page_intro_de'),
         index.SearchField('page_intro_en'),
         index.SearchField('page_intro_de_ls'),
+        index.SearchField('body_de'),
+        index.SearchField('body_en'),
+        index.SearchField('body_de_ls'),
+
     ]
 
 
@@ -346,9 +353,15 @@ class DetailPage(TranslatedMetadataPageMixin, Page):
     subpage_types = []
 
     search_fields = Page.search_fields + [
+        index.AutocompleteField('page_title_de'),
+        index.AutocompleteField('page_title_en'),
+        index.AutocompleteField('page_title_de_ls'),
         index.AutocompleteField('body_de'),
         index.AutocompleteField('body_en'),
         index.AutocompleteField('body_de_ls'),
+        index.SearchField('page_title_de'),
+        index.SearchField('page_title_en'),
+        index.SearchField('page_title_de_ls'),
         index.SearchField('body_de'),
         index.SearchField('body_en'),
         index.SearchField('body_de_ls')
@@ -417,9 +430,15 @@ class SimplePage(TranslatedMetadataPageMixin, Page):
     subpage_types = []
 
     search_fields = Page.search_fields + [
+        index.AutocompleteField('page_title_de'),
+        index.AutocompleteField('page_title_en'),
+        index.AutocompleteField('page_title_de_ls'),
         index.AutocompleteField('body_de'),
         index.AutocompleteField('body_en'),
         index.AutocompleteField('body_de_ls'),
+        index.SearchField('page_title_de'),
+        index.SearchField('page_title_en'),
+        index.SearchField('page_title_de_ls'),
         index.SearchField('body_de'),
         index.SearchField('body_en'),
         index.SearchField('body_de_ls')
@@ -543,12 +562,18 @@ class MicrositeOverviewPage(TranslatedMetadataPageMixin, Page):
         index.AutocompleteField('page_intro_de'),
         index.AutocompleteField('page_intro_en'),
         index.AutocompleteField('page_intro_de_ls'),
+        index.AutocompleteField('body_de'),
+        index.AutocompleteField('body_en'),
+        index.AutocompleteField('body_de_ls'),
         index.SearchField('page_title_de'),
         index.SearchField('page_title_en'),
         index.SearchField('page_title_de_ls'),
         index.SearchField('page_intro_de'),
         index.SearchField('page_intro_en'),
         index.SearchField('page_intro_de_ls'),
+        index.SearchField('body_de'),
+        index.SearchField('body_en'),
+        index.SearchField('body_de_ls')
     ]
 
 
@@ -630,9 +655,15 @@ class MicrositeDetailPage(TranslatedMetadataPageMixin, Page):
     subpage_types = []
 
     search_fields = Page.search_fields + [
+        index.AutocompleteField('page_title_de'),
+        index.AutocompleteField('page_title_en'),
+        index.AutocompleteField('page_title_de_ls'),
         index.AutocompleteField('body_de'),
         index.AutocompleteField('body_en'),
         index.AutocompleteField('body_de_ls'),
+        index.SearchField('page_title_de'),
+        index.SearchField('page_title_en'),
+        index.SearchField('page_title_de_ls'),
         index.SearchField('body_de'),
         index.SearchField('body_en'),
         index.SearchField('body_de_ls')

apps/measures/models.py

@@ -356,12 +356,24 @@ def get_context(self, request):
         index.AutocompleteField('page_intro_de'),
         index.AutocompleteField('page_intro_en'),
         index.AutocompleteField('page_intro_de_ls'),
+        index.AutocompleteField('swiper_de'),
+        index.AutocompleteField('swiper_en'),
+        index.AutocompleteField('swiper_de_ls'),
+        index.AutocompleteField('teaser_de'),
+        index.AutocompleteField('teaser_en'),
+        index.AutocompleteField('teaser_de_ls'),
         index.SearchField('page_title_de'),
         index.SearchField('page_title_en'),
         index.SearchField('page_title_de_ls'),
         index.SearchField('page_intro_de'),
         index.SearchField('page_intro_en'),
         index.SearchField('page_intro_de_ls'),
+        index.SearchField('swiper_de'),
+        index.SearchField('swiper_en'),
+        index.SearchField('swiper_de_ls'),
+        index.SearchField('teaser_de'),
+        index.SearchField('teaser_en'),
+        index.SearchField('teaser_de_ls'),
     ]
 
 

changelog/852.md

@@ -0,0 +1,3 @@
+### Fixed
+
+- search results show page title in wrong language (#852)

changelog/_1111.md

@@ -0,0 +1,8 @@
+### Fixed
+
+- search: add missing search indices for some model fields
+- make search results better by using search with Fuzzy instead of autocomplete
+
+### Removed
+
+- removed the no longer needed ElasticSearchCustomQueryCompiler

changelog/_11112.md

@@ -0,0 +1,3 @@
+### Changed
+
+- update to elasticsearch 8

digitalstrategie/templates/search_results.html

@@ -57,7 +57,7 @@ <h2 class="title">
                {% if result.highlight %}
                     <div class="searchresult-item">
                         <h3 class="title">
-                            <a href="{% pageurl result %}">{{ result }}</a>
+                            <a href="{% pageurl result %}">{{ result.page_title }}</a>
                         </h3>
                         <p class="text">
                             {{ result.highlight | safe }}
@@ -68,7 +68,7 @@ <h3 class="title">
                               aria-hidden="true"
                               href="{% pageurl result %}"
                             >
-                                {{ result }}
+                                {{ result.page_title }}
                             </a>
                         </p>
                     </div>

requirements/base.txt

@@ -7,6 +7,6 @@ django-widget-tweaks==1.4.12
 sentry-sdk==1.32.0
 wagtail==5.1.3
 whitenoise==6.6.0
-elasticsearch==7.17.9
+elasticsearch==8.10.1
 wagtail-metadata==4.0.3
 sib-api-v3-sdk==7.6.0

tests/contrib/test_translations.py

@@ -5,12 +5,12 @@
 
 def test_get_search_fields():
     with translation.override("en"):
-        fields = get_search_fields()
-        assert all(map(lambda field: field.endswith("_en_edgengrams"), fields))
+        fields = get_search_fields(["body", "page_title"])
+        assert all(map(lambda field: field.endswith("_en"), fields))
     with translation.override("de"):
-        fields = get_search_fields()
-        assert all(map(lambda field: field.endswith("_de_edgengrams"), fields))
+        fields = get_search_fields(["body", "page_title"])
+        assert all(map(lambda field: field.endswith("_de"), fields))
     with translation.override("de_ls"):
-        fields = get_search_fields()
-        assert all(map(lambda field: field.endswith("_de-ls_edgengrams"),
+        fields = get_search_fields(["body", "page_title"])
+        assert all(map(lambda field: field.endswith("_de-ls"),
                        fields))