search: refactor search to improve results

* add some missing fields to index
* remove no longer needed ElasticSearchCustomQueryCompiler

apps/contrib/elasticsearch.py

@@ -1,21 +1,8 @@
-from wagtail.search.backends.elasticsearch7 import \
-    Elasticsearch7AutocompleteQueryCompiler
 from wagtail.search.backends.elasticsearch7 import Elasticsearch7SearchBackend
 from wagtail.search.backends.elasticsearch7 import Elasticsearch7SearchResults
 
 
 # Code below taken and modified from wagtails elasticsearch backend
-class ElasticsearchCustomQueryCompiler(
-    Elasticsearch7AutocompleteQueryCompiler
-):
-    # we override this to remove the check if the field exists on the model
-    # as we use a glob pattern (e.g. *page_title_en)
-    def check(self):
-        self._get_filters_from_where_node(self.queryset.query.where,
-                                          check_only=True)
-        list(self._get_order_by())
-
-
 class ElasticsearchResults(Elasticsearch7SearchResults):
     def _get_es_body(self, for_count=False):
         body = {
@@ -66,7 +53,6 @@ def _get_results_from_hits(self, hits):
 
 class ElasticsearchCustomSearchBackend(Elasticsearch7SearchBackend):
     results_class = ElasticsearchResults
-    autocomplete_query_compiler_class = ElasticsearchCustomQueryCompiler
 
 
 SearchBackend = ElasticsearchCustomSearchBackend

apps/contrib/translations.py

@@ -35,23 +35,13 @@ def __get__(self, instance, owner):
             return de
 
 
-def get_search_fields() -> List[str]:
+def get_search_fields(fields: List[str]) -> List[str]:
     """Create a list of fields to search in the current language of the user.
 
     Adds _edgengrams as otherwise autocomplete() won't work.
 
     Returns:
         List of fields with the correct language code set
     """
-    fields = [
-        '*page_title_',
-        '*page_intro_'
-        '*subtitle_',
-        '*body_',
-    ]
     lang = translation.get_language()
-    localized_fields = []
-    for f in fields:
-        localized_fields.append(f + lang)
-    localized_fields.append(f + lang + "_edgengrams")
-    return localized_fields
+    return [f + "_" + lang for f in fields]

apps/contrib/views.py

@@ -4,6 +4,7 @@
 
 from django.views.generic import ListView
 from wagtail.search.backends import get_search_backend
+from wagtail.search.query import Fuzzy
 
 from apps.contrib.translations import get_search_fields
 from apps.gruenbuch.models import GruenbuchDetailPage
@@ -26,7 +27,7 @@ def get_queryset(self) -> List[Any]:
         """Return list with all search results from the different pages.
 
         This is an inefficient workaround for
-        `sb.autocomplete(query, Page, fields=get_search_fields())` not
+        `sb.search(query, Page, fields=get_search_fields())` not
         returning the title in the correct language as the wagtail.models.Page
         doesn't know about it. The proper way would be to subclass Page with
         the translated title field and make all custom models inherit from that
@@ -41,23 +42,36 @@ def get_queryset(self) -> List[Any]:
         # FIXME: Should use one query over a common page model inherited by all
         # models below
         res = list(chain(
-            sb.autocomplete(query, HomePage, fields=get_search_fields()),
-            sb.autocomplete(query, OverviewPage, fields=get_search_fields()),
-            sb.autocomplete(query, DetailPage, fields=get_search_fields()),
-            sb.autocomplete(query, MicrositeOverviewPage,
-                            fields=get_search_fields()),
-            sb.autocomplete(query, MicrositeDetailPage,
-                            fields=get_search_fields()),
-            sb.autocomplete(query, SimplePage, fields=get_search_fields()),
-            sb.autocomplete(query, GruenbuchOverviewPage,
-                            fields=get_search_fields()),
-            sb.autocomplete(query, GruenbuchDetailPage,
-                            fields=get_search_fields()),
-            sb.autocomplete(query, GruenbuchIndexPage,
-                            fields=get_search_fields()),
-            sb.autocomplete(query, MeasuresOverviewPage,
-                            fields=get_search_fields()),
-            sb.autocomplete(query, MeasuresDetailPage,
-                            fields=get_search_fields())
+            sb.search(Fuzzy(query), HomePage, fields=get_search_fields(
+                ['page_title', 'page_subtitle', 'body'])),
+            sb.search(Fuzzy(query), OverviewPage, fields=get_search_fields(
+                ['page_title', 'page_intro', 'body'])),
+            sb.search(Fuzzy(query), DetailPage, fields=get_search_fields(
+                ['page_title', 'body'])),
+            sb.search(Fuzzy(query), SimplePage, fields=get_search_fields(
+                ['page_title', 'body'])),
+            sb.search(Fuzzy(query), MicrositeOverviewPage,
+                      fields=get_search_fields(
+                      ['page_title', 'page_intro', 'body'])),
+            sb.search(Fuzzy(query), MicrositeDetailPage,
+                      fields=get_search_fields(
+                      ['page_title', 'body'])),
+            sb.search(Fuzzy(query), GruenbuchOverviewPage,
+                      fields=get_search_fields(
+                      ['page_title', 'page_intro', 'body'])),
+            sb.search(Fuzzy(query), GruenbuchIndexPage,
+                      fields=get_search_fields(
+                      ['page_title', 'page_intro', 'body'])),
+            sb.search(Fuzzy(query), GruenbuchDetailPage,
+                      fields=get_search_fields(
+                      ['page_title', 'subtitle', 'body'])),
+            sb.search(Fuzzy(query), MeasuresOverviewPage,
+                      fields=get_search_fields(
+                      ['page_title', 'page_intro', 'swiper', 'teaser'])),
+            sb.search(Fuzzy(query), MeasuresDetailPage,
+                      fields=get_search_fields(
+                      ['page_title', 'body', 'body_participation',
+                       'body_effect', 'contact_organisation_name']) +
+                      ['contact_name']),
         ))
         return res

apps/gruenbuch/models.py

@@ -124,12 +124,18 @@ class GruenbuchOverviewPage(TranslatedMetadataPageMixin, Page):
         index.AutocompleteField('page_intro_de'),
         index.AutocompleteField('page_intro_en'),
         index.AutocompleteField('page_intro_de_ls'),
+        index.AutocompleteField('body_de'),
+        index.AutocompleteField('body_en'),
+        index.AutocompleteField('body_de_ls'),
         index.SearchField('page_title_de'),
         index.SearchField('page_title_en'),
         index.SearchField('page_title_de_ls'),
         index.SearchField('page_intro_de'),
         index.SearchField('page_intro_en'),
         index.SearchField('page_intro_de_ls'),
+        index.SearchField('body_de'),
+        index.SearchField('body_en'),
+        index.SearchField('body_de_ls'),
     ]
 
 

apps/home/models.py

@@ -265,12 +265,19 @@ class OverviewPage(TranslatedMetadataPageMixin, Page):
         index.AutocompleteField('page_intro_de'),
         index.AutocompleteField('page_intro_en'),
         index.AutocompleteField('page_intro_de_ls'),
+        index.AutocompleteField('body_de'),
+        index.AutocompleteField('body_en'),
+        index.AutocompleteField('body_de_ls'),
         index.SearchField('page_title_de'),
         index.SearchField('page_title_en'),
         index.SearchField('page_title_de_ls'),
         index.SearchField('page_intro_de'),
         index.SearchField('page_intro_en'),
         index.SearchField('page_intro_de_ls'),
+        index.SearchField('body_de'),
+        index.SearchField('body_en'),
+        index.SearchField('body_de_ls'),
+
     ]
 
 
@@ -342,9 +349,15 @@ class DetailPage(TranslatedMetadataPageMixin, Page):
     subpage_types = []
 
     search_fields = Page.search_fields + [
+        index.AutocompleteField('page_title_de'),
+        index.AutocompleteField('page_title_en'),
+        index.AutocompleteField('page_title_de_ls'),
         index.AutocompleteField('body_de'),
         index.AutocompleteField('body_en'),
         index.AutocompleteField('body_de_ls'),
+        index.SearchField('page_title_de'),
+        index.SearchField('page_title_en'),
+        index.SearchField('page_title_de_ls'),
         index.SearchField('body_de'),
         index.SearchField('body_en'),
         index.SearchField('body_de_ls')
@@ -413,9 +426,15 @@ class SimplePage(TranslatedMetadataPageMixin, Page):
     subpage_types = []
 
     search_fields = Page.search_fields + [
+        index.AutocompleteField('page_title_de'),
+        index.AutocompleteField('page_title_en'),
+        index.AutocompleteField('page_title_de_ls'),
         index.AutocompleteField('body_de'),
         index.AutocompleteField('body_en'),
         index.AutocompleteField('body_de_ls'),
+        index.SearchField('page_title_de'),
+        index.SearchField('page_title_en'),
+        index.SearchField('page_title_de_ls'),
         index.SearchField('body_de'),
         index.SearchField('body_en'),
         index.SearchField('body_de_ls')
@@ -536,12 +555,18 @@ class MicrositeOverviewPage(TranslatedMetadataPageMixin, Page):
         index.AutocompleteField('page_intro_de'),
         index.AutocompleteField('page_intro_en'),
         index.AutocompleteField('page_intro_de_ls'),
+        index.AutocompleteField('body_de'),
+        index.AutocompleteField('body_en'),
+        index.AutocompleteField('body_de_ls'),
         index.SearchField('page_title_de'),
         index.SearchField('page_title_en'),
         index.SearchField('page_title_de_ls'),
         index.SearchField('page_intro_de'),
         index.SearchField('page_intro_en'),
         index.SearchField('page_intro_de_ls'),
+        index.SearchField('body_de'),
+        index.SearchField('body_en'),
+        index.SearchField('body_de_ls')
     ]
 
 
@@ -622,9 +647,15 @@ class MicrositeDetailPage(TranslatedMetadataPageMixin, Page):
     subpage_types = []
 
     search_fields = Page.search_fields + [
+        index.AutocompleteField('page_title_de'),
+        index.AutocompleteField('page_title_en'),
+        index.AutocompleteField('page_title_de_ls'),
         index.AutocompleteField('body_de'),
         index.AutocompleteField('body_en'),
         index.AutocompleteField('body_de_ls'),
+        index.SearchField('page_title_de'),
+        index.SearchField('page_title_en'),
+        index.SearchField('page_title_de_ls'),
         index.SearchField('body_de'),
         index.SearchField('body_en'),
         index.SearchField('body_de_ls')

apps/measures/models.py

@@ -356,12 +356,24 @@ def get_context(self, request):
         index.AutocompleteField('page_intro_de'),
         index.AutocompleteField('page_intro_en'),
         index.AutocompleteField('page_intro_de_ls'),
+        index.AutocompleteField('swiper_de'),
+        index.AutocompleteField('swiper_en'),
+        index.AutocompleteField('swiper_de_ls'),
+        index.AutocompleteField('teaser_de'),
+        index.AutocompleteField('teaser_en'),
+        index.AutocompleteField('teaser_de_ls'),
         index.SearchField('page_title_de'),
         index.SearchField('page_title_en'),
         index.SearchField('page_title_de_ls'),
         index.SearchField('page_intro_de'),
         index.SearchField('page_intro_en'),
         index.SearchField('page_intro_de_ls'),
+        index.SearchField('swiper_de'),
+        index.SearchField('swiper_en'),
+        index.SearchField('swiper_de_ls'),
+        index.SearchField('teaser_de'),
+        index.SearchField('teaser_en'),
+        index.SearchField('teaser_de_ls'),
     ]
 
 

changelog/_1111.md

@@ -0,0 +1,8 @@
+### Fixed
+
+- search: add missing search indices for some model fields
+- make search results better by using search with Fuzzy instead of autocomplete
+
+### Removed
+
+- removed the no longer needed ElasticSearchCustomQueryCompiler