From 2c684ca07934cfbdd83b2f0310bb4966351cbb09 Mon Sep 17 00:00:00 2001 From: Arthur Pemberton Date: Fri, 17 Apr 2020 01:55:43 -0400 Subject: [PATCH 1/7] initial commit of the web api --- dbdb/api/api_v202004/__init__.py | 0 dbdb/api/api_v202004/apps.py | 5 + dbdb/api/api_v202004/serializers.py | 140 ++++++++++++++++++++++++++++ dbdb/api/api_v202004/urls.py | 15 +++ dbdb/api/api_v202004/views.py | 34 +++++++ dbdb/api/pagination.py | 12 +++ dbdb/api/urls.py | 9 ++ dbdb/core/models.py | 11 +++ dbdb/settings.py | 15 ++- dbdb/urls.py | 1 + 10 files changed, 239 insertions(+), 3 deletions(-) create mode 100644 dbdb/api/api_v202004/__init__.py create mode 100644 dbdb/api/api_v202004/apps.py create mode 100644 dbdb/api/api_v202004/serializers.py create mode 100644 dbdb/api/api_v202004/urls.py create mode 100644 dbdb/api/api_v202004/views.py create mode 100644 dbdb/api/pagination.py create mode 100644 dbdb/api/urls.py diff --git a/dbdb/api/api_v202004/__init__.py b/dbdb/api/api_v202004/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/dbdb/api/api_v202004/apps.py b/dbdb/api/api_v202004/apps.py new file mode 100644 index 00000000..86aae4f9 --- /dev/null +++ b/dbdb/api/api_v202004/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class ApiV202004Config(AppConfig): + name = 'api_v202004' diff --git a/dbdb/api/api_v202004/serializers.py b/dbdb/api/api_v202004/serializers.py new file mode 100644 index 00000000..7a8e0241 --- /dev/null +++ b/dbdb/api/api_v202004/serializers.py @@ -0,0 +1,140 @@ +# stdlib imports +import collections +# third-party imports +from rest_framework import serializers +from rest_framework.reverse import reverse +# project imports +from dbdb.core.models import FeatureOption +from dbdb.core.models import System +from dbdb.core.models import SystemFeature + + +# serializers + +class SystemSerializer(serializers.HyperlinkedModelSerializer): + + class Meta: + model = System + fields = [ + 'href', + 'name', + 'former_names', + 'start_year', + 'end_year', + 'description', + 'history', + 'acquired_by', + 'developer', + 'countries', + 'features', + 'project_types', + 'urls', + 'version', + ] + + href = serializers.SerializerMethodField() + former_names = serializers.SerializerMethodField() + start_year = serializers.SerializerMethodField() + end_year = serializers.SerializerMethodField() + description = serializers.SerializerMethodField() + history = serializers.SerializerMethodField() + acquired_by = serializers.SerializerMethodField() + developer = serializers.SerializerMethodField() + countries = serializers.SerializerMethodField() + features = serializers.SerializerMethodField() + project_types = serializers.SerializerMethodField() + urls = serializers.SerializerMethodField() + version = serializers.IntegerField(source='ver', read_only=True) + + def get_acquired_by(self, obj): + current = obj.get_current() + return current.acquired_by + + def get_countries(self, obj): + current = obj.get_current() + return list( map(str, current.countries) ) + + def get_description(self, obj): + current = obj.get_current() + return current.description + + def get_developer(self, obj): + current = obj.get_current() + return current.developer + + def get_features(self, obj): + current = obj.get_current() + + sysfeatures = SystemFeature.objects \ + .filter(system=current) \ + .select_related('feature') \ + .order_by('feature__slug') + + items = collections.OrderedDict() + for sysfeature in sysfeatures: + empty = True + + f = { + 'name': sysfeature.feature.label, + } + + if sysfeature.feature.multivalued: + f['values'] = [] + pass + else: + f['value'] = None + pass + + if sysfeature.feature.multivalued: + for option in sysfeature.options.all().order_by('slug'): + f['values'].append(option.value) + empty = False + else: + option = sysfeature.options.first() + if option: + f['value'] = option.value + empty = False + pass + + if not empty: + items[sysfeature.feature_id] = f + pass + + return list( items.values() ) + + def get_former_names(self, obj): + current = obj.get_current() + return current.former_names if current.former_names else None + + def get_history(self, obj): + current = obj.get_current() + return current.history + + def get_href(self, obj): + request = self.context['request'] + return reverse('system', args=[obj.slug], request=request) + + def get_end_year(self, obj): + current = obj.get_current() + return current.end_year + + def get_project_types(self, obj): + current = obj.get_current() + return list( map(str, current.project_types.all()) ) + + def get_urls(self, obj): + current = obj.get_current() + data = { + 'docs': None if not current.tech_docs else current.tech_docs, + 'homepage': None if not current.url else current.url, + 'source': None if not current.source_url else current.source_url, + 'wikipedia': None if not current.wikipedia_url else current.wikipedia_url, + } + return data + + def get_start_year(self, obj): + current = obj.get_current() + return current.start_year + + pass + diff --git a/dbdb/api/api_v202004/urls.py b/dbdb/api/api_v202004/urls.py new file mode 100644 index 00000000..a274ef7a --- /dev/null +++ b/dbdb/api/api_v202004/urls.py @@ -0,0 +1,15 @@ +# django imports +from django.urls import path +# third-party imports +from rest_framework.urlpatterns import format_suffix_patterns +# local imports +from . import views + + +app_name = 'api_v202004' + +urlpatterns = [ + path('systems', views.SystemsView.as_view(), name='systems'), +] + +urlpatterns = format_suffix_patterns(urlpatterns) diff --git a/dbdb/api/api_v202004/views.py b/dbdb/api/api_v202004/views.py new file mode 100644 index 00000000..ab2c5672 --- /dev/null +++ b/dbdb/api/api_v202004/views.py @@ -0,0 +1,34 @@ +# django imports +from django.shortcuts import render +# third-party imports +from rest_framework import generics +# project imports +from dbdb.core.models import System +from dbdb.core.models import SystemVersion +# local imports +from .serializers import SystemSerializer + + +# class based views + +class SystemsView(generics.ListAPIView): + + queryset = System.objects.all() + serializer_class = SystemSerializer + + def paginate_queryset(self, queryset): + if self.paginator is None: + return None + + items = self.paginator.paginate_queryset(queryset, self.request, view=self) + + # db optimizations - versions + ids = { item.id for item in items } + versions = SystemVersion.objects.filter(system_id__in=ids, is_current=True) + versions_map = { sv.system_id : sv for sv in versions } + for item in items: + item._current = versions_map[item.id] + + return items + + pass diff --git a/dbdb/api/pagination.py b/dbdb/api/pagination.py new file mode 100644 index 00000000..8376c3b3 --- /dev/null +++ b/dbdb/api/pagination.py @@ -0,0 +1,12 @@ +# third-party imports +from rest_framework.pagination import LimitOffsetPagination + + +# classes + +class StandardPagination(LimitOffsetPagination): + + default_limit = 10 + max_limit = 100 + + pass diff --git a/dbdb/api/urls.py b/dbdb/api/urls.py new file mode 100644 index 00000000..a798cb8a --- /dev/null +++ b/dbdb/api/urls.py @@ -0,0 +1,9 @@ +# django imports +from django.urls import path +from django.urls import include +from django.conf.urls import url + + +urlpatterns = [ + path('v202004/', include('dbdb.api.api_v202004.urls')), +] diff --git a/dbdb/core/models.py b/dbdb/core/models.py index eaeb6088..a84c02bc 100644 --- a/dbdb/core/models.py +++ b/dbdb/core/models.py @@ -208,6 +208,17 @@ def current(self): def get_absolute_url(self): return reverse('system', args=[self.slug]) + + def get_current(self): + if not hasattr(self, '_current'): + if self.id is None: + self._current = SystemVersion(system=self) + else: + print('get version', self.slug) + self._current = self.versions.get(is_current=True) + pass + + return self._current pass diff --git a/dbdb/settings.py b/dbdb/settings.py index 33e5b6e7..4c173207 100644 --- a/dbdb/settings.py +++ b/dbdb/settings.py @@ -21,13 +21,14 @@ 'django.contrib.messages', 'django.contrib.staticfiles', - #'autoslug', 'bootstrap4', - 'easy_thumbnails', 'django_countries', + 'easy_thumbnails', 'haystack', # django-haystack + 'rest_framework', # djangorestframework - 'dbdb.core' + 'dbdb.core', + 'dbdb.api.api_v202004', ] MIDDLEWARE = [ @@ -157,6 +158,14 @@ MEDIA_ROOT = root.path('media')() MEDIA_URL = '/media/' + +# Rest Framework + +REST_FRAMEWORK = { + 'DEFAULT_PAGINATION_CLASS': 'dbdb.api.pagination.StandardPagination', +} + + # Security ALLOWED_HOSTS = env.list('ALLOWED_HOSTS', default=['*']) diff --git a/dbdb/urls.py b/dbdb/urls.py index 1962aff1..9261b007 100644 --- a/dbdb/urls.py +++ b/dbdb/urls.py @@ -8,6 +8,7 @@ urlpatterns = [ url(r'^', include('django.contrib.auth.urls')), url(r'^', include('dbdb.core.urls')), + url(r'^api/', include('dbdb.api.urls')), url(r'^admin/', admin.site.urls), ] From 86b962ec136d1350eafde91655980eef298cdf4a Mon Sep 17 00:00:00 2001 From: Arthur Pemberton Date: Fri, 17 Apr 2020 02:01:07 -0400 Subject: [PATCH 2/7] renaming api module --- dbdb/api/urls.py | 2 +- dbdb/api/{api_v202004 => v202004}/__init__.py | 0 dbdb/api/{api_v202004 => v202004}/apps.py | 0 .../{api_v202004 => v202004}/serializers.py | 0 dbdb/api/{api_v202004 => v202004}/urls.py | 0 dbdb/api/{api_v202004 => v202004}/views.py | 0 dbdb/settings.py | 26 +++++++------------ 7 files changed, 10 insertions(+), 18 deletions(-) rename dbdb/api/{api_v202004 => v202004}/__init__.py (100%) rename dbdb/api/{api_v202004 => v202004}/apps.py (100%) rename dbdb/api/{api_v202004 => v202004}/serializers.py (100%) rename dbdb/api/{api_v202004 => v202004}/urls.py (100%) rename dbdb/api/{api_v202004 => v202004}/views.py (100%) diff --git a/dbdb/api/urls.py b/dbdb/api/urls.py index a798cb8a..328c1ad2 100644 --- a/dbdb/api/urls.py +++ b/dbdb/api/urls.py @@ -5,5 +5,5 @@ urlpatterns = [ - path('v202004/', include('dbdb.api.api_v202004.urls')), + path('v202004/', include('dbdb.api.v202004.urls')), ] diff --git a/dbdb/api/api_v202004/__init__.py b/dbdb/api/v202004/__init__.py similarity index 100% rename from dbdb/api/api_v202004/__init__.py rename to dbdb/api/v202004/__init__.py diff --git a/dbdb/api/api_v202004/apps.py b/dbdb/api/v202004/apps.py similarity index 100% rename from dbdb/api/api_v202004/apps.py rename to dbdb/api/v202004/apps.py diff --git a/dbdb/api/api_v202004/serializers.py b/dbdb/api/v202004/serializers.py similarity index 100% rename from dbdb/api/api_v202004/serializers.py rename to dbdb/api/v202004/serializers.py diff --git a/dbdb/api/api_v202004/urls.py b/dbdb/api/v202004/urls.py similarity index 100% rename from dbdb/api/api_v202004/urls.py rename to dbdb/api/v202004/urls.py diff --git a/dbdb/api/api_v202004/views.py b/dbdb/api/v202004/views.py similarity index 100% rename from dbdb/api/api_v202004/views.py rename to dbdb/api/v202004/views.py diff --git a/dbdb/settings.py b/dbdb/settings.py index 4c173207..58e18edf 100644 --- a/dbdb/settings.py +++ b/dbdb/settings.py @@ -28,7 +28,7 @@ 'rest_framework', # djangorestframework 'dbdb.core', - 'dbdb.api.api_v202004', + 'dbdb.api.v202004', ] MIDDLEWARE = [ @@ -82,24 +82,18 @@ 'default': env.db( default='sqlite:///{}'.format( root.path('data/db.sqlite3') ) ) } + # Password validation # https://docs.djangoproject.com/en/2.0/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ - { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', - }, - { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', - }, + { 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator' }, + { 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator' }, + { 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator' }, + { 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator' }, ] + # CACHE CACHES = { 'default': { @@ -132,10 +126,6 @@ 'PATH': root.path('data/xapian')(), 'FLAGS': HAYSTACK_XAPIAN_FLAGS, }, - # 'default': { - # 'ENGINE': 'haystack.backends.whoosh_backend.WhooshEngine', - # 'PATH': root.path('data/whoosh')(), - # }, } @@ -209,9 +199,11 @@ }, } + # Django Countries COUNTRIES_FIRST = ['US'] + # Django Invisible reCaptcha NORECAPTCHA_SITE_KEY = '6Lfo8VwUAAAAAEHNqeL01PSkiRul7ImQ8Bsw8Nqc' NORECAPTCHA_SECRET_KEY = '6Lfo8VwUAAAAALFGUrGKqrzCR94pfgFahtd56WY9' From 0a4c75221dafc5cf6a37844f312af970176b2cd9 Mon Sep 17 00:00:00 2001 From: Arthur Pemberton Date: Fri, 17 Apr 2020 02:21:02 -0400 Subject: [PATCH 3/7] added throttling to API --- dbdb/settings.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/dbdb/settings.py b/dbdb/settings.py index 58e18edf..1542e6b2 100644 --- a/dbdb/settings.py +++ b/dbdb/settings.py @@ -95,16 +95,14 @@ # CACHE + CACHES = { 'default': { - #'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', - #'LOCATION': 'dbdb_io_cache', + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', } } - # Haystack # https://django-haystack.readthedocs.io/ @@ -152,7 +150,18 @@ # Rest Framework REST_FRAMEWORK = { + # https://www.django-rest-framework.org/api-guide/pagination/ 'DEFAULT_PAGINATION_CLASS': 'dbdb.api.pagination.StandardPagination', + + # https://www.django-rest-framework.org/api-guide/throttling/ + 'DEFAULT_THROTTLE_CLASSES': [ + 'rest_framework.throttling.AnonRateThrottle', + 'rest_framework.throttling.UserRateThrottle' + ], + 'DEFAULT_THROTTLE_RATES': { + 'anon': '4/second', + 'user': '4/second' + } } From 23637f864441d997d321fb6814556f4104c66c18 Mon Sep 17 00:00:00 2001 From: Arthur Pemberton Date: Thu, 25 Jun 2020 09:37:38 -0400 Subject: [PATCH 4/7] fixing tests and travis --- .travis.yml | 11 +- dbdb/core/tests.py | 69 +- dbdb/core/views.py | 4 - requirements-dev.txt | 3 +- requirements.txt | 58 +- templates/base.html | 2 +- templates/core/create-database.html | 2 +- templates/core/database-browse.html | 2 +- templates/core/database_form_base.html | 2 +- templates/core/databases-edit.html | 2 +- templates/core/edit-database.html | 2 +- templates/core/empty-fields.html | 2 +- templates/core/home-listitem.html | 2 +- templates/core/home.html | 2 +- templates/core/recent.html | 2 +- templates/core/revision_view.html | 2 +- templates/core/stats.html | 2 +- templates/core/system-sys-sidebar.html | 2 +- templates/core/system.html | 2 +- templates/core/system_base.html | 2 +- templates/core/version-card.html | 2 +- xapian_backend.py | 1695 ++++++++++++++++++++++++ 22 files changed, 1781 insertions(+), 91 deletions(-) create mode 100644 xapian_backend.py diff --git a/.travis.yml b/.travis.yml index 62682475..f4c5daa9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,17 +2,14 @@ language: python python: - "3.6" install: - - mkdir -p media/logos media/twitter - - sudo apt-get install python3-sphinx - - pip3 install Sphinx - - ./install_xapian.sh 1.4.9 - - pip install git+https://github.com/notanumber/xapian-haystack.git - mkdir -p data/xapian + - mkdir -p media/logos + - mkdir -p media/twitter + - pip3 install -U pip setuptools wheel + - ./install_xapian.sh 1.4.16 - pip3 install -r requirements.txt - pip3 install -r requirements-test.txt before_script: - python manage.py migrate - script: - python manage.py test - diff --git a/dbdb/core/tests.py b/dbdb/core/tests.py index e3f84bb5..370d64d6 100644 --- a/dbdb/core/tests.py +++ b/dbdb/core/tests.py @@ -1,5 +1,6 @@ +import tempfile # django imports -from django.test import TestCase, Client, override_settings +from django.test import TestCase, override_settings from django.contrib.auth import get_user from django.urls import reverse from django.core import management @@ -15,8 +16,6 @@ from .models import Feature from .views import CounterView -import tempfile -from pprint import pprint # ============================================== # HAYSTACK CONFIG @@ -75,19 +74,19 @@ class SearchTestCase(BaseTestCase): #response = self.client.get(reverse('search')) #self.assertRedirects(response, reverse('home')) #return - + def test_haystack_contents(self): """Make sure we are setting up haystack correctly.""" sqs = SearchQuerySet() num_results = len(sqs) self.assertEquals(num_results, 2) - + expected = ["SQLite", "XXX"] for i in range(num_results): res = sqs[i] self.assertTrue(res.name in expected) return - + def test_search_no_parameters(self): query = {'q': 'sql'} response = self.client.get(reverse('browse')) @@ -97,8 +96,8 @@ def test_search_no_parameters(self): def test_search_valid_parameters(self): query = {'q': 'sql'} response = self.client.get(reverse('browse'), data=query) - #print(response.content) - self.assertContains(response, 'Found 1 database for \"sql\"', html=True) + + self.assertContains(response, 'Found 1 database', html=False) self.assertContains(response, 'SQLite', html=True) # self.assertContains(response, '

Nice description

', html=True) return @@ -106,7 +105,7 @@ def test_search_valid_parameters(self): def test_search_invalid_parameters(self): query = {'q': 'dock'} response = self.client.get(reverse('browse'), data=query) - self.assertContains(response, 'No databases found for \"dock\"', html=True) + self.assertContains(response, 'No databases found', html=False) return pass @@ -127,19 +126,19 @@ def test_autocom_valid_parameters(self): target = "SQLite" for i in range(1, len(target)): query = {'q': target[:i+1]} - #pprint(query) + response = self.client.get(reverse('search_autocomplete'), data=query) - #pprint(response.json()) + self.assertContains(response, 'SQLite', html=False) return - + def test_autocom_invalid_parameters(self): query = {'q': "YYY"} response = self.client.get(reverse('search_autocomplete'), data=query) - #pprint(response.json()) + self.assertEquals(len(response.json()), 0) return - + def test_autocom_no_parameters(self): response = self.client.get(reverse('search_autocomplete')) self.assertEquals(len(response.json()), 0) @@ -161,46 +160,21 @@ class SystemViewTestCase(BaseTestCase): def test_counter(self): target = "SQLite" system = System.objects.get(name=target) - orig_count = system.view_count + orig_visits = SystemVisit.objects.filter(system=system).count() - + data = {"token": CounterView.build_token('system', pk=system.id)} response = self.client.post(reverse('counter'), data) result = response.json(); self.assertTrue("status" in result) self.assertEquals(result["status"], "ok") - - # Make sure count increased by one - system = System.objects.get(name=target) - new_count = system.view_count - self.assertEquals(new_count, orig_count+1) - + # Check that we got added a SystemVisit new_visits = SystemVisit.objects.filter(system=system).count() self.assertEquals(new_visits, orig_visits+1) - - return - - def test_bot_block(self): - target = "SQLite" - system = System.objects.get(name=target) - orig_count = system.view_count - - c = Client(HTTP_USER_AGENT='(KHTML, like Gecko; compatible; Googlebot/2.1)') - - data = {"token": CounterView.build_token('system', pk=system.id)} - response = c.post(reverse('counter'), data) - result = response.json(); - self.assertTrue("status" in result) - self.assertEquals(result["status"], "bot") - - # Make sure count is the same - system = System.objects.get(name=target) - new_count = system.view_count - self.assertEquals(new_count, orig_count) + return - - + pass # ============================================== @@ -227,7 +201,7 @@ def test_inputs_quantity(self): filtergroups = d('div.filter-group') # Add two for the year filtergroups # Add nine for country, OS, project type, PL, inspired, derived, embedded compatiable, licenses - #pprint(filtergroups) + self.assertEquals(quantity + 2 + 9, len(filtergroups)) return @@ -236,7 +210,7 @@ def test_search_with_insuficient_data(self): 'feature1': ['option1'], } response = self.client.get(reverse('browse'), data=data) - #pprint(response.content) + self.assertContains(response, 'No databases found') return @@ -301,7 +275,7 @@ class CreateDatabaseTestCase(BaseTestCase): def test_cant_access_not_authenticated(self): response = self.client.get(reverse('create_database')) - self.assertEquals(response.status_code, 404) + self.assertEquals(response.status_code, 302) return def test_cant_access_not_superuser(self): @@ -344,6 +318,7 @@ def test_can_create_database(self): response = self.client.post(reverse('create_database'), data=data) self.assertRedirects(response, reverse('system', kwargs={'slug': 'testdb'})) return + pass # ============================================== diff --git a/dbdb/core/views.py b/dbdb/core/views.py index 3e6308c5..bbd72412 100644 --- a/dbdb/core/views.py +++ b/dbdb/core/views.py @@ -128,7 +128,6 @@ def get_removal_url(self): query = [] for key,values in self.query.lists(): - print(key, values) for value in values: if key == self.group_slug and value == self.tag_slug: continue @@ -660,8 +659,6 @@ def do_search(self, request): for row in FeatureOption.objects.filter(id__in=filter_option_ids).values_list('feature__slug','feature__label','slug','value') ) - #for st in search_tags: - #print('-', st) return (sqs, search_mapping, search_tags) def handle_old_urls(self, request): @@ -959,7 +956,6 @@ def build_features(self, feature_form): @never_cache def get(self, request, slug=None): - # If there is no slug, then they are trying to create a new database. # Only superusers are allowed to do that. if slug is None: diff --git a/requirements-dev.txt b/requirements-dev.txt index c6234e35..bcb7d9eb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,2 @@ -django-debug-toolbar==1.8django-debug-toolbar==1.8 +django-debug-toolbar==1.8 django-extensions==1.9.6 -django-extensions==1.9.6 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index bfca0851..e7225421 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,45 @@ -Django==2.0.13 -django-autoslug==1.9.3 -django-bootstrap4==0.0.7 -django-countries==5.3.2 +alabaster==0.7.12 +asgiref==3.2.7 +Babel==2.8.0 +beautifulsoup4==4.9.0 +certifi==2020.4.5.1 +chardet==3.0.4 +cssselect==1.1.0 +Django==3.0.5 +django-autoslug==1.9.7 +django-bootstrap4==1.1.1 +django-countries==6.1.2 django-environ==0.4.5 -django-haystack==2.8.1 +django-haystack==3.0b1 django-nocaptcha-recaptcha==0.0.20 -easy-thumbnails==2.5 -lxml==4.2.5 -mysqlclient==1.3.14 -Pillow==5.3.0 -PyJWT==1.7.0 -pytz==2018.7 -six==1.11.0 -Whoosh==2.7.4 -xapian-haystack==2.1.1 -python-dateutil==2.8.0 +djangorestframework==3.11.0 +docutils==0.16 +easy-thumbnails==2.7 +idna==2.9 +imagesize==1.2.0 +Jinja2==2.11.2 +lxml==4.5.0 +MarkupSafe==1.1.1 +mysqlclient==1.4.6 +packaging==20.3 +Pillow==7.1.1 +Pygments==2.6.1 +PyJWT==1.7.1 +pyparsing==2.4.7 +pyquery==1.4.1 +python-dateutil==2.8.1 +pytz==2019.3 +requests==2.23.0 +six==1.14.0 +snowballstemmer==2.0.0 +soupsieve==2.0 +Sphinx==3.0.1 +sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-htmlhelp==1.0.3 +sphinxcontrib-jsmath==1.0.1 +sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-serializinghtml==1.1.4 +sphinxcontrib-websupport==1.2.1 +sqlparse==0.3.1 +urllib3==1.25.9 diff --git a/templates/base.html b/templates/base.html index 789f06b6..82a1f475 100644 --- a/templates/base.html +++ b/templates/base.html @@ -1,4 +1,4 @@ -{% load staticfiles %} +{% load static %} diff --git a/templates/core/create-database.html b/templates/core/create-database.html index 88eadc17..a6342818 100644 --- a/templates/core/create-database.html +++ b/templates/core/create-database.html @@ -1,5 +1,5 @@ {% extends 'core/database_form_base.html' %} -{% load staticfiles %} +{% load static %} {% load bootstrap4 %} {% block title %} diff --git a/templates/core/database-browse.html b/templates/core/database-browse.html index 16484328..673e76d3 100644 --- a/templates/core/database-browse.html +++ b/templates/core/database-browse.html @@ -1,6 +1,6 @@ {% extends 'base.html' %} {% load humanize %} -{% load staticfiles %} +{% load static %} {% load countries %} {% load bootstrap4 %} diff --git a/templates/core/database_form_base.html b/templates/core/database_form_base.html index 58b246bc..ff6090c1 100644 --- a/templates/core/database_form_base.html +++ b/templates/core/database_form_base.html @@ -1,5 +1,5 @@ {% extends 'base.html' %} -{% load staticfiles %} +{% load static %} {% load bootstrap4 %} {% block css %} diff --git a/templates/core/databases-edit.html b/templates/core/databases-edit.html index 4c255450..b6b3cac5 100644 --- a/templates/core/databases-edit.html +++ b/templates/core/databases-edit.html @@ -1,6 +1,6 @@ {% extends 'base.html' %} {% load bootstrap4 %} -{% load staticfiles %} +{% load static %} {% block title %} diff --git a/templates/core/edit-database.html b/templates/core/edit-database.html index 9b986e52..3db0235a 100644 --- a/templates/core/edit-database.html +++ b/templates/core/edit-database.html @@ -1,5 +1,5 @@ {% extends 'core/database_form_base.html' %} -{% load staticfiles %} +{% load static %} {% load bootstrap4 %} {% block title %} diff --git a/templates/core/empty-fields.html b/templates/core/empty-fields.html index 7cc6d14d..d7a84e18 100644 --- a/templates/core/empty-fields.html +++ b/templates/core/empty-fields.html @@ -1,5 +1,5 @@ {% extends 'base.html' %} -{% load staticfiles %} +{% load static %} {% load bootstrap4 %} {% block title %} diff --git a/templates/core/home-listitem.html b/templates/core/home-listitem.html index f6cea3f9..e27bee78 100644 --- a/templates/core/home-listitem.html +++ b/templates/core/home-listitem.html @@ -1,4 +1,4 @@ -{% load staticfiles %} +{% load static %} {% load thumbnail %} diff --git a/templates/core/home.html b/templates/core/home.html index 2e320b1a..9725660c 100644 --- a/templates/core/home.html +++ b/templates/core/home.html @@ -1,6 +1,6 @@ {% extends 'base.html' %} {% load humanize %} -{% load staticfiles %} +{% load static %} {% load thumbnail %} {% block title %}Home{% endblock %} diff --git a/templates/core/recent.html b/templates/core/recent.html index 884948b1..c0551a59 100644 --- a/templates/core/recent.html +++ b/templates/core/recent.html @@ -1,5 +1,5 @@ {% extends 'base.html' %} -{% load staticfiles %} +{% load static %} {% load thumbnail %} {% load bootstrap4 %} diff --git a/templates/core/revision_view.html b/templates/core/revision_view.html index ae61742d..0acd2298 100644 --- a/templates/core/revision_view.html +++ b/templates/core/revision_view.html @@ -1,5 +1,5 @@ {% extends 'core/system_base.html' %} -{% load staticfiles %} +{% load static %} {% block navbuttons %}
  • diff --git a/templates/core/stats.html b/templates/core/stats.html index b7a3d7e4..7c7f161c 100644 --- a/templates/core/stats.html +++ b/templates/core/stats.html @@ -1,5 +1,5 @@ {% extends 'base.html' %} -{% load staticfiles %} +{% load static %} {% load countries %} {% load thumbnail %} {% load bootstrap4 %} diff --git a/templates/core/system-sys-sidebar.html b/templates/core/system-sys-sidebar.html index cbe6d9f6..bb7c8cb0 100644 --- a/templates/core/system-sys-sidebar.html +++ b/templates/core/system-sys-sidebar.html @@ -1,4 +1,4 @@ -{% load staticfiles %} +{% load static %} {% load thumbnail %}
    diff --git a/templates/core/system.html b/templates/core/system.html index a0ab0d91..5b9935c8 100644 --- a/templates/core/system.html +++ b/templates/core/system.html @@ -1,5 +1,5 @@ {% extends 'core/system_base.html' %} -{% load staticfiles %} +{% load static %} {% block navbuttons %} {% if user.is_authenticated and user_can_edit %} diff --git a/templates/core/system_base.html b/templates/core/system_base.html index 245e7aaf..f1edddae 100644 --- a/templates/core/system_base.html +++ b/templates/core/system_base.html @@ -1,5 +1,5 @@ {% extends 'base.html' %} -{% load staticfiles %} +{% load static %} {% load thumbnail %} {% block css %} diff --git a/templates/core/version-card.html b/templates/core/version-card.html index 3e9d9982..d563d2b9 100644 --- a/templates/core/version-card.html +++ b/templates/core/version-card.html @@ -1,4 +1,4 @@ -{% load staticfiles %} +{% load static %} {% load thumbnail %} {% with logo=result.logo|thumbnail_url:'search' %} diff --git a/xapian_backend.py b/xapian_backend.py new file mode 100644 index 00000000..b0eb8591 --- /dev/null +++ b/xapian_backend.py @@ -0,0 +1,1695 @@ +from __future__ import unicode_literals + +import datetime +import pickle +import os +import re +import shutil +import sys + +import six +from django.conf import settings +from django.core.exceptions import ImproperlyConfigured +from django.utils.encoding import force_text + +from haystack import connections +from haystack.backends import BaseEngine, BaseSearchBackend, BaseSearchQuery, SearchNode, log_query +from haystack.constants import ID, DJANGO_ID, DJANGO_CT, DEFAULT_OPERATOR +from haystack.exceptions import HaystackError, MissingDependency +from haystack.inputs import AutoQuery +from haystack.models import SearchResult +from haystack.utils import get_identifier, get_model_ct + +NGRAM_MIN_LENGTH = 2 +NGRAM_MAX_LENGTH = 15 + +try: + import xapian +except ImportError: + raise MissingDependency("The 'xapian' backend requires the installation of 'Xapian'. " + "Please refer to the documentation.") + +if sys.version_info[0] == 2: + DirectoryExistsException = OSError +elif sys.version_info[0] == 3: + DirectoryExistsException = FileExistsError + + +class NotSupportedError(Exception): + """ + When the installed version of Xapian doesn't support something and we have + the old implementation. + """ + pass + +# this maps the different reserved fields to prefixes used to +# create the database: +# id str: unique document id. +# django_id int: id of the django model instance. +# django_ct str: of the content type of the django model. +# field str: name of the field of the index. +TERM_PREFIXES = { + ID: 'Q', + DJANGO_ID: 'QQ', + DJANGO_CT: 'CONTENTTYPE', + 'field': 'X' +} + +MEMORY_DB_NAME = ':memory:' + +DEFAULT_XAPIAN_FLAGS = ( + xapian.QueryParser.FLAG_PHRASE | + xapian.QueryParser.FLAG_BOOLEAN | + xapian.QueryParser.FLAG_LOVEHATE | + xapian.QueryParser.FLAG_WILDCARD | + xapian.QueryParser.FLAG_PURE_NOT +) + +# Mapping from `HAYSTACK_DEFAULT_OPERATOR` to Xapian operators +XAPIAN_OPTS = {'AND': xapian.Query.OP_AND, + 'OR': xapian.Query.OP_OR, + 'PHRASE': xapian.Query.OP_PHRASE, + 'NEAR': xapian.Query.OP_NEAR + } + +# number of documents checked by default when building facets +# this must be improved to be relative to the total number of docs. +DEFAULT_CHECK_AT_LEAST = 1000 + +# field types accepted to be serialized as values in Xapian +FIELD_TYPES = {'text', 'integer', 'date', 'datetime', 'float', 'boolean', + 'edge_ngram', 'ngram'} + +# defines the format used to store types in Xapian +# this format ensures datetimes are sorted correctly +DATETIME_FORMAT = '%Y%m%d%H%M%S' +INTEGER_FORMAT = '%012d' + +# defines the distance given between +# texts with positional information +TERMPOS_DISTANCE = 100 + +class InvalidIndexError(HaystackError): + """Raised when an index can not be opened.""" + pass + + +class XHValueRangeProcessor(xapian.ValueRangeProcessor): + """ + A Processor to construct ranges of values + """ + def __init__(self, backend): + self.backend = backend + xapian.ValueRangeProcessor.__init__(self) + + def __call__(self, begin, end): + """ + Construct a tuple for value range processing. + `begin` -- a string in the format ':[low_range]' + If 'low_range' is omitted, assume the smallest possible value. + `end` -- a string in the the format '[high_range|*]'. If '*', assume + the highest possible value. + Return a tuple of three strings: (column, low, high) + """ + colon = begin.find(':') + field_name = begin[:colon] + begin = begin[colon + 1:len(begin)] + for field_dict in self.backend.schema: + if field_dict['field_name'] == field_name: + field_type = field_dict['type'] + + if not begin: + if field_type == 'text': + begin = 'a' # TODO: A better way of getting a min text value? + elif field_type == 'integer': + begin = -sys.maxsize - 1 + elif field_type == 'float': + begin = float('-inf') + elif field_type == 'date' or field_type == 'datetime': + begin = '00010101000000' + elif end == '*': + if field_type == 'text': + end = 'z' * 100 # TODO: A better way of getting a max text value? + elif field_type == 'integer': + end = sys.maxsize + elif field_type == 'float': + end = float('inf') + elif field_type == 'date' or field_type == 'datetime': + end = '99990101000000' + + if field_type == 'float': + begin = _term_to_xapian_value(float(begin), field_type) + end = _term_to_xapian_value(float(end), field_type) + elif field_type == 'integer': + begin = _term_to_xapian_value(int(begin), field_type) + end = _term_to_xapian_value(int(end), field_type) + return field_dict['column'], str(begin), str(end) + + +class XHExpandDecider(xapian.ExpandDecider): + def __call__(self, term): + """ + Return True if the term should be used for expanding the search + query, False otherwise. + + Ignore terms related with the content type of objects. + """ + if term.decode('utf-8').startswith(TERM_PREFIXES[DJANGO_CT]): + return False + return True + + +class XapianSearchBackend(BaseSearchBackend): + """ + `SearchBackend` defines the Xapian search backend for use with the Haystack + API for Django search. + + It uses the Xapian Python bindings to interface with Xapian, and as + such is subject to this bug: when + Django is running with mod_python or mod_wsgi under Apache. + + Until this issue has been fixed by Xapian, it is neccessary to set + `WSGIApplicationGroup to %{GLOBAL}` when using mod_wsgi, or + `PythonInterpreter main_interpreter` when using mod_python. + + In order to use this backend, `PATH` must be included in the + `connection_options`. This should point to a location where you would your + indexes to reside. + """ + inmemory_db = None + + def __init__(self, connection_alias, **connection_options): + """ + Instantiates an instance of `SearchBackend`. + + Optional arguments: + `connection_alias` -- The name of the connection + `language` -- The stemming language (default = 'english') + `**connection_options` -- The various options needed to setup + the backend. + + Also sets the stemming language to be used to `language`. + """ + super(XapianSearchBackend, self).__init__(connection_alias, **connection_options) + + if not 'PATH' in connection_options: + raise ImproperlyConfigured("You must specify a 'PATH' in your settings for connection '%s'." + % connection_alias) + + self.path = connection_options.get('PATH') + + if self.path != MEMORY_DB_NAME: + try: + os.makedirs(self.path) + except DirectoryExistsException: + pass + + self.flags = connection_options.get('FLAGS', DEFAULT_XAPIAN_FLAGS) + self.language = getattr(settings, 'HAYSTACK_XAPIAN_LANGUAGE', 'english') + + stemming_strategy_string = getattr(settings, 'HAYSTACK_XAPIAN_STEMMING_STRATEGY', 'STEM_SOME') + self.stemming_strategy = getattr(xapian.QueryParser, stemming_strategy_string, xapian.QueryParser.STEM_SOME) + + # these 4 attributes are caches populated in `build_schema` + # they are checked in `_update_cache` + # use property to retrieve them + self._fields = {} + self._schema = [] + self._content_field_name = None + self._columns = {} + + def _update_cache(self): + """ + To avoid build_schema every time, we cache + some values: they only change when a SearchIndex + changes, which typically restarts the Python. + """ + fields = connections[self.connection_alias].get_unified_index().all_searchfields() + if self._fields != fields: + self._fields = fields + self._content_field_name, self._schema = self.build_schema(self._fields) + + @property + def schema(self): + self._update_cache() + return self._schema + + @property + def content_field_name(self): + self._update_cache() + return self._content_field_name + + @property + def column(self): + """ + Returns the column in the database of a given field name. + """ + self._update_cache() + return self._columns + + def update(self, index, iterable, commit=True): + """ + Updates the `index` with any objects in `iterable` by adding/updating + the database as needed. + + Required arguments: + `index` -- The `SearchIndex` to process + `iterable` -- An iterable of model instances to index + Optional arguments: + `commit` -- ignored + + For each object in `iterable`, a document is created containing all + of the terms extracted from `index.full_prepare(obj)` with field prefixes, + and 'as-is' as needed. Also, if the field type is 'text' it will be + stemmed and stored with the 'Z' prefix as well. + + eg. `content:Testing` ==> `testing, Ztest, ZXCONTENTtest, XCONTENTtest` + + Each document also contains an extra term in the format: + + `XCONTENTTYPE.` + + As well as a unique identifier in the the format: + + `Q..` + + eg.: foo.bar (pk=1) ==> `Qfoo.bar.1`, `XCONTENTTYPEfoo.bar` + + This is useful for querying for a specific document corresponding to + a model instance. + + The document also contains a pickled version of the object itself and + the document ID in the document data field. + + Finally, we also store field values to be used for sorting data. We + store these in the document value slots (position zero is reserver + for the document ID). All values are stored as unicode strings with + conversion of float, int, double, values being done by Xapian itself + through the use of the :method:xapian.sortable_serialise method. + """ + database = self._database(writable=True) + + try: + term_generator = xapian.TermGenerator() + term_generator.set_database(database) + term_generator.set_stemmer(xapian.Stem(self.language)) + try: + term_generator.set_stemming_strategy(self.stemming_strategy) + except AttributeError: + # Versions before Xapian 1.2.11 do not support stemming strategies for TermGenerator + pass + if self.include_spelling is True: + term_generator.set_flags(xapian.TermGenerator.FLAG_SPELLING) + + def _add_text(termpos, text, weight, prefix=''): + """ + indexes text appending 2 extra terms + to identify beginning and ending of the text. + """ + term_generator.set_termpos(termpos) + + start_term = '%s^' % prefix + end_term = '%s$' % prefix + # add begin + document.add_posting(start_term, termpos, weight) + # add text + term_generator.index_text(text, weight, prefix) + termpos = term_generator.get_termpos() + # add ending + termpos += 1 + document.add_posting(end_term, termpos, weight) + + # increase termpos + term_generator.set_termpos(termpos) + term_generator.increase_termpos(TERMPOS_DISTANCE) + + return term_generator.get_termpos() + + def _add_literal_text(termpos, text, weight, prefix=''): + """ + Adds sentence to the document with positional information + but without processing. + + The sentence is bounded by "^" "$" to allow exact matches. + """ + text = '^ %s $' % text + for word in text.split(): + term = '%s%s' % (prefix, word) + document.add_posting(term, termpos, weight) + termpos += 1 + termpos += TERMPOS_DISTANCE + return termpos + + def add_text(termpos, prefix, text, weight): + """ + Adds text to the document with positional information + and processing (e.g. stemming). + """ + termpos = _add_text(termpos, text, weight, prefix=prefix) + termpos = _add_text(termpos, text, weight, prefix='') + termpos = _add_literal_text(termpos, text, weight, prefix=prefix) + termpos = _add_literal_text(termpos, text, weight, prefix='') + return termpos + + def _get_ngram_lengths(value): + values = value.split() + for item in values: + for ngram_length in six.moves.range(NGRAM_MIN_LENGTH, NGRAM_MAX_LENGTH + 1): + yield item, ngram_length + + for obj in iterable: + document = xapian.Document() + term_generator.set_document(document) + + def ngram_terms(value): + for item, length in _get_ngram_lengths(value): + item_length = len(item) + for start in six.moves.range(0, item_length - length + 1): + for size in six.moves.range(length, length + 1): + end = start + size + if end > item_length: + continue + yield _to_xapian_term(item[start:end]) + + def edge_ngram_terms(value): + for item, length in _get_ngram_lengths(value): + yield _to_xapian_term(item[0:length]) + + def add_edge_ngram_to_document(prefix, value, weight): + """ + Splits the term in ngrams and adds each ngram to the index. + The minimum and maximum size of the ngram is respectively + NGRAM_MIN_LENGTH and NGRAM_MAX_LENGTH. + """ + for term in edge_ngram_terms(value): + document.add_term(term, weight) + document.add_term(prefix + term, weight) + + def add_ngram_to_document(prefix, value, weight): + """ + Splits the term in ngrams and adds each ngram to the index. + The minimum and maximum size of the ngram is respectively + NGRAM_MIN_LENGTH and NGRAM_MAX_LENGTH. + """ + for term in ngram_terms(value): + document.add_term(term, weight) + document.add_term(prefix + term, weight) + + def add_non_text_to_document(prefix, term, weight): + """ + Adds term to the document without positional information + and without processing. + + If the term is alone, also adds it as "^$" + to allow exact matches on single terms. + """ + document.add_term(term, weight) + document.add_term(prefix + term, weight) + + def add_datetime_to_document(termpos, prefix, term, weight): + """ + Adds a datetime to document with positional order + to allow exact matches on it. + """ + date, time = term.split() + document.add_posting(date, termpos, weight) + termpos += 1 + document.add_posting(time, termpos, weight) + termpos += 1 + document.add_posting(prefix + date, termpos, weight) + termpos += 1 + document.add_posting(prefix + time, termpos, weight) + termpos += TERMPOS_DISTANCE + 1 + return termpos + + data = index.full_prepare(obj) + weights = index.get_field_weights() + + termpos = term_generator.get_termpos() # identifies the current position in the document. + for field in self.schema: + if field['field_name'] not in list(data.keys()): + # not supported fields are ignored. + continue + + if field['field_name'] in weights: + weight = int(weights[field['field_name']]) + else: + weight = 1 + + value = data[field['field_name']] + + if field['field_name'] in (ID, DJANGO_ID, DJANGO_CT): + # Private fields are indexed in a different way: + # `django_id` is an int and `django_ct` is text; + # besides, they are indexed by their (unstemmed) value. + if field['field_name'] == DJANGO_ID: + value = int(value) + value = _term_to_xapian_value(value, field['type']) + + document.add_term(TERM_PREFIXES[field['field_name']] + value, weight) + document.add_value(field['column'], value) + continue + else: + prefix = TERM_PREFIXES['field'] + field['field_name'].upper() + + # if not multi_valued, we add as a document value + # for sorting and facets + if field['multi_valued'] == 'false': + document.add_value(field['column'], _term_to_xapian_value(value, field['type'])) + else: + for t in value: + # add the exact match of each value + term = _to_xapian_term(t) + termpos = add_text(termpos, prefix, term, weight) + continue + + term = _to_xapian_term(value) + if term == '': + continue + # from here on the term is a string; + # we now decide how it is indexed + + if field['type'] == 'text': + # text is indexed with positional information + termpos = add_text(termpos, prefix, term, weight) + elif field['type'] == 'datetime': + termpos = add_datetime_to_document(termpos, prefix, term, weight) + elif field['type'] == 'ngram': + add_ngram_to_document(prefix, value, weight) + elif field['type'] == 'edge_ngram': + add_edge_ngram_to_document(prefix, value, weight) + else: + # all other terms are added without positional information + add_non_text_to_document(prefix, term, weight) + + # store data without indexing it + document.set_data(pickle.dumps( + (obj._meta.app_label, obj._meta.model_name, obj.pk, data), + pickle.HIGHEST_PROTOCOL + )) + + # add the id of the document + document_id = TERM_PREFIXES[ID] + get_identifier(obj) + document.add_term(document_id) + + # finally, replace or add the document to the database + database.replace_document(document_id, document) + + except UnicodeDecodeError: + sys.stderr.write('Chunk failed.\n') + pass + + finally: + database.close() + + def remove(self, obj, commit=True): + """ + Remove indexes for `obj` from the database. + + We delete all instances of `Q..` which + should be unique to this object. + + Optional arguments: + `commit` -- ignored + """ + database = self._database(writable=True) + database.delete_document(TERM_PREFIXES[ID] + get_identifier(obj)) + database.close() + + def clear(self, models=(), commit=True): + """ + Clear all instances of `models` from the database or all models, if + not specified. + + Optional Arguments: + `models` -- Models to clear from the database (default = []) + + If `models` is empty, an empty query is executed which matches all + documents in the database. Afterwards, each match is deleted. + + Otherwise, for each model, a `delete_document` call is issued with + the term `XCONTENTTYPE.`. This will delete + all documents with the specified model type. + """ + if not models: + # Because there does not appear to be a "clear all" method, + # it's much quicker to remove the contents of the `self.path` + # folder than it is to remove each document one at a time. + if os.path.exists(self.path): + shutil.rmtree(self.path) + else: + database = self._database(writable=True) + for model in models: + database.delete_document(TERM_PREFIXES[DJANGO_CT] + get_model_ct(model)) + database.close() + + def document_count(self): + try: + return self._database().get_doccount() + except InvalidIndexError: + return 0 + + def _build_models_query(self, query): + """ + Builds a query from `query` that filters to documents only from registered models. + """ + registered_models_ct = self.build_models_list() + if registered_models_ct: + restrictions = [xapian.Query('%s%s' % (TERM_PREFIXES[DJANGO_CT], model_ct)) + for model_ct in registered_models_ct] + limit_query = xapian.Query(xapian.Query.OP_OR, restrictions) + + query = xapian.Query(xapian.Query.OP_AND, query, limit_query) + + return query + + def _check_field_names(self, field_names): + """ + Raises InvalidIndexError if any of a field_name in field_names is + not indexed. + """ + if field_names: + for field_name in field_names: + try: + self.column[field_name] + except KeyError: + raise InvalidIndexError('Trying to use non indexed field "%s"' % field_name) + + @log_query + def search(self, query, sort_by=None, start_offset=0, end_offset=None, + fields='', highlight=False, facets=None, date_facets=None, + query_facets=None, narrow_queries=None, spelling_query=None, + limit_to_registered_models=None, result_class=None, **kwargs): + """ + Executes the Xapian::query as defined in `query`. + + Required arguments: + `query` -- Search query to execute + + Optional arguments: + `sort_by` -- Sort results by specified field (default = None) + `start_offset` -- Slice results from `start_offset` (default = 0) + `end_offset` -- Slice results at `end_offset` (default = None), if None, then all documents + `fields` -- Filter results on `fields` (default = '') + `highlight` -- Highlight terms in results (default = False) + `facets` -- Facet results on fields (default = None) + `date_facets` -- Facet results on date ranges (default = None) + `query_facets` -- Facet results on queries (default = None) + `narrow_queries` -- Narrow queries (default = None) + `spelling_query` -- An optional query to execute spelling suggestion on + `limit_to_registered_models` -- Limit returned results to models registered in + the current `SearchSite` (default = True) + + Returns: + A dictionary with the following keys: + `results` -- A list of `SearchResult` + `hits` -- The total available results + `facets` - A dictionary of facets with the following keys: + `fields` -- A list of field facets + `dates` -- A list of date facets + `queries` -- A list of query facets + If faceting was not used, the `facets` key will not be present + + If `query` is None, returns no results. + + If `INCLUDE_SPELLING` was enabled in the connection options, the + extra flag `FLAG_SPELLING_CORRECTION` will be passed to the query parser + and any suggestions for spell correction will be returned as well as + the results. + """ + if xapian.Query.empty(query): + return { + 'results': [], + 'hits': 0, + } + + self._check_field_names(facets) + self._check_field_names(date_facets) + self._check_field_names(query_facets) + + database = self._database() + + if limit_to_registered_models is None: + limit_to_registered_models = getattr(settings, 'HAYSTACK_LIMIT_TO_REGISTERED_MODELS', True) + + if result_class is None: + result_class = SearchResult + + if self.include_spelling is True: + spelling_suggestion = self._do_spelling_suggestion(database, query, spelling_query) + else: + spelling_suggestion = '' + + if narrow_queries is not None: + query = xapian.Query( + xapian.Query.OP_AND, query, xapian.Query( + xapian.Query.OP_AND, [self.parse_query(narrow_query) for narrow_query in narrow_queries] + ) + ) + + if limit_to_registered_models: + query = self._build_models_query(query) + + enquire = xapian.Enquire(database) + if hasattr(settings, 'HAYSTACK_XAPIAN_WEIGHTING_SCHEME'): + enquire.set_weighting_scheme(xapian.BM25Weight(*settings.HAYSTACK_XAPIAN_WEIGHTING_SCHEME)) + enquire.set_query(query) + + if sort_by: + try: + _xapian_sort(enquire, sort_by, self.column) + except NotSupportedError: + _old_xapian_sort(enquire, sort_by, self.column) + + results = [] + facets_dict = { + 'fields': {}, + 'dates': {}, + 'queries': {}, + } + + if not end_offset: + end_offset = database.get_doccount() - start_offset + + ## prepare spies in case of facets + if facets: + facets_spies = self._prepare_facet_field_spies(facets) + for spy in facets_spies: + enquire.add_matchspy(spy) + + # print enquire.get_query() + + matches = self._get_enquire_mset(database, enquire, start_offset, end_offset) + + for match in matches: + app_label, model_name, pk, model_data = pickle.loads(self._get_document_data(database, match.document)) + if highlight: + model_data['highlighted'] = { + self.content_field_name: self._do_highlight( + model_data.get(self.content_field_name), query + ) + } + results.append( + result_class(app_label, model_name, pk, match.percent, **model_data) + ) + + if facets: + # pick single valued facets from spies + single_facets_dict = self._process_facet_field_spies(facets_spies) + + # pick multivalued valued facets from results + multi_facets_dict = self._do_multivalued_field_facets(results, facets) + + # merge both results (http://stackoverflow.com/a/38990/931303) + facets_dict['fields'] = dict(list(single_facets_dict.items()) + list(multi_facets_dict.items())) + + if date_facets: + facets_dict['dates'] = self._do_date_facets(results, date_facets) + + if query_facets: + facets_dict['queries'] = self._do_query_facets(results, query_facets) + + return { + 'results': results, + 'hits': self._get_hit_count(database, enquire), + 'facets': facets_dict, + 'spelling_suggestion': spelling_suggestion, + } + + def more_like_this(self, model_instance, additional_query=None, + start_offset=0, end_offset=None, + limit_to_registered_models=True, result_class=None, **kwargs): + """ + Given a model instance, returns a result set of similar documents. + + Required arguments: + `model_instance` -- The model instance to use as a basis for + retrieving similar documents. + + Optional arguments: + `additional_query` -- An additional query to narrow results + `start_offset` -- The starting offset (default=0) + `end_offset` -- The ending offset (default=None), if None, then all documents + `limit_to_registered_models` -- Limit returned results to models registered in the search (default = True) + + Returns: + A dictionary with the following keys: + `results` -- A list of `SearchResult` + `hits` -- The total available results + + Opens a database connection, then builds a simple query using the + `model_instance` to build the unique identifier. + + For each document retrieved(should always be one), adds an entry into + an RSet (relevance set) with the document id, then, uses the RSet + to query for an ESet (A set of terms that can be used to suggest + expansions to the original query), omitting any document that was in + the original query. + + Finally, processes the resulting matches and returns. + """ + database = self._database() + + if result_class is None: + result_class = SearchResult + + query = xapian.Query(TERM_PREFIXES[ID] + get_identifier(model_instance)) + + enquire = xapian.Enquire(database) + enquire.set_query(query) + + rset = xapian.RSet() + + if not end_offset: + end_offset = database.get_doccount() + + match = None + for match in self._get_enquire_mset(database, enquire, 0, end_offset): + rset.add_document(match.docid) + + if match is None: + if not self.silently_fail: + raise InvalidIndexError('Instance %s with id "%d" not indexed' % + (get_identifier(model_instance), model_instance.id)) + else: + return {'results': [], + 'hits': 0} + + query = xapian.Query( + xapian.Query.OP_ELITE_SET, + [expand.term for expand in enquire.get_eset(match.document.termlist_count(), rset, XHExpandDecider())], + match.document.termlist_count() + ) + query = xapian.Query( + xapian.Query.OP_AND_NOT, [query, TERM_PREFIXES[ID] + get_identifier(model_instance)] + ) + + if limit_to_registered_models: + query = self._build_models_query(query) + + if additional_query: + query = xapian.Query( + xapian.Query.OP_AND, query, additional_query + ) + + enquire.set_query(query) + + results = [] + matches = self._get_enquire_mset(database, enquire, start_offset, end_offset) + + for match in matches: + app_label, model_name, pk, model_data = pickle.loads(self._get_document_data(database, match.document)) + results.append( + result_class(app_label, model_name, pk, match.percent, **model_data) + ) + + return { + 'results': results, + 'hits': self._get_hit_count(database, enquire), + 'facets': { + 'fields': {}, + 'dates': {}, + 'queries': {}, + }, + 'spelling_suggestion': None, + } + + def parse_query(self, query_string): + """ + Given a `query_string`, will attempt to return a xapian.Query + + Required arguments: + ``query_string`` -- A query string to parse + + Returns a xapian.Query + """ + if query_string == '*': + return xapian.Query('') # Match everything + elif query_string == '': + return xapian.Query() # Match nothing + + qp = xapian.QueryParser() + qp.set_database(self._database()) + qp.set_stemmer(xapian.Stem(self.language)) + qp.set_stemming_strategy(self.stemming_strategy) + qp.set_default_op(XAPIAN_OPTS[DEFAULT_OPERATOR]) + qp.add_boolean_prefix(DJANGO_CT, TERM_PREFIXES[DJANGO_CT]) + + for field_dict in self.schema: + # since 'django_ct' has a boolean_prefix, + # we ignore it here. + if field_dict['field_name'] == DJANGO_CT: + continue + + qp.add_prefix( + field_dict['field_name'], + TERM_PREFIXES['field'] + field_dict['field_name'].upper() + ) + + vrp = XHValueRangeProcessor(self) + qp.add_valuerangeprocessor(vrp) + + return qp.parse_query(query_string, self.flags) + + def build_schema(self, fields): + """ + Build the schema from fields. + + :param fields: A list of fields in the index + :returns: list of dictionaries + + Each dictionary has the keys + field_name: The name of the field index + type: what type of value it is + 'multi_valued': if it allows more than one value + 'column': a number identifying it + 'type': the type of the field + 'multi_valued': 'false', 'column': 0} + """ + content_field_name = '' + schema_fields = [ + {'field_name': ID, + 'type': 'text', + 'multi_valued': 'false', + 'column': 0}, + {'field_name': DJANGO_ID, + 'type': 'integer', + 'multi_valued': 'false', + 'column': 1}, + {'field_name': DJANGO_CT, + 'type': 'text', + 'multi_valued': 'false', + 'column': 2}, + ] + self._columns[ID] = 0 + self._columns[DJANGO_ID] = 1 + self._columns[DJANGO_CT] = 2 + + column = len(schema_fields) + + for field_name, field_class in sorted(list(fields.items()), key=lambda n: n[0]): + if field_class.document is True: + content_field_name = field_class.index_fieldname + + if field_class.indexed is True: + field_data = { + 'field_name': field_class.index_fieldname, + 'type': 'text', + 'multi_valued': 'false', + 'column': column, + } + + if field_class.field_type == 'date': + field_data['type'] = 'date' + elif field_class.field_type == 'datetime': + field_data['type'] = 'datetime' + elif field_class.field_type == 'integer': + field_data['type'] = 'integer' + elif field_class.field_type == 'float': + field_data['type'] = 'float' + elif field_class.field_type == 'boolean': + field_data['type'] = 'boolean' + elif field_class.field_type == 'ngram': + field_data['type'] = 'ngram' + elif field_class.field_type == 'edge_ngram': + field_data['type'] = 'edge_ngram' + + if field_class.is_multivalued: + field_data['multi_valued'] = 'true' + + schema_fields.append(field_data) + self._columns[field_data['field_name']] = column + column += 1 + + return content_field_name, schema_fields + + @staticmethod + def _do_highlight(content, query, tag='em'): + """ + Highlight `query` terms in `content` with html `tag`. + + This method assumes that the input text (`content`) does not contain + any special formatting. That is, it does not contain any html tags + or similar markup that could be screwed up by the highlighting. + + Required arguments: + `content` -- Content to search for instances of `text` + `text` -- The text to be highlighted + """ + for term in query: + term = term.decode('utf-8') + for match in re.findall('[^A-Z]+', term): # Ignore field identifiers + match_re = re.compile(match, re.I) + content = match_re.sub('<%s>%s' % (tag, term, tag), content) + + return content + + def _prepare_facet_field_spies(self, facets): + """ + Returns a list of spies based on the facets + used to count frequencies. + """ + spies = [] + for facet in facets: + slot = self.column[facet] + spy = xapian.ValueCountMatchSpy(slot) + # add attribute "slot" to know which column this spy is targeting. + spy.slot = slot + spies.append(spy) + return spies + + def _process_facet_field_spies(self, spies): + """ + Returns a dict of facet names with lists of + tuples of the form (term, term_frequency) + from a list of spies that observed the enquire. + """ + facet_dict = {} + for spy in spies: + field = self.schema[spy.slot] + field_name, field_type = field['field_name'], field['type'] + + facet_dict[field_name] = [] + for facet in list(spy.values()): + if field_type == 'float': + # the float term is a Xapian serialized object, which is + # in bytes. + term = facet.term + else: + term = facet.term.decode('utf-8') + + facet_dict[field_name].append((_from_xapian_value(term, field_type), + facet.termfreq)) + return facet_dict + + def _do_multivalued_field_facets(self, results, field_facets): + """ + Implements a multivalued field facet on the results. + + This is implemented using brute force - O(N^2) - + because Xapian does not have it implemented yet + (see http://trac.xapian.org/ticket/199) + """ + facet_dict = {} + + for field in field_facets: + facet_list = {} + if not self._multi_value_field(field): + continue + + for result in results: + field_value = getattr(result, field) + for item in field_value: # Facet each item in a MultiValueField + facet_list[item] = facet_list.get(item, 0) + 1 + + facet_dict[field] = list(facet_list.items()) + return facet_dict + + @staticmethod + def _do_date_facets(results, date_facets): + """ + Private method that facets a document by date ranges + + Required arguments: + `results` -- A list SearchResults to facet + `date_facets` -- A dictionary containing facet parameters: + {'field': {'start_date': ..., 'end_date': ...: 'gap_by': '...', 'gap_amount': n}} + nb., gap must be one of the following: + year|month|day|hour|minute|second + + For each date facet field in `date_facets`, generates a list + of date ranges (from `start_date` to `end_date` by `gap_by`) then + iterates through `results` and tallies the count for each date_facet. + + Returns a dictionary of date facets (fields) containing a list with + entries for each range and a count of documents matching the range. + + eg. { + 'pub_date': [ + (datetime.datetime(2009, 1, 1, 0, 0), 5), + (datetime.datetime(2009, 2, 1, 0, 0), 0), + (datetime.datetime(2009, 3, 1, 0, 0), 0), + (datetime.datetime(2008, 4, 1, 0, 0), 1), + (datetime.datetime(2008, 5, 1, 0, 0), 2), + ], + } + """ + def next_datetime(previous, gap_value, gap_type): + year = previous.year + month = previous.month + + if gap_type == 'year': + next = previous.replace(year=year + gap_value) + elif gap_type == 'month': + if month + gap_value <= 12: + next = previous.replace(month=month + gap_value) + else: + next = previous.replace( + month=((month + gap_value) % 12), + year=(year + (month + gap_value) // 12) + ) + elif gap_type == 'day': + next = previous + datetime.timedelta(days=gap_value) + elif gap_type == 'hour': + return previous + datetime.timedelta(hours=gap_value) + elif gap_type == 'minute': + next = previous + datetime.timedelta(minutes=gap_value) + elif gap_type == 'second': + next = previous + datetime.timedelta(seconds=gap_value) + else: + raise TypeError('\'gap_by\' must be ' + '{second, minute, day, month, year}') + return next + + facet_dict = {} + + for date_facet, facet_params in list(date_facets.items()): + gap_type = facet_params.get('gap_by') + gap_value = facet_params.get('gap_amount', 1) + date_range = facet_params['start_date'] + + # construct the bins of the histogram + facet_list = [] + while date_range < facet_params['end_date']: + facet_list.append((date_range, 0)) + date_range = next_datetime(date_range, gap_value, gap_type) + + facet_list = sorted(facet_list, key=lambda x: x[0], reverse=True) + + for result in results: + result_date = getattr(result, date_facet) + + # convert date to datetime + if not isinstance(result_date, datetime.datetime): + result_date = datetime.datetime(result_date.year, + result_date.month, + result_date.day) + + # ignore results outside the boundaries. + if facet_list[0][0] < result_date < facet_list[-1][0]: + continue + + # populate the histogram by putting the result on the right bin. + for n, facet_date in enumerate(facet_list): + if result_date > facet_date[0]: + # equal to facet_list[n][1] += 1, but for a tuple + facet_list[n] = (facet_list[n][0], (facet_list[n][1] + 1)) + break # bin found; go to next result + + facet_dict[date_facet] = facet_list + + return facet_dict + + def _do_query_facets(self, results, query_facets): + """ + Private method that facets a document by query + + Required arguments: + `results` -- A list SearchResults to facet + `query_facets` -- A dictionary containing facet parameters: + {'field': 'query', [...]} + + For each query in `query_facets`, generates a dictionary entry with + the field name as the key and a tuple with the query and result count + as the value. + + eg. {'name': ('a*', 5)} + """ + facet_dict = {} + for field, query in list(dict(query_facets).items()): + facet_dict[field] = (query, self.search(self.parse_query(query))['hits']) + + return facet_dict + + @staticmethod + def _do_spelling_suggestion(database, query, spelling_query): + """ + Private method that returns a single spelling suggestion based on + `spelling_query` or `query`. + + Required arguments: + `database` -- The database to check spelling against + `query` -- The query to check + `spelling_query` -- If not None, this will be checked instead of `query` + + Returns a string with a suggested spelling + """ + if spelling_query: + if ' ' in spelling_query: + return ' '.join([database.get_spelling_suggestion(term).decode('utf-8') for term in spelling_query.split()]) + else: + return database.get_spelling_suggestion(spelling_query).decode('utf-8') + + term_set = set() + for term in query: + for match in re.findall('[^A-Z]+', term.decode('utf-8')): # Ignore field identifiers + term_set.add(database.get_spelling_suggestion(match).decode('utf-8')) + + return ' '.join(term_set) + + def _database(self, writable=False): + """ + Private method that returns a xapian.Database for use. + + Optional arguments: + ``writable`` -- Open the database in read/write mode (default=False) + + Returns an instance of a xapian.Database or xapian.WritableDatabase + """ + if self.path == MEMORY_DB_NAME: + if not self.inmemory_db: + self.inmemory_db = xapian.inmemory_open() + return self.inmemory_db + if writable: + database = xapian.WritableDatabase(self.path, xapian.DB_CREATE_OR_OPEN) + else: + try: + database = xapian.Database(self.path) + except xapian.DatabaseOpeningError: + raise InvalidIndexError('Unable to open index at %s' % self.path) + + return database + + @staticmethod + def _get_enquire_mset(database, enquire, start_offset, end_offset, checkatleast=DEFAULT_CHECK_AT_LEAST): + """ + A safer version of Xapian.enquire.get_mset + + Simply wraps the Xapian version and catches any `Xapian.DatabaseModifiedError`, + attempting a `database.reopen` as needed. + + Required arguments: + `database` -- The database to be read + `enquire` -- An instance of an Xapian.enquire object + `start_offset` -- The start offset to pass to `enquire.get_mset` + `end_offset` -- The end offset to pass to `enquire.get_mset` + """ + try: + return enquire.get_mset(start_offset, end_offset, checkatleast) + except xapian.DatabaseModifiedError: + database.reopen() + return enquire.get_mset(start_offset, end_offset, checkatleast) + + @staticmethod + def _get_document_data(database, document): + """ + A safer version of Xapian.document.get_data + + Simply wraps the Xapian version and catches any `Xapian.DatabaseModifiedError`, + attempting a `database.reopen` as needed. + + Required arguments: + `database` -- The database to be read + `document` -- An instance of an Xapian.document object + """ + try: + return document.get_data() + except xapian.DatabaseModifiedError: + database.reopen() + return document.get_data() + + def _get_hit_count(self, database, enquire): + """ + Given a database and enquire instance, returns the estimated number + of matches. + + Required arguments: + `database` -- The database to be queried + `enquire` -- The enquire instance + """ + return self._get_enquire_mset( + database, enquire, 0, database.get_doccount() + ).size() + + def _multi_value_field(self, field): + """ + Private method that returns `True` if a field is multi-valued, else + `False`. + + Required arguemnts: + `field` -- The field to lookup + + Returns a boolean value indicating whether the field is multi-valued. + """ + for field_dict in self.schema: + if field_dict['field_name'] == field: + return field_dict['multi_valued'] == 'true' + return False + + +class XapianSearchQuery(BaseSearchQuery): + """ + This class is the Xapian specific version of the SearchQuery class. + It acts as an intermediary between the ``SearchQuerySet`` and the + ``SearchBackend`` itself. + """ + def build_params(self, *args, **kwargs): + kwargs = super(XapianSearchQuery, self).build_params(*args, **kwargs) + + if self.end_offset is not None: + kwargs['end_offset'] = self.end_offset - self.start_offset + + return kwargs + + def build_query(self): + if not self.query_filter: + query = xapian.Query('') + else: + query = self._query_from_search_node(self.query_filter) + + if self.models: + subqueries = [ + xapian.Query( + xapian.Query.OP_SCALE_WEIGHT, + xapian.Query('%s%s' % (TERM_PREFIXES[DJANGO_CT], get_model_ct(model))), + 0 # Pure boolean sub-query + ) for model in self.models + ] + query = xapian.Query( + xapian.Query.OP_AND, query, + xapian.Query(xapian.Query.OP_OR, subqueries) + ) + + if self.boost: + subqueries = [ + xapian.Query( + xapian.Query.OP_SCALE_WEIGHT, + self._term_query(term, None, None), value + ) for term, value in list(self.boost.items()) + ] + query = xapian.Query( + xapian.Query.OP_AND_MAYBE, query, + xapian.Query(xapian.Query.OP_OR, subqueries) + ) + + return query + + def _query_from_search_node(self, search_node, is_not=False): + query_list = [] + + for child in search_node.children: + if isinstance(child, SearchNode): + query_list.append( + self._query_from_search_node(child, child.negated) + ) + else: + expression, term = child + field_name, filter_type = search_node.split_expression(expression) + + constructed_query_list = self._query_from_term(term, field_name, filter_type, is_not) + query_list.extend(constructed_query_list) + + if search_node.connector == 'OR': + return xapian.Query(xapian.Query.OP_OR, query_list) + else: + return xapian.Query(xapian.Query.OP_AND, query_list) + + def _query_from_term(self, term, field_name, filter_type, is_not): + """ + Uses arguments to construct a list of xapian.Query's. + """ + if field_name != 'content' and field_name not in self.backend.column: + raise InvalidIndexError('field "%s" not indexed' % field_name) + + # It it is an AutoQuery, it has no filters + # or others, thus we short-circuit the procedure. + if isinstance(term, AutoQuery): + if field_name != 'content': + query = '%s:%s' % (field_name, term.prepare(self)) + else: + query = term.prepare(self) + return [self.backend.parse_query(query)] + query_list = [] + + # Handle `ValuesListQuerySet`. + if hasattr(term, 'values_list'): + term = list(term) + + if field_name == 'content': + # content is the generic search: + # force no field_name search + # and the field_type to be 'text'. + field_name = None + field_type = 'text' + + # we don't know what is the type(term), so we parse it. + # Ideally this would not be required, but + # some filters currently depend on the term to make decisions. + term = _to_xapian_term(term) + + query_list.append(self._filter_contains(term, field_name, field_type, is_not)) + # when filter has no filter_type, haystack uses + # filter_type = 'content'. Here we remove it + # since the above query is already doing this + if filter_type == 'content': + filter_type = None + else: + # get the field_type from the backend + field_type = self.backend.schema[self.backend.column[field_name]]['type'] + + # private fields don't accept 'contains' or 'startswith' + # since they have no meaning. + if filter_type in ('contains', 'startswith') and field_name in (ID, DJANGO_ID, DJANGO_CT): + filter_type = 'exact' + + if field_type == 'text': + # we don't know what type "term" is, but we know we are searching as text + # so we parse it like that. + # Ideally this would not be required since _term_query does it, but + # some filters currently depend on the term to make decisions. + if isinstance(term, list): + term = [_to_xapian_term(term) for term in term] + else: + term = _to_xapian_term(term) + + # todo: we should check that the filter is valid for this field_type or raise InvalidIndexError + if filter_type == 'contains': + query_list.append(self._filter_contains(term, field_name, field_type, is_not)) + elif filter_type in ('content', 'exact'): + query_list.append(self._filter_exact(term, field_name, field_type, is_not)) + elif filter_type == 'in': + query_list.append(self._filter_in(term, field_name, field_type, is_not)) + elif filter_type == 'startswith': + query_list.append(self._filter_startswith(term, field_name, field_type, is_not)) + elif filter_type == 'endswith': + raise NotImplementedError("The Xapian search backend doesn't support endswith queries.") + elif filter_type == 'gt': + query_list.append(self._filter_gt(term, field_name, field_type, is_not)) + elif filter_type == 'gte': + query_list.append(self._filter_gte(term, field_name, field_type, is_not)) + elif filter_type == 'lt': + query_list.append(self._filter_lt(term, field_name, field_type, is_not)) + elif filter_type == 'lte': + query_list.append(self._filter_lte(term, field_name, field_type, is_not)) + elif filter_type == 'range': + query_list.append(self._filter_range(term, field_name, field_type, is_not)) + return query_list + + def _all_query(self): + """ + Returns a match all query. + """ + return xapian.Query('') + + def _filter_contains(self, term, field_name, field_type, is_not): + """ + Splits the sentence in terms and join them with OR, + using stemmed and un-stemmed. + + Assumes term is not a list. + """ + if field_type == 'text': + term_list = term.split() + else: + term_list = [term] + + query = self._or_query(term_list, field_name, field_type) + if is_not: + return xapian.Query(xapian.Query.OP_AND_NOT, self._all_query(), query) + else: + return query + + def _filter_in(self, term_list, field_name, field_type, is_not): + """ + Returns a query that matches exactly ANY term in term_list. + + Notice that: + A in {B,C} <=> (A = B or A = C) + ~(A in {B,C}) <=> ~(A = B or A = C) + Because OP_AND_NOT(C, D) <=> (C and ~D), then D=(A in {B,C}) requires `is_not=False`. + + Assumes term is a list. + """ + query_list = [self._filter_exact(term, field_name, field_type, is_not=False) + for term in term_list] + + if is_not: + return xapian.Query(xapian.Query.OP_AND_NOT, self._all_query(), + xapian.Query(xapian.Query.OP_OR, query_list)) + else: + return xapian.Query(xapian.Query.OP_OR, query_list) + + def _filter_exact(self, term, field_name, field_type, is_not): + """ + Returns a query that matches exactly the un-stemmed term + with positional order. + + Assumes term is not a list. + """ + if field_type == 'text' and field_name not in (DJANGO_CT,): + term = '^ %s $' % term + query = self._phrase_query(term.split(), field_name, field_type) + else: + query = self._term_query(term, field_name, field_type, stemmed=False) + + if is_not: + return xapian.Query(xapian.Query.OP_AND_NOT, self._all_query(), query) + else: + return query + + def _filter_startswith(self, term, field_name, field_type, is_not): + """ + Returns a startswith query on the un-stemmed term. + + Assumes term is not a list. + """ + if field_type == 'text': + if len(term.split()) == 1: + term = '^ %s*' % term + query = self.backend.parse_query(term) + else: + term = '^ %s' % term + query = self._phrase_query(term.split(), field_name, field_type) + else: + term = '^%s*' % term + query = self.backend.parse_query(term) + + if is_not: + return xapian.Query(xapian.Query.OP_AND_NOT, self._all_query(), query) + return query + + def _or_query(self, term_list, field, field_type): + """ + Joins each item of term_list decorated by _term_query with an OR. + """ + term_list = [self._term_query(term, field, field_type) for term in term_list] + return xapian.Query(xapian.Query.OP_OR, term_list) + + def _phrase_query(self, term_list, field_name, field_type): + """ + Returns a query that matches exact terms with + positional order (i.e. ["this", "thing"] != ["thing", "this"]) + and no stem. + + If `field_name` is not `None`, restrict to the field. + """ + term_list = [self._term_query(term, field_name, field_type, + stemmed=False) for term in term_list] + + query = xapian.Query(xapian.Query.OP_PHRASE, term_list) + return query + + def _term_query(self, term, field_name, field_type, stemmed=True): + """ + Constructs a query of a single term. + + If `field_name` is not `None`, the term is search on that field only. + If exact is `True`, the search is restricted to boolean matches. + """ + constructor = '{prefix}{term}' + + # construct the prefix to be used. + prefix = '' + if field_name: + prefix = TERM_PREFIXES['field'] + field_name.upper() + term = _to_xapian_term(term) + + if field_name in (ID, DJANGO_ID, DJANGO_CT): + # to ensure the value is serialized correctly. + if field_name == DJANGO_ID: + term = int(term) + term = _term_to_xapian_value(term, field_type) + return xapian.Query('%s%s' % (TERM_PREFIXES[field_name], term)) + + # we construct the query dates in a slightly different way + if field_type == 'datetime': + date, time = term.split() + return xapian.Query(xapian.Query.OP_AND_MAYBE, + constructor.format(prefix=prefix, term=date), + constructor.format(prefix=prefix, term=time) + ) + + # only use stem if field is text or "None" + if field_type not in ('text', None): + stemmed = False + + unstemmed_term = constructor.format(prefix=prefix, term=term) + if stemmed: + stem = xapian.Stem(self.backend.language) + stemmed_term = 'Z' + constructor.format(prefix=prefix, term=stem(term).decode('utf-8')) + + return xapian.Query(xapian.Query.OP_OR, + xapian.Query(stemmed_term), + xapian.Query(unstemmed_term) + ) + else: + return xapian.Query(unstemmed_term) + + def _filter_gt(self, term, field_name, field_type, is_not): + return self._filter_lte(term, field_name, field_type, is_not=not is_not) + + def _filter_lt(self, term, field_name, field_type, is_not): + return self._filter_gte(term, field_name, field_type, is_not=not is_not) + + def _filter_gte(self, term, field_name, field_type, is_not): + """ + Private method that returns a xapian.Query that searches for any term + that is greater than `term` in a specified `field`. + """ + vrp = XHValueRangeProcessor(self.backend) + pos, begin, end = vrp('%s:%s' % (field_name, _term_to_xapian_value(term, field_type)), '*') + if is_not: + return xapian.Query(xapian.Query.OP_AND_NOT, + self._all_query(), + xapian.Query(xapian.Query.OP_VALUE_RANGE, pos, begin, end) + ) + return xapian.Query(xapian.Query.OP_VALUE_RANGE, pos, begin, end) + + def _filter_lte(self, term, field_name, field_type, is_not): + """ + Private method that returns a xapian.Query that searches for any term + that is less than `term` in a specified `field`. + """ + vrp = XHValueRangeProcessor(self.backend) + pos, begin, end = vrp('%s:' % field_name, '%s' % _term_to_xapian_value(term, field_type)) + if is_not: + return xapian.Query(xapian.Query.OP_AND_NOT, + self._all_query(), + xapian.Query(xapian.Query.OP_VALUE_RANGE, pos, begin, end) + ) + return xapian.Query(xapian.Query.OP_VALUE_RANGE, pos, begin, end) + + def _filter_range(self, term, field_name, field_type, is_not): + """ + Private method that returns a xapian.Query that searches for any term + that is between the values from the `term` list. + """ + vrp = XHValueRangeProcessor(self.backend) + pos, begin, end = vrp('%s:%s' % (field_name, _term_to_xapian_value(term[0], field_type)), + '%s' % _term_to_xapian_value(term[1], field_type)) + if is_not: + return xapian.Query(xapian.Query.OP_AND_NOT, + self._all_query(), + xapian.Query(xapian.Query.OP_VALUE_RANGE, pos, begin, end) + ) + return xapian.Query(xapian.Query.OP_VALUE_RANGE, pos, begin, end) + + +def _term_to_xapian_value(term, field_type): + """ + Converts a term to a serialized + Xapian value based on the field_type. + """ + assert field_type in FIELD_TYPES + + def strf(dt): + """ + Equivalent to datetime.datetime.strptime(dt, DATETIME_FORMAT) + but accepts years below 1900 (see http://stackoverflow.com/q/10263956/931303) + """ + return '%04d%02d%02d%02d%02d%02d' % ( + dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second) + + if field_type == 'boolean': + assert isinstance(term, bool) + if term: + value = 't' + else: + value = 'f' + + elif field_type == 'integer': + value = INTEGER_FORMAT % term + elif field_type == 'float': + value = xapian.sortable_serialise(term) + elif field_type == 'date' or field_type == 'datetime': + if field_type == 'date': + # http://stackoverflow.com/a/1937636/931303 and comments + term = datetime.datetime.combine(term, datetime.time()) + value = strf(term) + else: # field_type == 'text' + value = _to_xapian_term(term) + + return value + + +def _to_xapian_term(term): + """ + Converts a Python type to a + Xapian term that can be indexed. + """ + return force_text(term).lower() + + +def _from_xapian_value(value, field_type): + """ + Converts a serialized Xapian value + to Python equivalent based on the field_type. + + Doesn't accept multivalued fields. + """ + assert field_type in FIELD_TYPES + if field_type == 'boolean': + if value == 't': + return True + elif value == 'f': + return False + else: + InvalidIndexError('Field type "%d" does not accept value "%s"' % (field_type, value)) + elif field_type == 'integer': + return int(value) + elif field_type == 'float': + return xapian.sortable_unserialise(value) + elif field_type == 'date' or field_type == 'datetime': + datetime_value = datetime.datetime.strptime(value, DATETIME_FORMAT) + if field_type == 'datetime': + return datetime_value + else: + return datetime_value.date() + else: # field_type == 'text' + return value + + +def _old_xapian_sort(enquire, sort_by, column): + sorter = xapian.MultiValueSorter() + + for sort_field in sort_by: + if sort_field.startswith('-'): + reverse = True + sort_field = sort_field[1:] # Strip the '-' + else: + reverse = False # Reverse is inverted in Xapian -- http://trac.xapian.org/ticket/311 + sorter.add(column[sort_field], reverse) + + enquire.set_sort_by_key_then_relevance(sorter, True) + + +def _xapian_sort(enquire, sort_by, column): + try: + sorter = xapian.MultiValueKeyMaker() + except AttributeError: + raise NotSupportedError + + for sort_field in sort_by: + if sort_field.startswith('-'): + reverse = False + sort_field = sort_field[1:] # Strip the '-' + else: + reverse = True + sorter.add_value(column[sort_field], reverse) + + enquire.set_sort_by_key_then_relevance(sorter, True) + + +class XapianEngine(BaseEngine): + backend = XapianSearchBackend + query = XapianSearchQuery From ebdef285d55eb7c0e2576aa3bc7e8c107c540fe6 Mon Sep 17 00:00:00 2001 From: Arthur Pemberton Date: Thu, 3 Sep 2020 00:08:00 -0400 Subject: [PATCH 5/7] removing debug code --- dbdb/core/models.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dbdb/core/models.py b/dbdb/core/models.py index a84c02bc..43833496 100644 --- a/dbdb/core/models.py +++ b/dbdb/core/models.py @@ -214,7 +214,6 @@ def get_current(self): if self.id is None: self._current = SystemVersion(system=self) else: - print('get version', self.slug) self._current = self.versions.get(is_current=True) pass @@ -470,11 +469,10 @@ def create_twitter_card(self): text_size = [0, 0] for line in name.split("\n"): line_size = font.getsize(line) - #print("'%s' -> %s" % (line, str(line_size))) + text_size[0] = max(text_size[0], line_size[0]) text_size[1] += line_size[1] + 5 - #print("text_size =", text_size) - + logo = Image.new('RGBA', text_size) text_draw = ImageDraw.Draw(logo) text_draw.text((0, 0), name, font=font, fill=(70,70,70,255)) From 961c821219666777d7e9aa1eb96d8bba205840b7 Mon Sep 17 00:00:00 2001 From: Arthur Pemberton Date: Thu, 3 Sep 2020 00:13:03 -0400 Subject: [PATCH 6/7] added REST API to get a single system --- dbdb/api/v202004/serializers.py | 107 +++++++++++++++++++++----------- dbdb/api/v202004/urls.py | 1 + dbdb/api/v202004/views.py | 11 +++- 3 files changed, 83 insertions(+), 36 deletions(-) diff --git a/dbdb/api/v202004/serializers.py b/dbdb/api/v202004/serializers.py index 7a8e0241..8ee2ca37 100644 --- a/dbdb/api/v202004/serializers.py +++ b/dbdb/api/v202004/serializers.py @@ -9,43 +9,11 @@ from dbdb.core.models import SystemFeature -# serializers - -class SystemSerializer(serializers.HyperlinkedModelSerializer): +# abstracts - class Meta: - model = System - fields = [ - 'href', - 'name', - 'former_names', - 'start_year', - 'end_year', - 'description', - 'history', - 'acquired_by', - 'developer', - 'countries', - 'features', - 'project_types', - 'urls', - 'version', - ] - - href = serializers.SerializerMethodField() - former_names = serializers.SerializerMethodField() - start_year = serializers.SerializerMethodField() - end_year = serializers.SerializerMethodField() - description = serializers.SerializerMethodField() - history = serializers.SerializerMethodField() - acquired_by = serializers.SerializerMethodField() - developer = serializers.SerializerMethodField() - countries = serializers.SerializerMethodField() - features = serializers.SerializerMethodField() - project_types = serializers.SerializerMethodField() - urls = serializers.SerializerMethodField() - version = serializers.IntegerField(source='ver', read_only=True) +class AbstractSystemSerializer: + def get_acquired_by(self, obj): current = obj.get_current() return current.acquired_by @@ -138,3 +106,72 @@ def get_start_year(self, obj): pass + +# serializers + +class SystemBriefSerializer(AbstractSystemSerializer, serializers.HyperlinkedModelSerializer): + + class Meta: + model = System + fields = [ + 'url', + 'href', + 'name', + 'start_year', + 'end_year', + 'description', + 'history', + 'acquired_by', + 'developer', + 'project_types', + ] + + href = serializers.SerializerMethodField() + start_year = serializers.SerializerMethodField() + end_year = serializers.SerializerMethodField() + description = serializers.SerializerMethodField() + history = serializers.SerializerMethodField() + acquired_by = serializers.SerializerMethodField() + developer = serializers.SerializerMethodField() + project_types = serializers.SerializerMethodField() + url = serializers.HyperlinkedIdentityField(lookup_field='slug', read_only=True, view_name='api_v202004:systems_view') + + pass + +class SystemSerializer(AbstractSystemSerializer, serializers.HyperlinkedModelSerializer): + + class Meta: + model = System + fields = [ + 'href', + 'name', + 'former_names', + 'start_year', + 'end_year', + 'description', + 'history', + 'acquired_by', + 'developer', + 'countries', + 'features', + 'project_types', + 'urls', + 'version', + ] + + href = serializers.SerializerMethodField() + former_names = serializers.SerializerMethodField() + start_year = serializers.SerializerMethodField() + end_year = serializers.SerializerMethodField() + description = serializers.SerializerMethodField() + history = serializers.SerializerMethodField() + acquired_by = serializers.SerializerMethodField() + developer = serializers.SerializerMethodField() + countries = serializers.SerializerMethodField() + features = serializers.SerializerMethodField() + project_types = serializers.SerializerMethodField() + urls = serializers.SerializerMethodField() + version = serializers.IntegerField(source='ver', read_only=True) + + pass + diff --git a/dbdb/api/v202004/urls.py b/dbdb/api/v202004/urls.py index a274ef7a..3affc256 100644 --- a/dbdb/api/v202004/urls.py +++ b/dbdb/api/v202004/urls.py @@ -10,6 +10,7 @@ urlpatterns = [ path('systems', views.SystemsView.as_view(), name='systems'), + path('systems/', views.SystemView.as_view(), name='systems_view'), ] urlpatterns = format_suffix_patterns(urlpatterns) diff --git a/dbdb/api/v202004/views.py b/dbdb/api/v202004/views.py index ab2c5672..6ed96314 100644 --- a/dbdb/api/v202004/views.py +++ b/dbdb/api/v202004/views.py @@ -6,16 +6,25 @@ from dbdb.core.models import System from dbdb.core.models import SystemVersion # local imports +from .serializers import SystemBriefSerializer from .serializers import SystemSerializer # class based views -class SystemsView(generics.ListAPIView): +class SystemView(generics.RetrieveAPIView): + lookup_field = 'slug' queryset = System.objects.all() serializer_class = SystemSerializer + pass + +class SystemsView(generics.ListAPIView): + + queryset = System.objects.all() + serializer_class = SystemBriefSerializer + def paginate_queryset(self, queryset): if self.paginator is None: return None From e5ad71769530a016216f1aaafdb0da6682dcb10e Mon Sep 17 00:00:00 2001 From: Arthur Pemberton Date: Thu, 3 Sep 2020 00:35:47 -0400 Subject: [PATCH 7/7] Updating brand of browseable API --- dbdb/api/v202004/urls.py | 1 + dbdb/api/v202004/views.py | 16 ++++++++++++++++ templates/rest_framework/api.html | 11 +++++++++++ 3 files changed, 28 insertions(+) create mode 100644 templates/rest_framework/api.html diff --git a/dbdb/api/v202004/urls.py b/dbdb/api/v202004/urls.py index 3affc256..c00c542d 100644 --- a/dbdb/api/v202004/urls.py +++ b/dbdb/api/v202004/urls.py @@ -9,6 +9,7 @@ app_name = 'api_v202004' urlpatterns = [ + path('', views.APIRootView.as_view(), name='root'), path('systems', views.SystemsView.as_view(), name='systems'), path('systems/', views.SystemView.as_view(), name='systems_view'), ] diff --git a/dbdb/api/v202004/views.py b/dbdb/api/v202004/views.py index 6ed96314..a71b7a01 100644 --- a/dbdb/api/v202004/views.py +++ b/dbdb/api/v202004/views.py @@ -2,6 +2,9 @@ from django.shortcuts import render # third-party imports from rest_framework import generics +from rest_framework.response import Response +from rest_framework.reverse import reverse +from rest_framework.views import APIView # project imports from dbdb.core.models import System from dbdb.core.models import SystemVersion @@ -12,6 +15,19 @@ # class based views +class APIRootView(APIView): + """ + Welcome to Database of Databases RESTful API, a public API for accessing out data. + """ + + def get(self, request): + + data = { + 'systems': reverse('api_v202004:systems', request=request) + } + + return Response(data) + class SystemView(generics.RetrieveAPIView): lookup_field = 'slug' diff --git a/templates/rest_framework/api.html b/templates/rest_framework/api.html new file mode 100644 index 00000000..bc976783 --- /dev/null +++ b/templates/rest_framework/api.html @@ -0,0 +1,11 @@ +{% extends "rest_framework/base.html" %} + + +{% block title %}{{ name }} – Database of Databases RESTful API{% endblock %} + + +{% block branding %} + + Database of Databases RESTful API (v{{ request.path|slice:"6:"|slice:":6" }}) + +{% endblock %}