Optimize map query page load

Now we save geotags data returned from solr in the cache, and load it when computing the bytearray. In this way we only make one solr query (and no DB queries) for showing results in the map, which makes page load quite fast.

freesound/settings.py

@@ -653,7 +653,7 @@
 USE_SEARCH_ENGINE_SIMILARITY = False  # Does not currently apply to API
 
 SEARCH_ALLOW_DISPLAY_RESULTS_IN_MAP = True
-MAX_SEARCH_RESULTS_IN_MAP_DISPLAY = 1000  # This is the maximum number of sounds that will be shown when using "display results in map" mode
+MAX_SEARCH_RESULTS_IN_MAP_DISPLAY = 10000  # This is the maximum number of sounds that will be shown when using "display results in map" mode
 
 # -------------------------------------------------------------------------------
 # Similarity client settings

geotags/urls.py

@@ -26,7 +26,7 @@
     path('sounds_barray/user_latest/<username>/', geotags.geotags_for_user_latest_barray, name="geotags-for-user-latest-barray"),
     path('sounds_barray/pack/<int:pack_id>/', geotags.geotags_for_pack_barray, name="geotags-for-pack-barray"),
     path('sounds_barray/sound/<int:sound_id>/', geotags.geotag_for_sound_barray, name="geotags-for-sound-barray"),
-    path('sounds_barray/query/', geotags.gelotags_for_query_barray, name="geotags-for-query-barray"),
+    path('sounds_barray/query/', geotags.geotags_for_query_barray, name="geotags-for-query-barray"),
     re_path(r'^sounds_barray/(?P<tag>[\w-]+)?/?$', geotags.geotags_barray, name="geotags-barray"),
     path('geotags_box_barray/', geotags.geotags_box_barray, name="geotags-box-barray"),
     path('infowindow/<int:sound_id>/', geotags.infowindow, name="geotags-infowindow"),

geotags/views.py

@@ -59,24 +59,37 @@ def update_query_params_for_map_query(query_params, preserve_facets=False):
         'num_sounds': settings.MAX_SEARCH_RESULTS_IN_MAP_DISPLAY,
         'group_by_pack': False,
         'only_sounds_with_pack': False,
+        'field_list': ['id', 'score', 'geotag']
     })
     if not preserve_facets:
         # No need to compute facets for the bytearray, but it might be needed for the main query
         if 'facets' in query_params:
             del query_params['facets']
 
 
-def generate_bytearray(sound_queryset):
+def generate_bytearray(sound_queryset_or_list):
     # sounds as bytearray
     packed_sounds = io.BytesIO()
     num_sounds_in_bytearray = 0
-    for s in sound_queryset:
-        if not math.isnan(s.geotag.lat) and not math.isnan(s.geotag.lon):
-            packed_sounds.write(struct.pack("i", s.id))
-            packed_sounds.write(struct.pack("i", int(s.geotag.lat*1000000)))
-            packed_sounds.write(struct.pack("i", int(s.geotag.lon*1000000)))
-            num_sounds_in_bytearray += 1
-
+    for s in sound_queryset_or_list:
+        if type(s) == Sound:
+            if not math.isnan(s.geotag.lat) and not math.isnan(s.geotag.lon):
+                packed_sounds.write(struct.pack("i", s.id))
+                packed_sounds.write(struct.pack("i", int(s.geotag.lat * 1000000)))
+                packed_sounds.write(struct.pack("i", int(s.geotag.lon * 1000000)))
+                num_sounds_in_bytearray += 1
+        elif type(s) == dict:
+            try:
+                lon, lat = s['geotag'][0].split(' ')
+                lat = max(min(float(lat), 90), -90) 
+                lon = max(min(float(lon), 180), -180) 
+                packed_sounds.write(struct.pack("i", s['id']))
+                packed_sounds.write(struct.pack("i", int(lat * 1000000)))
+                packed_sounds.write(struct.pack("i", int(lon * 1000000)))
+                num_sounds_in_bytearray += 1
+            except:
+                pass
+
     return packed_sounds.getvalue(), num_sounds_in_bytearray
 
 
@@ -161,12 +174,10 @@ def geotag_for_sound_barray(request, sound_id):
     return HttpResponse(generated_bytearray, content_type='application/octet-stream')
 
 
-def gelotags_for_query_barray(request):
-    query_params, _, _ = search_prepare_parameters(request)
-    update_query_params_for_map_query(query_params)
-    results, _ = perform_search_engine_query(query_params)
-    resultids = [d.get("id") for d in results.docs]
-    generated_bytearray, num_geotags = generate_bytearray(Sound.objects.select_related('geotag').filter(id__in=resultids))
+def geotags_for_query_barray(request):
+    results_cache_key = request.GET.get('key', None)
+    results_docs = cache.get(results_cache_key)
+    generated_bytearray, num_geotags = generate_bytearray(results_docs)
     if num_geotags > 0:
         log_map_load('query', num_geotags, request)
     return HttpResponse(generated_bytearray, content_type='application/octet-stream')

search/views.py

@@ -22,6 +22,7 @@
 import json
 import logging
 import re
+import uuid
 import sentry_sdk
 from collections import defaultdict, Counter
 
@@ -118,12 +119,14 @@ def search_view_helper(request, tags_mode=False):
     disable_display_results_in_grid_option = False
     map_bytearray_url = ''
     use_map_mode = settings.SEARCH_ALLOW_DISPLAY_RESULTS_IN_MAP and request.GET.get("mm", "0") == "1"
+    map_mode_query_results_cache_key = None
     if use_map_mode:
         disable_group_by_pack_option = True
         disable_only_sounds_by_pack_option = True
         disable_display_results_in_grid_option = True
         geotags.views.update_query_params_for_map_query(query_params, preserve_facets=True)
-        map_bytearray_url = reverse('geotags-for-query-barray') + '?' + request.get_full_path().split('?')[-1]
+        map_mode_query_results_cache_key = f'map-query-results-{str(uuid.uuid4())[0:10]}'
+        map_bytearray_url = reverse('geotags-for-query-barray') + f'?key={map_mode_query_results_cache_key}'
 
     tvars = {
         'error_text': None,
@@ -194,8 +197,13 @@ def search_view_helper(request, tags_mode=False):
                 for d in docs:
                     d["pack"] = allpacks[int(d.get("group_name").split('_')[0])]
         else:
-            # In map mode we don't need to retrieve any information about sounds as we'll make another query
-            # to generate the geotags bytearray
+            # In map we configure the search query to already return geotags data. Here we collect all this data
+            # and save it to the cache so we can collect it in the 'geotags_for_query_barray' view which prepares
+            # data points for the map of sounds. 
+            cache.set(map_mode_query_results_cache_key, results.docs, 60)  # cache for 1 minute
+
+            # Nevertheless we set docs to empty list as we won't displat anything in the search results page (the map
+            # will make an extra request that will load the cached data and display it in the map)
             docs = []
 
         search_logger.info('Search (%s)' % json.dumps({

templates/search/search.html

@@ -225,7 +225,7 @@ <h3>
                                 </li>
                                 {% if allow_map_mode %}
                                 <li class="bw-search__filter-value v-padding-1">
-                                    <label class="between w-100" title="Display search results in a map. Note that a maximum of {{ max_search_results_map_mode }} sounds will be displayed.">
+                                    <label class="between w-100" title="Display search results in a map">
                                         <div class="bw-search__filter-checkbox">
                                             <input id="use_map_mode" type="checkbox" class="bw-checkbox" {% if use_map_mode %}checked{% endif %} />
                                             <input id="use_map_mode_hidden" type="hidden" name="mm" value="{{ use_map_mode|yesno:"1,0" }}"/>
@@ -376,6 +376,11 @@ <h5>No results... &#128543</h5>
                      data-access-token="pk.eyJ1IjoiZnJlZXNvdW5kIiwiYSI6ImNrd3E0Mm9lbjBqM2Qyb2wwdmwxaWI3a3oifQ.MZkgLSByRuk_Xql67CySAg"
                 ><span id="mapLoadingIndicator">Loading map... <img width="12px" height="12px" src="{% static 'bw-frontend/public/bw_indicator.gif' %}"></span></div>
             </div>
+            {% if paginator.count > max_search_results_map_mode %}
+            <div class="right">
+                <p class="text-grey">{% bw_icon 'notification' %} Note that only the first {{ max_search_results_map_mode|bw_intcomma }} search results are shown on the map</p>
+            </div>
+            {% endif %}
             {% endif %}
         </main>
     </div>