diff --git a/apiv2/views.py b/apiv2/views.py index 2ef40833e2..275caa3a0b 100755 --- a/apiv2/views.py +++ b/apiv2/views.py @@ -61,7 +61,7 @@ ApiSearchPaginator, get_sounds_descriptors, prepend_base, get_formatted_examples_for_view from bookmarks.models import Bookmark, BookmarkCategory from geotags.models import GeoTag -from similarity.client import Similarity +from similarity.client import similarity_client from sounds.models import Sound, Pack, License from comments.models import Comment from ratings.models import SoundRating @@ -1203,7 +1203,7 @@ def get_description(cls): def get(self, request, *args, **kwargs): logger.info(self.log_message('available_audio_descriptors')) try: - descriptor_names = Similarity.get_descriptor_names() + descriptor_names = similarity_client.get_descriptor_names() del descriptor_names['all'] for key, value in descriptor_names.items(): descriptor_names[key] = [item[1:] for item in value] # remove initial dot from descriptor names diff --git a/freesound/settings.py b/freesound/settings.py index 5715c9b603..cfcd98dc25 100644 --- a/freesound/settings.py +++ b/freesound/settings.py @@ -376,9 +376,8 @@ # ------------------------------------------------------------------------------- # Similarity client settings -SIMILARITY_ADDRESS = 'similarity' -SIMILARITY_PORT = 8008 -SIMILARITY_INDEXING_SERVER_PORT = 8009 +SIMILARITY_ADDRESS = 'similarity:8008' +INDEXING_SIMILARITY_ADDRESS = 'similarity:8009' # ------------------------------------------------------------------------------- # Tag recommendation client settings diff --git a/general/management/commands/report_index_statuses.py b/general/management/commands/report_index_statuses.py index eca90299da..6e87b94918 100644 --- a/general/management/commands/report_index_statuses.py +++ b/general/management/commands/report_index_statuses.py @@ -20,7 +20,7 @@ from django.core.management.base import BaseCommand from utils.search.search_general import get_all_sound_ids_from_solr, delete_sounds_from_solr -from utils.similarity_utilities import Similarity +from similarity.client import similarity_client from sounds.models import Sound import sys @@ -49,7 +49,7 @@ def handle(self, *args, **options): # Get ell gaia ids print "Getting gaia ids...", - gaia_ids = Similarity.get_all_sound_ids() + gaia_ids = similarity_client.get_all_sound_ids() print "done!" print "Getting freesound db data..." @@ -109,4 +109,4 @@ def handle(self, *args, **options): for count, sid in enumerate(in_gaia_not_in_fs): sys.stdout.write('\r\tDeleting sound %i of %i ' % (count+1, N)) sys.stdout.flush() - Similarity.delete(sid) + similarity_client.delete(sid) diff --git a/general/management/commands/similarity_save_index.py b/general/management/commands/similarity_save_index.py index bc03440db4..9a1d56acd3 100644 --- a/general/management/commands/similarity_save_index.py +++ b/general/management/commands/similarity_save_index.py @@ -19,8 +19,7 @@ # from django.core.management.base import BaseCommand -from similarity.client import Similarity -from optparse import make_option +from similarity.client import indexing_similarity_client, similarity_client import logging logger = logging.getLogger("web") @@ -40,7 +39,7 @@ def handle(self, *args, **options): logger.info('Saving current similarity index') if options['indexing_server']: - Similarity.save_indexing_server() + indexing_similarity_client.save() else: - Similarity.save() + similarity_client.save() diff --git a/general/management/commands/similarity_update.py b/general/management/commands/similarity_update.py index 81d11348b5..3d36300b86 100644 --- a/general/management/commands/similarity_update.py +++ b/general/management/commands/similarity_update.py @@ -23,7 +23,7 @@ import yaml from django.core.management.base import BaseCommand -from similarity.client import Similarity +from similarity.client import indexing_similarity_client, similarity_client from sounds.models import Sound logger_console = logging.getLogger('console') @@ -106,9 +106,9 @@ def handle(self, *args, **options): try: if options['indexing_server']: - result = Similarity.add_to_indexing_server(sound.id, sound.locations('analysis.statistics.path')) + result = indexing_similarity_client.add(sound.id, sound.locations('analysis.statistics.path')) else: - result = Similarity.add(sound.id, sound.locations('analysis.statistics.path')) + result = similarity_client.add(sound.id, sound.locations('analysis.statistics.path')) sound.set_similarity_state('OK') sound.invalidate_template_caches() logger_console.info("%s (%i of %i)" % (result, count+1, N)) @@ -116,9 +116,9 @@ def handle(self, *args, **options): # Every 2000 added sounds, save the index #if count % 2000 == 0: # if options['indexing_server']: - # Similarity.save_indexing_server() + # indexing_similarity_client.save() # else: - # Similarity.save() + # similarity_client.save() except Exception as e: if not options['indexing_server']: @@ -129,6 +129,6 @@ def handle(self, *args, **options): logger.info("Finished similarity update. %i sounds added to the similarity index" % to_be_added.count()) # At the end save the index #if options['indexing_server']: - # Similarity.save_indexing_server() + # indexing_similarity_client.save() #else: - # Similarity.save() + # similarity_client.save() diff --git a/similarity/client/__init__.py b/similarity/client/__init__.py index 49c2c24c2c..d22e6fa709 100644 --- a/similarity/client/__init__.py +++ b/similarity/client/__init__.py @@ -19,18 +19,19 @@ # import requests +from django.conf import settings -_URL_ADD_POINT = 'add_point/' -_URL_DELETE_POINT = 'delete_point/' -_URL_GET_DESCRIPTOR_NAMES = 'get_descriptor_names/' -_URL_GET_ALL_SOUND_IDS = 'get_all_point_names/' -_URL_CONTAINS_POINT = 'contains/' -_URL_NNSEARCH = 'nnsearch/' -_URL_API_SEARCH = 'api_search/' -_URL_SOUNDS_DESCRIPTORS = 'get_sounds_descriptors/' -_URL_SAVE = 'save/' -_URL_RELOAD_GAIA_WRAPPER = 'reload_gaia_wrapper/' -_URL_CLEAR_MEMORY = 'clear_memory/' +_URL_ADD_POINT = 'add_point/' +_URL_DELETE_POINT = 'delete_point/' +_URL_GET_DESCRIPTOR_NAMES = 'get_descriptor_names/' +_URL_GET_ALL_SOUND_IDS = 'get_all_point_names/' +_URL_CONTAINS_POINT = 'contains/' +_URL_NNSEARCH = 'nnsearch/' +_URL_API_SEARCH = 'api_search/' +_URL_SOUNDS_DESCRIPTORS = 'get_sounds_descriptors/' +_URL_SAVE = 'save/' +_URL_RELOAD_GAIA_WRAPPER = 'reload_gaia_wrapper/' +_URL_CLEAR_MEMORY = 'clear_memory/' class SimilarityException(Exception): @@ -42,6 +43,7 @@ def __init__(self, *args, **kwargs): def _get_url_as_json(url, data=None, timeout=None): + # TODO: (requests): If no timeout is specified explicitly, requests do not time out. kwargs = dict() if data is not None: kwargs['data'] = data @@ -66,7 +68,6 @@ class Similarity(object): def __init__(self, host): self.base_url = 'http://%s/similarity/' % host - self.base_indexing_server_url = 'http://%s/similarity/' % host def search(self, sound_id, num_results = None, preset = None, offset = None): url = self.base_url + _URL_NNSEARCH + '?' + 'sound_id=' + str(sound_id) @@ -78,7 +79,8 @@ def search(self, sound_id, num_results = None, preset = None, offset = None): url += '&offset=' + str(offset) return _result_or_exception(_get_url_as_json(url)) - def api_search(self, target_type=None, target=None, filter=None, preset=None, metric_descriptor_names=None, num_results=None, offset=None, file=None, in_ids=None): + def api_search(self, target_type=None, target=None, filter=None, preset=None, metric_descriptor_names=None, + num_results=None, offset=None, file=None, in_ids=None): url = self.base_url + _URL_API_SEARCH + '?' if target_type: url += '&target_type=' + str(target_type) @@ -106,10 +108,6 @@ def add(self, sound_id, yaml_path): url = self.base_url + _URL_ADD_POINT + '?' + 'sound_id=' + str(sound_id) + '&location=' + str(yaml_path) return _result_or_exception(_get_url_as_json(url)) - def add_to_indexing_server(self, sound_id, yaml_path): - url = self.base_indexing_server_url + _URL_ADD_POINT + '?' + 'sound_id=' + str(sound_id) + '&location=' + str(yaml_path) - return _result_or_exception(_get_url_as_json(url)) - def get_all_sound_ids(self): url = self.base_url + _URL_GET_ALL_SOUND_IDS return _result_or_exception(_get_url_as_json(url)) @@ -122,32 +120,15 @@ def delete(self, sound_id): url = self.base_url + _URL_DELETE_POINT + '?' + 'sound_id=' + str(sound_id) return _result_or_exception(_get_url_as_json(url)) - def contains(self, sound_id): - url = self.base_url + _URL_CONTAINS_POINT + '?' + 'sound_id=' + str(sound_id) - return _result_or_exception(_get_url_as_json(url)) - - def save(self, filename = None): + def save(self, filename=None): url = self.base_url + _URL_SAVE if filename: url += '?' + 'filename=' + str(filename) return _result_or_exception(_get_url_as_json(url, timeout=60 * 5)) - def save_indexing_server(self, filename = None): - url = self.base_indexing_server_url + _URL_SAVE - if filename: - url += '?' + 'filename=' + str(filename) - return _result_or_exception(_get_url_as_json(url)) - - def clear_indexing_server_memory(self): - url = self.base_indexing_server_url + _URL_CLEAR_MEMORY - return _result_or_exception(_get_url_as_json(url)) - - def reload_indexing_server_gaia_wrapper(self): - url = self.base_indexing_server_url + _URL_RELOAD_GAIA_WRAPPER - return _result_or_exception(_get_url_as_json(url)) - def get_sounds_descriptors(self, sound_ids, descriptor_names=None, normalization=True, only_leaf_descriptors=False): - url = self.base_url + _URL_SOUNDS_DESCRIPTORS + '?' + 'sound_ids=' + ','.join([str(sound_id) for sound_id in sound_ids]) + url = self.base_url + _URL_SOUNDS_DESCRIPTORS + '?' + 'sound_ids=' + ','.join( + [str(sound_id) for sound_id in sound_ids]) if descriptor_names: url += '&descriptor_names=' + ','.join(descriptor_names) if normalization: @@ -156,3 +137,7 @@ def get_sounds_descriptors(self, sound_ids, descriptor_names=None, normalization url += '&only_leaf_descriptors=1' return _result_or_exception(_get_url_as_json(url)) + + +similarity_client = Similarity(settings.SIMILARITY_ADDRESS) +indexing_similarity_client = Similarity(settings.INDEXING_SIMILARITY_ADDRESS) diff --git a/similarity/test/__init__.py b/similarity/test/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/similarity/test/test_client.py b/similarity/test/test_client.py new file mode 100644 index 0000000000..48c6463849 --- /dev/null +++ b/similarity/test/test_client.py @@ -0,0 +1,83 @@ +import unittest + +import mock + +from similarity import client + + +class SimilarityClientTest(unittest.TestCase): + + def setUp(self): + self.client = client.Similarity("localhost") + + @mock.patch('similarity.client._get_url_as_json') + def test_search(self, get_url): + get_url.return_value = {"error": False, "result": {"data": "here"}} + + r = self.client.search(1) + + self.assertEqual(r, {"data": "here"}) + # requests.Request('GET', 'url', params=params).prepare().url + get_url.assert_called_with("http://localhost/similarity/nnsearch/?sound_id=1") + get_url.reset_mock() + + self.client.search(1, num_results=10, preset='pr', offset=3) + get_url.assert_called_with("http://localhost/similarity/nnsearch/?sound_id=1&num_results=10&preset=pr&offset=3") + + @mock.patch('similarity.client._get_url_as_json') + def test_api_search(self, get_url): + get_url.return_value = {"error": False, "result": {"data": "here"}} + + self.client.api_search() + get_url.assert_called_with("http://localhost/similarity/api_search/?", data=None) + get_url.reset_mock() + + self.client.api_search(target_type=1, target=2, filter=3, preset=4, metric_descriptor_names=5, + num_results=6, offset=7, file='x', in_ids="7,8,9") + get_url.assert_called_with("http://localhost/similarity/api_search/?&target_type=1&target=2&filter=3&preset=4&metric_descriptor_names=5&num_results=6&offset=7&in_ids=7,8,9", data="x") + + @mock.patch('similarity.client._get_url_as_json') + def test_add(self, get_url): + get_url.return_value = {"error": False, "result": {"data": "here"}} + + self.client.add(10, 'data.yaml') + get_url.assert_called_with("http://localhost/similarity/add_point/?sound_id=10&location=data.yaml") + + @mock.patch('similarity.client._get_url_as_json') + def test_get_all_sound_ids(self, get_url): + get_url.return_value = {"error": False, "result": {"data": "here"}} + + self.client.get_all_sound_ids() + get_url.assert_called_with("http://localhost/similarity/get_all_point_names/") + + @mock.patch('similarity.client._get_url_as_json') + def test_get_descriptor_names(self, get_url): + get_url.return_value = {"error": False, "result": {"data": "here"}} + + self.client.get_descriptor_names() + get_url.assert_called_with("http://localhost/similarity/get_descriptor_names/") + + @mock.patch('similarity.client._get_url_as_json') + def test_delete(self, get_url): + get_url.return_value = {"error": False, "result": {"data": "here"}} + + self.client.delete(123) + get_url.assert_called_with("http://localhost/similarity/delete_point/?sound_id=123") + + @mock.patch('similarity.client._get_url_as_json') + def test_save(self, get_url): + get_url.return_value = {"error": False, "result": {"data": "here"}} + + self.client.save() + get_url.assert_called_with("http://localhost/similarity/save/", timeout=300) + get_url.reset_mock() + + self.client.save('database.db') + get_url.assert_called_with("http://localhost/similarity/save/?filename=database.db", timeout=300) + + @mock.patch('similarity.client._get_url_as_json') + def test_get_sounds_descriptors(self, get_url): + get_url.return_value = {"error": False, "result": {"data": "here"}} + + self.client.get_sounds_descriptors([1, 2, 3]) + get_url.assert_called_with("http://localhost/similarity/get_sounds_descriptors/?sound_ids=1,2,3&normalization=1") diff --git a/sounds/tests/test_sound.py b/sounds/tests/test_sound.py index 886bd6a4a1..8658a73e9d 100644 --- a/sounds/tests/test_sound.py +++ b/sounds/tests/test_sound.py @@ -767,7 +767,7 @@ def test_download(self, sendfile): self.assertInHTML('1 download', resp.content) # Similarity link (cached in display and view) - @mock.patch('general.management.commands.similarity_update.Similarity.add', return_value='Dummy response') + @mock.patch('general.management.commands.similarity_update.similarity_client.add', return_value='Dummy response') def _test_similarity_update(self, cache_keys, check_present, similarity_add): # Default analysis_state is 'PE', but for similarity update it should be 'OK', otherwise sound gets ignored self.sound.analysis_state = 'OK' diff --git a/utils/similarity_utilities.py b/utils/similarity_utilities.py index a65cc78118..1e067c2bf4 100644 --- a/utils/similarity_utilities.py +++ b/utils/similarity_utilities.py @@ -21,7 +21,7 @@ import traceback, logging from django.conf import settings from django.core.cache import cache -from similarity.client import Similarity +from similarity.client import similarity_client from similarity.similarity_settings import PRESETS, DEFAULT_PRESET, SIMILAR_SOUNDS_TO_CACHE, SIMILARITY_CACHE_TIME from utils.encryption import create_hash @@ -51,7 +51,7 @@ def get_similar_sounds(sound, preset=DEFAULT_PRESET, num_results=settings.SOUNDS if not similar_sounds: try: - result = Similarity.search(sound.id, preset=preset, num_results=num_results, offset=offset) + result = similarity_client.search(sound.id, preset=preset, num_results=num_results, offset=offset) similar_sounds = [[int(x[0]), float(x[1])] for x in result['results']] count = result['count'] except Exception as e: @@ -90,7 +90,7 @@ def api_search(target=None, filter=None, preset=None, metric_descriptor_names=No if not returned_sounds or target_file: if target_file: - # If there is a file attahced, set the file as the target + # If there is a file attached, set the file as the target target_type = 'file' target = None # If target is given as a file, we set target to None (just in case) else: @@ -100,7 +100,7 @@ def api_search(target=None, filter=None, preset=None, metric_descriptor_names=No else: target_type = 'descriptor_values' - result = Similarity.api_search( + result = similarity_client.api_search( target_type=target_type, target=target, filter=filter, @@ -136,7 +136,7 @@ def get_sounds_descriptors(sound_ids, descriptor_names, normalization=True, only # remove id form list so it is not included in similarity request not_cached_sound_ids.remove(id) try: - returned_data = Similarity.get_sounds_descriptors(not_cached_sound_ids, descriptor_names, normalization, only_leaf_descriptors) + returned_data = similarity_client.get_sounds_descriptors(not_cached_sound_ids, descriptor_names, normalization, only_leaf_descriptors) except Exception as e: logger.info('Something wrong occurred with the "get sound descriptors" request (%s)\n\t%s' %\ (e, traceback.format_exc())) @@ -155,7 +155,7 @@ def get_sounds_descriptors(sound_ids, descriptor_names, normalization=True, only def delete_sound_from_gaia(sound): logger.info("Deleting sound from gaia with id %d" % sound.id) try: - Similarity.delete(sound.id) + similarity_client.delete(sound.id) except Exception as e: logger.warn("Could not delete sound from gaia with id %d (%s)" % (sound.id, str(e)))