From 1aeb708a29a4aaba594c22e1e9c9eac3c3fdaac1 Mon Sep 17 00:00:00 2001 From: Alastair Porter Date: Tue, 19 Sep 2023 17:55:35 +0200 Subject: [PATCH 1/5] Add yapf checker --- .github/workflows/python-yapf.yml | 12 ++++++++++++ pyproject.toml | 17 +++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 .github/workflows/python-yapf.yml create mode 100644 pyproject.toml diff --git a/.github/workflows/python-yapf.yml b/.github/workflows/python-yapf.yml new file mode 100644 index 000000000..9639171db --- /dev/null +++ b/.github/workflows/python-yapf.yml @@ -0,0 +1,12 @@ +name: YAPF Formatting Check +on: [push] +jobs: + formatting-check: + name: Formatting Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: run YAPF to test if python code is correctly formatted + uses: AlexanderMelde/yapf-action@master + with: + args: --verbose diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..e156d344b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,17 @@ +[tool.yapfignore] +ignore_patterns = [ + "env/*", + "**/migrations/*", +] + +[tool.yapf] +based_on_style = "google" +spaces_before_comment = 4 +split_before_logical_operator = true +split_before_expression_after_opening_paren = true +split_before_first_argument = true +dedent_closing_brackets = true +coalesce_brackets = true +column_limit = 120 +align_closing_bracket_with_visual_indent = true + From d5ebbd0725f79ea4c994312bd508059fa5b8ce51 Mon Sep 17 00:00:00 2001 From: Alastair Porter Date: Wed, 20 Dec 2023 12:30:22 +0100 Subject: [PATCH 2/5] Format code with yapf --- .../add_examples_to_api_docs_source_files.py | 3 + .../generate_analysis_rst.py | 87 ++- .../generate_plots.py | 36 +- _docs/api/source/conf.py | 23 +- accounts/admin.py | 266 ++++--- accounts/forms.py | 139 ++-- .../commands/check_async_deleted_users.py | 20 +- .../commands/clean_old_tmp_upload_files.py | 5 +- .../commands/cleanup_unactivated_users.py | 5 +- .../commands/process_email_bounces.py | 26 +- accounts/models.py | 176 ++--- accounts/templatetags/display_user.py | 1 - accounts/templatetags/filefunctions.py | 3 +- accounts/templatetags/flag_user.py | 27 +- accounts/tests/test_profile.py | 122 +-- accounts/tests/test_spam.py | 148 ++-- accounts/tests/test_upload.py | 171 +++-- accounts/tests/test_user.py | 639 ++++++++++------ accounts/tests/test_views.py | 104 +-- accounts/urls.py | 62 +- accounts/views.py | 393 +++++----- apiv2/admin.py | 4 - apiv2/apiv2_utils.py | 67 +- apiv2/authentication.py | 1 + apiv2/combined_search_strategies.py | 84 ++- apiv2/examples.py | 259 +++++-- apiv2/exceptions.py | 124 +++- apiv2/forms.py | 101 ++- apiv2/management/commands/basic_api_tests.py | 25 +- .../commands/consolidate_api_usage_data.py | 1 - apiv2/models.py | 14 +- apiv2/oauth2_urls.py | 12 +- apiv2/oauth2_validators.py | 2 +- apiv2/serializers.py | 591 +++++++++------ apiv2/templatetags/apiv2_templatetags.py | 6 +- apiv2/tests.py | 232 +++--- apiv2/throttling.py | 31 +- apiv2/urls.py | 26 +- apiv2/views.py | 697 +++++++++++------- bookmarks/forms.py | 23 +- bookmarks/models.py | 9 +- bookmarks/tests.py | 19 +- bookmarks/views.py | 42 +- clustering/clustering.py | 95 ++- clustering/clustering_settings.py | 13 +- clustering/features_store.py | 19 +- clustering/interface.py | 14 +- clustering/tasks.py | 5 +- comments/admin.py | 4 +- comments/forms.py | 19 +- comments/models.py | 7 +- comments/tests.py | 22 +- comments/views.py | 23 +- donations/admin.py | 14 +- .../commands/check_missing_donations.py | 15 +- .../commands/send_donation_request_emails.py | 53 +- donations/models.py | 24 +- donations/tests.py | 222 +++--- donations/urls.py | 1 - donations/views.py | 68 +- follow/follow_utils.py | 10 +- .../management/commands/send_stream_emails.py | 33 +- follow/tests.py | 16 +- follow/views.py | 31 +- forum/admin.py | 6 +- forum/forms.py | 53 +- forum/models.py | 22 +- forum/templatetags/display_forum_objects.py | 25 +- .../display_forum_search_results.py | 50 +- forum/templatetags/smileys.py | 9 +- forum/tests.py | 207 ++++-- forum/urls.py | 23 +- forum/views.py | 153 ++-- freesound/celery.py | 13 +- freesound/context_processor.py | 16 +- freesound/local_settings.example.py | 1 - freesound/logger.py | 2 +- freesound/middleware.py | 17 +- freesound/settings.py | 274 +++---- freesound/storage.py | 1 + freesound/test_settings.py | 11 +- freesound/urls.py | 67 +- freesound/wsgi.py | 1 + general/admin.py | 8 +- general/management/commands/build_static.py | 1 - .../management/commands/clean_data_volume.py | 21 +- .../commands/clear_sound_template_caches.py | 2 +- .../commands/create_front_page_caches.py | 35 +- .../post_sounds_to_tagrecommendation.py | 12 +- general/management/commands/prune_database.py | 42 +- .../commands/report_count_statuses.py | 67 +- .../commands/report_index_statuses.py | 12 +- .../commands/similarity_save_index.py | 6 +- .../management/commands/similarity_update.py | 45 +- general/tasks.py | 344 ++++++--- general/templatetags/absurl.py | 10 +- general/templatetags/bw_templatetags.py | 63 +- general/templatetags/filter_img.py | 2 +- general/templatetags/rss.py | 9 +- general/templatetags/util.py | 4 +- general/tests.py | 22 +- .../commands/retrieve_geotag_names.py | 7 +- geotags/models.py | 11 +- geotags/tests.py | 11 +- geotags/urls.py | 6 +- geotags/views.py | 25 +- manage.py | 5 +- messages/admin.py | 25 +- messages/apps.py | 7 +- messages/forms.py | 18 +- messages/models.py | 10 +- messages/templatetags/display_message.py | 1 - messages/tests/test_message_notifications.py | 26 +- messages/tests/test_message_write.py | 54 +- messages/views.py | 84 ++- monitor/management/commands/generate_stats.py | 56 +- monitor/tests.py | 12 +- monitor/urls.py | 4 - monitor/views.py | 39 +- ratings/admin.py | 4 +- ratings/models.py | 8 +- ratings/templatetags/ratings.py | 10 +- ratings/tests.py | 12 +- ratings/urls.py | 6 +- ratings/views.py | 20 +- search/forms.py | 30 +- .../post_dirty_sounds_to_search_engine.py | 28 +- .../commands/reindex_search_engine_forum.py | 15 +- .../commands/reindex_search_engine_sounds.py | 19 +- .../commands/test_search_engine_backend.py | 61 +- search/templatetags/search.py | 38 +- search/tests.py | 52 +- search/views.py | 169 +++-- similarity/client/__init__.py | 55 +- similarity/gaia_wrapper.py | 202 +++-- similarity/similarity_indexing_server.py | 18 +- similarity/similarity_server.py | 116 ++- similarity/similarity_server_utils.py | 223 +++--- similarity/similarity_settings.py | 11 +- sounds/admin.py | 150 ++-- sounds/forms.py | 187 +++-- sounds/management.py | 12 +- .../management/commands/check_sound_paths.py | 10 +- .../commands/clean_tmp_pcm_analysis_files.py | 12 +- sounds/management/commands/copy_downloads.py | 36 +- .../commands/create_random_sounds.py | 2 +- .../commands/create_remix_groups.py | 58 +- sounds/management/commands/csv_bulk_upload.py | 6 +- .../commands/orchestrate_analysis.py | 70 +- .../commands/reprocess_all_sounds.py | 64 +- .../commands/reprocess_failed_sounds.py | 17 +- .../management/commands/test_color_schemes.py | 12 +- .../management/commands/update_cdn_sounds.py | 54 +- sounds/models.py | 466 +++++++----- sounds/templatetags/display_pack.py | 5 +- sounds/templatetags/display_remix.py | 66 +- sounds/templatetags/display_sound.py | 174 ++++- sounds/templatetags/sound_signature.py | 4 +- sounds/templatetags/sounds_selector.py | 8 +- sounds/tests/test_manager.py | 6 +- sounds/tests/test_random_sound.py | 11 +- sounds/tests/test_sound.py | 366 +++++---- sounds/tests/test_templatetags.py | 54 +- sounds/views.py | 293 ++++---- support/tests.py | 10 +- support/views.py | 29 +- tagrecommendation/client/__init__.py | 15 +- .../tag_recommendation/community_detector.py | 15 +- .../community_tag_recommender.py | 32 +- .../tag_recommendation/data_processor.py | 82 ++- .../tag_recommendation/heuristics.py | 12 +- .../tag_recommendation_utils.py | 31 +- .../tag_recommendation/tag_recommender.py | 12 +- tagrecommendation/tagrecommendation_server.py | 39 +- tagrecommendation/utils.py | 12 +- tags/models.py | 2 + tags/templatetags/tags.py | 8 +- tags/tests.py | 38 +- tags/views.py | 2 +- tickets/__init__.py | 224 +++--- tickets/admin.py | 16 +- tickets/forms.py | 46 +- tickets/models.py | 79 +- tickets/templatetags/display_ticket.py | 5 +- tickets/templatetags/sound_tickets_count.py | 1 + tickets/tests.py | 57 +- tickets/urls.py | 78 +- tickets/views.py | 228 +++--- utils/admin_helpers.py | 13 +- utils/audioprocessing/color_schemes.py | 79 +- utils/audioprocessing/convert_to_wav.py | 2 +- .../freesound_audio_processing.py | 147 ++-- utils/audioprocessing/processing.py | 69 +- utils/audioprocessing/wav2png.py | 35 +- utils/aws.py | 11 +- utils/cache.py | 1 - utils/chunks.py | 3 +- utils/downloads.py | 7 +- utils/filesystem.py | 1 - utils/forms.py | 18 +- utils/locations.py | 14 +- utils/logging_filters.py | 3 +- utils/mail.py | 90 ++- utils/management_commands.py | 1 - utils/mirror_files.py | 51 +- utils/nginxsendfile.py | 6 +- utils/onlineusers.py | 46 +- utils/paypal/wrapper.py | 47 +- utils/search/__init__.py | 48 +- utils/search/backends/solr555pysolr.py | 158 ++-- utils/search/backends/solr9pysolr.py | 9 +- utils/search/backends/solr_common.py | 87 ++- .../backends/test_search_engine_backend.py | 297 +++++--- .../backends/tests/test_solr555pysolr.py | 2 + .../search/backends/tests/test_solr_common.py | 2 + utils/search/lucene_parser.py | 18 +- utils/search/search_forum.py | 5 +- utils/search/search_sounds.py | 51 +- utils/session_checks.py | 8 +- utils/similarity_utilities.py | 36 +- utils/sound_upload.py | 140 ++-- utils/spam.py | 2 +- utils/tagrecommendation_utilities.py | 8 +- utils/tags.py | 3 +- utils/test_helpers.py | 50 +- utils/tests/test_filesystem.py | 1 + utils/tests/test_forms.py | 12 +- utils/tests/test_logging_filters.py | 17 +- utils/tests/test_processing.py | 38 +- utils/tests/test_search_general.py | 169 +++-- utils/tests/test_spam.py | 5 +- utils/tests/test_text.py | 82 ++- utils/tests/tests.py | 309 ++++---- utils/text.py | 20 +- utils/urlpatterns.py | 7 +- utils/username.py | 1 - wiki/admin.py | 12 +- wiki/models.py | 5 +- wiki/tests.py | 9 +- wiki/views.py | 20 +- 240 files changed, 8485 insertions(+), 5694 deletions(-) diff --git a/_docs/api/add_examples_to_api_docs_source_files.py b/_docs/api/add_examples_to_api_docs_source_files.py index 66c34ebee..9e30337c5 100644 --- a/_docs/api/add_examples_to_api_docs_source_files.py +++ b/_docs/api/add_examples_to_api_docs_source_files.py @@ -1,9 +1,12 @@ import sys import urllib.request, urllib.parse, urllib.error + sys.path.append("../../apiv2") from examples import examples + base_url = 'https://freesound.org/' + def get_formatted_examples_for_view(view_name): try: data = examples[view_name] diff --git a/_docs/api/generate_analysis_documentation/generate_analysis_rst.py b/_docs/api/generate_analysis_documentation/generate_analysis_rst.py index 49c66de43..b63e82dd0 100644 --- a/_docs/api/generate_analysis_documentation/generate_analysis_rst.py +++ b/_docs/api/generate_analysis_documentation/generate_analysis_rst.py @@ -1,9 +1,7 @@ -# Generate skeleton for documentation, +# Generate skeleton for documentation, # add essentia documentation links by hand - -import urllib.request, urllib.error, urllib.parse,json - +import urllib.request, urllib.error, urllib.parse, json header = """ .. _analysis-docs: @@ -50,8 +48,8 @@ image_str = " .. image:: _static/descriptors/" height_str = " :height: 300px" algorithm_doc_str = "http://essentia.upf.edu/documentation/reference/streaming_" -sorted_namespaces = ["metadata","lowlevel","rhythm","tonal","sfx"] -desc_exceptions = ["metadata.audio_properties","metadata.version","rhythm.onset_rate"] +sorted_namespaces = ["metadata", "lowlevel", "rhythm", "tonal", "sfx"] +desc_exceptions = ["metadata.audio_properties", "metadata.version", "rhythm.onset_rate"] example_url = "https://freesound.org/api/sounds/1234/analysis/?api_key=53b80e4d8a674ccaa80b780372103680&all=True" @@ -59,48 +57,47 @@ resp = urllib.request.urlopen(req) top = json.loads(resp.read()) - mapping = dict() for line in open("algorithm_mapping.csv"): - desc,alg = line[:-1].split(",") - mapping[desc] = alg + desc, alg = line[:-1].split(",") + mapping[desc] = alg print(header) for k in sorted_namespaces: - ns = k[0].upper()+k[1:] - print(ns+ " Descriptors") - print(">>>>>>>>>>>>>>>>>>>>\n\n") - for d in top[k].keys(): - descriptor = k+"."+d - print(descriptor) - print("-------------------------") - print("\n::\n") - print(curl_str+k+"/"+d) - if mapping[descriptor] !="None": - print("\n**Essentia Algorithm**\n") - print(algorithm_doc_str+mapping[descriptor]+".html") - stats = top[k][d] - if descriptor in desc_exceptions: - print("\n") - continue - if isinstance(stats, dict): - print("\n\n**Stats**::\n\n") - for s in stats.keys(): - print("/"+s) - - print("\n\n**Distribution in Freesound**\n") - - if "mean" in stats.keys(): - if isinstance(stats['mean'], list): - for i in range(len(stats['mean'])): - img = image_str+descriptor+".mean.%03d"%i - print(img+".png") - print(height_str) - else: - print(image_str+descriptor+".mean.png") - print(height_str) - elif isinstance(stats, float) or isinstance(stats, int): - print(image_str+descriptor+".png") - print(height_str) - print("\n\n") + ns = k[0].upper() + k[1:] + print(ns + " Descriptors") + print(">>>>>>>>>>>>>>>>>>>>\n\n") + for d in top[k].keys(): + descriptor = k + "." + d + print(descriptor) + print("-------------------------") + print("\n::\n") + print(curl_str + k + "/" + d) + if mapping[descriptor] != "None": + print("\n**Essentia Algorithm**\n") + print(algorithm_doc_str + mapping[descriptor] + ".html") + stats = top[k][d] + if descriptor in desc_exceptions: + print("\n") + continue + if isinstance(stats, dict): + print("\n\n**Stats**::\n\n") + for s in stats.keys(): + print("/" + s) + + print("\n\n**Distribution in Freesound**\n") + + if "mean" in stats.keys(): + if isinstance(stats['mean'], list): + for i in range(len(stats['mean'])): + img = image_str + descriptor + ".mean.%03d" % i + print(img + ".png") + print(height_str) + else: + print(image_str + descriptor + ".mean.png") + print(height_str) + elif isinstance(stats, float) or isinstance(stats, int): + print(image_str + descriptor + ".png") + print(height_str) + print("\n\n") diff --git a/_docs/api/generate_analysis_documentation/generate_plots.py b/_docs/api/generate_analysis_documentation/generate_plots.py index db5f88922..46decf8e8 100644 --- a/_docs/api/generate_analysis_documentation/generate_plots.py +++ b/_docs/api/generate_analysis_documentation/generate_plots.py @@ -4,18 +4,18 @@ import gaia2 import pylab as pl -OUT_FOLDER = 'out_plots' # Must be created -GAIA_INDEX_FILE = 'fs_index.db' # File with gaia index -BINS = 100 # Bins per histogram plot +OUT_FOLDER = 'out_plots' # Must be created +GAIA_INDEX_FILE = 'fs_index.db' # File with gaia index +BINS = 100 # Bins per histogram plot -def plot_histogram(pool, label, x_label_ticks = False): +def plot_histogram(pool, label, x_label_ticks=False): fig = pl.figure() ax = fig.add_subplot(111) if not x_label_ticks: - range_min = min(pool) #percentile(pool, 10) - range_max = max(pool) #percentile(pool, 90) + range_min = min(pool) #percentile(pool, 10) + range_max = max(pool) #percentile(pool, 90) else: range_min = min(pool) range_max = max(pool) + 1 @@ -23,38 +23,40 @@ def plot_histogram(pool, label, x_label_ticks = False): n_bins = BINS if x_label_ticks: n_bins = len(x_label_ticks) - n, bins, patches = ax.hist(pool, bins=n_bins, range=(float(range_min), float(range_max)), log=False, histtype='stepfilled') + n, bins, patches = ax.hist( + pool, bins=n_bins, range=(float(range_min), float(range_max)), log=False, histtype='stepfilled' + ) pl.title('Distribution: %s' % label) if not x_label_ticks: - ax.ticklabel_format(axis='x', style='sci', scilimits=(-3,3)) + ax.ticklabel_format(axis='x', style='sci', scilimits=(-3, 3)) else: - pl.xticks(list(range(0, len(x_label_ticks))),[' %s'%tick for tick in x_label_ticks]) - ax.ticklabel_format(axis='y', style='sci', scilimits=(-2,2)) + pl.xticks(list(range(0, len(x_label_ticks))), [' %s' % tick for tick in x_label_ticks]) + ax.ticklabel_format(axis='y', style='sci', scilimits=(-2, 2)) ax.set_xlabel('Value') ax.set_ylabel('Frequency of occurrence') ax.grid(True) pl.savefig('%s/%s.png' % (OUT_FOLDER, label[1:])) pl.close() + ds = gaia2.DataSet() dataset_path = GAIA_INDEX_FILE ds.load(dataset_path) transformation_history = ds.history().toPython() normalization_coeffs = None -for i in range(0,len(transformation_history)): - if transformation_history[-(i+1)]['Analyzer name'] == 'normalize': - normalization_coeffs = transformation_history[-(i+1)]['Applier parameters']['coeffs'] +for i in range(0, len(transformation_history)): + if transformation_history[-(i + 1)]['Analyzer name'] == 'normalize': + normalization_coeffs = transformation_history[-(i + 1)]['Applier parameters']['coeffs'] print([x for x in normalization_coeffs.keys() if (".tonal" in x and "chords" in x)]) descriptor_names = ds.layout().descriptorNames() point_names = ds.pointNames() example_point = ds.point(point_names[0]) -reject_stats = ['dmean', 'dmean2', 'dvar', 'dvar2', 'max', 'min', 'var'] - +reject_stats = ['dmean', 'dmean2', 'dvar', 'dvar2', 'max', 'min', 'var'] for descriptor_name in descriptor_names: region = ds.layout().descriptorLocation(descriptor_name) if region.lengthType() == gaia2.VariableLength or descriptor_name.split('.')[-1] in reject_stats: - continue + continue try: example_value = example_point.value(descriptor_name) @@ -112,4 +114,4 @@ def plot_histogram(pool, label, x_label_ticks = False): key_ids[key] = j pool = [key_ids[value] for value in pool] - plot_histogram(pool, descriptor_name, x_label_ticks = keys) + plot_histogram(pool, descriptor_name, x_label_ticks=keys) diff --git a/_docs/api/source/conf.py b/_docs/api/source/conf.py index 3b0cd0fcd..9f24cc59f 100644 --- a/_docs/api/source/conf.py +++ b/_docs/api/source/conf.py @@ -85,7 +85,6 @@ # A list of ignored prefixes for module index sorting. #modindex_common_prefix = [] - # -- Options for HTML output --------------------------------------------------- # The theme to use for HTML and HTML Help pages. Major themes that come with @@ -159,7 +158,6 @@ # Output file base name for HTML help builder. htmlhelp_basename = 'FreesoundAPIdoc' - # -- Options for LaTeX output -------------------------------------------------- # The paper size ('letter' or 'a4'). @@ -171,23 +169,22 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'FreesoundAPI.tex', 'Freesound API Documentation', - 'Frederic Font', 'manual'), + ('index', 'FreesoundAPI.tex', 'Freesound API Documentation', 'Frederic Font', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of # the title page. #latex_logo = None -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False + # For "manual" documents, if this is true, then toplevel headings are parts, + # not chapters. + #latex_use_parts = False -# Additional stuff for the LaTeX preamble. -#latex_preamble = '' + # Additional stuff for the LaTeX preamble. + #latex_preamble = '' -# Documents to append as an appendix to all manuals. -#latex_appendices = [] + # Documents to append as an appendix to all manuals. + #latex_appendices = [] -# If false, no module index is generated. -#latex_use_modindex = True + # If false, no module index is generated. + #latex_use_modindex = True diff --git a/accounts/admin.py b/accounts/admin.py index d6b98c90c..02953434d 100644 --- a/accounts/admin.py +++ b/accounts/admin.py @@ -42,12 +42,12 @@ @admin.register(Profile) class ProfileAdmin(admin.ModelAdmin): - readonly_fields = ('user', ) - raw_id_fields = ('geotag', ) + readonly_fields = ('user',) + raw_id_fields = ('geotag',) list_display = ('user', 'home_page', 'signature', 'is_whitelisted') - ordering = ('id', ) - list_filter = ('is_whitelisted', ) - search_fields = ('=user__username', ) + ordering = ('id',) + list_filter = ('is_whitelisted',) + search_fields = ('=user__username',) def has_add_permission(self, request): return False @@ -70,7 +70,7 @@ def has_change_permission(self, request, obj=None): @admin.register(DeletedUser) class DeletedUserAdmin(admin.ModelAdmin): - list_filter = ('reason', ) + list_filter = ('reason',) readonly_fields = ('user', 'username', 'email', 'date_joined', 'last_login', 'deletion_date', 'reason') list_display = ('get_object_link', 'get_view_link', 'deletion_date', 'reason') search_fields = ('=username',) @@ -80,23 +80,24 @@ class DeletedUserAdmin(admin.ModelAdmin): ordering='username', ) def get_object_link(self, obj): - return mark_safe('{}'.format( - reverse('admin:accounts_deleteduser_change', args=[obj.id]), - f'DeletedUser: {obj.username}')) + return mark_safe( + '{}'.format( + reverse('admin:accounts_deleteduser_change', args=[obj.id]), f'DeletedUser: {obj.username}' + ) + ) - @admin.display( - description='View on site' - ) + @admin.display(description='View on site') def get_view_link(self, obj): if obj.user is None: return '-' else: - return mark_safe('{}'.format( - reverse('account', args=[obj.user.username]), obj.user.username)) + return mark_safe( + '{}'.format( + reverse('account', args=[obj.user.username]), obj.user.username + ) + ) - @admin.display( - description='# sounds' - ) + @admin.display(description='# sounds') def get_num_sounds(self, obj): return f'{obj.profile.num_sounds}' @@ -113,8 +114,10 @@ def clean_username(self): return username if not username_taken_by_other_user(username): return username - raise ValidationError("This username is already taken or has been in used in the past by this or some other " - "user.") + raise ValidationError( + "This username is already taken or has been in used in the past by this or some other " + "user." + ) def clean_email(self): # Check that email is not being used by another user (case insensitive) @@ -132,15 +135,23 @@ class FreesoundUserAdmin(DjangoObjectActions, UserAdmin): actions = () list_display = ('username', 'email', 'get_num_sounds', 'get_num_posts', 'date_joined', 'get_view_link') list_filter = () - ordering = ('id', ) + ordering = ('id',) show_full_result_count = False form = AdminUserForm fieldsets = ( - (None, {'fields': ('username', 'password')}), - ('Personal info', {'fields': ('email', )}), - ('Permissions', {'fields': ('is_active', 'is_staff', 'is_superuser', 'groups')}), - ('Important dates', {'fields': ('last_login', 'date_joined')}), - ) + (None, { + 'fields': ('username', 'password') + }), + ('Personal info', { + 'fields': ('email',) + }), + ('Permissions', { + 'fields': ('is_active', 'is_staff', 'is_superuser', 'groups') + }), + ('Important dates', { + 'fields': ('last_login', 'date_joined') + }), + ) paginator = LargeTablePaginator @@ -155,22 +166,17 @@ def has_delete_permission(self, request, obj=None): # We want to disable that button in favour of the custom asynchronous delete change actions return False - @admin.display( - description='View on site' - ) + @admin.display(description='View on site') def get_view_link(self, obj): - return mark_safe('{}'.format( - reverse('account', args=[obj.username]), obj.username)) + return mark_safe( + '{}'.format(reverse('account', args=[obj.username]), obj.username) + ) - @admin.display( - description='# sounds' - ) + @admin.display(description='# sounds') def get_num_sounds(self, obj): return f'{obj.profile.num_sounds}' - @admin.display( - description='# posts' - ) + @admin.display(description='# posts') def get_num_posts(self, obj): return f'{obj.profile.num_posts}' @@ -181,9 +187,7 @@ def get_actions(self, request): del actions['delete_selected'] return actions - @admin.action( - description="Delete the user but keep the sounds available" - ) + @admin.action(description="Delete the user but keep the sounds available") def delete_preserve_sounds(self, request, obj): username = obj.username if request.method == "POST": @@ -192,19 +196,23 @@ def delete_preserve_sounds(self, request, obj): web_logger.info(f'Requested async deletion of user {obj.id} - {delete_action}') # Create a UserDeletionRequest with a status of 'Deletion action was triggered' - UserDeletionRequest.objects.create(user_from=request.user, - user_to=obj, - status=UserDeletionRequest.DELETION_REQUEST_STATUS_DELETION_TRIGGERED, - triggered_deletion_action=delete_action, - triggered_deletion_reason=delete_reason) + UserDeletionRequest.objects.create( + user_from=request.user, + user_to=obj, + status=UserDeletionRequest.DELETION_REQUEST_STATUS_DELETION_TRIGGERED, + triggered_deletion_action=delete_action, + triggered_deletion_reason=delete_reason + ) # Trigger async task so user gets deleted asynchronously tasks.delete_user.delay(user_id=obj.id, deletion_action=delete_action, deletion_reason=delete_reason) # Show message to admin user - messages.add_message(request, messages.INFO, - 'User \'%s\' will be deleted asynchronously. Sounds, comments and other related ' - 'user content will still be available but appear under anonymised account' % username) + messages.add_message( + request, messages.INFO, + 'User \'%s\' will be deleted asynchronously. Sounds, comments and other related ' + 'user content will still be available but appear under anonymised account' % username + ) return HttpResponseRedirect(reverse('admin:auth_user_changelist')) user_info = obj.profile.get_info_before_delete_user(include_sounds=False, include_other_related_objects=False) @@ -215,9 +223,7 @@ def delete_preserve_sounds(self, request, obj): delete_preserve_sounds.label = "Delete user only" - @admin.action( - description="Delete the user and the sounds" - ) + @admin.action(description="Delete the user and the sounds") def delete_include_sounds(self, request, obj): username = obj.username if request.method == "POST": @@ -226,26 +232,30 @@ def delete_include_sounds(self, request, obj): web_logger.info(f'Requested async deletion of user {obj.id} - {delete_action}') # Create a UserDeletionRequest with a status of 'Deletion action was triggered' - UserDeletionRequest.objects.create(user_from=request.user, - user_to=obj, - status=UserDeletionRequest.DELETION_REQUEST_STATUS_DELETION_TRIGGERED, - triggered_deletion_action=delete_action, - triggered_deletion_reason=delete_reason) + UserDeletionRequest.objects.create( + user_from=request.user, + user_to=obj, + status=UserDeletionRequest.DELETION_REQUEST_STATUS_DELETION_TRIGGERED, + triggered_deletion_action=delete_action, + triggered_deletion_reason=delete_reason + ) # Trigger async task so user gets deleted asynchronously tasks.delete_user.delay(user_id=obj.id, deletion_action=delete_action, deletion_reason=delete_reason) # Show message to admin user - messages.add_message(request, messages.INFO, - 'User \'%s\' will be deleted asynchronously. Sounds will be deleted as well. ' - 'Comments and other related user content will still be available but appear under ' - 'anonymised account' % username) + messages.add_message( + request, messages.INFO, 'User \'%s\' will be deleted asynchronously. Sounds will be deleted as well. ' + 'Comments and other related user content will still be available but appear under ' + 'anonymised account' % username + ) return HttpResponseRedirect(reverse('admin:auth_user_changelist')) user_info = obj.profile.get_info_before_delete_user(include_sounds=True, include_other_related_objects=False) user_info['deleted_objects_details'] = {} - model_count = {model._meta.verbose_name_plural: len(objs) for - model, objs in user_info['deleted'].model_objs.items()} + model_count = { + model._meta.verbose_name_plural: len(objs) for model, objs in user_info['deleted'].model_objs.items() + } user_info['deleted_objects_details']['model_count'] = list(dict(model_count).items()) tvars = {'users_to_delete': [], 'type': 'delete_include_sounds'} @@ -255,9 +265,7 @@ def delete_include_sounds(self, request, obj): delete_include_sounds.label = "Delete user and sounds" - @admin.action( - description="Delete the user and the sounds, mark deleted user as spammer" - ) + @admin.action(description="Delete the user and the sounds, mark deleted user as spammer") def delete_spammer(self, request, obj): username = obj.username if request.method == "POST": @@ -266,25 +274,30 @@ def delete_spammer(self, request, obj): web_logger.info(f'Requested async deletion of user {obj.id} - {delete_action}') # Create a UserDeletionRequest with a status of 'Deletion action was triggered' - UserDeletionRequest.objects.create(user_from=request.user, - user_to=obj, - status=UserDeletionRequest.DELETION_REQUEST_STATUS_DELETION_TRIGGERED, - triggered_deletion_action=delete_action, - triggered_deletion_reason=delete_reason) + UserDeletionRequest.objects.create( + user_from=request.user, + user_to=obj, + status=UserDeletionRequest.DELETION_REQUEST_STATUS_DELETION_TRIGGERED, + triggered_deletion_action=delete_action, + triggered_deletion_reason=delete_reason + ) # Trigger async task so user gets deleted asynchronously tasks.delete_user.delay(user_id=obj.id, deletion_action=delete_action, deletion_reason=delete_reason) # Show message to admin user - messages.add_message(request, messages.INFO, - 'User \'%s\' will be deleted asynchronously including sounds and all of its related ' - 'content.' % username) + messages.add_message( + request, messages.INFO, + 'User \'%s\' will be deleted asynchronously including sounds and all of its related ' + 'content.' % username + ) return HttpResponseRedirect(reverse('admin:auth_user_changelist')) user_info = obj.profile.get_info_before_delete_user(include_sounds=True, include_other_related_objects=True) user_info['deleted_objects_details'] = {} - model_count = {model._meta.verbose_name_plural: len(objs) for - model, objs in user_info['deleted'].model_objs.items()} + model_count = { + model._meta.verbose_name_plural: len(objs) for model, objs in user_info['deleted'].model_objs.items() + } user_info['deleted_objects_details']['model_count'] = list(dict(model_count).items()) tvars = {'users_to_delete': [], 'type': 'delete_spammer'} @@ -294,9 +307,7 @@ def delete_spammer(self, request, obj): delete_spammer.label = "Delete as spammer" - @admin.action( - description='Completely delete user from db' - ) + @admin.action(description='Completely delete user from db') def full_delete(self, request, obj): username = obj.username if request.method == "POST": @@ -305,25 +316,30 @@ def full_delete(self, request, obj): web_logger.info(f'Requested async deletion of user {obj.id} - {delete_action}') # Create a UserDeletionRequest with a status of 'Deletion action was triggered' - UserDeletionRequest.objects.create(user_from=request.user, - user_to=obj, - status=UserDeletionRequest.DELETION_REQUEST_STATUS_DELETION_TRIGGERED, - triggered_deletion_action=delete_action, - triggered_deletion_reason=delete_reason) + UserDeletionRequest.objects.create( + user_from=request.user, + user_to=obj, + status=UserDeletionRequest.DELETION_REQUEST_STATUS_DELETION_TRIGGERED, + triggered_deletion_action=delete_action, + triggered_deletion_reason=delete_reason + ) # Trigger async task so user gets deleted asynchronously tasks.delete_user.delay(user_id=obj.id, deletion_action=delete_action, deletion_reason=delete_reason) # Show message to admin user - messages.add_message(request, messages.INFO, - 'User \'%s\' will be deleted asynchronously including sounds and all of its related' - 'content.' % username) + messages.add_message( + request, messages.INFO, + 'User \'%s\' will be deleted asynchronously including sounds and all of its related' + 'content.' % username + ) return HttpResponseRedirect(reverse('admin:auth_user_changelist')) user_info = obj.profile.get_info_before_delete_user(include_sounds=True, include_other_related_objects=True) user_info['deleted_objects_details'] = {} - model_count = {model._meta.verbose_name_plural: len(objs) for - model, objs in user_info['deleted'].model_objs.items()} + model_count = { + model._meta.verbose_name_plural: len(objs) for model, objs in user_info['deleted'].model_objs.items() + } user_info['deleted_objects_details']['model_count'] = list(dict(model_count).items()) tvars = {'users_to_delete': [], 'type': 'full_delete'} @@ -333,30 +349,25 @@ def full_delete(self, request, obj): full_delete.label = "Full delete" - @admin.action( - description='Clear all user flags for of spam reports and akismet' - ) + @admin.action(description='Clear all user flags for of spam reports and akismet') def clear_spam_flags(self, request, obj): num_akismet, _ = obj.akismetspam_set.all().delete() num_reports, _ = obj.flags.all().delete() - messages.add_message(request, messages.INFO, - 'User \'%s\' flags have been cleared: %i akismet flags and %i user reports.' - % (obj.username, num_akismet, num_reports)) + messages.add_message( + request, messages.INFO, 'User \'%s\' flags have been cleared: %i akismet flags and %i user reports.' % + (obj.username, num_akismet, num_reports) + ) return HttpResponseRedirect(reverse('admin:auth_user_change', args=[obj.id])) clear_spam_flags.label = "Clear spam flags" - @admin.action( - description='Open user on site' - ) + @admin.action(description='Open user on site') def view_on_site_action(self, request, obj): return HttpResponseRedirect(reverse('account', args=[obj.username])) view_on_site_action.label = "View on site" - @admin.action( - description='Edit profile in admin' - ) + @admin.action(description='Edit profile in admin') def edit_profile_admin(self, request, obj): return HttpResponseRedirect(reverse('admin:accounts_profile_change', args=[obj.profile.id])) @@ -365,14 +376,20 @@ def edit_profile_admin(self, request, obj): # NOTE: in the line below we removed the 'full_delete' option as ideally we should never need to use it. In for # some unexpected reason we happen to need it, we can call the .delete() method on a user object using the terminal. # If we observe a real need for that, we can re-add the option to the admin. - change_actions = ('edit_profile_admin', 'view_on_site_action', 'clear_spam_flags', - 'delete_spammer', 'delete_include_sounds', 'delete_preserve_sounds', ) + change_actions = ( + 'edit_profile_admin', + 'view_on_site_action', + 'clear_spam_flags', + 'delete_spammer', + 'delete_include_sounds', + 'delete_preserve_sounds', + ) @admin.register(OldUsername) class OldUsernameAdmin(admin.ModelAdmin): readonly_fields = ('user', 'username') - search_fields = ('=username', ) + search_fields = ('=username',) list_display = ('user', 'username') def has_add_permission(self, request): @@ -384,16 +401,19 @@ def has_change_permission(self, request, obj=None): @admin.register(UserDeletionRequest) class UserDeletionRequestAdmin(admin.ModelAdmin): - list_filter = ('status', ) + list_filter = ('status',) search_fields = ('=username_to', '=email_from') - raw_id_fields = ('user_from', 'user_to' ) - list_display = ('status', 'email_from', 'username_from', 'username_to', 'user_to_link', 'deleted_user_link', - 'get_reason', 'last_updated') - fieldsets = ( - (None, {'fields': - ('status', 'email_from', 'user_to', 'username_to', - 'deleted_user', 'status_history', 'last_updated')}), + raw_id_fields = ('user_from', 'user_to') + list_display = ( + 'status', 'email_from', 'username_from', 'username_to', 'user_to_link', 'deleted_user_link', 'get_reason', + 'last_updated' ) + fieldsets = (( + None, { + 'fields': + ('status', 'email_from', 'user_to', 'username_to', 'deleted_user', 'status_history', 'last_updated') + } + ),) readonly_fields = ('status_history', 'user_from', 'username_from', 'username_to', 'last_updated', 'deleted_user') def get_queryset(self, request): @@ -409,10 +429,12 @@ def get_queryset(self, request): def deleted_user_link(self, obj): if obj.deleted_user is None: return '-' - return mark_safe('{}'.format( - reverse('admin:accounts_deleteduser_change', args=[obj.deleted_user_id]), - f'DeletedUser: {obj.deleted_user.username}')) - + return mark_safe( + '{}'.format( + reverse('admin:accounts_deleteduser_change', args=[obj.deleted_user_id]), + f'DeletedUser: {obj.deleted_user.username}' + ) + ) @admin.display( description='User to', @@ -421,16 +443,18 @@ def deleted_user_link(self, obj): def user_to_link(self, obj): if obj.user_to is None: return '-' - return mark_safe('{}'.format( - reverse('admin:auth_user_change', args=[obj.user_to_id]), obj.user_to.username)) + return mark_safe( + '{}'.format( + reverse('admin:auth_user_change', args=[obj.user_to_id]), obj.user_to.username + ) + ) - - @admin.display( - description='Reason' - ) + @admin.display(description='Reason') def get_reason(self, obj): if obj.triggered_deletion_reason: - return [label for key, label in DeletedUser.DELETION_REASON_CHOICES if key == obj.triggered_deletion_reason][0] + return [ + label for key, label in DeletedUser.DELETION_REASON_CHOICES if key == obj.triggered_deletion_reason + ][0] else: return '-' diff --git a/accounts/forms.py b/accounts/forms.py index 78647b554..518c759b0 100644 --- a/accounts/forms.py +++ b/accounts/forms.py @@ -63,7 +63,9 @@ def validate_file_extension(audiofiles): # Firefox seems to set wrong mime type for ogg files to video/ogg instead of audio/ogg. # Also safari seems to use 'application/octet-stream'. # For these reasons we also allow extra mime types for ogg files. - if not content_type.startswith("audio") and not content_type == 'video/ogg' and not content_type == 'application/octet-stream': + if not content_type.startswith( + "audio" + ) and not content_type == 'video/ogg' and not content_type == 'application/octet-stream': raise forms.ValidationError('Uploaded file format not supported or not an audio file.') elif ext == 'wv': # All major browsers seem to use 'application/octet-stream' for wv (wavpack) files. @@ -90,6 +92,7 @@ def validate_csvfile_extension(csv_file): class BulkDescribeForm(forms.Form): csv_file = forms.FileField(label='', validators=[validate_csvfile_extension]) + class UploadFileForm(forms.Form): files = MultiFileField(min_num=1, validators=[validate_file_extension], label="", required=False) @@ -102,15 +105,16 @@ class TermsOfServiceForm(forms.Form): accepted_tos = forms.BooleanField( label='', help_text='Check this box to accept the terms of use ' - 'and the privacy policy of Freesound (required)', + 'and the privacy policy of Freesound (required)', required=True, - error_messages={'required': 'You must accept the terms of use and the privacy poclicy in order to continue ' - 'using Freesound.'} + error_messages={ + 'required': + 'You must accept the terms of use and the privacy poclicy in order to continue ' + 'using Freesound.' + } ) accepted_license_change = forms.BooleanField( - label='', - help_text='Check this box to upgrade your Creative Commons 3.0 licenses to 4.0', - required=False + label='', help_text='Check this box to upgrade your Creative Commons 3.0 licenses to 4.0', required=False ) next = forms.CharField(widget=forms.HiddenInput(), required=False) @@ -152,16 +156,21 @@ def get_user_by_email(email): class UsernameField(forms.CharField): """ Username field, 3~30 characters, allows only alphanumeric chars, required by default """ + def __init__(self, required=True): super().__init__( label="Username", min_length=3, max_length=30, - validators=[RegexValidator(r'^[\w.+-]+$')], # is the same as Django UsernameValidator except for '@' symbol + validators=[RegexValidator(r'^[\w.+-]+$') + ], # is the same as Django UsernameValidator except for '@' symbol help_text="30 characters or fewer. Can contain: letters, digits, underscores, dots, dashes and plus signs.", - error_messages={'invalid': "This value must contain only letters, digits, underscores, dots, dashes and " - "plus signs."}, - required=required) + error_messages={ + 'invalid': "This value must contain only letters, digits, underscores, dots, dashes and " + "plus signs." + }, + required=required + ) def username_taken_by_other_user(username): @@ -198,12 +207,14 @@ class RegistrationForm(forms.Form): email2 = forms.EmailField(label=False, help_text=False, max_length=254) password1 = forms.CharField(label=False, help_text=False, widget=forms.PasswordInput) accepted_tos = forms.BooleanField( - label=mark_safe('Check this box to accept our terms of ' - 'use and the privacy policy'), + label=mark_safe( + 'Check this box to accept our terms of ' + 'use and the privacy policy' + ), required=True, error_messages={'required': 'You must accept the terms of use in order to register to Freesound'} ) - recaptcha = ReCaptchaField(label="") # Note that this field needs to be the last to appear last in the form + recaptcha = ReCaptchaField(label="") # Note that this field needs to be the last to appear last in the form def __init__(self, *args, **kwargs): kwargs.update(dict(auto_id='id_%s_registration')) @@ -251,13 +262,9 @@ def save(self): email = self.cleaned_data["email1"] password = self.cleaned_data["password1"] - # NOTE: we create user "manually" instead of using "create_user" as we don't want + # NOTE: we create user "manually" instead of using "create_user" as we don't want # is_active to be set to True automatically - user = User(username=username, - email=email, - is_staff=False, - is_active=False, - is_superuser=False) + user = User(username=username, email=email, is_staff=False, is_active=False, is_superuser=False) user.set_password(password) user.save() return user @@ -280,10 +287,14 @@ class FsAuthenticationForm(AuthenticationForm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.error_messages.update({ - 'inactive': mark_safe("You are trying to log in with an inactive account, please activate " - "your account first." % reverse("accounts-resend-activation")), - 'invalid_login': "Please enter a correct username/email and password. " - "Note that passwords are case-sensitive.", + 'inactive': + mark_safe( + "You are trying to log in with an inactive account, please activate " + "your account first." % reverse("accounts-resend-activation") + ), + 'invalid_login': + "Please enter a correct username/email and password. " + "Note that passwords are case-sensitive.", }) self.fields['username'].label = False self.fields['username'].widget.attrs['placeholder'] = 'Enter your email or username' @@ -299,9 +310,10 @@ class ProfileForm(forms.ModelForm): username = UsernameField(required=False) about = HtmlCleaningCharFieldWithCenterTag( - widget=forms.Textarea(attrs=dict(rows=20, cols=70)), - required=False, - help_text=HtmlCleaningCharFieldWithCenterTag.make_help_text()) + widget=forms.Textarea(attrs=dict(rows=20, cols=70)), + required=False, + help_text=HtmlCleaningCharFieldWithCenterTag.make_help_text() + ) signature = HtmlCleaningCharField( label="Forum signature", help_text=HtmlCleaningCharField.make_help_text(), @@ -315,28 +327,32 @@ class ProfileForm(forms.ModelForm): help_text="""Your sound signature is added to the end of each of your sound descriptions. If you change the sound signature it will be automatically updated on all of your sounds. Use the special text ${sound_url} to refer to the URL of the current sound being displayed - and ${sound_id} to refer to the id of the current sound. """ + HtmlCleaningCharField.make_help_text(), + and ${sound_id} to refer to the id of the current sound. """ + + HtmlCleaningCharField.make_help_text(), required=False, max_length=256, ) - is_adult = forms.BooleanField(label="I'm an adult, I don't want to see inappropriate content warnings", - help_text=False, required=False) + is_adult = forms.BooleanField( + label="I'm an adult, I don't want to see inappropriate content warnings", help_text=False, required=False + ) not_shown_in_online_users_list = forms.BooleanField( - help_text="Hide from \"users currently online\" list in the People page", - label="", - required=False + help_text="Hide from \"users currently online\" list in the People page", label="", required=False ) allow_simultaneous_playback = forms.BooleanField( - label="Allow simultaneous audio playback", required=False, widget=forms.CheckboxInput(attrs={'class': 'bw-checkbox'})) + label="Allow simultaneous audio playback", + required=False, + widget=forms.CheckboxInput(attrs={'class': 'bw-checkbox'}) + ) prefer_spectrograms = forms.BooleanField( - label="Show spectrograms in sound players by default", required=False, widget=forms.CheckboxInput(attrs={'class': 'bw-checkbox'})) + label="Show spectrograms in sound players by default", + required=False, + widget=forms.CheckboxInput(attrs={'class': 'bw-checkbox'}) + ) def __init__(self, request, *args, **kwargs): self.request = request - kwargs.update(initial={ - 'username': request.user.username - }) + kwargs.update(initial={'username': request.user.username}) kwargs.update(dict(label_suffix='')) super().__init__(*args, **kwargs) @@ -426,8 +442,10 @@ def clean_sound_signature(self): class Meta: model = Profile - fields = ('username', 'home_page', 'about', 'signature', 'sound_signature', 'is_adult', - 'allow_simultaneous_playback', 'prefer_spectrograms', 'ui_theme_preference' ) + fields = ( + 'username', 'home_page', 'about', 'signature', 'sound_signature', 'is_adult', 'allow_simultaneous_playback', + 'prefer_spectrograms', 'ui_theme_preference' + ) def get_img_check_fields(self): """ Returns fields that should show JS notification for unsafe `img` sources links (http://) """ @@ -455,7 +473,7 @@ def clean_password(self): class DeleteUserForm(forms.Form): encrypted_link = forms.CharField(widget=forms.HiddenInput()) - delete_sounds = forms.ChoiceField(label=False, choices=DELETE_CHOICES, widget=forms.RadioSelect()) + delete_sounds = forms.ChoiceField(label=False, choices=DELETE_CHOICES, widget=forms.RadioSelect()) password = forms.CharField(label="Confirm your password", widget=forms.PasswordInput) def clean_password(self): @@ -485,10 +503,7 @@ def reset_encrypted_link(self, user_id): def __init__(self, *args, **kwargs): self.user_id = kwargs.pop('user_id') encrypted_link = sign_with_timestamp(self.user_id) - kwargs['initial'] = { - 'delete_sounds': 'only_user', - 'encrypted_link': encrypted_link - } + kwargs['initial'] = {'delete_sounds': 'only_user', 'encrypted_link': encrypted_link} kwargs.update(dict(label_suffix='')) super().__init__(*args, **kwargs) @@ -501,10 +516,7 @@ def __init__(self, *args, **kwargs): class EmailSettingsForm(forms.Form): email_types = forms.ModelMultipleChoiceField( - queryset=EmailPreferenceType.objects.all(), - widget=forms.CheckboxSelectMultiple, - required=False, - label=False + queryset=EmailPreferenceType.objects.all(), widget=forms.CheckboxSelectMultiple, required=False, label=False ) def __init__(self, *args, **kwargs): @@ -536,7 +548,8 @@ def get_users(self, username_or_email): password hash that django understands) """ UserModel = get_user_model() - active_users = UserModel._default_manager.filter(Q(**{ + active_users = UserModel._default_manager.filter( + Q(**{ f'{UserModel.get_email_field_name()}__iexact': username_or_email, 'is_active': True, }) | Q(**{ @@ -546,12 +559,18 @@ def get_users(self, username_or_email): ) return (u for u in active_users) - def save(self, domain_override=None, - subject_template_name='emails/password_reset_subject.txt', - email_template_name='emails/password_reset_email.html', - use_https=False, token_generator=default_token_generator, - from_email=None, request=None, html_email_template_name=None, - extra_email_context=None): + def save( + self, + domain_override=None, + subject_template_name='emails/password_reset_subject.txt', + email_template_name='emails/password_reset_email.html', + use_https=False, + token_generator=default_token_generator, + from_email=None, + request=None, + html_email_template_name=None, + extra_email_context=None + ): """ Generates a one-use only link for resetting password and sends to the user. @@ -577,8 +596,12 @@ def save(self, domain_override=None, context.update(extra_email_context) dj_auth_form = PasswordResetForm() dj_auth_form.send_mail( - subject_template_name, email_template_name, context, from_email, - user.email, html_email_template_name=html_email_template_name, + subject_template_name, + email_template_name, + context, + from_email, + user.email, + html_email_template_name=html_email_template_name, ) diff --git a/accounts/management/commands/check_async_deleted_users.py b/accounts/management/commands/check_async_deleted_users.py index f7bde804c..70b1b54ba 100644 --- a/accounts/management/commands/check_async_deleted_users.py +++ b/accounts/management/commands/check_async_deleted_users.py @@ -43,8 +43,8 @@ def handle(self, *args, **options): user_ids_not_properly_deleted = [] for user_deletion_request in UserDeletionRequest.objects.filter( status=UserDeletionRequest.DELETION_REQUEST_STATUS_DELETION_TRIGGERED, - last_updated__lt=datetime.datetime.now() - - datetime.timedelta(hours=settings.CHECK_ASYNC_DELETED_USERS_HOURS_BACK)): + last_updated__lt=datetime.datetime.now() - + datetime.timedelta(hours=settings.CHECK_ASYNC_DELETED_USERS_HOURS_BACK)): if user_deletion_request.user_to is not None and \ not user_deletion_request.user_to.profile.is_anonymized_user: @@ -57,16 +57,18 @@ def handle(self, *args, **options): user_deletion_request.status = UserDeletionRequest.DELETION_REQUEST_STATUS_USER_WAS_DELETED user_deletion_request.save() - user_ids_not_properly_deleted = list(set(user_ids_not_properly_deleted)) - console_logger.info('Found {} users that should have been deleted and were not'.format( - len(user_ids_not_properly_deleted))) + console_logger.info( + 'Found {} users that should have been deleted and were not'.format(len(user_ids_not_properly_deleted)) + ) for user_id in user_ids_not_properly_deleted: # It could be that there are several requests per user, just display info about the first one user_deletion_request = UserDeletionRequest.objects.filter(user_to_id=user_id).first() - console_logger.info('- User "{0}" with id {1} should have been deleted. Action: "{2}". Reason: "{2}".' - .format(user_deletion_request.user_to.username, user_deletion_request.user_to.id, - user_deletion_request.triggered_deletion_action, user_deletion_request.triggered_deletion_reason - )) + console_logger.info( + '- User "{0}" with id {1} should have been deleted. Action: "{2}". Reason: "{2}".'.format( + user_deletion_request.user_to.username, user_deletion_request.user_to.id, + user_deletion_request.triggered_deletion_action, user_deletion_request.triggered_deletion_reason + ) + ) self.log_end({'n_users_should_have_been_deleted': len(user_ids_not_properly_deleted)}) diff --git a/accounts/management/commands/clean_old_tmp_upload_files.py b/accounts/management/commands/clean_old_tmp_upload_files.py index dcfcd9569..162b4d17e 100644 --- a/accounts/management/commands/clean_old_tmp_upload_files.py +++ b/accounts/management/commands/clean_old_tmp_upload_files.py @@ -18,7 +18,6 @@ # See AUTHORS file. # - from django.core.management.base import BaseCommand from django.conf import settings import os @@ -28,10 +27,10 @@ class Command(BaseCommand): help = "Cleans all files in FILE_UPLOAD_TEMP_DIR which are older than 24 hours" - def handle(self, *args, **options): + def handle(self, *args, **options): for f in os.listdir(settings.FILE_UPLOAD_TEMP_DIR): f_mod_date = datetime.datetime.fromtimestamp(os.path.getmtime(settings.FILE_UPLOAD_TEMP_DIR + f)) now = datetime.datetime.now() - if (now - f_mod_date).total_seconds() > 3600*24: + if (now - f_mod_date).total_seconds() > 3600 * 24: print(f"Deleting {f}") os.remove(settings.FILE_UPLOAD_TEMP_DIR + f) diff --git a/accounts/management/commands/cleanup_unactivated_users.py b/accounts/management/commands/cleanup_unactivated_users.py index 6a762ddbc..065f01480 100644 --- a/accounts/management/commands/cleanup_unactivated_users.py +++ b/accounts/management/commands/cleanup_unactivated_users.py @@ -42,8 +42,9 @@ def add_arguments(self, parser): ) def handle(self, *args, **options): - bounces = EmailBounce.objects.filter(type__in=EmailBounce.TYPES_INVALID, user__is_active=False, - user__last_login=None) + bounces = EmailBounce.objects.filter( + type__in=EmailBounce.TYPES_INVALID, user__is_active=False, user__last_login=None + ) users = User.objects.filter(id__in=bounces.values_list('user', flat=True)) if not options['fast']: diff --git a/accounts/management/commands/process_email_bounces.py b/accounts/management/commands/process_email_bounces.py index b92fddd4e..fd4d8ea49 100644 --- a/accounts/management/commands/process_email_bounces.py +++ b/accounts/management/commands/process_email_bounces.py @@ -48,9 +48,9 @@ def process_message(data): user = User.objects.get(email__iexact=email) EmailBounce.objects.create(user=user, type=bounce_type, timestamp=timestamp) n_recipients += 1 - except User.DoesNotExist: # user probably got deleted + except User.DoesNotExist: # user probably got deleted console_logger.info(f'User {email} not found in database (probably deleted)') - except IntegrityError: # message duplicated + except IntegrityError: # message duplicated is_duplicate = True return is_duplicate, n_recipients @@ -65,12 +65,13 @@ def create_dump_file(): def decode_idna_email(email): """Takes email (unicode) with IDNA encoded domain and returns true unicode representation""" user, domain = email.split('@') - domain = domain.encode().decode('idna') # explicit encode to bytestring for python 2/3 compatability + domain = domain.encode().decode('idna') # explicit encode to bytestring for python 2/3 compatability return user + '@' + domain class Command(LoggingBaseCommand): help = 'Retrieves email bounce info from AWS SQS and updates db.' + # At the time of implementation AWS has two queue types: standard and FIFO. Standard queue will not guarantee the # uniqueness of messages that we are receiving, thus it is possible to get duplicates of the same message. # Configuration parameter AWS_SQS_MESSAGES_PER_CALL can take values from 1 to 10 and indicates number of messages @@ -112,8 +113,9 @@ def handle(self, *args, **options): messages_per_call = settings.AWS_SQS_MESSAGES_PER_CALL if not 1 <= settings.AWS_SQS_MESSAGES_PER_CALL <= 10: - console_logger.warn('Invalid value for number messages to process per call: {}, using 1' - .format(messages_per_call)) + console_logger.warn( + 'Invalid value for number messages to process per call: {}, using 1'.format(messages_per_call) + ) messages_per_call = 1 save_messages = options['save'] @@ -122,7 +124,7 @@ def handle(self, *args, **options): console_logger.info('Running without deleting messages from queue (one batch)') total_messages = 0 - total_bounces = 0 # counts multiple recipients of the same mail + total_bounces = 0 # counts multiple recipients of the same mail has_messages = True all_messages = [] @@ -130,15 +132,12 @@ def handle(self, *args, **options): # Receive message from SQS queue try: response = sqs.receive_message( - QueueUrl=queue_url, - MaxNumberOfMessages=messages_per_call, - VisibilityTimeout=0, - WaitTimeSeconds=0 + QueueUrl=queue_url, MaxNumberOfMessages=messages_per_call, VisibilityTimeout=0, WaitTimeSeconds=0 ) except EndpointConnectionError as e: console_logger.info(str(e)) return - + messages = response.get('Messages', []) has_messages = not no_delete and len(messages) > 0 @@ -152,10 +151,7 @@ def handle(self, *args, **options): is_duplicate, n_recipients = process_message(data['bounce']) if not no_delete: - sqs.delete_message( - QueueUrl=queue_url, - ReceiptHandle=receipt_handle - ) + sqs.delete_message(QueueUrl=queue_url, ReceiptHandle=receipt_handle) if not is_duplicate: total_messages += 1 diff --git a/accounts/models.py b/accounts/models.py index a5272f023..b36f591c3 100644 --- a/accounts/models.py +++ b/accounts/models.py @@ -76,11 +76,9 @@ class DeletedUser(models.Model): DELETION_REASON_SPAMMER = 'sp' DELETION_REASON_DELETED_BY_ADMIN = 'ad' DELETION_REASON_SELF_DELETED = 'sd' - DELETION_REASON_CHOICES = ( - (DELETION_REASON_SPAMMER, 'Spammer'), - (DELETION_REASON_DELETED_BY_ADMIN, 'Deleted by an admin'), - (DELETION_REASON_SELF_DELETED, 'Self deleted') - ) + DELETION_REASON_CHOICES = ((DELETION_REASON_SPAMMER, + 'Spammer'), (DELETION_REASON_DELETED_BY_ADMIN, 'Deleted by an admin'), + (DELETION_REASON_SELF_DELETED, 'Self deleted')) reason = models.CharField(max_length=2, choices=DELETION_REASON_CHOICES) def __str__(self): @@ -94,7 +92,7 @@ def random_uploader(): user_count = User.objects.filter(profile__num_sounds__gte=1).count() if user_count: offset = random.randint(0, user_count - 1) - return User.objects.filter(profile__num_sounds__gte=1)[offset:offset+1][0] + return User.objects.filter(profile__num_sounds__gte=1)[offset:offset + 1][0] else: return None @@ -110,7 +108,9 @@ class Profile(models.Model): is_whitelisted = models.BooleanField(default=False, db_index=True) has_old_license = models.BooleanField(null=False, default=False) not_shown_in_online_users_list = models.BooleanField(null=False, default=False) - accepted_tos = models.BooleanField(default=False) # This legacy field referring to old (pre-GDPR) terms of service + accepted_tos = models.BooleanField( + default=False + ) # This legacy field referring to old (pre-GDPR) terms of service last_stream_email_sent = models.DateTimeField(db_index=True, null=True, default=None, blank=True) last_attempt_of_sending_stream_email = models.DateTimeField(db_index=True, null=True, default=None, blank=True) @@ -123,10 +123,18 @@ class Profile(models.Model): # The following 4 fields are updated using django signals (methods 'update_num_downloads*') num_sounds = models.PositiveIntegerField(editable=False, default=0) num_posts = models.PositiveIntegerField(editable=False, default=0) - num_sound_downloads = models.PositiveIntegerField(editable=False, default=0) # Number of sounds the user has downloaded - num_pack_downloads = models.PositiveIntegerField(editable=False, default=0) # Number of packs the user has downloaded - num_user_sounds_downloads = models.PositiveIntegerField(editable=False, default=0) # Number of times user's sounds have been downloaded - num_user_packs_downloads = models.PositiveIntegerField(editable=False, default=0) # Number of times user's packs have been downloaded + num_sound_downloads = models.PositiveIntegerField( + editable=False, default=0 + ) # Number of sounds the user has downloaded + num_pack_downloads = models.PositiveIntegerField( + editable=False, default=0 + ) # Number of packs the user has downloaded + num_user_sounds_downloads = models.PositiveIntegerField( + editable=False, default=0 + ) # Number of times user's sounds have been downloaded + num_user_packs_downloads = models.PositiveIntegerField( + editable=False, default=0 + ) # Number of times user's packs have been downloaded # "is_anonymized_user" indicates that the user account has been anonimized and no longer contains personal data # This is what we do when we delete a user to still preserve statistics and information and downloads @@ -224,10 +232,10 @@ def get_absolute_url(self): def get_user_sounds_in_search_url(self): return f'{reverse("sounds-search")}?f=username:"{ self.user.username }"&s=Date+added+(newest+first)&g=0' - + def get_user_packs_in_search_url(self): return f'{reverse("sounds-search")}?f=username:"{ self.user.username }"&s=Date+added+(newest+first)&g=1&only_p=1' - + def get_latest_packs_for_profile_page(self): latest_pack_ids = Pack.objects.select_related().filter(user=self.user, num_sounds__gt=0).exclude(is_deleted=True) \ .order_by("-last_updated").values_list('id', flat=True)[0:15] @@ -248,22 +256,10 @@ def locations_static(user_id, has_avatar): xl_avatar = None return dict( avatar=dict( - S=dict( - path=os.path.join(settings.AVATARS_PATH, id_folder, "%d_S.jpg" % user_id), - url=s_avatar - ), - M=dict( - path=os.path.join(settings.AVATARS_PATH, id_folder, "%d_M.jpg" % user_id), - url=m_avatar - ), - L=dict( - path=os.path.join(settings.AVATARS_PATH, id_folder, "%d_L.jpg" % user_id), - url=l_avatar - ), - XL=dict( - path=os.path.join(settings.AVATARS_PATH, id_folder, "%d_XL.jpg" % user_id), - url=xl_avatar - ) + S=dict(path=os.path.join(settings.AVATARS_PATH, id_folder, "%d_S.jpg" % user_id), url=s_avatar), + M=dict(path=os.path.join(settings.AVATARS_PATH, id_folder, "%d_M.jpg" % user_id), url=m_avatar), + L=dict(path=os.path.join(settings.AVATARS_PATH, id_folder, "%d_L.jpg" % user_id), url=l_avatar), + XL=dict(path=os.path.join(settings.AVATARS_PATH, id_folder, "%d_XL.jpg" % user_id), url=xl_avatar) ), uploads_dir=os.path.join(settings.UPLOADS_PATH, str(user_id)) ) @@ -355,8 +351,11 @@ def get_user_tags(self): try: search_engine = get_search_engine() tags_counts = search_engine.get_user_tags(self.user.username) - return [{'name': tag, 'count': count, 'browse_url': reverse('tags', args=[tag]) + f'?username_flt={self.user.username}'} for tag, count in - tags_counts] + return [{ + 'name': tag, + 'count': count, + 'browse_url': reverse('tags', args=[tag]) + f'?username_flt={self.user.username}' + } for tag, count in tags_counts] except SearchEngineException as e: return False except Exception as e: @@ -418,7 +417,8 @@ def can_do_bulk_upload(self): self.user.has_perm('sounds.can_describe_in_bulk') def is_blocked_for_spam_reports(self): - reports_count = UserFlag.objects.filter(user__username=self.user.username).values('reporting_user').distinct().count() + reports_count = UserFlag.objects.filter(user__username=self.user.username + ).values('reporting_user').distinct().count() if reports_count < settings.USERFLAG_THRESHOLD_FOR_AUTOMATIC_BLOCKING or self.user.sounds.all().count() > 0: return False else: @@ -447,10 +447,13 @@ def get_info_before_delete_user(self, include_sounds=False, include_other_relate ret['profile'] = self return ret - def delete_user(self, remove_sounds=False, - delete_user_object_from_db=False, - deletion_reason=DeletedUser.DELETION_REASON_DELETED_BY_ADMIN, - chunk_size=100): + def delete_user( + self, + remove_sounds=False, + delete_user_object_from_db=False, + deletion_reason=DeletedUser.DELETION_REASON_DELETED_BY_ADMIN, + chunk_size=100 + ): """ Custom method for deleting a user from Freesound. @@ -479,7 +482,7 @@ def delete_user(self, remove_sounds=False, chunk_size (int): size of the chunks in which sounds will be deleted inside atomic trasactions. """ - # If required, start by deleting all user's sounds and packs + # If required, start by deleting all user's sounds and packs if remove_sounds: Pack.objects.filter(user=self.user).update(is_deleted=True) num_sounds = Sound.objects.filter(user=self.user).count() @@ -493,7 +496,7 @@ def delete_user(self, remove_sounds=False, except (IntegrityError, ForeignKeyViolation) as e: num_errors += 1 num_sounds = Sound.objects.filter(user=self.user).count() - + if Sound.objects.filter(user=self.user).count() > 0: raise Exception("Could not delete all sounds from user {0}".format(self.user.username)) @@ -514,7 +517,8 @@ def delete_user(self, remove_sounds=False, date_joined=self.user.date_joined, last_login=self.user.last_login, sounds_were_also_deleted=remove_sounds, - reason=deletion_reason) + reason=deletion_reason + ) # Before deleting the user from db or anonymizing it, get a list of all UserDeletionRequest that will need # to be updated once the user has been deleted (we do that because if the user gets deleted from DB, the @@ -557,7 +561,7 @@ def delete_user(self, remove_sounds=False, # If UserDeletionRequest object(s) exist for that user, update the status and set deleted_user property # NOTE: don't use QuerySet.update method because this won't trigger the pre_save/post_save signals - for udr in UserDeletionRequest.objects.filter(id__in=udr_to_update_ids): + for udr in UserDeletionRequest.objects.filter(id__in=udr_to_update_ids): udr.status = UserDeletionRequest.DELETION_REQUEST_STATUS_USER_WAS_DELETED udr.deleted_user = deleted_user_object udr.save() @@ -567,18 +571,15 @@ def has_content(self): Checks if the user has created any content or used Freesound in any way that leaves any significant data. Typically should be used to check if it is safe to hard delete the user. """ - return (Sound.objects.filter(user=self.user).exists() or - Pack.objects.filter(user=self.user).exists() or - SoundRating.objects.filter(user=self.user).exists() or - Download.objects.filter(user=self.user).exists() or - PackDownload.objects.filter(user=self.user).exists() or - Bookmark.objects.filter(user=self.user).exists() or - Post.objects.filter(author=self.user).exists() or - Comment.objects.filter(user=self.user).exists() or - Donation.objects.filter(user=self.user).exists() or - Message.objects.filter(user_from=self.user).exists() or - BulkUploadProgress.objects.filter(user=self.user).exists() or - ApiV2Client.objects.filter(user=self.user).exists()) + return ( + Sound.objects.filter(user=self.user).exists() or Pack.objects.filter(user=self.user).exists() + or SoundRating.objects.filter(user=self.user).exists() or Download.objects.filter(user=self.user).exists() + or PackDownload.objects.filter(user=self.user).exists() or Bookmark.objects.filter(user=self.user).exists() + or Post.objects.filter(author=self.user).exists() or Comment.objects.filter(user=self.user).exists() + or Donation.objects.filter(user=self.user).exists() or Message.objects.filter(user_from=self.user).exists() + or BulkUploadProgress.objects.filter(user=self.user).exists() + or ApiV2Client.objects.filter(user=self.user).exists() + ) def update_num_sounds(self, commit=True): """ @@ -626,15 +627,15 @@ def get_total_uploaded_sounds_length(self): def num_packs(self): # Return the number of packs for which at least one sound has been published return Sound.public.filter(user_id=self.user_id).exclude(pack=None).order_by('pack_id').distinct('pack').count() - + def get_stats_for_profile_page(self): # Return a dictionary of user statistics to show on the user profile page # Because some stats are expensive to compute, we cache them stats_from_db = { - 'num_sounds': self.num_sounds, - 'num_downloads': self.num_downloads_on_sounds_and_packs, - 'num_posts': self.num_posts - } + 'num_sounds': self.num_sounds, + 'num_downloads': self.num_downloads_on_sounds_and_packs, + 'num_posts': self.num_posts + } stats_from_cache = cache.get(settings.USER_STATS_CACHE_KEY.format(self.user_id), None) if stats_from_cache is None: stats_from_cache = { @@ -642,13 +643,12 @@ def get_stats_for_profile_page(self): 'total_uploaded_sounds_length': self.get_total_uploaded_sounds_length(), 'avg_rating_0_5': self.avg_rating_0_5, } - cache.set(settings.USER_STATS_CACHE_KEY.format(self.user_id), stats_from_cache, 60*60*24) + cache.set(settings.USER_STATS_CACHE_KEY.format(self.user_id), stats_from_cache, 60 * 60 * 24) stats_from_db.update(stats_from_cache) return stats_from_db - class Meta: - ordering = ('-user__date_joined', ) + ordering = ('-user__date_joined',) class GdprAcceptance(models.Model): @@ -690,7 +690,7 @@ def orig_email(self): @property def main_trans_email(self): - return self.main_orig_email # email of main user remained unchanged + return self.main_orig_email # email of main user remained unchanged @property def secondary_trans_email(self): @@ -740,9 +740,11 @@ class EmailPreferenceType(models.Model): description = models.TextField(max_length=1024, null=True, blank=True) name = models.CharField(max_length=255, unique=True) display_name = models.CharField(max_length=255) - send_by_default = models.BooleanField(default=True, + send_by_default = models.BooleanField( + default=True, help_text="Indicates if the user should receive an email, if " + - "UserEmailSetting exists for the user then the behavior is the opposite") + "UserEmailSetting exists for the user then the behavior is the opposite" + ) def __str__(self): return self.display_name @@ -769,11 +771,7 @@ class EmailBounce(models.Model): UNDETERMINED = 'UD' PERMANENT = 'PE' TRANSIENT = 'TR' - TYPE_CHOICES = ( - (UNDETERMINED, 'Undetermined'), - (PERMANENT, 'Permanent'), - (TRANSIENT, 'Transient') - ) + TYPE_CHOICES = ((UNDETERMINED, 'Undetermined'), (PERMANENT, 'Permanent'), (TRANSIENT, 'Transient')) TYPES_INVALID = [UNDETERMINED, PERMANENT] type = models.CharField(db_index=True, max_length=2, choices=TYPE_CHOICES, default=UNDETERMINED) type_map = {t[1]: t[0] for t in TYPE_CHOICES} @@ -817,14 +815,22 @@ class UserDeletionRequest(models.Model): NOTE: username_from and username_to are filled in automatically when UserDeletionRequest object is saved. """ - email_from = models.CharField(max_length=200, help_text="The email from which the user deletion request" - "was received.") + email_from = models.CharField( + max_length=200, help_text="The email from which the user deletion request" + "was received." + ) user_from = models.ForeignKey( - User, null=True, on_delete=models.SET_NULL, related_name='deletion_requests_from', blank=True) + User, null=True, on_delete=models.SET_NULL, related_name='deletion_requests_from', blank=True + ) username_from = models.CharField(max_length=150, null=True, blank=True) - user_to = models.ForeignKey(User, null=True, on_delete=models.SET_NULL, related_name='deletion_requests_to', - help_text="The user account that should be deleted if this request proceeds. " - "Note that you can click on the magnifying glass icon and search by email.") + user_to = models.ForeignKey( + User, + null=True, + on_delete=models.SET_NULL, + related_name='deletion_requests_to', + help_text="The user account that should be deleted if this request proceeds. " + "Note that you can click on the magnifying glass icon and search by email." + ) username_to = models.CharField(max_length=150, null=True, blank=True) deleted_user = models.ForeignKey(DeletedUser, null=True, on_delete=models.SET_NULL) DELETION_REQUEST_STATUS_RECEIVED_REQUEST = 're' @@ -832,13 +838,11 @@ class UserDeletionRequest(models.Model): DELETION_REQUEST_STATUS_DELETION_CANCELLED = 'ca' DELETION_REQUEST_STATUS_DELETION_TRIGGERED = 'tr' DELETION_REQUEST_STATUS_USER_WAS_DELETED = 'de' - DELETION_REQUEST_STATUSES = ( - (DELETION_REQUEST_STATUS_RECEIVED_REQUEST, 'Received email deletion request'), - (DELETION_REQUEST_STATUS_WAITING_FOR_USER, 'Waiting for user action'), - (DELETION_REQUEST_STATUS_DELETION_CANCELLED, 'Request was cancelled'), - (DELETION_REQUEST_STATUS_DELETION_TRIGGERED, 'Deletion action was triggered'), - (DELETION_REQUEST_STATUS_USER_WAS_DELETED, 'User has been deleted or anonymized') - ) + DELETION_REQUEST_STATUSES = ((DELETION_REQUEST_STATUS_RECEIVED_REQUEST, 'Received email deletion request'), + (DELETION_REQUEST_STATUS_WAITING_FOR_USER, 'Waiting for user action'), + (DELETION_REQUEST_STATUS_DELETION_CANCELLED, 'Request was cancelled'), + (DELETION_REQUEST_STATUS_DELETION_TRIGGERED, 'Deletion action was triggered'), + (DELETION_REQUEST_STATUS_USER_WAS_DELETED, 'User has been deleted or anonymized')) status = models.CharField(max_length=2, choices=DELETION_REQUEST_STATUSES, db_index=True, default='re') last_updated = models.DateTimeField(auto_now=True) @@ -849,6 +853,7 @@ class UserDeletionRequest(models.Model): triggered_deletion_action = models.CharField(max_length=100) triggered_deletion_reason = models.CharField(max_length=100) + @receiver(pre_save, sender=UserDeletionRequest) def update_status_history(sender, instance, **kwargs): should_update_status_history = False @@ -862,9 +867,12 @@ def update_status_history(sender, instance, **kwargs): should_update_status_history = True if should_update_status_history: - instance.status_history += ['{}: {} ({})'.format(pytz.utc.localize(datetime.datetime.utcnow()), - instance.get_status_display(), - instance.status)] + instance.status_history += [ + '{}: {} ({})'.format( + pytz.utc.localize(datetime.datetime.utcnow()), instance.get_status_display(), instance.status + ) + ] + @receiver(pre_save, sender=UserDeletionRequest) def updated_status_history(sender, instance, **kwargs): diff --git a/accounts/templatetags/display_user.py b/accounts/templatetags/display_user.py index 7a731a109..9823f936d 100644 --- a/accounts/templatetags/display_user.py +++ b/accounts/templatetags/display_user.py @@ -18,7 +18,6 @@ # See AUTHORS file. # - from django import template from django.contrib.auth.models import User diff --git a/accounts/templatetags/filefunctions.py b/accounts/templatetags/filefunctions.py index f12e7b91e..947808b7d 100644 --- a/accounts/templatetags/filefunctions.py +++ b/accounts/templatetags/filefunctions.py @@ -22,6 +22,7 @@ register = template.Library() + @register.inclusion_tag('accounts/recursive_file.html') def show_file(file_structure, non_recursive=False): - return {'file': file_structure, 'non_recursive': non_recursive} \ No newline at end of file + return {'file': file_structure, 'non_recursive': non_recursive} diff --git a/accounts/templatetags/flag_user.py b/accounts/templatetags/flag_user.py index 3128766e1..b105745e4 100644 --- a/accounts/templatetags/flag_user.py +++ b/accounts/templatetags/flag_user.py @@ -20,11 +20,12 @@ from django import template from accounts.models import UserFlag + register = template.Library() @register.inclusion_tag("accounts/flag_user.html", takes_context=True) -def flag_user(context, flag_type, username, content_id, text = None, user_sounds = None): +def flag_user(context, flag_type, username, content_id, text=None, user_sounds=None): no_show = False link_text = "Report spam/offensive" @@ -33,17 +34,19 @@ def flag_user(context, flag_type, username, content_id, text = None, user_sounds no_show = True flagged = [] else: - flagged = UserFlag.objects.filter(user__username=username, - reporting_user=context['request'].user, - object_id=content_id).values('reporting_user').distinct() + flagged = UserFlag.objects.filter( + user__username=username, reporting_user=context['request'].user, object_id=content_id + ).values('reporting_user').distinct() if text: link_text = text - return {'user_sounds': user_sounds, - 'done_text': "Marked as spam/offensive", # Not used in BW - 'flagged': len(flagged), - 'flag_type': flag_type, - 'username': username, - 'content_obj_id': content_id, - 'link_text': link_text, - 'no_show': no_show} + return { + 'user_sounds': user_sounds, + 'done_text': "Marked as spam/offensive", # Not used in BW + 'flagged': len(flagged), + 'flag_type': flag_type, + 'username': username, + 'content_obj_id': content_id, + 'link_text': link_text, + 'no_show': no_show + } diff --git a/accounts/tests/test_profile.py b/accounts/tests/test_profile.py index 14b77b5f4..2be1fc99c 100644 --- a/accounts/tests/test_profile.py +++ b/accounts/tests/test_profile.py @@ -51,18 +51,9 @@ def test_user_tagcloud_solr(self): user = User.objects.get(username="Anton") mock_search_engine = mock.Mock() conf = { - 'get_user_tags.return_value': [ - ('conversation', 1), - ('dutch', 1), - ('glas', 1), - ('glass', 1), - ('instrument', 2), - ('laughter', 1), - ('sine-like', 1), - ('struck', 1), - ('tone', 1), - ('water', 1) - ] + 'get_user_tags.return_value': [('conversation', 1), ('dutch', 1), ('glas', 1), ('glass', 1), + ('instrument', 2), ('laughter', 1), ('sine-like', 1), ('struck', 1), + ('tone', 1), ('water', 1)] } mock_search_engine.return_value.configure_mock(**conf) accounts.models.get_search_engine = mock_search_engine @@ -100,13 +91,15 @@ def test_handle_uploaded_image(self): def test_edit_user_profile(self): user = User.objects.create_user("testuser") self.client.force_login(user) - resp = self.client.post("/home/edit/", { - 'profile-home_page': 'http://www.example.com/', - 'profile-username': 'testuser', - 'profile-about': 'About test text', - 'profile-signature': 'Signature test text', - 'profile-ui_theme_preference': 'd', - }) + resp = self.client.post( + "/home/edit/", { + 'profile-home_page': 'http://www.example.com/', + 'profile-username': 'testuser', + 'profile-about': 'About test text', + 'profile-signature': 'Signature test text', + 'profile-ui_theme_preference': 'd', + } + ) user = User.objects.select_related('profile').get(username="testuser") self.assertEqual(user.profile.home_page, 'http://www.example.com/') @@ -116,13 +109,15 @@ def test_edit_user_profile(self): # Now we change the username the maximum allowed times for i in range(settings.USERNAME_CHANGE_MAX_TIMES): - self.client.post("/home/edit/", { - 'profile-home_page': 'http://www.example.com/', - 'profile-username': 'testuser%d' % i, - 'profile-about': 'About test text', - 'profile-signature': 'Signature test text', - 'profile-ui_theme_preference': 'd', - }) + self.client.post( + "/home/edit/", { + 'profile-home_page': 'http://www.example.com/', + 'profile-username': 'testuser%d' % i, + 'profile-about': 'About test text', + 'profile-signature': 'Signature test text', + 'profile-ui_theme_preference': 'd', + } + ) user.refresh_from_db() self.assertEqual(user.old_usernames.count(), i + 1) @@ -130,13 +125,15 @@ def test_edit_user_profile(self): # Now the "username" field in the form will be "disabled" because maximum number of username changes has been # reached. Therefore, the contents of "profile-username" in the POST request should have no effect and username # should not be changed any further - self.client.post("/home/edit/", { - 'profile-home_page': 'http://www.example.com/', - 'profile-username': 'testuser-error', - 'profile-about': 'About test text', - 'profile-signature': 'Signature test text', - 'profile-ui_theme_preference': 'd', - }) + self.client.post( + "/home/edit/", { + 'profile-home_page': 'http://www.example.com/', + 'profile-username': 'testuser-error', + 'profile-about': 'About test text', + 'profile-signature': 'Signature test text', + 'profile-ui_theme_preference': 'd', + } + ) user.refresh_from_db() self.assertEqual(user.old_usernames.count(), settings.USERNAME_CHANGE_MAX_TIMES) @@ -290,11 +287,11 @@ def test_unactivated_user_with_content_not_deleted(self): def test_request_email_change(self): user = User.objects.create_user('user', email='user@freesound.org') self.client.force_login(user) - cache.clear() # Need to clear cache here to avoid 'random_sound' cache key being set + cache.clear() # Need to clear cache here to avoid 'random_sound' cache key being set resp = self.client.get(reverse('front-page')) self.assertEqual(resp.status_code, 200) EmailBounce.objects.create(user=user, type=EmailBounce.PERMANENT) - cache.clear() # Need to clear cache here to avoid 'random_sound' cache key being set + cache.clear() # Need to clear cache here to avoid 'random_sound' cache key being set resp = self.client.get(reverse('front-page')) self.assertRedirects(resp, reverse('accounts-email-reset')) @@ -302,7 +299,9 @@ def test_populate_bounce(self): message_body = { "bounceType": "Permanent", "bounceSubType": "Suppressed", - "bouncedRecipients": [{"emailAddress": "user@freesound.org"}], + "bouncedRecipients": [{ + "emailAddress": "user@freesound.org" + }], "timestamp": "2018-05-20T13:54:37.821Z" } @@ -322,10 +321,12 @@ def test_email_email_bounce_removed_when_resetting_email(self): # User fills in email reset form self.client.force_login(user) - resp = self.client.post(reverse('accounts-email-reset'), { - 'email': 'new_email@freesound.org', - 'password': '12345', - }) + resp = self.client.post( + reverse('accounts-email-reset'), { + 'email': 'new_email@freesound.org', + 'password': '12345', + } + ) self.assertRedirects(resp, reverse('accounts-email-reset-done')) # User goes to link to complete email reset (which is sent by email) @@ -342,15 +343,19 @@ def test_email_email_bounce_removed_when_resetting_email_via_admin(self): EmailBounce.objects.create(user=user, type=EmailBounce.PERMANENT) # Admin changes user's email address via admin page - admin_user = User.objects.create_user('admin_user', email='admin_user@freesound.org', - is_staff=True, is_superuser=True) + admin_user = User.objects.create_user( + 'admin_user', email='admin_user@freesound.org', is_staff=True, is_superuser=True + ) self.client.force_login(admin_user) - resp = self.client.post(reverse('admin:auth_user_change', args=[user.id]), data={ - 'username': user.username, - 'email': 'new_email@freesound.org', - 'date_joined_0': "2015-10-06", - 'date_joined_1': "16:42:00" - }) + resp = self.client.post( + reverse('admin:auth_user_change', args=[user.id]), + data={ + 'username': user.username, + 'email': 'new_email@freesound.org', + 'date_joined_0': "2015-10-06", + 'date_joined_1': "16:42:00" + } + ) user.refresh_from_db() self.assertEqual(user.email, 'new_email@freesound.org') @@ -377,7 +382,7 @@ def test_email_is_valid(self): email_bounce = EmailBounce.objects.create(user=user, type=EmailBounce.PERMANENT) self.assertEqual(user.profile.email_is_valid(), False) email_bounce.delete() - self.assertEqual(user.profile.email_is_valid(), True) # Back to normal + self.assertEqual(user.profile.email_is_valid(), True) # Back to normal # Test email becomes invalid when user is deleted (anonymized) user.profile.delete_user() @@ -445,7 +450,7 @@ def test_can_post_in_forum_numposts(self): today = parse_date("2019-02-05 01:50:00") for i in range(9): post = Post.objects.create(thread=self.thread, body="", author=self.user, moderation_state="OK") - today = today + datetime.timedelta(minutes=i+10) + today = today + datetime.timedelta(minutes=i + 10) post.created = today post.save() self.user.profile.refresh_from_db() @@ -539,8 +544,9 @@ class ProfileTestDownloadCountFields(TestCase): fixtures = ['licenses'] def setUp(self): - self.user, self.packs, self.sounds = create_user_and_sounds(num_sounds=3, num_packs=3, - processing_state="OK", moderation_state="OK") + self.user, self.packs, self.sounds = create_user_and_sounds( + num_sounds=3, num_packs=3, processing_state="OK", moderation_state="OK" + ) @mock.patch('sounds.models.delete_sound_from_gaia') @mock.patch('sounds.models.delete_sounds_from_search_engine') @@ -555,16 +561,16 @@ def test_download_sound_count_field_is_updated(self, delete_sounds_from_search_e # Test deleting downloaded sounds decreases the "num_sound_downloads" field # Delete 2 of the 3 downloaded sounds for i in range(0, len(self.sounds) - 1): - self.sounds[i].delete() # This should decrease "num_sound_downloads" field + self.sounds[i].delete() # This should decrease "num_sound_downloads" field self.user.profile.refresh_from_db() self.assertEqual(self.user.profile.num_sound_downloads, len(self.sounds) - 1 - i) self.assertEqual(self.sounds[0].user.profile.num_user_sounds_downloads, len(self.sounds) - 1 - i) # Now test that if the "num_sound_downloads" field is out of sync and deleting a sound would set it to # -1, we will set it to 0 instead to avoid DB check constraint error - self.user.profile.num_sound_downloads = 0 # Set num_sound_downloads out of sync (should be 1 instead of 0) + self.user.profile.num_sound_downloads = 0 # Set num_sound_downloads out of sync (should be 1 instead of 0) self.user.profile.save() - self.sounds[2].delete() # Delete the remaining sound + self.sounds[2].delete() # Delete the remaining sound self.user.profile.refresh_from_db() self.assertEqual(self.user.profile.num_sound_downloads, 0) self.assertEqual(self.sounds[2].user.profile.num_user_sounds_downloads, 0) @@ -580,16 +586,16 @@ def test_download_pack_count_field_is_updated(self): # Test deleting downloaded packs decreases the "num_pack_downloads" field # Delete 2 of the 3 downloaded packs for i in range(0, len(self.packs) - 1): - self.packs[i].delete() # This should decrease "num_sound_downloads" field + self.packs[i].delete() # This should decrease "num_sound_downloads" field self.user.profile.refresh_from_db() self.assertEqual(self.user.profile.num_pack_downloads, len(self.packs) - 1 - i) self.assertEqual(self.packs[i].user.profile.num_user_packs_downloads, len(self.packs) - 1 - i) # Now test that if the "num_pack_downloads" field is out of sync and deleting a pack would set it to # -1, we will set it to 0 instead to avoid DB check constraint error - self.user.profile.num_pack_downloads = 0 # Set num_sound_downloads out of sync (should be 1 instead of 0) + self.user.profile.num_pack_downloads = 0 # Set num_sound_downloads out of sync (should be 1 instead of 0) self.user.profile.save() - self.packs[2].delete() # Delete the remaining sound + self.packs[2].delete() # Delete the remaining sound self.user.profile.refresh_from_db() self.assertEqual(self.user.profile.num_pack_downloads, 0) self.assertEqual(self.packs[2].user.profile.num_user_packs_downloads, 0) diff --git a/accounts/tests/test_spam.py b/accounts/tests/test_spam.py index a08c0a63d..033022bc5 100644 --- a/accounts/tests/test_spam.py +++ b/accounts/tests/test_spam.py @@ -42,8 +42,7 @@ def setUp(self): # Create users reporting spam self.reporters = [] for i in range(0, settings.USERFLAG_THRESHOLD_FOR_AUTOMATIC_BLOCKING + 1): - reporter = User.objects.create_user(username=f'reporter_{i}', - email=f'reporter_{i}@example.com') + reporter = User.objects.create_user(username=f'reporter_{i}', email=f'reporter_{i}@example.com') self.reporters.append(reporter) # Create user posting spam @@ -58,61 +57,75 @@ def __test_report_object(self, flag_type, object): # Flag the comment (no email to admins should be sent yet as only one reporter) reporter = self.get_reporter_as_logged_in_user(0) - resp = self.client.post(reverse('flag-user', kwargs={'username': self.spammer.username}), data={ - 'object_id': object.id, - 'flag_type': flag_type, - }) + resp = self.client.post( + reverse('flag-user', kwargs={'username': self.spammer.username}), + data={ + 'object_id': object.id, + 'flag_type': flag_type, + } + ) self.assertEqual(resp.status_code, 200) - self.assertEqual(UserFlag.objects.count(), 1) # One flag object created - self.assertEqual(UserFlag.objects.first().reporting_user, reporter) # Flag object created by reporter - self.assertEqual(len(mail.outbox), 0) # No email sent + self.assertEqual(UserFlag.objects.count(), 1) # One flag object created + self.assertEqual(UserFlag.objects.first().reporting_user, reporter) # Flag object created by reporter + self.assertEqual(len(mail.outbox), 0) # No email sent # Flag the same comment by other users so email is sent - for i in range(1, settings.USERFLAG_THRESHOLD_FOR_NOTIFICATION): # Start at 1 as first flag already done + for i in range(1, settings.USERFLAG_THRESHOLD_FOR_NOTIFICATION): # Start at 1 as first flag already done reporter = self.get_reporter_as_logged_in_user(i) - resp = self.client.post(reverse('flag-user', kwargs={'username': self.spammer.username}), data={ - 'object_id': object.id, - 'flag_type': flag_type, - }) + resp = self.client.post( + reverse('flag-user', kwargs={'username': self.spammer.username}), + data={ + 'object_id': object.id, + 'flag_type': flag_type, + } + ) self.assertEqual(resp.status_code, 200) - self.assertEqual(UserFlag.objects.count(), i + 1) # Now we have more flags - self.assertEqual(UserFlag.objects.all().order_by('id')[i].reporting_user, - reporter) # Flag object created by reporter + self.assertEqual(UserFlag.objects.count(), i + 1) # Now we have more flags + self.assertEqual( + UserFlag.objects.all().order_by('id')[i].reporting_user, reporter + ) # Flag object created by reporter # Now check that after the new flags an email was sent - self.assertEqual(len(mail.outbox), 1) # Notification email sent + self.assertEqual(len(mail.outbox), 1) # Notification email sent self.assertTrue(settings.EMAIL_SUBJECT_PREFIX in mail.outbox[0].subject) self.assertTrue(settings.EMAIL_SUBJECT_USER_SPAM_REPORT in mail.outbox[0].subject) self.assertTrue("has been reported" in mail.outbox[0].body) # Continue flagging object until it reaches blocked state for i in range(settings.USERFLAG_THRESHOLD_FOR_NOTIFICATION, - settings.USERFLAG_THRESHOLD_FOR_AUTOMATIC_BLOCKING): # Start at 1 as first flag already done + settings.USERFLAG_THRESHOLD_FOR_AUTOMATIC_BLOCKING): # Start at 1 as first flag already done reporter = self.get_reporter_as_logged_in_user(i) - resp = self.client.post(reverse('flag-user', kwargs={'username': self.spammer.username}), data={ - 'object_id': object.id, - 'flag_type': flag_type, - }) + resp = self.client.post( + reverse('flag-user', kwargs={'username': self.spammer.username}), + data={ + 'object_id': object.id, + 'flag_type': flag_type, + } + ) self.assertEqual(resp.status_code, 200) - self.assertEqual(UserFlag.objects.count(), i + 1) # Now we have more flags - self.assertEqual(UserFlag.objects.all().order_by('id')[i].reporting_user, - reporter) # Flag object created by reporter + self.assertEqual(UserFlag.objects.count(), i + 1) # Now we have more flags + self.assertEqual( + UserFlag.objects.all().order_by('id')[i].reporting_user, reporter + ) # Flag object created by reporter # Now check that an extra mail was now sent (the email notifying user is blocked) - self.assertEqual(len(mail.outbox), 2) # New notification email sent + self.assertEqual(len(mail.outbox), 2) # New notification email sent self.assertTrue(settings.EMAIL_SUBJECT_PREFIX in mail.outbox[1].subject) self.assertTrue(settings.EMAIL_SUBJECT_USER_SPAM_REPORT in mail.outbox[1].subject) self.assertTrue("has been blocked" in mail.outbox[1].body) # Flag the object again and check that no new notification emails are sent _ = self.get_reporter_as_logged_in_user(settings.USERFLAG_THRESHOLD_FOR_AUTOMATIC_BLOCKING) - resp = self.client.post(reverse('flag-user', kwargs={'username': self.spammer.username}), data={ - 'object_id': object.id, - 'flag_type': flag_type, - }) + resp = self.client.post( + reverse('flag-user', kwargs={'username': self.spammer.username}), + data={ + 'object_id': object.id, + 'flag_type': flag_type, + } + ) self.assertEqual(resp.status_code, 200) self.assertEqual(UserFlag.objects.count(), settings.USERFLAG_THRESHOLD_FOR_AUTOMATIC_BLOCKING + 1) - self.assertEqual(len(mail.outbox), 2) # New notification email sent + self.assertEqual(len(mail.outbox), 2) # New notification email sent def test_report_sound_comment(self): sound = Sound.objects.first() @@ -121,15 +134,22 @@ def test_report_sound_comment(self): self.__test_report_object('SC', comment) def test_report_forum_post(self): - thread = Thread.objects.create(author=self.spammer, title="Span thread", - forum=Forum.objects.create(name="Test forum")) + thread = Thread.objects.create( + author=self.spammer, title="Span thread", forum=Forum.objects.create(name="Test forum") + ) object = Post.objects.create(author=self.spammer, thread=thread, body="Spam post post body") self.__test_report_object('FP', object) def test_report_private_message(self): - object = Message.objects.create(user_from=self.spammer, user_to=self.reporters[0], subject='Spam subject', - body=MessageBody.objects.create(body='Spam body'), is_sent=True, - is_archived=False, is_read=False) + object = Message.objects.create( + user_from=self.spammer, + user_to=self.reporters[0], + subject='Spam subject', + body=MessageBody.objects.create(body='Spam body'), + is_sent=True, + is_archived=False, + is_read=False + ) self.__test_report_object('PM', object) def test_report_object_same_user(self): @@ -144,25 +164,35 @@ def test_report_object_same_user(self): # Flag the comment many times by the same user (no email to admins should be sent yet as only one reporter) reporter = self.get_reporter_as_logged_in_user(0) for i in range(0, settings.USERFLAG_THRESHOLD_FOR_AUTOMATIC_BLOCKING + 1): - resp = self.client.post(reverse('flag-user', kwargs={'username': self.spammer.username}), data={ - 'object_id': comment.id, - 'flag_type': 'SC', # Sound comment - }) + resp = self.client.post( + reverse('flag-user', kwargs={'username': self.spammer.username}), + data={ + 'object_id': comment.id, + 'flag_type': 'SC', # Sound comment + } + ) self.assertEqual(resp.status_code, 200) self.assertEqual(UserFlag.objects.count(), i + 1) - self.assertEqual(len(mail.outbox), 0) # No email sent - + self.assertEqual(len(mail.outbox), 0) # No email sent + def test_report_multiple_objects(self): # Make spammy objects sound = Sound.objects.first() sound.add_comment(self.spammer, 'This is a spammy comment') comment = self.spammer.comment_set.first() - thread = Thread.objects.create(author=self.spammer, title="Span thread", - forum=Forum.objects.create(name="Test forum")) + thread = Thread.objects.create( + author=self.spammer, title="Span thread", forum=Forum.objects.create(name="Test forum") + ) post = Post.objects.create(author=self.spammer, thread=thread, body="Spam post post body") - message = Message.objects.create(user_from=self.spammer, user_to=self.reporters[0], subject='Spam subject', - body=MessageBody.objects.create(body='Spam body'), is_sent=True, - is_archived=False, is_read=False) + message = Message.objects.create( + user_from=self.spammer, + user_to=self.reporters[0], + subject='Spam subject', + body=MessageBody.objects.create(body='Spam body'), + is_sent=True, + is_archived=False, + is_read=False + ) objects_flag_types = [ (comment, 'SC'), (post, 'FP'), @@ -173,22 +203,26 @@ def test_report_multiple_objects(self): for i in range(0, settings.USERFLAG_THRESHOLD_FOR_AUTOMATIC_BLOCKING + 1): reporter = self.get_reporter_as_logged_in_user(i) object, flag_type = objects_flag_types[i % len(objects_flag_types)] - resp = self.client.post(reverse('flag-user', kwargs={'username': self.spammer.username}), data={ - 'object_id': object.id, - 'flag_type': flag_type, - }) + resp = self.client.post( + reverse('flag-user', kwargs={'username': self.spammer.username}), + data={ + 'object_id': object.id, + 'flag_type': flag_type, + } + ) self.assertEqual(resp.status_code, 200) self.assertEqual(UserFlag.objects.count(), i + 1) - self.assertEqual(UserFlag.objects.all().order_by('id')[i].reporting_user, - reporter) # Flag object created by reporter + self.assertEqual( + UserFlag.objects.all().order_by('id')[i].reporting_user, reporter + ) # Flag object created by reporter - if i == settings.USERFLAG_THRESHOLD_FOR_NOTIFICATION - 1: # Last iteration - self.assertEqual(len(mail.outbox), 1) # New notification email sent + if i == settings.USERFLAG_THRESHOLD_FOR_NOTIFICATION - 1: # Last iteration + self.assertEqual(len(mail.outbox), 1) # New notification email sent self.assertTrue(settings.EMAIL_SUBJECT_PREFIX in mail.outbox[0].subject) self.assertTrue(settings.EMAIL_SUBJECT_USER_SPAM_REPORT in mail.outbox[0].subject) self.assertTrue("has been reported" in mail.outbox[0].body) elif i == settings.USERFLAG_THRESHOLD_FOR_AUTOMATIC_BLOCKING - 1: - self.assertEqual(len(mail.outbox), 2) # New notification email sent + self.assertEqual(len(mail.outbox), 2) # New notification email sent self.assertTrue(settings.EMAIL_SUBJECT_PREFIX in mail.outbox[1].subject) self.assertTrue(settings.EMAIL_SUBJECT_USER_SPAM_REPORT in mail.outbox[1].subject) self.assertTrue("has been blocked" in mail.outbox[1].body) diff --git a/accounts/tests/test_upload.py b/accounts/tests/test_upload.py index 81490d2d0..f348cbd4d 100644 --- a/accounts/tests/test_upload.py +++ b/accounts/tests/test_upload.py @@ -73,36 +73,62 @@ def test_select_uploaded_files_to_describe(self): # Check that files are displayed in the template resp = self.client.get(reverse('accounts-manage-sounds', args=['pending_description'])) self.assertEqual(resp.status_code, 200) - self.assertListEqual(sorted([os.path.basename(f.full_path) for f in resp.context['file_structure'].children]), sorted(filenames)) + self.assertListEqual( + sorted([os.path.basename(f.full_path) for f in resp.context['file_structure'].children]), sorted(filenames) + ) # Selecting one file redirects to /home/describe/sounds/ sounds_to_describe_idx = [0] - resp = self.client.post(reverse('accounts-manage-sounds', args=['pending_description']), { - 'describe': 'describe', - 'sound-files': [f'file{idx}' for idx in sounds_to_describe_idx], # Note this is not the filename but the value of the "select" option - }) + resp = self.client.post( + reverse('accounts-manage-sounds', args=['pending_description']), + { + 'describe': 'describe', + 'sound-files': [f'file{idx}' for idx in sounds_to_describe_idx + ], # Note this is not the filename but the value of the "select" option + } + ) sesison_key_prefix = resp.url.split('session=')[1] self.assertRedirects(resp, reverse('accounts-describe-sounds') + f'?session={sesison_key_prefix}') - self.assertEqual(self.client.session[f'{sesison_key_prefix}-len_original_describe_sounds'], len(sounds_to_describe_idx)) - self.assertListEqual(sorted([os.path.basename(f.full_path) for f in self.client.session[f'{sesison_key_prefix}-describe_sounds']]), sorted([filenames[idx] for idx in sounds_to_describe_idx])) - + self.assertEqual( + self.client.session[f'{sesison_key_prefix}-len_original_describe_sounds'], len(sounds_to_describe_idx) + ) + self.assertListEqual( + sorted([ + os.path.basename(f.full_path) for f in self.client.session[f'{sesison_key_prefix}-describe_sounds'] + ]), sorted([filenames[idx] for idx in sounds_to_describe_idx]) + ) + # Selecting multiple file redirects to /home/describe/license/ sounds_to_describe_idx = [1, 2, 3] - resp = self.client.post(reverse('accounts-manage-sounds', args=['pending_description']), { - 'describe': 'describe', - 'sound-files': [f'file{idx}' for idx in sounds_to_describe_idx], # Note this is not the filename but the value of the "select" option - }) + resp = self.client.post( + reverse('accounts-manage-sounds', args=['pending_description']), + { + 'describe': 'describe', + 'sound-files': [f'file{idx}' for idx in sounds_to_describe_idx + ], # Note this is not the filename but the value of the "select" option + } + ) sesison_key_prefix = resp.url.split('session=')[1] self.assertRedirects(resp, reverse('accounts-describe-license') + f'?session={sesison_key_prefix}') - self.assertEqual(self.client.session[f'{sesison_key_prefix}-len_original_describe_sounds'], len(sounds_to_describe_idx)) - self.assertListEqual(sorted([os.path.basename(f.full_path) for f in self.client.session[f'{sesison_key_prefix}-describe_sounds']]), sorted([filenames[idx] for idx in sounds_to_describe_idx])) - + self.assertEqual( + self.client.session[f'{sesison_key_prefix}-len_original_describe_sounds'], len(sounds_to_describe_idx) + ) + self.assertListEqual( + sorted([ + os.path.basename(f.full_path) for f in self.client.session[f'{sesison_key_prefix}-describe_sounds'] + ]), sorted([filenames[idx] for idx in sounds_to_describe_idx]) + ) + # Selecting files to delete, deletes the files sounds_to_delete_idx = [1, 2, 3] - resp = self.client.post(reverse('accounts-manage-sounds', args=['pending_description']), { - 'delete_confirm': 'delete_confirm', - 'sound-files': [f'file{idx}' for idx in sounds_to_delete_idx], # Note this is not the filename but the value of the "select" option, - }) + resp = self.client.post( + reverse('accounts-manage-sounds', args=['pending_description']), + { + 'delete_confirm': 'delete_confirm', + 'sound-files': [f'file{idx}' for idx in sounds_to_delete_idx + ], # Note this is not the filename but the value of the "select" option, + } + ) self.assertRedirects(resp, reverse('accounts-manage-sounds', args=['pending_description'])) self.assertEqual(len(os.listdir(user_upload_path)), len(filenames) - len(sounds_to_delete_idx)) @@ -117,8 +143,10 @@ def test_describe_selected_files(self): os.makedirs(user_upload_path, exist_ok=True) create_test_files(filenames, user_upload_path) _, _, sound_sources = create_user_and_sounds( - num_sounds=3, num_packs=0, - user=User.objects.create_user("testuser2", email="2@xmpl.com", password="testpass")) # These sounds will be used as sources for an uploaded sound + num_sounds=3, + num_packs=0, + user=User.objects.create_user("testuser2", email="2@xmpl.com", password="testpass") + ) # These sounds will be used as sources for an uploaded sound # Set license and pack data in session session = self.client.session @@ -126,44 +154,51 @@ def test_describe_selected_files(self): session[f'{session_key_prefix}-describe_license'] = License.objects.all()[0] session[f'{session_key_prefix}-describe_pack'] = False session[f'{session_key_prefix}-len_original_describe_sounds'] = 2 - session[f'{session_key_prefix}-describe_sounds'] = [File(1, filenames[0], user_upload_path + filenames[0], False), - File(2, filenames[1], user_upload_path + filenames[1], False)] + session[f'{session_key_prefix}-describe_sounds'] = [ + File(1, filenames[0], user_upload_path + filenames[0], False), + File(2, filenames[1], user_upload_path + filenames[1], False) + ] session.save() # Post description information - resp = self.client.post(f'/home/describe/sounds/?session={session_key_prefix}', { - '0-audio_filename': filenames[0], - '0-lat': '46.31658418182218', - '0-lon': '3.515625', - '0-zoom': '16', - '0-tags': 'testtag1 testtag2 testtag3', - '0-pack': PackForm.NO_PACK_CHOICE_VALUE, - '0-license': '3', - '0-description': 'a test description for the sound file', - '0-new_pack': '', - '0-name': filenames[0], - '1-audio_filename': filenames[1], - '1-license': '3', - '1-description': 'another test description', - '1-lat': '', - '1-pack': PackForm.NO_PACK_CHOICE_VALUE, - '1-lon': '', - '1-name': filenames[1], - '1-new_pack': 'Name of a new pack', - '1-zoom': '', - '1-tags': 'testtag1 testtag4 testtag5', - '1-sources': ','.join([f'{s.id}' for s in sound_sources]), - }, follow=True) - + resp = self.client.post( + f'/home/describe/sounds/?session={session_key_prefix}', { + '0-audio_filename': filenames[0], + '0-lat': '46.31658418182218', + '0-lon': '3.515625', + '0-zoom': '16', + '0-tags': 'testtag1 testtag2 testtag3', + '0-pack': PackForm.NO_PACK_CHOICE_VALUE, + '0-license': '3', + '0-description': 'a test description for the sound file', + '0-new_pack': '', + '0-name': filenames[0], + '1-audio_filename': filenames[1], + '1-license': '3', + '1-description': 'another test description', + '1-lat': '', + '1-pack': PackForm.NO_PACK_CHOICE_VALUE, + '1-lon': '', + '1-name': filenames[1], + '1-new_pack': 'Name of a new pack', + '1-zoom': '', + '1-tags': 'testtag1 testtag4 testtag5', + '1-sources': ','.join([f'{s.id}' for s in sound_sources]), + }, + follow=True + ) + # Check that post redirected to first describe page with confirmation message on sounds described self.assertRedirects(resp, '/home/sounds/manage/processing/') - self.assertEqual('Successfully finished sound description round' in list(resp.context['messages'])[2].message, True) + self.assertEqual( + 'Successfully finished sound description round' in list(resp.context['messages'])[2].message, True + ) # Check that sounds have been created along with related tags, geotags and packs self.assertEqual(user.sounds.all().count(), 2) self.assertListEqual( - sorted(list(user.sounds.values_list('original_filename', flat=True))), - sorted([f for f in filenames])) + sorted(list(user.sounds.values_list('original_filename', flat=True))), sorted([f for f in filenames]) + ) self.assertEqual(Pack.objects.filter(name='Name of a new pack').exists(), True) self.assertEqual(Tag.objects.filter(name__contains="testtag").count(), 5) self.assertNotEqual(user.sounds.get(original_filename=filenames[0]).geotag, None) @@ -226,18 +261,17 @@ def test_bulk_describe_view_permissions(self): bulk = BulkUploadProgress.objects.create(progress_type="N", user=user, original_csv_filename="test.csv") resp = self.client.get(reverse('accounts-bulk-describe', args=[bulk.id])) - expected_redirect_url = reverse('login') + '?next=%s' % reverse('accounts-bulk-describe', - args=[bulk.id]) - self.assertRedirects(resp, expected_redirect_url) # If user not logged in, redirect to login page + expected_redirect_url = reverse('login') + '?next=%s' % reverse('accounts-bulk-describe', args=[bulk.id]) + self.assertRedirects(resp, expected_redirect_url) # If user not logged in, redirect to login page self.client.force_login(user) resp = self.client.get(reverse('accounts-bulk-describe', args=[bulk.id])) - self.assertEqual(resp.status_code, 200) # After login, page loads normally (200 OK) + self.assertEqual(resp.status_code, 200) # After login, page loads normally (200 OK) User.objects.create_user("testuser2", password="testpass", email='another_email@example.com') self.client.login(username='testuser2', password='testpass') resp = self.client.get(reverse('accounts-bulk-describe', args=[bulk.id])) - self.assertEqual(resp.status_code, 404) # User without permission (not owner of object) gets 404 + self.assertEqual(resp.status_code, 404) # User without permission (not owner of object) gets 404 with self.settings(BULK_UPLOAD_MIN_SOUNDS=10): # Now user is not allowed to load the page as user.profile.can_do_bulk_upload() returns False @@ -267,7 +301,9 @@ def test_bulk_describe_state_finished_validation(self, submit_job): # Test that chosing option to delete existing BulkUploadProgress really does it resp = self.client.post(reverse('accounts-bulk-describe', args=[bulk.id]), data={'delete': True}) - self.assertRedirects(resp, reverse('accounts-manage-sounds', args=['pending_description'])) # Redirects to describe page after delete + self.assertRedirects( + resp, reverse('accounts-manage-sounds', args=['pending_description']) + ) # Redirects to describe page after delete self.assertEqual(BulkUploadProgress.objects.filter(user=user).count(), 0) # Test that chosing option to start describing files triggers bulk describe gearmnan job @@ -295,12 +331,13 @@ def test_bulk_describe_state_description_in_progress(self): # show that info to the users. First we fake some data for the bulk object bulk.progress_type = 'F' bulk.validation_output = { - 'lines_ok': list(range(5)), # NOTE: we only use the length of these lists, so we fill them with irrelevant data + 'lines_ok': list(range(5) + ), # NOTE: we only use the length of these lists, so we fill them with irrelevant data 'lines_with_errors': list(range(2)), 'global_errors': [], } bulk.description_output = { - '1': 1, # NOTE: we only use the length of the dict so we fill it with irrelevant values/keys + '1': 1, # NOTE: we only use the length of the dict so we fill it with irrelevant values/keys '2': 2, '3': 3, } @@ -309,18 +346,20 @@ def test_bulk_describe_state_description_in_progress(self): self.assertContains(resp, 'Your sounds are being described and processed') # Test that when both description and processing have finished we show correct info to users - for i in range(0, 5): # First create the sound objects so BulkUploadProgress can properly compute progress - Sound.objects.create(user=user, - original_filename="Test sound %i" % i, - license=License.objects.all()[0], - md5="fakemd5%i" % i, - moderation_state="OK", - processing_state="OK") + for i in range(0, 5): # First create the sound objects so BulkUploadProgress can properly compute progress + Sound.objects.create( + user=user, + original_filename="Test sound %i" % i, + license=License.objects.all()[0], + md5="fakemd5%i" % i, + moderation_state="OK", + processing_state="OK" + ) bulk.progress_type = 'F' bulk.description_output = {} for count, sound in enumerate(user.sounds.all()): - bulk.description_output[count] = sound.id # Fill bulk.description_output with real sound IDs + bulk.description_output[count] = sound.id # Fill bulk.description_output with real sound IDs bulk.save() resp = self.client.get(reverse('accounts-bulk-describe', args=[bulk.id])) self.assertContains(resp, 'The bulk description process has finished!') diff --git a/accounts/tests/test_user.py b/accounts/tests/test_user.py index bb81c6957..b414f24cd 100644 --- a/accounts/tests/test_user.py +++ b/accounts/tests/test_user.py @@ -49,117 +49,141 @@ class UserRegistrationAndActivation(TestCase): def test_user_save(self): u = User.objects.create_user("testuser2", password="testpass") self.assertEqual(Profile.objects.filter(user=u).exists(), True) - u.save() # Check saving user again (with existing profile) does not fail + u.save() # Check saving user again (with existing profile) does not fail @mock.patch("captcha.fields.ReCaptchaField.validate") def test_user_registration(self, magic_mock_function): username = 'new_user' # Try registration without accepting tos - resp = self.client.post(reverse('accounts-registration-modal'), data={ - 'username': [username], - 'password1': ['123456!@'], - 'accepted_tos': [''], - 'email1': ['example@email.com'], - 'email2': ['example@email.com'] - }) + resp = self.client.post( + reverse('accounts-registration-modal'), + data={ + 'username': [username], + 'password1': ['123456!@'], + 'accepted_tos': [''], + 'email1': ['example@email.com'], + 'email2': ['example@email.com'] + } + ) self.assertEqual(resp.status_code, 200) self.assertContains(resp, 'You must accept the terms of use') self.assertEqual(User.objects.filter(username=username).count(), 0) - self.assertEqual(len(mail.outbox), 0) # No email sent + self.assertEqual(len(mail.outbox), 0) # No email sent # Try registration with bad email - resp = self.client.post(reverse('accounts-registration-modal'), data={ - 'username': [username], - 'password1': ['12345678'], - 'accepted_tos': ['on'], - 'email1': ['example@email.com'], - 'email2': ['example@email.com'] - }) + resp = self.client.post( + reverse('accounts-registration-modal'), + data={ + 'username': [username], + 'password1': ['12345678'], + 'accepted_tos': ['on'], + 'email1': ['example@email.com'], + 'email2': ['example@email.com'] + } + ) self.assertEqual(resp.status_code, 200) self.assertContains(resp, 'This password is entirely numeric') self.assertEqual(User.objects.filter(username=username).count(), 0) - self.assertEqual(len(mail.outbox), 0) # No email sent + self.assertEqual(len(mail.outbox), 0) # No email sent # Try registration with bad password - resp = self.client.post(reverse('accounts-registration-modal'), data={ - 'username': [username], - 'password1': ['123456!@'], - 'accepted_tos': ['on'], - 'email1': ['exampleemail.com'], - 'email2': ['exampleemail.com'] - }) + resp = self.client.post( + reverse('accounts-registration-modal'), + data={ + 'username': [username], + 'password1': ['123456!@'], + 'accepted_tos': ['on'], + 'email1': ['exampleemail.com'], + 'email2': ['exampleemail.com'] + } + ) self.assertEqual(resp.status_code, 200) self.assertContains(resp, 'Enter a valid email') self.assertEqual(User.objects.filter(username=username).count(), 0) - self.assertEqual(len(mail.outbox), 0) # No email sent + self.assertEqual(len(mail.outbox), 0) # No email sent # Try registration with no username - resp = self.client.post(reverse('accounts-registration-modal'), data={ - 'username': [''], - 'password1': ['123456!@'], - 'accepted_tos': ['on'], - 'email1': ['example@email.com'], - 'email2': ['example@email.com'] - }) + resp = self.client.post( + reverse('accounts-registration-modal'), + data={ + 'username': [''], + 'password1': ['123456!@'], + 'accepted_tos': ['on'], + 'email1': ['example@email.com'], + 'email2': ['example@email.com'] + } + ) self.assertEqual(resp.status_code, 200) self.assertContains(resp, 'This field is required') self.assertEqual(User.objects.filter(username=username).count(), 0) - self.assertEqual(len(mail.outbox), 0) # No email sent + self.assertEqual(len(mail.outbox), 0) # No email sent # Try registration with different email addresses - resp = self.client.post(reverse('accounts-registration-modal'), data={ - 'username': [''], - 'password1': ['123456!@'], - 'accepted_tos': ['on'], - 'email1': ['example@email.com'], - 'email2': ['exampl@email.net'] - }) + resp = self.client.post( + reverse('accounts-registration-modal'), + data={ + 'username': [''], + 'password1': ['123456!@'], + 'accepted_tos': ['on'], + 'email1': ['example@email.com'], + 'email2': ['exampl@email.net'] + } + ) self.assertEqual(resp.status_code, 200) self.assertContains(resp, 'Please confirm that your email address is the same') self.assertEqual(User.objects.filter(username=username).count(), 0) - self.assertEqual(len(mail.outbox), 0) # No email sent + self.assertEqual(len(mail.outbox), 0) # No email sent # Try successful registration - resp = self.client.post(reverse('accounts-registration-modal'), data={ - 'username': [username], - 'password1': ['123456!@'], - 'accepted_tos': ['on'], - 'email1': ['example@email.com'], - 'email2': ['example@email.com'] - }) + resp = self.client.post( + reverse('accounts-registration-modal'), + data={ + 'username': [username], + 'password1': ['123456!@'], + 'accepted_tos': ['on'], + 'email1': ['example@email.com'], + 'email2': ['example@email.com'] + } + ) self.assertEqual(resp.status_code, 200) self.assertContains(resp, 'feedbackRegistration=1') self.assertEqual(User.objects.filter(username=username).count(), 1) - self.assertEqual(len(mail.outbox), 1) # An email was sent! + self.assertEqual(len(mail.outbox), 1) # An email was sent! self.assertTrue(settings.EMAIL_SUBJECT_PREFIX in mail.outbox[0].subject) self.assertTrue(settings.EMAIL_SUBJECT_ACTIVATION_LINK in mail.outbox[0].subject) # Try register again with same username - resp = self.client.post(reverse('accounts-registration-modal'), data={ - 'username': [username], - 'password1': ['123456!@'], - 'accepted_tos': ['on'], - 'email1': ['example@email.com'], - 'email2': ['example@email.com'] - }) + resp = self.client.post( + reverse('accounts-registration-modal'), + data={ + 'username': [username], + 'password1': ['123456!@'], + 'accepted_tos': ['on'], + 'email1': ['example@email.com'], + 'email2': ['example@email.com'] + } + ) self.assertEqual(resp.status_code, 200) self.assertContains(resp, 'You cannot use this username to create an account') self.assertEqual(User.objects.filter(username=username).count(), 1) - self.assertEqual(len(mail.outbox), 1) # No new email sent + self.assertEqual(len(mail.outbox), 1) # No new email sent # Try with repeated email address - resp = self.client.post(reverse('accounts-registration-modal'), data={ - 'username': ['a_different_username'], - 'password1': ['123456!@'], - 'accepted_tos': ['on'], - 'email1': ['example@email.com'], - 'email2': ['example@email.com'] - }) + resp = self.client.post( + reverse('accounts-registration-modal'), + data={ + 'username': ['a_different_username'], + 'password1': ['123456!@'], + 'accepted_tos': ['on'], + 'email1': ['example@email.com'], + 'email2': ['example@email.com'] + } + ) self.assertEqual(resp.status_code, 200) self.assertContains(resp, 'You cannot use this email address to create an account') self.assertEqual(User.objects.filter(username=username).count(), 1) - self.assertEqual(len(mail.outbox), 1) # No new email sent + self.assertEqual(len(mail.outbox), 1) # No new email sent activation_code = re.search(r"home/activate/.+/(.+)/", mail.outbox[0].body).group(1) # Test calling accounts-activate with good hash, user should be activated @@ -169,7 +193,7 @@ def test_user_registration(self, magic_mock_function): self.assertEqual(User.objects.get(username=username).is_active, True) def test_user_activation_fails(self): - user = User.objects.get(username="User6Inactive") # Inactive user in fixture + user = User.objects.get(username="User6Inactive") # Inactive user in fixture # Test calling accounts-activate with wrong hash, user should not be activated bad_hash = '4dad3dft' @@ -195,7 +219,7 @@ def create_user_and_content(self, username="testuser", is_index_dirty=True): target_sound = Sound.objects.all()[0] for i in range(0, 3): target_sound.add_comment(user, f"{username} comment {i}") - # Create threads and posts + # Create threads and posts forum, _ = Forum.objects.get_or_create(name="Test forum") self.forum = forum thread = Thread.objects.create(author=user, title=f"Test thread by {username}", forum=forum) @@ -204,14 +228,16 @@ def create_user_and_content(self, username="testuser", is_index_dirty=True): # Create sounds and packs pack = Pack.objects.create(user=user, name=f"Test pack by {username}") for i in range(0, 3): - Sound.objects.create(user=user, - original_filename=f"Test sound {i} by {username}", - pack=pack, - is_index_dirty=is_index_dirty, - license=License.objects.all()[0], - md5=f"fake_unique_md5_{i}_{username}", - moderation_state="OK", - processing_state="OK") + Sound.objects.create( + user=user, + original_filename=f"Test sound {i} by {username}", + pack=pack, + is_index_dirty=is_index_dirty, + license=License.objects.all()[0], + md5=f"fake_unique_md5_{i}_{username}", + moderation_state="OK", + processing_state="OK" + ) return user def test_user_delete_make_invalid_password(self): @@ -324,7 +350,6 @@ def test_user_full_delete(self, delete_sounds_from_search_engine, delete_sound_f delete_sounds_from_search_engine.assert_has_calls(calls, any_order=True) delete_sound_from_gaia.assert_has_calls(calls, any_order=True) - @mock.patch('general.tasks.delete_user.delay') def test_user_delete_include_sounds_using_web_form(self, submit_job): # Test user's option to delete user account including the sounds @@ -335,19 +360,32 @@ def test_user_delete_include_sounds_using_web_form(self, submit_job): encr_link = form.initial['encrypted_link'] resp = self.client.post( reverse('accounts-delete'), - {'encrypted_link': encr_link, 'password': 'testpass', 'delete_sounds': 'delete_sounds'}, + { + 'encrypted_link': encr_link, + 'password': 'testpass', + 'delete_sounds': 'delete_sounds' + }, follow=True, ) # Test job is triggered - data = json.dumps({'user_id': user.id, 'action': DELETE_USER_DELETE_SOUNDS_ACTION_NAME, - 'deletion_reason': DeletedUser.DELETION_REASON_SELF_DELETED}) - submit_job.assert_called_once_with(user_id=user.id, deletion_action=DELETE_USER_DELETE_SOUNDS_ACTION_NAME, deletion_reason=DeletedUser.DELETION_REASON_SELF_DELETED) + data = json.dumps({ + 'user_id': user.id, + 'action': DELETE_USER_DELETE_SOUNDS_ACTION_NAME, + 'deletion_reason': DeletedUser.DELETION_REASON_SELF_DELETED + }) + submit_job.assert_called_once_with( + user_id=user.id, + deletion_action=DELETE_USER_DELETE_SOUNDS_ACTION_NAME, + deletion_reason=DeletedUser.DELETION_REASON_SELF_DELETED + ) # Test UserDeletionRequest object is created with status "tr" (Deletion action was triggered) - self.assertTrue(UserDeletionRequest.objects.filter( - user_from=user, user_to=user, - status=UserDeletionRequest.DELETION_REQUEST_STATUS_DELETION_TRIGGERED).exists()) + self.assertTrue( + UserDeletionRequest.objects.filter( + user_from=user, user_to=user, status=UserDeletionRequest.DELETION_REQUEST_STATUS_DELETION_TRIGGERED + ).exists() + ) # Assert user is redirected to front page self.assertRedirects(resp, reverse('front-page')) @@ -365,19 +403,32 @@ def test_user_delete_keep_sounds_using_web_form(self, submit_job): encr_link = form.initial['encrypted_link'] resp = self.client.post( reverse('accounts-delete'), - {'encrypted_link': encr_link, 'password': 'testpass', 'delete_sounds': 'only_user'}, + { + 'encrypted_link': encr_link, + 'password': 'testpass', + 'delete_sounds': 'only_user' + }, follow=True, ) # Test job is triggered - data = json.dumps({'user_id': user.id, 'action': DELETE_USER_KEEP_SOUNDS_ACTION_NAME, - 'deletion_reason': DeletedUser.DELETION_REASON_SELF_DELETED}) - submit_job.assert_called_once_with(user_id=user.id, deletion_action=DELETE_USER_KEEP_SOUNDS_ACTION_NAME, deletion_reason=DeletedUser.DELETION_REASON_SELF_DELETED) - + data = json.dumps({ + 'user_id': user.id, + 'action': DELETE_USER_KEEP_SOUNDS_ACTION_NAME, + 'deletion_reason': DeletedUser.DELETION_REASON_SELF_DELETED + }) + submit_job.assert_called_once_with( + user_id=user.id, + deletion_action=DELETE_USER_KEEP_SOUNDS_ACTION_NAME, + deletion_reason=DeletedUser.DELETION_REASON_SELF_DELETED + ) + # Test UserDeletionRequest object is created with status "tr" (Deletion action was triggered) - self.assertTrue(UserDeletionRequest.objects.filter( - user_from=user, user_to=user, - status=UserDeletionRequest.DELETION_REQUEST_STATUS_DELETION_TRIGGERED).exists()) + self.assertTrue( + UserDeletionRequest.objects.filter( + user_from=user, user_to=user, status=UserDeletionRequest.DELETION_REQUEST_STATUS_DELETION_TRIGGERED + ).exists() + ) # Assert user is redirected to front page self.assertRedirects(resp, reverse('front-page')) @@ -394,8 +445,12 @@ def test_fail_user_delete_include_sounds_using_web_form(self): form = DeleteUserForm(user_id=user.id) encr_link = form.initial['encrypted_link'] resp = self.client.post( - reverse('accounts-delete'), - {'encrypted_link': encr_link, 'password': 'wrong_pass', 'delete_sounds': 'delete_sounds'}) + reverse('accounts-delete'), { + 'encrypted_link': encr_link, + 'password': 'wrong_pass', + 'delete_sounds': 'delete_sounds' + } + ) # Check user is reported incorrect password self.assertContains(resp, 'Incorrect password') @@ -411,8 +466,7 @@ def test_fail_user_delete_include_sounds_using_web_form(self): def test_delete_user_reasons(self): # Tests that the reasons passed to Profile.delete_user() are effectively added to the created DeletedUser object - for reason in [DeletedUser.DELETION_REASON_SELF_DELETED, - DeletedUser.DELETION_REASON_DELETED_BY_ADMIN, + for reason in [DeletedUser.DELETION_REASON_SELF_DELETED, DeletedUser.DELETION_REASON_DELETED_BY_ADMIN, DeletedUser.DELETION_REASON_SPAMMER]: user = User.objects.create_user("testuser", password="testpass", email='email@freesound.org') user.profile.delete_user(deletion_reason=reason) @@ -421,8 +475,9 @@ def test_delete_user_reasons(self): @mock.patch('sounds.models.delete_sound_from_gaia') @mock.patch('sounds.models.delete_sounds_from_search_engine') @mock.patch('forum.models.delete_posts_from_search_engine') - def test_delete_user_with_count_fields_out_of_sync(self, delete_posts_from_search_engine, delete_sounds_from_search_engine, - delete_sound_from_gaia): + def test_delete_user_with_count_fields_out_of_sync( + self, delete_posts_from_search_engine, delete_sounds_from_search_engine, delete_sound_from_gaia + ): # Test that deleting a user work properly even when the profile count fields (num_sounds, num_posts, # num_sound_downloads and num_pack_downloads) are out of sync. This is a potential issue because if the # count fields are not right, deleting sound/download/post objects related to that user might trigger @@ -471,8 +526,12 @@ def delete_user_using_web_view(user): form = DeleteUserForm(user_id=user.id) encr_link = form.initial['encrypted_link'] resp = self.client.post( - reverse('accounts-delete'), - {'encrypted_link': encr_link, 'password': 'testpass', 'delete_sounds': 'only_user'}) + reverse('accounts-delete'), { + 'encrypted_link': encr_link, + 'password': 'testpass', + 'delete_sounds': 'only_user' + } + ) self.assertRedirects(resp, reverse('front-page')) def call_command_get_console_log_output(command): @@ -517,8 +576,10 @@ def call_command_get_console_log_output(command): user2.profile.delete_user() user2.refresh_from_db() self.assertEqual(user2.profile.is_anonymized_user, True) - self.assertEqual(UserDeletionRequest.objects.get(user_to=user2).status, - UserDeletionRequest.DELETION_REQUEST_STATUS_USER_WAS_DELETED) + self.assertEqual( + UserDeletionRequest.objects.get(user_to=user2).status, + UserDeletionRequest.DELETION_REQUEST_STATUS_USER_WAS_DELETED + ) # Run the command again, it should say that there is 1 user that should have been deleted cmd_output = call_command_get_console_log_output("check_async_deleted_users") @@ -532,8 +593,10 @@ def call_command_get_console_log_output(command): # have updated the UserDeletionRequest status for user 1 cmd_output = call_command_get_console_log_output("check_async_deleted_users") self.assertIn('Found 0 users that should have been deleted and were not', cmd_output) - self.assertEqual(UserDeletionRequest.objects.get(user_to=user1).status, - UserDeletionRequest.DELETION_REQUEST_STATUS_USER_WAS_DELETED) + self.assertEqual( + UserDeletionRequest.objects.get(user_to=user1).status, + UserDeletionRequest.DELETION_REQUEST_STATUS_USER_WAS_DELETED + ) class UserDeletionRequestTestCase(TestCase): @@ -583,39 +646,47 @@ def test_user_deletion_request_update_status_history(self): deletion_request = UserDeletionRequest.objects.create(user_to=user, username_to=username, status="re") self.assertEqual(deletion_request.status, UserDeletionRequest.DELETION_REQUEST_STATUS_RECEIVED_REQUEST) self.assertEqual(len(deletion_request.status_history), 1) - self.assertTrue(UserDeletionRequest.DELETION_REQUEST_STATUS_RECEIVED_REQUEST in - deletion_request.status_history[0]) + self.assertTrue( + UserDeletionRequest.DELETION_REQUEST_STATUS_RECEIVED_REQUEST in deletion_request.status_history[0] + ) # Change status: status history should be updated deletion_request.status = UserDeletionRequest.DELETION_REQUEST_STATUS_WAITING_FOR_USER deletion_request.save() self.assertEqual(deletion_request.status, UserDeletionRequest.DELETION_REQUEST_STATUS_WAITING_FOR_USER) self.assertEqual(len(deletion_request.status_history), 2) - self.assertTrue(UserDeletionRequest.DELETION_REQUEST_STATUS_RECEIVED_REQUEST in - deletion_request.status_history[0]) - self.assertTrue(UserDeletionRequest.DELETION_REQUEST_STATUS_WAITING_FOR_USER in - deletion_request.status_history[1]) + self.assertTrue( + UserDeletionRequest.DELETION_REQUEST_STATUS_RECEIVED_REQUEST in deletion_request.status_history[0] + ) + self.assertTrue( + UserDeletionRequest.DELETION_REQUEST_STATUS_WAITING_FOR_USER in deletion_request.status_history[1] + ) # Now save again, but don't change status: status history should not be updated deletion_request.save() self.assertEqual(deletion_request.status, UserDeletionRequest.DELETION_REQUEST_STATUS_WAITING_FOR_USER) self.assertEqual(len(deletion_request.status_history), 2) - self.assertTrue(UserDeletionRequest.DELETION_REQUEST_STATUS_RECEIVED_REQUEST in - deletion_request.status_history[0]) - self.assertTrue(UserDeletionRequest.DELETION_REQUEST_STATUS_WAITING_FOR_USER in - deletion_request.status_history[1]) + self.assertTrue( + UserDeletionRequest.DELETION_REQUEST_STATUS_RECEIVED_REQUEST in deletion_request.status_history[0] + ) + self.assertTrue( + UserDeletionRequest.DELETION_REQUEST_STATUS_WAITING_FOR_USER in deletion_request.status_history[1] + ) # Now change status again: status history should be updated deletion_request.status = UserDeletionRequest.DELETION_REQUEST_STATUS_USER_WAS_DELETED deletion_request.save() self.assertEqual(deletion_request.status, UserDeletionRequest.DELETION_REQUEST_STATUS_USER_WAS_DELETED) self.assertEqual(len(deletion_request.status_history), 3) - self.assertTrue(UserDeletionRequest.DELETION_REQUEST_STATUS_RECEIVED_REQUEST in - deletion_request.status_history[0]) - self.assertTrue(UserDeletionRequest.DELETION_REQUEST_STATUS_WAITING_FOR_USER in - deletion_request.status_history[1]) - self.assertTrue(UserDeletionRequest.DELETION_REQUEST_STATUS_USER_WAS_DELETED in - deletion_request.status_history[2]) + self.assertTrue( + UserDeletionRequest.DELETION_REQUEST_STATUS_RECEIVED_REQUEST in deletion_request.status_history[0] + ) + self.assertTrue( + UserDeletionRequest.DELETION_REQUEST_STATUS_WAITING_FOR_USER in deletion_request.status_history[1] + ) + self.assertTrue( + UserDeletionRequest.DELETION_REQUEST_STATUS_USER_WAS_DELETED in deletion_request.status_history[2] + ) class UserEmailsUniqueTestCase(TestCase): @@ -624,13 +695,14 @@ def setUp(self): self.user_a = User.objects.create_user("user_a", password="12345", email='a@b.com') self.original_shared_email = 'c@d.com' self.user_b = User.objects.create_user("user_b", password="12345", email=self.original_shared_email) - self.user_c = User.objects.create_user("user_c", password="12345", - email=transform_unique_email(self.original_shared_email)) + self.user_c = User.objects.create_user( + "user_c", password="12345", email=transform_unique_email(self.original_shared_email) + ) SameUser.objects.create( main_user=self.user_b, main_orig_email=self.user_b.email, secondary_user=self.user_c, - secondary_orig_email=self.user_b.email, # Must be same email (original) + secondary_orig_email=self.user_b.email, # Must be same email (original) ) # User a never had problems with email # User b and c had the same email, but user_c's was automaitcally changed to avoid duplicates @@ -640,30 +712,54 @@ def test_redirects_when_shared_emails(self): # User a is not in same users table, so redirect should be plain and simple to messages # NOTE: in the following tests we don't use `self.client.login` because what we want to test # is in fact in the login view logic. - resp = self.client.post(reverse('login'), - {'username': self.user_a, 'password': '12345', 'next': reverse('messages')}) + resp = self.client.post( + reverse('login'), { + 'username': self.user_a, + 'password': '12345', + 'next': reverse('messages') + } + ) self.assertRedirects(resp, reverse('messages')) resp = self.client.get(reverse('logout')) # Now try with user_b and user_c. User b had a shared email with user_c. Even if user_b's email was # not changed, he is still redirected to the duplicate email cleanup page - resp = self.client.post(reverse('login'), - {'username': self.user_b, 'password': '12345', 'next': reverse('messages')}) + resp = self.client.post( + reverse('login'), { + 'username': self.user_b, + 'password': '12345', + 'next': reverse('messages') + } + ) self.assertRedirects(resp, reverse('accounts-multi-email-cleanup') + f"?next={reverse('messages')}") resp = self.client.get(reverse('logout')) - resp = self.client.post(reverse('login'), - {'username': self.user_c, 'password': '12345', 'next': reverse('messages')}) + resp = self.client.post( + reverse('login'), { + 'username': self.user_c, + 'password': '12345', + 'next': reverse('messages') + } + ) self.assertRedirects(resp, reverse('accounts-multi-email-cleanup') + f"?next={reverse('messages')}") def test_fix_email_issues_with_secondary_user_email_change(self): # user_c changes his email and tries to login, redirect should go to email cleanup page and from there # directly to messages (2 redirect steps) - self.user_c.email = 'new@email.com' # Must be different than transform_unique_email('c@d.com') + self.user_c.email = 'new@email.com' # Must be different than transform_unique_email('c@d.com') self.user_c.save() - resp = self.client.post(reverse('login'), follow=True, - data={'username': self.user_c, 'password': '12345', 'next': reverse('messages')}) - self.assertEqual(resp.redirect_chain[0][0], - reverse('accounts-multi-email-cleanup') + f"?next={reverse('messages')}") + resp = self.client.post( + reverse('login'), + follow=True, + data={ + 'username': self.user_c, + 'password': '12345', + 'next': reverse('messages') + } + ) + self.assertEqual( + resp.redirect_chain[0][0], + reverse('accounts-multi-email-cleanup') + f"?next={reverse('messages')}" + ) self.assertEqual(resp.redirect_chain[1][0], reverse('messages')) # Also check that related SameUser objects have been removed @@ -671,30 +767,49 @@ def test_fix_email_issues_with_secondary_user_email_change(self): resp = self.client.get(reverse('logout')) # Now next time user_c tries to go to messages again, there is only one redirect (like for user_a) - resp = self.client.post(reverse('login'), - {'username': self.user_c, 'password': '12345', 'next': reverse('messages')}) + resp = self.client.post( + reverse('login'), { + 'username': self.user_c, + 'password': '12345', + 'next': reverse('messages') + } + ) self.assertRedirects(resp, reverse('messages')) resp = self.client.get(reverse('logout')) # Also if user_b logs in, redirect goes straight to messages - resp = self.client.post(reverse('login'), - {'username': self.user_b, 'password': '12345', 'next': reverse('messages')}) + resp = self.client.post( + reverse('login'), { + 'username': self.user_b, + 'password': '12345', + 'next': reverse('messages') + } + ) self.assertRedirects(resp, reverse('messages')) def test_fix_email_issues_with_main_user_email_change(self): # user_b changes his email and tries to login, redirect should go to email cleanup page and from there # directly to messages (2 redirect steps). Also user_c email should be changed to the original email of # both users - self.user_b.email = 'new@email.com' # Must be different than transform_unique_email('c@d.com') + self.user_b.email = 'new@email.com' # Must be different than transform_unique_email('c@d.com') self.user_b.save() - resp = self.client.post(reverse('login'), follow=True, - data={'username': self.user_b, 'password': '12345', 'next': reverse('messages')}) - self.assertEqual(resp.redirect_chain[0][0], - reverse('accounts-multi-email-cleanup') + f"?next={reverse('messages')}") + resp = self.client.post( + reverse('login'), + follow=True, + data={ + 'username': self.user_b, + 'password': '12345', + 'next': reverse('messages') + } + ) + self.assertEqual( + resp.redirect_chain[0][0], + reverse('accounts-multi-email-cleanup') + f"?next={reverse('messages')}" + ) self.assertEqual(resp.redirect_chain[1][0], reverse('messages')) # Check that user_c email was changed - self.user_c = User.objects.get(id=self.user_c.id) # Reload user from db + self.user_c = User.objects.get(id=self.user_c.id) # Reload user from db self.assertEqual(self.user_c.email, self.original_shared_email) # Also check that related SameUser objects have been removed @@ -702,14 +817,24 @@ def test_fix_email_issues_with_main_user_email_change(self): resp = self.client.get(reverse('logout')) # Now next time user_b tries to go to messages again, there is only one redirect (like for user_a) - resp = self.client.post(reverse('login'), - {'username': self.user_b, 'password': '12345', 'next': reverse('messages')}) + resp = self.client.post( + reverse('login'), { + 'username': self.user_b, + 'password': '12345', + 'next': reverse('messages') + } + ) self.assertRedirects(resp, reverse('messages')) resp = self.client.get(reverse('logout')) # Also if user_c logs in, redirect goes straight to messages - resp = self.client.post(reverse('login'), - {'username': self.user_c, 'password': '12345', 'next': reverse('messages')}) + resp = self.client.post( + reverse('login'), { + 'username': self.user_c, + 'password': '12345', + 'next': reverse('messages') + } + ) self.assertRedirects(resp, reverse('messages')) def test_fix_email_issues_with_both_users_email_change(self): @@ -719,14 +844,23 @@ def test_fix_email_issues_with_both_users_email_change(self): self.user_b.save() self.user_c.email = 'new2w@email.com' self.user_c.save() - resp = self.client.post(reverse('login'), follow=True, - data={'username': self.user_b, 'password': '12345', 'next': reverse('messages')}) - self.assertEqual(resp.redirect_chain[0][0], - reverse('accounts-multi-email-cleanup') + f"?next={reverse('messages')}") + resp = self.client.post( + reverse('login'), + follow=True, + data={ + 'username': self.user_b, + 'password': '12345', + 'next': reverse('messages') + } + ) + self.assertEqual( + resp.redirect_chain[0][0], + reverse('accounts-multi-email-cleanup') + f"?next={reverse('messages')}" + ) self.assertEqual(resp.redirect_chain[1][0], reverse('messages')) # Check that user_c email was not changed - self.user_c = User.objects.get(id=self.user_c.id) # Reload user from db + self.user_c = User.objects.get(id=self.user_c.id) # Reload user from db self.assertEqual(self.user_c.email, 'new2w@email.com') # Also check that related SameUser objects have been removed @@ -734,14 +868,24 @@ def test_fix_email_issues_with_both_users_email_change(self): resp = self.client.get(reverse('logout')) # Now next time user_b tries to go to messages again, there is only one redirect (like for user_a) - resp = self.client.post(reverse('login'), - {'username': self.user_b, 'password': '12345', 'next': reverse('messages')}) + resp = self.client.post( + reverse('login'), { + 'username': self.user_b, + 'password': '12345', + 'next': reverse('messages') + } + ) self.assertRedirects(resp, reverse('messages')) resp = self.client.get(reverse('logout')) # Also if user_c logs in, redirect goes straight to messages - resp = self.client.post(reverse('login'), - {'username': self.user_c, 'password': '12345', 'next': reverse('messages')}) + resp = self.client.post( + reverse('login'), { + 'username': self.user_c, + 'password': '12345', + 'next': reverse('messages') + } + ) self.assertRedirects(resp, reverse('messages')) def test_user_profile_get_email(self): @@ -762,6 +906,7 @@ def test_user_profile_get_email(self): class PasswordReset(TestCase): + def test_reset_form_get_users(self): """Check that a user with an unknown password hash can reset their password""" @@ -799,16 +944,19 @@ def test_reset_view_with_username(self): class EmailResetTestCase(TestCase): + def test_reset_email_form(self): """ Check that reset email with the right parameters """ user = User.objects.create_user("testuser", email="testuser@freesound.org") user.set_password('12345') user.save() self.client.force_login(user) - resp = self.client.post(reverse('accounts-email-reset'), { - 'email': 'new_email@freesound.org', - 'password': '12345', - }) + resp = self.client.post( + reverse('accounts-email-reset'), { + 'email': 'new_email@freesound.org', + 'password': '12345', + } + ) self.assertRedirects(resp, reverse('accounts-email-reset-done')) self.assertEqual(ResetEmailRequest.objects.filter(user=user, email="new_email@freesound.org").count(), 1) @@ -819,10 +967,12 @@ def test_reset_email_form_existing_email(self): user.set_password('12345') user.save() self.client.force_login(user) - resp = self.client.post(reverse('accounts-email-reset'), { - 'email': 'new_email@freesound.org', - 'password': '12345', - }) + resp = self.client.post( + reverse('accounts-email-reset'), { + 'email': 'new_email@freesound.org', + 'password': '12345', + } + ) self.assertRedirects(resp, reverse('accounts-email-reset-done')) self.assertEqual(ResetEmailRequest.objects.filter(user=user, email="new_email@freesound.org").count(), 0) @@ -842,6 +992,7 @@ def test_reset_long_email(self): class ReSendActivationTestCase(TestCase): + def test_resend_activation_code_from_email(self): """ Check that resend activation code doesn't return an error with post request (use email to identify user) @@ -851,7 +1002,7 @@ def test_resend_activation_code_from_email(self): 'username_or_email': 'testuser@freesound.org', }) self.assertEqual(resp.status_code, 200) - self.assertEqual(len(mail.outbox), 1) # Check email was sent + self.assertEqual(len(mail.outbox), 1) # Check email was sent self.assertTrue(settings.EMAIL_SUBJECT_PREFIX in mail.outbox[0].subject) self.assertTrue(settings.EMAIL_SUBJECT_ACTIVATION_LINK in mail.outbox[0].subject) @@ -859,7 +1010,7 @@ def test_resend_activation_code_from_email(self): 'username_or_email': 'new_email@freesound.org', }) self.assertEqual(resp.status_code, 200) - self.assertEqual(len(mail.outbox), 1) # Check no new email was sent (len() is same as before) + self.assertEqual(len(mail.outbox), 1) # Check no new email was sent (len() is same as before) def test_resend_activation_code_from_username(self): """ @@ -870,7 +1021,7 @@ def test_resend_activation_code_from_username(self): 'username_or_email': 'testuser', }) self.assertEqual(resp.status_code, 200) - self.assertEqual(len(mail.outbox), 1) # Check email was sent + self.assertEqual(len(mail.outbox), 1) # Check email was sent self.assertTrue(settings.EMAIL_SUBJECT_PREFIX in mail.outbox[0].subject) self.assertTrue(settings.EMAIL_SUBJECT_ACTIVATION_LINK in mail.outbox[0].subject) @@ -878,7 +1029,7 @@ def test_resend_activation_code_from_username(self): 'username_or_email': 'testuser_does_not_exist', }) self.assertEqual(resp.status_code, 200) - self.assertEqual(len(mail.outbox), 1) # Check no new email was sent (len() is same as before) + self.assertEqual(len(mail.outbox), 1) # Check no new email was sent (len() is same as before) class ChangeUsernameTest(TestCase): @@ -922,40 +1073,70 @@ def test_change_username_form_profile_page(self): self.client.login(username='userA', password='testpass') # Test save profile without changing username (note we set all mandatory fields) - resp = self.client.post(reverse('accounts-edit'), data={'profile-username': ['userA'], 'profile-ui_theme_preference': 'f'}) + resp = self.client.post( + reverse('accounts-edit'), data={ + 'profile-username': ['userA'], + 'profile-ui_theme_preference': 'f' + } + ) self.assertRedirects(resp, reverse('accounts-edit')) self.assertEqual(OldUsername.objects.filter(user=userA).count(), 0) # Try rename user with an existing username from another user userB = User.objects.create_user('userB', email='userB@freesound.org') - resp = self.client.post(reverse('accounts-edit'), data={'profile-username': [userB.username], 'profile-ui_theme_preference': 'f'}) + resp = self.client.post( + reverse('accounts-edit'), data={ + 'profile-username': [userB.username], + 'profile-ui_theme_preference': 'f' + } + ) self.assertEqual(resp.status_code, 200) - self.assertEqual(resp.context['profile_form'].has_error('username'), True) # Error in username field - self.assertIn('This username is already taken or has been in used in the past', - str(resp.context['profile_form']['username'].errors)) + self.assertEqual(resp.context['profile_form'].has_error('username'), True) # Error in username field + self.assertIn( + 'This username is already taken or has been in used in the past', + str(resp.context['profile_form']['username'].errors) + ) userA.refresh_from_db() - self.assertEqual(userA.username, 'userA') # Username has not changed + self.assertEqual(userA.username, 'userA') # Username has not changed self.assertEqual(OldUsername.objects.filter(user=userA).count(), 0) # Now rename user for the first time - resp = self.client.post(reverse('accounts-edit'), data={'profile-username': ['userANewName'], 'profile-ui_theme_preference': 'f'}) + resp = self.client.post( + reverse('accounts-edit'), data={ + 'profile-username': ['userANewName'], + 'profile-ui_theme_preference': 'f' + } + ) self.assertRedirects(resp, reverse('accounts-edit')) userA.refresh_from_db() self.assertEqual(userA.username, 'userANewName') self.assertEqual(OldUsername.objects.filter(user=userA).count(), 1) # Try rename again user with a username that was already used by the same user in the past - resp = self.client.post(reverse('accounts-edit'), data={'profile-username': ['userA'], 'profile-ui_theme_preference': 'f'}) + resp = self.client.post( + reverse('accounts-edit'), data={ + 'profile-username': ['userA'], + 'profile-ui_theme_preference': 'f' + } + ) self.assertEqual(resp.status_code, 200) - self.assertEqual(resp.context['profile_form'].has_error('username'), True) # Error in username field - self.assertIn('This username is already taken or has been in used in the past', - str(resp.context['profile_form']['username'].errors)) + self.assertEqual(resp.context['profile_form'].has_error('username'), True) # Error in username field + self.assertIn( + 'This username is already taken or has been in used in the past', + str(resp.context['profile_form']['username'].errors) + ) userA.refresh_from_db() - self.assertEqual(userA.username, 'userANewName') # Username has not changed + self.assertEqual(userA.username, 'userANewName') # Username has not changed self.assertEqual(OldUsername.objects.filter(user=userA).count(), 1) # Now rename user for the second time - resp = self.client.post(reverse('accounts-edit'), data={'profile-username': ['userANewNewName'], 'profile-ui_theme_preference': 'f'}) + resp = self.client.post( + reverse('accounts-edit'), + data={ + 'profile-username': ['userANewNewName'], + 'profile-ui_theme_preference': 'f' + } + ) self.assertRedirects(resp, reverse('accounts-edit')) userA.refresh_from_db() self.assertEqual(userA.username, 'userANewNewName') @@ -966,11 +1147,19 @@ def test_change_username_form_profile_page(self): # NOTE: when USERNAME_CHANGE_MAX_TIMES is reached, the form renders the "username" field as "disabled" and # therefore the username can't be changed. Other than that the form behaves normally, therefore no # form errors will be raised because the field is ignored - resp = self.client.post(reverse('accounts-edit'), data={'profile-username': ['userANewNewNewName'], 'profile-ui_theme_preference': 'f'}) - self.assertRedirects(resp, reverse('accounts-edit')) # Successful edit redirects to home but... + resp = self.client.post( + reverse('accounts-edit'), + data={ + 'profile-username': ['userANewNewNewName'], + 'profile-ui_theme_preference': 'f' + } + ) + self.assertRedirects(resp, reverse('accounts-edit')) # Successful edit redirects to home but... userA.refresh_from_db() - self.assertEqual(userA.username, 'userANewNewName') # ...username has not changed... - self.assertEqual(OldUsername.objects.filter(user=userA).count(), 2) # ...and no new OldUsername objects created + self.assertEqual(userA.username, 'userANewNewName') # ...username has not changed... + self.assertEqual( + OldUsername.objects.filter(user=userA).count(), 2 + ) # ...and no new OldUsername objects created @override_settings(USERNAME_CHANGE_MAX_TIMES=2) def test_change_username_form_admin(self): @@ -981,25 +1170,28 @@ def test_change_username_form_admin(self): userA = User.objects.create_user('userA', email='userA@freesound.org', password='testpass') admin_change_url = reverse('admin:auth_user_change', args=[userA.id]) - post_data = {'username': 'userA', - 'email': userA.email, # Required to avoid breaking unique constraint with empty email - 'date_joined_0': "2015-10-06", 'date_joined_1': "16:42:00"} # date_joined required + post_data = { + 'username': 'userA', + 'email': userA.email, # Required to avoid breaking unique constraint with empty email + 'date_joined_0': "2015-10-06", + 'date_joined_1': "16:42:00" + } # date_joined required # Test save user without changing username resp = self.client.post(admin_change_url, data=post_data) - self.assertRedirects(resp, reverse('admin:auth_user_changelist')) # Successful edit redirects to users list + self.assertRedirects(resp, reverse('admin:auth_user_changelist')) # Successful edit redirects to users list self.assertEqual(OldUsername.objects.filter(user=userA).count(), 0) # Now rename user for the first time post_data.update({'username': 'userANewName'}) resp = self.client.post(admin_change_url, data=post_data) - self.assertRedirects(resp, reverse('admin:auth_user_changelist')) # Successful edit redirects to users list + self.assertRedirects(resp, reverse('admin:auth_user_changelist')) # Successful edit redirects to users list self.assertEqual(OldUsername.objects.filter(username='userA', user=userA).count(), 1) # Now rename user for the second time post_data.update({'username': 'userANewNewName'}) resp = self.client.post(admin_change_url, data=post_data) - self.assertRedirects(resp, reverse('admin:auth_user_changelist')) # Successful edit redirects to users list + self.assertRedirects(resp, reverse('admin:auth_user_changelist')) # Successful edit redirects to users list self.assertEqual(OldUsername.objects.filter(username='userANewName', user=userA).count(), 1) self.assertEqual(OldUsername.objects.filter(user=userA).count(), 2) @@ -1008,22 +1200,24 @@ def test_change_username_form_admin(self): post_data.update({'username': userB.username}) resp = self.client.post(admin_change_url, data=post_data) self.assertEqual(resp.status_code, 200) - self.assertEqual(bool(resp.context['adminform'].errors), True) # Error in username field - self.assertIn('This username is already taken or has been in used in the past', - str(resp.context['adminform'].errors)) + self.assertEqual(bool(resp.context['adminform'].errors), True) # Error in username field + self.assertIn( + 'This username is already taken or has been in used in the past', str(resp.context['adminform'].errors) + ) userA.refresh_from_db() - self.assertEqual(userA.username, 'userANewNewName') # Username has not changed + self.assertEqual(userA.username, 'userANewNewName') # Username has not changed self.assertEqual(OldUsername.objects.filter(user=userA).count(), 2) # Try rename user with a username that was already used by the same user in the past post_data.update({'username': 'userA'}) resp = self.client.post(admin_change_url, data=post_data) self.assertEqual(resp.status_code, 200) - self.assertEqual(bool(resp.context['adminform'].errors), True) # Error in username field - self.assertIn('This username is already taken or has been in used in the past', - str(resp.context['adminform'].errors)) + self.assertEqual(bool(resp.context['adminform'].errors), True) # Error in username field + self.assertIn( + 'This username is already taken or has been in used in the past', str(resp.context['adminform'].errors) + ) userA.refresh_from_db() - self.assertEqual(userA.username, 'userANewNewName') # Username has not changed + self.assertEqual(userA.username, 'userANewNewName') # Username has not changed self.assertEqual(OldUsername.objects.filter(user=userA).count(), 2) # Try rename user with a username that was already used by the same user in the past, but using different @@ -1031,18 +1225,19 @@ def test_change_username_form_admin(self): post_data.update({'username': 'uSeRA'}) resp = self.client.post(admin_change_url, data=post_data) self.assertEqual(resp.status_code, 200) - self.assertEqual(bool(resp.context['adminform'].errors), True) # Error in username field - self.assertIn('This username is already taken or has been in used in the past', - str(resp.context['adminform'].errors)) + self.assertEqual(bool(resp.context['adminform'].errors), True) # Error in username field + self.assertIn( + 'This username is already taken or has been in used in the past', str(resp.context['adminform'].errors) + ) userA.refresh_from_db() - self.assertEqual(userA.username, 'userANewNewName') # Username has not changed + self.assertEqual(userA.username, 'userANewNewName') # Username has not changed self.assertEqual(OldUsername.objects.filter(user=userA).count(), 2) # Try to rename for a third time to a valid username. Because we are in admin now, the USERNAME_CHANGE_MAX_TIMES # restriction does not apply so rename should work correctly post_data.update({'username': 'userANewNewNewName'}) resp = self.client.post(admin_change_url, data=post_data) - self.assertRedirects(resp, reverse('admin:auth_user_changelist')) # Successful edit redirects to users list + self.assertRedirects(resp, reverse('admin:auth_user_changelist')) # Successful edit redirects to users list self.assertEqual(OldUsername.objects.filter(username='userANewNewName', user=userA).count(), 1) self.assertEqual(OldUsername.objects.filter(user=userA).count(), 3) @@ -1056,11 +1251,16 @@ def test_change_username_case_insensitiveness(self): self.client.login(username='userA', password='testpass') # Rename "userA" to "UserA", should not create OldUsername object - resp = self.client.post(reverse('accounts-edit'), data={'profile-username': ['UserA'], 'profile-ui_theme_preference': 'f'}) + resp = self.client.post( + reverse('accounts-edit'), data={ + 'profile-username': ['UserA'], + 'profile-ui_theme_preference': 'f' + } + ) self.assertRedirects(resp, reverse('accounts-edit')) userA.refresh_from_db() - self.assertEqual(userA.username, 'UserA') # Username capitalization was changed ... - self.assertEqual(OldUsername.objects.filter(user=userA).count(), 0) # ... but not OldUsername was created + self.assertEqual(userA.username, 'UserA') # Username capitalization was changed ... + self.assertEqual(OldUsername.objects.filter(user=userA).count(), 0) # ... but not OldUsername was created def test_oldusername_username_unique_case_insensitiveness(self): """Test that OldUsername.username is case insensitive at the DB level, and that we can't create objects @@ -1081,28 +1281,33 @@ def test_change_email_form_admin(self): userA = User.objects.create_user('userA', email='userA@freesound.org', password='testpass') admin_change_url = reverse('admin:auth_user_change', args=[userA.id]) - # Try to change email to some other (unused email) new_email = 'aNewEmail@freesound.org' - post_data = {'username': userA.username, - 'email': new_email, - 'date_joined_0': "2015-10-06", 'date_joined_1': "16:42:00"} # date_joined required + post_data = { + 'username': userA.username, + 'email': new_email, + 'date_joined_0': "2015-10-06", + 'date_joined_1': "16:42:00" + } # date_joined required resp = self.client.post(admin_change_url, data=post_data) - self.assertRedirects(resp, reverse('admin:auth_user_changelist')) # Successful edit redirects to users list + self.assertRedirects(resp, reverse('admin:auth_user_changelist')) # Successful edit redirects to users list userA.refresh_from_db() self.assertEqual(userA.email, new_email) # Now create another user with a different email, and try to change userA email to the email of the new user userB = User.objects.create_user('userB', email='userBA@freesound.org', password='testpass') - post_data = {'username': userA.username, - 'email': userB.email, - 'date_joined_0': "2015-10-06", 'date_joined_1': "16:42:00"} # date_joined required + post_data = { + 'username': userA.username, + 'email': userB.email, + 'date_joined_0': "2015-10-06", + 'date_joined_1': "16:42:00" + } # date_joined required resp = self.client.post(admin_change_url, data=post_data) self.assertEqual(resp.status_code, 200) - self.assertEqual(bool(resp.context['adminform'].errors), True) # Error in email field + self.assertEqual(bool(resp.context['adminform'].errors), True) # Error in email field self.assertIn('This email is already being used by another user', str(resp.context['adminform'].errors)) userA.refresh_from_db() - self.assertEqual(userA.email, new_email) # Email has not been changed + self.assertEqual(userA.email, new_email) # Email has not been changed class UsernameValidatorTest(TestCase): diff --git a/accounts/tests/test_views.py b/accounts/tests/test_views.py index e29b3e3ed..6b564d3da 100644 --- a/accounts/tests/test_views.py +++ b/accounts/tests/test_views.py @@ -49,13 +49,14 @@ def setUp(self): self.sound.geotag = GeoTag.objects.create(user=user, lat=45.8498, lon=-62.6879, zoom=9) self.sound.save() SoundOfTheDay.objects.create(sound=self.sound, date_display=datetime.date.today()) - self.download = Download.objects.create(user=self.user, sound=self.sound, license=self.sound.license, - created=self.sound.created) + self.download = Download.objects.create( + user=self.user, sound=self.sound, license=self.sound.license, created=self.sound.created + ) self.pack_download = PackDownload.objects.create(user=self.user, pack=self.pack, created=self.pack.created) def test_old_ng_redirects(self): # Test that some pages which used to have its own "URL" in NG now redirect to other pages and open as a modal - + # Comments on user sounds resp = self.client.get(reverse('comments-for-user', kwargs={'username': self.user.username})) self.assertRedirects(resp, reverse('account', args=[self.user.username]) + '?comments=1') @@ -74,12 +75,20 @@ def test_old_ng_redirects(self): # Users that downloaded a sound resp = self.client.get( - reverse('sound-downloaders', kwargs={'username': self.user.username, "sound_id": self.sound.id})) + reverse('sound-downloaders', kwargs={ + 'username': self.user.username, + "sound_id": self.sound.id + }) + ) self.assertRedirects(resp, reverse('sound', args=[self.user.username, self.sound.id]) + '?downloaders=1') # Users that downloaded a pack resp = self.client.get( - reverse('pack-downloaders', kwargs={'username': self.user.username, "pack_id": self.pack.id})) + reverse('pack-downloaders', kwargs={ + 'username': self.user.username, + "pack_id": self.pack.id + }) + ) self.assertRedirects(resp, reverse('pack', args=[self.user.username, self.pack.id]) + '?downloaders=1') # Users following user @@ -117,10 +126,9 @@ def test_old_ng_redirects(self): self.assertTrue(reverse('sounds-search') in resp.url and self.user.username in resp.url) self.client.force_login(self.user) - + # Sound edit page - resp = self.client.get( - reverse('sound-edit-sources', args=[self.user.username, self.sound.id])) + resp = self.client.get(reverse('sound-edit-sources', args=[self.user.username, self.sound.id])) self.assertRedirects(resp, reverse('sound-edit', args=[self.user.username, self.sound.id])) # Flag sound @@ -200,10 +208,12 @@ def test_download_attribution_csv(self): resp = self.client.get(reverse('accounts-download-attribution') + '?dl=csv') self.assertEqual(resp.status_code, 200) # response content as expected - self.assertContains(resp, - 'Download Type,File Name,User,License,Timestamp\r\nP,{0},{1},{0},{6}\r\nS,{2},{3},{4},{5}\r\n'.format( - self.pack.name, self.user.username, self.sound.original_filename, self.user.username, - self.sound.license, self.download.created, self.pack_download.created)) + self.assertContains( + resp, 'Download Type,File Name,User,License,Timestamp\r\nP,{0},{1},{0},{6}\r\nS,{2},{3},{4},{5}\r\n'.format( + self.pack.name, self.user.username, self.sound.original_filename, self.user.username, + self.sound.license, self.download.created, self.pack_download.created + ) + ) def test_download_attribution_txt(self): self.client.force_login(self.user) @@ -211,11 +221,13 @@ def test_download_attribution_txt(self): resp = self.client.get(reverse('accounts-download-attribution') + '?dl=txt') self.assertEqual(resp.status_code, 200) # response content as expected - self.assertContains(resp, - 'P: {0} by {1} | License: {0} | Timestamp: {6}\nS: {2} by {3} | License: {4} | Timestamp: {5}\n'.format( - self.pack.name, self.user.username, self.sound.original_filename, self.user.username, - self.sound.license, self.download.created, self.pack_download.created)) - + self.assertContains( + resp, + 'P: {0} by {1} | License: {0} | Timestamp: {6}\nS: {2} by {3} | License: {4} | Timestamp: {5}\n'.format( + self.pack.name, self.user.username, self.sound.original_filename, self.user.username, + self.sound.license, self.download.created, self.pack_download.created + ) + ) # If user is deleted, get 404 self.user.profile.delete_user() @@ -231,18 +243,23 @@ def test_sounds_response(self): resp = self.client.get(reverse('sounds')) self.assertEqual(resp.status_code, 302) self.assertTrue(reverse('sounds-search') in resp.url) - + # Test other sound related views. Nota that since BW many of these will include redirects user = self.sound.user user.set_password('12345') user.is_superuser = True user.save() self.client.force_login(user) - + resp = self.client.get(reverse('sound', kwargs={'username': user.username, "sound_id": self.sound.id})) self.assertEqual(resp.status_code, 200) - - resp = self.client.get(reverse('sound-flag', kwargs={'username': user.username, "sound_id": self.sound.id}) + '?ajax=1') + + resp = self.client.get( + reverse('sound-flag', kwargs={ + 'username': user.username, + "sound_id": self.sound.id + }) + '?ajax=1' + ) self.assertEqual(resp.status_code, 200) resp = self.client.get(reverse('sound-edit', kwargs={'username': user.username, "sound_id": self.sound.id})) @@ -251,15 +268,28 @@ def test_sounds_response(self): resp = self.client.get(reverse('sound-geotag', kwargs={'username': user.username, "sound_id": self.sound.id})) self.assertEqual(resp.status_code, 200) - resp = self.client.get(reverse('sound-similar', kwargs={'username': user.username, "sound_id": self.sound.id}) + '?ajax=1') + resp = self.client.get( + reverse('sound-similar', kwargs={ + 'username': user.username, + "sound_id": self.sound.id + }) + '?ajax=1' + ) self.assertEqual(resp.status_code, 200) resp = self.client.get( - reverse('sound-downloaders', kwargs={'username': user.username, "sound_id": self.sound.id}) + '?ajax=1') + reverse('sound-downloaders', kwargs={ + 'username': user.username, + "sound_id": self.sound.id + }) + '?ajax=1' + ) self.assertEqual(resp.status_code, 200) resp = self.client.get( - reverse('pack-downloaders', kwargs={'username': user.username, "pack_id": self.pack.id}) + '?ajax=1') + reverse('pack-downloaders', kwargs={ + 'username': user.username, + "pack_id": self.pack.id + }) + '?ajax=1' + ) self.assertEqual(resp.status_code, 200) @mock.patch('search.views.perform_search_engine_query') @@ -329,7 +359,7 @@ def test_accounts_manage_pages(self): resp = self.client.get(reverse('accounts-home')) self.assertEqual(resp.status_code, 302) self.assertEqual(resp.url, reverse('account', args=[user.username])) - + # 200 response on Account edit page resp = self.client.get(reverse('accounts-edit')) self.assertEqual(resp.status_code, 200) @@ -380,14 +410,12 @@ def test_accounts_manage_pages(self): def test_username_check(self): username = 'test_user_new' - resp = self.client.get(reverse('check_username'), - {'username': username}) + resp = self.client.get(reverse('check_username'), {'username': username}) self.assertEqual(resp.status_code, 200) self.assertEqual(resp.json()['result'], True) user = User.objects.create_user(username, password="testpass") - resp = self.client.get(reverse('check_username'), - {'username': username}) + resp = self.client.get(reverse('check_username'), {'username': username}) self.assertEqual(resp.status_code, 200) self.assertEqual(resp.json()['result'], False) @@ -399,38 +427,32 @@ def test_username_check(self): self.assertEqual(OldUsername.objects.filter(username=username, user=user).count(), 1) # Now check that check_username will return false for both old and new usernames - resp = self.client.get(reverse('check_username'), - {'username': username}) + resp = self.client.get(reverse('check_username'), {'username': username}) self.assertEqual(resp.status_code, 200) self.assertEqual(resp.json()['result'], False) - resp = self.client.get(reverse('check_username'), - {'username': user.username}) + resp = self.client.get(reverse('check_username'), {'username': user.username}) self.assertEqual(resp.status_code, 200) self.assertEqual(resp.json()['result'], False) # Now delete user and check that the username before deleting is still not available because we also # forbid reuse of usernames in DeletedUser objects user.profile.delete_user() - resp = self.client.get(reverse('check_username'), - {'username': user.username}) + resp = self.client.get(reverse('check_username'), {'username': user.username}) self.assertEqual(resp.status_code, 200) self.assertEqual(resp.json()['result'], False) # Check that a username that doesn't fit the registration guidelines returns "False", even # if the username doesn't exist - resp = self.client.get(reverse('check_username'), - {'username': 'username@withat'}) + resp = self.client.get(reverse('check_username'), {'username': 'username@withat'}) self.assertEqual(resp.status_code, 200) self.assertEqual(resp.json()['result'], False) - resp = self.client.get(reverse('check_username'), - {'username': 'username^withcaret'}) + resp = self.client.get(reverse('check_username'), {'username': 'username^withcaret'}) self.assertEqual(resp.status_code, 200) self.assertEqual(resp.json()['result'], False) - resp = self.client.get(reverse('check_username'), - {'username': 'username_withunderscore'}) + resp = self.client.get(reverse('check_username'), {'username': 'username_withunderscore'}) self.assertEqual(resp.status_code, 200) self.assertEqual(resp.json()['result'], True) diff --git a/accounts/urls.py b/accounts/urls.py index 564f2968c..3aae3b332 100644 --- a/accounts/urls.py +++ b/accounts/urls.py @@ -29,29 +29,37 @@ import apiv2.views as api from utils.urlpatterns import redirect_inline - - # By putting some URLs at the top that are the same as the ones listed in # django.contrib.auth.urls, we can override some configuration: # https://docs.djangoproject.com/en/1.11/topics/http/urls/#how-django-processes-a-request # 3. Django runs through each URL pattern, in order, and stops at the first one that matches the requested URL. urlpatterns = [ - path('login/', login_redirect(accounts.login), {'template_name': 'registration/login.html', - 'authentication_form': FsAuthenticationForm}, name="login"), + path( + 'login/', + login_redirect(accounts.login), { + 'template_name': 'registration/login.html', + 'authentication_form': FsAuthenticationForm + }, + name="login" + ), path('logout/', auth_views.LogoutView.as_view(), name='logout'), path('cleanup/', accounts.multi_email_cleanup, name="accounts-multi-email-cleanup"), - path('password_reset/', + path( + 'password_reset/', login_redirect( redirect_inline( auth_views.PasswordResetView.as_view(form_class=FsPasswordResetForm), redirect_url_name='front-page', - query_string='loginProblems=1')), - name='password_reset'), - path('password_reset/done/', - redirect_inline( - auth_views.PasswordResetDoneView.as_view(), - redirect_url_name='front-page'), - name='password_reset_done'), + query_string='loginProblems=1' + ) + ), + name='password_reset' + ), + path( + 'password_reset/done/', + redirect_inline(auth_views.PasswordResetDoneView.as_view(), redirect_url_name='front-page'), + name='password_reset_done' + ), path('password_change/', accounts.password_change_form, name='password_change'), path('password_change/done/', accounts.password_change_done, name='password_change_done'), path('reset///', accounts.password_reset_confirm, name='password_reset_confirm'), @@ -59,16 +67,23 @@ path('registration_modal/', login_redirect(accounts.registration_modal), name="accounts-registration-modal"), path('reactivate/', login_redirect(accounts.resend_activation), name="accounts-resend-activation"), path('username/', login_redirect(accounts.username_reminder), name="accounts-username-reminder"), - re_path(r'^activate/(?P[^\/]+)/(?P[^\/]+)/.*$', login_redirect(accounts.activate_user), name="accounts-activate"), + re_path( + r'^activate/(?P[^\/]+)/(?P[^\/]+)/.*$', + login_redirect(accounts.activate_user), + name="accounts-activate" + ), path('resetemail/', accounts.email_reset, name="accounts-email-reset"), path('resetemail/sent/', accounts.email_reset_done, name="accounts-email-reset-done"), - re_path(r'^resetemail/complete/(?P[0-9A-Za-z]+)-(?P.+)/$', accounts.email_reset_complete, name="accounts-email-reset-complete"), + re_path( + r'^resetemail/complete/(?P[0-9A-Za-z]+)-(?P.+)/$', + accounts.email_reset_complete, + name="accounts-email-reset-complete" + ), path('problems/', accounts.problems_logging_in, name="problems-logging-in"), path('bulklicensechange/', accounts.bulk_license_change, name="bulk-license-change"), path('tosacceptance/', accounts.tos_acceptance, name="tos-acceptance"), path('check_username/', accounts.check_username, name="check_username"), path('update_old_cc_licenses/', accounts.update_old_cc_licenses, name="update-old-cc-licenses"), - path('', accounts.home, name="accounts-home"), path('edit/', accounts.edit, name="accounts-edit"), path('email-settings/', accounts.edit_email_settings, name="accounts-email-settings"), @@ -76,26 +91,30 @@ path('attribution/', accounts.attribution, name="accounts-attribution"), path('download-attribution/', accounts.download_attribution, name="accounts-download-attribution"), path('stream/', follow.stream, name='stream'), - path('upload/', accounts.upload, name="accounts-upload", kwargs=dict(no_flash=True)), path('upload/html/', accounts.upload, name="accounts-upload-html", kwargs=dict(no_flash=True)), path('upload/flash/', accounts.upload, name="accounts-upload-flash"), path('upload/file/', accounts.upload_file, name="accounts-upload-file"), path('upload/bulk-describe//', accounts.bulk_describe, name="accounts-bulk-describe"), - path('sounds/manage//', accounts.manage_sounds, name="accounts-manage-sounds"), path('sounds/edit/', accounts.edit_sounds, name="accounts-edit-sounds"), path('describe/license/', accounts.describe_license, name="accounts-describe-license"), path('describe/pack/', accounts.describe_pack, name="accounts-describe-pack"), path('describe/sounds/', accounts.describe_sounds, name="accounts-describe-sounds"), - path('bookmarks/', bookmarks.bookmarks, name="bookmarks"), path('bookmarks/category//', bookmarks.bookmarks, name="bookmarks-category"), path('bookmarks/add//', bookmarks.add_bookmark, name="add-bookmark"), - path('bookmarks/get_form_for_sound//', bookmarks.get_form_for_sound, name="bookmarks-add-form-for-sound"), - path('bookmarks/category//delete/', bookmarks.delete_bookmark_category, name="delete-bookmark-category"), + path( + 'bookmarks/get_form_for_sound//', + bookmarks.get_form_for_sound, + name="bookmarks-add-form-for-sound" + ), + path( + 'bookmarks/category//delete/', + bookmarks.delete_bookmark_category, + name="delete-bookmark-category" + ), path('bookmarks//delete/', bookmarks.delete_bookmark, name="delete-bookmark"), - path('messages/', messages.inbox, name='messages'), path('messages/sent/', messages.sent_messages, name='messages-sent'), path('messages/archived/', messages.archived_messages, name='messages-archived'), @@ -105,7 +124,6 @@ path('messages/new/', messages.new_message, name='messages-new'), path('messages/new//', messages.new_message, name='messages-new', kwargs=dict(message_id=None)), path('messages/new/username_lookup', messages.username_lookup, name='messages-username_lookup'), - path('app_permissions/', api.granted_permissions, name='access-tokens'), path('app_permissions/revoke_permission//', api.revoke_permission, name='revoke-permission'), path('app_permissions/permission_granted/', api.permission_granted, name='permission-granted'), diff --git a/accounts/views.py b/accounts/views.py index d737edbdd..69d469eee 100644 --- a/accounts/views.py +++ b/accounts/views.py @@ -116,9 +116,7 @@ def login(request, template_name, authentication_form): # Freesound-specific login view to check if a user has multiple accounts # with the same email address. We can switch back to the regular django view # once all accounts are adapted - response = LoginView.as_view( - template_name='accounts/login.html', - authentication_form=authentication_form)(request) + response = LoginView.as_view(template_name='accounts/login.html', authentication_form=authentication_form)(request) if isinstance(response, HttpResponseRedirect): # If there is a redirect it's because the login was successful # Now we check if the logged in user has shared email problems @@ -161,8 +159,10 @@ def password_reset_complete(request): instead of staying in PasswordResetCompleteView (the current path). """ response = PasswordResetCompleteView.as_view( - template_name='accounts/password_reset_complete.html', - extra_context={'next_path': reverse('accounts-home')})(request) + template_name='accounts/password_reset_complete.html', extra_context={'next_path': reverse('accounts-home')} + )( + request + ) return response @@ -174,7 +174,10 @@ def password_change_form(request): response = PasswordChangeView.as_view( form_class=FsPasswordChangeForm, template_name='accounts/password_change_form.html', - extra_context={'activePage': 'password'})(request) + extra_context={'activePage': 'password'} + )( + request + ) return response @@ -184,8 +187,10 @@ def password_change_done(request): This view is called when user has successfully changed the password by filling in the password change form. """ response = PasswordChangeDoneView.as_view( - template_name='accounts/password_change_done.html', - extra_context={'activePage': 'password'})(request) + template_name='accounts/password_change_done.html', extra_context={'activePage': 'password'} + )( + request + ) return response @@ -221,8 +226,12 @@ def multi_email_cleanup(request): return HttpResponseRedirect(request.GET.get('next', reverse('accounts-home'))) else: # If email issues are still valid, then we show the email cleanup page with the instructions - return render(request, 'accounts/multi_email_cleanup.html', { - 'same_user': same_user, 'next': request.GET.get('next', reverse('accounts-home'))}) + return render( + request, 'accounts/multi_email_cleanup.html', { + 'same_user': same_user, + 'next': request.GET.get('next', reverse('accounts-home')) + } + ) def check_username(request): @@ -331,12 +340,15 @@ def activate_user(request, username, uid_hash): try: user = User.objects.get(username__iexact=username) except User.DoesNotExist: - return render(request, 'accounts/activate.html', {'user_does_not_exist': True, - 'next_path': reverse('accounts-home')}) + return render( + request, 'accounts/activate.html', { + 'user_does_not_exist': True, + 'next_path': reverse('accounts-home') + } + ) if not default_token_generator.check_token(user, uid_hash): - return render(request, 'accounts/activate.html', {'decode_error': True, - 'next_path': reverse('accounts-home')}) + return render(request, 'accounts/activate.html', {'decode_error': True, 'next_path': reverse('accounts-home')}) user.is_active = True user.save() @@ -346,11 +358,7 @@ def activate_user(request, username, uid_hash): def send_activation(user): token = default_token_generator.make_token(user) username = user.username - tvars = { - 'user': user, - 'username': username, - 'hash': token - } + tvars = {'user': user, 'username': username, 'hash': token} send_mail_template(settings.EMAIL_SUBJECT_ACTIVATION_LINK, 'emails/email_activation.txt', tvars, user_to=user) @@ -360,7 +368,7 @@ def resend_activation(request): def username_reminder(request): return HttpResponseRedirect(reverse('front-page') + '?loginProblems=1') - + @login_required def home(request): @@ -383,11 +391,8 @@ def edit_email_settings(request): all_emails = request.user.profile.get_enabled_email_types() form = EmailSettingsForm(initial={ 'email_types': all_emails, - }) - tvars = { - 'form': form, - 'activePage': 'notifications' - } + }) + tvars = {'form': form, 'activePage': 'notifications'} return render(request, 'accounts/edit_email_settings.html', tvars) @@ -457,7 +462,7 @@ def is_selected(prefix): 'has_granted_permissions': has_granted_permissions, 'has_old_avatar': has_old_avatar, 'uploads_enabled': settings.UPLOAD_AND_DESCRIPTION_ENABLED, - 'activePage': 'profile', + 'activePage': 'profile', } return render(request, 'accounts/edit.html', tvars) @@ -534,7 +539,6 @@ def process_filter_and_sort_options(request, sort_options, tab): 'filter_query': filter_query, 'sort_options': sort_options, }, sort_by_db, filter_db - # First do some stuff common to all tabs sounds_published_base_qs = Sound.public.filter(user=request.user) @@ -577,13 +581,15 @@ def process_filter_and_sort_options(request, sort_options, tab): packs = Pack.objects.ordered_ids(pack_ids) # Just as a sanity check, filter out packs not owned by the user packs = [pack for pack in packs if pack.user == pack.user] - + if packs: if 'edit' in request.POST: # There will be only one pack selected (otherwise the button is disabled) # Redirect to the edit pack page pack = packs[0] - return HttpResponseRedirect(reverse('pack-edit', args=[pack.user.username, pack.id]) + '?next=' + request.path) + return HttpResponseRedirect( + reverse('pack-edit', args=[pack.user.username, pack.id]) + '?next=' + request.path + ) elif 'delete_confirm' in request.POST: # Delete the selected packs n_packs_deleted = 0 @@ -591,9 +597,10 @@ def process_filter_and_sort_options(request, sort_options, tab): web_logger.info(f"User {request.user.username} requested to delete pack {pack.id}") pack.delete_pack(remove_sounds=False) n_packs_deleted += 1 - messages.add_message(request, messages.INFO, - f'Successfully deleted {n_packs_deleted} ' - f'pack{"s" if n_packs_deleted != 1 else ""}') + messages.add_message( + request, messages.INFO, f'Successfully deleted {n_packs_deleted} ' + f'pack{"s" if n_packs_deleted != 1 else ""}' + ) return HttpResponseRedirect(reverse('accounts-manage-sounds', args=[tab])) sort_options = [ @@ -607,7 +614,8 @@ def process_filter_and_sort_options(request, sort_options, tab): extra_tvars, sort_by_db, filter_db = process_filter_and_sort_options(request, sort_options, tab) tvars.update(extra_tvars) if filter_db is not None: - packs_base_qs = packs_base_qs.annotate(search=SearchVector('name', 'id', 'description')).filter(search=filter_db).distinct() + packs_base_qs = packs_base_qs.annotate(search=SearchVector('name', 'id', 'description') + ).filter(search=filter_db).distinct() packs = packs_base_qs.order_by(sort_by_db) pack_ids = list(packs.values_list('id', flat=True)) paginator = paginate(request, pack_ids, 12) @@ -630,10 +638,15 @@ def process_filter_and_sort_options(request, sort_options, tab): if sounds: if 'edit' in request.POST: # Edit the selected sounds - session_key_prefix = str(uuid.uuid4())[0:8] # Use a new so we don't interfere with other active description/editing processes - request.session[f'{session_key_prefix}-edit_sounds'] = sounds # Add the list of sounds to edit in the session object + session_key_prefix = str( + uuid.uuid4() + )[0:8] # Use a new so we don't interfere with other active description/editing processes + request.session[f'{session_key_prefix}-edit_sounds' + ] = sounds # Add the list of sounds to edit in the session object request.session[f'{session_key_prefix}-len_original_edit_sounds'] = len(sounds) - return HttpResponseRedirect(reverse('accounts-edit-sounds') + f'?next={request.path}&session={session_key_prefix}') + return HttpResponseRedirect( + reverse('accounts-edit-sounds') + f'?next={request.path}&session={session_key_prefix}' + ) elif 'delete_confirm' in request.POST: # Delete the selected sounds n_sounds_deleted = 0 @@ -645,15 +658,17 @@ def process_filter_and_sort_options(request, sort_options, tab): sender=request.user, text=f"User {request.user} deleted the sound", ticket=ticket, - moderator_only=False) + moderator_only=False + ) tc.save() except Ticket.DoesNotExist: pass sound.delete() n_sounds_deleted += 1 - messages.add_message(request, messages.INFO, - f'Successfully deleted {n_sounds_deleted} ' - f'sound{"s" if n_sounds_deleted != 1 else ""}') + messages.add_message( + request, messages.INFO, f'Successfully deleted {n_sounds_deleted} ' + f'sound{"s" if n_sounds_deleted != 1 else ""}' + ) return HttpResponseRedirect(reverse('accounts-manage-sounds', args=[tab])) elif 'process' in request.POST: @@ -666,12 +681,13 @@ def process_filter_and_sort_options(request, sort_options, tab): if n_send_to_processing != len(sounds): sounds_skipped_msg_part = f' {len(sounds) - n_send_to_processing} sounds were not send to ' \ f'processing due to many failed processing attempts.' - messages.add_message(request, messages.INFO, - f'Sent { n_send_to_processing } ' - f'sound{ "s" if n_send_to_processing != 1 else "" } ' - f'to re-process.{ sounds_skipped_msg_part }') + messages.add_message( + request, messages.INFO, f'Sent { n_send_to_processing } ' + f'sound{ "s" if n_send_to_processing != 1 else "" } ' + f'to re-process.{ sounds_skipped_msg_part }' + ) return HttpResponseRedirect(reverse('accounts-manage-sounds', args=[tab])) - + # Process query and filter options sort_options = [ ('created_desc', 'Date added (newest first)', '-created'), @@ -689,7 +705,8 @@ def process_filter_and_sort_options(request, sort_options, tab): elif tab == 'processing': sounds = sounds_processing_base_qs if filter_db is not None: - sounds = sounds.annotate(search=SearchVector('original_filename', 'id', 'description', 'tags__tag__name')).filter(search=filter_db).distinct() + sounds = sounds.annotate(search=SearchVector('original_filename', 'id', 'description', 'tags__tag__name') + ).filter(search=filter_db).distinct() sounds = sounds.order_by(sort_by_db) sound_ids = list(sounds.values_list('id', flat=True)) @@ -705,7 +722,7 @@ def process_filter_and_sort_options(request, sort_options, tab): sound.show_processing_status = True tvars['sounds_to_select'] = sounds_to_select else: - raise Http404 # Non-existing tab + raise Http404 # Non-existing tab return render(request, 'accounts/manage_sounds.html', tvars) @@ -714,7 +731,9 @@ def process_filter_and_sort_options(request, sort_options, tab): @transaction.atomic() def edit_sounds(request): session_key_prefix = request.GET.get('session', '') - return edit_and_describe_sounds_helper(request, session_key_prefix=session_key_prefix) # Note that the list of sounds to describe is stored in the session object + return edit_and_describe_sounds_helper( + request, session_key_prefix=session_key_prefix + ) # Note that the list of sounds to describe is stored in the session object def sounds_pending_description_helper(request, file_structure, files): @@ -736,8 +755,9 @@ def sounds_pending_description_helper(request, file_structure, files): destination.write(chunk) destination.close() - bulk = BulkUploadProgress.objects.create(user=request.user, csv_filename=new_csv_filename, - original_csv_filename=f.name) + bulk = BulkUploadProgress.objects.create( + user=request.user, csv_filename=new_csv_filename, original_csv_filename=f.name + ) tasks.validate_bulk_describe_csv.delay(bulk_upload_progress_object_id=bulk.id) return HttpResponseRedirect(reverse("accounts-bulk-describe", args=[bulk.id])) elif form.is_valid(): @@ -759,9 +779,15 @@ def sounds_pending_description_helper(request, file_structure, files): remove_empty_user_directory_from_mirror_locations(user_uploads_dir) return HttpResponseRedirect(reverse('accounts-manage-sounds', args=['pending_description'])) elif "describe" in request.POST: - session_key_prefix = str(uuid.uuid4())[0:8] # Use a new so we don't interfere with other active description/editing processes - request.session[f'{session_key_prefix}-describe_sounds'] = [files[x] for x in form.cleaned_data["files"]] - request.session[f'{session_key_prefix}-len_original_describe_sounds'] = len(request.session[f'{session_key_prefix}-describe_sounds']) + session_key_prefix = str( + uuid.uuid4() + )[0:8] # Use a new so we don't interfere with other active description/editing processes + request.session[f'{session_key_prefix}-describe_sounds'] = [ + files[x] for x in form.cleaned_data["files"] + ] + request.session[f'{session_key_prefix}-len_original_describe_sounds'] = len( + request.session[f'{session_key_prefix}-describe_sounds'] + ) # If only one file is choosen, go straight to the last step of the describe process, otherwise go to license selection step if len(request.session[f'{session_key_prefix}-describe_sounds']) > 1: return HttpResponseRedirect(reverse('accounts-describe-license') + f'?session={session_key_prefix}') @@ -795,8 +821,8 @@ def describe_license(request): else: form = LicenseForm(hide_old_license_versions=True) tvars = { - 'form': form, - 'num_files': request.session.get(f'{session_key_prefix}-len_original_describe_sounds', 0), + 'form': form, + 'num_files': request.session.get(f'{session_key_prefix}-len_original_describe_sounds', 0), 'session_key_prefix': session_key_prefix } return render(request, 'accounts/describe_license.html', tvars) @@ -821,8 +847,8 @@ def describe_pack(request): else: form = PackForm(packs, prefix="pack") tvars = { - 'form': form, - 'num_files': request.session.get(f'{session_key_prefix}-len_original_describe_sounds', 0), + 'form': form, + 'num_files': request.session.get(f'{session_key_prefix}-len_original_describe_sounds', 0), 'session_key_prefix': session_key_prefix } return render(request, 'accounts/describe_pack.html', tvars) @@ -832,9 +858,11 @@ def describe_pack(request): @transaction.atomic() def describe_sounds(request): session_key_prefix = request.GET.get('session', '') - return edit_and_describe_sounds_helper(request, describing=True, session_key_prefix=session_key_prefix) # Note that the list of sounds to describe is stored in the session object + return edit_and_describe_sounds_helper( + request, describing=True, session_key_prefix=session_key_prefix + ) # Note that the list of sounds to describe is stored in the session object + - @login_required def attribution(request): qs_sounds = Download.objects.annotate(download_type=Value("sound", CharField()))\ @@ -881,19 +909,24 @@ def download_attribution(request): output.write('Download Type,File Name,User,License,Timestamp\r\n') csv_writer = csv.writer(output, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL) for row in qs: - csv_writer.writerow( - [row['download_type'][0].upper(), row['sound__original_filename'], - row['sound__user__username'], - license_with_version(row['license__name'] or row['sound__license__name'], - row['license__deed_url'] or row['sound__license__deed_url']), - row['created']]) + csv_writer.writerow([ + row['download_type'][0].upper(), row['sound__original_filename'], row['sound__user__username'], + license_with_version( + row['license__name'] or row['sound__license__name'], row['license__deed_url'] + or row['sound__license__deed_url'] + ), row['created'] + ]) elif download == 'txt': for row in qs: - output.write("{}: {} by {} | License: {} | Timestamp: {}\n".format(row['download_type'][0].upper(), - row['sound__original_filename'], row['sound__user__username'], - license_with_version(row['license__name'] or row['sound__license__name'], - row['license__deed_url'] or row['sound__license__deed_url']), - row['created'])) + output.write( + "{}: {} by {} | License: {} | Timestamp: {}\n".format( + row['download_type'][0].upper(), row['sound__original_filename'], row['sound__user__username'], + license_with_version( + row['license__name'] or row['sound__license__name'], row['license__deed_url'] + or row['sound__license__deed_url'] + ), row['created'] + ) + ) response.writelines(output.getvalue()) return response else: @@ -918,10 +951,7 @@ def downloaded_sounds(request, username): sound = sounds_dict.get(d.sound_id, None) if sound is not None: download_list.append({"created": d.created, "sound": sound}) - tvars = {"username": username, - "user": user, - "download_list": download_list, - "type_sounds": True} + tvars = {"username": username, "user": user, "download_list": download_list, "type_sounds": True} tvars.update(paginator) return render(request, 'accounts/modal_downloads.html', tvars) @@ -944,9 +974,7 @@ def downloaded_packs(request, username): pack = packs_dict.get(d.pack_id, None) if pack is not None: download_list.append({"created": d.created, "pack": pack}) - tvars = {"username": username, - "download_list": download_list, - "type_sounds": False} + tvars = {"username": username, "download_list": download_list, "type_sounds": False} tvars.update(paginator) return render(request, 'accounts/modal_downloads.html', tvars) @@ -1006,16 +1034,17 @@ def compute_charts_stats(): most_active_users = User.objects.select_related("profile")\ .filter(id__in=[u[1] for u in sorted(sort_list, reverse=True)[:num_items]]) most_active_users_display = [[u, user_rank[u.id]] for u in most_active_users] - most_active_users_display = sorted(most_active_users_display, - key=lambda usr: user_rank[usr[0].id]['score'], - reverse=True) + most_active_users_display = sorted( + most_active_users_display, key=lambda usr: user_rank[usr[0].id]['score'], reverse=True + ) # Newest active users new_user_in_rank_ids = User.objects.filter(date_joined__gte=last_time, id__in=list(user_rank.keys()))\ .values_list('id', flat=True) - new_user_objects = {user.id: user for user in - User.objects.select_related("profile").filter(date_joined__gte=last_time) - .filter(id__in=new_user_in_rank_ids)} + new_user_objects = { + user.id: user for user in User.objects.select_related("profile").filter(date_joined__gte=last_time + ).filter(id__in=new_user_in_rank_ids) + } new_users_display = [(new_user_objects[user_id], user_rank[user_id]) for user_id in new_user_in_rank_ids] new_users_display = sorted(new_users_display, key=lambda x: x[1]['score'], reverse=True)[:num_items] @@ -1024,44 +1053,52 @@ def compute_charts_stats(): .filter(created__gte=last_time) \ .values('user_id').annotate(n_sounds=Count('user_id')) \ .order_by('-n_sounds')[0:num_items] - user_objects = {user.id: user for user in - User.objects.filter(id__in=[item['user_id'] for item in top_recent_uploaders_by_count])} - top_recent_uploaders_by_count_display = [ - (user_objects[item['user_id']].profile.locations("avatar.M.url"), - user_objects[item['user_id']].username, - item['n_sounds']) for item in top_recent_uploaders_by_count] + user_objects = { + user.id: user + for user in User.objects.filter(id__in=[item['user_id'] for item in top_recent_uploaders_by_count]) + } + top_recent_uploaders_by_count_display = [( + user_objects[item['user_id']].profile.locations("avatar.M.url"), user_objects[item['user_id']].username, + item['n_sounds'] + ) for item in top_recent_uploaders_by_count] top_recent_uploaders_by_length = Sound.public \ .filter(created__gte=last_time) \ .values('user_id').annotate(total_duration=Sum('duration')) \ .order_by('-total_duration')[0:num_items] - user_objects = {user.id: user for user in - User.objects.filter(id__in=[item['user_id'] for item in top_recent_uploaders_by_length])} - top_recent_uploaders_by_length_display = [ - (user_objects[item['user_id']].profile.locations("avatar.M.url"), - user_objects[item['user_id']].username, - item['total_duration']) for item in top_recent_uploaders_by_length] + user_objects = { + user.id: user + for user in User.objects.filter(id__in=[item['user_id'] for item in top_recent_uploaders_by_length]) + } + top_recent_uploaders_by_length_display = [( + user_objects[item['user_id']].profile.locations("avatar.M.url"), user_objects[item['user_id']].username, + item['total_duration'] + ) for item in top_recent_uploaders_by_length] # All time top uploaders (by count and by length) all_time_top_uploaders_by_count = Sound.public \ .values('user_id').annotate(n_sounds=Count('user_id')) \ .order_by('-n_sounds')[0:num_items] - user_objects = {user.id: user for user in - User.objects.filter(id__in=[item['user_id'] for item in all_time_top_uploaders_by_count])} - all_time_top_uploaders_by_count_display = [ - (user_objects[item['user_id']].profile.locations("avatar.M.url"), - user_objects[item['user_id']].username, - item['n_sounds']) for item in all_time_top_uploaders_by_count] + user_objects = { + user.id: user + for user in User.objects.filter(id__in=[item['user_id'] for item in all_time_top_uploaders_by_count]) + } + all_time_top_uploaders_by_count_display = [( + user_objects[item['user_id']].profile.locations("avatar.M.url"), user_objects[item['user_id']].username, + item['n_sounds'] + ) for item in all_time_top_uploaders_by_count] all_time_top_uploaders_by_length = Sound.public \ .values('user_id').annotate(total_duration=Sum('duration')) \ .order_by('-total_duration')[0:num_items] - user_objects = {user.id: user for user in - User.objects.filter(id__in=[item['user_id'] for item in all_time_top_uploaders_by_length])} - all_time_top_uploaders_by_length_display = [ - (user_objects[item['user_id']].profile.locations("avatar.M.url"), - user_objects[item['user_id']].username, - item['total_duration']) for item in all_time_top_uploaders_by_length] + user_objects = { + user.id: user + for user in User.objects.filter(id__in=[item['user_id'] for item in all_time_top_uploaders_by_length]) + } + all_time_top_uploaders_by_length_display = [( + user_objects[item['user_id']].profile.locations("avatar.M.url"), user_objects[item['user_id']].username, + item['total_duration'] + ) for item in all_time_top_uploaders_by_length] return { 'num_days': num_days, @@ -1107,15 +1144,16 @@ def account(request, username): num_sounds_pending = None num_mod_annotations = None - show_about = ((request.user == user) # user is looking at own page - or request.user.is_superuser # admins should always see about fields - or user.is_superuser # no reason to hide admin's about fields - or user.profile.get_total_downloads > 0 # user has downloads - or user.profile.num_sounds > 0) # user has uploads + show_about = ((request.user == user) # user is looking at own page + or request.user.is_superuser # admins should always see about fields + or user.is_superuser # no reason to hide admin's about fields + or user.profile.get_total_downloads > 0 # user has downloads + or user.profile.num_sounds > 0) # user has uploads last_geotags_serialized = [] if user.profile.has_geotags and settings.MAPBOX_USE_STATIC_MAPS_BEFORE_LOADING: - for sound in Sound.public.select_related('geotag').filter(user__username__iexact=username).exclude(geotag=None)[0:10]: + for sound in Sound.public.select_related('geotag').filter(user__username__iexact=username).exclude(geotag=None + )[0:10]: last_geotags_serialized.append({'lon': sound.geotag.lon, 'lat': sound.geotag.lat}) last_geotags_serialized = json.dumps(last_geotags_serialized) @@ -1136,7 +1174,7 @@ def account(request, username): 'following_modal_page': request.GET.get('following', 1), 'followers_modal_page': request.GET.get('followers', 1), 'following_tags_modal_page': request.GET.get('followingTags', 1), - 'last_geotags_serialized': last_geotags_serialized, + 'last_geotags_serialized': last_geotags_serialized, 'user_downloads_public': settings.USER_DOWNLOADS_PUBLIC, } return render(request, 'accounts/account.html', tvars) @@ -1145,7 +1183,7 @@ def account(request, username): @redirect_if_old_username_or_404 def account_stats_section(request, username): if not request.GET.get('ajax'): - raise Http404 # Only accessible via ajax + raise Http404 # Only accessible via ajax user = request.parameter_user tvars = { 'user': user, @@ -1157,13 +1195,13 @@ def account_stats_section(request, username): @redirect_if_old_username_or_404 def account_latest_packs_section(request, username): if not request.GET.get('ajax'): - raise Http404 # Only accessible via ajax - + raise Http404 # Only accessible via ajax + user = request.parameter_user tvars = { 'user': user, - # Note we don't pass latest packs data because it is requested from the template - # if there is no cache available + # Note we don't pass latest packs data because it is requested from the template + # if there is no cache available } return render(request, 'accounts/account_latest_packs_section.html', tvars) @@ -1181,7 +1219,7 @@ def handle_uploaded_file(user_id, f): # file instead of copying it try: os.rename(f.temporary_file_path(), dest_path) - os.chmod(dest_path, 0o644) # Set appropriate permissions so that file can be downloaded from nginx + os.chmod(dest_path, 0o644) # Set appropriate permissions so that file can be downloaded from nginx except Exception as e: upload_logger.warning("failed moving TemporaryUploadedFile error: %s", str(e)) return False @@ -1213,7 +1251,7 @@ def upload_file(request): the user login """ upload_logger.info("start uploading file") - engine = __import__(settings.SESSION_ENGINE, {}, {}, ['']) # get the current session engine + engine = __import__(settings.SESSION_ENGINE, {}, {}, ['']) # get the current session engine session_data = engine.SessionStore(request.POST.get('sessionid', '')) try: user_id = session_data['_auth_user_id'] @@ -1269,7 +1307,9 @@ def upload(request, no_flash=False): 'no_flash': no_flash, 'max_file_size': settings.UPLOAD_MAX_FILE_SIZE_COMBINED, 'max_file_size_in_MB': int(round(settings.UPLOAD_MAX_FILE_SIZE_COMBINED * 1.0 / (1024 * 1024))), - 'lossless_file_extensions': [ext for ext in settings.ALLOWED_AUDIOFILE_EXTENSIONS if ext not in settings.LOSSY_FILE_EXTENSIONS], + 'lossless_file_extensions': [ + ext for ext in settings.ALLOWED_AUDIOFILE_EXTENSIONS if ext not in settings.LOSSY_FILE_EXTENSIONS + ], 'lossy_file_extensions': settings.LOSSY_FILE_EXTENSIONS, 'all_file_extensions': settings.ALLOWED_AUDIOFILE_EXTENSIONS, 'uploads_enabled': settings.UPLOAD_AND_DESCRIPTION_ENABLED @@ -1280,9 +1320,11 @@ def upload(request, no_flash=False): @login_required def bulk_describe(request, bulk_id): if not request.user.profile.can_do_bulk_upload(): - messages.add_message(request, messages.INFO, "Your user does not have permission to use the bulk describe " - "feature. You must upload at least %i sounds before being able" - "to use that feature." % settings.BULK_UPLOAD_MIN_SOUNDS) + messages.add_message( + request, messages.INFO, "Your user does not have permission to use the bulk describe " + "feature. You must upload at least %i sounds before being able" + "to use that feature." % settings.BULK_UPLOAD_MIN_SOUNDS + ) return HttpResponseRedirect(reverse('accounts-manage-sounds', args=['pending_description'])) bulk = get_object_or_404(BulkUploadProgress, id=int(bulk_id), user=request.user) @@ -1341,12 +1383,14 @@ def delete(request): cutoff_date = datetime.datetime.today() - datetime.timedelta(days=1) recent_pending_deletion_requests_exist = UserDeletionRequest.objects\ .filter(user_to_id=request.user.id, last_updated__gt=cutoff_date)\ - .filter(status=UserDeletionRequest.DELETION_REQUEST_STATUS_DELETION_TRIGGERED).exists() + .filter(status=UserDeletionRequest.DELETION_REQUEST_STATUS_DELETION_TRIGGERED).exists() if recent_pending_deletion_requests_exist: - messages.add_message(request, messages.INFO, + messages.add_message( + request, messages.INFO, f'It looks like a deletion action was already triggered for your user account and ' f'your account should be deleted shortly. If you see the account not being deleted, ' - f'please contact us using the contact form.') + f'please contact us using the contact form.' + ) else: delete_sounds =\ form.cleaned_data['delete_sounds'] == 'delete_sounds' @@ -1356,19 +1400,25 @@ def delete(request): web_logger.info(f'Requested async deletion of user {request.user.id} - {delete_action}') # Create a UserDeletionRequest with a status of 'Deletion action was triggered' - UserDeletionRequest.objects.create(user_from=request.user, - user_to=request.user, - status=UserDeletionRequest.DELETION_REQUEST_STATUS_DELETION_TRIGGERED, - triggered_deletion_action=delete_action, - triggered_deletion_reason=delete_reason) + UserDeletionRequest.objects.create( + user_from=request.user, + user_to=request.user, + status=UserDeletionRequest.DELETION_REQUEST_STATUS_DELETION_TRIGGERED, + triggered_deletion_action=delete_action, + triggered_deletion_reason=delete_reason + ) # Trigger async task so user gets deleted asynchronously - tasks.delete_user.delay(user_id=request.user.id, deletion_action=delete_action, deletion_reason=delete_reason) + tasks.delete_user.delay( + user_id=request.user.id, deletion_action=delete_action, deletion_reason=delete_reason + ) # Show a message to the user that the account will be deleted shortly - messages.add_message(request, messages.INFO, - 'Your user account will be deleted in a few moments. Note that this process could ' - 'take up to several hours for users with many uploaded sounds.') + messages.add_message( + request, messages.INFO, + 'Your user account will be deleted in a few moments. Note that this process could ' + 'take up to several hours for users with many uploaded sounds.' + ) # Logout user, mark account inctive, set unusable password and change email to a dummy one so that # user can't recover the account while it is being delete asynchronously @@ -1385,9 +1435,9 @@ def delete(request): form = DeleteUserForm(user_id=request.user.id) tvars = { - 'delete_form': form, - 'num_sounds': num_sounds, - 'activePage': 'account', + 'delete_form': form, + 'num_sounds': num_sounds, + 'activePage': 'account', } return render(request, 'accounts/delete.html', tvars) @@ -1428,30 +1478,20 @@ def email_reset(request): # Send email to the new address user = request.user email = form.cleaned_data["email"] - tvars = { - 'uid': int_to_base36(user.id), - 'user': user, - 'token': default_token_generator.make_token(user) - } - send_mail_template(settings.EMAIL_SUBJECT_EMAIL_CHANGED, - 'emails/email_reset_email.txt', tvars, - email_to=email) + tvars = {'uid': int_to_base36(user.id), 'user': user, 'token': default_token_generator.make_token(user)} + send_mail_template( + settings.EMAIL_SUBJECT_EMAIL_CHANGED, 'emails/email_reset_email.txt', tvars, email_to=email + ) return HttpResponseRedirect(reverse('accounts-email-reset-done')) else: form = EmailResetForm(user=request.user, label_suffix='') - tvars = { - 'form': form, - 'user': request.user, - 'activePage': 'email' - } + tvars = {'form': form, 'user': request.user, 'activePage': 'email'} return render(request, 'accounts/email_reset_form.html', tvars) def email_reset_done(request): - return render(request, 'accounts/email_reset_done.html', { - 'activePage': 'email' - }) + return render(request, 'accounts/email_reset_done.html', {'activePage': 'email'}) @never_cache @@ -1459,7 +1499,7 @@ def email_reset_done(request): @transaction.atomic() def email_reset_complete(request, uidb36=None, token=None): # Check that the link is valid and the base36 corresponds to a user id - assert uidb36 is not None and token is not None # checked by URLconf + assert uidb36 is not None and token is not None # checked by URLconf try: uid_int = base36_to_int(uidb36) user = User.objects.get(id=uid_int) @@ -1488,18 +1528,17 @@ def email_reset_complete(request, uidb36=None, token=None): # a User deletion pre_save hook if we detect that email has changed # Send email to the old address notifying about the change - tvars = { - 'old_email': old_email, - 'user': user, - 'activePage': 'email' - } - send_mail_template(settings.EMAIL_SUBJECT_EMAIL_CHANGED, - 'emails/email_reset_complete_old_address_notification.txt', tvars, email_to=old_email) + tvars = {'old_email': old_email, 'user': user, 'activePage': 'email'} + send_mail_template( + settings.EMAIL_SUBJECT_EMAIL_CHANGED, + 'emails/email_reset_complete_old_address_notification.txt', + tvars, + email_to=old_email + ) return render(request, 'accounts/email_reset_complete.html', tvars) - def problems_logging_in(request): """This view gets a User object from ProblemsLoggingInForm form contents and then either sends email instructions to re-activate the user (if the user is not active) or sends instructions to re-set the password (if the user @@ -1579,8 +1618,7 @@ def flag_user(request, username): added_objects.append(key) try: obj = f_object.content_type.get_object_for_this_type(id=f_object.object_id) - url = reverse('admin:%s_%s_change' % - (obj._meta.app_label, obj._meta.model_name), args=[obj.id]) + url = reverse('admin:%s_%s_change' % (obj._meta.app_label, obj._meta.model_name), args=[obj.id]) if isinstance(obj, Comment): content = obj.comment elif isinstance(obj, Post): @@ -1592,8 +1630,10 @@ def flag_user(request, username): objects_data.append([str(f_object.content_type), request.build_absolute_uri(url), content]) except ObjectDoesNotExist: objects_data.append([str(f_object.content_type), "url not available", ""]) - user_url = reverse('admin:%s_%s_delete' % - (flagged_user._meta.app_label, flagged_user._meta.model_name), args=[flagged_user.id]) + user_url = reverse( + 'admin:%s_%s_delete' % (flagged_user._meta.app_label, flagged_user._meta.model_name), + args=[flagged_user.id] + ) user_url = request.build_absolute_uri(user_url) clear_url = reverse("clear-flags-user", args=[flagged_user.username]) clear_url = request.build_absolute_uri(clear_url) @@ -1602,12 +1642,15 @@ def flag_user(request, username): else: template_to_use = 'emails/email_report_blocked_spammer_admins.txt' - tvars = {'flagged_user': flagged_user, - 'objects_data': objects_data, - 'user_url': user_url, - 'clear_url': clear_url} + tvars = { + 'flagged_user': flagged_user, + 'objects_data': objects_data, + 'user_url': user_url, + 'clear_url': clear_url + } send_mail_template_to_support( - settings.EMAIL_SUBJECT_USER_SPAM_REPORT, template_to_use, tvars, extra_subject=flagged_user.username) + settings.EMAIL_SUBJECT_USER_SPAM_REPORT, template_to_use, tvars, extra_subject=flagged_user.username + ) return HttpResponse(json.dumps({"errors": None}), content_type='application/javascript') else: return HttpResponse(json.dumps({"errors": True}), content_type='application/javascript') @@ -1616,7 +1659,7 @@ def flag_user(request, username): @login_required def clear_flags_user(request, username): if request.user.is_superuser or request.user.is_staff: - flags = UserFlag.objects.filter(user__username = username) + flags = UserFlag.objects.filter(user__username=username) num = len(flags) for flag in flags: flag.delete() diff --git a/apiv2/admin.py b/apiv2/admin.py index 61322be5e..8fb4c43fd 100644 --- a/apiv2/admin.py +++ b/apiv2/admin.py @@ -31,7 +31,3 @@ class ApiV2ClientAdmin(admin.ModelAdmin): def has_add_permission(self, request): return False - - - - diff --git a/apiv2/apiv2_utils.py b/apiv2/apiv2_utils.py index a4086e310..a2a196bad 100644 --- a/apiv2/apiv2_utils.py +++ b/apiv2/apiv2_utils.py @@ -58,12 +58,13 @@ error_logger = logging.getLogger("api_errors") cache_api_monitoring = caches["api_monitoring"] - ########################################## # oauth 2 provider generator for client id ########################################## + class FsClientIdGenerator(BaseHashGenerator): + def hash(self): """ Override ClientIdGenerator from oauth_provider2 as it does not allow to change length of id with @@ -76,6 +77,7 @@ def hash(self): # Rest Framework custom function for getting descriptions from function instead docstring ######################################################################################### + def get_view_description(cls, html=False): description = '' if getattr(cls, 'get_description', None): @@ -85,7 +87,7 @@ def get_view_description(cls, html=False): description = cls.get_description() description = formatting.dedent(smart_str(description)) # Cache for 1 hour (if we update description, it will take 1 hour to show) - cache.set(cache_key, description, 60*60) + cache.set(cache_key, description, 60 * 60) else: description = cached_description if html: @@ -127,7 +129,7 @@ def store_monitor_usage(self): now = datetime.datetime.now().date() monitoring_key = f'{now.year}-{now.month}-{now.day}_{self.client_id}' current_value = cache_api_monitoring.get(monitoring_key, 0) - cache_api_monitoring.set(monitoring_key, current_value + 1, 60 * 60 * 24 * 3) # Expire in 3 days + cache_api_monitoring.set(monitoring_key, current_value + 1, 60 * 60 * 24 * 3) # Expire in 3 days def get_request_information(self, request): # Get request information and store it as class variable @@ -255,7 +257,9 @@ def get_serializer(self, *args, **kwargs): if 'SoundListSerializer' in str(serializer_class): # If we are trying to serialize sounds, check if we should and sound analysis data to them and add it if isinstance(args[0], collections.abc.Iterable): - sound_analysis_data = get_analysis_data_for_sound_ids(kwargs['context']['request'], sound_ids=[s.id for s in args[0]]) + sound_analysis_data = get_analysis_data_for_sound_ids( + kwargs['context']['request'], sound_ids=[s.id for s in args[0]] + ) if sound_analysis_data: kwargs['sound_analysis_data'] = sound_analysis_data return serializer_class(*args, **kwargs) @@ -287,8 +291,10 @@ def finalize_response(self, request, response, *args, **kwargs): # Search utilities ################## + def api_search( - search_form, target_file=None, extra_parameters=False, merging_strategy='merge_optimized', resource=None): + search_form, target_file=None, extra_parameters=False, merging_strategy='merge_optimized', resource=None +): if search_form.cleaned_data['query'] is None \ and search_form.cleaned_data['filter'] is None \ @@ -306,7 +312,8 @@ def api_search( filter=search_form.cleaned_data['descriptors_filter'], num_results=search_form.cleaned_data['page_size'], offset=(search_form.cleaned_data['page'] - 1) * search_form.cleaned_data['page_size'], - target_file=target_file) + target_file=target_file + ) gaia_ids = [result[0] for result in results] distance_to_target_data = None @@ -328,7 +335,8 @@ def api_search( raise ServerErrorException(msg=f'Similarity server error: {str(e)}', resource=resource) except Exception as e: raise ServerErrorException( - msg='The similarity server could not be reached or some unexpected error occurred.', resource=resource) + msg='The similarity server could not be reached or some unexpected error occurred.', resource=resource + ) elif not search_form.cleaned_data['descriptors_filter'] \ and not search_form.cleaned_data['target'] \ @@ -362,13 +370,17 @@ def api_search( except SearchEngineException as e: if search_form.cleaned_data['filter'] is not None: - raise BadRequestException(msg='Search server error: %s (please check that your filter syntax and field ' - 'names are correct)' % str(e), resource=resource) + raise BadRequestException( + msg='Search server error: %s (please check that your filter syntax and field ' + 'names are correct)' % str(e), + resource=resource + ) raise BadRequestException(msg=f'Search server error: {str(e)}', resource=resource) except Exception as e: print(e) raise ServerErrorException( - msg='The search server could not be reached or some unexpected error occurred.', resource=resource) + msg='The search server could not be reached or some unexpected error occurred.', resource=resource + ) else: # Combined search (there is at least one of query/filter and one of descriptors_filter/target) @@ -396,7 +408,7 @@ def log_message_helper(message, data_dict=None, info_dict=None, resource=None, r if resource is not None: data_dict = resource.request.query_params.copy() data_dict = {key: urllib.parse.quote(value, safe=",:") for key, value in data_dict.items()} - data_dict.pop('token', None) # Remove token from req params if it exists (we don't need it) + data_dict.pop('token', None) # Remove token from req params if it exists (we don't need it) if info_dict is None: if resource is not None: info_dict = build_info_dict(resource=resource) @@ -445,7 +457,7 @@ def prepend_base(rel, dynamic_resolve=True, use_https=False, request_is_secure=F if request_is_secure: use_https = True - dynamic_resolve = False # don't need to dynamic resolve is request is https + dynamic_resolve = False # don't need to dynamic resolve is request is https if dynamic_resolve: use_https = True @@ -491,6 +503,7 @@ def request_parameters_info_for_log_message(get_parameters): class ApiSearchPaginator: + def __init__(self, results, count, num_per_page): self.num_per_page = num_per_page self.count = count @@ -502,18 +515,21 @@ def page(self, page_num): has_next = page_num < self.num_pages has_previous = 1 < page_num <= self.num_pages - return {'object_list': self.results, - 'has_next': has_next, - 'has_previous': has_previous, - 'has_other_pages': has_next or has_previous, - 'next_page_number': page_num + 1, - 'previous_page_number': page_num - 1, - 'page_num': page_num} + return { + 'object_list': self.results, + 'has_next': has_next, + 'has_previous': has_previous, + 'has_other_pages': has_next or has_previous, + 'next_page_number': page_num + 1, + 'previous_page_number': page_num - 1, + 'page_num': page_num + } # Docs examples utils ##################### + def get_formatted_examples_for_view(view_name, url_name, max=10): try: data = examples[view_name] @@ -545,8 +561,9 @@ def get_formatted_examples_for_view(view_name, url_name, max=10): # Similarity utils ################## + def get_analysis_data_for_sound_ids(request, sound_ids=[]): - # Get analysis data for all requested sounds and return it as a dictionary + # Get analysis data for all requested sounds and return it as a dictionary sound_analysis_data = {} analysis_data_is_requested = 'analysis' in request.query_params.get('fields', '').split(',') if analysis_data_is_requested: @@ -555,7 +572,9 @@ def get_analysis_data_for_sound_ids(request, sound_ids=[]): ids = [int(sid) for sid in sound_ids] if descriptors: try: - sound_analysis_data = get_sounds_descriptors(ids, descriptors.split(','), normalized, only_leaf_descriptors=True) + sound_analysis_data = get_sounds_descriptors( + ids, descriptors.split(','), normalized, only_leaf_descriptors=True + ) except: pass else: @@ -565,7 +584,6 @@ def get_analysis_data_for_sound_ids(request, sound_ids=[]): return sound_analysis_data - # APIv1 end of life ################### @@ -575,7 +593,8 @@ def get_analysis_data_for_sound_ids(request, sound_ids=[]): def apiv1_end_of_life_message(request): apiv1_logger.info('410 API error: End of life') content = { - "explanation": "Freesound APIv1 has reached its end of life and is no longer available." - "Please, upgrade to Freesound APIv2. More information: https://freesound.org/docs/api/" + "explanation": + "Freesound APIv1 has reached its end of life and is no longer available." + "Please, upgrade to Freesound APIv2. More information: https://freesound.org/docs/api/" } return JsonResponse(content, status=410) diff --git a/apiv2/authentication.py b/apiv2/authentication.py index 42b6adc5b..9f785ef0f 100644 --- a/apiv2/authentication.py +++ b/apiv2/authentication.py @@ -57,6 +57,7 @@ def authenticate(self, request): raise exceptions.AuthenticationFailed('Suspended token or token pending for approval') return super_response + class TokenAuthentication(BaseAuthentication): """ Simple token based authentication. diff --git a/apiv2/combined_search_strategies.py b/apiv2/combined_search_strategies.py index a42facbc3..69ba6a5bd 100644 --- a/apiv2/combined_search_strategies.py +++ b/apiv2/combined_search_strategies.py @@ -38,11 +38,13 @@ def merge_all(search_form, target_file=None, extra_parameters=None): extra_parameters = dict() solr_page_size = extra_parameters.get('cs_solr_page_size', 500) max_solr_pages = extra_parameters.get('cs_max_solr_pages', 10) - gaia_page_size = extra_parameters.get('cs_gaia_page_size', 9999999) # We can get ALL gaia results at once + gaia_page_size = extra_parameters.get('cs_gaia_page_size', 9999999) # We can get ALL gaia results at once max_gaia_pages = extra_parameters.get('cs_max_gaia_pages', 1) # Get all gaia results - gaia_ids, gaia_count, distance_to_target_data, note = get_gaia_results(search_form, target_file, page_size=gaia_page_size, max_pages=max_gaia_pages) + gaia_ids, gaia_count, distance_to_target_data, note = get_gaia_results( + search_form, target_file, page_size=gaia_page_size, max_pages=max_gaia_pages + ) # Get 'max_pages' pages of size 'page_size' from solr results solr_ids, solr_count = get_solr_results(search_form, page_size=solr_page_size, max_pages=max_solr_pages) @@ -85,16 +87,28 @@ def filter_both(search_form, target_file=None, extra_parameters=None): gaia_filter_id_block_size = extra_parameters.get('cs_gaia_filter_id_block_size', 350) gaia_filter_id_max_pages = extra_parameters.get('cs_gaia_filter_id_max_pages', 7) gaia_max_pages = extra_parameters.get('cs_max_gaia_pages', 1) - gaia_page_size = extra_parameters.get('cs_gaia_page_size', 9999999) # We can get ALL gaia results at once + gaia_page_size = extra_parameters.get('cs_gaia_page_size', 9999999) # We can get ALL gaia results at once if search_form.cleaned_data['target'] or target_file: # First search into gaia and then into solr (get all gaia results) - gaia_ids, gaia_count, distance_to_target_data, note = get_gaia_results(search_form, target_file, page_size=gaia_page_size, max_pages=gaia_max_pages) - valid_ids_pages = [gaia_ids[i:i+solr_filter_id_block_size] for i in range(0, len(gaia_ids), solr_filter_id_block_size) if (old_div(i,solr_filter_id_block_size)) < solr_filter_id_max_pages] + gaia_ids, gaia_count, distance_to_target_data, note = get_gaia_results( + search_form, target_file, page_size=gaia_page_size, max_pages=gaia_max_pages + ) + valid_ids_pages = [ + gaia_ids[i:i + solr_filter_id_block_size] + for i in range(0, len(gaia_ids), solr_filter_id_block_size) + if (old_div(i, solr_filter_id_block_size)) < solr_filter_id_max_pages + ] solr_ids = list() search_engine = get_search_engine() for valid_ids_page in valid_ids_pages: - page_solr_ids, solr_count = get_solr_results(search_form, page_size=len(valid_ids_page), max_pages=1, valid_ids=valid_ids_page, search_engine=search_engine) + page_solr_ids, solr_count = get_solr_results( + search_form, + page_size=len(valid_ids_page), + max_pages=1, + valid_ids=valid_ids_page, + search_engine=search_engine + ) solr_ids += page_solr_ids if gaia_count <= solr_filter_id_block_size * solr_filter_id_max_pages: @@ -110,7 +124,9 @@ def filter_both(search_form, target_file=None, extra_parameters=None): # present in the current block. However given that gaia results can be retrieved # all at once very quickly, we optimize this bit by retrieving them all at once and avoiding many requests # to similarity server. - gaia_ids, gaia_count, distance_to_target_data, note = get_gaia_results(search_form, target_file, page_size=gaia_page_size, max_pages=gaia_max_pages) + gaia_ids, gaia_count, distance_to_target_data, note = get_gaia_results( + search_form, target_file, page_size=gaia_page_size, max_pages=gaia_max_pages + ) ''' # That would be the code without the optimization: valid_ids_pages = [solr_ids[i:i+gaia_filter_id_block_size] for i in range(0, len(solr_ids), gaia_filter_id_block_size) if (i/gaia_filter_id_block_size) < gaia_filter_id_max_pages] @@ -126,7 +142,6 @@ def filter_both(search_form, target_file=None, extra_parameters=None): #print 'COMPLETE results (starting with solr)' pass - if search_form.cleaned_data['target'] or target_file: # Combined search, sort by gaia_ids results_a = gaia_ids @@ -159,7 +174,7 @@ def merge_optimized(search_form, target_file=None, extra_parameters=None): solr_max_requests = extra_parameters.get('cs_max_solr_requests', 20) solr_page_size = extra_parameters.get('cs_solr_page_size', 200) gaia_max_pages = extra_parameters.get('cs_max_gaia_pages', 1) - gaia_page_size = extra_parameters.get('cs_gaia_page_size', 9999999) # We can get ALL gaia results at once + gaia_page_size = extra_parameters.get('cs_gaia_page_size', 9999999) # We can get ALL gaia results at once num_requested_results = search_form.cleaned_data['page_size'] params_for_next_page = dict() @@ -171,18 +186,32 @@ def merge_optimized(search_form, target_file=None, extra_parameters=None): last_checked_valid_id_position = extra_parameters.get('cs_lcvidp', 0) if last_checked_valid_id_position < 0: last_checked_valid_id_position = 0 - gaia_ids, gaia_count, distance_to_target_data, note = get_gaia_results(search_form, target_file, page_size=gaia_page_size, max_pages=gaia_max_pages, offset=last_checked_valid_id_position) + gaia_ids, gaia_count, distance_to_target_data, note = get_gaia_results( + search_form, + target_file, + page_size=gaia_page_size, + max_pages=gaia_max_pages, + offset=last_checked_valid_id_position + ) if len(gaia_ids): # Now divide gaia results in blocks of "solr_filter_id_block_size" results and iteratively query solr limiting the # results to those ids in the common block to obtain common results for the search. # Once we get as many results as "num_requested_results" or we exceed a maximum number # of iterations (solr_filter_id_max_pages), return what we got and update 'cs_lcvidp' parameter for further calls. - valid_ids_pages = [gaia_ids[i:i+solr_filter_id_block_size] for i in range(0, len(gaia_ids), solr_filter_id_block_size)] + valid_ids_pages = [ + gaia_ids[i:i + solr_filter_id_block_size] for i in range(0, len(gaia_ids), solr_filter_id_block_size) + ] solr_ids = list() checked_gaia_ids = list() search_engine = get_search_engine() for count, valid_ids_page in enumerate(valid_ids_pages): - page_solr_ids, solr_count = get_solr_results(search_form, page_size=len(valid_ids_page), max_pages=1, valid_ids=valid_ids_page, search_engine=search_engine) + page_solr_ids, solr_count = get_solr_results( + search_form, + page_size=len(valid_ids_page), + max_pages=1, + valid_ids=valid_ids_page, + search_engine=search_engine + ) solr_ids += page_solr_ids checked_gaia_ids += valid_ids_page if len(solr_ids) >= num_requested_results: @@ -214,14 +243,22 @@ def merge_optimized(search_form, target_file=None, extra_parameters=None): else: # First search into gaia to obtain a list of all sounds that match content-based query parameters - gaia_ids, gaia_count, distance_to_target_data, note = get_gaia_results(search_form, target_file, page_size=gaia_page_size, max_pages=gaia_max_pages) + gaia_ids, gaia_count, distance_to_target_data, note = get_gaia_results( + search_form, target_file, page_size=gaia_page_size, max_pages=gaia_max_pages + ) last_retrieved_solr_id_pos = extra_parameters.get('cs_lrsidp', 0) if last_retrieved_solr_id_pos < 0: last_retrieved_solr_id_pos = 0 if len(gaia_ids) < solr_filter_id_block_size: # optimization, if there are few gaia_ids, we can get all results in one query - solr_ids, solr_count = get_solr_results(search_form, page_size=len(gaia_ids), max_pages=1, valid_ids=gaia_ids, offset=last_retrieved_solr_id_pos) + solr_ids, solr_count = get_solr_results( + search_form, + page_size=len(gaia_ids), + max_pages=1, + valid_ids=gaia_ids, + offset=last_retrieved_solr_id_pos + ) combined_ids = solr_ids[:num_requested_results] params_for_next_page['cs_lrsidp'] = last_retrieved_solr_id_pos + num_requested_results if len(combined_ids) < num_requested_results: @@ -239,7 +276,9 @@ def merge_optimized(search_form, target_file=None, extra_parameters=None): if stop_main_for_loop: continue offset = last_retrieved_solr_id_pos + i * solr_page_size - solr_ids, solr_count = get_solr_results(search_form, page_size=solr_page_size, max_pages=1, offset=offset) + solr_ids, solr_count = get_solr_results( + search_form, page_size=solr_page_size, max_pages=1, offset=offset + ) n_requests_made += 1 common_ids = list(set(solr_ids).intersection(gaia_ids)) for index, sid in enumerate(solr_ids): @@ -278,12 +317,14 @@ def get_gaia_results(search_form, target_file, page_size, max_pages, start_page= while (gaia_count is None or len(gaia_ids) < gaia_count) and n_page_requests <= max_pages: if not offset: offset = (current_page - 1) * page_size - results, count, note = similarity_api_search(target=search_form.cleaned_data['target'], - filter=search_form.cleaned_data['descriptors_filter'], - num_results=page_size, - offset=offset, - target_file=target_file, - in_ids=valid_ids) + results, count, note = similarity_api_search( + target=search_form.cleaned_data['target'], + filter=search_form.cleaned_data['descriptors_filter'], + num_results=page_size, + offset=offset, + target_file=target_file, + in_ids=valid_ids + ) gaia_ids += [id[0] for id in results] gaia_count = count @@ -342,7 +383,6 @@ def get_solr_results(search_form, page_size, max_pages, start_page=1, valid_ids= offset=(current_page - 1) * page_size, num_sounds=page_size, group_by_pack=False - ) solr_ids += [element['id'] for element in result.docs] solr_count = result.num_found diff --git a/apiv2/examples.py b/apiv2/examples.py index e2e02c4f0..c4e736aaf 100644 --- a/apiv2/examples.py +++ b/apiv2/examples.py @@ -18,113 +18,240 @@ # See AUTHORS file. # - examples = { # Search 'TextSearch': [ - ('Simple search', ['apiv2/search/text/?query=cars', 'apiv2/search/text/?query=piano&page=2', 'apiv2/search/text/?query=bass -drum', 'apiv2/search/text/?query="bass drum" -double']), - ('Search with a filter', ['apiv2/search/text/?query=music&filter=tag:guitar','apiv2/search/text/?query=music&filter=type:(wav OR aiff)','apiv2/search/text/?query=music&filter=tag:bass tag:drum','apiv2/search/text/?query=music&filter=tag:bass description:"heavy distortion"','apiv2/search/text/?query=music&filter=is_geotagged:true tag:field-recording duration:[60 TO 120]','apiv2/search/text/?query=music&filter=samplerate:44100 type:wav channels:2','apiv2/search/text/?query=music&filter=duration:[0.1 TO 0.3] avg_rating:[3 TO *]']), - ('Simple search and selection of sound fields to return in the results', ['apiv2/search/text/?query=alarm&fields=name,previews', 'apiv2/search/text/?query=alarm&fields=name,previews,analysis&descriptors=lowlevel.spectral_centroid.mean,lowlevel.pitch.mean', 'apiv2/search/text/?query=loop&fields=uri,analysis&descriptors=rhythm.onset_times']), + ( + 'Simple search', [ + 'apiv2/search/text/?query=cars', 'apiv2/search/text/?query=piano&page=2', + 'apiv2/search/text/?query=bass -drum', 'apiv2/search/text/?query="bass drum" -double' + ] + ), + ( + 'Search with a filter', [ + 'apiv2/search/text/?query=music&filter=tag:guitar', + 'apiv2/search/text/?query=music&filter=type:(wav OR aiff)', + 'apiv2/search/text/?query=music&filter=tag:bass tag:drum', + 'apiv2/search/text/?query=music&filter=tag:bass description:"heavy distortion"', + 'apiv2/search/text/?query=music&filter=is_geotagged:true tag:field-recording duration:[60 TO 120]', + 'apiv2/search/text/?query=music&filter=samplerate:44100 type:wav channels:2', + 'apiv2/search/text/?query=music&filter=duration:[0.1 TO 0.3] avg_rating:[3 TO *]' + ] + ), + ( + 'Simple search and selection of sound fields to return in the results', [ + 'apiv2/search/text/?query=alarm&fields=name,previews', + 'apiv2/search/text/?query=alarm&fields=name,previews,analysis&descriptors=lowlevel.spectral_centroid.mean,lowlevel.pitch.mean', + 'apiv2/search/text/?query=loop&fields=uri,analysis&descriptors=rhythm.onset_times' + ] + ), ('Group search results by pack', ['apiv2/search/text/?query=piano&group_by_pack=1']), - ('Get geotagged sounds with tag field-recording. Return only geotag and tags for each result', ['apiv2/search/text/?filter=is_geotagged:1 tag:field-recording&fields=geotag,tags']), - ('Basic geospatial filtering', ['apiv2/search/text/?filter=geotag:"Intersects(-74.093 41.042 -69.347 44.558)"', 'apiv2/search/text/?filter=geotag:"IsDisjointTo(-74.093 41.042 -69.347 44.558)"']), - ('Geospatial with customizable max error parameter (in degrees) and combinations of filters', - ['apiv2/search/text/?filter=geotag:"Intersects(-74.093 41.042 -69.347 44.558) distErr=20"', - 'apiv2/search/text/?filter=geotag:"Intersects(-80 40 -60 50)" OR geotag:"Intersects(60 40 100 50)"&fields=id,geotag,tags', - 'apiv2/search/text/?filter=(geotag:"Intersects(-80 40 -60 50)" OR geotag:"Intersects(60 40 100 50)") AND tag:field-recording&fields=id,geotag,tags']), - ('Geospatial search for points at a maximum distance d (in km) from a latitude,longitude position and with a particular tag', ['apiv2/search/text/?filter={!geofilt sfield=geotag pt=41.3833,2.1833 d=10} tag:barcelona&fields=id,geotag,tags',]), + ( + 'Get geotagged sounds with tag field-recording. Return only geotag and tags for each result', + ['apiv2/search/text/?filter=is_geotagged:1 tag:field-recording&fields=geotag,tags'] + ), + ( + 'Basic geospatial filtering', [ + 'apiv2/search/text/?filter=geotag:"Intersects(-74.093 41.042 -69.347 44.558)"', + 'apiv2/search/text/?filter=geotag:"IsDisjointTo(-74.093 41.042 -69.347 44.558)"' + ] + ), + ( + 'Geospatial with customizable max error parameter (in degrees) and combinations of filters', [ + 'apiv2/search/text/?filter=geotag:"Intersects(-74.093 41.042 -69.347 44.558) distErr=20"', + 'apiv2/search/text/?filter=geotag:"Intersects(-80 40 -60 50)" OR geotag:"Intersects(60 40 100 50)"&fields=id,geotag,tags', + 'apiv2/search/text/?filter=(geotag:"Intersects(-80 40 -60 50)" OR geotag:"Intersects(60 40 100 50)") AND tag:field-recording&fields=id,geotag,tags' + ] + ), + ( + 'Geospatial search for points at a maximum distance d (in km) from a latitude,longitude position and with a particular tag', + [ + 'apiv2/search/text/?filter={!geofilt sfield=geotag pt=41.3833,2.1833 d=10} tag:barcelona&fields=id,geotag,tags', + ] + ), ], 'ContentSearch': [ - ('Setting a target as some descriptor values', ['apiv2/search/content/?target=lowlevel.pitch.mean:220', 'apiv2/search/content/?target=lowlevel.pitch.mean:220 AND lowlevel.pitch.var:0']), - ('Using multidimensional descriptors in the target', ['apiv2/search/content/?target=sfx.tristimulus.mean:0,1,0&fields=id,analysis&descriptors=sfx.tristimulus.mean']), + ( + 'Setting a target as some descriptor values', [ + 'apiv2/search/content/?target=lowlevel.pitch.mean:220', + 'apiv2/search/content/?target=lowlevel.pitch.mean:220 AND lowlevel.pitch.var:0' + ] + ), + ( + 'Using multidimensional descriptors in the target', [ + 'apiv2/search/content/?target=sfx.tristimulus.mean:0,1,0&fields=id,analysis&descriptors=sfx.tristimulus.mean' + ] + ), ('Using a Freesound sound id as target', ['apiv2/search/content/?target=1234']), - ('Using an Essentia analysis file as target', ['curl -X POST -H "Authorization: Token {{your_api_key}}" -F analysis_file=@"/path/to/your_file.json" \'%s/apiv2/search/content/\'']), - ('Using descriptors filter', ['apiv2/search/content/?descriptors_filter=lowlevel.pitch.mean:[219.9 TO 220.1]', 'apiv2/search/content/?descriptors_filter=lowlevel.pitch.mean:[219.9 TO 220.1] AND lowlevel.pitch_salience.mean:[0.6 TO *]', 'apiv2/search/content/?descriptors_filter=lowlevel.mfcc.mean[0]:[-1124 TO -1121]', 'apiv2/search/content/?descriptors_filter=lowlevel.mfcc.mean[1]:[17 TO 20] AND lowlevel.mfcc.mean[4]:[0 TO 20]', 'apiv2/search/content/?descriptors_filter=tonal.key_key:"Asharp"', 'apiv2/search/content/?descriptors_filter=tonal.key_scale:"major"', 'apiv2/search/content/?descriptors_filter=(tonal.key_key:"C" AND tonal.key_scale:"major") OR (tonal.key_key:"A" AND tonal.key_scale:"minor")', 'apiv2/search/content/?descriptors_filter=tonal.key_key:"C" tonal.key_scale="major" tonal.key_strength:[0.8 TO *]']), + ( + 'Using an Essentia analysis file as target', [ + 'curl -X POST -H "Authorization: Token {{your_api_key}}" -F analysis_file=@"/path/to/your_file.json" \'%s/apiv2/search/content/\'' + ] + ), + ( + 'Using descriptors filter', [ + 'apiv2/search/content/?descriptors_filter=lowlevel.pitch.mean:[219.9 TO 220.1]', + 'apiv2/search/content/?descriptors_filter=lowlevel.pitch.mean:[219.9 TO 220.1] AND lowlevel.pitch_salience.mean:[0.6 TO *]', + 'apiv2/search/content/?descriptors_filter=lowlevel.mfcc.mean[0]:[-1124 TO -1121]', + 'apiv2/search/content/?descriptors_filter=lowlevel.mfcc.mean[1]:[17 TO 20] AND lowlevel.mfcc.mean[4]:[0 TO 20]', + 'apiv2/search/content/?descriptors_filter=tonal.key_key:"Asharp"', + 'apiv2/search/content/?descriptors_filter=tonal.key_scale:"major"', + 'apiv2/search/content/?descriptors_filter=(tonal.key_key:"C" AND tonal.key_scale:"major") OR (tonal.key_key:"A" AND tonal.key_scale:"minor")', + 'apiv2/search/content/?descriptors_filter=tonal.key_key:"C" tonal.key_scale="major" tonal.key_strength:[0.8 TO *]' + ] + ), ], 'CombinedSearch': [ - ('Combining query with target descriptors and textual filter', ['apiv2/search/combined/?target=rhythm.bpm:120&filter=tag:loop']), - ('Combining textual query with descriptors filter', ['apiv2/search/combined/?filter=tag:loop&descriptors_filter=rhythm.bpm:[119 TO 121]']), - ('Combining two filters (textual and descriptors)', ['apiv2/search/combined/?descriptors_filter=tonal.key_key:"A" tonal.key_scale:"major"&filter=tag:chord']), - ('Combining textual query with multidimensional descriptors filter', ['apiv2/search/combined/?query=music&fields=id,analysis&descriptors=lowlevel.mfcc.mean&descriptors_filter=lowlevel.mfcc.mean[1]:[17 TO 20] AND lowlevel.mfcc.mean[4]:[0 TO 20]']), + ( + 'Combining query with target descriptors and textual filter', + ['apiv2/search/combined/?target=rhythm.bpm:120&filter=tag:loop'] + ), + ( + 'Combining textual query with descriptors filter', + ['apiv2/search/combined/?filter=tag:loop&descriptors_filter=rhythm.bpm:[119 TO 121]'] + ), + ( + 'Combining two filters (textual and descriptors)', + ['apiv2/search/combined/?descriptors_filter=tonal.key_key:"A" tonal.key_scale:"major"&filter=tag:chord'] + ), + ( + 'Combining textual query with multidimensional descriptors filter', [ + 'apiv2/search/combined/?query=music&fields=id,analysis&descriptors=lowlevel.mfcc.mean&descriptors_filter=lowlevel.mfcc.mean[1]:[17 TO 20] AND lowlevel.mfcc.mean[4]:[0 TO 20]' + ] + ), ], # Sounds 'SoundInstance': [ ('Complete sound information', ['apiv2/sounds/1234/']), - ('Complete sound information plus some descriptors', ['apiv2/sounds/213524/?descriptors=lowlevel.mfcc,rhythm.bpm']), + ( + 'Complete sound information plus some descriptors', + ['apiv2/sounds/213524/?descriptors=lowlevel.mfcc,rhythm.bpm'] + ), ('Getting only id and tags for a particular sound', ['apiv2/sounds/1234/?fields=id,tags']), - ('Getting sound name and spectral centroid values (second example gets normalized centroid values)', ['apiv2/sounds/1234/?fields=name,analysis&descriptors=lowlevel.spectral_centroid', 'apiv2/sounds/1234/?fields=name,analysis&descriptors=lowlevel.spectral_centroid&normalized=1']), + ( + 'Getting sound name and spectral centroid values (second example gets normalized centroid values)', [ + 'apiv2/sounds/1234/?fields=name,analysis&descriptors=lowlevel.spectral_centroid', + 'apiv2/sounds/1234/?fields=name,analysis&descriptors=lowlevel.spectral_centroid&normalized=1' + ] + ), ], 'SoundAnalysis': [ ('Full analysis information', ['apiv2/sounds/1234/analysis/']), ('Getting only tristimulus descriptor', ['apiv2/sounds/1234/analysis/?descriptors=sfx.tristimulus']), - ('Getting normalized mean mfcc descriptors', ['apiv2/sounds/1234/analysis/?descriptors=lowlevel.mfcc.mean&normalized=1']), - ], - 'SimilarSounds': [ - ('Getting similar sounds', ['apiv2/sounds/80408/similar/', 'apiv2/sounds/80408/similar/?page=2', 'apiv2/sounds/1234/similar/?fields=name,analysis&descriptors=lowlevel.pitch.mean&descriptors_filter=.lowlevel.pitch.mean:[90 TO 110]']), - ], - 'SoundComments': [ - ('Get sound comments', ['apiv2/sounds/14854/comments/', 'apiv2/sounds/14854/comments/?page=2']), + ( + 'Getting normalized mean mfcc descriptors', + ['apiv2/sounds/1234/analysis/?descriptors=lowlevel.mfcc.mean&normalized=1'] + ), ], + 'SimilarSounds': [( + 'Getting similar sounds', [ + 'apiv2/sounds/80408/similar/', 'apiv2/sounds/80408/similar/?page=2', + 'apiv2/sounds/1234/similar/?fields=name,analysis&descriptors=lowlevel.pitch.mean&descriptors_filter=.lowlevel.pitch.mean:[90 TO 110]' + ] + ),], + 'SoundComments': [('Get sound comments', ['apiv2/sounds/14854/comments/', 'apiv2/sounds/14854/comments/?page=2']),], 'DownloadSound': [ ('Download a sound', ['curl -H "Authorization: Bearer {{access_token}}" \'%s/apiv2/sounds/14854/download/\'']), ], 'UploadSound': [ - ('Upload a sound (audiofile only, no description)', ['curl -X POST -H "Authorization: Bearer {{access_token}}" -F audiofile=@"/path/to/your_file.wav" \'%s/apiv2/sounds/upload/\'']), - ('Upload and describe a sound all at once', ['curl -X POST -H "Authorization: Bearer {{access_token}}" -F audiofile=@"/path/to/your_file.wav" -F "tags=field-recording birds nature h4n" -F "description=This sound was recorded...
bla bla bla..." -F "license=Attribution" \'%s/apiv2/sounds/upload/\'']), - ('Upload and describe a sound with name, pack and geotag', ['curl -X POST -H "Authorization: Bearer {{access_token}}" -F audiofile=@"/path/to/your_file.wav" -F "name=Another cool sound" -F "tags=field-recording birds nature h4n" -F "description=This sound was recorded...
bla bla bla..." -F "license=Attribution" -F "pack=A birds pack" -F "geotag=2.145677,3.22345,14" \'%s/apiv2/sounds/upload/\'']), - ], - 'PendingUploads': [ - ('Get uploaded sounds that are pending description, processing or moderation', ['curl -H "Authorization: Bearer {{access_token}}" \'%s/apiv2/sounds/pending_uploads/\'']), + ( + 'Upload a sound (audiofile only, no description)', [ + 'curl -X POST -H "Authorization: Bearer {{access_token}}" -F audiofile=@"/path/to/your_file.wav" \'%s/apiv2/sounds/upload/\'' + ] + ), + ( + 'Upload and describe a sound all at once', [ + 'curl -X POST -H "Authorization: Bearer {{access_token}}" -F audiofile=@"/path/to/your_file.wav" -F "tags=field-recording birds nature h4n" -F "description=This sound was recorded...
bla bla bla..." -F "license=Attribution" \'%s/apiv2/sounds/upload/\'' + ] + ), + ( + 'Upload and describe a sound with name, pack and geotag', [ + 'curl -X POST -H "Authorization: Bearer {{access_token}}" -F audiofile=@"/path/to/your_file.wav" -F "name=Another cool sound" -F "tags=field-recording birds nature h4n" -F "description=This sound was recorded...
bla bla bla..." -F "license=Attribution" -F "pack=A birds pack" -F "geotag=2.145677,3.22345,14" \'%s/apiv2/sounds/upload/\'' + ] + ), ], + 'PendingUploads': [( + 'Get uploaded sounds that are pending description, processing or moderation', + ['curl -H "Authorization: Bearer {{access_token}}" \'%s/apiv2/sounds/pending_uploads/\''] + ),], 'DescribeSound': [ - ('Describe a sound (only with required fields)', ['curl -X POST -H "Authorization: Bearer {{access_token}}" --data "upload_filename=your_file.wav&tags=field-recording birds nature h4n&description=This sound was recorded...
bla bla bla...&license=Attribution" \'%s/apiv2/sounds/describe/\'']), - ('Also add a name to the sound', ['curl -X POST -H "Authorization: Bearer {{access_token}}" --data "upload_filename=your_file.wav&name=A cool bird sound&tags=field-recording birds nature h4n&description=This sound was recorded...
bla bla bla...&license=Attribution" \'%s/apiv2/sounds/describe/\'']), - ('Include geotag and pack information', ['curl -X POST -H "Authorization: Bearer {{access_token}}" --data "upload_filename=your_file.wav&name=A cool bird sound&tags=field-recording birds nature h4n&description=This sound was recorded...
bla bla bla...&license=Attribution&pack=A birds pack&geotag=2.145677,3.22345,14" \'%s/apiv2/sounds/describe/\'']), + ( + 'Describe a sound (only with required fields)', [ + 'curl -X POST -H "Authorization: Bearer {{access_token}}" --data "upload_filename=your_file.wav&tags=field-recording birds nature h4n&description=This sound was recorded...
bla bla bla...&license=Attribution" \'%s/apiv2/sounds/describe/\'' + ] + ), + ( + 'Also add a name to the sound', [ + 'curl -X POST -H "Authorization: Bearer {{access_token}}" --data "upload_filename=your_file.wav&name=A cool bird sound&tags=field-recording birds nature h4n&description=This sound was recorded...
bla bla bla...&license=Attribution" \'%s/apiv2/sounds/describe/\'' + ] + ), + ( + 'Include geotag and pack information', [ + 'curl -X POST -H "Authorization: Bearer {{access_token}}" --data "upload_filename=your_file.wav&name=A cool bird sound&tags=field-recording birds nature h4n&description=This sound was recorded...
bla bla bla...&license=Attribution&pack=A birds pack&geotag=2.145677,3.22345,14" \'%s/apiv2/sounds/describe/\'' + ] + ), ], #'EditSoundDescription': [ # ('Setting tags of an existing sound to be "new tags for the sound" and description to "New sound description..."', ['curl -X POST -H "Authorization: Bearer {{access_token}}" --data "tags=new tags for the sound&description=New sound description..." \'%s/apiv2/sounds/1234/edit/\'']), #], 'BookmarkSound': [ - ('Simple bookmark', ['curl -X POST -H "Authorization: Bearer {{access_token}}" --data "name=Classic thunderstorm" \'%s/apiv2/sounds/2523/bookmark/\'']), - ('Bookmark with category', ['curl -X POST -H "Authorization: Bearer {{access_token}}" --data "name=Nice loop&category=Nice loops" \'%s/apiv2/sounds/1234/bookmark/\'']), - ], - 'RateSound': [ - ('Rate sounds', ['curl -X POST -H "Authorization: Bearer {{access_token}}" --data "rating=5" \'%s/apiv2/sounds/2523/rate/\'', 'curl -X POST -H "Authorization: Bearer {{access_token}}" --data "rating=4" \'%s/apiv2/sounds/1234/rate/\'']), - ], - 'CommentSound': [ - ('Comment sounds', ['curl -X POST -H "Authorization: Bearer {{access_token}}" --data "comment=Cool! I understand now why this is the most downloaded sound in Freesound..." \'%s/apiv2/sounds/2523/comment/\'', 'curl -X POST -H "Authorization: Bearer {{access_token}}" --data "comment=A very cool sound!" \'%s/apiv2/sounds/1234/comment/\'']), + ( + 'Simple bookmark', [ + 'curl -X POST -H "Authorization: Bearer {{access_token}}" --data "name=Classic thunderstorm" \'%s/apiv2/sounds/2523/bookmark/\'' + ] + ), + ( + 'Bookmark with category', [ + 'curl -X POST -H "Authorization: Bearer {{access_token}}" --data "name=Nice loop&category=Nice loops" \'%s/apiv2/sounds/1234/bookmark/\'' + ] + ), ], + 'RateSound': [( + 'Rate sounds', [ + 'curl -X POST -H "Authorization: Bearer {{access_token}}" --data "rating=5" \'%s/apiv2/sounds/2523/rate/\'', + 'curl -X POST -H "Authorization: Bearer {{access_token}}" --data "rating=4" \'%s/apiv2/sounds/1234/rate/\'' + ] + ),], + 'CommentSound': [( + 'Comment sounds', [ + 'curl -X POST -H "Authorization: Bearer {{access_token}}" --data "comment=Cool! I understand now why this is the most downloaded sound in Freesound..." \'%s/apiv2/sounds/2523/comment/\'', + 'curl -X POST -H "Authorization: Bearer {{access_token}}" --data "comment=A very cool sound!" \'%s/apiv2/sounds/1234/comment/\'' + ] + ),], # Users - 'UserInstance': [ - ('User information', ['apiv2/users/reinsamba/', 'apiv2/users/Freed/']), - ], - 'UserSounds': [ - ('Getting user sounds', ['apiv2/users/Jovica/sounds/', 'apiv2/users/Jovica/sounds/?page=2', 'apiv2/users/Jovica/sounds/?fields=id,bitdepth,type,samplerate']), - ], - 'UserPacks': [ - ('Getting user packs', ['apiv2/users/reinsamba/packs/', 'apiv2/users/reinsamba/packs/?page=2']), - ], + 'UserInstance': [('User information', ['apiv2/users/reinsamba/', 'apiv2/users/Freed/']),], + 'UserSounds': [( + 'Getting user sounds', [ + 'apiv2/users/Jovica/sounds/', 'apiv2/users/Jovica/sounds/?page=2', + 'apiv2/users/Jovica/sounds/?fields=id,bitdepth,type,samplerate' + ] + ),], + 'UserPacks': [('Getting user packs', ['apiv2/users/reinsamba/packs/', 'apiv2/users/reinsamba/packs/?page=2']),], # Packs - 'PackInstance': [ - ('Getting a pack', ['apiv2/packs/9678/']), - ], - 'PackSounds': [ - ('Getting pack sounds', ['apiv2/packs/9678/sounds/','apiv2/packs/9678/sounds/?fields=id,name']), - ], + 'PackInstance': [('Getting a pack', ['apiv2/packs/9678/']),], + 'PackSounds': [('Getting pack sounds', ['apiv2/packs/9678/sounds/', 'apiv2/packs/9678/sounds/?fields=id,name']),], 'DownloadPack': [ ('Download a pack', ['curl -H "Authorization: Bearer {{access_token}}" \'%s/apiv2/packs/9678/download/\'']), ], # Me - 'MeBookmarkCategories': [ - ('Users bookmark categories', ['curl -H "Authorization: Bearer {{access_token}}" \'%s/apiv2/me/bookmark_categories/\'']), - ], + 'MeBookmarkCategories': [( + 'Users bookmark categories', + ['curl -H "Authorization: Bearer {{access_token}}" \'%s/apiv2/me/bookmark_categories/\''] + ),], 'MeBookmarkCategorySounds': [ - ('Getting uncategorized bookmarks', ['curl -H "Authorization: Bearer {{access_token}}" \'%s/apiv2/me/bookmark_categories/0/sounds/\'']), - ('Getting sounds of a particular bookmark cateogry', ['curl -H "Authorization: Bearer {{access_token}}" \'%s/apiv2/me/bookmark_categories/11819/sounds/\'', 'curl -H "Authorization: Bearer {{access_token}}" \'%s/apiv2/me/bookmark_categories/11819/sounds/?fields=duration,previews\'']), + ( + 'Getting uncategorized bookmarks', + ['curl -H "Authorization: Bearer {{access_token}}" \'%s/apiv2/me/bookmark_categories/0/sounds/\''] + ), + ( + 'Getting sounds of a particular bookmark cateogry', [ + 'curl -H "Authorization: Bearer {{access_token}}" \'%s/apiv2/me/bookmark_categories/11819/sounds/\'', + 'curl -H "Authorization: Bearer {{access_token}}" \'%s/apiv2/me/bookmark_categories/11819/sounds/?fields=duration,previews\'' + ] + ), ], - } diff --git a/apiv2/exceptions.py b/apiv2/exceptions.py index 86711e6a5..9c658e7ee 100644 --- a/apiv2/exceptions.py +++ b/apiv2/exceptions.py @@ -18,14 +18,13 @@ # See AUTHORS file. # - import logging import sentry_sdk from rest_framework import status from rest_framework.exceptions import APIException -import apiv2.apiv2_utils # absolute import because of mutual imports of this module and apiv2_utils +import apiv2.apiv2_utils # absolute import because of mutual imports of this module and apiv2_utils errors_logger = logging.getLogger("api_errors") @@ -36,8 +35,17 @@ class NotFoundException(APIException): def __init__(self, msg="Not found", resource=None): summary_message = '%i Not found' % self.status_code - errors_logger.info(apiv2.apiv2_utils.log_message_helper(summary_message, data_dict={ - 'summary_message': summary_message, 'long_message': msg, 'status': self.status_code}, resource=resource)) + errors_logger.info( + apiv2.apiv2_utils.log_message_helper( + summary_message, + data_dict={ + 'summary_message': summary_message, + 'long_message': msg, + 'status': self.status_code + }, + resource=resource + ) + ) self.detail = msg @@ -47,8 +55,17 @@ class InvalidUrlException(APIException): def __init__(self, msg="Invalid url", request=None): summary_message = '%i Invalid url' % self.status_code - errors_logger.info(apiv2.apiv2_utils.log_message_helper(summary_message, data_dict={ - 'summary_message': summary_message, 'long_message': msg, 'status': self.status_code}, request=request)) + errors_logger.info( + apiv2.apiv2_utils.log_message_helper( + summary_message, + data_dict={ + 'summary_message': summary_message, + 'long_message': msg, + 'status': self.status_code + }, + request=request + ) + ) self.detail = msg @@ -58,8 +75,17 @@ class BadRequestException(APIException): def __init__(self, msg="Bad request", resource=None): summary_message = '%i Bad request' % self.status_code - errors_logger.info(apiv2.apiv2_utils.log_message_helper(summary_message, data_dict={ - 'summary_message': summary_message, 'long_message': msg, 'status': self.status_code}, resource=resource)) + errors_logger.info( + apiv2.apiv2_utils.log_message_helper( + summary_message, + data_dict={ + 'summary_message': summary_message, + 'long_message': msg, + 'status': self.status_code + }, + resource=resource + ) + ) self.detail = msg @@ -69,8 +95,17 @@ class ConflictException(APIException): def __init__(self, msg="Conflict", resource=None): summary_message = '%i Conflict' % self.status_code - errors_logger.info(apiv2.apiv2_utils.log_message_helper(summary_message, data_dict={ - 'summary_message': summary_message, 'long_message': msg, 'status': self.status_code}, resource=resource)) + errors_logger.info( + apiv2.apiv2_utils.log_message_helper( + summary_message, + data_dict={ + 'summary_message': summary_message, + 'long_message': msg, + 'status': self.status_code + }, + resource=resource + ) + ) self.detail = msg @@ -80,8 +115,17 @@ class UnauthorizedException(APIException): def __init__(self, msg="Not authorized", resource=None): summary_message = '%i Not authorized' % self.status_code - errors_logger.info(apiv2.apiv2_utils.log_message_helper(summary_message, data_dict={ - 'summary_message': summary_message, 'long_message': msg, 'status': self.status_code}, resource=resource)) + errors_logger.info( + apiv2.apiv2_utils.log_message_helper( + summary_message, + data_dict={ + 'summary_message': summary_message, + 'long_message': msg, + 'status': self.status_code + }, + resource=resource + ) + ) self.detail = msg @@ -91,8 +135,17 @@ class RequiresHttpsException(APIException): def __init__(self, msg="This resource requires a secure connection (https)", request=None): summary_message = '%i Requires Https' % self.status_code - errors_logger.info(apiv2.apiv2_utils.log_message_helper(summary_message, data_dict={ - 'summary_message': summary_message, 'long_message': msg, 'status': self.status_code}, request=request)) + errors_logger.info( + apiv2.apiv2_utils.log_message_helper( + summary_message, + data_dict={ + 'summary_message': summary_message, + 'long_message': msg, + 'status': self.status_code + }, + request=request + ) + ) self.detail = msg @@ -102,10 +155,21 @@ class ServerErrorException(APIException): def __init__(self, msg="Server error", resource=None): summary_message = '%i Server error' % self.status_code - errors_logger.info(apiv2.apiv2_utils.log_message_helper(summary_message, data_dict={ - 'summary_message': summary_message, 'long_message': msg, 'status': self.status_code}, resource=resource)) + errors_logger.info( + apiv2.apiv2_utils.log_message_helper( + summary_message, + data_dict={ + 'summary_message': summary_message, + 'long_message': msg, + 'status': self.status_code + }, + resource=resource + ) + ) self.detail = msg - sentry_sdk.capture_exception(self) # Manually capture exception so it has mroe info and Sentry can organize it properly + sentry_sdk.capture_exception( + self + ) # Manually capture exception so it has mroe info and Sentry can organize it properly class OtherException(APIException): @@ -114,8 +178,17 @@ class OtherException(APIException): def __init__(self, msg="Bad request", status=status.HTTP_400_BAD_REQUEST, resource=None): summary_message = '%i Other exception' % status - errors_logger.info(apiv2.apiv2_utils.log_message_helper(summary_message, data_dict={ - 'summary_message': summary_message, 'long_message': msg, 'status': status}, resource=resource)) + errors_logger.info( + apiv2.apiv2_utils.log_message_helper( + summary_message, + data_dict={ + 'summary_message': summary_message, + 'long_message': msg, + 'status': status + }, + resource=resource + ) + ) self.detail = msg self.status_code = status @@ -126,6 +199,15 @@ class Throttled(APIException): def __init__(self, msg="Request was throttled", request=None): summary_message = '%i Throttled' % self.status_code - errors_logger.info(apiv2.apiv2_utils.log_message_helper(summary_message, data_dict={ - 'summary_message': summary_message, 'long_message': msg, 'status': self.status_code}, request=request)) + errors_logger.info( + apiv2.apiv2_utils.log_message_helper( + summary_message, + data_dict={ + 'summary_message': summary_message, + 'long_message': msg, + 'status': self.status_code + }, + request=request + ) + ) self.detail = msg diff --git a/apiv2/forms.py b/apiv2/forms.py index 4316ba603..8a241d258 100644 --- a/apiv2/forms.py +++ b/apiv2/forms.py @@ -18,7 +18,6 @@ # See AUTHORS file. # - import django.forms as forms from django.conf import settings from django.utils.safestring import mark_safe @@ -28,45 +27,76 @@ class ApiV2ClientForm(forms.Form): - name = forms.CharField(label='Name*', max_length='60', widget=forms.TextInput(attrs={'style': 'width:100%', - 'placeholder': 'The name of the application or project where the credential will be used'})) - url = forms.URLField(required=False, label='URL', max_length=200, widget=forms.TextInput(attrs={'style': 'width:100%', - 'placeholder': 'URL of your application, project, institution or other related page'})) - redirect_uri = forms.URLField(required=False, label='Callback URL', max_length=200, widget=forms.TextInput( - attrs={'style': 'width:100%', 'placeholder': 'OAuth2 callback URL'}), - help_text="""
The Callback URL is only used for the authorization process when accessing resources that require OAuth2. + name = forms.CharField( + label='Name*', + max_length='60', + widget=forms.TextInput( + attrs={ + 'style': 'width:100%', + 'placeholder': 'The name of the application or project where the credential will be used' + } + ) + ) + url = forms.URLField( + required=False, + label='URL', + max_length=200, + widget=forms.TextInput( + attrs={ + 'style': 'width:100%', + 'placeholder': 'URL of your application, project, institution or other related page' + } + ) + ) + redirect_uri = forms.URLField( + required=False, + label='Callback URL', + max_length=200, + widget=forms.TextInput(attrs={ + 'style': 'width:100%', + 'placeholder': 'OAuth2 callback URL' + }), + help_text= + """
The Callback URL is only used for the authorization process when accessing resources that require OAuth2. At the end of the OAuth2 authorization process, Freesound will redirect the browser to the url specified in this field. In this way your application can be automatically notified when users have given the permissions to access their data. If your application does not support the use of a callback url (generally non web-based applications or non server-based), you must introduce the following url: http://freesound.org/home/app_permissions/permission_granted/. -
See the API docummentation for more information.
""") - description = forms.CharField(label='Description*', widget=forms.Textarea( - attrs={'style': 'width:100%', 'placeholder': 'Tell us something about what you\'re planning to do with this ' - 'API credential (i.e. what kind of project or application you\'re' - ' going to build)'})) - accepted_tos = forms.BooleanField(label=mark_safe('Check this box to accept the terms of use of the Freesound API'), - help_text=False, - required=True, - error_messages={'required': 'You must accept the terms of use in order to ' - 'get access to the API.'}) +
See the API docummentation for more information.
""" + ) + description = forms.CharField( + label='Description*', + widget=forms.Textarea( + attrs={ + 'style': 'width:100%', + 'placeholder': + 'Tell us something about what you\'re planning to do with this ' + 'API credential (i.e. what kind of project or application you\'re' + ' going to build)' + } + ) + ) + accepted_tos = forms.BooleanField( + label=mark_safe( + 'Check this box to accept the terms of use of the Freesound API' + ), + help_text=False, + required=True, + error_messages={'required': 'You must accept the terms of use in order to ' + 'get access to the API.'} + ) def __init__(self, *args, **kwargs): kwargs.update(dict(label_suffix='')) super().__init__(*args, **kwargs) self.fields['accepted_tos'].widget.attrs['class'] = 'bw-checkbox' - - -SEARCH_SORT_OPTIONS_API = [ - ("score", "score desc"), - ("duration_desc", "duration desc"), - ("duration_asc", "duration asc"), - ("created_desc", "created desc"), - ("created_asc", "created asc"), - ("downloads_desc", "num_downloads desc"), - ("downloads_asc", "num_downloads asc"), - ("rating_desc", "avg_rating desc"), - ("rating_asc", "avg_rating asc") - ] + + +SEARCH_SORT_OPTIONS_API = [("score", "score desc"), ("duration_desc", "duration desc"), + ("duration_asc", "duration asc"), ("created_desc", "created desc"), + ("created_asc", "created asc"), ("downloads_desc", "num_downloads desc"), + ("downloads_asc", "num_downloads asc"), ("rating_desc", "avg_rating desc"), + ("rating_asc", "avg_rating asc")] SEARCH_SOUNDS_SORT_DEFAULT_API = "score desc" @@ -161,7 +191,7 @@ def clean_page_size(self): requested_paginate_by = self.cleaned_data[settings.APIV2['PAGE_SIZE_QUERY_PARAM']] try: paginate_by = min(int(requested_paginate_by), settings.APIV2['MAX_PAGE_SIZE']) - except (ValueError, TypeError): # TypeError if None, ValueError if bad input + except (ValueError, TypeError): # TypeError if None, ValueError if bad input paginate_by = settings.APIV2['PAGE_SIZE'] return paginate_by @@ -188,12 +218,13 @@ def construct_link(self, base_url, page=None, filt=None, group_by_pack=None, inc link += f'&filter={my_quote(filt)}' if self.cleaned_data['weights'] is not None: link += f"&weights={self.cleaned_data['weights']}" - if self.original_url_sort_value and not self.original_url_sort_value == SEARCH_SOUNDS_SORT_DEFAULT_API.split(' ')[0]: + if self.original_url_sort_value and not self.original_url_sort_value == SEARCH_SOUNDS_SORT_DEFAULT_API.split( + ' ')[0]: link += f'&sort={self.original_url_sort_value}' if self.cleaned_data['descriptors_filter']: - link += f"&descriptors_filter={self.cleaned_data['descriptors_filter']}" + link += f"&descriptors_filter={self.cleaned_data['descriptors_filter']}" if self.cleaned_data['target']: - link += f"&target={self.cleaned_data['target']}" + link += f"&target={self.cleaned_data['target']}" if include_page: if not page: if self.cleaned_data['page'] and self.cleaned_data['page'] != 1: diff --git a/apiv2/management/commands/basic_api_tests.py b/apiv2/management/commands/basic_api_tests.py index d65d66b2e..476b146b2 100644 --- a/apiv2/management/commands/basic_api_tests.py +++ b/apiv2/management/commands/basic_api_tests.py @@ -49,7 +49,7 @@ def api_request(full_url, type='GET', post_data=None, auth='token', token=None): data = json.dumps(post_data) if auth == 'token': - headers = {'Authorization': f'Token {token}' } + headers = {'Authorization': f'Token {token}'} if type == 'GET': r = requests.get(url, params=params, headers=headers) @@ -65,25 +65,16 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument( - '--base_url', - action='store', - dest='base_url', - default=False, - help='base url where to run the tests') + '--base_url', action='store', dest='base_url', default=False, help='base url where to run the tests' + ) parser.add_argument( - '--token', - action='store', - dest='token', - default=False, - help='api token (client secret) to use') + '--token', action='store', dest='token', default=False, help='api token (client secret) to use' + ) parser.add_argument( - '--section', - action='store', - dest='section', - default=False, - help='section of the tests to run') + '--section', action='store', dest='section', default=False, help='section of the tests to run' + ) - def handle(self, *args, **options): + def handle(self, *args, **options): base_url = options['base_url'] token = options['token'] section = options['section'] diff --git a/apiv2/management/commands/consolidate_api_usage_data.py b/apiv2/management/commands/consolidate_api_usage_data.py index 4c290f31b..c7ff9fded 100644 --- a/apiv2/management/commands/consolidate_api_usage_data.py +++ b/apiv2/management/commands/consolidate_api_usage_data.py @@ -26,7 +26,6 @@ from utils.management_commands import LoggingBaseCommand from apiv2.models import ApiV2Client, APIClientDailyUsageHistory - console_logger = logging.getLogger("console") cache_api_monitoring = caches["api_monitoring"] diff --git a/apiv2/models.py b/apiv2/models.py index 5ed3cc1bb..d47e9c567 100755 --- a/apiv2/models.py +++ b/apiv2/models.py @@ -31,15 +31,13 @@ class ApiV2Client(models.Model): - STATUS_CHOICES = (('OK', 'Approved'), - ('REJ', 'Rejected'), - ('REV', 'Revoked'), - ('PEN', 'Pending')) + STATUS_CHOICES = (('OK', 'Approved'), ('REJ', 'Rejected'), ('REV', 'Revoked'), ('PEN', 'Pending')) DEFAULT_STATUS = 'OK' oauth_client = models.OneToOneField( - Application, related_name='apiv2_client', default=None, null=True, blank=True, on_delete=models.CASCADE) + Application, related_name='apiv2_client', default=None, null=True, blank=True, on_delete=models.CASCADE + ) key = models.CharField(max_length=40, blank=True) user = models.ForeignKey(User, related_name='apiv2_client', on_delete=models.CASCADE) status = models.CharField(max_length=3, default=DEFAULT_STATUS, choices=STATUS_CHOICES) @@ -122,7 +120,7 @@ def get_usage_history(self, n_days_back=30, year=None): try: number_of_requests = self.usage.get(date=date_filter).number_of_requests except APIClientDailyUsageHistory.DoesNotExist: - number_of_requests = 0 + number_of_requests = 0 usage.append((date_filter, number_of_requests)) return sorted(usage, reverse=True) @@ -144,8 +142,8 @@ def client_id(self): @property def client_secret(self): - return self.key # We can't use self.oauth_client.client_secret as it is hashed - + return self.key # We can't use self.oauth_client.client_secret as it is hashed + @property def version(self): return "V2" diff --git a/apiv2/oauth2_urls.py b/apiv2/oauth2_urls.py index 5c44d50ca..2de08b144 100644 --- a/apiv2/oauth2_urls.py +++ b/apiv2/oauth2_urls.py @@ -30,19 +30,23 @@ def https_required(view_func): + def _wrapped_view_func(request, *args, **kwargs): if not request.is_secure() and not settings.DEBUG: return HttpResponse('{"detail": "This resource requires a secure connection (https)"}', status=403) return view_func(request, *args, **kwargs) + return _wrapped_view_func def force_login(view_func): + def _wrapped_view_func(request, *args, **kwargs): logout(request) path = request.build_absolute_uri() - path = path.replace('logout_and_', '') # To avoid loop in this view + path = path.replace('logout_and_', '') # To avoid loop in this view return redirect_to_login(path, reverse('api-login'), REDIRECT_FIELD_NAME) + return _wrapped_view_func @@ -50,6 +54,10 @@ def _wrapped_view_func(request, *args, **kwargs): urlpatterns = ( re_path(r'^authorize[/]*$', https_required(AuthorizationView.as_view()), name="authorize"), - re_path(r'^logout_and_authorize[/]*$', https_required(force_login(AuthorizationView.as_view())), name="logout_and_authorize"), + re_path( + r'^logout_and_authorize[/]*$', + https_required(force_login(AuthorizationView.as_view())), + name="logout_and_authorize" + ), re_path(r'^access_token[/]*$', csrf_exempt(https_required(views.TokenView.as_view())), name="access_token"), ) diff --git a/apiv2/oauth2_validators.py b/apiv2/oauth2_validators.py index 79a5ac189..d2181ee5c 100644 --- a/apiv2/oauth2_validators.py +++ b/apiv2/oauth2_validators.py @@ -22,7 +22,7 @@ def validate_grant_type(self, client_id, grant_type, client, request, *args, **k you to define one allowed authorization grant type per client. Therefore we need to customise this method. """ - assert (grant_type in GRANT_TYPE_MAPPING) # mapping misconfiguration + assert (grant_type in GRANT_TYPE_MAPPING) # mapping misconfiguration if grant_type == AbstractApplication.GRANT_PASSWORD: if request.client.apiv2_client.allow_oauth_passoword_grant: return True diff --git a/apiv2/serializers.py b/apiv2/serializers.py index 5dc89baba..25b5bd416 100644 --- a/apiv2/serializers.py +++ b/apiv2/serializers.py @@ -33,17 +33,16 @@ from utils.similarity_utilities import get_sounds_descriptors from utils.tags import clean_and_split_tags - ################### # SOUND SERIALIZERS ################### -DEFAULT_FIELDS_IN_SOUND_LIST = 'id,name,tags,username,license' # Separated by commas (None = all) +DEFAULT_FIELDS_IN_SOUND_LIST = 'id,name,tags,username,license' # Separated by commas (None = all) DEFAULT_FIELDS_IN_SOUND_DETAIL = 'id,url,name,tags,description,geotag,created,license,type,channels,filesize,bitrate,' + \ 'bitdepth,duration,samplerate,username,pack,pack_name,download,bookmark,previews,images,' + \ 'num_downloads,avg_rating,num_ratings,rate,comments,num_comments,comment,similar_sounds,' + \ 'analysis,analysis_frames,analysis_stats,is_explicit' # All except for analyzers -DEFAULT_FIELDS_IN_PACK_DETAIL = None # Separated by commas (None = all) +DEFAULT_FIELDS_IN_PACK_DETAIL = None # Separated by commas (None = all) def get_sound_analyzers_output_helper(sound, fallback_to_db=True): @@ -81,10 +80,10 @@ def __init__(self, *args, **kwargs): self.sound_analysis_data = kwargs.pop('sound_analysis_data', {}) super().__init__(*args, **kwargs) requested_fields = self.context['request'].GET.get("fields", self.default_fields) - if not requested_fields: # If parameter is in url but parameter is empty, set to default + if not requested_fields: # If parameter is in url but parameter is empty, set to default requested_fields = self.default_fields - if requested_fields == '*': # If parameter is *, return all fields + if requested_fields == '*': # If parameter is *, return all fields requested_fields = ','.join(self.fields.keys()) if requested_fields: @@ -95,52 +94,56 @@ def __init__(self, *args, **kwargs): class Meta: model = Sound - fields = ('id', - 'url', - 'name', - 'tags', - 'description', - 'geotag', - 'created', - 'license', - 'type', - 'channels', - 'filesize', - 'bitrate', - 'bitdepth', - 'duration', - 'samplerate', - 'username', - 'pack', - 'pack_name', - 'download', - 'bookmark', - 'previews', - 'images', - 'num_downloads', - 'avg_rating', - 'num_ratings', - 'rate', - 'comments', - 'num_comments', - 'comment', - 'similar_sounds', - 'analysis', - 'analysis_frames', - 'analysis_stats', - 'ac_analysis', # Kept for legacy reasons only as it is also contained in 'analyzers_output' - 'analyzers_output', - 'is_explicit', - 'score', - ) + fields = ( + 'id', + 'url', + 'name', + 'tags', + 'description', + 'geotag', + 'created', + 'license', + 'type', + 'channels', + 'filesize', + 'bitrate', + 'bitdepth', + 'duration', + 'samplerate', + 'username', + 'pack', + 'pack_name', + 'download', + 'bookmark', + 'previews', + 'images', + 'num_downloads', + 'avg_rating', + 'num_ratings', + 'rate', + 'comments', + 'num_comments', + 'comment', + 'similar_sounds', + 'analysis', + 'analysis_frames', + 'analysis_stats', + 'ac_analysis', # Kept for legacy reasons only as it is also contained in 'analyzers_output' + 'analyzers_output', + 'is_explicit', + 'score', + ) url = serializers.SerializerMethodField() + def get_url(self, obj): username = self.get_username(obj) - return prepend_base(reverse('sound', args=[username, obj.id]), - request_is_secure=self.context['request'].is_secure()) + return prepend_base( + reverse('sound', args=[username, obj.id]), request_is_secure=self.context['request'].is_secure() + ) username = serializers.SerializerMethodField() + def get_username(self, obj): try: return obj.username @@ -148,6 +151,7 @@ def get_username(self, obj): return obj.user.username score = serializers.SerializerMethodField() + def get_score(self, obj): if self.score_map: return self.score_map.get(obj.id) @@ -155,14 +159,17 @@ def get_score(self, obj): return None name = serializers.SerializerMethodField() + def get_name(self, obj): return obj.original_filename created = serializers.SerializerMethodField() + def get_created(self, obj): return obj.created.replace(microsecond=0) tags = serializers.SerializerMethodField() + def get_tags(self, obj): try: return obj.tag_array @@ -170,6 +177,7 @@ def get_tags(self, obj): return [tagged.tag.name for tagged in obj.tags.select_related("tag").all()] license = serializers.SerializerMethodField() + def get_license(self, obj): try: return obj.license_deed_url @@ -177,17 +185,21 @@ def get_license(self, obj): return obj.license.deed_url pack = serializers.SerializerMethodField() + def get_pack(self, obj): try: if obj.pack_id: - return prepend_base(reverse('apiv2-pack-instance', args=[obj.pack_id]), - request_is_secure=self.context['request'].is_secure()) + return prepend_base( + reverse('apiv2-pack-instance', args=[obj.pack_id]), + request_is_secure=self.context['request'].is_secure() + ) else: return None except: return None pack_name = serializers.SerializerMethodField() + def get_pack_name(self, obj): try: return obj.pack_name @@ -195,105 +207,152 @@ def get_pack_name(self, obj): return None previews = serializers.SerializerMethodField() + def get_previews(self, obj): return { - 'preview-hq-mp3': prepend_base(obj.locations("preview.HQ.mp3.url"), - request_is_secure=self.context['request'].is_secure()), - 'preview-hq-ogg': prepend_base(obj.locations("preview.HQ.ogg.url"), - request_is_secure=self.context['request'].is_secure()), - 'preview-lq-mp3': prepend_base(obj.locations("preview.LQ.mp3.url"), - request_is_secure=self.context['request'].is_secure()), - 'preview-lq-ogg': prepend_base(obj.locations("preview.LQ.ogg.url"), - request_is_secure=self.context['request'].is_secure()), + 'preview-hq-mp3': + prepend_base( + obj.locations("preview.HQ.mp3.url"), request_is_secure=self.context['request'].is_secure() + ), + 'preview-hq-ogg': + prepend_base( + obj.locations("preview.HQ.ogg.url"), request_is_secure=self.context['request'].is_secure() + ), + 'preview-lq-mp3': + prepend_base( + obj.locations("preview.LQ.mp3.url"), request_is_secure=self.context['request'].is_secure() + ), + 'preview-lq-ogg': + prepend_base( + obj.locations("preview.LQ.ogg.url"), request_is_secure=self.context['request'].is_secure() + ), } images = serializers.SerializerMethodField() + def get_images(self, obj): return { - 'waveform_m': prepend_base(obj.locations("display.wave.M.url"), - request_is_secure=self.context['request'].is_secure()), - 'waveform_l': prepend_base(obj.locations("display.wave.L.url"), - request_is_secure=self.context['request'].is_secure()), - 'spectral_m': prepend_base(obj.locations("display.spectral.M.url"), - request_is_secure=self.context['request'].is_secure()), - 'spectral_l': prepend_base(obj.locations("display.spectral.L.url"), - request_is_secure=self.context['request'].is_secure()), - 'waveform_bw_m': prepend_base(obj.locations("display.wave_bw.M.url"), - request_is_secure=self.context['request'].is_secure()), - 'waveform_bw_l': prepend_base(obj.locations("display.wave_bw.L.url"), - request_is_secure=self.context['request'].is_secure()), - 'spectral_bw_m': prepend_base(obj.locations("display.spectral_bw.M.url"), - request_is_secure=self.context['request'].is_secure()), - 'spectral_bw_l': prepend_base(obj.locations("display.spectral_bw.L.url"), - request_is_secure=self.context['request'].is_secure()), + 'waveform_m': + prepend_base( + obj.locations("display.wave.M.url"), request_is_secure=self.context['request'].is_secure() + ), + 'waveform_l': + prepend_base( + obj.locations("display.wave.L.url"), request_is_secure=self.context['request'].is_secure() + ), + 'spectral_m': + prepend_base( + obj.locations("display.spectral.M.url"), request_is_secure=self.context['request'].is_secure() + ), + 'spectral_l': + prepend_base( + obj.locations("display.spectral.L.url"), request_is_secure=self.context['request'].is_secure() + ), + 'waveform_bw_m': + prepend_base( + obj.locations("display.wave_bw.M.url"), request_is_secure=self.context['request'].is_secure() + ), + 'waveform_bw_l': + prepend_base( + obj.locations("display.wave_bw.L.url"), request_is_secure=self.context['request'].is_secure() + ), + 'spectral_bw_m': + prepend_base( + obj.locations("display.spectral_bw.M.url"), request_is_secure=self.context['request'].is_secure() + ), + 'spectral_bw_l': + prepend_base( + obj.locations("display.spectral_bw.L.url"), request_is_secure=self.context['request'].is_secure() + ), } def get_or_compute_analysis_state_essentia_exists(self, sound_obj): if hasattr(sound_obj, 'analysis_state_essentia_exists'): return sound_obj.analysis_state_essentia_exists else: - return SoundAnalysis.objects.filter(analyzer=settings.FREESOUND_ESSENTIA_EXTRACTOR_NAME, analysis_status="OK", sound_id=sound_obj.id).exists() + return SoundAnalysis.objects.filter( + analyzer=settings.FREESOUND_ESSENTIA_EXTRACTOR_NAME, analysis_status="OK", sound_id=sound_obj.id + ).exists() analysis = serializers.SerializerMethodField() + def get_analysis(self, obj): - raise NotImplementedError # Should be implemented in subclasses + raise NotImplementedError # Should be implemented in subclasses analysis_frames = serializers.SerializerMethodField() + def get_analysis_frames(self, obj): if not self.get_or_compute_analysis_state_essentia_exists(obj): return None - return prepend_base(obj.locations('analysis.frames.url'), - request_is_secure=self.context['request'].is_secure()) + return prepend_base(obj.locations('analysis.frames.url'), request_is_secure=self.context['request'].is_secure()) analysis_stats = serializers.SerializerMethodField() + def get_analysis_stats(self, obj): if not self.get_or_compute_analysis_state_essentia_exists(obj): return None - return prepend_base(reverse('apiv2-sound-analysis', args=[obj.id]), - request_is_secure=self.context['request'].is_secure()) + return prepend_base( + reverse('apiv2-sound-analysis', args=[obj.id]), request_is_secure=self.context['request'].is_secure() + ) similar_sounds = serializers.SerializerMethodField() + def get_similar_sounds(self, obj): if obj.similarity_state != 'OK': return None - return prepend_base(reverse('apiv2-similarity-sound', args=[obj.id]), - request_is_secure=self.context['request'].is_secure()) + return prepend_base( + reverse('apiv2-similarity-sound', args=[obj.id]), request_is_secure=self.context['request'].is_secure() + ) download = serializers.SerializerMethodField() + def get_download(self, obj): - return prepend_base(reverse('apiv2-sound-download', args=[obj.id]), - request_is_secure=self.context['request'].is_secure()) + return prepend_base( + reverse('apiv2-sound-download', args=[obj.id]), request_is_secure=self.context['request'].is_secure() + ) rate = serializers.SerializerMethodField() + def get_rate(self, obj): - return prepend_base(reverse('apiv2-user-create-rating', args=[obj.id]), - request_is_secure=self.context['request'].is_secure()) + return prepend_base( + reverse('apiv2-user-create-rating', args=[obj.id]), request_is_secure=self.context['request'].is_secure() + ) bookmark = serializers.SerializerMethodField() + def get_bookmark(self, obj): - return prepend_base(reverse('apiv2-user-create-bookmark', args=[obj.id]), - request_is_secure=self.context['request'].is_secure()) + return prepend_base( + reverse('apiv2-user-create-bookmark', args=[obj.id]), request_is_secure=self.context['request'].is_secure() + ) comment = serializers.SerializerMethodField() + def get_comment(self, obj): - return prepend_base(reverse('apiv2-user-create-comment', args=[obj.id]), - request_is_secure=self.context['request'].is_secure()) + return prepend_base( + reverse('apiv2-user-create-comment', args=[obj.id]), request_is_secure=self.context['request'].is_secure() + ) ratings = serializers.SerializerMethodField() + def get_ratings(self, obj): - return prepend_base(reverse('apiv2-sound-ratings', args=[obj.id]), - request_is_secure=self.context['request'].is_secure()) + return prepend_base( + reverse('apiv2-sound-ratings', args=[obj.id]), request_is_secure=self.context['request'].is_secure() + ) avg_rating = serializers.SerializerMethodField() + def get_avg_rating(self, obj): return obj.avg_rating / 2 comments = serializers.SerializerMethodField() + def get_comments(self, obj): - return prepend_base(reverse('apiv2-sound-comments', args=[obj.id]), - request_is_secure=self.context['request'].is_secure()) + return prepend_base( + reverse('apiv2-sound-comments', args=[obj.id]), request_is_secure=self.context['request'].is_secure() + ) geotag = serializers.SerializerMethodField() + def get_geotag(self, obj): if obj.geotag: return str(obj.geotag.lat) + " " + str(obj.geotag.lon) @@ -301,14 +360,17 @@ def get_geotag(self, obj): return None ac_analysis = serializers.SerializerMethodField() + def get_ac_analysis(self, obj): - raise NotImplementedError # Should be implemented in subclasses + raise NotImplementedError # Should be implemented in subclasses analyzers_output = serializers.SerializerMethodField() + def get_analyzers_output(self, obj): - raise NotImplementedError # Should be implemented in subclasses - + raise NotImplementedError # Should be implemented in subclasses + is_explicit = serializers.SerializerMethodField() + def get_is_explicit(self, obj): return obj.is_explicit @@ -403,36 +465,45 @@ class UserSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = User - fields = ('url', - 'username', - 'about', - 'home_page', - 'avatar', - 'date_joined', - 'num_sounds', - 'sounds', - 'num_packs', - 'packs', - 'num_posts', - 'num_comments', - ) + fields = ( + 'url', + 'username', + 'about', + 'home_page', + 'avatar', + 'date_joined', + 'num_sounds', + 'sounds', + 'num_packs', + 'packs', + 'num_posts', + 'num_comments', + ) url = serializers.SerializerMethodField() + def get_url(self, obj): - return prepend_base(reverse('account', args=[obj.username]), - request_is_secure=self.context['request'].is_secure()) + return prepend_base( + reverse('account', args=[obj.username]), request_is_secure=self.context['request'].is_secure() + ) sounds = serializers.SerializerMethodField() + def get_sounds(self, obj): - return prepend_base(reverse('apiv2-user-sound-list', args=[obj.username]), - request_is_secure=self.context['request'].is_secure()) + return prepend_base( + reverse('apiv2-user-sound-list', args=[obj.username]), + request_is_secure=self.context['request'].is_secure() + ) packs = serializers.SerializerMethodField() + def get_packs(self, obj): - return prepend_base(reverse('apiv2-user-packs', args=[obj.username]), - request_is_secure=self.context['request'].is_secure()) + return prepend_base( + reverse('apiv2-user-packs', args=[obj.username]), request_is_secure=self.context['request'].is_secure() + ) avatar = serializers.SerializerMethodField() + def get_avatar(self, obj): if obj.profile.locations()['avatar']['S']['url'] is None: # User has no avatar, return None in paths @@ -443,35 +514,50 @@ def get_avatar(self, obj): } else: return { - 'small': prepend_base(obj.profile.locations()['avatar']['S']['url'], - request_is_secure=self.context['request'].is_secure()), - 'medium': prepend_base(obj.profile.locations()['avatar']['M']['url'], - request_is_secure=self.context['request'].is_secure()), - 'large': prepend_base(obj.profile.locations()['avatar']['L']['url'], - request_is_secure=self.context['request'].is_secure()), + 'small': + prepend_base( + obj.profile.locations()['avatar']['S']['url'], + request_is_secure=self.context['request'].is_secure() + ), + 'medium': + prepend_base( + obj.profile.locations()['avatar']['M']['url'], + request_is_secure=self.context['request'].is_secure() + ), + 'large': + prepend_base( + obj.profile.locations()['avatar']['L']['url'], + request_is_secure=self.context['request'].is_secure() + ), } about = serializers.SerializerMethodField() + def get_about(self, obj): return obj.profile.about or "" home_page = serializers.SerializerMethodField() + def get_home_page(self, obj): return obj.profile.home_page or "" num_sounds = serializers.SerializerMethodField() + def get_num_sounds(self, obj): return obj.sounds.filter(moderation_state="OK", processing_state="OK").count() num_packs = serializers.SerializerMethodField() + def get_num_packs(self, obj): return obj.pack_set.all().count() num_posts = serializers.SerializerMethodField() + def get_num_posts(self, obj): return obj.profile.num_posts num_comments = serializers.SerializerMethodField() + def get_num_comments(self, obj): return obj.comment_set.all().count() @@ -485,35 +571,34 @@ class PackSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Pack - fields = ('id', - 'url', - 'description', - 'created', - 'name', - 'username', - 'num_sounds', - 'sounds', - 'num_downloads') + fields = ('id', 'url', 'description', 'created', 'name', 'username', 'num_sounds', 'sounds', 'num_downloads') url = serializers.SerializerMethodField() + def get_url(self, obj): - return prepend_base(reverse('pack', args=[obj.user.username, obj.id]), - request_is_secure=self.context['request'].is_secure()) + return prepend_base( + reverse('pack', args=[obj.user.username, obj.id]), request_is_secure=self.context['request'].is_secure() + ) sounds = serializers.SerializerMethodField() + def get_sounds(self, obj): - return prepend_base(reverse('apiv2-pack-sound-list', args=[obj.id]), - request_is_secure=self.context['request'].is_secure()) + return prepend_base( + reverse('apiv2-pack-sound-list', args=[obj.id]), request_is_secure=self.context['request'].is_secure() + ) username = serializers.SerializerMethodField() + def get_username(self, obj): return obj.user.username description = serializers.SerializerMethodField() + def get_description(self, obj): return obj.description or "" created = serializers.SerializerMethodField() + def get_created(self, obj): return obj.created.replace(microsecond=0) @@ -527,39 +612,48 @@ class BookmarkCategorySerializer(serializers.HyperlinkedModelSerializer): class Meta: model = BookmarkCategory - fields = ('id', - 'url', - 'name', - 'num_sounds', - 'sounds') + fields = ('id', 'url', 'name', 'num_sounds', 'sounds') url = serializers.SerializerMethodField() + def get_url(self, obj): if obj.id != 0: - return prepend_base(reverse('bookmarks-for-user-for-category', args=[obj.user.username, obj.id]), - request_is_secure=self.context['request'].is_secure()) + return prepend_base( + reverse('bookmarks-for-user-for-category', args=[obj.user.username, obj.id]), + request_is_secure=self.context['request'].is_secure() + ) else: - return prepend_base(reverse('bookmarks-for-user', args=[obj.user.username]), - request_is_secure=self.context['request'].is_secure()) + return prepend_base( + reverse('bookmarks-for-user', args=[obj.user.username]), + request_is_secure=self.context['request'].is_secure() + ) num_sounds = serializers.SerializerMethodField() + def get_num_sounds(self, obj): - if obj.id != 0: # Category is not 'uncategorized' + if obj.id != 0: # Category is not 'uncategorized' return obj.bookmarks.filter(sound__processing_state="OK", sound__moderation_state="OK").count() else: - return Bookmark.objects.select_related("sound").filter(user__username=obj.user.username, - category=None).count() + return Bookmark.objects.select_related("sound").filter( + user__username=obj.user.username, category=None + ).count() sounds = serializers.SerializerMethodField() + def get_sounds(self, obj): - return prepend_base(reverse('apiv2-me-bookmark-category-sounds', args=[obj.id]), - request_is_secure=self.context['request'].is_secure()) + return prepend_base( + reverse('apiv2-me-bookmark-category-sounds', args=[obj.id]), + request_is_secure=self.context['request'].is_secure() + ) class CreateBookmarkSerializer(serializers.Serializer): - category = serializers.CharField(max_length=128, required=False, - help_text='Not required. Name you want to give to the category under which the ' - 'bookmark will be classified (leave empty for no category).') + category = serializers.CharField( + max_length=128, + required=False, + help_text='Not required. Name you want to give to the category under which the ' + 'bookmark will be classified (leave empty for no category).' + ) def validate_category(self, value): if value.isspace(): @@ -573,8 +667,9 @@ def validate_category(self, value): class CreateRatingSerializer(serializers.Serializer): - rating = serializers.IntegerField(required=True, - help_text='Required. Chose an integer rating between 0 and 5 (both included).') + rating = serializers.IntegerField( + required=True, help_text='Required. Chose an integer rating between 0 and 5 (both included).' + ) def validate_rating(self, value): if (value not in [0, 1, 2, 3, 4, 5]): @@ -591,15 +686,15 @@ class SoundCommentsSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = Comment - fields = ('username', - 'comment', - 'created') + fields = ('username', 'comment', 'created') username = serializers.SerializerMethodField() + def get_username(self, obj): return obj.user.username created = serializers.SerializerMethodField() + def get_created(self, obj): return obj.created.replace(microsecond=0) @@ -620,8 +715,10 @@ def validate_comment(self, value): def validate_license(value): if value not in [key for key, name in LICENSE_CHOICES]: - raise serializers.ValidationError('Invalid License, must be either \'Attribution\', \'Attribution ' - 'NonCommercial\' or \'Creative Commons 0\'.') + raise serializers.ValidationError( + 'Invalid License, must be either \'Attribution\', \'Attribution ' + 'NonCommercial\' or \'Creative Commons 0\'.' + ) return value @@ -668,8 +765,10 @@ def validate_geotag(value): except: fails = True if fails: - raise serializers.ValidationError('Geotag should have the format \'float,float,integer\' (for latitude, ' - 'longitude and zoom respectively).') + raise serializers.ValidationError( + 'Geotag should have the format \'float,float,integer\' (for latitude, ' + 'longitude and zoom respectively).' + ) else: # Check that ranges are corrent if float(data[0]) > 90 or float(data[0]) < -90: @@ -682,34 +781,50 @@ def validate_geotag(value): LICENSE_CHOICES = ( - ('Attribution', 'Attribution'), - ('Attribution NonCommercial', 'Attribution NonCommercial'), - ('Creative Commons 0', 'Creative Commons 0'),) + ('Attribution', 'Attribution'), + ('Attribution NonCommercial', 'Attribution NonCommercial'), + ('Creative Commons 0', 'Creative Commons 0'), +) class SoundDescriptionSerializer(serializers.Serializer): - upload_filename = serializers.CharField(max_length=512, - help_text='Must match a filename from \'Pending Uploads\' resource.') - name = serializers.CharField(max_length=512, required=False, - help_text='Not required. Name you want to give to the sound (by default it will be ' - 'the original filename).') - tags = serializers.CharField(max_length=512, - help_text='Separate tags with spaces. Join multi-word tags with dashes.') + upload_filename = serializers.CharField( + max_length=512, help_text='Must match a filename from \'Pending Uploads\' resource.' + ) + name = serializers.CharField( + max_length=512, + required=False, + help_text='Not required. Name you want to give to the sound (by default it will be ' + 'the original filename).' + ) + tags = serializers.CharField( + max_length=512, help_text='Separate tags with spaces. Join multi-word tags with dashes.' + ) description = serializers.CharField(help_text='Textual description of the sound.') - license = serializers.ChoiceField(choices=LICENSE_CHOICES, - help_text='License for the sound. Must be either \'Attribution\', \'Attribution ' - 'NonCommercial\' or \'Creative Commons 0\'.') - pack = serializers.CharField(required=False, help_text='Not required. Pack name (if there is no such pack with ' - 'that name, a new one will be created).') - geotag = serializers.CharField(max_length=100, required=False, - help_text='Not required. Latitude, longitude and zoom values in the form ' - 'lat,lon,zoom (ex: \'2.145677,3.22345,14\').') + license = serializers.ChoiceField( + choices=LICENSE_CHOICES, + help_text='License for the sound. Must be either \'Attribution\', \'Attribution ' + 'NonCommercial\' or \'Creative Commons 0\'.' + ) + pack = serializers.CharField( + required=False, + help_text='Not required. Pack name (if there is no such pack with ' + 'that name, a new one will be created).' + ) + geotag = serializers.CharField( + max_length=100, + required=False, + help_text='Not required. Latitude, longitude and zoom values in the form ' + 'lat,lon,zoom (ex: \'2.145677,3.22345,14\').' + ) def validate_upload_filename(self, value): if 'not_yet_described_audio_files' in self.context: if value not in self.context['not_yet_described_audio_files']: - raise serializers.ValidationError('Upload filename (%s) must match with a filename from ' - '\'Pending Uploads\' resource.' % value) + raise serializers.ValidationError( + 'Upload filename (%s) must match with a filename from ' + '\'Pending Uploads\' resource.' % value + ) return value def validate_geotag(self, value): @@ -729,23 +844,38 @@ def validate_pack(self, value): class EditSoundDescriptionSerializer(serializers.Serializer): - name = serializers.CharField(max_length=512, required=False, - help_text='Not required. New name you want to give to the sound.') - tags = serializers.CharField(max_length=512, required=False, - help_text='Not required. Tags that should be assigned to the sound (note that ' - 'existing ones will be deleted). Separate tags with spaces. Join multi-word ' - 'tags with dashes.') - description = serializers.CharField(required=False, - help_text='Not required. New textual description for the sound.') - license = serializers.ChoiceField(required=False, allow_blank=True, choices=LICENSE_CHOICES, - help_text='Not required. New license for the sound. Must be either ' - '\'Attribution\', \'Attribution NonCommercial\' or ' - '\'Creative Commons 0\'.') - pack = serializers.CharField(required=False, help_text='Not required. New pack name for the sound (if there is no ' - 'such pack with that name, a new one will be created).') - geotag = serializers.CharField(required=False, max_length=100, - help_text='Not required. New geotag for the sound. Latitude, longitude and zoom ' - 'values in the form lat,lon,zoom (ex: \'2.145677,3.22345,14\').') + name = serializers.CharField( + max_length=512, required=False, help_text='Not required. New name you want to give to the sound.' + ) + tags = serializers.CharField( + max_length=512, + required=False, + help_text='Not required. Tags that should be assigned to the sound (note that ' + 'existing ones will be deleted). Separate tags with spaces. Join multi-word ' + 'tags with dashes.' + ) + description = serializers.CharField( + required=False, help_text='Not required. New textual description for the sound.' + ) + license = serializers.ChoiceField( + required=False, + allow_blank=True, + choices=LICENSE_CHOICES, + help_text='Not required. New license for the sound. Must be either ' + '\'Attribution\', \'Attribution NonCommercial\' or ' + '\'Creative Commons 0\'.' + ) + pack = serializers.CharField( + required=False, + help_text='Not required. New pack name for the sound (if there is no ' + 'such pack with that name, a new one will be created).' + ) + geotag = serializers.CharField( + required=False, + max_length=100, + help_text='Not required. New geotag for the sound. Latitude, longitude and zoom ' + 'values in the form lat,lon,zoom (ex: \'2.145677,3.22345,14\').' + ) def validate_geotag(self, value): return validate_geotag(value) @@ -764,26 +894,46 @@ def validate_pack(self, value): class UploadAndDescribeAudioFileSerializer(serializers.Serializer): - audiofile = serializers.FileField(max_length=100, allow_empty_file=False, - help_text='Required. Must be in .wav, .aif, .flac, .ogg or .mp3 format.') - name = serializers.CharField(max_length=512, required=False, - help_text='Not required. Name you want to give to the sound (by default it will be ' - 'the original filename).') - tags = serializers.CharField(max_length=512, required=False, - help_text='Only required if providing file description. Separate tags with spaces. ' - 'Join multi-word tags with dashes.') - description = serializers.CharField(required=False, - help_text='Only required if providing file description. Textual ' - 'description of the sound.') - license = serializers.ChoiceField(required=False, allow_blank=True, choices=LICENSE_CHOICES, - help_text='Only required if providing file description. License for the sound. ' - 'Must be either \'Attribution\', \'Attribution NonCommercial\' ' - 'or \'Creative Commons 0\'.') - pack = serializers.CharField(help_text='Not required. Pack name (if there is no such pack with that name, a new ' - 'one will be created).', required=False) - geotag = serializers.CharField(max_length=100, - help_text='Not required. Latitude, longitude and zoom values in the form ' - 'lat,lon,zoom (ex: \'2.145677,3.22345,14\').', required=False) + audiofile = serializers.FileField( + max_length=100, + allow_empty_file=False, + help_text='Required. Must be in .wav, .aif, .flac, .ogg or .mp3 format.' + ) + name = serializers.CharField( + max_length=512, + required=False, + help_text='Not required. Name you want to give to the sound (by default it will be ' + 'the original filename).' + ) + tags = serializers.CharField( + max_length=512, + required=False, + help_text='Only required if providing file description. Separate tags with spaces. ' + 'Join multi-word tags with dashes.' + ) + description = serializers.CharField( + required=False, help_text='Only required if providing file description. Textual ' + 'description of the sound.' + ) + license = serializers.ChoiceField( + required=False, + allow_blank=True, + choices=LICENSE_CHOICES, + help_text='Only required if providing file description. License for the sound. ' + 'Must be either \'Attribution\', \'Attribution NonCommercial\' ' + 'or \'Creative Commons 0\'.' + ) + pack = serializers.CharField( + help_text='Not required. Pack name (if there is no such pack with that name, a new ' + 'one will be created).', + required=False + ) + geotag = serializers.CharField( + max_length=100, + help_text='Not required. Latitude, longitude and zoom values in the form ' + 'lat,lon,zoom (ex: \'2.145677,3.22345,14\').', + required=False + ) def is_providing_description(self, attrs): if 'name' in attrs or 'license' in attrs or 'tags' in attrs or 'geotag' in attrs or 'pack' in attrs \ @@ -841,13 +991,16 @@ def validate(self, data): # SIMILARITY SERIALIZERS ######################## - ALLOWED_ANALYSIS_EXTENSIONS = ['json'] + class SimilarityFileSerializer(serializers.Serializer): - analysis_file = serializers.FileField(max_length=100, allow_empty_file=False, - help_text='Analysis file created with the latest freesound extractor. ' - 'Must be in .json format.') + analysis_file = serializers.FileField( + max_length=100, + allow_empty_file=False, + help_text='Analysis file created with the latest freesound extractor. ' + 'Must be in .json format.' + ) def validate_analysis_file(self, value): try: diff --git a/apiv2/templatetags/apiv2_templatetags.py b/apiv2/templatetags/apiv2_templatetags.py index a5a20e87f..e4b0d3e4e 100644 --- a/apiv2/templatetags/apiv2_templatetags.py +++ b/apiv2/templatetags/apiv2_templatetags.py @@ -7,5 +7,7 @@ @register.simple_tag def next_url_for_login(client_id, response_type, state): - return quote("%s?client_id=%s&response_type=%s&state=%s" % - (reverse('oauth2_provider:authorize'), client_id, response_type, state)) + return quote( + "%s?client_id=%s&response_type=%s&state=%s" % + (reverse('oauth2_provider:authorize'), client_id, response_type, state) + ) diff --git a/apiv2/tests.py b/apiv2/tests.py index e77144471..9a0ce7942 100755 --- a/apiv2/tests.py +++ b/apiv2/tests.py @@ -52,16 +52,14 @@ def test_pack_views_response_ok(self): self.assertEqual(resp.status_code, 200) # 200 response on pack instance sounds list (make it return all fields) - resp = self.client.get(reverse('apiv2-pack-sound-list', kwargs={'pk': packs[0].id}) + '?fields=*') + resp = self.client.get(reverse('apiv2-pack-sound-list', kwargs={'pk': packs[0].id}) + '?fields=*') self.assertEqual(resp.status_code, 200) # 200 response on pack instance download # This test uses a https connection. - resp = self.client.get(reverse('apiv2-pack-download', - kwargs={'pk': packs[0].id}), secure=True) + resp = self.client.get(reverse('apiv2-pack-download', kwargs={'pk': packs[0].id}), secure=True) self.assertEqual(resp.status_code, 200) - def test_basic_user_response_ok(self): user, packs, sounds = create_user_and_sounds(num_sounds=5, num_packs=1) @@ -79,7 +77,6 @@ def test_basic_user_response_ok(self): resp = self.client.get(reverse('api-logout'), secure=True) self.assertEqual(resp.status_code, 302) - def test_user_views_response_ok(self): user, packs, sounds = create_user_and_sounds(num_sounds=5, num_packs=1) for sound in sounds: @@ -95,7 +92,6 @@ def test_user_views_response_ok(self): resp = self.client.get(reverse('apiv2-user-sound-list', kwargs={'username': user.username}) + '?fields=*') self.assertEqual(resp.status_code, 200) - def test_sound_views_response_ok(self): user, packs, sounds = create_user_and_sounds(num_sounds=5, num_packs=1) for sound in sounds: @@ -108,7 +104,6 @@ def test_sound_views_response_ok(self): self.assertEqual(resp.status_code, 200) - class TestAPI(TestCase): fixtures = ['licenses'] @@ -116,20 +111,17 @@ def test_cors_header(self): # Create App to login using token user, packs, sounds = create_user_and_sounds(num_sounds=5, num_packs=1) - c = ApiV2Client(user=user, status='OK', redirect_uri="https://freesound.com", - url="https://freesound.com", name="test") + c = ApiV2Client( + user=user, status='OK', redirect_uri="https://freesound.com", url="https://freesound.com", name="test" + ) c.save() sound = sounds[0] sound.change_processing_state("OK") sound.change_moderation_state("OK") - headers = { - 'HTTP_AUTHORIZATION': f'Token {c.key}', - 'HTTP_ORIGIN': 'https://www.google.com' - } - resp = self.client.options(reverse('apiv2-sound-instance', - kwargs={'pk': sound.id}), secure=True, **headers) + headers = {'HTTP_AUTHORIZATION': f'Token {c.key}', 'HTTP_ORIGIN': 'https://www.google.com'} + resp = self.client.options(reverse('apiv2-sound-instance', kwargs={'pk': sound.id}), secure=True, **headers) self.assertEqual(resp.status_code, 200) # Check if header is present self.assertEqual(resp['ACCESS-CONTROL-ALLOW-ORIGIN'], '*') @@ -138,8 +130,9 @@ def test_encoding(self): # Create App to login using token user, packs, sounds = create_user_and_sounds(num_sounds=5, num_packs=1) - c = ApiV2Client(user=user, status='OK', redirect_uri="https://freesound.com", - url="https://freesound.com", name="test") + c = ApiV2Client( + user=user, status='OK', redirect_uri="https://freesound.com", url="https://freesound.com", name="test" + ) c.save() sound = sounds[0] @@ -150,22 +143,29 @@ def test_encoding(self): 'HTTP_AUTHORIZATION': f'Token {c.key}', } # make query that can't be decoded - resp = self.client.options("/apiv2/search/text/?query=ambient&filter=tag:(rain%20OR%CAfe)", secure=True, **headers) + resp = self.client.options( + "/apiv2/search/text/?query=ambient&filter=tag:(rain%20OR%CAfe)", secure=True, **headers + ) self.assertEqual(resp.status_code, 200) class ApiSearchPaginatorTest(TestCase): + def test_page(self): paginator = ApiSearchPaginator([1, 2, 3, 4, 5], 5, 2) page = paginator.page(2) - self.assertEqual(page, {'object_list': [1, 2, 3, 4, 5], - 'has_next': True, - 'has_previous': True, - 'has_other_pages': True, - 'next_page_number': 3, - 'previous_page_number': 1, - 'page_num': 2}) + self.assertEqual( + page, { + 'object_list': [1, 2, 3, 4, 5], + 'has_next': True, + 'has_previous': True, + 'has_other_pages': True, + 'next_page_number': 3, + 'previous_page_number': 1, + 'page_num': 2 + } + ) class TestSoundCombinedSearchFormAPI(SimpleTestCase): @@ -335,8 +335,8 @@ def test_num_queries(self): Site.objects.get_current() field_sets = [ - '', # default fields - ','.join(SoundListSerializer.Meta.fields), # all fields + '', # default fields + ','.join(SoundListSerializer.Meta.fields), # all fields ] # Test when serializing a single sound @@ -388,17 +388,22 @@ def test_urls_length_validation(self): """ user = User.objects.create_user("testuser") self.client.force_login(user) - resp = self.client.post(reverse('apiv2-apply'), data={ - 'name': 'Name for the app', - 'url': 'http://example.com/a/super/long/paaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaath', - 'redirect_uri': 'http://example.com/a/super/long/paaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' - 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaath', - 'description': 'test description', - 'accepted_tos': '1', - }) + resp = self.client.post( + reverse('apiv2-apply'), + data={ + 'name': 'Name for the app', + 'url': + 'http://example.com/a/super/long/paaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaath', + 'redirect_uri': + 'http://example.com/a/super/long/paaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaath', + 'description': 'test description', + 'accepted_tos': '1', + } + ) self.assertEqual(resp.status_code, 200) self.assertIn('redirect_uri', resp.context['form'].errors) self.assertIn('url', resp.context['form'].errors) @@ -423,17 +428,18 @@ def setUp(self): # Get access token for end_user resp = self.client.post( - reverse('oauth2_provider:access_token'), - { + reverse('oauth2_provider:access_token'), { 'client_id': client.client_id, 'grant_type': 'password', 'username': self.end_user.username, 'password': self.end_user_password, - }, secure=True) + }, + secure=True + ) self.auth_headers = { 'HTTP_AUTHORIZATION': f'Bearer {resp.json()["access_token"]}', } - + # Create sounds and content _, _, sounds = create_user_and_sounds(user=self.end_user, num_sounds=5, num_packs=1) for sound in sounds: @@ -451,18 +457,26 @@ def test_me_resource(self): resp = self.client.get(reverse('apiv2-me'), secure=True, **self.auth_headers) self.assertEqual(resp.status_code, 200) self.assertEqual(resp.json()['username'], self.end_user.username) - + def test_bookmark_resources(self): # 200 response on list of bookmark categories resp = self.client.get(reverse('apiv2-me-bookmark-categories'), secure=True, **self.auth_headers) self.assertEqual(resp.status_code, 200) # 200 response on getting sounds for bookmark category without name - resp = self.client.get(reverse('apiv2-me-bookmark-category-sounds', kwargs={'category_id': 0}) + '?fields=*', secure=True, **self.auth_headers) + resp = self.client.get( + reverse('apiv2-me-bookmark-category-sounds', kwargs={'category_id': 0}) + '?fields=*', + secure=True, + **self.auth_headers + ) self.assertEqual(resp.status_code, 200) # 200 response on getting sounds for bookmark category without name - resp = self.client.get(reverse('apiv2-me-bookmark-category-sounds', kwargs={'category_id': self.category.id}) + '?fields=*', secure=True, **self.auth_headers) + resp = self.client.get( + reverse('apiv2-me-bookmark-category-sounds', kwargs={'category_id': self.category.id}) + '?fields=*', + secure=True, + **self.auth_headers + ) self.assertEqual(resp.status_code, 200) @@ -501,12 +515,10 @@ def check_dict_has_fields(self, dictionary, fields): self.assertIn(field, dictionary) def check_access_token_response_fields(self, resp): - self.check_dict_has_fields( - resp.json(), ['expires_in', 'scope', 'refresh_token', 'access_token', 'token_type']) + self.check_dict_has_fields(resp.json(), ['expires_in', 'scope', 'refresh_token', 'access_token', 'token_type']) def check_redirect_uri_access_token_frag_params(self, params): - self.check_dict_has_fields( - params, ['expires_in', 'scope', 'access_token', 'token_type']) + self.check_dict_has_fields(params, ['expires_in', 'scope', 'access_token', 'token_type']) def test_oauth2_password_grant_flow(self): @@ -514,13 +526,14 @@ def test_oauth2_password_grant_flow(self): # to false client = ApiV2Client.objects.get(name='AuthorizationCodeClient') resp = self.client.post( - reverse('oauth2_provider:access_token'), - { + reverse('oauth2_provider:access_token'), { 'client_id': client.client_id, 'grant_type': 'password', 'username': self.end_user.username, 'password': self.end_user_password, - }, secure=True) + }, + secure=True + ) self.assertEqual(resp.status_code, 400) self.assertEqual(resp.json()['error'], 'unauthorized_client') @@ -528,13 +541,14 @@ def test_oauth2_password_grant_flow(self): client.allow_oauth_passoword_grant = True client.save() resp = self.client.post( - reverse('oauth2_provider:access_token'), - { + reverse('oauth2_provider:access_token'), { 'client_id': client.client_id, 'grant_type': 'password', 'username': self.end_user.username, 'password': self.end_user_password, - }, secure=True) + }, + secure=True + ) self.assertEqual(resp.status_code, 200) self.check_access_token_response_fields(resp) @@ -542,35 +556,39 @@ def test_oauth2_password_grant_flow(self): resp = self.client.post( reverse('oauth2_provider:access_token'), { - #'client_id': client.client_id, + #'client_id': client.client_id, 'grant_type': 'password', 'username': self.end_user.username, 'password': self.end_user_password, - }, secure=True) + }, + secure=True + ) self.assertEqual(resp.status_code, 401) self.assertEqual(resp.json()['error'], 'invalid_client') # Return 'invalid_client' when client_id does not exist in db resp = self.client.post( - reverse('oauth2_provider:access_token'), - { + reverse('oauth2_provider:access_token'), { 'client_id': 'thi5i5aninv3nt3dcli3ntid', 'grant_type': 'password', 'username': self.end_user.username, 'password': self.end_user_password, - }, secure=True) + }, + secure=True + ) self.assertEqual(resp.status_code, 401) self.assertEqual(resp.json()['error'], 'invalid_client') # Return 'unsupported_grant_type' when grant type does not exist resp = self.client.post( - reverse('oauth2_provider:access_token'), - { + reverse('oauth2_provider:access_token'), { 'client_id': client.client_id, 'grant_type': 'invented_grant', 'username': self.end_user.username, 'password': self.end_user_password, - }, secure=True) + }, + secure=True + ) self.assertEqual(resp.status_code, 400) self.assertEqual(resp.json()['error'], 'unsupported_grant_type') @@ -580,9 +598,11 @@ def test_oauth2_password_grant_flow(self): { 'client_id': client.client_id, 'grant_type': 'password', - #'username': self.end_user.username, + #'username': self.end_user.username, 'password': self.end_user_password, - }, secure=True) + }, + secure=True + ) self.assertEqual(resp.status_code, 400) self.assertEqual(resp.json()['error'], 'invalid_request') @@ -593,8 +613,10 @@ def test_oauth2_password_grant_flow(self): 'client_id': client.client_id, 'grant_type': 'password', 'username': self.end_user.username, - #'password': self.end_user_password, - }, secure=True) + #'password': self.end_user_password, + }, + secure=True + ) self.assertEqual(resp.status_code, 400) self.assertEqual(resp.json()['error'], 'invalid_request') @@ -603,11 +625,13 @@ def test_oauth2_authorization_code_grant_flow(self): # Redirect to login page when visiting authorize page with an AnonymousUser client = ApiV2Client.objects.get(name='AuthorizationCodeClient') resp = self.client.get( - reverse('oauth2_provider:authorize'), - { + reverse('oauth2_provider:authorize'), { 'client_id': client.client_id, 'response_type': 'code', - }, secure=True, follow=True) + }, + secure=True, + follow=True + ) response_path = resp.request['PATH_INFO'] self.assertEqual(resp.status_code, 200) self.assertIn('/login', response_path) @@ -615,11 +639,12 @@ def test_oauth2_authorization_code_grant_flow(self): # Redirect includes 'error' param when using non-existing response type self.client.force_login(self.end_user) resp = self.client.get( - reverse('oauth2_provider:authorize'), - { + reverse('oauth2_provider:authorize'), { 'client_id': client.client_id, 'response_type': 'non_existing_response_type', - }, secure=True) + }, + secure=True + ) self.assertEqual(resp.status_code, 302) resp = self.client.get(resp.request['PATH_INFO'] + '?' + resp.request['QUERY_STRING'], secure=True) self.assertTrue(resp.url.startswith(client.get_default_redirect_uri())) @@ -629,11 +654,12 @@ def test_oauth2_authorization_code_grant_flow(self): # Redirect includes 'error' param when using non-supported response type resp = self.client.get( - reverse('oauth2_provider:authorize'), - { + reverse('oauth2_provider:authorize'), { 'client_id': client.client_id, 'response_type': 'token', - }, secure=True) + }, + secure=True + ) self.assertEqual(resp.status_code, 302) resp = self.client.get(resp.request['PATH_INFO'] + '?' + resp.request['QUERY_STRING'], secure=True) self.assertEquals(resp.url.startswith(client.get_default_redirect_uri()), True) @@ -643,51 +669,55 @@ def test_oauth2_authorization_code_grant_flow(self): # Authorization page is displayed with errors with non-existing client_id resp = self.client.get( - reverse('oauth2_provider:authorize'), - { + reverse('oauth2_provider:authorize'), { 'client_id': 'thi5i5aninv3nt3dcli3ntid', 'response_type': 'code', - }, secure=True) + }, + secure=True + ) self.assertEqual(resp.status_code, 400) self.assertIn('Invalid client_id parameter value', str(resp.content)) # Authorization page is displayed correctly when correct response_type and client_id resp = self.client.get( - reverse('oauth2_provider:authorize'), - { + reverse('oauth2_provider:authorize'), { 'client_id': client.client_id, 'response_type': 'code', - }, secure=True) + }, + secure=True + ) self.assertEqual(resp.status_code, 200) self.assertIn('name="allow" value="Authorize', str(resp.content)) # Redirect includes 'code' and 'state' params resp = self.client.post( - reverse('oauth2_provider:authorize'), - { + reverse('oauth2_provider:authorize'), { 'client_id': client.client_id, 'response_type': 'code', 'redirect_uri': client.get_default_redirect_uri(), 'scope': 'read', 'state': 'an_optional_state', 'allow': 'Authorize', - }, secure=True) + }, + secure=True + ) self.assertTrue(resp.url.startswith(client.get_default_redirect_uri())) resp_params = self.get_params_from_url(resp.url) - self.assertEquals(resp_params['state'], 'an_optional_state') # Check state is returned and preserved - self.check_dict_has_fields(resp_params, ['code']) # Check code is there + self.assertEquals(resp_params['state'], 'an_optional_state') # Check state is returned and preserved + self.check_dict_has_fields(resp_params, ['code']) # Check code is there # Return 200 OK when requesting access token setting client_id and client_secret in body params code = resp_params['code'] resp = self.client.post( - reverse('oauth2_provider:access_token'), - { + reverse('oauth2_provider:access_token'), { 'client_id': client.client_id, 'client_secret': client.client_secret, 'grant_type': 'authorization_code', 'code': code, 'redirect_uri': client.get_default_redirect_uri() - }, secure=True) + }, + secure=True + ) self.assertEqual(resp.status_code, 200) self.check_access_token_response_fields(resp) @@ -696,18 +726,21 @@ def test_oauth2_authorization_code_grant_flow(self): reverse('oauth2_provider:access_token'), { 'client_id': client.client_id, - #'client_secret': client.client_secret, + #'client_secret': client.client_secret, 'grant_type': 'authorization_code', 'code': code, 'redirect_uri': client.get_default_redirect_uri() - }, secure=True) + }, + secure=True + ) self.assertEqual(resp.status_code, 400) def test_token_authentication_with_header(self): user = User.objects.create_user("testuser") - c = ApiV2Client(user=user, status='OK', redirect_uri="https://freesound.com", - url="https://freesound.com", name="test") + c = ApiV2Client( + user=user, status='OK', redirect_uri="https://freesound.com", url="https://freesound.com", name="test" + ) c.save() headers = { 'HTTP_AUTHORIZATION': f'Token {c.key}', @@ -717,17 +750,18 @@ def test_token_authentication_with_header(self): def test_token_authentication_with_query_param(self): user = User.objects.create_user("testuser") - c = ApiV2Client(user=user, status='OK', redirect_uri="https://freesound.com", - url="https://freesound.com", name="test") + c = ApiV2Client( + user=user, status='OK', redirect_uri="https://freesound.com", url="https://freesound.com", name="test" + ) c.save() resp = self.client.get(f"/apiv2/?token={c.key}", secure=True) self.assertEqual(resp.status_code, 200) def test_token_authentication_disabled_client(self): user = User.objects.create_user("testuser") - c = ApiV2Client(user=user, status='REV', redirect_uri="https://freesound.com", - url="https://freesound.com", name="test") + c = ApiV2Client( + user=user, status='REV', redirect_uri="https://freesound.com", url="https://freesound.com", name="test" + ) c.save() resp = self.client.get(f"/apiv2/?token={c.key}", secure=True) self.assertEqual(resp.status_code, 401) - diff --git a/apiv2/throttling.py b/apiv2/throttling.py index d992245a1..62a5607a6 100644 --- a/apiv2/throttling.py +++ b/apiv2/throttling.py @@ -18,7 +18,6 @@ # See AUTHORS file. # - from django.conf import settings from rest_framework.throttling import SimpleRateThrottle @@ -61,7 +60,7 @@ def allow_request(self, request, view): # Apply the burst limit rate (the first of the list if there are limit rates. # No limit rates means unlimited api usage) if limit_rates: - rate = limit_rates[0] # Get burst limit + rate = limit_rates[0] # Get burst limit self.rate = rate self.num_requests, self.duration = self.parse_rate(rate) passes_throttle = super().allow_request(request, view) @@ -75,14 +74,10 @@ def allow_request(self, request, view): def get_cache_key(self, request, view): if self.client: - return self.cache_format % { - 'identity': self.client.client_id - } + return self.cache_format % {'identity': self.client.client_id} else: # If using session based auth, we use the user id as identity for throttling cache - return self.cache_format % { - 'identity': request.user.id - } + return self.cache_format % {'identity': request.user.id} class ClientBasedThrottlingSustained(SimpleRateThrottle): @@ -121,7 +116,7 @@ def allow_request(self, request, view): # Apply the sustained limit rate (the second of the list if there are limit rates. # No limit rates means unlimited api usage) if limit_rates: - rate = limit_rates[1] # Get sustained limit + rate = limit_rates[1] # Get sustained limit self.rate = rate self.num_requests, self.duration = self.parse_rate(rate) passes_throttle = super().allow_request(request, view) @@ -135,14 +130,10 @@ def allow_request(self, request, view): def get_cache_key(self, request, view): if self.client: - return self.cache_format % { - 'identity': self.client.client_id - } + return self.cache_format % {'identity': self.client.client_id} else: # If using session based auth, we use the user id as identity for throttling cache - return self.cache_format % { - 'identity': request.user.id - } + return self.cache_format % {'identity': request.user.id} class IpBasedThrottling(SimpleRateThrottle): @@ -189,7 +180,7 @@ def allow_request(self, request, view): # Apply the ip limit rate (No limit rates means unlimited api usage) if limit_rates: - rate = limit_rates[2] # Get sustained limit + rate = limit_rates[2] # Get sustained limit self.rate = rate self.num_requests, self.duration = self.parse_rate(rate) @@ -236,11 +227,7 @@ def throttle_success(self): def get_cache_key(self, request, view): if self.client: - return self.cache_format % { - 'identity': self.client.client_id - } + return self.cache_format % {'identity': self.client.client_id} else: # If using session based auth, we use the user id as identity for throttling cache - return self.cache_format % { - 'identity': request.user.id - } + return self.cache_format % {'identity': request.user.id} diff --git a/apiv2/urls.py b/apiv2/urls.py index 9436f3808..a515de09e 100644 --- a/apiv2/urls.py +++ b/apiv2/urls.py @@ -23,7 +23,6 @@ # - djangorestframework ('2.3.8') # - markdown (for browseable api) - from django.urls import include, path, re_path from django.contrib.auth.views import LogoutView from apiv2 import views @@ -42,7 +41,11 @@ # Me path('me/', views.Me.as_view(), name="apiv2-me"), path('me/bookmark_categories/', views.MeBookmarkCategories.as_view(), name='apiv2-me-bookmark-categories'), - path('me/bookmark_categories//sounds/', views.MeBookmarkCategorySounds.as_view(), name='apiv2-me-bookmark-category-sounds'), + path( + 'me/bookmark_categories//sounds/', + views.MeBookmarkCategorySounds.as_view(), + name='apiv2-me-bookmark-category-sounds' + ), # Available audio descriptors path('descriptors/', views.AvailableAudioDescriptors.as_view(), name="apiv2-available-descriptors"), @@ -83,7 +86,6 @@ # Download item from link path('download//', views.download_from_token, name="apiv2-download_from_token"), - ######################### # MANAGEMENT AND OAUTH2 # ######################### @@ -91,14 +93,24 @@ # Client management # use apply[/]* for backwards compatibility with links to /apiv2/apply re_path(r'^apply[/]*$', views.create_apiv2_key, name="apiv2-apply"), - re_path(r'^apply/credentials/(?P[^//]+)/monitor/$', views.monitor_api_credential, name="apiv2-monitor-credential"), - re_path(r'^apply/credentials/(?P[^//]+)/delete/$', views.delete_api_credential, name="apiv2-delete-credential"), + re_path( + r'^apply/credentials/(?P[^//]+)/monitor/$', views.monitor_api_credential, name="apiv2-monitor-credential" + ), + re_path( + r'^apply/credentials/(?P[^//]+)/delete/$', views.delete_api_credential, name="apiv2-delete-credential" + ), re_path(r'^apply/credentials/(?P[^//]+)/edit/$', views.edit_api_credential, name="apiv2-edit-credential"), # Oauth2 path('oauth2/', include('apiv2.oauth2_urls', namespace='oauth2_provider')), - path('login/', login, {'template_name': 'oauth2_provider/oauth_login.html', - 'authentication_form': FsAuthenticationForm}, name="api-login"), + path( + 'login/', + login, { + 'template_name': 'oauth2_provider/oauth_login.html', + 'authentication_form': FsAuthenticationForm + }, + name="api-login" + ), path('logout/', LogoutView.as_view(next_page='/apiv2/'), name="api-logout"), ######### diff --git a/apiv2/views.py b/apiv2/views.py index f9d2bd926..f59dc7e18 100755 --- a/apiv2/views.py +++ b/apiv2/views.py @@ -34,7 +34,7 @@ from django.db import IntegrityError from django.db.models import Exists, OuterRef from django.http import HttpResponseRedirect, Http404 -from django.shortcuts import render +from django.shortcuts import render from django.urls import reverse from oauth2_provider.models import Grant, AccessToken from oauth2_provider.views import AuthorizationView as ProviderAuthorizationView @@ -76,6 +76,7 @@ class AuthorizationView(ProviderAuthorizationView): login_url = '/apiv2/login/' + #################################### # SEARCH AND SIMILARITY SEARCH VIEWS #################################### @@ -90,7 +91,7 @@ def get_description(cls): % (prepend_base('/docs/api'), '%s#text-search' % resources_doc_filename, get_formatted_examples_for_view('TextSearch', 'apiv2-sound-search', max=5)) - def get(self, request, *args, **kwargs): + def get(self, request, *args, **kwargs): api_logger.info(self.log_message('search')) # Validate search form and check page 0 @@ -98,8 +99,11 @@ def get(self, request, *args, **kwargs): if not search_form.is_valid(): raise BadRequestException(msg='Malformed request.', resource=self) if search_form.cleaned_data['query'] is None and search_form.cleaned_data['filter'] is None: - raise BadRequestException(msg='At least one request parameter from Text Search should be included ' - 'in the request.', resource=self) + raise BadRequestException( + msg='At least one request parameter from Text Search should be included ' + 'in the request.', + resource=self + ) if search_form.cleaned_data['page'] < 1: raise NotFoundException(resource=self) if search_form.cleaned_data['page_size'] < 1: @@ -124,12 +128,14 @@ def get(self, request, *args, **kwargs): response_data['previous'] = None response_data['next'] = None if page['has_other_pages']: - if page['has_previous']: - response_data['previous'] = search_form.construct_link(reverse('apiv2-sound-text-search'), - page=page['previous_page_number']) - if page['has_next']: - response_data['next'] = search_form.construct_link(reverse('apiv2-sound-text-search'), - page=page['next_page_number']) + if page['has_previous']: + response_data['previous'] = search_form.construct_link( + reverse('apiv2-sound-text-search'), page=page['previous_page_number'] + ) + if page['has_next']: + response_data['next'] = search_form.construct_link( + reverse('apiv2-sound-text-search'), page=page['next_page_number'] + ) # Get analysis data and serialize sound results object_list = [ob for ob in page['object_list']] @@ -137,22 +143,31 @@ def get(self, request, *args, **kwargs): sound_ids = [ob[0] for ob in object_list] sound_analysis_data = get_analysis_data_for_sound_ids(request, sound_ids=sound_ids) # In search queries, only include audio analyers's output if requested through the fields parameter - needs_analyzers_ouptut = 'analyzers_output' in search_form.cleaned_data.get('fields', '') or 'ac_analysis' in search_form.cleaned_data.get('fields', '') + needs_analyzers_ouptut = 'analyzers_output' in search_form.cleaned_data.get( + 'fields', '' + ) or 'ac_analysis' in search_form.cleaned_data.get('fields', '') sounds_dict = Sound.objects.dict_ids(sound_ids=sound_ids, include_analyzers_output=needs_analyzers_ouptut) sounds = [] for i, sid in enumerate(sound_ids): try: - sound = SoundListSerializer(sounds_dict[sid], context=self.get_serializer_context(), score_map=id_score_map, sound_analysis_data=sound_analysis_data).data + sound = SoundListSerializer( + sounds_dict[sid], + context=self.get_serializer_context(), + score_map=id_score_map, + sound_analysis_data=sound_analysis_data + ).data if more_from_pack_data: if more_from_pack_data[sid][0]: - pack_id = more_from_pack_data[sid][1][:more_from_pack_data[sid][1].find("_")] - pack_name = more_from_pack_data[sid][1][more_from_pack_data[sid][1].find("_")+1:] + pack_id = more_from_pack_data[sid][1][:more_from_pack_data[sid][1].find("_")] + pack_name = more_from_pack_data[sid][1][more_from_pack_data[sid][1].find("_") + 1:] sound['more_from_same_pack'] = search_form.construct_link( reverse('apiv2-sound-text-search'), page=1, filt='grouping_pack:"%i_%s"' % (int(pack_id), pack_name), - group_by_pack='0') - sound['n_from_same_pack'] = more_from_pack_data[sid][0] + 1 # we add one as is the sound itself + group_by_pack='0' + ) + sound['n_from_same_pack' + ] = more_from_pack_data[sid][0] + 1 # we add one as is the sound itself sounds.append(sound) except KeyError: # This will happen if there are synchronization errors between solr index, gaia and the database. @@ -178,7 +193,7 @@ def get_description(cls): serializer_class = SimilarityFileSerializer analysis_file = None - def get(self, request, *args, **kwargs): + def get(self, request, *args, **kwargs): api_logger.info(self.log_message('content_search')) # Validate search form and check page 0 @@ -188,8 +203,10 @@ def get(self, request, *args, **kwargs): if not search_form.cleaned_data['target'] and \ not search_form.cleaned_data['descriptors_filter'] and \ not self.analysis_file: - raise BadRequestException(msg='At least one parameter from Content Search should be included ' - 'in the request.', resource=self) + raise BadRequestException( + msg='At least one parameter from Content Search should be included ' + 'in the request.', resource=self + ) if search_form.cleaned_data['page'] < 1: raise NotFoundException(resource=self) @@ -201,7 +218,7 @@ def get(self, request, *args, **kwargs): results, count, distance_to_target_data, more_from_pack_data, note, params_for_next_page, debug_note = \ api_search(search_form, target_file=analysis_file, resource=self) except APIException as e: - raise e # TODO pass correct exception message + raise e # TODO pass correct exception message except Exception as e: raise ServerErrorException(msg='Unexpected error', resource=self) @@ -212,29 +229,36 @@ def get(self, request, *args, **kwargs): page = paginator.page(search_form.cleaned_data['page']) response_data = dict() if self.analysis_file: - response_data['target_analysis_file'] = f'{self.analysis_file.name} ({self.analysis_file.size // 1024:d} KB)' + response_data['target_analysis_file' + ] = f'{self.analysis_file.name} ({self.analysis_file.size // 1024:d} KB)' response_data['count'] = paginator.count response_data['previous'] = None response_data['next'] = None if page['has_other_pages']: - if page['has_previous']: - response_data['previous'] = search_form.construct_link(reverse('apiv2-sound-content-search'), - page=page['previous_page_number']) - if page['has_next']: - response_data['next'] = search_form.construct_link(reverse('apiv2-sound-content-search'), - page=page['next_page_number']) + if page['has_previous']: + response_data['previous'] = search_form.construct_link( + reverse('apiv2-sound-content-search'), page=page['previous_page_number'] + ) + if page['has_next']: + response_data['next'] = search_form.construct_link( + reverse('apiv2-sound-content-search'), page=page['next_page_number'] + ) # Get analysis data and serialize sound results ids = [id for id in page['object_list']] sound_analysis_data = get_analysis_data_for_sound_ids(request, sound_ids=ids) # In search queries, only include audio analyers's output if requested through the fields parameter - needs_analyzers_ouptut = 'analyzers_output' in search_form.cleaned_data.get('fields', '') or 'ac_analysis' in search_form.cleaned_data.get('fields', '') + needs_analyzers_ouptut = 'analyzers_output' in search_form.cleaned_data.get( + 'fields', '' + ) or 'ac_analysis' in search_form.cleaned_data.get('fields', '') sounds_dict = Sound.objects.dict_ids(sound_ids=ids, include_analyzers_output=needs_analyzers_ouptut) sounds = [] for i, sid in enumerate(ids): try: - sound = SoundListSerializer(sounds_dict[sid], context=self.get_serializer_context(), sound_analysis_data=sound_analysis_data).data + sound = SoundListSerializer( + sounds_dict[sid], context=self.get_serializer_context(), sound_analysis_data=sound_analysis_data + ).data # Distance to target is present we add it to the serialized sound if distance_to_target_data: sound['distance_to_target'] = distance_to_target_data[sid] @@ -250,12 +274,12 @@ def get(self, request, *args, **kwargs): return Response(response_data, status=status.HTTP_200_OK) - def post(self, request, *args, **kwargs): + def post(self, request, *args, **kwargs): # This view has a post version to handle analysis file uploads serializer = SimilarityFileSerializer(data=request.data) if serializer.is_valid(): self.analysis_file = request.FILES['analysis_file'] - return self.get(request, *args, **kwargs) + return self.get(request, *args, **kwargs) else: return Response({'detail': serializer.errors}, status=status.HTTP_400_BAD_REQUEST) @@ -271,9 +295,9 @@ def get_description(cls): serializer_class = SimilarityFileSerializer analysis_file = None - merging_strategy = 'merge_optimized' # 'filter_both', 'merge_all' + merging_strategy = 'merge_optimized' # 'filter_both', 'merge_all' - def get(self, request, *args, **kwargs): + def get(self, request, *args, **kwargs): api_logger.info(self.log_message('combined_search')) # Validate search form and check page 0 @@ -284,11 +308,16 @@ def get(self, request, *args, **kwargs): not search_form.cleaned_data['descriptors_filter'] and not self.analysis_file) \ or (search_form.cleaned_data['query'] is None and search_form.cleaned_data['filter'] is None): - raise BadRequestException(msg='At least one parameter from Text Search and one parameter from ' - 'Content Search should be included in the request.', resource=self) + raise BadRequestException( + msg='At least one parameter from Text Search and one parameter from ' + 'Content Search should be included in the request.', + resource=self + ) if search_form.cleaned_data['target'] and search_form.cleaned_data['query']: - raise BadRequestException(msg='Request parameters \'target\' and \'query\' can not be used at ' - 'the same time.', resource=self) + raise BadRequestException( + msg='Request parameters \'target\' and \'query\' can not be used at ' + 'the same time.', resource=self + ) if search_form.cleaned_data['page'] < 1: raise NotFoundException(resource=self) @@ -309,7 +338,7 @@ def get(self, request, *args, **kwargs): merging_strategy=self.merging_strategy, resource=self) except APIException as e: - raise e # TODO pass correct resource parameter + raise e # TODO pass correct resource parameter except Exception as e: raise ServerErrorException(msg='Unexpected error', resource=self) @@ -324,18 +353,21 @@ def get(self, request, *args, **kwargs): response_data = dict() if self.analysis_file: - response_data['target_analysis_file'] = f'{self.analysis_file._name} ({self.analysis_file._size // 1024:d} KB)' + response_data['target_analysis_file' + ] = f'{self.analysis_file._name} ({self.analysis_file._size // 1024:d} KB)' # Build 'more' link (only add it if we know there might be more results) if 'no_more_results' not in extra_parameters: if self.merging_strategy == 'merge_optimized': - response_data['more'] = search_form.construct_link(reverse('apiv2-sound-combined-search'), - include_page=False) + response_data['more'] = search_form.construct_link( + reverse('apiv2-sound-combined-search'), include_page=False + ) else: num_pages = math.ceil(count / search_form.cleaned_data['page_size']) if search_form.cleaned_data['page'] < num_pages: - response_data['more'] = search_form.construct_link(reverse('apiv2-sound-combined-search'), - page=search_form.cleaned_data['page'] + 1) + response_data['more'] = search_form.construct_link( + reverse('apiv2-sound-combined-search'), page=search_form.cleaned_data['page'] + 1 + ) else: response_data['more'] = None if extra_parameters_string: @@ -347,13 +379,17 @@ def get(self, request, *args, **kwargs): ids = results sound_analysis_data = get_analysis_data_for_sound_ids(request, sound_ids=ids) # In search queries, only include audio analyers's output if requested through the fields parameter - needs_analyzers_ouptut = 'analyzers_output' in search_form.cleaned_data.get('fields', '') or 'ac_analysis' in search_form.cleaned_data.get('fields', '') + needs_analyzers_ouptut = 'analyzers_output' in search_form.cleaned_data.get( + 'fields', '' + ) or 'ac_analysis' in search_form.cleaned_data.get('fields', '') sounds_dict = Sound.objects.dict_ids(sound_ids=ids, include_analyzers_output=needs_analyzers_ouptut) sounds = [] for i, sid in enumerate(ids): try: - sound = SoundListSerializer(sounds_dict[sid], context=self.get_serializer_context(), sound_analysis_data=sound_analysis_data).data + sound = SoundListSerializer( + sounds_dict[sid], context=self.get_serializer_context(), sound_analysis_data=sound_analysis_data + ).data # Distance to target is present we add it to the serialized sound if distance_to_target_data: sound['distance_to_target'] = distance_to_target_data[sid] @@ -372,13 +408,13 @@ def get(self, request, *args, **kwargs): return Response(response_data, status=status.HTTP_200_OK) - def post(self, request, *args, **kwargs): + def post(self, request, *args, **kwargs): # This view has a post version to handle analysis file uploads serializer = SimilarityFileSerializer(data=request.data) if serializer.is_valid(): analysis_file = request.FILES['analysis_file'] self.analysis_file = analysis_file - return self.get(request, *args, **kwargs) + return self.get(request, *args, **kwargs) else: return Response({'detail': serializer.errors}, status=status.HTTP_400_BAD_REQUEST) @@ -387,6 +423,7 @@ def post(self, request, *args, **kwargs): # SOUND VIEWS ############# + class SoundInstance(RetrieveAPIView): @classmethod @@ -397,14 +434,22 @@ def get_description(cls): get_formatted_examples_for_view('SoundInstance', 'apiv2-sound-instance', max=5)) serializer_class = SoundSerializer - queryset = Sound.objects.filter(moderation_state="OK", processing_state="OK").annotate(analysis_state_essentia_exists=Exists(SoundAnalysis.objects.filter(analyzer=settings.FREESOUND_ESSENTIA_EXTRACTOR_NAME, analysis_status="OK", sound=OuterRef('id')))) + queryset = Sound.objects.filter( + moderation_state="OK", processing_state="OK" + ).annotate( + analysis_state_essentia_exists=Exists( + SoundAnalysis.objects. + filter(analyzer=settings.FREESOUND_ESSENTIA_EXTRACTOR_NAME, analysis_status="OK", sound=OuterRef('id')) + ) + ) - def get(self, request, *args, **kwargs): + def get(self, request, *args, **kwargs): api_logger.info(self.log_message('sound:%i instance' % (int(kwargs['pk'])))) return super().get(request, *args, **kwargs) -class SoundAnalysisView(GenericAPIView): # Needs to be named SoundAnalysisView so it does not overlap with SoundAnalysis +class SoundAnalysisView(GenericAPIView + ): # Needs to be named SoundAnalysisView so it does not overlap with SoundAnalysis @classmethod def get_description(cls): @@ -413,16 +458,16 @@ def get_description(cls): % (prepend_base('/docs/api'), '%s#sound-analysis' % resources_doc_filename, get_formatted_examples_for_view('SoundAnalysis', 'apiv2-sound-analysis', max=5)) - def get(self, request, *args, **kwargs): + def get(self, request, *args, **kwargs): sound_id = kwargs['pk'] descriptors = [] if request.query_params.get('descriptors', False): descriptors = request.query_params['descriptors'].split(',') api_logger.info(self.log_message('sound:%i analysis' % (int(sound_id)))) response_data = get_sounds_descriptors([sound_id], - descriptors, - request.query_params.get('normalized', '0') == '1', - only_leaf_descriptors=True) + descriptors, + request.query_params.get('normalized', '0') == '1', + only_leaf_descriptors=True) if response_data: return Response(response_data[str(sound_id)], status=status.HTTP_200_OK) else: @@ -438,7 +483,7 @@ def get_description(cls): % (prepend_base('/docs/api'), '%s#similar-sounds' % resources_doc_filename, get_formatted_examples_for_view('SimilarSounds', 'apiv2-similarity-sound', max=5)) - def get(self, request, *args, **kwargs): + def get(self, request, *args, **kwargs): sound_id = self.kwargs['pk'] api_logger.info(self.log_message('sound:%i similar_sounds' % (int(sound_id)))) @@ -465,24 +510,30 @@ def get(self, request, *args, **kwargs): response_data['previous'] = None response_data['next'] = None if page['has_other_pages']: - if page['has_previous']: - response_data['previous'] = similarity_sound_form.construct_link( - reverse('apiv2-similarity-sound', args=[sound_id]), page=page['previous_page_number']) - if page['has_next']: - response_data['next'] = similarity_sound_form.construct_link( - reverse('apiv2-similarity-sound', args=[sound_id]), page=page['next_page_number']) + if page['has_previous']: + response_data['previous'] = similarity_sound_form.construct_link( + reverse('apiv2-similarity-sound', args=[sound_id]), page=page['previous_page_number'] + ) + if page['has_next']: + response_data['next'] = similarity_sound_form.construct_link( + reverse('apiv2-similarity-sound', args=[sound_id]), page=page['next_page_number'] + ) # Get analysis data and serialize sound results ids = [id for id in page['object_list']] sound_analysis_data = get_analysis_data_for_sound_ids(request, sound_ids=ids) # In search queries, only include audio analyers's output if requested through the fields parameter - needs_analyzers_ouptut = 'analyzers_output' in similarity_sound_form.cleaned_data.get('fields', '') or 'ac_analysis' in similarity_sound_form.cleaned_data.get('fields', '') + needs_analyzers_ouptut = 'analyzers_output' in similarity_sound_form.cleaned_data.get( + 'fields', '' + ) or 'ac_analysis' in similarity_sound_form.cleaned_data.get('fields', '') sounds_dict = Sound.objects.dict_ids(sound_ids=ids, include_analyzers_output=needs_analyzers_ouptut) sounds = [] for i, sid in enumerate(ids): try: - sound = SoundListSerializer(sounds_dict[sid], context=self.get_serializer_context(), sound_analysis_data=sound_analysis_data).data + sound = SoundListSerializer( + sounds_dict[sid], context=self.get_serializer_context(), sound_analysis_data=sound_analysis_data + ).data # Distance to target is present we add it to the serialized sound if distance_to_target_data: sound['distance_to_target'] = distance_to_target_data[sid] @@ -506,7 +557,7 @@ def get_description(cls): % (prepend_base('/docs/api'), '%s#sound-comments' % resources_doc_filename, get_formatted_examples_for_view('SoundComments', 'apiv2-sound-comments', max=5)) - def get(self, request, *args, **kwargs): + def get(self, request, *args, **kwargs): api_logger.info(self.log_message('sound:%i comments' % (int(self.kwargs['pk'])))) return super().get(request, *args, **kwargs) @@ -523,7 +574,7 @@ def get_description(cls): % (prepend_base('/docs/api'), '%s#download-sound-oauth2-required' % resources_doc_filename, get_formatted_examples_for_view('DownloadSound', 'apiv2-sound-download', max=5)) - def get(self, request, *args, **kwargs): + def get(self, request, *args, **kwargs): sound_id = kwargs['pk'] api_logger.info(self.log_message('sound:%i download' % (int(sound_id)))) try: @@ -556,9 +607,12 @@ def get(self, request, *args, **kwargs): 'sound_id': sound.id, 'client_id': self.client_id, 'exp': datetime.datetime.utcnow() + datetime.timedelta(seconds=settings.API_DOWNLOAD_TOKEN_LIFETIME), - }, settings.SECRET_KEY, algorithm='HS256') - download_link = prepend_base(reverse('apiv2-download_from_token', args=[download_token]), - request_is_secure=request.is_secure()) + }, + settings.SECRET_KEY, + algorithm='HS256') + download_link = prepend_base( + reverse('apiv2-download_from_token', args=[download_token]), request_is_secure=request.is_secure() + ) return Response({'download_link': download_link}, status=status.HTTP_200_OK) @@ -566,6 +620,7 @@ def get(self, request, *args, **kwargs): # USER VIEWS ############ + class UserInstance(RetrieveAPIView): @classmethod @@ -579,7 +634,7 @@ def get_description(cls): serializer_class = UserSerializer queryset = User.objects.filter(is_active=True) - def get(self, request, *args, **kwargs): + def get(self, request, *args, **kwargs): api_logger.info(self.log_message(f"user:{self.kwargs['username']} instance")) return super().get(request, *args, **kwargs) @@ -595,7 +650,7 @@ def get_description(cls): % (prepend_base('/docs/api'), '%s#user-sounds' % resources_doc_filename, get_formatted_examples_for_view('UserSounds', 'apiv2-user-sound-list', max=5)) - def get(self, request, *args, **kwargs): + def get(self, request, *args, **kwargs): api_logger.info(self.log_message(f"user:{self.kwargs['username']} sounds")) return super().get(request, *args, **kwargs) @@ -604,7 +659,9 @@ def get_queryset(self): user = User.objects.get(username=self.kwargs['username'], is_active=True) except User.DoesNotExist: raise NotFoundException(resource=self) - needs_analyzers_ouptut = 'analyzers_output' in self.request.GET.get('fields', '') or 'ac_analysis' in self.request.GET.get('fields', '') + needs_analyzers_ouptut = 'analyzers_output' in self.request.GET.get( + 'fields', '' + ) or 'ac_analysis' in self.request.GET.get('fields', '') queryset = Sound.objects.bulk_sounds_for_user(user_id=user.id, include_analyzers_output=needs_analyzers_ouptut) return queryset @@ -620,7 +677,7 @@ def get_description(cls): % (prepend_base('/docs/api'), '%s#user-packs' % resources_doc_filename, get_formatted_examples_for_view('UserPacks', 'apiv2-user-packs', max=5)) - def get(self, request, *args, **kwargs): + def get(self, request, *args, **kwargs): api_logger.info(self.log_message(f"user:{self.kwargs['username']} packs")) return super().get(request, *args, **kwargs) @@ -639,6 +696,7 @@ def get_queryset(self): # PACK VIEWS ############ + class PackInstance(RetrieveAPIView): serializer_class = PackSerializer queryset = Pack.objects.exclude(is_deleted=True) @@ -650,7 +708,7 @@ def get_description(cls): % (prepend_base('/docs/api'), '%s#pack-instance' % resources_doc_filename, get_formatted_examples_for_view('PackInstance', 'apiv2-pack-instance', max=5)) - def get(self, request, *args, **kwargs): + def get(self, request, *args, **kwargs): api_logger.info(self.log_message('pack:%i instance' % (int(kwargs['pk'])))) return super().get(request, *args, **kwargs) @@ -665,7 +723,7 @@ def get_description(cls): % (prepend_base('/docs/api'), '%s#pack-sounds' % resources_doc_filename, get_formatted_examples_for_view('PackSounds', 'apiv2-pack-sound-list', max=5)) - def get(self, request, *args, **kwargs): + def get(self, request, *args, **kwargs): api_logger.info(self.log_message('pack:%i sounds' % (int(kwargs['pk'])))) return super().get(request, *args, **kwargs) @@ -674,8 +732,12 @@ def get_queryset(self): Pack.objects.get(id=self.kwargs['pk'], is_deleted=False) except Pack.DoesNotExist: raise NotFoundException(resource=self) - needs_analyzers_ouptut = 'analyzers_output' in self.request.GET.get('fields', '') or 'ac_analysis' in self.request.GET.get('fields', '') - queryset = Sound.objects.bulk_sounds_for_pack(pack_id=self.kwargs['pk'], include_analyzers_output=needs_analyzers_ouptut) + needs_analyzers_ouptut = 'analyzers_output' in self.request.GET.get( + 'fields', '' + ) or 'ac_analysis' in self.request.GET.get('fields', '') + queryset = Sound.objects.bulk_sounds_for_pack( + pack_id=self.kwargs['pk'], include_analyzers_output=needs_analyzers_ouptut + ) return queryset @@ -688,7 +750,7 @@ def get_description(cls): % (prepend_base('/docs/api'), '%s#download-pack-oauth2-required' % resources_doc_filename, get_formatted_examples_for_view('DownloadPack', 'apiv2-pack-download', max=5)) - def get(self, request, *args, **kwargs): + def get(self, request, *args, **kwargs): pack_id = kwargs['pk'] api_logger.info(self.log_message('pack:%i download' % (int(pack_id)))) try: @@ -699,7 +761,8 @@ def get(self, request, *args, **kwargs): sounds = pack.sounds.filter(processing_state="OK", moderation_state="OK") if not sounds: raise NotFoundException( - msg='Sounds in pack %i have not yet been described or moderated' % int(pack_id), resource=self) + msg='Sounds in pack %i have not yet been described or moderated' % int(pack_id), resource=self + ) licenses_url = (reverse('pack-licenses', args=[pack.user.username, pack.id])) return download_sounds(licenses_url, pack) @@ -721,7 +784,7 @@ def get_description(cls): % (prepend_base('/docs/api'), '%s#upload-sound-oauth2-required' % resources_doc_filename, get_formatted_examples_for_view('UploadSound', 'apiv2-uploads-upload', max=5)) - def post(self, request, *args, **kwargs): + def post(self, request, *args, **kwargs): api_logger.info(self.log_message('upload_sound')) serializer = UploadAndDescribeAudioFileSerializer(data=request.data) is_providing_description = serializer.is_providing_description(serializer.initial_data) @@ -737,11 +800,16 @@ def post(self, request, *args, **kwargs): msg = 'Audio file successfully uploaded and described (now pending processing and moderation).' else: msg = 'Audio file successfully uploaded (%i Bytes, now pending description).' % audiofile.size - return Response(data={'detail': msg, - 'id': None, - 'note': 'Sound has not been saved in the database as browseable API is only ' - 'for testing purposes.'}, - status=status.HTTP_201_CREATED) + return Response( + data={ + 'detail': msg, + 'id': None, + 'note': + 'Sound has not been saved in the database as browseable API is only ' + 'for testing purposes.' + }, + status=status.HTTP_201_CREATED + ) else: if is_providing_description: try: @@ -769,17 +837,17 @@ def post(self, request, *args, **kwargs): sound_fields['tags'] = clean_and_split_tags(sound_fields['tags']) try: - sound = utils.sound_upload.create_sound( - self.user, - sound_fields, - apiv2_client=apiv2_client - ) + sound = utils.sound_upload.create_sound(self.user, sound_fields, apiv2_client=apiv2_client) except utils.sound_upload.NoAudioException: - raise OtherException('Something went wrong with accessing the file %s.' - % sound_fields['name']) + raise OtherException( + 'Something went wrong with accessing the file %s.' % sound_fields['name'] + ) except utils.sound_upload.AlreadyExistsException: - raise OtherException("Sound could not be created because the uploaded file is " - "already part of freesound.", resource=self) + raise OtherException( + "Sound could not be created because the uploaded file is " + "already part of freesound.", + resource=self + ) except utils.sound_upload.CantMoveException: if settings.DEBUG: msg = "File could not be copied to the correct destination." @@ -788,18 +856,29 @@ def post(self, request, *args, **kwargs): raise ServerErrorException(msg=msg, resource=self) except APIException as e: - raise e # TODO pass correct resource variable + raise e # TODO pass correct resource variable except Exception as e: raise ServerErrorException(msg='Unexpected error', resource=self) - return Response(data={'detail': 'Audio file successfully uploaded and described (now pending ' - 'processing and moderation).', 'id': int(sound.id) }, - status=status.HTTP_201_CREATED) + return Response( + data={ + 'detail': + 'Audio file successfully uploaded and described (now pending ' + 'processing and moderation).', + 'id': int(sound.id) + }, + status=status.HTTP_201_CREATED + ) else: - return Response(data={'filename': audiofile.name, - 'detail': 'Audio file successfully uploaded (%i Bytes, ' - 'now pending description).' % audiofile.size}, - status=status.HTTP_201_CREATED) + return Response( + data={ + 'filename': audiofile.name, + 'detail': + 'Audio file successfully uploaded (%i Bytes, ' + 'now pending description).' % audiofile.size + }, + status=status.HTTP_201_CREATED + ) else: return Response({'detail': serializer.errors}, status=status.HTTP_400_BAD_REQUEST) @@ -813,7 +892,7 @@ def get_description(cls): % (prepend_base('/docs/api'), '%s#pending-uploads-oauth2-required' % resources_doc_filename, get_formatted_examples_for_view('PendingUploads', 'apiv2-uploads-pending', max=5)) - def get(self, request, *args, **kwargs): + def get(self, request, *args, **kwargs): api_logger.info(self.log_message('pending_uploads')) # Look for sounds pending description @@ -865,29 +944,40 @@ def get_description(cls): % (prepend_base('/docs/api'), '%s#describe-sound-oauth2-required' % resources_doc_filename, get_formatted_examples_for_view('DescribeSound', 'apiv2-uploads-describe', max=5)) - def post(self, request, *args, **kwargs): + def post(self, request, *args, **kwargs): api_logger.info(self.log_message('describe_sound')) file_structure, files = generate_tree(self.user.profile.locations()['uploads_dir']) filenames = [file_instance.name for file_id, file_instance in files.items()] serializer = SoundDescriptionSerializer(data=request.data, context={'not_yet_described_audio_files': filenames}) if serializer.is_valid(): if not settings.ALLOW_WRITE_WHEN_SESSION_BASED_AUTHENTICATION and self.auth_method_name == 'Session': - return Response(data={'detail': 'Sound successfully described (now pending processing and moderation).', - 'id': None, - 'note': 'Sound has not been saved in the database as browseable API is only for ' - 'testing purposes.'}, - status=status.HTTP_201_CREATED) + return Response( + data={ + 'detail': 'Sound successfully described (now pending processing and moderation).', + 'id': None, + 'note': + 'Sound has not been saved in the database as browseable API is only for ' + 'testing purposes.' + }, + status=status.HTTP_201_CREATED + ) else: apiv2_client = None # This will always be true as long as settings.ALLOW_WRITE_WHEN_SESSION_BASED_AUTHENTICATION is False if self.auth_method_name == 'OAuth2': apiv2_client = request.auth.client.apiv2_client sound_fields = serializer.data.copy() - sound_fields['dest_path'] = os.path.join(self.user.profile.locations()['uploads_dir'], - sound_fields['upload_filename']) + sound_fields['dest_path'] = os.path.join( + self.user.profile.locations()['uploads_dir'], sound_fields['upload_filename'] + ) sound = utils.sound_upload.create_sound(self.user, sound_fields, apiv2_client=apiv2_client) - return Response(data={'detail': 'Sound successfully described (now pending processing and moderation).', - 'id': int(sound.id)}, status=status.HTTP_201_CREATED) + return Response( + data={ + 'detail': 'Sound successfully described (now pending processing and moderation).', + 'id': int(sound.id) + }, + status=status.HTTP_201_CREATED + ) else: return Response({'detail': serializer.errors}, status=status.HTTP_400_BAD_REQUEST) @@ -902,7 +992,7 @@ def get_description(cls): % (prepend_base('/docs/api'), '%s#edit-sound-description-oauth2-required' % resources_doc_filename, get_formatted_examples_for_view('EditSoundDescription', 'apiv2-sound-edit', max=5)) - def post(self, request, *args, **kwargs): + def post(self, request, *args, **kwargs): sound_id = kwargs['pk'] # Check that sound exists try: @@ -911,17 +1001,25 @@ def post(self, request, *args, **kwargs): raise NotFoundException(resource=self) # Check that sound belongs to current end user if sound.user != self.user: - raise UnauthorizedException(msg='Not authorized. The sound you\'re trying to edit is not owned by ' - 'the OAuth2 logged in user.', resource=self) + raise UnauthorizedException( + msg='Not authorized. The sound you\'re trying to edit is not owned by ' + 'the OAuth2 logged in user.', + resource=self + ) api_logger.info(self.log_message(f'sound:{sound_id} edit_description')) serializer = EditSoundDescriptionSerializer(data=request.data) if serializer.is_valid(): if not settings.ALLOW_WRITE_WHEN_SESSION_BASED_AUTHENTICATION and self.auth_method_name == 'Session': - return Response(data={'detail': f'Description of sound {sound_id} successfully edited.', - 'note': 'Description of sound %s has not been saved in the database as ' - 'browseable API is only for testing purposes.' % sound_id}, - status=status.HTTP_200_OK) + return Response( + data={ + 'detail': f'Description of sound {sound_id} successfully edited.', + 'note': + 'Description of sound %s has not been saved in the database as ' + 'browseable API is only for testing purposes.' % sound_id + }, + status=status.HTTP_200_OK + ) else: if 'name' in serializer.data: if serializer.data['name']: @@ -941,10 +1039,7 @@ def post(self, request, *args, **kwargs): if 'geotag' in serializer.data: if serializer.data['geotag']: lat, lon, zoom = serializer.data['geotag'].split(',') - geotag = GeoTag(user=self.user, - lat=float(lat), - lon=float(lon), - zoom=int(zoom)) + geotag = GeoTag(user=self.user, lat=float(lat), lon=float(lon), zoom=int(zoom)) geotag.save() sound.geotag = geotag if 'pack' in serializer.data: @@ -961,8 +1056,9 @@ def post(self, request, *args, **kwargs): # Invalidate caches sound.invalidate_template_caches() - return Response(data={'detail': f'Description of sound {sound_id} successfully edited.'}, - status=status.HTTP_200_OK) + return Response( + data={'detail': f'Description of sound {sound_id} successfully edited.'}, status=status.HTTP_200_OK + ) else: return Response({'detail': serializer.errors}, status=status.HTTP_400_BAD_REQUEST) @@ -977,7 +1073,7 @@ def get_description(cls): % (prepend_base('/docs/api'), '%s#bookmark-sound-oauth2-required' % resources_doc_filename, get_formatted_examples_for_view('BookmarkSound', 'apiv2-user-create-bookmark', max=5)) - def post(self, request, *args, **kwargs): + def post(self, request, *args, **kwargs): sound_id = kwargs['pk'] try: sound = Sound.objects.get(id=sound_id, moderation_state="OK", processing_state="OK") @@ -987,10 +1083,15 @@ def post(self, request, *args, **kwargs): serializer = CreateBookmarkSerializer(data=request.data) if serializer.is_valid(): if not settings.ALLOW_WRITE_WHEN_SESSION_BASED_AUTHENTICATION and self.auth_method_name == 'Session': - return Response(data={'detail': f'Successfully bookmarked sound {sound_id}.', - 'note': 'This bookmark has not been saved in the database as browseable API is ' - 'only for testing purposes.'}, - status=status.HTTP_201_CREATED) + return Response( + data={ + 'detail': f'Successfully bookmarked sound {sound_id}.', + 'note': + 'This bookmark has not been saved in the database as browseable API is ' + 'only for testing purposes.' + }, + status=status.HTTP_201_CREATED + ) else: name = serializer.data.get('name', sound.original_filename) category_name = serializer.data.get('category', None) @@ -1000,8 +1101,9 @@ def post(self, request, *args, **kwargs): else: bookmark = Bookmark(user=self.user, sound_id=sound_id) bookmark.save() - return Response(data={'detail': f'Successfully bookmarked sound {sound_id}.'}, - status=status.HTTP_201_CREATED) + return Response( + data={'detail': f'Successfully bookmarked sound {sound_id}.'}, status=status.HTTP_201_CREATED + ) else: return Response({'detail': serializer.errors}, status=status.HTTP_400_BAD_REQUEST) @@ -1016,7 +1118,7 @@ def get_description(cls): % (prepend_base('/docs/api'), '%s#rate-sound-oauth2-required' % resources_doc_filename, get_formatted_examples_for_view('RateSound', 'apiv2-user-create-rating', max=5)) - def post(self, request, *args, **kwargs): + def post(self, request, *args, **kwargs): sound_id = kwargs['pk'] try: sound = Sound.objects.get(id=sound_id, moderation_state="OK", processing_state="OK") @@ -1027,16 +1129,23 @@ def post(self, request, *args, **kwargs): if serializer.is_valid(): try: if not settings.ALLOW_WRITE_WHEN_SESSION_BASED_AUTHENTICATION and self.auth_method_name == 'Session': - return Response(data={'detail': f'Successfully rated sound {sound_id}.', - 'note': 'This rating has not been saved in the database as browseable API ' - 'is only for testing purposes.'}, - status=status.HTTP_201_CREATED) + return Response( + data={ + 'detail': f'Successfully rated sound {sound_id}.', + 'note': + 'This rating has not been saved in the database as browseable API ' + 'is only for testing purposes.' + }, + status=status.HTTP_201_CREATED + ) else: SoundRating.objects.create( - user=self.user, sound_id=sound_id, rating=int(request.data['rating']) * 2) + user=self.user, sound_id=sound_id, rating=int(request.data['rating']) * 2 + ) Sound.objects.filter(id=sound_id).update(is_index_dirty=True) - return Response(data={'detail': f'Successfully rated sound {sound_id}.'}, - status=status.HTTP_201_CREATED) + return Response( + data={'detail': f'Successfully rated sound {sound_id}.'}, status=status.HTTP_201_CREATED + ) except IntegrityError: raise ConflictException(msg=f'User has already rated sound {sound_id}', resource=self) except: @@ -1055,7 +1164,7 @@ def get_description(cls): % (prepend_base('/docs/api'), '%s#comment-sound-oauth2-required' % resources_doc_filename, get_formatted_examples_for_view('CommentSound', 'apiv2-user-create-comment', max=5)) - def post(self, request, *args, **kwargs): + def post(self, request, *args, **kwargs): sound_id = kwargs['pk'] try: sound = Sound.objects.get(id=sound_id, moderation_state="OK", processing_state="OK") @@ -1065,14 +1174,20 @@ def post(self, request, *args, **kwargs): serializer = CreateCommentSerializer(data=request.data) if serializer.is_valid(): if not settings.ALLOW_WRITE_WHEN_SESSION_BASED_AUTHENTICATION and self.auth_method_name == 'Session': - return Response(data={'detail': f'Successfully commented sound {sound_id}.', - 'note': 'This comment has not been saved in the database as browseable API is ' - 'only for testing purposes.'}, - status=status.HTTP_201_CREATED) + return Response( + data={ + 'detail': f'Successfully commented sound {sound_id}.', + 'note': + 'This comment has not been saved in the database as browseable API is ' + 'only for testing purposes.' + }, + status=status.HTTP_201_CREATED + ) else: sound.add_comment(self.user, request.data['comment']) - return Response(data={'detail': f'Successfully commented sound {sound_id}.'}, - status=status.HTTP_201_CREATED) + return Response( + data={'detail': f'Successfully commented sound {sound_id}.'}, status=status.HTTP_201_CREATED + ) else: return Response({'detail': serializer.errors}, status=status.HTTP_400_BAD_REQUEST) @@ -1081,6 +1196,7 @@ def post(self, request, *args, **kwargs): # OTHER VIEWS ############# + @api_view(['GET']) @throttle_classes([]) @authentication_classes([]) @@ -1112,14 +1228,14 @@ def get_description(cls): % (prepend_base('/docs/api'), '%s#me-information-about-user-authenticated-using-oauth2-oauth2-required' % resources_doc_filename) - def get(self, request, *args, **kwargs): + def get(self, request, *args, **kwargs): api_logger.info(self.log_message('me')) if self.user: response_data = UserSerializer(self.user, context=self.get_serializer_context()).data response_data.update({ - 'email': self.user.profile.get_email_for_delivery(), - 'unique_id': self.user.id, - 'bookmark_categories': prepend_base(reverse('apiv2-me-bookmark-categories')), + 'email': self.user.profile.get_email_for_delivery(), + 'unique_id': self.user.id, + 'bookmark_categories': prepend_base(reverse('apiv2-me-bookmark-categories')), }) return Response(response_data, status=status.HTTP_200_OK) else: @@ -1136,7 +1252,7 @@ def get_description(cls): % (prepend_base('/docs/api'), '%s#me-bookmark-categories' % resources_doc_filename, get_formatted_examples_for_view('MeBookmarkCategories', 'apiv2-me-bookmark-categories', max=5)) - def get(self, request, *args, **kwargs): + def get(self, request, *args, **kwargs): api_logger.info(self.log_message(f"user:{self.user.username} bookmark_categories")) return super().get(request, *args, **kwargs) @@ -1156,7 +1272,7 @@ def get_queryset(self): return list(categories) else: raise ServerErrorException(resource=self) - + class MeBookmarkCategorySounds(OauthRequiredAPIView, ListAPIView): serializer_class = SoundListSerializer @@ -1169,9 +1285,13 @@ def get_description(cls): get_formatted_examples_for_view('MeBookmarkCategorySounds', 'apiv2-me-bookmark-category-sounds', max=5)) - def get(self, request, *args, **kwargs): - api_logger.info(self.log_message('user:%s sounds_for_bookmark_category:%s' - % (self.user.username, str(self.kwargs.get('category_id', None))))) + def get(self, request, *args, **kwargs): + api_logger.info( + self.log_message( + 'user:%s sounds_for_bookmark_category:%s' % + (self.user.username, str(self.kwargs.get('category_id', None))) + ) + ) return super().get(request, *args, **kwargs) def get_queryset(self): @@ -1196,7 +1316,9 @@ def get_queryset(self): else: raise ServerErrorException(resource=self) + class AvailableAudioDescriptors(GenericAPIView): + @classmethod def get_description(cls): return 'Get a list of valid audio descriptor names that can be used in content/combined search, in sound ' \ @@ -1204,27 +1326,31 @@ def get_description(cls): 'Full documentation can be found here.' \ % (prepend_base('/docs/api'), '%s#available-audio-descriptors' % resources_doc_filename) - def get(self, request, *args, **kwargs): + def get(self, request, *args, **kwargs): api_logger.info(self.log_message('available_audio_descriptors')) try: descriptor_names = Similarity.get_descriptor_names() del descriptor_names['all'] for key, value in descriptor_names.items(): - descriptor_names[key] = [item[1:] for item in value] # remove initial dot from descriptor names + descriptor_names[key] = [item[1:] for item in value] # remove initial dot from descriptor names return Response({ 'fixed-length': { - 'one-dimensional': [item for item in descriptor_names['fixed-length'] - if item not in descriptor_names['multidimensional']], + 'one-dimensional': [ + item for item in descriptor_names['fixed-length'] + if item not in descriptor_names['multidimensional'] + ], 'multi-dimensional': descriptor_names['multidimensional'] }, 'variable-length': descriptor_names['variable-length'] - }, status=status.HTTP_200_OK) + }, + status=status.HTTP_200_OK) except Exception as e: raise ServerErrorException(resource=self) class FreesoundApiV2Resources(GenericAPIView): + @classmethod def get_description(cls): return 'List of resources available in the Freesound APIv2. ' \ @@ -1232,74 +1358,137 @@ def get_description(cls): 'urls containing elements in brackets (<>) should be replaced with the corresponding variables.' \ % (prepend_base('/docs/api'), 'index.html') - def get(self, request, *args, **kwargs): + def get(self, request, *args, **kwargs): api_logger.info(self.log_message('api_root')) api_index = [ - {'Search resources': OrderedDict(sorted({ - '01 Text Search': prepend_base( - reverse('apiv2-sound-text-search'), request_is_secure=request.is_secure()), - '02 Content Search': prepend_base( - reverse('apiv2-sound-content-search'), request_is_secure=request.is_secure()), - '03 Combined Search': prepend_base( - reverse('apiv2-sound-combined-search'), request_is_secure=request.is_secure()), - }.items(), key=lambda t: t[0]))}, - {'Sound resources': OrderedDict(sorted({ - '01 Sound instance': prepend_base( - reverse('apiv2-sound-instance', args=[0]).replace('0', ''), - request_is_secure=request.is_secure()), - '02 Similar sounds': prepend_base( - reverse('apiv2-similarity-sound', args=[0]).replace('0', ''), - request_is_secure=request.is_secure()), - '03 Sound analysis': prepend_base( - reverse('apiv2-sound-analysis', args=[0]).replace('0', ''), - request_is_secure=request.is_secure()), - '04 Sound comments': prepend_base( - reverse('apiv2-sound-comments', args=[0]).replace('0', ''), - request_is_secure=request.is_secure()), - '05 Download sound': prepend_base( - reverse('apiv2-sound-download', args=[0]).replace('0', '')), - '06 Bookmark sound': prepend_base( - reverse('apiv2-user-create-bookmark', args=[0]).replace('0', '')), - '07 Rate sound': prepend_base( - reverse('apiv2-user-create-rating', args=[0]).replace('0', '')), - '08 Comment sound': prepend_base( - reverse('apiv2-user-create-comment', args=[0]).replace('0', '')), - '09 Upload sound': prepend_base(reverse('apiv2-uploads-upload')), - '10 Describe sound': prepend_base(reverse('apiv2-uploads-describe')), - '11 Pending uploads': prepend_base(reverse('apiv2-uploads-pending')), - '12 Edit sound description': prepend_base( - reverse('apiv2-sound-edit', args=[0]).replace('0', '')), - }.items(), key=lambda t: t[0]))}, - {'User resources': OrderedDict(sorted({ - '01 User instance': prepend_base( - reverse('apiv2-user-instance', args=['uname']).replace('uname', ''), - request_is_secure=request.is_secure()), - '02 User sounds': prepend_base( - reverse('apiv2-user-sound-list', args=['uname']).replace('uname', ''), - request_is_secure=request.is_secure()), - '03 User packs': prepend_base( - reverse('apiv2-user-packs', args=['uname']).replace('uname', ''), - request_is_secure=request.is_secure()), - }.items(), key=lambda t: t[0]))}, - {'Pack resources': OrderedDict(sorted({ - '01 Pack instance': prepend_base( - reverse('apiv2-pack-instance', args=[0]).replace('0', ''), - request_is_secure=request.is_secure()), - '02 Pack sounds': prepend_base( - reverse('apiv2-pack-sound-list', args=[0]).replace('0', ''), - request_is_secure=request.is_secure()), - '03 Download pack': prepend_base( - reverse('apiv2-pack-download', args=[0]).replace('0', '')), - }.items(), key=lambda t: t[0]))}, - {'Other resources': OrderedDict(sorted({ - '01 Me (information about user authenticated using oauth)': prepend_base(reverse('apiv2-me')), - '02 My bookmark categories': prepend_base(reverse('apiv2-me-bookmark-categories')), - '03 My bookmark category sounds': prepend_base( - reverse('apiv2-me-bookmark-category-sounds', args=[0]).replace('0', ''), - request_is_secure=request.is_secure()), - '04 Available audio descriptors': prepend_base(reverse('apiv2-available-descriptors')), - }.items(), key=lambda t: t[0]))}, - ] + { + 'Search resources': + OrderedDict( + sorted({ + '01 Text Search': + prepend_base(reverse('apiv2-sound-text-search'), request_is_secure=request.is_secure()), + '02 Content Search': + prepend_base( + reverse('apiv2-sound-content-search'), request_is_secure=request.is_secure() + ), + '03 Combined Search': + prepend_base( + reverse('apiv2-sound-combined-search'), request_is_secure=request.is_secure() + ), + }.items(), + key=lambda t: t[0]) + ) + }, + { + 'Sound resources': + OrderedDict( + sorted({ + '01 Sound instance': + prepend_base( + reverse('apiv2-sound-instance', args=[0]).replace('0', ''), + request_is_secure=request.is_secure() + ), + '02 Similar sounds': + prepend_base( + reverse('apiv2-similarity-sound', args=[0]).replace('0', ''), + request_is_secure=request.is_secure() + ), + '03 Sound analysis': + prepend_base( + reverse('apiv2-sound-analysis', args=[0]).replace('0', ''), + request_is_secure=request.is_secure() + ), + '04 Sound comments': + prepend_base( + reverse('apiv2-sound-comments', args=[0]).replace('0', ''), + request_is_secure=request.is_secure() + ), + '05 Download sound': + prepend_base(reverse('apiv2-sound-download', args=[0]).replace('0', '')), + '06 Bookmark sound': + prepend_base( + reverse('apiv2-user-create-bookmark', args=[0]).replace('0', '') + ), + '07 Rate sound': + prepend_base(reverse('apiv2-user-create-rating', args=[0]).replace('0', '')), + '08 Comment sound': + prepend_base(reverse('apiv2-user-create-comment', args=[0]).replace('0', '')), + '09 Upload sound': + prepend_base(reverse('apiv2-uploads-upload')), + '10 Describe sound': + prepend_base(reverse('apiv2-uploads-describe')), + '11 Pending uploads': + prepend_base(reverse('apiv2-uploads-pending')), + '12 Edit sound description': + prepend_base(reverse('apiv2-sound-edit', args=[0]).replace('0', '')), + }.items(), + key=lambda t: t[0]) + ) + }, + { + 'User resources': + OrderedDict( + sorted({ + '01 User instance': + prepend_base( + reverse('apiv2-user-instance', args=['uname']).replace('uname', ''), + request_is_secure=request.is_secure() + ), + '02 User sounds': + prepend_base( + reverse('apiv2-user-sound-list', args=['uname']).replace('uname', ''), + request_is_secure=request.is_secure() + ), + '03 User packs': + prepend_base( + reverse('apiv2-user-packs', args=['uname']).replace('uname', ''), + request_is_secure=request.is_secure() + ), + }.items(), + key=lambda t: t[0]) + ) + }, + { + 'Pack resources': + OrderedDict( + sorted({ + '01 Pack instance': + prepend_base( + reverse('apiv2-pack-instance', args=[0]).replace('0', ''), + request_is_secure=request.is_secure() + ), + '02 Pack sounds': + prepend_base( + reverse('apiv2-pack-sound-list', args=[0]).replace('0', ''), + request_is_secure=request.is_secure() + ), + '03 Download pack': + prepend_base(reverse('apiv2-pack-download', args=[0]).replace('0', '')), + }.items(), + key=lambda t: t[0]) + ) + }, + { + 'Other resources': + OrderedDict( + sorted({ + '01 Me (information about user authenticated using oauth)': + prepend_base(reverse('apiv2-me')), + '02 My bookmark categories': + prepend_base(reverse('apiv2-me-bookmark-categories')), + '03 My bookmark category sounds': + prepend_base( + reverse('apiv2-me-bookmark-category-sounds', + args=[0]).replace('0', ''), + request_is_secure=request.is_secure() + ), + '04 Available audio descriptors': + prepend_base(reverse('apiv2-available-descriptors')), + }.items(), + key=lambda t: t[0]) + ) + }, + ] # Yaml format can not represent ordered dicts, so turn ordered dict to dict if these formats are requested if request.accepted_renderer.format in ['yaml']: @@ -1350,8 +1539,10 @@ def create_apiv2_key(request): api_client.accepted_tos = form.cleaned_data['accepted_tos'] api_client.save() form = ApiV2ClientForm() - api_logger.info('new_credential <> (ApiV2 Auth:%s Dev:%s User:%s Client:%s)' % - (None, request.user.username, None, api_client.client_id)) + api_logger.info( + 'new_credential <> (ApiV2 Auth:%s Dev:%s User:%s Client:%s)' % + (None, request.user.username, None, api_client.client_id) + ) else: form = ApiV2ClientForm() @@ -1394,12 +1585,15 @@ def edit_api_credential(request, key): messages.add_message(request, messages.INFO, f"Credentials with name {client.name} have been updated.") return HttpResponseRedirect(reverse("apiv2-apply")) else: - form = ApiV2ClientForm(initial={'name': client.name, - 'url': client.url, - 'redirect_uri': client.redirect_uri, - 'description': client.description, - 'accepted_tos': client.accepted_tos - }) + form = ApiV2ClientForm( + initial={ + 'name': client.name, + 'url': client.url, + 'redirect_uri': client.redirect_uri, + 'description': client.description, + 'accepted_tos': client.accepted_tos + } + ) use_https_in_callback = True if settings.DEBUG: use_https_in_callback = False @@ -1428,12 +1622,8 @@ def monitor_api_credential(request, key): last_year = datetime.datetime.now().year - 1 tvars = { 'n_days': n_days, - 'n_days_options': [ - (30, '1 month'), - (93, '3 months'), - (182, '6 months'), - (365, '1 year') - ], 'client': client, + 'n_days_options': [(30, '1 month'), (93, '3 months'), (182, '6 months'), (365, '1 year')], + 'client': client, 'data': json.dumps([(str(date), count) for date, count in usage_history]), 'total_in_range': sum([count for _, count in usage_history]), 'total_in_range_above_5000': sum([count - 5000 for _, count in usage_history if count > 5000]), @@ -1503,13 +1693,15 @@ def granted_permissions(request): }) grant_and_token_names.append(grant.application.apiv2_client.name) - return render(request, 'accounts/manage_api_permissions.html', { - 'user': request.user, - 'tokens': tokens, - 'grants': grants, - 'show_expiration_date': False, - 'activePage': 'api', - }) + return render( + request, 'accounts/manage_api_permissions.html', { + 'user': request.user, + 'tokens': tokens, + 'grants': grants, + 'show_expiration_date': False, + 'activePage': 'api', + } + ) @login_required @@ -1542,5 +1734,10 @@ def permission_granted(request): logout_next = quote(logout_next) else: logout_next = reverse('api-login') - return render(request, 'oauth2_provider/app_authorized.html', - {'code': code, 'app_name': app_name, 'logout_next': logout_next}) + return render( + request, 'oauth2_provider/app_authorized.html', { + 'code': code, + 'app_name': app_name, + 'logout_next': logout_next + } + ) diff --git a/bookmarks/forms.py b/bookmarks/forms.py index 478327554..068a0acb7 100644 --- a/bookmarks/forms.py +++ b/bookmarks/forms.py @@ -24,6 +24,7 @@ class BookmarkCategoryForm(forms.ModelForm): + class Meta: model = BookmarkCategory fields = ('name',) @@ -33,15 +34,8 @@ class Meta: class BookmarkForm(forms.Form): - category = forms.ChoiceField( - label=False, - choices=[], - required=False) - new_category_name = forms.CharField( - label=False, - help_text=None, - max_length=128, - required=False) + category = forms.ChoiceField(label=False, choices=[], required=False) + new_category_name = forms.CharField(label=False, help_text=None, max_length=128, required=False) use_last_category = forms.BooleanField(widget=forms.HiddenInput(), required=False, initial=False) user_bookmark_categories = None @@ -57,11 +51,11 @@ def __init__(self, *args, **kwargs): (self.NEW_CATEGORY_CHOICE_VALUE, 'Create a new category...')] + \ ([(category.id, category.name) for category in self.user_bookmark_categories] if self.user_bookmark_categories else []) - + self.fields['new_category_name'].widget.attrs['placeholder'] = "Fill in the name for the new category" self.fields['category'].widget.attrs = { - 'data-grey-items': f'{self.NO_CATEGORY_CHOICE_VALUE},{self.NEW_CATEGORY_CHOICE_VALUE}'} - + 'data-grey-items': f'{self.NO_CATEGORY_CHOICE_VALUE},{self.NEW_CATEGORY_CHOICE_VALUE}' + } def save(self, *args, **kwargs): category_to_use = None @@ -90,7 +84,6 @@ def save(self, *args, **kwargs): # If bookmark already exists, don't save it and return the existing one bookmark, _ = Bookmark.objects.get_or_create( - sound_id=self.sound_id, user=self.user_saving_bookmark, category=category_to_use) + sound_id=self.sound_id, user=self.user_saving_bookmark, category=category_to_use + ) return bookmark - - diff --git a/bookmarks/models.py b/bookmarks/models.py index 1673abb22..0225be870 100755 --- a/bookmarks/models.py +++ b/bookmarks/models.py @@ -27,7 +27,7 @@ class BookmarkCategory(models.Model): user = models.ForeignKey(User, on_delete=models.CASCADE) name = models.CharField(max_length=128, default="") - + def __str__(self): return f"{self.name}" @@ -35,10 +35,11 @@ def __str__(self): class Bookmark(models.Model): user = models.ForeignKey(User, on_delete=models.CASCADE) category = models.ForeignKey( - BookmarkCategory, blank=True, null=True, default=None, related_name='bookmarks', on_delete=models.SET_NULL) + BookmarkCategory, blank=True, null=True, default=None, related_name='bookmarks', on_delete=models.SET_NULL + ) sound = models.ForeignKey(Sound, on_delete=models.CASCADE) created = models.DateTimeField(db_index=True, auto_now_add=True) - + def __str__(self): return f"Bookmark: {self.name}" @@ -54,5 +55,5 @@ def sound_name(self): return self.sound.original_filename class Meta: - ordering = ("-created", ) + ordering = ("-created",) unique_together = (('user_id', 'category_id', 'sound_id'),) diff --git a/bookmarks/tests.py b/bookmarks/tests.py index b71117fc1..ce604f0c8 100644 --- a/bookmarks/tests.py +++ b/bookmarks/tests.py @@ -38,16 +38,21 @@ def test_old_bookmarks_for_user_redirect(self): # User not logged in, redirect raises 404 resp = self.client.get(reverse('bookmarks-for-user', kwargs={'username': 'Anton'})) self.assertEqual(404, resp.status_code) - + # User logged in, redirect to home/bookmarks page self.client.force_login(user) resp = self.client.get(reverse('bookmarks-for-user', kwargs={'username': 'Anton'})) self.assertRedirects(resp, reverse('bookmarks')) # User logged in, redirect to home/bookmarks/category page - resp = self.client.get(reverse('bookmarks-for-user-for-category', kwargs={'username': 'Anton', 'category_id': category.id})) + resp = self.client.get( + reverse('bookmarks-for-user-for-category', kwargs={ + 'username': 'Anton', + 'category_id': category.id + }) + ) self.assertRedirects(resp, reverse('bookmarks-category', kwargs={'category_id': category.id})) - + def test_bookmarks(self): user = User.objects.get(username='Anton') self.client.force_login(user) @@ -56,7 +61,7 @@ def test_bookmarks(self): response = self.client.get(reverse('bookmarks')) self.assertEqual(200, response.status_code) self.assertContains(response, 'There are no uncategorized bookmarks') - + # Create bookmarks category = bookmarks.models.BookmarkCategory.objects.create(name='Category1', user=user) bookmarks.models.Bookmark.objects.create(user=user, sound_id=10) @@ -66,13 +71,13 @@ def test_bookmarks(self): # Test main bookmarks page response = self.client.get(reverse('bookmarks')) self.assertEqual(200, response.status_code) - self.assertEqual(1, len(response.context['page'].object_list)) # 1 bookmark uncategorized - self.assertEqual(1, len(response.context['bookmark_categories'])) # 1 bookmark cateogry + self.assertEqual(1, len(response.context['page'].object_list)) # 1 bookmark uncategorized + self.assertEqual(1, len(response.context['bookmark_categories'])) # 1 bookmark cateogry # Test bookmark cateogry page response = self.client.get(reverse('bookmarks-category', kwargs={'category_id': category.id})) self.assertEqual(200, response.status_code) - self.assertEqual(2, len(response.context['page'].object_list)) # 2 sounds in cateogry + self.assertEqual(2, len(response.context['page'].object_list)) # 2 sounds in cateogry self.assertContains(response, category.name) # Test category does not exist diff --git a/bookmarks/views.py b/bookmarks/views.py index 75dbd625a..532c7c40c 100755 --- a/bookmarks/views.py +++ b/bookmarks/views.py @@ -46,11 +46,13 @@ def bookmarks(request, category_id=None): category = get_object_or_404(BookmarkCategory, id=category_id, user=user) bookmarked_sounds = category.bookmarks.select_related("sound", "sound__user").all() bookmark_categories = BookmarkCategory.objects.filter(user=user).annotate(num_bookmarks=Count('bookmarks')) - tvars = {'user': user, - 'is_owner': is_owner, - 'n_uncat': n_uncat, - 'category': category, - 'bookmark_categories': bookmark_categories} + tvars = { + 'user': user, + 'is_owner': is_owner, + 'n_uncat': n_uncat, + 'category': category, + 'bookmark_categories': bookmark_categories + } tvars.update(paginate(request, bookmarked_sounds, settings.BOOKMARKS_PER_PAGE)) return render(request, 'bookmarks/bookmarks.html', tvars) @@ -92,10 +94,12 @@ def add_bookmark(request, sound_id): msg_to_return = '' if request.method == 'POST': user_bookmark_categories = BookmarkCategory.objects.filter(user=request.user) - form = BookmarkForm(request.POST, - user_bookmark_categories=user_bookmark_categories, - sound_id=sound_id, - user_saving_bookmark=request.user) + form = BookmarkForm( + request.POST, + user_bookmark_categories=user_bookmark_categories, + sound_id=sound_id, + user_saving_bookmark=request.user + ) if form.is_valid(): saved_bookmark = form.save() msg_to_return = f'Bookmark created with name "{saved_bookmark.sound_name}"' @@ -145,14 +149,20 @@ def get_form_for_sound(request, sound_id): except IndexError: last_category = None user_bookmark_categories = BookmarkCategory.objects.filter(user=request.user) - form = BookmarkForm(initial={'category': last_category.id if last_category else BookmarkForm.NO_CATEGORY_CHOICE_VALUE}, - prefix=sound.id, - user_bookmark_categories=user_bookmark_categories) - categories_already_containing_sound = BookmarkCategory.objects.filter(user=request.user, - bookmarks__sound=sound).distinct() - sound_has_bookmark_without_category = Bookmark.objects.filter(user=request.user, sound=sound, category=None).exists() + form = BookmarkForm( + initial={'category': last_category.id if last_category else BookmarkForm.NO_CATEGORY_CHOICE_VALUE}, + prefix=sound.id, + user_bookmark_categories=user_bookmark_categories + ) + categories_already_containing_sound = BookmarkCategory.objects.filter( + user=request.user, bookmarks__sound=sound + ).distinct() + sound_has_bookmark_without_category = Bookmark.objects.filter( + user=request.user, sound=sound, category=None + ).exists() add_bookmark_url = '/'.join( - request.build_absolute_uri(reverse('add-bookmark', args=[sound_id])).split('/')[:-2]) + '/' + request.build_absolute_uri(reverse('add-bookmark', args=[sound_id])).split('/')[:-2] + ) + '/' tvars = { 'bookmarks': Bookmark.objects.filter(user=request.user, sound=sound).exists(), 'sound_id': sound.id, diff --git a/clustering/clustering.py b/clustering/clustering.py index ccee9b738..bbd509e95 100644 --- a/clustering/clustering.py +++ b/clustering/clustering.py @@ -35,7 +35,7 @@ from . import clustering_settings as clust_settings -# The following packages are only needed if the running process is configured to be a Celery worker. +# The following packages are only needed if the running process is configured to be a Celery worker. # We avoid importing them in appservers to avoid having to install unneeded dependencies. if settings.IS_CELERY_WORKER: import community as com @@ -65,6 +65,7 @@ class ClusteringEngine(object): method. Moreover, a few unsued alternative methods for performing some intermediate steps are left here for developement and research purpose. """ + def __init__(self): self.feature_store = FeaturesStore() @@ -129,7 +130,7 @@ def _calinski_idx_reference_features_clusters(self, partition): """ reference_features, clusters = self._prepare_clustering_result_and_reference_features_for_evaluation(partition) return metrics.calinski_harabaz_score(reference_features, clusters) - + def _davies_idx_reference_features_clusters(self, partition): """Computes the Davies-Bouldin score between reference features and the given clustering classes. @@ -158,7 +159,9 @@ def _evaluation_metrics(self, partition): # we compute the evaluation metrics only if some reference features are available for evaluation # we return None when they are not available not to break the following part of the code if clust_settings.REFERENCE_FEATURES in clust_settings.AVAILABLE_FEATURES: - reference_features, clusters = self._prepare_clustering_result_and_reference_features_for_evaluation(partition) + reference_features, clusters = self._prepare_clustering_result_and_reference_features_for_evaluation( + partition + ) ami = np.average(mutual_info_classif(reference_features, clusters, discrete_features=True)) ss = metrics.silhouette_score(reference_features, clusters, metric='euclidean') ci = metrics.calinski_harabaz_score(reference_features, clusters) @@ -183,10 +186,15 @@ def _ratio_intra_community_edges(self, graph, communities): # counts the number of edges inside a community intra_community_edges = [graph.subgraph(block).size() for block in communities] # counts the number of edges from nodes in a community to nodes inside or outside the same community - total_community_edges = [sum([graph.degree(node_id) for node_id in community])-intra_community_edges[i] - for i, community in enumerate(communities)] + total_community_edges = [ + sum([graph.degree(node_id) + for node_id in community]) - intra_community_edges[i] + for i, community in enumerate(communities) + ] # ratio (high value -> good cluster) - ratio_intra_community_edges = [round(a/float(b), 2) for a,b in zip(intra_community_edges, total_community_edges)] + ratio_intra_community_edges = [ + round(a / float(b), 2) for a, b in zip(intra_community_edges, total_community_edges) + ] return ratio_intra_community_edges @@ -204,17 +212,21 @@ def _point_centralities(self, graph, communities): Dict{Int: Float}: Dict containing the community centrality value for each sound ({: }). """ - # + # subgraphs = [graph.subgraph(community) for community in communities] communities_centralities = [nx.algorithms.centrality.degree_centrality(subgraph) for subgraph in subgraphs] # merge and normalize in each community - node_community_centralities = {k: old_div(v,max(d.values())) for d in communities_centralities for k, v in d.items()} + node_community_centralities = { + k: old_div(v, max(d.values())) for d in communities_centralities for k, v in d.items() + } return node_community_centralities - - def _save_results_to_file(self, query_params, features, graph_json, sound_ids, modularity, - num_communities, ratio_intra_community_edges, ami, ss, ci, communities): + + def _save_results_to_file( + self, query_params, features, graph_json, sound_ids, modularity, num_communities, ratio_intra_community_edges, + ami, ss, ci, communities + ): """Saves a json file to disk containing the clustering results information listed below. This is used when developing the clustering method. The results and the evaluation metrics are made accessible @@ -236,7 +248,7 @@ def _save_results_to_file(self, query_params, features, graph_json, sound_ids, m """ if clust_settings.SAVE_RESULTS_FOLDER: result = { - 'query_params' : query_params, + 'query_params': query_params, 'sound_ids': sound_ids, 'num_clusters': num_communities, 'graph': graph_json, @@ -248,10 +260,7 @@ def _save_results_to_file(self, query_params, features, graph_json, sound_ids, m 'calinski_harabaz_score': ci, 'communities': communities } - with open(os.path.join( - clust_settings.SAVE_RESULTS_FOLDER, - f'{query_params}.json' - ), 'w') as f: + with open(os.path.join(clust_settings.SAVE_RESULTS_FOLDER, f'{query_params}.json'), 'w') as f: json.dump(result, f) def create_knn_graph(self, sound_ids_list, features=clust_settings.DEFAULT_FEATURES): @@ -265,10 +274,10 @@ def create_knn_graph(self, sound_ids_list, features=clust_settings.DEFAULT_FEATU Returns: (nx.Graph): NetworkX graph representation of sounds. """ - # Create k nearest neighbors graph + # Create k nearest neighbors graph graph = nx.Graph() graph.add_nodes_from(sound_ids_list) - # we set k to log2(N), where N is the number of elements to cluster. This allows us to reach a sufficient number of + # we set k to log2(N), where N is the number of elements to cluster. This allows us to reach a sufficient number of # neighbors for small collections, while limiting it for larger collections, which ensures low-computational complexity. k = int(np.ceil(np.log2(len(sound_ids_list)))) @@ -305,7 +314,9 @@ def create_common_nn_graph(self, sound_ids_list, features=clust_settings.DEFAULT for i, node_i in enumerate(knn_graph.nodes): for j, node_j in enumerate(knn_graph.nodes): if j > i: - num_common_neighbors = len(set(knn_graph.neighbors(node_i)).intersection(knn_graph.neighbors(node_j))) + num_common_neighbors = len( + set(knn_graph.neighbors(node_i)).intersection(knn_graph.neighbors(node_j)) + ) if num_common_neighbors > 0: graph.add_edge(node_i, node_j, weight=num_common_neighbors) @@ -339,14 +350,14 @@ def cluster_graph(self, graph): {: }, the number of communities (clusters), the sound ids in the communities and the modularity of the graph partition. - """ + """ # Community detection in the graph - partition = com.best_partition(graph) + partition = com.best_partition(graph) num_communities = max(partition.values()) + 1 - communities = [[key for key, value in six.iteritems(partition ) if value == i] for i in range(num_communities)] + communities = [[key for key, value in six.iteritems(partition) if value == i] for i in range(num_communities)] # overall quality (modularity of the partition) - modularity = com.modularity(partition , graph) + modularity = com.modularity(partition, graph) return partition, num_communities, communities, modularity @@ -364,13 +375,13 @@ def cluster_graph_overlap(self, graph, k=5): Tuple(Dict{Int: Int}, int, List[List[Int]], None): 4-element tuple containing the clustering classes for each sound {: }, the number of communities (clusters), the sound ids in the communities and None. - """ + """ communities = [list(community) for community in k_clique_communities(graph, k)] # communities = [list(community) for community in greedy_modularity_communities(graph)] num_communities = len(communities) partition = {sound_id: cluster_id for cluster_id, cluster in enumerate(communities) for sound_id in cluster} - return partition, num_communities, communities, None + return partition, num_communities, communities, None def remove_lowest_quality_cluster(self, graph, partition, communities, ratio_intra_community_edges): """Removes the lowest quality cluster in the given graph. @@ -417,12 +428,15 @@ def cluster_points(self, query_params, features, sound_ids): """ start_time = time() sound_ids = [str(s) for s in sound_ids] - logger.info('Request clustering of {} points: {} ... from the query "{}"' - .format(len(sound_ids), ', '.join(sound_ids[:20]), json.dumps(query_params))) + logger.info( + 'Request clustering of {} points: {} ... from the query "{}"'.format( + len(sound_ids), ', '.join(sound_ids[:20]), json.dumps(query_params) + ) + ) graph = self.create_knn_graph(sound_ids, features=features) - if len(graph.nodes) == 0: # the graph does not contain any node + if len(graph.nodes) == 0: # the graph does not contain any node return {'error': False, 'result': None, 'graph': None} partition, num_communities, communities, modularity = self.cluster_graph(graph) @@ -447,20 +461,25 @@ def cluster_points(self, query_params, features, sound_ids): ami, ss, ci = self._evaluation_metrics(partition) end_time = time() - logger.info('Clustering done! It took {} seconds. ' - 'Modularity: {}, ' - 'Average ratio_intra_community_edges: {}, ' - 'Average Mutual Information with reference: {}, ' - 'Silouhette Coefficient with reference: {}, ' - 'Calinski Index with reference: {}, ' - 'Davies Index with reference: {}' - .format(end_time-start_time, modularity, np.mean(ratio_intra_community_edges), ami, ss, ci, None)) + logger.info( + 'Clustering done! It took {} seconds. ' + 'Modularity: {}, ' + 'Average ratio_intra_community_edges: {}, ' + 'Average Mutual Information with reference: {}, ' + 'Silouhette Coefficient with reference: {}, ' + 'Calinski Index with reference: {}, ' + 'Davies Index with reference: {}'.format( + end_time - start_time, modularity, np.mean(ratio_intra_community_edges), ami, ss, ci, None + ) + ) # Export graph as json graph_json = json_graph.node_link_data(graph) # Save results to file if SAVE_RESULTS_FOLDER is configured in clustering settings - self._save_results_to_file(query_params, features, graph_json, sound_ids, modularity, - num_communities, ratio_intra_community_edges, ami, ss, ci, communities) + self._save_results_to_file( + query_params, features, graph_json, sound_ids, modularity, num_communities, ratio_intra_community_edges, + ami, ss, ci, communities + ) return {'error': False, 'result': communities, 'graph': graph_json} diff --git a/clustering/clustering_settings.py b/clustering/clustering_settings.py index 2e63cccbc..cf6a9e735 100644 --- a/clustering/clustering_settings.py +++ b/clustering/clustering_settings.py @@ -18,7 +18,6 @@ # See AUTHORS file. # - # Directory where the Gaia dataset index files are located. INDEX_DIR = '/freesound-data/clustering_index/' @@ -46,15 +45,15 @@ MAX_RESULTS_FOR_CLUSTERING = 1000 # Cache settings -# One day timeout for keeping clustering results. The cache timer is reset when the clustering is +# One day timeout for keeping clustering results. The cache timer is reset when the clustering is # requested so that popular queries that are performed once a day minimum will always stay in cache # and won't be recomputed. -CLUSTERING_CACHE_TIME = 24*60*60*1 -# One minute timeout for keeping the pending state. When a clustering is being performed async in a -# Celery worker, we consider the clustering as pending for only 1 minute. This may be useful if a -# worker task got stuck. There should be a settings in celery to stop a worker task if it is running +CLUSTERING_CACHE_TIME = 24 * 60 * 60 * 1 +# One minute timeout for keeping the pending state. When a clustering is being performed async in a +# Celery worker, we consider the clustering as pending for only 1 minute. This may be useful if a +# worker task got stuck. There should be a settings in celery to stop a worker task if it is running # for too long. -CLUSTERING_PENDING_CACHE_TIME = 60*1 +CLUSTERING_PENDING_CACHE_TIME = 60 * 1 # Folder for saving the clustering results with evaluation (dev/debug/research purpose) SAVE_RESULTS_FOLDER = None diff --git a/clustering/features_store.py b/clustering/features_store.py index 51915ce2d..2f78a67e1 100644 --- a/clustering/features_store.py +++ b/clustering/features_store.py @@ -36,9 +36,11 @@ class RedisStore(object): + def __init__(self): self.r = redis.StrictRedis( - host=settings.REDIS_HOST, port=settings.REDIS_PORT, db=settings.AUDIO_FEATURES_REDIS_STORE_ID) + host=settings.REDIS_HOST, port=settings.REDIS_PORT, db=settings.AUDIO_FEATURES_REDIS_STORE_ID + ) def set_feature(self, sound_id, feature): self.r.set(str(sound_id), json.dumps(feature)) @@ -58,15 +60,20 @@ def get_features(self, sound_ids): class FeaturesStore(object): """Method for storing and retrieving audio features """ + def __init__(self): self.redis = RedisStore() self.__load_features() def __load_features(self): - self.AS_features = json.load(open(os.path.join( - clust_settings.INDEX_DIR, - clust_settings.AVAILABLE_FEATURES[clust_settings.DEFAULT_FEATURES]['DATASET_FILE'] - ), 'r')) + self.AS_features = json.load( + open( + os.path.join( + clust_settings.INDEX_DIR, + clust_settings.AVAILABLE_FEATURES[clust_settings.DEFAULT_FEATURES]['DATASET_FILE'] + ), 'r' + ) + ) self.redis.set_features(self.AS_features) def return_features(self, sound_ids): @@ -77,5 +84,5 @@ def return_features(self, sound_ids): if feature: features.append(json.loads(feature)) sound_ids_out.append(sound_id) - + return np.array(features).astype('float32'), sound_ids_out diff --git a/clustering/interface.py b/clustering/interface.py index 064b8d905..08b0f16df 100644 --- a/clustering/interface.py +++ b/clustering/interface.py @@ -100,11 +100,15 @@ def cluster_sound_results(request, features=DEFAULT_FEATURES): sound_ids = get_sound_ids_from_search_engine_query(query_params) # launch clustering with celery async task - celery_app.send_task('cluster_sounds', kwargs={ - 'cache_key_hashed': cache_key_hashed, - 'sound_ids': sound_ids, - 'features': features - }, queue='clustering') + celery_app.send_task( + 'cluster_sounds', + kwargs={ + 'cache_key_hashed': cache_key_hashed, + 'sound_ids': sound_ids, + 'features': features + }, + queue='clustering' + ) return {'finished': False, 'error': False} diff --git a/clustering/tasks.py b/clustering/tasks.py index d9d79dc23..ff6f9d852 100644 --- a/clustering/tasks.py +++ b/clustering/tasks.py @@ -38,10 +38,11 @@ class ClusteringTask(Task): """ Task Class used for defining the clustering engine only required in celery workers """ + def __init__(self): if settings.IS_CELERY_WORKER: self.engine = ClusteringEngine() - + @shared_task(name="cluster_sounds", base=ClusteringTask) def cluster_sounds(cache_key_hashed, sound_ids, features): @@ -65,7 +66,7 @@ def cluster_sounds(cache_key_hashed, sound_ids, features): # store result in cache cache_clustering.set(cache_key_hashed, result, CLUSTERING_CACHE_TIME) - except Exception as e: + except Exception as e: # delete pending state if exception raised during clustering cache_clustering.set(cache_key_hashed, CLUSTERING_RESULT_STATUS_FAILED, CLUSTERING_PENDING_CACHE_TIME) logger.info("Exception raised while clustering sounds", exc_info=True) diff --git a/comments/admin.py b/comments/admin.py index 6408f5023..1a35da8ce 100644 --- a/comments/admin.py +++ b/comments/admin.py @@ -41,10 +41,10 @@ def get_comment_summary(self, obj): def has_add_permission(self, request): return False - + def has_change_permission(self, request, obj=None): return False - + def get_changelist(self, request): # Use custom change list class to avoid ordering by '-pk' in addition to '-created' # That would cause a slow query as we don't have a combined db index on both fields diff --git a/comments/forms.py b/comments/forms.py index 60bbb7503..b18ea8218 100644 --- a/comments/forms.py +++ b/comments/forms.py @@ -22,11 +22,16 @@ from utils.forms import HtmlCleaningCharField from utils.spam import is_spam + class CommentForm(forms.Form): comment = HtmlCleaningCharField( - widget=forms.Textarea, max_length=4000, label='', - help_text="You can add comments with a timestamp using the syntax #minute:second (e.g., \"The sound in #1:34 is really neat\").") - + widget=forms.Textarea, + max_length=4000, + label='', + help_text= + "You can add comments with a timestamp using the syntax #minute:second (e.g., \"The sound in #1:34 is really neat\")." + ) + def __init__(self, request, *args, **kwargs): self.request = request kwargs.update(dict(label_suffix='')) @@ -35,11 +40,13 @@ def __init__(self, request, *args, **kwargs): self.fields['comment'].widget.attrs['placeholder'] = 'Write your comment here...' self.fields['comment'].widget.attrs['rows'] = False self.fields['comment'].widget.attrs['cols'] = False - + def clean_comment(self): comment = self.cleaned_data['comment'] if is_spam(self.request, comment): - raise forms.ValidationError("Your comment was considered spam, please edit and repost. If it keeps failing please contact the admins.") - + raise forms.ValidationError( + "Your comment was considered spam, please edit and repost. If it keeps failing please contact the admins." + ) + return comment diff --git a/comments/models.py b/comments/models.py index 2a6615543..3caed69e7 100644 --- a/comments/models.py +++ b/comments/models.py @@ -32,7 +32,8 @@ class Comment(models.Model): sound = models.ForeignKey('sounds.Sound', null=True, related_name='comments', on_delete=models.CASCADE) comment = models.TextField() parent = models.ForeignKey( - 'self', null=True, blank=True, related_name='replies', default=None, on_delete=models.SET_NULL) + 'self', null=True, blank=True, related_name='replies', default=None, on_delete=models.SET_NULL + ) created = models.DateTimeField(db_index=True, auto_now_add=True) contains_hyperlink = models.BooleanField(db_index=True, default=False) @@ -40,7 +41,7 @@ def __str__(self): return f"{self.user} comment on {self.sound}" class Meta: - ordering = ('-created', ) + ordering = ('-created',) def set_has_hyperlink(self, commit=False): if text_has_hyperlink(self.comment): @@ -63,4 +64,4 @@ def on_delete_comment(sender, instance, **kwargs): @receiver(pre_save, sender=Comment) def update_hyperlink_field(sender, instance, **kwargs): - instance.set_has_hyperlink() \ No newline at end of file + instance.set_has_hyperlink() diff --git a/comments/tests.py b/comments/tests.py index e22219533..4fc31f16b 100644 --- a/comments/tests.py +++ b/comments/tests.py @@ -34,23 +34,33 @@ def setUp(self): def test_save_comment_with_hyperlinks(self): """Test that 'contains_hyperlink' boolean field is properly set when saving comments""" - - comment = Comment.objects.create(user=self.user, sound=self.sound, comment="This is a comment with no hyperlinks") + + comment = Comment.objects.create( + user=self.user, sound=self.sound, comment="This is a comment with no hyperlinks" + ) comment.refresh_from_db() self.assertFalse(comment.contains_hyperlink) - - comment = Comment.objects.create(user=self.user, sound=self.sound, comment="This is a comment with a link to http://www.freesound.org") + + comment = Comment.objects.create( + user=self.user, sound=self.sound, comment="This is a comment with a link to http://www.freesound.org" + ) comment.refresh_from_db() self.assertTrue(comment.contains_hyperlink) - comment = Comment.objects.create(user=self.user, sound=self.sound, comment="This is a comment with a https link to https://www.freesound.org") + comment = Comment.objects.create( + user=self.user, + sound=self.sound, + comment="This is a comment with a https link to https://www.freesound.org" + ) comment.refresh_from_db() self.assertTrue(comment.contains_hyperlink) def test_update_comment_with_hyperlinks(self): """Test that 'contains_hyperlink' boolean field is properly set when updating comments""" - comment = Comment.objects.create(user=self.user, sound=self.sound, comment="This is a comment with no hyperlinks") + comment = Comment.objects.create( + user=self.user, sound=self.sound, comment="This is a comment with no hyperlinks" + ) comment.refresh_from_db() self.assertFalse(comment.contains_hyperlink) diff --git a/comments/views.py b/comments/views.py index 10bebee19..81b93cfe7 100644 --- a/comments/views.py +++ b/comments/views.py @@ -49,7 +49,7 @@ def delete(request, comment_id): next = request.GET.get("next") page = request.GET.get("page", None) if page is not None: - next = next+"?page="+page + next = next + "?page=" + page return HttpResponseRedirect(next + "#comments") @@ -60,11 +60,11 @@ def for_user(request, username): if not request.GET.get('ajax'): # If not loading as a modal, redirect to account page with parameter to open modal return HttpResponseRedirect(reverse('account', args=[username]) + '?comments=1') - + user = request.parameter_user sounds = Sound.objects.filter(user=user) - qs = Comment.objects.filter(sound__in=sounds).select_related("user", "user__profile", - "sound__user", "sound__user__profile") + qs = Comment.objects.filter(sound__in=sounds + ).select_related("user", "user__profile", "sound__user", "sound__user__profile") num_items_per_page = settings.COMMENTS_IN_MODAL_PER_PAGE paginator = paginate(request, qs, num_items_per_page) page = paginator["page"] @@ -87,10 +87,10 @@ def by_user(request, username): if not request.GET.get('ajax'): # If not loaded as a modal, redirect to account page with parameter to open modal return HttpResponseRedirect(reverse('account', args=[username]) + '?comments_by=1') - + user = request.parameter_user - qs = Comment.objects.filter(user=user).select_related("user", "user__profile", - "sound__user", "sound__user__profile") + qs = Comment.objects.filter(user=user + ).select_related("user", "user__profile", "sound__user", "sound__user__profile") num_items_per_page = settings.COMMENTS_IN_MODAL_PER_PAGE paginator = paginate(request, qs, num_items_per_page) page = paginator["page"] @@ -113,13 +113,13 @@ def for_sound(request, username, sound_id): if not request.GET.get('ajax'): # If not loaded as a modal, redirect to account page with parameter to open modal return HttpResponseRedirect(reverse('sound', args=[username, sound_id]) + '#comments') - + sound = get_object_or_404(Sound, id=sound_id) if sound.user.username.lower() != username.lower(): raise Http404 - - qs = Comment.objects.filter(sound=sound).select_related("user", "user__profile", - "sound__user", "sound__user__profile") + + qs = Comment.objects.filter(sound=sound + ).select_related("user", "user__profile", "sound__user", "sound__user__profile") num_items_per_page = settings.SOUND_COMMENTS_PER_PAGE paginator = paginate(request, qs, num_items_per_page) tvars = { @@ -130,4 +130,3 @@ def for_sound(request, username, sound_id): } tvars.update(paginator) return render(request, 'accounts/modal_comments.html', tvars) - diff --git a/donations/admin.py b/donations/admin.py index a53ee6873..767aa96dc 100644 --- a/donations/admin.py +++ b/donations/admin.py @@ -29,8 +29,18 @@ def has_add_permission(self, request): @admin.register(Donation) class DonationAdmin(admin.ModelAdmin): raw_id_fields = ("user",) - list_display = ('id', 'email', 'user', 'amount', 'currency', 'created', ) - search_fields = ('=user__username', '=email', ) + list_display = ( + 'id', + 'email', + 'user', + 'amount', + 'currency', + 'created', + ) + search_fields = ( + '=user__username', + '=email', + ) def has_change_permission(self, request, obj=None): return False diff --git a/donations/management/commands/check_missing_donations.py b/donations/management/commands/check_missing_donations.py index 351951eb3..5b80030f4 100644 --- a/donations/management/commands/check_missing_donations.py +++ b/donations/management/commands/check_missing_donations.py @@ -36,11 +36,13 @@ class Command(LoggingBaseCommand): def add_arguments(self, parser): parser.add_argument( - '-d', '--days', + '-d', + '--days', dest='days', default=1, type=int, - help='Use this option to get the donations older than 1 day.') + help='Use this option to get the donations older than 1 day.' + ) def handle(self, **options): self.log_start() @@ -76,7 +78,7 @@ def handle(self, **options): if raw_rsp['L_TYPE%d' % i][0] in ['Donation', 'Payment']: amount = raw_rsp['L_AMT%d' % i][0] if float(amount) < 0: - continue # Don't create objects for donations with negative amounts + continue # Don't create objects for donations with negative amounts created_dt = datetime.datetime.strptime(raw_rsp['L_TIMESTAMP%d' % i][0], '%Y-%m-%dT%H:%M:%SZ') donation_data = { 'email': raw_rsp['L_EMAIL%d' % i][0], @@ -93,16 +95,17 @@ def handle(self, **options): user = User.objects.get(email=raw_rsp['L_EMAIL%d' % i][0]) donation_data['user'] = user except User.DoesNotExist: - pass # Don't link donation object to user object + pass # Don't link donation object to user object obj, created = Donation.objects.get_or_create( - transaction_id=raw_rsp['L_TRANSACTIONID%d'%i][0], defaults=donation_data) + transaction_id=raw_rsp['L_TRANSACTIONID%d' % i][0], defaults=donation_data + ) if created: n_donations_created += 1 del donation_data['campaign'] donation_data['created'] = raw_rsp['L_TIMESTAMP%d' % i][0] if 'user' in donation_data: - donation_data['user'] = donation_data['user'].username # Only log username in graylog + donation_data['user'] = donation_data['user'].username # Only log username in graylog commands_logger.info(f'Created donation object ({json.dumps(donation_data)})') start = start + one_day diff --git a/donations/management/commands/send_donation_request_emails.py b/donations/management/commands/send_donation_request_emails.py index 078a1e5a7..be8d5d8a8 100644 --- a/donations/management/commands/send_donation_request_emails.py +++ b/donations/management/commands/send_donation_request_emails.py @@ -55,7 +55,8 @@ def handle(self, **options): days=donation_settings.minimum_days_since_last_donation_email) user_received_donation_email_within_email_timespan = User.objects.filter( - profile__last_donation_email_sent__gt=email_timespan).values_list('id') + profile__last_donation_email_sent__gt=email_timespan + ).values_list('id') if donation_settings.never_send_email_to_uploaders: uploaders = User.objects.filter(profile__num_sounds__gt=0).values_list('id') else: @@ -82,17 +83,27 @@ def handle(self, **options): email_sent_successfully = send_mail_template( settings.EMAIL_SUBJECT_DONATION_REMINDER, 'emails/email_donation_reminder.txt', {'user': user}, - user_to=user, email_type_preference_check='donation_request') + user_to=user, + email_type_preference_check='donation_request' + ) if email_sent_successfully: user.profile.last_donation_email_sent = datetime.datetime.now() user.profile.donations_reminder_email_sent = True user.profile.save() - commands_logger.info("Sent donation email (%s)" % - json.dumps({'user_id': user.id, 'donation_email_type': 'reminder'})) + commands_logger.info( + "Sent donation email (%s)" % json.dumps({ + 'user_id': user.id, + 'donation_email_type': 'reminder' + }) + ) else: - commands_logger.info("Didn't send donation email due to email address being invalid or donation" - "emails preference disabled (%s)" % - json.dumps({'user_id': user.id, 'donation_email_type': 'reminder'})) + commands_logger.info( + "Didn't send donation email due to email address being invalid or donation" + "emails preference disabled (%s)" % json.dumps({ + 'user_id': user.id, + 'donation_email_type': 'reminder' + }) + ) # 2) Send email to users that download a lot of sounds without donating # potential_users -> All users that: @@ -131,9 +142,8 @@ def handle(self, **options): send_email = True else: relevant_period = max( - last_donation.created + datetime.timedelta( - days=donation_settings.minimum_days_since_last_donation), - email_timespan + last_donation.created + + datetime.timedelta(days=donation_settings.minimum_days_since_last_donation), email_timespan ) user_sound_downloads = Download.objects.filter(created__gte=relevant_period, user=user).count() user_pack_downloads = PackDownload.objects.filter(created__gte=relevant_period, user=user).count() @@ -147,16 +157,27 @@ def handle(self, **options): settings.EMAIL_SUBJECT_DONATION_REQUEST, 'emails/email_donation_request.txt', { 'user': user, - }, user_to=user, email_type_preference_check='donation_request') + }, + user_to=user, + email_type_preference_check='donation_request' + ) if email_sent_successfully: user.profile.last_donation_email_sent = datetime.datetime.now() user.profile.save() - commands_logger.info("Sent donation email (%s)" % - json.dumps({'user_id': user.id, 'donation_email_type': 'request'})) + commands_logger.info( + "Sent donation email (%s)" % json.dumps({ + 'user_id': user.id, + 'donation_email_type': 'request' + }) + ) else: - commands_logger.info("Didn't send donation email due to email address being invalid or donation" - "emails preference disabled (%s)" % - json.dumps({'user_id': user.id, 'donation_email_type': 'request'})) + commands_logger.info( + "Didn't send donation email due to email address being invalid or donation" + "emails preference disabled (%s)" % json.dumps({ + 'user_id': user.id, + 'donation_email_type': 'request' + }) + ) self.log_end() diff --git a/donations/models.py b/donations/models.py index 5d94ffee0..b07792062 100644 --- a/donations/models.py +++ b/donations/models.py @@ -14,13 +14,13 @@ class Donation(models.Model): display_name = models.CharField(max_length=255, null=True) amount = models.DecimalField(max_digits=7, decimal_places=2) transaction_id = models.CharField(max_length=255, blank=True) - currency = models.CharField(max_length=100) # Should always be EUR + currency = models.CharField(max_length=100) # Should always be EUR created = models.DateTimeField(auto_now_add=True) campaign = models.ForeignKey(DonationCampaign, null=True, on_delete=models.SET_NULL) - is_anonymous= models.BooleanField(default=True) + is_anonymous = models.BooleanField(default=True) display_amount = models.BooleanField(default=True) - DONATION_CHOICES =( + DONATION_CHOICES = ( ('p', 'paypal'), ('s', 'stripe'), ('t', 'transfer'), @@ -32,13 +32,16 @@ class DonationsModalSettings(models.Model): enabled = models.BooleanField(default=False) never_show_modal_to_uploaders = models.BooleanField(default=True) days_after_donation = models.PositiveIntegerField( - default=365, help_text='If user made a donation in the last X days, no modal is shown') + default=365, help_text='If user made a donation in the last X days, no modal is shown' + ) downloads_in_period = models.PositiveIntegerField(default=5, help_text='After user has download Z sounds...') download_days = models.PositiveIntegerField(default=7, help_text='...in Y days, we display the modal') display_probability = models.FloatField( - default=0.25, help_text='probabily of the modal being shown once all previous requirements are met') + default=0.25, help_text='probabily of the modal being shown once all previous requirements are met' + ) max_times_display_a_day = models.PositiveIntegerField( - default=10, help_text='max number of times we display the modal during a single day') + default=10, help_text='max number of times we display the modal during a single day' + ) DONATION_MODAL_SETTINGS_CACHE_KEY = 'donation-modal-settings' @@ -51,7 +54,7 @@ def get_donation_modal_settings(cls): """ instance = cache.get(cls.DONATION_MODAL_SETTINGS_CACHE_KEY, None) if instance is None: - instance, _ = cls.objects.get_or_create() # Gets existing object or creates it + instance, _ = cls.objects.get_or_create() # Gets existing object or creates it cache.set(cls.DONATION_MODAL_SETTINGS_CACHE_KEY, instance, timeout=3600) return instance @@ -60,8 +63,9 @@ class DonationsEmailSettings(models.Model): enabled = models.BooleanField(default=False) never_send_email_to_uploaders = models.BooleanField(default=True) minimum_days_since_last_donation = models.PositiveIntegerField( - default=365, help_text="Send emails to user only if didn't made a donation in the last X days") + default=365, help_text="Send emails to user only if didn't made a donation in the last X days" + ) minimum_days_since_last_donation_email = models.PositiveIntegerField( - default=30*3, help_text="Don't send a donation email if the last one was sent in less than X days") + default=30 * 3, help_text="Don't send a donation email if the last one was sent in less than X days" + ) downloads_in_period = models.PositiveIntegerField(default=100, help_text='After user has download Z sounds...') - diff --git a/donations/tests.py b/donations/tests.py index 5033620fa..1589100d2 100644 --- a/donations/tests.py +++ b/donations/tests.py @@ -21,24 +21,30 @@ class DonationTest(TestCase): fixtures = ['licenses', 'email_preference_type'] def test_non_annon_donation_with_name_paypal(self): - donations.models.DonationCampaign.objects.create( - goal=200, date_start=datetime.datetime.now(), id=1) - self.user = User.objects.create_user( - username='jacob', email='j@test.com', password='top', id='46280') - custom = base64.b64encode(json.dumps({'display_amount': True, 'user_id': 46280, 'campaign_id': 1, 'name': 'test'}).encode()).decode() - params = {'txn_id': '8B703020T00352816', - 'payer_email': 'fs@freesound.org', - 'custom': custom, - 'mc_currency': 'EUR', - 'mc_gross': '1.00'} + donations.models.DonationCampaign.objects.create(goal=200, date_start=datetime.datetime.now(), id=1) + self.user = User.objects.create_user(username='jacob', email='j@test.com', password='top', id='46280') + custom = base64.b64encode( + json.dumps({ + 'display_amount': True, + 'user_id': 46280, + 'campaign_id': 1, + 'name': 'test' + }).encode() + ).decode() + params = { + 'txn_id': '8B703020T00352816', + 'payer_email': 'fs@freesound.org', + 'custom': custom, + 'mc_currency': 'EUR', + 'mc_gross': '1.00' + } with mock.patch('donations.views.requests') as mock_requests: mock_response = mock.Mock(text='VERIFIED') mock_requests.post.return_value = mock_response resp = self.client.post(reverse('donation-complete-paypal'), params) self.assertEqual(resp.status_code, 200) - donations_query = donations.models.Donation.objects.filter( - transaction_id='8B703020T00352816') + donations_query = donations.models.Donation.objects.filter(transaction_id='8B703020T00352816') self.assertEqual(donations_query.exists(), True) self.assertEqual(donations_query[0].campaign_id, 1) self.assertEqual(donations_query[0].display_name, 'test') @@ -47,24 +53,27 @@ def test_non_annon_donation_with_name_paypal(self): self.assertEqual(donations_query[0].source, 'p') def test_non_annon_donation_paypal(self): - donations.models.DonationCampaign.objects.create( - goal=200, date_start=datetime.datetime.now(), id=1) - self.user = User.objects.create_user( - username='jacob', email='j@test.com', password='top', id='46280') - custom = base64.b64encode(json.dumps({'campaign_id': 1, 'user_id': 46280, 'display_amount': True}).encode()).decode() - params = {'txn_id': '8B703020T00352816', - 'payer_email': 'fs@freesound.org', - 'custom': custom, - 'mc_currency': 'EUR', - 'mc_gross': '1.00'} + donations.models.DonationCampaign.objects.create(goal=200, date_start=datetime.datetime.now(), id=1) + self.user = User.objects.create_user(username='jacob', email='j@test.com', password='top', id='46280') + custom = base64.b64encode(json.dumps({ + 'campaign_id': 1, + 'user_id': 46280, + 'display_amount': True + }).encode()).decode() + params = { + 'txn_id': '8B703020T00352816', + 'payer_email': 'fs@freesound.org', + 'custom': custom, + 'mc_currency': 'EUR', + 'mc_gross': '1.00' + } with mock.patch('donations.views.requests') as mock_requests: mock_response = mock.Mock(text='VERIFIED') mock_requests.post.return_value = mock_response resp = self.client.post(reverse('donation-complete-paypal'), params) self.assertEqual(resp.status_code, 200) - donations_query = donations.models.Donation.objects.filter( - transaction_id='8B703020T00352816') + donations_query = donations.models.Donation.objects.filter(transaction_id='8B703020T00352816') self.assertEqual(donations_query.exists(), True) self.assertEqual(donations_query[0].campaign_id, 1) self.assertEqual(donations_query[0].display_name, None) @@ -73,42 +82,55 @@ def test_non_annon_donation_paypal(self): self.assertEqual(donations_query[0].source, 'p') def test_annon_donation_paypal(self): - donations.models.DonationCampaign.objects.create( - goal=200, date_start=datetime.datetime.now(), id=1) - - custom = base64.b64encode(json.dumps({'campaign_id': 1, 'name': 'Anonymous', 'display_amount': True}).encode()).decode() - params = {'txn_id': '8B703020T00352816', - 'payer_email': 'fs@freesound.org', - 'custom': custom, - 'mc_currency': 'EUR', - 'mc_gross': '1.00'} + donations.models.DonationCampaign.objects.create(goal=200, date_start=datetime.datetime.now(), id=1) + + custom = base64.b64encode(json.dumps({ + 'campaign_id': 1, + 'name': 'Anonymous', + 'display_amount': True + }).encode()).decode() + params = { + 'txn_id': '8B703020T00352816', + 'payer_email': 'fs@freesound.org', + 'custom': custom, + 'mc_currency': 'EUR', + 'mc_gross': '1.00' + } with mock.patch('donations.views.requests') as mock_requests: mock_response = mock.Mock(text='VERIFIED') mock_requests.post.return_value = mock_response resp = self.client.post(reverse('donation-complete-paypal'), params) self.assertEqual(resp.status_code, 200) - donations_query = donations.models.Donation.objects.filter( - transaction_id='8B703020T00352816') + donations_query = donations.models.Donation.objects.filter(transaction_id='8B703020T00352816') self.assertEqual(donations_query.exists(), True) self.assertEqual(donations_query[0].is_anonymous, True) self.assertEqual(donations_query[0].source, 'p') def test_non_annon_donation_with_name_stripe(self): - donations.models.DonationCampaign.objects.create( - goal=200, date_start=datetime.datetime.now(), id=1) - self.user = User.objects.create_user( - username='fsuser', email='j@test.com', password='top', id='46280') + donations.models.DonationCampaign.objects.create(goal=200, date_start=datetime.datetime.now(), id=1) + self.user = User.objects.create_user(username='fsuser', email='j@test.com', password='top', id='46280') self.client.force_login(self.user) - custom = base64.b64encode(json.dumps({'display_amount': True, 'user_id': 46280, 'campaign_id': 1, 'name': 'test'}).encode()).decode() - params = {"data": {"object" :{"id": "txn123", - "customer_email": "donor@freesound.org", - "display_items": [{ - "amount": 1510, - "currency": "eur", - }], - "success_url": "https://example.com/success?token="+custom - }}, + custom = base64.b64encode( + json.dumps({ + 'display_amount': True, + 'user_id': 46280, + 'campaign_id': 1, + 'name': 'test' + }).encode() + ).decode() + params = { + "data": { + "object": { + "id": "txn123", + "customer_email": "donor@freesound.org", + "display_items": [{ + "amount": 1510, + "currency": "eur", + }], + "success_url": "https://example.com/success?token=" + custom + } + }, "type": "checkout.session.completed" } with mock.patch('stripe.Webhook.construct_event') as mock_create: @@ -122,23 +144,29 @@ def test_non_annon_donation_with_name_stripe(self): self.assertEqual(donations_query[0].user_id, 46280) self.assertEqual(donations_query[0].is_anonymous, True) self.assertEqual(donations_query[0].source, 's') - self.assertEqual(donations_query[0].amount*100, 1510) + self.assertEqual(donations_query[0].amount * 100, 1510) def test_non_annon_donation_stripe(self): - donations.models.DonationCampaign.objects.create( - goal=200, date_start=datetime.datetime.now(), id=1) - self.user = User.objects.create_user( - username='fsuser', email='j@test.com', password='top', id='46280') + donations.models.DonationCampaign.objects.create(goal=200, date_start=datetime.datetime.now(), id=1) + self.user = User.objects.create_user(username='fsuser', email='j@test.com', password='top', id='46280') self.client.force_login(self.user) - custom = base64.b64encode(json.dumps({'campaign_id': 1, 'user_id': 46280, 'display_amount': True}).encode()).decode() - params = {"data": {"object" :{"id": "txn123", - "customer_email": "donor@freesound.org", - "display_items": [{ - "amount": 1500, - "currency": "eur", - }], - "success_url": "https://example.com/success?token="+custom - }}, + custom = base64.b64encode(json.dumps({ + 'campaign_id': 1, + 'user_id': 46280, + 'display_amount': True + }).encode()).decode() + params = { + "data": { + "object": { + "id": "txn123", + "customer_email": "donor@freesound.org", + "display_items": [{ + "amount": 1500, + "currency": "eur", + }], + "success_url": "https://example.com/success?token=" + custom + } + }, "type": "checkout.session.completed" } with mock.patch('stripe.Webhook.construct_event') as mock_create: @@ -155,17 +183,24 @@ def test_non_annon_donation_stripe(self): self.assertEqual(donations_query[0].amount, 15.0) def test_annon_donation_stripe(self): - donations.models.DonationCampaign.objects.create( - goal=200, date_start=datetime.datetime.now(), id=1) - custom = base64.b64encode(json.dumps({'campaign_id': 1, 'name': 'Anonymous', 'display_amount': True}).encode()).decode() - params = {"data": {"object" :{"id": "txn123", - "customer_email": "donor@freesound.org", - "display_items": [{ - "amount": 1500, - "currency": "eur", - }], - "success_url": "https://example.com/success?token="+custom - }}, + donations.models.DonationCampaign.objects.create(goal=200, date_start=datetime.datetime.now(), id=1) + custom = base64.b64encode(json.dumps({ + 'campaign_id': 1, + 'name': 'Anonymous', + 'display_amount': True + }).encode()).decode() + params = { + "data": { + "object": { + "id": "txn123", + "customer_email": "donor@freesound.org", + "display_items": [{ + "amount": 1500, + "currency": "eur", + }], + "success_url": "https://example.com/success?token=" + custom + } + }, "type": "checkout.session.completed" } with mock.patch('stripe.Webhook.construct_event') as mock_create: @@ -199,7 +234,7 @@ def test_donation_form_stripe(self): with mock.patch('stripe.checkout.Session.create') as mock_create: mock_create.return_value = session ret = self.client.post("/donations/donation-session-stripe/", data) - response = ret.json() + response = ret.json() # Decimals must have '.' and not ',' self.assertTrue('errors' in response) @@ -219,7 +254,7 @@ def test_donation_form_stripe(self): mock_create.return_value = session data['amount'] = '0.1' ret = self.client.post("/donations/donation-session-stripe/", data) - response = ret.json() + response = ret.json() # amount must be greater than 1 self.assertTrue('errors' in response) @@ -227,16 +262,16 @@ def test_donation_form_stripe(self): mock_create.return_value = session data['amount'] = '5.1' ret = self.client.post("/donations/donation-session-stripe/", data) - response = ret.json() + response = ret.json() self.assertFalse('errors' in response) with mock.patch('stripe.checkout.Session.create') as mock_create: mock_create.return_value = session - long_mail = ('1'*256) + '@freesound.org' + long_mail = ('1' * 256) + '@freesound.org' data['name_option'] = long_mail data['donation_type'] = '2' ret = self.client.post("/donations/donation-session-stripe/", data) - response = ret.json() + response = ret.json() self.assertTrue('errors' in response) def test_donation_form_paypal(self): @@ -248,26 +283,26 @@ def test_donation_form_paypal(self): 'donation_type': '1', } ret = self.client.post("/donations/donation-session-paypal/", data) - response = ret.json() + response = ret.json() # Decimals must have '.' and not ',' self.assertTrue('errors' in response) data['amount'] = '0.1' ret = self.client.post("/donations/donation-session-paypal/", data) - response = ret.json() + response = ret.json() # amount must be greater than 1 self.assertTrue('errors' in response) data['amount'] = '5.1' ret = self.client.post("/donations/donation-session-paypal/", data) - response = ret.json() + response = ret.json() self.assertFalse('errors' in response) - long_mail = ('1'*256) + '@freesound.org' + long_mail = ('1' * 256) + '@freesound.org' data['name_option'] = long_mail data['donation_type'] = '2' ret = self.client.post("/donations/donation-session-paypal/", data) - response = ret.json() + response = ret.json() self.assertTrue('errors' in response) def test_donation_response(self): @@ -300,9 +335,11 @@ def test_donation_emails(self): # Simulate a donation from the user (older than donation_settings.minimum_days_since_last_donation) old_donation_date = datetime.datetime.now() - datetime.timedelta( - days=donation_settings.minimum_days_since_last_donation + 100) + days=donation_settings.minimum_days_since_last_donation + 100 + ) donation = donations.models.Donation.objects.create( - user=self.user_a, amount=50.25, email=self.user_a.email, currency='EUR') + user=self.user_a, amount=50.25, email=self.user_a.email, currency='EUR' + ) # NOTE: use .update(created=...) to avoid field auto_now to take over donations.models.Donation.objects.filter(pk=donation.pk).update(created=old_donation_date) @@ -322,7 +359,8 @@ def test_donation_emails(self): original_filename="Test sound %i" % i, base_filename_slug="test_sound_%i" % i, license=sounds.models.License.objects.all()[0], - md5="fakemd5_%i" % i) + md5="fakemd5_%i" % i + ) self.user_c.profile.num_sounds = TEST_DOWNLOADS_IN_PERIOD + 1 self.user_c.profile.save() @@ -451,7 +489,8 @@ def test_donation_emails(self): # Simulate user_a makes a new donation and then downloads some sounds donations.models.Donation.objects.create( - user=self.user_a, amount=50.25, email=self.user_a.email, currency='EUR') + user=self.user_a, amount=50.25, email=self.user_a.email, currency='EUR' + ) # Reset the reminder flag to False so that in a year time user is reminded to donate self.user_a.profile.donations_reminder_email_sent = False self.user_a.profile.save() @@ -486,9 +525,11 @@ def test_donation_emails_not_sent_when_preference_disabled(self): # Simulate a donation from the user (older than donation_settings.minimum_days_since_last_donation) old_donation_date = datetime.datetime.now() - datetime.timedelta( - days=donation_settings.minimum_days_since_last_donation + 100) + days=donation_settings.minimum_days_since_last_donation + 100 + ) donation = donations.models.Donation.objects.create( - user=self.user_a, amount=50.25, email=self.user_a.email, currency='EUR') + user=self.user_a, amount=50.25, email=self.user_a.email, currency='EUR' + ) # NOTE: use .update(created=...) to avoid field auto_now to take over donations.models.Donation.objects.filter(pk=donation.pk).update(created=old_donation_date) @@ -499,7 +540,8 @@ def test_donation_emails_not_sent_when_preference_disabled(self): original_filename="Test sound %i" % i, base_filename_slug="test_sound_%i" % i, license=sounds.models.License.objects.all()[0], - md5="fakemd5_%i" % i) + md5="fakemd5_%i" % i + ) self.user_c.profile.num_sounds = TEST_DOWNLOADS_IN_PERIOD + 1 self.user_c.profile.save() diff --git a/donations/urls.py b/donations/urls.py index 568cdab98..95d024a7e 100644 --- a/donations/urls.py +++ b/donations/urls.py @@ -29,5 +29,4 @@ path('donation-success/', views.donation_success, name="donation-success"), path('donation-complete-stripe/', views.donation_complete_stripe, name="donation-complete-stripe"), path('donation-complete-paypal/', views.donation_complete_paypal, name="donation-complete-paypal"), - ] diff --git a/donations/views.py b/donations/views.py index 3dc5d4586..0fa3a5062 100644 --- a/donations/views.py +++ b/donations/views.py @@ -58,18 +58,21 @@ def _save_donation(encoded_data, email, amount, currency, transaction_id, source if created: email_to = None if user is not None else email send_mail_template( - settings.EMAIL_SUBJECT_DONATION_THANK_YOU, - 'emails/email_donation.txt', { - 'user': user, - 'amount': amount, - 'display_name': display_name - }, user_to=user, email_to=email_to) + settings.EMAIL_SUBJECT_DONATION_THANK_YOU, + 'emails/email_donation.txt', { + 'user': user, + 'amount': amount, + 'display_name': display_name + }, + user_to=user, + email_to=email_to + ) log_data = donation_data log_data.update({'user_id': user_id}) log_data.update({'created': str(donation.created)}) - del log_data['user'] # Don't want to serialize user - del log_data['campaign'] # Don't want to serialize campaign + del log_data['user'] # Don't want to serialize user + del log_data['campaign'] # Don't want to serialize campaign log_data['amount_float'] = float(log_data['amount']) web_logger.info(f'Recevied donation ({json.dumps(log_data)})') return True @@ -89,9 +92,7 @@ def donation_complete_stripe(request): event = None try: - event = stripe.Webhook.construct_event( - payload, sig_header, endpoint_secret - ) + event = stripe.Webhook.construct_event(payload, sig_header, endpoint_secret) except ValueError as e: # Invalid payload return HttpResponse(status=400) @@ -104,7 +105,7 @@ def donation_complete_stripe(request): session = event['data']['object'] # Fulfill the purchase... - amount = int(session['display_items'][0]['amount'])/100.0 + amount = int(session['display_items'][0]['amount']) / 100.0 encoded_data = session['success_url'].split('?')[1].replace("token=", "") if encoded_data.startswith("b'"): encoded_data = encoded_data[2:-1] @@ -150,12 +151,10 @@ def donation_complete_paypal(request): return HttpResponse("FAIL") if req.text == 'VERIFIED': - _save_donation(params['custom'], - params['payer_email'], - params['mc_gross'], - params['mc_currency'], - params['txn_id'], - 'p') + _save_donation( + params['custom'], params['payer_email'], params['mc_gross'], params['mc_currency'], params['txn_id'], + 'p' + ) return HttpResponse("OK") @@ -180,14 +179,14 @@ def donation_session_stripe(request): 'name': 'Freesound donation', 'description': 'Donation for freesound.org', 'images': ['https://freesound.org/media/images/logo.png'], - 'amount': int(amount*100), + 'amount': int(amount * 100), 'currency': 'eur', 'quantity': 1, }], - success_url=return_url_success, - cancel_url=return_url_cancel, + success_url=return_url_success, + cancel_url=return_url_cancel, ) - return JsonResponse({"session_id":session.id}) + return JsonResponse({"session_id": session.id}) else: return JsonResponse({'errors': form.errors}) # If request is GET return an error 400 @@ -206,18 +205,19 @@ def donation_session_paypal(request): amount = form.cleaned_data['amount'] domain = f"https://{Site.objects.get_current().domain}" return_url = urllib.parse.urljoin(domain, reverse('donation-complete-paypal')) - data = {"url": settings.PAYPAL_VALIDATION_URL, - "params": { - "cmd": "_donations", - "currency_code": "EUR", - "business": settings.PAYPAL_EMAIL, - "item_name": "Freesound donation", - "custom": form.encoded_data, - "notify_url": return_url, - "no_shipping": 1, - "lc": "en_US" - } - } + data = { + "url": settings.PAYPAL_VALIDATION_URL, + "params": { + "cmd": "_donations", + "currency_code": "EUR", + "business": settings.PAYPAL_EMAIL, + "item_name": "Freesound donation", + "custom": form.encoded_data, + "notify_url": return_url, + "no_shipping": 1, + "lc": "en_US" + } + } if form.cleaned_data['recurring']: data['params']['cmd'] = '_xclick-subscriptions' diff --git a/follow/follow_utils.py b/follow/follow_utils.py index 4db0e9c59..eba9d0ac5 100644 --- a/follow/follow_utils.py +++ b/follow/follow_utils.py @@ -87,7 +87,10 @@ def get_stream_sounds(user, time_lapse, num_results_per_grup=3): more_count = max(0, result.num_found - num_results_per_grup) # the sorting only works if done like this! - more_url_params = [urllib.parse.quote(filter_str), urllib.parse.quote(settings.SEARCH_SOUNDS_SORT_OPTION_DATE_NEW_FIRST)] + more_url_params = [ + urllib.parse.quote(filter_str), + urllib.parse.quote(settings.SEARCH_SOUNDS_SORT_OPTION_DATE_NEW_FIRST) + ] # this is the same link but for the email has to be "quoted" more_url = "?f=" + filter_str + "&s=" + settings.SEARCH_SOUNDS_SORT_OPTION_DATE_NEW_FIRST @@ -129,7 +132,10 @@ def get_stream_sounds(user, time_lapse, num_results_per_grup=3): more_count = max(0, result.num_found - num_results_per_grup) # the sorting only works if done like this! - more_url_params = [urllib.parse.quote(tag_filter_str), urllib.parse.quote(settings.SEARCH_SOUNDS_SORT_OPTION_DATE_NEW_FIRST)] + more_url_params = [ + urllib.parse.quote(tag_filter_str), + urllib.parse.quote(settings.SEARCH_SOUNDS_SORT_OPTION_DATE_NEW_FIRST) + ] # this is the same link but for the email has to be "quoted" more_url = "?f=" + tag_filter_str + "&s=" + settings.SEARCH_SOUNDS_SORT_OPTION_DATE_NEW_FIRST diff --git a/follow/management/commands/send_stream_emails.py b/follow/management/commands/send_stream_emails.py index 491fd4603..54ab9cf86 100644 --- a/follow/management/commands/send_stream_emails.py +++ b/follow/management/commands/send_stream_emails.py @@ -40,7 +40,8 @@ class Command(LoggingBaseCommand): """ help = 'Send stream notifications to users who have not been notified for the last ' \ 'settings.NOTIFICATION_TIMEDELTA_PERIOD period and whose stream has new sounds for that period' - args = True # For backwards compatibility mode + args = True # For backwards compatibility mode + # See: http://stackoverflow.com/questions/30244288/django-management-command-cannot-see-arguments def handle(self, *args, **options): @@ -55,8 +56,8 @@ def handle(self, *args, **options): user_ids = email_type.useremailsetting_set.values_list('user_id') users_enabled_notifications = Profile.objects.filter(user_id__in=user_ids).exclude( - last_stream_email_sent__gt=date_today_minus_notification_timedelta).order_by( - "-last_attempt_of_sending_stream_email")[:settings.MAX_EMAILS_PER_COMMAND_RUN] + last_stream_email_sent__gt=date_today_minus_notification_timedelta + ).order_by("-last_attempt_of_sending_stream_email")[:settings.MAX_EMAILS_PER_COMMAND_RUN] n_emails_sent = 0 for profile in users_enabled_notifications: @@ -84,30 +85,32 @@ def handle(self, *args, **options): except Exception as e: # If error occur do not send the email console_logger.info(f"could not get new sounds data for {username.encode('utf-8')}") - profile.save() # Save last_attempt_of_sending_stream_email + profile.save() # Save last_attempt_of_sending_stream_email continue if not users_sounds and not tags_sounds: console_logger.info(f"no news sounds for {username.encode('utf-8')}") - profile.save() # Save last_attempt_of_sending_stream_email + profile.save() # Save last_attempt_of_sending_stream_email continue - tvars = {'username': username, - 'users_sounds': users_sounds, - 'tags_sounds': tags_sounds} + tvars = {'username': username, 'users_sounds': users_sounds, 'tags_sounds': tags_sounds} text_content = render_mail_template('emails/email_stream.txt', tvars) # Send email try: - send_mail(settings.EMAIL_SUBJECT_STREAM_EMAILS, text_content, - extra_subject=extra_email_subject, user_to=user) + send_mail( + settings.EMAIL_SUBJECT_STREAM_EMAILS, text_content, extra_subject=extra_email_subject, user_to=user + ) except Exception as e: # Do not send the email and do not update the last email sent field in the profile - profile.save() # Save last_attempt_of_sending_stream_email - commands_logger.info("Unexpected error while sending stream notification email (%s)" % json.dumps( - {'email_to': profile.get_email_for_delivery(), - 'username': profile.user.username, - 'error': str(e)})) + profile.save() # Save last_attempt_of_sending_stream_email + commands_logger.info( + "Unexpected error while sending stream notification email (%s)" % json.dumps({ + 'email_to': profile.get_email_for_delivery(), + 'username': profile.user.username, + 'error': str(e) + }) + ) continue n_emails_sent += 1 diff --git a/follow/tests.py b/follow/tests.py index dbd3870d0..1216a7ee1 100644 --- a/follow/tests.py +++ b/follow/tests.py @@ -48,7 +48,7 @@ def test_following_users_oldusername(self): # If we get following users for someone who exists by it's old username resp = self.client.get(reverse('user-following-users', args=['User2']) + '?ajax=1') self.assertEqual(resp.status_code, 301) - + def test_followers_modal(self): # If we get following users for someone who exists, OK resp = self.client.get(reverse('user-followers', args=['User2']) + '?ajax=1') @@ -99,7 +99,8 @@ def test_follow_user(self): # Check that user is actually following the other user self.assertEqual( - FollowingUserItem.objects.filter(user_from__username='testuser', user_to__username='User1').exists(), True) + FollowingUserItem.objects.filter(user_from__username='testuser', user_to__username='User1').exists(), True + ) # Stop following unexisting user resp = self.client.get(reverse('unfollow-user', args=['nouser'])) @@ -115,7 +116,8 @@ def test_follow_user(self): # Check that user is no longer following the other user self.assertEqual( - FollowingUserItem.objects.filter(user_from__username='testuser', user_to__username='User1').exists(), False) + FollowingUserItem.objects.filter(user_from__username='testuser', user_to__username='User1').exists(), False + ) def test_follow_tags(self): # Start following group of tags @@ -128,7 +130,9 @@ def test_follow_tags(self): # Check that user is actually following the tags self.assertEqual( - FollowingQueryItem.objects.filter(user__username='testuser', query='field-recording another_tag').exists(), True) + FollowingQueryItem.objects.filter(user__username='testuser', query='field-recording another_tag').exists(), + True + ) # Stop following group of tags you do not already follow resp = self.client.get(reverse('unfollow-tags', args=['a-tag/another_tag'])) @@ -140,7 +144,9 @@ def test_follow_tags(self): # Check that user is no longer following the tags self.assertEqual( - FollowingQueryItem.objects.filter(user__username='testuser', query='field-recording another_tag').exists(), False) + FollowingQueryItem.objects.filter(user__username='testuser', query='field-recording another_tag').exists(), + False + ) def test_stream(self): # Stream should return OK diff --git a/follow/views.py b/follow/views.py index ae40a0d0a..04b0a413c 100644 --- a/follow/views.py +++ b/follow/views.py @@ -48,9 +48,7 @@ def following_users(request, username): user = request.parameter_user following = follow_utils.get_users_following_qs(user) - tvars = { - 'user': user - } + tvars = {'user': user} # NOTE: 'next_path' tvar below is used for follow/unfollow buttons. We overwrite default value of next_path # given by the context processor so the redirects go to the user profile page URL instead of the follow modal @@ -75,9 +73,7 @@ def followers(request, username): user = request.parameter_user followers = follow_utils.get_users_followers_qs(user) - tvars = { - 'user': user - } + tvars = {'user': user} # NOTE: 'next_path' tvar below is used for follow/unfollow buttons. We overwrite default value of next_path # given by the context processor so the redirects go to the user profile page URL instead of the follow modal @@ -112,7 +108,7 @@ def following_tags(request, username): tvars.update(paginator) tvars.update({ 'next_path': reverse('account', args=[username]) + f"?followingTags={paginator['current_page']}", - 'follow_page': 'tags' # Used in BW + 'follow_page': 'tags' # Used in BW }) return render(request, 'accounts/modal_follow.html', tvars) @@ -196,17 +192,10 @@ def unfollow_tags(request, slash_tags): @transaction.atomic() def stream(request): - SELECT_OPTIONS = OrderedDict([ - ("last_week", "Last week"), - ("last_month", "Last month"), - ("specific_dates", "Specific dates...") - ]) + SELECT_OPTIONS = OrderedDict([("last_week", "Last week"), ("last_month", "Last month"), + ("specific_dates", "Specific dates...")]) - SELECT_OPTIONS_DAYS = { - "last_week": 7, - "last_month": 30, - "specific_dates": 0 - } + SELECT_OPTIONS_DAYS = {"last_week": 7, "last_month": 30, "specific_dates": 0} user = request.user @@ -222,14 +211,16 @@ def stream(request): date_from = request.POST.get("date_from") date_to = request.POST.get("date_to") if not date_from or not date_to: - if not date_from and not date_to: # Set it to last week (default) + if not date_from and not date_to: # Set it to last week (default) date_to = datetime.now().strftime("%Y-%m-%d") date_from = (datetime.now() - timedelta(days=7)).strftime("%Y-%m-%d") else: if not date_from: - date_from = (datetime.strptime(date_to,"%Y-%m-%d") - timedelta(days=7)).strftime("%Y-%m-%d") # A week before date to + date_from = (datetime.strptime(date_to, "%Y-%m-%d") - + timedelta(days=7)).strftime("%Y-%m-%d") # A week before date to if not date_to: - date_to = (datetime.strptime(date_from,"%Y-%m-%d") + timedelta(days=7)).strftime("%Y-%m-%d") # A week after date from + date_to = (datetime.strptime(date_from, "%Y-%m-%d") + + timedelta(days=7)).strftime("%Y-%m-%d") # A week after date from time_lapse = f'["{date_from}T00:00:00Z" TO "{date_to}T23:59:59.999Z"]' # if first time going into the page, the default is last week diff --git a/forum/admin.py b/forum/admin.py index 7dab61578..ebd6ac4c7 100644 --- a/forum/admin.py +++ b/forum/admin.py @@ -26,20 +26,18 @@ @admin.register(Forum) class ForumAdmin(SortableAdmin): - raw_id_fields = ('last_post', ) + raw_id_fields = ('last_post',) list_display = ('name', 'num_threads') - @admin.register(Thread) class ThreadAdmin(admin.ModelAdmin): - raw_id_fields = ('author', 'last_post','first_post' ) + raw_id_fields = ('author', 'last_post', 'first_post') list_display = ('forum', 'author', 'title', 'status', 'num_posts', 'created') list_filters = ('status',) search_fields = ('=author__username', "title") - @admin.register(Post) class PostAdmin(admin.ModelAdmin): raw_id_fields = ('author', 'thread') diff --git a/forum/forms.py b/forum/forms.py index efdbe1998..a2ce8f56c 100644 --- a/forum/forms.py +++ b/forum/forms.py @@ -25,10 +25,17 @@ class PostReplyForm(forms.Form): - body = HtmlCleaningCharField(widget=forms.Textarea(attrs={'cols': 100, 'rows': 30}), label="Message", - help_text=HtmlCleaningCharField.make_help_text()) - subscribe = forms.BooleanField(label="Send me an email notification when new posts are added in this thread.", - required=False, initial=True) + body = HtmlCleaningCharField( + widget=forms.Textarea(attrs={ + 'cols': 100, + 'rows': 30 + }), + label="Message", + help_text=HtmlCleaningCharField.make_help_text() + ) + subscribe = forms.BooleanField( + label="Send me an email notification when new posts are added in this thread.", required=False, initial=True + ) def __init__(self, request, quote, *args, **kwargs): self.request = request @@ -44,7 +51,6 @@ def __init__(self, request, quote, *args, **kwargs): self.fields['body'].widget.attrs['class'] = 'unsecure-image-check' self.fields['subscribe'].widget.attrs['class'] = 'bw-checkbox' - def clean_body(self): body = self.cleaned_data['body'] @@ -52,18 +58,27 @@ def clean_body(self): raise forms.ValidationError("You should type something...") if is_spam(self.request, body): - raise forms.ValidationError("Your post was considered spam, please edit and repost. " - "If it keeps failing please contact the admins.") + raise forms.ValidationError( + "Your post was considered spam, please edit and repost. " + "If it keeps failing please contact the admins." + ) - return body + return body class NewThreadForm(forms.Form): - title = forms.CharField(max_length=250, - widget=forms.TextInput(attrs={'size': 100})) - body = HtmlCleaningCharField(widget=forms.Textarea(attrs={'cols': 100, 'rows': 30}), label="Message", - help_text=HtmlCleaningCharField.make_help_text()) - subscribe = forms.BooleanField(label="Send me an email notification when new posts are added in this thread.", required=False, initial=True) + title = forms.CharField(max_length=250, widget=forms.TextInput(attrs={'size': 100})) + body = HtmlCleaningCharField( + widget=forms.Textarea(attrs={ + 'cols': 100, + 'rows': 30 + }), + label="Message", + help_text=HtmlCleaningCharField.make_help_text() + ) + subscribe = forms.BooleanField( + label="Send me an email notification when new posts are added in this thread.", required=False, initial=True + ) def __init__(self, *args, **kwargs): kwargs.update(dict(label_suffix='')) @@ -79,21 +94,13 @@ def __init__(self, *args, **kwargs): self.fields['subscribe'].widget.attrs['class'] = 'bw-checkbox' -MODERATION_CHOICES = [(x, x) for x in - ['Approve', - 'Delete User', - 'Delete Post']] +MODERATION_CHOICES = [(x, x) for x in ['Approve', 'Delete User', 'Delete Post']] class PostModerationForm(forms.Form): - action = forms.ChoiceField(choices=MODERATION_CHOICES, - required=True, - widget=forms.RadioSelect(), - label='') + action = forms.ChoiceField(choices=MODERATION_CHOICES, required=True, widget=forms.RadioSelect(), label='') post = forms.IntegerField(widget=forms.widgets.HiddenInput) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.fields['action'].widget.attrs['class'] = 'bw-radio' - - \ No newline at end of file diff --git a/forum/models.py b/forum/models.py index 0c106ce7f..cd746f251 100644 --- a/forum/models.py +++ b/forum/models.py @@ -50,9 +50,9 @@ class Forum(SortableMixin): num_threads = models.PositiveIntegerField(default=0) num_posts = models.PositiveIntegerField(default=0) - last_post = models.OneToOneField('Post', null=True, blank=True, default=None, - related_name="latest_in_forum", - on_delete=models.SET_NULL) + last_post = models.OneToOneField( + 'Post', null=True, blank=True, default=None, related_name="latest_in_forum", on_delete=models.SET_NULL + ) def set_last_post(self, commit=False): """ @@ -108,12 +108,12 @@ class Thread(models.Model): status = models.PositiveSmallIntegerField(choices=THREAD_STATUS_CHOICES, default=1, db_index=True) num_posts = models.PositiveIntegerField(default=0) - last_post = models.OneToOneField('Post', null=True, blank=True, default=None, - related_name="latest_in_thread", - on_delete=models.SET_NULL) - first_post = models.OneToOneField('Post', null=True, blank=True, default=None, - related_name="first_in_thread", - on_delete=models.SET_NULL) + last_post = models.OneToOneField( + 'Post', null=True, blank=True, default=None, related_name="latest_in_thread", on_delete=models.SET_NULL + ) + first_post = models.OneToOneField( + 'Post', null=True, blank=True, default=None, related_name="first_in_thread", on_delete=models.SET_NULL + ) created = models.DateTimeField(db_index=True, auto_now_add=True) @@ -231,9 +231,7 @@ class Post(models.Model): class Meta: ordering = ('created',) - permissions = ( - ("can_moderate_forum", "Can moderate posts."), - ) + permissions = (("can_moderate_forum", "Can moderate posts."),) def __str__(self): return f"Post by {self.author} in {self.thread}" diff --git a/forum/templatetags/display_forum_objects.py b/forum/templatetags/display_forum_objects.py index 14cb1435a..2abf3d3e0 100644 --- a/forum/templatetags/display_forum_objects.py +++ b/forum/templatetags/display_forum_objects.py @@ -18,7 +18,6 @@ # See AUTHORS file. # - from django import template register = template.Library() @@ -39,10 +38,7 @@ def display_forum(context, forum): dict: dictionary with the variables needed for rendering the forum with the display_forum.html template """ - return { - 'forum': forum, - 'request': context['request'] - } + return {'forum': forum, 'request': context['request']} @register.inclusion_tag('forum/display_thread.html', takes_context=True) @@ -61,15 +57,20 @@ def display_thread(context, thread): dict: dictionary with the variables needed for rendering the thread with the display_thread.html template """ - return { - 'thread': thread, - 'request': context['request'] - } + return {'thread': thread, 'request': context['request']} @register.inclusion_tag('forum/display_post.html', takes_context=True) -def display_post(context, post, forloop_counter=0, post_number_offset=0, show_post_location=False, - show_action_icons=True, show_report_actions=True, results_highlighted=None): +def display_post( + context, + post, + forloop_counter=0, + post_number_offset=0, + show_post_location=False, + show_action_icons=True, + show_report_actions=True, + results_highlighted=None +): """This templatetag is used to display a post in a list of posts. It prepares some variables that are then passed to the display_post.html template to show post information. @@ -94,7 +95,7 @@ def display_post(context, post, forloop_counter=0, post_number_offset=0, show_po try: highlighted_content = results_highlighted[str(post.id)]['post_body'][0] except KeyError: - highlighted_content = False + highlighted_content = False else: highlighted_content = False return { diff --git a/forum/templatetags/display_forum_search_results.py b/forum/templatetags/display_forum_search_results.py index eba7a1bfd..54e60897e 100644 --- a/forum/templatetags/display_forum_search_results.py +++ b/forum/templatetags/display_forum_search_results.py @@ -7,6 +7,7 @@ register = template.Library() + # TODO: do we need takes_context??? @register.inclusion_tag('forum/display_forum_result.html', takes_context=True) def display_forum_search_results(context, results_docs, highlight): @@ -19,27 +20,36 @@ def display_forum_search_results(context, results_docs, highlight): for p in first_docs: # highlighted result if str(p['id']) in highlight: - posts.append({'post_id': p['id'], - 'post_body': highlight[str(p['id'])]['post_body'][0], - 'post_info': ' - '.join(['Post by: ' + p['post_author'], - 'Date: ' + str(p['post_created'])])}) + posts.append({ + 'post_id': p['id'], + 'post_body': highlight[str(p['id'])]['post_body'][0], + 'post_info': ' - '.join(['Post by: ' + p['post_author'], 'Date: ' + str(p['post_created'])]) + }) else: - posts.append({'post_id': p['id'], - 'post_body': p['post_body'], - 'post_info': ' - '.join(['Post by: ' + p['post_author'], - 'Date: ' + str(p['post_created'])])}) + posts.append({ + 'post_id': p['id'], + 'post_body': p['post_body'], + 'post_info': ' - '.join(['Post by: ' + p['post_author'], 'Date: ' + str(p['post_created'])]) + }) results.append({ - 'thread_id': first_docs[0]['thread_id'], - 'thread_title': first_docs[0]['thread_title'], - 'forum_name': first_docs[0]['forum_name'], - 'forum_name_slug': first_docs[0]['forum_name_slug'], - 'post_id': first_docs[0]['id'], - 'posts': posts, - 'thread_info': ' - '.join(['Forum: ' + first_docs[0]['forum_name'], - 'Thread by: ' + first_docs[0]['thread_author'], - 'Posts: ' + str(first_docs[0]['num_posts']), - 'Date: ' + str(first_docs[0]['thread_created'])]), - }) + 'thread_id': + first_docs[0]['thread_id'], + 'thread_title': + first_docs[0]['thread_title'], + 'forum_name': + first_docs[0]['forum_name'], + 'forum_name_slug': + first_docs[0]['forum_name_slug'], + 'post_id': + first_docs[0]['id'], + 'posts': + posts, + 'thread_info': + ' - '.join([ + 'Forum: ' + first_docs[0]['forum_name'], 'Thread by: ' + first_docs[0]['thread_author'], + 'Posts: ' + str(first_docs[0]['num_posts']), 'Date: ' + str(first_docs[0]['thread_created']) + ]), + }) - return { 'results': results } + return {'results': results} diff --git a/forum/templatetags/smileys.py b/forum/templatetags/smileys.py index cd90bfdc9..c04f7ee63 100644 --- a/forum/templatetags/smileys.py +++ b/forum/templatetags/smileys.py @@ -20,10 +20,11 @@ d = [] for emoticons, name in [(x[:-1], x[-1]) for x in [x.split() for x in mapping.lower().split("\n")]]: for emoticon in emoticons: - d.append((emoticon,name)) + d.append((emoticon, name)) emoticons = dict(d) + def smiley_replace(matchobj): try: expression = emoticons[matchobj.group(0).lower()] @@ -32,9 +33,13 @@ def smiley_replace(matchobj): except KeyError: return matchobj.group(0) + smiley_replacer = re.compile(r"=\)|;\-?\)|8\-?\)|:'\(|:\-?[OoPpSsDd\)\(\|]") + @register.filter(is_safe=True) def smileys(string): return smiley_replacer.sub(smiley_replace, string) -#smileys.is_safe = True # Moved to filter definition (for Django 1.4 upgrade) \ No newline at end of file + + +#smileys.is_safe = True # Moved to filter definition (for Django 1.4 upgrade) diff --git a/forum/tests.py b/forum/tests.py index 478fdbe46..da85173b0 100644 --- a/forum/tests.py +++ b/forum/tests.py @@ -282,29 +282,25 @@ def test_cant_view_unmoderated_post(self): Post.objects.create(thread=thread, author=user, body="", moderation_state="NM") - res = self.client.get(reverse("forums-thread", - kwargs={"forum_name_slug": forum.name_slug, "thread_id": thread.id})) + res = self.client.get( + reverse("forums-thread", kwargs={ + "forum_name_slug": forum.name_slug, + "thread_id": thread.id + }) + ) self.assertEqual(res.status_code, 404) def _create_forums_threads_posts(author, n_forums=1, n_threads=1, n_posts=5): for i in range(0, n_forums): forum = Forum.objects.create( - name='Forum %i' % i, - name_slug='forum_%i' % i, - description="Description of forum %i" % i + name='Forum %i' % i, name_slug='forum_%i' % i, description="Description of forum %i" % i ) for j in range(0, n_threads): - thread = Thread.objects.create( - author=author, - title='Thread %i of forum %i' % (j, i), - forum=forum - ) + thread = Thread.objects.create(author=author, title='Thread %i of forum %i' % (j, i), forum=forum) for k in range(0, n_posts): post = Post.objects.create( - author=author, - thread=thread, - body='Text of the post %i for thread %i and forum %i' % (k, j, i) + author=author, thread=thread, body='Text of the post %i for thread %i and forum %i' % (k, j, i) ) if k == 0: thread.first_post = post @@ -325,30 +321,41 @@ def setUp(self): def test_forums_response_ok(self): resp = self.client.get(reverse('forums-forums')) self.assertEqual(resp.status_code, 200) - self.assertEqual(len(resp.context['forums']), self.N_FORUMS) # Check that N_FORUMS are passed to context + self.assertEqual(len(resp.context['forums']), self.N_FORUMS) # Check that N_FORUMS are passed to context def test_forum_response_ok(self): forum = Forum.objects.first() resp = self.client.get(reverse('forums-forum', args=[forum.name_slug])) self.assertEqual(resp.status_code, 200) - self.assertEqual(len(resp.context['page'].object_list), self.N_THREADS) # Check N_THREADS in context + self.assertEqual(len(resp.context['page'].object_list), self.N_THREADS) # Check N_THREADS in context @override_settings(LAST_FORUM_POST_MINIMUM_TIME=0) def test_new_thread_response_ok(self): forum = Forum.objects.first() # Assert non-logged in user is redirected to login page - resp = self.client.post(reverse('forums-new-thread', args=[forum.name_slug]), data={ - 'body': ['New thread body (first post)'], 'subscribe': ['on'], 'title': ['New thread title'] - }) - self.assertRedirects(resp, '{}?next={}'.format( - reverse('login'), reverse('forums-new-thread', args=[forum.name_slug]))) + resp = self.client.post( + reverse('forums-new-thread', args=[forum.name_slug]), + data={ + 'body': ['New thread body (first post)'], + 'subscribe': ['on'], + 'title': ['New thread title'] + } + ) + self.assertRedirects( + resp, '{}?next={}'.format(reverse('login'), reverse('forums-new-thread', args=[forum.name_slug])) + ) # Assert logged in user can create new thread self.client.force_login(self.user) - resp = self.client.post(reverse('forums-new-thread', args=[forum.name_slug]), data={ - 'body': ['New thread body (first post)'], 'subscribe': ['on'], 'title': ['New thread title'] - }) + resp = self.client.post( + reverse('forums-new-thread', args=[forum.name_slug]), + data={ + 'body': ['New thread body (first post)'], + 'subscribe': ['on'], + 'title': ['New thread title'] + } + ) post = Post.objects.get(body='New thread body (first post)') self.assertRedirects(resp, post.get_absolute_url(), target_status_code=302) @@ -359,9 +366,14 @@ def test_new_thread_title_length(self): # Assert logged in user fails creating thread long_title = 255 * '1' self.client.force_login(self.user) - resp = self.client.post(reverse('forums-new-thread', args=[forum.name_slug]), data={ - 'body': ['New thread body (first post)'], 'subscribe': ['on'], 'title': [long_title] - }) + resp = self.client.post( + reverse('forums-new-thread', args=[forum.name_slug]), + data={ + 'body': ['New thread body (first post)'], + 'subscribe': ['on'], + 'title': [long_title] + } + ) self.assertNotEqual(resp.context['form'].errors, None) def test_thread_response_ok(self): @@ -369,7 +381,7 @@ def test_thread_response_ok(self): thread = forum.thread_set.first() resp = self.client.get(reverse('forums-thread', args=[forum.name_slug, thread.id])) self.assertEqual(resp.status_code, 200) - self.assertEqual(len(resp.context['page'].object_list), self.N_POSTS) # Check N_POSTS in context + self.assertEqual(len(resp.context['page'].object_list), self.N_POSTS) # Check N_POSTS in context def test_post_response_ok(self): forum = Forum.objects.first() @@ -385,17 +397,26 @@ def test_thread_reply_response_ok(self): thread = forum.thread_set.first() # Assert non-logged in user is redirected to login page - resp = self.client.post(reverse('forums-reply', args=[forum.name_slug, thread.id]), data={ - 'body': ['Reply post body'], 'subscribe': ['on'], - }) - self.assertRedirects(resp, '{}?next={}'.format( - reverse('login'), reverse('forums-reply', args=[forum.name_slug, thread.id]))) + resp = self.client.post( + reverse('forums-reply', args=[forum.name_slug, thread.id]), + data={ + 'body': ['Reply post body'], + 'subscribe': ['on'], + } + ) + self.assertRedirects( + resp, '{}?next={}'.format(reverse('login'), reverse('forums-reply', args=[forum.name_slug, thread.id])) + ) # Assert logged in user can reply self.client.force_login(self.user) - resp = self.client.post(reverse('forums-reply', args=[forum.name_slug, thread.id]), data={ - 'body': ['Reply post body'], 'subscribe': ['on'], - }) + resp = self.client.post( + reverse('forums-reply', args=[forum.name_slug, thread.id]), + data={ + 'body': ['Reply post body'], + 'subscribe': ['on'], + } + ) post = Post.objects.get(body='Reply post body') self.assertRedirects(resp, post.get_absolute_url(), target_status_code=302) @@ -406,17 +427,28 @@ def test_thread_reply_quote_post_response_ok(self): post = thread.post_set.first() # Assert non-logged in user is redirected to login page - resp = self.client.post(reverse('forums-reply-quote', args=[forum.name_slug, thread.id, post.id]), data={ - 'body': ['Reply post body'], 'subscribe': ['on'], - }) - self.assertRedirects(resp, '{}?next={}'.format( - reverse('login'), reverse('forums-reply-quote', args=[forum.name_slug, thread.id, post.id]))) + resp = self.client.post( + reverse('forums-reply-quote', args=[forum.name_slug, thread.id, post.id]), + data={ + 'body': ['Reply post body'], + 'subscribe': ['on'], + } + ) + self.assertRedirects( + resp, '{}?next={}'.format( + reverse('login'), reverse('forums-reply-quote', args=[forum.name_slug, thread.id, post.id]) + ) + ) # Assert logged in user can reply self.client.force_login(self.user) - resp = self.client.post(reverse('forums-reply-quote', args=[forum.name_slug, thread.id, post.id]), data={ - 'body': ['Reply post body'], 'subscribe': ['on'], - }) + resp = self.client.post( + reverse('forums-reply-quote', args=[forum.name_slug, thread.id, post.id]), + data={ + 'body': ['Reply post body'], + 'subscribe': ['on'], + } + ) post = Post.objects.get(body='Reply post body') self.assertRedirects(resp, post.get_absolute_url(), target_status_code=302) @@ -426,24 +458,18 @@ def test_edit_post_response_ok(self): post = thread.post_set.first() # Assert non-logged in user can't edit post - resp = self.client.post(reverse('forums-post-edit', args=[post.id]), data={ - 'body': ['Edited post body'] - }) + resp = self.client.post(reverse('forums-post-edit', args=[post.id]), data={'body': ['Edited post body']}) self.assertRedirects(resp, f"{reverse('login')}?next={reverse('forums-post-edit', args=[post.id])}") # Assert logged in user which is not author of post can't edit post user2 = User.objects.create_user(username='testuser2', email='email2@example.com', password='12345') self.client.force_login(user2) - resp = self.client.post(reverse('forums-post-edit', args=[post.id]), data={ - 'body': ['Edited post body'] - }) + resp = self.client.post(reverse('forums-post-edit', args=[post.id]), data={'body': ['Edited post body']}) self.assertEqual(resp.status_code, 404) # Assert logged in user can edit post self.client.force_login(self.user) - resp = self.client.post(reverse('forums-post-edit', args=[post.id]), data={ - 'body': ['Edited post body'] - }) + resp = self.client.post(reverse('forums-post-edit', args=[post.id]), data={'body': ['Edited post body']}) self.assertRedirects(resp, post.get_absolute_url(), target_status_code=302) edited_post = Post.objects.get(id=post.id) self.assertEqual(edited_post.body, 'Edited post body') @@ -509,7 +535,9 @@ def test_user_subscribe_to_thread(self): # Assert non-logged in user can't subscribe resp = self.client.get(reverse('forums-thread-subscribe', args=[forum.name_slug, thread.id])) - self.assertRedirects(resp, f"{reverse('login')}?next={reverse('forums-thread-subscribe', args=[forum.name_slug, thread.id])}") + self.assertRedirects( + resp, f"{reverse('login')}?next={reverse('forums-thread-subscribe', args=[forum.name_slug, thread.id])}" + ) # Assert logged in user can subscribe user2 = User.objects.create_user(username='testuser2', email='email2@example.com', password='12345') @@ -547,9 +575,13 @@ def test_emails_sent_for_subscription_to_thread(self): resp = self.client.get(reverse('forums-thread-subscribe', args=[forum.name_slug, thread.id])) self.assertEqual(Subscription.objects.filter(thread=thread, subscriber=user2).count(), 1) - resp = self.client.post(reverse('forums-reply-quote', args=[forum.name_slug, thread.id, post.id]), data={ - 'body': ['Reply post body'], 'subscribe': ['on'], - }) + resp = self.client.post( + reverse('forums-reply-quote', args=[forum.name_slug, thread.id, post.id]), + data={ + 'body': ['Reply post body'], + 'subscribe': ['on'], + } + ) post = Post.objects.get(body='Reply post body') self.assertRedirects(resp, post.get_absolute_url(), target_status_code=302) @@ -576,13 +608,18 @@ def test_emails_not_sent_for_subscription_to_thread_if_preference_disabled(self) # A second user replies to that thread user2 = User.objects.create_user(username='testuser2', email='email2@example.com', password='12345') self.client.force_login(user2) - self.client.post(reverse('forums-reply-quote', args=[forum.name_slug, thread.id, post.id]), data={ - 'body': ['Reply post body'], 'subscribe': ['on'], - }) + self.client.post( + reverse('forums-reply-quote', args=[forum.name_slug, thread.id, post.id]), + data={ + 'body': ['Reply post body'], + 'subscribe': ['on'], + } + ) # No emails sent self.assertEqual(len(mail.outbox), 0) + class ForumModerationTestCase(TestCase): fixtures = ["user_groups"] @@ -602,18 +639,25 @@ def setUp(self): def test_user_no_permissions(self): """If the user doesn't have forum.can_moderate_forum permission, they're redirected to login screen""" self.client.force_login(self.regular_user) - resp = self.client.post(reverse('forums-moderate'), data={ - 'action': ['Delete'], 'post': ['1'], - }) + resp = self.client.post( + reverse('forums-moderate'), data={ + 'action': ['Delete'], + 'post': ['1'], + } + ) self.assertEqual(resp.status_code, 302) def test_approve_post(self): """Approve a post""" self.client.force_login(self.admin_user) - resp = self.client.post(reverse('forums-moderate'), data={ - f'{self.post.id}-action': ['Approve'], f'{self.post.id}-post': [f'{self.post.id}'], - }) + resp = self.client.post( + reverse('forums-moderate'), + data={ + f'{self.post.id}-action': ['Approve'], + f'{self.post.id}-post': [f'{self.post.id}'], + } + ) self.assertEqual(resp.status_code, 200) self.post.refresh_from_db() self.assertEqual(self.post.moderation_state, "OK") @@ -622,9 +666,13 @@ def test_delete_user(self): """The user is spammy, delete it. The post will also be deleted""" self.client.force_login(self.admin_user) - resp = self.client.post(reverse('forums-moderate'), data={ - f'{self.post.id}-action': ['Delete User'], f'{self.post.id}-post': [f'{self.post.id}'], - }) + resp = self.client.post( + reverse('forums-moderate'), + data={ + f'{self.post.id}-action': ['Delete User'], + f'{self.post.id}-post': [f'{self.post.id}'], + } + ) self.assertEqual(resp.status_code, 200) with self.assertRaises(Post.DoesNotExist): self.post.refresh_from_db() @@ -637,9 +685,13 @@ def test_delete_post(self): """The post is spammy. Delete it, but keep the user""" self.client.force_login(self.admin_user) - resp = self.client.post(reverse('forums-moderate'), data={ - f'{self.post.id}-action': ['Delete Post'], f'{self.post.id}-post': [f'{self.post.id}'], - }) + resp = self.client.post( + reverse('forums-moderate'), + data={ + f'{self.post.id}-action': ['Delete Post'], + f'{self.post.id}-post': [f'{self.post.id}'], + } + ) self.assertEqual(resp.status_code, 200) with self.assertRaises(Post.DoesNotExist): self.post.refresh_from_db() @@ -652,9 +704,14 @@ def test_no_such_post(self): self.admin_user.groups.add(group) self.client.force_login(self.admin_user) - resp = self.client.post(reverse('forums-moderate'), data={ - f'1234-action': ['Delete Post'], f'1234-post': [f'1234'], - }) + resp = self.client.post( + reverse('forums-moderate'), data={ + f'1234-action': ['Delete Post'], + f'1234-post': [f'1234'], + } + ) self.assertEqual(resp.status_code, 200) - self.assertEqual(list(resp.context['messages'])[0].message, "This post no longer exists. It may have already been deleted.") + self.assertEqual( + list(resp.context['messages'])[0].message, "This post no longer exists. It may have already been deleted." + ) diff --git a/forum/urls.py b/forum/urls.py index f428e2a30..174b14e39 100644 --- a/forum/urls.py +++ b/forum/urls.py @@ -30,12 +30,25 @@ re_path(r'^(?P[\w\-]+)/$', forum_views.forum, name="forums-forum"), re_path(r'^(?P[\w\-]+)/new-thread/$', forum_views.new_thread, name="forums-new-thread"), re_path(r'^(?P[\w-]+)/(?P\d+)/$', forum_views.thread, name="forums-thread"), - re_path(r'^(?P[\w-]+)/(?P\d+)/unsubscribe/$', forum_views.unsubscribe_from_thread, name="forums-thread-unsubscribe"), - re_path(r'^(?P[\w-]+)/(?P\d+)/subscribe/$', forum_views.subscribe_to_thread, name="forums-thread-subscribe"), - re_path(r'^(?P[\w-]+)/(?P\d+)/(?P\d+)/$', forum_views.post, name="forums-post"), + re_path( + r'^(?P[\w-]+)/(?P\d+)/unsubscribe/$', + forum_views.unsubscribe_from_thread, + name="forums-thread-unsubscribe" + ), + re_path( + r'^(?P[\w-]+)/(?P\d+)/subscribe/$', + forum_views.subscribe_to_thread, + name="forums-thread-subscribe" + ), + re_path( + r'^(?P[\w-]+)/(?P\d+)/(?P\d+)/$', forum_views.post, name="forums-post" + ), re_path(r'^(?P[\w-]+)/(?P\d+)/reply/$', forum_views.reply, name="forums-reply"), - re_path(r'^(?P[\w-]+)/(?P\d+)/(?P\d+)/reply/$', forum_views.reply, name="forums-reply-quote"), - + re_path( + r'^(?P[\w-]+)/(?P\d+)/(?P\d+)/reply/$', + forum_views.reply, + name="forums-reply-quote" + ), path('post//edit/', forum_views.post_edit, name="forums-post-edit"), path('post//delete-confirm/', forum_views.post_delete_confirm, name="forums-post-delete-confirm"), ] diff --git a/forum/views.py b/forum/views.py index e3679e99a..de89fe5af 100644 --- a/forum/views.py +++ b/forum/views.py @@ -18,7 +18,6 @@ # See AUTHORS file. # - import datetime import re import functools @@ -81,7 +80,7 @@ def inner(request, *args, **kwargs): reply_object = view_func(request, *args, **kwargs) - reply_object.set_cookie(key, now_as_string, 60*60*24*30) # 30 days + reply_object.set_cookie(key, now_as_string, 60 * 60 * 24 * 30) # 30 days return reply_object @@ -91,7 +90,8 @@ def inner(request, *args, **kwargs): @last_action def forums(request): forums = Forum.objects.select_related( - 'last_post', 'last_post__author', 'last_post__author__profile', 'last_post__thread').all() + 'last_post', 'last_post__author', 'last_post__author__profile', 'last_post__thread' + ).all() tvars = {'forums': forums} return render(request, 'forum/index.html', tvars) @@ -104,10 +104,13 @@ def forum(request, forum_name_slug): raise Http404 tvars = {'forum': forum} - paginator = paginate(request, Thread.objects.filter(forum=forum, first_post__moderation_state="OK") - .select_related('last_post', 'last_post__author', 'last_post__author__profile', - 'author', 'author__profile', 'first_post', 'forum'), - settings.FORUM_THREADS_PER_PAGE) + paginator = paginate( + request, + Thread.objects.filter(forum=forum, first_post__moderation_state="OK").select_related( + 'last_post', 'last_post__author', 'last_post__author__profile', 'author', 'author__profile', 'first_post', + 'forum' + ), settings.FORUM_THREADS_PER_PAGE + ) tvars.update(paginator) return render(request, 'forum/threads.html', tvars) @@ -117,13 +120,22 @@ def forum(request, forum_name_slug): @transaction.atomic() def thread(request, forum_name_slug, thread_id): forum = get_object_or_404(Forum, name_slug=forum_name_slug) - thread = get_object_or_404(Thread.objects.select_related('last_post', 'last_post__author'), - forum=forum, id=thread_id, first_post__moderation_state="OK") - - paginator = paginate(request, Post.objects.select_related( - 'author', 'author__profile', 'thread', 'thread__forum', - ).filter( - thread=thread, moderation_state="OK"), settings.FORUM_POSTS_PER_PAGE) + thread = get_object_or_404( + Thread.objects.select_related('last_post', 'last_post__author'), + forum=forum, + id=thread_id, + first_post__moderation_state="OK" + ) + + paginator = paginate( + request, + Post.objects.select_related( + 'author', + 'author__profile', + 'thread', + 'thread__forum', + ).filter(thread=thread, moderation_state="OK"), settings.FORUM_POSTS_PER_PAGE + ) has_subscription = False # a logged in user watching a thread can activate his subscription to that thread! @@ -139,10 +151,12 @@ def thread(request, forum_name_slug, thread_id): except Subscription.DoesNotExist: pass - tvars = {'thread': thread, - 'forum': forum, - 'post_counter_offset': settings.FORUM_POSTS_PER_PAGE * (paginator['current_page'] - 1), # Only used in BW - 'has_subscription': has_subscription} + tvars = { + 'thread': thread, + 'forum': forum, + 'post_counter_offset': settings.FORUM_POSTS_PER_PAGE * (paginator['current_page'] - 1), # Only used in BW + 'has_subscription': has_subscription + } tvars.update(paginator) return render(request, 'forum/thread.html', tvars) @@ -174,8 +188,9 @@ def hot_threads(request): @last_action def post(request, forum_name_slug, thread_id, post_id): - post = get_object_or_404(Post, id=post_id, thread__id=thread_id, thread__forum__name_slug=forum_name_slug, - moderation_state="OK") + post = get_object_or_404( + Post, id=post_id, thread__id=thread_id, thread__forum__name_slug=forum_name_slug, moderation_state="OK" + ) posts_before = Post.objects.filter(thread=post.thread, moderation_state="OK", created__lt=post.created).count() page = 1 + (posts_before // settings.FORUM_POSTS_PER_PAGE) @@ -212,7 +227,8 @@ def reply(request, forum_name_slug, thread_id, post_id=None): text_may_be_spam(form.cleaned_data.get("title", '')) if not request.user.posts.filter(moderation_state="OK").count() and may_be_spam: post = Post.objects.create( - author=request.user, body=form.cleaned_data["body"], thread=thread, moderation_state="NM") + author=request.user, body=form.cleaned_data["body"], thread=thread, moderation_state="NM" + ) # DO NOT add the post to the search engine, only do it when it is moderated set_to_moderation = True else: @@ -244,17 +260,23 @@ def reply(request, forum_name_slug, thread_id, post_id=None): if users_to_notify and post.thread.get_status_display() != 'Sunk': send_mail_template( settings.EMAIL_SUBJECT_TOPIC_REPLY, - "emails/email_new_post_notification.txt", - {'post': post, 'thread': thread, 'forum': forum}, + "emails/email_new_post_notification.txt", { + 'post': post, + 'thread': thread, + 'forum': forum + }, extra_subject=thread.title, - user_to=users_to_notify, email_type_preference_check="new_post" + user_to=users_to_notify, + email_type_preference_check="new_post" ) if not set_to_moderation: return HttpResponseRedirect(post.get_absolute_url()) else: - messages.add_message(request, messages.INFO, "Your post won't be shown until it is manually " - "approved by moderators") + messages.add_message( + request, messages.INFO, "Your post won't be shown until it is manually " + "approved by moderators" + ) return HttpResponseRedirect(post.thread.get_absolute_url()) else: initial = {'subscribe': thread.is_user_subscribed(request.user)} @@ -266,13 +288,12 @@ def reply(request, forum_name_slug, thread_id, post_id=None): messages.add_message(request, messages.INFO, user_can_post_message) if user_is_blocked_for_spam_reports: - messages.add_message(request, messages.INFO, "You're not allowed to post in the forums because your account " - "has been temporaly blocked after multiple spam reports") + messages.add_message( + request, messages.INFO, "You're not allowed to post in the forums because your account " + "has been temporaly blocked after multiple spam reports" + ) - tvars = {'forum': forum, - 'thread': thread, - 'form': form, - 'latest_posts': latest_posts} + tvars = {'forum': forum, 'thread': thread, 'form': form, 'latest_posts': latest_posts} return render(request, 'forum/reply.html', tvars) @@ -295,8 +316,9 @@ def new_thread(request, forum_name_slug): post_body = remove_control_chars(post_body) if not request.user.posts.filter(moderation_state="OK").count() and may_be_spam: - post = Post.objects.create(author=request.user, body=post_body, thread=thread, - moderation_state="NM") + post = Post.objects.create( + author=request.user, body=post_body, thread=thread, moderation_state="NM" + ) # DO NOT add the post to the search engine, only do it when it is moderated set_to_moderation = True else: @@ -318,8 +340,10 @@ def new_thread(request, forum_name_slug): if not set_to_moderation: return HttpResponseRedirect(post.get_absolute_url()) else: - messages.add_message(request, messages.INFO, "Your post won't be shown until it is manually " - "approved by moderators") + messages.add_message( + request, messages.INFO, "Your post won't be shown until it is manually " + "approved by moderators" + ) return HttpResponseRedirect(post.thread.forum.get_absolute_url()) else: form = NewThreadForm() @@ -328,11 +352,12 @@ def new_thread(request, forum_name_slug): messages.add_message(request, messages.INFO, user_can_post_message) if user_is_blocked_for_spam_reports: - messages.add_message(request, messages.INFO, "You're not allowed to post in the forums because your account " - "has been temporarily blocked after multiple spam reports") + messages.add_message( + request, messages.INFO, "You're not allowed to post in the forums because your account " + "has been temporarily blocked after multiple spam reports" + ) - tvars = {'forum': forum, - 'form': form} + tvars = {'forum': forum, 'form': form} return render(request, 'forum/new_thread.html', tvars) @@ -350,8 +375,10 @@ def subscribe_to_thread(request, forum_name_slug, thread_id): forum = get_object_or_404(Forum, name_slug=forum_name_slug) thread = get_object_or_404(Thread, forum=forum, id=thread_id, first_post__moderation_state="OK") subscription, created = Subscription.objects.get_or_create(thread=thread, subscriber=request.user) - messages.add_message(request, messages.INFO, "You have been subscribed to this thread. You will receive an " - "email notification every time someone makes a reply to this thread.") + messages.add_message( + request, messages.INFO, "You have been subscribed to this thread. You will receive an " + "email notification every time someone makes a reply to this thread." + ) return HttpResponseRedirect(reverse('forums-thread', args=[forum.name_slug, thread.id])) @@ -364,7 +391,8 @@ def old_topic_link_redirect(request): except ValueError: raise Http404 return HttpResponsePermanentRedirect( - reverse('forums-post', args=[post.thread.forum.name_slug, post.thread.id, post.id])) + reverse('forums-post', args=[post.thread.forum.name_slug, post.thread.id, post.id]) + ) thread_id = request.GET.get("t", False) if thread_id: @@ -426,8 +454,9 @@ def post_edit(request, post_id): add_posts_to_search_engine([post]) if form.cleaned_data["subscribe"]: - subscription, created = Subscription.objects.get_or_create(thread=post.thread, - subscriber=request.user) + subscription, created = Subscription.objects.get_or_create( + thread=post.thread, subscriber=request.user + ) if not subscription.is_active: subscription.is_active = True subscription.save() @@ -436,22 +465,22 @@ def post_edit(request, post_id): Subscription.objects.filter(thread=post.thread, subscriber=request.user).delete() return HttpResponseRedirect( - reverse('forums-post', args=[post.thread.forum.name_slug, post.thread.id, post.id])) + reverse('forums-post', args=[post.thread.forum.name_slug, post.thread.id, post.id]) + ) else: - initial = { - 'subscribe': post.thread.is_user_subscribed(request.user), - 'body': post.body - } + initial = {'subscribe': post.thread.is_user_subscribed(request.user), 'body': post.body} form = FromToUse(request, '', initial=initial) latest_posts = Post.objects.select_related('author', 'author__profile', 'thread', 'thread__forum') \ .order_by('-created') \ .filter(thread=post.thread, moderation_state="OK", created__lt=post.created)[0:15] - tvars = {'forum': post.thread.forum, - 'thread': post.thread, - 'form': form, - 'latest_posts': latest_posts, - 'editing': True} + tvars = { + 'forum': post.thread.forum, + 'thread': post.thread, + 'form': form, + 'latest_posts': latest_posts, + 'editing': True + } return render(request, 'forum/reply.html', tvars) else: raise Http404 @@ -466,7 +495,7 @@ def moderate_posts(request): for key in request.POST.keys(): if key.endswith('-post'): post_id = int(key.split('-')[0]) - + mod_form = PostModerationForm(request.POST, prefix=f'{post_id}') if mod_form.is_valid(): action = mod_form.cleaned_data.get("action") @@ -483,8 +512,9 @@ def moderate_posts(request): elif action == "Delete User": try: deletion_reason = DeletedUser.DELETION_REASON_SPAMMER - post.author.profile.delete_user(delete_user_object_from_db=True, - deletion_reason=deletion_reason) + post.author.profile.delete_user( + delete_user_object_from_db=True, deletion_reason=deletion_reason + ) messages.add_message(request, messages.INFO, 'The user has been successfully deleted.') except User.DoesNotExist: messages.add_message(request, messages.INFO, 'The user has already been deleted.') @@ -492,7 +522,9 @@ def moderate_posts(request): post.delete() messages.add_message(request, messages.INFO, 'The post has been successfully deleted.') except Post.DoesNotExist: - messages.add_message(request, messages.INFO, 'This post no longer exists. It may have already been deleted.') + messages.add_message( + request, messages.INFO, 'This post no longer exists. It may have already been deleted.' + ) invalidate_all_moderators_header_cache() @@ -502,6 +534,5 @@ def moderate_posts(request): f = PostModerationForm(initial={'action': 'Approve', 'post': p.id}, prefix=f'{p.id}') post_list.append({'post': p, 'form': f}) - tvars = {'post_list': post_list, - 'hide_search': True} + tvars = {'post_list': post_list, 'hide_search': True} return render(request, 'forum/moderate.html', tvars) diff --git a/freesound/celery.py b/freesound/celery.py index 70741ed2b..308c340a3 100644 --- a/freesound/celery.py +++ b/freesound/celery.py @@ -22,7 +22,8 @@ def get_queues_task_counts(): raw_data = requests.get( f'http://{settings.RABBITMQ_HOST}:{settings.RABBITMQ_API_PORT}/rabbitmq-admin/api/queues', - auth=(settings.RABBITMQ_USER, settings.RABBITMQ_PASS)).json() + auth=(settings.RABBITMQ_USER, settings.RABBITMQ_PASS) + ).json() data = [] if 'error' not in raw_data: for queue_data in raw_data: @@ -33,12 +34,10 @@ def get_queues_task_counts(): message_rate = queue_data['message_stats']['ack_details']['rate'] except KeyError: message_rate = -1 - data.append((queue_name, - queue_data['messages_ready'], - queue_data['messages_unacknowledged'], - queue_data['consumers'], - message_rate - )) + data.append(( + queue_name, queue_data['messages_ready'], queue_data['messages_unacknowledged'], + queue_data['consumers'], message_rate + )) data = sorted(data, key=lambda x: x[0]) return data diff --git a/freesound/context_processor.py b/freesound/context_processor.py index c34988944..c7649ced2 100644 --- a/freesound/context_processor.py +++ b/freesound/context_processor.py @@ -33,10 +33,10 @@ def context_extra(request): tvars = { 'request': request, } - + # Determine if extra context needs to be computed (this will allways be true expect for most of api calls and embeds) # There'll be other places in which the extra context is not needed, but this will serve as an approximation - should_compute_extra_context = True + should_compute_extra_context = True if request.path.startswith('/apiv2/') and \ 'apply' not in request.path and \ 'login' not in request.path and \ @@ -46,7 +46,7 @@ def context_extra(request): should_compute_extra_context = False if should_compute_extra_context: - new_tickets_count = -1 # Initially set to -1 (to distinguish later users that can not moderate) + new_tickets_count = -1 # Initially set to -1 (to distinguish later users that can not moderate) num_pending_sounds = 0 num_messages = 0 new_posts_pending_moderation = 0 @@ -57,7 +57,9 @@ def context_extra(request): if request.user.has_perm('forum.can_moderate_forum'): new_posts_pending_moderation = Post.objects.filter(moderation_state='NM').count() num_pending_sounds = request.user.profile.num_sounds_pending_moderation() - num_messages = Message.objects.filter(user_to=request.user, is_archived=False, is_sent=False, is_read=False).count() + num_messages = Message.objects.filter( + user_to=request.user, is_archived=False, is_sent=False, is_read=False + ).count() # Determine if anniversary special css and js content should be loaded # Animations will only be shown during the day of the anniversary @@ -76,7 +78,9 @@ def context_extra(request): 'next_path': request.GET.get('next', request.get_full_path()), 'login_form': FsAuthenticationForm(), 'problems_logging_in_form': ProblemsLoggingInForm(), - 'system_prefers_dark_theme': request.COOKIES.get('systemPrefersDarkTheme', 'no') == 'yes' # Determine the user's system preference for dark/light theme (for non authenticated users, always use light theme) + 'system_prefers_dark_theme': + request.COOKIES.get('systemPrefersDarkTheme', 'no') == + 'yes' # Determine the user's system preference for dark/light theme (for non authenticated users, always use light theme) }) - + return tvars diff --git a/freesound/local_settings.example.py b/freesound/local_settings.example.py index 72f4e4168..b110a7a94 100644 --- a/freesound/local_settings.example.py +++ b/freesound/local_settings.example.py @@ -13,7 +13,6 @@ EMAIL_BACKEND = 'django.core.mail.backends.filebased.EmailBackend' EMAIL_FILE_PATH = os.path.join(os.path.abspath(os.path.join(os.path.dirname(__file__), '../freesound-data/')), "_mail/") - WORKER_MIN_FREE_DISK_SPACE_PERCENTAGE = 0.0 BULK_UPLOAD_MIN_SOUNDS = 0 diff --git a/freesound/logger.py b/freesound/logger.py index 83b343aa2..ad1626d8f 100755 --- a/freesound/logger.py +++ b/freesound/logger.py @@ -28,7 +28,7 @@ }, 'django.request': { 'handlers': ['stdout'], - 'level': 'ERROR', # only catches 5xx not 4xx messages + 'level': 'ERROR', # only catches 5xx not 4xx messages 'propagate': True, }, 'sounds': { diff --git a/freesound/middleware.py b/freesound/middleware.py index 3dda4db0e..da9391478 100644 --- a/freesound/middleware.py +++ b/freesound/middleware.py @@ -45,6 +45,7 @@ def dont_redirect(path): class OnlineUsersHandler: + def __init__(self, get_response): self.get_response = get_response @@ -55,6 +56,7 @@ def __call__(self, request): class BulkChangeLicenseHandler: + def __init__(self, get_response): self.get_response = get_response @@ -115,7 +117,7 @@ def __call__(self, request): and 'contact' not in request.get_full_path() \ and 'bulklicensechange' not in request.get_full_path() \ and 'resetemail' not in request.get_full_path(): - # replace with dont_redirect() and add resetemail to it after merge with gdpr_acceptance pr + # replace with dont_redirect() and add resetemail to it after merge with gdpr_acceptance pr user = request.user @@ -133,17 +135,12 @@ class ModelAdminReorderWithNav(ModelAdminReorder): def process_template_response(self, request, response): - if ( - getattr(response, 'context_data', None) - and not response.context_data.get('app_list') - and response.context_data.get('available_apps') - ): + if (getattr(response, 'context_data', None) and not response.context_data.get('app_list') + and response.context_data.get('available_apps')): available_apps = response.context_data.get('available_apps') response.context_data['app_list'] = available_apps response = super().process_template_response(request, response) - response.context_data['available_apps'] = response.context_data[ - 'app_list' - ] + response.context_data['available_apps'] = response.context_data['app_list'] return response - return super().process_template_response(request, response) \ No newline at end of file + return super().process_template_response(request, response) diff --git a/freesound/settings.py b/freesound/settings.py index 2223ddd38..7a731c3c2 100644 --- a/freesound/settings.py +++ b/freesound/settings.py @@ -13,14 +13,14 @@ DISPLAY_DEBUG_TOOLBAR = False DEBUGGER_HOST = "0.0.0.0" -DEBUGGER_PORT = 3000 # This port should match the one in docker compose +DEBUGGER_PORT = 3000 # This port should match the one in docker compose SECRET_KEY = os.getenv('DJANGO_SECRET_KEY', '___this_is_a_secret_key_that_should_not_be_used___') default_url = 'postgres://postgres@db/postgres' DATABASES = {'default': dj_database_url.config('DJANGO_DATABASE_URL', default=default_url)} -DEFAULT_AUTO_FIELD='django.db.models.AutoField' +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', @@ -82,48 +82,69 @@ # Specify custom ordering of models in Django Admin index ADMIN_REORDER = ( - - {'app': 'accounts', 'models': ( - 'auth.User', - 'accounts.Profile', - 'accounts.DeletedUser', - 'accounts.UserDeletionRequest', - 'accounts.UserFlag', - 'accounts.OldUsername', - 'accounts.EmailBounce', - 'auth.Groups', - 'fsmessages.Message', - 'accounts.GdprAcceptance', - )}, - {'app': 'sounds', 'models': ( - 'sounds.Sound', - {'model': 'sounds.SoundAnalysis', 'label': 'Sound analyses'}, - 'sounds.Pack', - 'sounds.DeletedSound', - 'sounds.License', - {'model': 'sounds.Flag', 'label': 'Sound flags'}, - 'sounds.BulkUploadProgress', - {'model': 'sounds.SoundOfTheDay', 'label': 'Sound of the day'} - )}, - {'app': 'apiv2', 'label': 'API', 'models': ( - {'model': 'apiv2.ApiV2Client', 'label': 'API V2 Application'}, - 'oauth2_provider.AccessToken', - 'oauth2_provider.RefreshToken', - 'oauth2_provider.Grant', - )}, + { + 'app': + 'accounts', + 'models': ( + 'auth.User', + 'accounts.Profile', + 'accounts.DeletedUser', + 'accounts.UserDeletionRequest', + 'accounts.UserFlag', + 'accounts.OldUsername', + 'accounts.EmailBounce', + 'auth.Groups', + 'fsmessages.Message', + 'accounts.GdprAcceptance', + ) + }, + { + 'app': + 'sounds', + 'models': ( + 'sounds.Sound', { + 'model': 'sounds.SoundAnalysis', + 'label': 'Sound analyses' + }, 'sounds.Pack', 'sounds.DeletedSound', 'sounds.License', { + 'model': 'sounds.Flag', + 'label': 'Sound flags' + }, 'sounds.BulkUploadProgress', { + 'model': 'sounds.SoundOfTheDay', + 'label': 'Sound of the day' + } + ) + }, + { + 'app': + 'apiv2', + 'label': + 'API', + 'models': ( + { + 'model': 'apiv2.ApiV2Client', + 'label': 'API V2 Application' + }, + 'oauth2_provider.AccessToken', + 'oauth2_provider.RefreshToken', + 'oauth2_provider.Grant', + ) + }, 'forum', - {'app': 'donations', 'models': ( - 'donations.Donation', - 'donations.DonationsEmailSettings', - 'donations.DonationsModalSettings', - )}, + { + 'app': 'donations', + 'models': ( + 'donations.Donation', + 'donations.DonationsEmailSettings', + 'donations.DonationsModalSettings', + ) + }, 'sites', ) # Silk is the Request/SQL logging platform. We install it but leave it disabled # It can be activated in local_settings by changing INTERCEPT_FUNC -SILKY_AUTHENTICATION = True # User must login -SILKY_AUTHORISATION = True # User must have permissions +SILKY_AUTHENTICATION = True # User must login +SILKY_AUTHORISATION = True # User must have permissions SILKY_PERMISSIONS = lambda user: user.is_superuser SILKY_INTERCEPT_FUNC = lambda request: False @@ -278,11 +299,11 @@ AWS_SQS_ACCESS_KEY_ID = '' AWS_SQS_SECRET_ACCESS_KEY = '' AWS_SQS_QUEUE_URL = '' -AWS_SQS_MESSAGES_PER_CALL = 1 # between 1 and 10, see accounts management command `process_email_bounces` for more +AWS_SQS_MESSAGES_PER_CALL = 1 # between 1 and 10, see accounts management command `process_email_bounces` for more # Email stats retrieval parameters (see utils.aws.report_ses_stats for more details) -AWS_SES_BOUNCE_RATE_SAMPLE_SIZE = 10500 # should be ~ 10000-11000 -AWS_SES_SHORT_BOUNCE_RATE_DATAPOINTS = 4 # cron period (1hr) / AWS stats period (15min) +AWS_SES_BOUNCE_RATE_SAMPLE_SIZE = 10500 # should be ~ 10000-11000 +AWS_SES_SHORT_BOUNCE_RATE_DATAPOINTS = 4 # cron period (1hr) / AWS stats period (15min) # If ALLOWED emails is not empty, only emails going to these destinations will be actually sent ALLOWED_EMAILS = [] @@ -310,23 +331,19 @@ # Static settings # Add freesound/static/ to STATICFILES_DIRS as it won't be added by default (freesound/ is not an installed Django app) -STATICFILES_DIRS = [os.path.join(os.path.dirname(__file__), 'static'), ] +STATICFILES_DIRS = [ + os.path.join(os.path.dirname(__file__), 'static'), +] STATIC_URL = '/static/' STATIC_ROOT = 'bw_static' STATICFILES_STORAGE = 'freesound.storage.NoStrictManifestStaticFilesStorage' - # ------------------------------------------------------------------------------- # Freesound miscellaneous settings SUPPORT = () -IFRAME_PLAYER_SIZE = { - 'large': [920, 245], - 'medium': [481, 86], - 'small': [375, 30], - 'twitter_card': [440, 132] -} +IFRAME_PLAYER_SIZE = {'large': [920, 245], 'medium': [481, 86], 'small': [375, 30], 'twitter_card': [440, 132]} FREESOUND_RSS = '' @@ -334,12 +351,12 @@ FORUM_POSTS_PER_PAGE = 20 FORUM_THREADS_PER_PAGE = 15 SOUND_COMMENTS_PER_PAGE = 5 -SOUNDS_PER_PAGE = 15 # In search page -SOUNDS_PER_PAGE_COMPACT_MODE = 30 # In search page -PACKS_PER_PAGE = 15 # In search page +SOUNDS_PER_PAGE = 15 # In search page +SOUNDS_PER_PAGE_COMPACT_MODE = 30 # In search page +PACKS_PER_PAGE = 15 # In search page DOWNLOADED_SOUNDS_PACKS_PER_PAGE = 12 USERS_PER_DOWNLOADS_MODAL_PAGE = 15 -COMMENTS_IN_MODAL_PER_PAGE = 15 +COMMENTS_IN_MODAL_PER_PAGE = 15 REMIXES_PER_PAGE = 10 MAX_TICKETS_IN_MODERATION_ASSIGNED_PAGE = 60 SOUNDS_PER_DESCRIBE_ROUND = 10 @@ -348,12 +365,11 @@ DONATIONS_PER_PAGE = 40 FOLLOW_ITEMS_PER_PAGE = 5 MESSAGES_PER_PAGE = 10 -BOOKMARKS_PER_PAGE = 12 +BOOKMARKS_PER_PAGE = 12 SOUNDS_PER_PAGE_PROFILE_PACK_PAGE = 12 NUM_SIMILAR_SOUNDS_PER_PAGE = 9 NUM_SIMILAR_SOUNDS_PAGES = 5 - # Weights using to compute charts BW_CHARTS_ACTIVE_USERS_WEIGHTS = {'upload': 1, 'post': 0.8, 'comment': 0.05} CHARTS_DATA_CACHE_KEY = 'bw-charts-data' @@ -416,6 +432,7 @@ # Avatar background colors (only BW) from utils.audioprocessing.processing import interpolate_colors from utils.audioprocessing.color_schemes import BEASTWHOOSH_COLOR_SCHEME, COLOR_SCHEMES + AVATAR_BG_COLORS = interpolate_colors(COLOR_SCHEMES[BEASTWHOOSH_COLOR_SCHEME]['wave_colors'][1:], num_colors=10) # Number of ratings of a sound to start showing average @@ -428,7 +445,7 @@ UPLOAD_AND_DESCRIPTION_ENABLED = True # Maximum combined file size for uploading files. This is set in nginx configuration -UPLOAD_MAX_FILE_SIZE_COMBINED = 1024 * 1024 * 1024 # 1 GB +UPLOAD_MAX_FILE_SIZE_COMBINED = 1024 * 1024 * 1024 # 1 GB MOVE_TMP_UPLOAD_FILES_INSTEAD_OF_COPYING = True # Minimum number of sounds that a user has to upload before enabling bulk upload feature for that user @@ -471,15 +488,14 @@ # Locations where sounds, previews and other "static" content will be mirrored (if specified) # If locations do not exist, they will be created -MIRROR_SOUNDS = None # list of locations to mirror contents of SOUNDS_PATH, set to None to turn off -MIRROR_PREVIEWS = None # list of locations to mirror contents of SOUNDS_PATH, set to None to turn off -MIRROR_DISPLAYS = None # list of locations to mirror contents of SOUNDS_PATH, set to None to turn off -MIRROR_ANALYSIS = None # list of locations to mirror contents of SOUNDS_PATH, set to None to turn off -MIRROR_AVATARS = None # list of locations to mirror contents of AVATARS_PATH, set to None to turn off -MIRROR_UPLOADS = None # list of locations to mirror contents of MIRROR_UPLOADS, set to None to turn off +MIRROR_SOUNDS = None # list of locations to mirror contents of SOUNDS_PATH, set to None to turn off +MIRROR_PREVIEWS = None # list of locations to mirror contents of SOUNDS_PATH, set to None to turn off +MIRROR_DISPLAYS = None # list of locations to mirror contents of SOUNDS_PATH, set to None to turn off +MIRROR_ANALYSIS = None # list of locations to mirror contents of SOUNDS_PATH, set to None to turn off +MIRROR_AVATARS = None # list of locations to mirror contents of AVATARS_PATH, set to None to turn off +MIRROR_UPLOADS = None # list of locations to mirror contents of MIRROR_UPLOADS, set to None to turn off LOG_START_AND_END_COPYING_FILES = True - # ------------------------------------------------------------------------------- # Donations @@ -498,63 +514,50 @@ PAYPAL_USERNAME = '' PAYPAL_SIGNATURE = '' - # ------------------------------------------------------------------------------- # New Analysis options ORCHESTRATE_ANALYSIS_MAX_JOBS_PER_QUEUE_DEFAULT = 500 ORCHESTRATE_ANALYSIS_MAX_NUM_ANALYSIS_ATTEMPTS = 3 -ORCHESTRATE_ANALYSIS_MAX_TIME_IN_QUEUED_STATUS = 24 * 2 # in hours -ORCHESTRATE_ANALYSIS_MAX_TIME_CONVERTED_FILES_IN_DISK = 24 * 7 # in hours +ORCHESTRATE_ANALYSIS_MAX_TIME_IN_QUEUED_STATUS = 24 * 2 # in hours +ORCHESTRATE_ANALYSIS_MAX_TIME_CONVERTED_FILES_IN_DISK = 24 * 7 # in hours AUDIOCOMMONS_ANALYZER_NAME = 'ac-extractor_v3' FREESOUND_ESSENTIA_EXTRACTOR_NAME = 'fs-essentia-extractor_legacy' AUDIOSET_YAMNET_ANALYZER_NAME = 'audioset-yamnet_v1' BIRDNET_ANALYZER_NAME = 'birdnet_v1' FSDSINET_ANALYZER_NAME = 'fsd-sinet_v1' - + ANALYZERS_CONFIGURATION = { AUDIOCOMMONS_ANALYZER_NAME: { - 'descriptors_map': [ - ('loudness', 'ac_loudness', float), - ('dynamic_range', 'ac_dynamic_range', float), - ('temporal_centroid', 'ac_temporal_centroid', float), - ('log_attack_time', 'ac_log_attack_time', float), - ('single_event', 'ac_single_event', bool), - ('tonality', 'ac_tonality', str), - ('tonality_confidence', 'ac_tonality_confidence', float), - ('loop', 'ac_loop', bool), - ('tempo', 'ac_tempo', int), - ('tempo_confidence', 'ac_tempo_confidence', float), - ('note_midi', 'ac_note_midi', int), - ('note_name', 'ac_note_name', str), - ('note_frequency', 'ac_note_frequency', float), - ('note_confidence', 'ac_note_confidence', float), - ('brightness', 'ac_brightness', float), - ('depth', 'ac_depth', float), - ('hardness', 'ac_hardness', float), - ('roughness', 'ac_roughness', float), - ('boominess', 'ac_boominess', float), - ('warmth', 'ac_warmth', float), - ('sharpness', 'ac_sharpness', float), - ('reverb', 'ac_reverb', bool) - ] + 'descriptors_map': [('loudness', 'ac_loudness', float), ('dynamic_range', 'ac_dynamic_range', float), + ('temporal_centroid', 'ac_temporal_centroid', float), + ('log_attack_time', 'ac_log_attack_time', float), ('single_event', 'ac_single_event', bool), + ('tonality', 'ac_tonality', str), ('tonality_confidence', 'ac_tonality_confidence', float), + ('loop', 'ac_loop', bool), ('tempo', 'ac_tempo', int), + ('tempo_confidence', 'ac_tempo_confidence', float), ('note_midi', 'ac_note_midi', int), + ('note_name', 'ac_note_name', str), ('note_frequency', 'ac_note_frequency', float), + ('note_confidence', 'ac_note_confidence', float), ('brightness', 'ac_brightness', float), + ('depth', 'ac_depth', float), ('hardness', 'ac_hardness', float), + ('roughness', 'ac_roughness', float), ('boominess', 'ac_boominess', float), + ('warmth', 'ac_warmth', float), ('sharpness', 'ac_sharpness', float), + ('reverb', 'ac_reverb', bool)] }, FREESOUND_ESSENTIA_EXTRACTOR_NAME: {}, AUDIOSET_YAMNET_ANALYZER_NAME: { - 'descriptors_map': [ - ('classes', 'yamnet_class', list) - ] + 'descriptors_map': [('classes', 'yamnet_class', list)] }, BIRDNET_ANALYZER_NAME: { 'descriptors_map': [ - ('detections', 'birdnet_detections', None), # Use None so detections are not indexed in solr but stored in database + ('detections', 'birdnet_detections', + None), # Use None so detections are not indexed in solr but stored in database ('detected_classes', 'birdnet_detected_class', list), ('num_detections', 'birdnet_detections_count', int), ] }, FSDSINET_ANALYZER_NAME: { 'descriptors_map': [ - ('detections', 'fsdsinet_detections', None), # Use None so detections are not indexed in solr but stored in database + ('detections', 'fsdsinet_detections', + None), # Use None so detections are not indexed in solr but stored in database ('detected_classes', 'fsdsinet_detected_class', list), ('num_detections', 'fsdsinet_detections_count', int), ] @@ -616,22 +619,29 @@ SEARCH_SOUNDS_DEFAULT_FACETS = { SEARCH_SOUNDS_FIELD_SAMPLERATE: {}, - SEARCH_SOUNDS_FIELD_PACK_GROUPING: {'limit': 10}, - SEARCH_SOUNDS_FIELD_USER_NAME: {'limit': 30}, - SEARCH_SOUNDS_FIELD_TAGS: {'limit': 30}, + SEARCH_SOUNDS_FIELD_PACK_GROUPING: { + 'limit': 10 + }, + SEARCH_SOUNDS_FIELD_USER_NAME: { + 'limit': 30 + }, + SEARCH_SOUNDS_FIELD_TAGS: { + 'limit': 30 + }, SEARCH_SOUNDS_FIELD_BITRATE: {}, SEARCH_SOUNDS_FIELD_BITDEPTH: {}, - SEARCH_SOUNDS_FIELD_TYPE: {'limit': len(SOUND_TYPE_CHOICES)}, + SEARCH_SOUNDS_FIELD_TYPE: { + 'limit': len(SOUND_TYPE_CHOICES) + }, SEARCH_SOUNDS_FIELD_CHANNELS: {}, - SEARCH_SOUNDS_FIELD_LICENSE_NAME: {'limit': 10}, + SEARCH_SOUNDS_FIELD_LICENSE_NAME: { + 'limit': 10 + }, } SEARCH_FORUM_SORT_OPTION_THREAD_DATE_FIRST = "Thread creation (newest first)" SEARCH_FORUM_SORT_OPTION_DATE_NEW_FIRST = "Post creation (newest first)" -SEARCH_FORUM_SORT_OPTIONS_WEB = [ - SEARCH_FORUM_SORT_OPTION_THREAD_DATE_FIRST, - SEARCH_FORUM_SORT_OPTION_DATE_NEW_FIRST -] +SEARCH_FORUM_SORT_OPTIONS_WEB = [SEARCH_FORUM_SORT_OPTION_THREAD_DATE_FIRST, SEARCH_FORUM_SORT_OPTION_DATE_NEW_FIRST] SEARCH_FORUM_SORT_DEFAULT = SEARCH_FORUM_SORT_OPTION_THREAD_DATE_FIRST SEARCH_ENGINE_BACKEND_CLASS = 'utils.search.backends.solr9pysolr.Solr9PySolrSearchEngine' @@ -678,7 +688,6 @@ MAPBOX_ACCESS_TOKEN = '' MAPBOX_USE_STATIC_MAPS_BEFORE_LOADING = True - # ------------------------------------------------------------------------------- # Recaptcha settings @@ -693,7 +702,6 @@ SILENCED_SYSTEM_CHECKS += ['captcha.recaptcha_test_key_error'] - # ------------------------------------------------------------------------------- # Akismet @@ -711,7 +719,9 @@ WORKER_TIMEOUT = 60 * 60 # Used to configure output formats in newer FreesoundExtractor versions -ESSENTIA_PROFILE_FILE_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '../utils/audioprocessing/essentia_profile.yaml')) +ESSENTIA_PROFILE_FILE_PATH = os.path.abspath( + os.path.join(os.path.dirname(__file__), '../utils/audioprocessing/essentia_profile.yaml') +) # Sound previews quality (for mp3, quality is in bitrate, for ogg, quality is in a number from 1 to 10 ) MP3_LQ_PREVIEW_QUALITY = 70 @@ -741,13 +751,10 @@ RATELIMIT_SEARCH_GROUP = 'search' RATELIMIT_SIMILARITY_GROUP = 'similarity' RATELIMIT_DEFAULT_GROUP_RATELIMIT = '2/s' -RATELIMITS = { - RATELIMIT_SEARCH_GROUP: '2/s', - RATELIMIT_SIMILARITY_GROUP: '2/s' -} +RATELIMITS = {RATELIMIT_SEARCH_GROUP: '2/s', RATELIMIT_SIMILARITY_GROUP: '2/s'} BLOCKED_IPS = [] CACHED_BLOCKED_IPS_KEY = 'cached_blocked_ips' -CACHED_BLOCKED_IPS_TIME = 60 * 5 # 5 minutes +CACHED_BLOCKED_IPS_TIME = 60 * 5 # 5 minutes # ------------------------------------------------------------------------------- # API settings @@ -778,12 +785,12 @@ 'VIEW_DESCRIPTION_FUNCTION': 'apiv2.apiv2_utils.get_view_description', } -API_DOWNLOAD_TOKEN_LIFETIME = 60*60 # 1 hour +API_DOWNLOAD_TOKEN_LIFETIME = 60 * 60 # 1 hour OAUTH2_PROVIDER = { - 'ACCESS_TOKEN_EXPIRE_SECONDS': 60*60*24, + 'ACCESS_TOKEN_EXPIRE_SECONDS': 60 * 60 * 24, 'CLIENT_SECRET_GENERATOR_LENGTH': 40, - 'AUTHORIZATION_CODE_EXPIRE_SECONDS': 10*60, + 'AUTHORIZATION_CODE_EXPIRE_SECONDS': 10 * 60, 'OAUTH2_VALIDATOR_CLASS': 'apiv2.oauth2_validators.OAuth2Validator', 'REQUEST_APPROVAL_PROMPT': 'auto', 'CLIENT_ID_GENERATOR_CLASS': 'apiv2.apiv2_utils.FsClientIdGenerator', @@ -799,30 +806,27 @@ # Sustained limit sets the maximum number of requests that an api client can do in a day # Ip limit sets the maximum number of requests from different ips that a client can do in an hour APIV2_BASIC_THROTTLING_RATES_PER_LEVELS = { - 0: ['0/minute', '0/day', '0/hour'], # Client 'disabled' - 1: ['60/minute', '2000/day', None], # Ip limit not yet enabled - 2: ['300/minute', '5000/day', None], # Ip limit not yet enabled - 3: ['300/minute', '15000/day', None], # Ip limit not yet enabled - 99: [], # No limit of requests + 0: ['0/minute', '0/day', '0/hour'], # Client 'disabled' + 1: ['60/minute', '2000/day', None], # Ip limit not yet enabled + 2: ['300/minute', '5000/day', None], # Ip limit not yet enabled + 3: ['300/minute', '15000/day', None], # Ip limit not yet enabled + 99: [], # No limit of requests } APIV2_POST_THROTTLING_RATES_PER_LEVELS = { - 0: ['0/minute', '0/day', '0/hour'], # Client 'disabled' - 1: ['30/minute', '500/day', None], # Ip limit not yet enabled - 2: ['60/minute', '1000/day', None], # Ip limit not yet enabled - 3: ['60/minute', '3000/day', None], # Ip limit not yet enabled - 99: [], # No limit of requests + 0: ['0/minute', '0/day', '0/hour'], # Client 'disabled' + 1: ['30/minute', '500/day', None], # Ip limit not yet enabled + 2: ['60/minute', '1000/day', None], # Ip limit not yet enabled + 3: ['60/minute', '3000/day', None], # Ip limit not yet enabled + 99: [], # No limit of requests } - # ------------------------------------------------------------------------------- # Frontend handling TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [ - os.path.join(os.path.dirname(__file__), '../templates'), - ], + 'DIRS': [os.path.join(os.path.dirname(__file__), '../templates'),], 'APP_DIRS': True, 'OPTIONS': { 'context_processors': [ @@ -851,20 +855,18 @@ PLAUSIBLE_AGGREGATE_PAGEVIEWS = True # ------------------------------------------------------------------------------- -# Rabbit MQ +# Rabbit MQ RABBITMQ_USER = "guest" RABBITMQ_PASS = "guest" RABBITMQ_HOST = 'rabbitmq' RABBITMQ_PORT = '5672' RABBITMQ_API_PORT = '5673' - # ------------------------------------------------------------------------------- # Import local settings # Important: place settings which depend on other settings potentially modified in local_settings.py BELOW the import from .local_settings import * - # ------------------------------------------------------------------------------- # Celery CELERY_BROKER_URL = f'amqp://{RABBITMQ_USER}:{RABBITMQ_PASS}@{RABBITMQ_HOST}:{RABBITMQ_PORT}//' @@ -875,7 +877,6 @@ CELERY_ASYNC_TASKS_QUEUE_NAME = 'async_tasks_queue' CELERY_SOUND_PROCESSING_QUEUE_NAME = 'sound_processing_queue' - # ------------------------------------------------------------------------------- # Sentry @@ -887,7 +888,6 @@ traces_sample_rate=TRACES_SAMPLE_RATE, ) - # ------------------------------------------------------------------------------- # Extra Freesound settings @@ -939,8 +939,8 @@ DEBUG_TOOLBAR_CONFIG = { 'INTERCEPT_REDIRECTS': False, - # This normally checks the running host with the request url, but this doesn't - # work in docker. Unconditionally show the toolbar when DEBUG is True + # This normally checks the running host with the request url, but this doesn't + # work in docker. Unconditionally show the toolbar when DEBUG is True 'SHOW_TOOLBAR_CALLBACK': lambda request: DEBUG } diff --git a/freesound/storage.py b/freesound/storage.py index aa1b40544..111041bbc 100644 --- a/freesound/storage.py +++ b/freesound/storage.py @@ -1,4 +1,5 @@ from django.contrib.staticfiles.storage import ManifestStaticFilesStorage + class NoStrictManifestStaticFilesStorage(ManifestStaticFilesStorage): manifest_strict = False diff --git a/freesound/test_settings.py b/freesound/test_settings.py index d95458f5a..1cd47be53 100644 --- a/freesound/test_settings.py +++ b/freesound/test_settings.py @@ -33,13 +33,14 @@ } from .logger import LOGGING + LOGGING['handlers']['stdout']['class'] = 'logging.NullHandler' -SOLR5_SOUNDS_URL = "http://fakehost:8080/fs2/" # Avoid making accidental queries to "real" search server if running -SOLR5_FORUM_URL = "http://fakehost:8080/forum/" # Avoid making accidental requests to "real" search server if running -SEARCH_ENGINE_BACKEND_CLASS = 'utils.search.backends.solr555pysolr.Solr555PySolrSearchEngine' # Test with our own custom search engine functions -SIMILARITY_ADDRESS = 'fakehost' # Avoid making accidental requests to "real" similarity server if running -TAGRECOMMENDATION_ADDRESS = 'fakehost' # Avoid making accidental requests to "real" tag rec server if running +SOLR5_SOUNDS_URL = "http://fakehost:8080/fs2/" # Avoid making accidental queries to "real" search server if running +SOLR5_FORUM_URL = "http://fakehost:8080/forum/" # Avoid making accidental requests to "real" search server if running +SEARCH_ENGINE_BACKEND_CLASS = 'utils.search.backends.solr555pysolr.Solr555PySolrSearchEngine' # Test with our own custom search engine functions +SIMILARITY_ADDRESS = 'fakehost' # Avoid making accidental requests to "real" similarity server if running +TAGRECOMMENDATION_ADDRESS = 'fakehost' # Avoid making accidental requests to "real" tag rec server if running # Disable debug toolbar (it will have been enabled because when importing settings and checking local_settings, the # DISPLAY_DEBUG_TOOLBAR is most probably True, so we undo this change here) diff --git a/freesound/urls.py b/freesound/urls.py index 532294fae..4f1b1ec6b 100644 --- a/freesound/urls.py +++ b/freesound/urls.py @@ -41,11 +41,14 @@ urlpatterns = [ path('', sounds.views.front_page, name='front-page'), - path('people/', accounts.views.accounts, name="accounts"), path('people//', accounts.views.account, name="account"), path('people//section/stats/', accounts.views.account_stats_section, name="account-stats-section"), - path('people//section/latest_packs/', accounts.views.account_latest_packs_section, name="account-latest-packs-section"), + path( + 'people//section/latest_packs/', + accounts.views.account_latest_packs_section, + name="account-latest-packs-section" + ), path('people//sounds/', sounds.views.for_user, name="sounds-for-user"), path('people//flag/', accounts.views.flag_user, name="flag-user"), path('people//clear_flags/', accounts.views.clear_flags_user, name="clear-flags-user"), @@ -53,9 +56,17 @@ path('people//comments_by/', comments.views.by_user, name="comments-by-user"), path('people//geotags/', geotags.views.for_user, name="geotags-for-user"), path('people//sounds//', sounds.views.sound, name="sound"), - re_path(r'^people/(?P[^//]+)/sounds/(?P\d+)/download/.*$', sounds.views.sound_download, name="sound-download"), + re_path( + r'^people/(?P[^//]+)/sounds/(?P\d+)/download/.*$', + sounds.views.sound_download, + name="sound-download" + ), path('people//sounds//flag/', sounds.views.flag, name="sound-flag"), - path('people//sounds//edit/sources/', sounds.views.sound_edit_sources, name="sound-edit-sources"), + path( + 'people//sounds//edit/sources/', + sounds.views.sound_edit_sources, + name="sound-edit-sources" + ), path('people//sounds//edit/', sounds.views.sound_edit, name="sound-edit"), path('people//sounds//remixes/', sounds.views.remixes, name="sound-remixes"), path('people//sounds//geotag/', geotags.views.for_sound, name="sound-geotag"), @@ -64,9 +75,17 @@ path('people//sounds//comments/', comments.views.for_sound, name="sound-comments"), path('people//packs/', sounds.views.packs_for_user, name="packs-for-user"), path('people//packs//', sounds.views.pack, name="pack"), - path('people//packs//section/stats/', sounds.views.pack_stats_section, name="pack-stats-section"), + path( + 'people//packs//section/stats/', + sounds.views.pack_stats_section, + name="pack-stats-section" + ), path('people//packs//edit/', sounds.views.pack_edit, name="pack-edit"), - re_path(r'^people/(?P[^//]+)/packs/(?P\d+)/download/.*$', sounds.views.pack_download, name="pack-download"), + re_path( + r'^people/(?P[^//]+)/packs/(?P\d+)/download/.*$', + sounds.views.pack_download, + name="pack-download" + ), path('people//packs//downloaders/', sounds.views.pack_downloaders, name="pack-downloaders"), path('people//packs//licenses/', sounds.views.pack_licenses, name="pack-licenses"), path('people//packs//geotags/', geotags.views.for_pack, name="pack-geotags"), @@ -74,19 +93,23 @@ path('people//downloaded_sounds/', accounts.views.downloaded_sounds, name="user-downloaded-sounds"), path('people//downloaded_packs/', accounts.views.downloaded_packs, name="user-downloaded-packs"), path('people//bookmarks/', bookmarks.views.bookmarks_for_user, name="bookmarks-for-user"), - path('people//bookmarks/category//', bookmarks.views.bookmarks_for_user, name="bookmarks-for-user-for-category"), + path( + 'people//bookmarks/category//', + bookmarks.views.bookmarks_for_user, + name="bookmarks-for-user-for-category" + ), path('people//following_users/', follow.views.following_users, name="user-following-users"), path('people//followers/', follow.views.followers, name="user-followers"), path('people//following_tags/', follow.views.following_tags, name="user-following-tags"), - path('charts/', accounts.views.charts, name="charts"), - - path('embed/sound/iframe//simple//', sounds.views.embed_iframe, name="embed-simple-sound-iframe"), + path( + 'embed/sound/iframe//simple//', + sounds.views.embed_iframe, + name="embed-simple-sound-iframe" + ), path('embed/geotags_box/iframe/', geotags.views.embed_iframe, name="embed-geotags-box-iframe"), path('oembed/', sounds.views.oembed, name="oembed-sound"), - path('after-download-modal/', sounds.views.after_download_modal, name="after-download-modal"), - path('browse/', sounds.views.sounds, name="sounds"), path('browse/tags/', tags.views.tags, name="tags"), re_path(r'^browse/tags/(?P[\w//-]+)/$', tags.views.tags, name="tags"), @@ -94,17 +117,17 @@ path('browse/random/', sounds.views.random, name="sounds-random"), re_path(r'^browse/geotags/(?P[\w-]+)?/?$', geotags.views.geotags, name="geotags"), path('browse/geotags_box/', geotags.views.geotags_box, name="geotags-box"), - path('contact/', support.views.contact, name="contact"), - path('search/', search.views.search, name='sounds-search'), path('clustering_facet/', search.views.clustering_facet, name='clustering-facet'), path('clustered_graph/', search.views.clustered_graph, name='clustered-graph-json'), path('query_suggestions/', search.views.query_suggestions, name='query-suggestions'), - path('add_sounds_modal/sources/', sounds.views.add_sounds_modal_for_edit_sources, name="add-sounds-modal-sources"), - path('add_sounds_modal/pack//', sounds.views.add_sounds_modal_for_pack_edit, name="add-sounds-modal-pack"), - + path( + 'add_sounds_modal/pack//', + sounds.views.add_sounds_modal_for_pack_edit, + name="add-sounds-modal-pack" + ), path('', include('ratings.urls')), path('comments/', include('comments.urls')), path('help/', include('wiki.urls')), @@ -115,7 +138,6 @@ path('tickets/', include('tickets.urls')), path('monitor/', include('monitor.urls')), path('follow/', include('follow.urls')), - path('blog/', RedirectView.as_view(url='https://blog.freesound.org/'), name="blog"), # admin views @@ -131,7 +153,6 @@ # 500 view path('crash_me/', accounts.views.crash_me, name="crash-me"), - path('donate/', donations.views.donate_redirect, name="donate-redirect"), path('s//', sounds.views.sound_short_link, name="short-sound-link"), path('p//', sounds.views.pack_short_link, name="short-pack-link"), @@ -158,8 +179,12 @@ def serve_source_map_files(request): return serve(request, path, document_root=document_root, show_indexes=False) urlpatterns += [ - re_path(r'^%s/(?P.*)$' % settings.DATA_URL.strip('/'), serve, - {'document_root': settings.DATA_PATH, 'show_indexes': True}), + re_path( + r'^%s/(?P.*)$' % settings.DATA_URL.strip('/'), serve, { + 'document_root': settings.DATA_PATH, + 'show_indexes': True + } + ), path('__debug__/', include(debug_toolbar.urls)), re_path(r'^.*\.map$', serve_source_map_files), ] diff --git a/freesound/wsgi.py b/freesound/wsgi.py index f264762e5..ae8b59766 100644 --- a/freesound/wsgi.py +++ b/freesound/wsgi.py @@ -21,6 +21,7 @@ # file. This includes Django's development server, if the WSGI_APPLICATION # setting points here. from django.core.wsgi import get_wsgi_application + application = get_wsgi_application() # Apply WSGI middleware here. diff --git a/general/admin.py b/general/admin.py index bbc604f2b..ce63a97f5 100644 --- a/general/admin.py +++ b/general/admin.py @@ -21,10 +21,10 @@ from django.contrib import admin from general.models import AkismetSpam + @admin.register(AkismetSpam) class AkismetSpamAdmin(admin.ModelAdmin): - raw_id_fields = ('user', ) + raw_id_fields = ('user',) list_display = ('user', 'created') - ordering = ('-created', ) - search_fields = ('=user__username', ) - + ordering = ('-created',) + search_fields = ('=user__username',) diff --git a/general/management/commands/build_static.py b/general/management/commands/build_static.py index 30e609fa6..8a6667eea 100644 --- a/general/management/commands/build_static.py +++ b/general/management/commands/build_static.py @@ -23,7 +23,6 @@ from django.core.management.base import BaseCommand - console_logger = logging.getLogger("console") diff --git a/general/management/commands/clean_data_volume.py b/general/management/commands/clean_data_volume.py index 652b69706..180af244a 100644 --- a/general/management/commands/clean_data_volume.py +++ b/general/management/commands/clean_data_volume.py @@ -34,8 +34,8 @@ def remove_folder(folderpath, recursively=False): if not recursively: # First delete files inside folder for filename in os.listdir(folderpath): - os.remove(os.path.join(folderpath, filename)) - # Then delete the folder itself + os.remove(os.path.join(folderpath, filename)) + # Then delete the folder itself shutil.rmtree(folderpath) except Exception as e: console_logger.info(f'ERROR removing folder {folderpath}: {e}') @@ -48,16 +48,12 @@ def add_arguments(self, parser): parser.add_argument( '--dry-run', action="store_true", - help="Using this flag files will not be deleted but only information printed on screen.") + help="Using this flag files will not be deleted but only information printed on screen." + ) def handle(self, **options): self.log_start() - cleaned_files = { - 'tmp_uploads': 0, - 'tmp_processing': 0, - 'uploads': 0, - 'processing_before_describe': 0 - } + cleaned_files = {'tmp_uploads': 0, 'tmp_processing': 0, 'uploads': 0, 'processing_before_describe': 0} one_day_ago = datetime.datetime.today() - datetime.timedelta(days=1) one_year_ago = datetime.datetime.today() - datetime.timedelta(days=365) @@ -81,7 +77,8 @@ def handle(self, **options): if not files_in_folder: should_delete = True else: - if all([datetime.datetime.fromtimestamp(os.path.getmtime(os.path.join(folderpath, filename))) < one_day_ago for filename in files_in_folder]): + if all([datetime.datetime.fromtimestamp(os.path.getmtime(os.path.join(folderpath, filename))) + < one_day_ago for filename in files_in_folder]): should_delete = True if should_delete: # Delete directory and contents @@ -99,7 +96,8 @@ def handle(self, **options): if not files_in_folder: should_delete = True else: - if all([datetime.datetime.fromtimestamp(os.path.getmtime(os.path.join(folderpath, sound_filename))) < one_year_ago for sound_filename in files_in_folder]): + if all([datetime.datetime.fromtimestamp(os.path.getmtime(os.path.join(folderpath, sound_filename))) + < one_year_ago for sound_filename in files_in_folder]): should_delete = True if should_delete: # Delete directory and contents @@ -118,5 +116,4 @@ def handle(self, **options): if not options['dry_run']: remove_folder(folderpath, recursively=True) - self.log_end(cleaned_files) diff --git a/general/management/commands/clear_sound_template_caches.py b/general/management/commands/clear_sound_template_caches.py index 06f049d83..123845f8f 100644 --- a/general/management/commands/clear_sound_template_caches.py +++ b/general/management/commands/clear_sound_template_caches.py @@ -33,7 +33,7 @@ class Command(BaseCommand): def handle(self, **options): # Get default cache, as this is where sound template entries are stored cache = caches['default'] - all_keys= cache.keys('*') + all_keys = cache.keys('*') keys_to_delete = [key for key in all_keys if 'template' in key and ('sound' in key or 'pack' in key)] total = len(keys_to_delete) console_logger.info(f'Will clear {total} keys from cache') diff --git a/general/management/commands/create_front_page_caches.py b/general/management/commands/create_front_page_caches.py index cb5e482e8..1c91dd77c 100644 --- a/general/management/commands/create_front_page_caches.py +++ b/general/management/commands/create_front_page_caches.py @@ -44,10 +44,10 @@ def handle(self, **options): NUM_ITEMS_PER_SECTION = 9 - last_week = get_n_weeks_back_datetime(n_weeks=1) # Use later to filter queries - last_two_weeks = get_n_weeks_back_datetime(n_weeks=2) # Use later to filter queries + last_week = get_n_weeks_back_datetime(n_weeks=1) # Use later to filter queries + last_two_weeks = get_n_weeks_back_datetime(n_weeks=2) # Use later to filter queries - cache_time = 24 * 60 * 60 # 1 day cache time + cache_time = 24 * 60 * 60 # 1 day cache time # NOTE: The specific cache time is not important as long as it is bigger than the frequency with which we run # create_front_page_caches management command @@ -61,9 +61,11 @@ def handle(self, **options): # Generate popular searches cache # TODO: implement this properly if we want to add this functionality - popular_searches = ['wind', 'music', 'footsteps', 'woosh', 'explosion', 'scream', 'click', 'whoosh', 'piano', - 'swoosh', 'rain', 'fire'] - cache.set("popular_searches", popular_searches, cache_time) + popular_searches = [ + 'wind', 'music', 'footsteps', 'woosh', 'explosion', 'scream', 'click', 'whoosh', 'piano', 'swoosh', 'rain', + 'fire' + ] + cache.set("popular_searches", popular_searches, cache_time) # Generate trending sounds cache (most downloaded sounds during last week) trending_sound_ids = Download.objects \ @@ -71,8 +73,8 @@ def handle(self, **options): .values('sound_id').annotate(n_downloads=Count('sound_id')) \ .order_by('-n_downloads').values_list('sound_id', flat=True)[0:NUM_ITEMS_PER_SECTION * 5] trending_sound_ids = list(trending_sound_ids) - random.shuffle(trending_sound_ids) # Randomize the order of the sounds - cache.set("trending_sound_ids", trending_sound_ids[0:NUM_ITEMS_PER_SECTION], cache_time) + random.shuffle(trending_sound_ids) # Randomize the order of the sounds + cache.set("trending_sound_ids", trending_sound_ids[0:NUM_ITEMS_PER_SECTION], cache_time) # Generate trending new sounds cache (most downloaded sounds from those created last week) trending_new_sound_ids = Sound.public.select_related('license', 'user') \ @@ -80,15 +82,15 @@ def handle(self, **options): .filter(greatest_date__gte=last_week).exclude(is_explicit=True) \ .order_by("-num_downloads").values_list('id', flat=True)[0:NUM_ITEMS_PER_SECTION * 5] trending_new_sound_ids = list(trending_new_sound_ids) - random.shuffle(trending_new_sound_ids) # Randomize the order of the sounds - cache.set("trending_new_sound_ids", trending_new_sound_ids[0:NUM_ITEMS_PER_SECTION], cache_time) + random.shuffle(trending_new_sound_ids) # Randomize the order of the sounds + cache.set("trending_new_sound_ids", trending_new_sound_ids[0:NUM_ITEMS_PER_SECTION], cache_time) # Generate trending new packs cache (most downloaded packs from those created last week) trending_new_pack_ids = Pack.objects.select_related('user') \ .filter(created__gte=last_week, num_sounds__gt=0).exclude(is_deleted=True) \ .order_by("-num_downloads").values_list('id', flat=True)[0:NUM_ITEMS_PER_SECTION * 5] trending_new_pack_ids = list(trending_new_pack_ids) - random.shuffle(trending_new_pack_ids) # Randomize the order of the packs + random.shuffle(trending_new_pack_ids) # Randomize the order of the packs cache.set("trending_new_pack_ids", trending_new_pack_ids[0:NUM_ITEMS_PER_SECTION], cache_time) # Generate top rated new sounds cache (top rated sounds from those created last two weeks) @@ -99,11 +101,14 @@ def handle(self, **options): .filter(num_ratings__gt=settings.MIN_NUMBER_RATINGS) \ .order_by("-avg_rating", "-num_ratings").values_list('id', flat=True)[0:NUM_ITEMS_PER_SECTION * 5] top_rated_new_sound_ids = list(top_rated_new_sound_ids) - random.shuffle(top_rated_new_sound_ids) # Randomize the order of the sounds - cache.set("top_rated_new_sound_ids", top_rated_new_sound_ids[0:NUM_ITEMS_PER_SECTION], cache_time) - + random.shuffle(top_rated_new_sound_ids) # Randomize the order of the sounds + cache.set("top_rated_new_sound_ids", top_rated_new_sound_ids[0:NUM_ITEMS_PER_SECTION], cache_time) + # Generate latest "random sound of the day" ids - recent_random_sound_ids = [sd.sound_id for sd in SoundOfTheDay.objects.filter(date_display__lt=datetime.datetime.today()).order_by('-date_display')[:NUM_ITEMS_PER_SECTION]] + recent_random_sound_ids = [ + sd.sound_id for sd in SoundOfTheDay.objects.filter(date_display__lt=datetime.datetime.today() + ).order_by('-date_display')[:NUM_ITEMS_PER_SECTION] + ] cache.set("recent_random_sound_ids", list(recent_random_sound_ids), cache_time) # Add total number of sounds in Freesound to the cache diff --git a/general/management/commands/post_sounds_to_tagrecommendation.py b/general/management/commands/post_sounds_to_tagrecommendation.py index b3f4856d7..b9231c10f 100644 --- a/general/management/commands/post_sounds_to_tagrecommendation.py +++ b/general/management/commands/post_sounds_to_tagrecommendation.py @@ -18,7 +18,6 @@ # See AUTHORS file. # - from django.core.management.base import BaseCommand from sounds.models import Sound @@ -28,13 +27,16 @@ class Command(BaseCommand): args = '' help = 'Get the id of the last indexed sound in tag recommendation service and send tag information of the older ones' + def add_arguments(self, parser): parser.add_argument( - '-a','--all', + '-a', + '--all', action='store_true', dest='all', default=False, - help='Repost all sounds to tag recommendation even if they were already indexed') + help='Repost all sounds to tag recommendation even if they were already indexed' + ) def handle(self, *args, **options): @@ -44,5 +46,7 @@ def handle(self, *args, **options): last_indexed_id = get_id_of_last_indexed_sound() print("Starting at id %i" % last_indexed_id) - sound_qs = Sound.objects.filter(moderation_state='OK', processing_state='OK', id__gt=last_indexed_id).order_by("id") + sound_qs = Sound.objects.filter( + moderation_state='OK', processing_state='OK', id__gt=last_indexed_id + ).order_by("id") post_sounds_to_tagrecommendation_service(sound_qs) diff --git a/general/management/commands/prune_database.py b/general/management/commands/prune_database.py index 4b0d85167..af5e9acfe 100644 --- a/general/management/commands/prune_database.py +++ b/general/management/commands/prune_database.py @@ -18,7 +18,6 @@ # See AUTHORS file. # - import datetime import logging import os @@ -52,25 +51,21 @@ console_logger = logging.getLogger('console') - - class Command(BaseCommand): help = "Delete most of the database to make it smaller for development, and anonymise it" def add_arguments(self, parser): parser.add_argument( - '-d', '--keep-downloaders', + '-d', + '--keep-downloaders', dest='downloaders', default=100000, type=int, - help='The number of downloaders to keep') + help='The number of downloaders to keep' + ) parser.add_argument( - '-s', '--keep-sounds', - dest='sounds', - default=5000, - type=int, - help='Number of sounds to keep' + '-s', '--keep-sounds', dest='sounds', default=5000, type=int, help='Number of sounds to keep' ) def disconnect_signals(self): @@ -105,7 +100,10 @@ def delete_some_users(self, userids): # table and rename. with connection.cursor() as cursor: cursor.execute("delete from sounds_download where user_id in %s", [tuple(userids)]) - cursor.execute("delete from sounds_packdownloadsound where pack_download_id in (select id from sounds_packdownload where user_id in %s)", [tuple(userids)]) + cursor.execute( + "delete from sounds_packdownloadsound where pack_download_id in (select id from sounds_packdownload where user_id in %s)", + [tuple(userids)] + ) cursor.execute("delete from sounds_packdownload where user_id in %s", [tuple(userids)]) console_logger.info(' - done, user objects') User.objects.filter(id__in=userids).delete() @@ -139,7 +137,7 @@ def delete_sound_uploaders(self, numkeep): break console_logger.info(f"Keeping {len(userids)} users with {totalsounds} sounds") - + users_not_in_userids = list(set(all_users_with_sounds) - set(userids)) ch = [c for c in chunks(sorted(list(users_not_in_userids)), 1000)] tot = len(ch) @@ -177,13 +175,21 @@ def anonymise_database(self): user.is_staff = False user.is_superuser = False users_to_update.append(user) - User.objects.bulk_update(users_to_update, ['email', 'first_name', 'last_name', 'password', 'is_staff', 'is_superuser']) + User.objects.bulk_update( + users_to_update, ['email', 'first_name', 'last_name', 'password', 'is_staff', 'is_superuser'] + ) - MessageBody.objects.all().update(body='(message body) Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.') - TicketComment.objects.all().update(text='(ticket comment) Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.') + MessageBody.objects.all().update( + body= + '(message body) Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.' + ) + TicketComment.objects.all().update( + text= + '(ticket comment) Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.' + ) GdprAcceptance.objects.all().update(date_accepted=datetime.datetime.now()) - + Profile.objects.all().update( is_whitelisted=False, not_shown_in_online_users_list=False, @@ -234,7 +240,9 @@ def delete_unneeded_tables(self): def handle(self, *args, **options): if os.environ.get('FREESOUND_PRUNE_DATABASE') != '1': - raise Exception('Run this command with env FREESOUND_PRUNE_DATABASE=1 to confirm you want to prune the database') + raise Exception( + 'Run this command with env FREESOUND_PRUNE_DATABASE=1 to confirm you want to prune the database' + ) self.disconnect_signals() self.delete_unneeded_tables() diff --git a/general/management/commands/report_count_statuses.py b/general/management/commands/report_count_statuses.py index 730a2778a..94d8795ed 100644 --- a/general/management/commands/report_count_statuses.py +++ b/general/management/commands/report_count_statuses.py @@ -18,7 +18,6 @@ # See AUTHORS file. # - import logging import pprint from collections import defaultdict @@ -42,20 +41,24 @@ class Command(LoggingBaseCommand): def add_arguments(self, parser): parser.add_argument( - '-n', '--no-changes', + '-n', + '--no-changes', action='store_true', dest='no-changes', default=False, - help='Using the option --no-changes the original objects will not be modified.') + help='Using the option --no-changes the original objects will not be modified.' + ) parser.add_argument( - '-d', '--skip-downloads', + '-d', + '--skip-downloads', action='store_true', dest='skip-downloads', default=False, help='Using the option --skip-downloads the command will not checked for mismatched downloads ' - '(to save time).') + '(to save time).' + ) - def handle(self, *args, **options): + def handle(self, *args, **options): self.log_start() def report_progress(message, total, count, scale=10000): @@ -76,7 +79,8 @@ def report_progress(message, total, count, scale=10000): total = Sound.objects.all().count() # Look at number of comments - for count, sound in enumerate(Sound.objects.all().annotate(real_num_comments=Count('comments')).order_by('id').iterator()): + for count, sound in enumerate(Sound.objects.all().annotate(real_num_comments=Count('comments') + ).order_by('id').iterator()): real_num_comments = sound.real_num_comments if real_num_comments != sound.num_comments: mismatches_report['Sound.num_comments'] += 1 @@ -89,7 +93,8 @@ def report_progress(message, total, count, scale=10000): # Look at number of ratings and average rating for count, sound in enumerate(Sound.objects.all().annotate( - real_num_ratings=Count('ratings'), real_avg_rating=Coalesce(Avg('ratings__rating'), 0.0)).order_by('id').iterator()): + real_num_ratings=Count('ratings'), real_avg_rating=Coalesce(Avg('ratings__rating'), + 0.0)).order_by('id').iterator()): real_num_ratings = sound.real_num_ratings if real_num_ratings != sound.num_ratings: mismatches_report['Sound.num_ratings'] += 1 @@ -103,8 +108,8 @@ def report_progress(message, total, count, scale=10000): # Look at number of downloads if not options['skip-downloads']: - for count, sound in enumerate(Sound.objects.all().annotate( - real_num_downloads=Count('downloads')).order_by('id').iterator()): + for count, sound in enumerate(Sound.objects.all().annotate(real_num_downloads=Count('downloads') + ).order_by('id').iterator()): real_num_downloads = sound.real_num_downloads if real_num_downloads != sound.num_downloads: @@ -120,14 +125,12 @@ def report_progress(message, total, count, scale=10000): total = Pack.objects.all().count() # Look at number of sounds - for count, pack in enumerate(Pack.objects.all().extra(select={ - 'real_num_sounds': """ + for count, pack in enumerate(Pack.objects.all().extra(select={'real_num_sounds': """ SELECT COUNT(U0."id") AS "count" FROM "sounds_sound" U0 WHERE U0."pack_id" = ("sounds_pack"."id") AND U0."processing_state" = 'OK' AND U0."moderation_state" = 'OK' - """ - }).iterator()): + """}).iterator()): real_num_sounds = pack.real_num_sounds if real_num_sounds != pack.num_sounds: mismatches_report['Pack.num_sounds'] += 1 @@ -139,7 +142,8 @@ def report_progress(message, total, count, scale=10000): # Look at number of downloads if not options['skip-downloads']: - for count, pack in enumerate(Pack.objects.all().annotate(real_num_downloads=Count('downloads')).order_by('id').iterator()): + for count, pack in enumerate(Pack.objects.all().annotate(real_num_downloads=Count('downloads') + ).order_by('id').iterator()): real_num_downloads = pack.real_num_downloads if real_num_downloads != pack.num_downloads: mismatches_report['Pack.num_downloads'] += 1 @@ -151,20 +155,19 @@ def report_progress(message, total, count, scale=10000): # Users potential_user_ids = set() - potential_user_ids.update(Sound.objects.all().values_list('user_id', flat=True)) # Add ids of uploaders - potential_user_ids.update(Post.objects.all().values_list('author_id', flat=True)) # Add ids of forum posters + potential_user_ids.update(Sound.objects.all().values_list('user_id', flat=True)) # Add ids of uploaders + potential_user_ids.update(Post.objects.all().values_list('author_id', flat=True)) # Add ids of forum posters total = len(potential_user_ids) # Look at number of sounds - for count, user in enumerate(User.objects.filter(id__in=potential_user_ids).select_related('profile').extra( - select={ - 'real_num_sounds': """ + for count, user in enumerate( + User.objects.filter(id__in=potential_user_ids + ).select_related('profile').extra(select={'real_num_sounds': """ SELECT COUNT(U0."id") AS "count" FROM "sounds_sound" U0 WHERE U0."user_id" = ("auth_user"."id") AND U0."processing_state" = 'OK' AND U0."moderation_state" = 'OK' - """ - }).iterator()): + """}).iterator()): user_profile = user.profile real_num_sounds = user.real_num_sounds if real_num_sounds != user_profile.num_sounds: @@ -197,7 +200,7 @@ def report_progress(message, total, count, scale=10000): report_progress('Checking number of posts in %i users... %.2f%%', total, count) if not options['skip-downloads']: - + total = User.objects.all().count() # Look at number of sound downloads for all active users # NOTE: a possible optimization here would be to first get user candidates that have downloaded sounds. @@ -205,8 +208,8 @@ def report_progress(message, total, count, scale=10000): # for 1/8th less of the time. Nevertheless, because we only run this very ocasionally and the performance # is not severely impacted when running, we decided that the optimization is probably not worth right now. # Same thing applies to pack downloads below. - for count, user in enumerate(User.objects.filter(is_active=True).select_related('profile'). - annotate(real_num_sound_downloads=Count('sound_downloads'),).order_by('id').iterator()): + for count, user in enumerate(User.objects.filter(is_active=True).select_related('profile').annotate( + real_num_sound_downloads=Count('sound_downloads'),).order_by('id').iterator()): user_profile = user.profile real_num_sound_downloads = user.real_num_sound_downloads @@ -221,8 +224,8 @@ def report_progress(message, total, count, scale=10000): report_progress('Checking number of downloaded sounds in %i users... %.2f%%', total, count) # Look at number of pack downloads for all active users (see note above) - for count, user in enumerate(User.objects.filter(is_active=True).select_related('profile'). - annotate(real_num_pack_downloads=Count('pack_downloads'),).order_by('id').iterator()): + for count, user in enumerate(User.objects.filter(is_active=True).select_related('profile').annotate( + real_num_pack_downloads=Count('pack_downloads'),).order_by('id').iterator()): user_profile = user.profile real_num_pack_downloads = user.real_num_pack_downloads @@ -235,17 +238,17 @@ def report_progress(message, total, count, scale=10000): user_profile.save() report_progress('Checking number of downloaded packs in %i users... %.2f%%', total, count) - + # Look at counts of sounds/packs downloaded from a user (i.e. for a given profile, the number of times her sounds/packs # have been downloaded by other users) qs = Profile.objects.filter(num_sounds__gt=0).all().only('user_id') total = qs.count() for count, profile in enumerate(qs): - real_num_user_sounds_downloads = Download.objects.filter(sound__user_id=profile.user_id).count() + real_num_user_sounds_downloads = Download.objects.filter(sound__user_id=profile.user_id).count() real_num_user_packs_downloads = PackDownload.objects.filter(pack__user_id=profile.user_id).count() if real_num_user_sounds_downloads != profile.num_user_sounds_downloads or real_num_user_packs_downloads != profile.num_user_packs_downloads: - + if real_num_user_sounds_downloads != profile.num_user_sounds_downloads: mismatches_report['User.num_user_sounds_downloads'] += 1 mismatches_object_ids['User.num_user_sounds_downloads'].append(profile.user_id) @@ -258,7 +261,9 @@ def report_progress(message, total, count, scale=10000): profile.save() - report_progress('Checking number of downloaded sounds and packs from %i users... %.2f%%', total, count, scale=1000) + report_progress( + 'Checking number of downloaded sounds and packs from %i users... %.2f%%', total, count, scale=1000 + ) console_logger.info("Number of mismatched counts: ") console_logger.info('\n' + pprint.pformat(mismatches_report)) diff --git a/general/management/commands/report_index_statuses.py b/general/management/commands/report_index_statuses.py index 1a64923e7..8327cf76b 100644 --- a/general/management/commands/report_index_statuses.py +++ b/general/management/commands/report_index_statuses.py @@ -39,14 +39,16 @@ class Command(LoggingBaseCommand): def add_arguments(self, parser): parser.add_argument( - '-n', '--no-changes', + '-n', + '--no-changes', action='store_true', dest='no-changes', default=False, help='Using the option --no-changes the is_index_dirty and similarity_state sound fields will not ' - 'be modified.') + 'be modified.' + ) - def handle(self, *args, **options): + def handle(self, *args, **options): self.log_start() # Get all solr ids @@ -108,7 +110,7 @@ def handle(self, *args, **options): console_logger.info("Changing similarity_state of sounds that require it") N = len(in_fs_not_in_gaia) for count, sid in enumerate(in_fs_not_in_gaia): - console_logger.info('\r\tChanging state of sound %i of %i ' % (count+1, N)) + console_logger.info('\r\tChanging state of sound %i of %i ' % (count + 1, N)) sound = Sound.objects.get(id=sid) sound.set_similarity_state('PE') @@ -117,7 +119,7 @@ def handle(self, *args, **options): console_logger.info("\nDeleting sounds that should not be in gaia") N = len(in_gaia_not_in_fs) for count, sid in enumerate(in_gaia_not_in_fs): - console_logger.info('\r\tDeleting sound %i of %i ' % (count+1, N)) + console_logger.info('\r\tDeleting sound %i of %i ' % (count + 1, N)) Similarity.delete(sid) self.log_end({ diff --git a/general/management/commands/similarity_save_index.py b/general/management/commands/similarity_save_index.py index f588abebb..dc435989c 100644 --- a/general/management/commands/similarity_save_index.py +++ b/general/management/commands/similarity_save_index.py @@ -28,11 +28,13 @@ class Command(LoggingBaseCommand): def add_arguments(self, parser): parser.add_argument( - '-i', '--indexing_server', + '-i', + '--indexing_server', action='store_true', dest='indexing_server', default=False, - help='Save the index of the indexing server instead of the index of the main similarity server') + help='Save the index of the indexing server instead of the index of the main similarity server' + ) def handle(self, *args, **options): self.log_start() diff --git a/general/management/commands/similarity_update.py b/general/management/commands/similarity_update.py index ca8ecc564..fd7ee9400 100644 --- a/general/management/commands/similarity_update.py +++ b/general/management/commands/similarity_update.py @@ -38,34 +38,37 @@ class Command(LoggingBaseCommand): def add_arguments(self, parser): parser.add_argument( - '-a', '--analyzer', + '-a', + '--analyzer', action='store', dest='analyzer', default=settings.FREESOUND_ESSENTIA_EXTRACTOR_NAME, - help='Only index sounds analyzed with specific anayzer name/version') + help='Only index sounds analyzed with specific anayzer name/version' + ) parser.add_argument( - '-l', '--limit', - action='store', - dest='limit', - default=1000, - help='Maximum number of sounds to index') + '-l', '--limit', action='store', dest='limit', default=1000, help='Maximum number of sounds to index' + ) parser.add_argument( - '-f', '--force', + '-f', + '--force', action='store_true', dest='force', default=False, - help='Reindex all sounds regardless of their similarity state') + help='Reindex all sounds regardless of their similarity state' + ) parser.add_argument( - '-i', '--indexing_server', + '-i', + '--indexing_server', action='store_true', dest='indexing_server', default=False, - help='Send files to the indexing server instead of the main similarity server') + help='Send files to the indexing server instead of the main similarity server' + ) - def handle(self, *args, **options): + def handle(self, *args, **options): self.log_start() limit = int(options['limit']) @@ -78,8 +81,10 @@ def handle(self, *args, **options): if options['force']: sound_ids_to_be_added = sound_ids_analyzed_with_analyzer_ok[:limit] else: - sound_ids_similarity_pending = list(Sound.public.filter(similarity_state='PE').values_list('id', flat=True)) - sound_ids_to_be_added = list(set(sound_ids_similarity_pending).intersection(sound_ids_analyzed_with_analyzer_ok))[:limit] + sound_ids_similarity_pending = list(Sound.public.filter(similarity_state='PE').values_list('id', flat=True)) + sound_ids_to_be_added = list( + set(sound_ids_similarity_pending).intersection(sound_ids_analyzed_with_analyzer_ok) + )[:limit] N = len(sound_ids_to_be_added) to_be_added = sorted(Sound.objects.filter(id__in=sound_ids_to_be_added), key=lambda x: x.id) @@ -94,14 +99,18 @@ def handle(self, *args, **options): sound.set_similarity_state('OK') sound.invalidate_template_caches() n_added += 1 - console_logger.info("%s (%i of %i)" % (result, count+1, N)) + console_logger.info("%s (%i of %i)" % (result, count + 1, N)) except Exception as e: if not options['indexing_server']: sound.set_similarity_state('FA') n_failed += 1 - console_logger.info('Unexpected error while trying to add sound (id: %i, %i of %i): \n\t%s' - % (sound.id, count+1, N, str(e))) - sentry_sdk.capture_exception(e) # Manually capture exception so it has mroe info and Sentry can organize it properly + console_logger.info( + 'Unexpected error while trying to add sound (id: %i, %i of %i): \n\t%s' % + (sound.id, count + 1, N, str(e)) + ) + sentry_sdk.capture_exception( + e + ) # Manually capture exception so it has mroe info and Sentry can organize it properly self.log_end({'n_sounds_added': n_added, 'n_sounds_failed': n_failed}) diff --git a/general/tasks.py b/general/tasks.py index 2dea3c913..cb1eb8ff2 100644 --- a/general/tasks.py +++ b/general/tasks.py @@ -25,7 +25,6 @@ import time import sentry_sdk - from celery import shared_task from django.apps import apps from django.conf import settings @@ -37,7 +36,6 @@ FreesoundAudioProcessor, WorkerException, cancel_timeout_alarm, FreesoundAudioProcessorBeforeDescription from utils.cache import invalidate_user_template_caches, invalidate_all_moderators_header_cache - workers_logger = logging.getLogger("workers") WHITELIST_USER_TASK_NAME = 'whitelist_user' @@ -57,10 +55,13 @@ @shared_task(name=WHITELIST_USER_TASK_NAME, queue=settings.CELERY_ASYNC_TASKS_QUEUE_NAME) def whitelist_user(ticket_ids=None, user_id=None): # Whitelist "sender" users from the tickets with given ids - workers_logger.info("Start whitelisting users from tickets (%s)" % json.dumps({ - 'task_name': WHITELIST_USER_TASK_NAME, - 'n_tickets': len(ticket_ids) if ticket_ids is not None else 0, - 'user_id': user_id if user_id is not None else ''})) + workers_logger.info( + "Start whitelisting users from tickets (%s)" % json.dumps({ + 'task_name': WHITELIST_USER_TASK_NAME, + 'n_tickets': len(ticket_ids) if ticket_ids is not None else 0, + 'user_id': user_id if user_id is not None else '' + }) + ) start_time = time.time() count_done = 0 @@ -74,7 +75,7 @@ def whitelist_user(ticket_ids=None, user_id=None): if user_id is not None: users_to_whitelist_ids.append(user_id) - users_to_whitelist_ids = list(set(users_to_whitelist_ids)) + users_to_whitelist_ids = list(set(users_to_whitelist_ids)) users_to_whitelist = User.objects.filter(id__in=users_to_whitelist_ids).select_related('profile') for whitelist_user in users_to_whitelist: if not whitelist_user.profile.is_whitelisted: @@ -97,21 +98,27 @@ def whitelist_user(ticket_ids=None, user_id=None): # Invalidate template caches for sender user invalidate_user_template_caches(whitelist_user.id) - workers_logger.info("Whitelisted user (%s)" % json.dumps( - {'user_id': whitelist_user.id, - 'username': whitelist_user.username, - 'work_time': round(time.time() - local_start_time)})) + workers_logger.info( + "Whitelisted user (%s)" % json.dumps({ + 'user_id': whitelist_user.id, + 'username': whitelist_user.username, + 'work_time': round(time.time() - local_start_time) + }) + ) count_done = count_done + 1 # Invalidate template caches for moderators invalidate_all_moderators_header_cache() - workers_logger.info("Finished whitelisting users from tickets (%s)" % json.dumps( - {'task_name': WHITELIST_USER_TASK_NAME, - 'n_tickets': len(ticket_ids) if ticket_ids is not None else 0, - 'user_id': user_id if user_id is not None else '', - 'work_time': round(time.time() - start_time)})) + workers_logger.info( + "Finished whitelisting users from tickets (%s)" % json.dumps({ + 'task_name': WHITELIST_USER_TASK_NAME, + 'n_tickets': len(ticket_ids) if ticket_ids is not None else 0, + 'user_id': user_id if user_id is not None else '', + 'work_time': round(time.time() - start_time) + }) + ) @shared_task(name=DELETE_USER_TASK_NAME, queue=settings.CELERY_ASYNC_TASKS_QUEUE_NAME) @@ -119,19 +126,29 @@ def delete_user(user_id, deletion_action, deletion_reason): try: user = User.objects.get(id=user_id) except User.DoesNotExist: - workers_logger.info("Can't delete user as it does not exist (%s)" % json.dumps( - {'task_name': deletion_action, 'user_id': user_id, 'username': '-', - 'deletion_reason': deletion_reason})) + workers_logger.info( + "Can't delete user as it does not exist (%s)" % json.dumps({ + 'task_name': deletion_action, + 'user_id': user_id, + 'username': '-', + 'deletion_reason': deletion_reason + }) + ) return username_before_deletion = user.username - workers_logger.info("Start deleting user (%s)" % json.dumps( - {'task_name': deletion_action, 'user_id': user_id, 'username': username_before_deletion, - 'deletion_reason': deletion_reason})) + workers_logger.info( + "Start deleting user (%s)" % json.dumps({ + 'task_name': deletion_action, + 'user_id': user_id, + 'username': username_before_deletion, + 'deletion_reason': deletion_reason + }) + ) start_time = time.time() try: if deletion_action in [FULL_DELETE_USER_ACTION_NAME, DELETE_USER_KEEP_SOUNDS_ACTION_NAME, - DELETE_USER_DELETE_SOUNDS_ACTION_NAME, DELETE_SPAMMER_USER_ACTION_NAME]: + DELETE_USER_DELETE_SOUNDS_ACTION_NAME, DELETE_SPAMMER_USER_ACTION_NAME]: if deletion_action == DELETE_USER_KEEP_SOUNDS_ACTION_NAME: # This will anonymize the user and will keep the sounds publicly availabe under a "deleted user" @@ -145,15 +162,13 @@ def delete_user(user_id, deletion_action, deletion_reason): # as well as DeletedSound objects for each deleted sound, but sounds will no longer be # publicly available. Extra user content (posts, comments, etc) will be preserved but shown as # being authored by a "deleted user". - user.profile.delete_user(remove_sounds=True, - deletion_reason=deletion_reason) + user.profile.delete_user(remove_sounds=True, deletion_reason=deletion_reason) elif deletion_action == DELETE_SPAMMER_USER_ACTION_NAME: # This will completely remove the user object and all of its related data (including sounds) # from the database. A DeletedUser object will be creaetd to keep a record of a user having been # deleted. - user.profile.delete_user(delete_user_object_from_db=True, - deletion_reason=deletion_reason) + user.profile.delete_user(delete_user_object_from_db=True, deletion_reason=deletion_reason) elif deletion_action == FULL_DELETE_USER_ACTION_NAME: # This will fully delete the user and the sounds from the database. @@ -161,17 +176,32 @@ def delete_user(user_id, deletion_action, deletion_reason): # absolutely no trace about the user. user.delete() - workers_logger.info("Finished deleting user (%s)" % json.dumps( - {'task_name': deletion_action, 'user_id': user.id, 'username': username_before_deletion, - 'deletion_reason': deletion_reason, 'work_time': round(time.time() - start_time)})) + workers_logger.info( + "Finished deleting user (%s)" % json.dumps({ + 'task_name': deletion_action, + 'user_id': user.id, + 'username': username_before_deletion, + 'deletion_reason': deletion_reason, + 'work_time': round(time.time() - start_time) + }) + ) except Exception as e: # This exception is broad but we catch it so that we can log that an error happened. # TODO: catching more specific exceptions would be desirable - workers_logger.info("Unexpected error while deleting user (%s)" % json.dumps( - {'task_name': deletion_action, 'user_id': user.id, 'username': username_before_deletion, - 'deletion_reason': deletion_reason, 'error': str(e), 'work_time': round(time.time() - start_time)})) - sentry_sdk.capture_exception(e) # Manually capture exception so it has mroe info and Sentry can organize it properly + workers_logger.info( + "Unexpected error while deleting user (%s)" % json.dumps({ + 'task_name': deletion_action, + 'user_id': user.id, + 'username': username_before_deletion, + 'deletion_reason': deletion_reason, + 'error': str(e), + 'work_time': round(time.time() - start_time) + }) + ) + sentry_sdk.capture_exception( + e + ) # Manually capture exception so it has mroe info and Sentry can organize it properly @shared_task(name=VALIDATE_BULK_DESCRIBE_CSV_TASK_NAME, queue=settings.CELERY_ASYNC_TASKS_QUEUE_NAME) @@ -179,22 +209,36 @@ def validate_bulk_describe_csv(bulk_upload_progress_object_id): # Import BulkUploadProgress model from apps to avoid circular dependency BulkUploadProgress = apps.get_model('sounds.BulkUploadProgress') - workers_logger.info("Starting validation of BulkUploadProgress (%s)" % json.dumps( - {'task_name': VALIDATE_BULK_DESCRIBE_CSV_TASK_NAME, 'bulk_upload_progress_id': bulk_upload_progress_object_id})) + workers_logger.info( + "Starting validation of BulkUploadProgress (%s)" % json.dumps({ + 'task_name': VALIDATE_BULK_DESCRIBE_CSV_TASK_NAME, + 'bulk_upload_progress_id': bulk_upload_progress_object_id + }) + ) start_time = time.time() try: bulk = BulkUploadProgress.objects.get(id=bulk_upload_progress_object_id) bulk.validate_csv_file() - workers_logger.info("Finished validation of BulkUploadProgress (%s)" % json.dumps( - {'task_name': VALIDATE_BULK_DESCRIBE_CSV_TASK_NAME, 'bulk_upload_progress_id': bulk_upload_progress_object_id, - 'work_time': round(time.time() - start_time)})) - + workers_logger.info( + "Finished validation of BulkUploadProgress (%s)" % json.dumps({ + 'task_name': VALIDATE_BULK_DESCRIBE_CSV_TASK_NAME, + 'bulk_upload_progress_id': bulk_upload_progress_object_id, + 'work_time': round(time.time() - start_time) + }) + ) + except BulkUploadProgress.DoesNotExist as e: - workers_logger.info("Error validating of BulkUploadProgress (%s)" % json.dumps( - {'task_name': VALIDATE_BULK_DESCRIBE_CSV_TASK_NAME, 'bulk_upload_progress_id': bulk_upload_progress_object_id, + workers_logger.info( + "Error validating of BulkUploadProgress (%s)" % json.dumps({ + 'task_name': VALIDATE_BULK_DESCRIBE_CSV_TASK_NAME, + 'bulk_upload_progress_id': bulk_upload_progress_object_id, 'error': str(e), - 'work_time': round(time.time() - start_time)})) - sentry_sdk.capture_exception(e) # Manually capture exception so it has mroe info and Sentry can organize it properly + 'work_time': round(time.time() - start_time) + }) + ) + sentry_sdk.capture_exception( + e + ) # Manually capture exception so it has mroe info and Sentry can organize it properly @shared_task(name=BULK_DESCRIBE_TASK_NAME, queue=settings.CELERY_ASYNC_TASKS_QUEUE_NAME) @@ -202,25 +246,39 @@ def bulk_describe(bulk_upload_progress_object_id): # Import BulkUploadProgress model from apps to avoid circular dependency BulkUploadProgress = apps.get_model('sounds.BulkUploadProgress') - workers_logger.info("Starting describing sounds of BulkUploadProgress (%s)" % json.dumps( - {'task_name': BULK_DESCRIBE_TASK_NAME, 'bulk_upload_progress_id': bulk_upload_progress_object_id})) + workers_logger.info( + "Starting describing sounds of BulkUploadProgress (%s)" % json.dumps({ + 'task_name': BULK_DESCRIBE_TASK_NAME, + 'bulk_upload_progress_id': bulk_upload_progress_object_id + }) + ) start_time = time.time() try: bulk = BulkUploadProgress.objects.get(id=bulk_upload_progress_object_id) bulk.describe_sounds() - bulk.refresh_from_db() # Refresh from db as describe_sounds() method will change fields of bulk - bulk.progress_type = 'F' # Set to finished when one + bulk.refresh_from_db() # Refresh from db as describe_sounds() method will change fields of bulk + bulk.progress_type = 'F' # Set to finished when one bulk.save() - workers_logger.info("Finished describing sounds of BulkUploadProgress (%s)" % json.dumps( - {'task_name': BULK_DESCRIBE_TASK_NAME, 'bulk_upload_progress_id': bulk_upload_progress_object_id, - 'work_time': round(time.time() - start_time)})) - + workers_logger.info( + "Finished describing sounds of BulkUploadProgress (%s)" % json.dumps({ + 'task_name': BULK_DESCRIBE_TASK_NAME, + 'bulk_upload_progress_id': bulk_upload_progress_object_id, + 'work_time': round(time.time() - start_time) + }) + ) + except BulkUploadProgress.DoesNotExist as e: - workers_logger.info("Error describing sounds of BulkUploadProgress (%s)" % json.dumps( - {'task_name': BULK_DESCRIBE_TASK_NAME, 'bulk_upload_progress_id': bulk_upload_progress_object_id, + workers_logger.info( + "Error describing sounds of BulkUploadProgress (%s)" % json.dumps({ + 'task_name': BULK_DESCRIBE_TASK_NAME, + 'bulk_upload_progress_id': bulk_upload_progress_object_id, 'error': str(e), - 'work_time': round(time.time() - start_time)})) - sentry_sdk.capture_exception(e) # Manually capture exception so it has mroe info and Sentry can organize it properly + 'work_time': round(time.time() - start_time) + }) + ) + sentry_sdk.capture_exception( + e + ) # Manually capture exception so it has mroe info and Sentry can organize it properly @shared_task(name=PROCESS_ANALYSIS_RESULTS_TASK_NAME, queue=settings.CELERY_ASYNC_TASKS_QUEUE_NAME) @@ -243,8 +301,14 @@ def process_analysis_results(sound_id, analyzer, status, analysis_time, exceptio # Import SoundAnalysis model from apps to avoid circular dependency SoundAnalysis = apps.get_model('sounds.SoundAnalysis') - workers_logger.info("Starting processing analysis results (%s)" % json.dumps( - {'task_name': PROCESS_ANALYSIS_RESULTS_TASK_NAME, 'sound_id': sound_id, 'analyzer': analyzer, 'status': status})) + workers_logger.info( + "Starting processing analysis results (%s)" % json.dumps({ + 'task_name': PROCESS_ANALYSIS_RESULTS_TASK_NAME, + 'sound_id': sound_id, + 'analyzer': analyzer, + 'status': status + }) + ) start_time = time.time() try: # Analysis happens in a different celery worker, here we just save the results in a SoundAnalysis object @@ -256,23 +320,45 @@ def process_analysis_results(sound_id, analyzer, status, analysis_time, exceptio a.last_analyzer_finished = datetime.datetime.now() a.save(update_fields=['analysis_status', 'last_analyzer_finished', 'analysis_time']) if exception: - workers_logger.info("Finished processing analysis results (%s)" % json.dumps( - {'task_name': PROCESS_ANALYSIS_RESULTS_TASK_NAME, 'sound_id': sound_id, 'analyzer': analyzer, 'status': status, - 'exception': str(exception), 'work_time': round(time.time() - start_time)})) + workers_logger.info( + "Finished processing analysis results (%s)" % json.dumps({ + 'task_name': PROCESS_ANALYSIS_RESULTS_TASK_NAME, + 'sound_id': sound_id, + 'analyzer': analyzer, + 'status': status, + 'exception': str(exception), + 'work_time': round(time.time() - start_time) + }) + ) else: # Load analysis output to database field (following configuration in settings.ANALYZERS_CONFIGURATION) a.load_analysis_data_from_file_to_db() # Set sound to index dirty so that the sound gets reindexed with updated analysis fields a.sound.mark_index_dirty(commit=True) - workers_logger.info("Finished processing analysis results (%s)" % json.dumps( - {'task_name': PROCESS_ANALYSIS_RESULTS_TASK_NAME, 'sound_id': sound_id, 'analyzer': analyzer, 'status': status, - 'work_time': round(time.time() - start_time)})) + workers_logger.info( + "Finished processing analysis results (%s)" % json.dumps({ + 'task_name': PROCESS_ANALYSIS_RESULTS_TASK_NAME, + 'sound_id': sound_id, + 'analyzer': analyzer, + 'status': status, + 'work_time': round(time.time() - start_time) + }) + ) except (SoundAnalysis.DoesNotExist, Exception) as e: - workers_logger.info("Error processing analysis results (%s)" % json.dumps( - {'task_name': PROCESS_ANALYSIS_RESULTS_TASK_NAME, 'sound_id': sound_id, 'analyzer': analyzer, 'status': status, - 'error': str(e), 'work_time': round(time.time() - start_time)})) - sentry_sdk.capture_exception(e) # Manually capture exception so it has mroe info and Sentry can organize it properly + workers_logger.info( + "Error processing analysis results (%s)" % json.dumps({ + 'task_name': PROCESS_ANALYSIS_RESULTS_TASK_NAME, + 'sound_id': sound_id, + 'analyzer': analyzer, + 'status': status, + 'error': str(e), + 'work_time': round(time.time() - start_time) + }) + ) + sentry_sdk.capture_exception( + e + ) # Manually capture exception so it has mroe info and Sentry can organize it properly @shared_task(name=SOUND_PROCESSING_TASK_NAME, queue=settings.CELERY_SOUND_PROCESSING_QUEUE_NAME) @@ -288,21 +374,35 @@ def process_sound(sound_id, skip_previews=False, skip_displays=False): Sound = apps.get_model('sounds.Sound') set_timeout_alarm(settings.WORKER_TIMEOUT, f'Processing of sound {sound_id} timed out') - workers_logger.info("Starting processing of sound (%s)" % json.dumps({ - 'task_name': SOUND_PROCESSING_TASK_NAME, 'sound_id': sound_id})) + workers_logger.info( + "Starting processing of sound (%s)" % json.dumps({ + 'task_name': SOUND_PROCESSING_TASK_NAME, + 'sound_id': sound_id + }) + ) start_time = time.time() try: check_if_free_space() result = FreesoundAudioProcessor(sound_id=sound_id) \ .process(skip_displays=skip_displays, skip_previews=skip_previews) if result: - workers_logger.info("Finished processing of sound (%s)" % json.dumps( - {'task_name': SOUND_PROCESSING_TASK_NAME, 'sound_id': sound_id, 'result': 'success', - 'work_time': round(time.time() - start_time)})) + workers_logger.info( + "Finished processing of sound (%s)" % json.dumps({ + 'task_name': SOUND_PROCESSING_TASK_NAME, + 'sound_id': sound_id, + 'result': 'success', + 'work_time': round(time.time() - start_time) + }) + ) else: - workers_logger.info("Finished processing of sound (%s)" % json.dumps( - {'task_name': SOUND_PROCESSING_TASK_NAME, 'sound_id': sound_id, 'result': 'failure', - 'work_time': round(time.time() - start_time)})) + workers_logger.info( + "Finished processing of sound (%s)" % json.dumps({ + 'task_name': SOUND_PROCESSING_TASK_NAME, + 'sound_id': sound_id, + 'result': 'failure', + 'work_time': round(time.time() - start_time) + }) + ) except WorkerException as e: try: @@ -311,10 +411,17 @@ def process_sound(sound_id, skip_previews=False, skip_displays=False): sound.change_processing_state("FA", processing_log=str(e)) except Sound.DoesNotExist: pass - workers_logger.info("WorkerException while processing sound (%s)" % json.dumps( - {'task_name': SOUND_PROCESSING_TASK_NAME, 'sound_id': sound_id, 'error': str(e), - 'work_time': round(time.time() - start_time)})) - sentry_sdk.capture_exception(e) # Manually capture exception so it has mroe info and Sentry can organize it properly + workers_logger.info( + "WorkerException while processing sound (%s)" % json.dumps({ + 'task_name': SOUND_PROCESSING_TASK_NAME, + 'sound_id': sound_id, + 'error': str(e), + 'work_time': round(time.time() - start_time) + }) + ) + sentry_sdk.capture_exception( + e + ) # Manually capture exception so it has mroe info and Sentry can organize it properly except Exception as e: try: @@ -323,10 +430,17 @@ def process_sound(sound_id, skip_previews=False, skip_displays=False): sound.change_processing_state("FA", processing_log=str(e)) except Sound.DoesNotExist: pass - workers_logger.info("Unexpected error while processing sound (%s)" % json.dumps( - {'task_name': SOUND_PROCESSING_TASK_NAME, 'sound_id': sound_id, 'error': str(e), - 'work_time': round(time.time() - start_time)})) - sentry_sdk.capture_exception(e) # Manually capture exception so it has mroe info and Sentry can organize it properly + workers_logger.info( + "Unexpected error while processing sound (%s)" % json.dumps({ + 'task_name': SOUND_PROCESSING_TASK_NAME, + 'sound_id': sound_id, + 'error': str(e), + 'work_time': round(time.time() - start_time) + }) + ) + sentry_sdk.capture_exception( + e + ) # Manually capture exception so it has mroe info and Sentry can organize it properly cancel_timeout_alarm() @@ -341,31 +455,59 @@ def process_before_description(audio_file_path): audio_file_path (str): path to the uploaded file """ set_timeout_alarm(settings.WORKER_TIMEOUT, f'Processing-before-describe of sound {audio_file_path} timed out') - workers_logger.info("Starting processing-before-describe of sound (%s)" % json.dumps({ - 'task_name': PROCESS_BEFORE_DESCRIPTION_TASK_NAME, 'audio_file_path': audio_file_path})) + workers_logger.info( + "Starting processing-before-describe of sound (%s)" % json.dumps({ + 'task_name': PROCESS_BEFORE_DESCRIPTION_TASK_NAME, + 'audio_file_path': audio_file_path + }) + ) start_time = time.time() try: check_if_free_space() result = FreesoundAudioProcessorBeforeDescription(audio_file_path=audio_file_path).process() if result: - workers_logger.info("Finished processing-before-describe of sound (%s)" % json.dumps( - {'task_name': PROCESS_BEFORE_DESCRIPTION_TASK_NAME, 'audio_file_path': audio_file_path, 'result': 'success', - 'work_time': round(time.time() - start_time)})) + workers_logger.info( + "Finished processing-before-describe of sound (%s)" % json.dumps({ + 'task_name': PROCESS_BEFORE_DESCRIPTION_TASK_NAME, + 'audio_file_path': audio_file_path, + 'result': 'success', + 'work_time': round(time.time() - start_time) + }) + ) else: - workers_logger.info("Finished processing-before-describe of sound (%s)" % json.dumps( - {'task_name': PROCESS_BEFORE_DESCRIPTION_TASK_NAME, 'audio_file_path': audio_file_path, 'result': 'failure', - 'work_time': round(time.time() - start_time)})) + workers_logger.info( + "Finished processing-before-describe of sound (%s)" % json.dumps({ + 'task_name': PROCESS_BEFORE_DESCRIPTION_TASK_NAME, + 'audio_file_path': audio_file_path, + 'result': 'failure', + 'work_time': round(time.time() - start_time) + }) + ) except WorkerException as e: - workers_logger.info("WorkerException while processing-before-describe sound (%s)" % json.dumps( - {'task_name': PROCESS_BEFORE_DESCRIPTION_TASK_NAME, 'audio_file_path': audio_file_path, 'error': str(e), - 'work_time': round(time.time() - start_time)})) - sentry_sdk.capture_exception(e) # Manually capture exception so it has mroe info and Sentry can organize it properly + workers_logger.info( + "WorkerException while processing-before-describe sound (%s)" % json.dumps({ + 'task_name': PROCESS_BEFORE_DESCRIPTION_TASK_NAME, + 'audio_file_path': audio_file_path, + 'error': str(e), + 'work_time': round(time.time() - start_time) + }) + ) + sentry_sdk.capture_exception( + e + ) # Manually capture exception so it has mroe info and Sentry can organize it properly except Exception as e: - workers_logger.info("Unexpected error while processing-before-describe sound (%s)" % json.dumps( - {'task_name': PROCESS_BEFORE_DESCRIPTION_TASK_NAME, 'audio_file_path': audio_file_path, 'error': str(e), - 'work_time': round(time.time() - start_time)})) - sentry_sdk.capture_exception(e) # Manually capture exception so it has mroe info and Sentry can organize it properly + workers_logger.info( + "Unexpected error while processing-before-describe sound (%s)" % json.dumps({ + 'task_name': PROCESS_BEFORE_DESCRIPTION_TASK_NAME, + 'audio_file_path': audio_file_path, + 'error': str(e), + 'work_time': round(time.time() - start_time) + }) + ) + sentry_sdk.capture_exception( + e + ) # Manually capture exception so it has mroe info and Sentry can organize it properly - cancel_timeout_alarm() \ No newline at end of file + cancel_timeout_alarm() diff --git a/general/templatetags/absurl.py b/general/templatetags/absurl.py index 85f17e43f..ac5967d3d 100644 --- a/general/templatetags/absurl.py +++ b/general/templatetags/absurl.py @@ -25,18 +25,24 @@ register = Library() + class AbsoluteURLNode(URLNode): + def render(self, context): path = super().render(context) domain = f"https://{Site.objects.get_current().domain}" return urllib.parse.urljoin(domain, path) + def absurl(parser, token, node_cls=AbsoluteURLNode): """Just like {% url %} but ads the domain of the current site.""" node_instance = url(parser, token) - return node_cls(view_name=node_instance.view_name, + return node_cls( + view_name=node_instance.view_name, args=node_instance.args, kwargs=node_instance.kwargs, - asvar=node_instance.asvar) + asvar=node_instance.asvar + ) + absurl = register.tag(absurl) diff --git a/general/templatetags/bw_templatetags.py b/general/templatetags/bw_templatetags.py index a63a7ce68..05e27d599 100644 --- a/general/templatetags/bw_templatetags.py +++ b/general/templatetags/bw_templatetags.py @@ -55,12 +55,14 @@ def bw_tag(tag_name, size=1, class_name="", url=None, weight=None): line_height_class = 'line-height-38' if size < 4 else 'line-height-fs-1' - return {'tag_name': tag_name, - 'size': size, - 'class_name': class_name, - 'line_height_class': line_height_class, - 'url': url, - 'opacity_class': opacity_class} + return { + 'tag_name': tag_name, + 'size': size, + 'class_name': class_name, + 'line_height_class': line_height_class, + 'url': url, + 'opacity_class': opacity_class + } @register.inclusion_tag('atoms/avatar.html') @@ -72,17 +74,19 @@ def bw_user_avatar(avatar_url, username, size=40, extra_class=''): decorator of the Profile model might return something different if user has no avatar. """ if len(username) > 1: - no_avatar_bg_color = settings.AVATAR_BG_COLORS[(ord(username[0]) + ord(username[1])) % len(settings.AVATAR_BG_COLORS)] + no_avatar_bg_color = settings.AVATAR_BG_COLORS[(ord(username[0]) + ord(username[1])) % + len(settings.AVATAR_BG_COLORS)] else: no_avatar_bg_color = settings.AVATAR_BG_COLORS[ord(username[0]) % len(settings.AVATAR_BG_COLORS)] return { 'size': size, - 'avatar_url':avatar_url, + 'avatar_url': avatar_url, 'username': username, 'font_size': int(size * 0.4), 'extra_class': extra_class, - 'no_avatar_bg_color': no_avatar_bg_color} + 'no_avatar_bg_color': no_avatar_bg_color + } @register.inclusion_tag('atoms/stars.html', takes_context=True) @@ -130,16 +134,18 @@ def bw_sound_stars(context, sound, allow_rating=True, use_request_user_rating=Fa else: stars_5.append('half') - return {'sound_user': sound_user, - 'allow_rating': allow_rating, - 'sound': sound, - 'sound_rating_0_5': sound_rating/2, - 'user_has_rated_this_sound': user_has_rated_this_sound, - 'has_min_ratings': has_min_ratings, - 'show_added_rating_on_save': show_added_rating_on_save, - 'use_request_user_rating': use_request_user_rating, - 'fill_class': 'text-red' if not use_request_user_rating else 'text-yellow', - 'stars_range': list(zip(stars_5, list(range(1, 6))))} + return { + 'sound_user': sound_user, + 'allow_rating': allow_rating, + 'sound': sound, + 'sound_rating_0_5': sound_rating / 2, + 'user_has_rated_this_sound': user_has_rated_this_sound, + 'has_min_ratings': has_min_ratings, + 'show_added_rating_on_save': show_added_rating_on_save, + 'use_request_user_rating': use_request_user_rating, + 'fill_class': 'text-red' if not use_request_user_rating else 'text-yellow', + 'stars_range': list(zip(stars_5, list(range(1, 6)))) + } @register.inclusion_tag('atoms/stars.html', takes_context=True) @@ -182,7 +188,7 @@ def bw_paginator(context, paginator, page, current_page, request, anchor="", non # If paginator object is None, don't go ahead as below calculations will fail. This can happen if show_paginator # is called and no paginator object is present in view return {} - + adjacent_pages = 3 total_wanted = adjacent_pages * 2 + 1 min_page_num = max(current_page - adjacent_pages, 1) @@ -200,8 +206,9 @@ def bw_paginator(context, paginator, page, current_page, request, anchor="", non # although paginator objects are 0-based, we use 1-based paging page_numbers = [n for n in range(min_page_num, max_page_num) if 0 < n <= paginator.num_pages] - params = urllib.parse.urlencode([(key.encode('utf-8'), value.encode('utf-8')) for (key, value) in request.GET.items() - if key.lower() != "page"]) + params = urllib.parse.urlencode([ + (key.encode('utf-8'), value.encode('utf-8')) for (key, value) in request.GET.items() if key.lower() != "page" + ]) if params == "": url = request.path + "?page=" @@ -212,15 +219,15 @@ def bw_paginator(context, paginator, page, current_page, request, anchor="", non # if it's the case a query to the DB or a dict if it's the case of a query to solr if isinstance(page, dict): url_prev_page = url + str(page['previous_page_number']) - url_next_page = url + str(page['next_page_number']) + url_next_page = url + str(page['next_page_number']) url_first_page = url + '1' else: url_prev_page = None if page.has_previous(): - url_prev_page = url + str(page.previous_page_number()) + url_prev_page = url + str(page.previous_page_number()) url_next_page = None if page.has_next(): - url_next_page = url + str(page.next_page_number()) + url_next_page = url + str(page.next_page_number()) url_first_page = url + '1' url_last_page = url + str(paginator.num_pages) @@ -237,7 +244,7 @@ def bw_paginator(context, paginator, page, current_page, request, anchor="", non "show_first": 1 not in page_numbers, "show_last": paginator.num_pages not in page_numbers, "last_is_next": last_is_next, - "url" : url, + "url": url, "url_prev_page": url_prev_page, "url_next_page": url_next_page, "url_first_page": url_first_page, @@ -273,7 +280,7 @@ def bw_intcomma(value): @register.inclusion_tag('molecules/carousel.html', takes_context=True) def sound_carousel(context, sounds, show_timesince=False): # Update context and pass it to templatetag so nested template tags also have it - context.update({'elements': sounds, 'type': 'sound', 'show_timesince': show_timesince}) + context.update({'elements': sounds, 'type': 'sound', 'show_timesince': show_timesince}) return context @@ -286,4 +293,4 @@ def sound_carousel_with_timesince(context, sounds): def pack_carousel(context, packs): # Update context and pass it to templatetag so nested template tags also have it context.update({'elements': packs, 'type': 'pack'}) - return context \ No newline at end of file + return context diff --git a/general/templatetags/filter_img.py b/general/templatetags/filter_img.py index bbec36ecd..c4350b41c 100644 --- a/general/templatetags/filter_img.py +++ b/general/templatetags/filter_img.py @@ -9,7 +9,7 @@ def replace_img(string): if not string or not " length: - return value[:length-3] + "..." + return value[:length - 3] + "..." else: return value @@ -68,7 +68,7 @@ def formatnumber(number): @register.filter -def in_list(value,arg): +def in_list(value, arg): return value in arg diff --git a/general/tests.py b/general/tests.py index d0493aa5e..bdaaba6c0 100644 --- a/general/tests.py +++ b/general/tests.py @@ -43,18 +43,18 @@ def test_report_count_statuses(self): sound.change_processing_state("OK") sound.change_moderation_state("OK") SoundRating.objects.create(sound=sound, user=user, rating=4) - sound.refresh_from_db() # Refresh from db after methods that use F-expressions + sound.refresh_from_db() # Refresh from db after methods that use F-expressions sound.add_comment(user=user, comment="testComment") - sound.refresh_from_db() # Refresh from db after methods that use F-expressions + sound.refresh_from_db() # Refresh from db after methods that use F-expressions forum = Forum.objects.create(name="testForum", name_slug="test_forum", description="test") thread = Thread.objects.create(forum=forum, title="testThread", author=user) Post.objects.create(author=user, body="testBody", thread=thread) Post.objects.create(author=user, body="testBody unnmoderated", thread=thread, moderation_state="NM") - user.profile.refresh_from_db() # Refresh from db after methods that use F-expressions + user.profile.refresh_from_db() # Refresh from db after methods that use F-expressions # Assert initial counts are ok self.assertEqual(user.profile.num_sounds, 1) - self.assertEqual(user.profile.num_posts, 1) # Note that count is 1 because one of the posts is not moderated + self.assertEqual(user.profile.num_posts, 1) # Note that count is 1 because one of the posts is not moderated self.assertEqual(pack.num_sounds, 1) self.assertEqual(pack.num_downloads, 0) self.assertEqual(sound.num_ratings, 1) @@ -106,7 +106,7 @@ def test_report_count_statuses(self): sound.refresh_from_db() pack.refresh_from_db() self.assertEqual(user.profile.num_sounds, 1) - self.assertEqual(user.profile.num_posts, 1) # Note this is still 1 as unmoderated posts do not count + self.assertEqual(user.profile.num_posts, 1) # Note this is still 1 as unmoderated posts do not count self.assertEqual(pack.num_sounds, 1) self.assertNotEqual(pack.num_downloads, 0) self.assertEqual(sound.num_ratings, 1) @@ -140,10 +140,12 @@ def test_url_with_non_ascii_characters(self): or values, paginator does not break. """ text_with_non_ascii = '�textèé' - dummy_request = RequestFactory().get(reverse('sounds'), { - text_with_non_ascii: '1', - 'param_name': text_with_non_ascii, - 'param2_name': 'ok_value', - }) + dummy_request = RequestFactory().get( + reverse('sounds'), { + text_with_non_ascii: '1', + 'param_name': text_with_non_ascii, + 'param2_name': 'ok_value', + } + ) paginator = paginate(dummy_request, Sound.objects.all(), 10) bw_paginator({}, paginator['paginator'], paginator['page'], paginator['current_page'], dummy_request) diff --git a/geotags/management/commands/retrieve_geotag_names.py b/geotags/management/commands/retrieve_geotag_names.py index a1beb74c8..0448e0a64 100644 --- a/geotags/management/commands/retrieve_geotag_names.py +++ b/geotags/management/commands/retrieve_geotag_names.py @@ -34,11 +34,8 @@ class Command(LoggingBaseCommand): def add_arguments(self, parser): parser.add_argument( - '-l', '--limit', - action='store', - dest='limit', - default=5000, - help='Maximum number of geotags to update') + '-l', '--limit', action='store', dest='limit', default=5000, help='Maximum number of geotags to update' + ) def handle(self, *args, **options): self.log_start() diff --git a/geotags/models.py b/geotags/models.py index a81272bd8..b73137ffa 100644 --- a/geotags/models.py +++ b/geotags/models.py @@ -59,21 +59,24 @@ def retrieve_location_information(self): self.save() except Exception as e: pass - + if self.information is not None: features = self.information.get('features', []) if features: try: # Try with "place" feature - self.location_name = [feature for feature in features if 'place' in feature['place_type']][0]['place_name'] + self.location_name = [feature for feature in features if 'place' in feature['place_type'] + ][0]['place_name'] except IndexError: # If "place" feature is not avialable, use "locality" feature try: - self.location_name = [feature for feature in features if 'locality' in feature['place_type']][0]['place_name'] + self.location_name = [feature for feature in features if 'locality' in feature['place_type'] + ][0]['place_name'] except IndexError: # If "place" nor "locality" features are avialable, use "region" try: - self.location_name = [feature for feature in features if 'region' in feature['place_type']][0]['place_name'] + self.location_name = [feature for feature in features if 'region' in feature['place_type'] + ][0]['place_name'] except: # It is not possible to derive a name... pass diff --git a/geotags/tests.py b/geotags/tests.py index 53c0fb7c4..f56d3e696 100644 --- a/geotags/tests.py +++ b/geotags/tests.py @@ -47,8 +47,15 @@ def test_browse_geotags_box(self): def test_geotags_box_iframe(self): resp = self.client.get(reverse('embed-geotags-box-iframe')) - check_values = {'m_width': 942, 'm_height': 600, 'cluster': True, 'center_lat': None, 'center_lon': None, - 'zoom': None, 'username': None} + check_values = { + 'm_width': 942, + 'm_height': 600, + 'cluster': True, + 'center_lat': None, + 'center_lon': None, + 'zoom': None, + 'username': None + } self.check_context(resp.context, check_values) def test_browse_geotags_for_user(self): diff --git a/geotags/urls.py b/geotags/urls.py index 2778b48a8..05a3f6e64 100644 --- a/geotags/urls.py +++ b/geotags/urls.py @@ -23,7 +23,11 @@ urlpatterns = [ path('sounds_barray/user//', geotags.geotags_for_user_barray, name="geotags-for-user-barray"), - path('sounds_barray/user_latest//', geotags.geotags_for_user_latest_barray, name="geotags-for-user-latest-barray"), + path( + 'sounds_barray/user_latest//', + geotags.geotags_for_user_latest_barray, + name="geotags-for-user-latest-barray" + ), path('sounds_barray/pack//', geotags.geotags_for_pack_barray, name="geotags-for-pack-barray"), path('sounds_barray/sound//', geotags.geotag_for_sound_barray, name="geotags-for-sound-barray"), re_path(r'^sounds_barray/(?P[\w-]+)?/?$', geotags.geotags_barray, name="geotags-barray"), diff --git a/geotags/views.py b/geotags/views.py index 4df4ab535..f7b46a785 100644 --- a/geotags/views.py +++ b/geotags/views.py @@ -40,8 +40,13 @@ def log_map_load(map_type, num_geotags, request): - web_logger.info('Map load (%s)' % json.dumps({ - 'map_type': map_type, 'num_geotags': num_geotags, 'ip': get_client_ip(request)})) + web_logger.info( + 'Map load (%s)' % json.dumps({ + 'map_type': map_type, + 'num_geotags': num_geotags, + 'ip': get_client_ip(request) + }) + ) def generate_bytearray(sound_queryset): @@ -51,8 +56,8 @@ def generate_bytearray(sound_queryset): for s in sound_queryset: if not math.isnan(s.geotag.lat) and not math.isnan(s.geotag.lon): packed_sounds.write(struct.pack("i", s.id)) - packed_sounds.write(struct.pack("i", int(s.geotag.lat*1000000))) - packed_sounds.write(struct.pack("i", int(s.geotag.lon*1000000))) + packed_sounds.write(struct.pack("i", int(s.geotag.lat * 1000000))) + packed_sounds.write(struct.pack("i", int(s.geotag.lon * 1000000))) num_sounds_in_bytearray += 1 return packed_sounds.getvalue(), num_sounds_in_bytearray @@ -82,7 +87,9 @@ def geotags_box_barray(request): is_embed = request.GET.get("embed", "0") == "1" try: min_lat, min_lon, max_lat, max_lon = box.split(",") - qs = Sound.objects.select_related("geotag").exclude(geotag=None).filter(moderation_state="OK", processing_state="OK") + qs = Sound.objects.select_related("geotag").exclude(geotag=None).filter( + moderation_state="OK", processing_state="OK" + ) sounds = [] if min_lat <= max_lat and min_lon <= max_lon: sounds = qs.filter(geotag__lat__range=(min_lat, max_lat)).filter(geotag__lon__range=(min_lon, max_lon)) @@ -189,8 +196,7 @@ def for_user(request, username): @redirect_if_old_username_or_404 def for_sound(request, username, sound_id): - sound = get_object_or_404( - Sound.objects.select_related('geotag', 'user'), id=sound_id) + sound = get_object_or_404(Sound.objects.select_related('geotag', 'user'), id=sound_id) if sound.user.username.lower() != username.lower() or sound.geotag is None: raise Http404 tvars = _get_geotags_query_params(request) @@ -259,10 +265,7 @@ def infowindow(request, sound_id): sound = Sound.objects.select_related('user', 'geotag').get(id=sound_id) except Sound.DoesNotExist: raise Http404 - tvars = { - 'sound': sound, - 'minimal': request.GET.get('minimal', False) - } + tvars = {'sound': sound, 'minimal': request.GET.get('minimal', False)} if request.GET.get('embed', False): # When loading infowindow for an embed, use the template for old UI as embeds have not been updated to new UI return render(request, 'embeds/geotags_infowindow.html', tvars) diff --git a/manage.py b/manage.py index 2251ba02c..6e54abd69 100755 --- a/manage.py +++ b/manage.py @@ -10,10 +10,11 @@ if '--settings=freesound.test_settings' not in sys.argv: from django.conf import settings if settings.DEBUG: - if (os.environ.get('RUN_MAIN') or os.environ.get('WERKZEUG_RUN_MAIN')) and not os.environ.get('DISABLE_DEBUGPY'): + if (os.environ.get('RUN_MAIN') + or os.environ.get('WERKZEUG_RUN_MAIN')) and not os.environ.get('DISABLE_DEBUGPY'): import debugpy debugpy.listen((settings.DEBUGGER_HOST, settings.DEBUGGER_PORT)) print(f'Debugger ready and listening at http://{settings.DEBUGGER_HOST}:{settings.DEBUGGER_PORT}/') - + from django.core.management import execute_from_command_line execute_from_command_line(sys.argv) diff --git a/messages/admin.py b/messages/admin.py index 78063ae6e..69efd75f6 100644 --- a/messages/admin.py +++ b/messages/admin.py @@ -21,12 +21,29 @@ from django.contrib import admin from messages.models import Message, MessageBody + @admin.register(Message) class MessageAdmin(admin.ModelAdmin): raw_id_fields = ('user_from', 'user_to', 'body') - list_display = ('user_from', 'user_to', 'subject', 'is_sent', 'is_read', 'is_archived', 'created', ) - search_fields = ('=user_from__username', '=user_to__username', 'subject',) - list_filter = ('is_sent', 'is_read', 'is_archived', ) + list_display = ( + 'user_from', + 'user_to', + 'subject', + 'is_sent', + 'is_read', + 'is_archived', + 'created', + ) + search_fields = ( + '=user_from__username', + '=user_to__username', + 'subject', + ) + list_filter = ( + 'is_sent', + 'is_read', + 'is_archived', + ) -admin.site.register(MessageBody) \ No newline at end of file +admin.site.register(MessageBody) diff --git a/messages/apps.py b/messages/apps.py index c2df1ffc5..4f91d7c95 100644 --- a/messages/apps.py +++ b/messages/apps.py @@ -1,6 +1,7 @@ from django.apps import AppConfig + class MessagesConfig(AppConfig): - name = 'messages' - label = 'fsmessages' - verbose_name = "Messages" + name = 'messages' + label = 'fsmessages' + verbose_name = "Messages" diff --git a/messages/forms.py b/messages/forms.py index 6241cff4a..7088beeda 100644 --- a/messages/forms.py +++ b/messages/forms.py @@ -28,28 +28,30 @@ class ManualUserField(forms.CharField): + def clean(self, value): if not value: raise forms.ValidationError('Please enter a username.') try: return User.objects.get(username__iexact=value) - except User.DoesNotExist: # @UndefinedVariable + except User.DoesNotExist: # @UndefinedVariable raise forms.ValidationError("We are sorry, but this username does not exist...") class MessageReplyForm(forms.Form): to = ManualUserField(widget=forms.TextInput(attrs={'size': '40'})) subject = forms.CharField(min_length=3, max_length=128, widget=forms.TextInput(attrs={'size': '80'})) - body = HtmlCleaningCharField(widget=forms.Textarea(attrs=dict(cols=100, rows=30)), - help_text=HtmlCleaningCharField.make_help_text()) + body = HtmlCleaningCharField( + widget=forms.Textarea(attrs=dict(cols=100, rows=30)), help_text=HtmlCleaningCharField.make_help_text() + ) def __init__(self, request, *args, **kwargs): - self.request = request # This is used by MessageReplyFormWithCaptcha to be able to call is_spam function + self.request = request # This is used by MessageReplyFormWithCaptcha to be able to call is_spam function kwargs.update(dict(label_suffix='')) super().__init__(*args, **kwargs) self.fields['to'].widget.attrs['placeholder'] = "Username of the user to send the message to" - self.fields['to'].widget.attrs['data-typeahead'] = 'true' + self.fields['to'].widget.attrs['data-typeahead'] = 'true' self.fields['to'].widget.attrs['data-typeahead-suggestions-url'] = reverse('messages-username_lookup') self.fields['to'].widget.attrs['data-check-username-url'] = reverse('check_username') self.fields['to'].widget.attrs['id'] = "username-to-field" @@ -67,6 +69,8 @@ class MessageReplyFormWithCaptcha(MessageReplyForm): def clean_body(self): body = self.cleaned_data['body'] if is_spam(self.request, body): - raise forms.ValidationError("Your message was considered spam. If your message is not spam and the " - "check keeps failing, please contact the admins.") + raise forms.ValidationError( + "Your message was considered spam. If your message is not spam and the " + "check keeps failing, please contact the admins." + ) return body diff --git a/messages/models.py b/messages/models.py index f85e8a957..623b3764d 100644 --- a/messages/models.py +++ b/messages/models.py @@ -84,22 +84,22 @@ class Message(models.Model): user_from = models.ForeignKey(User, related_name='messages_sent', on_delete=models.CASCADE) user_to = models.ForeignKey(User, related_name='messages_received', on_delete=models.CASCADE) - + subject = models.CharField(max_length=128) - + body = models.ForeignKey(MessageBody, on_delete=models.CASCADE) is_sent = models.BooleanField(default=True, db_index=True) is_read = models.BooleanField(default=False, db_index=True) is_archived = models.BooleanField(default=False, db_index=True) - + created = models.DateTimeField(db_index=True, auto_now_add=True) - + def get_absolute_url(self): return "message", (smart_str(self.id),) def __str__(self): return f"from: [{self.user_from}] to: [{self.user_to}]" - + class Meta: ordering = ('-created',) diff --git a/messages/templatetags/display_message.py b/messages/templatetags/display_message.py index 9d71506f2..fba2f8109 100644 --- a/messages/templatetags/display_message.py +++ b/messages/templatetags/display_message.py @@ -18,7 +18,6 @@ # See AUTHORS file. # - from django import template from messages.models import Message diff --git a/messages/tests/test_message_notifications.py b/messages/tests/test_message_notifications.py index bdc8baeb9..75502b0e7 100644 --- a/messages/tests/test_message_notifications.py +++ b/messages/tests/test_message_notifications.py @@ -42,11 +42,14 @@ def setUp(self): @mock.patch("captcha.fields.ReCaptchaField.validate") def test_message_email_preference_enabled(self, magic_mock): self.client.force_login(user=self.sender) - resp = self.client.post(reverse('messages-new'), data={ - 'body': ['test message body'], - 'to': ['receiver'], - 'subject': ['test message'], - }) + resp = self.client.post( + reverse('messages-new'), + data={ + 'body': ['test message body'], + 'to': ['receiver'], + 'subject': ['test message'], + } + ) self.assertRedirects(resp, reverse('messages')) self.assertEqual(len(mail.outbox), 1) self.assertTrue(settings.EMAIL_SUBJECT_PREFIX in mail.outbox[0].subject) @@ -60,10 +63,13 @@ def test_message_email_preference_disabled(self, magic_mock): UserEmailSetting.objects.create(user=self.receiver, email_type=email_pref) self.client.force_login(user=self.sender) - resp = self.client.post(reverse('messages-new'), data={ - 'body': ['test message body'], - 'to': ['receiver'], - 'subject': ['test message'], - }) + resp = self.client.post( + reverse('messages-new'), + data={ + 'body': ['test message body'], + 'to': ['receiver'], + 'subject': ['test message'], + } + ) self.assertRedirects(resp, reverse('messages')) self.assertEqual(len(mail.outbox), 0) diff --git a/messages/tests/test_message_write.py b/messages/tests/test_message_write.py index 5b02c0d81..3ae942098 100644 --- a/messages/tests/test_message_write.py +++ b/messages/tests/test_message_write.py @@ -37,11 +37,12 @@ class RecaptchaPresenceInMessageForms(TestCase): def setUp(self): # Create one user which is a potential spammer and one which is not self.no_spammer = User.objects.create_user(username='noSpammer', email='noSpammer@example.com') - self.no_spammer.profile.num_sounds = 4 # Having sounds will make user "trustable" + self.no_spammer.profile.num_sounds = 4 # Having sounds will make user "trustable" self.no_spammer.profile.save() self.potential_spammer = User.objects.create_user( - username='potentialSpammer', email='potentialSpammer@example.com') + username='potentialSpammer', email='potentialSpammer@example.com' + ) self.message_sender = User.objects.create_user(username='sender', email='sender@example.com') @@ -61,16 +62,28 @@ def test_captcha_presence_in_reply_message_form(self): # Test non spammer does not see recaptcha field in reply form message = Message.objects.create( - user_from=self.message_sender, user_to=self.no_spammer, subject='Message subject', - body=MessageBody.objects.create(body='Message body'), is_sent=True, is_archived=False, is_read=False) + user_from=self.message_sender, + user_to=self.no_spammer, + subject='Message subject', + body=MessageBody.objects.create(body='Message body'), + is_sent=True, + is_archived=False, + is_read=False + ) self.client.force_login(user=self.no_spammer) resp = self.client.get(reverse('messages-new', args=[message.id])) self.assertNotContains(resp, 'recaptcha') # Potential spammer (has no uploaded sounds), recaptcha field should be shown message = Message.objects.create( - user_from=self.message_sender, user_to=self.potential_spammer, subject='Message subject', - body=MessageBody.objects.create(body='Message body'), is_sent=True, is_archived=False, is_read=False) + user_from=self.message_sender, + user_to=self.potential_spammer, + subject='Message subject', + body=MessageBody.objects.create(body='Message body'), + is_sent=True, + is_archived=False, + is_read=False + ) self.client.force_login(user=self.potential_spammer) resp = self.client.get(reverse('messages-new', args=[message.id])) self.assertContains(resp, 'recaptcha') @@ -93,15 +106,25 @@ def setUp(self): for count, receiver in enumerate([self.receiver1, self.receiver2, self.receiver3]): for _ in range(0, count + 1): Message.objects.create( - user_from=self.sender, user_to=receiver, subject='Message subject', + user_from=self.sender, + user_to=receiver, + subject='Message subject', body=MessageBody.objects.create(body='Message body'), - is_sent=True, is_archived=False, is_read=False) + is_sent=True, + is_archived=False, + is_read=False + ) # Send one message from sender2 to sender1 Message.objects.create( - user_from=self.sender2, user_to=self.sender, subject='Message subject', + user_from=self.sender2, + user_to=self.sender, + subject='Message subject', body=MessageBody.objects.create(body='Message body'), - is_sent=True, is_archived=False, is_read=False) + is_sent=True, + is_archived=False, + is_read=False + ) def test_username_lookup_num_queries(self): # Check that username lookup view only makes 1 query @@ -111,9 +134,10 @@ def test_username_lookup_num_queries(self): def test_get_previously_contacted_usernames(self): # Check get_previously_contacted_usernames helper function returns userames of users previously contacted by # the sender or users who previously contacted the sender - self.assertCountEqual([self.receiver3.username, self.receiver2.username, self.receiver1.username, - self.sender2.username, self.sender.username], - get_previously_contacted_usernames(self.sender)) + self.assertCountEqual([ + self.receiver3.username, self.receiver2.username, self.receiver1.username, self.sender2.username, + self.sender.username + ], get_previously_contacted_usernames(self.sender)) def test_username_lookup_response(self): # Check username lookup view returns userames of users previously contacted by the sender or users who @@ -123,10 +147,11 @@ def test_username_lookup_response(self): response_json = json.loads(resp.content) self.assertEqual(resp.status_code, 200) self.assertCountEqual([self.receiver3.username, self.receiver2.username, self.receiver1.username], - response_json) + response_json) class QuoteMessageTestCase(TestCase): + def test_oneline(self): body = "This is a message" username = "testuser" @@ -143,7 +168,6 @@ def test_manylines(self): expected = "> --- testuser wrote:\n>\n> This is a message\n> with multiple lines" self.assertEqual(new_body, expected) - def test_alreadyquoted(self): body = "This is a message\n> with already quoted lines" username = "testuser" diff --git a/messages/views.py b/messages/views.py index f090ef029..456534bf1 100644 --- a/messages/views.py +++ b/messages/views.py @@ -29,7 +29,7 @@ from django.db.models import Q from django.http import HttpResponse from django.http import HttpResponseRedirect, Http404 -from django.shortcuts import render +from django.shortcuts import render from django.urls import reverse from messages.forms import MessageReplyForm, MessageReplyFormWithCaptcha @@ -45,8 +45,9 @@ def messages_change_state(request): choice = request.POST.get("choice", False) message_ids = [int(mid) for mid in request.POST.get("ids", "").split(',')] if choice and message_ids: - fs_messages = Message.objects.filter(Q(user_to=request.user, is_sent=False) | - Q(user_from=request.user, is_sent=True)).filter(id__in=message_ids) + fs_messages = Message.objects.filter( + Q(user_to=request.user, is_sent=False) | Q(user_from=request.user, is_sent=True) + ).filter(id__in=message_ids) if choice == "a": for message in fs_messages: message.is_archived = not message.is_archived @@ -68,24 +69,16 @@ def messages_change_state(request): @login_required def inbox(request): qs = base_qs.filter(user_to=request.user, is_archived=False, is_sent=False) - tvars = {'list_type': 'inbox'} - tvars.update(paginate( - request, qs, - items_per_page=settings.MESSAGES_PER_PAGE)) + tvars = {'list_type': 'inbox'} + tvars.update(paginate(request, qs, items_per_page=settings.MESSAGES_PER_PAGE)) return render(request, 'messages/inbox.html', tvars) @login_required def sent_messages(request): qs = base_qs.filter(user_from=request.user, is_archived=False, is_sent=True) - tvars = { - 'list_type': 'sent', - 'hide_toggle_read_unread': True, - 'hide_archive_unarchive': True - } - tvars.update(paginate( - request, qs, - items_per_page=settings.MESSAGES_PER_PAGE)) + tvars = {'list_type': 'sent', 'hide_toggle_read_unread': True, 'hide_archive_unarchive': True} + tvars.update(paginate(request, qs, items_per_page=settings.MESSAGES_PER_PAGE)) return render(request, 'messages/sent.html', tvars) @@ -93,9 +86,7 @@ def sent_messages(request): def archived_messages(request): qs = base_qs.filter(user_to=request.user, is_archived=True, is_sent=False) tvars = {'list_type': 'archived'} - tvars.update(paginate( - request, qs, - items_per_page=settings.MESSAGES_PER_PAGE)) + tvars.update(paginate(request, qs, items_per_page=settings.MESSAGES_PER_PAGE)) return render(request, 'messages/archived.html', tvars) @@ -127,13 +118,15 @@ def new_message(request, username=None, message_id=None): form_class = MessageReplyForm else: form_class = MessageReplyFormWithCaptcha - + if request.method == 'POST': form = form_class(request, request.POST) if request.user.profile.is_blocked_for_spam_reports(): - messages.add_message(request, messages.INFO, "You're not allowed to send the message because your account " - "has been temporally blocked after multiple spam reports") + messages.add_message( + request, messages.INFO, "You're not allowed to send the message because your account " + "has been temporally blocked after multiple spam reports" + ) else: if form.is_valid(): user_from = request.user @@ -141,19 +134,37 @@ def new_message(request, username=None, message_id=None): subject = form.cleaned_data["subject"] body = MessageBody.objects.create(body=form.cleaned_data["body"]) - Message.objects.create(user_from=user_from, user_to=user_to, subject=subject, body=body, is_sent=True, - is_archived=False, is_read=False) - Message.objects.create(user_from=user_from, user_to=user_to, subject=subject, body=body, is_sent=False, - is_archived=False, is_read=False) + Message.objects.create( + user_from=user_from, + user_to=user_to, + subject=subject, + body=body, + is_sent=True, + is_archived=False, + is_read=False + ) + Message.objects.create( + user_from=user_from, + user_to=user_to, + subject=subject, + body=body, + is_sent=False, + is_archived=False, + is_read=False + ) invalidate_user_template_caches(user_to.id) try: # send the user an email to notify him of the sent message! - tvars = {'user_to': user_to, - 'user_from': user_from} - send_mail_template(settings.EMAIL_SUBJECT_PRIVATE_MESSAGE, 'emails/email_new_message.txt', tvars, - user_to=user_to, email_type_preference_check="private_message") + tvars = {'user_to': user_to, 'user_from': user_from} + send_mail_template( + settings.EMAIL_SUBJECT_PRIVATE_MESSAGE, + 'emails/email_new_message.txt', + tvars, + user_to=user_to, + email_type_preference_check="private_message" + ) except: # if the email sending fails, ignore... pass @@ -166,7 +177,7 @@ def new_message(request, username=None, message_id=None): if message.user_from != request.user and message.user_to != request.user: raise Http404 - + body = message.body.body.replace("\r\n", "\n").replace("\r", "\n") body = quote_message_for_reply(body, message.user_from.username) @@ -187,17 +198,20 @@ def new_message(request, username=None, message_id=None): def quote_message_for_reply(body, username): body = ''.join(BeautifulSoup(body, "html.parser").find_all(string=True)) - body = "\n".join([(">" if line.startswith(">") else "> ") + "\n> ".join(wrap(line.strip(), 60)) - for line in body.split("\n")]) + body = "\n".join([ + (">" if line.startswith(">") else "> ") + "\n> ".join(wrap(line.strip(), 60)) for line in body.split("\n") + ]) body = "> --- " + username + " wrote:\n>\n" + body return body def get_previously_contacted_usernames(user): # Get a list of previously contacted usernames (in no particular order) - usernames = list(Message.objects.select_related('user_from', 'user_to') - .filter(Q(user_from=user) | Q(user_to=user)) - .values_list('user_to__username', 'user_from__username')) + usernames = list( + Message.objects.select_related('user_from', + 'user_to').filter(Q(user_from=user) | Q(user_to=user) + ).values_list('user_to__username', 'user_from__username') + ) return list({item for sublist in usernames for item in sublist}) diff --git a/monitor/management/commands/generate_stats.py b/monitor/management/commands/generate_stats.py index 0c3de82f7..2329d5921 100644 --- a/monitor/management/commands/generate_stats.py +++ b/monitor/management/commands/generate_stats.py @@ -40,7 +40,7 @@ class Command(LoggingBaseCommand): def handle(self, **options): self.log_start() - time_span = datetime.datetime.now()-datetime.timedelta(weeks=2) + time_span = datetime.datetime.now() - datetime.timedelta(weeks=2) # Compute stats relatad with sounds: @@ -54,10 +54,7 @@ def handle(self, **options): .extra(select={'day': 'date(processing_date)'}).values('day')\ .order_by().annotate(Count('id')) - sounds_stats = { - "new_sounds_mod": list(new_sounds_mod), - "new_sounds": list(new_sounds) - } + sounds_stats = {"new_sounds_mod": list(new_sounds_mod), "new_sounds": list(new_sounds)} cache.set("sounds_stats", sounds_stats, 60 * 60 * 24) # Compute stats related with downloads: @@ -85,15 +82,33 @@ def handle(self, **options): cache.set("users_stats", {"new_users": list(new_users)}, 60 * 60 * 24) - time_span = datetime.datetime.now()-datetime.timedelta(days=365) + time_span = datetime.datetime.now() - datetime.timedelta(days=365) active_users = { - 'sounds': {'obj': sounds.models.Sound.objects, 'attr': 'user_id'}, - 'comments': {'obj': comments.models.Comment.objects, 'attr': 'user_id'}, - 'posts': {'obj': forum.models.Post.objects, 'attr': 'author_id'}, - 'sound_downloads': {'obj': sounds.models.Download.objects, 'attr': 'user_id'}, - 'pack_downloads': {'obj': sounds.models.PackDownload.objects, 'attr': 'user_id'}, - 'rate': {'obj': ratings.models.SoundRating.objects, 'attr': 'user_id'}, + 'sounds': { + 'obj': sounds.models.Sound.objects, + 'attr': 'user_id' + }, + 'comments': { + 'obj': comments.models.Comment.objects, + 'attr': 'user_id' + }, + 'posts': { + 'obj': forum.models.Post.objects, + 'attr': 'author_id' + }, + 'sound_downloads': { + 'obj': sounds.models.Download.objects, + 'attr': 'user_id' + }, + 'pack_downloads': { + 'obj': sounds.models.PackDownload.objects, + 'attr': 'user_id' + }, + 'rate': { + 'obj': ratings.models.SoundRating.objects, + 'attr': 'user_id' + }, } for i in active_users.keys(): qq = active_users[i]['obj'].filter(created__gt=time_span)\ @@ -102,8 +117,8 @@ def handle(self, **options): .annotate(Count(active_users[i]['attr'], distinct=True)) converted_weeks = [{ - 'week': str(datetime.datetime.strptime(d['week']+ '-0', "%W-%Y-%w").date()), - 'amount__sum': d[active_users[i]['attr']+'__count'] + 'week': str(datetime.datetime.strptime(d['week'] + '-0', "%W-%Y-%w").date()), + 'amount__sum': d[active_users[i]['attr'] + '__count'] } for d in qq] active_users[i] = converted_weeks @@ -115,10 +130,10 @@ def handle(self, **options): .extra({'day': 'date(created)'}).values('day').order_by()\ .annotate(Sum('amount')) - cache.set('donations_stats', {'new_donations': list(query_donations)}, 60*60*24) + cache.set('donations_stats', {'new_donations': list(query_donations)}, 60 * 60 * 24) # Compute stats related with Tags: - time_span = datetime.datetime.now()-datetime.timedelta(weeks=2) + time_span = datetime.datetime.now() - datetime.timedelta(weeks=2) tags_stats = TaggedItem.objects.values('tag_id')\ .filter(created__gt=time_span).annotate(num=Count('tag_id'))\ @@ -145,7 +160,7 @@ def handle(self, **options): "downloads_tags": list(downloads_tags) } - cache.set('tags_stats', tags_stats, 60*60*24) + cache.set('tags_stats', tags_stats, 60 * 60 * 24) # Compute stats for Totals table: @@ -155,12 +170,11 @@ def handle(self, **options): num_donations = donations.models.Donation.objects\ .aggregate(Sum('amount'))['amount__sum'] - time_span = datetime.datetime.now()-datetime.timedelta(30) + time_span = datetime.datetime.now() - datetime.timedelta(30) sum_donations_month = donations.models.Donation.objects\ .filter(created__gt=time_span).aggregate(Sum('amount'))['amount__sum'] - num_sounds = sounds.models.Sound.objects.filter(processing_state="OK", - moderation_state="OK").count() + num_sounds = sounds.models.Sound.objects.filter(processing_state="OK", moderation_state="OK").count() packs = sounds.models.Pack.objects.all().count() downloads_sounds = sounds.models.Download.objects.count() @@ -191,5 +205,5 @@ def handle(self, **options): "threads": threads, } - cache.set('totals_stats', totals_stats, 60*60*24) + cache.set('totals_stats', totals_stats, 60 * 60 * 24) self.log_end() diff --git a/monitor/tests.py b/monitor/tests.py index 3723d9bcd..bbcb7a986 100644 --- a/monitor/tests.py +++ b/monitor/tests.py @@ -17,7 +17,9 @@ def test_monitor_queries_stats_ajax_error(self, mock_get): resp = self.client.get(reverse('monitor-queries-stats-ajax')) self.assertEqual(resp.status_code, 500) - mock_get.assert_called_with('http://graylog/graylog/api/search/universal/relative/terms', auth=mock.ANY, params=mock.ANY) + mock_get.assert_called_with( + 'http://graylog/graylog/api/search/universal/relative/terms', auth=mock.ANY, params=mock.ANY + ) @override_settings(GRAYLOG_DOMAIN='http://graylog') @mock.patch('requests.get') @@ -29,7 +31,9 @@ def test_monitor_queries_stats_ajax_bad_data(self, mock_get): resp = self.client.get(reverse('monitor-queries-stats-ajax')) self.assertEqual(resp.status_code, 500) - mock_get.assert_called_with('http://graylog/graylog/api/search/universal/relative/terms', auth=mock.ANY, params=mock.ANY) + mock_get.assert_called_with( + 'http://graylog/graylog/api/search/universal/relative/terms', auth=mock.ANY, params=mock.ANY + ) @override_settings(GRAYLOG_DOMAIN='http://graylog') @mock.patch('requests.get') @@ -43,4 +47,6 @@ def test_monitor_queries_stats_ajax_ok(self, mock_get): self.assertEqual(resp.status_code, 200) self.assertJSONEqual(resp.content, {'response': 'ok'}) - mock_get.assert_called_with('http://graylog/graylog/api/search/universal/relative/terms', auth=mock.ANY, params=mock.ANY) + mock_get.assert_called_with( + 'http://graylog/graylog/api/search/universal/relative/terms', auth=mock.ANY, params=mock.ANY + ) diff --git a/monitor/urls.py b/monitor/urls.py index 3e0b7b160..4fbb34dae 100644 --- a/monitor/urls.py +++ b/monitor/urls.py @@ -22,10 +22,7 @@ import monitor.views urlpatterns = [ - path('', monitor.views.monitor_home, name='monitor-home'), - - path('stats/', monitor.views.monitor_stats, name='monitor-stats'), path('analysis/', monitor.views.monitor_analysis, name='monitor-analysis'), path('processing/', monitor.views.monitor_processing, name='monitor-processing'), @@ -42,5 +39,4 @@ path('ajax_users_stats/', monitor.views.users_stats_ajax, name='monitor-users-stats-ajax'), path('ajax_active_users_stats/', monitor.views.active_users_stats_ajax, name='monitor-active-users-stats-ajax'), path('ajax_moderator_stats/', monitor.views.moderator_stats_ajax, name='monitor-moderator-stats-ajax'), - ] diff --git a/monitor/views.py b/monitor/views.py index 81674c1f6..3903062b6 100644 --- a/monitor/views.py +++ b/monitor/views.py @@ -44,8 +44,7 @@ def get_queues_status(request): celery_task_counts = get_queues_task_counts() except Exception: celery_task_counts = [] - return render(request, 'monitor/queues_status.html', - {'celery_task_counts': celery_task_counts}) + return render(request, 'monitor/queues_status.html', {'celery_task_counts': celery_task_counts}) @login_required @@ -58,19 +57,15 @@ def monitor_home(request): @user_passes_test(lambda u: u.is_staff, login_url='/') def monitor_processing(request): # Processing - sounds_queued_count = Sound.objects.filter( - processing_ongoing_state='QU').count() + sounds_queued_count = Sound.objects.filter(processing_ongoing_state='QU').count() sounds_pending_count = Sound.objects.\ filter(processing_state='PE')\ .exclude(processing_ongoing_state='PR')\ .exclude(processing_ongoing_state='QU')\ .count() - sounds_processing_count = Sound.objects.filter( - processing_ongoing_state='PR').count() - sounds_failed_count = Sound.objects.filter( - processing_state='FA').count() - sounds_ok_count = Sound.objects.filter( - processing_state='OK').count() + sounds_processing_count = Sound.objects.filter(processing_ongoing_state='PR').count() + sounds_failed_count = Sound.objects.filter(processing_state='FA').count() + sounds_ok_count = Sound.objects.filter(processing_state='OK').count() tvars = { "sounds_queued_count": sounds_queued_count, "sounds_pending_count": sounds_pending_count, @@ -82,11 +77,12 @@ def monitor_processing(request): } return render(request, 'monitor/processing.html', tvars) + @login_required @user_passes_test(lambda u: u.is_staff, login_url='/') def monitor_analysis(request): # Analysis - analyzers_data = {} + analyzers_data = {} all_sound_ids = Sound.objects.all().values_list('id', flat=True).order_by('id') n_sounds = len(all_sound_ids) for analyzer_name in settings.ANALYZERS_CONFIGURATION.keys(): @@ -95,7 +91,7 @@ def monitor_analysis(request): fa = SoundAnalysis.objects.filter(analyzer=analyzer_name, analysis_status="FA").count() qu = SoundAnalysis.objects.filter(analyzer=analyzer_name, analysis_status="QU").count() missing = n_sounds - (ok + sk + fa + qu) - percentage_done = (ok + sk + fa) * 100.0/n_sounds + percentage_done = (ok + sk + fa) * 100.0 / n_sounds analyzers_data[analyzer_name] = { 'OK': ok, 'SK': sk, @@ -124,16 +120,15 @@ def monitor_moderation(request): time_span = datetime.datetime.now() - datetime.timedelta((6 * 365) // 12) #Maybe we should user created and not modified user_ids = tickets.models.Ticket.objects.filter( - status=TICKET_STATUS_CLOSED, - created__gt=time_span, - assignee__isnull=False - ).values_list("assignee_id", flat=True) + status=TICKET_STATUS_CLOSED, created__gt=time_span, assignee__isnull=False + ).values_list( + "assignee_id", flat=True + ) counter = Counter(user_ids) moderators = User.objects.filter(id__in=list(counter.keys())) moderators = [(counter.get(m.id), m) for m in moderators.all()] ordered = sorted(moderators, key=lambda m: m[0], reverse=True) - tvars = { "new_upload_count": new_upload_count, "tardy_moderator_sounds_count": tardy_moderator_sounds_count, @@ -157,7 +152,7 @@ def monitor_stats(request): def moderators_stats(request): return HttpResponseRedirect(reverse('monitor-moderation')) - + def queries_stats_ajax(request): try: auth = (settings.GRAYLOG_USERNAME, settings.GRAYLOG_PASSWORD) @@ -167,8 +162,9 @@ def queries_stats_ajax(request): 'filter': f'streams:{settings.GRAYLOG_SEARCH_STREAM_ID}', 'field': 'query' } - req = requests.get(settings.GRAYLOG_DOMAIN + '/graylog/api/search/universal/relative/terms', - auth=auth, params=params) + req = requests.get( + settings.GRAYLOG_DOMAIN + '/graylog/api/search/universal/relative/terms', auth=auth, params=params + ) req.raise_for_status() return JsonResponse(req.json()) except requests.HTTPError: @@ -240,9 +236,8 @@ def process_sounds(request): if sounds_to_process: for sound in sounds_to_process: sound.process(force=True) - + return HttpResponseRedirect(reverse("monitor-processing")) - def moderator_stats_ajax(request): diff --git a/ratings/admin.py b/ratings/admin.py index 664f84384..962350247 100644 --- a/ratings/admin.py +++ b/ratings/admin.py @@ -21,10 +21,10 @@ from django.contrib import admin from ratings.models import SoundRating + @admin.register(SoundRating) class SoundRatingAdmin(admin.ModelAdmin): raw_id_fields = ('user',) list_display = ('user', 'rating', 'created') - search_fields = ('=user__username', ) + search_fields = ('=user__username',) list_filter = ('rating',) - diff --git a/ratings/models.py b/ratings/models.py index 975478c5a..272263875 100644 --- a/ratings/models.py +++ b/ratings/models.py @@ -49,8 +49,8 @@ def post_delete_rating(sender, instance, **kwargs): try: with transaction.atomic(): instance.sound.num_ratings = F('num_ratings') - 1 - avg_rating = SoundRating.objects.filter( - sound_id=instance.sound_id).aggregate(average_rating=Coalesce(Avg('rating'), 0.0)) + avg_rating = SoundRating.objects.filter(sound_id=instance.sound_id + ).aggregate(average_rating=Coalesce(Avg('rating'), 0.0)) rating = avg_rating['average_rating'] instance.sound.avg_rating = rating instance.sound.save() @@ -66,8 +66,8 @@ def update_num_ratings_on_post_save(sender, instance, created, **kwargs): if created: instance.sound.num_ratings = F('num_ratings') + 1 - avg_rating = SoundRating.objects.filter( - sound_id=instance.sound_id).aggregate(average_rating=Coalesce(Avg('rating'), 0.0)) + avg_rating = SoundRating.objects.filter(sound_id=instance.sound_id + ).aggregate(average_rating=Coalesce(Avg('rating'), 0.0)) rating = avg_rating['average_rating'] instance.sound.avg_rating = rating instance.sound.save() diff --git a/ratings/templatetags/ratings.py b/ratings/templatetags/ratings.py index 5bac57330..e4db8ae96 100644 --- a/ratings/templatetags/ratings.py +++ b/ratings/templatetags/ratings.py @@ -34,7 +34,9 @@ def sound_ratings(context): request_user = request.user.username is_authenticated = request.user.is_authenticated - return {'sound_user': sound_user, - 'request_user': request_user, - 'is_authenticated': is_authenticated, - 'sound': sound} + return { + 'sound_user': sound_user, + 'request_user': request_user, + 'is_authenticated': is_authenticated, + 'sound': sound + } diff --git a/ratings/tests.py b/ratings/tests.py index f5aa044e4..c36f7d8bf 100644 --- a/ratings/tests.py +++ b/ratings/tests.py @@ -55,7 +55,7 @@ def test_rating_normal(self): self.assertEqual(ratings.models.SoundRating.objects.count(), 2) r = ratings.models.SoundRating.objects.get(sound_id=self.sound.id, user_id=self.user1.id) # Ratings in the database are 2x the value from the web call - self.assertEqual(r.rating, 2*RATING_VALUE) + self.assertEqual(r.rating, 2 * RATING_VALUE) # Check that signal updated sound.avg_rating and sound.num_ratings self.sound.refresh_from_db() @@ -133,15 +133,19 @@ def test_rating_link_logged_in(self): self.client.force_login(self.user1) resp = self.client.get(self.sound.get_absolute_url()) self.assertContains(resp, f'