From a47fb9e90a05ab82beb691bb7d13e4fc4ab2cd0e Mon Sep 17 00:00:00 2001 From: legaltextai <144342123+legaltextai@users.noreply.github.com> Date: Tue, 17 Sep 2024 17:12:56 +0000 Subject: [PATCH 1/2] fix(search): implements changes to search to capture user queries Makes changes to models, views, adds migrations Fixes: #4230 --- .../migrations/0035_capture_search_query.py | 37 ++++++++++ .../migrations/0035_capture_search_query.sql | 10 +++ .../0035_capture_search_query_customers.sql | 10 +++ cl/search/models.py | 10 +++ cl/search/views.py | 74 ++++++++++--------- 5 files changed, 107 insertions(+), 34 deletions(-) create mode 100644 cl/search/migrations/0035_capture_search_query.py create mode 100644 cl/search/migrations/0035_capture_search_query.sql create mode 100644 cl/search/migrations/0035_capture_search_query_customers.sql diff --git a/cl/search/migrations/0035_capture_search_query.py b/cl/search/migrations/0035_capture_search_query.py new file mode 100644 index 0000000000..5baf04abf9 --- /dev/null +++ b/cl/search/migrations/0035_capture_search_query.py @@ -0,0 +1,37 @@ +# Generated by Django 5.1.1 on 2024-09-16 20:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("search", "0034_add_harvard_pdf_to_opinioncluster"), + ] + + operations = [ + migrations.CreateModel( + name="SearchQuery", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "date_created", + models.DateTimeField(auto_now_add=True), + ), + ( + "date_modified", + models.DateTimeField(auto_now=True), + ), + ("get_params", models.CharField(max_length=255)), + ("query_time_ms", models.IntegerField()), + ("hit_cache", models.BooleanField()), + ], + ), + ] diff --git a/cl/search/migrations/0035_capture_search_query.sql b/cl/search/migrations/0035_capture_search_query.sql new file mode 100644 index 0000000000..fb6f19d3bd --- /dev/null +++ b/cl/search/migrations/0035_capture_search_query.sql @@ -0,0 +1,10 @@ +CREATE TABLE "search_searchquery" ( + "id" integer NOT NULL PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, + "date_created" timestamp with time zone NOT NULL, + "date_modified" timestamp with time zone NOT NULL, + "get_params" varchar(255) NOT NULL, + "query_time_ms" integer NOT NULL, + "hit_cache" boolean NOT NULL +); + +COMMIT; \ No newline at end of file diff --git a/cl/search/migrations/0035_capture_search_query_customers.sql b/cl/search/migrations/0035_capture_search_query_customers.sql new file mode 100644 index 0000000000..fb6f19d3bd --- /dev/null +++ b/cl/search/migrations/0035_capture_search_query_customers.sql @@ -0,0 +1,10 @@ +CREATE TABLE "search_searchquery" ( + "id" integer NOT NULL PRIMARY KEY GENERATED BY DEFAULT AS IDENTITY, + "date_created" timestamp with time zone NOT NULL, + "date_modified" timestamp with time zone NOT NULL, + "get_params" varchar(255) NOT NULL, + "query_time_ms" integer NOT NULL, + "hit_cache" boolean NOT NULL +); + +COMMIT; \ No newline at end of file diff --git a/cl/search/models.py b/cl/search/models.py index a0c808f3d3..1bf7afa549 100644 --- a/cl/search/models.py +++ b/cl/search/models.py @@ -49,6 +49,16 @@ HYPERSCAN_TOKENIZER = HyperscanTokenizer(cache_dir=".hyperscan") +class SearchQuery(models.Model): + date_created = models.DateTimeField(auto_now_add=True, db_index=True) + date_modified = models.DateTimeField(auto_now=True, db_index=True) + get_params = models.CharField(max_length=255) + query_time_ms = models.IntegerField() + hit_cache = models.BooleanField() + + def str(self): + return f"Query: {self.get_params} at {self.date_created}" + class PRECEDENTIAL_STATUS: PUBLISHED = "Published" UNPUBLISHED = "Unpublished" diff --git a/cl/search/views.py b/cl/search/views.py index 539efaaed3..295d0325bb 100644 --- a/cl/search/views.py +++ b/cl/search/views.py @@ -79,14 +79,13 @@ UnbalancedQuotesQuery, ) from cl.search.forms import SearchForm, _clean_form -from cl.search.models import SEARCH_TYPES, Court, Opinion, OpinionCluster +from cl.search.models import SEARCH_TYPES, Court, Opinion, OpinionCluster, SearchQuery from cl.stats.models import Stat from cl.stats.utils import tally_stat from cl.visualizations.models import SCOTUSMap logger = logging.getLogger(__name__) - def check_pagination_depth(page_number): """Check if the pagination is too deep (indicating a crawler)""" @@ -347,7 +346,6 @@ def show_results(request: HttpRequest) -> HttpResponse: All of these paths have tests. """ # Create a search string that does not contain the page numbers - get_string = make_get_string(request) get_string_sans_alert = make_get_string( request, ["page", "edit_alert", "show_alert_modal"] @@ -513,42 +511,51 @@ def show_results(request: HttpRequest) -> HttpResponse: # Just a regular search if not is_bot(request): async_to_sync(tally_stat)("search.results") + + # Perform the search + search_type = request.GET.get("type", SEARCH_TYPES.OPINION) + + if search_type == SEARCH_TYPES.PARENTHETICAL: + render_dict.update(do_es_search(request.GET.copy())) + elif search_type == SEARCH_TYPES.ORAL_ARGUMENT: + # Check if waffle flag is active. + if waffle.flag_is_active(request, "oa-es-active"): + render_dict.update(do_es_search(request.GET.copy())) + else: + render_dict.update(do_search(request.GET.copy())) + elif search_type == SEARCH_TYPES.PEOPLE: + if waffle.flag_is_active(request, "p-es-active"): + render_dict.update(do_es_search(request.GET.copy())) + else: + render_dict.update(do_search(request.GET.copy())) + elif search_type in [SEARCH_TYPES.RECAP, SEARCH_TYPES.DOCKETS]: + if waffle.flag_is_active(request, "r-es-active"): + search_results = do_es_search(request.GET.copy()) + else: + search_results = do_search(request.GET.copy()) + render_dict.update(search_results) + elif search_type == SEARCH_TYPES.OPINION: + if waffle.flag_is_active(request, "o-es-active"): + render_dict.update(do_es_search(request.GET.copy())) + else: + render_dict.update(do_search(request.GET.copy())) + elif search_type == SEARCH_TYPES.RECAP_DOCUMENT: + render_dict.update(do_es_search(request.GET.copy())) + else: + render_dict.update(do_search(request.GET.copy())) + + # Create and save the SearchQuery object + SearchQuery.objects.create( + get_params=request.GET.urlencode(), + query_time_ms=render_dict.get('query_time', 0), + hit_cache=render_dict.get('hit_cache', False) + ) # Create bare-bones alert form. alert_form = CreateAlertForm( initial={"query": get_string, "rate": "dly"}, user=request.user, ) - search_type = request.GET.get("type", SEARCH_TYPES.OPINION) - match search_type: - case SEARCH_TYPES.PARENTHETICAL: - render_dict.update(do_es_search(request.GET.copy())) - case SEARCH_TYPES.ORAL_ARGUMENT: - # Check if waffle flag is active. - if waffle.flag_is_active(request, "oa-es-active"): - render_dict.update(do_es_search(request.GET.copy())) - else: - render_dict.update(do_search(request.GET.copy())) - case SEARCH_TYPES.PEOPLE: - if waffle.flag_is_active(request, "p-es-active"): - render_dict.update(do_es_search(request.GET.copy())) - else: - render_dict.update(do_search(request.GET.copy())) - case SEARCH_TYPES.RECAP | SEARCH_TYPES.DOCKETS: - if waffle.flag_is_active(request, "r-es-active"): - search_results = do_es_search(request.GET.copy()) - else: - search_results = do_search(request.GET.copy()) - render_dict.update(search_results) - case SEARCH_TYPES.OPINION: - if waffle.flag_is_active(request, "o-es-active"): - render_dict.update(do_es_search(request.GET.copy())) - else: - render_dict.update(do_search(request.GET.copy())) - case SEARCH_TYPES.RECAP_DOCUMENT: - render_dict.update(do_es_search(request.GET.copy())) - case _: - render_dict.update(do_search(request.GET.copy())) # Set the value to the query as a convenience alert_form.fields["name"].widget.attrs["value"] = render_dict[ @@ -558,7 +565,6 @@ def show_results(request: HttpRequest) -> HttpResponse: return TemplateResponse(request, "search.html", render_dict) - def advanced(request: HttpRequest) -> HttpResponse: render_dict = {"private": False} From 204075f5168ba3ee953e2f623beb4f465c5d373d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 17 Sep 2024 17:15:23 +0000 Subject: [PATCH 2/2] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- cl/search/models.py | 1 + cl/search/views.py | 33 +++++++++++++++++++++++++-------- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/cl/search/models.py b/cl/search/models.py index 1bf7afa549..97dae76ce9 100644 --- a/cl/search/models.py +++ b/cl/search/models.py @@ -59,6 +59,7 @@ class SearchQuery(models.Model): def str(self): return f"Query: {self.get_params} at {self.date_created}" + class PRECEDENTIAL_STATUS: PUBLISHED = "Published" UNPUBLISHED = "Unpublished" diff --git a/cl/search/views.py b/cl/search/views.py index 295d0325bb..2af4a8fc78 100644 --- a/cl/search/views.py +++ b/cl/search/views.py @@ -79,13 +79,20 @@ UnbalancedQuotesQuery, ) from cl.search.forms import SearchForm, _clean_form -from cl.search.models import SEARCH_TYPES, Court, Opinion, OpinionCluster, SearchQuery +from cl.search.models import ( + SEARCH_TYPES, + Court, + Opinion, + OpinionCluster, + SearchQuery, +) from cl.stats.models import Stat from cl.stats.utils import tally_stat from cl.visualizations.models import SCOTUSMap logger = logging.getLogger(__name__) + def check_pagination_depth(page_number): """Check if the pagination is too deep (indicating a crawler)""" @@ -511,7 +518,7 @@ def show_results(request: HttpRequest) -> HttpResponse: # Just a regular search if not is_bot(request): async_to_sync(tally_stat)("search.results") - + # Perform the search search_type = request.GET.get("type", SEARCH_TYPES.OPINION) @@ -520,15 +527,22 @@ def show_results(request: HttpRequest) -> HttpResponse: elif search_type == SEARCH_TYPES.ORAL_ARGUMENT: # Check if waffle flag is active. if waffle.flag_is_active(request, "oa-es-active"): - render_dict.update(do_es_search(request.GET.copy())) + render_dict.update( + do_es_search(request.GET.copy()) + ) else: render_dict.update(do_search(request.GET.copy())) elif search_type == SEARCH_TYPES.PEOPLE: if waffle.flag_is_active(request, "p-es-active"): - render_dict.update(do_es_search(request.GET.copy())) + render_dict.update( + do_es_search(request.GET.copy()) + ) else: render_dict.update(do_search(request.GET.copy())) - elif search_type in [SEARCH_TYPES.RECAP, SEARCH_TYPES.DOCKETS]: + elif search_type in [ + SEARCH_TYPES.RECAP, + SEARCH_TYPES.DOCKETS, + ]: if waffle.flag_is_active(request, "r-es-active"): search_results = do_es_search(request.GET.copy()) else: @@ -536,7 +550,9 @@ def show_results(request: HttpRequest) -> HttpResponse: render_dict.update(search_results) elif search_type == SEARCH_TYPES.OPINION: if waffle.flag_is_active(request, "o-es-active"): - render_dict.update(do_es_search(request.GET.copy())) + render_dict.update( + do_es_search(request.GET.copy()) + ) else: render_dict.update(do_search(request.GET.copy())) elif search_type == SEARCH_TYPES.RECAP_DOCUMENT: @@ -547,8 +563,8 @@ def show_results(request: HttpRequest) -> HttpResponse: # Create and save the SearchQuery object SearchQuery.objects.create( get_params=request.GET.urlencode(), - query_time_ms=render_dict.get('query_time', 0), - hit_cache=render_dict.get('hit_cache', False) + query_time_ms=render_dict.get("query_time", 0), + hit_cache=render_dict.get("hit_cache", False), ) # Create bare-bones alert form. @@ -565,6 +581,7 @@ def show_results(request: HttpRequest) -> HttpResponse: return TemplateResponse(request, "search.html", render_dict) + def advanced(request: HttpRequest) -> HttpResponse: render_dict = {"private": False}