Skip to content

Commit

Permalink
Merge branch 'main' into 4664-recap-attachment-doppelganger-edge-case
Browse files Browse the repository at this point in the history
  • Loading branch information
@albertisfu
albertisfu committed Dec 4, 2024
2 parents e532157 + 53646b2 commit d5fb59e
Show file tree
Hide file tree
Showing 100 changed files with 5,395 additions and 464 deletions.
5 changes: 5 additions & 0 deletions .github/workflows/docker-build.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,11 @@ jobs:
- name: Watch cl-es-sweep-indexer rollout status
run: kubectl rollout status -n ${{ env.EKS_NAMESPACE }} deployment/cl-es-sweep-indexer

- name: Rollout cl-iquery-probe
run: kubectl set image -n ${{ env.EKS_NAMESPACE }} deployment/cl-iquery-probe cl-iquery-probe=freelawproject/courtlistener:${{ steps.vars.outputs.sha_short }}-prod
- name: Watch cl-iquery-probe rollout status
run: kubectl rollout status -n ${{ env.EKS_NAMESPACE }} deployment/cl-iquery-probe


# Watch "cronjobs" in k9s
- name: Update cronjobs
Expand Down
110 changes: 85 additions & 25 deletions cl/alerts/management/commands/cl_send_alerts.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,25 @@
from django.template import loader
from django.urls import reverse
from django.utils.timezone import now
from elasticsearch_dsl import MultiSearch
from elasticsearch_dsl import Q as ES_Q
from elasticsearch_dsl.response import Response

from cl.alerts.models import Alert, RealTimeQueue
from cl.alerts.utils import InvalidDateError
from cl.api.models import WebhookEventType
from cl.api.models import WebhookEventType, WebhookVersions
from cl.api.webhooks import send_search_alert_webhook
from cl.lib import search_utils
from cl.lib.command_utils import VerboseCommand, logger
from cl.lib.elasticsearch_utils import do_es_api_query
from cl.lib.elasticsearch_utils import (
do_es_api_query,
limit_inner_hits,
set_child_docs_and_score,
set_results_highlights,
)
from cl.lib.scorched_utils import ExtraSolrInterface
from cl.lib.search_utils import regroup_snippets
from cl.lib.types import CleanData
from cl.search.constants import ALERTS_HL_TAG, SEARCH_ALERTS_OPINION_HL_FIELDS
from cl.search.documents import OpinionDocument
from cl.search.forms import SearchForm
Expand Down Expand Up @@ -106,6 +114,59 @@ def send_alert(user_profile, hits):
msg.send(fail_silently=False)


def query_alerts_es(
cd: CleanData, v1_webhook: bool = False
) -> tuple[Response, Response | None]:
"""Query ES for opinion alerts, optionally handling a V1 webhook query.
:param cd: A CleanData object containing the query parameters.
:param v1_webhook: A boolean indicating whether to include a V1 webhook query.
:return: A tuple containing the main search response and an optional V1
query response.
"""

v1_results = None
search_query = OpinionDocument.search()
cd["highlight"] = True
main_query, _ = do_es_api_query(
search_query,
cd,
SEARCH_ALERTS_OPINION_HL_FIELDS,
ALERTS_HL_TAG,
"v4",
)
main_query = main_query.extra(
from_=0,
size=settings.SCHEDULED_ALERT_HITS_LIMIT,
)
multi_search = MultiSearch()
multi_search = multi_search.add(main_query)

if v1_webhook:
search_query = OpinionDocument.search()
v1_query, _ = do_es_api_query(
search_query,
cd,
SEARCH_ALERTS_OPINION_HL_FIELDS,
ALERTS_HL_TAG,
"v3",
)
v1_query = v1_query.extra(
from_=0,
size=settings.SCHEDULED_ALERT_HITS_LIMIT,
)
multi_search = multi_search.add(v1_query)

responses = multi_search.execute()
results = responses[0]
limit_inner_hits({}, results, cd["type"])
set_results_highlights(results, cd["type"])
set_child_docs_and_score(results)
if v1_webhook:
v1_results = responses[1]
return results, v1_results


class Command(VerboseCommand):
help = (
"Sends the alert emails on a real time, daily, weekly or monthly "
Expand Down Expand Up @@ -152,10 +213,9 @@ def handle(self, *args, **options):
if options["rate"] == Alert.REAL_TIME:
self.clean_rt_queue()

def run_query(self, alert, rate):
def run_query(self, alert, rate, v1_webhook=False):
results = []
cd = {}
main_params = {}
v1_results = None
logger.info(f"Now running the query: {alert.query}\n")

# Make a dict from the query string.
Expand All @@ -175,7 +235,7 @@ def run_query(self, alert, rate):
if waffle.switch_is_active("oa-es-alerts-active"):
# Return empty results for OA alerts. They are now handled
# by Elasticsearch.
return query_type, results
return query_type, results, v1_results

logger.info(f"Data sent to SearchForm is: {qd}\n")
search_form = SearchForm(qd, is_es_form=self.o_es_alerts)
Expand All @@ -187,7 +247,7 @@ def run_query(self, alert, rate):
and len(self.valid_ids[query_type]) == 0
):
# Bail out. No results will be found if no valid_ids.
return query_type, results
return query_type, results, v1_results

main_params = search_utils.build_main_query(
cd,
Expand Down Expand Up @@ -220,19 +280,7 @@ def run_query(self, alert, rate):
)

if self.o_es_alerts:
search_query = OpinionDocument.search()
s, _ = do_es_api_query(
search_query,
cd,
SEARCH_ALERTS_OPINION_HL_FIELDS,
ALERTS_HL_TAG,
"v3",
)
s = s.extra(
from_=0,
size=settings.SCHEDULED_ALERT_HITS_LIMIT,
)
results = s.execute()
results, v1_results = query_alerts_es(cd, v1_webhook)
else:
# Ignore warnings from this bit of code. Otherwise, it complains
# about the query URL being too long and having to POST it instead
Expand All @@ -248,7 +296,7 @@ def run_query(self, alert, rate):
regroup_snippets(results)

logger.info(f"There were {len(results)} results.")
return qd, results
return qd, results, v1_results

def send_emails_and_webhooks(self, rate):
"""Send out an email and webhook events to every user whose alert has a
Expand All @@ -261,14 +309,23 @@ def send_emails_and_webhooks(self, rate):
alerts = user.alerts.filter(rate=rate)
logger.info(f"Running alerts for user '{user}': {alerts}")

# Query user's webhooks.
user_webhooks = user.webhooks.filter(
event_type=WebhookEventType.SEARCH_ALERT, enabled=True
)
v1_webhook = WebhookVersions.v1 in {
webhook.version for webhook in user_webhooks
}
if rate == Alert.REAL_TIME:
if not user.profile.is_member:
continue

hits = []
for alert in alerts:
try:
qd, results = self.run_query(alert, rate)
qd, results, v1_results = self.run_query(
alert, rate, v1_webhook
)
except:
traceback.print_exc()
logger.info(
Expand All @@ -293,10 +350,13 @@ def send_emails_and_webhooks(self, rate):

# Send webhook event if the user has a SEARCH_ALERT
# endpoint enabled.
user_webhooks = user.webhooks.filter(
event_type=WebhookEventType.SEARCH_ALERT, enabled=True
)
for user_webhook in user_webhooks:
results = (
v1_results
if alert.alert_type == SEARCH_TYPES.OPINION
and user_webhook.version == WebhookVersions.v1
else results
)
send_search_alert_webhook(
self.sis[search_type], results, user_webhook, alert
)
Expand Down
67 changes: 51 additions & 16 deletions cl/alerts/templates/alert_email_es.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,26 +75,63 @@ <h3 class="alt bottom" style="font-size: 1.5em; font-weight: normal; line-height
</a>
<br>
{% endif %}
{% else %}
<p style="font-size: 100%; font-weight: inherit; font-family: inherit; border: 0; vertical-align: baseline; font-style: inherit; margin: 0; padding: 0;">
<strong style="font-weight: bold;">
View original:
</strong>
{% if result.download_url %}
<a href="{{result.download_url}}" style="font-size: 100%; font-weight: inherit; font-family: inherit; color: #009; border: 0; font-style: inherit; padding: 0; text-decoration: none; vertical-align: baseline; margin: 0;">
From the court
</a>
&nbsp;&nbsp;|&nbsp;&nbsp;
{% endif %}
{% endif %}
{% if type == 'o' %}
<ul>
{% for doc in result.child_docs %}
{% with doc=doc|get_es_doc_content:True %}
<li style="margin-bottom: 5px;">
{% if result.child_docs|length > 1 or doc.type != 'combined-opinion' %}
{% if doc.text %}
<strong>{{ doc.type_text }}</strong>
{% endif %}
{% endif %}
{% if doc.text %}
{% contains_highlights doc.text.0 True as highlighted %}
<span style="display: block;">{% if highlighted %}&hellip; {% endif %}{{ doc.text|render_string_or_list|safe|underscore_to_space }} &hellip;</span>
{% endif %}
{% if doc.download_url or doc.local_path %}
<p style="font-size: 100%; font-weight: inherit; font-family: inherit; border: 0; vertical-align: baseline; font-style: inherit; margin: 0; padding: 0;">
<strong style="font-weight: bold;">
View original:
</strong>
{% if doc.download_url %}
<a href="{{doc.download_url}}" style="font-size: 100%; font-weight: inherit; font-family: inherit; color: #009; border: 0; font-style: inherit; padding: 0; text-decoration: none; vertical-align: baseline; margin: 0;">
From the court
</a>
&nbsp;&nbsp;|&nbsp;&nbsp;
{% endif %}
{% if doc.local_path %}
{# Provide link to S3. #}
<a href="https://storage.courtlistener.com/{{doc.local_path}}" style="font-size: 100%; font-weight: inherit; font-family: inherit; color: #009; border: 0; font-style: inherit; padding: 0; text-decoration: none; vertical-align: baseline; margin: 0;">
Our backup
</a>
{% endif %}
</p>
{% endif %}
</li>
{% endwith %}
{% endfor %}
</ul>
{% endif %}
{% if type == 'oa' %}
<p style="font-size: 100%; font-weight: inherit; font-family: inherit; border: 0; vertical-align: baseline; font-style: inherit; margin: 0; padding: 0;">
<strong style="font-weight: bold;">
View original:
</strong>
{% if result.download_url %}
<a href="{{result.download_url}}" style="font-size: 100%; font-weight: inherit; font-family: inherit; color: #009; border: 0; font-style: inherit; padding: 0; text-decoration: none; vertical-align: baseline; margin: 0;">
From the court
</a>
&nbsp;&nbsp;|&nbsp;&nbsp;
{% endif %}
{% if result.local_path %}
{# Provide link to S3. #}
<a href="https://storage.courtlistener.com/{{result.local_path}}" style="font-size: 100%; font-weight: inherit; font-family: inherit; color: #009; border: 0; font-style: inherit; padding: 0; text-decoration: none; vertical-align: baseline; margin: 0;">
Our backup
</a>
{% endif %}
</p>
{% endif %}
{% if type == 'oa' %}
<p style="font-size: 100%; font-weight: inherit; font-family: inherit; border: 0; vertical-align: baseline; font-style: inherit; margin: 0 0 0; padding: 0;">
<strong style="font-weight: bold;">Date Argued: </strong>
{% if result.dateArgued %}
Expand All @@ -116,9 +153,7 @@ <h3 class="alt bottom" style="font-size: 1.5em; font-weight: normal; line-height
{{ result|get_highlight:"judge"|safe|underscore_to_space }}
{% endif %}
</p>
{% endif %}
{% if type == 'o' or type == 'oa' %}
<p style="font-size: 100%; font-weight: inherit; font-family: inherit; border: 0; vertical-align: baseline; font-style: inherit; margin: 0 0 1.5em; padding: 0;">
<p style="font-size: 100%; font-weight: inherit; font-family: inherit; border: 0; vertical-align: baseline; font-style: inherit; margin: 0 0 1.5em; padding: 0;">
{% if result|get_highlight:"text" %}
&hellip;{{ result|get_highlight:"text"|safe|underscore_to_space }}&hellip;
{% endif %}
Expand Down
15 changes: 10 additions & 5 deletions cl/alerts/templates/alert_email_es.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,14 @@ View Full Results / Edit this Alert: https://www.courtlistener.com/?{{ alert.que
Disable this Alert (one click): https://www.courtlistener.com{% url "disable_alert" alert.secret_key %}{% endif %}

{{forloop.counter}}. {{ result.caseName|render_string_or_list|safe|striptags }} ({% if result.court_id != 'scotus' %}{{ result.court_citation_string|render_string_or_list|striptags }} {% endif %}{% if type == 'o' or type == 'r' %}{{ result.dateFiled|date:"Y" }}{% elif type == 'oa' %}{{ result.dateArgued|date:"Y" }}{% endif %})
{% if type == 'oa' %}{% if result.dateArgued %}Date Argued: {{ result.dateArgued|date:"F jS, Y" }}{% else %}Date Argued: Unknown Date {% endif %}{% if result.docketNumber %} | Docket Number: {{ result.docketNumber|render_string_or_list|safe|striptags }}{% endif %} | Duration: {{ result.duration|naturalduration }}{% if result.judge %} | Judge: {{ result.judge|render_string_or_list|safe|striptags|underscore_to_space }}{% endif %}{% endif %}
{% if type == 'o' or type == 'oa' %}{% if result|get_highlight:"text" %}...{{ result|get_highlight:"text"|safe|striptags|underscore_to_space|compress_whitespace }}...{% endif %}{% endif %}
{% if type == 'oa' %}{% if result.dateArgued %}Date Argued: {{ result.dateArgued|date:"F jS, Y" }}{% else %}Date Argued: Unknown Date {% endif %}{% if result.docketNumber %} | Docket Number: {{ result.docketNumber|render_string_or_list|safe|striptags }}{% endif %} | Duration: {{ result.duration|naturalduration }}{% if result.judge %} | Judge: {{ result.judge|render_string_or_list|safe|striptags|underscore_to_space }}{% endif %}
{% if result|get_highlight:"text" %}...{{ result|get_highlight:"text"|safe|striptags|underscore_to_space|compress_whitespace }}...{% endif %}
{% endif %}
{% if type == 'o' %}{% for doc in result.child_docs %}{% with doc=doc|get_es_doc_content:True %}{% if result.child_docs|length > 1 or doc.type != 'combined-opinion' %}{% if doc.text %}{{ doc.type_text }}{% endif %}{% endif %}
{% if doc.text %}...{{ doc.text|render_string_or_list|safe|striptags|underscore_to_space|compress_whitespace }}...{% endif %}
{% if doc.download_url %} - Download original from the court: {{doc.download_url}}{% endif %}
{% if doc.local_path %} - Download the original from our backup: https://storage.courtlistener.com/{{ doc.local_path }}{% endif %}
{% endwith %}{% endfor %}{% endif %}
{% if type == 'r' %}{% if result.dateFiled %}Date Filed: {{ result.dateFiled|date:"F jS, Y" }}{% else %}Date Filed: Unknown Date {% endif %}{% if result.docketNumber %} | Docket Number: {{ result.docketNumber|render_string_or_list|safe|striptags }}{% endif %}
{% for doc in result.child_docs %}{% with doc=doc|get_es_doc_content:scheduled_alert %} - {% if doc.short_description %}{{ doc.short_description|render_string_or_list|safe|striptags }} - {% endif %}Document #{% if doc.document_number %}{{ doc.document_number }}{% endif %}{% if doc.attachment_number %}, Attachment #{{ doc.attachment_number }}{% endif %}
{% if doc.description %}Description: {{ doc.description|render_string_or_list|safe|striptags }}{% endif %}
Expand All @@ -27,9 +33,8 @@ Disable this Alert (one click): https://www.courtlistener.com{% url "disable_ale
{% if result.child_docs and result.child_remaining %}{% extract_q_value alert.query_run as q_value %}View Additional Results for this Case: https://www.courtlistener.com/?type={{ type|urlencode }}&q={% if q_value %}({{ q_value|urlencode }})%20AND%20{% endif %}docket_id%3A{{ result.docket_id|urlencode }}{% endif %}
{% endif %}~~~~~
- View this item on our site: https://www.courtlistener.com{% if type == 'r' %}{{result.docket_absolute_url}}{% else %}{{result.absolute_url}}{% endif %}
{% if result.download_url %} - Download original from the court: {{result.download_url}}
{% endif %}{% if result.local_path %} - Download the original from our backup: https://storage.courtlistener.com/{{ result.local_path }}{% endif %}{% endfor %}

{% if type == 'oa' %}{% if result.download_url %} - Download original from the court: {{result.download_url}}
{% endif %}{% if result.local_path %} - Download the original from our backup: https://storage.courtlistener.com/{{ result.local_path }}{% endif %}{% endif %}{% endfor %}
{% endfor %}
************************
This alert brought to you by the 501(c)(3) non-profit Free Law Project
Expand Down
Loading

0 comments on commit d5fb59e

Please sign in to comment.