-
-
Notifications
You must be signed in to change notification settings - Fork 152
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'main' into fix-front-end-case-name
- Loading branch information
Showing
6 changed files
with
187 additions
and
11 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
17 changes: 17 additions & 0 deletions
17
cl/scrapers/management/commands/refresh_scrapers_status_view.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| from django.db import connection | ||
|
|
||
| from cl.lib.command_utils import VerboseCommand, logger | ||
|
|
||
|
|
||
| class Command(VerboseCommand): | ||
| help = """Refreshes the `scrapers_mv_latest_opinion` materialized view. | ||
| Check the cl.scrapers.admin.py file for more info about the view | ||
| """ | ||
|
|
||
| def handle(self, *args, **options): | ||
| query = "REFRESH MATERIALIZED VIEW scrapers_mv_latest_opinion;" | ||
| with connection.cursor() as cursor: | ||
| cursor.execute(query) | ||
|
|
||
| logger.info("View refresh completed successfully") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| # Generated by Django 5.1.2 on 2024-11-25 15:27 | ||
|
|
||
| from django.db import migrations, models | ||
|
|
||
|
|
||
| class Migration(migrations.Migration): | ||
|
|
||
| dependencies = [ | ||
| ("scrapers", "0003_delete_errorlog"), | ||
| ] | ||
|
|
||
| operations = [ | ||
| migrations.CreateModel( | ||
| name="MVLatestOpinion", | ||
| fields=[ | ||
| ( | ||
| "court_id", | ||
| models.TextField(primary_key=True, serialize=False), | ||
| ), | ||
| ("latest_creation_date", models.DateTimeField()), | ||
| ("time_since", models.TextField()), | ||
| ("view_last_updated", models.DateTimeField()), | ||
| ], | ||
| options={ | ||
| "db_table": "scrapers_mv_latest_opinion", | ||
| "managed": False, | ||
| }, | ||
| ), | ||
| migrations.RunSQL(""" | ||
| CREATE MATERIALIZED VIEW IF NOT EXISTS | ||
| scrapers_mv_latest_opinion | ||
| AS | ||
| ( | ||
| SELECT | ||
| court_id, | ||
| max(so.date_created) as latest_creation_date, | ||
| DATE_TRUNC('minutes', (now() - max(so.date_created)))::text as time_since, | ||
| now() as view_last_updated | ||
| FROM | ||
| ( | ||
| SELECT id, court_id | ||
| FROM search_docket | ||
| WHERE court_id IN ( | ||
| SELECT id | ||
| FROM search_court | ||
| /* | ||
| Only check courts with scrapers in use | ||
| */ | ||
| WHERE | ||
| has_opinion_scraper | ||
| AND in_use | ||
| ) | ||
| ) sd | ||
| INNER JOIN | ||
| (SELECT id, docket_id FROM search_opinioncluster) soc ON soc.docket_id = sd.id | ||
| INNER JOIN | ||
| search_opinion so ON so.cluster_id = soc.id | ||
| GROUP BY | ||
| sd.court_id | ||
| HAVING | ||
| /* | ||
| Only return results for courts with no updates in a week | ||
| */ | ||
| now() - max(so.date_created) > interval '7 days' | ||
| ORDER BY | ||
| 2 DESC | ||
| ) | ||
| """) | ||
| ] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| BEGIN; | ||
| -- | ||
| -- Create model MVLatestOpinion | ||
| -- | ||
| -- (no-op) | ||
| -- | ||
| -- Raw SQL operation | ||
| -- | ||
|
|
||
| CREATE MATERIALIZED VIEW IF NOT EXISTS | ||
| scrapers_mv_latest_opinion | ||
| AS | ||
| ( | ||
| SELECT | ||
| court_id, | ||
| max(so.date_created) as latest_creation_date, | ||
| DATE_TRUNC('minutes', (now() - max(so.date_created)))::text as time_since, | ||
| now() as view_last_updated | ||
| FROM | ||
| ( | ||
| SELECT id, court_id | ||
| FROM search_docket | ||
| WHERE court_id IN ( | ||
| SELECT id | ||
| FROM search_court | ||
| /* | ||
| Only check courts with scrapers in use | ||
| */ | ||
| WHERE | ||
| has_opinion_scraper | ||
| AND in_use | ||
| ) | ||
| ) sd | ||
| INNER JOIN | ||
| (SELECT id, docket_id FROM search_opinioncluster) soc ON soc.docket_id = sd.id | ||
| INNER JOIN | ||
| search_opinion so ON so.cluster_id = soc.id | ||
| GROUP BY | ||
| sd.court_id | ||
| HAVING | ||
| /* | ||
| Only return results for courts with no updates in a week | ||
| */ | ||
| now() - max(so.date_created) > interval '7 days' | ||
| ORDER BY | ||
| 2 DESC | ||
| ) | ||
| ; | ||
| COMMIT; |