From 0d2b4cef651b3c305a0684ce847269f1cc2d58e5 Mon Sep 17 00:00:00 2001 From: Ayyub I Date: Tue, 10 Oct 2023 18:06:26 -0500 Subject: [PATCH] fixed news article filenames; renewed cache --- .../backend/src/cache/faiss_index_general.dvc | 4 +- .../src/cache/faiss_index_in_depth.dvc | 4 +- ...ndemic_relief_aid_A_new_debate_begins.json | 2 +- ...r_3_hours,_a_judge_defers_the_ruling..json | 2 +- ...chards_Disposal_over_garbage_failures.json | 0 ...mayor's consent decree hearing stance.json | 2 +- packages/backend/src/preprocessor.py | 247 ++++++++++++++++++ .../getanswer/cache/faiss_index_general.dvc | 4 +- .../getanswer/cache/faiss_index_in_depth.dvc | 4 +- 9 files changed, 258 insertions(+), 11 deletions(-) rename "packages/backend/src/news_directory/\342\200\230No_excuses\342\200\231:_Frustrated_council_members_press_Richard\342\200\231s_Disposal_over_garbage_failures.json" => packages/backend/src/news_directory/No_excuses_Frustrated_council_members_press_Richards_Disposal_over_garbage_failures.json (100%) create mode 100644 packages/backend/src/preprocessor.py diff --git a/packages/backend/src/cache/faiss_index_general.dvc b/packages/backend/src/cache/faiss_index_general.dvc index 5d01f4a8..d2d8fed2 100644 --- a/packages/backend/src/cache/faiss_index_general.dvc +++ b/packages/backend/src/cache/faiss_index_general.dvc @@ -1,6 +1,6 @@ outs: -- md5: f47bd32f6216a08d934a604429bda914.dir - size: 61410252 +- md5: f86af62933bae171911ae891560b2407.dir + size: 61663926 nfiles: 2 hash: md5 path: faiss_index_general diff --git a/packages/backend/src/cache/faiss_index_in_depth.dvc b/packages/backend/src/cache/faiss_index_in_depth.dvc index 4b64a5d9..83865e58 100644 --- a/packages/backend/src/cache/faiss_index_in_depth.dvc +++ b/packages/backend/src/cache/faiss_index_in_depth.dvc @@ -1,6 +1,6 @@ outs: -- md5: 70eb12798f3bc495a8de4963f7595e81.dir - size: 61410252 +- md5: ed5520e1a6f903d4d38105dd3d9d034b.dir + size: 61663926 nfiles: 2 hash: md5 path: faiss_index_in_depth diff --git a/packages/backend/src/news_directory/How_should_New_Orleans_spend_the_millions_left_in_pandemic_relief_aid_A_new_debate_begins.json b/packages/backend/src/news_directory/How_should_New_Orleans_spend_the_millions_left_in_pandemic_relief_aid_A_new_debate_begins.json index ee1af2a2..298ddd62 100644 --- a/packages/backend/src/news_directory/How_should_New_Orleans_spend_the_millions_left_in_pandemic_relief_aid_A_new_debate_begins.json +++ b/packages/backend/src/news_directory/How_should_New_Orleans_spend_the_millions_left_in_pandemic_relief_aid_A_new_debate_begins.json @@ -1,7 +1,7 @@ { "messages": [ { - "page_content": "Skip to main content\nHomes\nCalendar\nStore\nPublic Notices\nE-Edition\nNewsletters\nSubscribe for $1\nHow should New Orleans spend the millions left in pandemic relief aid? A new debate begins\n\n\n\n\n\n\n\n\nThis website stores data such as cookies to enable essential website functionality, marketing, personalization and analytics. By remaining on this website you indicate your consent. See updated terms and conditions.\n This story is for subscribers.\nSign up for full digital access.\nAlready a print or digital subscriber? Log in here.\n$1/5 MONTHS UNLIMITED DIGITAL ACCESS\n$1\n1-YEAR UNLIMITED DIGITAL ACCESS\n$119.88\nCancel anytime. $1/5 months offer auto renews at $19.95, billed every four weeks.", + "page_content": "Skip to main content\nHomes\nCalendar\nStore\nPublic Notices\nE-Edition\nNewsletters\nSubscribe for $1\nHow should New Orleans spend the millions left in pandemic relief aid? A new debate begins\nBY MATT SLEDGE | Staff writer\nApr 4, 2023\n3 min to read\n1 of 2\nPeople who live on the streets are fed a hot meal in New Orleans, Sunday, Dec. 25, 2022.\nSTAFF PHOTO BY DAVID GRUNFELD\nFacebook\nTwitter\nEmail\nPrint\nCopy article link\nSave\nSolutions for crime and homelessness would claim tens of millions of the city\u2019s remaining federal relief dollars under a plan unveiled by Mayor LaToya Cantrell's administration on Tuesday, while progressive groups made a much larger pitch that would also call for the use of money from the general fund.\nIn the city\u2019s vision, the New Orleans Health Department would quarterback programs designed to prevent violence, such as mental health services for school children and a rejuvenated street team of violence \u201cinterrupters.\u201d\nAlong with a $10 million plan targeting homelessness, the plan would use $38.4 million of the remaining $53.6 million in pandemic relief funds from the American Rescue Plan Act.\nA coalition of progressive groups, by contrast, told the City Council that the city\u2019s one-time spending should rise much higher. They proposed a $147 million package, including $107 million for housing, that would also draw from a growing pot of unspent general fund dollars.\nThe council members who will have the final say sounded receptive notes but didn\u2019t make firm commitments. Next week, they will hear from the city\u2019s chief administrative officer, Gilbert Monta\u00f1o, who prefers keeping more money socked away.\nThe debate that will play out in the coming weeks is in many ways a repeat of the 2023 budget process, which culminated in a last-minute, $262 million amendment to use one-time funds. Advocates said they were caught off guard in December, and this time around, they\u2019re keen to make their voices heard.\n\u201cNow the question is, how do you reconcile what the city wants to do with the (relief) money with what we heard today from advocacy groups and from people interested in how that money is spent?\u201d said City Council member Joe Giarrusso, who chairs the budget committee.\nCity plan\nThe late-breaking budget amendment last year aimed lots of one-time money at the most urgent issue for many voters -- violent crime. Big spends included $22.5 million for police recruitment and retention, $30 million for new criminal justice software and $4.9 million for juvenile probation and parole.\nBut some advocates complained that far too many of those funds were directed at punitive measures instead of social services. The Cantrell administration is now proposing a package aimed at preventing and interrupting violence through non-policing measures.\nThe city\u2019s plan includes $10 million for a joint program between NOLA Public Schools and Children\u2019s Hospital to place mental health professionals, case managers and expanded services in the \u201chighest risk\u201d schools where kids need help to address trauma. Separately, the school system would partner with the city's Office of Workforce Development for an $8.1 million program to put at-risk high schoolers on career paths.\nMayor LaToya Cantrell and Dr. Jennifer Avegno, pictured in 2020 \nPHOTO BY MAX BECHERER / THE TIMES-PICAYUNE/NEW ORLEANS ADVOCATE\nOther expenditures would include $500,000 for a rebooted violence interruption street team, $1 million for a blight remediation partnership between Tulane public health researchers and city code enforcement, $2.5 million for pocket parks, $5 million for financial assistance programs and $1 million for food security programs.\nThe city\u2019s package is designed to put a \u201cdown payment\u201d on projects that have been proven to reduce crime, said Dr. Jennifer Avegno, health department director. The City Council in January passed a motion mandating that her department create a violence reduction program.\n\u201cIf you don\u2019t make significant investments in the root causes of crime and violence \u2013 and certainly, from the public health perspective, shootings and killings, which are a major cause of mortality in this city \u2013 then you\u2019re not getting a lot for your money,\u201d she said.\nSeparately, the city would spend $10 million on reducing homelessness.\nThe city\u2019s proposals draw only from American Rescue Plan Act funds. However, advocates note that the city also has a growing pot of unspent general fund dollars \u2013 pegged at $274 million in a recent city report.\nCAO Gilbert Montano, right \nPHOTO BY BRETT DUKE / THE TIMES-PICAYUNE\nMonta\u00f1o said that number represents an unaudited point-in-time figure. He acknowledged that the city\u2019s fund balance is \u201csizable\u201d but said he wants to keep a healthy reserve in place for emergencies and economic downturns.\n\u201cIf we don\u2019t protect the city for the future, then I don't think we\u2019re being good stewards,\u201d he said.\nAdvocates\u2019 plan\nThe community groups pitching the council on Tuesday proposed a much more aggressive spending plan. They said it would be a mistake to continue using the federal pandemic relief dollars to build up the city\u2019s fund balance.\nPresenters from the Louisiana Fair Housing Action Center, New Orleans Workers Center for Racial Justice and the Vera Institute said they had contacted dozens of community groups for input and conducted their own vetting project to come up with a wish-list of mostly existing projects that could benefit from infusions of one-time cash.\nThe top item on that list was $107 million for housing, including $70 million for the construction of affordable rental housing, $15 million for reducing street homelessness and $10 million for the renovation of existing but substandard rental units.\nOther proposals include a $20 million investment in youth development through community organizations and $18 million for \"community equity\" projects including $5 million for free public transit fares, $5 million for food banks and food producers, and $5.2 million in direct cash assistance for people who were left out of earlier rounds of government assistance during the pandemic.\nWill Snowden, director of Vera Institute of Justice's New Orleans office\nPROVIDED PHOTO\nWhile last year\u2019s budget appropriated tens of millions to agencies that respond to crime, Will Snowden, the director of the Vera Institute in New Orleans, said it was time to pivot to addressing root causes. He saw the groups\u2019 proposal as a floor, rather than a ceiling.\n\u201cThere can be more. When we\u2019re having conversations in New Orleans about what provides public safety and what creates public safety, it is these kinds of investments,\u201d he said.\nEmail Matt Sledge at msledge@theadvocate.com.\nFacebook\nTwitter\nEmail\nPrint\nCopy article link\nSave\nCity ARPA plan\nCity of New Orleans Apr 5, 2023\nMORE INFORMATION\nThe Nightly Top 5: Catch up on today's top stories\nDebate continues over how New Orleans should spend millions in pandemic relief aid, S&WB walks back its threat to cut water to tenants wit\u2026\nTags\nHardwall\nThis Day in History\nSponsored by Connatix\nRecommended for you\nRecommended by\nSECTIONS\nHOME\nNEWS\nOPINION\nSPORTS\nENTERTAINMENT/LIFE\nNEWSLETTERS\nGAMES\nSERVICES\nCLASSIFIEDS\nSEARCH\nSUBSCRIBE | GROUPS\nDIGITAL ADVERTISING\nHELP/CONTACT US\nRSS FEEDS\nMEDIA KIT\nEEDITION\nCAREERS\nTEACHER'S LINK\nREVIEWING THE RECORD\nOUR SITES\nOBITUARIES\nJOBS\nCELEBRATIONS\nNIE\nCLASSIFIEDS\nHOMES\nPETS\nARCHIVES\nSTORE\nCONTACT INFORMATION\nnola.com\n840 St. Charles Avenue\nNew Orleans, LA 70130\nPhone: 504-529-0522\n\nNews Tips:\nnolanewstips@theadvocate.com\nOther questions:\nsubscriberservices@theadvocate.com\nNeed help?\nReport a delivery issue\nCreate a temporary stop\nSign up for recurring payments\nPay your bill\nUpdate your billing info\n\n\u00a9 Copyright 2023 NOLA.com 840 St. Charles Avenue, New Orleans, LA | Terms of Use | Privacy Policy\nPowered by BLOX Content Management System from BLOX Digital.\n This website stores data such as cookies to enable essential website functionality, marketing, personalization and analytics. By remaining on this website you indicate your consent. See updated terms and conditions.", "url": "https://www.nola.com/news/politics/how-should-new-orleans-spend-millions-of-one-time-funds-new-debate-begins/article_ef36af06-d320-11ed-b9fe-978f03c2909b.html", "title": "How should New Orleans spend the millions left in pandemic relief aid? A new debate begins" } diff --git a/packages/backend/src/news_directory/New_Orleans_argues_case_to_exit_NOPD_consent_decree._After_3_hours,_a_judge_defers_the_ruling..json b/packages/backend/src/news_directory/New_Orleans_argues_case_to_exit_NOPD_consent_decree._After_3_hours,_a_judge_defers_the_ruling..json index ee3e2934..07348303 100644 --- a/packages/backend/src/news_directory/New_Orleans_argues_case_to_exit_NOPD_consent_decree._After_3_hours,_a_judge_defers_the_ruling..json +++ b/packages/backend/src/news_directory/New_Orleans_argues_case_to_exit_NOPD_consent_decree._After_3_hours,_a_judge_defers_the_ruling..json @@ -1,7 +1,7 @@ { "messages": [ { - "page_content": "Skip to main content\nHomes\nCalendar\nStore\nPublic Notices\nE-Edition\nNewsletters\nSubscribe for $1\nNew Orleans argues case to exit NOPD consent decree. After 3 hours, a judge defers the ruling.\nBY MISSY WILKINSON and JOHN SIMERMAN | Staff writers Jun 28, 2023\n2 min to read\nBuy Now\nThe U.S. District Courthouse in New Orleans.\nSTAFF FILE PHOTO\nFacebook\nTwitter\nEmail\nPrint\nCopy article link\nSave\nAlmost a year after filing a motion to end the sprawling reform agreement that has governed the New Orleans Police Department for a decade, Mayor LaToya Cantrell had her day in court.\nAttorneys for the city on Wednesday argued that problems the original federal consent decree was designed to address, including corruption, bias and shoddy policing, have long been ameliorated, despite consistent assertions that the police force should do more. \nU.S. Department of Justice lawyers disagreed, citing ongoing shortcomings that include improper use of force by officers and patterns of unlawful racial discrimination in stops and searches. \nAfter three hours of back-and-forth, U.S. District Judge Susie Morgan said she would take the matter under advisement.\nThe hearing culminated a legal gambit by Cantrell that began last August, when the city filed its motion to exit the deal, casting doubt on the assessments by the federal monitors who report to Morgan. In recent reports, the monitors identified backsliding in areas they'd previously checked off as compliant.\nMorgan had projected that the city would reach a two-year offramp from federal oversight last summer. But that timetable dissolved as the judge and the monitors began to find progress stifled, in part by a steep loss of police officers from the force. Morgan pumped the brakes, stoking a bitter response from Cantrell, who has since taken an aggressive tack against the judge and monitors, in court and public statements. \nCharles Zimmer, outside counsel for the city, told Morgan Wednesday that the NOPD has shown substantial compliance with the demands of the 2012 consent decree, which at the time was the most wide-reaching blueprint for police reform in the country.\nZimmer argued that changes have rendered compliance more onerous, and that perfection shouldn't be the goal. \n\"The city believes the pattern that existed in 2011 has been remedied for years, and we are legally entitled to exit,\" said Daniel Davillier, outside counsel for the city.\nAccording to a review by Independent Police Monitor Stella Cziment, 17 of the 28 instances in which the NOPD used force in 2021 weren't justified, and 37% of police pursuits that year resulted in formal disciplinary investigations. The NOPD's own audit revealed 35% of pat-downs from March 2022 weren't properly documented\u2014a potential Fourth Amendment violation.\nThough the department has exhibited enormous progress, that doesn't equate to full and effective compliance, a DOJ representative argued. \nFormer Mayor Mitch Landrieu agreed to the reforms after a blistering federal investigation portrayed a police department replete with misconduct. Morgan and the monitors have touted improvements since then, though doubts over the will to sustain those efforts remain strong. \n\"This is supposed to be a durable remedy,\" Morgan told the city's attorney. \"Once you're in compliance, the concept is you stay in compliance\u2014not that you can be compliant for two years and then forget about it.\"\nThe exchange between Morgan and Zimmer grew heated when Zimmer likened federal oversight\u2014and the accompanying $16 million paid out to federal monitors over the years\u2014to \"a hostage situation.\"\n\"I do not want you to impugn the integrity of the court to my face,\" Morgan replied.\nCziment, whose office is slated to take on a bigger watchdog role once the feds leave, cited fear in the community over a premature exit from federal oversight, and uncertainty over an NOPD left to its own devices.\n\"There isn't a clear vision of what constitutional policing will look like after the consent decree is done,\" Cziment stated after the hearing. \"As for the use of force and stop, searches and arrests, we share the concerns highlighted by the Department of Justice and the NOPD themselves (via departmental audits.)\"\nMorgan provided no timeline for her ruling.\nFacebook\nTwitter\nEmail\nPrint\nCopy article link\nSave\nMORE INFORMATION\nMayor defends reinstatement of Jeffrey Vappie to security detail\nSeven months after questions arose about his time spent with Mayor LaToya Cantrell in a French Quarter apartment, New Orleans police officer J\u2026\nNOPD says officer on Cantrell detail violated policies, but monitor says it could've looked harder\nFederal monitors cast a dim view of the New Orleans Police Department\u2019s internal investigation of Officer Jeffrey Vappie over his timesheets while serving on a team securing Mayor LaToya Cantrell, in a report released Thursday.\nNOPD to sell reforms straight to public after federal judge caves on hearing order\nNew Orleans Police Department brass were set to launch a series of public meetings on Wednesday to tout progress on a slew of reforms to the f\u2026\nIn police reform quarrel, New Orleans objects to 'unattainable' goal\nCantrell administration says Justice Department, \"faced with losing control\" is changing the rules\nMonitor slams NOPD over internal affairs, probe of Cantrell security guard\nA New Orleans police internal affairs probe into alleged payroll irregularities involving an officer who was assigned to protect Mayor LaToya \u2026\nCity must defend NOPD probe of mayor's bodyguard, appeals court rules\nA federal appeals court on Monday opposed the city\u2019s objection to a rule to show cause hearing regarding alleged violations of the consent decree that stemmed from its handling of the Vappie probe. U.S. District Judge Susie Morgan ordered the city to defend its investigation on Aug. 31.\nThree-day federal court hearing ends over NOPD's investigation of mayor's bodyguard\nMonitors who report to U.S. Judge Susie Morgan claim the city violated several reform mandates in its probe of Officer Jeffrey Vappie\nThis Day in History\nSponsored by Connatix\nRecommended for you\nRecommended by\nSECTIONS\nHOME\nNEWS\nOPINION\nSPORTS\nENTERTAINMENT/LIFE\nNEWSLETTERS\nGAMES\nSERVICES\nCLASSIFIEDS\nSEARCH\nSUBSCRIBE | GROUPS\nDIGITAL ADVERTISING\nHELP/CONTACT US\nRSS FEEDS\nMEDIA KIT\nEEDITION\nCAREERS\nTEACHER'S LINK\nREVIEWING THE RECORD\nOUR SITES\nOBITUARIES\nJOBS\nCELEBRATIONS\nNIE\nCLASSIFIEDS\nHOMES\nPETS\nARCHIVES\nSTORE\nCONTACT INFORMATION\nnola.com\n840 St. Charles Avenue\nNew Orleans, LA 70130\nPhone: 504-529-0522\n\nNews Tips:\nnolanewstips@theadvocate.com\nOther questions:\nsubscriberservices@theadvocate.com\nNeed help?\nReport a delivery issue\nCreate a temporary stop\nSign up for recurring payments\nPay your bill\nUpdate your billing info\n\n\u00a9 Copyright 2023 NOLA.com 840 St. Charles Avenue, New Orleans, LA | Terms of Use | Privacy Policy\nPowered by BLOX Content Management System from BLOX Digital.\nThis website stores data such as cookies to enable essential website functionality, marketing, personalization and analytics. By remaining on this website you indicate your consent. See updated terms and conditions.\n ", + "page_content": "Skip to main content\nHomes\nCalendar\nStore\nPublic Notices\nE-Edition\nNewsletters\nSubscribe for $1\nNew Orleans argues case to exit NOPD consent decree. After 3 hours, a judge defers the ruling.\nBY MISSY WILKINSON and JOHN SIMERMAN | Staff writers Jun 28, 2023\n2 min to read\nBuy Now\nThe U.S. District Courthouse in New Orleans.\nSTAFF FILE PHOTO\nFacebook\nTwitter\nEmail\nPrint\nCopy article link\nSave\nAlmost a year after filing a motion to end the sprawling reform agreement that has governed the New Orleans Police Department for a decade, Mayor LaToya Cantrell had her day in court.\nAttorneys for the city on Wednesday argued that problems the original federal consent decree was designed to address, including corruption, bias and shoddy policing, have long been ameliorated, despite consistent assertions that the police force should do more. \nU.S. Department of Justice lawyers disagreed, citing ongoing shortcomings that include improper use of force by officers and patterns of unlawful racial discrimination in stops and searches. \nAfter three hours of back-and-forth, U.S. District Judge Susie Morgan said she would take the matter under advisement.\nThe hearing culminated a legal gambit by Cantrell that began last August, when the city filed its motion to exit the deal, casting doubt on the assessments by the federal monitors who report to Morgan. In recent reports, the monitors identified backsliding in areas they'd previously checked off as compliant.\nMorgan had projected that the city would reach a two-year offramp from federal oversight last summer. But that timetable dissolved as the judge and the monitors began to find progress stifled, in part by a steep loss of police officers from the force. Morgan pumped the brakes, stoking a bitter response from Cantrell, who has since taken an aggressive tack against the judge and monitors, in court and public statements. \nCharles Zimmer, outside counsel for the city, told Morgan Wednesday that the NOPD has shown substantial compliance with the demands of the 2012 consent decree, which at the time was the most wide-reaching blueprint for police reform in the country.\nZimmer argued that changes have rendered compliance more onerous, and that perfection shouldn't be the goal. \n\"The city believes the pattern that existed in 2011 has been remedied for years, and we are legally entitled to exit,\" said Daniel Davillier, outside counsel for the city.\nAccording to a review by Independent Police Monitor Stella Cziment, 17 of the 28 instances in which the NOPD used force in 2021 weren't justified, and 37% of police pursuits that year resulted in formal disciplinary investigations. The NOPD's own audit revealed 35% of pat-downs from March 2022 weren't properly documented\u2014a potential Fourth Amendment violation.\nThough the department has exhibited enormous progress, that doesn't equate to full and effective compliance, a DOJ representative argued. \nFormer Mayor Mitch Landrieu agreed to the reforms after a blistering federal investigation portrayed a police department replete with misconduct. Morgan and the monitors have touted improvements since then, though doubts over the will to sustain those efforts remain strong. \n\"This is supposed to be a durable remedy,\" Morgan told the city's attorney. \"Once you're in compliance, the concept is you stay in compliance\u2014not that you can be compliant for two years and then forget about it.\"\nThe exchange between Morgan and Zimmer grew heated when Zimmer likened federal oversight\u2014and the accompanying $16 million paid out to federal monitors over the years\u2014to \"a hostage situation.\"\n\"I do not want you to impugn the integrity of the court to my face,\" Morgan replied.\nCziment, whose office is slated to take on a bigger watchdog role once the feds leave, cited fear in the community over a premature exit from federal oversight, and uncertainty over an NOPD left to its own devices.\n\"There isn't a clear vision of what constitutional policing will look like after the consent decree is done,\" Cziment stated after the hearing. \"As for the use of force and stop, searches and arrests, we share the concerns highlighted by the Department of Justice and the NOPD themselves (via departmental audits.)\"\nMorgan provided no timeline for her ruling.\nFacebook\nTwitter\nEmail\nPrint\nCopy article link\nSave\nMORE INFORMATION\nMayor defends reinstatement of Jeffrey Vappie to security detail\nSeven months after questions arose about his time spent with Mayor LaToya Cantrell in a French Quarter apartment, New Orleans police officer J\u2026\nNOPD says officer on Cantrell detail violated policies, but monitor says it could've looked harder\nFederal monitors cast a dim view of the New Orleans Police Department\u2019s internal investigation of Officer Jeffrey Vappie over his timesheets while serving on a team securing Mayor LaToya Cantrell, in a report released Thursday.\nNOPD to sell reforms straight to public after federal judge caves on hearing order\nNew Orleans Police Department brass were set to launch a series of public meetings on Wednesday to tout progress on a slew of reforms to the f\u2026\nIn police reform quarrel, New Orleans objects to 'unattainable' goal\nCantrell administration says Justice Department, \"faced with losing control\" is changing the rules\nMonitor slams NOPD over internal affairs, probe of Cantrell security guard\nA New Orleans police internal affairs probe into alleged payroll irregularities involving an officer who was assigned to protect Mayor LaToya \u2026\nCity must defend NOPD probe of mayor's bodyguard, appeals court rules\nA federal appeals court on Monday opposed the city\u2019s objection to a rule to show cause hearing regarding alleged violations of the consent decree that stemmed from its handling of the Vappie probe. U.S. District Judge Susie Morgan ordered the city to defend its investigation on Aug. 31.\nThree-day federal court hearing ends over NOPD's investigation of mayor's bodyguard\nMonitors who report to U.S. Judge Susie Morgan claim the city violated several reform mandates in its probe of Officer Jeffrey Vappie\nThis Day in History\nSponsored by Connatix\nRecommended for you\nRecommended by\nSECTIONS\nHOME\nNEWS\nOPINION\nSPORTS\nENTERTAINMENT/LIFE\nNEWSLETTERS\nGAMES\nSERVICES\nCLASSIFIEDS\nSEARCH\nSUBSCRIBE | GROUPS\nDIGITAL ADVERTISING\nHELP/CONTACT US\nRSS FEEDS\nMEDIA KIT\nEEDITION\nCAREERS\nTEACHER'S LINK\nREVIEWING THE RECORD\nOUR SITES\nOBITUARIES\nJOBS\nCELEBRATIONS\nNIE\nCLASSIFIEDS\nHOMES\nPETS\nARCHIVES\nSTORE\nCONTACT INFORMATION\nnola.com\n840 St. Charles Avenue\nNew Orleans, LA 70130\nPhone: 504-529-0522\n\nNews Tips:\nnolanewstips@theadvocate.com\nOther questions:\nsubscriberservices@theadvocate.com\nNeed help?\nReport a delivery issue\nCreate a temporary stop\nSign up for recurring payments\nPay your bill\nUpdate your billing info\n\n\u00a9 Copyright 2023 NOLA.com 840 St. Charles Avenue, New Orleans, LA | Terms of Use | Privacy Policy\nPowered by BLOX Content Management System from BLOX Digital.\n This website stores data such as cookies to enable essential website functionality, marketing, personalization and analytics. By remaining on this website you indicate your consent. See updated terms and conditions.", "url": "https://www.nola.com/news/crime_police/judge-offers-no-ruling-on-citys-motion-to-exit-consent-decree/article_1871f7dc-15c9-11ee-ad6c-afaeccd8e9dc.html", "title": "New Orleans argues case to exit NOPD consent decree. After 3 hours, a judge defers the ruling." } diff --git "a/packages/backend/src/news_directory/\342\200\230No_excuses\342\200\231:_Frustrated_council_members_press_Richard\342\200\231s_Disposal_over_garbage_failures.json" b/packages/backend/src/news_directory/No_excuses_Frustrated_council_members_press_Richards_Disposal_over_garbage_failures.json similarity index 100% rename from "packages/backend/src/news_directory/\342\200\230No_excuses\342\200\231:_Frustrated_council_members_press_Richard\342\200\231s_Disposal_over_garbage_failures.json" rename to packages/backend/src/news_directory/No_excuses_Frustrated_council_members_press_Richards_Disposal_over_garbage_failures.json diff --git a/packages/backend/src/news_directory/Some New Orleans City Council members voice concerns over mayor's consent decree hearing stance.json b/packages/backend/src/news_directory/Some New Orleans City Council members voice concerns over mayor's consent decree hearing stance.json index c535b789..4fceacf7 100644 --- a/packages/backend/src/news_directory/Some New Orleans City Council members voice concerns over mayor's consent decree hearing stance.json +++ b/packages/backend/src/news_directory/Some New Orleans City Council members voice concerns over mayor's consent decree hearing stance.json @@ -1,7 +1,7 @@ { "messages": [ { - "page_content": "NOWCAST\nWDSU News at Noon\nWatch on Demand\nMENU\n76\u00b0\nWEATHER\nSEVERE WEATHER There are currently 3 active weather alerts\n1 / 2\nAdvertisement\nSome New Orleans City Council members voice concerns over mayor's consent decree hearing stance\nShare\nUpdated: 8:55 AM CDT Apr 11, 2023\nInfinite Scroll Enabled\nCassie Schirm\nInvestigative Reporter\nPlay Video\nGET LOCAL BREAKING NEWS ALERTS\nThe latest breaking updates, delivered straight to your email inbox.\nYour Email Address\nSUBMIT\nPrivacy Notice\nNEW ORLEANS \u2014\nSome New Orleans City Council members are voicing their concerns over Mayor LaToya Cantrell's stance on city employees attending public consent decree hearings.\nMayor Cantrell issued a statement a few weeks ago barring any city employees from attending public consent decree meetings without her consent.\nAdvertisement\nThis was a move that Interim Superintendent Michelle Woodfork asked for the mayor's support on, saying the hearings didn't focus on constitutional policing and wasted resources.\nThe federal judge overseeing the hearings ordered certain city employees to attend the next hearing after the last meeting was canceled due to the mayor's decision on attendance.\nNew Orleans Council President J.P. Morrell, Vice President Helena Moreno, Joseph I. Giarrusso and Leslie Harris issued a letter to several judges voicing their concerns over the mayor's actions.\nThe letter says the fighting is a \"direct attack on democratic values,\" and \"a waste of resources.\nThey asked for a \"greater collaboration\" between the courts and the city to avoid unnecessary fees.\nRecommended\nTaylor Swift 'Eras Tour:' Everything you need to know before tickets go on sale\nThe letter goes on to say that although the council could not control the mayor, they would consider how public funding is used if the problem surrounding attendance at the meetings persists.\nThe next public consent decree hearing is scheduled for April 12.\nTOP PICKS\nAmazon Prime Big Deal Days 2023: The best deals we're seeing for October Prime Day event\nA mom shielding her son from gunmen and a 'pro-peace' academic are among the Americans killed in Israel\nWhat is Hamas?\nPumpkin weighing 2,749 pounds wins World Championship Pumpkin Weigh-Off in California, sets world record\nWDSU NEW ORLEANS\nContact Us\nNews Team\nApps & Social\nEmail Alerts\nCareers\nInternships\nAdvertise\nDigital Advertising Terms & Conditions\nBroadcast Terms & Conditions\nRSS\nEEO Reports\nCaptioning Contacts\nPublic Inspection File\nPublic File Assistance\nFCC Applications\nNews Policy Statements\nHearst Television participates in various affiliate marketing programs, which means we may get paid commissions on editorially chosen products purchased through our links to retailer sites.\n\u00a92023, Hearst Television Inc. on behalf of WDSU-TV.\nPrivacy Notice\nYour California Privacy Rights\nInterest-Based Ads\nTerms of Use\nSite Map", + "page_content": "NOWCAST\nWDSU News at 5pm\nWatch on Demand\nMENU\n74\u00b0\nWEATHER\nSEVERE WEATHER There are currently 3 active weather alerts\n1 / 2\nAdvertisement\nSome New Orleans City Council members voice concerns over mayor's consent decree hearing stance\nShare\nUpdated: 8:55 AM CDT Apr 11, 2023\nInfinite Scroll Enabled\nCassie Schirm\nInvestigative Reporter\nPlay Video\nGET LOCAL BREAKING NEWS ALERTS\nThe latest breaking updates, delivered straight to your email inbox.\nYour Email Address\nSUBMIT\nPrivacy Notice\nNEW ORLEANS \u2014\nSome New Orleans City Council members are voicing their concerns over Mayor LaToya Cantrell's stance on city employees attending public consent decree hearings.\nMayor Cantrell issued a statement a few weeks ago barring any city employees from attending public consent decree meetings without her consent.\nAdvertisement\nThis was a move that Interim Superintendent Michelle Woodfork asked for the mayor's support on, saying the hearings didn't focus on constitutional policing and wasted resources.\nThe federal judge overseeing the hearings ordered certain city employees to attend the next hearing after the last meeting was canceled due to the mayor's decision on attendance.\nNew Orleans Council President J.P. Morrell, Vice President Helena Moreno, Joseph I. Giarrusso and Leslie Harris issued a letter to several judges voicing their concerns over the mayor's actions.\nThe letter says the fighting is a \"direct attack on democratic values,\" and \"a waste of resources.\nThey asked for a \"greater collaboration\" between the courts and the city to avoid unnecessary fees.\nRecommended\nVoter registation deadlines for October elections\nThe letter goes on to say that although the council could not control the mayor, they would consider how public funding is used if the problem surrounding attendance at the meetings persists.\nThe next public consent decree hearing is scheduled for April 12.\nTOP PICKS\nAmazon Prime Big Deal Days 2023: The best deals we're seeing for October Prime Day event\nRossen Reports: 5 things in your house you didn\u2019t know expire\nA mom shielding her son from gunmen and a 'pro-peace' academic are among the Americans killed in Israel\nWhat is Hamas?\nWDSU NEW ORLEANS\nContact Us\nNews Team\nApps & Social\nEmail Alerts\nCareers\nInternships\nAdvertise\nDigital Advertising Terms & Conditions\nBroadcast Terms & Conditions\nRSS\nEEO Reports\nCaptioning Contacts\nPublic Inspection File\nPublic File Assistance\nFCC Applications\nNews Policy Statements\nHearst Television participates in various affiliate marketing programs, which means we may get paid commissions on editorially chosen products purchased through our links to retailer sites.\n\u00a92023, Hearst Television Inc. on behalf of WDSU-TV.\nPrivacy Notice\nYour California Privacy Rights\nInterest-Based Ads\nTerms of Use\nSite Map", "url": "https://www.wdsu.com/article/new-orleans-council-members-consent-decree-mayor-concerns/43556031", "title": "Some New Orleans City Council members voice concerns over mayor's consent decree hearing stance" } diff --git a/packages/backend/src/preprocessor.py b/packages/backend/src/preprocessor.py new file mode 100644 index 00000000..f14f8a77 --- /dev/null +++ b/packages/backend/src/preprocessor.py @@ -0,0 +1,247 @@ +import logging +import os +from langchain.document_loaders import ( + Docx2txtLoader, + JSONLoader, +) +from langchain.text_splitter import RecursiveCharacterTextSplitter +from langchain.embeddings import OpenAIEmbeddings +from langchain.chains import LLMChain, HypotheticalDocumentEmbedder +from langchain.prompts import PromptTemplate +from langchain.vectorstores.faiss import FAISS +from langchain.llms import OpenAI +from pathlib import Path +import shutil + + +logger = logging.getLogger(__name__) +dir = Path(__file__).parent.absolute() + + +def create_embeddings(): + llm = OpenAI() + + base_embeddings = OpenAIEmbeddings() + + general_prompt_template = """ + As an AI assistant, your role is to provide concise, balanced summaries from the transcripts of New Orleans City Council meetings in response to the user's query "{user_query}". Your response should not exceed one paragraph in length. If the available information from the transcripts is insufficient to accurately summarize the issue, respond with 'Insufficient information available.' If the user's query extends beyond the scope of information contained in the transcripts, state 'I don't know.' + Answer:""" + + in_depth_prompt_template = """ + As an AI assistant, use the New Orleans City Council transcript data that you were trained on to provide an in-depth and balanced response to the following query: "{user_query}" + Answer:""" + + general_prompt = PromptTemplate( + input_variables=["user_query"], template=general_prompt_template + ) + in_depth_prompt = PromptTemplate( + input_variables=["user_query"], template=in_depth_prompt_template + ) + + llm_chain_general = LLMChain(llm=llm, prompt=general_prompt) + llm_chain_in_depth = LLMChain(llm=llm, prompt=in_depth_prompt) + + general_embeddings = HypotheticalDocumentEmbedder( + llm_chain=llm_chain_general, + base_embeddings=base_embeddings, + ) + in_depth_embeddings = HypotheticalDocumentEmbedder( + llm_chain=llm_chain_in_depth, base_embeddings=base_embeddings + ) + + return base_embeddings, base_embeddings + + +def metadata_func_minutes_and_agendas(record: dict, metadata: dict) -> dict: + metadata["title"] = record.get("title") + metadata["page_number"] = record.get("page_number") + return metadata + + +def create_db_from_minutes_and_agendas(doc_directory): + logger.info("Creating database from minutes...") + all_docs = [] + for doc_file in os.listdir(doc_directory): + if not doc_file.endswith(".json"): + continue + doc_path = os.path.join(doc_directory, doc_file) + loader = JSONLoader( + file_path=doc_path, + jq_schema=".messages[]", + content_key="page_content", + metadata_func=metadata_func_minutes_and_agendas, + ) + + data = loader.load() + text_splitter = RecursiveCharacterTextSplitter( + chunk_size=15000, chunk_overlap=10000 + ) + docs = text_splitter.split_documents(data) + all_docs.extend(docs) + logger.info("Finished database from minutes...") + return all_docs + + +def metadata_func(record: dict, metadata: dict) -> dict: + metadata["timestamp"] = record.get("timestamp") + metadata["url"] = record.get("url") + metadata["title"] = record.get("title") + metadata["publish_date"] = record.get("publish_date") + + return metadata + + +def metadata_news(record: dict, metadata: dict) -> dict: + metadata["url"] = record.get("url") + metadata["title"] = record.get("title") + return metadata + + +def create_db_from_news_transcripts(news_json_directory): + logger.info("Creating database from CJ transcripts...") + all_docs = [] + for doc_file in os.listdir(news_json_directory): + if not doc_file.endswith(".json"): + continue + doc_path = os.path.join(news_json_directory, doc_file) + loader = JSONLoader( + file_path=doc_path, + jq_schema=".messages[]", + content_key="page_content", + metadata_func=metadata_news, + ) + + data = loader.load() + text_splitter = RecursiveCharacterTextSplitter( + chunk_size=500, chunk_overlap=250 + ) + docs = text_splitter.split_documents(data) + all_docs.extend(docs) + logger.info("Finished database from news transcripts...") + return all_docs + + +def create_db_from_cj_transcripts(cj_json_directory): + logger.info("Creating database from CJ transcripts...") + all_docs = [] + for doc_file in os.listdir(cj_json_directory): + if not doc_file.endswith(".json"): + continue + doc_path = os.path.join(cj_json_directory, doc_file) + loader = JSONLoader( + file_path=doc_path, + jq_schema=".messages[]", + content_key="page_content", + metadata_func=metadata_func, + ) + + data = loader.load() + text_splitter = RecursiveCharacterTextSplitter( + chunk_size=20000, chunk_overlap=10000 + ) + docs = text_splitter.split_documents(data) + all_docs.extend(docs) + logger.info("Finished database from CJ transcripts...") + return all_docs + + +def create_db_from_fc_transcripts(fc_json_directory): + logger.info("Creating database from FC transcripts...") + all_docs = [] + for doc_file in os.listdir(fc_json_directory): + if not doc_file.endswith(".json"): + continue + doc_path = os.path.join(fc_json_directory, doc_file) + loader = JSONLoader( + file_path=doc_path, + jq_schema=".messages[]", + content_key="page_content", + metadata_func=metadata_func, + ) + + data = loader.load() + text_splitter = RecursiveCharacterTextSplitter( + chunk_size=20000, chunk_overlap=10000 + ) + docs = text_splitter.split_documents(data) + all_docs.extend(docs) + logger.info("Finished database from FC transcripts...") + return all_docs + + +def create_db_from_public_comments(pc_json_directory): + logger.info("Creating database from FC transcripts...") + all_docs = [] + for doc_file in os.listdir(pc_json_directory): + if not doc_file.endswith(".json"): + continue + doc_path = os.path.join(pc_json_directory, doc_file) + loader = JSONLoader( + file_path=doc_path, + jq_schema=".messages[]", + content_key="page_content", + metadata_func=metadata_func, + ) + + data = loader.load() + text_splitter = RecursiveCharacterTextSplitter( + chunk_size=1000, chunk_overlap=500 + ) + docs = text_splitter.split_documents(data) + all_docs.extend(docs) + logger.info("Finished database from Public Comments...") + return all_docs + + +def create_db_from_youtube_urls_and_pdfs( + fc_json_directory, + cj_json_directory, + doc_directory, + pc_directory, + news_directory, + general_embeddings, + in_depth_embeddings, +): + fc_video_docs = create_db_from_fc_transcripts(fc_json_directory) + cj_video_docs = create_db_from_cj_transcripts(cj_json_directory) + pdf_docs = create_db_from_minutes_and_agendas(doc_directory) + pc_docs = create_db_from_public_comments(pc_directory) + news_docs = create_db_from_news_transcripts(news_directory) + + all_docs = fc_video_docs + cj_video_docs + pc_docs + news_docs + + db_general = FAISS.from_documents(all_docs, general_embeddings) + db_in_depth = FAISS.from_documents(all_docs, in_depth_embeddings) + + cache_dir = dir.joinpath("cache") + if not os.path.exists(cache_dir): + os.makedirs(cache_dir) + + save_dir_general = cache_dir.joinpath("faiss_index_general") + save_dir_in_depth = cache_dir.joinpath("faiss_index_in_depth") + + db_general.save_local(save_dir_general) + db_in_depth.save_local(save_dir_in_depth) + + db_general.save_local(save_dir_general) + db_in_depth.save_local(save_dir_in_depth) + + logger.info( + f"Combined database for general model transcripts created frfom all video URLs and PDF files saved to {save_dir_general}" + ) + logger.info( + f"Combined database for in-depth model transcripts created from all video URLs and PDF files saved to {save_dir_in_depth}" + ) + + # copy results to cloud function + dest_dir_general = dir.parent.parent.joinpath( + "googlecloud/functions/getanswer/cache/faiss_index_general" + ) + dest_dir_in_depth = dir.parent.parent.joinpath( + "googlecloud/functions/getanswer/cache/faiss_index_in_depth" + ) + + shutil.copytree(save_dir_general, dest_dir_general, dirs_exist_ok=True) + shutil.copytree(save_dir_in_depth, dest_dir_in_depth, dirs_exist_ok=True) + + return db_general, db_in_depth diff --git a/packages/googlecloud/functions/getanswer/cache/faiss_index_general.dvc b/packages/googlecloud/functions/getanswer/cache/faiss_index_general.dvc index 5d01f4a8..d2d8fed2 100644 --- a/packages/googlecloud/functions/getanswer/cache/faiss_index_general.dvc +++ b/packages/googlecloud/functions/getanswer/cache/faiss_index_general.dvc @@ -1,6 +1,6 @@ outs: -- md5: f47bd32f6216a08d934a604429bda914.dir - size: 61410252 +- md5: f86af62933bae171911ae891560b2407.dir + size: 61663926 nfiles: 2 hash: md5 path: faiss_index_general diff --git a/packages/googlecloud/functions/getanswer/cache/faiss_index_in_depth.dvc b/packages/googlecloud/functions/getanswer/cache/faiss_index_in_depth.dvc index 4b64a5d9..83865e58 100644 --- a/packages/googlecloud/functions/getanswer/cache/faiss_index_in_depth.dvc +++ b/packages/googlecloud/functions/getanswer/cache/faiss_index_in_depth.dvc @@ -1,6 +1,6 @@ outs: -- md5: 70eb12798f3bc495a8de4963f7595e81.dir - size: 61410252 +- md5: ed5520e1a6f903d4d38105dd3d9d034b.dir + size: 61663926 nfiles: 2 hash: md5 path: faiss_index_in_depth