Merge remote-tracking branch 'origin' into prod

.gitignore

@@ -8,4 +8,8 @@ txt/
 *.csv
 package-lock.json
 package.json
-*.DS_Store
+*.DS_Store
+*.mp4
+*.mp3
+*.wav
+*.jpg

packages/backend/src/__main__.py

@@ -1,7 +1,7 @@
 import logging
 
 from dotenv import find_dotenv, load_dotenv
-from preprocessor import create_db_from_youtube_urls_and_pdfs, create_embeddings
+from preprocessor import create_vector_dbs, create_embeddings
 
 load_dotenv(find_dotenv())
 
@@ -20,13 +20,12 @@ def main():
     general_embeddings, in_depth_embeddings = create_embeddings()
 
     # create_db_from_fc_youtube_urls(FC_INPUT_VIDEO_URLS)
-    create_db_from_youtube_urls_and_pdfs(
+    create_vector_dbs(
         fc_json_directory,
         cj_json_directory,
         pdf_directory,
         pc_json_directory,
         news_json_directory,
-        general_embeddings,
         in_depth_embeddings,
     )
 

packages/backend/src/cache/.gitignore

@@ -1,2 +1,7 @@
 /faiss_index_general
 /faiss_index_in_depth
+/faiss_index_in_depth_pdf
+/faiss_index_in_depth_pc
+/faiss_index_in_depth_news
+/faiss_index_in_depth_fc
+/faiss_index_in_depth_cj

packages/backend/src/cache/faiss_index_in_depth_cj.dvc

@@ -0,0 +1,6 @@
+outs:
+- md5: 9c2090ec6ee84ddf3792866364b35b0f.dir
+  size: 23116757
+  nfiles: 2
+  hash: md5
+  path: faiss_index_in_depth_cj

packages/backend/src/cache/faiss_index_in_depth_fc.dvc

@@ -0,0 +1,6 @@
+outs:
+- md5: ad7c688f8fff83d65f8728a403c864dd.dir
+  size: 63799934
+  nfiles: 2
+  hash: md5
+  path: faiss_index_in_depth_fc

packages/backend/src/cache/faiss_index_in_depth_news.dvc

@@ -0,0 +1,6 @@
+outs:
+- md5: d08e6dfb04e5afcff49028b2bffed7ad.dir
+  size: 526418
+  nfiles: 2
+  hash: md5
+  path: faiss_index_in_depth_news

packages/backend/src/cache/faiss_index_in_depth_pc.dvc

@@ -0,0 +1,6 @@
+outs:
+- md5: 75d23bd607ac91d67d994f624fb2e4c2.dir
+  size: 2097501
+  nfiles: 2
+  hash: md5
+  path: faiss_index_in_depth_pc

packages/backend/src/cache/faiss_index_in_depth_pdf.dvc

@@ -0,0 +1,6 @@
+outs:
+- md5: 28bfb00c7d3ebfbb4de620e9915e75b3.dir
+  size: 53418928
+  nfiles: 2
+  hash: md5
+  path: faiss_index_in_depth_pdf

packages/backend/src/minutes_agendas_directory/At lincoln beach volunteer caretakers plead for help from new orleans city hall.json

@@ -0,0 +1,9 @@
+{
+    "messages": [
+        {
+            "page_content": "Skip to main content\nHomes\nCalendar\nStore\nPublic Notices\nE-Edition\nNewsletters\nSubscribe for $1\nAt Lincoln Beach, volunteer caretakers plead for help from New Orleans City Hall\nResidents, officials try to cooperate as redevelopment unfold\nBy BEN MYERS | Staff writer\nSep 4, 2022\n3 min to read\n1 of 3\nReggie Ford takes a picture from a sailboat that washed up at Lincoln Beach in New Orleans on Tuesday, Aug. 30, 2022.\nSTAFF PHOTO BY SOPHIA GERMER\nFacebook\nTwitter\nEmail\nPrint\nCopy article link\nSave\nWhen it comes to Lincoln Beach, the long-neglected New Orleans East recreation spot that\u2019s recently had a quiet revival, city officials and the volunteers who tend to it say they want the same thing: a newly restored, pristine Lake Pontchartrain beachfront for everyone to enjoy.\nBut in recent months, as City Hall has begun developing plans to rehabilitate the area, Lincoln Beach advocates say they\u2019ve been frustrated by the slow pace of government action.\nSick of the lack of progress over four mayoral administrations, residents adopted the beach in the early days of the COVID-19 pandemic. They took it upon themselves to clear thickets, build tables and walking trails, gather thousands of bags of garbage and even install a rudimentary drainage system.\nA pipe that is part of the draining and irrigation system helps move water under a walkway at Lincoln Beach in New Orleans on Tuesday, Aug. 30, 2022.\nSTAFF PHOTO BY SOPHIA GERMER\nOfficials in Mayor LaToya Cantrell\u2019s administration, meanwhile, say they are pursuing a development plan filled with contracting procedures, permitting requirements and grant applications. Cantrell has set aside $5 million in money borrowed via bond issues, and the Department of Public Works has submitted a formal proposal for money from the BP oil disaster settlement.\nAmid the plodding bureacratic process, garbage continues to accumulate on the 15-acre site. Volunteers have hung signs directing visitors to dispose of waste properly, in strategically placed bins, and warning that glass is not allowed. They have pleaded with City Hall to clear a massive pile of garbage bags - large enough to fill a small apartment - that they have collected under a pavilion.\nBut officials have resisted. They are quick to point out that, technically, nothing is allowed at Lincoln Beach: Setting foot on it is trespassing.\nSpace is cleared of debris at Lincoln Beach in New Orleans on Tuesday, Aug. 30, 2022.\nSTAFF PHOTO BY SOPHIA GERMER\n\u201cLincoln Beach is closed. It\u2019s not a safe place to be right now,\u201d Cheryn Robles, the Cantrell administration\u2019s project manager, said during a City Council committee meeting on Wednesday. \u201cIn terms of priorities of how we should be addressing illegal dumping, I would say it's more important for us to be addressing it on roads that are open.\u201d\nThe latest plans \nThe Cantrell administration\u2019s Lincoln Beach revival plans are the latest in a succession of fitful efforts dating from the 1990s. Plans call for a natural recreation environment, and roughly half of the needed money has been committed. The administration produced a site assessment in April 2021, and Robles said designs are in the works. A request for proposals for a master planner could be issued soon, and a construction timeline ready by spring.\nBut residents have grown frustrated with the lack of visible progress and clear information, especially as they continue to maintain the site on their own time and dime.\nReggie Ford walks along a wall next to a ramp that has crumbled in the past few years at Lincoln Beach in New Orleans on Tuesday, Aug. 30, 2022.\nSTAFF PHOTO BY SOPHIA GERMER\n\u201cI just bought a $600 lawn mower, and it broke in a week because it hit some old rebar. said artist Reggie Ford, who has worked with community activist Michael Pellet on the site\u2019s upkeep. \"I\u2019m about to buy another one. This is like my fourth lawn mower,\u201d \nFord said he spends as many as six hours a day working at Lincoln Beach, and has spent more than $20,000 on it. Pellet, who also goes by Sage Michael, said he started organizing volunteers in March 2020 so New Orleans East could \u201chave something.\u201d\n\u201cFor this community that's been suffering from disaster after disaster, after false promises after lack of investment,\u201d Michael said.\nClashing priorities \nThe Cantrell administration formed the Lincoln Beach Community Advisory Committee to encourage the volunteers\u2019 commitment, even as officials tiptoe around the appearance of inviting liability. The idea is to consult with residents and report on the latest developments in planning.\nBut there hasn\u2019t been much to report over the last 1\u00bd years, said Blyss Wallace, the group\u2019s president. Getting clear information has been \u201clike pulling teeth,\u201d she said.\nA sign asks visitors to dispose of trash at Lincoln Beach in New Orleans on Tuesday, Aug. 30, 2022.\nSTAFF PHOTO BY SOPHIA GERMER\nStill, Wallace said she is hopeful for renewed momentum, and better relations with city officials, after the City Council committee meeting on Wednesday. That will start with clearing the trash, which officials finally agreed to help with this month.\nFollowing through on that promise will show good faith, Wallace said.\n\u201cEverybody's so zoned in on [the garbage], and with every right to be,\u201d Wallace said. \u201cRemoving that will show that they're actually working with us.\u201d\nFears of getting left behind\nPart of the reason for concerns about Lincoln Beach\u2019s progress relates to the news last month that Pontchartrain Beach had been turned over to a nonprofit for redevelopment, with plans for a wetlands preserve, dog park and a small marina near the University of New Orleans.\nPontchartrain Beach was a popular attraction but was off limits to Black residents during the Jim Crow era. Lincoln Beach, located far from where most people lived at the time, was offered up as a consolation prize. It was hard to reach, but it hosted a full-fledged amusement park, with a Ferris wheel, roller coaster, swimming pools and a performance venue that drew big-name artists.\nReggie Ford picks up concrete blocks so visitors do not hurt themselves near a sailboat that washed up at Lincoln Beach in New Orleans on Tuesday, Aug. 30, 2022.\nSTAFF PHOTO BY SOPHIA GERMER\nCity Hall left Lincoln Beach to nature once Pontchartrain Beach was desegregated with the 1964 Civil Rights Act. But even with the end of legal segregation at Pontchartrain Beach Black residents were never entirely welcome there.\nSome Black New Orleanians worry that Lincoln Beach will get left behind again, said Dawn Hebert, president of the East New Orleans Neighborhood Advisory Commission.\n\u201cLincoln Beach is more of an African American historical site that the community wants to restore,\u201d Hebert said. \u201cThe city just decided to abandon Lincoln Beach completely.\u201d\nEmail Ben Myers at [email protected]. Follow Ben Myers on Twitter, @blevimyers.\nFacebook\nTwitter\nEmail\nPrint\nCopy article link\nSave\nThis Day in History\nSponsored by Connatix\nRecommended for you\nRecommended by\nSECTIONS\nHOME\nNEWS\nOPINION\nSPORTS\nENTERTAINMENT/LIFE\nNEWSLETTERS\nGAMES\nSERVICES\nCLASSIFIEDS\nSEARCH\nSUBSCRIBE | GROUPS\nDIGITAL ADVERTISING\nHELP/CONTACT US\nRSS FEEDS\nMEDIA KIT\nEEDITION\nCAREERS\nTEACHER'S LINK\nREVIEWING THE RECORD\nOUR SITES\nOBITUARIES\nJOBS\nCELEBRATIONS\nNIE\nCLASSIFIEDS\nHOMES\nPETS\nARCHIVES\nSTORE\nCONTACT INFORMATION\nnola.com\n840 St. Charles Avenue\nNew Orleans, LA 70130\nPhone: 504-529-0522\n\nNews Tips:\[email protected]\nOther questions:\[email protected]\nNeed help?\nReport a delivery issue\nCreate a temporary stop\nSign up for recurring payments\nPay your bill\nUpdate your billing info\n\n\u00a9 Copyright 2023 NOLA.com 840 St. Charles Avenue, New Orleans, LA  | Terms of Use | Privacy Policy\nPowered by BLOX Content Management System from BLOX Digital.\nThis website stores data such as cookies to enable essential website functionality, marketing, personalization and analytics. By remaining on this website you indicate your consent. See updated terms and conditions.\n ",
+            "url": "https://www.nola.com/news/politics/at-lincoln-beach-volunteer-caretakers-plead-for-help-from-new-orleans-city-hall/article_2cd4c6e4-2b98-11ed-b0b7-1b46c1cb095d.html",
+            "title": "At lincoln beach volunteer caretakers plead for help from new orleans city hall"
+        }
+    ]
+}

packages/backend/src/minutes_agendas_directory/Cantrell administration should boost transparency on 388 million in federal funds BGR says.json

@@ -0,0 +1,9 @@
+{
+    "messages": [
+        {
+            "page_content": "Skip to main content\nHomes\nCalendar\nStore\nPublic Notices\nE-Edition\nNewsletters\nSubscribe for $1\nCantrell administration should boost transparency on $388 million in federal funds, BGR says\nBY MATT SLEDGE | Staff writer\nDec 18, 2022\n4 min to read\n1 of 8\nMayor LaToya Cantrell answers questions during the Community Budget Meeting at Lakeview Christian Center in New Orleans, Thursday, Aug. 18, 2022. (Photo by Sophia Germer, NOLA.com, The Times-Picayune | The New Orleans Advocate)\nSophia Germer\nFacebook\nTwitter\nEmail\nPrint\nCopy article link\nSave\nWhen the New Orleans City Council passed a massive, $262 million amendment to the 2023 budget in the waning hours before a Dec. 1 deadline, local activists were surprised.\nThere\u2019d been no formal notice that Mayor LaToya Cantrell and the council had hammered out a deal to tap hundreds of millions of dollars in federal pandemic relief funds and other money, said Maxwell Ciardullo, the spokesperson for the Louisiana Fair Housing Action Center.\n\u201cEveryone was totally caught off guard,\u201d Ciardullo said.\nNow, organizers are preparing for another debate early next year over how to use the rest of the city\u2019s one-time funds.\nWhen that happens, the Cantrell administration should open the curtain on how it is spending American Rescue Plan Act funds and a huge general fund balance, the Bureau of Governmental Research said in a report this week.\nThat report faults the administration for a lack of transparency thus far. But it also notes that there is time for a course correction given the huge sums yet to be spent.\nA growing surplus\nWhen the pandemic walloped the city\u2019s tourism industry in 2020, the Cantrell administration prepared for a downturn in sales and hotel taxes for years to come. The city planned employee furloughs in 2021 that would save $26 million, plus another $92 million in general budget cuts, according to the BGR report.\nHowever, in March 2021, the U.S. Congress passed into law the American Rescue Plan Act, which was designed to provide a lifeline to local governments. Suddenly, New Orleans was in line to receive $388 million in federal funds in two installments.\nThe federal law placed some limits on how local governments could spend the money. The city plugged $187 million from the first installment into the police and fire departments, freeing up general fund dollars that would have been spent there as the city saw fit.\nThat didn\u2019t violate federal law. But the fact that the city only explained how it was using the money on the front end in an online dashboard, the Bureau of Governmental Research said, \u201cdoes not provide the public with the information necessary to understand ARPA\u2019s real impact on funding for other departments or the city\u2019s finances.\"\nMayor LaToya Cantrell answers questions during the Community Budget Meeting at Lakeview Christian Center in New Orleans, Thursday, Aug. 18, 2022. (Photo by Sophia Germer, NOLA.com, The Times-Picayune | The New Orleans Advocate)\nSophia Germer\nIn addition to the federal aid, another factor bulked up the city\u2019s bank account. Hundreds of unfilled positions, many in the New Orleans Police Department, contributed to a huge balance of leftover funds at the end of 2021. In total, the city spent $532 million in 2021, $100 million short of its trimmed-down budget.\nThe BGR report found that even without the federal funds, last year the city would have been able to meet all of its expenses while generating a $25 million surplus. In effect, the federal relief dollars helped grow the city's fund balance.\nThe Cantrell administration has portrayed its conservative budgeting as a prudent response to the uncertainty of the pandemic. Even after Congress passed ARPA it warned of budget shortfalls lasting into 2025. It has also said that the savings have allowed it to sock money into a much-needed rainy-day fund. Yet the drop in spending coincided with widespread complaints about the quality of municipal services.\nRebecca Mowbray, the president and CEO of BGR, said the group didn\u2019t try to determine how the two factors were related.\n\u201cOur task here really was just to follow the money and see where it went. We didn\u2019t so much make judgments about how they were spending it,\u201d she said.\nSpending the pot\nThis summer, the federal government sent the city its second, $194 million block of pandemic relief funds. Heading into the November budget season, Cantrell held a series of town halls.\nIn some settings, Cantrell didn't typically distinguish between sources of funds and the timing of City Council votes. But administration officials also stated that the city would allocate general funds through the ordinary budget process before turning to the federal dollars.\nInstead, the Cantrell administration and the City Council passed the last-minute amendments to the mayor\u2019s proposed budget that caught organizers off-guard. Those amendments allocated $124 million in ARPA funds and $151 million in fund balance dollars, according to BGR.\n\u201cUltimately, they went ahead and passed everything all at once, not really giving citizens the opportunity to know the details,\u201d said Susie Dudis, a BGR research analyst.\nIn a statement, a city spokesperson pushed back at the idea that there wasn\u2019t advance notice, pointing to the town halls and budget hearings. John Lawson, the spokesperson, said the amendment included priorities that had emerged as consensus priorities, like fighting crime and blight.\n\u201cThe first set of appropriations invested in projects and initiatives in response to what we heard from the public as core/vital basic needs,\u201d said Lawson.\nGilbert Monta\u00f1o, bottom center, chief administrative officer for New Orleans, talks about the details of Mayor LaToya Cantrell's 2023 operating budget to the New Orleans City Council on Tuesday, October 25, 2022 at City Hall. (Photo by Chris Granger | The Times-Picayune | The New Orleans Advocate)\nSTAFF PHOTO BY CHRIS GRANGER\nThe budget amendment, unlike previous appropriations, sends the federal relief funds to specific departments rather than flowing it through police and fire. But even with that added level of detail, BGR\u2019s report faulted the administration for falling short on transparency.\nA $5 million outlay for \u201cunhoused populations,\u201d for example, was described in an internal memo as going to a \u201cfull-time staff member and consulting team to develop a comprehensive program to support unhoused populations in the city.\u201d\nThe BGR, Dudis said, is \u201choping we will see more of the actual plan details... their time frame, what are the objectives, what is the population of people being served. That, we don\u2019t have at this point.\u201d\nMoney in the bank\nDistrict A Council member Joe Giarrusso, the chair of the City Council budget committee, agreed with the Cantrell administration that the budget amendments reflected a broad consensus. But he also agreed with BGR that the city and council could have done a better job of communicating how they planned to allocate ARPA funds.\nThe issues raised in the BGR report, he said, pointed to the larger problem of compressing complex budget discussions into November. He favors expanding budget season to include October.\nGiarrusso said he expects to hold hearings on how to spend the remainder of one-time dollars from ARPA and the fund balance in January and February. That amounts to about $70 million in federal dollars plus at least as many fund-balance dollars, he\u2019s said previously.\nCiardullo and other activists are pushing for the city to spend the money on a wide variety of needs. His group\u2019s priority is up to $90 million to tackle what he called \u201ca tremendous shortage of affordable rental housing.\u201d\nGiarrusso said that under a new approach he is pioneering next year, there will also be quarterly hearings on how the city is spending its money. The Cantrell administration said that it will also be expanding an online dashboard in early 2023 to include spending outcomes and the use of fund balance dollars.\nEmail Matt Sledge at [email protected].\nFacebook\nTwitter\nEmail\nPrint\nCopy article link\nSave\nThis Day in History\nSponsored by Connatix\nRecommended for you\nRecommended by\nSECTIONS\nHOME\nNEWS\nOPINION\nSPORTS\nENTERTAINMENT/LIFE\nNEWSLETTERS\nGAMES\nSERVICES\nCLASSIFIEDS\nSEARCH\nSUBSCRIBE | GROUPS\nDIGITAL ADVERTISING\nHELP/CONTACT US\nRSS FEEDS\nMEDIA KIT\nEEDITION\nCAREERS\nTEACHER'S LINK\nREVIEWING THE RECORD\nOUR SITES\nOBITUARIES\nJOBS\nCELEBRATIONS\nNIE\nCLASSIFIEDS\nHOMES\nPETS\nARCHIVES\nSTORE\nCONTACT INFORMATION\nnola.com\n840 St. Charles Avenue\nNew Orleans, LA 70130\nPhone: 504-529-0522\n\nNews Tips:\[email protected]\nOther questions:\[email protected]\nNeed help?\nReport a delivery issue\nCreate a temporary stop\nSign up for recurring payments\nPay your bill\nUpdate your billing info\n\n\u00a9 Copyright 2023 NOLA.com 840 St. Charles Avenue, New Orleans, LA  | Terms of Use | Privacy Policy\nPowered by BLOX Content Management System from BLOX Digital.\nThis website stores data such as cookies to enable essential website functionality, marketing, personalization and analytics. By remaining on this website you indicate your consent. See updated terms and conditions.\n ",
+            "url": "https://www.nola.com/news/politics/cantrell-administration-should-boost-transparency-on-388-million-in-federal-funds-bgr-says/article_6b591a1a-7d7c-11ed-b58f-0371d99e2538.html",
+            "title": "Cantrell administration should boost transparency on 388 million in federal funds BGR says"
+        }
+    ]
+}