update arch diagram, fix search querying

backend/server.js

@@ -1,9 +1,12 @@
 import sqlite3 from 'sqlite3'
-import { open } from 'sqlite'
+import * as path from 'path'
 import * as dotenv from 'dotenv'
+import fs from 'fs/promises'
+import { open } from 'sqlite'
 dotenv.config()
 
 const db_path = process.env.DB_PATH
+const frigate_clips_path = process.env.FRIGATE_CLIPS_PATH
 
 let db
 
@@ -103,14 +106,23 @@ function max_of_tfidf_scores(events_scores) {
 fastify.get(
     '/events',
     async (request, reply) => {
-        // const { start_time, end_time } = request.query
-
-        const start_time = 1723259761
-        const end_time = 1724123761
-        let query = `SELECT e.id, e.*, t.transcript FROM event e LEFT JOIN transcribed t ON e.id = t.id WHERE e.start_time >= ${start_time} AND e.start_time <= ${end_time}`
+        const { start_time, end_time, limit, search_term } = request.query
+
+        let query = `
+        SELECT e.id, e.*, t.transcript
+        FROM event e
+        LEFT JOIN transcribed t ON e.id = t.id
+        WHERE e.start_time >= ${start_time} AND e.start_time <= ${end_time}
+    `
+
+    // Add search filter if search_expr is provided
+    if (search_term && search_term.trim() !== '') {
+        query += ` AND t.transcript LIKE '%${search_term.trim()}%'`
+    }
 
-        let results = await db.all(query)
+    query += ` ORDER BY RANDOM() LIMIT ${limit}`
 
+    let results = await db.all(query)
         // Reduce list of events into one object
         const events = results.reduce(
             (obj, event) => {
@@ -125,14 +137,40 @@ fastify.get(
         const total_tfidf_scores = max_of_tfidf_scores(Object.values(tfidf_results))
         const top_n = Object.keys(total_tfidf_scores).sort((a, b) => total_tfidf_scores[b] - total_tfidf_scores[a]).slice(0, 100)
 
-        // console.log(events)
         return {
             events,
             total_tfidf_scores,
             top_n
         }
     })
 
+    fastify.get('/events/:camera/:id/image', async (request, reply) => {
+        const { camera, id } = request.params;
+
+        try {
+          // Assuming your images are stored in an 'images' directory
+          const imagePath = path.join(frigate_clips_path, `${camera}-${id}-clean.png`);
+
+          // Read the file
+          const imageBuffer = await fs.readFile(imagePath);
+
+          // Convert buffer to base64
+          const base64Image = imageBuffer.toString('base64');
+
+          // Send the response
+          reply
+            .code(200)
+            .header('Content-Type', 'application/json')
+            .send({ image: base64Image });
+        } catch (error) {
+          console.error(`Error reading image for event ${id}:`, error);
+          reply
+            .code(404)
+            .send({ error: 'Image not found' });
+        }
+
+      });
+
 // Run the server!
 try {
     await fastify.listen({ port: 3000 })

frontend/src/App.vue

@@ -12,25 +12,21 @@ import WordCloud from './components/WordCloud.vue';
 import CameraRoll from './components/CameraRoll.vue';
 import Scrubber from './components/Scrubber.vue';
 
-import { onMounted, ref, watch } from 'vue';
+import { onMounted } from 'vue';
+import { storeToRefs } from 'pinia';
 
 import { useEventStore } from './stores/event_store';
 const store = useEventStore();
+const { start_time, end_time } = storeToRefs(store)
 
-const end_time = ref(0);
-const start_time = ref(0);
 
-end_time.value = Date.now() / 1000;
-start_time.value = end_time.value - 60 * 60 * 24;
 
-onMounted(async () => {
-    await store.fetchEvents(start_time.value, end_time.value);
-    return
-});
+// To set the time range:
+// store.setTimeRange(newStartTime, newEndTime)
 
-watch([start_time, end_time], async () => {
-    await store.fetchEvents(start_time.value, end_time.value);
-    console.log("start_time or end_time changed");
+onMounted(async () => {
+    // store.setTimeRange(start_time.value)
+    await store.fetchEvents();
     return
 });
 

frontend/src/components/CameraRoll.vue

@@ -3,40 +3,104 @@
 
     </div>
     <div class="image-grid">
-        <div class="image-item" v-for="(event_data, event_id, idx) in events" :key="event_id">
+        <div class="image-item" v-for="(event_data, event_id, idx) in store.events" :key="event_id"  @click="expandImage(event_id, event_data)">
             <img :src="`data:image/png;base64,${event_data.thumbnail}`" alt="event.name">
         </div>
     </div>
+    <div v-if="expandedImage" class="expanded-image-overlay" @click="closeExpandedImage">
+        <img :src="expandedImage" alt="Expanded Image">
+      <div v-if="expandedCaption" class="expanded-image-caption">
+        {{ expandedCaption }}
+      </div>
+    </div>
 </template>
 
 <script setup>
-import { ref, onMounted, watch, computed, defineProps } from 'vue';
+import { watch, ref } from 'vue';
 import { useEventStore } from '../stores/event_store';
 
-onMounted(async ()=> {
-    const store = useEventStore();
-    await store.fetchEvents();
+const store = useEventStore()
+const eventsRef = ref(null)
+const expandedImage = ref(null)
+const expandedCaption = ref(null)
+
+watch(store.events, (newEvents) => {
+    eventsRef.value = store.events
+});
 
-    const events = computed(() => store.events);
+const expandImage = async (event_id, event_data) => {
+    try {
+        const cameraName = event_data.camera
+        const imageData = await store.fetchImage(event_id, cameraName);
+        expandedImage.value = `data:image/png;base64,${imageData}`;
+        expandedCaption.value = event_data.transcript || 'No transcription available';
+    } catch (error) {
+        console.error('Error fetching full image:', error);
+    }
+}
 
-    watch(events, (newEvents) => {
-        // This will re-render the image grid when events change
-    });
-})
+const closeExpandedImage = () => {
+    expandedImage.value = null;
+    expandedCaption.value = null;
+}
 </script>
 
 <style scoped>
 .image-grid {
     width: 100%;
     display: grid;
-    /* justify-content: start; */
     grid-template-columns: repeat(auto-fill, minmax(60px, 1fr));
     gap: 1px;
 }
 
+.image-item {
+    cursor: pointer;
+}
+
 .image-item img {
     width: 100%;
     height: auto;
     display: block;
 }
+
+.expanded-image-overlay {
+  position: fixed;
+  display: flex;
+  flex-direction: column;
+  top: 0;
+  left: 0;
+  width: 100%;
+  height: 100%;
+  background-color: rgba(0, 0, 0, 0.8);
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  z-index: 1000;
+  cursor: pointer;
+}
+
+.expanded-image-container {
+  align-items: center;
+  max-width: 90%;
+  max-height: 90%;
+  background-color: rgba(0, 0, 0, 0.7);
+  padding: 20px;
+  border-radius: 10px;
+}
+
+.expanded-image-container img {
+  max-width: 100%;
+  max-height: calc(80vh - 100px);
+  object-fit: contain;
+  margin-bottom: 20px;
+}
+
+.expanded-image-caption {
+  color: white;
+  padding: 10px;
+  width: 80%;
+  text-align: center;
+  font-size: 16px;
+  line-height: 1.4;
+}
 </style>

frontend/src/components/Scrubber.vue

@@ -13,32 +13,43 @@ import { Timeline } from "vis-timeline";
 import { DataSet } from "vis-data";
 import { onMounted, ref, watch } from 'vue';
 import { useEventStore } from '../stores/event_store';
-
 // Get events from the event store
 const store = useEventStore();
+let container = null;
+let timeline = null;
+// Configuration for the Timeline
+const options = {
+    height: '150px',
+    width: '90vw'
+};
 
-function renderScrubber() {
+onMounted(() => {
     // DOM element where the Timeline will be attached
-    const container = document.getElementById('visualization');
+    container = document.getElementById('visualization');
+    timeline = new Timeline(container, null, options);
+
+    timeline.on("rangechanged", async e => {
+        store.setTimeRange(e.start.valueOf()/1000 , e.end.valueOf()/1000)
+        await store.fetchEvents();
+    });
+})
+
+// timeline.on
+
+function renderScrubber() {
     const events = store.events;
 
     // Create a DataSet for the timeline items
     const items = new DataSet(
         Object.values(events).map(event => ({
             id: event.id,
             content: event.label || 'No Label',
+            group:  event.label || 'No Label',
             start: new Date(event.start_time * 1000) // Convert Unix timestamp to Date object
         }))
     );
 
-    // Configuration for the Timeline
-    const options = {
-        height: '150px',
-        width: '90vw'
-    };
-
-    // Create and render the Timeline
-    new Timeline(container, items, options);
+    timeline.setData({items})
 }
 
 watch(() => store.events, (newDict) => {

frontend/src/components/WordCloud.vue

@@ -1,6 +1,6 @@
 <template>
   <div>
-    <input type="text" v-model="searchTerm" placeholder="Search..." />
+    <input type="text" v-model="localSearchTerm" placeholder="Search..." @input="updateSearchTerm" />
     <div ref="wordCloud" id="wordcloud" style="width:100%;  display: block"></div>
   </div>
 
@@ -12,20 +12,27 @@
   import cloud from 'd3-cloud';
 
   import { useEventStore } from '../stores/event_store';
+  import { storeToRefs } from 'pinia';
   const store = useEventStore();
+  const { start_time, end_time } = storeToRefs(store)
 
   // import transcribed from '../../../transcribed/analyze_2024-07-26T00:48:28.144473.json';
   // const props = defineProps(['total_tfidf_scores' , 'top_n', 'events']);
 
   const wordCloud = ref(null);
-  const searchTerm = ref('');
+  const localSearchTerm = ref('');
+
+  const updateSearchTerm = () => {
+    store.setSearchTerm(localSearchTerm.value)
+    store.debouncedFetchEvents();
+  };
 
   const drawWordCloud = (eventsDict) => {
     // let words = Object.values(eventsDict).reduce((acc: any, event: any) => acc += event)
     const layout = cloud()
       .size([window.innerWidth * 0.9, window.innerHeight * 0.28])
       // .spiral(rectangularSpiral)
-      .words(Object.keys(store.total_tfidf_scores).map(k => ({ text: k, size: (store.total_tfidf_scores[k]) * 300 })) )
+      .words(Object.keys(store.total_tfidf_scores).map(k => ({ text: k, size: ((store.total_tfidf_scores[k]) * 800) ^ 2 })) )
       .padding(0.2)
       .rotate(() => 0)
       .font('Impact')
@@ -35,9 +42,6 @@
     layout.start();
 
     function draw(words) {
-      console.log(words)
-
-
       d3.select(wordCloud.value)
         .append('svg')
         .attr('width', layout.size()[0])
@@ -55,23 +59,26 @@
         .style('width', '100%')
         .style('height', '20px')
         .text(d => d.text)
-        .on("click", function (d, i){
-            // window.open(d.url, "_blank");
-            console.log(d)
-            console.log(i)
+        .on("click", function (word, i){
+            store.setSearchTerm(i.text)
+            store.fetchEvents()
         });
     }
   };
 
   onMounted(() => {
     drawWordCloud(store.total_tfidf_scores);
+    localSearchTerm.value = store.search_term;
   });
 
   watch(() => store.total_tfidf_scores, (newDict) => {
     d3.select(wordCloud.value).selectAll('*').remove();
-    // console.log("Newdict" + Object.keys(newDict))
     drawWordCloud(store.total_tfidf_scores);
   });
+
+  watch(() => store.search_term, (newSearchTerm) => {
+    localSearchTerm.value = newSearchTerm;
+  });
   </script>
 
   <style scoped>