feat: fixed graph populating

src/components/App/index.tsx

@@ -80,6 +80,7 @@ export const App = () => {
     splashDataLoading,
     runningProjectId,
     setRunningProjectMessages,
+    isFetching,
   } = useDataStore((s) => s)
 
   const { setAiSummaryAnswer, getKeyExist, aiRefId } = useAiSummaryStore((s) => s)
@@ -138,6 +139,10 @@ export const App = () => {
 
   const handleNewNodeCreated = useCallback(
     (data: FetchDataResponse) => {
+      if (isFetching) {
+        return
+      }
+
       if (!queueRef.current) {
         queueRef.current = { nodes: [], edges: [] }
       }
@@ -162,7 +167,7 @@ export const App = () => {
         addNewNode(batchedData) // Call the original addNewNode function with batched data
       }, 3000) // Adjust delay as necessary
     },
-    [addNewNode],
+    [addNewNode, isFetching],
   )
 
   const handleAiSummaryAnswer = useCallback(

src/stores/useDataStore/index.ts

@@ -9,6 +9,18 @@ import { FilterParams, GraphData, Link, NodeExtended, NodeType, Sources, Trendin
 import { useAiSummaryStore } from '../useAiSummaryStore'
 import { useAppStore } from '../useAppStore'
 
+const deduplicateByRefId = (items) => {
+  const uniqueMap = new Map()
+
+  items.forEach((item) => {
+    if (!uniqueMap.has(item.ref_id)) {
+      uniqueMap.set(item.ref_id, item)
+    }
+  })
+
+  return Array.from(uniqueMap.values())
+}
+
 export type GraphStyle = 'sphere' | 'force' | 'split' | 'earth'
 
 export const graphStyles: GraphStyle[] = ['sphere', 'force', 'split', 'earth']
@@ -312,34 +324,48 @@ export const useDataStore = create<DataStore>()(
         return
       }
 
-      const currentNodes = [...(existingData?.nodes || [])]
-      const currentLinks = [...(existingData?.links || [])]
+      const uniqueIncomingNodes = deduplicateByRefId(data.nodes || [])
+      const uniqueIncomingEdges = deduplicateByRefId(data.edges || [])
 
-      const newNodes = (data?.nodes || []).filter((n) => !currentNodes.some((c) => c.ref_id === n.ref_id))
+      // Step 2: Existing nodes and links from the current state
+      const currentNodes = existingData?.nodes ? [...existingData.nodes] : []
+      const currentLinks = existingData?.links ? [...existingData.links] : []
 
-      currentNodes.push(...newNodes)
+      // Step 3: Use Sets to track unique ref_ids of existing data
+      const nodeRefIds = new Set(currentNodes.map((node) => node.ref_id))
+      const linkRefIds = new Set(currentLinks.map((link) => link.ref_id))
 
-      const newLinks = (data?.edges || [])
-        .filter((n) => !currentLinks.some((c) => c.ref_id === n.ref_id))
-        .filter((c) => {
-          const { target, source } = c
+      // Step 4: Filter new nodes and add only unique ones
+      const newNodes = uniqueIncomingNodes.filter((node) => !nodeRefIds.has(node.ref_id))
+      const updatedNodes = [...currentNodes, ...newNodes]
 
-          return currentNodes.some((n) => n.ref_id === target) && currentNodes.some((n) => n.ref_id === source)
-        })
+      // Update `nodeRefIds` with the new nodes added
+      newNodes.forEach((node) => nodeRefIds.add(node.ref_id))
 
-      currentLinks.push(...newLinks)
+      // Step 5: Filter new links based on unique ref_ids and node presence
+      const newLinks = uniqueIncomingEdges
+        .filter((link) => !linkRefIds.has(link.ref_id)) // Check for unique `ref_id`
+        .filter((link) => {
+          const { source, target } = link
+
+          return nodeRefIds.has(source) && nodeRefIds.has(target) // Ensure nodes exist
+        })
 
-      const nodeTypes = [...new Set(currentNodes.map((i) => i.node_type))]
+      const updatedLinks = [...currentLinks, ...newLinks]
 
-      const sidebarFilters = ['all', ...nodeTypes.map((i) => i.toLowerCase())]
+      // Step 6: Extract unique node types and create sidebar filters
+      const nodeTypes = [...new Set(updatedNodes.map((node) => node.node_type))]
+      const sidebarFilters = ['all', ...nodeTypes.map((type) => type.toLowerCase())]
 
+      // Step 7: Calculate sidebar filter counts
       const sidebarFilterCounts = sidebarFilters.map((filter) => ({
         name: filter,
-        count: currentNodes.filter((node) => filter === 'all' || node.node_type?.toLowerCase() === filter).length,
+        count: updatedNodes.filter((node) => filter === 'all' || node.node_type?.toLowerCase() === filter).length,
       }))
 
+      // Step 8: Update the state with the new data
       set({
-        dataInitial: { nodes: currentNodes, links: currentLinks },
+        dataInitial: { nodes: updatedNodes, links: updatedLinks },
         dataNew: { nodes: newNodes, links: newLinks },
         nodeTypes,
         sidebarFilters,