Loading faster, and a loading screen

demos/linearlite/db/generate_data.js

@@ -35,7 +35,7 @@ function generateIssue(kanbanKey) {
     username: faker.internet.userName(),
     comments: faker.helpers.multiple(
       () => generateComment(issueId, createdAt),
-      { count: faker.number.int({ min: 0, max: 10 }) }
+      { count: faker.number.int({ min: 0, max: 1 }) }
     ),
   }
 }

demos/linearlite/db/load_data.js

@@ -34,7 +34,7 @@ try {
   // Process data in batches
   for (let i = 0; i < issues.length; i += BATCH_SIZE) {
     const issueBatch = issues.slice(i, i + BATCH_SIZE)
-    
+
     await sql.begin(async (sql) => {
       // Disable FK checks
       await sql`SET CONSTRAINTS ALL DEFERRED`
@@ -47,12 +47,20 @@ try {
       // Insert related comments
       const batchComments = issueBatch.flatMap((issue) => issue.comments)
       const commentColumns = Object.keys(batchComments[0])
-      await batchInsert(sql, 'comment', commentColumns, batchComments, BATCH_SIZE)
+      await batchInsert(
+        sql,
+        'comment',
+        commentColumns,
+        batchComments,
+        BATCH_SIZE
+      )
 
       commentCount += batchComments.length
     })
 
-    process.stdout.write(`\nProcessed batch ${Math.floor(i / BATCH_SIZE) + 1}: ${Math.min(i + BATCH_SIZE, issues.length)} of ${issues.length} issues\n`)
+    process.stdout.write(
+      `\nProcessed batch ${Math.floor(i / BATCH_SIZE) + 1}: ${Math.min(i + BATCH_SIZE, issues.length)} of ${issues.length} issues\n`
+    )
   }
 
   console.info(`Loaded ${issueCount} issues with ${commentCount} comments.`)

demos/linearlite/db/migrations-client/01-create_tables.sql

@@ -15,10 +15,6 @@ CREATE TABLE IF NOT EXISTS "issue" (
     "sent_to_server" BOOLEAN NOT NULL DEFAULT FALSE, -- Flag to track if the row has been sent to the server
     "synced" BOOLEAN GENERATED ALWAYS AS (ARRAY_LENGTH(modified_columns, 1) IS NULL AND NOT deleted AND NOT new) STORED,
     "backup" JSONB, -- JSONB column to store the backup of the row data for modified columns
-    "search_vector" tsvector GENERATED ALWAYS AS (
-        setweight(to_tsvector('simple', coalesce(title, '')), 'A') ||
-        setweight(to_tsvector('simple', coalesce(description, '')), 'B')
-    ) STORED,
     CONSTRAINT "issue_pkey" PRIMARY KEY ("id")
 );
 
@@ -39,20 +35,8 @@ CREATE TABLE IF NOT EXISTS "comment" (
 );
 
 CREATE INDEX IF NOT EXISTS "issue_id_idx" ON "issue" ("id");
-CREATE INDEX IF NOT EXISTS "issue_priority_idx" ON "issue" ("priority");
-CREATE INDEX IF NOT EXISTS "issue_status_idx" ON "issue" ("status");
-CREATE INDEX IF NOT EXISTS "issue_modified_idx" ON "issue" ("modified");
-CREATE INDEX IF NOT EXISTS "issue_created_idx" ON "issue" ("created");
-CREATE INDEX IF NOT EXISTS "issue_kanbanorder_idx" ON "issue" ("kanbanorder");
-CREATE INDEX IF NOT EXISTS "issue_deleted_idx" ON "issue" ("deleted");
-CREATE INDEX IF NOT EXISTS "issue_synced_idx" ON "issue" ("synced");
-CREATE INDEX IF NOT EXISTS "issue_search_idx" ON "issue" USING GIN ("search_vector");
 
 CREATE INDEX IF NOT EXISTS "comment_id_idx" ON "comment" ("id");
-CREATE INDEX IF NOT EXISTS "comment_issue_id_idx" ON "comment" ("issue_id");
-CREATE INDEX IF NOT EXISTS "comment_created_idx" ON "comment" ("created");
-CREATE INDEX IF NOT EXISTS "comment_deleted_idx" ON "comment" ("deleted");
-CREATE INDEX IF NOT EXISTS "comment_synced_idx" ON "comment" ("synced");
 
 -- During sync the electric.syncing config var is set to true
 -- We can use this in triggers to determine the action that should be performed
@@ -297,3 +281,7 @@ $$ LANGUAGE plpgsql;
 -- Example usage:
 -- SELECT revert_local_changes('issue', '123e4567-e89b-12d3-a456-426614174000');
 -- SELECT revert_local_changes('comment', '123e4567-e89b-12d3-a456-426614174001');
+
+
+ALTER TABLE issue DISABLE TRIGGER ALL;
+ALTER TABLE comment DISABLE TRIGGER ALL;

demos/linearlite/db/migrations-client/post-initial-sync-fts-index.sql

@@ -0,0 +1 @@
+CREATE INDEX IF NOT EXISTS "issue_search_idx" ON "issue" USING GIN ((setweight(to_tsvector('simple', coalesce(title, '')), 'A') || setweight(to_tsvector('simple', coalesce(description, '')), 'B')));

demos/linearlite/db/migrations-client/post-initial-sync-indexes.sql

@@ -0,0 +1,11 @@
+CREATE INDEX IF NOT EXISTS "issue_priority_idx" ON "issue" ("priority");
+CREATE INDEX IF NOT EXISTS "issue_status_idx" ON "issue" ("status");
+CREATE INDEX IF NOT EXISTS "issue_modified_idx" ON "issue" ("modified");
+CREATE INDEX IF NOT EXISTS "issue_created_idx" ON "issue" ("created");
+CREATE INDEX IF NOT EXISTS "issue_kanbanorder_idx" ON "issue" ("kanbanorder");
+CREATE INDEX IF NOT EXISTS "issue_deleted_idx" ON "issue" ("deleted");
+CREATE INDEX IF NOT EXISTS "issue_synced_idx" ON "issue" ("synced");
+CREATE INDEX IF NOT EXISTS "comment_issue_id_idxx" ON "comment" ("issue_id");
+CREATE INDEX IF NOT EXISTS "comment_created_idx" ON "comment" ("created");
+CREATE INDEX IF NOT EXISTS "comment_deleted_idx" ON "comment" ("deleted");
+CREATE INDEX IF NOT EXISTS "comment_synced_idx" ON "comment" ("synced");

demos/linearlite/src/App.tsx

@@ -20,6 +20,9 @@ import {
   FilterState,
 } from './utils/filterState'
 import { Issue as IssueType, Status, StatusValue } from './types/types'
+import { startSync, useSyncStatus, waitForInitialSyncDone } from './sync'
+import { electricSync } from '@electric-sql/pglite-sync'
+import { ImSpinner8 } from 'react-icons/im'
 
 interface MenuContextInterface {
   showMenu: boolean
@@ -30,13 +33,38 @@ export const MenuContext = createContext(null as MenuContextInterface | null)
 
 type PGliteWorkerWithLive = PGliteWorker & { live: LiveNamespace }
 
-const pgPromise = PGliteWorker.create(new PGWorker(), {
-  extensions: {
-    live,
-  },
+async function createPGlite() {
+  return PGliteWorker.create(new PGWorker(), {
+    extensions: {
+      live,
+      sync: electricSync({ debug: true }),
+    },
+  })
+}
+
+const pgPromise = createPGlite()
+
+let syncStarted = false
+pgPromise.then(async (pg) => {
+  console.log('PGlite worker started', pg.isLeader)
+  pg.onLeaderChange(() => {
+    console.log('Leader changed', pg.isLeader)
+    if (pg.isLeader && !syncStarted) {
+      syncStarted = true
+      startSync(pg)
+    }
+  })
+})
+
+let resolveFirstLoaderPromise: (value: void | PromiseLike<void>) => void
+const firstLoaderPromise = new Promise<void>((resolve) => {
+  resolveFirstLoaderPromise = resolve
 })
 
 async function issueListLoader({ request }: { request: Request }) {
+  console.log('issueListLoader: waiting for initial sync to complete')
+  await waitForInitialSyncDone()
+  console.log('issueListLoader: initial sync complete')
   const pg = await pgPromise
   const url = new URL(request.url)
   const filterState = getFilterStateFromSearchParams(url.searchParams)
@@ -48,10 +76,13 @@ async function issueListLoader({ request }: { request: Request }) {
     offset: 0,
     limit: 100,
   })
+  console.log('issueListLoader: liveIssues', liveIssues)
+  resolveFirstLoaderPromise()
   return { liveIssues, filterState }
 }
 
 async function boardIssueListLoader({ request }: { request: Request }) {
+  await waitForInitialSyncDone()
   const pg = await pgPromise
   const url = new URL(request.url)
   const filterState = getFilterStateFromSearchParams(url.searchParams)
@@ -78,6 +109,8 @@ async function boardIssueListLoader({ request }: { request: Request }) {
     columnsLiveIssues[status] = colLiveIssues
   }
 
+  resolveFirstLoaderPromise()
+
   return {
     columnsLiveIssues: columnsLiveIssues as Record<
       StatusValue,
@@ -132,21 +165,52 @@ const router = createBrowserRouter([
   },
 ])
 
+const LoadingScreen = ({ children }: { children: React.ReactNode }) => {
+  return (
+    <div className="h-screen w-screen flex flex-col items-center justify-center gap-4">
+      <ImSpinner8 className="w-8 h-8 animate-spin text-blue-500" />
+      <div className="text-gray-600 text-center" style={{ minHeight: '100px' }}>
+        {children}
+      </div>
+    </div>
+  )
+}
+
 const App = () => {
   const [showMenu, setShowMenu] = useState(false)
   const [pgForProvider, setPgForProvider] =
     useState<PGliteWorkerWithLive | null>(null)
+  const [syncStatus, syncMessage] = useSyncStatus()
+  const [firstLoaderDone, setFirstLoaderDone] = useState(false)
 
   useEffect(() => {
     pgPromise.then(setPgForProvider)
   }, [])
 
+  useEffect(() => {
+    if (firstLoaderDone) return
+    firstLoaderPromise.then(() => {
+      setFirstLoaderDone(true)
+    })
+  }, [firstLoaderDone])
+
   const menuContextValue = useMemo(
     () => ({ showMenu, setShowMenu }),
     [showMenu]
   )
 
-  if (!pgForProvider) return <div>Loading...</div>
+  if (!pgForProvider) return <LoadingScreen>Starting PGlite...</LoadingScreen>
+
+  if (syncStatus === 'initial-sync')
+    return (
+      <LoadingScreen>
+        Performing initial sync...
+        <br />
+        {syncMessage}
+      </LoadingScreen>
+    )
+
+  if (!firstLoaderDone) return <LoadingScreen>Loading...</LoadingScreen>
 
   return (
     <PGliteProvider db={pgForProvider}>

demos/linearlite/src/components/TopFilter.tsx

@@ -1,13 +1,14 @@
 import { ReactComponent as MenuIcon } from '../assets/icons/menu.svg'
-import { useState, useContext } from 'react'
+import { useState, useContext, useEffect } from 'react'
 import { BsSortUp, BsPlus, BsX, BsSearch as SearchIcon } from 'react-icons/bs'
-import { useLiveQuery } from '@electric-sql/pglite-react'
+import { useLiveQuery, usePGlite } from '@electric-sql/pglite-react'
 import ViewOptionMenu from './ViewOptionMenu'
 import { MenuContext } from '../App'
 import FilterMenu from './contextmenu/FilterMenu'
 import { FilterState, useFilterState } from '../utils/filterState'
 import { PriorityDisplay, StatusDisplay } from '../types/types'
 import debounce from 'lodash.debounce'
+import { createFTSIndex } from '../migrations'
 
 interface Props {
   filteredIssuesCount: number
@@ -28,6 +29,8 @@ export default function ({
   const [showViewOption, setShowViewOption] = useState(false)
   const { showMenu, setShowMenu } = useContext(MenuContext)!
   const [searchQuery, setSearchQuery] = useState(``)
+  const [FTSIndexReady, setFTSIndexReady] = useState(true)
+  const pg = usePGlite()
 
   filterState ??= usedFilterState
 
@@ -63,6 +66,24 @@ export default function ({
     }
   }
 
+  useEffect(() => {
+    if (!showSearch) return
+    const checkFTSIndex = async () => {
+      const res = await pg.query(
+        `SELECT 1 FROM pg_indexes WHERE indexname = 'issue_search_idx';`
+      )
+      const indexReady = res.rows.length > 0
+      if (!indexReady) {
+        setFTSIndexReady(false)
+        await createFTSIndex(pg)
+      }
+      setFTSIndexReady(true)
+    }
+    checkFTSIndex()
+  }, [showSearch, pg])
+
+  const showFTSIndexProgress = showSearch && !FTSIndexReady
+
   return (
     <>
       <div className="flex justify-between flex-shrink-0 pl-2 pr-6 border-b border-gray-200 h-14 lg:pl-9">
@@ -177,6 +198,15 @@ export default function ({
         </div>
       )}
 
+      {showFTSIndexProgress && (
+        <div className="flex items-center justify-between flex-shrink-0 pl-2 pr-6 border-b border-gray-200 lg:pl-9 py-2">
+          <div className="flex items-center gap-2">
+            <div className="animate-spin h-4 w-4 border-2 border-gray-500 rounded-full border-t-transparent"></div>
+            <span>Building full text search index... (only happens once)</span>
+          </div>
+        </div>
+      )}
+
       <ViewOptionMenu
         isOpen={showViewOption}
         onDismiss={() => setShowViewOption(false)}

demos/linearlite/src/migrations.ts

@@ -1,6 +1,30 @@
-import type { PGlite } from '@electric-sql/pglite'
+import type { PGlite, PGliteInterface } from '@electric-sql/pglite'
 import m1 from '../db/migrations-client/01-create_tables.sql?raw'
+import postInitialSyncIndexes from '../db/migrations-client/post-initial-sync-indexes.sql?raw'
+import postInitialSyncFtsIndex from '../db/migrations-client/post-initial-sync-fts-index.sql?raw'
 
 export async function migrate(pg: PGlite) {
-  await pg.exec(m1)
+  const tables = await pg.query(
+    `SELECT table_name FROM information_schema.tables WHERE table_schema='public'`
+  )
+  if (tables.rows.length === 0) {
+    await pg.exec(m1)
+  }
+}
+
+export async function postInitialSync(pg: PGliteInterface) {
+  const commands = postInitialSyncIndexes
+    .split('\n')
+    .map((c) => c.trim())
+    .filter((c) => c.length > 0)
+  for (const command of commands) {
+    // wait 100ms between commands
+    console.time(`command: ${command}`)
+    await pg.exec(command)
+    console.timeEnd(`command: ${command}`)
+  }
+}
+
+export async function createFTSIndex(pg: PGliteInterface) {
+  await pg.exec(postInitialSyncFtsIndex)
 }