Create and use a separate table (parts_by_lcsc) to index FTS5 parts table by rowid to speed up.

library.py

@@ -382,6 +382,26 @@ def get_part_details(self, lcsc):
         with contextlib.closing(sqlite3.connect(self.partsdb_file)) as con, con as cur:
             numbers = ",".join([f'"{n}"' for n in lcsc])
 
+            # try retrieving from the cached index first (LCSC Part indexing from FTS5 parts is sloooooow)
+            try:
+                rows = cur.execute(f"SELECT lcsc, partsId FROM parts_by_lcsc where lcsc IN ({numbers})").fetchall()
+
+                # orphaned parts found
+                if len(rows) != len(lcsc):
+                    rowid_by_lcsc = dict(rows)
+                    for lc in lcsc:
+                        if lc not in rowid_by_lcsc:
+                            self.logger.debug("LCSC Part `%s` not found in the database.", lc)
+
+                numbers = ",".join([f'"{r[0]}"' for r in rows])
+                return cur.execute(
+                    f'SELECT "LCSC Part", "Stock", "Library Type" FROM parts where rowid IN ({numbers})'
+                ).fetchall()
+            except Exception as e:
+                self.logger.debug(repr(e))
+                pass
+
+            # fall back to the direct approach
             try:
                 return cur.execute(
                     f'SELECT "LCSC Part", "Stock", "Library Type" FROM parts where "LCSC Part" IN ({numbers})'
@@ -530,6 +550,30 @@ def download(self):
                 self.create_tables(["placeholder_invalid_column_fix_errors"])
                 return
         else:
+            self.logger.debug("Indexing parts table...")
+            wx.PostEvent(self.parent, UpdateGaugeEvent(value=0))
+            with contextlib.closing(sqlite3.connect(self.partsdb_file)) as con:
+                con.execute("DROP TABLE IF EXISTS parts_by_lcsc;")
+                con.execute("CREATE TABLE IF NOT EXISTS parts_by_lcsc (partsId INTEGER, lcsc TEXT);")
+                con.execute("DROP INDEX IF EXISTS LCSCpartIdx;")
+                cur = con.execute("SELECT rowid, `LCSC Part` FROM parts")
+                indexedParts = cur.fetchall()
+                howMany = len(indexedParts)
+                progress = 0
+                for i, r in enumerate(indexedParts):
+                    con.execute("INSERT OR REPLACE INTO parts_by_lcsc (partsId, lcsc) VALUES (?, ?)", (r[0], r[1]))
+                    p = int(i / howMany * 100)
+                    if p > progress:
+                        wx.PostEvent(self.parent, UpdateGaugeEvent(value=p))
+                        progress = p
+
+                wx.PostEvent(self.parent, UpdateGaugeEvent(value=100))
+                self.logger.debug("Finalising database index...")
+                con.commit()
+                con.execute("CREATE INDEX IF NOT EXISTS LCSCpartIdx ON parts_by_lcsc(lcsc);")
+                con.commit()
+                self.logger.debug("Indexing parts table done.")
+
             wx.PostEvent(self.parent, ResetGaugeEvent())
             end = time.time()
             wx.PostEvent(self.parent, PopulateFootprintListEvent())