linearlite: make transaction batching configurable when loading seed …

…data (#2169)

This allows for easier benchmarking of transaction processing in
Electric.

examples/linearlite/db/load_data.js

@@ -7,6 +7,9 @@ if (!process.env.DATABASE_URL) {
 
 const DATABASE_URL = process.env.DATABASE_URL
 const ISSUES_TO_LOAD = process.env.ISSUES_TO_LOAD || 512
+const TX_BATCH_SIZE = process.env.TX_BATCH_SIZE
+  ? Number(process.env.TX_BATCH_SIZE)
+  : 10000
 const BATCH_SIZE = 1000
 const issues = generateIssues(ISSUES_TO_LOAD)
 
@@ -30,10 +33,13 @@ async function batchInsert(sql, table, columns, dataArray, batchSize = 1000) {
 const issueCount = issues.length
 let commentCount = 0
 
+// Insert all issues with comments in a single transaction when TX_BATCH_SIZE = 0
+const issueBatchSize = TX_BATCH_SIZE > 0 ? TX_BATCH_SIZE : issueCount
+
 try {
   // Process data in batches
-  for (let i = 0; i < issues.length; i += BATCH_SIZE) {
-    const issueBatch = issues.slice(i, i + BATCH_SIZE)
+  for (let i = 0; i < issues.length; i += issueBatchSize) {
+    const issueBatch = issues.slice(i, i + issueBatchSize)
 
     await sql.begin(async (sql) => {
       // Disable FK checks
@@ -46,21 +52,25 @@ try {
 
       // Insert related comments
       const batchComments = issueBatch.flatMap((issue) => issue.comments)
-      const commentColumns = Object.keys(batchComments[0])
-      await batchInsert(
-        sql,
-        'comment',
-        commentColumns,
-        batchComments,
-        BATCH_SIZE
-      )
+      if (batchComments.length > 0) {
+        const commentColumns = Object.keys(batchComments[0])
+        await batchInsert(
+          sql,
+          'comment',
+          commentColumns,
+          batchComments,
+          BATCH_SIZE
+        )
+      }
 
       commentCount += batchComments.length
     })
 
-    process.stdout.write(
-      `\nProcessed batch ${Math.floor(i / BATCH_SIZE) + 1}: ${Math.min(i + BATCH_SIZE, issues.length)} of ${issues.length} issues\n`
-    )
+    if (issueBatchSize < issues.length) {
+      process.stdout.write(
+        `Processed batch ${Math.floor(i / issueBatchSize) + 1}: ${Math.min(i + issueBatchSize, issues.length)} of ${issues.length} issues\n`
+      )
+    }
   }
 
   console.info(`Loaded ${issueCount} issues with ${commentCount} comments.`)