diff --git a/examples/linearlite/db/load_data.js b/examples/linearlite/db/load_data.js index 4a4f494e62..604b923b4f 100644 --- a/examples/linearlite/db/load_data.js +++ b/examples/linearlite/db/load_data.js @@ -7,6 +7,9 @@ if (!process.env.DATABASE_URL) { const DATABASE_URL = process.env.DATABASE_URL const ISSUES_TO_LOAD = process.env.ISSUES_TO_LOAD || 512 +const TX_BATCH_SIZE = process.env.TX_BATCH_SIZE + ? Number(process.env.TX_BATCH_SIZE) + : 10000 const BATCH_SIZE = 1000 const issues = generateIssues(ISSUES_TO_LOAD) @@ -30,10 +33,13 @@ async function batchInsert(sql, table, columns, dataArray, batchSize = 1000) { const issueCount = issues.length let commentCount = 0 +// Insert all issues with comments in a single transaction when TX_BATCH_SIZE = 0 +const issueBatchSize = TX_BATCH_SIZE > 0 ? TX_BATCH_SIZE : issueCount + try { // Process data in batches - for (let i = 0; i < issues.length; i += BATCH_SIZE) { - const issueBatch = issues.slice(i, i + BATCH_SIZE) + for (let i = 0; i < issues.length; i += issueBatchSize) { + const issueBatch = issues.slice(i, i + issueBatchSize) await sql.begin(async (sql) => { // Disable FK checks @@ -46,21 +52,25 @@ try { // Insert related comments const batchComments = issueBatch.flatMap((issue) => issue.comments) - const commentColumns = Object.keys(batchComments[0]) - await batchInsert( - sql, - 'comment', - commentColumns, - batchComments, - BATCH_SIZE - ) + if (batchComments.length > 0) { + const commentColumns = Object.keys(batchComments[0]) + await batchInsert( + sql, + 'comment', + commentColumns, + batchComments, + BATCH_SIZE + ) + } commentCount += batchComments.length }) - process.stdout.write( - `\nProcessed batch ${Math.floor(i / BATCH_SIZE) + 1}: ${Math.min(i + BATCH_SIZE, issues.length)} of ${issues.length} issues\n` - ) + if (issueBatchSize < issues.length) { + process.stdout.write( + `Processed batch ${Math.floor(i / issueBatchSize) + 1}: ${Math.min(i + issueBatchSize, issues.length)} of ${issues.length} issues\n` + ) + } } console.info(`Loaded ${issueCount} issues with ${commentCount} comments.`)