diff --git a/bank/README.md b/bank/README.md index 68233fe4..abf591b7 100644 --- a/bank/README.md +++ b/bank/README.md @@ -406,6 +406,43 @@ npm start DBOS Cloud comes with a [monitoring dashboard](https://docs.dbos.dev/cloud-tutorials/monitoring-dashboard) automatically. To find the URL for your dashboard, execute `npx dbos-cloud dashboard url`. The dashboard includes execution traces. +## A Demo of Workflow Recovery + +We can simulate a catastrophic failure during a transfer. Namely the following case: +1. we try to send a transfer from Bank A to Bank B +1. we make the transfer fail, as if bank B went offline +2. we crash Bank A before it has a chance to recover and undo the transfer + +DBOS workflows are guaranteed to continue where they left off. This means that when Bank A is restarted, it will continue undoing the transfer. Shortly after restart, Bank A returns to a consistent state with the funds back in the source account. + +To replicate this, perform the following: +1. in `withdrawWorkflow` (bank-backend/src/workflows/txnhistory.workflows.ts) uncomment the sleep block like so: +```ts + if (!remoteRes) { + // Sleep for 10 seconds before undoing the transaction + for (let i = 0; i < 10; i++) { + ctxt.logger.info("Sleeping") + await ctxt.sleepms(1000) + } + + // Undo withdrawal with a deposit. + const undoRes = await ctxt.invoke(BankTransactionHistory).updateAcctTransactionFunc(data.fromAccountId, data, true, result); + if (undoRes !== result) { + throw new Error(`Mismatch: Original txnId: ${result}, undo txnId: ${undoRes}`); + } + throw new Error("Failed to deposit to remote bank; transaction reversed"); + } +``` +2. Restart or redeploy Bank A with this change. +3. Stop Bank B: `CTRL+C` the app if running locally or something like `npx dbos-cloud app delete bank_b` if in the cloud. + +This demo is time sensitive as you'll have a 10-second sleep window to crash Bank A. Adjust the above `sleep` loop if you nee more time. Read these steps ahead of time to get a sense for what to do: +1. Initiate a withdrawal from Bank A to Bank B +2. Because of the above `sleep` loop, you won't see the effect. You can quickly close the "Withdraw" window and refresh the browser to see that the funds left the account. Now, quickly press the red "Crash!" button. +3. If running in DBOS cloud, wait a few seconds. If you are running locally, restart Bank A by hand. +4. After restarting the app, log in again to `https://localhost:8089`. Observe the funds returning to the sender account. +5. In the log for the app, you should see the failed transfer. The 10 `Sleeping` statements, interrupted by a crash, then restart, workflow recovery and finally the "transaction reversed" error message. + ## Further Reading To get started with DBOS Transact, check out the [quickstart](https://docs.dbos.dev/getting-started/quickstart) and [docs](https://docs.dbos.dev/). diff --git a/bank/bank-backend/package-lock.json b/bank/bank-backend/package-lock.json index bcffad2e..41d77a12 100644 --- a/bank/bank-backend/package-lock.json +++ b/bank/bank-backend/package-lock.json @@ -9,7 +9,7 @@ "version": "0.0.2", "license": "ISC", "dependencies": { - "@dbos-inc/dbos-sdk": "^1.15.9", + "@dbos-inc/dbos-sdk": "^1.16.12", "@koa/bodyparser": "^5.0.0", "@koa/cors": "^5.0.0", "@koa/router": "^12.0.0", @@ -771,12 +771,9 @@ } }, "node_modules/@dbos-inc/dbos-sdk": { - "version": "1.15.9", - "resolved": "https://registry.npmjs.org/@dbos-inc/dbos-sdk/-/dbos-sdk-1.15.9.tgz", - "integrity": "sha512-2omgN7TVdUpq6zFHQfbxMUvlCyi02aNNKa//4iTWCzQE9VychF9qgwpORjRh6WqiWVE0D41QewriVGFKURY1HA==", - "workspaces": [ - "packages/*" - ], + "version": "1.16.12", + "resolved": "https://registry.npmjs.org/@dbos-inc/dbos-sdk/-/dbos-sdk-1.16.12.tgz", + "integrity": "sha512-RifvaH7hfBii8qEts1nH/1w59lBe/vo5rXwBE6UE8S7b23W6I2HbouqwOAR7dGenXTMuuRPohLgmldoj86LNHQ==", "dependencies": { "@koa/bodyparser": "5.0.0", "@koa/cors": "5.0.0", diff --git a/bank/bank-backend/package.json b/bank/bank-backend/package.json index 9a683365..64ac545a 100644 --- a/bank/bank-backend/package.json +++ b/bank/bank-backend/package.json @@ -14,7 +14,7 @@ "license": "ISC", "private": true, "dependencies": { - "@dbos-inc/dbos-sdk": "^1.15.9", + "@dbos-inc/dbos-sdk": "^1.16.12", "@koa/bodyparser": "^5.0.0", "@koa/cors": "^5.0.0", "@koa/router": "^12.0.0", diff --git a/bank/bank-backend/src/router.ts b/bank/bank-backend/src/router.ts index bb41c6fa..7ba42c53 100644 --- a/bank/bank-backend/src/router.ts +++ b/bank/bank-backend/src/router.ts @@ -64,6 +64,16 @@ export class BankEndpoints { } } +// For demo purposes +export class CrashEndpoint { + @GetApi('/crash_application') + static async crashApplication(_ctx: HandlerContext) { + // For testing and demo purposes :) + process.exit(1); + return Promise.resolve(); + } +} + // Helper functions to convert to the correct data types. // Especially convert the bigint. export function convertTransactionHistory(data: TransactionHistory): TransactionHistory { diff --git a/bank/bank-backend/src/workflows/txnhistory.workflows.ts b/bank/bank-backend/src/workflows/txnhistory.workflows.ts index b8ea60ef..9cb336a8 100644 --- a/bank/bank-backend/src/workflows/txnhistory.workflows.ts +++ b/bank/bank-backend/src/workflows/txnhistory.workflows.ts @@ -226,7 +226,6 @@ export class BankTransactionHistory { static async withdrawWorkflow(ctxt: WorkflowContext, data: TransactionHistory) { // Withdraw first. const result = await ctxt.invoke(BankTransactionHistory).updateAcctTransactionFunc(data.fromAccountId, data, false); - // Then, contact remote DB to deposit. if (data.toLocation && !(data.toLocation === "cash") && !data.toLocation.startsWith(REMOTEDB_PREFIX)) { ctxt.logger.info("Deposit to another DB: " + data.toLocation + ", account: " + data.toAccountId); @@ -240,12 +239,26 @@ export class BankTransactionHistory { }; const remoteRes: boolean = await ctxt.invoke(BankTransactionHistory).remoteTransferComm(remoteUrl, thReq as TransactionHistory, ctxt.workflowUUID + '-deposit'); if (!remoteRes) { - // Undo transaction is a deposit. + + /////////////////////////////// + // Example sleep Window. For a reliability test, uncomment the below + // Then, start a transfer to a nonexistent bank, (i.e. stop bank b). + // Wait for app to go into sleep and then crash it. The DBOS workflow + // recovery will ensure the undo transaction below is executed when + // the app restarts + // + // for (let i = 0; i < 10; i++) { + // ctxt.logger.info("Sleeping") + // await ctxt.sleepms(1000) + // } + /////////////////////////////// + + // Undo withdrawal with a deposit. const undoRes = await ctxt.invoke(BankTransactionHistory).updateAcctTransactionFunc(data.fromAccountId, data, true, result); if (undoRes !== result) { throw new Error(`Mismatch: Original txnId: ${result}, undo txnId: ${undoRes}`); } - throw new Error("Failed to deposit to remote bank."); + throw new Error("Failed to deposit to remote bank; transaction reversed"); } } else { ctxt.logger.info("Deposit to: " + data.fromLocation); diff --git a/bank/bank-frontend/src/app/bank.component.ts b/bank/bank-frontend/src/app/bank.component.ts index 4ab94444..0ef8ea6c 100644 --- a/bank/bank-frontend/src/app/bank.component.ts +++ b/bank/bank-frontend/src/app/bank.component.ts @@ -20,7 +20,7 @@ import { OwnerNameDialogComponent } from './owner-name-dialog/owner-name-dialog. - + @@ -431,4 +431,9 @@ export class BankComponent { error: (err: any) => { this.bankmsg = 'Failed to transfer! '; } }); } + + //In response to the "Crash" button - for demo purposes + crashApp(){ + this._service.getResource(this.bankUrl + "/crash_application").subscribe() + } } diff --git a/bank/bank-frontend/src/styles.css b/bank/bank-frontend/src/styles.css index 34a3bdeb..68b24066 100644 --- a/bank/bank-frontend/src/styles.css +++ b/bank/bank-frontend/src/styles.css @@ -33,6 +33,18 @@ position: relative; } +.button-red { + margin: 10px; + padding: 15px 20px; + background-color: #ff1111; + color: white; + border: none; + cursor: pointer; + font-size: 16px; + border-radius: 10px; /* Rounded corners */ + position: relative; +} + .button-grey { margin: 10px; padding: 15px 20px;