From 341c41ba41d92d1b9c308300b3f6d7e090793e57 Mon Sep 17 00:00:00 2001 From: apoliakov Date: Fri, 5 Jul 2024 17:09:09 -0400 Subject: [PATCH 1/3] Add the crash button and instructions for a demo. The demo uses sleep and crash to show how DBOS workflow recovery works --- bank/README.md | 37 +++++++++++++++++++ bank/bank-backend/package-lock.json | 11 ++---- bank/bank-backend/package.json | 2 +- bank/bank-backend/src/router.ts | 9 +++++ .../src/workflows/txnhistory.workflows.ts | 19 ++++++++-- bank/bank-frontend/src/app/bank.component.ts | 7 +++- bank/bank-frontend/src/styles.css | 12 ++++++ 7 files changed, 85 insertions(+), 12 deletions(-) diff --git a/bank/README.md b/bank/README.md index 68233fe4..abf591b7 100644 --- a/bank/README.md +++ b/bank/README.md @@ -406,6 +406,43 @@ npm start DBOS Cloud comes with a [monitoring dashboard](https://docs.dbos.dev/cloud-tutorials/monitoring-dashboard) automatically. To find the URL for your dashboard, execute `npx dbos-cloud dashboard url`. The dashboard includes execution traces. +## A Demo of Workflow Recovery + +We can simulate a catastrophic failure during a transfer. Namely the following case: +1. we try to send a transfer from Bank A to Bank B +1. we make the transfer fail, as if bank B went offline +2. we crash Bank A before it has a chance to recover and undo the transfer + +DBOS workflows are guaranteed to continue where they left off. This means that when Bank A is restarted, it will continue undoing the transfer. Shortly after restart, Bank A returns to a consistent state with the funds back in the source account. + +To replicate this, perform the following: +1. in `withdrawWorkflow` (bank-backend/src/workflows/txnhistory.workflows.ts) uncomment the sleep block like so: +```ts + if (!remoteRes) { + // Sleep for 10 seconds before undoing the transaction + for (let i = 0; i < 10; i++) { + ctxt.logger.info("Sleeping") + await ctxt.sleepms(1000) + } + + // Undo withdrawal with a deposit. + const undoRes = await ctxt.invoke(BankTransactionHistory).updateAcctTransactionFunc(data.fromAccountId, data, true, result); + if (undoRes !== result) { + throw new Error(`Mismatch: Original txnId: ${result}, undo txnId: ${undoRes}`); + } + throw new Error("Failed to deposit to remote bank; transaction reversed"); + } +``` +2. Restart or redeploy Bank A with this change. +3. Stop Bank B: `CTRL+C` the app if running locally or something like `npx dbos-cloud app delete bank_b` if in the cloud. + +This demo is time sensitive as you'll have a 10-second sleep window to crash Bank A. Adjust the above `sleep` loop if you nee more time. Read these steps ahead of time to get a sense for what to do: +1. Initiate a withdrawal from Bank A to Bank B +2. Because of the above `sleep` loop, you won't see the effect. You can quickly close the "Withdraw" window and refresh the browser to see that the funds left the account. Now, quickly press the red "Crash!" button. +3. If running in DBOS cloud, wait a few seconds. If you are running locally, restart Bank A by hand. +4. After restarting the app, log in again to `https://localhost:8089`. Observe the funds returning to the sender account. +5. In the log for the app, you should see the failed transfer. The 10 `Sleeping` statements, interrupted by a crash, then restart, workflow recovery and finally the "transaction reversed" error message. + ## Further Reading To get started with DBOS Transact, check out the [quickstart](https://docs.dbos.dev/getting-started/quickstart) and [docs](https://docs.dbos.dev/). diff --git a/bank/bank-backend/package-lock.json b/bank/bank-backend/package-lock.json index bcffad2e..41d77a12 100644 --- a/bank/bank-backend/package-lock.json +++ b/bank/bank-backend/package-lock.json @@ -9,7 +9,7 @@ "version": "0.0.2", "license": "ISC", "dependencies": { - "@dbos-inc/dbos-sdk": "^1.15.9", + "@dbos-inc/dbos-sdk": "^1.16.12", "@koa/bodyparser": "^5.0.0", "@koa/cors": "^5.0.0", "@koa/router": "^12.0.0", @@ -771,12 +771,9 @@ } }, "node_modules/@dbos-inc/dbos-sdk": { - "version": "1.15.9", - "resolved": "https://registry.npmjs.org/@dbos-inc/dbos-sdk/-/dbos-sdk-1.15.9.tgz", - "integrity": "sha512-2omgN7TVdUpq6zFHQfbxMUvlCyi02aNNKa//4iTWCzQE9VychF9qgwpORjRh6WqiWVE0D41QewriVGFKURY1HA==", - "workspaces": [ - "packages/*" - ], + "version": "1.16.12", + "resolved": "https://registry.npmjs.org/@dbos-inc/dbos-sdk/-/dbos-sdk-1.16.12.tgz", + "integrity": "sha512-RifvaH7hfBii8qEts1nH/1w59lBe/vo5rXwBE6UE8S7b23W6I2HbouqwOAR7dGenXTMuuRPohLgmldoj86LNHQ==", "dependencies": { "@koa/bodyparser": "5.0.0", "@koa/cors": "5.0.0", diff --git a/bank/bank-backend/package.json b/bank/bank-backend/package.json index 9a683365..64ac545a 100644 --- a/bank/bank-backend/package.json +++ b/bank/bank-backend/package.json @@ -14,7 +14,7 @@ "license": "ISC", "private": true, "dependencies": { - "@dbos-inc/dbos-sdk": "^1.15.9", + "@dbos-inc/dbos-sdk": "^1.16.12", "@koa/bodyparser": "^5.0.0", "@koa/cors": "^5.0.0", "@koa/router": "^12.0.0", diff --git a/bank/bank-backend/src/router.ts b/bank/bank-backend/src/router.ts index bb41c6fa..e46ad9ba 100644 --- a/bank/bank-backend/src/router.ts +++ b/bank/bank-backend/src/router.ts @@ -64,6 +64,15 @@ export class BankEndpoints { } } +// For demo purposes +export class CrashEndpoint { + @GetApi('/crash_application') + static async crashApplication(ctx: HandlerContext) { + // For testing and demo purposes :) + process.exit(1); + } +} + // Helper functions to convert to the correct data types. // Especially convert the bigint. export function convertTransactionHistory(data: TransactionHistory): TransactionHistory { diff --git a/bank/bank-backend/src/workflows/txnhistory.workflows.ts b/bank/bank-backend/src/workflows/txnhistory.workflows.ts index b8ea60ef..9cb336a8 100644 --- a/bank/bank-backend/src/workflows/txnhistory.workflows.ts +++ b/bank/bank-backend/src/workflows/txnhistory.workflows.ts @@ -226,7 +226,6 @@ export class BankTransactionHistory { static async withdrawWorkflow(ctxt: WorkflowContext, data: TransactionHistory) { // Withdraw first. const result = await ctxt.invoke(BankTransactionHistory).updateAcctTransactionFunc(data.fromAccountId, data, false); - // Then, contact remote DB to deposit. if (data.toLocation && !(data.toLocation === "cash") && !data.toLocation.startsWith(REMOTEDB_PREFIX)) { ctxt.logger.info("Deposit to another DB: " + data.toLocation + ", account: " + data.toAccountId); @@ -240,12 +239,26 @@ export class BankTransactionHistory { }; const remoteRes: boolean = await ctxt.invoke(BankTransactionHistory).remoteTransferComm(remoteUrl, thReq as TransactionHistory, ctxt.workflowUUID + '-deposit'); if (!remoteRes) { - // Undo transaction is a deposit. + + /////////////////////////////// + // Example sleep Window. For a reliability test, uncomment the below + // Then, start a transfer to a nonexistent bank, (i.e. stop bank b). + // Wait for app to go into sleep and then crash it. The DBOS workflow + // recovery will ensure the undo transaction below is executed when + // the app restarts + // + // for (let i = 0; i < 10; i++) { + // ctxt.logger.info("Sleeping") + // await ctxt.sleepms(1000) + // } + /////////////////////////////// + + // Undo withdrawal with a deposit. const undoRes = await ctxt.invoke(BankTransactionHistory).updateAcctTransactionFunc(data.fromAccountId, data, true, result); if (undoRes !== result) { throw new Error(`Mismatch: Original txnId: ${result}, undo txnId: ${undoRes}`); } - throw new Error("Failed to deposit to remote bank."); + throw new Error("Failed to deposit to remote bank; transaction reversed"); } } else { ctxt.logger.info("Deposit to: " + data.fromLocation); diff --git a/bank/bank-frontend/src/app/bank.component.ts b/bank/bank-frontend/src/app/bank.component.ts index 4ab94444..0ef8ea6c 100644 --- a/bank/bank-frontend/src/app/bank.component.ts +++ b/bank/bank-frontend/src/app/bank.component.ts @@ -20,7 +20,7 @@ import { OwnerNameDialogComponent } from './owner-name-dialog/owner-name-dialog. - + @@ -431,4 +431,9 @@ export class BankComponent { error: (err: any) => { this.bankmsg = 'Failed to transfer! '; } }); } + + //In response to the "Crash" button - for demo purposes + crashApp(){ + this._service.getResource(this.bankUrl + "/crash_application").subscribe() + } } diff --git a/bank/bank-frontend/src/styles.css b/bank/bank-frontend/src/styles.css index 34a3bdeb..68b24066 100644 --- a/bank/bank-frontend/src/styles.css +++ b/bank/bank-frontend/src/styles.css @@ -33,6 +33,18 @@ position: relative; } +.button-red { + margin: 10px; + padding: 15px 20px; + background-color: #ff1111; + color: white; + border: none; + cursor: pointer; + font-size: 16px; + border-radius: 10px; /* Rounded corners */ + position: relative; +} + .button-grey { margin: 10px; padding: 15px 20px; From 70cb0685a4c82603713ece8ab325a409750a7ca9 Mon Sep 17 00:00:00 2001 From: apoliakov Date: Fri, 5 Jul 2024 17:13:45 -0400 Subject: [PATCH 2/3] MAke linter happy --- bank/bank-backend/src/router.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/bank/bank-backend/src/router.ts b/bank/bank-backend/src/router.ts index e46ad9ba..45b17e00 100644 --- a/bank/bank-backend/src/router.ts +++ b/bank/bank-backend/src/router.ts @@ -70,6 +70,7 @@ export class CrashEndpoint { static async crashApplication(ctx: HandlerContext) { // For testing and demo purposes :) process.exit(1); + return Promise.resolve(); } } From 9230a208148276e12150e9dad13abcc3f4b60b65 Mon Sep 17 00:00:00 2001 From: apoliakov Date: Fri, 5 Jul 2024 17:15:56 -0400 Subject: [PATCH 3/3] Make linter happier --- bank/bank-backend/src/router.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bank/bank-backend/src/router.ts b/bank/bank-backend/src/router.ts index 45b17e00..7ba42c53 100644 --- a/bank/bank-backend/src/router.ts +++ b/bank/bank-backend/src/router.ts @@ -67,7 +67,7 @@ export class BankEndpoints { // For demo purposes export class CrashEndpoint { @GetApi('/crash_application') - static async crashApplication(ctx: HandlerContext) { + static async crashApplication(_ctx: HandlerContext) { // For testing and demo purposes :) process.exit(1); return Promise.resolve();