From db0ce4f612f82be5f15875c35a4d25963f6029f7 Mon Sep 17 00:00:00 2001 From: gagik Date: Fri, 20 Dec 2024 17:35:22 +0100 Subject: [PATCH 1/7] add fix and better tracking --- src/participant/participant.ts | 100 ++++++++++++------ src/telemetry/telemetryService.ts | 14 +++ .../suite/participant/participant.test.ts | 31 ++++++ src/views/webview-app/overview-page.tsx | 1 + 4 files changed, 113 insertions(+), 33 deletions(-) diff --git a/src/participant/participant.ts b/src/participant/participant.ts index 3c1260f2..de751bab 100644 --- a/src/participant/participant.ts +++ b/src/participant/participant.ts @@ -345,26 +345,31 @@ export default class ParticipantController { token: vscode.CancellationToken; language?: string; }): Promise { - const chatResponse = await this._getChatResponse({ - modelInput, - token, - }); + try { + const chatResponse = await this._getChatResponse({ + modelInput, + token, + }); - const languageCodeBlockIdentifier = { - start: `\`\`\`${language ? language : 'javascript'}`, - end: '```', - }; + const languageCodeBlockIdentifier = { + start: `\`\`\`${language ? language : 'javascript'}`, + end: '```', + }; - const runnableContent: string[] = []; - await processStreamWithIdentifiers({ - processStreamFragment: () => {}, - onStreamIdentifier: (content: string) => { - runnableContent.push(content.trim()); - }, - inputIterable: chatResponse.text, - identifier: languageCodeBlockIdentifier, - }); - return runnableContent.length ? runnableContent.join('') : null; + const runnableContent: string[] = []; + await processStreamWithIdentifiers({ + processStreamFragment: () => {}, + onStreamIdentifier: (content: string) => { + runnableContent.push(content.trim()); + }, + inputIterable: chatResponse.text, + identifier: languageCodeBlockIdentifier, + }); + return runnableContent.length ? runnableContent.join('') : null; + } catch (error) { + /** If anything goes wrong with the response or the stream, return null instead of throwing. */ + return null; + } } async streamChatResponseContentWithCodeActions({ @@ -1784,8 +1789,7 @@ export default class ParticipantController { } async exportCodeToPlayground(): Promise { - const selectedText = getSelectedText(); - const codeToExport = selectedText || getAllText(); + const codeToExport = getSelectedText() || getAllText(); try { const content = await vscode.window.withProgress( @@ -1795,27 +1799,50 @@ export default class ParticipantController { cancellable: true, }, async (progress, token): Promise => { - const modelInput = await Prompts.exportToPlayground.buildMessages({ - request: { prompt: codeToExport }, - }); + let modelInput: ModelInput | undefined; + try { + modelInput = await Prompts.exportToPlayground.buildMessages({ + request: { prompt: codeToExport }, + }); + } catch (error) { + void vscode.window.showErrorMessage( + 'Failed to generate a MongoDB Playground. Please ensure your code block contains a MongoDB query.' + ); + + this._telemetryService.trackExportToPlaygroundFailed({ + input_length: codeToExport?.length, + details: 'modelInput', + }); + return null; + } const result = await Promise.race([ this.streamChatResponseWithExportToLanguage({ modelInput, token, }), - new Promise((resolve) => + new Promise<'cancelled'>((resolve) => token.onCancellationRequested(() => { log.info('The export to a playground operation was canceled.'); - resolve(null); + resolve('cancelled'); }) ), ]); - if (result?.includes("Sorry, I can't assist with that.")) { + if (result === 'cancelled') { + return null; + } + + if (!result || result?.includes("Sorry, I can't assist with that.")) { void vscode.window.showErrorMessage( - 'Sorry, we were unable to generate the playground, please try again. If the error persists, try changing your selected code.' + 'Failed to generate a MongoDB Playground. Please ensure your code block contains a MongoDB query.' ); + + this._telemetryService.trackExportToPlaygroundFailed({ + input_length: codeToExport?.length, + details: 'streamChatResponseWithExportToLanguage', + }); + return null; } @@ -1827,12 +1854,19 @@ export default class ParticipantController { return true; } - await vscode.commands.executeCommand( - EXTENSION_COMMANDS.OPEN_PARTICIPANT_CODE_IN_PLAYGROUND, - { - runnableContent: content, - } - ); + try { + await vscode.commands.executeCommand( + EXTENSION_COMMANDS.OPEN_PARTICIPANT_CODE_IN_PLAYGROUND, + { + runnableContent: content, + } + ); + } catch (error) { + this._telemetryService.trackExportToPlaygroundFailed({ + input_length: content.length, + details: 'executeCommand', + }); + } return true; } catch (error) { diff --git a/src/telemetry/telemetryService.ts b/src/telemetry/telemetryService.ts index bccfc1b7..4d5c0ff0 100644 --- a/src/telemetry/telemetryService.ts +++ b/src/telemetry/telemetryService.ts @@ -54,6 +54,11 @@ type DocumentEditedTelemetryEventProperties = { source: DocumentSource; }; +type ExportToPlaygroundFailedEventProperties = { + input_length: number | undefined; + details: string; +}; + type PlaygroundExportedToLanguageTelemetryEventProperties = { language?: string; exported_code_length: number; @@ -106,6 +111,7 @@ type ParticipantResponseFailedProperties = { command: ParticipantResponseType; error_code?: string; error_name: ParticipantErrorTypes; + error_details?: string; }; export type InternalPromptPurpose = 'intent' | 'namespace' | undefined; @@ -171,6 +177,7 @@ type TelemetryEventProperties = | PlaygroundSavedTelemetryEventProperties | PlaygroundLoadedTelemetryEventProperties | KeytarSecretsMigrationFailedProperties + | ExportToPlaygroundFailedEventProperties | SavedConnectionsLoadedProperties | ParticipantFeedbackProperties | ParticipantResponseFailedProperties @@ -193,6 +200,7 @@ export enum TelemetryEventTypes { PLAYGROUND_EXPORTED_TO_LANGUAGE = 'Playground Exported To Language', PLAYGROUND_CREATED = 'Playground Created', KEYTAR_SECRETS_MIGRATION_FAILED = 'Keytar Secrets Migration Failed', + EXPORT_TO_PLAYGROUND_FAILED = 'Export To Playground Failed', SAVED_CONNECTIONS_LOADED = 'Saved Connections Loaded', PARTICIPANT_FEEDBACK = 'Participant Feedback', PARTICIPANT_WELCOME_SHOWN = 'Participant Welcome Shown', @@ -346,6 +354,12 @@ export default class TelemetryService { ); } + trackExportToPlaygroundFailed( + props: ExportToPlaygroundFailedEventProperties + ): void { + this.track(TelemetryEventTypes.EXPORT_TO_PLAYGROUND_FAILED, props); + } + trackCommandRun(command: ExtensionCommand): void { this.track(TelemetryEventTypes.EXTENSION_COMMAND_RUN, { command }); } diff --git a/src/test/suite/participant/participant.test.ts b/src/test/suite/participant/participant.test.ts index 78572f75..74e52d76 100644 --- a/src/test/suite/participant/participant.test.ts +++ b/src/test/suite/participant/participant.test.ts @@ -1818,6 +1818,37 @@ Schema: ); }); + test('tracks failures with export to playground and not as a failed prompt', async function () { + const editor = vscode.window.activeTextEditor; + if (!editor) { + throw new Error('Window active text editor is undefined'); + } + + const testDocumentUri = editor.document.uri; + const edit = new vscode.WorkspaceEdit(); + const code = ` + THIS IS SOME ERROR CAUSING CODE. +`; + edit.replace(testDocumentUri, getFullRange(editor.document), code); + await vscode.workspace.applyEdit(edit); + + await testParticipantController.exportCodeToPlayground(); + sendRequestStub.rejects(); + const messages = sendRequestStub.firstCall.args[0]; + expect(getMessageContent(messages[1])).to.equal(code.trim()); + expect(telemetryTrackStub).calledWith( + TelemetryEventTypes.EXPORT_TO_PLAYGROUND_FAILED, + { + input_length: code.trim().length, + details: 'streamChatResponseWithExportToLanguage', + } + ); + + expect(telemetryTrackStub).not.calledWith( + TelemetryEventTypes.PARTICIPANT_RESPONSE_FAILED + ); + }); + test('exports selected lines of code to a playground', async function () { const editor = vscode.window.activeTextEditor; if (!editor) { diff --git a/src/views/webview-app/overview-page.tsx b/src/views/webview-app/overview-page.tsx index ea5bfe38..b34f5d80 100644 --- a/src/views/webview-app/overview-page.tsx +++ b/src/views/webview-app/overview-page.tsx @@ -126,6 +126,7 @@ const OverviewPage: React.FC = () => { From 70d6aad7acbc37f6107eaccd49552f477cded22e Mon Sep 17 00:00:00 2001 From: gagik Date: Fri, 20 Dec 2024 17:41:39 +0100 Subject: [PATCH 2/7] ts discrepancy --- src/views/webview-app/overview-page.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/views/webview-app/overview-page.tsx b/src/views/webview-app/overview-page.tsx index b34f5d80..ea5bfe38 100644 --- a/src/views/webview-app/overview-page.tsx +++ b/src/views/webview-app/overview-page.tsx @@ -126,7 +126,6 @@ const OverviewPage: React.FC = () => { From 048e51b59a68f4bbf3de138c78ac7526ba8bb1a7 Mon Sep 17 00:00:00 2001 From: gagik Date: Fri, 20 Dec 2024 18:06:15 +0100 Subject: [PATCH 3/7] use enums --- src/participant/participant.ts | 77 +++++++++++++----------- src/participant/participantErrorTypes.ts | 6 ++ src/telemetry/telemetryService.ts | 2 +- 3 files changed, 49 insertions(+), 36 deletions(-) diff --git a/src/participant/participant.ts b/src/participant/participant.ts index de751bab..c105a3ae 100644 --- a/src/participant/participant.ts +++ b/src/participant/participant.ts @@ -45,7 +45,10 @@ import { processStreamWithIdentifiers } from './streamParsing'; import type { PromptIntent } from './prompts/intent'; import { isPlayground, getSelectedText, getAllText } from '../utils/playground'; import type { DataService } from 'mongodb-data-service'; -import { ParticipantErrorTypes } from './participantErrorTypes'; +import { + ExportToPlaygroundFailure, + ParticipantErrorTypes, +} from './participantErrorTypes'; import type PlaygroundResultProvider from '../editors/playgroundResultProvider'; import { isExportToLanguageResult } from '../types/playgroundType'; import { PromptHistory } from './prompts/promptHistory'; @@ -1792,7 +1795,9 @@ export default class ParticipantController { const codeToExport = getSelectedText() || getAllText(); try { - const content = await vscode.window.withProgress( + const contentOrFailure = await vscode.window.withProgress< + string | ExportToPlaygroundFailure | null + >( { location: vscode.ProgressLocation.Notification, title: 'Exporting code to a playground...', @@ -1805,15 +1810,7 @@ export default class ParticipantController { request: { prompt: codeToExport }, }); } catch (error) { - void vscode.window.showErrorMessage( - 'Failed to generate a MongoDB Playground. Please ensure your code block contains a MongoDB query.' - ); - - this._telemetryService.trackExportToPlaygroundFailed({ - input_length: codeToExport?.length, - details: 'modelInput', - }); - return null; + return ExportToPlaygroundFailure.STREAM_CHAT_RESPONSE; } const result = await Promise.race([ @@ -1821,53 +1818,63 @@ export default class ParticipantController { modelInput, token, }), - new Promise<'cancelled'>((resolve) => + new Promise((resolve) => token.onCancellationRequested(() => { log.info('The export to a playground operation was canceled.'); - resolve('cancelled'); + resolve(ExportToPlaygroundFailure.CANCELLED); }) ), ]); - if (result === 'cancelled') { - return null; + if (result === ExportToPlaygroundFailure.CANCELLED) { + return ExportToPlaygroundFailure.CANCELLED; } if (!result || result?.includes("Sorry, I can't assist with that.")) { - void vscode.window.showErrorMessage( - 'Failed to generate a MongoDB Playground. Please ensure your code block contains a MongoDB query.' - ); - - this._telemetryService.trackExportToPlaygroundFailed({ - input_length: codeToExport?.length, - details: 'streamChatResponseWithExportToLanguage', - }); - - return null; + return ExportToPlaygroundFailure.STREAM_CHAT_RESPONSE; } return result; } ); - if (!content) { + if ( + !contentOrFailure || + contentOrFailure === ExportToPlaygroundFailure.CANCELLED + ) { return true; } - try { - await vscode.commands.executeCommand( - EXTENSION_COMMANDS.OPEN_PARTICIPANT_CODE_IN_PLAYGROUND, - { - runnableContent: content, - } + if ( + Object.values(ExportToPlaygroundFailure).includes( + contentOrFailure as ExportToPlaygroundFailure + ) + ) { + void vscode.window.showErrorMessage( + 'Failed to generate a MongoDB Playground. Please ensure your code block contains a MongoDB query.' + ); + + // Content in this case is already equal to the failureType; this is just to make it explicit + // and avoid accidentally sending actual contents of the message. + const failureType = Object.values(ExportToPlaygroundFailure).find( + (value) => value === contentOrFailure ); - } catch (error) { this._telemetryService.trackExportToPlaygroundFailed({ - input_length: content.length, - details: 'executeCommand', + input_length: codeToExport?.length, + + details: failureType, }); + return false; } + const content = contentOrFailure; + await vscode.commands.executeCommand( + EXTENSION_COMMANDS.OPEN_PARTICIPANT_CODE_IN_PLAYGROUND, + { + runnableContent: content, + } + ); + return true; } catch (error) { const message = formatError(error).message; diff --git a/src/participant/participantErrorTypes.ts b/src/participant/participantErrorTypes.ts index 1b2a6ce8..208c31bd 100644 --- a/src/participant/participantErrorTypes.ts +++ b/src/participant/participantErrorTypes.ts @@ -5,3 +5,9 @@ export enum ParticipantErrorTypes { OTHER = 'Other', DOCS_CHATBOT_API = 'Docs Chatbot API Issue', } + +export enum ExportToPlaygroundFailure { + CANCELLED = 'cancelled', + MODEL_INPUT = 'modelInput', + STREAM_CHAT_RESPONSE = 'streamChatResponseWithExportToLanguage', +} diff --git a/src/telemetry/telemetryService.ts b/src/telemetry/telemetryService.ts index 4d5c0ff0..d0e53def 100644 --- a/src/telemetry/telemetryService.ts +++ b/src/telemetry/telemetryService.ts @@ -56,7 +56,7 @@ type DocumentEditedTelemetryEventProperties = { type ExportToPlaygroundFailedEventProperties = { input_length: number | undefined; - details: string; + details?: string; }; type PlaygroundExportedToLanguageTelemetryEventProperties = { From 50cc763427544ac49ecddbc083ded3c6258c879a Mon Sep 17 00:00:00 2001 From: gagik Date: Fri, 20 Dec 2024 18:46:30 +0100 Subject: [PATCH 4/7] switch to wrapped objects --- src/participant/participant.ts | 47 +++++++------------ src/participant/participantErrorTypes.ts | 9 ++-- src/telemetry/telemetryService.ts | 3 +- .../suite/participant/participant.test.ts | 2 +- 4 files changed, 25 insertions(+), 36 deletions(-) diff --git a/src/participant/participant.ts b/src/participant/participant.ts index c105a3ae..b7cb5f16 100644 --- a/src/participant/participant.ts +++ b/src/participant/participant.ts @@ -46,8 +46,8 @@ import type { PromptIntent } from './prompts/intent'; import { isPlayground, getSelectedText, getAllText } from '../utils/playground'; import type { DataService } from 'mongodb-data-service'; import { - ExportToPlaygroundFailure, ParticipantErrorTypes, + type ExportToPlaygroundError, } from './participantErrorTypes'; import type PlaygroundResultProvider from '../editors/playgroundResultProvider'; import { isExportToLanguageResult } from '../types/playgroundType'; @@ -1795,22 +1795,25 @@ export default class ParticipantController { const codeToExport = getSelectedText() || getAllText(); try { - const contentOrFailure = await vscode.window.withProgress< - string | ExportToPlaygroundFailure | null + const contentOrError = await vscode.window.withProgress< + { value: string } | { error: ExportToPlaygroundError } >( { location: vscode.ProgressLocation.Notification, title: 'Exporting code to a playground...', cancellable: true, }, - async (progress, token): Promise => { + async ( + progress, + token + ): Promise<{ value: string } | { error: ExportToPlaygroundError }> => { let modelInput: ModelInput | undefined; try { modelInput = await Prompts.exportToPlayground.buildMessages({ request: { prompt: codeToExport }, }); } catch (error) { - return ExportToPlaygroundFailure.STREAM_CHAT_RESPONSE; + return { error: 'modelInput' }; } const result = await Promise.race([ @@ -1818,56 +1821,42 @@ export default class ParticipantController { modelInput, token, }), - new Promise((resolve) => + new Promise((resolve) => token.onCancellationRequested(() => { log.info('The export to a playground operation was canceled.'); - resolve(ExportToPlaygroundFailure.CANCELLED); + resolve('cancelled'); }) ), ]); - if (result === ExportToPlaygroundFailure.CANCELLED) { - return ExportToPlaygroundFailure.CANCELLED; + if (result === 'cancelled') { + return { error: 'cancelled' }; } if (!result || result?.includes("Sorry, I can't assist with that.")) { - return ExportToPlaygroundFailure.STREAM_CHAT_RESPONSE; + return { error: 'streamChatResponseWithExportToLanguage' }; } - return result; + return { value: result }; } ); - if ( - !contentOrFailure || - contentOrFailure === ExportToPlaygroundFailure.CANCELLED - ) { - return true; - } - - if ( - Object.values(ExportToPlaygroundFailure).includes( - contentOrFailure as ExportToPlaygroundFailure - ) - ) { + if ('error' in contentOrError) { + const { error } = contentOrError; void vscode.window.showErrorMessage( 'Failed to generate a MongoDB Playground. Please ensure your code block contains a MongoDB query.' ); // Content in this case is already equal to the failureType; this is just to make it explicit // and avoid accidentally sending actual contents of the message. - const failureType = Object.values(ExportToPlaygroundFailure).find( - (value) => value === contentOrFailure - ); this._telemetryService.trackExportToPlaygroundFailed({ input_length: codeToExport?.length, - - details: failureType, + error_name: error, }); return false; } - const content = contentOrFailure; + const content = contentOrError.value; await vscode.commands.executeCommand( EXTENSION_COMMANDS.OPEN_PARTICIPANT_CODE_IN_PLAYGROUND, { diff --git a/src/participant/participantErrorTypes.ts b/src/participant/participantErrorTypes.ts index 208c31bd..d83ce859 100644 --- a/src/participant/participantErrorTypes.ts +++ b/src/participant/participantErrorTypes.ts @@ -6,8 +6,7 @@ export enum ParticipantErrorTypes { DOCS_CHATBOT_API = 'Docs Chatbot API Issue', } -export enum ExportToPlaygroundFailure { - CANCELLED = 'cancelled', - MODEL_INPUT = 'modelInput', - STREAM_CHAT_RESPONSE = 'streamChatResponseWithExportToLanguage', -} +export type ExportToPlaygroundError = + | 'cancelled' + | 'modelInput' + | 'streamChatResponseWithExportToLanguage'; diff --git a/src/telemetry/telemetryService.ts b/src/telemetry/telemetryService.ts index d0e53def..23203199 100644 --- a/src/telemetry/telemetryService.ts +++ b/src/telemetry/telemetryService.ts @@ -12,6 +12,7 @@ import { getConnectionTelemetryProperties } from './connectionTelemetry'; import type { NewConnectionTelemetryEventProperties } from './connectionTelemetry'; import type { ShellEvaluateResult } from '../types/playgroundType'; import type { StorageController } from '../storage'; +import type { ExportToPlaygroundError } from '../participant/participantErrorTypes'; import { ParticipantErrorTypes } from '../participant/participantErrorTypes'; import type { ExtensionCommand } from '../commands'; import type { @@ -56,7 +57,7 @@ type DocumentEditedTelemetryEventProperties = { type ExportToPlaygroundFailedEventProperties = { input_length: number | undefined; - details?: string; + error_name?: ExportToPlaygroundError; }; type PlaygroundExportedToLanguageTelemetryEventProperties = { diff --git a/src/test/suite/participant/participant.test.ts b/src/test/suite/participant/participant.test.ts index 74e52d76..5d856579 100644 --- a/src/test/suite/participant/participant.test.ts +++ b/src/test/suite/participant/participant.test.ts @@ -1840,7 +1840,7 @@ Schema: TelemetryEventTypes.EXPORT_TO_PLAYGROUND_FAILED, { input_length: code.trim().length, - details: 'streamChatResponseWithExportToLanguage', + error_name: 'streamChatResponseWithExportToLanguage', } ); From fce56e9b8d6d022abd0ed5cc080105c57f3f9e4b Mon Sep 17 00:00:00 2001 From: gagik Date: Fri, 20 Dec 2024 18:48:20 +0100 Subject: [PATCH 5/7] Add error log --- src/participant/participant.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/participant/participant.ts b/src/participant/participant.ts index b7cb5f16..a87d7e27 100644 --- a/src/participant/participant.ts +++ b/src/participant/participant.ts @@ -371,6 +371,7 @@ export default class ParticipantController { return runnableContent.length ? runnableContent.join('') : null; } catch (error) { /** If anything goes wrong with the response or the stream, return null instead of throwing. */ + log.error('Error while exporting to playground', error); return null; } } From 65e991e278c862c8cdf34425de10b1afcdde8639 Mon Sep 17 00:00:00 2001 From: gagik Date: Fri, 20 Dec 2024 18:49:22 +0100 Subject: [PATCH 6/7] better wording --- src/participant/participant.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/participant/participant.ts b/src/participant/participant.ts index a87d7e27..7cfaa71c 100644 --- a/src/participant/participant.ts +++ b/src/participant/participant.ts @@ -371,7 +371,10 @@ export default class ParticipantController { return runnableContent.length ? runnableContent.join('') : null; } catch (error) { /** If anything goes wrong with the response or the stream, return null instead of throwing. */ - log.error('Error while exporting to playground', error); + log.error( + 'Error while streaming chat response with export to language', + error + ); return null; } } From b66ba990fcbd3c3ac13a6ae44b0ff0e7bd9cdcc3 Mon Sep 17 00:00:00 2001 From: gagik Date: Fri, 20 Dec 2024 18:52:23 +0100 Subject: [PATCH 7/7] don't show error when cancelling --- src/participant/participant.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/participant/participant.ts b/src/participant/participant.ts index 7cfaa71c..f54f5ff8 100644 --- a/src/participant/participant.ts +++ b/src/participant/participant.ts @@ -1847,6 +1847,10 @@ export default class ParticipantController { if ('error' in contentOrError) { const { error } = contentOrError; + if (error === 'cancelled') { + return true; + } + void vscode.window.showErrorMessage( 'Failed to generate a MongoDB Playground. Please ensure your code block contains a MongoDB query.' );