diff --git a/src/participant/participant.ts b/src/participant/participant.ts index 3c1260f2..f54f5ff8 100644 --- a/src/participant/participant.ts +++ b/src/participant/participant.ts @@ -45,7 +45,10 @@ import { processStreamWithIdentifiers } from './streamParsing'; import type { PromptIntent } from './prompts/intent'; import { isPlayground, getSelectedText, getAllText } from '../utils/playground'; import type { DataService } from 'mongodb-data-service'; -import { ParticipantErrorTypes } from './participantErrorTypes'; +import { + ParticipantErrorTypes, + type ExportToPlaygroundError, +} from './participantErrorTypes'; import type PlaygroundResultProvider from '../editors/playgroundResultProvider'; import { isExportToLanguageResult } from '../types/playgroundType'; import { PromptHistory } from './prompts/promptHistory'; @@ -345,26 +348,35 @@ export default class ParticipantController { token: vscode.CancellationToken; language?: string; }): Promise { - const chatResponse = await this._getChatResponse({ - modelInput, - token, - }); + try { + const chatResponse = await this._getChatResponse({ + modelInput, + token, + }); - const languageCodeBlockIdentifier = { - start: `\`\`\`${language ? language : 'javascript'}`, - end: '```', - }; + const languageCodeBlockIdentifier = { + start: `\`\`\`${language ? language : 'javascript'}`, + end: '```', + }; - const runnableContent: string[] = []; - await processStreamWithIdentifiers({ - processStreamFragment: () => {}, - onStreamIdentifier: (content: string) => { - runnableContent.push(content.trim()); - }, - inputIterable: chatResponse.text, - identifier: languageCodeBlockIdentifier, - }); - return runnableContent.length ? runnableContent.join('') : null; + const runnableContent: string[] = []; + await processStreamWithIdentifiers({ + processStreamFragment: () => {}, + onStreamIdentifier: (content: string) => { + runnableContent.push(content.trim()); + }, + inputIterable: chatResponse.text, + identifier: languageCodeBlockIdentifier, + }); + return runnableContent.length ? runnableContent.join('') : null; + } catch (error) { + /** If anything goes wrong with the response or the stream, return null instead of throwing. */ + log.error( + 'Error while streaming chat response with export to language', + error + ); + return null; + } } async streamChatResponseContentWithCodeActions({ @@ -1784,49 +1796,75 @@ export default class ParticipantController { } async exportCodeToPlayground(): Promise { - const selectedText = getSelectedText(); - const codeToExport = selectedText || getAllText(); + const codeToExport = getSelectedText() || getAllText(); try { - const content = await vscode.window.withProgress( + const contentOrError = await vscode.window.withProgress< + { value: string } | { error: ExportToPlaygroundError } + >( { location: vscode.ProgressLocation.Notification, title: 'Exporting code to a playground...', cancellable: true, }, - async (progress, token): Promise => { - const modelInput = await Prompts.exportToPlayground.buildMessages({ - request: { prompt: codeToExport }, - }); + async ( + progress, + token + ): Promise<{ value: string } | { error: ExportToPlaygroundError }> => { + let modelInput: ModelInput | undefined; + try { + modelInput = await Prompts.exportToPlayground.buildMessages({ + request: { prompt: codeToExport }, + }); + } catch (error) { + return { error: 'modelInput' }; + } const result = await Promise.race([ this.streamChatResponseWithExportToLanguage({ modelInput, token, }), - new Promise((resolve) => + new Promise((resolve) => token.onCancellationRequested(() => { log.info('The export to a playground operation was canceled.'); - resolve(null); + resolve('cancelled'); }) ), ]); - if (result?.includes("Sorry, I can't assist with that.")) { - void vscode.window.showErrorMessage( - 'Sorry, we were unable to generate the playground, please try again. If the error persists, try changing your selected code.' - ); - return null; + if (result === 'cancelled') { + return { error: 'cancelled' }; } - return result; + if (!result || result?.includes("Sorry, I can't assist with that.")) { + return { error: 'streamChatResponseWithExportToLanguage' }; + } + + return { value: result }; } ); - if (!content) { - return true; + if ('error' in contentOrError) { + const { error } = contentOrError; + if (error === 'cancelled') { + return true; + } + + void vscode.window.showErrorMessage( + 'Failed to generate a MongoDB Playground. Please ensure your code block contains a MongoDB query.' + ); + + // Content in this case is already equal to the failureType; this is just to make it explicit + // and avoid accidentally sending actual contents of the message. + this._telemetryService.trackExportToPlaygroundFailed({ + input_length: codeToExport?.length, + error_name: error, + }); + return false; } + const content = contentOrError.value; await vscode.commands.executeCommand( EXTENSION_COMMANDS.OPEN_PARTICIPANT_CODE_IN_PLAYGROUND, { diff --git a/src/participant/participantErrorTypes.ts b/src/participant/participantErrorTypes.ts index 1b2a6ce8..d83ce859 100644 --- a/src/participant/participantErrorTypes.ts +++ b/src/participant/participantErrorTypes.ts @@ -5,3 +5,8 @@ export enum ParticipantErrorTypes { OTHER = 'Other', DOCS_CHATBOT_API = 'Docs Chatbot API Issue', } + +export type ExportToPlaygroundError = + | 'cancelled' + | 'modelInput' + | 'streamChatResponseWithExportToLanguage'; diff --git a/src/telemetry/telemetryService.ts b/src/telemetry/telemetryService.ts index bccfc1b7..23203199 100644 --- a/src/telemetry/telemetryService.ts +++ b/src/telemetry/telemetryService.ts @@ -12,6 +12,7 @@ import { getConnectionTelemetryProperties } from './connectionTelemetry'; import type { NewConnectionTelemetryEventProperties } from './connectionTelemetry'; import type { ShellEvaluateResult } from '../types/playgroundType'; import type { StorageController } from '../storage'; +import type { ExportToPlaygroundError } from '../participant/participantErrorTypes'; import { ParticipantErrorTypes } from '../participant/participantErrorTypes'; import type { ExtensionCommand } from '../commands'; import type { @@ -54,6 +55,11 @@ type DocumentEditedTelemetryEventProperties = { source: DocumentSource; }; +type ExportToPlaygroundFailedEventProperties = { + input_length: number | undefined; + error_name?: ExportToPlaygroundError; +}; + type PlaygroundExportedToLanguageTelemetryEventProperties = { language?: string; exported_code_length: number; @@ -106,6 +112,7 @@ type ParticipantResponseFailedProperties = { command: ParticipantResponseType; error_code?: string; error_name: ParticipantErrorTypes; + error_details?: string; }; export type InternalPromptPurpose = 'intent' | 'namespace' | undefined; @@ -171,6 +178,7 @@ type TelemetryEventProperties = | PlaygroundSavedTelemetryEventProperties | PlaygroundLoadedTelemetryEventProperties | KeytarSecretsMigrationFailedProperties + | ExportToPlaygroundFailedEventProperties | SavedConnectionsLoadedProperties | ParticipantFeedbackProperties | ParticipantResponseFailedProperties @@ -193,6 +201,7 @@ export enum TelemetryEventTypes { PLAYGROUND_EXPORTED_TO_LANGUAGE = 'Playground Exported To Language', PLAYGROUND_CREATED = 'Playground Created', KEYTAR_SECRETS_MIGRATION_FAILED = 'Keytar Secrets Migration Failed', + EXPORT_TO_PLAYGROUND_FAILED = 'Export To Playground Failed', SAVED_CONNECTIONS_LOADED = 'Saved Connections Loaded', PARTICIPANT_FEEDBACK = 'Participant Feedback', PARTICIPANT_WELCOME_SHOWN = 'Participant Welcome Shown', @@ -346,6 +355,12 @@ export default class TelemetryService { ); } + trackExportToPlaygroundFailed( + props: ExportToPlaygroundFailedEventProperties + ): void { + this.track(TelemetryEventTypes.EXPORT_TO_PLAYGROUND_FAILED, props); + } + trackCommandRun(command: ExtensionCommand): void { this.track(TelemetryEventTypes.EXTENSION_COMMAND_RUN, { command }); } diff --git a/src/test/suite/participant/participant.test.ts b/src/test/suite/participant/participant.test.ts index 78572f75..5d856579 100644 --- a/src/test/suite/participant/participant.test.ts +++ b/src/test/suite/participant/participant.test.ts @@ -1818,6 +1818,37 @@ Schema: ); }); + test('tracks failures with export to playground and not as a failed prompt', async function () { + const editor = vscode.window.activeTextEditor; + if (!editor) { + throw new Error('Window active text editor is undefined'); + } + + const testDocumentUri = editor.document.uri; + const edit = new vscode.WorkspaceEdit(); + const code = ` + THIS IS SOME ERROR CAUSING CODE. +`; + edit.replace(testDocumentUri, getFullRange(editor.document), code); + await vscode.workspace.applyEdit(edit); + + await testParticipantController.exportCodeToPlayground(); + sendRequestStub.rejects(); + const messages = sendRequestStub.firstCall.args[0]; + expect(getMessageContent(messages[1])).to.equal(code.trim()); + expect(telemetryTrackStub).calledWith( + TelemetryEventTypes.EXPORT_TO_PLAYGROUND_FAILED, + { + input_length: code.trim().length, + error_name: 'streamChatResponseWithExportToLanguage', + } + ); + + expect(telemetryTrackStub).not.calledWith( + TelemetryEventTypes.PARTICIPANT_RESPONSE_FAILED + ); + }); + test('exports selected lines of code to a playground', async function () { const editor = vscode.window.activeTextEditor; if (!editor) {