add fix and better tracking

mongodb-js · Dec 20, 2024 · db0ce4f · db0ce4f
1 parent 30e1ace
commit db0ce4f
Show file tree

Hide file tree

Showing 4 changed files with 113 additions and 33 deletions.
diff --git a/src/participant/participant.ts b/src/participant/participant.ts
@@ -345,26 +345,31 @@ export default class ParticipantController {
     token: vscode.CancellationToken;
     language?: string;
   }): Promise<string | null> {
-    const chatResponse = await this._getChatResponse({
-      modelInput,
-      token,
-    });
+    try {
+      const chatResponse = await this._getChatResponse({
+        modelInput,
+        token,
+      });
 
-    const languageCodeBlockIdentifier = {
-      start: `\`\`\`${language ? language : 'javascript'}`,
-      end: '```',
-    };
+      const languageCodeBlockIdentifier = {
+        start: `\`\`\`${language ? language : 'javascript'}`,
+        end: '```',
+      };
 
-    const runnableContent: string[] = [];
-    await processStreamWithIdentifiers({
-      processStreamFragment: () => {},
-      onStreamIdentifier: (content: string) => {
-        runnableContent.push(content.trim());
-      },
-      inputIterable: chatResponse.text,
-      identifier: languageCodeBlockIdentifier,
-    });
-    return runnableContent.length ? runnableContent.join('') : null;
+      const runnableContent: string[] = [];
+      await processStreamWithIdentifiers({
+        processStreamFragment: () => {},
+        onStreamIdentifier: (content: string) => {
+          runnableContent.push(content.trim());
+        },
+        inputIterable: chatResponse.text,
+        identifier: languageCodeBlockIdentifier,
+      });
+      return runnableContent.length ? runnableContent.join('') : null;
+    } catch (error) {
+      /** If anything goes wrong with the response or the stream, return null instead of throwing. */
+      return null;
+    }
   }
 
   async streamChatResponseContentWithCodeActions({
@@ -1784,8 +1789,7 @@ export default class ParticipantController {
   }
 
   async exportCodeToPlayground(): Promise<boolean> {
-    const selectedText = getSelectedText();
-    const codeToExport = selectedText || getAllText();
+    const codeToExport = getSelectedText() || getAllText();
 
     try {
       const content = await vscode.window.withProgress(
@@ -1795,27 +1799,50 @@ export default class ParticipantController {
           cancellable: true,
         },
         async (progress, token): Promise<string | null> => {
-          const modelInput = await Prompts.exportToPlayground.buildMessages({
-            request: { prompt: codeToExport },
-          });
+          let modelInput: ModelInput | undefined;
+          try {
+            modelInput = await Prompts.exportToPlayground.buildMessages({
+              request: { prompt: codeToExport },
+            });
+          } catch (error) {
+            void vscode.window.showErrorMessage(
+              'Failed to generate a MongoDB Playground. Please ensure your code block contains a MongoDB query.'
+            );
+
+            this._telemetryService.trackExportToPlaygroundFailed({
+              input_length: codeToExport?.length,
+              details: 'modelInput',
+            });
+            return null;
+          }
 
           const result = await Promise.race([
             this.streamChatResponseWithExportToLanguage({
               modelInput,
               token,
             }),
-            new Promise<null>((resolve) =>
+            new Promise<'cancelled'>((resolve) =>
               token.onCancellationRequested(() => {
                 log.info('The export to a playground operation was canceled.');
-                resolve(null);
+                resolve('cancelled');
               })
             ),
           ]);
 
-          if (result?.includes("Sorry, I can't assist with that.")) {
+          if (result === 'cancelled') {
+            return null;
+          }
+
+          if (!result || result?.includes("Sorry, I can't assist with that.")) {
             void vscode.window.showErrorMessage(
-              'Sorry, we were unable to generate the playground, please try again. If the error persists, try changing your selected code.'
+              'Failed to generate a MongoDB Playground. Please ensure your code block contains a MongoDB query.'
             );
+
+            this._telemetryService.trackExportToPlaygroundFailed({
+              input_length: codeToExport?.length,
+              details: 'streamChatResponseWithExportToLanguage',
+            });
+
             return null;
           }
 
@@ -1827,12 +1854,19 @@ export default class ParticipantController {
         return true;
       }
 
-      await vscode.commands.executeCommand(
-        EXTENSION_COMMANDS.OPEN_PARTICIPANT_CODE_IN_PLAYGROUND,
-        {
-          runnableContent: content,
-        }
-      );
+      try {
+        await vscode.commands.executeCommand(
+          EXTENSION_COMMANDS.OPEN_PARTICIPANT_CODE_IN_PLAYGROUND,
+          {
+            runnableContent: content,
+          }
+        );
+      } catch (error) {
+        this._telemetryService.trackExportToPlaygroundFailed({
+          input_length: content.length,
+          details: 'executeCommand',
+        });
+      }
 
       return true;
     } catch (error) {

diff --git a/src/telemetry/telemetryService.ts b/src/telemetry/telemetryService.ts
@@ -54,6 +54,11 @@ type DocumentEditedTelemetryEventProperties = {
   source: DocumentSource;
 };
 
+type ExportToPlaygroundFailedEventProperties = {
+  input_length: number | undefined;
+  details: string;
+};
+
 type PlaygroundExportedToLanguageTelemetryEventProperties = {
   language?: string;
   exported_code_length: number;
@@ -106,6 +111,7 @@ type ParticipantResponseFailedProperties = {
   command: ParticipantResponseType;
   error_code?: string;
   error_name: ParticipantErrorTypes;
+  error_details?: string;
 };
 
 export type InternalPromptPurpose = 'intent' | 'namespace' | undefined;
@@ -171,6 +177,7 @@ type TelemetryEventProperties =
   | PlaygroundSavedTelemetryEventProperties
   | PlaygroundLoadedTelemetryEventProperties
   | KeytarSecretsMigrationFailedProperties
+  | ExportToPlaygroundFailedEventProperties
   | SavedConnectionsLoadedProperties
   | ParticipantFeedbackProperties
   | ParticipantResponseFailedProperties
@@ -193,6 +200,7 @@ export enum TelemetryEventTypes {
   PLAYGROUND_EXPORTED_TO_LANGUAGE = 'Playground Exported To Language',
   PLAYGROUND_CREATED = 'Playground Created',
   KEYTAR_SECRETS_MIGRATION_FAILED = 'Keytar Secrets Migration Failed',
+  EXPORT_TO_PLAYGROUND_FAILED = 'Export To Playground Failed',
   SAVED_CONNECTIONS_LOADED = 'Saved Connections Loaded',
   PARTICIPANT_FEEDBACK = 'Participant Feedback',
   PARTICIPANT_WELCOME_SHOWN = 'Participant Welcome Shown',
@@ -346,6 +354,12 @@ export default class TelemetryService {
     );
   }
 
+  trackExportToPlaygroundFailed(
+    props: ExportToPlaygroundFailedEventProperties
+  ): void {
+    this.track(TelemetryEventTypes.EXPORT_TO_PLAYGROUND_FAILED, props);
+  }
+
   trackCommandRun(command: ExtensionCommand): void {
     this.track(TelemetryEventTypes.EXTENSION_COMMAND_RUN, { command });
   }

diff --git a/src/test/suite/participant/participant.test.ts b/src/test/suite/participant/participant.test.ts
@@ -1818,6 +1818,37 @@ Schema:
           );
         });
 
+        test('tracks failures with export to playground and not as a failed prompt', async function () {
+          const editor = vscode.window.activeTextEditor;
+          if (!editor) {
+            throw new Error('Window active text editor is undefined');
+          }
+
+          const testDocumentUri = editor.document.uri;
+          const edit = new vscode.WorkspaceEdit();
+          const code = `
+          THIS IS SOME ERROR CAUSING CODE.
+`;
+          edit.replace(testDocumentUri, getFullRange(editor.document), code);
+          await vscode.workspace.applyEdit(edit);
+
+          await testParticipantController.exportCodeToPlayground();
+          sendRequestStub.rejects();
+          const messages = sendRequestStub.firstCall.args[0];
+          expect(getMessageContent(messages[1])).to.equal(code.trim());
+          expect(telemetryTrackStub).calledWith(
+            TelemetryEventTypes.EXPORT_TO_PLAYGROUND_FAILED,
+            {
+              input_length: code.trim().length,
+              details: 'streamChatResponseWithExportToLanguage',
+            }
+          );
+
+          expect(telemetryTrackStub).not.calledWith(
+            TelemetryEventTypes.PARTICIPANT_RESPONSE_FAILED
+          );
+        });
+
         test('exports selected lines of code to a playground', async function () {
           const editor = vscode.window.activeTextEditor;
           if (!editor) {

diff --git a/src/views/webview-app/overview-page.tsx b/src/views/webview-app/overview-page.tsx
@@ -126,6 +126,7 @@ const OverviewPage: React.FC = () => {
         <FileInputBackendProvider
           createFileInputBackend={createElectronFileInputBackend(
             dialogProvider,
+            // @ts-expect-error Type for this may not be accurate.
             null
           )}
         >