連続再生中にバックグラウンドで音声を合成する (VOICEVOX#1774)

* generate audio in background while continuously playing

* FETCH_AUDIO and PLAY_AUDIO_BLOB in ContinuousPlayer

* change private interfaces

* audioContinuousPlayer.tsから依存を失くす

* generation & playUntilComplete

---------

Co-authored-by: Hiroshiba <[email protected]>

src/store/audio.ts

@@ -36,6 +36,7 @@ import {
   handlePossiblyNotMorphableError,
   isMorphable,
 } from "./audioGenerate";
+import { ContinuousPlayer } from "./audioContinuousPlayer";
 import {
   AudioKey,
   CharacterInfo,
@@ -1730,23 +1731,40 @@ export const audioStore = createPartialStore<AudioStoreTypes>({
         index = state.audioKeys.findIndex((v) => v === currentAudioKey);
       }
 
-      commit("SET_NOW_PLAYING_CONTINUOUSLY", { nowPlaying: true });
-      try {
-        for (let i = index; i < state.audioKeys.length; ++i) {
-          const audioKey = state.audioKeys[i];
-          commit("SET_ACTIVE_AUDIO_KEY", { audioKey });
-          const isEnded = await dispatch("PLAY_AUDIO", { audioKey });
-          if (!isEnded) {
-            break;
-          }
-        }
-      } finally {
-        commit("SET_ACTIVE_AUDIO_KEY", { audioKey: currentAudioKey });
-        commit("SET_AUDIO_PLAY_START_POINT", {
-          startPoint: currentAudioPlayStartPoint,
+      const player = new ContinuousPlayer(state.audioKeys.slice(index), {
+        generateAudio: ({ audioKey }) =>
+          dispatch("FETCH_AUDIO", { audioKey }).then((result) => result.blob),
+        playAudioBlob: ({ audioBlob, audioKey }) =>
+          dispatch("PLAY_AUDIO_BLOB", { audioBlob, audioKey }),
+      });
+      player.addEventListener("playstart", (e) => {
+        commit("SET_ACTIVE_AUDIO_KEY", { audioKey: e.audioKey });
+      });
+      player.addEventListener("waitstart", (e) => {
+        dispatch("START_PROGRESS");
+        commit("SET_ACTIVE_AUDIO_KEY", { audioKey: e.audioKey });
+        commit("SET_AUDIO_NOW_GENERATING", {
+          audioKey: e.audioKey,
+          nowGenerating: true,
         });
-        commit("SET_NOW_PLAYING_CONTINUOUSLY", { nowPlaying: false });
-      }
+      });
+      player.addEventListener("waitend", (e) => {
+        dispatch("RESET_PROGRESS");
+        commit("SET_AUDIO_NOW_GENERATING", {
+          audioKey: e.audioKey,
+          nowGenerating: false,
+        });
+      });
+
+      commit("SET_NOW_PLAYING_CONTINUOUSLY", { nowPlaying: true });
+
+      await player.playUntilComplete();
+
+      commit("SET_ACTIVE_AUDIO_KEY", { audioKey: currentAudioKey });
+      commit("SET_AUDIO_PLAY_START_POINT", {
+        startPoint: currentAudioPlayStartPoint,
+      });
+      commit("SET_NOW_PLAYING_CONTINUOUSLY", { nowPlaying: false });
     }),
   },
 });

src/store/audioContinuousPlayer.ts

@@ -0,0 +1,172 @@
+import { AudioKey } from "@/type/preload";
+
+interface DI {
+  /**
+   * 音声を生成する
+   */
+  generateAudio({ audioKey }: { audioKey: AudioKey }): Promise<Blob>;
+
+  /**
+   * 音声を再生する。
+   * 再生が完了した場合trueを、途中で停止した場合falseを返す。
+   */
+  playAudioBlob({
+    audioBlob,
+    audioKey,
+  }: {
+    audioBlob: Blob;
+    audioKey: AudioKey;
+  }): Promise<boolean>;
+}
+
+/**
+ * 音声を生成しながら連続再生する。
+ * 生成の開始・完了、再生の開始・完了、生成待機の開始・完了のイベントを発行する。
+ */
+export class ContinuousPlayer extends EventTarget {
+  private generating?: AudioKey;
+  private playQueue: { audioKey: AudioKey; audioBlob: Blob }[] = [];
+  private playing?: { audioKey: AudioKey; audioBlob: Blob };
+
+  private finished = false;
+  private resolve!: () => void;
+  private promise: Promise<void>;
+
+  constructor(
+    private generationQueue: AudioKey[],
+    { generateAudio, playAudioBlob }: DI
+  ) {
+    super();
+
+    this.addEventListener("generatestart", (e) => {
+      this.generating = e.audioKey;
+    });
+    this.addEventListener("generatestart", async (e) => {
+      const audioBlob = await generateAudio({ audioKey: e.audioKey });
+      this.dispatchEvent(new GenerateEndEvent(e.audioKey, audioBlob));
+    });
+    this.addEventListener("generateend", (e) => {
+      delete this.generating;
+
+      const { audioKey, audioBlob } = e;
+      if (this.playing) {
+        this.playQueue.push({ audioKey, audioBlob });
+      } else {
+        this.dispatchEvent(new WaitEndEvent(e.audioKey));
+        if (this.finished) return;
+        this.dispatchEvent(new PlayStartEvent(audioKey, audioBlob));
+      }
+
+      const next = this.generationQueue.shift();
+      if (next) {
+        this.dispatchEvent(new GenerateStartEvent(next));
+      }
+    });
+    this.addEventListener("playstart", (e) => {
+      this.playing = { audioKey: e.audioKey, audioBlob: e.audioBlob };
+    });
+    this.addEventListener("playstart", async (e) => {
+      const isEnded = await playAudioBlob({
+        audioBlob: e.audioBlob,
+        audioKey: e.audioKey,
+      });
+      this.dispatchEvent(new PlayEndEvent(e.audioKey, !isEnded));
+    });
+    this.addEventListener("playend", (e) => {
+      delete this.playing;
+      if (e.forceFinish) {
+        this.finish();
+        return;
+      }
+
+      const next = this.playQueue.shift();
+      if (next) {
+        this.dispatchEvent(new PlayStartEvent(next.audioKey, next.audioBlob));
+      } else if (this.generating) {
+        this.dispatchEvent(new WaitStartEvent(this.generating));
+      } else {
+        this.finish();
+      }
+    });
+
+    this.promise = new Promise((resolve) => {
+      this.resolve = resolve;
+    });
+  }
+
+  private finish() {
+    this.finished = true;
+    this.resolve();
+  }
+
+  /**
+   * 音声の生成・再生を開始する。
+   * すべての音声の再生が完了するか、途中で停止されるとresolveする。
+   */
+  async playUntilComplete() {
+    const next = this.generationQueue.shift();
+    if (!next) return;
+    this.dispatchEvent(new WaitStartEvent(next));
+    this.dispatchEvent(new GenerateStartEvent(next));
+
+    await this.promise;
+  }
+}
+
+export interface ContinuousPlayer extends EventTarget {
+  addEventListener<K extends keyof ContinuousPlayerEvents>(
+    type: K,
+    listener: (this: ContinuousPlayer, ev: ContinuousPlayerEvents[K]) => void,
+    options?: boolean | AddEventListenerOptions
+  ): void;
+  addEventListener(
+    type: string,
+    listener: EventListenerOrEventListenerObject,
+    options?: boolean | AddEventListenerOptions
+  ): void;
+}
+
+interface ContinuousPlayerEvents {
+  generatestart: GenerateStartEvent;
+  generateend: GenerateEndEvent;
+  playstart: PlayStartEvent;
+  playend: PlayEndEvent;
+  waitstart: WaitStartEvent;
+  waitend: WaitEndEvent;
+}
+
+export class GenerateStartEvent extends Event {
+  constructor(public audioKey: AudioKey) {
+    super("generatestart");
+  }
+}
+
+export class GenerateEndEvent extends Event {
+  constructor(public audioKey: AudioKey, public audioBlob: Blob) {
+    super("generateend");
+  }
+}
+
+export class PlayStartEvent extends Event {
+  constructor(public audioKey: AudioKey, public audioBlob: Blob) {
+    super("playstart");
+  }
+}
+
+export class PlayEndEvent extends Event {
+  constructor(public audioKey: AudioKey, public forceFinish: boolean) {
+    super("playend");
+  }
+}
+
+export class WaitStartEvent extends Event {
+  constructor(public audioKey: AudioKey) {
+    super("waitstart");
+  }
+}
+
+export class WaitEndEvent extends Event {
+  constructor(public audioKey: AudioKey) {
+    super("waitend");
+  }
+}