From e0fc41ac3f4c9fad2d4435a2bc1321424bf2c77c Mon Sep 17 00:00:00 2001 From: Sig <62321214+sigprogramming@users.noreply.github.com> Date: Sun, 1 Dec 2024 01:09:35 +0900 Subject: [PATCH 1/6] =?UTF-8?q?lab=E3=83=95=E3=82=A1=E3=82=A4=E3=83=AB?= =?UTF-8?q?=E3=82=92=E6=9B=B8=E3=81=8D=E5=87=BA=E3=81=99=E6=A9=9F=E8=83=BD?= =?UTF-8?q?=E3=82=92=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/components/Sing/AudioExportOverlay.vue | 59 +++++ src/components/Sing/LabelExportOverlay.vue | 63 +++++ src/components/Sing/SingEditor.vue | 31 +-- src/components/Sing/menuBarData.ts | 25 +- src/sing/domain.ts | 23 +- ...tToWavFileData.ts => fileDataGenerator.ts} | 57 ++++- src/store/singing.ts | 236 +++++++++++++++++- src/store/type.ts | 18 ++ 8 files changed, 473 insertions(+), 39 deletions(-) create mode 100644 src/components/Sing/AudioExportOverlay.vue create mode 100644 src/components/Sing/LabelExportOverlay.vue rename src/sing/{convertToWavFileData.ts => fileDataGenerator.ts} (52%) diff --git a/src/components/Sing/AudioExportOverlay.vue b/src/components/Sing/AudioExportOverlay.vue new file mode 100644 index 0000000000..756cb5497b --- /dev/null +++ b/src/components/Sing/AudioExportOverlay.vue @@ -0,0 +1,59 @@ + + + + + diff --git a/src/components/Sing/LabelExportOverlay.vue b/src/components/Sing/LabelExportOverlay.vue new file mode 100644 index 0000000000..ffbf90c44f --- /dev/null +++ b/src/components/Sing/LabelExportOverlay.vue @@ -0,0 +1,63 @@ + + + + + diff --git a/src/components/Sing/SingEditor.vue b/src/components/Sing/SingEditor.vue index 4f5f4db6e2..147bb5964d 100644 --- a/src/components/Sing/SingEditor.vue +++ b/src/components/Sing/SingEditor.vue @@ -2,22 +2,8 @@
-
-
- -
- {{ nowRendering ? "レンダリング中・・・" : "音声を書き出し中・・・" }} -
- -
-
+ + { - return store.state.nowRendering; -}); -const nowAudioExporting = computed(() => { - return store.state.nowAudioExporting; -}); - -const cancelExport = () => { - void store.actions.CANCEL_AUDIO_EXPORT(); -}; - const isCompletedInitialStartup = ref(false); // TODO: Vueっぽくないので解体する onetimeWatch( diff --git a/src/components/Sing/menuBarData.ts b/src/components/Sing/menuBarData.ts index a32dbfe144..4fd22070e6 100644 --- a/src/components/Sing/menuBarData.ts +++ b/src/components/Sing/menuBarData.ts @@ -2,6 +2,7 @@ import { computed } from "vue"; import { useStore } from "@/store"; import { MenuItemData } from "@/components/Menu/type"; import { useRootMiscSetting } from "@/composables/useRootMiscSetting"; +import { notifyResult } from "@/components/Dialog/Dialog"; export const useMenuBarData = () => { const store = useStore(); @@ -24,16 +25,38 @@ export const useMenuBarData = () => { }); }; + const exportLabelFile = async () => { + const results = await store.actions.EXPORT_LABEL_FILES({}); + + if (results.length === 0) { + throw new Error("results.length is 0."); + } + notifyResult( + results[0], // TODO: SaveResultObject[] に対応する + "text", + store.actions, + store.state.confirmedTips.notifyOnGenerate, + ); + }; + // 「ファイル」メニュー const fileSubMenuData = computed(() => [ { type: "button", - label: "音声を出力", + label: "音声書き出し", onClick: () => { void exportAudioFile(); }, disableWhenUiLocked: true, }, + { + type: "button", + label: "labファイルを書き出し", + onClick: () => { + void exportLabelFile(); + }, + disableWhenUiLocked: true, + }, { type: "separator" }, { type: "button", diff --git a/src/sing/domain.ts b/src/sing/domain.ts index 850d691a75..aedd531d0e 100644 --- a/src/sing/domain.ts +++ b/src/sing/domain.ts @@ -513,7 +513,7 @@ export type PhonemeTimingEditData = Map; /** * 音素列を音素タイミング列に変換する。 */ -function phonemesToPhonemeTimings(phonemes: FramePhoneme[]) { +export function phonemesToPhonemeTimings(phonemes: FramePhoneme[]) { const phonemeTimings: PhonemeTiming[] = []; let cumulativeFrame = 0; for (const phoneme of phonemes) { @@ -531,7 +531,7 @@ function phonemesToPhonemeTimings(phonemes: FramePhoneme[]) { /** * 音素タイミング列を音素列に変換する。 */ -function phonemeTimingsToPhonemes(phonemeTimings: PhonemeTiming[]) { +export function phonemeTimingsToPhonemes(phonemeTimings: PhonemeTiming[]) { return phonemeTimings.map( (value): FramePhoneme => ({ phoneme: value.phoneme, @@ -544,7 +544,7 @@ function phonemeTimingsToPhonemes(phonemeTimings: PhonemeTiming[]) { /** * フレーズごとの音素列を全体の音素タイミング列に変換する。 */ -function toEntirePhonemeTimings( +export function toEntirePhonemeTimings( phrasePhonemeSequences: FramePhoneme[][], phraseStartFrames: number[], ) { @@ -725,7 +725,7 @@ function applyPhonemeTimingEditToPhonemeTimings( /** * 音素が重ならないように音素タイミングとフレーズの終了フレームを調整する。 */ -function adjustPhonemeTimingsAndPhraseEndFrames( +export function adjustPhonemeTimingsAndPhraseEndFrames( phonemeTimings: PhonemeTiming[], phraseStartFrames: number[], phraseEndFrames: number[], @@ -816,13 +816,24 @@ function adjustPhonemeTimingsAndPhraseEndFrames( } } -function calcPhraseStartFrames(phraseStartTimes: number[], frameRate: number) { +/** + * フレーズの開始フレームを算出する。 + * 開始フレームは整数。 + */ +export function calcPhraseStartFrames( + phraseStartTimes: number[], + frameRate: number, +) { return phraseStartTimes.map((value) => secondToRoundedFrame(value, frameRate), ); } -function calcPhraseEndFrames( +/** + * フレーズの終了フレームを算出する。 + * 終了フレームは整数。 + */ +export function calcPhraseEndFrames( phraseStartFrames: number[], phraseQueries: EditorFrameAudioQuery[], ) { diff --git a/src/sing/convertToWavFileData.ts b/src/sing/fileDataGenerator.ts similarity index 52% rename from src/sing/convertToWavFileData.ts rename to src/sing/fileDataGenerator.ts index 1ddfb637e4..0980bc2033 100644 --- a/src/sing/convertToWavFileData.ts +++ b/src/sing/fileDataGenerator.ts @@ -1,4 +1,8 @@ -export const convertToWavFileData = (audioBuffer: AudioBuffer) => { +import Encoding from "encoding-japanese"; +import { Encoding as EncodingType } from "@/type/preload"; +import { FramePhoneme } from "@/openapi"; + +export function generateWavFileData(audioBuffer: AudioBuffer) { const bytesPerSample = 4; // Float32 const formatCode = 3; // WAVE_FORMAT_IEEE_FLOAT @@ -53,4 +57,53 @@ export const convertToWavFileData = (audioBuffer: AudioBuffer) => { } return new Uint8Array(buffer); -}; +} + +export async function generateTextFileData(obj: { + text: string; + encoding?: EncodingType; +}) { + obj.encoding ??= "UTF-8"; + + const textBlob = { + "UTF-8": (text: string) => { + const bom = new Uint8Array([0xef, 0xbb, 0xbf]); + return new Blob([bom, text], { + type: "text/plain;charset=UTF-8", + }); + }, + Shift_JIS: (text: string) => { + const sjisArray = Encoding.convert(Encoding.stringToCode(text), { + to: "SJIS", + type: "arraybuffer", + }); + return new Blob([new Uint8Array(sjisArray)], { + type: "text/plain;charset=Shift_JIS", + }); + }, + }[obj.encoding](obj.text); + + return await textBlob.arrayBuffer(); +} + +export async function generateLabelFileData( + phonemes: FramePhoneme[], + frameRate: number, +) { + let labString = ""; + let timestamp = 0; + + const writeLine = (phonemeLengthSeconds: number, phoneme: string) => { + labString += timestamp.toFixed() + " "; + timestamp += phonemeLengthSeconds * 10e7; // 100ns単位に変換 + labString += timestamp.toFixed() + " "; + labString += phoneme + "\n"; + }; + + for (const phoneme of phonemes) { + // REVIEW: vowel != "N" のときに vowel.toLowerCase() する必要がある…？ + writeLine(phoneme.frameLength / frameRate, phoneme.phoneme); + } + + return await generateTextFileData({ text: labString }); +} diff --git a/src/store/singing.ts b/src/store/singing.ts index d343bfe912..1c5e55b7f6 100644 --- a/src/store/singing.ts +++ b/src/store/singing.ts @@ -87,11 +87,17 @@ import { shouldPlayTracks, decibelToLinear, applyPitchEdit, + calcPhraseStartFrames, + calcPhraseEndFrames, + toEntirePhonemeTimings, + adjustPhonemeTimingsAndPhraseEndFrames, + phonemeTimingsToPhonemes, } from "@/sing/domain"; import { getOverlappingNoteIds } from "@/sing/storeHelper"; import { AnimationTimer, calculateHash, + createArray, createPromiseThatResolvesWhen, linearInterpolation, round, @@ -103,8 +109,11 @@ import { getOrThrow } from "@/helpers/mapHelper"; import { cloneWithUnwrapProxy } from "@/helpers/cloneWithUnwrapProxy"; import { ufProjectToVoicevox } from "@/sing/utaformatixProject/toVoicevox"; import { uuid4 } from "@/helpers/random"; -import { convertToWavFileData } from "@/sing/convertToWavFileData"; import { generateWriteErrorMessage } from "@/helpers/fileHelper"; +import { + generateLabelFileData, + generateWavFileData, +} from "@/sing/fileDataGenerator"; import path from "@/helpers/path"; import { showAlertDialog } from "@/components/Dialog/Dialog"; @@ -784,7 +793,9 @@ export const singingStoreState: SingingStoreState = { stopRenderingRequested: false, nowRendering: false, nowAudioExporting: false, + nowLabelExporting: false, cancellationOfAudioExportRequested: false, + cancellationOfLabelExportRequested: false, isSongSidebarOpen: false, }; @@ -2671,12 +2682,19 @@ export const singingStore = createPartialStore({ }); }, }, + SET_NOW_AUDIO_EXPORTING: { mutation(state, { nowAudioExporting }) { state.nowAudioExporting = nowAudioExporting; }, }, + SET_NOW_LABEL_EXPORTING: { + mutation(state, { nowLabelExporting }) { + state.nowLabelExporting = nowLabelExporting; + }, + }, + SET_CANCELLATION_OF_AUDIO_EXPORT_REQUESTED: { mutation(state, { cancellationOfAudioExportRequested }) { state.cancellationOfAudioExportRequested = @@ -2684,6 +2702,13 @@ export const singingStore = createPartialStore({ }, }, + SET_CANCELLATION_OF_LABEL_EXPORT_REQUESTED: { + mutation(state, { cancellationOfLabelExportRequested }) { + state.cancellationOfLabelExportRequested = + cancellationOfLabelExportRequested; + }, + }, + EXPORT_AUDIO_FILE: { action: createUILockAction( async ({ state, mutations, getters, actions }, { filePath, setting }) => { @@ -2747,7 +2772,7 @@ export const singingStore = createPartialStore({ phraseSingingVoices, ); - const fileData = convertToWavFileData(audioBuffer); + const fileData = generateWavFileData(audioBuffer); const result = await actions.EXPORT_FILE({ filePath, @@ -2874,7 +2899,7 @@ export const singingStore = createPartialStore({ singingVoiceCache, ); - const fileData = convertToWavFileData(audioBuffer); + const fileData = generateWavFileData(audioBuffer); const result = await actions.EXPORT_FILE({ filePath, @@ -2903,6 +2928,199 @@ export const singingStore = createPartialStore({ ), }, + EXPORT_LABEL_FILES: { + action: createUILockAction( + async ({ actions, mutations, state, getters }, { dirPath }) => { + const exportLabelFile = async () => { + if (state.nowPlaying) { + await actions.SING_STOP_AUDIO(); + } + + if (state.savingSetting.fixedExportEnabled) { + dirPath = state.savingSetting.fixedExportDir; + } else { + dirPath ??= await window.backend.showSaveDirectoryDialog({ + title: "labファイルを保存", + }); + } + if (!dirPath) { + return createArray( + state.tracks.size, + (): SaveResultObject => ({ result: "CANCELED", path: "" }), + ); + } + + if (state.nowRendering) { + await createPromiseThatResolvesWhen(() => { + return ( + !state.nowRendering || state.cancellationOfLabelExportRequested + ); + }); + if (state.cancellationOfLabelExportRequested) { + return createArray( + state.tracks.size, + (): SaveResultObject => ({ result: "CANCELED", path: "" }), + ); + } + } + + const results: SaveResultObject[] = []; + + for (const [i, trackId] of state.trackOrder.entries()) { + const track = getOrThrow(state.tracks, trackId); + if (!track.singer) { + continue; + } + + const characterInfo = getters.CHARACTER_INFO( + track.singer.engineId, + track.singer.styleId, + ); + if (!characterInfo) { + continue; + } + + const style = characterInfo.metas.styles.find( + (style) => style.styleId === track.singer?.styleId, + ); + if (style == undefined) { + throw new Error("assert style != undefined"); + } + + const styleName = style.styleName || DEFAULT_STYLE_NAME; + const projectName = getters.PROJECT_NAME ?? DEFAULT_PROJECT_NAME; + + const trackFileName = buildSongTrackAudioFileNameFromRawData( + state.savingSetting.songTrackFileNamePattern, + { + characterName: characterInfo.metas.speakerName, + index: i, + styleName, + date: currentDateString(), + projectName, + trackName: track.name, + }, + ); + let filePath = path.join(dirPath, `${trackFileName}.lab`); + if (state.savingSetting.avoidOverwrite) { + let tail = 1; + const pathWithoutExt = filePath.slice(0, -4); + while (await window.backend.checkFileExists(filePath)) { + filePath = `${pathWithoutExt}[${tail}].lab`; + tail += 1; + } + } + + const frameRate = state.editorFrameRate; + const phrases = [...state.phrases.values()] + .filter((value) => value.trackId === trackId) + .filter((value) => value.queryKey != undefined) + .toSorted((a, b) => a.startTime - b.startTime); + + if (phrases.length === 0) { + continue; + } + + const phraseQueries = phrases.map((value) => { + const phraseQuery = + value.queryKey != undefined + ? state.phraseQueries.get(value.queryKey) + : undefined; + if (phraseQuery == undefined) { + throw new Error("phraseQuery is undefined."); + } + return phraseQuery; + }); + const phraseStartTimes = phrases.map((value) => value.startTime); + + for (const phraseQuery of phraseQueries) { + // フレーズのクエリのフレームレートとエディターのフレームレートが一致しない場合はエラー + // TODO: 補間するようにする + if (phraseQuery.frameRate != frameRate) { + throw new Error( + "The frame rate between the phrase query and the editor does not match.", + ); + } + } + + const phraseStartFrames = calcPhraseStartFrames( + phraseStartTimes, + frameRate, + ); + const phraseEndFrames = calcPhraseEndFrames( + phraseStartFrames, + phraseQueries, + ); + + const phrasePhonemeSequences = phraseQueries.map((query) => { + return query.phonemes; + }); + const entirePhonemeTimings = toEntirePhonemeTimings( + phrasePhonemeSequences, + phraseStartFrames, + ); + + // TODO: 音素タイミング編集データを取得して適用するようにする + + adjustPhonemeTimingsAndPhraseEndFrames( + entirePhonemeTimings, + phraseStartFrames, + phraseEndFrames, + ); + + const entirePhonemes = + phonemeTimingsToPhonemes(entirePhonemeTimings); + const labFileData = await generateLabelFileData( + entirePhonemes, + frameRate, + ); + + try { + await window.backend + .writeFile({ + filePath, + buffer: labFileData, + }) + .then(getValueOrThrow); + + results.push({ result: "SUCCESS", path: filePath }); + } catch (e) { + logger.error("Failed to export file.", e); + + if (e instanceof ResultError) { + results.push({ + result: "WRITE_ERROR", + path: filePath, + errorMessage: generateWriteErrorMessage( + e as ResultError, + ), + }); + } else { + results.push({ + result: "UNKNOWN_ERROR", + path: filePath, + errorMessage: + (e instanceof Error ? e.message : String(e)) || + "不明なエラーが発生しました。", + }); + break; // 想定外のエラーなので書き出しを中断 + } + } + } + return results; + }; + + mutations.SET_NOW_LABEL_EXPORTING({ nowLabelExporting: true }); + return exportLabelFile().finally(() => { + mutations.SET_CANCELLATION_OF_LABEL_EXPORT_REQUESTED({ + cancellationOfLabelExportRequested: false, + }); + mutations.SET_NOW_LABEL_EXPORTING({ nowLabelExporting: false }); + }); + }, + ), + }, + EXPORT_FILE: { async action(_, { filePath, content }) { try { @@ -2946,6 +3164,18 @@ export const singingStore = createPartialStore({ }, }, + CANCEL_LABEL_EXPORT: { + async action({ state, mutations }) { + if (!state.nowLabelExporting) { + logger.warn("CANCEL_LAB_EXPORT on !nowLabelExporting"); + return; + } + mutations.SET_CANCELLATION_OF_LABEL_EXPORT_REQUESTED({ + cancellationOfLabelExportRequested: true, + }); + }, + }, + COPY_NOTES_TO_CLIPBOARD: { async action({ getters }) { const selectedTrack = getters.SELECTED_TRACK; diff --git a/src/store/type.ts b/src/store/type.ts index 50c523d22a..419c58ec00 100644 --- a/src/store/type.ts +++ b/src/store/type.ts @@ -877,7 +877,9 @@ export type SingingStoreState = { stopRenderingRequested: boolean; nowRendering: boolean; nowAudioExporting: boolean; + nowLabelExporting: boolean; cancellationOfAudioExportRequested: boolean; + cancellationOfLabelExportRequested: boolean; isSongSidebarOpen: boolean; }; @@ -1118,6 +1120,10 @@ export type SingingStoreTypes = { action(payload: { isDrag: boolean }): void; }; + EXPORT_LABEL_FILES: { + action(payload: { dirPath?: string }): SaveResultObject[]; + }; + EXPORT_AUDIO_FILE: { action(payload: { filePath?: string; @@ -1143,6 +1149,10 @@ export type SingingStoreTypes = { action(): void; }; + CANCEL_LABEL_EXPORT: { + action(): void; + }; + FETCH_SING_FRAME_VOLUME: { action(palyoad: { notes: NoteForRequestToEngine[]; @@ -1209,10 +1219,18 @@ export type SingingStoreTypes = { mutation: { nowAudioExporting: boolean }; }; + SET_NOW_LABEL_EXPORTING: { + mutation: { nowLabelExporting: boolean }; + }; + SET_CANCELLATION_OF_AUDIO_EXPORT_REQUESTED: { mutation: { cancellationOfAudioExportRequested: boolean }; }; + SET_CANCELLATION_OF_LABEL_EXPORT_REQUESTED: { + mutation: { cancellationOfLabelExportRequested: boolean }; + }; + RENDER: { action(): void; }; From 61b9333f790be6b47df884001afe49be06d50f78 Mon Sep 17 00:00:00 2001 From: Sig <62321214+sigprogramming@users.noreply.github.com> Date: Fri, 6 Dec 2024 00:07:24 +0900 Subject: [PATCH 2/6] =?UTF-8?q?=E6=9B=B8=E3=81=8D=E5=87=BA=E3=81=97?= =?UTF-8?q?=E6=99=82=E3=81=AE=E3=82=B9=E3=83=86=E3=83=BC=E3=83=88=E3=81=A8?= =?UTF-8?q?=E3=82=B3=E3=83=B3=E3=83=9D=E3=83=BC=E3=83=8D=E3=83=B3=E3=83=88?= =?UTF-8?q?=E3=82=92=E5=85=B1=E9=80=9A=E5=8C=96=E3=80=81=E6=9B=B8=E3=81=8D?= =?UTF-8?q?=E5=87=BA=E3=81=97=E4=B8=AD=E3=81=8B=E3=81=AE=E3=83=81=E3=82=A7?= =?UTF-8?q?=E3=83=83=E3=82=AF=E3=82=92=E8=A1=8C=E3=81=A3=E3=81=A6=E3=81=84?= =?UTF-8?q?=E3=81=AA=E3=81=8B=E3=81=A3=E3=81=9F=E3=81=AE=E3=81=A7=E3=83=81?= =?UTF-8?q?=E3=82=A7=E3=83=83=E3=82=AF=E3=82=92=E8=A1=8C=E3=81=86=E3=82=88?= =?UTF-8?q?=E3=81=86=E3=81=AB=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/components/Sing/AudioExportOverlay.vue | 59 ---------- ...belExportOverlay.vue => ExportOverlay.vue} | 23 ++-- src/components/Sing/SingEditor.vue | 6 +- src/store/singing.ts | 102 +++++++----------- src/store/type.ts | 33 +++--- 5 files changed, 70 insertions(+), 153 deletions(-) delete mode 100644 src/components/Sing/AudioExportOverlay.vue rename src/components/Sing/{LabelExportOverlay.vue => ExportOverlay.vue} (64%) diff --git a/src/components/Sing/AudioExportOverlay.vue b/src/components/Sing/AudioExportOverlay.vue deleted file mode 100644 index 756cb5497b..0000000000 --- a/src/components/Sing/AudioExportOverlay.vue +++ /dev/null @@ -1,59 +0,0 @@ - - - - - diff --git a/src/components/Sing/LabelExportOverlay.vue b/src/components/Sing/ExportOverlay.vue similarity index 64% rename from src/components/Sing/LabelExportOverlay.vue rename to src/components/Sing/ExportOverlay.vue index ffbf90c44f..cb49e7ede3 100644 --- a/src/components/Sing/LabelExportOverlay.vue +++ b/src/components/Sing/ExportOverlay.vue @@ -1,18 +1,18 @@