From 72d14cce5cc88b81879d26f59f854983ecdd5ab8 Mon Sep 17 00:00:00 2001 From: Maiko Date: Sun, 26 Nov 2023 12:49:29 +0900 Subject: [PATCH 01/35] Fix creation of empty Oto from blank line in otoini --- OpenUtau.Core/Classic/VoicebankLoader.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/OpenUtau.Core/Classic/VoicebankLoader.cs b/OpenUtau.Core/Classic/VoicebankLoader.cs index 79e6d98e6..bb8614fe5 100644 --- a/OpenUtau.Core/Classic/VoicebankLoader.cs +++ b/OpenUtau.Core/Classic/VoicebankLoader.cs @@ -335,6 +335,10 @@ public static OtoSet ParseOtoSet(Stream stream, string filePath, Encoding encodi }; while (!reader.EndOfStream) { var line = reader.ReadLine().Trim(); + if (string.IsNullOrWhiteSpace(line) ) { + trace.lineNumber++; + continue; + } trace.line = line; try { Oto oto = ParseOto(line, trace); From f571c5030546c549ecf2aca8bdf89ad76f4922f4 Mon Sep 17 00:00:00 2001 From: Lotte V Date: Thu, 30 Nov 2023 02:28:03 +0100 Subject: [PATCH 02/35] Add Cantonese Syo Phonemizer --- .../CantoneseSyoPhonemizer.cs | 362 ++++++++++++++++++ 1 file changed, 362 insertions(+) create mode 100644 OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs diff --git a/OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs b/OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs new file mode 100644 index 000000000..7e2e066c4 --- /dev/null +++ b/OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs @@ -0,0 +1,362 @@ +using OpenUtau.Api; +using OpenUtau.Core.G2p; +using OpenUtau.Core.Ustx; +using System.Collections.Generic; +using System.Linq; + +namespace OpenUtau.Plugin.Builtin { + /// + /// Cantonese phonemizer for Syo-style banks. + /// Supports both full jyutping syllables as well as syllable fallbacks without a final consonant or falling diphthong. + /// Supports hanzi and jyutping input. + /// + [Phonemizer("Cantonese Syo-Style Phonemizer", "ZH-YUE SYO", "Lotte V", language: "ZH-YUE")] + public class CantoneseSyoPhonemizer : Phonemizer { + + /// + /// The consonant table. + /// + static readonly string consonants = "b,p,m,f,d,t,n,l,g,k,ng,h,gw,kw,w,z,c,s,j"; + + /// + /// The vowel split table. + /// + static readonly string vowels = "aap=aa p,aat=aa t,aak=aa k,aam=aa m,aan=aa n,aang=aa ng,aai=aa i,aau=aa u,ap=a p,at=a t,ak=a k,am=a m,an=a n,ang=a ng,ai=a i,au=a u,ok=o k,om=o m,on=o n,ong=o ng,oi=o i,ou=o u,oet=oe t,oek=oe k,oeng=oe ng,oei=oe i,eot=eo t,eon=eo n,eoi=eo i,ep=e p,et=e t,ek=e k,em=e m,en=e n,eng=e ng,ei=e i,eu=e u,up=u p,ut=u t,uk=uu k,um=um,un=u n,ung=uu ng,ui=u i,yut=yu t,yun=yu n,ip=i p,it=i t,ik=ii k,im=i m,in=i n,ing=ii ng,iu=i u"; + + /// + /// Check for vowel substitutes. + /// + static readonly string[] substitution = new string[] { + "aap,aat,aak,aam,aan,aang,aai,aau=aa", "ap,at,ak,am,an,ang,ai,au=a", "op,ot,ok,om,on,ong,oi,ou=o", "oet,oek,oen,oeng,oei=oe", "eot,eon,eoi=eo","ep,et,ek,em,en,eng,ei,eu=e", "uk,ung=uu", "up,ut,um,un,ui=u", "yut,yun=yu","ik,ing=ii", "ip,it,im,in,iu=i" + }; + + /// + /// Check for substitutes for finals. + /// + static readonly string[] finalSub = new string[] { + "ii ng=i ng", "ii k=i k", "uu k=u k", "uu ng=u ng", "oe t=eo t", "oe i=eo i" + }; + + static HashSet cSet; + static Dictionary vDict; + static readonly Dictionary substituteLookup; + static readonly Dictionary finalSubLookup; + + static CantoneseSyoPhonemizer() { + cSet = new HashSet(consonants.Split(',')); + vDict = vowels.Split(',') + .Select(s => s.Split('=')) + .ToDictionary(a => a[0], a => a[1]); + substituteLookup = substitution.ToList() + .SelectMany(line => { + var parts = line.Split('='); + return parts[0].Split(',').Select(orig => (orig, parts[1])); + }) + .ToDictionary(t => t.Item1, t => t.Item2); + finalSubLookup = finalSub.ToList() + .SelectMany(line => { + var parts = line.Split('='); + return parts[0].Split(',').Select(orig => (orig, parts[1])); + }) + .ToDictionary(t => t.Item1, t => t.Item2); + } + + private USinger singer; + + // Simply stores the singer in a field. + public override void SetSinger(USinger singer) => this.singer = singer; + + /// + /// Converts hanzi notes to jyutping phonemes. + /// + /// + public override void SetUp(Note[][] groups) { + JyutpingConversion.RomanizeNotes(groups); + } + public override Result Process(Note[] notes, Note? prev, Note? next, Note? prevNeighbour, Note? nextNeighbour, Note[] prevNeighbours) { + // The overall logic is: + // 1. Remove consonant: "jyut" -> "yut". + // 2. Lookup the trailing sound in vowel table: "yut" -> "yu t". + // 3. Split the total duration and returns "jyut"/"jyu" and "yu t". + var note = notes[0]; + var lyric = note.lyric; + string consonant = string.Empty; + string vowel = string.Empty; + + if (lyric.Length > 2 && cSet.Contains(lyric.Substring(0, 2))) { + // First try to find consonant "gw", "kw", "ng", and extract vowel. + consonant = lyric.Substring(0, 2); + vowel = lyric.Substring(2); + } else if (lyric.Length > 1 && cSet.Contains(lyric.Substring(0, 1)) && lyric != "ng") { + // Then try to find single character consonants, and extract vowel. + consonant = lyric.Substring(0, 1); + vowel = lyric.Substring(1); + } else { + // Otherwise the lyric is a vowel. + vowel = lyric; + } + + string phoneme0 = lyric; + + // Get color + string color = string.Empty; + int toneShift = 0; + int? alt = 0; + if (note.phonemeAttributes != null) { + var attr = note.phonemeAttributes.FirstOrDefault(attr0 => attr0.index == 0); + color = attr.voiceColor; + toneShift = attr.toneShift; + alt = attr.alternate; + } + + string fin = $"{vowel} -"; + // We will need to split the total duration for phonemes, so we compute it here. + int totalDuration = notes.Sum(n => n.duration); + // Lookup the vowel split table. For example, "yut" will match "yu t". + if (vDict.TryGetValue(vowel, out var phoneme1) && !string.IsNullOrEmpty(phoneme1)) { + // Now phoneme0="jyu" and phoneme1="yu t", + // try to give "yu t" 120 ticks, but no more than half of the total duration. + int length1 = 120; + + if (length1 > totalDuration / 2) { + length1 = totalDuration / 2; + } + var lyrics = new List { lyric }; + // find potential substitute symbol + if (substituteLookup.TryGetValue(vowel ?? string.Empty, out var sub)) { + if (!string.IsNullOrEmpty(consonant)) { + lyrics.Add($"{consonant}{sub}"); + } else { + lyrics.Add(sub); + } + } + + // Try initial and then a plain lyric + if (prevNeighbour == null) { + var initial = $"- {lyric}"; + var initial2 = $"- {lyrics[1]}"; + var tests = new List { initial, initial2, lyric, lyrics[1] }; + if (checkOtoUntilHit(tests, note, out var otoInit)) { + phoneme0 = otoInit.Alias; + } + } else { // nothing special necessary + if (checkOtoUntilHit(lyrics, note, out var otoLyric)) { + phoneme0 = otoLyric.Alias; + } else { + return MakeSimpleResult(phoneme0); + } + } + + var tails = new List { phoneme1 }; + if (!string.IsNullOrEmpty(phoneme1)) { + // find potential substitute symbol + if (finalSubLookup.TryGetValue(phoneme1 ?? string.Empty, out var finSub)) { + tails.Add(finSub); + } + if (checkOtoUntilHitFinal(tails, note, out var otoTail)) { + phoneme1 = otoTail.Alias; + } else { + // Check for vowel ending if final does not exist and there's no next neighbor + if (nextNeighbour == null && !string.IsNullOrEmpty(fin)) { + // Vowel ending is minimum 60 ticks, maximum half of note + int length2 = 60; + + if (length2 > totalDuration / 2) { + length2 = totalDuration / 2; + } + + var finals = new List { fin }; + if (checkOtoUntilHitFinal(finals, note, out var otoFin)) { + phoneme1 = otoFin.Alias; + } else { + return MakeSimpleResult(phoneme0); + } + return new Result { + phonemes = new Phoneme[] { + new Phoneme() { + phoneme = phoneme0, + }, + new Phoneme() { + phoneme = phoneme1, + position = totalDuration - length2, + } + }, + }; + } else { + return MakeSimpleResult(phoneme0); + } + } + } + + return new Result { + phonemes = new Phoneme[] { + new Phoneme() { + phoneme = phoneme0, + }, + new Phoneme() { + phoneme = phoneme1, + position = totalDuration - length1, + } + }, + }; + } + + // Check for vowel ending on open syllables. + // If a vowel ending does not exist, it will not be inserted. + if (nextNeighbour == null && string.IsNullOrEmpty(phoneme1) && !string.IsNullOrEmpty(fin)) { + // Vowel ending is minimum 60 ticks, maximum half of note + int length1 = 60; + + if (length1 > totalDuration / 2) { + length1 = totalDuration / 2; + } + // Try initial and then a plain lyric + var lyrics = new List { lyric }; + if (prevNeighbour == null) { + var initial = $"- {lyric}"; + var tests = new List { initial, lyric }; + if (checkOtoUntilHit(tests, note, out var otoInit)) { + phoneme0 = otoInit.Alias; + } + } else { // nothing special necessary + if (checkOtoUntilHit(lyrics, note, out var otoLyric)) { + phoneme0 = otoLyric.Alias; + } else { + return MakeSimpleResult(phoneme0); + } + } + + // Map vowel ending + var tails = new List { fin }; + if (checkOtoUntilHitFinal(tails, note, out var otoTail)) { + fin = otoTail.Alias; + } else { + return MakeSimpleResult(phoneme0); + } + + return new Result { + phonemes = new Phoneme[] { + new Phoneme() { + phoneme = phoneme0, + }, + new Phoneme() { + phoneme = fin, + position = totalDuration - length1, + } + }, + }; + } + + // Try initial and then a plain lyric + if (prevNeighbour == null) { + var simpleInitial = $"- {lyric}"; + var tests = new List { simpleInitial, lyric }; + if (checkOtoUntilHit(tests, note, out var otoInit)) { + phoneme0 = otoInit.Alias; + } else { + return MakeSimpleResult(phoneme0); + } + } else { // nothing special necessary + var tests = new List { lyric }; + if (checkOtoUntilHit(tests, note, out var otoLyric)) { + phoneme0 = otoLyric.Alias; + } else { + return MakeSimpleResult(phoneme0); + } + } + // Not spliting is needed. Return as is. + return new Result { + phonemes = new Phoneme[] { + new Phoneme() { + phoneme = phoneme0, + } + }, + }; + + } + + /// + /// Converts hanzi to jyutping, based on G2P. + /// + public class JyutpingConversion { + public static Note[] ChangeLyric(Note[] group, string lyric) { + var oldNote = group[0]; + group[0] = new Note { + lyric = lyric, + phoneticHint = oldNote.phoneticHint, + tone = oldNote.tone, + position = oldNote.position, + duration = oldNote.duration, + phonemeAttributes = oldNote.phonemeAttributes, + }; + return group; + } + + public static string[] Romanize(IEnumerable lyrics) { + return ZhG2p.CantoneseInstance.Convert(lyrics.ToList(), false, true).Split(' '); + } + + public static void RomanizeNotes(Note[][] groups) { + var ResultLyrics = Romanize(groups.Select(group => group[0].lyric)); + Enumerable.Zip(groups, ResultLyrics, ChangeLyric).Last(); + } + + public void SetUp(Note[][] groups) { + RomanizeNotes(groups); + } + } + + // make it quicker to check multiple oto occurrences at once rather than spamming if else if + private bool checkOtoUntilHit(List input, Note note, out UOto oto) { + oto = default; + var attr = note.phonemeAttributes?.FirstOrDefault(attrCheck => attrCheck.index == 0) ?? default; + + var otos = new List(); + foreach (string test in input) { + if (singer.TryGetMappedOto(test + attr.alternate, note.tone + attr.toneShift, attr.voiceColor, out var otoAlt)) { + otos.Add(otoAlt); + } else if (singer.TryGetMappedOto(test, note.tone + attr.toneShift, attr.voiceColor, out var otoCandidacy)) { + otos.Add(otoCandidacy); + } + } + + string color = attr.voiceColor ?? ""; + if (otos.Count > 0) { + if (otos.Any(otoCheck => (otoCheck.Color ?? string.Empty) == color)) { + oto = otos.Find(otoCheck => (otoCheck.Color ?? string.Empty) == color); + return true; + } else { + oto = otos.First(); + return true; + } + } + return false; + } + + // Check for final consonant or vowel ending + private bool checkOtoUntilHitFinal(List input, Note note, out UOto oto) { + oto = default; + var attr = note.phonemeAttributes?.FirstOrDefault(attrCheck => attrCheck.index == 1) ?? default; + + var otos = new List(); + foreach (string test in input) { + if (singer.TryGetMappedOto(test + attr.alternate, note.tone + attr.toneShift, attr.voiceColor, out var otoAlt)) { + otos.Add(otoAlt); + } else if (singer.TryGetMappedOto(test, note.tone + attr.toneShift, attr.voiceColor, out var otoCandidacy)) { + otos.Add(otoCandidacy); + } + } + + string color = attr.voiceColor ?? ""; + if (otos.Count > 0) { + if (otos.Any(otoCheck => (otoCheck.Color ?? string.Empty) == color)) { + oto = otos.Find(otoCheck => (otoCheck.Color ?? string.Empty) == color); + return true; + } else { + return false; + } + } + return false; + } + } +} From 72154367e79d7fc66d597e02f3c5e90baa2ce5bf Mon Sep 17 00:00:00 2001 From: Lotte V Date: Thu, 30 Nov 2023 02:48:55 +0100 Subject: [PATCH 03/35] Small fix --- OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs b/OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs index 7e2e066c4..7d6e6d5fa 100644 --- a/OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs +++ b/OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs @@ -142,8 +142,6 @@ public override Result Process(Note[] notes, Note? prev, Note? next, Note? prevN } else { // nothing special necessary if (checkOtoUntilHit(lyrics, note, out var otoLyric)) { phoneme0 = otoLyric.Alias; - } else { - return MakeSimpleResult(phoneme0); } } From eac046741737ca85a2c733cf54f2b6c0c6d164c2 Mon Sep 17 00:00:00 2001 From: Lotte V Date: Thu, 30 Nov 2023 03:05:28 +0100 Subject: [PATCH 04/35] Forgot a few endings --- OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs b/OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs index 7d6e6d5fa..144c28dfc 100644 --- a/OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs +++ b/OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs @@ -21,7 +21,7 @@ public class CantoneseSyoPhonemizer : Phonemizer { /// /// The vowel split table. /// - static readonly string vowels = "aap=aa p,aat=aa t,aak=aa k,aam=aa m,aan=aa n,aang=aa ng,aai=aa i,aau=aa u,ap=a p,at=a t,ak=a k,am=a m,an=a n,ang=a ng,ai=a i,au=a u,ok=o k,om=o m,on=o n,ong=o ng,oi=o i,ou=o u,oet=oe t,oek=oe k,oeng=oe ng,oei=oe i,eot=eo t,eon=eo n,eoi=eo i,ep=e p,et=e t,ek=e k,em=e m,en=e n,eng=e ng,ei=e i,eu=e u,up=u p,ut=u t,uk=uu k,um=um,un=u n,ung=uu ng,ui=u i,yut=yu t,yun=yu n,ip=i p,it=i t,ik=ii k,im=i m,in=i n,ing=ii ng,iu=i u"; + static readonly string vowels = "aap=aa p,aat=aa t,aak=aa k,aam=aa m,aan=aa n,aang=aa ng,aai=aa i,aau=aa u,ap=a p,at=a t,ak=a k,am=a m,an=a n,ang=a ng,ai=a i,au=a u,op=o p,ot=o kok=o k,om=o m,on=o n,ong=o ng,oi=o i,ou=o u,oet=oe t,oek=oe k,oeng=oe ng,oei=oe i,eot=eo t,eon=eo n,eoi=eo i,ep=e p,et=e t,ek=e k,em=e m,en=e n,eng=e ng,ei=e i,eu=e u,up=u p,ut=u t,uk=uu k,um=um,un=u n,ung=uu ng,ui=u i,yut=yu t,yun=yu n,ip=i p,it=i t,ik=ii k,im=i m,in=i n,ing=ii ng,iu=i u"; /// /// Check for vowel substitutes. From 3a0672612149d7386c97471552873bad8a578ebb Mon Sep 17 00:00:00 2001 From: Lotte V Date: Thu, 30 Nov 2023 13:29:35 +0100 Subject: [PATCH 05/35] Use initial when end stop is used on previous note --- OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs b/OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs index 144c28dfc..2498dd3d1 100644 --- a/OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs +++ b/OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs @@ -132,7 +132,7 @@ public override Result Process(Note[] notes, Note? prev, Note? next, Note? prevN } // Try initial and then a plain lyric - if (prevNeighbour == null) { + if (prevNeighbour == null || (prevNeighbour != null && (prevNeighbour.Value.lyric.EndsWith("p") || prevNeighbour.Value.lyric.EndsWith("t") || prevNeighbour.Value.lyric.EndsWith("k")))) { var initial = $"- {lyric}"; var initial2 = $"- {lyrics[1]}"; var tests = new List { initial, initial2, lyric, lyrics[1] }; @@ -210,7 +210,7 @@ public override Result Process(Note[] notes, Note? prev, Note? next, Note? prevN } // Try initial and then a plain lyric var lyrics = new List { lyric }; - if (prevNeighbour == null) { + if (prevNeighbour == null || (prevNeighbour != null && (prevNeighbour.Value.lyric.EndsWith("p") || prevNeighbour.Value.lyric.EndsWith("t") || prevNeighbour.Value.lyric.EndsWith("k")))) { var initial = $"- {lyric}"; var tests = new List { initial, lyric }; if (checkOtoUntilHit(tests, note, out var otoInit)) { @@ -246,7 +246,7 @@ public override Result Process(Note[] notes, Note? prev, Note? next, Note? prevN } // Try initial and then a plain lyric - if (prevNeighbour == null) { + if (prevNeighbour == null || (prevNeighbour != null && (prevNeighbour.Value.lyric.EndsWith("p") || prevNeighbour.Value.lyric.EndsWith("t") || prevNeighbour.Value.lyric.EndsWith("k")))) { var simpleInitial = $"- {lyric}"; var tests = new List { simpleInitial, lyric }; if (checkOtoUntilHit(tests, note, out var otoInit)) { From c8d4aaa98a961eba2076213e3132968d3c3f4f41 Mon Sep 17 00:00:00 2001 From: Lotte V Date: Thu, 30 Nov 2023 13:34:51 +0100 Subject: [PATCH 06/35] Improve jyutping conversion --- .../CantoneseSyoPhonemizer.cs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs b/OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs index 2498dd3d1..90deba2b1 100644 --- a/OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs +++ b/OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs @@ -291,7 +291,22 @@ public static Note[] ChangeLyric(Note[] group, string lyric) { } public static string[] Romanize(IEnumerable lyrics) { - return ZhG2p.CantoneseInstance.Convert(lyrics.ToList(), false, true).Split(' '); + var lyricsArray = lyrics.ToArray(); + var hanziLyrics = lyricsArray + .Where(ZhG2p.CantoneseInstance.IsHanzi) + .ToList(); + var pinyinResult = ZhG2p.CantoneseInstance.Convert(hanziLyrics, false, false).ToLower().Split(); + if (pinyinResult == null) { + return lyricsArray; + } + var pinyinIndex = 0; + for (int i = 0; i < lyricsArray.Length; i++) { + if (lyricsArray[i].Length == 1 && ZhG2p.CantoneseInstance.IsHanzi(lyricsArray[i])) { + lyricsArray[i] = pinyinResult[pinyinIndex]; + pinyinIndex++; + } + } + return lyricsArray; } public static void RomanizeNotes(Note[][] groups) { From 140da21db906a1919af6237cafa3f8f74b1ec651 Mon Sep 17 00:00:00 2001 From: Lotte V Date: Thu, 30 Nov 2023 18:53:33 +0100 Subject: [PATCH 07/35] Refinement --- OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs b/OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs index 90deba2b1..7b8ddc221 100644 --- a/OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs +++ b/OpenUtau.Plugin.Builtin/CantoneseSyoPhonemizer.cs @@ -21,7 +21,7 @@ public class CantoneseSyoPhonemizer : Phonemizer { /// /// The vowel split table. /// - static readonly string vowels = "aap=aa p,aat=aa t,aak=aa k,aam=aa m,aan=aa n,aang=aa ng,aai=aa i,aau=aa u,ap=a p,at=a t,ak=a k,am=a m,an=a n,ang=a ng,ai=a i,au=a u,op=o p,ot=o kok=o k,om=o m,on=o n,ong=o ng,oi=o i,ou=o u,oet=oe t,oek=oe k,oeng=oe ng,oei=oe i,eot=eo t,eon=eo n,eoi=eo i,ep=e p,et=e t,ek=e k,em=e m,en=e n,eng=e ng,ei=e i,eu=e u,up=u p,ut=u t,uk=uu k,um=um,un=u n,ung=uu ng,ui=u i,yut=yu t,yun=yu n,ip=i p,it=i t,ik=ii k,im=i m,in=i n,ing=ii ng,iu=i u"; + static readonly string vowels = "aap=aa p,aat=aa t,aak=aa k,aam=aa m,aan=aa n,aang=aa ng,aai=aa i,aau=aa u,ap=a p,at=a t,ak=a k,am=a m,an=a n,ang=a ng,ai=a i,au=a u,op=o p,ot=o t,ok=o k,om=o m,on=o n,ong=o ng,oi=o i,ou=o u,oet=oe t,oek=oe k,oeng=oe ng,oei=oe i,eot=eo t,eon=eo n,eoi=eo i,ep=e p,et=e t,ek=e k,em=e m,en=e n,eng=e ng,ei=e i,eu=e u,up=u p,ut=u t,uk=uu k,um=um,un=u n,ung=uu ng,ui=u i,yut=yu t,yun=yu n,ip=i p,it=i t,ik=ii k,im=i m,in=i n,ing=ii ng,iu=i u"; /// /// Check for vowel substitutes. @@ -295,15 +295,15 @@ public static string[] Romanize(IEnumerable lyrics) { var hanziLyrics = lyricsArray .Where(ZhG2p.CantoneseInstance.IsHanzi) .ToList(); - var pinyinResult = ZhG2p.CantoneseInstance.Convert(hanziLyrics, false, false).ToLower().Split(); - if (pinyinResult == null) { + var jyutpingResult = ZhG2p.CantoneseInstance.Convert(hanziLyrics, false, false).ToLower().Split(); + if (jyutpingResult == null) { return lyricsArray; } - var pinyinIndex = 0; + var jyutpingIndex = 0; for (int i = 0; i < lyricsArray.Length; i++) { if (lyricsArray[i].Length == 1 && ZhG2p.CantoneseInstance.IsHanzi(lyricsArray[i])) { - lyricsArray[i] = pinyinResult[pinyinIndex]; - pinyinIndex++; + lyricsArray[i] = jyutpingResult[jyutpingIndex]; + jyutpingIndex++; } } return lyricsArray; From 9ad4752feb8fb91af5ce43071f227d8f353d9ba8 Mon Sep 17 00:00:00 2001 From: Lotte V Date: Thu, 30 Nov 2023 19:27:24 +0100 Subject: [PATCH 08/35] Add Cantonese CVVC Phonemizer --- .../CantoneseCVVCPhonemizer.cs | 186 ++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100644 OpenUtau.Plugin.Builtin/CantoneseCVVCPhonemizer.cs diff --git a/OpenUtau.Plugin.Builtin/CantoneseCVVCPhonemizer.cs b/OpenUtau.Plugin.Builtin/CantoneseCVVCPhonemizer.cs new file mode 100644 index 000000000..847f707e0 --- /dev/null +++ b/OpenUtau.Plugin.Builtin/CantoneseCVVCPhonemizer.cs @@ -0,0 +1,186 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using OpenUtau.Api; +using OpenUtau.Classic; +using OpenUtau.Core.G2p; +using OpenUtau.Core.Ustx; +using Serilog; + +namespace OpenUtau.Plugin.Builtin { + /// + /// Cantonese CVVC phonemizer. + /// It works similarly to the Chinese CVVC phonemizer, including presamp.ini requirement. + /// The big difference is that it converts hanzi to jyutping instead of pinyin. + /// + [Phonemizer("Cantonese CVVC Phonemizer", "ZH-YUE CVVC", language: "ZH-YUE")] + public class CantoneseCVVCPhonemizer : Phonemizer { + private Dictionary vowels = new Dictionary(); + private Dictionary consonants = new Dictionary(); + private USinger singer; + public override Result Process(Note[] notes, Note? prev, Note? next, Note? prevNeighbour, Note? nextNeighbour, Note[] prevNeighbours) { + var lyric = notes[0].lyric; + string consonant = consonants.TryGetValue(lyric, out consonant) ? consonant : lyric; + string prevVowel = "-"; + if (prevNeighbour != null) { + var prevLyric = prevNeighbour.Value.lyric; + if (vowels.TryGetValue(prevLyric, out var vowel)) { + prevVowel = vowel; + } + }; + var attr0 = notes[0].phonemeAttributes?.FirstOrDefault(attr => attr.index == 0) ?? default; + var attr1 = notes[0].phonemeAttributes?.FirstOrDefault(attr => attr.index == 1) ?? default; + if (lyric == "-" || lyric.ToLowerInvariant() == "r") { + if (singer.TryGetMappedOto($"{prevVowel} R", notes[0].tone + attr0.toneShift, attr0.voiceColor, out var oto1)) { + return MakeSimpleResult(oto1.Alias); + } + return MakeSimpleResult($"{prevVowel} R"); + } + if (singer.TryGetMappedOto($"{prevVowel} {lyric}", notes[0].tone + attr0.toneShift, attr0.voiceColor, out var oto)) { + return MakeSimpleResult(oto.Alias); + } + int vcLen = 120; + if (singer.TryGetMappedOto(lyric, notes[0].tone + attr1.toneShift, attr1.voiceColor, out var cvOto)) { + vcLen = MsToTick(cvOto.Preutter); + if (cvOto.Overlap == 0 && vcLen < 120) { + vcLen = Math.Min(120, vcLen * 2); // explosive consonant with short preutter. + } + if (cvOto.Overlap < 0) { + vcLen = MsToTick(cvOto.Preutter - cvOto.Overlap); + } + } + var vcPhoneme = $"{prevVowel} {consonant}"; + if (prevNeighbour != null) { + if (singer.TryGetMappedOto(vcPhoneme, prevNeighbour.Value.tone + attr0.toneShift, attr0.voiceColor, out oto)) { + vcPhoneme = oto.Alias; + } + // totalDuration calculated on basis of previous note length + int totalDuration = prevNeighbour.Value.duration; + // vcLength depends on the Vel of the current base note + vcLen = Convert.ToInt32(Math.Min(totalDuration / 1.5, Math.Max(30, vcLen * (attr1.consonantStretchRatio ?? 1)))); + } else { + if (singer.TryGetMappedOto(vcPhoneme, notes[0].tone + attr0.toneShift, attr0.voiceColor, out oto)) { + vcPhoneme = oto.Alias; + } + // no previous note, so length can be minimum velocity regardless of oto + vcLen = Convert.ToInt32(Math.Min(vcLen * 2, Math.Max(30, vcLen * (attr1.consonantStretchRatio ?? 1)))); + } + + if (singer.TryGetMappedOto(vcPhoneme, notes[0].tone + attr0.toneShift, attr0.voiceColor, out oto)) { + return new Result { + phonemes = new Phoneme[] { + new Phoneme() { + phoneme = vcPhoneme, + position = -vcLen, + }, + new Phoneme() { + phoneme = cvOto?.Alias ?? lyric, + }, + }, + }; + } + return MakeSimpleResult(cvOto?.Alias ?? lyric); + } + + public override void SetSinger(USinger singer) { + if (this.singer == singer) { + return; + } + this.singer = singer; + vowels.Clear(); + consonants.Clear(); + if (this.singer == null) { + return; + } + try { + string file = Path.Combine(singer.Location, "presamp.ini"); + using (var reader = new StreamReader(file, singer.TextFileEncoding)) { + var blocks = Ini.ReadBlocks(reader, file, @"\[\w+\]"); + var vowelLines = blocks.Find(block => block.header == "[VOWEL]").lines; + foreach (var iniLine in vowelLines) { + var parts = iniLine.line.Split('='); + if (parts.Length >= 3) { + string vowelLower = parts[0]; + string vowelUpper = parts[1]; + string[] sounds = parts[2].Split(','); + foreach (var sound in sounds) { + vowels[sound] = vowelLower; + } + } + } + var consonantLines = blocks.Find(block => block.header == "[CONSONANT]").lines; + foreach (var iniLine in consonantLines) { + var parts = iniLine.line.Split('='); + if (parts.Length >= 3) { + string consonant = parts[0]; + string[] sounds = parts[1].Split(','); + foreach (var sound in sounds) { + consonants[sound] = consonant; + } + } + } + var priority = blocks.Find(block => block.header == "PRIORITY"); + var replace = blocks.Find(block => block.header == "REPLACE"); + var alias = blocks.Find(block => block.header == "ALIAS"); + } + } catch (Exception e) { + Log.Error(e, "failed to load presamp.ini"); + } + } + + /// + /// Converts hanzi notes to jyutping phonemes. + /// + /// + public override void SetUp(Note[][] groups) { + JyutpingConversion.RomanizeNotes(groups); + } + + /// + /// Converts hanzi to jyutping, based on G2P. + /// + public class JyutpingConversion { + public static Note[] ChangeLyric(Note[] group, string lyric) { + var oldNote = group[0]; + group[0] = new Note { + lyric = lyric, + phoneticHint = oldNote.phoneticHint, + tone = oldNote.tone, + position = oldNote.position, + duration = oldNote.duration, + phonemeAttributes = oldNote.phonemeAttributes, + }; + return group; + } + + public static string[] Romanize(IEnumerable lyrics) { + var lyricsArray = lyrics.ToArray(); + var hanziLyrics = lyricsArray + .Where(ZhG2p.CantoneseInstance.IsHanzi) + .ToList(); + var jyutpingResult = ZhG2p.CantoneseInstance.Convert(hanziLyrics, false, false).ToLower().Split(); + if (jyutpingResult == null) { + return lyricsArray; + } + var jyutpingIndex = 0; + for (int i = 0; i < lyricsArray.Length; i++) { + if (lyricsArray[i].Length == 1 && ZhG2p.CantoneseInstance.IsHanzi(lyricsArray[i])) { + lyricsArray[i] = jyutpingResult[jyutpingIndex]; + jyutpingIndex++; + } + } + return lyricsArray; + } + + public static void RomanizeNotes(Note[][] groups) { + var ResultLyrics = Romanize(groups.Select(group => group[0].lyric)); + Enumerable.Zip(groups, ResultLyrics, ChangeLyric).Last(); + } + + public void SetUp(Note[][] groups) { + RomanizeNotes(groups); + } + } + } +} From aae2fa04248826e0b160bf04bb2756c31609fa10 Mon Sep 17 00:00:00 2001 From: Lotte V Date: Thu, 30 Nov 2023 19:38:54 +0100 Subject: [PATCH 09/35] Add contributor credit --- OpenUtau.Plugin.Builtin/CantoneseCVVCPhonemizer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/OpenUtau.Plugin.Builtin/CantoneseCVVCPhonemizer.cs b/OpenUtau.Plugin.Builtin/CantoneseCVVCPhonemizer.cs index 847f707e0..33624caad 100644 --- a/OpenUtau.Plugin.Builtin/CantoneseCVVCPhonemizer.cs +++ b/OpenUtau.Plugin.Builtin/CantoneseCVVCPhonemizer.cs @@ -14,7 +14,7 @@ namespace OpenUtau.Plugin.Builtin { /// It works similarly to the Chinese CVVC phonemizer, including presamp.ini requirement. /// The big difference is that it converts hanzi to jyutping instead of pinyin. /// - [Phonemizer("Cantonese CVVC Phonemizer", "ZH-YUE CVVC", language: "ZH-YUE")] + [Phonemizer("Cantonese CVVC Phonemizer", "ZH-YUE CVVC", "Lotte V", language: "ZH-YUE")] public class CantoneseCVVCPhonemizer : Phonemizer { private Dictionary vowels = new Dictionary(); private Dictionary consonants = new Dictionary(); From 14037e405bb5bb7dbf2914f5108bab26d50205bc Mon Sep 17 00:00:00 2001 From: SoulMelody Date: Fri, 1 Dec 2023 12:20:59 +0800 Subject: [PATCH 10/35] Fix default font family name on linux for CJK locales --- OpenUtau/Program.cs | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/OpenUtau/Program.cs b/OpenUtau/Program.cs index 717cc7efb..d81148fc8 100644 --- a/OpenUtau/Program.cs +++ b/OpenUtau/Program.cs @@ -7,6 +7,7 @@ using System.Text; using Avalonia; using Avalonia.Controls; +using Avalonia.Media; using Avalonia.ReactiveUI; using OpenUtau.App.ViewModels; using OpenUtau.Core; @@ -50,11 +51,26 @@ public static void Main(string[] args) { } // Avalonia configuration, don't remove; also used by visual designer. - public static AppBuilder BuildAvaloniaApp() - => AppBuilder.Configure() + public static AppBuilder BuildAvaloniaApp() { + FontManagerOptions fontOptions = new(); + if (OS.IsLinux()) { + using Process process = Process.Start(new ProcessStartInfo("/usr/bin/fc-match", "--format %{family}") + { + RedirectStandardOutput = true + })!; + process.WaitForExit(); + + string fontFamily = process.StandardOutput.ReadToEnd(); + if (!string.IsNullOrEmpty(fontFamily)) { + fontOptions.DefaultFamilyName = fontFamily; + } + } + return AppBuilder.Configure() .UsePlatformDetect() .LogToTrace() - .UseReactiveUI(); + .UseReactiveUI() + .With(fontOptions); + } public static void Run(string[] args) => BuildAvaloniaApp() From 8851a5f1a3c60a66aa57aed02f67ea90b57e803f Mon Sep 17 00:00:00 2001 From: SoulMelody Date: Fri, 1 Dec 2023 13:49:08 +0800 Subject: [PATCH 11/35] specify ArgumentList --- OpenUtau/Program.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/OpenUtau/Program.cs b/OpenUtau/Program.cs index d81148fc8..39c1ba7d0 100644 --- a/OpenUtau/Program.cs +++ b/OpenUtau/Program.cs @@ -54,8 +54,9 @@ public static void Main(string[] args) { public static AppBuilder BuildAvaloniaApp() { FontManagerOptions fontOptions = new(); if (OS.IsLinux()) { - using Process process = Process.Start(new ProcessStartInfo("/usr/bin/fc-match", "--format %{family}") + using Process process = Process.Start(new ProcessStartInfo("/usr/bin/fc-match") { + ArgumentList = { "-f", "%{family}" }, RedirectStandardOutput = true })!; process.WaitForExit(); From 82976973d12340dbe24b24e7f207c3f58745c571 Mon Sep 17 00:00:00 2001 From: oxygen-dioxide <54425948+oxygen-dioxide@users.noreply.github.com> Date: Fri, 1 Dec 2023 14:16:43 +0800 Subject: [PATCH 12/35] windows file case check --- .../Classic/VoicebankErrorChecker.cs | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/OpenUtau.Core/Classic/VoicebankErrorChecker.cs b/OpenUtau.Core/Classic/VoicebankErrorChecker.cs index 9b003057b..39d3d6b93 100644 --- a/OpenUtau.Core/Classic/VoicebankErrorChecker.cs +++ b/OpenUtau.Core/Classic/VoicebankErrorChecker.cs @@ -101,6 +101,20 @@ public void Check() { message = $"There are duplicate aliases.{message}" }); } + //Cross platform check + //Windows path is case insensitive, while MacOS path and Linux path are case sensitive. + //On Windows, check if the wave filename in oto.ini is the same as the filename in the file system. + if(OS.IsWindows()){ + foreach(var otoSet in voicebank.OtoSets) { + WindowsCaseCheck(otoSet); + } + WindowsCaseCheck(voicebank.BasePath, new string[]{ + "chatacter.txt", + "character.yaml", + "prefix.map", + }); + } + //TODO: On MacOS and Linux, check if there are files that have the same name but different case. } bool TryGetFileDuration(string filePath, Oto oto, out double fileDuration) { @@ -240,5 +254,41 @@ bool FindDuplication(out List duplicates) { return duplicates.Count > 0; } + + /// + /// Check if the file names in the oto.ini are the same as the file names in the file system. + /// + /// otoSet to be checked + /// + bool WindowsCaseCheck(OtoSet otoSet) { + return WindowsCaseCheck( + Directory.GetParent(otoSet.File).FullName, + otoSet.Otos + .Select(oto => oto.Wav) + .Append(otoSet.File)//oto.ini itself + .ToHashSet()); + } + + bool WindowsCaseCheck(string folder, IEnumerable correctFileNames){ + bool valid = true; + Dictionary fileNamesLowerToActual = Directory.GetFiles(folder) + .Select(Path.GetFileName) + .ToDictionary(x => x.ToLower(), x => x); + foreach(string fileName in correctFileNames) { + if(!fileNamesLowerToActual.ContainsKey(fileName.ToLower())) { + continue; + } + if (fileNamesLowerToActual[fileName.ToLower()] != fileName) { + valid = false; + Infos.Add(new VoicebankError() { + message = $"Wrong case in file name: \n" + + $"expected: {Path.Join(folder,fileName)}\n" + + $"Actual: {Path.Join(folder,fileNamesLowerToActual[fileName.ToLower()])}\n" + + $"voicebank may not work on another OS." + }); + } + } + return valid; + } } } From d5dbb1bce7b4afd4209a61d2269a41c39a35b39e Mon Sep 17 00:00:00 2001 From: Sugita Akira Date: Sat, 2 Dec 2023 01:55:39 -0800 Subject: [PATCH 13/35] tweaks note params visual --- OpenUtau/Controls/NotePropertiesControl.axaml | 25 ++--- OpenUtau/Styles/Styles.axaml | 16 ++- OpenUtau/Views/PianoRollWindow.axaml | 105 ++++++++++-------- 3 files changed, 79 insertions(+), 67 deletions(-) diff --git a/OpenUtau/Controls/NotePropertiesControl.axaml b/OpenUtau/Controls/NotePropertiesControl.axaml index 29593262e..d2f3dba01 100644 --- a/OpenUtau/Controls/NotePropertiesControl.axaml +++ b/OpenUtau/Controls/NotePropertiesControl.axaml @@ -18,11 +18,10 @@ - - - - - + + + + - + @@ -59,7 +58,7 @@ - + @@ -146,14 +145,14 @@ - + diff --git a/OpenUtau/Styles/Styles.axaml b/OpenUtau/Styles/Styles.axaml index 2c36efe5f..ad7b47fe6 100644 --- a/OpenUtau/Styles/Styles.axaml +++ b/OpenUtau/Styles/Styles.axaml @@ -431,14 +431,20 @@ + + diff --git a/OpenUtau/Views/PianoRollWindow.axaml b/OpenUtau/Views/PianoRollWindow.axaml index 5e2815b0f..84d300720 100644 --- a/OpenUtau/Views/PianoRollWindow.axaml +++ b/OpenUtau/Views/PianoRollWindow.axaml @@ -164,10 +164,17 @@ Fill="{DynamicResource TickLineBrushLow}" ZIndex="1000" IsHitTestVisible="False" IsVisible="False"/> - - - + + + + + + + + +