lpc

huailiang · May 28, 2020 · 7623c9f · 7623c9f
1 parent 38e1c17
commit 7623c9f
Show file tree

Hide file tree

Showing 3 changed files with 53 additions and 11 deletions.
diff --git a/Assets/LipSync/Editor/LpcEditor.cs b/Assets/LipSync/Editor/LpcEditor.cs
@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Linq;
 using UnityEditor;
 using UnityEngine;
@@ -11,6 +12,7 @@ public class LpcEditor : EditorWindow
         static void LpcShow()
         {
             var win = GetWindow<LpcEditor>();
+            win.titleContent = new GUIContent("lPC");
             win.Show();
         }
 
@@ -19,6 +21,11 @@ static void LpcShow()
         private float[] audioBuffer;
         private int window, step, fs;
         private Texture2D tex;
+        private ERecognizerLanguage language;
+        string[] selectedVowels = null;
+        float[] currentVowelFormantCeilValues;
+        string vowelsInfo = "";
+
 
         private void OnEnable()
         {
@@ -28,6 +35,7 @@ private void OnEnable()
                 window = 30;
                 step = 15;
             }
+            language = ERecognizerLanguage.Japanese;
         }
 
         private void OnGUI()
@@ -64,15 +72,25 @@ private void OnGUI()
             EditorGUILayout.EndVertical();
             window = EditorGUILayout.IntField("window", window);
             step = EditorGUILayout.IntField("step", step);
+            language = (ERecognizerLanguage)EditorGUILayout.EnumPopup("language", language);
+            switch (language)
+            {
+                case ERecognizerLanguage.Japanese:
+                    selectedVowels = LipSyncRecognizer.vowelsByFormantJP;
+                    currentVowelFormantCeilValues = LipSyncRecognizer.vowelFormantFloorJP;
+                    break;
+                case ERecognizerLanguage.Chinese:
+                    selectedVowels = LipSyncRecognizer.vowelsByFormantCN;
+                    currentVowelFormantCeilValues = LipSyncRecognizer.vowelFormantFloorCN;
+                    break;
+            }
             GUILayout.Space(4);
+
             if (GUILayout.Button("Analy"))
             {
                 Normalize();
                 var rst = model.Analy(audioBuffer, window, step);
-                for (int i = 0; i < rst.Count; i++)
-                {
-                    Debug.Log(string.Format("{0}: {1:f2} {2:f2} {3:f2}", i, rst[i][0], rst[i][1], rst[i][2]));
-                }
+                VowelsInfo(rst);
             }
             GUILayout.Space(4);
             GUILayout.BeginHorizontal();
@@ -124,6 +142,9 @@ private void OnGUI()
                 Debug.Log(msg);
             }
             GUILayout.EndHorizontal();
+
+            GUILayout.Space(8);
+            GUILayout.Label(vowelsInfo);
             GUILayout.EndVertical();
         }
 
@@ -168,6 +189,25 @@ private int CulSteps(out float w, out float sec)
             return step;
         }
 
+        public void VowelsInfo(List<double[]> formants)
+        {
+            vowelsInfo = "";
+            for (int j = 0; j < formants.Count; j++)
+            {
+                string reslt = "- ";
+                int len = currentVowelFormantCeilValues.Length;
+                for (int i = 0; i < len; ++i)
+                {
+                    if (formants[j][0] > currentVowelFormantCeilValues[i])
+                    {
+                        reslt = selectedVowels[i] + " ";
+                    }
+                }
+                vowelsInfo += reslt;
+                if (j % 50 == 49) vowelsInfo += "\n";
+            }
+        }
+
     }
 
 }
diff --git a/Assets/LipSync/Scripts/Core/LipSyncRecognizer.cs b/Assets/LipSync/Scripts/Core/LipSyncRecognizer.cs
@@ -10,10 +10,10 @@ public class LipSyncRecognizer
         protected const float FILTER_DEVIATION_SQUARE = 5.0f;
         protected const int FORMANT_COUNT = 1;
 
-        protected string[] vowelsByFormantJP = { "i", "u", "e", "o", "a" };
-        protected float[] vowelFormantFloorJP = { 0.0f, 500.0f, 600.0f, 900.0f, 1200.0f };
-        protected string[] vowelsByFormantCN = { "i", "v", "u", "e", "o", "a" };
-        protected float[] vowelFormantFloorCN = { 0.0f, 200.0f, 500.0f, 600.0f, 900.0f, 1200.0f };
+        public static string[] vowelsByFormantJP = { "i", "u", "e", "o", "a" };
+        public static float[] vowelFormantFloorJP = { 0.0f, 500.0f, 600.0f, 900.0f, 1200.0f };
+        public static string[] vowelsByFormantCN = { "i", "v", "u", "e", "o", "a" };
+        public static float[] vowelFormantFloorCN = { 0.0f, 200.0f, 500.0f, 600.0f, 900.0f, 1200.0f };
 
         protected string[] currentVowels;
         protected float[] currentVowelFormantCeilValues;

diff --git a/Assets/LipSync/Scripts/Core/LpcModel.cs b/Assets/LipSync/Scripts/Core/LpcModel.cs
@@ -256,7 +256,7 @@ private List<double[]> Formants(List<float[]> splitting)
                 var rts = FindCRoots(coefficients).Where(x => x.imag >= 0.0);
                 var frqs = rts.Select(x => x.arg * (fs / (2 * Mathf.PI))).ToList();
                 frqs.Sort();
-                double[] fmts = { frqs[1], frqs[2], frqs[3] };
+                double[] fmts = { frqs[1], frqs[2] };
                 ret.Add(fmts);
                 i++;
             }
@@ -265,11 +265,13 @@ private List<double[]> Formants(List<float[]> splitting)
 
         public List<double[]> Analy(float[] buffer, int window, int step)
         {
-            this.step = (step * fs);
-            this.window = (window * fs);
+            this.step = (step * fs) / 1000;
+            this.window = (window * fs) / 1000;
             this.audioBuffer = buffer;
             var split = MakeFrame();
             return Formants(split);
         }
+
+
     }
 }