lipc sync

huailiang · May 28, 2020 · be42f48 · be42f48
1 parent 7623c9f
commit be42f48
Show file tree

Hide file tree

Showing 3 changed files with 48 additions and 48 deletions.
diff --git a/Assets/LipSync/Editor/LpcEditor.cs b/Assets/LipSync/Editor/LpcEditor.cs
@@ -26,6 +26,7 @@ static void LpcShow()
         float[] currentVowelFormantCeilValues;
         string vowelsInfo = "";
 
+        private bool debugFolder;
 
         private void OnEnable()
         {
@@ -36,12 +37,12 @@ private void OnEnable()
                 step = 15;
             }
             language = ERecognizerLanguage.Japanese;
+            debugFolder = true;
         }
 
         private void OnGUI()
         {
             GUILayout.BeginVertical();
-
             GUILayout.Space(10);
             audioClip = (AudioClip)EditorGUILayout.ObjectField("Audio Clip", audioClip, typeof(AudioClip), false);
             GUILayout.Space(4);
@@ -93,55 +94,59 @@ private void OnGUI()
                 VowelsInfo(rst);
             }
             GUILayout.Space(4);
-            GUILayout.BeginHorizontal();
-            if (GUILayout.Button("root"))
+            debugFolder = EditorGUILayout.Foldout(debugFolder, "debug tools");
+            if (debugFolder)
             {
-                float[] poly = new float[] { -4, 0, 1 };
-                var ret = model.FindRoots(poly);
-                foreach (var it in ret)
+                GUILayout.BeginHorizontal();
+                if (GUILayout.Button("root"))
                 {
-                    Debug.Log(it);
+                    float[] poly = new float[] { -4, 0, 1 };
+                    var ret = model.FindRoots(poly);
+                    foreach (var it in ret)
+                    {
+                        Debug.Log(it);
+                    }
                 }
-            }
-            if (GUILayout.Button("c-root"))
-            {
-                double[] poly = new Double[] { 4, 0, 1 };
-                var roots = model.FindCRoots(poly);
-                for (int i = 0; i < roots.Length; i++)
+                if (GUILayout.Button("c-root"))
                 {
-                    Debug.Log("i: " + roots[i]);
+                    double[] poly = new Double[] { 4, 0, 1 };
+                    var roots = model.FindCRoots(poly);
+                    for (int i = 0; i < roots.Length; i++)
+                    {
+                        Debug.Log("i: " + roots[i]);
+                    }
                 }
-            }
-            if (GUILayout.Button("correlate"))
-            {
-                var a = new float[] { 0.3f, 0.1f, 0.2f, 0.4f, 0.3f, 0.5f, -1.6f, -2.5f, 1.6f, 3.2f, 1.34f, -4.1f, -5.34f };
-                var t = model.Correlate(a, a);
-                string str = "";
-                for (int i = 0; i < t.Length; i++)
+                if (GUILayout.Button("correlate"))
                 {
-                    str += t[i].ToString("f3") + " ";
+                    var a = new float[] { 0.3f, 0.1f, 0.2f, 0.4f, 0.3f, 0.5f, -1.6f, -2.5f, 1.6f, 3.2f, 1.34f, -4.1f, -5.34f };
+                    var t = model.Correlate(a, a);
+                    string str = "";
+                    for (int i = 0; i < t.Length; i++)
+                    {
+                        str += t[i].ToString("f3") + " ";
+                    }
+                    Debug.Log(str);
                 }
-                Debug.Log(str);
-            }
-            if (GUILayout.Button("toeplitz"))
-            {
-                var c = new double[]
+                if (GUILayout.Button("toeplitz"))
                 {
+                    var c = new double[]
+                    {
                     4, -2.6, 1.7, 4.3, 11, 21, 1.3, -3, 4, 11, 9, -4, 7, 12, 0.3, -7.0
-                };
-                ToeplitzMtrix toeplitzMtrix = new ToeplitzMtrix(c);
-                Debug.Log(toeplitzMtrix);
-                var t = toeplitzMtrix.Inverse();
-                int n = (int)Math.Sqrt((double)t.Length);
-                string msg = "size: " + n;
-                for (int i = 0; i < n; i++)
-                {
-                    msg += "\n";
-                    for (int j = 0; j < n; j++) msg += t[i, j].ToString("f3") + "\t";
+                    };
+                    ToeplitzMtrix toeplitzMtrix = new ToeplitzMtrix(c);
+                    Debug.Log(toeplitzMtrix);
+                    var t = toeplitzMtrix.Inverse();
+                    int n = (int)Math.Sqrt((double)t.Length);
+                    string msg = "size: " + n;
+                    for (int i = 0; i < n; i++)
+                    {
+                        msg += "\n";
+                        for (int j = 0; j < n; j++) msg += t[i, j].ToString("f3") + "\t";
+                    }
+                    Debug.Log(msg);
                 }
-                Debug.Log(msg);
+                GUILayout.EndHorizontal();
             }
-            GUILayout.EndHorizontal();
 
             GUILayout.Space(8);
             GUILayout.Label(vowelsInfo);
@@ -210,4 +215,4 @@ public void VowelsInfo(List<double[]> formants)
 
     }
 
-}
+}
diff --git a/Assets/LipSync/Scripts/Core/LpcModel.cs b/Assets/LipSync/Scripts/Core/LpcModel.cs
@@ -41,7 +41,8 @@ public double[] Estimate(float[] signal, int order)
             }
             ToeplitzMtrix mtrix = new ToeplitzMtrix(r1);
             var inv = mtrix.Inverse();
-            mtrix = new ToeplitzMtrix(inv);
+            if (inv != null)
+                mtrix = new ToeplitzMtrix(inv);
             var phi = mtrix.Dot(r2);
             var ret = new double[phi.Length + 1];
             ret[0] = 1;
@@ -252,6 +253,7 @@ private List<double[]> Formants(List<float[]> splitting)
                 {
                     FL[j] = FL[j] * w[j];
                 }
+                Debug.Log(i);
                 var coefficients = Estimate(FL, 2 + fs / 1000);
                 var rts = FindCRoots(coefficients).Where(x => x.imag >= 0.0);
                 var frqs = rts.Select(x => x.arg * (fs / (2 * Mathf.PI))).ToList();

diff --git a/Assets/LipSync/Scripts/LipSync.cs b/Assets/LipSync/Scripts/LipSync.cs
@@ -27,7 +27,6 @@ public class LipSync : MonoBehaviour
         protected string[] currentVowels;
         protected Dictionary<string, int> vowelToIndexDict = new Dictionary<string, int>();
         protected int[] propertyIndexs = new int[MAX_BLEND_VALUE_COUNT];
-        protected float blendValuesSum;
 
         protected string recognizeResult;
         protected float[] targetBlendValues = new float[MAX_BLEND_VALUE_COUNT];
@@ -36,7 +35,6 @@ public class LipSync : MonoBehaviour
         public Text recognizeText;
 
 
-
         public void InitializeRecognizer()
         {
             switch (recognizerLanguage)
@@ -75,11 +73,6 @@ protected void UpdateForward()
             {
                 targetBlendValues[vowelToIndexDict[recognizeResult]] = 1.0f;
             }
-            blendValuesSum = 0.0f;
-            for (int j = 0; j < currentBlendValues.Length; ++j)
-            {
-                blendValuesSum += currentBlendValues[j];
-            }
             for (int k = 0; k < currentBlendValues.Length; ++k)
             {
                 if (propertyIndexs[k] != -1)