From be42f4801f510382fc031682b01878b4b2116bb6 Mon Sep 17 00:00:00 2001 From: penghuailiang Date: Thu, 28 May 2020 15:27:39 +0800 Subject: [PATCH] lipc sync --- Assets/LipSync/Editor/LpcEditor.cs | 85 +++++++++++++------------ Assets/LipSync/Scripts/Core/LpcModel.cs | 4 +- Assets/LipSync/Scripts/LipSync.cs | 7 -- 3 files changed, 48 insertions(+), 48 deletions(-) diff --git a/Assets/LipSync/Editor/LpcEditor.cs b/Assets/LipSync/Editor/LpcEditor.cs index 11122c9..67c7892 100644 --- a/Assets/LipSync/Editor/LpcEditor.cs +++ b/Assets/LipSync/Editor/LpcEditor.cs @@ -26,6 +26,7 @@ static void LpcShow() float[] currentVowelFormantCeilValues; string vowelsInfo = ""; + private bool debugFolder; private void OnEnable() { @@ -36,12 +37,12 @@ private void OnEnable() step = 15; } language = ERecognizerLanguage.Japanese; + debugFolder = true; } private void OnGUI() { GUILayout.BeginVertical(); - GUILayout.Space(10); audioClip = (AudioClip)EditorGUILayout.ObjectField("Audio Clip", audioClip, typeof(AudioClip), false); GUILayout.Space(4); @@ -93,55 +94,59 @@ private void OnGUI() VowelsInfo(rst); } GUILayout.Space(4); - GUILayout.BeginHorizontal(); - if (GUILayout.Button("root")) + debugFolder = EditorGUILayout.Foldout(debugFolder, "debug tools"); + if (debugFolder) { - float[] poly = new float[] { -4, 0, 1 }; - var ret = model.FindRoots(poly); - foreach (var it in ret) + GUILayout.BeginHorizontal(); + if (GUILayout.Button("root")) { - Debug.Log(it); + float[] poly = new float[] { -4, 0, 1 }; + var ret = model.FindRoots(poly); + foreach (var it in ret) + { + Debug.Log(it); + } } - } - if (GUILayout.Button("c-root")) - { - double[] poly = new Double[] { 4, 0, 1 }; - var roots = model.FindCRoots(poly); - for (int i = 0; i < roots.Length; i++) + if (GUILayout.Button("c-root")) { - Debug.Log("i: " + roots[i]); + double[] poly = new Double[] { 4, 0, 1 }; + var roots = model.FindCRoots(poly); + for (int i = 0; i < roots.Length; i++) + { + Debug.Log("i: " + roots[i]); + } } - } - if (GUILayout.Button("correlate")) - { - var a = new float[] { 0.3f, 0.1f, 0.2f, 0.4f, 0.3f, 0.5f, -1.6f, -2.5f, 1.6f, 3.2f, 1.34f, -4.1f, -5.34f }; - var t = model.Correlate(a, a); - string str = ""; - for (int i = 0; i < t.Length; i++) + if (GUILayout.Button("correlate")) { - str += t[i].ToString("f3") + " "; + var a = new float[] { 0.3f, 0.1f, 0.2f, 0.4f, 0.3f, 0.5f, -1.6f, -2.5f, 1.6f, 3.2f, 1.34f, -4.1f, -5.34f }; + var t = model.Correlate(a, a); + string str = ""; + for (int i = 0; i < t.Length; i++) + { + str += t[i].ToString("f3") + " "; + } + Debug.Log(str); } - Debug.Log(str); - } - if (GUILayout.Button("toeplitz")) - { - var c = new double[] + if (GUILayout.Button("toeplitz")) { + var c = new double[] + { 4, -2.6, 1.7, 4.3, 11, 21, 1.3, -3, 4, 11, 9, -4, 7, 12, 0.3, -7.0 - }; - ToeplitzMtrix toeplitzMtrix = new ToeplitzMtrix(c); - Debug.Log(toeplitzMtrix); - var t = toeplitzMtrix.Inverse(); - int n = (int)Math.Sqrt((double)t.Length); - string msg = "size: " + n; - for (int i = 0; i < n; i++) - { - msg += "\n"; - for (int j = 0; j < n; j++) msg += t[i, j].ToString("f3") + "\t"; + }; + ToeplitzMtrix toeplitzMtrix = new ToeplitzMtrix(c); + Debug.Log(toeplitzMtrix); + var t = toeplitzMtrix.Inverse(); + int n = (int)Math.Sqrt((double)t.Length); + string msg = "size: " + n; + for (int i = 0; i < n; i++) + { + msg += "\n"; + for (int j = 0; j < n; j++) msg += t[i, j].ToString("f3") + "\t"; + } + Debug.Log(msg); } - Debug.Log(msg); + GUILayout.EndHorizontal(); } - GUILayout.EndHorizontal(); GUILayout.Space(8); GUILayout.Label(vowelsInfo); @@ -210,4 +215,4 @@ public void VowelsInfo(List formants) } -} +} \ No newline at end of file diff --git a/Assets/LipSync/Scripts/Core/LpcModel.cs b/Assets/LipSync/Scripts/Core/LpcModel.cs index bf54370..8cc9349 100644 --- a/Assets/LipSync/Scripts/Core/LpcModel.cs +++ b/Assets/LipSync/Scripts/Core/LpcModel.cs @@ -41,7 +41,8 @@ public double[] Estimate(float[] signal, int order) } ToeplitzMtrix mtrix = new ToeplitzMtrix(r1); var inv = mtrix.Inverse(); - mtrix = new ToeplitzMtrix(inv); + if (inv != null) + mtrix = new ToeplitzMtrix(inv); var phi = mtrix.Dot(r2); var ret = new double[phi.Length + 1]; ret[0] = 1; @@ -252,6 +253,7 @@ private List Formants(List splitting) { FL[j] = FL[j] * w[j]; } + Debug.Log(i); var coefficients = Estimate(FL, 2 + fs / 1000); var rts = FindCRoots(coefficients).Where(x => x.imag >= 0.0); var frqs = rts.Select(x => x.arg * (fs / (2 * Mathf.PI))).ToList(); diff --git a/Assets/LipSync/Scripts/LipSync.cs b/Assets/LipSync/Scripts/LipSync.cs index 4b7cfdb..0ac933a 100644 --- a/Assets/LipSync/Scripts/LipSync.cs +++ b/Assets/LipSync/Scripts/LipSync.cs @@ -27,7 +27,6 @@ public class LipSync : MonoBehaviour protected string[] currentVowels; protected Dictionary vowelToIndexDict = new Dictionary(); protected int[] propertyIndexs = new int[MAX_BLEND_VALUE_COUNT]; - protected float blendValuesSum; protected string recognizeResult; protected float[] targetBlendValues = new float[MAX_BLEND_VALUE_COUNT]; @@ -36,7 +35,6 @@ public class LipSync : MonoBehaviour public Text recognizeText; - public void InitializeRecognizer() { switch (recognizerLanguage) @@ -75,11 +73,6 @@ protected void UpdateForward() { targetBlendValues[vowelToIndexDict[recognizeResult]] = 1.0f; } - blendValuesSum = 0.0f; - for (int j = 0; j < currentBlendValues.Length; ++j) - { - blendValuesSum += currentBlendValues[j]; - } for (int k = 0; k < currentBlendValues.Length; ++k) { if (propertyIndexs[k] != -1)