Skip to content

Commit

Permalink
lipsync
Browse files Browse the repository at this point in the history
  • Loading branch information
@huailiang
huailiang committed May 27, 2020
1 parent 27703c1 commit b1c2fc6
Show file tree
Hide file tree
Showing 3 changed files with 100 additions and 78 deletions.
2 changes: 1 addition & 1 deletion Assets/LipSync/Editor/BakingEditorWindow.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,7 +192,7 @@ private void Bake()
}

List<AnimationClip> tempClipList = new List<AnimationClip>(audioClipToBake.Count);
for (int j = 0; j < /*audioClipToBake.Count*/1; ++j)
for (int j = 0; j < audioClipToBake.Count; ++j)
{
AnimationClip clip = new AnimationClip();
AnimationCurve[] curveArray = new AnimationCurve[currentVowels.Length];
Expand Down
81 changes: 18 additions & 63 deletions Assets/LipSync/Editor/LpcEditor.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
using System;
using System.Collections.Generic;
using System.Linq;
using UnityEditor;
using UnityEngine;
Expand Down Expand Up @@ -35,12 +34,25 @@ private void OnGUI()

GUILayout.Space(10);
audioClip = (AudioClip)EditorGUILayout.ObjectField("Audio Clip", audioClip, typeof(AudioClip), false);

EditorGUILayout.BeginVertical(EditorStyles.textField);
{
if (audioClip)
{
var pat = AssetDatabase.GetAssetPath(audioClip);
pat = pat.Substring(pat.LastIndexOf('/') + 1);
EditorGUILayout.LabelField(pat);
EditorGUILayout.LabelField("采样率: " + audioClip.frequency);
EditorGUILayout.LabelField("声 道 : " + audioClip.channels);
model.fs = audioClip.frequency;
}
}
EditorGUILayout.EndVertical();
model.window = EditorGUILayout.IntField("window", model.window);
model.step = EditorGUILayout.IntField("step", model.step);
if (GUILayout.Button("Analy"))
{
Normalize();
var split = MakeFrame();
Formant(split);
model.Analy(audioBuffer);
}
GUILayout.Space(4);
GUILayout.BeginHorizontal();
Expand All @@ -55,7 +67,7 @@ private void OnGUI()
}
if (GUILayout.Button("c-root"))
{
double[] poly = new Double[] { -4, 0, 1 };
double[] poly = new Double[] { 4, 0, 1 };
var roots = model.FindCRoots(poly);
for (int i = 0; i < roots.Length; i++)
{
Expand Down Expand Up @@ -118,64 +130,7 @@ private void Normalize()
}
}


private List<float[]> MakeFrame()
{
step = (15 * fs) / 1000;
window = (30 * fs) / 1000;
List<float[]> splitting = new List<float[]>();
int i = 0;
while (i <= audioBuffer.Length - window)
{
float[] arr = new float[window];
for (int j = i; j < i + window; j++)
{
arr[j - i] = audioBuffer[j];
}
splitting.Add(arr);
i += step;
}
return splitting;
}

private float[] PreEmphasis(float[] x, float a)
{
var temp = new float[window];
int i = 1;
while (i <= window - 2)
{
temp[i - 1] = x[i] - a * x[i - 1];
i++;
}
return temp;
}

private void Formant(List<float[]> splitting)
{
int i = 0;
float a = 0.67f;
info = String.Empty;
List<double[]> ret = new List<double[]>();
while (i < splitting.Count())
{
float[] FL = PreEmphasis(splitting[i], a);
float[] w = MathToolBox.GenerateWindow(window, MathToolBox.EWindowType.Hamming);
for (int j = 0; j < window; j++)
{
FL[j] = FL[j] * w[j];
}
var coefficients = model.Estimate(FL, 2 + fs / 1000);
coefficients = coefficients.Reverse().ToArray();
var rts = model.FindCRoots(coefficients).Where(x => x.imag >= 0.0);
var frqs = rts.Select(x => x.arg * (fs / (2 * Mathf.PI))).ToList();
frqs.Sort();
double[] fmts = { frqs[1], frqs[2], frqs[3] };
Debug.Log(frqs[1] + " " + frqs[2] + " " + frqs[3]);
ret.Add(fmts);
i++;
}
}



private void AppendInfo(int idx, double[] formants)
{
Expand Down
95 changes: 81 additions & 14 deletions Assets/LipSync/Scripts/Core/LpcModel.cs
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,21 @@
using System;
using System.Collections.Generic;
using System.Linq;
using UnityEngine;

namespace LipSync
{
public class LpcModel
{
public int window, step, fs;
private float[] audioBuffer;

public LpcModel()
{
window = 30;
step = 15;
}

public double[] Estimate(float[] signal, int order)
{
if (order > signal.Length)
Expand Down Expand Up @@ -46,8 +56,7 @@ public double[] Estimate(float[] signal, int order)
}
return ret;
}



private Complex RandomFloat(Complex low, Complex high)
{
float rand = UnityEngine.Random.Range(0.0f, 1.0f);
Expand Down Expand Up @@ -126,23 +135,18 @@ private Complex[] Deflate(Complex root, Complex[] a, int n)

public Complex[] FindRoots(float[] poly)
{
Complex[] cs = new Complex[poly.Length];
for (int i = 0; i < cs.Length; i++) cs[i] = new Complex(poly[i]);
int len = poly.Length;
Complex[] cs = new Complex[len];
for (int i = 0; i < len; i++) cs[len - i - 1] = new Complex(poly[i]);
return FindRoots(cs);
}

public Complex[] FindCRoots(float[] poly)
{
double[] dpoly = poly.Select(x => (double) x).ToArray();
return FindCRoots(dpoly);
}

public Complex[] FindCRoots(double[] dpoly)
public Complex[] FindCRoots(IEnumerable<double> dpoly)
{
int len = dpoly.Length;
int len = dpoly.Count();
int len2 = (len - 1) * 2;
double[] ret = new double[len2];
MathToolBox.poly_roots(len, dpoly, ret);
MathToolBox.poly_roots(len, dpoly.Reverse().ToArray(), ret);
Complex[] cpx = new Complex[len - 1];
for (int i = 0; i < len - 1; i++)
{
Expand All @@ -151,7 +155,7 @@ public Complex[] FindCRoots(double[] dpoly)
return cpx;
}

public Complex[] FindRoots(Complex[] poly)
private Complex[] FindRoots(Complex[] poly)
{
int n = poly.Length - 1;
int N = n;
Expand Down Expand Up @@ -209,5 +213,68 @@ public double[] Correlate(float[] a, float[] v)
}
return ret;
}


private List<float[]> MakeFrame()
{
step = (15 * fs) / 1000;
window = (30 * fs) / 1000;
List<float[]> splitting = new List<float[]>();
int i = 0;
while (i <= audioBuffer.Length - window)
{
float[] arr = new float[window];
for (int j = i; j < i + window; j++)
{
arr[j - i] = audioBuffer[j];
}
splitting.Add(arr);
i += step;
}
return splitting;
}

private float[] PreEmphasis(float[] x, float a)
{
var temp = new float[window];
int i = 1;
while (i <= window - 2)
{
temp[i - 1] = x[i] - a * x[i - 1];
i++;
}
return temp;
}

private void Formants(List<float[]> splitting)
{
int i = 0;
float a = 0.67f;
List<double[]> ret = new List<double[]>();
while (i < splitting.Count())
{
float[] FL = PreEmphasis(splitting[i], a);
float[] w = MathToolBox.GenerateWindow(window, MathToolBox.EWindowType.Hamming);
for (int j = 0; j < window; j++)
{
FL[j] = FL[j] * w[j];
}
var coefficients = Estimate(FL, 2 + fs / 1000);
var rts = FindCRoots(coefficients).Where(x => x.imag >= 0.0);
var frqs = rts.Select(x => x.arg * (fs / (2 * Mathf.PI))).ToList();
frqs.Sort();
double[] fmts = { frqs[1], frqs[2], frqs[3] };
Debug.Log(frqs[1] + " " + frqs[2] + " " + frqs[3]);
ret.Add(fmts);
i++;
}
}

public void Analy(float[] buffer)
{
this.audioBuffer = buffer;
var split = MakeFrame();
Formants(split);
}
}
}

0 comments on commit b1c2fc6

Please sign in to comment.