From 2cbf79cf0f2593e647fa04ef4db9337aa8b7597e Mon Sep 17 00:00:00 2001 From: huailiang Date: Sun, 24 May 2020 18:46:29 +0800 Subject: [PATCH] lipsync lpc --- Assets/LipSync/Editor/LpcEditor.cs | 46 ++++--- Assets/LipSync/Plugins.meta | 8 ++ Assets/LipSync/Plugins/ZSolver.bundle.meta | 33 +++++ .../Plugins/ZSolver.bundle/Contents.meta | 8 ++ .../ZSolver.bundle/Contents/Info.plist | 46 +++++++ .../ZSolver.bundle/Contents/Info.plist.meta | 7 ++ .../ZSolver.bundle/Contents/MacOS.meta | 8 ++ .../ZSolver.bundle/Contents/MacOS/ZSolver | Bin 0 -> 35616 bytes .../Contents/MacOS/ZSolver.meta | 7 ++ .../Contents/_CodeSignature.meta | 8 ++ .../Contents/_CodeSignature/CodeResources | 115 ++++++++++++++++++ .../_CodeSignature/CodeResources.meta | 7 ++ Assets/LipSync/Scripts/Core/LpcModel.cs | 76 ++++++++---- Assets/LipSync/Scripts/Core/MathToolBox.cs | 21 +++- Assets/LipSync/Scripts/Core/ToeplitzMatrix.cs | 80 ++++++------ README.md | 2 +- zsolve/generate_android.sh | 32 +++++ 17 files changed, 431 insertions(+), 73 deletions(-) create mode 100644 Assets/LipSync/Plugins.meta create mode 100644 Assets/LipSync/Plugins/ZSolver.bundle.meta create mode 100644 Assets/LipSync/Plugins/ZSolver.bundle/Contents.meta create mode 100644 Assets/LipSync/Plugins/ZSolver.bundle/Contents/Info.plist create mode 100644 Assets/LipSync/Plugins/ZSolver.bundle/Contents/Info.plist.meta create mode 100644 Assets/LipSync/Plugins/ZSolver.bundle/Contents/MacOS.meta create mode 100755 Assets/LipSync/Plugins/ZSolver.bundle/Contents/MacOS/ZSolver create mode 100644 Assets/LipSync/Plugins/ZSolver.bundle/Contents/MacOS/ZSolver.meta create mode 100644 Assets/LipSync/Plugins/ZSolver.bundle/Contents/_CodeSignature.meta create mode 100644 Assets/LipSync/Plugins/ZSolver.bundle/Contents/_CodeSignature/CodeResources create mode 100644 Assets/LipSync/Plugins/ZSolver.bundle/Contents/_CodeSignature/CodeResources.meta create mode 100644 zsolve/generate_android.sh diff --git a/Assets/LipSync/Editor/LpcEditor.cs b/Assets/LipSync/Editor/LpcEditor.cs index 79eb33d..fa56657 100644 --- a/Assets/LipSync/Editor/LpcEditor.cs +++ b/Assets/LipSync/Editor/LpcEditor.cs @@ -23,7 +23,7 @@ static void LpcShow() private void OnEnable() { - if(model==null) + if (model == null) { model = new LpcModel(); } @@ -34,7 +34,7 @@ private void OnGUI() GUILayout.BeginVertical(); GUILayout.Space(10); - audioClip = (AudioClip)EditorGUILayout.ObjectField("Audio Clip", audioClip, typeof(AudioClip), false); + audioClip = (AudioClip) EditorGUILayout.ObjectField("Audio Clip", audioClip, typeof(AudioClip), false); if (GUILayout.Button("Analy")) { @@ -42,19 +42,29 @@ private void OnGUI() var split = MakeFrame(); Formant(split); } + GUILayout.BeginHorizontal(); if (GUILayout.Button("root")) { - float[] poly = new float[] { -8, 12, -6, 1}; + float[] poly = new float[] {-8, 12, -6, 1}; var ret = model.FindRoots(poly); foreach (var it in ret) { Debug.Log(it); } } + if (GUILayout.Button("c-root")) + { + double[] poly = new Double[] {-8, 12, -6, 1}; + var roots = model.FindCRoots(poly); + for (int i = 0; i < roots.Length; i++) + { + Debug.Log("i: " + roots[i]); + } + } if (GUILayout.Button("correlate")) { - var a = new float[] { 3, 1, 2, 4, 3, 5, 6, 5, 6, 2 }; - var v = new float[] { 3, 1, 4, 2 }; + var a = new float[] {3, 1, 2, 4, 3, 5, 6, 5, 6, 2}; + var v = new float[] {3, 1, 4, 2}; var t = model.Correlate(a, v); string str = ""; for (int i = 0; i < t.Length; i++) @@ -65,21 +75,23 @@ private void OnGUI() } if (GUILayout.Button("toeplitz")) { - float[] c = new float[] { 3, 4, 2, -6, 7, 3, 1, -3 }; - ToeplitzMtrix toeplitzMtrix; - toeplitzMtrix = new ToeplitzMtrix(c); + float[] c = new float[] + { + 3, 4, 2, -2.6f, 1.7f, 4.3f, 121, 321, 1.3f, 1, -3, 3, 4, 11, 9, -4, 7, 12, 0.3f, -7.0f + }; + ToeplitzMtrix toeplitzMtrix = new ToeplitzMtrix(c); Debug.Log(toeplitzMtrix); var t = toeplitzMtrix.Inverse(); - int n = (int)Math.Sqrt((double)t.Length); + int n = (int) Math.Sqrt((double) t.Length); string msg = "size: " + n; for (int i = 0; i < n; i++) { msg += "\n"; - for (int j = 0; j < n; j++) - msg += t[i, j].ToString("f3") + "\t"; + for (int j = 0; j < n; j++) msg += t[i, j].ToString("f3") + "\t"; } Debug.Log(msg); } + GUILayout.EndHorizontal(); GUILayout.Space(10); if (!string.IsNullOrEmpty(info)) { @@ -144,6 +156,7 @@ private void Formant(List splitting) int i = 0; float a = 0.67f; info = String.Empty; + List ret = new List(); while (i < splitting.Count()) { float[] FL = PreEmphasis(splitting[i], a); @@ -152,9 +165,14 @@ private void Formant(List splitting) { FL[i] = FL[i] * w[i]; } - var coefficients = model.EstimateLpcCoefficients(FL, 2 + fs / 1000); - var formants = model.FindFormants(coefficients, fs); - AppendInfo(i, formants); + var coefficients = model.Estimate(FL, 2 + fs / 1000); + var rts = model.FindCRoots(coefficients); + rts = rts.Where(x => x.imag >= 0).ToArray(); + var frqs = rts.Select(x => x.arg * (fs / (2 * Mathf.PI))).ToList(); + frqs.Sort(); + double[] fmts = {frqs[1], frqs[2], frqs[3]}; + Debug.Log(frqs[1] + " " + frqs[2] + " " + frqs[3]); + ret.Add(fmts); i++; } } diff --git a/Assets/LipSync/Plugins.meta b/Assets/LipSync/Plugins.meta new file mode 100644 index 0000000..d358f48 --- /dev/null +++ b/Assets/LipSync/Plugins.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 3dcf1f52c059b4ddba72120d01651e60 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/LipSync/Plugins/ZSolver.bundle.meta b/Assets/LipSync/Plugins/ZSolver.bundle.meta new file mode 100644 index 0000000..8f9e0fa --- /dev/null +++ b/Assets/LipSync/Plugins/ZSolver.bundle.meta @@ -0,0 +1,33 @@ +fileFormatVersion: 2 +guid: 1cec1dacc344f4b7f9a7d7e9ecf91b71 +folderAsset: yes +PluginImporter: + externalObjects: {} + serializedVersion: 2 + iconMap: {} + executionOrder: {} + defineConstraints: [] + isPreloaded: 0 + isOverridable: 0 + isExplicitlyReferenced: 0 + validateReferences: 1 + platformData: + - first: + Any: + second: + enabled: 0 + settings: {} + - first: + Editor: Editor + second: + enabled: 1 + settings: + DefaultValueInitialized: true + - first: + Standalone: OSXUniversal + second: + enabled: 1 + settings: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/LipSync/Plugins/ZSolver.bundle/Contents.meta b/Assets/LipSync/Plugins/ZSolver.bundle/Contents.meta new file mode 100644 index 0000000..863010f --- /dev/null +++ b/Assets/LipSync/Plugins/ZSolver.bundle/Contents.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: b3b30a3fa35d64fbdaee7c864055b230 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/LipSync/Plugins/ZSolver.bundle/Contents/Info.plist b/Assets/LipSync/Plugins/ZSolver.bundle/Contents/Info.plist new file mode 100644 index 0000000..05f3ef6 --- /dev/null +++ b/Assets/LipSync/Plugins/ZSolver.bundle/Contents/Info.plist @@ -0,0 +1,46 @@ + + + + + BuildMachineOSBuild + 18G3020 + CFBundleDevelopmentRegion + English + CFBundleExecutable + ZSolver + CFBundleInfoDictionaryVersion + 6.0 + CFBundleLongVersionString + + CFBundlePackageType + APPL + CFBundleSignature + ???? + CFBundleSupportedPlatforms + + MacOSX + + CFBundleVersion + + CSResourcesFileMapped + + DTCompiler + com.apple.compilers.llvm.clang.1_0 + DTPlatformBuild + 11C504 + DTPlatformVersion + GM + DTSDKBuild + 19B90 + DTSDKName + macosx10.15 + DTXcode + 1131 + DTXcodeBuild + 11C504 + LSMinimumSystemVersion + 10.14 + NSHumanReadableCopyright + + + diff --git a/Assets/LipSync/Plugins/ZSolver.bundle/Contents/Info.plist.meta b/Assets/LipSync/Plugins/ZSolver.bundle/Contents/Info.plist.meta new file mode 100644 index 0000000..0d38f6c --- /dev/null +++ b/Assets/LipSync/Plugins/ZSolver.bundle/Contents/Info.plist.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 4c1c85bef87fe4f51ac0fa605c84754d +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/LipSync/Plugins/ZSolver.bundle/Contents/MacOS.meta b/Assets/LipSync/Plugins/ZSolver.bundle/Contents/MacOS.meta new file mode 100644 index 0000000..df89b33 --- /dev/null +++ b/Assets/LipSync/Plugins/ZSolver.bundle/Contents/MacOS.meta @@ -0,0 +1,8 @@ +fileFormatVersion: 2 +guid: 9ee20ae5102a8462f8ff2b0481d763c8 +folderAsset: yes +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/LipSync/Plugins/ZSolver.bundle/Contents/MacOS/ZSolver b/Assets/LipSync/Plugins/ZSolver.bundle/Contents/MacOS/ZSolver new file mode 100755 index 0000000000000000000000000000000000000000..68bbd20a1bd28bf6b64728b6f4187bae14314832 GIT binary patch literal 35616 zcmeHPdw3L8maob~A_gjn!9m2Ob%O>6A^}vwV>d0ySPj*JOF|M%X!GcXNFJDUhv3K< z?KE2U)J~ikXPH^uWyNus&*cMmWEWk94#6adLPW#}ND^LljA9am1S0AEomfUqCJ?EbLJLlfIebde1tKSTc5d<+@5QHe)<8XIJ2!aPCUAN*+3=;&KEh9ZU zgRw?FW{xBUgN_87?i&oT&E~XMJBQkiwzq7eF8l9zy(AiL3|$dI%@9HUR61P+l_Az> zdr>oVQG(HeXff0pV>Rpz`A7C_YwRWK>=kG?+TL`-o?_S_8n6sq*ZEgiURF6Y;L-MK z40}Gq0?{MJJ7k&FY_`HmXGL+@YJ)%8UTTsqSZNG0NC(Y~@D)s(%~iI(xU9%lT;?ce z^vL+We6OxqWEdcNv<*YV3d)nY=-XyXGiR8GNrL$iWAH}Nly||KTG(u5CAP{9r3K|B zwsp>mAGJrh$}ZR={oqCM491jrHrwj*p(tGMUzFib&^@9@+r#__T{c@u{!jTg(XeL} z0@1_mk*ym;*7W%$gSOce%fn03(<~W=Fx4oRjigOWbeh6!gqkt#6oeYqXR1DrqfiEY zpaye6KZbHM%5^9ymNexiB39#2COjwzEhuloy$tv8D>#=!m=6~kbwBJ#Kfu2YJnAR8 zYwXtY59eps{chamk9u#j&H~Sbn~h=oSSXKWbsej?uzMhj!5zG!QJIUIZ0UI@!Yxs_ z*Hu|@Z%J_h-OD#rI_;%N4<;3D0Htf*g@Wuv!9Oae88nTXuG?|5eip$W3IxA60vrL3 z07rl$z!BgGa0EC490861M}Q;15#R`L1ULd5fgeR6Q&!KJS1gy+0ofC9jhD5B$0y+Q z&*%Ru4)*ubNt=HwY8JIo_V}2*Lzn*!Tsd&iqW+C6?KmOVp9z=MHvf9o_7_pq_!dR& z^FKz-w1q9Y(kf7j+Wn|`RbGQbnvw>YC*U-Z6U͚kJv{4+>JZIj*mqaMxcoC2!B zkt9v2Wn*CeW|>!*GtC*x2V|cQZz!Sp&TfN-=3&{SGPXfI~{EhklDI?#UQ5^TAL59yF7^))q^O)PIejwvC|v1WI-{L@D7Lx5+7aTU@n!~O+IDmJ2?~@J)UYCIg|%wMeMQ!_Pf53y?HINdmv2l{+Upay{_HTvpX;rvzC@H zXwfnfuDRR8-)6YFuZC9?Sk*I@`ZJM>wdiwf$XbNOs&-iFJ0cWqS(tU&X}#?`B*b68 z{vN|dGU@rTSk3sd9}kHhW9TuZUlH3ZTJG2g*^ZJX+oz}^`FZN0?(Tzy7X5`!|<$OPiSmjJN*h*v6a6Z^K`C&Lr76;^%eXd5U z2MO?j^G-C4APbStz#>^mJKv80q_@aoORrgqjcBxL@y9H{@|vusZ&5#Ue;H1*VOjv$ z)sm+6xI387s+LWeIqi_5x^^n+dAa_4q@q=|2Vq9$ZC13&7DYXS*&MN0i#XR$;ZoCg z%PLZ!uMNfXjx;H8!Wei$m2rbuAROsZ)>#^{NL(p#>Wz9iBbNM4lVWZ+8I{N>s3_Xn z?)2od>B$E=>kx7b?1UAw$@4Bi@9Z!*iZ`=4JsB}d-#jY6Fyfeuux^K0%J{ff%Kn7t z+(~|-KL>rhcM;^1+DFP~wT9#tw=cpf_F7Y#TyM$ljoWCV@3yE{o%bt*=@KnspX|jj zgZTvXV@UZ38JERoInd^$GgV8<$Ic0emcgD= zPuyM@JcdfrNy#HZUwUBx2v`F3%6zOw)t!HfznWMyv&f7p=$P5eSy10A#V)8fzYCR6 z>&2ayKqmWiuIzq~A}vU@hr!kL)qRQxILWP`38O2imS$6EyztdEU-lf4Y6}Kw#rg~& zZ3&m$4p0;;ipnt9B+vU0bY5msTFMWDCfQ}#*=SJ7m-&34r$)F{;;Z_4WW5y{1M}xlgmnlGKM!5EZGgviVnvzQL zYeX^mXXrZTghQGTfbBN0k4u|gpx(?*oHf(z5hTywQ44S8W~=&%r4R2JYct}}kC)P80NDI4 z{Xx(=?I_LI%WT@x^j!EN{+>v)d4>rU;zT}294X5O7t`w$Z+Z&|^!&D^ zfL5MmOdhw45wrEHhgZdrYA4b`Br zSWENyA!#Q3A(3`8Jgk-Yx-DAj_gL~tsg;2VGp{7-N)2GPfC=4t*h=5FM4ODoU5g^I zj$#a|$bZ!%qk^~dPH0oMc1;P*A3|U8>PUGRQWOQ*`w}zb-9;vlUC9CCeGYsqYL7)d zX4MwVwYURe6-$sh+mIIVY~2B@nOi_jX4K~}M56^GfiXIe?Pn;fzr>p-popzYv^jSy z(chLua`((cBsX3MX;St{n%c0HGvR;mH*2V12B%9o^gc*D2$eN70JTX;>6e~Orq!~^2cQt4 z50G$)B=o*VNr4_C=8@7suat|8VKAV+sOIP$?^rNbt0Z!uomA<6adeDAKLxBQk)OmRo!w*y;Pft zzof%lOxiuH6<%2HQ8vTmWai@qhZO%c>3>Cb&pV{XGa`|v9a8qWV4uec46K3i*of!s z2e%dLBJQMC$$cX_khNb0K@KL;gu5Bk9CZ}{??ie?XmAwcU>WG#!{Rhih9 zVBEVNBkZs*k=4fG3ILdW8$b>;vdA`8U4UY^lCp38-O#2@O@Bxm+nzeP38?~a0(gP7 zpc&c+*z}X@}`3VcFQ2atTuNl?V} zs3wm#0)##cpqZLY9zayvA64$9%&~h&~o2?5w&KA zl=&fAW+9m}AX?D@6r5AxZq7*>)E9&T27(ZXjr96M_c!PEb$8u`c)H)C7zr3LMu^O% z$k1dMB!lleq?;oK(eL+ANo<3~RVR`E-{D3X*Tlwwbd#R@mXbm}C8xAXwVP3=v79IG z&nI6VN;n6t5-|}yG{{CgvLh}NBlool9FV6P@#n{=NdRyQ9p`0_Aj|#!FQ5Ekrasa z0a($=r~uAbShfTjWO2}%(pB|RR|!0!_Yi%C_2%uyhJkJMLf+GU4gEzV10~A7nCRK) zjBwNAgl_b(5H{#$w%%CpD&p8m4*E-QZzfS4lik} z3N3PddjvLrHDJb07G*6h4$~#;1mV-z*+W60e9S$FQDH&>sIwUfX!oJcKB*kX zlslnYhHldSs%n@yOJ-;)p?w;giJ&q2GS5Je^#vXott73-gv4aYZo{Pcgmlgz8mIA5 z$`^7^icJvZrZka#L5xxo%MgY4p?ob#wT*PLfw zWwzxt;Nx|&PyZk+c`kMX`IsIcAIcavca7PU_uFwU{!U>N6EU9hMWp3#r8 zTIm(9HUy8dZqb`GgKN>AO28XpK9;S3T>oXbe>-csNI%JLtA)*uJe0P>@4Oy zeg0Q_{Dt2jRlL89@|Q8iHW~$up>q}JA&<#lI=9+Q>z6n3z6dJdQ+hW(K`#Xi(zNt; zROjf`7F2K5t3Ff*VG9j*%9;!JOQ}G3F9p$mNnb@h#KEl4*v3q|{W++@_vFhv!~Cbw z4kCSThbH>2xorbph04fE>5q3%=$bD zFfbC77y?=d@u>&`>@7&J+z`+P%DRmN^fgcj=Ai&Gx4|+YGe5oC70q&kAbT{g!B1bw ztP4yXiw1$I(d;>%J!9CD;vSe9kEf#D{t?_aFAq%p6lL8SC<>kD;rSZ+>SO0gs@+Ys zBUGDBwPvcBskVn|OR4r2YAAg`;h>-i1-$-O$A=+n#Dsd`D@7Nu)WDl3FpdRddEm%dDc|3GSKKE9P)ZK^DO(rzj< zmAWdOrUJWZU3q1(v$)D`DlT){SK|xSj{M>hdy&amZptqyDKE@-+D(<~@(b-IM|p(_ zUuB|?ckzlaPv?G>da z=nRQM<1H&MoAIQ*qFgAQBf+*5=CQ1itJ;yN5V$KDhn zew-aC-XVlh7e9C{n4zZ#HQNlC-wP-Yi)&V*4kFP@+*pjN1lkPOT~|Ai`b9tZ3Xzf zIL>nFsQnWYnMMIS^{5k`FpV-mXD(>@L?ZZ(3cH=^Gf^*P=RZWv1+@;pXm>h@N+rtW ztVCVEL>o>p!dF+~XW%%Bu@#n=t}C%u+l1+iq?4XX0oIDkgd!m|`jt`i)=~5uqv*emqTe1x(NBMgGP$YEtf2swGj2=Th$VOCcHa)FI zWF$+d5ga{7_3R4fRWPIUT)HlGG00KZ)l<G-c=#(lWvq|BK!JK0_^Ge3X!{f@%I*#-B_O_`Z8 zduCC}+`=NqtmJ|@MS^3~shC$^xVPiB`w#9eJ@_BZcOTyWRqR^p8$GqFR$eIhmw(>8 z!BqF~8}{+8>R(4L-m>!_9)7%LZ_}i$Gy0D_@cPddUD@>Q{^H0lJkxVlf4$ZkxZ{Ci zE5F&$bm{TRHIv@?^?^5&>hAo<`QdMFIP&xw*Wtgo?)vVF7h@+ho0rzS^-jWb=UTmY zE_nQu?YE~NTo|$W(>HIadA;>FcV=C@rAczEow@#@e{XLz|EZ_wwTqcA4=Qh!JrXPI znOD63)PGi-yQy|l?j89rFFUli=e0jrA9}oM`?%80Z~yYujYsBXmfYC9ymRl1uRou# zI4Sv*`P$Jd-47T3>mL(VT>Q7~1q&1V>_XGoh+l=5SvLHpqin0Y768-4pd1vgp1O(Z z>`zHV^NS + + + + files + + files2 + + rules + + ^Resources/ + + ^Resources/.*\.lproj/ + + optional + + weight + 1000 + + ^Resources/.*\.lproj/locversion.plist$ + + omit + + weight + 1100 + + ^Resources/Base\.lproj/ + + weight + 1010 + + ^version.plist$ + + + rules2 + + .*\.dSYM($|/) + + weight + 11 + + ^(.*/)?\.DS_Store$ + + omit + + weight + 2000 + + ^(Frameworks|SharedFrameworks|PlugIns|Plug-ins|XPCServices|Helpers|MacOS|Library/(Automator|Spotlight|LoginItems))/ + + nested + + weight + 10 + + ^.* + + ^Info\.plist$ + + omit + + weight + 20 + + ^PkgInfo$ + + omit + + weight + 20 + + ^Resources/ + + weight + 20 + + ^Resources/.*\.lproj/ + + optional + + weight + 1000 + + ^Resources/.*\.lproj/locversion.plist$ + + omit + + weight + 1100 + + ^Resources/Base\.lproj/ + + weight + 1010 + + ^[^/]+$ + + nested + + weight + 10 + + ^embedded\.provisionprofile$ + + weight + 20 + + ^version\.plist$ + + weight + 20 + + + + diff --git a/Assets/LipSync/Plugins/ZSolver.bundle/Contents/_CodeSignature/CodeResources.meta b/Assets/LipSync/Plugins/ZSolver.bundle/Contents/_CodeSignature/CodeResources.meta new file mode 100644 index 0000000..2f83679 --- /dev/null +++ b/Assets/LipSync/Plugins/ZSolver.bundle/Contents/_CodeSignature/CodeResources.meta @@ -0,0 +1,7 @@ +fileFormatVersion: 2 +guid: 9308e398912784c8dbadef356f0e919f +DefaultImporter: + externalObjects: {} + userData: + assetBundleName: + assetBundleVariant: diff --git a/Assets/LipSync/Scripts/Core/LpcModel.cs b/Assets/LipSync/Scripts/Core/LpcModel.cs index 50f26d7..7651ed5 100644 --- a/Assets/LipSync/Scripts/Core/LpcModel.cs +++ b/Assets/LipSync/Scripts/Core/LpcModel.cs @@ -69,14 +69,14 @@ public float[] SynthesizeResponseForLPC(float[] coefficients, int samplingRate, int index = 0; foreach (var frequency in frequencies) { - var radians = frequency / (double)samplingRate * Mathf.PI * 2; + var radians = frequency / (double) samplingRate * Mathf.PI * 2; Complex response = new Complex(0.0, 0.0); for (int i = 0; i < coefficients.Length; i++) { var c = coefficients[i]; response += new Complex(c < 0 ? -c : c, i * radians); } - float v = 20 * Mathf.Log10((float)(1.0 / response.abs)); + float v = 20 * Mathf.Log10((float) (1.0 / response.abs)); retval[index++] = v; } return retval; @@ -93,7 +93,7 @@ public Complex LaguerreRoot(Complex[] polynomial, Complex guess) int maximumIterations = MR * MT; const double EPSS = 1.0e-7; double abx, abp, abm, err; - var frac = new Double[] { 0.0, 0.5, 0.25, 0.75, 0.125, 0.375, 0.625, 0.875, 1.0 }; + var frac = new Double[] {0.0, 0.5, 0.25, 0.75, 0.125, 0.375, 0.625, 0.875, 1.0}; Complex x = guess; for (int iteration = 1; iteration <= maximumIterations; iteration++) { @@ -122,7 +122,8 @@ public Complex LaguerreRoot(Complex[] polynomial, Complex guess) abm = gm.abs; if (abp < abm) gp = gm; - var dx = Math.Max(abp, abm) > 0.0 ? m / gp + var dx = Math.Max(abp, abm) > 0.0 + ? m / gp : (1 + abx) * new Complex(Mathf.Cos(iteration), Mathf.Sin(iteration)); Complex x1 = x - dx; if (x == x1) @@ -203,22 +204,36 @@ public double[] Estimate(float[] signal, int order) { throw new Exception("Input signal must have a lenght >= lpc order"); } - double[] ret = new double[0]; - if (order > 0) + if (order <= 0) { - int p = order + 1; + throw new Exception("lpc order must greater 0"); + } - var nx = Math.Min(p, signal.Length); - var x = Correlate(signal, signal); - var r = new double[nx]; - for (int i = 0; i < nx; i++) + int p = order + 1; + var nx = Math.Min(p, signal.Length); + var x = Correlate(signal, signal); + var r = new float[nx - 1]; + var r2 = new float[nx - 1]; + for (int i = 0; i < nx; i++) + { + if (i > 0) + { + r[i - 1] = x[signal.Length - 1 + i]; + } + if (i < nx - 1) { - r[i] = x[signal.Length - 1 + i]; + r2[i] = x[signal.Length - 1 + i]; } } - else + ToeplitzMtrix mtrix = new ToeplitzMtrix(r); + var inv = mtrix.Inverse(); + mtrix = new ToeplitzMtrix(inv); + var phi = mtrix.Dot(r2); + var ret = new double[phi.Length]; + ret[0] = 1; + for (int i = 0; i < phi.Length; i++) { - ret = new double[] { 1 }; + ret[i + 1] = phi[i]; } return ret; } @@ -226,7 +241,7 @@ public double[] Estimate(float[] signal, int order) private Complex RandomFloat(Complex low, Complex high) { - float rand = 0.5f;// UnityEngine.Random.Range(0.0f, 1.0f); + float rand = 0.5f; // UnityEngine.Random.Range(0.0f, 1.0f); Complex d = new Complex(rand); Complex k = d * (high - low + 1); return low + k; @@ -247,7 +262,7 @@ private Complex[] PolyDerivative(Complex[] a, int n) { if (n == 0) { - return new Complex[] { Complex.zero }; + return new Complex[] {Complex.zero}; } Complex[] b = new Complex[n]; b[0] = a[1]; @@ -286,8 +301,7 @@ private Complex Laguerre(Complex[] a, int n, double tol) dx = n / (g - f); } x = x - dx; - if (dx.abs < tol) - return x; + if (dx.abs < tol) return x; } Debug.LogError("ERROR: Too many iterations!"); return Complex.zero; @@ -307,11 +321,30 @@ private Complex[] Deflate(Complex root, Complex[] a, int n) public Complex[] FindRoots(float[] poly) { Complex[] cs = new Complex[poly.Length]; - for (int i = 0; i < cs.Length; i++) - cs[i] = new Complex(poly[i]); + for (int i = 0; i < cs.Length; i++) cs[i] = new Complex(poly[i]); return FindRoots(cs); } + public Complex[] FindCRoots(float[] poly) + { + double[] dpoly = poly.Select(x => (double) x).ToArray(); + return FindCRoots(dpoly); + } + + public Complex[] FindCRoots(double[] dpoly) + { + int len = dpoly.Length; + int len2 = (len - 1) * 2; + double[] ret = new double[len2]; + MathToolBox.poly_roots(len, dpoly, ret); + Complex[] cpx = new Complex[len - 1]; + for (int i = 0; i < len - 1; i += 2) + { + cpx[i] = new Complex(ret[2 * i], ret[2 * i + 1]); + } + return cpx; + } + public Complex[] FindRoots(Complex[] poly) { int n = poly.Length - 1; @@ -370,6 +403,5 @@ public float[] Correlate(float[] a, float[] v) } return ret; } - } -} \ No newline at end of file +} diff --git a/Assets/LipSync/Scripts/Core/MathToolBox.cs b/Assets/LipSync/Scripts/Core/MathToolBox.cs index 728004c..7ad0ff9 100644 --- a/Assets/LipSync/Scripts/Core/MathToolBox.cs +++ b/Assets/LipSync/Scripts/Core/MathToolBox.cs @@ -1,7 +1,13 @@ -using UnityEngine; +using System; +using System.Runtime.InteropServices; +using System.Security; +using UnityEngine; namespace LipSync { +#if !UNITY_IPHONE + [SuppressUnmanagedCodeSecurity] +#endif public class MathToolBox { public enum EPaddleType @@ -226,5 +232,18 @@ public static float GetValueFromArray(float[] data, int index, EPaddleType paddl } } } + +#if UNITY_IPHONE +#if UNITY_EDITOR + const string ZSolverDLL = "ZSolver"; +#else + const string ZSolverDLL = "__Internal"; +#endif +#else + const string ZSolverDLL = "ZSolver"; +#endif + + [DllImport(ZSolverDLL, CallingConvention = CallingConvention.Cdecl)] + public static extern void poly_roots(int size, double[] c, double[] z); } } \ No newline at end of file diff --git a/Assets/LipSync/Scripts/Core/ToeplitzMatrix.cs b/Assets/LipSync/Scripts/Core/ToeplitzMatrix.cs index d804a81..e3293a9 100644 --- a/Assets/LipSync/Scripts/Core/ToeplitzMatrix.cs +++ b/Assets/LipSync/Scripts/Core/ToeplitzMatrix.cs @@ -1,38 +1,49 @@ -namespace LipSync +using System; +using UnityEngine; + +namespace LipSync { public class ToeplitzMtrix { - private float[,] data; private int size; public int Size { - get { return size; } + get + { + return size; + } } public float this[int x, int y] { get { - return data != null ? data[x, y] : 0; + return data?[x, y] ?? 0; } } + public ToeplitzMtrix(float[,] c) + { + data = c; + size = (int)Mathf.Sqrt(c.Length); + } + public ToeplitzMtrix(float[] c) { size = c.Length; int n = size; data = new float[n, n]; for (int i = 0; i < n; i++) - for (int j = 0; j < n; j++) - { - if (i <= j) - data[i, j] = c[j - i]; - else - data[i, j] = c[i - j]; - } + for (int j = 0; j < n; j++) + { + if (i <= j) + data[i, j] = c[j - i]; + else + data[i, j] = c[i - j]; + } } public override string ToString() @@ -41,8 +52,7 @@ public override string ToString() for (int i = 0; i < size; i++) { rt += "\n"; - for (int j = 0; j < size; j++) - rt += this[i, j].ToString("f2") + "\t"; + for (int j = 0; j < size; j++) rt += this[i, j].ToString("f2") + "\t"; } return rt; } @@ -60,7 +70,8 @@ public override string ToString() { if (j < size) dReverseMatrix[i, j] = data[i, j]; - else dReverseMatrix[i, j] = 0; + else + dReverseMatrix[i, j] = 0; } dReverseMatrix[i, size + i] = 1; } @@ -68,17 +79,19 @@ public override string ToString() { if (dReverseMatrix[i, j] == 0) { - int m = i; for (; data[m, j] == 0; m++) ; - if (m == size) return null; + int m = i; + for (; data[m, j] == 0; m++) ; + if (m == size) + return null; else { // Add i-row with m-row - for (int n = j; n < 2 * size; n++) - dReverseMatrix[i, n] += dReverseMatrix[m, n]; + for (int n = j; n < 2 * size; n++) dReverseMatrix[i, n] += dReverseMatrix[m, n]; } } // Format the i-row with "1" start - x = dReverseMatrix[i, j]; if (x != 1) + x = dReverseMatrix[i, j]; + if (x != 1) { for (int n = j; n < 2 * size; n++) if (dReverseMatrix[i, n] != 0) @@ -88,8 +101,7 @@ public override string ToString() for (int s = size - 1; s > i; s--) { x = dReverseMatrix[s, j]; - for (int t = j; t < 2 * size; t++) - dReverseMatrix[s, t] -= (dReverseMatrix[i, t] * x); + for (int t = j; t < 2 * size; t++) dReverseMatrix[s, t] -= (dReverseMatrix[i, t] * x); } } // Format the first matrix into unit-matrix @@ -99,14 +111,13 @@ public override string ToString() if (dReverseMatrix[i, j] != 0) { c = dReverseMatrix[i, j]; - for (int n = j; n < 2 * size; n++) - dReverseMatrix[i, n] -= (c * dReverseMatrix[j, n]); + for (int n = j; n < 2 * size; n++) dReverseMatrix[i, n] -= (c * dReverseMatrix[j, n]); } } float[,] dReturn = new float[size, size]; for (int i = 0; i < size; i++) - for (int j = 0; j < size; j++) - dReturn[i, j] = dReverseMatrix[i, j + size]; + for (int j = 0; j < size; j++) + dReturn[i, j] = dReverseMatrix[i, j + size]; return dReturn; } @@ -115,9 +126,10 @@ private float MatrixValue(float[,] MatrixList, int Level) { float[,] dMatrix = new float[Level, Level]; for (int i = 0; i < Level; i++) - for (int j = 0; j < Level; j++) - dMatrix[i, j] = MatrixList[i, j]; - float c, x; int k = 1; + for (int j = 0; j < Level; j++) + dMatrix[i, j] = MatrixList[i, j]; + float c, x; + int k = 1; for (int i = 0, j = 0; i < Level && j < Level; i++, j++) { if (dMatrix[i, j] == 0) @@ -143,8 +155,7 @@ private float MatrixValue(float[,] MatrixList, int Level) for (int s = Level - 1; s > i; s--) { x = dMatrix[s, j]; - for (int t = j; t < Level; t++) - dMatrix[s, t] -= dMatrix[i, t] * (x / dMatrix[i, j]); + for (int t = j; t < Level; t++) dMatrix[s, t] -= dMatrix[i, t] * (x / dMatrix[i, j]); } } float sn = 1; @@ -152,12 +163,13 @@ private float MatrixValue(float[,] MatrixList, int Level) { if (dMatrix[i, i] != 0) sn *= dMatrix[i, i]; - else return 0; + else + return 0; } return k * sn; } - private float[] Dot(float[] v) + public float[] Dot(float[] v) { if (data == null) { @@ -178,7 +190,5 @@ private float[] Dot(float[] v) } return ret; } - } - -} \ No newline at end of file +} diff --git a/README.md b/README.md index 3988a39..10edfb3 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ LipSync是一个基于Unity的独立、轻量化口型匹配解决方案。它 * 分析噪声过于严重的语音数据 * 分辨某一段声音是语音还是和语音不相关的其他声音 -LipSync文件夹中的内容是本插件的主体部分，而UnityChan文件夹中的内容并不是本插件的一部分，它是为了演示LipSync的效果而附带的一套模型资源。她其实是Unity Technology Japan为Unity开发的一个官方形象。如果你想进一步了解UnityChan，可以在[这里][i2]获得她的信息。 +LipSync文件夹中的内容是本插件的主体部分，而UnityChan文件夹中的内容并不是本插件的一部分,是Unity Technology Japan为Unity开发的一个官方形象，它是为了演示LipSync的效果而附带的一套模型资源。 查看实现效果的展示视频， 点击[这里][i1], 笔者已经上传到B站。 diff --git a/zsolve/generate_android.sh b/zsolve/generate_android.sh new file mode 100644 index 0000000..6630e41 --- /dev/null +++ b/zsolve/generate_android.sh @@ -0,0 +1,32 @@ +# MIT License + +# Copyright (c) 2020 huailiang +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +export ANDROID_NDK=~/Documents/software/eclipse/android-ndk-r20b + +if [[ ! -d ${ANDROID_NDK} ]]; then + echo "ndk file not found" + exit 1 +fi + +mkdir -p build_v7a && cd build_v7a +cmake -DANDROID_ABI=armeabi-v7a -DCMAKE_TOOLCHAIN_FILE=../cmake/android.toolchain.cmake -DANDROID_TOOLCHAIN_NAME=arm-linux-androideabi-clang3.6 -DANDROID_NATIVE_API_LEVEL=android-9 ../ +cd .. +cmake --build build_v7a --config Release