From 0161435940c25c3fb3421c2e6bcadd2a84deb8ff Mon Sep 17 00:00:00 2001 From: Yazhou Cao Date: Mon, 14 Oct 2024 11:12:24 -0700 Subject: [PATCH 1/2] fix: handle edge cases gracefully in extract_frames_from_video --- tests/data/video/test.mp4 | Bin 17317 -> 0 bytes tests/unit/tools/test_video.py | 59 +++++++++++++++++++++++++++++++-- vision_agent/utils/video.py | 27 +++++++++++++-- 3 files changed, 80 insertions(+), 6 deletions(-) delete mode 100644 tests/data/video/test.mp4 diff --git a/tests/data/video/test.mp4 b/tests/data/video/test.mp4 deleted file mode 100644 index 596eea389407ac08915148bbd2212567a1382b4a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 17317 zcmeIaV|3-));G9gvtqkq+jfN&RczZfDy+C-I~CiuZB)g!(L3kd``+{1en)qY{?s3O zjr`Z-Z_YL6+sRO$YgED$^-xapiLZ%jR63D zJZmF;C!kLSB*goB>6+MK``)r-V-oEW(K6}Txf?S(D-j)$p{;{45fhMgU}5B9BQnr8 zHfCjH1R5mifes8`6vQOxSc!yGMSz}0#)d$Hh^?Kwm9dEv5i=tr3mr2fGc!PqioZe(n0rEf!T>tM?8k%ivO$=V9&V{7MRZfoPnO=PHVpl`^>MC4#>!pA~n zWNct%YiP;G#LdXfNThG0Z{_Z2%*W`)!p-Q$#Kb~mZOmt8>_+71Yyjjqi0mBQfl%PC z>tMvkM9&BW0XHIRb2non-H(V&K!mP?zKy9d9}_!~p_zlNwZ1M8%0%SkU~FY&?g->K z-8hX5oq&v?y)_>r@Cf=w9=0~de9TO=OiV;3`i@Szc8-?jb{`&p2e7x(wKXwuGn zVwgBdIKwG2Yq*4LtAS*eJ7x72yBs)gTA>9Fa*%iLI15Ocy$|vpe|C8W9f16NMBINBNbSLF@c`0qSj9Fp_dSX7Og(pm zAl;Fd(rIs}rBki(Y?(IwL6VgpoYDcqKc%a>#4B~NY1DI$*^MwixZw$^dlE~k###zP zm8OGwwFxQ3@cq-C=#MQU-CI0`)#@OQVe7ItII_e*7}P*DF#V(9^7pN#n+=^qYPdB1 z0sQlRA*2H&IOrEs?Z=vdDB-OP(~68WeRnL05wr7CBLQ;?>JtvTB<~cxKJd8Y@6o6A z#ha&5EEhthmfmU&%syXP5+&P_VyashV6>VI7>Kewra-;>$5lfpefOU15Cf`u{#YfX z_79R}xuV@kodiZAJB9GY1-{;axx-bIHubGj&N~FQR!^rR-&riy5o3%qmFYA&uk7|6Lkqj7F|iiHX2Au80n@)~5K| zW`$1oK9>0&vN`CX@pg^EdgSJQk8p+_VoM`AeFKD4+Ufz8ztV*8s0{;HaCT z)_()aEfc3WD&I4;A2WuBq!r(-svUHJO~BCu6(Q0O>x%T?e8J;BC_wc}+@kH}r_9a> zpYG9T)j3MrmJU$F)-$k{wXJ)~h5_a-S{I3R7Wum73GuV!tSFLXJWh=WwntIPc%*N8 zO%zo`W@31@R|8PldIf(x3)Ie>#wW8gDudZcU%$$)F+9HpXS^yxbSPp_IpxdfF$mm{ z=D!kk-b&P!Q|(5eRTzpLmYRcW22LoLym8pYKesALfg>LSDYrsU*2AlEI`Qs2}HD z6aC7cO5Sm!PVi(>u?p>bf^*mr$mm((>UP*e0dn2pUSlGc_|Ahk9NO-1#mgg~LS0$B z7Zs7kDWslkx)psDTUw!+R28PGr{$bN(LD>S_WbAtoRNegDdB6-K9jvo3EH5L2quqn z#ZiKnF>Q5mntUff&{6YbG-u{E#5(#6dgX;zr6JOX!Jm>rYiQ*D&ddp|#2H>#IS(DF z{SpuLsm^>Mx0Q~oVZjVCp^RG8b``Bs_NI?_s7t$9cED#rH1@}w(OaF~z3BJ)qQchJ zR8Or=S!sr7;^HF38NnG2WWtx{{l=juuCZ>g!wx)^aAW&0p6a>c-H1JEK}Z;C9v`MX zRt4~gl-aF7IE`2~Zwlj73RjO2yI7%0U1cj-YcT!Plo+nPX`8DCz{Qm3A(2#qbR6O*c??DwNMH4Hopd25`+9g4aOiLpT68RpCTgQ&2`_AVmC ztk9+Mk1=k0otVZuoW(`qM&pKnVtCk0b+xk6JWq-^f8afQH|Sa+9qzIUUIb@BXeK7=o_ zxcQa;+N2+9#B&)Pj;gD2B+bn;oox;mXy8{4;aT(vt~8>f=%$!fRkp(1tecGa+Qre4 z*23$ztw4{J+-!Z^QUW9j-!p2SejeI&;SrlNx@kH!M?lWdasE)$SSvuwBhN^kIfMHBW*g zcNXKMaM4M**Fd88CRQP|X*MHl_qrAo`F$S*c&@U9NZpCR<1|h}$MX>+kzX}XhoHLp zc6b7AZrKz%T-U4l#pMN|r#-66cK-a4m+hMA*tdt7Q)L*=Q=>m6_becrwWcAIw1SpvM$_cUh!; zSfBtL*eyxjMX+vs-Ov8$!n3Lyc3FwwwzZ^vEuA96t@n5gk=%nO!K8>%>8Of~Km&S)F z*){IH1j>4l@-$z+TAte%y2%)@nRapBFvF@Esc(Z3qHO#jCg})Mf5+on=o3Ml>O_{h{`~c-9LQ=fyUVcQZ9pu>5q`y08PH_7T3H=u+UrR zd?vkPCI|Ni4ar2erKALN?92gfLXd+4 zuZebH)wrdLatArJ2-ByZ5T{c~Atd-K-4_bAbaC1C=9=_M8B*J;&FUoG_&R6&7I8t_9gev+b8d^#6^`g3F*87LTC}w zk+!C(b%F^yQK`8%hXiqgT=H2SsYCH*pN^*-Z_9*SRdtOFK8P^2+}Ac`h~H0`%|4T#<7*t)_Z5}lSaP;b<9GG~aZc8+9j6VqOrtl@ z-=ci zOFF@PO>X-R{T?QgflL)#OV*sWvp@rfcR9Ab=7nI`H23-;TYLX6pj1pcNa1qM~yX%UWs&^^JbICR#Wd{9=Q~B=<7UP);L% zHy&G%ahNrbZwK#ebER0GTH3&Nd@qm-vQG|{Ul2b?uosY|?o57byV&)5#Gyet;0k^i z8~(gBhA#JsQn0%5H)KC=<>bqv-K#YH90$MiI0!1O1!GI}x4Wn`5-SV{E_|IYH2e|uiwB{;EsRI! z;-8wEmM`24Pi10I+#r@c zeH(TJ+uUe5$jvtm`x&|79VbOFVr8x{11FKU=TNAZh(C4ka6G)S40pt(bNl^)J+&?l%!<_RCDiVJMm@T;9WSY4c}BsB z(=#tdRw$!YgjK&-OyZpLrzQ>70zQB1sghxov^Ar98-%DF&RY(NSJGgv2&EyidZGF} z9-RfD)1X0+BM>V#Qo`lYZ9ljtBX(qRxFSe#plw_OXm!CEZQ{P}NS#|seU()roQ9SS z$*dpH5rDOTh9GNsTPjRlCufvYjwBC>FP9p zo{?WYZZTA%UJSvA19OCTti|K?7PTgXGG9BTN1sf`Il;xFO5d^5(UZ{~U!K&a5=aE= zg6Vn_w_n7M9rC^~Ksh4W5$h$!f>UCHI(KRGyO(44d;|@^l>}B?r4EzGb(@FQ0FJ}N zyYy{-*yi`pn#!5^fNTr9sv(^X3QsC&e^l?+Wb7@7K?V!m@9n1+S-)|w;>mrM6Pv)* z`{p=1wYP2k#@hY;dk5pudt8^$pB6|Vz)~Mlk%iGg87-^4G;5q<10b8=SMub)7+DC> zb+!6$Qf2JJK@Z7`gIdx>i8>=TWe39=Ur;YimWlAM``<5n;MBcCL^zM4zO|2aaltG8 z$@97!cv{=gbm~fype~D3Ivq)h6Fe>{uJz+`?`@p7cR_Z_KrQ@Q?JTY_N3)?MCiD4X zSlcJSKku{^$y$9ab*YFR*R5s@0mJHR)2H2~JJ8vVm6dYq*18`aI=*|eE6UK07I#LS zEjY__J|YN@23a0}Gc@J2_gQAvL8;`+iSa)xk*(nqXucgJ>OR^2#h&GE_%^WRmV73) zq0p)RZ}mrt8Pu=_u2BnU>93+AKk>lS!lsgiTXRMnU;bD#tCF06_;2rRURp52>5I1gT$^=l)kGgn%(T>zjr4Q3lVI}Tau8pgH4Sr{W z--zo3{bW^K1r%*oYDFvTqXnuRjWc^4H9O(cWg)t|@#+C$#WUQa(b$*OZ zuHrdJPd~spTZ3s}KiG2sjE8w>&-zq(Z^Wf4i}U-JOWO6MBj3e)*>L)xfT&Rp1&<<#JCh-EUObD(RamS%FD(|whq_6bbx?r%PaVN4W6 zvdZzt+i#HNeUWD?Y{Ono{OWw08jkRXYK=2m@eBqU-(~m;`uD8srLgFF%Xd|SpP944o_(n}BE_?UJu4PPQ8pqQ{+;-G0qsQ|QB&i6&ctc=90TUOW!c zdzOa&#H)2*qesm;aepx?8uDRWm*@FH1M(A-|6oC|buaF&z_nvV?A7yJl}-nuz1p_h z{ZB)FVu3RYKb&JlrtdCZjz=gG-n@0KbGia;>CaLEwa?iMO0rbK`?MEYReZ`C3uXhK ze4CpdFycgS<>`l!i>afU%4x238gZNEu_I3zDOrJnXkd87D5vRS8-KR=>8Mb06f$jRE}tM*py01QCR(kGwz+Mj5N7ivL4({^h8T%pA(g#3 z9l0-}B>Nj+@JIHGIB-(#SgxD)vw~CToQ?5W7ELyZP-MA#n(z zo1S4;(Bb9;7OgUBU>+Gy!-CyA-fBkf6z3^OBsveFZK=w5~SJIxp4HCa-MRLWii*pa~3;1vy2q?sU$3VS|a3 zv0ABf;L_*PPK=^ul!xNFp^zr$Q#%()y^=&e=gk-C_78-+i#P-puv?ap?beEb8H=Nv z2d6T)=R}EL7^~?SaqPvqI~$4lFz@IiyyFSQv;l|(tMEO$C-^rwU4|s|W`Wk6EIcT> zu)?dWhmF_r{$4jn4z0U}@PeHgdQ_~jzgjuJbb}XPG*^N1{~=WZ7%oF+H(pcx?(p=T z2tr#p#s}eDL88HR#WuOjJacn0Q19kTdcD3dyiIV=WcZwzs(6S}pLc+8k5z16=Y+Bc z{*HR(!8#!|sjEiyuB(lYZVTN|n;SJxq9WeKDvt@lz_759XCB%5&Vd)~wX^m{^W=GV zMC2NvkO~TE<`SjEHY#jH^p4s817SIo^6?MeH9Di##k%J0Imymx3jnTD)lTIJ6eD%MMVy|e6hlcFeD>Umf0GNRO-HUBji)AB)?^T7_BJq%AiU_uw27Pq#$J+ z%@HBYFIl74T zAErhxzB<($!l*6WB=N=zZ5>TcE|bO~ZVPSPJ%{9L=-}cM7Qnu=;}$BEKNt!n%U7IL z?ufcy)VHNbD?(CcZ5bolob!Z*v(FM9_bvNStNo=Quw@lLhtxG%(H8U47vk<^`f3Qs zo-`7WSHuyJTV2}AOB{rB*;45m#g9M3mAM|$=`ch)>lR8%-v;d0R{WYq6Ij3A1(Zk2 zzFUM;R=u>Z<#^AY{UrN>W;!5Ymy&(z-ox?oXQ<@R;re}paPzP~vQ?ahRI1+tdJuz+ zUmx<)Zo)m)E&5Zxj*I-J$6R>_*4~)kY!R!j_%ti)@kp=eVQu!{j4cq zvWWF+V1i5B@A}GmZw;SKYd!Hsyynw zo+>A zE@4*&PSqPRTTEee3|>9_fvahRD_N}A@Ku#R?APgI5qHP74Q(V8-SK{fYm1J1RC_|- zVS~HG5MmRwte*ty6VFxOrK%BbbGb>!zW0xE9z)*9OUC3*ntBj0D$vVC5N?Mj6t?&! zX3hbA>TEE3n2rK7ymL(fULL^8JU(<-udt(!(`;GlS7$&-aj=|m#UQ>)WVQCP0Wx|XZRPJrgzG97j$bXezN!Sp2%q;>Pv=@sQny?J9{M(r(p zVwmtVe$ioGvEn#KSmaX#(wt$XSv{zGLcQ42)r2iR!m44|fr$$&tzbRIZ~vNP)Cy1z zn7G0q5Vt+y@A!fIEH27u^ zX67&r5uQ)*8lIBh?9$q^WWOi(J#OvhK?>AwIn~!iuJP0kM>Cs*-(lBNC1ykF6zkQM zoFygBR~u6TuA%a@Mla{>V9aSGaq*YZ$NdUdaOARuIlJnKiV->8z6_ONMmVL+4YmoEr9a4?%pMkj z{6c}3D6018n4F-)+r`;?b%pYd0bz)p@p#sVGJr{;aFjKh7Qg5L1$~Fxs>$m570+4z z#p3y4UAUB?(X`k#PC}*F2+X!?hb3)y?+-b?U_Nrw7bI9nEt_(GiBrTF73q#3p>sUK zF)lYDv<>i#LRI6)$tCoG*Ohr4Ikk(r*f$C-Cco98R`46J?%vm(Z-l{1HBn|V*~;QD zlM(h_S^m#B!r^C4Mdvw_JWVXHf*cCA$!#j$x&B&#ND#C}UZULOHM6~;0_Iz=#5kc< zm|qxK=hSK+wc&Ky>4?b$Aw_XW`ax5@^R+#UkD{Q6L(l5nTP8WDLKpoY+Bm;pfZ>cq z6UY$FVh;lh6rmJbk?%5#8#XXRPtOgAvwZj(t=qft@2yl~G3|wsbgsAuYKNlkOTd)P z&7iA$$7<2`UE1b_E_OxX*45VYh!{9JqlT^!sC+2;cefQXb^`9#z~M^sF1sK*-W)F1 zaI21*r>pSR--)n|6!%w3g3ZxH_n*bA@UIVJP73V?e5P2yqCj;Q+fyk<}wVR;=V$w)JKSD;j!GkiT3P2Y+O z0WtQ1_5#P=BYh0oY|2*qWi}1ue1?7Kg|^gJtp7em)Leq zaxa_*{s||m`psO1(8G{XNoeT2J5R294y{|uF}qdz9?r)`<(>)*NnO7SybPnaH(2tJ zk@x%LrRPLOGm2NsFFTe@erX>1)9h~}eJRt3WFiQMWdeHxl~s?r%mN2jxIb4dkHFe7 zIDZq-{2Hb*`T1_w*|);fI(h+XCa;}SsY5?)i;OnF-_v)J)53n=2Ii`a(&d1aR#1?V z6W^Dn5l`&0CY9^Nyx_by9~?Q`v@@ak^L@5a*KyJ;Do=j)|0RDUWe?G0{AAIZyZdFQy>2W0z$1 z-;-)coSaK5rD$$1w?DsVu84z0o4_8IFiVD5py&u8){;N4@5^T>zNqkwcu7&+kg;)P z2XRQt2o2gw{$Z4 z-s?|-kJL%;{)OB`cIWTJo;>dEmS2#?_9910*aAAQt0#Go#2{EM6^Vu>4x7{5v2Q~{ zMZOcQI+(20Xa^NXqI{}*XaI7qSEF{*?UdEiD{b`xR85?w4DZmwLlfAQ4t_A1Oq_Gm zYAL55O~Zb-W5VpMh?e_|0cR-@Yjb5n@;VzM8jr4q2rc}ZC1uF)dpIR^420%;QxX-5 za*O@G@Mz-FgC6W_RT7QwPQWAlN4_Nm`V0C=McE-;XJGFQm2kppq z2F1h6(KGx4Mn)_YRI5o{Rv0#wdAi*Ico=ov;qC--IE3pv3#o@ zl|2o?ZJ{@l_vwt)dJH5DVx}dr8Djeqp}%`CM1mVGb>n&ve^6NMovZVHfmNR>NvT@q z2ehzAX`sZo9M#bTJ)gUW@8z})?Tz&JD)6PyJF@n-I0R@X7|;j76E(gO!_Vs4IvdUK z!aCV5aC>TbgG}Gw|IXyG<@3Wuncf9e#PfUj%!ps%(+%oAjXt0bQ2OSx!s;ZRSNJZ041|t1HZIR=`s+p$;fpDKk2Xr}{Mcx^* zFbjila>9>c9qG%?$!QRpus2HL_glq3n=@8}^|MLsq3-f(Jqq8Rx0fd6E=2QG-6+27 zB+2>5dD@ejsBOrXq_@1`C!XAM-#ryrpu>f5lNvz#31RL!VeI~p(JzU1QpVFyW?9=Z|0C?F#nMszMIg10j=OY4wozh5Bdrd(j9o$q@?Z3qCv~pr~XyPtM*4#)BpANq*Da2c_+g9ul~G50uP!!p^CGM@+~b`Xh9r-CMCin_|=ZgB0$ zDeSw}O#Nl6k2Tg*53G3K`;P*L<8|jROuaV^c_F&ydbD=Z`Q|6r**D_??KAbfo{=2H z2YISr2j0O{DA&|mSidwULE}&Pj&z6b+Lk9JfZ|W>=cqTcH|2hT&ubs4i3PY~<9vpf$e)rY3A<-s$S2kb@5b>^)0p=_^|A!#q!Z5AUkDjcVNh3&b! z9cks8%V#+dKztk=kK!ZPU#ytzn&|9yO@*b>pQhU^%ITR%S}qDJ^4n=>a>s8CRP3XXqwzlZ6+r>x*b74tB!o^S?9h~Vbc>0F`}xJeVi^skRN9B)jk&Ei zb`czD)_}8xa;gqBAZcn2dz5L1m${=fkzH)6loGru!=Tx-oT$Z=Uc4| znTHiKg?I!}Sm_=XO*B`746?kQF?`ScrEuq-v*+!`PiJ?$0DEND-9}Of>;#n=S!~c} zow={(y`Lfu7b(Xz2;1We}UE@(xw9uM$63B6o#X;CBkSzuO4 zjMU1#yv>(E8s?yenmP@mz@%!S{%E3&yZzS7b7|1SKM8l?fP0LlGW`-UM1)%2(`8TZ zWt#0I=4BfKP?mjt;tSY*kj;~e9aw!GG)tfM!XVNfT!_SWtIDHZrPFjkGQ{<|+jPrd zr%zBdp3iFreGIU_;H}kmH&~0%;?danQmx{xLvH2R(#jAENET(c z`C0Hoe0Tn}l2JxRCe$t8Sr+9y*UWABWNERapceTuDz(*l@}4WDtmpGv$J2qX*)01(p%jdl@ zy16Z>2%TNae$tbyFSC(S&2@CQ4WxOu&_wQ@hMG8x7|biw27e;z5}7RBZW+UTNaD6+ zb{0*WQ&I0f!PzSquO}+avT(*9h?8yQCeNU&AoUT|3plXm%V%||SM??bKvQ7HuW?El zzfM*xre-7zy5x^zCTsR6f45iR76@sqYh`HcUaVfhT>Kg@a*>TB*IR`#U_XsOnWtxs zamDm)dyNiu=UsyZMp>%zo~6MHF2o+tjB-`=z4^|u)FfB6#M1XktOmmTdYDtzJi&w0 z2i^`H)Kc;M`$o$N^O6YCPD?5z(XfCwLblv=?1L4nO`2!Spp{oUizzuvj%n4q^~`F! z?I-w=8m2mUNng2AkBAX}H_9^ffPLOiIf#+21>d}P==kljPyCkW(KFbQ7y#mIV|CwU z&|}{P_eE#D?G@s>1N9!fuM`?;=Nv)Az{U?wIIBo@+#MENx4+n-kN+%(OpoEY3ZMU6 zQDFJmKg^ld7#J1<0l z)j_dE58wQz)ptm%fRhBic02T}>ZK^Ve2G-r3g;WMgQwlYqb84&SXW(g2Xd&QN@$n@x3LtC z9x{#Cp*fGzBVF}6L~SHFI8UH#eiz0VO3~RbKa)<<78ZmYrQvUi6}LlsK{2l#gJTaP z3$_vNve*)3Eyk)z2tN2~ZqhS>Ut@Y_+C6kpr44^u1v1TYRD8%Sz_ZG`Vq+k4~*=xrGwI=prZKcqrKP1lD`jG`QpX6O;elox-QCG{V z;_@3V8Ry}KLM&fCeBc+gsZs0f8OofG{K;$SsV<$37vbQzlM^Hf^2Kqlb;r1Bks>KF z9jbkPFTD`qZ%QgKgUyDDoYs{+h}mD=XBF0rUsD*{*sD}E7PK-C;)g}-ift9!9|++Y ztV$$*mB~ND9D@|-t2%aQHbmz>gjaRGj;r0|#nVokW3sA!={ti`o!N#9U-}e#{pWJ-4+-Qs=z%Nbw`R?~X`^Vc9{gNnfmZIVwj7UBKDNWU4>$LD=eBfK|S0Yxl>Nb!6kU)7QPx61e?gp2!ipwcE^XTSs?PDuVf;&lV?us84>FHD zumH9Xdz{HbD7WlUhe@X+V_3I#PU7c2SprzgYfx5B4g-91+)UAmpT-`@R4HHfwaE|5oVsw(;V_)t-zOKjcmsJk`$~$AU@ejLrWKMah~&ouujVX z+@AG$z3Qb5efJSi(>qJOzVW%QEU-gCWF6llzwC;f*v=kW_!l_|%7bKkgK2gk=|w!4 z6p15BK4qlFD5#Q$e4&*bt~W_M?*GN;@a>M|o{yWd%b!m@@9sP}@oBv8^FCHxIEYOv zrkfLT;xkkbFa4}zqo)0NBM&6jAL-~z);Tvj3#HMqntI@%8qtPqMmmgA0i1){ z@bi5p6`8R5##yC8LE=p+Hfp}tWETCD=*^cm-P3n~){Emzh_6GR_nEGp3Z^fvn)m6_ zr-)moN$;^;3W=uymHUv0Go z))^DL{hz#{mH-p_Q~(zTF))HbuRN*B3hQQfKgLyI4B6pGc_Nq|hue#ckw=HV(YliZ*anigUY_3(BAnHT>6n zAhk%b`Pq%0%9wWw54F+#9Au88Gh+ULB?Kp$>aq69TAQ&S-At+>Zwkd}>lUN__y~wM zy89J%*jXW#QP#4fW?$~0d_sBjZC2BX3-7MY{e6+eOr}*kR_-erH_=g-hq-DCTlDax zDLl6m6HbL2`SRJ_Caw>@<$z<-Ay2{hbc|F!=t)_?h{p`QYt zssR-J4rWGkDTx=$H%D_6dLxt7X@MatNnR$1#=>^7LqJ~b_i(%t(@SoT4DSrIVLS`< zAB6E~%={uR!OC+j?8G7R&X@VruNP(6w=<+sOl-CFLiD7ETgyDF?Dt^vs&k1n0deb( z zGY8lYCRdTIDc4OA&?yf=$U!4wp)7+@0-|7jDs3<>1bG6?H*U}k_3H;Y`z^>^+NK}n zoJk7WBb0==zPa1BihLcNAAB7wvJR4Kgoma<9jjb-&pTU7rXy>LgP~pK4Jp%#fM{5W@{!0VXS^WK?2M zF`l}aTh$EfFi{bcDNn{pmV-;qti8$9EIxDjTHTyA79Ge)FyJ`k`xb8NL;A<=G1zg# z7J9LLXX=JjI=fxa$&}zG!9)|Ge(k014<46y<>y9N<1+!l<9%?|oWHIhI62+Ptcp%R z&U)++CtZHjQG5|Fd=vkoR7!rl^}9LlE#T)wg$2wq`3t@nqQsYq1psKE!Upqc^Urk^ z=Nb8({jIX^=ml+vnEL7wKkeV+z-h~Fx{+*Xu zZpzh_JRMZ6KS}qxvDfmk`v^Rp&PP;!MRpMVhQ2|2f}Q3{*CVc+b-QlWBd{p>9DHQ+ zEIaMbpo5<~k7Wf}Gxi4yDcwv20I!p*fz2a$%e(r61GLiubxCt=dO2m*XZqJ_$Y>|~ ziw8?s*O9>(WRU(Gr(8_|u_wVePWVgQk0#krmIeG01Id{rivoRnp-bA0d9{#8c`NMD z3rqUB$H(vyveM1E9Bk=<7s2`Q2mPoZ{ZS6fDX0a@l#3?vThY9$)q+`6N{UDO7Jr-H zs?_USq6H0(?NHT=Xuh8Q$~JJ5eELhHAv5<*5-O2+Nq!T}5d~i(leMyBB`1-Lgttyn zZ4_n$Dp&YG=X-{C73Wzl;oF&1sWDB(%JVtpcq?H&KQG6t5>~NKm?JhGB4?$u{C+{dJX+lU8!Wy8!(eMtP;olg{}Oe)3QE=)<9jJh1{& z&C96O#pO&8a{~J>rthMP8CzfZkqK~IyW!6AzOIN7wniGJ=i)PUM~tK@-4Tk)NX?Vw z5m}9=Fw}cq9(NKF|5%l~#OJ#7S;zk@`mw>IEopRvFWV>$K5EH6h2 z1P#Alr#;#m9r*W{7$9^sS_V_e@Kkr3+hwnLld8%Nah^ospb-X{uEJgmQoD#({DsQP zCaQ{#bF7|022qei!qFnN>3pA=d>L4e^(*LTsUpfFZ^}bTm-87 zW}3^L`TcX!8|uTgEANBRwsDq_wXQgemv4BVxRM698I}TZz1)?<30^x!%SHLGfu8 z?5sT#+~jdUUZ1F{H!4AbLr)~N`g}dJY@fWE-3$i@$ABUjvej5&Ivp>R$f@!!J|kG? z$ImJ3m*`hEeIp7Ad&mTqx;@mUMUl7TT# z2J{>08?g_Bd~r+R2x5Yo6LJI|%Gcg!@#UESPLdK3+J<{E1Go{8LN|POy88o`?T>r8 zLaAg{z5r0fzxYgHw;%yb#Sb1NQ{PEA3(xLv-p~Tb+XHz-p;R-eVj;5>S)_LWC=yTu zE0h@`hC)mH{%>5t2kyfc7^CO|_Ynh#D+l7N{>FV|1L8)+(Eouu{71gyf8Z|ugZn#Q z7UDl~S#-Z`}X0`JMlLPH5$KklDX* zSqQ*<J(#>jKnSS-Y4S0qH*$*Z}|lH3a|xe0=?9{SN`C{x4bZ zKc4?P4io?Yj{sgLwgeiroGky!3HA@ie?5Hga+VQaocT zN8sJ}zZNjUM_llKof#a=+Q?iVXd$vT`tRBqfoeT~O5a~NIx{0Hhrc-B_HnEI-~0zv z{kY{Wsc&OsW&Gg-Rsp={YyxD^T&({J`p;vN8U1a;GXh?w{s05lhaKM8!HVc_9t?Qt z+rSFQS2#L3{(B%FKA9gE`+@Xh{WIp@8b4s!l0f$3`B?vFkH7W**|FZ=fdVTqstAY? z0PcMofN2V(<;%k0W`e*v0Kg3%tRL3c4+H>dcKm0Qr0ezJ;RBBbyyyLQOF>_=0f2{( zw3#4?XY!2{ZhUFT>wD;7ITmBF~Fn`4Y z(*NK7cR&8$_6Pc-pTK7R)lWR&Yz#jX!1b@$e9Y}Hehz5!_{;M52GsL}0a^I}dQgG# z$2XRL>;Jzj17C}Vw!pJ~tdDv>YBg{+a?%IVZ`Q_tQ6LK-{^#2r_z~ftZ)f-Mj{IMX n&iwDs6!H=$+mC?wz<;mt5#%E%Kvv(ygpG@ynS-8*k@5ckH212{ diff --git a/tests/unit/tools/test_video.py b/tests/unit/tools/test_video.py index 2ef1fe21..81b1699e 100644 --- a/tests/unit/tools/test_video.py +++ b/tests/unit/tools/test_video.py @@ -1,10 +1,63 @@ +import tempfile +from typing import Optional + +import cv2 +import numpy as np + from vision_agent.utils.video import extract_frames_from_video def test_extract_frames_from_video(): - # TODO: consider generating a video on the fly instead - video_path = "tests/data/video/test.mp4" - + video_path = _create_video(duration=2) # there are 48 frames at 24 fps in this video file res = extract_frames_from_video(video_path, fps=24) assert len(res) == 48 + + res = extract_frames_from_video(video_path, fps=2) + assert len(res) == 4 + + res = extract_frames_from_video(video_path, fps=1) + assert len(res) == 2 + + +def test_extract_frames_from_invalid_uri(): + uri = "https://www.youtube.com/watch?v=HjGJvNRkuqY&ab_channel=TheSAHDStudio" + res = extract_frames_from_video(uri, 1.0) + assert len(res) == 0 + + +def test_extract_frames_with_illegal_fps(): + video_path = _create_video(duration=1) + res = extract_frames_from_video(video_path, -1.0) + assert len(res) == 1 + + res = extract_frames_from_video(video_path, None) + assert len(res) == 1 + + res = extract_frames_from_video(video_path, 0.0) + assert len(res) == 1 + + +def test_extract_frames_with_input_video_has_no_fps(): + video_path = _create_video(fps_video_prop=None) + res = extract_frames_from_video(video_path, 1.0) + assert len(res) == 0 + + +def _create_video( + *, duration: int = 3, fps: int = 24, fps_video_prop: Optional[int] = 24 +) -> str: + # Create a temporary file for the video + with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as temp_video: + video_path = temp_video.name + # Set video properties + width, height = 640, 480 + # Create a VideoWriter object without setting FPS + fourcc = cv2.VideoWriter_fourcc(*"mp4v") + out = cv2.VideoWriter(video_path, fourcc, fps_video_prop, (width, height)) + # Generate and write random frames + for _ in range(duration * fps): + frame = np.random.randint(0, 256, (height, width, 3), dtype=np.uint8) + out.write(frame) + out.release() + return video_path diff --git a/vision_agent/utils/video.py b/vision_agent/utils/video.py index 0bb6fb18..6e8605c2 100644 --- a/vision_agent/utils/video.py +++ b/vision_agent/utils/video.py @@ -1,5 +1,6 @@ import base64 import logging +import math import tempfile from functools import lru_cache from typing import List, Optional, Tuple @@ -11,6 +12,9 @@ _LOGGER = logging.getLogger(__name__) # The maximum length of the clip to extract frames from, in seconds +_DEFAULT_VIDEO_FPS = 24 +_DEFAULT_INPUT_FPS = 1.0 + def play_video(video_base64: str) -> None: """Play a video file""" @@ -51,7 +55,9 @@ def _resize_frame(frame: np.ndarray) -> np.ndarray: def video_writer( - frames: List[np.ndarray], fps: float = 1.0, filename: Optional[str] = None + frames: List[np.ndarray], + fps: float = _DEFAULT_INPUT_FPS, + filename: Optional[str] = None, ) -> str: if filename is None: filename = tempfile.NamedTemporaryFile(delete=False, suffix=".mp4").name @@ -78,7 +84,7 @@ def video_writer( def frames_to_bytes( - frames: List[np.ndarray], fps: float = 1.0, file_ext: str = ".mp4" + frames: List[np.ndarray], fps: float = _DEFAULT_INPUT_FPS, file_ext: str = ".mp4" ) -> bytes: r"""Convert a list of frames to a video file encoded into a byte string. @@ -101,7 +107,7 @@ def frames_to_bytes( # same file name and the time savings are very large. @lru_cache(maxsize=8) def extract_frames_from_video( - video_uri: str, fps: float = 1.0 + video_uri: str, fps: float = _DEFAULT_INPUT_FPS ) -> List[Tuple[np.ndarray, float]]: """Extract frames from a video along with the timestamp in seconds. @@ -118,6 +124,16 @@ def extract_frames_from_video( cap = cv2.VideoCapture(video_uri) orig_fps = cap.get(cv2.CAP_PROP_FPS) + if not orig_fps or orig_fps <= 0: + _LOGGER.warning( + f"Input video, {video_uri}, has no fps, using the default value {_DEFAULT_VIDEO_FPS}" + ) + orig_fps = _DEFAULT_VIDEO_FPS + if not fps or fps <= 0: + _LOGGER.warning( + f"Input fps, {fps}, is illegal, using the default value: {_DEFAULT_INPUT_FPS}" + ) + fps = _DEFAULT_INPUT_FPS orig_frame_time = 1 / orig_fps targ_frame_time = 1 / fps frames: List[Tuple[np.ndarray, float]] = [] @@ -129,10 +145,15 @@ def extract_frames_from_video( break elapsed_time += orig_frame_time + # This is to prevent float point precision loss issue, which can cause + # the elapsed time to be slightly less than the target frame time, which + # causes the last frame to be skipped + elapsed_time = round(elapsed_time, 8) if elapsed_time >= targ_frame_time: frames.append((cv2.cvtColor(frame, cv2.COLOR_BGR2RGB), i / orig_fps)) elapsed_time -= targ_frame_time i += 1 cap.release() + _LOGGER.info(f"Extracted {len(frames)} frames from {video_uri}") return frames From 63dcd5e369631b5557b56c99feb70a529d112e93 Mon Sep 17 00:00:00 2001 From: Yazhou Cao Date: Mon, 14 Oct 2024 11:26:35 -0700 Subject: [PATCH 2/2] Fix lint error --- vision_agent/utils/video.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vision_agent/utils/video.py b/vision_agent/utils/video.py index 6e8605c2..384dc6dc 100644 --- a/vision_agent/utils/video.py +++ b/vision_agent/utils/video.py @@ -1,6 +1,5 @@ import base64 import logging -import math import tempfile from functools import lru_cache from typing import List, Optional, Tuple