From c0dde366ad5348f2332eae6ccee2732b88135ae7 Mon Sep 17 00:00:00 2001
From: Dillon Laird <dillonalaird@gmail.com>
Date: Tue, 23 Apr 2024 17:24:23 -0700
Subject: [PATCH] Add Custom Tools (#61)

* added custom tools

* updated readme

* register tool returns tool'

* Add a new tool: determine if a bbox is contained within another bbox (#59)

* Add a new bounding box contains tool

* Fix format

* [skip ci] chore(release): vision-agent 0.1.5

* Add Count tools (#56)

* Adding counting tools to vision agent

* fixed heatmap overlay and addressesessed PR comments

* adding the counting tool to take both absolute coordinate and normalized coordinates, refactoring code, adding llm generate counter tool

* fix linting

* Remove torch and cuda dependencies (#60)

Resolve merge conflicts

* [skip ci] chore(release): vision-agent 0.2.1

* make it easier to use custom tools

* ran isort

* fix linting error

* added OCR

* added example template matching use case

* formatting and typing fix

* round scores

* fix readme typo

---------

Co-authored-by: Asia <92344512+AsiaCao@users.noreply.github.com>
Co-authored-by: GitHub Actions Bot <yazhou.cao@landing.ai>
Co-authored-by: Shankar <90070882+shankar-landing-ai@users.noreply.github.com>
---
 README.md                                |  30 ++++++-
 examples/custom_tools/pid.png            | Bin 0 -> 9070 bytes
 examples/custom_tools/pid_template.png   | Bin 0 -> 4274 bytes
 examples/custom_tools/run_custom_tool.py |  49 ++++++++++++
 examples/custom_tools/template_match.py  |  96 +++++++++++++++++++++++
 tests/tools/test_tools.py                |  70 +++++++++++++++++
 vision_agent/agent/vision_agent.py       |  23 +++---
 vision_agent/tools/__init__.py           |   6 +-
 vision_agent/tools/tools.py              |  87 +++++++++++++++++++-
 9 files changed, 343 insertions(+), 18 deletions(-)
 create mode 100644 examples/custom_tools/pid.png
 create mode 100644 examples/custom_tools/pid_template.png
 create mode 100644 examples/custom_tools/run_custom_tool.py
 create mode 100644 examples/custom_tools/template_match.py

diff --git a/README.md b/README.md
index 48675938..835b8b99 100644
--- a/README.md
+++ b/README.md
@@ -31,7 +31,7 @@ pip install vision-agent
 ```
 
 Ensure you have an OpenAI API key and set it as an environment variable (if you are
-using Azure OpenAI please see the additional setup section):
+using Azure OpenAI please see the Azure setup section):
 
 ```bash
 export OPENAI_API_KEY="your-api-key"
@@ -96,6 +96,31 @@ you. For example:
 }]
 ```
 
+#### Custom Tools
+You can also add your own custom tools for your vision agent to use:
+    
+```python
+>>> from vision_agent.tools import Tool, register_tool
+>>> @register_tool
+>>> class NumItems(Tool):
+>>>    name = "num_items_"
+>>>    description = "Returns the number of items in a list."
+>>>    usage = {
+>>>        "required_parameters": [{"name": "prompt", "type": "list"}],
+>>>        "examples": [
+>>>            {
+>>>                "scenario": "How many items are in this list? ['a', 'b', 'c']",
+>>>                "parameters": {"prompt": "['a', 'b', 'c']"},
+>>>            }
+>>>        ],
+>>>    }
+>>>    def __call__(self, prompt: list[str]) -> int:
+>>>        return len(prompt)
+```
+This will register it with the list of tools Vision Agent has access to. It will be able
+to pick it based on the tool description and use it based on the usage provided.
+
+#### Tool List
 | Tool | Description |
 | --- | --- |
 | CLIP | CLIP is a tool that can classify or tag any image given a set of input classes or tags. |
@@ -114,11 +139,12 @@ you. For example:
 | ExtractFrames | ExtractFrames extracts frames with motion from a video. |
 | ZeroShotCounting | ZeroShotCounting returns the total number of objects belonging to a single class in a given image |
 | VisualPromptCounting | VisualPromptCounting returns the total number of objects belonging to a single class given an image and visual prompt |
+| OCR | OCR returns the text detected in an image along with the location. |
 
 
 It also has a basic set of calculate tools such as add, subtract, multiply and divide.
 
-### Additional Setup
+### Azure Setup
 If you want to use Azure OpenAI models, you can set the environment variable:
 
 ```bash
diff --git a/examples/custom_tools/pid.png b/examples/custom_tools/pid.png
new file mode 100644
index 0000000000000000000000000000000000000000..713b731716a42104aaad5d17de26db26c053da67
GIT binary patch
literal 9070
zcmch5Ra70_*5x@!2<{LdcyM=jcXx;2esBv1x8MYqgS)#+aM$1#+(~e6zI*R~yZfPE
zd)KH@bIm>1TzgeL)Yv;hSy2iZ0UrSX03gdqi>m?v!0Go}@8dfG5EcOsy+2?qMHEB;
zfZABZXA{_W9gMY@n6j;yl$fKPBS_83#LQgE+`$}VZ6*aW@scvJw>M!mVg&%SLkhLv
z^b;3xx;KXb2{pyvYvRp?0|2aKY~WMUmT(!805sDNP`^bZu`BdGbJNmB>gr<$@Tx><
z9uHQc+LS`(O&J&~x-wwU%Flc*?z*1OJ>E724%S>I1rD-(puy*oG2&#RK!BHND?3^I
z4?!A%AjLMg;hs-c&>oP9?a?n4_xGQMq52<81@CvgoS{iZU(+){RZ2&1_h_EJc(}Jz
zCUn0?TD$-xJ3x{QX9`sw0Dj_D%Nc#Ec!V4Y=dgkzQh+DVZZ6U&EphqBu2d#$&=I<-
zBDz3eV#-jiPn*x|PT_-&=!|8>tA7rEys0*q9hQ!RD*Ner?u$9UVhP5Rh;Q5&yj)-^
zyt88fGY8IZV}LSWgs&)BzY5wFRx1sCxB+R^R6=O6@aK<z2m{DeBZh$BG%Un~ITFqw
zZb~j8T^d_x=VVVwyG&i&TM8|l1Yj*@KLeDY9v&NCII-Ln<`4c!z!p1^5Id5o72G7&
z=<-NVi@s(YR#6jXw;}z<2#pgmWe~T@A2dGU(;i%jFJ%X${nERCMtQcma)s$5qg9?H
zxQ9|opF=jq>-6J3!`D*vkASQ}c`HPwUdhav$j;pT?Xa`M;*H(?f)Ua`ES(gj&p%sh
z2YV|>hp0-JMy^QBx_5C)@Cc2g&qBvVIsu4xgYYgVU`?mt4~0JjLxe->fN;CLl3_S4
zeK^3@{u&w5Fr?iw?gEHys^FL0Bxh(0=`lE+I4u~8u)!lRoTaiHt>s9V!b4$Zs~*Q7
z;rt<rUicmg1Z)H~jd%nVP8o=@AWho9jCxDzqz<%RMvI-#OODmjDQ^l8wDO;VB>Vng
z$TVq1Z3+=?`-JrqeL2ptg=Mm5sBb@ZXbZ?KMnMtsS)xaQL(q_ev*0G^Kz7qRv~e=H
zDdzgyZX|N$ZyIWfuuZbVTu&#zuhZ$x%9I|;k4)bfxzxC2zG%o{F3s62kQdG#iJljl
z`&Cj%DF9vu6A}-Ae$|R|ttJFVXP!Grwn@Jd{KJonoBTd6w8VSmkAB&p@`83lufkx&
zBtol#<~{YNY`|<IZIf$(^KTA2dq9J}SwY$hUSVVmL|*L)OqZawA+i(2j<w?^LI~fx
z%qmQ9f0Cmo6Z~WKMZjO%htuxCztM!*?4I@|bE<0Utxim`;%l;JYnnJ~Wz$zMlS*|M
ztNXJ=Et9E8DV>>6eoAdtZOWg_KU-qYpSbY4Sy%A%NcZ5I5cs|m{!Fj2s6d|H)?#x<
zuvN|NTvEB`yX4{{dt&1gO*2{RC4>}=!nru@8sZ(X*^09#J&W0<UU5ALu!besK5Xi+
zgR*1kKEC~AEL@)DxB8OKHW4+LN6Kz{e^mf0_EoMvcVPNEDM!M-G-6>ie!RzVc*F-b
z>3bCcWxY(4Qn55jO$~CCU3HMthnJV~*z4F1b#KbgX#^j!_d}h;kstKW^86bXRUs8<
z<EkTL>I)(j35mW*o2(iOf#kzqHIRhN2xejrrIE;azfFrFQAlF^G`YzAOcg^aDRNrA
z+sELHnu%Pl_ZhN7zlMlP)PEewrGBNZG7v<<frdJeHj4i+qqy}dr6hGO>bV$qg}65w
zB_iEbAWb`tGkiqv$@Y70*|e&iZar}us3g88Am``jnGI<ZB9*k)4`~n<2&FTEfxurF
zf9S!|hSKvaYZoyP>><vsH@W=&9eRYhZVuuGWfhH8We=|n7!{$=M)?IsHbz`RITjY+
z{ul$!uc7rvWdx5w2-iZWlyN<w9(xd(!6YBybIBgVX-q?w$Qi?_`oy8)nZJCH5uc5h
z|3ZLHj!*!j6Ru817mqxwS%AU=W+yF<yBfyc<(!SUCg~=3R7NU<d?<lE2FHk`Q6ol6
zCeD;vaq?m^NRCYx9`ZJPSdZd_6i*{s@vQ_1iafD^txY6TIrc+YYSHLX#(L_QX*hAq
zd&+9?D~L1$G*9t=1>BfcJCjw#&N*{^%8=%VJ^0P`r}ZAg7rG14v1e*)@lvq^)n5vN
zF%HiNBLtPf91*9GDnU)9j&1`V25)Jya8A}if)OFf%;am_Re<RdULPSBo)P{OP8FdQ
zjvDdu1!E*>R}_I{AfEdRPibaDc0-D5pzDu$32%vCWP0%eBip-3b*cKYRb&^G#8gHU
zLS(2`hgM6r_tyQ^m+X=3$<}s@R=-*bW(rv#rR7H@c#yP;wER|Kmn@6UeVqxP^r^;%
z2D=8|MvR7F-K)l5)@O^n#S5i3>3LGlMZwBN%2-K-Nm)sWNvlZ*>a+_1`0^8w(Neim
zxB0nwxOpvhzjVH6g`JuGnKEnShVq8u75tUk75X#lm4OwFm9PdT>$_vCL+-=iLnqh$
zWAzjEqqX^?#f8P2>FBwMIfJ}*y}J&FW+Is>5hnTAVy&WPg^Pv_F+cfDO(F4|@@{4S
z_E()3M(DhNeBdg{5M*|XASmjCU#NHJ4w4!&B62x$ES~!(6G;?FJM#D$*8CtzuJ0S4
z^gh*pe;AC4K8dcz?`Fp{$FS?;DrH`_oOSp+nB;(Ok#5`8cO8D@$F?@UOy6slylhcl
zZ_$~LB>52gJErWDo6|g@%lz6h|BP?b1Lqaz)i*@BsE>#d1Kv?{IGxFo$-T)u$>j{O
zdbip~da%0im7024HQw@E1@(tykGzj~-Z_n}ZUv2f-sLwECqM$J46F>*3_s30dk*`w
zCY`2odp*0S(~48dl^;uIwe4orCQ}W#OVy2zjs8blhjIHg!`tCv&G`}JtF=qcu{#1(
zWomJA^2PYYC;1WiY56Wvj`)7dcy6~K9BwT>Banzehe0r7hJIBGhQ6P{Kov(7*E+QW
zeKS<EfJ4@~<T=ZE;kndWPE0L9B8$a9>MxdyPlib7{F~?AhhE}dN#5)}&*w*{BR3Vd
z9fHZcoW7?Wwb5od(JpvH#LZr73`~C25r$D6x*f0Aw||no7k%#Tmma=8+FiGfup1Au
zr7%xDVE9tqPhJTeKdnryDxG^wUvT+zz3IQLzpn2P4qXj762CF`P=YaQI3VP4M5%c9
zZpKn^ZfH~`l!wK}tHy>-_Lt<ptTa(<nNE&i24RkZe_|Sj-iM6C^Wa5>NCs;L8zRji
z%E7Oq-IA<P^7HvBc^U<4^`C|(#s?1{4r9l2E#ufRw#TgF?qPCJ9m}&*@X~mTT`RFn
zvdpILM;!K-M3hr*L^Z}@4-rK`JD;{_IN$9M?pO}`rwGf3E0rkTQ_PUpeyQb<5wiB4
zJuko=!AwY@*i>+qRVipFuqjrQsg(0e!%1&qtu&iWm2sgokmi!ZE#k6>V^w9dVSA(s
zQ!r#}wT)a%`b)!A#H+}V>HRHznSG|gY_74`rHFX+%hw|F^GV;ah@E+-N6Q*ZyUdv(
z<n5|G!EM+p%`5MF;Pa1XDIK5=U42_Ube)#@-N!%}jX*<_aHpV49mTJc$(bpw&aHed
zn^CjkPk)EK(!3N*1)Qty<-5$knY?00L>1`IvdlT|r>FVNT3J^j#Vh>a7)9R7ndJ;g
z{EaAKDPk<*)?2PqT91N#j1!5t8kre|gDXIb!nDCckTa3Bu4-yf2Ptqtv7P+YI=qJG
zqPM?1Ntdz6A!Pbm1)Git!hO~rr8dxc9O`QbnJ~vOzZxU@hW=+D8>HLX@Vrm<CZX5W
z-;MRn%lXgJzzJ@5`Ig4B@<>TVnQwb=`)*h@f0ypv!c(D+924V}VUrh+bF1e=G`0wx
zd*!@=PRHNvvWRkg%{eWtidVCVYmS5RK@EGI@>b2-CNqQaw!s_slay-zc9ELgPH)kh
zuIPONTu0y5h~}f?{jG@`J^Chf2d}Hf%M(34@+QuC?s;{4atFGl+p*jEvr4{HzGIsS
z8#>#!HJXj(S~};&G~S%Hw)Oq?lrxb#>f7tnth&3uT)CU<zuFfZ*DV{WK|)K_OK<nW
z`vC-S`KV2wO^M3he!Z?nhuxrRv6CQ6Ap1$dNb%y&5md8Fa5S<%aS^{xzO-jaRmN%(
zv+*~bw(S3@+qqD}$~hFG;aT>feVYH7e8k|Pq{Qp@BHx!5Wtl&f{UrGOG*?;i3-bVK
zl2G8nqw5H6A61Ic)5yx|%7*Vbe=#<U$=k@TW!I(jPu<1dKHX0B=Wcx0oB76R+f5@&
zqwZ;<)o*)}qtSX?#cQW61MLfbdah5)S%G}e9tS_8{0988o-|O$iRYhfue)#EJUu?2
zAf4Gg(Hs@Gqqd^vJkvit*lR5_%^Q^w=?Ija+@1=qU~afP{Mj8cQ2L_u!F$Yq;l*u7
zt+;se73Wz`Z?mAQ%5v87!r$oc+V#Vv(BR9=Eb->smc&L>KnixXywREWgzx$z<=I4j
zfR3Y#<Khd?<J7*#%*A=<l+WAiCOqoBsp76_$ou|?#7tUM0RZr(1OULn0Knrr1wI4-
z+?WA?V`Bh-Hx&TDb;@j0;eVeHH`kW2P*4ESzT=MoC?Ey^`W*q^FF=?z0Ons90C;zL
zA5aje|6zdB5a|D6y?-V#`Xu$=6LqZBwOzFp<ax~;?HNtJI+~g@dfGevBLMJw^1efR
zb5|1*PkTED7hX>RvVSRf-|;^*6B)_BB(AmsWZDYKBw~&ra}rKQW=3W*K?D*K5`NHE
z3tm-m$^WFkM*?J4uC7kJOiUgg9*iDrjE*2nCKetZ9wug1CRSF4cLjrsmxHT`Cxe3v
z`M*v6$4A`U#SCQa<ZA8cK=O~TiK(NTs{k3<KZX7~{=H6DYm5I~$-(76ZM`?h^v?_v
z3nMerf4$#R`Tt>gl|a_!@0I`Y7i8i8m-7GO{!@pa>7U~N$7257>A%?bRs|9Gng09Q
z1QGBIx8VQ)EOr@j5p_@CSyq($DQ%U+R9F7nTZ+qo9=L};7%^zpPxVu9+^>cjgVL{W
zPfto+imF?0in5ZRhYKN~_mU2YFzlfT!sp}|^%8)9e8A7ycm!-n7>$!QuWD`>@MY8F
zp(ATjC>eHQ)on|TYwaY?wse|w`8MkdHt$HNFVotB{ikuC={5&6H)Jl@!y4DZh!0A}
zrDtn~2tgy#BNN8mcCA5u*v0Yu(Kk6ibu;;eNU(XUNpB&}$+l>cuf1=hAC?xC^}r}2
znyCd&g1wJ=QR-lvrSGoIm+Hg4=fo2Rxc9@e-(vK|I<e-EAtqG`O4&&Z{Jh@SrOLV~
zuZ9xgjC_5#r|r8x)cI0P8L(xC1y&ZuU0ee~y;O6N$r@8jUO4Mr8k1v75Usa))4h(4
z(I5t(AKzs%w|Pr@ij?Kr9jqe;zb%6ZLLMX2)T!ZU+JxOy()||BI%iDf+zm>nrFUwB
zpr)iOYY-C)h^aJ@V5}rEZ*#AqmbuxoZ!N3q&l_iC>?noJ1SDgM%8-=Yeb-N<nW0?8
z8(k-vfvMpUMc?3JU=P7of3PgbYOl`8fh!C;f+j7(SH*K{UY4<dGc0;Gs?EfSib%ra
zV(UyJs3f%sWMBAcnU6>}16yQYNQHxNf+{2EN_3q;-uMjw1?-3WTEGi;54$fgown{U
z)ytAWl14ac_7dq+T;mk}pFA8${hFuOxFoqhthsKl7hFYgu=*i?TqXa<=}M<@IC5bs
zj<4%XRAwr3u+%}}e64Y)5el=$e&p_~I2;DFNh1AO`qCkavs*2pAk9|w!!Oh8MTpZf
zWSI*bc#+wR!aVA0!(o6I0i<7KDH&v%kkIH(l|bmZy8gA7x!s}BW1w8fuNyBSietK?
zGGbQ$V<p;aJe_t}NDylSfJI6x^Q=mep-@!nNB#i~D3oYF!d~uW$s&(In5y3=6@Qeq
zLH|Keo{%`1!9$;q5A&rIMnB_YUT!$wblx~N9d={`q4j>MY_O}4@0jKH3cYl4VY$(o
z+qZ?+Us!Y|Kwd<~7yJB<4;%_=sShM~Vx{_GmH~afuGp16YVB1oQ^|$ArY{Ky=0DV~
zOxHzyequs8Bh7DnnC9^sX!(D}*)L4g7vl4N9kWL(T@Qu{cK<pN|5S$;LWPY#!r}@@
z5{*zLx~4YAxO#*0&vki}11rjfZ**%21LOu_y-nRMRLUGVKLdHoA?c2`^Fi0hXZmOL
zSbSwlN~sT&!j|*4ndSa>@<xILRPw1K3Nyi;CS{L&-yb%qP?5wC_fw%Y22?yQIp-1(
zvQ(JHHLQL|bAFGnl;GbB`c<$3odrF>$-KwITsmzTy+RLA`sSmTUPhmCnS{U~v$1LP
z6d?OsVO#&_l~5~<Y(qA`ofmG&iS8;vtX*qoq{gY+g1~Rp59P;K-6wSOBU25)GUAxP
zu{y3!L$4N46FbU!#~yF0Y#D8cBtv7`W7tFw!NholbnFeiYHcSrb*S~;jKP&@5Zq-!
z=0MkJq_btn_!ipZ%p@cXRgrDQVm3zF;XpppSoBX<-7xQ(k#;U@W?A0#bo107N2yKc
zeO|SR3RNhjok^uD+avFlQ^?e19fC#+E&D{txBO0}LBEO%AC$@~^{+`GjPriXjZ&b6
zWW=OwXPdJX2I4{s*P9lcPirs@ax08Gk+WOHH}qwm*2HaAV`q{D+P>!}M35|gg1h}Q
zun}}aBGqZ)rS@YlekZIImr~K}TycNK!2rcnE{_sZ-G<(TO5FtG1FmP9Vp3iP-#N%7
z`w~)mnFf3@U{MHP+0OHvf*vh>lqJmAz}|FR1|hDH=BmsQh~<d=e&Tf02dZn4Aoq2V
zR$HlxAIb3DZ2%hj)bgDrq|sgsBi@5n$j7`17Y~*WfK{dNwv3tzePSk^iq;d8VFZ(x
zUGDKvxdQT|;0qq9|NJs_4s}+{%0x9lcWxn;rt6gi<5Dfw{Q#Qbn<=;mH&-1_jUwGk
zZ^OBlqd`Q>AnQc%c?}JiG4e-vwJJ$>D$m*(j0~NVzY~zm_+47G{VutN!(n20n<)bq
zU>}cbob1^oU%lM%%*G8wV1|5XJl!mlc%<hgw9;G0OQ6Uu{H8lYe3$*Uq>9E}ghgzw
zCV2hD+dIU<PVoW_5{52FeE8GB`;*c`J|5IHJ8b%2>vkOnP~+0!GlNGhxV;q@(vF_?
zT?)(zf1kxyeiiWT;P0ZVjR0O2>a|S3&%tV+6D8b=Rgp72@2YS&kFS>kD=EOnLnP@T
zbhRj?XD}bocc?3RE?__BV_nXV*mz^p0uj3nmtcdtae{`<b9{uyzBdh0Rak`e-X0lU
zX4juSorZ8&BmRTMeafO`UXRI$G?GH&;shi%$2J^8EsIt+rBDKI6Az!__(0Vk(qM~+
zTY-0x$bvIn5K%UieyM8gZsmy5Uaa+b9UK(Lp@+rk(;^PAi)vd`DPaWiZ*ng4)y1oT
zz-xQ?NH^`zOYz6p!sOcl4q3`})9$R|bkEmYP!N``Pq?{$7HCsc7Iq76K;>z^B%M&X
z=ycR(g8Y7x@ihcJo>T&`Xao-<JY#+olZe?ND)ve7OKZhxtfT|r(P=_`6q<YJS&F13
zxpR+)y~E&Z&o*m=IHAW^z{Gr<+`Yy((Y$iarM`<#X|*~seSZf<$-+Oo(>M)qU~wvX
zb`RSX5+2uJSq*v97o`9e7W)MAoS<5oVRVwdjf`*z8_ixFjIBX8xGcWxenTvt6A*59
zUynU+23#i}C^us_35Z`C*k?KZ*5LDQTw?M<anq#>Al{_4Vg(V&RpXnp-nW2Eg+QKo
z%1<ap5<19vtQ%x?K@sc9ehgjS_Epd}oAm>_W)q*hisZBltho%ca{Y*v(I%ubXvXqA
zUy_vbm-~=DcSg|#co!Ld*i6N0S5&K>HU6ECXSH^{6u6h$Tid}Q-hLY;gyhrSJ6MTr
zvp2h!w?fa_d|q7XPA<;`hma7^M$g{P|A!%6wA*b=!OfWaJOHLRj&=_A{f8(wPn7Mg
zrY)upW(A(@w8oSG_JAv6E_gGm8<_{aDPE+W`+PQ_i^hGU`Xq#jEhJYoeu57AR<;BL
zd*yG#;tToy)Ys9ZS^|k^%6IvqH#V!@tJCpmmW{P+wN%+kE4+K&L0UQba<|)txM_a&
zV~sh9V@w~?Qrh{7VA9j&4-KIa_Wh6770x*#m<2aD^euj3Mg=Ye{H@kP+HHqUJb2o@
zR_Ag>RtP(leaLZ;MV?{Na^2)jq|7^0#f3Mp)1;e+8U4#vXbNaX{pt^8sv0L7662N4
zt+yViZ+`w<=||W(fUHuxaU>~hEBk9u)b6g)eXg)=ZzJpg4v)>wn)|p!H2UH`Q%$PO
zD0V<>X==dUf-F&1HbfC&m?f)Tu}s*_?c5CtbU1Vrln5)QuK}T_3r~cD^`xV03mv3e
z%~`bfKp9ktA=+*b^$}iM!4ldh$lvCi@+q12|4tlf8zX~6g;k<n@=AlB*>RC&fa&#J
zUI8=1Kg#$E0vh)zni<L^J<R-Gv7!B{oB7bv0L>YTPN#Ur@MX=O1IAt;Gc}s}W?*4T
zTL(@_7sS|l2m#AyDbJ5dIZnvLQ#8QvM(X`L@CF(s%o+H@Qpgdd99BCxUqSZ09X3GD
z5tQv0nI+ws(L{Q7iku351*^vg)#uMIVcBUlMBVZM)@Qx0+Wx!~aR^X0op|n{RVJlo
zoST=FA(~PL9#$x{!E03xuMe>Up(U1!uEm14W*y!6QZVXF@E5m7Q>{I~A4RnhLHVan
zq6E!1US+?d*zlvZX}070fVaoVvTfFBQqa_!du7TkLOtxC_sSx)9S1oQF!_1D>W4dF
zYm^XaX^o%-@>N2rGX>;j;DX>xXR_K{b-kX?NKFmC6KC)xll*{dA98Q;uP|sEtCSF~
zOp;nS3HaOAF$I$bx7oq`qtFF^P!thY?utwxlma3k*zZmb3xeFcN?2;%ct@VfJ1mVz
zF5`%eU{LFFqKfAzHNX8rl`l%#))#<67%SW4rh6zht$o;QkyA{E^%_!!H5@5t`v}c)
zSM2S3p=(YCp?vNO3~|V*L)C_R`+8RV(gh4ql8D08S}qtWQ+mP4c|9BVWxi0s4DAL%
ze(a_$5}~BKm{J&YnH%c#2NMW1eLZQy6nBK5J)q9vea^1y;7bn2(7F<?@6X|x58b^|
zG2N);Hc8hiVIfnN)bi@X&hEn}ul*%yFmfg~i8+7}^vEdT&-U`gl)bQ72kQ8lJy9ey
zwc^U<nohQ5^a>e0k71F?>JA;NgqW7Ik8gcBd*`6UwN**x!nYz9)bDuv-K%sG;e0|T
z7W*)vr7e<>UcID^YAnZC89Sj%!IgVs4asgPaail<D4?Q<tnaE9oixE+sY+z^PCRD?
z2V7*~J0#xaJ2}MvCq(_XSl?&FTWg#ZP=3dCQMu3Sb&Saeh4!ubI;D2;N}1kXrnWP8
z4EKQm5)wK#BWJc#G6n<3mD1>|$$*d3Nv0uv`85V|oh{}nxS-z|uAJXK)oDm))la|G
z|5EF*4-gR@M7MVc8J{!Eg)LL&O^-1&@E5m`(aP2y!aps_oGIvfu5~=T+#%op9*NCc
z7vuG%ydUwPto#Z6?bKzhG?`z{y&9T^Pm9pSL{_34MQSOjUi632F^&hl!RNiUCN0-v
zS_I>*w4Ns$1g$KfA6xobou<`Xq#izlauewkcKKf@19X!h=6p&O739-2PXp=Bv7ZSq
zI8dT`RZ6I^MU0H;7{_{bYR2{(HanU7#3gO+>l)l6Mj<_3wDxm5ksE|X*i0A5wHMUU
zQ0(c^9|LIFv$GFuXgMxo_qRP<^8;Q`Of~SJv^36}TV=bvx8Vrs=1#;8@$t{s69;~e
z)4j@bPV8eVT_8p8>qIIFB^;qrCecHTVyk^T#f0fZ)|iu~3FUP7O0#F=#xMh80?HP`
zox&MuHy3JEZDa1xGtA7g8#(Obaro1S?W*aN!jX)`*0T&jRiYjzYRzfOHU^j$588~a
za{eIeX@IZoowjr{(Rqe`L}pBY<Of=C*MR-7e4BW01&<Scy^n_nc|Vk20*bgMOr$Rd
zIqMAT8dKhYk$v46gKPxW0Vr9w45s8U#4w6Nl@*y0pB>I2k9RcKiwcjd%rqtTqw3XJ
z-pw%hCejVhd5(vwi{?0!e*cLE4H)7QNk5Rh%F6CM0pFeDt|y5FT=1D<M&U=_US~aX
z35pDeIlx2jw;D1S9Y=>v$jIC3LjzgiJGf?*g8pn1tV4RZ^yIPAH%BEXo)qi_@!X)W
z1h4XEdloEa>!5SS|CB-I7#8!8#0}^OdRWc(mTDw+9%e1nNJ^(ox?_Bt;zUryW_62&
zb#uk&fPB4N1)tx(R3Yre%JZYp^r+^VbWdEkhvg;6{6vrrL6K`d=qC&qC7<nYts__d
z&EiS+bF9b#{+jhe#SwzCCXfGxJoq&mXbS_Cq%pb4hY9Jn?7<HJnE$xCn}j*Mz^joY
z5!TtmvdjZhlOW7v4GZoVvL)BPV8_`WzJ6~b!+PH%pE=SC-&xLM=K{_WZ2irEZ9@4~
zA7QOh^q&Y2r(f{gB&dbkcULyG!RJ#YbfU>qd+wF@c1SsWs5#euoVbV;V0CKE9dHvs
z06?ZF0ygD9u|k}NCi#s8^Z$o7$v3*Ecnhub&BmOwv62hn48NvG>=d-h`bDnOV3ct8
zqTO7jyuIu%Y6pWg%0`}=Z08@?JhAlA>K2g!Z)0`q`))g?a=4HQHSEfSD;<PI=mD3q
z&AKnR5*+h|#KjcW9UA|3Z90-XIlm0b2N->~c3+Ynio^?h3x&YdR>iztj;L<9inWA9
z4Ra<V8_-1hz0w`}n0l1KVZ8LgiVHFk5Jg(qW9T?}@@YZo`nd~Hph|l}>2drcC;VG_
z4V`0R*$ukKEbZs{c0%PaIG2w?4H|Lt(U{j(MZXXT__&Q_-P*U+jP6!J9MzF)T&iWm
z@hE3-SY#F0DZd}6l+|KhJ;fmSX{gFvugfPUhZDSg%2qF92O+&;dIV6F)a$`4>6=32
z>93tZXAY6n@axeS97Ru9oTdbvDQ2dS!^0<NqtfXESviJlze@w=g`*ZyWeLi{D}A1a
z=!8%&!WrZM8&4LM9t@8!;5Sfa!~jRw6A%`}lTDSkc<C+gDg@a*i=N20gsx1!&S%Ww
zSVAe;*)mj1YTNAW)cdint7$(;u!d4XGc8L4RSe@EA=_>~@)PdK!w1E!;`KGeW4Ezl
zUq{_fixS?6kM4+LYi<%%^cddjCxY8HU(@XpxOX#)^u<QRa8T{#vrRD*_z=pbP4OJ-
ziKy02D>vqV=&vtRWG8sA-Ol$~z}_F@kD>6-&?r42^!@TE2UvE-kxRVw>d2?4mS(7Q
zcr?t(&_Z%8V=4%ksf{8M<?nQ1nCg23uSBbV<fCd#4w-+Iy%8vMy}LJbe1Y#is&~#u
z$eNV+42FWbu9-8gzXLV49j%|6BvdO%(YjW4Rr}yVs^z-eMSyLZn~R-@(~A^F?GS)L
z`Nhcw?BPj5wSh9YYT?f90_xgrWRa0tc=7nD%>($@wrh)v1+l&HTYrkV*z02KlR=aX
z$ri^1p+lgqnPE}nq-fQ_L`6L&N75k((?7Ae=?P1ff!?W5VTI(_w8E%rTQ#l)qW|yH
hy#Fz|W`?|>rjrp8w(Qm~{c{v6BcUi>C2Ab_zW~(r0Y3l$

literal 0
HcmV?d00001

diff --git a/examples/custom_tools/pid_template.png b/examples/custom_tools/pid_template.png
new file mode 100644
index 0000000000000000000000000000000000000000..c736c6cb43dae2643c328e2ad5f691adbe7bfb8b
GIT binary patch
literal 4274
zcmZ`+cQ{;I*B>Q%iy9@2(GqQxVHifQSB)!5Mi*hCw-^(mL>nRq(Gw$x=v@qgo9Lo<
zAqde+MDO3oz2Cj}dEWQk=Q(Guv)1po>i%bA4D~gsDcC3g006bNmYNZMXU4z6WcUw2
zO@T!izY#ks>nQ^O6>*ejw%71DVx+37p|h%{s=JH3*Y6&-_70j3ZVp~ZdrdD}e@$Cg
zS6c~d2>`$>JkONWB5|C#dA1YqqCEdud4hvd2tWcP8N9(=7o`mhp|c|){H7AkRE*$|
z5fqHRXTcN#yAxxw)?PyUumC$^Cn8$hltD~awE1}Zu<2~e?{XHtJnh{FUw-|VD0oZb
zxf)1?0N`(jgMboVD%^%Y)vqV*Y`Nh?<cIB@@8&H&I=azG*m`WIaJ1;}Nd)Y^NYC&p
zHCVkoqC*9+vK$E53VrLU^9KMI2!I)$d`7YWw#2!*O<|lGMh7U}si#jBf|6a#rs@{t
zDk5u21v7cAUN_Re4i8OC>BxRu|9EI2@7P>r(6RU;C`&%U&J60pXzpeN*_g>bcaYbA
zM}Mju5Z^<t6Pily=^i2`#r(|%U?_(PPyw~xq1$D|-4>3r<Sy-h5s|ONLAG-<1Z0Hi
zAP7!lq<k?9lzuA14^_N(+nLBS8KvQpd5`6Q&y@KEK?Osr2%!RkRZ=dBOJ|qirF<V?
z4#KGjp|W!#?PKho?0Q;fVG_^yww9sUN|+3DcO7KtC39zoPEKi~g+-mWaGASRYjLMr
zb{?9oBpe-k=Tu#$Bc<Rhd{(W}BA$c1LZG$F3+eS{iqp<NSuB&<Q>L}y+EAxjO>?UP
zM(d@cho06J2huDIt)~@kq;3~Ir8)GdNk&~(@$>Z|qWEYJ0N3JE)*rps>;}kJlt{v`
zO5v3Rq>F!OL^9WXWG2A1mTPlIQY{wB<YJvu6?n6gJc;PFdPvRVO^Nv;+gF209SwB^
z9lIj+j`K2c2&wj)BP%qBsP8n`U~9qw%2mQB{!p4xE$(VD+5>J6V*=zPUEIi<<Tz9F
zqS_Dzr=Wgpa==TGOj{gnZMahX4aQH`C*$4g#QR!0J}$*|%n`__^6`apsJG}zDOgEK
z=k7mU{%sZ=QPUSz`+V=oVl;Khx7)Y)B4<G>*(i^|`vd9k4f*{HPry$^p})&$^WN2A
zm>7OI%9A&=s<M^m5Lm*esRuZ3f04LM5Ll)^jI(85-9#-UIVX+12#Q+EuMOlFtxNFF
z>E`_^;7{axy_8;*fs?M32(}T#U(Ha@U9VF|8uUJL^O%lke~RjR*p#SfDD{_?&~$ad
zhny}dOmQwUoD@;tf4s_j@#s`X7^JY{#0w8Hdo1m898_Z~W`87jp4ng8e_5%jQG7oc
zg-cVDD5<?448BtqDe>q`y#j2fTp(nxn3GcRsv>14b7xNV>;{yzSz?M6!Tp`QmO}2?
z%}?p&4~nS==1nCZQ8*iAf0(#)B=-X<2RfCMQ%SR({^KS#x0}@4W6_FrRnl1vl60o(
zoVp7=hD$^yIUmoOL%d$c36WiX63v?&l6T@wm+Xycf5Q!NKHAN_rg~qeD!Xmq8MoAn
zB`wOlShfVewI~dUq}I_LxFI5wra(1~-{dY2&7!fF2g&()QQTf!qcNJFBaNMmX(_@(
zjr!PP^G#6oxDmFP?z2%>kMXE-@r%TOq*;l(qoF*V_wQ0E+OrSFt!Pp4z@7}KQt@ff
zf3n@q<`8(!t)aY8wD?iPlQxsO2*H6}5T2%_<!oJxh8pi0-)Vaal%k_;OY3Gk9@L+^
z&@|B8QbCQ!?^1s6j#1WX%GDD5%-p#ujFS8|JZV?jAT*OWPhg<-9j+4?h<c5!SLD`k
zwFs|Pf=38h6JyD@ZAe-e%?J^w!Uknag%+;YRklB#JtOK8s~pDq5<0~)mb#Lg0lJm>
z%xK1fsck4HXx5bAGGxz#bIJvSXhnm2us5f}1q@kGgeNVOV!=Q%@@&va)NQ-)2_DfX
zfsbm031Yk?+G;}yy1eYyc_?y;&7+J#LJ8EJCb=}S!4U5J_}xyXMd=~T9<Z6m-H<8|
zOY-j8TJUq_v|E(18i|7iPSP+25sjXKycJkA$xMtiHh+N9Np31cLHSfX7-_4KDzy|L
zUO^{J)6-HZc3YaIsHL!+t(em!#AJhQEM(uV%o9`^H|z<$k)b7jZTXwz4(^CPfT)Sm
zy`_I{{D*!cZIC9G{xi8Ku_9rH7$#mXRsEKL@%4w~k>rlHqg$XxAU#Gy3>;w7l*`~9
zRfW-s%7{9R62O>7(PGYd>AR8^RVaXM2{OF01)0^at5bYJeO`{Jqt(Yihy-}o{32Cl
zs>N@mplyCG0c$=*5UtaS(}eR8vK9FQ5)Da4x{N!G)#VQ6iDL_jR^PE=(~8q_a7x~<
z9(-6b?|q!!UsGM}QXNo3UmbRDw`L5vIS$JoE!a<gqv`oJ%<!!tV^UtytE9xFFG<VB
zf}<g9y1m%$0-XZik>L^25mQKDx?HT@!r;<iA(Fbfs5*a&ZR%i3coR9*Hg$I@vKov$
zTyt8HSqWS5@L5_jUWcqskF1W5j_(h|4)+dQzG*-lHoCp%)b3XX>&E4qzJ0H^UHw%x
zP<PfuQ7x;e*)XW#!u(v6=uJot!53f$c4&_MX$(nV1Uh1Y>UU~N>LThm)<-vNHE1+k
zcoGIBa-M2HpMAZ7xKZ`&xIHFzJ+_Rk8N%v7@A45^AU5eZ<o2^Y$&Kwnx^w-<y{Oed
z$?4CN!hg6VPd=!sdhj8KO5-^0+w;O3z8)hty+@`e<p%?5kEM5|cb`z|#E?;9+R!n>
z%pa09lK)7SO)e6NLmZf`BCg#dFEK&9Do5)=bE{TBC$JM%bXE<{H@D^^x@cd0oq%04
zgE7M>BT)L#RmwH3*1WdJ72$HaQM|!F^>SjfqQSn*w!fNXqO8WfCTMkTC4Q;Ab3RJ-
zeGZ1_OT~m|+yY#n@b~y(-F&wE^&Cu2T8_7-J6qr+tM7pqvy7>nwU@GGqh*+AhDB)|
zy+xp9Td7njbmo?u@O#4daJN@m8e8I9d0U#(S<frj6U862rH+Yj->{;(E<d}4Uh!A+
zPeMZ;pKYyfbnO=(G%6&+qysh@D`M@lV!c^AxZe9ui+}^mFjg^*_Zlzu4tA2!<Bt!I
zCXVl)xa{G&AU5riDPsM{^Z^1#eY^0r)2aS123vju+t48BrN!mU#mvIZj@=G-u1m2N
z{$Pf3DJ)MsXDaKFuZ?EBFP*?U{*`>2FE&;lu0MFjoJ?q(!RxEU?Zm5LpBStnj>12a
z%d$p?YlNAESy2sB>X3h-I{;4e%gY5Apsd48TQ{N-6T&)II++rnlguun4bNv-zB5P(
ztm#7dV7Jk#dj{ft;zOxRn3dLdm?HkKF*R{a9h?{<&yBjfo`(xJ7aZGzQj~O~4BqJ<
z@eT4+@K#7^D<aWDTe&P<3@=jnX7xOOyOUd;`!HW$yF@22jXAwmqQrhERok22QVXiX
z@)r6aUcyN7q2!4`q@I-|&N+HK>E~_eTbRB`Ci+SGBxJDKez+#z`z=>D@BOz9TYUjN
zn1vCK6UTB#m(0Nq>iN>|3iH=?O?J^o1ZOYLG|dUjg{tbSh$>AT4#`4^?}l30MtMB_
zVXl9_FF7*>=ZTZ^o{bq&JN?<|pXRS;2lp&J(rt2hVtc`aiOIDX5+8P7N>2+Mazd6+
zCFs4B>ZYE{8j=oA{6?wnsBEL``$v&^K@|<t8gn$|m*~tGW)`?04fv}#dsgqO86!K(
zN^Gt-jdR}^u5+5z8?iLmCzLTRrD%6idM%yOi{;F$`<A8oNyo?P@LmUIhut2|C)an{
zUVGicRi7<^F4YlDt<8*2{5^Lj+SXZ`i{|d08FsxZE(~Z0YgmjdlW)3rIC`3At^*d`
zwW{@(^~9l$W0{nN9+iw(nm7KOFT@nFnGBnn7GK!+?nx~dwcm9$FT$Br)Y@Bqu5aIe
zw4PEH)Sz6R{Q<4A-xRyV&f*?`!@OTzTbk?LM+ny%yZP_d{8&dIcxt6bWJZi#dEA61
z4tfqoHcRAE<<=hdJ`{4koWA{avO>snJPnprUq7?dkg}<Kc<W$q<5lJ1PiXcmWUOJ-
zea5l6%u8{iY~u1rX(@!AG>5j9!;Z7?a_r(uY~((zDFjHZP93O8tO;WqW-o)NOVvP5
zp<EYz=MAqkOX5tpB;{*2>XvpY7q$%;vsM&u%T7KPJRSLzyei^nU;qm|*Zr6l<CxR`
z`c&cUbhxB=jA5Cu?<RcPuW6NZiB?k-W$om&`%vyIXFM(vjJ9^ETl6m2soeg)B(zY*
z(ah$vKT<Q`JZtS}-8{hg<;i!A?pOpge|n>?tzk3};d46qDpZchZ~0S9U|Zm;)4Q~v
zxkk>M_nHrUQGOijRGTiRw^#EUXmPYzXTqn)uBMaV5$i%ub9mwU!G^*V!&mR)oy87I
z16~6XbWhOex$na7`T4UK%x4J1Y;IGj<B;QakoC{$z2iQ`_Vd9ZuGz~u^{+7@DNJR$
z)|=?wfSD8i&EA|4b9Zg`@pIXe{w2S`?X3^}k1sD~$!U-5^cU^I@%JO3y_S(40D$HP
z0D{8+fD^nbcm)9P6$1d)YybdQDgeOZky(F79&b={Fw=gZrw0(k%VYpT0(t-uULwFh
zfJh{O_?HX-;633NA(rrO4KtSLZ~5v@27tGa4C5grNMkb}Gd*3Hy}PTZ?R|GU2T_!(
z#}xn|kAmSvR|g+kAj;Ln%^QY-gMJ}kc=<{V1_6ITe4ODRGd)9~s=Jp1P+C+>R1Bm*
z0R#f&z3xAN8L4UfMaTcaK~6qC9xyQ2&(BZPPg2y~%MmOtD=P~YlK@Ldh~N<--u`Yr
zwkQ!dZ=OG${J$SH2XA{Xq=ygE-3@r<*VfM6*9Q&)T?P7k{h6l&3i(eYH}Aj7!WRg>
zvVg@!#lZiK=7W6jKWJB$KWV?}`ZJvTl{1(j66N4xu7-4VaP!8erXUG{O3VL>^B<;v
zGW{3T>>nybR$TVq)PI@&MZL-eX5fW%z_;nD84BX^;Q!G5g_j3kb?RT8`*SS6wD`Fw
zP{@P-p0NT&qG~Mu6ajj))s&5IG36yFGq(cCR7X!|CkY6-r}|tX)K96h;-@M)E~O4R
z-uH9Zr&WC@of2&xL7TOf51!6femeDuBeVW6qfSxrV(giIxG?HK**Vpwj^1pB*kSVC
zSE^6FdaB)<d>siMPa1LDy6PfRP?Kj?tA!@RQd~31Oj|FPn+@-|MM&5gdv!-jeIQyF
z(^J(+N!nG*gd|ZAnY$4kYd7;2RFD03n2YOy(cpwV=CMrA#rU16GGcf3+AFfweGVez
iU?qK}*4)5z^b^2(KA+xOiBOlT<E*W&uU4vJ6Z(Ji*R;_9

literal 0
HcmV?d00001

diff --git a/examples/custom_tools/run_custom_tool.py b/examples/custom_tools/run_custom_tool.py
new file mode 100644
index 00000000..beaa9eca
--- /dev/null
+++ b/examples/custom_tools/run_custom_tool.py
@@ -0,0 +1,49 @@
+from template_match import template_matching_with_rotation
+
+import vision_agent as va
+from vision_agent.image_utils import get_image_size, normalize_bbox
+from vision_agent.tools import Tool, register_tool
+
+
+@register_tool
+class TemplateMatch(Tool):
+    name = "template_match_"
+    description = "'template_match_' takes a template image and finds all locations where that template appears in the input image."
+    usage = {
+        "required_parameters": [
+            {"name": "target_image", "type": "str"},
+            {"name": "template_image", "type": "str"},
+        ],
+        "examples": [
+            {
+                "scenario": "Can you detect the location of the template in the target image? Image name: target.png Reference image: template.png",
+                "parameters": {
+                    "target_image": "target.png",
+                    "template_image": "template.png",
+                },
+            },
+        ],
+    }
+
+    def __call__(self, target_image: str, template_image: str) -> dict:
+        image_size = get_image_size(target_image)
+        matches = template_matching_with_rotation(target_image, template_image)
+        matches["bboxes"] = [
+            normalize_bbox(box, image_size) for box in matches["bboxes"]
+        ]
+        return matches
+
+
+if __name__ == "__main__":
+    agent = va.agent.VisionAgent(verbose=True)
+    resp, tools = agent.chat_with_workflow(
+        [
+            {
+                "role": "user",
+                "content": "Can you find the locations of the pid_template.png in pid.png and tell me if any are nearby 'NOTE 5'?",
+            }
+        ],
+        image="pid.png",
+        reference_data={"image": "pid_template.png"},
+        visualize_output=True,
+    )
diff --git a/examples/custom_tools/template_match.py b/examples/custom_tools/template_match.py
new file mode 100644
index 00000000..1dd9fbe0
--- /dev/null
+++ b/examples/custom_tools/template_match.py
@@ -0,0 +1,96 @@
+import cv2
+import numpy as np
+import torch
+from torchvision.ops import nms
+
+
+def rotate_image(mat, angle):
+    """
+    Rotates an image (angle in degrees) and expands image to avoid cropping
+    """
+
+    height, width = mat.shape[:2]  # image shape has 3 dimensions
+    image_center = (
+        width / 2,
+        height / 2,
+    )  # getRotationMatrix2D needs coordinates in reverse order (width, height) compared to shape
+
+    rotation_mat = cv2.getRotationMatrix2D(image_center, angle, 1.0)
+
+    # rotation calculates the cos and sin, taking absolutes of those.
+    abs_cos = abs(rotation_mat[0, 0])
+    abs_sin = abs(rotation_mat[0, 1])
+
+    # find the new width and height bounds
+    bound_w = int(height * abs_sin + width * abs_cos)
+    bound_h = int(height * abs_cos + width * abs_sin)
+
+    # subtract old image center (bringing image back to origo) and adding the new image center coordinates
+    rotation_mat[0, 2] += bound_w / 2 - image_center[0]
+    rotation_mat[1, 2] += bound_h / 2 - image_center[1]
+
+    # rotate image with the new bounds and translated rotation matrix
+    rotated_mat = cv2.warpAffine(mat, rotation_mat, (bound_w, bound_h))
+    return rotated_mat
+
+
+def template_matching_with_rotation(
+    main_image_path: str,
+    template_path: str,
+    max_rotation: int = 360,
+    step: int = 90,
+    threshold: float = 0.75,
+    visualize: bool = False,
+) -> dict:
+    main_image = cv2.imread(main_image_path)
+    template = cv2.imread(template_path)
+    template_height, template_width = template.shape[:2]
+
+    # Convert images to grayscale
+    main_image_gray = cv2.cvtColor(main_image, cv2.COLOR_BGR2GRAY)
+    template_gray = cv2.cvtColor(template, cv2.COLOR_BGR2GRAY)
+
+    boxes = []
+    scores = []
+
+    for angle in range(0, max_rotation, step):
+        # Rotate the template
+        rotated_template = rotate_image(template_gray, angle)
+
+        # Perform template matching
+        result = cv2.matchTemplate(
+            main_image_gray,
+            rotated_template,
+            cv2.TM_CCOEFF_NORMED,
+        )
+
+        y_coords, x_coords = np.where(result >= threshold)
+        for x, y in zip(x_coords, y_coords):
+            boxes.append(
+                (x, y, x + rotated_template.shape[1], y + rotated_template.shape[0])
+            )
+            scores.append(result[y, x])
+
+    indices = (
+        nms(
+            torch.tensor(boxes).float(),
+            torch.tensor(scores).float(),
+            0.2,
+        )
+        .numpy()
+        .tolist()
+    )
+    boxes = [boxes[i] for i in indices]
+    scores = [scores[i] for i in indices]
+
+    if visualize:
+        # Draw a rectangle around the best match
+        for box in boxes:
+            cv2.rectangle(main_image, (box[0], box[1]), (box[2], box[3]), 255, 2)
+
+        # Display the result
+        cv2.imshow("Best Match", main_image)
+        cv2.waitKey(0)
+        cv2.destroyAllWindows()
+
+    return {"bboxes": boxes, "scores": scores}
diff --git a/tests/tools/test_tools.py b/tests/tools/test_tools.py
index 12c21347..6de8d6c8 100644
--- a/tests/tools/test_tools.py
+++ b/tests/tools/test_tools.py
@@ -2,8 +2,10 @@
 import tempfile
 
 import numpy as np
+import pytest
 from PIL import Image
 
+from vision_agent.tools import TOOLS, Tool, register_tool
 from vision_agent.tools.tools import BboxIoU, BoxDistance, SegArea, SegIoU
 
 
@@ -65,3 +67,71 @@ def test_box_distance():
     box1 = [0, 0, 2, 2]
     box2 = [1, 1, 3, 3]
     assert box_dist(box1, box2) == 0.0
+
+
+def test_register_tool():
+    assert TOOLS[len(TOOLS) - 1]["name"] != "test_tool_"
+
+    @register_tool
+    class TestTool(Tool):
+        name = "test_tool_"
+        description = "Test Tool"
+        usage = {
+            "required_parameters": [{"name": "prompt", "type": "str"}],
+            "examples": [
+                {
+                    "scenario": "Test",
+                    "parameters": {"prompt": "Test Prompt"},
+                }
+            ],
+        }
+
+        def __call__(self, prompt: str) -> str:
+            return prompt
+
+    assert TOOLS[len(TOOLS) - 1]["name"] == "test_tool_"
+
+
+def test_register_tool_incorrect():
+    with pytest.raises(ValueError):
+
+        @register_tool
+        class NoAttributes(Tool):
+            pass
+
+    with pytest.raises(ValueError):
+
+        @register_tool
+        class NoName(Tool):
+            description = "Test Tool"
+            usage = {
+                "required_parameters": [{"name": "prompt", "type": "str"}],
+                "examples": [
+                    {
+                        "scenario": "Test",
+                        "parameters": {"prompt": "Test Prompt"},
+                    }
+                ],
+            }
+
+    with pytest.raises(ValueError):
+
+        @register_tool
+        class NoDescription(Tool):
+            name = "test_tool_"
+            usage = {
+                "required_parameters": [{"name": "prompt", "type": "str"}],
+                "examples": [
+                    {
+                        "scenario": "Test",
+                        "parameters": {"prompt": "Test Prompt"},
+                    }
+                ],
+            }
+
+    with pytest.raises(ValueError):
+
+        @register_tool
+        class NoUsage(Tool):
+            name = "test_tool_"
+            description = "Test Tool"
diff --git a/vision_agent/agent/vision_agent.py b/vision_agent/agent/vision_agent.py
index 6854ce43..93218e6c 100644
--- a/vision_agent/agent/vision_agent.py
+++ b/vision_agent/agent/vision_agent.py
@@ -377,6 +377,7 @@ def visualize_result(all_tool_results: List[Dict]) -> Sequence[Union[str, Path]]
             "dinov_",
             "zero_shot_counting_",
             "visual_prompt_counting_",
+            "ocr_",
         ]:
             continue
 
@@ -523,20 +524,20 @@ def chat_with_workflow(
         if image:
             question += f" Image name: {image}"
         if reference_data:
-            if not (
-                "image" in reference_data
-                and ("mask" in reference_data or "bbox" in reference_data)
-            ):
-                raise ValueError(
-                    f"Reference data must contain 'image' and a visual prompt which can be 'mask' or 'bbox'. but got {reference_data}"
-                )
-            visual_prompt_data = (
-                f"Reference mask: {reference_data['mask']}"
+            question += (
+                f" Reference image: {reference_data['image']}"
+                if "image" in reference_data
+                else ""
+            )
+            question += (
+                f" Reference mask: {reference_data['mask']}"
                 if "mask" in reference_data
-                else f"Reference bbox: {reference_data['bbox']}"
+                else ""
             )
             question += (
-                f" Reference image: {reference_data['image']}, {visual_prompt_data}"
+                f" Reference bbox: {reference_data['bbox']}"
+                if "bbox" in reference_data
+                else ""
             )
 
         reflections = ""
diff --git a/vision_agent/tools/__init__.py b/vision_agent/tools/__init__.py
index 38bb08d4..67248156 100644
--- a/vision_agent/tools/__init__.py
+++ b/vision_agent/tools/__init__.py
@@ -1,6 +1,7 @@
 from .prompts import CHOOSE_PARAMS, SYSTEM_PROMPT
 from .tools import (  # Counter,
     CLIP,
+    OCR,
     TOOLS,
     BboxArea,
     BboxIoU,
@@ -11,9 +12,10 @@
     GroundingDINO,
     GroundingSAM,
     ImageCaption,
-    ZeroShotCounting,
-    VisualPromptCounting,
     SegArea,
     SegIoU,
     Tool,
+    VisualPromptCounting,
+    ZeroShotCounting,
+    register_tool,
 )
diff --git a/vision_agent/tools/tools.py b/vision_agent/tools/tools.py
index 6528c795..a661aeb0 100644
--- a/vision_agent/tools/tools.py
+++ b/vision_agent/tools/tools.py
@@ -1,8 +1,9 @@
+import io
 import logging
 import tempfile
 from abc import ABC
 from pathlib import Path
-from typing import Any, Dict, List, Tuple, Union, cast
+from typing import Any, Dict, List, Tuple, Type, Union, cast
 
 import numpy as np
 import requests
@@ -11,10 +12,10 @@
 
 from vision_agent.image_utils import (
     convert_to_b64,
+    denormalize_bbox,
     get_image_size,
-    rle_decode,
     normalize_bbox,
-    denormalize_bbox,
+    rle_decode,
 )
 from vision_agent.tools.video import extract_frames_from_video
 from vision_agent.type_defs import LandingaiAPIKey
@@ -29,6 +30,9 @@ class Tool(ABC):
     description: str
     usage: Dict
 
+    def __call__(self, *args: Any, **kwargs: Any) -> Any:
+        raise NotImplementedError
+
 
 class NoOp(Tool):
     name = "noop_"
@@ -858,6 +862,57 @@ def __call__(self, video_uri: str) -> List[Tuple[str, float]]:
         return result
 
 
+class OCR(Tool):
+    name = "ocr_"
+    description = "'ocr_' extracts text from an image."
+    usage = {
+        "required_parameters": [
+            {"name": "image", "type": "str"},
+        ],
+        "examples": [
+            {
+                "scenario": "Can you extract the text from this image? Image name: image.png",
+                "parameters": {"image": "image.png"},
+            },
+        ],
+    }
+    _API_KEY = "land_sk_WVYwP00xA3iXely2vuar6YUDZ3MJT9yLX6oW5noUkwICzYLiDV"
+    _URL = "https://app.landing.ai/ocr/v1/detect-text"
+
+    def __call__(self, image: str) -> dict:
+        pil_image = Image.open(image).convert("RGB")
+        image_size = pil_image.size[::-1]
+        image_buffer = io.BytesIO()
+        pil_image.save(image_buffer, format="PNG")
+        buffer_bytes = image_buffer.getvalue()
+        image_buffer.close()
+
+        res = requests.post(
+            self._URL,
+            files={"images": buffer_bytes},
+            data={"language": "en"},
+            headers={"contentType": "multipart/form-data", "apikey": self._API_KEY},
+        )
+        if res.status_code != 200:
+            _LOGGER.error(f"Request failed: {res.text}")
+            raise ValueError(f"Request failed: {res.text}")
+
+        data = res.json()
+        output: Dict[str, List] = {"labels": [], "bboxes": [], "scores": []}
+        for det in data[0]:
+            output["labels"].append(det["text"])
+            box = [
+                det["location"][0]["x"],
+                det["location"][0]["y"],
+                det["location"][2]["x"],
+                det["location"][2]["y"],
+            ]
+            box = normalize_bbox(box, image_size)
+            output["bboxes"].append(box)
+            output["scores"].append(round(det["score"], 2))
+        return output
+
+
 class Calculator(Tool):
     r"""Calculator is a tool that can perform basic arithmetic operations."""
 
@@ -903,6 +958,7 @@ def __call__(self, equation: str) -> float:
             SegIoU,
             BboxContains,
             BoxDistance,
+            OCR,
             Calculator,
         ]
     )
@@ -910,6 +966,31 @@ def __call__(self, equation: str) -> float:
 }
 
 
+def register_tool(tool: Type[Tool]) -> Type[Tool]:
+    r"""Add a tool to the list of available tools.
+
+    Parameters:
+        tool: The tool to add.
+    """
+
+    if (
+        not hasattr(tool, "name")
+        or not hasattr(tool, "description")
+        or not hasattr(tool, "usage")
+    ):
+        raise ValueError(
+            "The tool must have 'name', 'description' and 'usage' attributes."
+        )
+
+    TOOLS[len(TOOLS)] = {
+        "name": tool.name,
+        "description": tool.description,
+        "usage": tool.usage,
+        "class": tool,
+    }
+    return tool
+
+
 def _send_inference_request(
     payload: Dict[str, Any], endpoint_name: str
 ) -> Dict[str, Any]: