From 9f53e010f2eda8bd29c710de78c1ea25eae8c27f Mon Sep 17 00:00:00 2001 From: Asia <2736300+humpydonkey@users.noreply.github.com> Date: Tue, 30 Jul 2024 14:39:12 +0800 Subject: [PATCH] Fix e2b websocket issue (#177) * Fix e2b Websocket issue Remove health/liveness checking code * Bump pydantic to 2.7.4 to include a bug fix; Rewrite save_video() using cv2 (get rid of moviepy); Add retry for common e2b exceptions * Update template name --- poetry.lock | 147 ++++----------------------------- pyproject.toml | 3 +- tests/unit/tools/test_tools.py | 31 +++++++ vision_agent/tools/tools.py | 25 +++--- vision_agent/utils/execute.py | 107 ++++++++++++++++-------- vision_agent/utils/sim.py | 1 + 6 files changed, 137 insertions(+), 177 deletions(-) create mode 100644 tests/unit/tools/test_tools.py diff --git a/poetry.lock b/poetry.lock index 705ae1ac..3a49cd85 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,16 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. - -[[package]] -name = "aenum" -version = "3.1.15" -description = "Advanced Enumerations (compatible with Python's stdlib Enum), NamedTuples, and NamedConstants" -optional = false -python-versions = "*" -files = [ - {file = "aenum-3.1.15-py2-none-any.whl", hash = "sha256:27b1710b9d084de6e2e695dab78fe9f269de924b51ae2850170ee7e1ca6288a5"}, - {file = "aenum-3.1.15-py3-none-any.whl", hash = "sha256:e0dfaeea4c2bd362144b87377e2c61d91958c5ed0b4daf89cb6f45ae23af6288"}, - {file = "aenum-3.1.15.tar.gz", hash = "sha256:8cbd76cd18c4f870ff39b24284d3ea028fbe8731a58df3aa581e434c575b9559"}, -] +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "annotated-types" @@ -478,42 +466,38 @@ files = [ [[package]] name = "e2b" -version = "0.17.2a23" +version = "0.17.2a34" description = "E2B SDK that give agents cloud environments" optional = false python-versions = "<4.0,>=3.8" files = [ - {file = "e2b-0.17.2a23-py3-none-any.whl", hash = "sha256:12e65f4f2c18c750c076f7bd22c24ed0b13ad0b59ecc9b99655990e7aeb7c93d"}, - {file = "e2b-0.17.2a23.tar.gz", hash = "sha256:948542564255d913da97a754765ea3ec1bf1482a897599ace9421e6857910cdd"}, + {file = "e2b-0.17.2a34-py3-none-any.whl", hash = "sha256:604e5ed6a0af2ee970b98b3806f2ff09b5f46226c70a0334c364d016f8de7201"}, + {file = "e2b-0.17.2a34.tar.gz", hash = "sha256:8c310878ef989682da479d894c83470f78f42712b9fcbad7445e256e8e09700e"}, ] [package.dependencies] -aenum = ">=3.1.11" +attrs = ">=23.2.0,<24.0.0" httpcore = ">=1.0.5,<2.0.0" httpx = ">=0.27.0,<0.28.0" -protobuf = ">=5.26.1,<6.0.0" -pydantic = "*" +packaging = ">=24.1,<25.0" +protobuf = ">=3.20.0,<6.0.0" python-dateutil = ">=2.8.2" -typing-extensions = ">=4.8.0" -urllib3 = ">=1.25.3" [[package]] name = "e2b-code-interpreter" -version = "0.0.11a2" +version = "0.0.11a17" description = "E2B Code Interpreter - Stateful code execution" optional = false python-versions = "<4.0,>=3.8" files = [ - {file = "e2b_code_interpreter-0.0.11a2-py3-none-any.whl", hash = "sha256:bc3ae199acfb8e4c6675c4fc57a1070bb767a1e9bf99ac6d3aa09347fdc29997"}, - {file = "e2b_code_interpreter-0.0.11a2.tar.gz", hash = "sha256:6280f553be373865c43b2bfb546a3953d3faa43f9ce4ab7ed2dba576db548301"}, + {file = "e2b_code_interpreter-0.0.11a17-py3-none-any.whl", hash = "sha256:b7bfcc32250b065c545d6fac0c117572cd30945bf264f3698a4cf0ff26ebbf72"}, + {file = "e2b_code_interpreter-0.0.11a17.tar.gz", hash = "sha256:5ccdd3ad49b9769730be4281f57d75335566ea44cbfdead02b1ee3a83c110dd1"}, ] [package.dependencies] -e2b = "0.17.2a23" -pydantic = "*" -requests = ">=2.32.3,<3.0.0" -websocket-client = ">=1.7.0,<2.0.0" -websockets = ">=12.0,<13.0" +attrs = ">=21.3.0" +e2b = "0.17.2a34" +httpx = ">=0.20.0,<0.28.0" [[package]] name = "exceptiongroup" @@ -2195,13 +2179,13 @@ files = [ [[package]] name = "pydantic" -version = "2.7.3" +version = "2.7.4" description = "Data validation using Python type hints" optional = false python-versions = ">=3.8" files = [ - {file = "pydantic-2.7.3-py3-none-any.whl", hash = "sha256:ea91b002777bf643bb20dd717c028ec43216b24a6001a280f83877fd2655d0b4"}, - {file = "pydantic-2.7.3.tar.gz", hash = "sha256:c46c76a40bb1296728d7a8b99aa73dd70a48c3510111ff290034f860c99c419e"}, + {file = "pydantic-2.7.4-py3-none-any.whl", hash = "sha256:ee8538d41ccb9c0a9ad3e0e5f07bf15ed8015b481ced539a1759d8cc89ae90d0"}, + {file = "pydantic-2.7.4.tar.gz", hash = "sha256:0c84efd9548d545f63ac0060c1e4d39bb9b14db8b3c0652338aecc07b5adec52"}, ] [package.dependencies] @@ -3432,103 +3416,6 @@ files = [ {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, ] -[[package]] -name = "websocket-client" -version = "1.8.0" -description = "WebSocket client for Python with low level API options" -optional = false -python-versions = ">=3.8" -files = [ - {file = "websocket_client-1.8.0-py3-none-any.whl", hash = "sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526"}, - {file = "websocket_client-1.8.0.tar.gz", hash = "sha256:3239df9f44da632f96012472805d40a23281a991027ce11d2f45a6f24ac4c3da"}, -] - -[package.extras] -docs = ["Sphinx (>=6.0)", "myst-parser (>=2.0.0)", "sphinx-rtd-theme (>=1.1.0)"] -optional = ["python-socks", "wsaccel"] -test = ["websockets"] - -[[package]] -name = "websockets" -version = "12.0" -description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" -optional = false -python-versions = ">=3.8" -files = [ - {file = "websockets-12.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d554236b2a2006e0ce16315c16eaa0d628dab009c33b63ea03f41c6107958374"}, - {file = "websockets-12.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2d225bb6886591b1746b17c0573e29804619c8f755b5598d875bb4235ea639be"}, - {file = "websockets-12.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:eb809e816916a3b210bed3c82fb88eaf16e8afcf9c115ebb2bacede1797d2547"}, - {file = "websockets-12.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c588f6abc13f78a67044c6b1273a99e1cf31038ad51815b3b016ce699f0d75c2"}, - {file = "websockets-12.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5aa9348186d79a5f232115ed3fa9020eab66d6c3437d72f9d2c8ac0c6858c558"}, - {file = "websockets-12.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6350b14a40c95ddd53e775dbdbbbc59b124a5c8ecd6fbb09c2e52029f7a9f480"}, - {file = "websockets-12.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:70ec754cc2a769bcd218ed8d7209055667b30860ffecb8633a834dde27d6307c"}, - {file = "websockets-12.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:6e96f5ed1b83a8ddb07909b45bd94833b0710f738115751cdaa9da1fb0cb66e8"}, - {file = "websockets-12.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4d87be612cbef86f994178d5186add3d94e9f31cc3cb499a0482b866ec477603"}, - {file = "websockets-12.0-cp310-cp310-win32.whl", hash = "sha256:befe90632d66caaf72e8b2ed4d7f02b348913813c8b0a32fae1cc5fe3730902f"}, - {file = "websockets-12.0-cp310-cp310-win_amd64.whl", hash = "sha256:363f57ca8bc8576195d0540c648aa58ac18cf85b76ad5202b9f976918f4219cf"}, - {file = "websockets-12.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:5d873c7de42dea355d73f170be0f23788cf3fa9f7bed718fd2830eefedce01b4"}, - {file = "websockets-12.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3f61726cae9f65b872502ff3c1496abc93ffbe31b278455c418492016e2afc8f"}, - {file = "websockets-12.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ed2fcf7a07334c77fc8a230755c2209223a7cc44fc27597729b8ef5425aa61a3"}, - {file = "websockets-12.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e332c210b14b57904869ca9f9bf4ca32f5427a03eeb625da9b616c85a3a506c"}, - {file = "websockets-12.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5693ef74233122f8ebab026817b1b37fe25c411ecfca084b29bc7d6efc548f45"}, - {file = "websockets-12.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e9e7db18b4539a29cc5ad8c8b252738a30e2b13f033c2d6e9d0549b45841c04"}, - {file = "websockets-12.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6e2df67b8014767d0f785baa98393725739287684b9f8d8a1001eb2839031447"}, - {file = "websockets-12.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:bea88d71630c5900690fcb03161ab18f8f244805c59e2e0dc4ffadae0a7ee0ca"}, - {file = "websockets-12.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:dff6cdf35e31d1315790149fee351f9e52978130cef6c87c4b6c9b3baf78bc53"}, - {file = "websockets-12.0-cp311-cp311-win32.whl", hash = "sha256:3e3aa8c468af01d70332a382350ee95f6986db479ce7af14d5e81ec52aa2b402"}, - {file = "websockets-12.0-cp311-cp311-win_amd64.whl", hash = "sha256:25eb766c8ad27da0f79420b2af4b85d29914ba0edf69f547cc4f06ca6f1d403b"}, - {file = "websockets-12.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0e6e2711d5a8e6e482cacb927a49a3d432345dfe7dea8ace7b5790df5932e4df"}, - {file = "websockets-12.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:dbcf72a37f0b3316e993e13ecf32f10c0e1259c28ffd0a85cee26e8549595fbc"}, - {file = "websockets-12.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:12743ab88ab2af1d17dd4acb4645677cb7063ef4db93abffbf164218a5d54c6b"}, - {file = "websockets-12.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b645f491f3c48d3f8a00d1fce07445fab7347fec54a3e65f0725d730d5b99cb"}, - {file = "websockets-12.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9893d1aa45a7f8b3bc4510f6ccf8db8c3b62120917af15e3de247f0780294b92"}, - {file = "websockets-12.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f38a7b376117ef7aff996e737583172bdf535932c9ca021746573bce40165ed"}, - {file = "websockets-12.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:f764ba54e33daf20e167915edc443b6f88956f37fb606449b4a5b10ba42235a5"}, - {file = "websockets-12.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:1e4b3f8ea6a9cfa8be8484c9221ec0257508e3a1ec43c36acdefb2a9c3b00aa2"}, - {file = "websockets-12.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:9fdf06fd06c32205a07e47328ab49c40fc1407cdec801d698a7c41167ea45113"}, - {file = "websockets-12.0-cp312-cp312-win32.whl", hash = "sha256:baa386875b70cbd81798fa9f71be689c1bf484f65fd6fb08d051a0ee4e79924d"}, - {file = "websockets-12.0-cp312-cp312-win_amd64.whl", hash = "sha256:ae0a5da8f35a5be197f328d4727dbcfafa53d1824fac3d96cdd3a642fe09394f"}, - {file = "websockets-12.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:5f6ffe2c6598f7f7207eef9a1228b6f5c818f9f4d53ee920aacd35cec8110438"}, - {file = "websockets-12.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9edf3fc590cc2ec20dc9d7a45108b5bbaf21c0d89f9fd3fd1685e223771dc0b2"}, - {file = "websockets-12.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:8572132c7be52632201a35f5e08348137f658e5ffd21f51f94572ca6c05ea81d"}, - {file = "websockets-12.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:604428d1b87edbf02b233e2c207d7d528460fa978f9e391bd8aaf9c8311de137"}, - {file = "websockets-12.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1a9d160fd080c6285e202327aba140fc9a0d910b09e423afff4ae5cbbf1c7205"}, - {file = "websockets-12.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87b4aafed34653e465eb77b7c93ef058516cb5acf3eb21e42f33928616172def"}, - {file = "websockets-12.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b2ee7288b85959797970114deae81ab41b731f19ebcd3bd499ae9ca0e3f1d2c8"}, - {file = "websockets-12.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7fa3d25e81bfe6a89718e9791128398a50dec6d57faf23770787ff441d851967"}, - {file = "websockets-12.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:a571f035a47212288e3b3519944f6bf4ac7bc7553243e41eac50dd48552b6df7"}, - {file = "websockets-12.0-cp38-cp38-win32.whl", hash = "sha256:3c6cc1360c10c17463aadd29dd3af332d4a1adaa8796f6b0e9f9df1fdb0bad62"}, - {file = "websockets-12.0-cp38-cp38-win_amd64.whl", hash = "sha256:1bf386089178ea69d720f8db6199a0504a406209a0fc23e603b27b300fdd6892"}, - {file = "websockets-12.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ab3d732ad50a4fbd04a4490ef08acd0517b6ae6b77eb967251f4c263011a990d"}, - {file = "websockets-12.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a1d9697f3337a89691e3bd8dc56dea45a6f6d975f92e7d5f773bc715c15dde28"}, - {file = "websockets-12.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1df2fbd2c8a98d38a66f5238484405b8d1d16f929bb7a33ed73e4801222a6f53"}, - {file = "websockets-12.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:23509452b3bc38e3a057382c2e941d5ac2e01e251acce7adc74011d7d8de434c"}, - {file = "websockets-12.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e5fc14ec6ea568200ea4ef46545073da81900a2b67b3e666f04adf53ad452ec"}, - {file = "websockets-12.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:46e71dbbd12850224243f5d2aeec90f0aaa0f2dde5aeeb8fc8df21e04d99eff9"}, - {file = "websockets-12.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b81f90dcc6c85a9b7f29873beb56c94c85d6f0dac2ea8b60d995bd18bf3e2aae"}, - {file = "websockets-12.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:a02413bc474feda2849c59ed2dfb2cddb4cd3d2f03a2fedec51d6e959d9b608b"}, - {file = "websockets-12.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bbe6013f9f791944ed31ca08b077e26249309639313fff132bfbf3ba105673b9"}, - {file = "websockets-12.0-cp39-cp39-win32.whl", hash = "sha256:cbe83a6bbdf207ff0541de01e11904827540aa069293696dd528a6640bd6a5f6"}, - {file = "websockets-12.0-cp39-cp39-win_amd64.whl", hash = "sha256:fc4e7fa5414512b481a2483775a8e8be7803a35b30ca805afa4998a84f9fd9e8"}, - {file = "websockets-12.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:248d8e2446e13c1d4326e0a6a4e9629cb13a11195051a73acf414812700badbd"}, - {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f44069528d45a933997a6fef143030d8ca8042f0dfaad753e2906398290e2870"}, - {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c4e37d36f0d19f0a4413d3e18c0d03d0c268ada2061868c1e6f5ab1a6d575077"}, - {file = "websockets-12.0-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3d829f975fc2e527a3ef2f9c8f25e553eb7bc779c6665e8e1d52aa22800bb38b"}, - {file = "websockets-12.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:2c71bd45a777433dd9113847af751aae36e448bc6b8c361a566cb043eda6ec30"}, - {file = "websockets-12.0-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:0bee75f400895aef54157b36ed6d3b308fcab62e5260703add87f44cee9c82a6"}, - {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:423fc1ed29f7512fceb727e2d2aecb952c46aa34895e9ed96071821309951123"}, - {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:27a5e9964ef509016759f2ef3f2c1e13f403725a5e6a1775555994966a66e931"}, - {file = "websockets-12.0-pp38-pypy38_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3181df4583c4d3994d31fb235dc681d2aaad744fbdbf94c4802485ececdecf2"}, - {file = "websockets-12.0-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:b067cb952ce8bf40115f6c19f478dc71c5e719b7fbaa511359795dfd9d1a6468"}, - {file = "websockets-12.0-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:00700340c6c7ab788f176d118775202aadea7602c5cc6be6ae127761c16d6b0b"}, - {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e469d01137942849cff40517c97a30a93ae79917752b34029f0ec72df6b46399"}, - {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffefa1374cd508d633646d51a8e9277763a9b78ae71324183693959cf94635a7"}, - {file = "websockets-12.0-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba0cab91b3956dfa9f512147860783a1829a8d905ee218a9837c18f683239611"}, - {file = "websockets-12.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2cb388a5bfb56df4d9a406783b7f9dbefb888c09b71629351cc6b036e9259370"}, - {file = "websockets-12.0-py3-none-any.whl", hash = "sha256:dc284bbc8d7c78a6c69e0c7325ab46ee5e40bb4d50e494d8131a07ef47500e9e"}, - {file = "websockets-12.0.tar.gz", hash = "sha256:81df9cbcbb6c260de1e007e58c011bfebe2dafc8435107b0537f393dd38c8b1b"}, -] - [[package]] name = "zipp" version = "3.19.2" @@ -3547,4 +3434,4 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", [metadata] lock-version = "2.0" python-versions = ">=3.9,<4.0" -content-hash = "67a7b88fd185a85e3351ef7ebe34ba7061a4eb95480728d3f7ab259c36813a00" +content-hash = "65ca2851faa9365881e4e9da33e3ad538b87d9f9be74aed47d447df0f660cb5d" diff --git a/pyproject.toml b/pyproject.toml index 1a1d350b..c3acb5a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,11 +35,12 @@ rich = "^13.7.1" langsmith = "^0.1.58" ipykernel = "^6.29.4" e2b = "^0.17.1" -e2b-code-interpreter = "0.0.11a2" +e2b-code-interpreter = "0.0.11a17" tenacity = "^8.3.0" pillow-heif = "^0.16.0" pytube = "15.0.0" anthropic = "^0.31.0" +pydantic = "2.7.4" [tool.poetry.group.dev.dependencies] autoflake = "1.*" diff --git a/tests/unit/tools/test_tools.py b/tests/unit/tools/test_tools.py new file mode 100644 index 00000000..4cc1e9af --- /dev/null +++ b/tests/unit/tools/test_tools.py @@ -0,0 +1,31 @@ +# Generated by CodiumAI +from pathlib import Path + +import numpy as np + +from vision_agent.tools.tools import save_video + + +class TestSaveVideo: + def test_saves_frames_without_output_path(self): + frames = [ + np.random.randint(0, 256, (480, 640, 3), dtype=np.uint8) for _ in range(10) + ] + output_path = save_video(frames) + assert Path(output_path).exists() + + def test_saves_frames_with_output_path(self, tmp_path): + frames = [ + np.random.randint(0, 256, (480, 640, 3), dtype=np.uint8) for _ in range(10) + ] + video_output_path = str(tmp_path / "output.mp4") + output_path = save_video(frames, video_output_path) + + assert output_path == video_output_path + assert Path(output_path).exists() + + # Handles an empty list of frames gracefully + def test_handles_empty_frames_list(self): + frames = [] + output_path = save_video(frames) + assert Path(output_path).exists() diff --git a/vision_agent/tools/tools.py b/vision_agent/tools/tools.py index 70dac144..27a2f10a 100644 --- a/vision_agent/tools/tools.py +++ b/vision_agent/tools/tools.py @@ -9,7 +9,6 @@ import cv2 import numpy as np import requests -from moviepy.editor import ImageSequenceClip from PIL import Image, ImageDraw, ImageFont from pillow_heif import register_heif_opener # type: ignore from pytube import YouTube # type: ignore @@ -1044,15 +1043,21 @@ def save_video( if fps <= 0: _LOGGER.warning(f"Invalid fps value: {fps}. Setting fps to 4 (default value).") fps = 4 - with ImageSequenceClip(frames, fps=fps) as video: - if output_video_path: - f = open(output_video_path, "wb") - else: - f = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) # type: ignore - video.write_videofile(f.name, codec="libx264") - f.close() - _save_video_to_result(f.name) - return f.name + + if not output_video_path: + output_video_path = tempfile.NamedTemporaryFile( + suffix=".mp4", delete=False + ).name + + height, width, layers = frames[0].shape if frames else (0, 0, 0) + fourcc = cv2.VideoWriter_fourcc(*"mp4v") # type: ignore + video = cv2.VideoWriter(output_video_path, fourcc, fps, (width, height)) + for frame in frames: + video.write(cv2.cvtColor(frame, cv2.COLOR_RGB2BGR)) + video.release() + + _save_video_to_result(output_video_path) + return output_video_path def _save_video_to_result(video_uri: str) -> None: diff --git a/vision_agent/utils/execute.py b/vision_agent/utils/execute.py index 2ba8b69d..0c058fe4 100644 --- a/vision_agent/utils/execute.py +++ b/vision_agent/utils/execute.py @@ -1,6 +1,5 @@ import abc import base64 -import copy import logging import os import platform @@ -17,9 +16,14 @@ import nbformat import tenacity from dotenv import load_dotenv +from e2b.exceptions import SandboxException from e2b_code_interpreter import CodeInterpreter as E2BCodeInterpreterImpl from e2b_code_interpreter import Execution as E2BExecution from e2b_code_interpreter import Result as E2BResult +from h11._util import LocalProtocolError +from httpx import ConnectError +from httpx import RemoteProtocolError as HttpcoreRemoteProtocolError +from httpx import RemoteProtocolError as HttpxRemoteProtocolError from nbclient import NotebookClient from nbclient import __version__ as nbclient_version from nbclient.exceptions import CellTimeoutError, DeadKernelError @@ -29,7 +33,6 @@ from typing_extensions import Self from vision_agent.utils.exceptions import ( - RemoteSandboxClosedError, RemoteSandboxCreationError, RemoteSandboxExecutionError, ) @@ -106,13 +109,8 @@ class Result: is_main_result: bool "Whether this data is the result of the cell. Data can be produced by display calls of which can be multiple in a cell." - raw: Dict[str, str] - "Dictionary that maps MIME types to their corresponding string representations of the data." - def __init__(self, is_main_result: bool, data: Dict[str, Any]): self.is_main_result = is_main_result - self.raw = copy.deepcopy(data) - self.text = data.pop(MimeType.TEXT_PLAIN, None) if self.text and (self.text.startswith("'") and self.text.endswith("'")): # This is a workaround for the issue that str result is wrapped with single quotes by notebook. @@ -136,13 +134,13 @@ def __init__(self, is_main_result: bool, data: Dict[str, Any]): # Allows to iterate over formats() def __getitem__(self, key: Any) -> Any: - return self.raw[key] if key in self.raw else getattr(self, key) + return getattr(self, key) def __str__(self) -> str: return repr(self) def __repr__(self) -> str: - return str(self.raw) + return str(self.text) def _repr_html_(self) -> Optional[str]: """Returns the HTML representation of the data.""" @@ -215,9 +213,16 @@ def from_e2b_result(result: E2BResult) -> "Result": # type: ignore """ Creates a Result object from an E2BResult object. """ + data = { + MimeType.TEXT_PLAIN.value: result.text, + MimeType.IMAGE_PNG.value: result.png, + MimeType.APPLICATION_JSON.value: result.json, + } + for k, v in result.extra.items(): + data[k] = v return Result( is_main_result=result.is_main_result, - data=result.raw, + data=data, ) @@ -367,7 +372,7 @@ def from_e2b_execution(exec: E2BExecution) -> "Execution": # type: ignore value=_remove_escape_and_color_codes(exec.error.value), traceback_raw=[ _remove_escape_and_color_codes(line) - for line in exec.error.traceback_raw + for line in exec.error.traceback.split("\n") ], ) if exec.error @@ -436,11 +441,12 @@ def __init__(self, *args: Any, **kwargs: Any) -> None: print(f"Vision Agent version: {va_version}")""" ) sys_versions = "\n".join(result.logs.stdout) - _LOGGER.info(f"E2BCodeInterpreter initialized:\n{sys_versions}") + _LOGGER.info( + f"E2BCodeInterpreter (sandbox id: {self.interpreter.sandbox_id}) initialized:\n{sys_versions}" + ) def close(self, *args: Any, **kwargs: Any) -> None: try: - self.interpreter.notebook.close() self.interpreter.kill(request_timeout=2) _LOGGER.info( f"The sandbox {self.interpreter.sandbox_id} is closed successfully." @@ -451,28 +457,67 @@ def close(self, *args: Any, **kwargs: Any) -> None: ) def restart_kernel(self) -> None: - self._check_sandbox_liveness() self.interpreter.notebook.restart_kernel() @tenacity.retry( wait=tenacity.wait_exponential_jitter(), - stop=tenacity.stop_after_attempt(2), - # TODO: change TimeoutError to a more specific exception when e2b team provides more granular retryable exceptions - retry=tenacity.retry_if_exception_type(TimeoutError), + stop=tenacity.stop_after_attempt(3), + retry=tenacity.retry_if_exception_type( + ( + LocalProtocolError, + HttpxRemoteProtocolError, + HttpcoreRemoteProtocolError, + ConnectError, + SandboxException, + ) + ), + before_sleep=tenacity.before_sleep_log(_LOGGER, logging.INFO), + after=tenacity.after_log(_LOGGER, logging.INFO), ) def exec_cell(self, code: str) -> Execution: - self._check_sandbox_liveness() self.interpreter.set_timeout(_SESSION_TIMEOUT) # Extend the life of the sandbox try: - execution = self.interpreter.notebook.exec_cell(code, timeout=self.timeout) + _LOGGER.info( + f"Start code execution in remote sandbox {self.interpreter.sandbox_id}. Timeout: {_SESSION_TIMEOUT}. Code hash: {hash(code)}" + ) + execution = self.interpreter.notebook.exec_cell( + code=code, + on_stdout=lambda msg: _LOGGER.info(msg), + on_stderr=lambda msg: _LOGGER.info(msg), + ) + _LOGGER.info( + f"Finished code execution in remote sandbox {self.interpreter.sandbox_id}. Code hash: {hash(code)}" + ) return Execution.from_e2b_execution(execution) + except ( + LocalProtocolError, + HttpxRemoteProtocolError, + HttpcoreRemoteProtocolError, + ConnectError, + SandboxException, + ) as e: + raise e except Exception as e: raise RemoteSandboxExecutionError( - f"Failed executing code in remote sandbox due to {e}: {code}" + f"Failed executing code in remote sandbox ({self.interpreter.sandbox_id}) due to error '{type(e).__name__} {str(e)}', code: {code}" ) from e + @tenacity.retry( + wait=tenacity.wait_exponential_jitter(), + stop=tenacity.stop_after_attempt(3), + retry=tenacity.retry_if_exception_type( + ( + LocalProtocolError, + HttpxRemoteProtocolError, + HttpcoreRemoteProtocolError, + ConnectError, + SandboxException, + ) + ), + before_sleep=tenacity.before_sleep_log(_LOGGER, logging.INFO), + after=tenacity.after_log(_LOGGER, logging.INFO), + ) def upload_file(self, file: Union[str, Path]) -> str: - self._check_sandbox_liveness() file_name = Path(file).name remote_path = f"/home/user/{file_name}" with open(file, "rb") as f: @@ -481,28 +526,18 @@ def upload_file(self, file: Union[str, Path]) -> str: return remote_path def download_file(self, file_path: str) -> Path: - self._check_sandbox_liveness() with tempfile.NamedTemporaryFile(mode="w+b", delete=False) as file: file.write(self.interpreter.files.read(path=file_path, format="bytes")) _LOGGER.info(f"File ({file_path}) is downloaded to: {file.name}") return Path(file.name) - def _check_sandbox_liveness(self) -> None: - try: - alive = self.interpreter.is_running(request_timeout=2) - except Exception as e: - _LOGGER.error( - f"Failed to check the health of the remote sandbox ({self.interpreter.sandbox_id}) due to {e}. Consider the sandbox as dead." - ) - alive = False - if not alive: - raise RemoteSandboxClosedError( - "Remote sandbox is closed unexpectedly. Please start a new VisionAgent instance." - ) - @staticmethod def _new_e2b_interpreter_impl(*args, **kwargs) -> E2BCodeInterpreterImpl: # type: ignore - return E2BCodeInterpreterImpl(template="va-sandbox", *args, **kwargs) + template_name = os.environ.get("E2B_TEMPLATE_NAME", "nx3fagq7sgdliww9cvm3") + _LOGGER.info( + f"Creating a new E2BCodeInterpreter using template: {template_name}" + ) + return E2BCodeInterpreterImpl(template=template_name, *args, **kwargs) class LocalCodeInterpreter(CodeInterpreter): diff --git a/vision_agent/utils/sim.py b/vision_agent/utils/sim.py index 6a2bbdca..c3b26403 100644 --- a/vision_agent/utils/sim.py +++ b/vision_agent/utils/sim.py @@ -9,6 +9,7 @@ from scipy.spatial.distance import cosine # type: ignore +@lru_cache(maxsize=512) def get_embedding( client: Client, text: str, model: str = "text-embedding-3-small" ) -> List[float]: