From 59d19fe4ebcb61027ce4b4155566615b5d330feb Mon Sep 17 00:00:00 2001 From: Shankar M S Date: Mon, 11 Nov 2019 15:08:48 +0100 Subject: [PATCH 01/14] test_file executable validation + tests --- .../kaos_backend/util/tests/test_validators.py | 12 ++++++++++-- backend/kaos_backend/util/validators.py | 16 ++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/backend/kaos_backend/util/tests/test_validators.py b/backend/kaos_backend/util/tests/test_validators.py index 411d679..d3f2a31 100644 --- a/backend/kaos_backend/util/tests/test_validators.py +++ b/backend/kaos_backend/util/tests/test_validators.py @@ -86,15 +86,23 @@ def test_validate_bundle_structure_missing_dockerfile_in_directory(): BundleValidator.validate_bundle_structure(temp_dir, []) temp.close() - def test_validate_bundle_structure_missing_model_directory(): - with pytest.raises(InvalidBundleError, match="Missing model directory in source-code bundle"): + with pytest.raises(InvalidBundleError, match=""): with TemporaryDirectory() as temp_dir: base_dir = tempfile.mkdtemp(dir=temp_dir) filename = os.path.join(base_dir, "Dockerfile") create_file(filename) BundleValidator.validate_bundle_structure(temp_dir, []) +def test_validate_bundle_structure_missing_shebang_line_in_train(): + with pytest.raises(InvalidBundleError, match="The train file cannot be executed. \n" + "Please add the line '#!/usr/bin/xenv python3' " + "in the beginning of the train file to make it executable"): + with TemporaryDirectory() as temp_dir: + base_dir = tempfile.mkdtemp(dir=temp_dir) + train_file = os.path.join(base_dir, "model", "train") + create_file(train_file) + BundleValidator.validate_bundle_structure(temp_dir, []) @pytest.mark.parametrize("files_include,file_exclude", inference_test_cases) def test_validate_inference_bundle_structure_missing_some_file(files_include, file_exclude): diff --git a/backend/kaos_backend/util/validators.py b/backend/kaos_backend/util/validators.py index 873cde5..04ea996 100644 --- a/backend/kaos_backend/util/validators.py +++ b/backend/kaos_backend/util/validators.py @@ -31,6 +31,10 @@ class BundleValidator: MODEL = "model" + TRAIN = "train" + + SHEBANG = "#!/usr/bin/env python3" + @classmethod def is_empty(cls, directory: str) -> bool: return all(len(files) == 0 for root, _, files in os.walk(directory)) @@ -45,6 +49,15 @@ def validate_model_directory(cls, dirs): if "model" not in dirs: raise InvalidBundleError("Missing model directory in source-code bundle") + @classmethod + def validate_train_file_is_executable(cls, train_file): + f = open(train_file) + first_line = f.readline() + if cls.SHEBANG not in first_line: + raise InvalidBundleError("The train file cannot be executed. \n" + "Please add the line '#!/usr/bin/xenv python3' " + "in the beginning of the train file to make it executable") + @classmethod def validate_dockerfile(cls, files): if "Dockerfile" not in files: @@ -77,11 +90,14 @@ def validate_bundle_structure(cls, directory, req_files): cls.validate_dockerfile(files) cls.validate_model_directory(dirs) model_dir = os.path.join(bundle_root, cls.MODEL) + train_file = os.path.join(model_dir, cls.TRAIN) + cls.validate_train_file_is_executable(train_file) if root == model_dir: for f in req_files: cls.validate_file(f, files) + @classmethod def validate_inference_bundle_structure(cls, directory: str): req_files = cls.REQUIRED_INFERENCE_FILES From 18cae3d69a9908eb056c1773597e4a0d57013dde Mon Sep 17 00:00:00 2001 From: Shankar M S Date: Mon, 11 Nov 2019 15:10:56 +0100 Subject: [PATCH 02/14] model directory validation correction --- backend/kaos_backend/util/tests/test_validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/kaos_backend/util/tests/test_validators.py b/backend/kaos_backend/util/tests/test_validators.py index d3f2a31..cb03a92 100644 --- a/backend/kaos_backend/util/tests/test_validators.py +++ b/backend/kaos_backend/util/tests/test_validators.py @@ -87,7 +87,7 @@ def test_validate_bundle_structure_missing_dockerfile_in_directory(): temp.close() def test_validate_bundle_structure_missing_model_directory(): - with pytest.raises(InvalidBundleError, match=""): + with pytest.raises(InvalidBundleError, match="Missing model directory in source-code bundle"): with TemporaryDirectory() as temp_dir: base_dir = tempfile.mkdtemp(dir=temp_dir) filename = os.path.join(base_dir, "Dockerfile") From 8d1483f2b2cd8c2874eb941779ed86b97a057268 Mon Sep 17 00:00:00 2001 From: Shankar M S Date: Mon, 11 Nov 2019 17:37:44 +0100 Subject: [PATCH 03/14] flake 8 validation fixed --- backend/kaos_backend/util/tests/test_validators.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/kaos_backend/util/tests/test_validators.py b/backend/kaos_backend/util/tests/test_validators.py index cb03a92..0aad3ae 100644 --- a/backend/kaos_backend/util/tests/test_validators.py +++ b/backend/kaos_backend/util/tests/test_validators.py @@ -86,6 +86,7 @@ def test_validate_bundle_structure_missing_dockerfile_in_directory(): BundleValidator.validate_bundle_structure(temp_dir, []) temp.close() + def test_validate_bundle_structure_missing_model_directory(): with pytest.raises(InvalidBundleError, match="Missing model directory in source-code bundle"): with TemporaryDirectory() as temp_dir: @@ -94,6 +95,7 @@ def test_validate_bundle_structure_missing_model_directory(): create_file(filename) BundleValidator.validate_bundle_structure(temp_dir, []) + def test_validate_bundle_structure_missing_shebang_line_in_train(): with pytest.raises(InvalidBundleError, match="The train file cannot be executed. \n" "Please add the line '#!/usr/bin/xenv python3' " @@ -104,6 +106,7 @@ def test_validate_bundle_structure_missing_shebang_line_in_train(): create_file(train_file) BundleValidator.validate_bundle_structure(temp_dir, []) + @pytest.mark.parametrize("files_include,file_exclude", inference_test_cases) def test_validate_inference_bundle_structure_missing_some_file(files_include, file_exclude): with pytest.raises(InvalidBundleError, From 15e724d64783c91557bd6458d69492b74c16bb8d Mon Sep 17 00:00:00 2001 From: Shankar M S Date: Mon, 11 Nov 2019 17:52:52 +0100 Subject: [PATCH 04/14] flake 8 fixed style --- backend/kaos_backend/util/tests/test_validators.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/kaos_backend/util/tests/test_validators.py b/backend/kaos_backend/util/tests/test_validators.py index 0aad3ae..200738d 100644 --- a/backend/kaos_backend/util/tests/test_validators.py +++ b/backend/kaos_backend/util/tests/test_validators.py @@ -97,9 +97,9 @@ def test_validate_bundle_structure_missing_model_directory(): def test_validate_bundle_structure_missing_shebang_line_in_train(): - with pytest.raises(InvalidBundleError, match="The train file cannot be executed. \n" - "Please add the line '#!/usr/bin/xenv python3' " - "in the beginning of the train file to make it executable"): + with pytest.raises(InvalidBundleError, + match="The train file cannot be executed. Please add the line '#!/usr/bin/xenv python3' " + "in the beginning of the train file to make it executable"): with TemporaryDirectory() as temp_dir: base_dir = tempfile.mkdtemp(dir=temp_dir) train_file = os.path.join(base_dir, "model", "train") From 86b11619c5510cbab1970ee0b258569ba49ae88d Mon Sep 17 00:00:00 2001 From: Shankar M S Date: Tue, 12 Nov 2019 15:04:06 +0100 Subject: [PATCH 05/14] More generic handling for both train and serve --- backend/kaos_backend/util/validators.py | 29 ++++++++++++------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/backend/kaos_backend/util/validators.py b/backend/kaos_backend/util/validators.py index 04ea996..819fb67 100644 --- a/backend/kaos_backend/util/validators.py +++ b/backend/kaos_backend/util/validators.py @@ -31,9 +31,9 @@ class BundleValidator: MODEL = "model" - TRAIN = "train" + MODE = "" - SHEBANG = "#!/usr/bin/env python3" + SHEBANG = "#!" @classmethod def is_empty(cls, directory: str) -> bool: @@ -50,13 +50,12 @@ def validate_model_directory(cls, dirs): raise InvalidBundleError("Missing model directory in source-code bundle") @classmethod - def validate_train_file_is_executable(cls, train_file): - f = open(train_file) + def validate_is_file_executable(cls, executable_file): + f = open(executable_file) first_line = f.readline() if cls.SHEBANG not in first_line: - raise InvalidBundleError("The train file cannot be executed. \n" - "Please add the line '#!/usr/bin/xenv python3' " - "in the beginning of the train file to make it executable") + raise InvalidBundleError(f"The {cls.MODE} file cannot be executed. \n" + "Please ensure that first line begins with the shebang '#!' to make it an executable") @classmethod def validate_dockerfile(cls, files): @@ -90,17 +89,17 @@ def validate_bundle_structure(cls, directory, req_files): cls.validate_dockerfile(files) cls.validate_model_directory(dirs) model_dir = os.path.join(bundle_root, cls.MODEL) - train_file = os.path.join(model_dir, cls.TRAIN) - cls.validate_train_file_is_executable(train_file) + executable_file = os.path.join(model_dir, cls.MODE) + cls.validate_is_file_executable(executable_file) if root == model_dir: for f in req_files: cls.validate_file(f, files) - @classmethod def validate_inference_bundle_structure(cls, directory: str): req_files = cls.REQUIRED_INFERENCE_FILES + cls.MODE = "serve" cls.validate_bundle_structure(directory, req_files) @classmethod @@ -110,8 +109,13 @@ def validate_notebook_bundle_structure(cls, directory): @classmethod def validate_train_bundle_structure(cls, directory): req_files = cls.REQUIRED_TRAINING_FILES + cls.MODE = "train" cls.validate_bundle_structure(directory, req_files) + @classmethod + def validate_source_bundle_structure(cls, directory): + req_files = cls.REQUIRED_TRAINING_FILES + cls.validate_bundle_structure(directory, req_files) def validate_cpu_request(cpu): if cpu and MAX_CPU: @@ -119,7 +123,6 @@ def validate_cpu_request(cpu): if cpu > MAX_CPU: raise CPURequestError(f" CPU request {cpu} too high. Maximum allowed is {MAX_CPU}.") - def validate_memory_request(memory): validate_memory_string(memory) @@ -130,14 +133,12 @@ def validate_memory_request(memory): if memory_request > memory_limit: raise MemoryRequestError(f'Memory request {memory} too high. Maximum allowed is {memory_gb}.') - def memory_to_bytes(memory): for k, v in TO_BYTES.items(): if memory.endswith(k): return v * float(memory.replace(k, "")) return float(memory) - def validate_memory_string(memory): """ Validates the input memory string according to the SI memory suffixes defined within kubernetes @@ -154,12 +155,10 @@ def validate_memory_string(memory): if not m: raise MemoryRequestError(f'Incorrect memory request {memory}\n\nPlease check memory specs: {SOURCE_URL}') - def validate_gpu_request(gpu): if gpu > MAX_GPU: raise GPURequestError("GPUs are not enabled") - def validate_resources(function): """ Validation of resource requests From ad6842a7b1d3b9999aa1e0a3e099ae51757208d9 Mon Sep 17 00:00:00 2001 From: Shankar M S Date: Tue, 12 Nov 2019 15:43:52 +0100 Subject: [PATCH 06/14] flake-8 styling issues --- backend/kaos_backend/util/validators.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/backend/kaos_backend/util/validators.py b/backend/kaos_backend/util/validators.py index 819fb67..f9b99e3 100644 --- a/backend/kaos_backend/util/validators.py +++ b/backend/kaos_backend/util/validators.py @@ -54,8 +54,9 @@ def validate_is_file_executable(cls, executable_file): f = open(executable_file) first_line = f.readline() if cls.SHEBANG not in first_line: - raise InvalidBundleError(f"The {cls.MODE} file cannot be executed. \n" - "Please ensure that first line begins with the shebang '#!' to make it an executable") + raise InvalidBundleError(f"The {cls.MODE} file cannot be executed. " + f"Please ensure that first line begins with the shebang '#!' " + f"to make it an executable") @classmethod def validate_dockerfile(cls, files): @@ -117,12 +118,14 @@ def validate_source_bundle_structure(cls, directory): req_files = cls.REQUIRED_TRAINING_FILES cls.validate_bundle_structure(directory, req_files) + def validate_cpu_request(cpu): if cpu and MAX_CPU: cpu = float(cpu) if cpu > MAX_CPU: raise CPURequestError(f" CPU request {cpu} too high. Maximum allowed is {MAX_CPU}.") + def validate_memory_request(memory): validate_memory_string(memory) @@ -133,12 +136,14 @@ def validate_memory_request(memory): if memory_request > memory_limit: raise MemoryRequestError(f'Memory request {memory} too high. Maximum allowed is {memory_gb}.') + def memory_to_bytes(memory): for k, v in TO_BYTES.items(): if memory.endswith(k): return v * float(memory.replace(k, "")) return float(memory) + def validate_memory_string(memory): """ Validates the input memory string according to the SI memory suffixes defined within kubernetes @@ -155,10 +160,12 @@ def validate_memory_string(memory): if not m: raise MemoryRequestError(f'Incorrect memory request {memory}\n\nPlease check memory specs: {SOURCE_URL}') + def validate_gpu_request(gpu): if gpu > MAX_GPU: raise GPURequestError("GPUs are not enabled") + def validate_resources(function): """ Validation of resource requests From 8068c0f6e48182fab657698e65a2b1bb207ba10e Mon Sep 17 00:00:00 2001 From: Shankar M S Date: Tue, 12 Nov 2019 16:45:03 +0100 Subject: [PATCH 07/14] Added test cases for serve --- .../kaos_backend/util/tests/test_validators.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/backend/kaos_backend/util/tests/test_validators.py b/backend/kaos_backend/util/tests/test_validators.py index 200738d..3022ad7 100644 --- a/backend/kaos_backend/util/tests/test_validators.py +++ b/backend/kaos_backend/util/tests/test_validators.py @@ -96,10 +96,9 @@ def test_validate_bundle_structure_missing_model_directory(): BundleValidator.validate_bundle_structure(temp_dir, []) -def test_validate_bundle_structure_missing_shebang_line_in_train(): +def test_validate_test_bundle_missing_executable_file(): with pytest.raises(InvalidBundleError, - match="The train file cannot be executed. Please add the line '#!/usr/bin/xenv python3' " - "in the beginning of the train file to make it executable"): + match="The train file cannot be executed. Please ensure that first line begins with the shebang '#!' to make it an executable"): with TemporaryDirectory() as temp_dir: base_dir = tempfile.mkdtemp(dir=temp_dir) train_file = os.path.join(base_dir, "model", "train") @@ -107,6 +106,16 @@ def test_validate_bundle_structure_missing_shebang_line_in_train(): BundleValidator.validate_bundle_structure(temp_dir, []) +def test_validate_serve_bundle_missing_executable_file(): + with pytest.raises(InvalidBundleError, + match="The serve file cannot be executed. Please ensure that first line begins with the shebang '#!' to make it an executable"): + with TemporaryDirectory() as temp_dir: + base_dir = tempfile.mkdtemp(dir=temp_dir) + serve_file = os.path.join(base_dir, "model", "serve") + create_file(serve_file) + BundleValidator.validate_bundle_structure(temp_dir, []) + + @pytest.mark.parametrize("files_include,file_exclude", inference_test_cases) def test_validate_inference_bundle_structure_missing_some_file(files_include, file_exclude): with pytest.raises(InvalidBundleError, From c3fecea5d0b137e6fc123ecbde0aff30a3b0ca1f Mon Sep 17 00:00:00 2001 From: Shankar M S Date: Tue, 12 Nov 2019 17:04:00 +0100 Subject: [PATCH 08/14] fixed tests --- backend/kaos_backend/util/validators.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/kaos_backend/util/validators.py b/backend/kaos_backend/util/validators.py index f9b99e3..bf6929f 100644 --- a/backend/kaos_backend/util/validators.py +++ b/backend/kaos_backend/util/validators.py @@ -31,7 +31,7 @@ class BundleValidator: MODEL = "model" - MODE = "" + MODE = None SHEBANG = "#!" @@ -90,8 +90,9 @@ def validate_bundle_structure(cls, directory, req_files): cls.validate_dockerfile(files) cls.validate_model_directory(dirs) model_dir = os.path.join(bundle_root, cls.MODEL) - executable_file = os.path.join(model_dir, cls.MODE) - cls.validate_is_file_executable(executable_file) + if cls.MODE: + executable_file = os.path.join(model_dir, cls.MODE) + cls.validate_is_file_executable(executable_file) if root == model_dir: for f in req_files: From 721a5ad5111db092b68f1efe540e22a6d70f28d3 Mon Sep 17 00:00:00 2001 From: Shankar M S Date: Wed, 13 Nov 2019 18:34:39 +0100 Subject: [PATCH 09/14] fixed test cases (modified building of temp dirs) --- Pipfile | 9 +- Pipfile.lock | 625 +----------------- .../controllers/tests/__init__.py | 8 +- .../util/tests/test_validators.py | 58 +- backend/kaos_backend/util/validators.py | 56 +- 5 files changed, 89 insertions(+), 667 deletions(-) diff --git a/Pipfile b/Pipfile index efae643..fd19f56 100644 --- a/Pipfile +++ b/Pipfile @@ -4,15 +4,8 @@ url = "https://pypi.org/simple" verify_ssl = true [dev-packages] -kaos-integration-tests = {editable = true, path = "./testing/integration"} -kaos-backend = {editable = true, path = "./backend"} [packages] -kaos-model = {editable = true, path = "./model"} -kaos = {editable = true, path = "./cli"} [requires] -python_version = "3.7" - -[pipenv] -allow_prereleases = true +python_version = "2.7" diff --git a/Pipfile.lock b/Pipfile.lock index 8e4e082..637b90f 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "cc580666520e49206414747789824998c089f82acbaa295c272a4313d508ede7" + "sha256": "ae4bdd7d4157baab65ae9d0e8389a6011e6b640995372c45ec81fa5d1ddfae9f" }, "pipfile-spec": 6, "requires": { - "python_version": "3.7" + "python_version": "2.7" }, "sources": [ { @@ -15,623 +15,6 @@ } ] }, - "default": { - "certifi": { - "hashes": [ - "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", - "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" - ], - "version": "==2019.9.11" - }, - "chardet": { - "hashes": [ - "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", - "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" - ], - "version": "==3.0.4" - }, - "click": { - "hashes": [ - "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", - "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" - ], - "version": "==7.0" - }, - "dataclasses-json": { - "hashes": [ - "sha256:16fdc36ec471fb6cc2054ff5ce2be78876d16f78cee6be3e49137c7fcf31667f", - "sha256:4fdbb0fa654e46dcad6555722d52f741b1414a4188a4d7fa89bc7abf15323a2d" - ], - "version": "==0.2.14" - }, - "docker": { - "hashes": [ - "sha256:2b1f48041cfdcc9f6b5da0e04e0e326ded225e736762ade2060000e708f4c9b7", - "sha256:c456ded5420af5860441219ff8e51cdec531d65f4a9e948ccd4133e063b72f50" - ], - "version": "==3.7.2" - }, - "docker-pycreds": { - "hashes": [ - "sha256:6ce3270bcaf404cc4c3e27e4b6c70d3521deae82fb508767870fdbf772d584d4", - "sha256:7266112468627868005106ec19cd0d722702d2b7d5912a28e19b826c3d37af49" - ], - "version": "==0.4.0" - }, - "graphviz": { - "hashes": [ - "sha256:0e1744a45b0d707bc44f99c7b8e5f25dc22cf96b6aaf2432ac308ed9822a9cb6", - "sha256:d311be4fddfe832a56986ac5e1d6e8715d7fcb0208560da79d1bb0f72abef41f" - ], - "version": "==0.10.1" - }, - "idna": { - "hashes": [ - "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", - "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" - ], - "version": "==2.8" - }, - "kaos": { - "editable": true, - "path": "./cli" - }, - "kaos-model": { - "editable": true, - "path": "./model" - }, - "marshmallow": { - "hashes": [ - "sha256:5e0b15f33c6e227b51e1fbe29a306676686f3c3a3bf7fe0aca005144c3e74d7c", - "sha256:8c6a22cfc9ca33945e2707c771c44030a035be96082a18643d431280b9b8f08e" - ], - "version": "==3.0.0rc6" - }, - "marshmallow-enum": { - "hashes": [ - "sha256:38e697e11f45a8e64b4a1e664000897c659b60aa57bfa18d44e226a9920b6e58", - "sha256:57161ab3dbfde4f57adeb12090f39592e992b9c86d206d02f6bd03ebec60f072" - ], - "version": "==1.5.1" - }, - "mypy-extensions": { - "hashes": [ - "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d", - "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8" - ], - "version": "==0.4.3" - }, - "ptable": { - "hashes": [ - "sha256:aa7fc151cb40f2dabcd2275ba6f7fd0ff8577a86be3365cd3fb297cbe09cc292" - ], - "version": "==0.9.2" - }, - "requests": { - "hashes": [ - "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", - "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" - ], - "version": "==2.21.0" - }, - "requests-toolbelt": { - "hashes": [ - "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f", - "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0" - ], - "version": "==0.9.1" - }, - "six": { - "hashes": [ - "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", - "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" - ], - "version": "==1.12.0" - }, - "stringcase": { - "hashes": [ - "sha256:48a06980661908efe8d9d34eab2b6c13aefa2163b3ced26972902e3bdfd87008" - ], - "version": "==1.2.0" - }, - "textdistance": { - "hashes": [ - "sha256:982f7f24696b9f668001b2c0d109c32d545585e7d641bdc9be6ecca0a69cc1dc", - "sha256:b9e06028210557c2d89234230da208ea868d6a2b2f82358204f4a7131713e71b" - ], - "version": "==4.1.3" - }, - "tqdm": { - "hashes": [ - "sha256:14a285392c32b6f8222ecfbcd217838f88e11630affe9006cd0e94c7eff3cb61", - "sha256:25d4c0ea02a305a688e7e9c2cdc8f862f989ef2a4701ab28ee963295f5b109ab" - ], - "version": "==4.32.2" - }, - "typing-extensions": { - "hashes": [ - "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2", - "sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d", - "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575" - ], - "version": "==3.7.4.1" - }, - "typing-inspect": { - "hashes": [ - "sha256:75c97b7854426a129f3184c68588db29091ff58e6908ed520add1d52fc44df6e", - "sha256:811b44f92e780b90cfe7bac94249a4fae87cfaa9b40312765489255045231d9c", - "sha256:c6ed1cd34860857c53c146a6704a96da12e1661087828ce350f34addc6e5eee3" - ], - "version": "==0.5.0" - }, - "urllib3": { - "hashes": [ - "sha256:2393a695cd12afedd0dcb26fe5d50d0cf248e5a66f75dbd89a3d4eb333a61af4", - "sha256:a637e5fae88995b256e3409dc4d52c2e2e0ba32c42a6365fee8bbd2238de3cfb" - ], - "version": "==1.24.3" - }, - "websocket-client": { - "hashes": [ - "sha256:1151d5fb3a62dc129164292e1227655e4bbc5dd5340a5165dfae61128ec50aa9", - "sha256:1fd5520878b68b84b5748bb30e592b10d0a91529d5383f74f4964e72b297fd3a" - ], - "version": "==0.56.0" - } - }, - "develop": { - "atomicwrites": { - "hashes": [ - "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", - "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" - ], - "version": "==1.3.0" - }, - "attrs": { - "hashes": [ - "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", - "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" - ], - "version": "==19.3.0" - }, - "bitstring": { - "hashes": [ - "sha256:c163a86fcef377c314690051885d86b47419e3e1770990c212e16723c1c08faa" - ], - "version": "==3.1.5" - }, - "boto3": { - "hashes": [ - "sha256:0bed0db8c10b88b3daa042adaa1fb6c3262caed39d28086e8548015405c71744", - "sha256:70e71e0192a68f65754ab9d2a335be3c6856a1e8a15f3bd6263ea12e2f442bc7" - ], - "version": "==1.9.93" - }, - "botocore": { - "hashes": [ - "sha256:3baf129118575602ada9926f5166d82d02273c250d0feb313fc270944b27c48b", - "sha256:dc080aed4f9b220a9e916ca29ca97a9d37e8e1d296fe89cbaeef929bf0c8066b" - ], - "version": "==1.12.253" - }, - "certifi": { - "hashes": [ - "sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50", - "sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef" - ], - "version": "==2019.9.11" - }, - "cgroupspy": { - "hashes": [ - "sha256:fb4aac7938499cff53c260112fb0759a54fb5f8bad89d9be0326c70a7619821a" - ], - "version": "==0.1.6" - }, - "chardet": { - "hashes": [ - "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", - "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" - ], - "version": "==3.0.4" - }, - "checksumdir": { - "hashes": [ - "sha256:aa1be031a0c51653aa7ae08c9991def5c639001976d09b7890284919652782d5" - ], - "version": "==1.1.7" - }, - "click": { - "hashes": [ - "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", - "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" - ], - "version": "==7.0" - }, - "dataclasses-json": { - "hashes": [ - "sha256:16fdc36ec471fb6cc2054ff5ce2be78876d16f78cee6be3e49137c7fcf31667f", - "sha256:4fdbb0fa654e46dcad6555722d52f741b1414a4188a4d7fa89bc7abf15323a2d" - ], - "version": "==0.2.14" - }, - "docker": { - "hashes": [ - "sha256:2b1f48041cfdcc9f6b5da0e04e0e326ded225e736762ade2060000e708f4c9b7", - "sha256:c456ded5420af5860441219ff8e51cdec531d65f4a9e948ccd4133e063b72f50" - ], - "version": "==3.7.2" - }, - "docker-pycreds": { - "hashes": [ - "sha256:6ce3270bcaf404cc4c3e27e4b6c70d3521deae82fb508767870fdbf772d584d4", - "sha256:7266112468627868005106ec19cd0d722702d2b7d5912a28e19b826c3d37af49" - ], - "version": "==0.4.0" - }, - "docutils": { - "hashes": [ - "sha256:6c4f696463b79f1fb8ba0c594b63840ebd41f059e92b31957c46b74a4599b6d0", - "sha256:9e4d7ecfc600058e07ba661411a2b7de2fd0fafa17d1a7f7361cd47b1175c827", - "sha256:a2aeea129088da402665e92e0b25b04b073c04b2dce4ab65caaa38b7ce2e1a99" - ], - "version": "==0.15.2" - }, - "flask": { - "hashes": [ - "sha256:2271c0070dbcb5275fad4a82e29f23ab92682dc45f9dfbc22c02ba9b9322ce48", - "sha256:a080b744b7e345ccfcbc77954861cb05b3c63786e93f2b3875e0913d44b43f05" - ], - "version": "==1.0.2" - }, - "graphviz": { - "hashes": [ - "sha256:0e1744a45b0d707bc44f99c7b8e5f25dc22cf96b6aaf2432ac308ed9822a9cb6", - "sha256:d311be4fddfe832a56986ac5e1d6e8715d7fcb0208560da79d1bb0f72abef41f" - ], - "version": "==0.10.1" - }, - "grpcio": { - "hashes": [ - "sha256:0f4ddd568e097ec454f557f5b8457357a2d589da736ed501d8cb9afcfa46e956", - "sha256:1163269299823cd1ce590a3c36deeec9e863bcf12bf28b05779f1029b7493133", - "sha256:15d8e0a6fd2c4c96504805a4488c1ea5d82d529625bf1ff76227477f998b0bd5", - "sha256:17a85cef12c47d5929f630517fe34c34ee6b20a0eb98bd4d6202d1206c02db88", - "sha256:1da84dd6445e59d7d70357289daf636fcbc2986c2e59c8f1e9c2b5072ae85961", - "sha256:23892c2c146b4d83487ac1c9fe14045c4167036d04d6488715fc3865025baf61", - "sha256:2454233d766dd3ad157744bd4faa2400e241d47ac9a84e7425ca940f4d5cca0b", - "sha256:25ceafff31e9b3e90e7a46fbddd4b57b74135e11458d2206313c6fb11920273a", - "sha256:283516f89708b11c8423ecdca1a80a4e920d2deb671106e079b3c17e88ac2378", - "sha256:31df91c1dfe328580702bcdfa1a77e1d18811939de95164f4ab348988664bea9", - "sha256:3c8c7c0b06da6aabde795626fdc08b8507a11b34031c8bca1940db00ad7386e6", - "sha256:3f68835f01b96b1fa200303d029c091c1e42e0a830d6c2f37ad28ea785db0bd2", - "sha256:4285fb0d8dfe6ceaae275373ff87611263026729ee36d2c6c49bfdbcf1cac4d2", - "sha256:484749d167357903021f66da9565a4f2ba635caa6fbfcb17caa8043fe7ee75ae", - "sha256:489bfeb7f1ed94f87ccf7e5808c7e5562d9b8d640945a5c0e445f16616f51943", - "sha256:4be8ccbc32ba091e42a51924c96d87c2b3317a08d87ae442b52e25bd458e3284", - "sha256:502d4bf405191ade18cd614e99e84ca8bccfdb7380d1c32d1592c022ffa7c0d2", - "sha256:5446ff52e6eb3f91723d83e48e76936dd85d631b981b36592a52672574df51ac", - "sha256:5a6de144dfe95712e1d40221648777d616902df7a9b501018738bb41eefadf34", - "sha256:5e8ea07ebeaa57879e11973940a03852501b64c39963533ef918f00457c949db", - "sha256:6462cfba6e3ac23e4d388efab0af8813548520a03dbcb307488360798e1352f5", - "sha256:6a724b1beaabc26d17b735e7213e98f99dc07a46c7139cad71f6e699e4fc4206", - "sha256:76b4ebc73a5cbee39a3754a6159bbf27b3c0da365836f2f216552c6c89e6d5ea", - "sha256:802006763ac2dd5c30a6791e0937e1045d9f103b14fc7860d380d6cfd515190a", - "sha256:80c290d477bd95f823809f01774ec96cc44a61df3efb0e0901441aa6fc843b78", - "sha256:89a729ab4255d88d3e7553b63907476ccaa141f3b036d059034fb94905114db4", - "sha256:8e859b8609e6bf9a4dd227bc8636de7ae4936d7821a8e74c37ac7b18fa5f5ee3", - "sha256:948793b791550313bdecce8ade74ed141bf3ec49d4d488ade9475a041e3d8ca0", - "sha256:95cf7fbfe00985cb034b788f6bd85501c71cff75aad0c2d3b4afe0a0df5ce784", - "sha256:9aded46e7197ef77a91cb486db453f843d7a93f4a3e488098b3b73a55f5835fe", - "sha256:9bdb843679a1701418ccd4eac10644859f456f013c08f1901698a740d3889bae", - "sha256:9bfba5dd2356e73454ed7477d0e801471135e1dc5f4634a4a429670168937915", - "sha256:9ccd990b8ea3fa8525c15da386b54cf7d17d74f3d4272d4402351df33a2c9db4", - "sha256:9ce5f0be4bdcd018fb0313cfa3587e1bfbb03336b6e27c8f0d42ec735c3d36dc", - "sha256:a581b89c5b35b45f66652429da0545739867d66c2cc0d003e1390f8ca4528772", - "sha256:add8d00c7522cc1ff7be233796deb0b78bdacb1435aeba1a59c60f49d1548997", - "sha256:b3d59349906b8af32b2fcb40b8a54ec0883b06fa778eba7db1ce48e29eb8e3b4", - "sha256:b75d09af4fb3aa9b188eb9e574bf60939c948e5dba0c71b76eb785c4c3831711", - "sha256:b80606790119e16d8b2f196ec588beb03818d96590c8ed063383521e87b4e376", - "sha256:bd5313a15e5dd0083fc114cb61b0e66811f11eb822f7d54ed5b08dd3a38aa139", - "sha256:c59ad06db6c069cbbe647d08a4117886e78f9552ccad354d7681ce17636af164", - "sha256:c75068a70743f27d91643746f4c3b7c057ba867a87314b6d108ef9ebf166cbcd", - "sha256:d396d1e50bef7eb9ac964a0a78028a8fdb45696378713d57996126a8e5f23710", - "sha256:e7398bbc6dc4ede73d4b4586421505345ff1f52c276d3e07d4438ebd3fd44031", - "sha256:fcb34e028d5087325cc4b8e365875bd661ec1151aa09a2802a6ee1d9d08730b8", - "sha256:fd3498fe389884d2db6f7db03dd66754501e5fa8df35d25700c7063317dbd9df", - "sha256:fd48e720df64d5d5d27af3a631f65e8d0a5e24811ec457f631ed43fac1e44eea", - "sha256:fe1e6037cad0bca6491d2f45e3a2836edf209d641bcd8b66dcbeb3d9f7e66dfa" - ], - "version": "==1.25.0rc1" - }, - "gunicorn": { - "hashes": [ - "sha256:aa8e0b40b4157b36a5df5e599f45c9c76d6af43845ba3b3b0efe2c70473c2471", - "sha256:fa2662097c66f920f53f70621c6c58ca4a3c4d3434205e608e121b5b3b71f4f3" - ], - "version": "==19.9.0" - }, - "idna": { - "hashes": [ - "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", - "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" - ], - "version": "==2.8" - }, - "importlib-metadata": { - "hashes": [ - "sha256:aa18d7378b00b40847790e7c27e11673d7fed219354109d0e7b9e5b25dc3ad26", - "sha256:d5f18a79777f3aa179c145737780282e27b508fc8fd688cb17c7a813e8bd39af" - ], - "markers": "python_version < '3.8'", - "version": "==0.23" - }, - "itsdangerous": { - "hashes": [ - "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", - "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749" - ], - "version": "==1.1.0" - }, - "jinja2": { - "hashes": [ - "sha256:74320bb91f31270f9551d46522e33af46a80c3d619f4a4bf42b3164d30b5911f", - "sha256:9fe95f19286cfefaa917656583d020be14e7859c6b0252588391e47db34527de" - ], - "version": "==2.10.3" - }, - "jmespath": { - "hashes": [ - "sha256:3720a4b1bd659dd2eecad0666459b9788813e032b83e7ba58578e48254e0a0e6", - "sha256:bde2aef6f44302dfb30320115b17d030798de8c4110e28d5cf6cf91a7a31074c" - ], - "version": "==0.9.4" - }, - "kaos-backend": { - "editable": true, - "path": "./backend" - }, - "kaos-integration-tests": { - "editable": true, - "path": "./testing/integration" - }, - "markupsafe": { - "hashes": [ - "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", - "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", - "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", - "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", - "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", - "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", - "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", - "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", - "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", - "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", - "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", - "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", - "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", - "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", - "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", - "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", - "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", - "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", - "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", - "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", - "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", - "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", - "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", - "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", - "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", - "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", - "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", - "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7" - ], - "version": "==1.1.1" - }, - "marshmallow": { - "hashes": [ - "sha256:5e0b15f33c6e227b51e1fbe29a306676686f3c3a3bf7fe0aca005144c3e74d7c", - "sha256:8c6a22cfc9ca33945e2707c771c44030a035be96082a18643d431280b9b8f08e" - ], - "version": "==3.0.0rc6" - }, - "marshmallow-enum": { - "hashes": [ - "sha256:38e697e11f45a8e64b4a1e664000897c659b60aa57bfa18d44e226a9920b6e58", - "sha256:57161ab3dbfde4f57adeb12090f39592e992b9c86d206d02f6bd03ebec60f072" - ], - "version": "==1.5.1" - }, - "more-itertools": { - "hashes": [ - "sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832", - "sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4" - ], - "version": "==7.2.0" - }, - "mypy-extensions": { - "hashes": [ - "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d", - "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8" - ], - "version": "==0.4.3" - }, - "packaging": { - "hashes": [ - "sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47", - "sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108" - ], - "version": "==19.2" - }, - "pluggy": { - "hashes": [ - "sha256:0db4b7601aae1d35b4a033282da476845aa19185c1e6964b25cf324b5e4ec3e6", - "sha256:fa5fa1622fa6dd5c030e9cad086fa19ef6a0cf6d7a2d12318e10cb49d6d68f34" - ], - "version": "==0.13.0" - }, - "protobuf": { - "hashes": [ - "sha256:03f43eac9d5b651f976e91cf46a25b75e5779d98f0f4114b0abfed83376d75f8", - "sha256:0c94b21e6de01362f91a86b372555d22a60b59708599ca9d5032ae9fdf8e3538", - "sha256:2d2a9f30f61f4063fadd7fb68a2510a6939b43c0d6ceeec5c4704f22225da28e", - "sha256:34a0b05fca061e4abb77dd180209f68d8637115ff319f51e28a6a9382d69853a", - "sha256:358710fd0db25372edcf1150fa691f48376a134a6c69ce29f38f185eea7699e6", - "sha256:41e47198b94c27ba05a08b4a95160656105745c462af574e4bcb0807164065c0", - "sha256:8c61cc8a76e9d381c665aecc5105fa0f1878cf7db8b5cd17202603bcb386d0fc", - "sha256:a6eebc4db759e58fdac02efcd3028b811effac881d8a5bad1996e4e8ee6acb47", - "sha256:a9c12f7c98093da0a46ba76ec40ace725daa1ac4038c41e4b1466afb5c45bb01", - "sha256:cb95068492ba0859b8c9e61fa8ba206a83c64e5d0916fb4543700b2e2b214115", - "sha256:cd98476ce7bb4dcd6a7b101f5eecdc073dafea19f311e36eb8fba1a349346277", - "sha256:ce64cfbea18c535176bdaa10ba740c0fc4c6d998a3f511c17bedb0ae4b3b167c", - "sha256:dcbb59eac73fd454e8f2c5fba9e3d3320fd4707ed6a9d3ea3717924a6f0903ea", - "sha256:dd67f34458ae716029e2a71ede998e9092493b62a519236ca52e3c5202096c87", - "sha256:e3c96056eb5b7284a20e256cb0bf783c8f36ad82a4ae5434a7b7cd02384144a7", - "sha256:f612d584d7a27e2f39e7b17878430a959c1bc09a74ba09db096b468558e5e126", - "sha256:f6de8a7d6122297b81566e5bd4df37fd5d62bec14f8f90ebff8ede1c9726cd0a", - "sha256:fa529d9261682b24c2aaa683667253175c9acebe0a31105394b221090da75832" - ], - "version": "==3.8.0" - }, - "psutil": { - "hashes": [ - "sha256:028a1ec3c6197eadd11e7b46e8cc2f0720dc18ac6d7aabdb8e8c0d6c9704f000", - "sha256:12542c3642909f4cd1928a2fba59e16fa27e47cbeea60928ebb62a8cbd1ce123", - "sha256:503e4b20fa9d3342bcf58191bbc20a4a5ef79ca7df8972e6197cc14c5513e73d", - "sha256:863a85c1c0a5103a12c05a35e59d336e1d665747e531256e061213e2e90f63f3", - "sha256:954f782608bfef9ae9f78e660e065bd8ffcfaea780f9f2c8a133bb7cb9e826d7", - "sha256:b6e08f965a305cd84c2d07409bc16fbef4417d67b70c53b299116c5b895e3f45", - "sha256:bc96d437dfbb8865fc8828cf363450001cb04056bbdcdd6fc152c436c8a74c61", - "sha256:cf49178021075d47c61c03c0229ac0c60d5e2830f8cab19e2d88e579b18cdb76", - "sha256:d5350cb66690915d60f8b233180f1e49938756fb2d501c93c44f8fb5b970cc63", - "sha256:eba238cf1989dfff7d483c029acb0ac4fcbfc15de295d682901f0e2497e6781a" - ], - "version": "==5.6.3" - }, - "py": { - "hashes": [ - "sha256:64f65755aee5b381cea27766a3a147c3f15b9b6b9ac88676de66ba2ae36793fa", - "sha256:dc639b046a6e2cff5bbe40194ad65936d6ba360b52b3c3fe1d08a82dd50b5e53" - ], - "version": "==1.8.0" - }, - "pyparsing": { - "hashes": [ - "sha256:6f98a7b9397e206d78cc01df10131398f1c8b8510a2f4d97d9abd82e1aacdd80", - "sha256:d9338df12903bbf5d65a0e4e87c2161968b10d2e489652bb47001d82a9b028b4" - ], - "version": "==2.4.2" - }, - "pypdf2": { - "hashes": [ - "sha256:e28f902f2f0a1603ea95ebe21dff311ef09be3d0f0ef29a3e44a932729564385" - ], - "version": "==1.26.0" - }, - "pytest": { - "hashes": [ - "sha256:27abc3fef618a01bebb1f0d6d303d2816a99aa87a5968ebc32fe971be91eb1e6", - "sha256:58cee9e09242937e136dbb3dab466116ba20d6b7828c7620f23947f37eb4dae4" - ], - "version": "==5.2.2" - }, - "pytest-mock": { - "hashes": [ - "sha256:43ce4e9dd5074993e7c021bb1c22cbb5363e612a2b5a76bc6d956775b10758b7", - "sha256:5bf5771b1db93beac965a7347dc81c675ec4090cb841e49d9d34637a25c30568" - ], - "version": "==1.10.4" - }, - "python-dateutil": { - "hashes": [ - "sha256:7e6584c74aeed623791615e26efd690f29817a27c73085b78e4bad02493df2fb", - "sha256:c89805f6f4d64db21ed966fda138f8a5ed7a4fdbc1a8ee329ce1b74e3c74da9e" - ], - "markers": "python_version >= '2.7'", - "version": "==2.8.0" - }, - "python-pachyderm": { - "hashes": [ - "sha256:624d692e55d51cb145f9cf7cb3675290f64ddf073fb4d5c77a6a80c62664e1f6" - ], - "version": "==1.9.0.post5" - }, - "requests": { - "hashes": [ - "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", - "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" - ], - "version": "==2.21.0" - }, - "s3transfer": { - "hashes": [ - "sha256:6efc926738a3cd576c2a79725fed9afde92378aa5c6a957e3af010cb019fac9d", - "sha256:b780f2411b824cb541dbcd2c713d0cb61c7d1bcadae204cdddda2b35cef493ba" - ], - "version": "==0.2.1" - }, - "six": { - "hashes": [ - "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", - "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" - ], - "version": "==1.12.0" - }, - "stringcase": { - "hashes": [ - "sha256:48a06980661908efe8d9d34eab2b6c13aefa2163b3ced26972902e3bdfd87008" - ], - "version": "==1.2.0" - }, - "typing-extensions": { - "hashes": [ - "sha256:091ecc894d5e908ac75209f10d5b4f118fbdb2eb1ede6a63544054bb1edb41f2", - "sha256:910f4656f54de5993ad9304959ce9bb903f90aadc7c67a0bef07e678014e892d", - "sha256:cf8b63fedea4d89bab840ecbb93e75578af28f76f66c35889bd7065f5af88575" - ], - "version": "==3.7.4.1" - }, - "typing-inspect": { - "hashes": [ - "sha256:75c97b7854426a129f3184c68588db29091ff58e6908ed520add1d52fc44df6e", - "sha256:811b44f92e780b90cfe7bac94249a4fae87cfaa9b40312765489255045231d9c", - "sha256:c6ed1cd34860857c53c146a6704a96da12e1661087828ce350f34addc6e5eee3" - ], - "version": "==0.5.0" - }, - "urllib3": { - "hashes": [ - "sha256:2393a695cd12afedd0dcb26fe5d50d0cf248e5a66f75dbd89a3d4eb333a61af4", - "sha256:a637e5fae88995b256e3409dc4d52c2e2e0ba32c42a6365fee8bbd2238de3cfb" - ], - "version": "==1.24.3" - }, - "wcwidth": { - "hashes": [ - "sha256:3df37372226d6e63e1b1e1eda15c594bca98a22d33a23832a90998faa96bc65e", - "sha256:f4ebe71925af7b40a864553f761ed559b43544f8f71746c2d756c7fe788ade7c" - ], - "version": "==0.1.7" - }, - "websocket-client": { - "hashes": [ - "sha256:1151d5fb3a62dc129164292e1227655e4bbc5dd5340a5165dfae61128ec50aa9", - "sha256:1fd5520878b68b84b5748bb30e592b10d0a91529d5383f74f4964e72b297fd3a" - ], - "version": "==0.56.0" - }, - "werkzeug": { - "hashes": [ - "sha256:97660b282aa7e29f94f3fe378e5c7162d7ab9d601a8dbb1cbb2ffc8f0e54607d", - "sha256:cfd1281b1748288e59762c0e174d64d8bcb2b70e7c57bc4a1203c8825af24ac3" - ], - "version": "==0.15.3" - }, - "zipp": { - "hashes": [ - "sha256:3718b1cbcd963c7d4c5511a8240812904164b7f381b647143a89d3b98f9bcd8e", - "sha256:f06903e9f1f43b12d371004b4ac7b06ab39a44adc747266928ae6debfa7b3335" - ], - "version": "==0.6.0" - } - } + "default": {}, + "develop": {} } diff --git a/backend/kaos_backend/controllers/tests/__init__.py b/backend/kaos_backend/controllers/tests/__init__.py index 3b3de47..160b670 100644 --- a/backend/kaos_backend/controllers/tests/__init__.py +++ b/backend/kaos_backend/controllers/tests/__init__.py @@ -39,8 +39,12 @@ def check_pipeline_exists_mock(pipeline_name): def create_test_file(dirname, filename): - f = open(os.path.join(dirname, filename), "w") - f.write("this is fake") + if filename in ['train', 'serve']: + f = open(os.path.join(dirname, filename), "w") + f.write("#!") + else: + f = open(os.path.join(dirname, filename), "w") + f.write("this is fake") f.close() return f diff --git a/backend/kaos_backend/util/tests/test_validators.py b/backend/kaos_backend/util/tests/test_validators.py index 3022ad7..2f3e3b3 100644 --- a/backend/kaos_backend/util/tests/test_validators.py +++ b/backend/kaos_backend/util/tests/test_validators.py @@ -17,6 +17,14 @@ def create_file(path): return temp +def make_executable_file(path): + with open(path, 'w') as executable_file: + line = "#!" + executable_file.write(line) + executable_file.close() + return executable_file + + def remove_el(l, el): ll = l[:] ll.remove(el) @@ -52,7 +60,7 @@ def test_is_not_empty(): def test_validate_bundle_structure_is_empty(): with pytest.raises(InvalidBundleError, match="Bundle must be non-empty"): with TemporaryDirectory() as temp_dir: - BundleValidator.validate_bundle_structure(temp_dir, []) + BundleValidator.validate_bundle_structure(temp_dir, [], mode=None) def test_validate_bundle_structure_missing_root_directory(): @@ -60,7 +68,7 @@ def test_validate_bundle_structure_missing_root_directory(): with TemporaryDirectory() as temp_dir: # temp file to avoid empty file exception temp = tempfile.NamedTemporaryFile(dir=temp_dir, delete=True) - BundleValidator.validate_bundle_structure(temp_dir, []) + BundleValidator.validate_bundle_structure(temp_dir, [], mode=None) temp.close() @@ -72,7 +80,7 @@ def test_validate_bundle_structure_too_many_directories(): tempfile.mkdtemp(dir=temp_dir) tempfile.mkdtemp(dir=temp_dir) - BundleValidator.validate_bundle_structure(temp_dir, []) + BundleValidator.validate_bundle_structure(temp_dir, [], mode=None) temp.close() @@ -83,7 +91,7 @@ def test_validate_bundle_structure_missing_dockerfile_in_directory(): temp = tempfile.NamedTemporaryFile(dir=temp_dir, delete=True) tempfile.mkdtemp(dir=temp_dir) - BundleValidator.validate_bundle_structure(temp_dir, []) + BundleValidator.validate_bundle_structure(temp_dir, [], mode=None) temp.close() @@ -93,27 +101,47 @@ def test_validate_bundle_structure_missing_model_directory(): base_dir = tempfile.mkdtemp(dir=temp_dir) filename = os.path.join(base_dir, "Dockerfile") create_file(filename) - BundleValidator.validate_bundle_structure(temp_dir, []) + BundleValidator.validate_bundle_structure(temp_dir, [], mode=None) -def test_validate_test_bundle_missing_executable_file(): +@pytest.mark.parametrize("files_include,file_exclude", training_test_cases) +def test_validate_train_bundle_missing_executable_file(files_include, file_exclude): with pytest.raises(InvalidBundleError, - match="The train file cannot be executed. Please ensure that first line begins with the shebang '#!' to make it an executable"): + match="The train file cannot be executed. " + "Please ensure that first line begins with the shebang '#!' to make it an executable"): with TemporaryDirectory() as temp_dir: base_dir = tempfile.mkdtemp(dir=temp_dir) - train_file = os.path.join(base_dir, "model", "train") - create_file(train_file) - BundleValidator.validate_bundle_structure(temp_dir, []) + dockerfile = os.path.join(base_dir, "Dockerfile") + create_file(dockerfile) + model_dir = os.path.join(base_dir, "model") + os.mkdir(model_dir) + mode = "train" + for f in files_include: + create_file(os.path.join(model_dir, f)) + train_file = os.path.join(model_dir, mode) + make_executable_file(train_file) + BundleValidator.validate_bundle_structure(temp_dir, [], mode=mode) -def test_validate_serve_bundle_missing_executable_file(): + +@pytest.mark.parametrize("files_include,file_exclude", inference_test_cases) +def test_validate_serve_bundle_missing_executable_file(files_include, file_exclude): with pytest.raises(InvalidBundleError, - match="The serve file cannot be executed. Please ensure that first line begins with the shebang '#!' to make it an executable"): + match="The serve file cannot be executed. " + "Please ensure that first line begins with the shebang '#!' to make it an executable"): with TemporaryDirectory() as temp_dir: base_dir = tempfile.mkdtemp(dir=temp_dir) - serve_file = os.path.join(base_dir, "model", "serve") - create_file(serve_file) - BundleValidator.validate_bundle_structure(temp_dir, []) + dockerfile = os.path.join(base_dir, "Dockerfile") + create_file(dockerfile) + model_dir = os.path.join(base_dir, "model") + os.mkdir(model_dir) + mode = "serve" + for f in files_include: + create_file(os.path.join(model_dir, f)) + + serve_file = os.path.join(model_dir, mode) + make_executable_file(serve_file) + BundleValidator.validate_bundle_structure(temp_dir, [], mode=mode) @pytest.mark.parametrize("files_include,file_exclude", inference_test_cases) diff --git a/backend/kaos_backend/util/validators.py b/backend/kaos_backend/util/validators.py index bf6929f..54fc754 100644 --- a/backend/kaos_backend/util/validators.py +++ b/backend/kaos_backend/util/validators.py @@ -21,6 +21,7 @@ class BundleValidator: + REQUIRED_INFERENCE_FILES = ["__init__.py", "serve", "web-requirements.txt"] @@ -31,8 +32,6 @@ class BundleValidator: MODEL = "model" - MODE = None - SHEBANG = "#!" @classmethod @@ -49,15 +48,6 @@ def validate_model_directory(cls, dirs): if "model" not in dirs: raise InvalidBundleError("Missing model directory in source-code bundle") - @classmethod - def validate_is_file_executable(cls, executable_file): - f = open(executable_file) - first_line = f.readline() - if cls.SHEBANG not in first_line: - raise InvalidBundleError(f"The {cls.MODE} file cannot be executed. " - f"Please ensure that first line begins with the shebang '#!' " - f"to make it an executable") - @classmethod def validate_dockerfile(cls, files): if "Dockerfile" not in files: @@ -76,7 +66,7 @@ def validate_file(cls, f, files): raise InvalidBundleError(f"Missing file {f} in model directory of source-code bundle") @classmethod - def validate_bundle_structure(cls, directory, req_files): + def validate_bundle_structure(cls, directory, req_files, mode): cls.validate_empty(directory) bundle_root, model_dir = None, None for root, dirs, files in os.walk(directory): @@ -90,34 +80,58 @@ def validate_bundle_structure(cls, directory, req_files): cls.validate_dockerfile(files) cls.validate_model_directory(dirs) model_dir = os.path.join(bundle_root, cls.MODEL) - if cls.MODE: - executable_file = os.path.join(model_dir, cls.MODE) - cls.validate_is_file_executable(executable_file) if root == model_dir: for f in req_files: cls.validate_file(f, files) + if mode: + for file in files: + if file == mode: + file_path = os.path.join(model_dir, mode) + cls.validate_is_file_executable(file_path, mode) + + @classmethod + def validate_is_file_executable(cls, executable_file, mode): + f = open(executable_file) + first_line = f.readline() + if cls.SHEBANG not in first_line: + raise InvalidBundleError(f"The {mode} file cannot be executed. " + f"Please ensure that first line begins with the shebang '#!' " + f"to make it an executable") + @classmethod def validate_inference_bundle_structure(cls, directory: str): req_files = cls.REQUIRED_INFERENCE_FILES - cls.MODE = "serve" - cls.validate_bundle_structure(directory, req_files) + mode = 'serve' + cls.validate_bundle_structure(directory, req_files, mode=mode) @classmethod def validate_notebook_bundle_structure(cls, directory): - cls.validate_bundle_structure(directory, []) + cls.validate_bundle_structure(directory, [], mode=None) @classmethod def validate_train_bundle_structure(cls, directory): req_files = cls.REQUIRED_TRAINING_FILES - cls.MODE = "train" - cls.validate_bundle_structure(directory, req_files) + mode = 'train' + cls.validate_bundle_structure(directory, req_files, mode=mode) @classmethod def validate_source_bundle_structure(cls, directory): req_files = cls.REQUIRED_TRAINING_FILES - cls.validate_bundle_structure(directory, req_files) + cls.validate_bundle_structure(directory, req_files, mode=None) + + # @classmethod + # def validate_is_serve_executable(cls, model_dir: str): + # mode = "serve" + # executable_file = os.path.join(model_dir, mode) + # cls.validate_file_executable(executable_file, mode) + # + # @classmethod + # def validate_is_train_executable(cls, model_dir: str): + # mode = "train" + # executable_file = os.path.join(model_dir, mode) + # cls.validate_file_executable(executable_file, mode) def validate_cpu_request(cpu): From 324826adf6cfc0a37d08b7131988be80f5861a53 Mon Sep 17 00:00:00 2001 From: Shankar M S Date: Thu, 14 Nov 2019 12:01:46 +0100 Subject: [PATCH 10/14] test cases modified --- Pipfile | 12 ++++++++---- backend/kaos_backend/util/tests/test_validators.py | 14 ++++++-------- backend/kaos_backend/util/validators.py | 12 ------------ 3 files changed, 14 insertions(+), 24 deletions(-) diff --git a/Pipfile b/Pipfile index fd19f56..608014f 100644 --- a/Pipfile +++ b/Pipfile @@ -1,11 +1,15 @@ -[[source]] -name = "pypi" -url = "https://pypi.org/simple" verify_ssl = true [dev-packages] +kaos-integration-tests = {editable = true, path = "./testing/integration"} +kaos-backend = {editable = true, path = "./backend"} [packages] +kaos-model = {editable = true, path = "./model"} +kaos = {editable = true, path = "./cli"} [requires] -python_version = "2.7" +python_version = "3.7" + +[pipenv] +allow_prereleases = true \ No newline at end of file diff --git a/backend/kaos_backend/util/tests/test_validators.py b/backend/kaos_backend/util/tests/test_validators.py index 2f3e3b3..d4e2379 100644 --- a/backend/kaos_backend/util/tests/test_validators.py +++ b/backend/kaos_backend/util/tests/test_validators.py @@ -104,8 +104,7 @@ def test_validate_bundle_structure_missing_model_directory(): BundleValidator.validate_bundle_structure(temp_dir, [], mode=None) -@pytest.mark.parametrize("files_include,file_exclude", training_test_cases) -def test_validate_train_bundle_missing_executable_file(files_include, file_exclude): +def test_validate_train_bundle_missing_executable_file(): with pytest.raises(InvalidBundleError, match="The train file cannot be executed. " "Please ensure that first line begins with the shebang '#!' to make it an executable"): @@ -116,16 +115,15 @@ def test_validate_train_bundle_missing_executable_file(files_include, file_exclu model_dir = os.path.join(base_dir, "model") os.mkdir(model_dir) mode = "train" - for f in files_include: + for f in REQ_TRAINING_FILES: create_file(os.path.join(model_dir, f)) train_file = os.path.join(model_dir, mode) - make_executable_file(train_file) + create_file(train_file) BundleValidator.validate_bundle_structure(temp_dir, [], mode=mode) -@pytest.mark.parametrize("files_include,file_exclude", inference_test_cases) -def test_validate_serve_bundle_missing_executable_file(files_include, file_exclude): +def test_validate_serve_bundle_missing_executable_file(): with pytest.raises(InvalidBundleError, match="The serve file cannot be executed. " "Please ensure that first line begins with the shebang '#!' to make it an executable"): @@ -136,11 +134,11 @@ def test_validate_serve_bundle_missing_executable_file(files_include, file_exclu model_dir = os.path.join(base_dir, "model") os.mkdir(model_dir) mode = "serve" - for f in files_include: + for f in REQ_INFERENCE_FILES: create_file(os.path.join(model_dir, f)) serve_file = os.path.join(model_dir, mode) - make_executable_file(serve_file) + create_file(serve_file) BundleValidator.validate_bundle_structure(temp_dir, [], mode=mode) diff --git a/backend/kaos_backend/util/validators.py b/backend/kaos_backend/util/validators.py index 54fc754..0234724 100644 --- a/backend/kaos_backend/util/validators.py +++ b/backend/kaos_backend/util/validators.py @@ -121,18 +121,6 @@ def validate_source_bundle_structure(cls, directory): req_files = cls.REQUIRED_TRAINING_FILES cls.validate_bundle_structure(directory, req_files, mode=None) - # @classmethod - # def validate_is_serve_executable(cls, model_dir: str): - # mode = "serve" - # executable_file = os.path.join(model_dir, mode) - # cls.validate_file_executable(executable_file, mode) - # - # @classmethod - # def validate_is_train_executable(cls, model_dir: str): - # mode = "train" - # executable_file = os.path.join(model_dir, mode) - # cls.validate_file_executable(executable_file, mode) - def validate_cpu_request(cpu): if cpu and MAX_CPU: From 99f7a534674c7c44c3bef290d89d2b20275148cd Mon Sep 17 00:00:00 2001 From: Shankar M S Date: Mon, 20 Jan 2020 13:34:22 +0100 Subject: [PATCH 11/14] used regular expressions to validate shebang --- backend/kaos_backend/util/validators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/kaos_backend/util/validators.py b/backend/kaos_backend/util/validators.py index 0234724..369e07b 100644 --- a/backend/kaos_backend/util/validators.py +++ b/backend/kaos_backend/util/validators.py @@ -32,7 +32,7 @@ class BundleValidator: MODEL = "model" - SHEBANG = "#!" + SHEBANG = re.compile(r"^(#!)") @classmethod def is_empty(cls, directory: str) -> bool: @@ -95,7 +95,7 @@ def validate_bundle_structure(cls, directory, req_files, mode): def validate_is_file_executable(cls, executable_file, mode): f = open(executable_file) first_line = f.readline() - if cls.SHEBANG not in first_line: + if not re.match(cls.SHEBANG, first_line): raise InvalidBundleError(f"The {mode} file cannot be executed. " f"Please ensure that first line begins with the shebang '#!' " f"to make it an executable") From 222822e31c7683fb0f36be8e3481f18a1bba48a2 Mon Sep 17 00:00:00 2001 From: Shankar M S Date: Mon, 20 Jan 2020 13:37:40 +0100 Subject: [PATCH 12/14] removed unused make_executable_file() --- backend/kaos_backend/util/tests/test_validators.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/backend/kaos_backend/util/tests/test_validators.py b/backend/kaos_backend/util/tests/test_validators.py index d4e2379..005def5 100644 --- a/backend/kaos_backend/util/tests/test_validators.py +++ b/backend/kaos_backend/util/tests/test_validators.py @@ -17,14 +17,6 @@ def create_file(path): return temp -def make_executable_file(path): - with open(path, 'w') as executable_file: - line = "#!" - executable_file.write(line) - executable_file.close() - return executable_file - - def remove_el(l, el): ll = l[:] ll.remove(el) From e9fb9a63a9a55d83a2cf139d442552b887a2a76f Mon Sep 17 00:00:00 2001 From: KI-labs Date: Mon, 20 Jan 2020 13:08:58 +0000 Subject: [PATCH 13/14] Version updated to: 1.1.4 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 9c1218c..1b87bcd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.3 \ No newline at end of file +1.1.4 \ No newline at end of file From 08a5ac1d382a5aa87a47f7d67f1891605bc78121 Mon Sep 17 00:00:00 2001 From: KI-labs Date: Tue, 4 Feb 2020 10:01:34 +0000 Subject: [PATCH 14/14] Version updated to: 1.1.5 --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 1b87bcd..314c3d7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.4 \ No newline at end of file +1.1.5 \ No newline at end of file