diff --git a/Pipfile b/Pipfile index efae643..608014f 100644 --- a/Pipfile +++ b/Pipfile @@ -1,6 +1,3 @@ -[[source]] -name = "pypi" -url = "https://pypi.org/simple" verify_ssl = true [dev-packages] @@ -15,4 +12,4 @@ kaos = {editable = true, path = "./cli"} python_version = "3.7" [pipenv] -allow_prereleases = true +allow_prereleases = true \ No newline at end of file diff --git a/Pipfile.lock b/Pipfile.lock index 8223728..4985674 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "cc580666520e49206414747789824998c089f82acbaa295c272a4313d508ede7" + "sha256": "ae4bdd7d4157baab65ae9d0e8389a6011e6b640995372c45ec81fa5d1ddfae9f" }, "pipfile-spec": 6, "requires": { - "python_version": "3.7" + "python_version": "2.7" }, "sources": [ { @@ -15,6 +15,7 @@ } ] }, + "default": { "certifi": { "hashes": [ diff --git a/VERSION b/VERSION index 1b87bcd..314c3d7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.1.4 \ No newline at end of file +1.1.5 \ No newline at end of file diff --git a/backend/kaos_backend/controllers/tests/__init__.py b/backend/kaos_backend/controllers/tests/__init__.py index 3b3de47..160b670 100644 --- a/backend/kaos_backend/controllers/tests/__init__.py +++ b/backend/kaos_backend/controllers/tests/__init__.py @@ -39,8 +39,12 @@ def check_pipeline_exists_mock(pipeline_name): def create_test_file(dirname, filename): - f = open(os.path.join(dirname, filename), "w") - f.write("this is fake") + if filename in ['train', 'serve']: + f = open(os.path.join(dirname, filename), "w") + f.write("#!") + else: + f = open(os.path.join(dirname, filename), "w") + f.write("this is fake") f.close() return f diff --git a/backend/kaos_backend/util/tests/test_validators.py b/backend/kaos_backend/util/tests/test_validators.py index 411d679..005def5 100644 --- a/backend/kaos_backend/util/tests/test_validators.py +++ b/backend/kaos_backend/util/tests/test_validators.py @@ -52,7 +52,7 @@ def test_is_not_empty(): def test_validate_bundle_structure_is_empty(): with pytest.raises(InvalidBundleError, match="Bundle must be non-empty"): with TemporaryDirectory() as temp_dir: - BundleValidator.validate_bundle_structure(temp_dir, []) + BundleValidator.validate_bundle_structure(temp_dir, [], mode=None) def test_validate_bundle_structure_missing_root_directory(): @@ -60,7 +60,7 @@ def test_validate_bundle_structure_missing_root_directory(): with TemporaryDirectory() as temp_dir: # temp file to avoid empty file exception temp = tempfile.NamedTemporaryFile(dir=temp_dir, delete=True) - BundleValidator.validate_bundle_structure(temp_dir, []) + BundleValidator.validate_bundle_structure(temp_dir, [], mode=None) temp.close() @@ -72,7 +72,7 @@ def test_validate_bundle_structure_too_many_directories(): tempfile.mkdtemp(dir=temp_dir) tempfile.mkdtemp(dir=temp_dir) - BundleValidator.validate_bundle_structure(temp_dir, []) + BundleValidator.validate_bundle_structure(temp_dir, [], mode=None) temp.close() @@ -83,7 +83,7 @@ def test_validate_bundle_structure_missing_dockerfile_in_directory(): temp = tempfile.NamedTemporaryFile(dir=temp_dir, delete=True) tempfile.mkdtemp(dir=temp_dir) - BundleValidator.validate_bundle_structure(temp_dir, []) + BundleValidator.validate_bundle_structure(temp_dir, [], mode=None) temp.close() @@ -93,7 +93,45 @@ def test_validate_bundle_structure_missing_model_directory(): base_dir = tempfile.mkdtemp(dir=temp_dir) filename = os.path.join(base_dir, "Dockerfile") create_file(filename) - BundleValidator.validate_bundle_structure(temp_dir, []) + BundleValidator.validate_bundle_structure(temp_dir, [], mode=None) + + +def test_validate_train_bundle_missing_executable_file(): + with pytest.raises(InvalidBundleError, + match="The train file cannot be executed. " + "Please ensure that first line begins with the shebang '#!' to make it an executable"): + with TemporaryDirectory() as temp_dir: + base_dir = tempfile.mkdtemp(dir=temp_dir) + dockerfile = os.path.join(base_dir, "Dockerfile") + create_file(dockerfile) + model_dir = os.path.join(base_dir, "model") + os.mkdir(model_dir) + mode = "train" + for f in REQ_TRAINING_FILES: + create_file(os.path.join(model_dir, f)) + + train_file = os.path.join(model_dir, mode) + create_file(train_file) + BundleValidator.validate_bundle_structure(temp_dir, [], mode=mode) + + +def test_validate_serve_bundle_missing_executable_file(): + with pytest.raises(InvalidBundleError, + match="The serve file cannot be executed. " + "Please ensure that first line begins with the shebang '#!' to make it an executable"): + with TemporaryDirectory() as temp_dir: + base_dir = tempfile.mkdtemp(dir=temp_dir) + dockerfile = os.path.join(base_dir, "Dockerfile") + create_file(dockerfile) + model_dir = os.path.join(base_dir, "model") + os.mkdir(model_dir) + mode = "serve" + for f in REQ_INFERENCE_FILES: + create_file(os.path.join(model_dir, f)) + + serve_file = os.path.join(model_dir, mode) + create_file(serve_file) + BundleValidator.validate_bundle_structure(temp_dir, [], mode=mode) @pytest.mark.parametrize("files_include,file_exclude", inference_test_cases) diff --git a/backend/kaos_backend/util/validators.py b/backend/kaos_backend/util/validators.py index 21a72d0..bf90880 100644 --- a/backend/kaos_backend/util/validators.py +++ b/backend/kaos_backend/util/validators.py @@ -24,6 +24,7 @@ class BundleValidator: + REQUIRED_INFERENCE_FILES = ["__init__.py", "serve", "web-requirements.txt"] @@ -34,6 +35,8 @@ class BundleValidator: MODEL = "model" + SHEBANG = re.compile(r"^(#!)") + @classmethod def is_empty(cls, directory: str) -> bool: return all(len(files) == 0 for root, _, files in os.walk(directory)) @@ -66,7 +69,7 @@ def validate_file(cls, f, files): raise InvalidBundleError(f"Missing file {f} in model directory of source-code bundle") @classmethod - def validate_bundle_structure(cls, directory, req_files): + def validate_bundle_structure(cls, directory, req_files, mode): cls.validate_empty(directory) bundle_root, model_dir = None, None for root, dirs, files in os.walk(directory): @@ -85,19 +88,41 @@ def validate_bundle_structure(cls, directory, req_files): for f in req_files: cls.validate_file(f, files) + if mode: + for file in files: + if file == mode: + file_path = os.path.join(model_dir, mode) + cls.validate_is_file_executable(file_path, mode) + + @classmethod + def validate_is_file_executable(cls, executable_file, mode): + f = open(executable_file) + first_line = f.readline() + if not re.match(cls.SHEBANG, first_line): + raise InvalidBundleError(f"The {mode} file cannot be executed. " + f"Please ensure that first line begins with the shebang '#!' " + f"to make it an executable") + @classmethod def validate_inference_bundle_structure(cls, directory: str): req_files = cls.REQUIRED_INFERENCE_FILES - cls.validate_bundle_structure(directory, req_files) + mode = 'serve' + cls.validate_bundle_structure(directory, req_files, mode=mode) @classmethod def validate_notebook_bundle_structure(cls, directory): - cls.validate_bundle_structure(directory, []) + cls.validate_bundle_structure(directory, [], mode=None) @classmethod def validate_train_bundle_structure(cls, directory): req_files = cls.REQUIRED_TRAINING_FILES - cls.validate_bundle_structure(directory, req_files) + mode = 'train' + cls.validate_bundle_structure(directory, req_files, mode=mode) + + @classmethod + def validate_source_bundle_structure(cls, directory): + req_files = cls.REQUIRED_TRAINING_FILES + cls.validate_bundle_structure(directory, req_files, mode=None) def validate_cpu_request(cpu):