From 8a3ef5b210e1a5897a9750833d9e2d7ad62bfe51 Mon Sep 17 00:00:00 2001 From: Eshaan Bansal Date: Wed, 22 Dec 2021 21:07:47 +0530 Subject: [PATCH] Fixes #38: allow deletion/cancellation of jobs (#39) * api: Shell2HttpAPI.delete method * tests: TestDeletion testcase * example: add deletion.py * docs: info about deletion.py --- README.md | 27 +++++++++++---------------- docs/source/Examples.md | 1 + docs/source/index.rst | 1 - examples/deletion.py | 29 +++++++++++++++++++++++++++++ flask_shell2http/api.py | 38 +++++++++++++++++++++++++++++++++++++- tests/test_deletion.py | 36 ++++++++++++++++++++++++++++++++++++ 6 files changed, 114 insertions(+), 18 deletions(-) create mode 100644 examples/deletion.py create mode 100644 tests/test_deletion.py diff --git a/README.md b/README.md index f173995..2212cd6 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ _For urgent issues and priority support, visit [https://xscode.com/eshaan7/flask [![codecov](https://codecov.io/gh/Eshaan7/Flask-Shell2HTTP/branch/master/graph/badge.svg?token=UQ43PYQPMR)](https://codecov.io/gh/Eshaan7/flask-shell2http/) [![CodeFactor](https://www.codefactor.io/repository/github/eshaan7/flask-shell2http/badge)](https://www.codefactor.io/repository/github/eshaan7/flask-shell2http) - Language grade: Python +Language grade: Python A minimalist [Flask](https://github.com/pallets/flask) extension that serves as a RESTful/HTTP wrapper for python's subprocess API. @@ -18,23 +18,20 @@ A minimalist [Flask](https://github.com/pallets/flask) extension that serves as - Execute pre-defined shell commands asynchronously and securely via flask's endpoints with dynamic arguments, file upload, callback function capabilities. - Designed for binary to binary/HTTP communication, development, prototyping, remote control and [more](https://flask-shell2http.readthedocs.io/en/stable/Examples.html). - ## Use Cases - Set a script that runs on a succesful POST request to an endpoint of your choice. See [Example code](examples/run_script.py). - Map a base command to an endpoint and pass dynamic arguments to it. See [Example code](examples/basic.py). - Can also process multiple uploaded files in one command. See [Example code](examples/multiple_files.py). - This is useful for internal docker-to-docker communications if you have different binaries distributed in micro-containers. See [real-life example](https://github.com/intelowlproject/IntelOwl/blob/master/integrations/static_analyzers/app.py). -- You can define a callback function/ use signals to listen for process completion. See [Example code](examples/with_callback.py). - * Maybe want to pass some additional context to the callback function ? - * Maybe intercept on completion and update the result ? See [Example code](examples/custom_save_fn.py) +- You can define a callback function/ use signals to listen for process completion. See [Example code](examples/with_callback.py). + - Maybe want to pass some additional context to the callback function ? + - Maybe intercept on completion and update the result ? See [Example code](examples/custom_save_fn.py) - You can also apply [View Decorators](https://flask.palletsprojects.com/en/1.1.x/patterns/viewdecorators/) to the exposed endpoint. See [Example code](examples/with_decorators.py). -- Currently, all commands run asynchronously (default timeout is 3600 seconds), so result is not available directly. An option _may_ be provided for this in future releases for commands that return immediately. > Note: This extension is primarily meant for executing long-running > shell commands/scripts (like nmap, code-analysis' tools) in background from an HTTP request and getting the result at a later time. - ## Documentation [![Documentation Status](https://readthedocs.org/projects/flask-shell2http/badge/?version=latest)](https://flask-shell2http.readthedocs.io/en/latest/?badge=latest) @@ -43,14 +40,13 @@ A minimalist [Flask](https://github.com/pallets/flask) extension that serves as - I also highly recommend the [Examples](https://flask-shell2http.readthedocs.io/en/stable/Examples.html) section. - [CHANGELOG](https://github.com/eshaan7/Flask-Shell2HTTP/blob/master/.github/CHANGELOG.md). - ## Quick Start ##### Dependencies -* Python: `>=v3.6` -* [Flask](https://pypi.org/project/Flask/) -* [Flask-Executor](https://pypi.org/project/Flask-Executor) +- Python: `>=v3.6` +- [Flask](https://pypi.org/project/Flask/) +- [Flask-Executor](https://pypi.org/project/Flask-Executor) ##### Installation @@ -109,9 +105,9 @@ returns JSON, ```json { - "key": "ddbe0a94", - "result_url": "http://localhost:4000/commands/saythis?key=ddbe0a94&wait=false", - "status": "running" + "key": "ddbe0a94", + "result_url": "http://localhost:4000/commands/saythis?key=ddbe0a94&wait=false", + "status": "running" } ``` @@ -131,11 +127,10 @@ Returns result in JSON, "end_time": 1593019807.782958, "process_time": 0.00748753547668457, "returncode": 0, - "error": null, + "error": null } ``` - ## Inspiration This was initially made to integrate various command-line tools easily with [Intel Owl](https://github.com/intelowlproject/IntelOwl), which I am working on as part of Google Summer of Code. diff --git a/docs/source/Examples.md b/docs/source/Examples.md index d16e169..c4b2b97 100644 --- a/docs/source/Examples.md +++ b/docs/source/Examples.md @@ -9,3 +9,4 @@ I have created some example python scripts to demonstrate various use-cases. The - [with_signals.py](https://github.com/Eshaan7/Flask-Shell2HTTP/blob/master/examples/with_signals.py): Using [Flask Signals](https://flask.palletsprojects.com/en/1.1.x/signals/) as callback function. - [with_decorators.py](https://github.com/Eshaan7/Flask-Shell2HTTP/blob/master/examples/with_decorators.py): Shows how to apply [View Decorators](https://flask.palletsprojects.com/en/1.1.x/patterns/viewdecorators/) to the exposed endpoint. Useful in case you wish to apply authentication, caching, etc. to the endpoint. - [custom_save_fn.py](https://github.com/Eshaan7/Flask-Shell2HTTP/blob/master/examples/custom_save_fn.py): There may be cases where the process doesn't print result to standard output but to a file/database. This example shows how to pass additional context to the callback function, intercept the future object after completion and update it's result attribute before it's ready to be consumed. +- [deletion.py](https://github.com/Eshaan7/Flask-Shell2HTTP/blob/master/examples/deletion.py): Example demonstrating how to request cancellation/deletion of an already running job. diff --git a/docs/source/index.rst b/docs/source/index.rst index 8f31194..c6922e3 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -23,7 +23,6 @@ A minimalist Flask_ extension that serves as a RESTful/HTTP wrapper for python's - This is useful for internal docker-to-docker communications if you have different binaries distributed in micro-containers. - You can define a callback function/ use signals to listen for process completion. - You can also apply View Decorators to the exposed endpoint. -- Currently, all commands run asynchronously (default timeout is 3600 seconds), so result is not available directly. An option _may_ be provided for this in future release. `Note: This extension is primarily meant for executing long-running shell commands/scripts (like nmap, code-analysis' tools) in background from an HTTP request and getting the result at a later time.` diff --git a/examples/deletion.py b/examples/deletion.py new file mode 100644 index 0000000..426a8f5 --- /dev/null +++ b/examples/deletion.py @@ -0,0 +1,29 @@ +# web imports +from flask import Flask +from flask_executor import Executor +from flask_shell2http import Shell2HTTP + +# Flask application instance +app = Flask(__name__) + +# application factory +executor = Executor(app) +shell2http = Shell2HTTP(app, executor) + + +shell2http.register_command( + endpoint="sleep", + command_name="sleep", +) + + +# Test Runner +if __name__ == "__main__": + app.testing = True + c = app.test_client() + # request new process + r1 = c.post("/sleep", json={"args": ["10"], "force_unique_key": True}) + print(r1) + # request cancellation + r2 = c.delete(f"/sleep?key={r1.get_json()['key']}") + print(r2) diff --git a/flask_shell2http/api.py b/flask_shell2http/api.py index 6e85e8c..4820c68 100644 --- a/flask_shell2http/api.py +++ b/flask_shell2http/api.py @@ -28,7 +28,7 @@ class Shell2HttpAPI(MethodView): """ - ``Flask.MethodView`` that registers ``GET`` and ``POST`` + ``Flask.MethodView`` that creates ``GET``, ``POST`` and ``DELETE`` methods for a given endpoint. This is invoked on ``Shell2HTTP.register_command``. @@ -37,6 +37,7 @@ class Shell2HttpAPI(MethodView): def get(self): """ + Get report by job key. Args: key (str): - Future key @@ -138,6 +139,41 @@ def post(self): response_dict["result_url"] = self.__build_result_url(key) return make_response(jsonify(response_dict), HTTPStatus.BAD_REQUEST) + def delete(self): + """ + Cancel (if running) and delete job by job key. + Args: + key (str): + - Future key + """ + try: + key = request.args.get("key") + logger.info( + f"Job: '{key}' --> deletion requested. " + f"Requester: '{request.remote_addr}'." + ) + if not key: + raise Exception("No key provided in arguments.") + + # get the future object + future: Future = self.executor.futures._futures.get(key) + if not future: + raise JobNotFoundException(f"No job exists for key: '{key}'.") + + # cancel and delete from memory + future.cancel() + self.executor.futures.pop(key) + + return make_response({}, HTTPStatus.NO_CONTENT) + + except JobNotFoundException as e: + logger.error(e) + return make_response(jsonify(error=str(e)), HTTPStatus.NOT_FOUND) + + except Exception as e: + logger.error(e) + return make_response(jsonify(error=str(e)), HTTPStatus.BAD_REQUEST) + @classmethod def __build_result_url(cls, key: str) -> str: return f"{request.base_url}?key={key}&wait=false" diff --git a/tests/test_deletion.py b/tests/test_deletion.py new file mode 100644 index 0000000..57f303b --- /dev/null +++ b/tests/test_deletion.py @@ -0,0 +1,36 @@ +from examples.deletion import app + +from tests._utils import CustomTestCase + + +class TestDeletion(CustomTestCase): + uri = "/sleep" + + def create_app(self): + app.config["TESTING"] = True + return app + + def test_delete__204(self): + # create command process + r1 = self.client.post(self.uri, json={"args": ["10"], "force_unique_key": True}) + r1_json = r1.get_json() + self.assertStatus(r1, 202) + # request cancellation: correct key + r2 = self.client.delete(f"{self.uri}?key={r1_json['key']}") + self.assertStatus(r2, 204) + + def test_delete__400(self): + # create command process + r1 = self.client.post(self.uri, json={"args": ["10"], "force_unique_key": True}) + self.assertStatus(r1, 202) + # request cancellation: no key + r2 = self.client.delete(f"{self.uri}?key=") + self.assertStatus(r2, 400) + + def test_delete__404(self): + # create command process + r1 = self.client.post(self.uri, json={"args": ["10"], "force_unique_key": True}) + self.assertStatus(r1, 202) + # request cancellation: invalid key + r2 = self.client.delete(f"{self.uri}?key=abcdefg") + self.assertStatus(r2, 404)