From 946b75c619beee93b7b6b52c1e6e73912deb09c0 Mon Sep 17 00:00:00 2001 From: AnsibleGuy Date: Wed, 17 Jan 2024 19:14:50 +0100 Subject: [PATCH] fixes for running the app via 'python3 -m ansible-webui' --- .github/workflows/test.yml | 34 ++++++++++++++++++- CONTRIBUTE.md | 5 +++ ansible-webui/__init__.py | 15 --------- ansible-webui/__main__.py | 24 +++++++++++++ ansible-webui/aw/config/main.py | 4 +++ ansible-webui/aw/execute/play.py | 2 +- ansible-webui/aw/execute/util.py | 5 ++- ansible-webui/aw/main.py | 6 ---- ansible-webui/aw/model/job.py | 4 ++- ansible-webui/aw/settings.py | 1 - ansible-webui/aw/utils/handlers.py | 2 +- ansible-webui/base/scheduler.py | 17 +++++----- ansible-webui/{aw => base}/serve_static.py | 0 ansible-webui/base/threader.py | 32 ++++++++++-------- ansible-webui/base/webserver.py | 6 ++-- ansible-webui/main.py | 27 +++++++++------ ansible-webui/manage.py | 14 +------- ansible-webui/route.py | 2 +- docs/source/usage/1_install.rst | 13 -------- requirements_build.txt | 1 + scripts/build.sh | 6 ++-- scripts/run_pip_build.sh | 39 ++++++++++++++++++++++ scripts/run_shared.sh | 13 ++++---- 23 files changed, 173 insertions(+), 99 deletions(-) create mode 100644 ansible-webui/__main__.py rename ansible-webui/{aw => base}/serve_static.py (100%) create mode 100644 scripts/run_pip_build.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index faa9b3a..41f004e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ on: jobs: build: runs-on: ubuntu-latest - timeout-minutes: 1 + timeout-minutes: 3 steps: - name: Checkout @@ -35,6 +35,7 @@ jobs: - name: Install dependencies run: | pip install -r requirements_test.txt + pip install -r requirements_build.txt pip install -r requirements.txt shell: bash @@ -54,6 +55,37 @@ jobs: shell: bash working-directory: ansible-webui/ + - name: Testing to start Ansible-WebUI + run: | + timeout 2 python3 ansible-webui + ec="$?" + if [[ "$ec" != "124" ]] + then + exit 1 + fi + shell: bash + - name: Running Tests run: python3 -m pytest shell: bash + + - name: Testing to build Ansible-WebUI with PIP + run: | + path_repo="$(pwd)" + cd /tmp + tmp_venv="/tmp/ansible-webui-venv/$(date +%s)" + python3 -m virtualenv "$tmp_venv" >/dev/null + source "${tmp_venv}/bin/activate" + python3 -m pip install -e "$path_repo" >/dev/null + + timeout 2 python3 -m ansible-webui + ec="$?" + + deactivate + rm -rf "$tmp_venv" + + if [[ "$ec" != "124" ]] + then + exit 1 + fi + shell: bash diff --git a/CONTRIBUTE.md b/CONTRIBUTE.md index 305eb9b..743d9f8 100644 --- a/CONTRIBUTE.md +++ b/CONTRIBUTE.md @@ -19,6 +19,11 @@ Admin user for testing: * User: `ansible` * Pwd: `automateMe` +Test to build the app using PIP: +```bash +bash ${REPO}/scripts/run_pip_build.sh +``` + Run tests and lint: ```bash diff --git a/ansible-webui/__init__.py b/ansible-webui/__init__.py index 1ef8918..e69de29 100644 --- a/ansible-webui/__init__.py +++ b/ansible-webui/__init__.py @@ -1,15 +0,0 @@ -#!/usr/bin/env python3 - -if __name__ == '__main__': - # pylint: disable=E0401 - from sys import argv as sys_argv - from sys import exit as sys_exit - from main import main - from aw.config.hardcoded import VERSION - - if len(sys_argv) > 1: - if sys_argv[1] == 'version': - print(VERSION) - sys_exit(0) - - main() diff --git a/ansible-webui/__main__.py b/ansible-webui/__main__.py new file mode 100644 index 0000000..02b0cf8 --- /dev/null +++ b/ansible-webui/__main__.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 + +if __name__ == '__main__': + # pylint: disable=E0401 + from sys import argv as sys_argv + from sys import exit as sys_exit + from sys import path as sys_path + from os import path as os_path + + try: + from main import main + + except ModuleNotFoundError: + sys_path.append(os_path.dirname(os_path.abspath(__file__))) + from main import main + + from aw.config.hardcoded import VERSION + + if len(sys_argv) > 1: + if sys_argv[1] == 'version': + print(VERSION) + sys_exit(0) + + main() diff --git a/ansible-webui/aw/config/main.py b/ansible-webui/aw/config/main.py index e3b323a..46d45e8 100644 --- a/ansible-webui/aw/config/main.py +++ b/ansible-webui/aw/config/main.py @@ -9,6 +9,10 @@ def init_globals(): global config config = {} + environ.setdefault('DJANGO_SETTINGS_MODULE', 'aw.settings') + environ['PYTHONIOENCODING'] = 'utf8' + environ['PYTHONUNBUFFERED'] = '1' + for cnf_key, values in ENVIRON_FALLBACK.items(): for env_key in values['keys']: if env_key in environ: diff --git a/ansible-webui/aw/execute/play.py b/ansible-webui/aw/execute/play.py index 4738fef..275d2f1 100644 --- a/ansible-webui/aw/execute/play.py +++ b/ansible-webui/aw/execute/play.py @@ -5,7 +5,7 @@ from aw.execute.util import runner_cleanup, runner_prep, parse_run_result -def ansible_playbook(job: Job, execution: JobExecution): +def ansible_playbook(job: Job, execution: (JobExecution, None)): time_start = datetime.now() opts = runner_prep(job=job, execution=execution) diff --git a/ansible-webui/aw/execute/util.py b/ansible-webui/aw/execute/util.py index 7c1684b..45f31ce 100644 --- a/ansible-webui/aw/execute/util.py +++ b/ansible-webui/aw/execute/util.py @@ -74,7 +74,10 @@ def _runner_options(job: Job, execution: JobExecution) -> dict: } -def runner_prep(job: Job, execution: JobExecution): +def runner_prep(job: Job, execution: (JobExecution, None)): + if execution is None: + execution = JobExecution(user=None, job=job, comment='Scheduled') + _update_execution_status(execution, status='Starting') opts = _runner_options(job=job, execution=execution) diff --git a/ansible-webui/aw/main.py b/ansible-webui/aw/main.py index 30db648..e08958d 100644 --- a/ansible-webui/aw/main.py +++ b/ansible-webui/aw/main.py @@ -1,9 +1,3 @@ -from os import environ - from django.core.wsgi import get_wsgi_application -environ.setdefault('DJANGO_SETTINGS_MODULE', 'aw.settings') -environ['PYTHONIOENCODING'] = 'utf8' -environ['PYTHONUNBUFFERED'] = '1' - app = get_wsgi_application() diff --git a/ansible-webui/aw/model/job.py b/ansible-webui/aw/model/job.py index 14a4006..a71ff0e 100644 --- a/ansible-webui/aw/model/job.py +++ b/ansible-webui/aw/model/job.py @@ -123,8 +123,9 @@ def __str__(self) -> str: class JobExecution(MetaJob): + # NOTE: scheduled execution will have no user user = models.ForeignKey( - settings.AUTH_USER_MODEL, on_delete=models.PROTECT, blank=True, null=True, + settings.AUTH_USER_MODEL, on_delete=models.PROTECT, null=True, related_name=f"jobexec_fk_user" ) job = models.ForeignKey(Job, on_delete=models.CASCADE, related_name=f"jobexec_fk_job") @@ -133,6 +134,7 @@ class JobExecution(MetaJob): null=True, default=None, # execution is created before result is available ) status = models.PositiveSmallIntegerField(default=0, choices=CHOICES_JOB_EXEC_STATUS) + comment = models.CharField(max_length=300, null=True, default=None) def __str__(self) -> str: status_name = CHOICES_JOB_EXEC_STATUS[int(self.status)][1] diff --git a/ansible-webui/aw/settings.py b/ansible-webui/aw/settings.py index b1d8542..d580eae 100644 --- a/ansible-webui/aw/settings.py +++ b/ansible-webui/aw/settings.py @@ -14,7 +14,6 @@ DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' X_FRAME_OPTIONS = 'SAMEORIGIN' -# Application definition INSTALLED_APPS = [ 'aw.apps.AwConfig', 'django.contrib.admin', diff --git a/ansible-webui/aw/utils/handlers.py b/ansible-webui/aw/utils/handlers.py index 8fb9bbb..e63f723 100644 --- a/ansible-webui/aw/utils/handlers.py +++ b/ansible-webui/aw/utils/handlers.py @@ -1,6 +1,6 @@ from django.shortcuts import render -from utils.debug import log +from aw.utils.debug import log class AnsibleConfigError(Exception): diff --git a/ansible-webui/base/scheduler.py b/ansible-webui/base/scheduler.py index e877fad..025c6aa 100644 --- a/ansible-webui/base/scheduler.py +++ b/ansible-webui/base/scheduler.py @@ -1,18 +1,16 @@ from time import sleep, time -from base.threader import Loop as Threader +from base.threader import ThreadManager from aw.utils.debug import log from aw.config.hardcoded import RELOAD_INTERVAL - -# NOTE: not able to use models here because of dependency on django-init -# from aw.model.job import Job +from aw.model.job import Job, JobExecution class Scheduler: WAIT_TIME = 1 def __init__(self): - self.threader = Threader() + self.thread_manager = ThreadManager() self.stopping = False self.reloading = False @@ -23,7 +21,7 @@ def stop(self, signum=None): self.stopping = True log('Stopping job-threads', level=6) - self.threader.stop() + self.thread_manager.stop() sleep(self.WAIT_TIME) log('Finished!') @@ -45,9 +43,10 @@ def reload(self, signum=None): def _signum_log(signum): log(f'Scheduler got signal {signum}') - def _thread(self, job): - self.threader.add_thread(job) - self.threader.start_thread(job) + def _thread(self, job: Job, execution: JobExecution = None): + # todo: creation of execution object for ad-hoc execution (with user-provided values) + self.thread_manager.add_thread(job) + self.thread_manager.start_thread(job) def start(self): log('Starting..', level=3) diff --git a/ansible-webui/aw/serve_static.py b/ansible-webui/base/serve_static.py similarity index 100% rename from ansible-webui/aw/serve_static.py rename to ansible-webui/base/serve_static.py diff --git a/ansible-webui/base/threader.py b/ansible-webui/base/threader.py index da132ac..04c613e 100644 --- a/ansible-webui/base/threader.py +++ b/ansible-webui/base/threader.py @@ -7,17 +7,16 @@ from aw.utils.debug import log from aw.config import hardcoded +from aw.model.job import Job, JobExecution from aw.execute.play import ansible_playbook -# NOTE: not able to use models here because of dependency on django-init -# from aw.model.job import Job - class Workload(Thread): - def __init__(self, job, loop_instance, name: str, once: bool = False, daemon: bool = True): + def __init__(self, job: Job, manager, name: str, execution: JobExecution, once: bool = False, daemon: bool = True): Thread.__init__(self, daemon=daemon, name=name) self.job = job - self.loop_instance = loop_instance + self.execution = execution + self.manager = manager self.once = once self.started = False self.state_stop = Event() @@ -40,13 +39,17 @@ def stop(self) -> bool: self.started = False return True + def run_playbook(self): + ansible_playbook(job=self.job, execution=self.execution) + def run(self) -> None: self.started = True log(f"Entering runtime of thread {self.log_name}", level=7) try: if self.once: - ansible_playbook(self.job) - Loop.stop_thread(self.loop_instance, self.job) + self.run_playbook() + self.stop() + self.manager.threads.remove(self.job) return else: @@ -59,14 +62,14 @@ def run(self) -> None: else: log(f"Starting job {self.log_name}", level=5) - ansible_playbook(self.job) + self.run_playbook() except ValueError as err: log(f"Got unexpected error while executing job {self.log_name}: '{err}'") self.run() -class Loop: +class ThreadManager: def __init__(self): self.threads = set() self.thread_nr = 0 @@ -79,13 +82,14 @@ def start(self) -> None: if not thread.started: thread.start() - def add_thread(self, job, once: bool = False): + def add_thread(self, job: Job, execution: JobExecution = None, once: bool = False): log(f"Adding thread for \"{job.name}\" with schedule \"{job.schedule}\"", level=7) self.thread_nr += 1 self.threads.add( Workload( job=job, - loop_instance=self, + execution=execution, + manager=self, once=once, name=f"Thread #{self.thread_nr}", ) @@ -108,7 +112,7 @@ def stop(self) -> bool: log('All threads stopped', level=3) return True - def stop_thread(self, job): + def stop_thread(self, job: Job): log(f"Stopping thread for \"{job.name}\"", level=6) for thread in self.threads: if thread.job.job_id == job.job_id: @@ -119,7 +123,7 @@ def stop_thread(self, job): del job break - def start_thread(self, job) -> None: + def start_thread(self, job: Job) -> None: for thread in self.threads: if thread.job.job_id == job.job_id: if not thread.started: @@ -127,7 +131,7 @@ def start_thread(self, job) -> None: log(f"Thread {job.name} started.", level=5) break - def replace_thread(self, job) -> None: + def replace_thread(self, job: Job) -> None: log(f"Reloading thread for \"{job.name}\"", level=6) self.stop_thread(job) self.add_thread(job) diff --git a/ansible-webui/base/webserver.py b/ansible-webui/base/webserver.py index a449501..5516ee2 100644 --- a/ansible-webui/base/webserver.py +++ b/ansible-webui/base/webserver.py @@ -37,7 +37,7 @@ def load_config(self): self.cfg.set(key.lower(), value) -def create_webserver() -> WSGIApplication: +def init_webserver(): gunicorn.SERVER = ''.join(random_choice(ascii_letters) for _ in range(10)) run_options = { 'workers': (cpu_count() * 2) + 1, @@ -47,7 +47,7 @@ def create_webserver() -> WSGIApplication: warn_if_development() run_options = {**run_options, **OPTIONS_DEV} - return StandaloneApplication( + StandaloneApplication( app_uri="aw.main:app", options=run_options - ) + ).run() diff --git a/ansible-webui/main.py b/ansible-webui/main.py index 4461181..081d40c 100644 --- a/ansible-webui/main.py +++ b/ansible-webui/main.py @@ -1,31 +1,43 @@ import signal -from platform import uname -from threading import Thread from os import getpid from os import kill as os_kill from time import sleep from sys import exit as sys_exit +from threading import Thread +from platform import uname # pylint: disable=E0401,C0413 from gunicorn.arbiter import Arbiter +from django import setup as django_setup from aw.config.main import init_globals init_globals() -from base.webserver import create_webserver -from base.scheduler import Scheduler +from base.webserver import init_webserver def main(): if uname().system.lower() != 'linux': raise SystemError('Currently only linux systems are supported!') + django_setup() + + init_scheduler() + init_webserver() + + +def init_scheduler(): + # pylint: disable=C0415 + from base.scheduler import Scheduler scheduler = Scheduler() scheduler_thread = Thread(target=scheduler.start) - webserver = create_webserver() # override gunicorn signal handling to allow for graceful shutdown + Arbiter.SIGNALS.remove(signal.SIGHUP) + Arbiter.SIGNALS.remove(signal.SIGINT) + Arbiter.SIGNALS.remove(signal.SIGTERM) + def signal_exit(signum=None, stack=None): del stack scheduler.stop(signum) @@ -37,13 +49,8 @@ def signal_reload(signum=None, stack=None): del stack scheduler.reload(signum) - Arbiter.SIGNALS.remove(signal.SIGHUP) - Arbiter.SIGNALS.remove(signal.SIGINT) - Arbiter.SIGNALS.remove(signal.SIGTERM) - signal.signal(signal.SIGHUP, signal_reload) signal.signal(signal.SIGINT, signal_exit) signal.signal(signal.SIGTERM, signal_exit) scheduler_thread.start() - webserver.run() diff --git a/ansible-webui/manage.py b/ansible-webui/manage.py index 15ed2ef..e5899a3 100644 --- a/ansible-webui/manage.py +++ b/ansible-webui/manage.py @@ -1,28 +1,16 @@ #!/usr/bin/env python3 -from os import environ from sys import argv as sys_argv def main(): - environ.setdefault('DJANGO_SETTINGS_MODULE', 'aw.settings') - # pylint: disable=E0401,C0415 from aw.config.main import init_globals init_globals() from aw.utils.deployment import warn_if_development warn_if_development() - try: - # pylint: disable=C0415 - from django.core.management import execute_from_command_line - - except ImportError as exc: - raise ImportError( - "Couldn't import Django. Are you sure it's installed and " - "available on your PYTHONPATH environment variable? Did you " - "forget to activate a virtual environment?" - ) from exc + from django.core.management import execute_from_command_line execute_from_command_line(sys_argv) diff --git a/ansible-webui/route.py b/ansible-webui/route.py index abdaee4..09874e6 100644 --- a/ansible-webui/route.py +++ b/ansible-webui/route.py @@ -5,9 +5,9 @@ from django.conf.urls import include from django.contrib import admin +from base.serve_static import urlpatterns_static from aw.route import ui, catchall, logout, not_implemented, manage from aw.config.hardcoded import ENV_KEY_SERVE_STATIC -from aw.serve_static import urlpatterns_static from aw.utils.deployment import deployment_dev urlpatterns = [] diff --git a/docs/source/usage/1_install.rst b/docs/source/usage/1_install.rst index 975fd11..15f3363 100644 --- a/docs/source/usage/1_install.rst +++ b/docs/source/usage/1_install.rst @@ -58,15 +58,9 @@ Without a virtual environment: [Service] Type=simple - Environment=AW_REPO="https://raw.githubusercontent.com/ansibleguy/ansible-webui" Environment=LANG="en_US.UTF-8" Environment=LC_ALL="en_US.UTF-8" - # making sure the current requirements are installed - ExecStartPre=/bin/bash -c 'rm -f /tmp/aw_requirements.txt \ - && wget -O /tmp/aw_requirements.txt "${AW_REPO}/$(python3 -m ansible-webui version)/requirements.txt" \ - && python3 -m pip install --upgrade -r /tmp/aw_requirements.txt' - ExecStart=/usr/bin/python3 -m ansible-webui User=ansible-webui @@ -95,16 +89,9 @@ When using a virtual environment: (recommended) [Service] Type=simple - Environment=AW_REPO="https://raw.githubusercontent.com/ansibleguy/ansible-webui" Environment=LANG="en_US.UTF-8" Environment=LC_ALL="en_US.UTF-8" - # making sure the current requirements are installed - ExecStartPre=/bin/bash -c 'source /home/ansible-webui/venv/bin/activate \ - && rm -f /tmp/aw_requirements.txt \ - && wget -O /tmp/aw_requirements.txt "${AW_REPO}/$(python3 -m ansible-webui version)/requirements.txt" \ - && python3 -m pip install --upgrade -r /tmp/aw_requirements.txt' - ExecStart=/bin/bash -c 'source /home/ansible-webui/venv/bin/activate \ && /usr/bin/python3 -m ansible-webui' diff --git a/requirements_build.txt b/requirements_build.txt index 378eac2..c1d9aaa 100644 --- a/requirements_build.txt +++ b/requirements_build.txt @@ -1 +1,2 @@ build +virtualenv diff --git a/scripts/build.sh b/scripts/build.sh index 27cffb4..23cae9e 100644 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -4,7 +4,7 @@ set -e cd "$(dirname "$0")/.." rm -rf dist/* -python3 -m build - -#python3 -m twine upload --repository pypi dist/* +python3 -m pip install -r ./requirements_build.txt >/dev/null +python3 -m build +# python3 -m twine upload --repository pypi dist/* diff --git a/scripts/run_pip_build.sh b/scripts/run_pip_build.sh new file mode 100644 index 0000000..51839f8 --- /dev/null +++ b/scripts/run_pip_build.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -e + +cd "$(dirname "$0")/.." +path_repo="$(pwd)" +tmp_venv="/tmp/ansible-webui-venv/$(date +%s)" + +echo '' +echo 'Installing requirements' +echo '' + +python3 -m pip install -r ./requirements_build.txt >/dev/null + +echo '' +echo "Creating virtualenv ${tmp_venv}" +echo '' + +python3 -m virtualenv "$tmp_venv" >/dev/null +source "${tmp_venv}/bin/activate" + +echo '' +echo 'Building & Installing Module using PIP' +echo '' + +python3 -m pip install -e "$path_repo" >/dev/null + +echo '' +echo 'Starting app' +echo '' + +python3 -m ansible-webui + +echo '' +echo "Removing virtualenv ${tmp_venv}" +echo '' + +deactivate +rm -rf "$tmp_venv" diff --git a/scripts/run_shared.sh b/scripts/run_shared.sh index a5d1eeb..e5b46d4 100644 --- a/scripts/run_shared.sh +++ b/scripts/run_shared.sh @@ -8,8 +8,8 @@ function log() { echo '' } -cd "$(pwd)/../ansible-webui" -TEST_DB="$(pwd)/aw.${AW_ENV}.db" +cd "$(pwd)/.." +TEST_DB="$(pwd)/ansible-webui/aw.${AW_ENV}.db" TEST_MIGRATE='' if [ -f "$TEST_DB" ] && [[ "$TEST_QUIET" != "1" ]] @@ -23,7 +23,8 @@ then rm "$TEST_DB" TEST_MIGRATE='clean' fi -else +elif ! [ -f "$TEST_DB" ] +then echo "Creating DB ${TEST_DB}" fi @@ -38,10 +39,10 @@ then fi log 'INITIALIZING DATABASE SCHEMA' -bash ../scripts/migrate_db.sh "$TEST_MIGRATE" +bash ./scripts/migrate_db.sh "$TEST_MIGRATE" log 'CREATING USERS' -python3 manage.py createsuperuser --noinput || true +python3 ansible-webui/manage.py createsuperuser --noinput || true log 'STARTING APP' -python3 __init__.py +python3 ansible-webui