From 2828cc24193e8bab72df35d4c4bca3e6b663fa89 Mon Sep 17 00:00:00 2001 From: Marie Backman Date: Fri, 15 Nov 2024 13:28:03 -0500 Subject: [PATCH 1/3] remove link to webref from run page --- src/webmon_app/reporting/report/views.py | 8 -------- .../reporting/reporting_app/settings/base.py | 5 +---- .../reporting_app/settings/envtest.py | 2 -- .../reporting/reporting_app/settings/prod.py | 2 -- .../reporting/templates/report/detail.html | 5 ----- .../IPTS-33077/nexus/REF_L_214746.nxs.h5 | 0 .../IPTS-33077/nexus/REF_L_214747.nxs.h5 | 0 .../REF_L/shared/autoreduce/reduce_REF_L.py | 20 +++++++++++++++++++ .../REF_M/IPTS-30794/nexus/REF_M_42112.nxs.h5 | 0 .../REF_M/IPTS-30794/nexus/REF_M_42113.nxs.h5 | 0 10 files changed, 21 insertions(+), 21 deletions(-) create mode 100644 tests/data/REF_L/IPTS-33077/nexus/REF_L_214746.nxs.h5 create mode 100644 tests/data/REF_L/IPTS-33077/nexus/REF_L_214747.nxs.h5 create mode 100644 tests/data/REF_L/shared/autoreduce/reduce_REF_L.py create mode 100644 tests/data/REF_M/IPTS-30794/nexus/REF_M_42112.nxs.h5 create mode 100644 tests/data/REF_M/IPTS-30794/nexus/REF_M_42113.nxs.h5 diff --git a/src/webmon_app/reporting/report/views.py b/src/webmon_app/reporting/report/views.py index 31d1b896..8506a7d0 100644 --- a/src/webmon_app/reporting/report/views.py +++ b/src/webmon_app/reporting/report/views.py @@ -8,7 +8,6 @@ import sys import logging import json -import string import datetime from django.http import HttpResponse, HttpResponseNotFound from django.shortcuts import render, get_object_or_404 @@ -250,19 +249,12 @@ def detail(request, instrument, run_id): except: # noqa: E722 prev_url = None - # Fitting URL - fitting_url = None - if hasattr(settings, "FITTING_URLS") and instrument.lower() in settings.FITTING_URLS: - url_template = string.Template(settings.FITTING_URLS[instrument.lower()]) - fitting_url = url_template.substitute(run_number=run_id) - template_values = { "instrument": instrument.upper(), "run_object": run_object, "status": status_objects, "breadcrumbs": breadcrumbs, "icat_info": icat_info, - "fitting_url": fitting_url, "reduce_url": reduce_url, "reduction_setup_url": reporting_view_util.reduction_setup_url(instrument), "prev_url": prev_url, diff --git a/src/webmon_app/reporting/reporting_app/settings/base.py b/src/webmon_app/reporting/reporting_app/settings/base.py index c17c1e4a..1336ca9c 100644 --- a/src/webmon_app/reporting/reporting_app/settings/base.py +++ b/src/webmon_app/reporting/reporting_app/settings/base.py @@ -152,6 +152,7 @@ def validate_ldap_settings(server_uri, user_dn_template): CATALOG_URL = environ.get("CATALOG_URL") CATALOG_ID = environ.get("CATALOG_ID") CATALOG_SECRET = environ.get("CATALOG_SECRET") +CATALOG_API_TOKEN = environ.get("CATALOG_API_TOKEN") # The DB settings are defined the same as in the workflow manager DATABASES = { @@ -365,10 +366,6 @@ def validate_ldap_settings(server_uri, user_dn_template): for instr, facility in additions.items(): FACILITY_INFO[instr] = facility -# Link out to fitting application -FITTING_URLS = {} - - # remote worker options # setting these will force all execution to happen as a single user TEST_REMOTE_USER = "" diff --git a/src/webmon_app/reporting/reporting_app/settings/envtest.py b/src/webmon_app/reporting/reporting_app/settings/envtest.py index 4c829093..4b4d2e90 100644 --- a/src/webmon_app/reporting/reporting_app/settings/envtest.py +++ b/src/webmon_app/reporting/reporting_app/settings/envtest.py @@ -11,6 +11,4 @@ GENERAL_USER_USERNAME = environ.get("GENERAL_USER_USERNAME", "GeneralUser") # noqa: F405 GENERAL_USER_PASSWORD = environ.get("GENERAL_USER_PASSWORD", "GeneralUser") # noqa: F405 -FITTING_URLS = {"ref_l": "https://reflectivity-test.ornl.gov/fit/ref_l/$run_number"} - validate_ldap_settings(server_uri=AUTH_LDAP_SERVER_URI, user_dn_template=AUTH_LDAP_USER_DN_TEMPLATE) # noqa: F405 diff --git a/src/webmon_app/reporting/reporting_app/settings/prod.py b/src/webmon_app/reporting/reporting_app/settings/prod.py index 61272614..8dbf4a19 100644 --- a/src/webmon_app/reporting/reporting_app/settings/prod.py +++ b/src/webmon_app/reporting/reporting_app/settings/prod.py @@ -12,6 +12,4 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = environ.get("DEBUG", False) -FITTING_URLS = {"ref_l": "https://reflectivity.sns.gov/fit/ref_l/$run_number"} - validate_ldap_settings(server_uri=AUTH_LDAP_SERVER_URI, user_dn_template=AUTH_LDAP_USER_DN_TEMPLATE) # noqa: F405 diff --git a/src/webmon_app/reporting/templates/report/detail.html b/src/webmon_app/reporting/templates/report/detail.html index 70a0ffdf..145ed61f 100644 --- a/src/webmon_app/reporting/templates/report/detail.html +++ b/src/webmon_app/reporting/templates/report/detail.html @@ -133,11 +133,6 @@

{% endif %} - {% if fitting_url %} -

- fit data | - {% endif %} - {% if html_data %} {% if data_url %} Data access: download plot data points diff --git a/tests/data/REF_L/IPTS-33077/nexus/REF_L_214746.nxs.h5 b/tests/data/REF_L/IPTS-33077/nexus/REF_L_214746.nxs.h5 new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/REF_L/IPTS-33077/nexus/REF_L_214747.nxs.h5 b/tests/data/REF_L/IPTS-33077/nexus/REF_L_214747.nxs.h5 new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/REF_L/shared/autoreduce/reduce_REF_L.py b/tests/data/REF_L/shared/autoreduce/reduce_REF_L.py new file mode 100644 index 00000000..b025ae49 --- /dev/null +++ b/tests/data/REF_L/shared/autoreduce/reduce_REF_L.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +import os +import sys +import socket +from datetime import datetime +from postprocessing.publish_plot import publish_plot + +if __name__ == "__main__": + time = datetime.isoformat(datetime.now()) + filename = sys.argv[1] + print("Running reduction for " + filename + " at " + time) + + publish_plot( + "REF_L", + os.path.basename(filename).split(".")[0].split("_")[-1], + files={ + "file": f"

Example Plot Data

Filename: {filename}

" + f"

Time: {time}

Hostname: {socket.gethostname()}

" + }, + ) diff --git a/tests/data/REF_M/IPTS-30794/nexus/REF_M_42112.nxs.h5 b/tests/data/REF_M/IPTS-30794/nexus/REF_M_42112.nxs.h5 new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/REF_M/IPTS-30794/nexus/REF_M_42113.nxs.h5 b/tests/data/REF_M/IPTS-30794/nexus/REF_M_42113.nxs.h5 new file mode 100644 index 00000000..e69de29b From de002c1a329d93b2072a76b64d654ed9d654d93b Mon Sep 17 00:00:00 2001 From: Marie Backman Date: Mon, 18 Nov 2024 08:33:58 -0500 Subject: [PATCH 2/3] change webmonchow version --- Dockerfile.webmonchow | 2 +- src/webmon_app/reporting/reporting_app/settings/base.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile.webmonchow b/Dockerfile.webmonchow index 82cf08be..314abd31 100644 --- a/Dockerfile.webmonchow +++ b/Dockerfile.webmonchow @@ -3,7 +3,7 @@ FROM continuumio/miniconda3:23.3.1-0 RUN conda install --yes -n base conda-libmamba-solver # Install webmonchow -RUN conda install --yes --solver=libmamba -n base -c conda-forge -c neutrons postgresql=14 webmonchow=1.0.1 +RUN conda install --yes --solver=libmamba -n base -c conda-forge -c neutrons/label/rc postgresql=14 webmonchow=1.0.2rc1 # Install webmonchow from a feature branch (only to test the latest changes in webmonchow) # RUN conda install --yes --solver=libmamba -n base -c conda-forge postgresql=14 diff --git a/src/webmon_app/reporting/reporting_app/settings/base.py b/src/webmon_app/reporting/reporting_app/settings/base.py index 1336ca9c..b1b13679 100644 --- a/src/webmon_app/reporting/reporting_app/settings/base.py +++ b/src/webmon_app/reporting/reporting_app/settings/base.py @@ -152,7 +152,6 @@ def validate_ldap_settings(server_uri, user_dn_template): CATALOG_URL = environ.get("CATALOG_URL") CATALOG_ID = environ.get("CATALOG_ID") CATALOG_SECRET = environ.get("CATALOG_SECRET") -CATALOG_API_TOKEN = environ.get("CATALOG_API_TOKEN") # The DB settings are defined the same as in the workflow manager DATABASES = { From 15107b0566c4b8f53fc406629bd00abcedd9bf63 Mon Sep 17 00:00:00 2001 From: Marie Backman Date: Mon, 18 Nov 2024 14:32:43 -0500 Subject: [PATCH 3/3] resolve certificate issues and publish plot from REF_L reduction script --- docker-compose.yml | 6 ++-- src/webmon_app/docker-entrypoint.sh | 2 ++ src/webmon_app/reporting/report/catalog.py | 10 +----- .../reporting/reporting_app/settings/base.py | 1 + .../REF_L/shared/autoreduce/reduce_REF_L.py | 33 ++++++++++++++++--- 5 files changed, 36 insertions(+), 16 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8134e69a..1dd8f916 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,7 +27,8 @@ services: dockerfile: Dockerfile.webmon volumes: - web-static:/var/www/workflow/static/ - - ./nginx/nginx.crt:/nginx.crt + # add in folder to be picked up by update-ca-certificates in docker entrypoint + - ./nginx/nginx.crt:/usr/local/share/ca-certificates/nginx.crt env_file: - .env - .env.ci @@ -39,8 +40,9 @@ services: - CATALOG_URL=${CATALOG_URL} - CATALOG_ID=${CATALOG_ID} - CATALOG_SECRET=${CATALOG_SECRET} + - CATALOG_API_TOKEN=${CATALOG_API_TOKEN} - GUNICORN_CMD_ARGS=--reload --workers=8 - - REQUESTS_CA_BUNDLE=/nginx.crt + - REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt healthcheck: test: wget --no-verbose --tries=1 --spider http://localhost:8000/ht || exit 1 interval: 60s diff --git a/src/webmon_app/docker-entrypoint.sh b/src/webmon_app/docker-entrypoint.sh index 22454a00..43683edd 100755 --- a/src/webmon_app/docker-entrypoint.sh +++ b/src/webmon_app/docker-entrypoint.sh @@ -26,6 +26,8 @@ if [[ "$DJANGO_SETTINGS_MODULE" != *".prod" ]]; then >&2 echo "Not in Production, setting up test users InstrumentScientist, and GeneralUser" python $MANAGE_PY_WEBMON ensure_adminuser --username="InstrumentScientist" --email='Instrument@Scientist.com' --password="InstrumentScientist" python $MANAGE_PY_WEBMON ensure_user --username="${GENERAL_USER_USERNAME}" --email='General@User.com' --password="${GENERAL_USER_PASSWORD}" + >&2 echo "Not in Production, updating certificates to add self-signed certificate" + update-ca-certificates fi # start up web-service diff --git a/src/webmon_app/reporting/report/catalog.py b/src/webmon_app/reporting/report/catalog.py index 0fbaae2c..f18e9c08 100644 --- a/src/webmon_app/reporting/report/catalog.py +++ b/src/webmon_app/reporting/report/catalog.py @@ -63,15 +63,7 @@ def _get_run_info(instrument, ipts, run_number, facility="SNS"): """ run_info = {} try: - oncat = pyoncat.ONCat( - settings.CATALOG_URL, - # Here we're using the machine-to-machine "Client Credentials" flow, - # which requires a client ID and secret, but no *user* credentials. - flow=pyoncat.CLIENT_CREDENTIALS_FLOW, - client_id=settings.CATALOG_ID, - client_secret=settings.CATALOG_SECRET, - ) - oncat.login() + oncat = pyoncat.ONCat(settings.CATALOG_URL, api_token=settings.CATALOG_API_TOKEN) datafiles = oncat.Datafile.list( facility=facility, diff --git a/src/webmon_app/reporting/reporting_app/settings/base.py b/src/webmon_app/reporting/reporting_app/settings/base.py index b1b13679..1336ca9c 100644 --- a/src/webmon_app/reporting/reporting_app/settings/base.py +++ b/src/webmon_app/reporting/reporting_app/settings/base.py @@ -152,6 +152,7 @@ def validate_ldap_settings(server_uri, user_dn_template): CATALOG_URL = environ.get("CATALOG_URL") CATALOG_ID = environ.get("CATALOG_ID") CATALOG_SECRET = environ.get("CATALOG_SECRET") +CATALOG_API_TOKEN = environ.get("CATALOG_API_TOKEN") # The DB settings are defined the same as in the workflow manager DATABASES = { diff --git a/tests/data/REF_L/shared/autoreduce/reduce_REF_L.py b/tests/data/REF_L/shared/autoreduce/reduce_REF_L.py index b025ae49..83582346 100644 --- a/tests/data/REF_L/shared/autoreduce/reduce_REF_L.py +++ b/tests/data/REF_L/shared/autoreduce/reduce_REF_L.py @@ -1,10 +1,36 @@ #!/usr/bin/env python import os import sys -import socket from datetime import datetime from postprocessing.publish_plot import publish_plot +plot_div = """ +
+
+ + +
+""" + if __name__ == "__main__": time = datetime.isoformat(datetime.now()) filename = sys.argv[1] @@ -13,8 +39,5 @@ publish_plot( "REF_L", os.path.basename(filename).split(".")[0].split("_")[-1], - files={ - "file": f"
Example Plot Data

Filename: {filename}

" - f"

Time: {time}

Hostname: {socket.gethostname()}

" - }, + files={"file": f"
{plot_div}
"}, )