Merge branch 'develop' into main

Updating to version 1.0.4 from develop

Author: pagrubel

.github/workflows/integration.yml

@@ -20,7 +20,7 @@ jobs:
     # available on 20.04
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Dependency Install
         run: ./ci/deps_install.sh
       - name: Slurm Setup and Install

.github/workflows/pylama.yml

@@ -15,8 +15,8 @@ jobs:
     name: PyLama Lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Lint
         run: |
-          pip install pylama pylint 2>&1 >/dev/null
+          pip install pylama==8.4.1 pyflakes==3.0.1 pylint==2.15.9 pydocstyle==6.1.1 2>&1 >/dev/null
           pylama beeflow/

.github/workflows/unit-tests.yml

@@ -20,7 +20,7 @@ jobs:
     # available on 20.04
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Dependency Install
         run: ./ci/deps_install.sh
       - name: Slurm Setup and Install

.gitignore

@@ -12,5 +12,7 @@ poetry.lock
 .vscode
 .env
 .venv
+docs/sphinx/_build
+src/beeflow/enhanced_client/node_modules
+**/dist
 _build
-dist

README.rst

@@ -35,6 +35,29 @@ Contact
 For bugs and problems report, suggestions and other general questions regarding the BEE project, email questions to `[email protected] <[email protected]>`_.
 
 
+Contributors:
+==========================
+
+* Steven Anaya - `Boogie3D <https://github.com/Boogie3D>`_
+* Paul Bryant - `paulbry <https://github.com/paulbry>`_
+* Rusty Davis - `rstyd <https://github.com/rstyd>`_
+* Jieyang Chen - `JieyangChen7 <https://github.com/JieyangChen7>`_
+* Patricia Grubel - `pagrubel <https://github.com/pagrubel>`_
+* Qiang Guan - `guanxyz <https://github.com/guanxyz>`_
+* Ragini Gupta - `raginigupta6 <https://github.com/raginigupta6>`_
+* Andres Quan - `aquan9 <https://github.com/aquan9>`_
+* Quincy Wofford - `qwofford <https://github.com/qwofford>`_
+* Tim Randles - `trandles-lanl <https://github.com/trandles-lanl>`_
+* Jacob Tronge - `jtronge <https://github.com/jtronge>`_
+
+Concept and Design Contributors
+
+* James Ahrens
+* Allen McPherson
+* Li-Ta Lo
+* Louis Vernon
+
+
 Contributing
 ==========================
 

RELEASE.rst

@@ -2,24 +2,27 @@ Publishing a new release
 ************************
 
 1. Change the version in pyproject.toml and verify docs build;
-   and get this change merged into develop.
-
+   and get this change merged into develop. (You may want to set the bypass as in step 2)
 2. On github site go to Settings; on the left under Code and Automation
    click on Branches; under Branch protection rules edit main;
     check Allow specified actors to bypass required pull requests; add yourself
-    and don'forget to save the setting
-3. Checkout main and merge develop into main. Make sure documentation will be 
-   published upon push to main (.github/workflows/docs.yml) and push.
-4. Once merged, create a tag
-   with something like ``git tag -a 0.1.1 -m "BEE version 0.1.1"``. You can see
-   existing tags with ``git tag``. Finally do ``git push origin --tags`` to
-   push the new tag.
-5. Create release.
+    and don't forget to save the setting
+3  Make sure documentation will be published upon push to main.
+   See: .github/workflows/docs.yml
+4. Checkout develop and pull for latest version then
+   checkout main and merge develop into main. Verify documentation was published.
+   See actions and site below.
+5. Once merged, on github web interface create a release and tag based on main branch
+   that matches the version in pyproject.toml
 6. Follow step 2 but uncheck Allow specified actors to bypass and don't forget save
 7. Finally, on the main branch, first run a ``poetry build`` and then a
    ``poetry publish``. The second command will ask for a username and password
    for PyPI.
 
+Check the documentation at: `https://lanl.github.io/BEE/ <https://lanl.github.io/BEE/>`_ 
+Also upgrade the pip version in your python or anaconda environment and check the version:
+ `` pip install --upgrade hpc-beeflow``
+
 **WARNING**: Once a version is pushed to PyPI, it cannot be undone. You can
 'delete' the version from the package settings, but you can no longer publish
 an update to that same version.

beeflow/cli.py

@@ -24,6 +24,11 @@
 from beeflow.common import cli_connection
 
 
+bc.init()
+# Max number of times a component can be restarted
+MAX_RESTARTS = bc.get('DEFAULT', 'max_restarts')
+
+
 class ComponentManager:
     """Component manager class."""
 
@@ -40,6 +45,8 @@ def wrap(fn):
             self.components[name] = {
                 'fn': fn,
                 'deps': deps,
+                'restart_count': 0,
+                'failed': False,
             }
 
         return wrap
@@ -87,12 +94,24 @@ def run(self, base_components):
             self.procs[name] = component['fn']()
 
     def poll(self):
-        """Poll each process to check for errors."""
-        for name, proc in self.procs.items():
-            returncode = proc.poll()
+        """Poll each process to check for errors, restart failed processes."""
+        for name in self.procs:  # noqa no need to iterate with items() since self.procs may be set
+            component = self.components[name]
+            if component['failed']:
+                continue
+            returncode = self.procs[name].poll()
             if returncode is not None:
                 log = log_fname(name)
                 print(f'Component "{name}" failed, check log "{log}"')
+                if component['restart_count'] >= MAX_RESTARTS:
+                    print(f'Component "{name}" has been restarted {MAX_RESTARTS} '
+                          'times, not restarting again')
+                    component['failed'] = True
+                else:
+                    restart_count = component['restart_count']
+                    print(f'Attempting restart {restart_count} of "{name}"...')
+                    self.procs[name] = component['fn']()
+                    component['restart_count'] += 1
 
     def status(self):
         """Return the statuses for each process in a dict."""
@@ -140,6 +159,12 @@ def open_log(component):
     return open(log, 'a', encoding='utf-8')
 
 
+# Slurmrestd will be started only if we're running with Slurm and
+# slurm::use_commands is not True
+NEED_SLURMRESTD = (bc.get('DEFAULT', 'workload_scheduler') == 'Slurm'
+                   and not bc.get('slurm', 'use_commands'))
+
+
 @MGR.component('wf_manager', ('scheduler',))
 def start_wfm():
     """Start the WFM."""
@@ -149,7 +174,12 @@ def start_wfm():
                                 sock_path, stdout=fp, stderr=fp)
 
 
-@MGR.component('task_manager', ('slurmrestd',))
+TM_DEPS = []
+if NEED_SLURMRESTD:
+    TM_DEPS.append('slurmrestd')
+
+
+@MGR.component('task_manager', TM_DEPS)
 def start_task_manager():
     """Start the TM."""
     fp = open_log('task_manager')
@@ -168,21 +198,22 @@ def start_scheduler():
 
 
 # Workflow manager and task manager need to be opened with PIPE for their stdout/stderr
-@MGR.component('slurmrestd')
-def start_slurm_restd():
-    """Start BEESlurmRestD. Returns a Popen process object."""
-    bee_workdir = bc.get('DEFAULT', 'bee_workdir')
-    slurmrestd_log = '/'.join([bee_workdir, 'logs', 'restd.log'])
-    slurm_socket = bc.get('slurmrestd', 'slurm_socket')
-    slurm_args = bc.get('slurmrestd', 'slurm_args')
-    slurm_args = slurm_args if slurm_args is not None else ''
-    subprocess.run(['rm', '-f', slurm_socket], check=True)
-    # log.info("Attempting to open socket: {}".format(slurm_socket))
-    fp = open(slurmrestd_log, 'w', encoding='utf-8') # noqa
-    cmd = ['slurmrestd']
-    cmd.extend(slurm_args.split())
-    cmd.append(f'unix:{slurm_socket}')
-    return subprocess.Popen(cmd, stdout=fp, stderr=fp)
+if NEED_SLURMRESTD:
+    @MGR.component('slurmrestd')
+    def start_slurm_restd():
+        """Start BEESlurmRestD. Returns a Popen process object."""
+        bee_workdir = bc.get('DEFAULT', 'bee_workdir')
+        slurmrestd_log = '/'.join([bee_workdir, 'logs', 'restd.log'])
+        slurm_socket = bc.get('slurm', 'slurmrestd_socket')
+        openapi_version = bc.get('slurm', 'openapi_version')
+        slurm_args = f'-s openapi/{openapi_version}'
+        subprocess.run(['rm', '-f', slurm_socket], check=True)
+        # log.info("Attempting to open socket: {}".format(slurm_socket))
+        fp = open(slurmrestd_log, 'w', encoding='utf-8') # noqa
+        cmd = ['slurmrestd']
+        cmd.extend(slurm_args.split())
+        cmd.append(f'unix:{slurm_socket}')
+        return subprocess.Popen(cmd, stdout=fp, stderr=fp)
 
 
 def handle_terminate(signum, stack): # noqa
@@ -192,7 +223,7 @@ def handle_terminate(signum, stack): # noqa
     sys.exit(1)
 
 
-MIN_CHARLIECLOUD_VERSION = (0, 27)
+MIN_CHARLIECLOUD_VERSION = (0, 32)
 
 
 def version_str(version):
@@ -288,13 +319,21 @@ def start(foreground: bool = typer.Option(False, '--foreground', '-F',
     beeflow_log = log_fname('beeflow')
     check_dependencies()
     sock_path = bc.get('DEFAULT', 'beeflow_socket')
+    if bc.get('DEFAULT', 'workload_scheduler') == 'Slurm' and not NEED_SLURMRESTD:
+        warn('Not using slurmrestd. Command-line interface will be used.')
     # Note: there is a possible race condition here, however unlikely
     if os.path.exists(sock_path):
         # Try to contact for a status
-        resp = cli_connection.send(sock_path, {'type': 'status'})
+        try:
+            resp = cli_connection.send(sock_path, {'type': 'status'})
+        except (ConnectionResetError, ConnectionRefusedError):
+            resp = None
         if resp is None:
             # Must be dead, so remove the socket path
-            os.remove(sock_path)
+            try:
+                os.remove(sock_path)
+            except FileNotFoundError:
+                pass
         else:
             # It's already running, so print an error and exit
             warn(f'Beeflow appears to be running. Check the beeflow log: "{beeflow_log}"')
@@ -352,6 +391,14 @@ def stop():
     print(f'Beeflow has stopped. Check the log at "{beeflow_log}".')
 
 
+@app.command()
+def restart(foreground: bool = typer.Option(False, '--foreground', '-F',
+            help='run in the foreground')):
+    """Attempt to stop and restart the beeflow daemon."""
+    stop()
+    start(foreground)
+
+
 @app.callback(invoke_without_command=True)
 def version_callback(version: bool = False):
     """Beeflow."""
@@ -364,7 +411,6 @@ def version_callback(version: bool = False):
 
 def main():
     """Start the beeflow app."""
-    bc.init()
     app()