Slurm changes for omnia 2.0

input/config/rhel/9.4/openmpi.json

@@ -0,0 +1,12 @@
+{
+  "openmpi": {
+    "cluster": [
+      { "package": "openmpi",
+        "type": "tarball",
+        "url": "https://download.open-mpi.org/release/open-mpi/v{{ openmpi_version.split('.')[:2] | join('.') }}/openmpi-{{ openmpi_version }}.tar.gz"
+      },
+      {"package": "gcc-c++", "type": "rpm", "repo_name": "appstream"},
+      {"package": "clang", "type": "rpm", "repo_name": "appstream"}
+    ]
+  }
+}

input/config/rhel/9.4/slurm.json

@@ -0,0 +1,32 @@
+{
+    "slurm": {
+        "cluster": [
+            {"package": "munge", "type": "rpm", "repo_name": "appstream"},
+            {"package": "firewalld", "type": "rpm", "repo_name": "baseos"},
+            {"package": "python3-firewall", "type": "rpm", "repo_name": "baseos"}
+        ]
+    },
+    "slurm_control_node": {
+        "cluster": [
+            {"package": "slurm-slurmctld", "type": "rpm", "repo_name": "epel"}
+        ]
+    },
+    "slurm_node": {
+        "cluster": [
+            {"package": "slurm-slurmd", "type": "rpm", "repo_name": "epel"}
+        ]
+    },
+    "slurmdbd":{
+        "cluster": [
+            {"package": "slurm-slurmdbd", "type": "rpm", "repo_name": "epel"},
+            {"package": "python3-PyMySQL", "type": "rpm", "repo_name": "appstream"},
+            {"package": "mysql-server", "type": "rpm", "repo_name": "appstream"},
+            {"package": "mariadb-server", "type": "rpm", "repo_name": "appstream"}
+        ]
+    },
+    "login":{
+        "cluster": [
+            {"package": "slurm", "type": "rpm", "repo_name": "epel"}
+        ]
+    }
+}

input/config/rhel/9.4/ucx.json

@@ -0,0 +1,11 @@
+{
+  "ucx": {
+    "cluster": [
+      { "package": "ucx",
+        "type": "tarball",
+        "url": "https://github.com/openucx/ucx/releases/download/v{{ ucx_version }}/ucx-{{ ucx_version }}.tar.gz"
+      },
+      {"package": "gcc-c++", "type": "rpm", "repo_name": "appstream"}
+    ]
+  }
+}

input/omnia_config.yml

@@ -26,10 +26,22 @@ ansible_config_file_path: "/etc/ansible"
 
 # -----------------------------SLURM------------------------------------------------
 
-# Password used for Slurm database.
+# Username and password used for Slurm database.
 # The Length of the password should be at least 8.
 # The password must not contain -,\, ',"
-mariadb_password: "password"
+slurm_db_username: root
+slurm_db_password: ""
+
+# Host and port of the Slurm database.
+# If no database is intalled on the given node, one will be created.
+# Defaults to the slurmdbd host if no host is given.
+# Defaults to 3306 if no port is given.
+slurm_db_host: 
+slurm_db_port: 3306
+
+# Type of database to be used by Slurm.
+# Options are mysql or mariadb. Defaults to mariadb.
+slurm_db_type: mariadb
 
 # This variable accepts whether slurm installation is supported in configless mode or slurm in nfs
 # Default value is "configless"

scheduler/add_node.yml

@@ -0,0 +1,23 @@
+---
+- name: Add nodes Slurm
+  hosts: slurm_control_node, slurm_node, login, slurm_dbd
+  any_errors_fatal: true
+  vars:
+    share_mounted_path: "{{ hostvars['localhost']['share_path'] | default('/home') }}" # from storage.yml
+    #TODO: nfs_client role here, slurm depends on a mandatory share path
+  pre_tasks:
+    - name: Include input project directory
+      ansible.builtin.import_role:
+        name: ../utils/roles/include_input_dir
+      run_once: true
+      delegate_to: localhost
+    - name: Include vars omnia_config.yml
+      ansible.builtin.include_vars:
+        file: "{{ input_project_dir }}/omnia_config.yml"
+      run_once: true
+      delegate_to: localhost
+  tasks:
+    - name: Add node
+      ansible.builtin.import_role:
+        name: slurm
+        tasks_from: add_node

scheduler/remove_node.yml

@@ -0,0 +1,23 @@
+---
+- name: Remove nodes Slurm
+  hosts: slurm_control_node, slurm_node
+  any_errors_fatal: true
+  vars:
+    share_mounted_path: "{{ hostvars['localhost']['share_path'] | default('/home') }}" # from storage.yml
+    #TODO: nfs_client role here, slurm depends on a mandatory share path
+  pre_tasks:
+    - name: Include input project directory
+      ansible.builtin.import_role:
+        name: ../utils/roles/include_input_dir
+      run_once: true
+      delegate_to: localhost
+    - name: Include vars omnia_config.yml
+      ansible.builtin.include_vars:
+        file: "{{ input_project_dir }}/omnia_config.yml"
+      run_once: true
+      delegate_to: localhost
+  tasks:
+    - name: Remove node
+      ansible.builtin.import_role:
+        name: slurm
+        tasks_from: rm_node

scheduler/roles/install_benchmarks_tools/tasks/compile_install_openmpi.yml

@@ -85,13 +85,9 @@
         - ucx_dir_data.stat.exists
         - ucx_cmd.rc == 0
 
-    - name: Construct the command to compile the openmpi when slurm support is true
-      when: slurm_support
-      ansible.builtin.include_tasks: openmpi_cmd_with_slurm.yml
-
-    - name: Construct the command to compile the openmpi when slurm support is false
-      when: not slurm_support
-      ansible.builtin.include_tasks: openmpi_cmd_without_slurm.yml
+    - name: Construct openmpi compile the command with ucx and slurm
+      ansible.builtin.set_fact:
+        openmpi_compile_cmd: "./configure --prefix={{ omnia_share_path }}/{{ benchmarks_dir_openmpi }} --enable-mpi1-compatibility --enable-prte-prefix-by-default {{ '--with-slurm=yes' if slurm_support else '--with-slurm=no' }}{{ ' --with-ucx='+ucx_dir_data.stat.path+' ' if ucx_installed else ' ' }}CC=clang  CXX=clang++ 2>&1 | tee config.out"
 
     - name: Create a build directory inside openmpi folder
       ansible.builtin.file:

scheduler/roles/slurm/defaults/main.yml

@@ -0,0 +1,37 @@
+---
+# defaults file for slurm
+mpi: {}
+cgroup: {}
+__cgroup_default_config:
+  CgroupPlugin: autodetect
+  ConstrainCores: true
+  ConstrainDevices: true
+  ConstrainRAMSpace: true
+  ConstrainSwapSpace: true
+__slurm_default_config:
+  SlurmUser: "{{ slurm_user }}"
+  SlurmctldPort: 6817
+  SlurmdPort: 6818
+  SrunPortRange: "60001-63000"
+  StateSaveLocation: "/var/spool/state"
+  SlurmdSpoolDir: "/var/spool/slurmd"
+  ReturnToService: 2
+  SchedulerType: sched/backfill
+  MpiDefault: None
+  ProctrackType: proctrack/cgroup
+  SelectType: select/linear
+  SlurmctldLogFile: /var/log/slurmctld.log
+  SlurmdLogFile: /var/log/slurmd.log
+  SlurmctldPidFile: /var/run/slurmctld.pid
+  SlurmdPidFile: /var/run/slurmd.pid
+  AuthType: auth/munge
+  CryptoType: crypto/munge
+  SlurmctldTimeout: 120
+  SlurmdTimeout: 300
+__slurm_dbd_default_config:
+  AuthType: auth/munge
+  LogFile: /var/log/slurmdbd.log
+  PidFile: /var/run/slurmdbd.pid
+  SlurmUser: "{{ slurm_user }}"
+  StorageType: accounting_storage/mysql
+  StorageLoc: slurm_acct_db

scheduler/roles/slurm/handlers/main.yml

@@ -0,0 +1,70 @@
+---
+# handlers file for slurm
+- name: Restart chrony
+  ansible.builtin.systemd_service:
+    name: "{{ 'chronyd' if ansible_os_family == 'RedHat' else 'chrony' }}"
+    state: restarted
+    enabled: true
+
+- name: Restart munge
+  ansible.builtin.systemd_service:
+    name: munge
+    state: restarted
+
+- name: Restart mysqld
+  ansible.builtin.systemd_service:
+    name: mysqld
+    state: restarted
+  run_once: true
+  when: restart_slurm_services
+  delegate_to: "{{ slurm_db_host }}"
+
+- name: Restart mariadb
+  ansible.builtin.systemd_service:
+    name: mariadb
+    state: restarted
+  run_once: true
+  when: restart_slurm_services
+  delegate_to: "{{ slurm_db_host }}"
+
+- name: Reload slurmdbd
+  ansible.builtin.systemd_service:
+    name: slurmdbd
+    state: reloaded
+    daemon_reload: true
+    enabled: true
+  when: restart_slurm_services and ('slurm_dbd' in group_names)
+
+- name: Restart slurmdbd
+  ansible.builtin.systemd_service:
+    name: slurmdbd
+    state: restarted
+  when: restart_slurm_services and ('slurm_dbd' in group_names)
+
+- name: Reload slurmctld
+  ansible.builtin.systemd_service:
+    name: slurmctld
+    state: reloaded
+    daemon_reload: true
+    enabled: true
+  when: restart_slurm_services and ('slurm_control_node' in group_names)
+
+- name: Restart slurmctld
+  ansible.builtin.systemd_service:
+    name: slurmctld
+    state: restarted
+  when: restart_slurm_services and ('slurm_control_node' in group_names)
+
+- name: Reload slurmd
+  ansible.builtin.systemd_service:
+    name: slurmd
+    state: reloaded
+    daemon_reload: true
+    enabled: true
+  when: restart_slurm_services and ('slurm_node' in group_names)
+
+- name: Restart slurmd
+  ansible.builtin.systemd_service:
+    name: slurmd
+    state: restarted
+  when: restart_slurm_services and ('slurm_node' in group_names)