Fix: scheduling: overhaul the whole thing

- prevent possible lockup when format in proc changes
- properly get and handle scheduler policy & prio
- recognize and try to handle cgroup-v2 similarly
- on SCHED_RR failing push to the max with SCHED_OTHER

Author: wenningerk

src/sbd-common.c

@@ -26,6 +26,9 @@
 #include <pwd.h>
 #include <unistd.h>
 #include <dirent.h>
+#include <sys/time.h>
+#include <sys/resource.h>
+#include <limits.h>
 
 #ifdef _POSIX_MEMLOCK
 #  include <sys/mman.h>
@@ -298,7 +301,7 @@ watchdog_populate_list(void)
 				FILE *file;
 
 				snprintf(entry_name, sizeof(entry_name),
-				         SYS_CLASS_WATCHDOG "/%s/dev", entry->d_name);
+						 SYS_CLASS_WATCHDOG "/%s/dev", entry->d_name);
 				file = fopen(entry_name, "r");
 				if (file) {
 					int major, minor;
@@ -667,7 +670,7 @@ static int get_realtime_budget(void)
 {
     FILE *f;
     char fname[PATH_MAX];
-    int res = -1, lnum = 0;
+    int res = -1, lnum = 0, num;
     char *cgroup = NULL, *namespecs = NULL;
 
     snprintf(fname, PATH_MAX, "/proc/%jd/cgroup", (intmax_t)getpid());
@@ -677,7 +680,8 @@ static int get_realtime_budget(void)
                             (intmax_t)getpid());
         goto exit_res;
     }
-    while( fscanf(f, "%d:%m[^:]:%m[^\n]", &lnum,  &namespecs, &cgroup) !=EOF ) {
+    while( (num = fscanf(f, "%d:%m[^:]:%m[^\n]\n", &lnum,
+                         &namespecs, &cgroup)) !=EOF ) {
         if (namespecs && strstr(namespecs, "cpuacct")) {
             free(namespecs);
             break;
@@ -690,6 +694,11 @@ static int get_realtime_budget(void)
             free(namespecs);
             namespecs = NULL;
         }
+        /* not to get stuck if format changes */
+        if ((num < 3) && ((fscanf(f, "%*[^\n]") == EOF) ||
+            (fscanf(f, "\n") == EOF))) {
+            break;
+        }
     }
     fclose(f);
     if (cgroup == NULL) {
@@ -720,9 +729,15 @@ static int get_realtime_budget(void)
 }
 
 /* stolen from corosync */
+
+#define LEGACY_CGROUP_PROC_PIDS "/sys/fs/cgroup/cpu/tasks"
+#define UNIFIED_CGROUP_PROC_PIDS "/sys/fs/cgroup/cgroup.procs"
+
 static int sbd_move_to_root_cgroup(bool enforce_root_cgroup) {
     FILE *f;
-    int res = -1;
+    int res = -1, num;
+    char *rt_rq_name = NULL;
+    const char *root_pids = LEGACY_CGROUP_PROC_PIDS;
 
     /*
      * /sys/fs/cgroup is hardcoded, because most of Linux distributions are now
@@ -731,13 +746,53 @@ static int sbd_move_to_root_cgroup(bool enforce_root_cgroup) {
      * This feature is expected to be removed as soon as systemd gets support
      * for managing RT configuration.
      */
-    f = fopen("/sys/fs/cgroup/cpu/cpu.rt_runtime_us", "rt");
-    if (f == NULL) {
-        cl_log(LOG_DEBUG, "cpu.rt_runtime_us doesn't exist -> "
+    do {
+        f = fopen("/sys/fs/cgroup/cpu/cpu.rt_runtime_us", "rt");
+        if (f) {
+            break;
+        }
+        /* CONFIG_RT_GROUP_SCHED might still be enabled with cgroup-v2
+           cgroup.procs on cgroup-toplevel tells us we have cgroup-v2
+           (handy as we already need that to be in selinux-policy)
+           and name of rt_rq(s) in /proc/sched_debug tells us that
+           CONFIG_RT_GROUP_SCHED is enabled
+           cgroup-v2 has been around for a while in the kernel and it
+           is no mutual exclusive compile-time-configuration - so
+           checking what is actually mounted to go with what is there
+        */
+        f = fopen(UNIFIED_CGROUP_PROC_PIDS, "rt");
+        if (f) {
+            fclose(f);
+            f = fopen("/proc/sched_debug", "rt");
+            if (f) {
+                while (((num = fscanf(f, "rt_rq[%*[^]]]:%m[^\n]\n",
+                                      &rt_rq_name)) != EOF) &&
+                       (rt_rq_name == NULL)) {
+                    /* consume a line */
+                    if ((num > 0) || (fscanf(f, "%*[^\n]") == EOF) ||
+                        (fscanf(f, "\n") == EOF)) {
+                        break;
+                    }
+                }
+                /* no hierarchical rt-budget distribution with
+                   cgroup-v2 so far - thus checking for budget is
+                   useless
+                 */
+                if (rt_rq_name) {
+                    free(rt_rq_name);
+                    enforce_root_cgroup = true;
+                    root_pids = UNIFIED_CGROUP_PROC_PIDS;
+                    break;
+                }
+                fclose(f);
+            }
+        }
+        cl_log(LOG_DEBUG, "cpu.rt_runtime_us doesn't exist & "
+            "/proc/sched_debug doesn't contain rt_rq[...]:/ -> "
             "system without cgroup or with disabled CONFIG_RT_GROUP_SCHED");
         res = 0;
         goto exit_res;
-    }
+    } while (0);
     fclose(f);
 
     if ((!enforce_root_cgroup) && (get_realtime_budget() > 0)) {
@@ -747,21 +802,23 @@ static int sbd_move_to_root_cgroup(bool enforce_root_cgroup) {
         goto exit_res;
     }
 
-    f = fopen("/sys/fs/cgroup/cpu/tasks", "w");
+    f = fopen(root_pids, "w");
     if (f == NULL) {
-        cl_log(LOG_WARNING, "Can't open cgroups tasks file for writing");
+        cl_log(LOG_WARNING, "Can't open %s for writing", root_pids);
 
         goto exit_res;
     }
 
     if (fprintf(f, "%jd\n", (intmax_t)getpid()) <= 0) {
-        cl_log(LOG_WARNING, "Can't write sbd pid into cgroups tasks file");
+        cl_log(LOG_WARNING, "Can't write sbd pid into %s", root_pids);
         goto close_and_exit_res;
     }
 
+    res = 0;
+
 close_and_exit_res:
     if (fclose(f) != 0) {
-        cl_log(LOG_WARNING, "Can't close cgroups tasks file");
+        cl_log(LOG_WARNING, "Can't close %s", root_pids);
         goto exit_res;
     }
 
@@ -776,15 +833,17 @@ sbd_make_realtime(int priority, int stackgrowK, int heapgrowK)
         return;
     }
 
+do {
 #ifdef SCHED_RR
     if (move_to_root_cgroup) {
         sbd_move_to_root_cgroup(enforce_moving_to_root_cgroup);
     }
 
     {
-        int pcurrent = 0;
         int pmin = sched_get_priority_min(SCHED_RR);
         int pmax = sched_get_priority_max(SCHED_RR);
+        struct sched_param sp;
+        int pcurrent;
 
         if (priority == 0) {
             priority = pmax;
@@ -794,26 +853,47 @@ sbd_make_realtime(int priority, int stackgrowK, int heapgrowK)
             priority = pmax;
         }
 
-        pcurrent = sched_getscheduler(0);
-        if (pcurrent < 0) {
+        if (sched_getparam(0, &sp) < 0) {
             cl_perror("Unable to get scheduler priority");
 
-        } else if(pcurrent < priority) {
-            struct sched_param sp;
+        } else if ((pcurrent = sched_getscheduler(0)) < 0) {
+            cl_perror("Unable to get scheduler policy");
 
+        } else if ((pcurrent == SCHED_RR) &&
+                   (sp.sched_priority >= priority)) {
+                cl_log(LOG_INFO,
+                       "Stay with priority (%d) for policy SCHED_RR",
+                       sp.sched_priority);
+                break;
+        } else {
             memset(&sp, 0, sizeof(sp));
             sp.sched_priority = priority;
 
             if (sched_setscheduler(0, SCHED_RR, &sp) < 0) {
-                cl_perror("Unable to set scheduler priority to %d", priority);
+                cl_perror(
+                    "Unable to set scheduler policy to SCHED_RR priority %d",
+                    priority);
             } else {
-                cl_log(LOG_INFO, "Scheduler priority is now %d", priority);
+                cl_log(LOG_INFO,
+                       "Scheduler policy is now SCHED_RR priority %d",
+                       priority);
+                break;
             }
         }
     }
 #else
-    cl_log(LOG_ERR, "System does not support updating the scheduler priority");
+    cl_log(LOG_ERR, "System does not support updating the scheduler policy");
+#endif
+#ifdef PRIO_PGRP
+    if (setpriority(PRIO_PGRP, 0, INT_MIN) < 0) {
+        cl_perror("Unable to raise the scheduler priority");
+    } else {
+        cl_log(LOG_INFO, "Scheduler priority raised to the maximum");
+	}
+#else
+    cl_perror("System does not support setting the scheduler priority");
 #endif
+} while (0);
 
     sbd_memlock(heapgrowK, stackgrowK);
 }
@@ -826,7 +906,7 @@ maximize_priority(void)
 		return;
 	}
 
-        sbd_make_realtime(0, 256, 256);
+	sbd_make_realtime(0, 256, 256);
 
 	if (ioprio_set(IOPRIO_WHO_PROCESS, getpid(),
 			IOPRIO_PRIO_VALUE(IOPRIO_CLASS_RT, 1)) != 0) {

src/sbd.sysconfig

@@ -104,6 +104,20 @@ SBD_TIMEOUT_ACTION=flush,reboot
 # If that is the case sbd will stay in that slice while it will
 # be moved to root-slice otherwise.
 #
+# With cgroup-v2 behavior is very much different.
+# With CONFIG_RT_GROUP_SCHED enabled and cpu-controller enabled
+# there currently is no way to configure RT-budget in any slice
+# but the root-slice. Otherway round if there is RT-budget used
+# in any but the root-slice enabling the cpu-controller is
+# inhibited.
+# Thus - unless strictly disabled by setting 'no' - with cgroup-v2
+# and CONFIG_RT_GROUP_SCHED enabled sbd is always moved
+# to the root-slice regardless if the cpu-controller is at the
+# moment enabled or not.
+# Reason is that subsequent services might enable the cpu-controller
+# or fail doing so if sbd was already using RT-budget in e.g. the
+# system-slice.
+#
 SBD_MOVE_TO_ROOT_CGROUP=auto
 
 ## Type: string