Use system limit on file descriptors for soft limit

If user tries to set the soft limit of file descriptors to unlimited on MacOS,
use the system limit instead.

Add unit  test.  Add diagnostics output to test.

Clean up incorrect return values.

Fixes eclipse-omr/omr#3579

Signed-off-by: Peter Bain <[email protected]>

Signed-off-by: Peter Bain <[email protected]>

Author: pdbain-ibm

fvtest/porttest/si.cpp

@@ -806,9 +806,10 @@ TEST(PortSysinfoTest, sysinfo_test_sysinfo_set_limit_CORE_FILE)
 	{
 		OMRPORT_ACCESS_FROM_OMRPORT(portTestEnv->getPortLibrary());
 		const char *testName = "omrsysinfo_test_sysinfo_set_limit_FILE_DESCRIPTORS";
-		intptr_t rc = -1;
+		uint32_t rc = OMRPORT_LIMIT_UNKNOWN;
 		uint64_t originalSoftLimit = 0;
 		uint64_t finalSoftLimit = 0;
+		uint64_t softSetToHardLimit = 0;
 		uint64_t originalHardLimit = 0;
 		uint64_t currentLimit = 0;
 		const uint64_t descriptorLimit = 256;
@@ -822,6 +823,7 @@ TEST(PortSysinfoTest, sysinfo_test_sysinfo_set_limit_CORE_FILE)
 			reportTestExit(OMRPORTLIB, testName);
 			return;
 		}
+		portTestEnv->log(LEVEL_ERROR, "originalSoftLimit=%llu\n", originalSoftLimit);
 		finalSoftLimit = originalSoftLimit;
 
 		rc = omrsysinfo_set_limit(OMRPORT_RESOURCE_FILE_DESCRIPTORS, descriptorLimit);
@@ -854,18 +856,54 @@ TEST(PortSysinfoTest, sysinfo_test_sysinfo_set_limit_CORE_FILE)
 			reportTestExit(OMRPORTLIB, testName);
 			return;
 		}
+		portTestEnv->log(LEVEL_ERROR, "originalHardLimit=%llu\n", originalHardLimit);
+
+		/* set soft limit to hard limit */
+		rc = omrsysinfo_set_limit(OMRPORT_RESOURCE_FILE_DESCRIPTORS, originalHardLimit);
+		if (0 != rc) {
+			outputErrorMessage(PORTTEST_ERROR_ARGS, "omrsysinfo_set_limit soft = hard FAILED rc=%d\n", rc);
+			reportTestExit(OMRPORTLIB, testName);
+			return;
+		}
+
+		/* get new soft limit */
+		rc = omrsysinfo_get_limit(OMRPORT_RESOURCE_FILE_DESCRIPTORS, &softSetToHardLimit);
+		if (OMRPORT_LIMIT_UNKNOWN == rc) {
+			outputErrorMessage(PORTTEST_ERROR_ARGS, "omrsysinfo_get_limit FAILED: OMRPORT_LIMIT_UNKNOWN\n");
+			reportTestExit(OMRPORTLIB, testName);
+			return;
+		}
+		portTestEnv->log(LEVEL_ERROR, "soft set to hard limit=%llu\n", softSetToHardLimit);
+
+		/* set soft limit to old value */
+		rc = omrsysinfo_set_limit(OMRPORT_RESOURCE_FILE_DESCRIPTORS, originalSoftLimit);
+		if (0 != rc) {
+			outputErrorMessage(PORTTEST_ERROR_ARGS, "omrsysinfo_set_limit reset soft FAILED rc=%d\n", rc);
+			reportTestExit(OMRPORTLIB, testName);
+			return;
+		}
+
+		rc = omrsysinfo_get_limit(OMRPORT_RESOURCE_FILE_DESCRIPTORS | OMRPORT_LIMIT_HARD, &currentLimit);
+		if (currentLimit != originalHardLimit) {
+			outputErrorMessage(PORTTEST_ERROR_ARGS, "omrsysinfo_get_limit FAILED: hard limit changed\n");
+			reportTestExit(OMRPORTLIB, testName);
+			return;
+		}
 
 		/* lowering the hard limit is irreversible unless privileged */
 		if (0 != geteuid()) { /* normal user */
 			/* setting the hard limit from unlimited to a finite value has unpredictable results:
 			 * the actual value may be much smaller than requested.
-			 * In that case, just try setting it to the same value.
+			 * In that case, just try setting it to its current value (softSetToHardLimit) or a value slightly lower.
+			 * Ensure that we don't try to set the hard limit to a value less than the current soft limit
+			 * (i.e. originalSoftLimit).
 			 */
-			uint64_t newHardLimit =  (OMRPORT_LIMIT_UNLIMITED == rc) ? originalHardLimit: originalHardLimit - 1;
+			uint64_t newHardLimit =  ((OMRPORT_LIMIT_UNLIMITED == rc) || (originalSoftLimit == softSetToHardLimit))
+					? softSetToHardLimit: softSetToHardLimit - 1;
 
 			rc = omrsysinfo_set_limit(OMRPORT_RESOURCE_FILE_DESCRIPTORS | OMRPORT_LIMIT_HARD, newHardLimit);
 			if (0 != rc) {
-				outputErrorMessage(PORTTEST_ERROR_ARGS, "omrsysinfo_set_limit set hard limit FAILED rc=%d\n", rc);
+				outputErrorMessage(PORTTEST_ERROR_ARGS, "omrsysinfo_set_limit set hard limit=%lld FAILED rc=%d\n", rc, newHardLimit);
 				reportTestExit(OMRPORTLIB, testName);
 				return;
 			}

port/unix/omrsysinfo.c

@@ -2375,7 +2375,7 @@ omrsysinfo_set_limit(struct OMRPortLibrary *portLibrary, uint32_t resourceID, ui
 #if !defined(OMRZTPF)
 		resource = RLIMIT_CORE;
 #else /* !defined(OMRZTPF) */
-		rc = -1;
+		rc = OMRPORT_LIMIT_UNKNOWN;
 #endif /* !defined(OMRZTPF) */
 		break;
 	default:
@@ -2400,6 +2400,22 @@ omrsysinfo_set_limit(struct OMRPortLibrary *portLibrary, uint32_t resourceID, ui
 			if (hardLimitRequested) {
 				lim.rlim_max = limit;
 			} else {
+#if defined(OSX)
+				/* MacOS doesn't allow the soft file limit to be unlimited */
+				if ((OMRPORT_RESOURCE_FILE_DESCRIPTORS == resourceRequested)
+						&& (RLIM_INFINITY == limit)) {
+					int32_t maxFiles = 0;
+					size_t resultSize = sizeof(maxFiles);
+					int name[] = {CTL_KERN, KERN_MAXFILESPERPROC};
+					rc = sysctl(name, 2, &maxFiles, &resultSize, NULL, 0);
+					if (-1 == rc) {
+						portLibrary->error_set_last_error(portLibrary, errno, findError(errno));
+						Trc_PRT_sysinfo_setrlimit_error(resource, limit, findError(errno));
+					} else {
+						limit = maxFiles;
+					}
+				}
+#endif
 				lim.rlim_cur = limit;
 			}
 
@@ -2409,7 +2425,7 @@ omrsysinfo_set_limit(struct OMRPortLibrary *portLibrary, uint32_t resourceID, ui
 				Trc_PRT_sysinfo_setrlimit_error(resource, limit, findError(errno));
 			}
 #else /* !defined(OMRZTPF) */
-			rc = -1;
+			rc = OMRPORT_LIMIT_UNKNOWN;
 #endif /* !defined(OMRZTPF) */
 			break;
 		}
@@ -2426,14 +2442,14 @@ omrsysinfo_set_limit(struct OMRPortLibrary *portLibrary, uint32_t resourceID, ui
 			}
 #else
 			/* unsupported so return error */
-			rc = -1;
+			rc = OMRPORT_LIMIT_UNKNOWN;
 #endif
 			break;
 		}
 
 		default:
 			Trc_PRT_sysinfo_setLimit_unrecognised_resourceID(resourceID);
-			rc = -1;
+			rc = OMRPORT_LIMIT_UNKNOWN;
 		}
 	}