From d95312c395d93ab41ece97b5e638aeff18676c3f Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Mon, 22 Apr 2024 11:04:33 -0400 Subject: [PATCH 01/44] zopen enhancements: - Override pthread_create to always set autocvt=on - Modify untagged heuristic to remove restriction on min required bytes (8) - Add tests --- include/pthread.h | 51 +++++++++++++++++++++++++++ src/zos-char-util.cc | 36 +++++++++----------- src/zos-io.cc | 29 +++++++++++++++- test/test-clib-override.cc | 70 +++++++++++++++++++++++++++++++++++++- test/test-thread.cc | 49 ++++++++++++++++++++++++++ 5 files changed, 214 insertions(+), 21 deletions(-) create mode 100644 include/pthread.h create mode 100644 test/test-thread.cc diff --git a/include/pthread.h b/include/pthread.h new file mode 100644 index 0000000..bf4b001 --- /dev/null +++ b/include/pthread.h @@ -0,0 +1,51 @@ +/////////////////////////////////////////////////////////////////////////////// +// Licensed Materials - Property of IBM +// ZOSLIB +// (C) Copyright IBM Corp. 2021. All Rights Reserved. +// US Government Users Restricted Rights - Use, duplication +// or disclosure restricted by GSA ADP Schedule Contract with IBM Corp. +/////////////////////////////////////////////////////////////////////////////// + +#ifndef ZOS_PTHREAD_H_ +#define ZOS_PTHREAD_H_ + +#define __XPLAT 1 +#include "zos-macros.h" +#include + + +#if defined(__cplusplus) +extern "C" { +#endif +__Z_EXPORT int __pthread_create_extended(pthread_t *thread, + const pthread_attr_t *attr, + void *(*start_routine)(void *), + void *arg); +#if defined(__cplusplus) +} +#endif + +#if defined(ZOSLIB_OVERRIDE_CLIB) || defined(ZOSLIB_OVERRIDE_CLIB_PTHREAD) +#define pthread_create __pthread_create_replaced +#endif + +#include_next + +#if defined(ZOSLIB_OVERRIDE_CLIB) || defined(ZOSLIB_OVERRIDE_CLIB_PTHREAD) + +#if defined(__cplusplus) +extern "C" { +#endif + +#undef pthread_create +__Z_EXPORT int pthread_create(pthread_t *thread, + const pthread_attr_t *attr, + void *(*start_routine)(void *), + void *arg) asm("__pthread_create_extended"); + +#if defined(__cplusplus) +} +#endif +#endif /* defined(ZOSLIB_OVERRIDE_CLIB) || defined(ZOSLIB_OVERRIDE_CLIB_PTHREAD) */ + +#endif diff --git a/src/zos-char-util.cc b/src/zos-char-util.cc index 557378e..4132de5 100644 --- a/src/zos-char-util.cc +++ b/src/zos-char-util.cc @@ -539,27 +539,25 @@ int __file_needs_conversion_init(const char *name, int fd) { close(fd); return 0; } - if (cnt > 8) { - int ccsid; - int am; - unsigned len = strlen_ae((unsigned char *)buf, &ccsid, cnt, &am); - if (ccsid == 1047 && len == cnt) { - if (no_tag_read_behaviour == __NO_TAG_READ_DEFAULT_WITHWARNING) { - if (name) { - len = strlen(name) + 1; - char filename[len]; - _convert_e2a(filename, name, len); - dprintf(2, "Warning: File \"%s\" is untagged and seems to contain " - "EBCDIC characters\n", filename); - } else { - dprintf(2, "Warning: File (null) is untagged and seems to contain " - "EBCDIC characters\n"); - } + int ccsid; + int am; + unsigned len = strlen_ae((unsigned char *)buf, &ccsid, cnt, &am); + if (ccsid == 1047 && len == cnt) { + if (no_tag_read_behaviour == __NO_TAG_READ_DEFAULT_WITHWARNING) { + if (name) { + len = strlen(name) + 1; + char filename[len]; + _convert_e2a(filename, name, len); + dprintf(2, "Warning: File \"%s\" is untagged and seems to contain " + "EBCDIC characters\n", filename); + } else { + dprintf(2, "Warning: File (null) is untagged and seems to contain " + "EBCDIC characters\n"); } - fdcache.set_attribute(fd, 0x0000000000020000UL); - return 1; } - } // cnt > 8 + fdcache.set_attribute(fd, 0x0000000000020000UL); + return 1; + } } // seekable files return 0; // not seekable } diff --git a/src/zos-io.cc b/src/zos-io.cc index 80f5580..80ecc2b 100644 --- a/src/zos-io.cc +++ b/src/zos-io.cc @@ -668,7 +668,7 @@ int __chgfdccsid(int fd, unsigned short ccsid) { memset(&attr, 0, sizeof(attr)); attr.att_filetagchg = 1; attr.att_filetag.ft_ccsid = ccsid; - if (ccsid != FT_BINARY) { + if (ccsid != FT_BINARY && ccsid != 0) { attr.att_filetag.ft_txtflag = 1; } return __fchattr(fd, &attr, sizeof(attr)); @@ -799,6 +799,8 @@ int __mkstemp_orig(char *) asm("@@A00184"); FILE *__fopen_orig(const char *filename, const char *mode) asm("@@A00246"); int __mkfifo_orig(const char *pathname, mode_t mode) asm("@@A00133"); struct utmpx *__getutxent_orig(void) asm("getutxent"); +int __pthread_create_orig(pthread_t *thread, const pthread_attr_t *attr, + void *(*start_routine)(void *), void *arg) asm("@@PT3C"); int utmpxname(char * file) { char buf[PATH_MAX]; @@ -1070,6 +1072,31 @@ bool __doLogMemoryWarning() { return __gLogMemoryAll || __gLogMemoryWarning; } +// pthread_create override to ensure that _CVTSTATE_OFF does not break multi-threaded programs +typedef struct { + void *(*start_routine)(void *); + void *arg; +} __ThreadArg; + +void *custom_start_routine(void *arg) { + __ThreadArg *threadArg = (__ThreadArg *)arg; + + int cvstate = __ae_autoconvert_state(_CVTSTATE_QUERY); + if (_CVTSTATE_OFF == cvstate) { + __ae_autoconvert_state(_CVTSTATE_ON); + } + + return threadArg->start_routine(threadArg->arg); +} + +int __pthread_create_extended(pthread_t *thread, const pthread_attr_t *attr, + void *(*start_routine)(void *), void *arg) { + __ThreadArg threadArg; + threadArg.start_routine = start_routine; + threadArg.arg = arg; + + return __pthread_create_orig(thread, attr, custom_start_routine, (void *)&threadArg); +} #ifdef __cplusplus } diff --git a/test/test-clib-override.cc b/test/test-clib-override.cc index a69cfc4..46148bd 100644 --- a/test/test-clib-override.cc +++ b/test/test-clib-override.cc @@ -12,7 +12,7 @@ namespace { class CLIBOverrides : public ::testing::Test { virtual void SetUp() { // Make sure default untagged read mode is set - setenv("UNTAGGED_READ_MODE", "AUTO", 1); + setenv("__UNTAGGED_READ_MODE", "AUTO", 1); temp_path = tmpnam(NULL); fd = open(temp_path, O_CREAT | O_WRONLY, 0660); } @@ -53,6 +53,74 @@ TEST_F(CLIBOverrides, open) { EXPECT_EQ(__getfdccsid(fd), 0x10000 + 819); close(fd); + // Test heuristic on 1 byte content + { + char buff[] = "A"; + __a2e_s(buff); + char* buff2 = (char*)malloc(sizeof(buff)); + remove(temp_path); + fd = open(temp_path, O_CREAT | O_WRONLY, 0777); + __setfdccsid(fd, 0); + write(fd, buff, sizeof(buff) - 1); + close(fd); + struct stat st; + stat(temp_path, &st); + + fd = open(temp_path, O_RDONLY); + int needs = __file_needs_conversion(fd); + EXPECT_EQ(needs, 1); + EXPECT_EQ(__getfdccsid(fd), 0x10000 + 1047); + memset(buff2, 0, sizeof(buff)); + read(fd, buff2, sizeof(buff)); + __e2a_s(buff); + EXPECT_EQ(strcmp(buff, buff2), 0); + close(fd); + free(buff2); + } + + // Test heuristic on several chars + { + char buff[] = "Hello world!"; + __a2e_s(buff); + char* buff2 = (char*)malloc(sizeof(buff)); + remove(temp_path); + fd = open(temp_path, O_CREAT | O_WRONLY, 0777); + __setfdccsid(fd, 0); + write(fd, buff, sizeof(buff) - 1); + close(fd); + struct stat st; + stat(temp_path, &st); + + fd = open(temp_path, O_RDONLY); + int needs = __file_needs_conversion(fd); + EXPECT_EQ(needs, 1); + EXPECT_EQ(__getfdccsid(fd), 0x10000 + 1047); + memset(buff2, 0, sizeof(buff)); + read(fd, buff2, sizeof(buff)); + __e2a_s(buff); + EXPECT_EQ(strcmp(buff, buff2), 0); + close(fd); + free(buff2); + } + + // Test heuristic on non-EBCDIC content + { + char buff[] = "Hello"; + remove(temp_path); + fd = open(temp_path, O_CREAT | O_WRONLY, 0777); + __setfdccsid(fd, 0); + write(fd, buff, sizeof(buff) - 1); + close(fd); + struct stat st; + stat(temp_path, &st); + + fd = open(temp_path, O_RDONLY); + int needs = __file_needs_conversion(fd); + EXPECT_EQ(needs, 0); + EXPECT_EQ(__getfdccsid(fd), 0); + close(fd); + } + char buff[] = "This is a test"; char* buff2 = (char*)malloc(sizeof(buff)); diff --git a/test/test-thread.cc b/test/test-thread.cc new file mode 100644 index 0000000..8ea98d4 --- /dev/null +++ b/test/test-thread.cc @@ -0,0 +1,49 @@ +// Enable CLIB overrides +#define ZOSLIB_OVERRIDE_CLIB 1 + +#include "zos.h" +#include "gtest/gtest.h" +#include +#include +#include +#include + +int cvsstate = 0; + +namespace { + +void* print_cvstate(void* arg) { + int* cvsstate_ptr = static_cast(arg); + *cvsstate_ptr = __ae_autoconvert_state(_CVTSTATE_QUERY); + + return NULL; +} + +TEST(CvstateTest, PrintCvstate) { + { + int cvsstate = 0; + unsetenv("_BPXK_AUTOCVT"); + + pthread_t thread; + pthread_create(&thread, NULL, &print_cvstate, &cvsstate); + + pthread_join(thread, NULL); + + EXPECT_EQ(cvsstate, _CVTSTATE_ON); + } + + { + int cvsstate = 0; + setenv("_BPXK_AUTOCVT", "ON", 1); + + pthread_t thread; + pthread_create(&thread, NULL, &print_cvstate, &cvsstate); + + pthread_join(thread, NULL); + + EXPECT_EQ(cvsstate, _CVTSTATE_ON); + } +} + +} + From 11eeb205e1205dcfcc05e201613fe3ee87dfb181 Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Tue, 25 Jun 2024 10:52:15 -0400 Subject: [PATCH 02/44] Add missing header netiniet --- include/netinet/in_systm.h | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 include/netinet/in_systm.h diff --git a/include/netinet/in_systm.h b/include/netinet/in_systm.h new file mode 100644 index 0000000..5fff894 --- /dev/null +++ b/include/netinet/in_systm.h @@ -0,0 +1,19 @@ +/////////////////////////////////////////////////////////////////////////////// +// Licensed Materials - Property of IBM +// ZOSLIB +// (C) Copyright IBM Corp. 2024. All Rights Reserved. +// US Government Users Restricted Rights - Use, duplication +// or disclosure restricted by GSA ADP Schedule Contract with IBM Corp. +/////////////////////////////////////////////////////////////////////////////// + +#ifndef ZOS_IN_SYSTM_H_ +#define ZOS_IN_SYSTM_H_ + +#include + +// Network types +typedef uint16_t n_short; +typedef uint32_t n_long; +typedef uint32_t n_time; + +#endif From ebe82e095c3b316154355b7e3b68b547c1ab20ce Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Wed, 26 Jun 2024 09:11:03 -0400 Subject: [PATCH 03/44] Add support for getline, getdelim, sysconf, memset --- include/stdio.h | 3 ++ include/stdlib.h | 3 ++ include/string.h | 1 + include/unistd.h | 12 ++++- src/zos-io.cc | 130 +++++++++++++++++++++++++++++++++++++++++++++-- src/zos-string.c | 14 +++++ src/zos.cc | 12 +++++ 7 files changed, 170 insertions(+), 5 deletions(-) diff --git a/include/stdio.h b/include/stdio.h index 7997a85..310f18e 100644 --- a/include/stdio.h +++ b/include/stdio.h @@ -10,6 +10,7 @@ #define ZOS_STDIO_H_ #include "zos-macros.h" +#include #define __XPLAT 1 @@ -39,6 +40,8 @@ extern "C" { #endif __Z_EXPORT extern FILE *fopen(const char *filename, const char *mode) __asm("__fopen_ascii"); +__Z_EXPORT ssize_t getline(char **lineptr, size_t *n, FILE *stream); +__Z_EXPORT ssize_t getdelim(char **lineptr, size_t *n, int delimiter, FILE *stream); #if defined(__cplusplus) } diff --git a/include/stdlib.h b/include/stdlib.h index c0d73da..43d2144 100644 --- a/include/stdlib.h +++ b/include/stdlib.h @@ -104,6 +104,9 @@ extern "C" { */ __Z_EXPORT int getloadavg(double loadavg[], int nelem); __Z_EXPORT const char * getprogname(void); +__Z_EXPORT int mkostemp(char *, int flags); +__Z_EXPORT int mkstemps(char *, int suffixlen); +__Z_EXPORT int mkostemps(char *, int suffixlen, int flags); #if defined(__cplusplus) } #endif diff --git a/include/string.h b/include/string.h index 918db1b..b3f40fd 100644 --- a/include/string.h +++ b/include/string.h @@ -24,6 +24,7 @@ __Z_EXPORT char *strndup(const char *s, size_t n); __Z_EXPORT char *strsignal(int ); __Z_EXPORT const char *sigdescr_np(int); __Z_EXPORT const char *sigabbrev_np(int); +__Z_EXPORT void *memmem(const void *haystack, size_t haystacklen, const void *needle, size_t needlelen); // Linux includes strings.h in string.h, this avoids the // warning - implicitly declaring library function 'strcasecmp' diff --git a/include/unistd.h b/include/unistd.h index 0a72208..43b80d7 100644 --- a/include/unistd.h +++ b/include/unistd.h @@ -17,6 +17,8 @@ extern "C" { #endif __Z_EXPORT int __pipe_ascii(int [2]); __Z_EXPORT int __close(int); +__Z_EXPORT int __sysconf(int name); + #if defined(__cplusplus) } #endif @@ -27,19 +29,23 @@ __Z_EXPORT int __close(int); #define pipe __pipe_replaced #undef close #define close __close_replaced +#undef sysconf +#define sysconf __sysconf_replaced #include_next #undef pipe #undef close +#undef sysconf #if defined(__cplusplus) extern "C" { #endif - /** * Same as C pipe but tags pipes as ASCII (819) */ __Z_EXPORT int pipe(int [2]) __asm("__pipe_ascii"); __Z_EXPORT int close(int) __asm("__close"); +__Z_EXPORT int close(int) __asm("__close"); +__Z_EXPORT int sysconf(int name) __asm("__sysconf"); #if defined(__cplusplus) } @@ -75,4 +81,8 @@ __Z_EXPORT int execvpe(const char *name, char *const argv[], } #endif +#ifndef _SC_NPROCESSORS_ONLN +#define _SC_NPROCESSORS_ONLN 58 /* match linux */ +#endif + #endif diff --git a/src/zos-io.cc b/src/zos-io.cc index 80ecc2b..fbe0a9b 100644 --- a/src/zos-io.cc +++ b/src/zos-io.cc @@ -1091,11 +1091,133 @@ void *custom_start_routine(void *arg) { int __pthread_create_extended(pthread_t *thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *arg) { - __ThreadArg threadArg; - threadArg.start_routine = start_routine; - threadArg.arg = arg; + __ThreadArg *threadArg = (__ThreadArg *)malloc(sizeof(__ThreadArg)); + threadArg->start_routine = start_routine; + threadArg->arg = arg; - return __pthread_create_orig(thread, attr, custom_start_routine, (void *)&threadArg); + return __pthread_create_orig(thread, attr, custom_start_routine, (void *)threadArg); +} + +ssize_t getline(char **lineptr, size_t *n, FILE *stream) { + if (lineptr == NULL || n == NULL || stream == NULL) { + return -1; + } + + int c; + size_t pos = 0; + + if (*lineptr == NULL || *n == 0) { + *n = 128; + *lineptr = (char *)malloc(*n); + if (*lineptr == NULL) { + return -1; + } + } + + while ((c = fgetc(stream)) != EOF) { + if (pos + 1 >= *n) { + // Resize buffer if needed + *n *= 2; + char *new_ptr = (char *)realloc(*lineptr, *n); + if (new_ptr == NULL) { + return -1; + } + *lineptr = new_ptr; + } + + (*lineptr)[pos++] = (char)c; + + if (c == '\n') { + break; + } + } + + if (pos == 0 && c == EOF) { + return -1; + } + + (*lineptr)[pos] = '\0'; + + return pos; +} + +ssize_t getdelim(char **lineptr, size_t *n, int delimiter, FILE *stream) { + char *buf = *lineptr; + size_t bufsize = *n; + size_t len = 0; + int c; + + if (buf == NULL || bufsize == 0) { + bufsize = 128; + buf = (char*)malloc(bufsize); + if (buf == NULL) { + return -1; + } + *lineptr = buf; + *n = bufsize; + } + + while (1) { + c = fgetc(stream); + if (c == EOF || c == delimiter) { + break; + } + + if (len + 1 >= bufsize) { + bufsize *= 2; + buf = (char*)realloc(buf, bufsize); + if (buf == NULL) { + return -1; + } + *lineptr = buf; + *n = bufsize; + } + + buf[len++] = c; + } + + buf[len] = '\0'; + + if (c == EOF && len == 0) { + free(buf); + *lineptr = NULL; + *n = 0; + return -1; + } + + return len; +} + +int mkostemp(char *tmpl, int flags) { + if (strlen(tmpl) < 6 || strcmp(&tmpl[strlen(tmpl) - 6], "XXXXXX") != 0) { + errno = EINVAL; + return -1; + } + return mkstemp(tmpl); +} + +int mkostemps(char *tmpl, int suffixlen, int flags) { + if (strlen(tmpl) < 6 + suffixlen || strcmp(&tmpl[strlen(tmpl) - 6 - suffixlen], "XXXXXX") != 0) { + errno = EINVAL; + return -1; + } + char *p = tmpl + strlen(tmpl) - suffixlen - 6; + *p = '\0'; + int fd = mkostemp(tmpl, flags); + *p = 'X'; + return fd; +} + +int mkstemps(char *tmpl, int suffixlen) { + if (strlen(tmpl) < 6 + suffixlen || strcmp(&tmpl[strlen(tmpl) - 6 - suffixlen], "XXXXXX") != 0) { + errno = EINVAL; + return -1; + } + char *p = tmpl + strlen(tmpl) - suffixlen - 6; + *p = '\0'; + int fd = __mkstemp_ascii(tmpl); + *p = 'X'; + return fd; } #ifdef __cplusplus diff --git a/src/zos-string.c b/src/zos-string.c index ae5310f..4f67713 100644 --- a/src/zos-string.c +++ b/src/zos-string.c @@ -139,6 +139,20 @@ char *strndup(const char *s, size_t n) { return dupStr; } +void *memmem(const void *haystack, size_t haystacklen, const void *needle, size_t needlelen) { + size_t i; + const unsigned char *haystack_bytes = (const unsigned char *)haystack; + const unsigned char *needle_bytes = (const unsigned char *)needle; + + for (i = 0; i <= haystacklen - needlelen; i++) { + if (memcmp(&haystack_bytes[i], needle_bytes, needlelen) == 0) { + return (void *)&haystack_bytes[i]; + } + } + + return NULL; +} + #if TEST int main() { printf("values: segv=%d total=%d\n", SIGSEGV, sigTotal); diff --git a/src/zos.cc b/src/zos.cc index e17433e..7e4c161 100644 --- a/src/zos.cc +++ b/src/zos.cc @@ -3133,6 +3133,18 @@ extern "C" void __aligned_free(void *ptr) { #else free((reinterpret_cast(ptr))[-1]); #endif + +// C Library overrides +int __sysconf_orig(int ) asm("sysconf"); + +// Add support for _SC_NPROCESSORS_ONLN +int __sysconf(int name) { + switch (name) { + case _SC_NPROCESSORS_ONLN: + return __get_num_online_cpus(); + default: + return __sysconf_orig(name); + } } #if defined(ZOSLIB_INITIALIZE) From b32dcd46e7dc73a84514cfd45d610f10962f5aef Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Thu, 27 Jun 2024 13:59:00 -0400 Subject: [PATCH 04/44] Add utimesat and PTHREAD_RWLOCK_INITIALIZER --- include/pthread.h | 4 ++++ include/zos-v2r5-symbolfixes.h | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/include/pthread.h b/include/pthread.h index bf4b001..c857a3c 100644 --- a/include/pthread.h +++ b/include/pthread.h @@ -48,4 +48,8 @@ __Z_EXPORT int pthread_create(pthread_t *thread, #endif #endif /* defined(ZOSLIB_OVERRIDE_CLIB) || defined(ZOSLIB_OVERRIDE_CLIB_PTHREAD) */ +#ifndef PTHREAD_RWLOCK_INITIALIZER +#define PTHREAD_RWLOCK_INITIALIZER PTHREAD_RWLOCK_INITIALIZER_NP +#endif + #endif diff --git a/include/zos-v2r5-symbolfixes.h b/include/zos-v2r5-symbolfixes.h index 1d9fd98..35f8a44 100644 --- a/include/zos-v2r5-symbolfixes.h +++ b/include/zos-v2r5-symbolfixes.h @@ -23,7 +23,6 @@ #pragma redefine_extname getrandom getrandom_undefined #pragma redefine_extname pipe2 pipe2_undefined #pragma redefine_extname fsstatfs fsstatfs_undefined -#pragma redefine_extname getline getline_undefined #pragma redefine_extname dprintf dprintf_undefined #pragma redefine_extname dirfd dirfd_undefined #pragma redefine_extname fchmodat fchmodat_undefined @@ -45,6 +44,7 @@ #pragma redefine_extname setresgid setresgid_undefined #pragma redefine_extname dup3 dup3_undefined #pragma redefine_extname shm_open shm_open_undefined +#pragma redefine_extname utimensat utimensat_undefined #endif #endif // ZOS_V2R5_SYMBOLFIXES_H From 7d497c6b6e151a049ad0f2bf6867e0dd8757b14e Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Thu, 11 Jul 2024 15:35:59 -0400 Subject: [PATCH 05/44] Add readlink override to resolve PARMLIB variables like --- include/unistd.h | 5 +++++ src/zos-bpx.cc | 25 +++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/include/unistd.h b/include/unistd.h index 43b80d7..4e90d69 100644 --- a/include/unistd.h +++ b/include/unistd.h @@ -31,10 +31,13 @@ __Z_EXPORT int __sysconf(int name); #define close __close_replaced #undef sysconf #define sysconf __sysconf_replaced +#undef readlink +#define readlink __readlink_replaced #include_next #undef pipe #undef close #undef sysconf +#undef readlink #if defined(__cplusplus) extern "C" { @@ -46,6 +49,8 @@ __Z_EXPORT int pipe(int [2]) __asm("__pipe_ascii"); __Z_EXPORT int close(int) __asm("__close"); __Z_EXPORT int close(int) __asm("__close"); __Z_EXPORT int sysconf(int name) __asm("__sysconf"); +__Z_EXPORT ssize_t readlink(const char *path, char *buf, size_t bufsiz) asm("__readlink"); + #if defined(__cplusplus) } diff --git a/src/zos-bpx.cc b/src/zos-bpx.cc index 3ed94ff..376bb8e 100644 --- a/src/zos-bpx.cc +++ b/src/zos-bpx.cc @@ -90,6 +90,31 @@ char *__realpath_extended(const char __restrict__ *path, char __restrict__ *reso return __realpath_orig(path, resolved_path); } +ssize_t __readlink_orig(const char *path, char *buf, size_t bufsiz) asm("@@A00202"); + +ssize_t __readlink(const char *path, char *buf, size_t bufsiz) { + ssize_t len = __readlink_orig(path, buf, bufsiz - 1); + if (len < 0) { + return -1; + } + + buf[len] = '\0'; // readlink doesn't null terminate + + if (buf[0] == '$' || (len > 1 && buf[0] == '/' && buf[1] == '$')) { + char resolved_path[PATH_MAX]; + if (realpath(path, resolved_path) == NULL) { + return -1; + } + + len = snprintf(buf, bufsiz, "%s", resolved_path); + if (len >= bufsiz) { + errno = ENAMETOOLONG; + return -1; + } + } + + return len; +} void __bpx4kil(int pid, int signal, void *signal_options, int *return_value, int *return_code, int *reason_code) { From 8253dfef6902968d30bd69b0dde59290e02baed6 Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Mon, 15 Jul 2024 21:48:46 -0400 Subject: [PATCH 06/44] Fix getdelim and readlink override --- src/zos-bpx.cc | 10 +++++--- src/zos-io.cc | 70 +++++++++++++++++++++++++------------------------- 2 files changed, 42 insertions(+), 38 deletions(-) diff --git a/src/zos-bpx.cc b/src/zos-bpx.cc index 376bb8e..9a9116e 100644 --- a/src/zos-bpx.cc +++ b/src/zos-bpx.cc @@ -98,9 +98,13 @@ ssize_t __readlink(const char *path, char *buf, size_t bufsiz) { return -1; } - buf[len] = '\0'; // readlink doesn't null terminate - - if (buf[0] == '$' || (len > 1 && buf[0] == '/' && buf[1] == '$')) { + if ((len > 0 buf[0] == '$') || (len > 1 && buf[0] == '/' && buf[1] == '$')) { + // Not sure if this is possible, but double check in case: + if (len < bufsiz) { + buf[len] = '\0'; + } else { + buf[bufsiz - 1] = '\0'; + } char resolved_path[PATH_MAX]; if (realpath(path, resolved_path) == NULL) { return -1; diff --git a/src/zos-io.cc b/src/zos-io.cc index fbe0a9b..4402957 100644 --- a/src/zos-io.cc +++ b/src/zos-io.cc @@ -1142,50 +1142,50 @@ ssize_t getline(char **lineptr, size_t *n, FILE *stream) { } ssize_t getdelim(char **lineptr, size_t *n, int delimiter, FILE *stream) { - char *buf = *lineptr; - size_t bufsize = *n; - size_t len = 0; - int c; + size_t pos; + int c; - if (buf == NULL || bufsize == 0) { - bufsize = 128; - buf = (char*)malloc(bufsize); - if (buf == NULL) { - return -1; + if (lineptr == NULL || n == NULL || stream == NULL) { + errno = EINVAL; + return -1; } - *lineptr = buf; - *n = bufsize; - } - while (1) { - c = fgetc(stream); - if (c == EOF || c == delimiter) { - break; + if (*lineptr == NULL) { + *n = 128; // Start with a reasonable buffer size + *lineptr = (char *)malloc(*n); + if (*lineptr == NULL) { + errno = ENOMEM; + return -1; + } } - if (len + 1 >= bufsize) { - bufsize *= 2; - buf = (char*)realloc(buf, bufsize); - if (buf == NULL) { - return -1; - } - *lineptr = buf; - *n = bufsize; - } + pos = 0; - buf[len++] = c; - } + while ((c = fgetc(stream)) != EOF) { + if (pos + 1 >= *n) { + size_t new_size = *n * 2; + char *new_ptr = (char *)realloc(*lineptr, new_size); + if (new_ptr == NULL) { + errno = ENOMEM; + return -1; + } + *lineptr = new_ptr; + *n = new_size; + } - buf[len] = '\0'; + (*lineptr)[pos++] = (char)c; - if (c == EOF && len == 0) { - free(buf); - *lineptr = NULL; - *n = 0; - return -1; - } + if (c == delimiter) { + break; + } + } - return len; + if (pos == 0 && c == EOF) { + return -1; + } + + (*lineptr)[pos] = '\0'; + return pos; } int mkostemp(char *tmpl, int flags) { From 153ef5f5ed3195712a4f4708ebd5be975570cd06 Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Mon, 15 Jul 2024 21:49:40 -0400 Subject: [PATCH 07/44] Fix typo --- src/zos-bpx.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zos-bpx.cc b/src/zos-bpx.cc index 9a9116e..9057ba0 100644 --- a/src/zos-bpx.cc +++ b/src/zos-bpx.cc @@ -98,7 +98,7 @@ ssize_t __readlink(const char *path, char *buf, size_t bufsiz) { return -1; } - if ((len > 0 buf[0] == '$') || (len > 1 && buf[0] == '/' && buf[1] == '$')) { + if ((len > 0 && buf[0] == '$') || (len > 1 && buf[0] == '/' && buf[1] == '$')) { // Not sure if this is possible, but double check in case: if (len < bufsiz) { buf[len] = '\0'; From b7074a5441aa87c30891a503a3dcff17efd41a84 Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Mon, 15 Jul 2024 22:25:41 -0400 Subject: [PATCH 08/44] Fix getdelim again --- src/zos-io.cc | 63 ++++++++++++++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 21 deletions(-) diff --git a/src/zos-io.cc b/src/zos-io.cc index 4402957..fd4d81f 100644 --- a/src/zos-io.cc +++ b/src/zos-io.cc @@ -1141,51 +1141,72 @@ ssize_t getline(char **lineptr, size_t *n, FILE *stream) { return pos; } -ssize_t getdelim(char **lineptr, size_t *n, int delimiter, FILE *stream) { - size_t pos; - int c; +ssize_t getdelim(char **lineptr, size_t *n, int delimiter, FILE *fp) { + ssize_t result; + size_t cur_len = 0; - if (lineptr == NULL || n == NULL || stream == NULL) { + if (lineptr == NULL || n == NULL || fp == NULL) { errno = EINVAL; return -1; } - if (*lineptr == NULL) { - *n = 128; // Start with a reasonable buffer size - *lineptr = (char *)malloc(*n); + if (*lineptr == NULL || *n == 0) { + *n = 120; + *lineptr = (char *)realloc(*lineptr, *n); if (*lineptr == NULL) { errno = ENOMEM; return -1; } } - pos = 0; + for (;;) { + int i = getc(fp); + if (i == EOF) { + if (cur_len == 0) { + result = -1; + } else { + result = cur_len; + } + break; + } - while ((c = fgetc(stream)) != EOF) { - if (pos + 1 >= *n) { - size_t new_size = *n * 2; - char *new_ptr = (char *)realloc(*lineptr, new_size); - if (new_ptr == NULL) { + if (cur_len + 1 >= *n) { + size_t needed_max = SSIZE_MAX < SIZE_MAX ? (size_t)SSIZE_MAX + 1 : SIZE_MAX; + size_t needed = 2 * *n + 1; + + if (needed_max < needed) { + needed = needed_max; + } + if (cur_len + 1 >= needed) { + result = -1; + errno = EOVERFLOW; + return result; + } + + char *new_lineptr = (char *)realloc(*lineptr, needed); + if (new_lineptr == NULL) { errno = ENOMEM; return -1; } - *lineptr = new_ptr; - *n = new_size; + + *lineptr = new_lineptr; + *n = needed; } - (*lineptr)[pos++] = (char)c; + (*lineptr)[cur_len] = i; + cur_len++; - if (c == delimiter) { + if (i == delimiter) { break; } } - if (pos == 0 && c == EOF) { - return -1; + if (cur_len > 0) { + (*lineptr)[cur_len] = '\0'; + result = cur_len; } - (*lineptr)[pos] = '\0'; - return pos; + return result; } int mkostemp(char *tmpl, int flags) { From b0cd5476adaebfc5fb2665ac60c0713cd355eeda Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Tue, 16 Jul 2024 09:11:07 -0400 Subject: [PATCH 09/44] Fix missing } --- src/zos.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/zos.cc b/src/zos.cc index 7e4c161..515ea9e 100644 --- a/src/zos.cc +++ b/src/zos.cc @@ -3133,6 +3133,7 @@ extern "C" void __aligned_free(void *ptr) { #else free((reinterpret_cast(ptr))[-1]); #endif +} // C Library overrides int __sysconf_orig(int ) asm("sysconf"); From 276feb4795bba552243e4dab3a4bbdb90721c71a Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Tue, 16 Jul 2024 13:40:57 -0400 Subject: [PATCH 10/44] Address comments --- src/zos-char-util.cc | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/zos-char-util.cc b/src/zos-char-util.cc index 4132de5..03a3cc1 100644 --- a/src/zos-char-util.cc +++ b/src/zos-char-util.cc @@ -496,6 +496,7 @@ class fdAttributeCache { }; fdAttributeCache fdcache; +#define FD_NEEDS_CONVERSION_ATTR 0x0000000000020000UL void __fd_close(int fd) { if (!bfdcache_destroyed) @@ -506,7 +507,7 @@ int __file_needs_conversion(int fd) { if (__get_no_tag_read_behaviour() == __NO_TAG_READ_STRICT) return 0; unsigned long attr = fdcache.get_attribute(fd); - if (attr == 0x0000000000020000UL) { + if (attr == FD_NEEDS_CONVERSION_ATTR) { return 1; } return 0; @@ -527,7 +528,7 @@ int __file_needs_conversion_init(const char *name, int fd) { if (no_tag_read_behaviour == __NO_TAG_READ_STRICT) return 0; if (no_tag_read_behaviour == __NO_TAG_READ_V6) { - fdcache.set_attribute(fd, 0x0000000000020000UL); + fdcache.set_attribute(fd, FD_NEEDS_CONVERSION_ATTR); return 1; } if (lseek(fd, 1, SEEK_SET) == 1 && lseek(fd, 0, SEEK_SET) == 0) { @@ -545,8 +546,8 @@ int __file_needs_conversion_init(const char *name, int fd) { if (ccsid == 1047 && len == cnt) { if (no_tag_read_behaviour == __NO_TAG_READ_DEFAULT_WITHWARNING) { if (name) { - len = strlen(name) + 1; - char filename[len]; + len = strlen(name); + char filename[len + 1]; _convert_e2a(filename, name, len); dprintf(2, "Warning: File \"%s\" is untagged and seems to contain " "EBCDIC characters\n", filename); @@ -555,7 +556,7 @@ int __file_needs_conversion_init(const char *name, int fd) { "EBCDIC characters\n"); } } - fdcache.set_attribute(fd, 0x0000000000020000UL); + fdcache.set_attribute(fd, FD_NEEDS_CONVERSION_ATTR); return 1; } } // seekable files From 827486bda02ff69603a9a21bde88fd931aa792cb Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Tue, 16 Jul 2024 13:41:15 -0400 Subject: [PATCH 11/44] Address comments --- src/zos-io.cc | 139 +++++++++++++++++--------------------------------- 1 file changed, 47 insertions(+), 92 deletions(-) diff --git a/src/zos-io.cc b/src/zos-io.cc index fd4d81f..06a546e 100644 --- a/src/zos-io.cc +++ b/src/zos-io.cc @@ -1098,115 +1098,70 @@ int __pthread_create_extended(pthread_t *thread, const pthread_attr_t *attr, return __pthread_create_orig(thread, attr, custom_start_routine, (void *)threadArg); } -ssize_t getline(char **lineptr, size_t *n, FILE *stream) { - if (lineptr == NULL || n == NULL || stream == NULL) { - return -1; - } - - int c; - size_t pos = 0; - - if (*lineptr == NULL || *n == 0) { - *n = 128; - *lineptr = (char *)malloc(*n); - if (*lineptr == NULL) { - return -1; - } - } - - while ((c = fgetc(stream)) != EOF) { - if (pos + 1 >= *n) { - // Resize buffer if needed - *n *= 2; - char *new_ptr = (char *)realloc(*lineptr, *n); - if (new_ptr == NULL) { - return -1; - } - *lineptr = new_ptr; - } +ssize_t getdelim(char **lineptr, size_t *n, int delimiter, FILE *fp) { + ssize_t result = 0; + size_t cur_len = 0; - (*lineptr)[pos++] = (char)c; + if (lineptr == NULL || n == NULL || fp == NULL) { + errno = EINVAL; + return -1; + } - if (c == '\n') { - break; - } + if (*lineptr == NULL || *n == 0) { + *n = 128; /* TODO: find a good initial value */ + *lineptr = (char *)malloc(*n); + if (*lineptr == NULL) { + errno = ENOMEM; + return -1; } + } - if (pos == 0 && c == EOF) { + while (1) { + int i = getc(fp); + if (i == EOF) { + if (cur_len == 0) { return -1; + } + break; } - (*lineptr)[pos] = '\0'; - - return pos; -} + if (cur_len + 1 >= *n) { + size_t needed_max = SSIZE_MAX < SIZE_MAX ? (size_t)SSIZE_MAX + 1 : SIZE_MAX; + size_t needed = 2 * *n + 1; /* double it */ -ssize_t getdelim(char **lineptr, size_t *n, int delimiter, FILE *fp) { - ssize_t result; - size_t cur_len = 0; + if (needed_max < needed) { + needed = needed_max; + } + if (cur_len + 1 >= needed) { + errno = EOVERFLOW; + return -1; + } - if (lineptr == NULL || n == NULL || fp == NULL) { - errno = EINVAL; + char *new_lineptr = (char *)realloc(*lineptr, needed); + if (new_lineptr == NULL) { + errno = ENOMEM; return -1; - } + } - if (*lineptr == NULL || *n == 0) { - *n = 120; - *lineptr = (char *)realloc(*lineptr, *n); - if (*lineptr == NULL) { - errno = ENOMEM; - return -1; - } + *lineptr = new_lineptr; + *n = needed; } - for (;;) { - int i = getc(fp); - if (i == EOF) { - if (cur_len == 0) { - result = -1; - } else { - result = cur_len; - } - break; - } + (*lineptr)[cur_len++] = i; - if (cur_len + 1 >= *n) { - size_t needed_max = SSIZE_MAX < SIZE_MAX ? (size_t)SSIZE_MAX + 1 : SIZE_MAX; - size_t needed = 2 * *n + 1; - - if (needed_max < needed) { - needed = needed_max; - } - if (cur_len + 1 >= needed) { - result = -1; - errno = EOVERFLOW; - return result; - } - - char *new_lineptr = (char *)realloc(*lineptr, needed); - if (new_lineptr == NULL) { - errno = ENOMEM; - return -1; - } - - *lineptr = new_lineptr; - *n = needed; - } - - (*lineptr)[cur_len] = i; - cur_len++; - - if (i == delimiter) { - break; - } + if (i == delimiter) { + break; } + } - if (cur_len > 0) { - (*lineptr)[cur_len] = '\0'; - result = cur_len; - } + (*lineptr)[cur_len] = '\0'; + result = cur_len; + + return result; +} - return result; +ssize_t getline(char **lineptr, size_t *n, FILE *stream) { + return getdelim(lineptr, n, '\n', stream); } int mkostemp(char *tmpl, int flags) { From 4145cf900603b19794f2012a5128e3e4c610f040 Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Fri, 19 Jul 2024 09:44:13 -0400 Subject: [PATCH 12/44] Add __threading_support - needed for boost and clang 1.1 builds --- include/c++/v1/__threading_support | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 include/c++/v1/__threading_support diff --git a/include/c++/v1/__threading_support b/include/c++/v1/__threading_support new file mode 100644 index 0000000..c53266e --- /dev/null +++ b/include/c++/v1/__threading_support @@ -0,0 +1,19 @@ +/////////////////////////////////////////////////////////////////////////////// +// Licensed Materials - Property of IBM +// ZOSLIB +// (C) Copyright IBM Corp. 2022. All Rights Reserved. +// US Government Users Restricted Rights - Use, duplication +// or disclosure restricted by GSA ADP Schedule Contract with IBM Corp. +/////////////////////////////////////////////////////////////////////////////// + +#ifndef ZOS_LIBCPP_THREADING_SUPPORT +#define ZOS_LIBCPP_THREADING_SUPPORT + +//TODO(itodorov) - zos: workaround for nanosleep WoZ conflict, revisit +// once WoZ allow overriding or LE provides a nanosleep definition +#include +#define nanosleep cpp_nanosleep +#include_next <__threading_support> +#undef nanosleep + +#endif From 040105dd67ffa8cb6effa5abbb150a763a860fa4 Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Mon, 22 Jul 2024 14:53:37 -0400 Subject: [PATCH 13/44] Update memmem --- src/zos-string.c | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/zos-string.c b/src/zos-string.c index 4f67713..1e014d2 100644 --- a/src/zos-string.c +++ b/src/zos-string.c @@ -139,16 +139,20 @@ char *strndup(const char *s, size_t n) { return dupStr; } -void *memmem(const void *haystack, size_t haystacklen, const void *needle, size_t needlelen) { - size_t i; - const unsigned char *haystack_bytes = (const unsigned char *)haystack; - const unsigned char *needle_bytes = (const unsigned char *)needle; - - for (i = 0; i <= haystacklen - needlelen; i++) { - if (memcmp(&haystack_bytes[i], needle_bytes, needlelen) == 0) { - return (void *)&haystack_bytes[i]; - } - } +void *memmem(const void *l, size_t l_len, const void *s, size_t s_len) { + const char *cl = (const char *)l; + const char *cs = (const char *)s; + + if (l_len == 0 || s_len == 0 || l_len < s_len) + return NULL; + + if (s_len == 1) + return memchr(l, (int)*cs, l_len); + + size_t len = l_len - s_len + 1; + for (size_t i = 0; i < len; i++) + if (cs[0] == cl[i] && memcmp(cl + i, cs, s_len) == 0) + return (void *)(cl + i); return NULL; } From 8c0764dd47922bb5d78df985849a08fa7acb4262 Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Tue, 23 Jul 2024 16:51:45 -0400 Subject: [PATCH 14/44] Add strverscmp from musl c --- include/string.h | 1 + src/zos-string.c | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/include/string.h b/include/string.h index b3f40fd..25f0145 100644 --- a/include/string.h +++ b/include/string.h @@ -25,6 +25,7 @@ __Z_EXPORT char *strsignal(int ); __Z_EXPORT const char *sigdescr_np(int); __Z_EXPORT const char *sigabbrev_np(int); __Z_EXPORT void *memmem(const void *haystack, size_t haystacklen, const void *needle, size_t needlelen); +__Z_EXPORT int strverscmp(const char *l0, const char *r0); // Linux includes strings.h in string.h, this avoids the // warning - implicitly declaring library function 'strcasecmp' diff --git a/src/zos-string.c b/src/zos-string.c index 1e014d2..d301458 100644 --- a/src/zos-string.c +++ b/src/zos-string.c @@ -157,6 +157,35 @@ void *memmem(const void *l, size_t l_len, const void *s, size_t s_len) { return NULL; } +// Adapted from Musl c's implementation (MIT license) +int strverscmp(const char *l0, const char *r0) +{ + const unsigned char *l = (const void *)l0; + const unsigned char *r = (const void *)r0; + size_t i, dp, j; + int z = 1; + /* Find maximal matching prefix and track its maximal digit + * suffix and whether those digits are all zeros. */ + for (dp=i=0; l[i]==r[i]; i++) { + int c = l[i]; + if (!c) return 0; + if (!isdigit(c)) dp=i+1, z=1; + else if (c!='0') z=0; + } + if (l[dp]!='0' && r[dp]!='0') { + /* If we're not looking at a digit sequence that began + * with a zero, longest digit string is greater. */ + for (j=i; isdigit(l[j]); j++) + if (!isdigit(r[j])) return 1; + if (isdigit(r[j])) return -1; + } else if (z && dp Date: Tue, 23 Jul 2024 22:52:38 -0400 Subject: [PATCH 15/44] Fix readlink and mkostemp - after further testing with zopen tools --- src/zos-io.cc | 47 +++++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/src/zos-io.cc b/src/zos-io.cc index 06a546e..29687e9 100644 --- a/src/zos-io.cc +++ b/src/zos-io.cc @@ -1164,38 +1164,45 @@ ssize_t getline(char **lineptr, size_t *n, FILE *stream) { return getdelim(lineptr, n, '\n', stream); } -int mkostemp(char *tmpl, int flags) { - if (strlen(tmpl) < 6 || strcmp(&tmpl[strlen(tmpl) - 6], "XXXXXX") != 0) { - errno = EINVAL; - return -1; +// Adapted from Musl C (MIT license) +void __randname(char *tmpl) { + const char charset[] = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + for (int i = 0; i < 6; ++i) { + tmpl[i] = charset[rand() % (sizeof(charset) - 1)]; } - return mkstemp(tmpl); } int mkostemps(char *tmpl, int suffixlen, int flags) { - if (strlen(tmpl) < 6 + suffixlen || strcmp(&tmpl[strlen(tmpl) - 6 - suffixlen], "XXXXXX") != 0) { + size_t l = strlen(tmpl); + if (l < 6 || suffixlen > l - 6 || memcmp(tmpl + l - suffixlen - 6, "XXXXXX", 6) != 0) { errno = EINVAL; return -1; } - char *p = tmpl + strlen(tmpl) - suffixlen - 6; - *p = '\0'; - int fd = mkostemp(tmpl, flags); - *p = 'X'; - return fd; + + int fd, retries = 100; + + do { + __randname(tmpl + l - suffixlen - 6); + fd = open(tmpl, flags | O_RDWR | O_CREAT | O_EXCL, 0600); + if (fd >= 0) { + __tag_new_file(fd); + return fd; + } + } while (--retries && errno == EEXIST); + + memcpy(tmpl + l - suffixlen - 6, "XXXXXX", 6); + return -1; } int mkstemps(char *tmpl, int suffixlen) { - if (strlen(tmpl) < 6 + suffixlen || strcmp(&tmpl[strlen(tmpl) - 6 - suffixlen], "XXXXXX") != 0) { - errno = EINVAL; - return -1; - } - char *p = tmpl + strlen(tmpl) - suffixlen - 6; - *p = '\0'; - int fd = __mkstemp_ascii(tmpl); - *p = 'X'; - return fd; + return mkostemps(tmpl, suffixlen, 0); } +int mkostemp(char *tmpl, int flags) { + return mkostemps(tmpl, 0, flags); +} + + #ifdef __cplusplus } #endif From 5a1b34b26394e0d75d62ca841be0694a2895b29a Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Tue, 23 Jul 2024 22:57:28 -0400 Subject: [PATCH 16/44] Fix readlink and mkostemp - after further testing with zopen tools --- src/zos-bpx.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zos-bpx.cc b/src/zos-bpx.cc index 9057ba0..597cff2 100644 --- a/src/zos-bpx.cc +++ b/src/zos-bpx.cc @@ -93,7 +93,7 @@ char *__realpath_extended(const char __restrict__ *path, char __restrict__ *reso ssize_t __readlink_orig(const char *path, char *buf, size_t bufsiz) asm("@@A00202"); ssize_t __readlink(const char *path, char *buf, size_t bufsiz) { - ssize_t len = __readlink_orig(path, buf, bufsiz - 1); + ssize_t len = __readlink_orig(path, buf, bufsiz); if (len < 0) { return -1; } From b36a272bbbab593ae467eefd716842ec397fd9db Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Mon, 19 Aug 2024 16:40:48 -0400 Subject: [PATCH 17/44] Add code for profiling instrumentation --- src/zos.cc | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) diff --git a/src/zos.cc b/src/zos.cc index 515ea9e..8a36b2a 100644 --- a/src/zos.cc +++ b/src/zos.cc @@ -3148,6 +3148,68 @@ int __sysconf(int name) { } } + +// Instrumentation code - for profiling +// When an application is built with zoslib and with the -finstrument-functions option +// It will generate a json file in the cwd which can be analyzed using perfetto (or https://ui.perfetto.dev/) +extern "C" { + +FILE* __prof_json_file = NULL; +pthread_mutex_t __prof_mutex = PTHREAD_MUTEX_INITIALIZER; +int __prof_isProfiling = 0; + +static inline uint64_t get_timestamp() { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t)ts.tv_nsec; +} + +static void write_json_object(const char* name, const char* phase) { + pthread_mutex_lock(&__prof_mutex); + fprintf(__prof_json_file, " {\"cat\": \"PERF\", \"name\": \"%s\", \"ph\": \"%s\", \"pid\": %d, \"tid\": \"%d\", \"ts\": %lu},\n", name, phase, getpid(), gettid(), get_timestamp()); + pthread_mutex_unlock(&__prof_mutex); +} + +__attribute__((destructor)) +void close_json_file() { + if (__prof_isProfiling) + if (__prof_json_file != NULL) { + fprintf(__prof_json_file, "]\n"); + fclose(__prof_json_file); + } +} + +__attribute__((no_instrument_function)) +void __cyg_profile_func_enter(void* this_fn, void* call_site) { + if (!__prof_isProfiling) { + char profiling_file[PATH_MAX]; + sprintf(profiling_file, "%s-%lu.json", getprogname(), get_timestamp()); + json_file = fopen(profiling_file, "w"); + if (json_file == NULL) { + perror("Error opening file profiling file %s for write.", profiling_file); + exit(1); + } + fprintf(json_file, "[\n"); + __prof_isProfiling = 1; + } + + __stack_info si; + void *cur_dsa = dsa(); + + __iterate_stack_and_get(cur_dsa, &si); + write_json_object(si.entry_name, "B"); +} + +__attribute__((no_instrument_function)) +void __cyg_profile_func_exit(void* this_fn, void* call_site) { + __stack_info si; + void *cur_dsa = dsa(); + + __iterate_stack_and_get(cur_dsa, &si); + write_json_object(si.entry_name, "E"); +} +} + #if defined(ZOSLIB_INITIALIZE) __init_zoslib __zoslib; #endif From 325ed896d34e3d64676e657d3a270be803550e18 Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Mon, 19 Aug 2024 16:43:31 -0400 Subject: [PATCH 18/44] Fix typos --- src/zos.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/zos.cc b/src/zos.cc index 8a36b2a..43b393f 100644 --- a/src/zos.cc +++ b/src/zos.cc @@ -3184,12 +3184,12 @@ void __cyg_profile_func_enter(void* this_fn, void* call_site) { if (!__prof_isProfiling) { char profiling_file[PATH_MAX]; sprintf(profiling_file, "%s-%lu.json", getprogname(), get_timestamp()); - json_file = fopen(profiling_file, "w"); - if (json_file == NULL) { - perror("Error opening file profiling file %s for write.", profiling_file); + __prof_json_file = fopen(profiling_file, "w"); + if (__prof_json_file == NULL) { + fprintf(stderr, "Error opening file profiling file %s for write, errno: %d", profiling_file, errno); exit(1); } - fprintf(json_file, "[\n"); + fprintf(__prof_json_file, "[\n"); __prof_isProfiling = 1; } From 6f5656388f7777b0316a4c37df3e92bfac87352a Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Tue, 20 Aug 2024 12:33:13 -0400 Subject: [PATCH 19/44] Fix typos --- src/CMakeLists.txt | 1 + src/zos-instrumentation.cc | 130 +++++++++++++++++++++++++++++++++++++ src/zos.cc | 61 ----------------- 3 files changed, 131 insertions(+), 61 deletions(-) create mode 100644 src/zos-instrumentation.cc diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7890133..f9c4442 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -20,6 +20,7 @@ set(libsrc zos.cc zos-mount.c zos-mkdtemp.c + zos-instrumentation.cc ) set(zoslib-help zoslib-help.cc) diff --git a/src/zos-instrumentation.cc b/src/zos-instrumentation.cc new file mode 100644 index 0000000..53d5639 --- /dev/null +++ b/src/zos-instrumentation.cc @@ -0,0 +1,130 @@ +/////////////////////////////////////////////////////////////////////////////// +// Licensed Materials - Property of IBM +// ZOSLIB +// (C) Copyright IBM Corp. 2020. All Rights Reserved. +// US Government Users Restricted Rights - Use, duplication +// or disclosure restricted by GSA ADP Schedule Contract with IBM Corp. +/////////////////////////////////////////////////////////////////////////////// + +#define _AE_BIMODAL 1 +#include "zos.h" +#include "edcwccwi.h" + +#include <_Ccsid.h> +#include <_Nascii.h> +#include <__le_api.h> +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Instrumentation code - for profiling +// When an application is built with zoslib and with the -finstrument-functions option +// It will generate a json file in the cwd which can be analyzed using perfetto (or https://ui.perfetto.dev/) +#ifndef dsa +#define dsa() ((unsigned long *)_gdsa()) +#endif + +namespace { +FILE* __prof_json_file = NULL; +pthread_mutex_t __prof_mutex = PTHREAD_MUTEX_INITIALIZER; +char __profiling_file[PATH_MAX] = {0}; +int __prof_isProfiling = 0; +} + +extern "C" { + +__attribute__((no_instrument_function)) +static inline uint64_t get_timens() { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t)ts.tv_nsec; +} + +__attribute__((no_instrument_function)) +static void check_env_vars() { + char* disable_prof = getenv("ZOSLIB_PROF_DISABLE"); + if (disable_prof != NULL) { + __prof_isProfiling = 0; + return; + } + + char* prof_path = getenv("ZOSLIB_PROF_PATH"); + if (prof_path != NULL) { + sprintf(__profiling_file, "%s", prof_path); + } else { + sprintf(__profiling_file, "%s-%lu.json", getprogname(), get_timens()); + } +} + +// Using the chrome trace event format: https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview +__attribute__((no_instrument_function)) +static void write_json_object(const char* name, const char* phase) { + pthread_mutex_lock(&__prof_mutex); + fprintf(__prof_json_file, " {\"cat\": \"PERF\", \"name\": \"%s\", \"ph\": \"%s\", \"pid\": %d, \"tid\": \"%d\", \"ts\": %lu},\n", name, phase, getpid(), gettid(), get_timens()); + pthread_mutex_unlock(&__prof_mutex); +} + +__attribute__((no_instrument_function)) +__attribute__((destructor)) +void close_json_file() { + if (__prof_isProfiling) + if (__prof_json_file != NULL) { + fprintf(__prof_json_file, "]\n"); + fclose(__prof_json_file); + } +} + +__attribute__((no_instrument_function)) +void __cyg_profile_func_enter(void* this_fn, void* call_site) { + if (!__prof_isProfiling) { + __prof_isProfiling = 1; + check_env_vars(); + if (!__prof_isProfiling) + return; + + __prof_json_file = fopen(__profiling_file, "w"); + if (__prof_json_file == NULL) { + fprintf(stderr, "Error opening file profiling file %s for write, errno: %d", __profiling_file, errno); + exit(1); + } + fprintf(__prof_json_file, "[\n"); + } + + __stack_info si; + void *cur_dsa = dsa(); + + __iterate_stack_and_get(cur_dsa, &si); + write_json_object(si.entry_name, "B"); +} + +__attribute__((no_instrument_function)) +void __cyg_profile_func_exit(void* this_fn, void* call_site) { + __stack_info si; + void *cur_dsa = dsa(); + + __iterate_stack_and_get(cur_dsa, &si); + write_json_object(si.entry_name, "E"); +} + +} diff --git a/src/zos.cc b/src/zos.cc index 43b393f..f7503ef 100644 --- a/src/zos.cc +++ b/src/zos.cc @@ -3149,67 +3149,6 @@ int __sysconf(int name) { } -// Instrumentation code - for profiling -// When an application is built with zoslib and with the -finstrument-functions option -// It will generate a json file in the cwd which can be analyzed using perfetto (or https://ui.perfetto.dev/) -extern "C" { - -FILE* __prof_json_file = NULL; -pthread_mutex_t __prof_mutex = PTHREAD_MUTEX_INITIALIZER; -int __prof_isProfiling = 0; - -static inline uint64_t get_timestamp() { - struct timespec ts; - clock_gettime(CLOCK_MONOTONIC, &ts); - return (uint64_t)ts.tv_nsec; -} - -static void write_json_object(const char* name, const char* phase) { - pthread_mutex_lock(&__prof_mutex); - fprintf(__prof_json_file, " {\"cat\": \"PERF\", \"name\": \"%s\", \"ph\": \"%s\", \"pid\": %d, \"tid\": \"%d\", \"ts\": %lu},\n", name, phase, getpid(), gettid(), get_timestamp()); - pthread_mutex_unlock(&__prof_mutex); -} - -__attribute__((destructor)) -void close_json_file() { - if (__prof_isProfiling) - if (__prof_json_file != NULL) { - fprintf(__prof_json_file, "]\n"); - fclose(__prof_json_file); - } -} - -__attribute__((no_instrument_function)) -void __cyg_profile_func_enter(void* this_fn, void* call_site) { - if (!__prof_isProfiling) { - char profiling_file[PATH_MAX]; - sprintf(profiling_file, "%s-%lu.json", getprogname(), get_timestamp()); - __prof_json_file = fopen(profiling_file, "w"); - if (__prof_json_file == NULL) { - fprintf(stderr, "Error opening file profiling file %s for write, errno: %d", profiling_file, errno); - exit(1); - } - fprintf(__prof_json_file, "[\n"); - __prof_isProfiling = 1; - } - - __stack_info si; - void *cur_dsa = dsa(); - - __iterate_stack_and_get(cur_dsa, &si); - write_json_object(si.entry_name, "B"); -} - -__attribute__((no_instrument_function)) -void __cyg_profile_func_exit(void* this_fn, void* call_site) { - __stack_info si; - void *cur_dsa = dsa(); - - __iterate_stack_and_get(cur_dsa, &si); - write_json_object(si.entry_name, "E"); -} -} - #if defined(ZOSLIB_INITIALIZE) __init_zoslib __zoslib; #endif From 63e621efe468720998d1605262ca2fe99e378a11 Mon Sep 17 00:00:00 2001 From: kishkulk Date: Tue, 20 Aug 2024 22:03:48 +0530 Subject: [PATCH 20/44] replace bzero with calloc (#68) --- src/zos-tls.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/zos-tls.cc b/src/zos-tls.cc index e2412f2..e518e59 100644 --- a/src/zos-tls.cc +++ b/src/zos-tls.cc @@ -94,9 +94,8 @@ void *__tlsPtrFromAnchor(struct __tlsanchor *anchor, const void *initvalue) { } void * __tlsValue(tls_t *a) { void *val = NULL; - char * initvalue = (char *)malloc(sizeof(char)*(a->sz)); + char *initvalue = (char *)calloc((sizeof(char)*(a->sz)), sizeof(char)); assert(initvalue != NULL); - bzero(initvalue,a->sz); val = __tlsPtrAlloc(a->sz, &(a->key), &(a->once),(void *)initvalue); free(initvalue); return val; From c09833536bdac2545b8f037b54414e38c27e3bd4 Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Tue, 20 Aug 2024 14:26:28 -0400 Subject: [PATCH 21/44] revise json --- src/zos-instrumentation.cc | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/zos-instrumentation.cc b/src/zos-instrumentation.cc index 53d5639..b0f1ff2 100644 --- a/src/zos-instrumentation.cc +++ b/src/zos-instrumentation.cc @@ -50,6 +50,7 @@ FILE* __prof_json_file = NULL; pthread_mutex_t __prof_mutex = PTHREAD_MUTEX_INITIALIZER; char __profiling_file[PATH_MAX] = {0}; int __prof_isProfiling = 0; +int __prof_isDisabled = 0; } extern "C" { @@ -65,7 +66,7 @@ __attribute__((no_instrument_function)) static void check_env_vars() { char* disable_prof = getenv("ZOSLIB_PROF_DISABLE"); if (disable_prof != NULL) { - __prof_isProfiling = 0; + __prof_isDisabled = 1; return; } @@ -90,17 +91,21 @@ __attribute__((destructor)) void close_json_file() { if (__prof_isProfiling) if (__prof_json_file != NULL) { - fprintf(__prof_json_file, "]\n"); + fprintf(__prof_json_file, "{} ]\n"); fclose(__prof_json_file); } } __attribute__((no_instrument_function)) void __cyg_profile_func_enter(void* this_fn, void* call_site) { + if (__prof_isDisabled) + return; + + // On first call, check envars if (!__prof_isProfiling) { __prof_isProfiling = 1; check_env_vars(); - if (!__prof_isProfiling) + if (__prof_isDisabled) return; __prof_json_file = fopen(__profiling_file, "w"); @@ -120,6 +125,9 @@ void __cyg_profile_func_enter(void* this_fn, void* call_site) { __attribute__((no_instrument_function)) void __cyg_profile_func_exit(void* this_fn, void* call_site) { + if (__prof_isDisabled) + return; + __stack_info si; void *cur_dsa = dsa(); From b548649b337055bfaffced82e1c63c7ab6c2b8a7 Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Tue, 20 Aug 2024 14:42:29 -0400 Subject: [PATCH 22/44] Fix tid in instrumentation json --- src/zos-instrumentation.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zos-instrumentation.cc b/src/zos-instrumentation.cc index b0f1ff2..98531b0 100644 --- a/src/zos-instrumentation.cc +++ b/src/zos-instrumentation.cc @@ -82,7 +82,7 @@ static void check_env_vars() { __attribute__((no_instrument_function)) static void write_json_object(const char* name, const char* phase) { pthread_mutex_lock(&__prof_mutex); - fprintf(__prof_json_file, " {\"cat\": \"PERF\", \"name\": \"%s\", \"ph\": \"%s\", \"pid\": %d, \"tid\": \"%d\", \"ts\": %lu},\n", name, phase, getpid(), gettid(), get_timens()); + fprintf(__prof_json_file, " {\"cat\": \"PERF\", \"name\": \"%s\", \"ph\": \"%s\", \"pid\": %d, \"tid\": %d, \"ts\": %lu},\n", name, phase, getpid(), gettid(), get_timens()); pthread_mutex_unlock(&__prof_mutex); } From b1ae73f9e9242367d3c0e6a70de4029e9aca0cff Mon Sep 17 00:00:00 2001 From: Igor Todorovski <39890068+IgorTodorovskiIBM@users.noreply.github.com> Date: Tue, 20 Aug 2024 15:40:52 -0400 Subject: [PATCH 23/44] Update zos-instrumentation.cc --- src/zos-instrumentation.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/zos-instrumentation.cc b/src/zos-instrumentation.cc index 98531b0..cfe973d 100644 --- a/src/zos-instrumentation.cc +++ b/src/zos-instrumentation.cc @@ -39,8 +39,9 @@ #include // Instrumentation code - for profiling -// When an application is built with zoslib and with the -finstrument-functions option -// It will generate a json file in the cwd which can be analyzed using perfetto (or https://ui.perfetto.dev/) +// when an application is built with zoslib, along with the -finstrument-functions option +// it will generate a json file in the cwd (set ZOSLIB_PROF_PATH to override), which can be +// analyzed using chrome tracing or perfetto (https://ui.perfetto.dev/) #ifndef dsa #define dsa() ((unsigned long *)_gdsa()) #endif From 51f5dd02239367c88df49a972948d7237ceee028 Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Wed, 21 Aug 2024 15:21:29 -0400 Subject: [PATCH 24/44] Resolve clang related build issues --- include/zos-base.h | 10 ++++++++++ src/zos-instrumentation.cc | 9 +++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/include/zos-base.h b/include/zos-base.h index adf8b6d..3667b48 100644 --- a/include/zos-base.h +++ b/include/zos-base.h @@ -39,6 +39,16 @@ #define __ZL_NR(attr,reg) attr "{" #reg "}" #endif +#if defined(__clang__) && !defined(__ibmxl__) +#define _gdsa __builtin_s390_gdsa +#else +extern "builtin" void *_gdsa(); +#endif + +#ifndef dsa +#define dsa() ((unsigned long *)_gdsa()) +#endif + #include "zos-macros.h" #include "zos-bpx.h" #include "zos-char-util.h" diff --git a/src/zos-instrumentation.cc b/src/zos-instrumentation.cc index cfe973d..57d52ae 100644 --- a/src/zos-instrumentation.cc +++ b/src/zos-instrumentation.cc @@ -37,15 +37,12 @@ #include #include #include +#include "zos-macros.h" // Instrumentation code - for profiling // when an application is built with zoslib, along with the -finstrument-functions option // it will generate a json file in the cwd (set ZOSLIB_PROF_PATH to override), which can be // analyzed using chrome tracing or perfetto (https://ui.perfetto.dev/) -#ifndef dsa -#define dsa() ((unsigned long *)_gdsa()) -#endif - namespace { FILE* __prof_json_file = NULL; pthread_mutex_t __prof_mutex = PTHREAD_MUTEX_INITIALIZER; @@ -98,7 +95,7 @@ void close_json_file() { } __attribute__((no_instrument_function)) -void __cyg_profile_func_enter(void* this_fn, void* call_site) { +__Z_EXPORT void __cyg_profile_func_enter(void* this_fn, void* call_site) { if (__prof_isDisabled) return; @@ -125,7 +122,7 @@ void __cyg_profile_func_enter(void* this_fn, void* call_site) { } __attribute__((no_instrument_function)) -void __cyg_profile_func_exit(void* this_fn, void* call_site) { +__Z_EXPORT void __cyg_profile_func_exit(void* this_fn, void* call_site) { if (__prof_isDisabled) return; From d847fc3aca1223e4fac01b9da179acc071005b86 Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Wed, 21 Aug 2024 15:21:56 -0400 Subject: [PATCH 25/44] Resolve clang related build issues --- src/zos.cc | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/zos.cc b/src/zos.cc index f7503ef..302abc3 100644 --- a/src/zos.cc +++ b/src/zos.cc @@ -53,17 +53,6 @@ #include #include -#if defined(__clang__) && !defined(__ibmxl__) -#define _gdsa __builtin_s390_gdsa -#else -extern "builtin" void *_gdsa(); -#endif - - -#ifndef dsa -#define dsa() ((unsigned long *)_gdsa()) -#endif - #if defined(ZOSLIB_ENABLE_V2R5_FEATURES) int (*epoll_create)(int) = 0; int (*epoll_create1)(int) = 0; From b87bc466e1dabc523886bd1164c885367caf6747 Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Wed, 21 Aug 2024 16:24:44 -0400 Subject: [PATCH 26/44] Change dsa to __dsa and use traceEvent objects to make ns the default timing --- include/zos-base.h | 4 ++-- src/zos-instrumentation.cc | 8 ++++---- src/zos.cc | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/include/zos-base.h b/include/zos-base.h index 3667b48..8030d52 100644 --- a/include/zos-base.h +++ b/include/zos-base.h @@ -45,8 +45,8 @@ extern "builtin" void *_gdsa(); #endif -#ifndef dsa -#define dsa() ((unsigned long *)_gdsa()) +#ifndef __dsa +#define __dsa() ((unsigned long *)_gdsa()) #endif #include "zos-macros.h" diff --git a/src/zos-instrumentation.cc b/src/zos-instrumentation.cc index 57d52ae..c19e113 100644 --- a/src/zos-instrumentation.cc +++ b/src/zos-instrumentation.cc @@ -89,7 +89,7 @@ __attribute__((destructor)) void close_json_file() { if (__prof_isProfiling) if (__prof_json_file != NULL) { - fprintf(__prof_json_file, "{} ]\n"); + fprintf(__prof_json_file, "null ]}\n"); fclose(__prof_json_file); } } @@ -111,11 +111,11 @@ __Z_EXPORT void __cyg_profile_func_enter(void* this_fn, void* call_site) { fprintf(stderr, "Error opening file profiling file %s for write, errno: %d", __profiling_file, errno); exit(1); } - fprintf(__prof_json_file, "[\n"); + fprintf(__prof_json_file, "{ \"traceEvents\": [\n"); } __stack_info si; - void *cur_dsa = dsa(); + void *cur_dsa = __dsa(); __iterate_stack_and_get(cur_dsa, &si); write_json_object(si.entry_name, "B"); @@ -127,7 +127,7 @@ __Z_EXPORT void __cyg_profile_func_exit(void* this_fn, void* call_site) { return; __stack_info si; - void *cur_dsa = dsa(); + void *cur_dsa = __dsa(); __iterate_stack_and_get(cur_dsa, &si); write_json_object(si.entry_name, "E"); diff --git a/src/zos.cc b/src/zos.cc index 302abc3..9c98a7b 100644 --- a/src/zos.cc +++ b/src/zos.cc @@ -2438,7 +2438,7 @@ int *__get_stack_start() { return __main_thread_stack_top_address; } __stack_info si; - void *cur_dsa = dsa(); + void *cur_dsa = __dsa(); while (__iterate_stack_and_get(cur_dsa, &si) != 0) { cur_dsa = si.prev_dsa; From 773810597c07e9edcf1faf0bdc99e171ab3a229f Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Wed, 21 Aug 2024 16:51:10 -0400 Subject: [PATCH 27/44] Gzip the json file once completed --- src/zos-instrumentation.cc | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/zos-instrumentation.cc b/src/zos-instrumentation.cc index c19e113..acbd3b9 100644 --- a/src/zos-instrumentation.cc +++ b/src/zos-instrumentation.cc @@ -91,6 +91,17 @@ void close_json_file() { if (__prof_json_file != NULL) { fprintf(__prof_json_file, "null ]}\n"); fclose(__prof_json_file); + + FILE *fp = popen("command -v gzip", "r"); + if (fp != NULL) { + char path[PATH_MAX]; + if (fgets(path, sizeof(path), fp) != NULL) { + char command[PATH_MAX]; + snprintf(command, sizeof(command), "gzip -f %s 2>/dev/null", __profiling_file); + system(command); // Should we handle this if it fails? + } + pclose(fp); + } } } From 0756b23747fda57aa1bfaadea533d7955ea10074 Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Wed, 28 Aug 2024 08:19:53 -0400 Subject: [PATCH 28/44] Only activate __get_stack_start on non-quick startup --- src/zos.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/zos.cc b/src/zos.cc index 9c98a7b..066f5ca 100644 --- a/src/zos.cc +++ b/src/zos.cc @@ -2605,7 +2605,9 @@ int __zinit::initialize(const zoslib_config_t &aconfig) { __ae_autoconvert_state(_CVTSTATE_ON); } +#ifndef ZOSLIB_QUICK_STARTUP __main_thread_stack_top_address = __get_stack_start(); +#endif if (setEnvarHelpMap() != 0) return -1; @@ -2653,8 +2655,9 @@ int __zinit::initialize(const zoslib_config_t &aconfig) { __set_autocvt_on_fd_stream(STDERR_FILENO, 1047, 1, true); setProcessEnvars(); - +#ifndef ZOSLIB_QUICK_STARTUP populateLEFunctionPointers(); +#endif _th = std::get_terminate(); std::set_terminate(abort); From 5d413b75ccd8b6a9fbfc4876cfa06af007dddd34 Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Wed, 28 Aug 2024 08:24:17 -0400 Subject: [PATCH 29/44] Propagate ZOSLIB_QUICK_STARTUP in cmake --- CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9e1d824..9e1f044 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -53,6 +53,11 @@ if(ZOSLIB_GENERIC) ZOSLIB_GENERIC) endif() +if(ZOSLIB_QUICK_STARTUP) + list(APPEND zoslib_defines + ZOSLIB_QUICK_STARTUP) +endif() + if(DEFINED ENV{BUILD_VERSION}) list(APPEND zoslib_defines BUILD_VERSION="$ENV{BUILD_VERSION}") From 659b1d5e0e168c3e3083db1f57144c9d0537733e Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Thu, 29 Aug 2024 13:24:25 -0400 Subject: [PATCH 30/44] Change the arch level to arch10 for clang (and arch14 only for zos-getentropy.cc) --- CMakeLists.txt | 2 +- src/CMakeLists.txt | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9e1f044..ce29af6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -86,7 +86,7 @@ else() list(APPEND zoslib_cflags -fgnu-keywords -m64 - -march=arch14 + -march=arch10 -mzos-target=zosv2r4 -fno-short-enums -fzos-le-char-mode=ascii) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f9c4442..e66e6a4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -36,6 +36,10 @@ COMMAND /bin/as -mgoff -o ${CELQUOPT_OBJECT} ${CELQUOPT_SOURCE} VERBATIM ) +if(${CMAKE_C_COMPILER} MATCHES clang) + set_source_files_properties(zos-getentropy.cc PROPERTIES COMPILE_FLAGS "-march=arch14") +endif() + add_library(libzoslib OBJECT ${libsrc}) add_library(zoslib SHARED $) From d2aed54351dc80b7e2a89008adc3e77dc6b8111b Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Thu, 19 Sep 2024 11:20:55 -0400 Subject: [PATCH 31/44] Add tracing for allocation --- src/zos-instrumentation.cc | 143 +++++++++++++++++++++++++++++-------- 1 file changed, 113 insertions(+), 30 deletions(-) diff --git a/src/zos-instrumentation.cc b/src/zos-instrumentation.cc index acbd3b9..9e7affb 100644 --- a/src/zos-instrumentation.cc +++ b/src/zos-instrumentation.cc @@ -7,9 +7,10 @@ /////////////////////////////////////////////////////////////////////////////// #define _AE_BIMODAL 1 -#include "zos.h" #include "edcwccwi.h" +#include "zos.h" +#include "zos-macros.h" #include <_Ccsid.h> #include <_Nascii.h> #include <__le_api.h> @@ -21,6 +22,8 @@ #include #include #include +#include +#include #include #include #include @@ -37,38 +40,40 @@ #include #include #include -#include "zos-macros.h" // Instrumentation code - for profiling -// when an application is built with zoslib, along with the -finstrument-functions option -// it will generate a json file in the cwd (set ZOSLIB_PROF_PATH to override), which can be -// analyzed using chrome tracing or perfetto (https://ui.perfetto.dev/) +// when an application is built with zoslib, along with the +// -finstrument-functions option it will generate a json file in the cwd (set +// ZOSLIB_PROF_PATH to override), which can be analyzed using chrome tracing or +// perfetto (https://ui.perfetto.dev/) namespace { -FILE* __prof_json_file = NULL; +FILE *__prof_json_file = NULL; pthread_mutex_t __prof_mutex = PTHREAD_MUTEX_INITIALIZER; +pthread_mutex_t __prof_mutex2 = PTHREAD_MUTEX_INITIALIZER; +pthread_mutex_t __prof_mutex3 = PTHREAD_MUTEX_INITIALIZER; char __profiling_file[PATH_MAX] = {0}; int __prof_isProfiling = 0; int __prof_isDisabled = 0; -} +uint64_t __total_allocated_memory = 0; +std::map allocation_map; +} // namespace extern "C" { -__attribute__((no_instrument_function)) -static inline uint64_t get_timens() { +__attribute__((no_instrument_function)) static inline uint64_t get_timens() { struct timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); return (uint64_t)ts.tv_nsec; } -__attribute__((no_instrument_function)) -static void check_env_vars() { - char* disable_prof = getenv("ZOSLIB_PROF_DISABLE"); +__attribute__((no_instrument_function)) static void check_env_vars() { + char *disable_prof = getenv("ZOSLIB_PROF_DISABLE"); if (disable_prof != NULL) { __prof_isDisabled = 1; return; } - char* prof_path = getenv("ZOSLIB_PROF_PATH"); + char *prof_path = getenv("ZOSLIB_PROF_PATH"); if (prof_path != NULL) { sprintf(__profiling_file, "%s", prof_path); } else { @@ -76,16 +81,30 @@ static void check_env_vars() { } } -// Using the chrome trace event format: https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview +// Using the chrome trace event format: +// https://docs.google.com/document/d/1CvAClvFfyA5R-PhYUmn5OOQtYMH4h6I0nSsKchNAySU/preview __attribute__((no_instrument_function)) -static void write_json_object(const char* name, const char* phase) { +static void __write_json_object(const char *name, const char *phase, + const char *cat = "PERF", int64_t delta_memory = 0) { pthread_mutex_lock(&__prof_mutex); - fprintf(__prof_json_file, " {\"cat\": \"PERF\", \"name\": \"%s\", \"ph\": \"%s\", \"pid\": %d, \"tid\": %d, \"ts\": %lu},\n", name, phase, getpid(), gettid(), get_timens()); + if (delta_memory != 0) { + __total_allocated_memory += delta_memory; + fprintf(__prof_json_file, + " {\"cat\": \"%s\", \"name\": \"%s\", \"ph\": \"%s\", \"pid\": " + "%d, \"tid\": %d, \"ts\": %lu, \"args\": {\"delta_memory\": %lld, " + "\"total_memory\": %zu}},\n", + cat, name, phase, getpid(), gettid(), get_timens(), delta_memory, + __total_allocated_memory); + } else { + fprintf(__prof_json_file, + " {\"cat\": \"%s\", \"name\": \"%s\", \"ph\": \"%s\", \"pid\": " + "%d, \"tid\": %d, \"ts\": %lu},\n", + cat, name, phase, getpid(), gettid(), get_timens()); + } pthread_mutex_unlock(&__prof_mutex); } -__attribute__((no_instrument_function)) -__attribute__((destructor)) +__attribute__((no_instrument_function)) __attribute__((destructor)) void close_json_file() { if (__prof_isProfiling) if (__prof_json_file != NULL) { @@ -97,7 +116,8 @@ void close_json_file() { char path[PATH_MAX]; if (fgets(path, sizeof(path), fp) != NULL) { char command[PATH_MAX]; - snprintf(command, sizeof(command), "gzip -f %s 2>/dev/null", __profiling_file); + snprintf(command, sizeof(command), "gzip -f %s 2>/dev/null", + __profiling_file); system(command); // Should we handle this if it fails? } pclose(fp); @@ -105,8 +125,8 @@ void close_json_file() { } } -__attribute__((no_instrument_function)) -__Z_EXPORT void __cyg_profile_func_enter(void* this_fn, void* call_site) { +__attribute__((no_instrument_function)) __Z_EXPORT void +__cyg_profile_func_enter(void *this_fn, void *call_site) { if (__prof_isDisabled) return; @@ -116,32 +136,95 @@ __Z_EXPORT void __cyg_profile_func_enter(void* this_fn, void* call_site) { check_env_vars(); if (__prof_isDisabled) return; - + __prof_json_file = fopen(__profiling_file, "w"); if (__prof_json_file == NULL) { - fprintf(stderr, "Error opening file profiling file %s for write, errno: %d", __profiling_file, errno); + fprintf(stderr, + "Error opening file profiling file %s for write, errno: %d", + __profiling_file, errno); exit(1); } fprintf(__prof_json_file, "{ \"traceEvents\": [\n"); } + int prevError = errno; __stack_info si; void *cur_dsa = __dsa(); - __iterate_stack_and_get(cur_dsa, &si); - write_json_object(si.entry_name, "B"); + if (__iterate_stack_and_get(cur_dsa, &si) != 0) + __write_json_object(si.entry_name, "B"); + errno = prevError; } -__attribute__((no_instrument_function)) -__Z_EXPORT void __cyg_profile_func_exit(void* this_fn, void* call_site) { +__attribute__((no_instrument_function)) __Z_EXPORT void +__cyg_profile_func_exit(void *this_fn, void *call_site) { if (__prof_isDisabled) return; + int prevError = errno; __stack_info si; void *cur_dsa = __dsa(); - __iterate_stack_and_get(cur_dsa, &si); - write_json_object(si.entry_name, "E"); + if (__iterate_stack_and_get(cur_dsa, &si) != 0) + __write_json_object(si.entry_name, "E"); + errno = prevError; +} + +void *__real_malloc(size_t size) asm("malloc"); +void __real_free(void *ptr) asm("free"); + +// Custom malloc with profiling +__attribute__((no_instrument_function)) void *__zoslib_malloc(size_t size) { + if (!__prof_isProfiling || __prof_isDisabled) { + return __real_malloc(size); + } + + void *ptr = __real_malloc(size); + int old_errno = errno; + if (ptr != NULL) { + pthread_mutex_lock(&__prof_mutex2); + allocation_map[ptr] = size; + pthread_mutex_unlock(&__prof_mutex2); + __write_json_object("Memory", "C", "malloc", size); + } + + errno = old_errno; + return ptr; +} + +__attribute__((no_instrument_function)) +size_t malloc_usable_size(void *ptr) { + if (ptr == NULL) { + return 0; + } + + auto it = allocation_map.find(ptr); + if (it != allocation_map.end()) { + return it->second; + } + + return 0; +} + +__attribute__((no_instrument_function)) void __zoslib_free(void *ptr) { + if (!__prof_isProfiling || __prof_isDisabled) { + __real_free(ptr); + return; + } + + int prevError = errno; + if (ptr != NULL) { + int64_t size = malloc_usable_size(ptr); + + pthread_mutex_lock(&__prof_mutex2); + allocation_map.erase(ptr); + pthread_mutex_unlock(&__prof_mutex2); + + __write_json_object("Memory", "C", "free", -size); + } + + errno = prevError; + __real_free(ptr); } -} +} // extern "C" From 3a4cb7ff0865884481ac60e65c1173a9aafd84bf Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Thu, 19 Sep 2024 11:21:55 -0400 Subject: [PATCH 32/44] Add malloc/free overrides --- include/stdlib.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/include/stdlib.h b/include/stdlib.h index 43d2144..7df9170 100644 --- a/include/stdlib.h +++ b/include/stdlib.h @@ -34,6 +34,10 @@ __Z_EXPORT int __mkstemp_ascii(char*); #define realpath __realpath_replaced #undef mkstemp #define mkstemp __mkstemp_replaced +#undef malloc +#define malloc __malloc_replaced +#undef free +#define free __free_replaced #endif #if defined(ZOSLIB_OVERRIDE_CLIB_GETENV) && defined(__NATIVE_ASCII_F) @@ -55,6 +59,8 @@ extern "C" { */ #undef realpath __Z_EXPORT char *realpath(const char * __restrict__, char * __restrict__) __asm("__realpath_extended"); +__Z_EXPORT void* malloc(size_t size) __asm("__zoslib_malloc"); +__Z_EXPORT void free(void* ptr) __asm("__zoslib_free"); #ifdef __NATIVE_ASCII_F /** @@ -127,6 +133,8 @@ __Z_EXPORT char *mkdtemp(char *); #endif #if defined(__cplusplus) +__Z_EXPORT void __zoslib_free(void* ptr); +__Z_EXPORT void* __zoslib_malloc(size_t size); } #endif From 9a69e8e095fa6665e876e1847936d3cf4919a8e3 Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Sun, 22 Sep 2024 21:49:22 -0400 Subject: [PATCH 33/44] Add export to __readlink --- src/zos-bpx.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zos-bpx.cc b/src/zos-bpx.cc index 597cff2..bbbdfc2 100644 --- a/src/zos-bpx.cc +++ b/src/zos-bpx.cc @@ -92,7 +92,7 @@ char *__realpath_extended(const char __restrict__ *path, char __restrict__ *reso ssize_t __readlink_orig(const char *path, char *buf, size_t bufsiz) asm("@@A00202"); -ssize_t __readlink(const char *path, char *buf, size_t bufsiz) { +__Z_EXPORT ssize_t __readlink(const char *path, char *buf, size_t bufsiz) { ssize_t len = __readlink_orig(path, buf, bufsiz); if (len < 0) { return -1; From 4e8ca924f574d0d01d2ca5b375e63d7b0b9d6f5f Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Mon, 23 Sep 2024 21:52:23 -0400 Subject: [PATCH 34/44] Update zos-v2r5-symbolfixes.h --- include/zos-v2r5-symbolfixes.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/zos-v2r5-symbolfixes.h b/include/zos-v2r5-symbolfixes.h index 35f8a44..89ff900 100644 --- a/include/zos-v2r5-symbolfixes.h +++ b/include/zos-v2r5-symbolfixes.h @@ -45,6 +45,7 @@ #pragma redefine_extname dup3 dup3_undefined #pragma redefine_extname shm_open shm_open_undefined #pragma redefine_extname utimensat utimensat_undefined +#pragma redefine_extname fdopendir fdopendir_undefined #endif #endif // ZOS_V2R5_SYMBOLFIXES_H From 2dc1f9bd4bf07bd23e48de137486be9e886efea7 Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Tue, 24 Sep 2024 10:57:37 -0400 Subject: [PATCH 35/44] Resolve issue when malloc/free are used as function pointers --- include/stdlib.h | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/include/stdlib.h b/include/stdlib.h index 7df9170..272c437 100644 --- a/include/stdlib.h +++ b/include/stdlib.h @@ -50,6 +50,11 @@ __Z_EXPORT int __mkstemp_ascii(char*); #if defined(ZOSLIB_OVERRIDE_CLIB) || defined(ZOSLIB_OVERRIDE_CLIB_STDLIB) +#undef realpath +#undef mkstemp +#undef malloc +#undef free + #if defined(__cplusplus) extern "C" { #endif From ac3adb83a62290b100820b06b2bae66f99d2f4a0 Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Wed, 9 Oct 2024 14:24:36 -0400 Subject: [PATCH 36/44] Add pthread_condattr_setclock to v2r5 symbol list --- include/zos-v2r5-symbolfixes.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/zos-v2r5-symbolfixes.h b/include/zos-v2r5-symbolfixes.h index 89ff900..4c7bc8e 100644 --- a/include/zos-v2r5-symbolfixes.h +++ b/include/zos-v2r5-symbolfixes.h @@ -46,6 +46,7 @@ #pragma redefine_extname shm_open shm_open_undefined #pragma redefine_extname utimensat utimensat_undefined #pragma redefine_extname fdopendir fdopendir_undefined +#pragma redefine_extname pthread_condattr_setclock pthread_condattr_setclock_undefined #endif #endif // ZOS_V2R5_SYMBOLFIXES_H From 6c843abf79b59fcbc8098eb5fce42ce0a21dcdb6 Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Wed, 9 Oct 2024 14:42:27 -0400 Subject: [PATCH 37/44] Add asprintf and vasprintf to v2r5 symbol list --- include/zos-v2r5-symbolfixes.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/include/zos-v2r5-symbolfixes.h b/include/zos-v2r5-symbolfixes.h index 4c7bc8e..a5faf17 100644 --- a/include/zos-v2r5-symbolfixes.h +++ b/include/zos-v2r5-symbolfixes.h @@ -47,6 +47,8 @@ #pragma redefine_extname utimensat utimensat_undefined #pragma redefine_extname fdopendir fdopendir_undefined #pragma redefine_extname pthread_condattr_setclock pthread_condattr_setclock_undefined +#pragma redefine_extname asprintf asprintf_undefined +#pragma redefine_extname vasprintf vasprintf_undefined #endif #endif // ZOS_V2R5_SYMBOLFIXES_H From ab0e6fa348f736e4bef74dec076f873cc99151d6 Mon Sep 17 00:00:00 2001 From: Igor Todorovski <39890068+IgorTodorovskiIBM@users.noreply.github.com> Date: Tue, 12 Nov 2024 22:32:32 -0500 Subject: [PATCH 38/44] Update unistd.h - use __asm --- include/unistd.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/unistd.h b/include/unistd.h index 4e90d69..ff7162f 100644 --- a/include/unistd.h +++ b/include/unistd.h @@ -49,7 +49,7 @@ __Z_EXPORT int pipe(int [2]) __asm("__pipe_ascii"); __Z_EXPORT int close(int) __asm("__close"); __Z_EXPORT int close(int) __asm("__close"); __Z_EXPORT int sysconf(int name) __asm("__sysconf"); -__Z_EXPORT ssize_t readlink(const char *path, char *buf, size_t bufsiz) asm("__readlink"); +__Z_EXPORT ssize_t readlink(const char *path, char *buf, size_t bufsiz) __asm("__readlink"); #if defined(__cplusplus) From cdd2f75278a9867aeceec1a0f9aac8433f7c9f04 Mon Sep 17 00:00:00 2001 From: Sean Perry Date: Thu, 14 Nov 2024 22:57:15 -0500 Subject: [PATCH 39/44] Fix up product name & remove the c++ copied dir (#70) * Fix up product name & remove the c++ copied dir * more places to undo * Add Openxl v2.1 too --- CMakeLists.txt | 6 +----- README.md | 3 ++- include/c++/v1/__threading_support | 19 ------------------- zoslib.gyp | 3 +-- 4 files changed, 4 insertions(+), 27 deletions(-) delete mode 100644 include/c++/v1/__threading_support diff --git a/CMakeLists.txt b/CMakeLists.txt index ce29af6..92b31bb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,11 +9,7 @@ cmake_minimum_required(VERSION 3.24) project(libzoslib CXX C ASM) -if(${CMAKE_C_COMPILER} MATCHES xlclang) - include_directories(BEFORE include) -else() - include_directories(BEFORE include include/c++/v1) -endif() +include_directories(BEFORE include) set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) diff --git a/README.md b/README.md index 513ef1d..e9d23fc 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,8 @@ ZOSLIB is supported on the following hardware: * CMake 3.24.2-zos+ * GNU Make 4.3+ * IBM XL C/C++ V2.3.1 for z/OS V2.3 web deliverable (xlclang/xlcang++) or - IBM Open XL C/C++ 2.0 (clang/clang++) + IBM C/C++ for Open Enterprise Languages on z/OS 2.0.0 (clang/clang++) or + IBM Open XL C/C++ 2.1 for z/OS, version 2.1 (ibm-clang64/ibm-clang++64) * Git * Ninja (optional) diff --git a/include/c++/v1/__threading_support b/include/c++/v1/__threading_support deleted file mode 100644 index c53266e..0000000 --- a/include/c++/v1/__threading_support +++ /dev/null @@ -1,19 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// Licensed Materials - Property of IBM -// ZOSLIB -// (C) Copyright IBM Corp. 2022. All Rights Reserved. -// US Government Users Restricted Rights - Use, duplication -// or disclosure restricted by GSA ADP Schedule Contract with IBM Corp. -/////////////////////////////////////////////////////////////////////////////// - -#ifndef ZOS_LIBCPP_THREADING_SUPPORT -#define ZOS_LIBCPP_THREADING_SUPPORT - -//TODO(itodorov) - zos: workaround for nanosleep WoZ conflict, revisit -// once WoZ allow overriding or LE provides a nanosleep definition -#include -#define nanosleep cpp_nanosleep -#include_next <__threading_support> -#undef nanosleep - -#endif diff --git a/zoslib.gyp b/zoslib.gyp index 8e9d02d..c3f7ad7 100644 --- a/zoslib.gyp +++ b/zoslib.gyp @@ -29,12 +29,11 @@ 'conditions': [ [ '" Date: Tue, 26 Nov 2024 10:49:32 +0530 Subject: [PATCH 40/44] Add dependant code for script command (#71) Co-authored-by: Sachin T --- include/paths.h | 13 +++++ include/pty.h | 38 +++++++++++++++ include/sys/signalfd.h | 65 +++++++++++++++++++++++++ include/sys/time.h | 28 +++++++++++ include/termios.h | 26 ++++++++++ src/CMakeLists.txt | 2 + src/zos-cfmakeraw.c | 24 +++++++++ src/zos-pty.c | 108 +++++++++++++++++++++++++++++++++++++++++ 8 files changed, 304 insertions(+) create mode 100644 include/paths.h create mode 100644 include/pty.h create mode 100644 include/sys/signalfd.h create mode 100644 include/termios.h create mode 100644 src/zos-cfmakeraw.c create mode 100644 src/zos-pty.c diff --git a/include/paths.h b/include/paths.h new file mode 100644 index 0000000..82911d6 --- /dev/null +++ b/include/paths.h @@ -0,0 +1,13 @@ +/////////////////////////////////////////////////////////////////////////////// +// Licensed Materials - Property of IBM +// ZOSLIB +// (C) Copyright IBM Corp. 2024. All Rights Reserved. +// US Government Users Restricted Rights - Use, duplication +// or disclosure restricted by GSA ADP Schedule Contract with IBM Corp. +/////////////////////////////////////////////////////////////////////////////// + +#ifndef ZOS_PATHS_H_ + +#define _PATH_BSHELL "/bin/sh" + +#endif diff --git a/include/pty.h b/include/pty.h new file mode 100644 index 0000000..e1273e6 --- /dev/null +++ b/include/pty.h @@ -0,0 +1,38 @@ +/////////////////////////////////////////////////////////////////////////////// +//// Licensed Materials - Property of IBM +//// ZOSLIB +//// (C) Copyright IBM Corp. 2022. All Rights Reserved. +//// US Government Users Restricted Rights - Use, duplication +//// or disclosure restricted by GSA ADP Schedule Contract with IBM Corp. +///////////////////////////////////////////////////////////////////////////////// +// +#ifndef ZOS_PTY_H +#define ZOS_PTY_H + +#include "zos-macros.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif + + +/* Create pseudo tty master slave pair with NAME and set terminal + attributes according to TERMP and WINP and return handles for both + ends in AMASTER and ASLAVE. */ +__Z_EXPORT extern int openpty (int *__amaster, int *__aslave, char *__name, + const struct termios *__termp, + const struct winsize *__winp); + +/* Create child process and establish the slave pseudo terminal as the + child's controlling terminal. */ +__Z_EXPORT extern int forkpty (int *__amaster, char *__name, + const struct termios *__termp, + const struct winsize *__winp); + + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/sys/signalfd.h b/include/sys/signalfd.h new file mode 100644 index 0000000..efbfc40 --- /dev/null +++ b/include/sys/signalfd.h @@ -0,0 +1,65 @@ +/////////////////////////////////////////////////////////////////////////////// +// Licensed Materials - Property of IBM +// ZOSLIB +// (C) Copyright IBM Corp. 2022. All Rights Reserved. +// US Government Users Restricted Rights - Use, duplication +// or disclosure restricted by GSA ADP Schedule Contract with IBM Corp. +/////////////////////////////////////////////////////////////////////////////// + +#ifndef ZOS_SYS_SIGNALFD_H +#define ZOS_SYS_SIGNALFD_H + +#include +#include + +/* Flags for signalfd. */ +enum { + SFD_CLOEXEC = 02000000, +#define SFD_CLOEXEC SFD_CLOEXEC + SFD_NONBLOCK = 00004000 +#define SFD_NONBLOCK SFD_NONBLOCK +}; + + +struct signalfd_siginfo { + uint32_t ssi_signo; + int32_t ssi_errno; + int32_t ssi_code; + uint32_t ssi_pid; + uint32_t ssi_uid; + int32_t ssi_fd; + uint32_t ssi_tid; + uint32_t ssi_band; + uint32_t ssi_overrun; + uint32_t ssi_trapno; + int32_t ssi_status; + uint32_t ssi_int; + uint64_t ssi_ptr; + uint64_t ssi_utime; + uint64_t ssi_stime; + uint64_t ssi_addr; + /* + * Pad strcture to 128 bytes. Remember to update the + * pad size when you add new members. We use a fixed + * size structure to avoid compatibility problems with + * future versions, and we leave extra space for additional + * members. We use fixed size members because this strcture + * comes out of a read(2) and we really don't want to have + * a compat on read(2). + */ + uint8_t __pad[48]; +}; + +#ifdef __cplusplus +extern "C" { +#endif + +__Z_EXPORT extern int signalfd(int fd, const sigset_t *mask, int flags); + +__Z_EXPORT int signalfd_close(int fd); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/sys/time.h b/include/sys/time.h index 25bbcb9..a124f07 100644 --- a/include/sys/time.h +++ b/include/sys/time.h @@ -65,4 +65,32 @@ __Z_EXPORT int lutimes(const char *filename, const struct timeval tv[2]); #endif +/* Convenience macros for operations on timevals. + NOTE: `timercmp' does not work for >= or <=. */ +#define timerisset(tvp) ((tvp)->tv_sec || (tvp)->tv_usec) +#define timerclear(tvp) ((tvp)->tv_sec = (tvp)->tv_usec = 0) +#define timercmp(a, b, CMP) \ + (((a)->tv_sec == (b)->tv_sec) \ + ? ((a)->tv_usec CMP (b)->tv_usec) \ + : ((a)->tv_sec CMP (b)->tv_sec)) +#define timeradd(a, b, result) \ + do { \ + (result)->tv_sec = (a)->tv_sec + (b)->tv_sec; \ + (result)->tv_usec = (a)->tv_usec + (b)->tv_usec; \ + if ((result)->tv_usec >= 1000000) \ + { \ + ++(result)->tv_sec; \ + (result)->tv_usec -= 1000000; \ + } \ + } while (0) +#define timersub(a, b, result) \ + do { \ + (result)->tv_sec = (a)->tv_sec - (b)->tv_sec; \ + (result)->tv_usec = (a)->tv_usec - (b)->tv_usec; \ + if ((result)->tv_usec < 0) { \ + --(result)->tv_sec; \ + (result)->tv_usec += 1000000; \ + } \ + } while (0) + #endif diff --git a/include/termios.h b/include/termios.h new file mode 100644 index 0000000..96c7919 --- /dev/null +++ b/include/termios.h @@ -0,0 +1,26 @@ +/////////////////////////////////////////////////////////////////////////////// +//// Licensed Materials - Property of IBM +//// ZOSLIB +//// (C) Copyright IBM Corp. 2022. All Rights Reserved. +//// US Government Users Restricted Rights - Use, duplication +//// or disclosure restricted by GSA ADP Schedule Contract with IBM Corp. +///////////////////////////////////////////////////////////////////////////////// +// +#ifndef ZOS_TERMIOS_H +#define ZOS_TERMIOS_H + +#include "zos-macros.h" + +#include_next + +#ifdef __cplusplus +extern "C" { +#endif + +__Z_EXPORT void cfmakeraw(struct termios *termios_p); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e66e6a4..0c0b8ea 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -21,6 +21,8 @@ set(libsrc zos-mount.c zos-mkdtemp.c zos-instrumentation.cc + zos-pty.c + zos-cfmakeraw.c ) set(zoslib-help zoslib-help.cc) diff --git a/src/zos-cfmakeraw.c b/src/zos-cfmakeraw.c new file mode 100644 index 0000000..0847ed4 --- /dev/null +++ b/src/zos-cfmakeraw.c @@ -0,0 +1,24 @@ +///////////////////////////////////////////////////////////////////////////////// +//// Licensed Materials - Property of IBM +//// ZOSLIB +//// (C) Copyright IBM Corp. 2022. All Rights Reserved. +//// US Government Users Restricted Rights - Use, duplication +//// or disclosure restricted by GSA ADP Schedule Contract with IBM Corp. +///////////////////////////////////////////////////////////////////////////////// + +#define _AE_BIMODAL 1 + +#include "zos-base.h" +#include + +void +cfmakeraw (struct termios *t) +{ + t->c_iflag &= ~(IGNBRK|BRKINT|PARMRK|ISTRIP|INLCR|IGNCR|ICRNL|IXON); + t->c_oflag &= ~OPOST; + t->c_lflag &= ~(ECHO|ECHONL|ICANON|ISIG|IEXTEN); + t->c_cflag &= ~(CSIZE|PARENB); + t->c_cflag |= CS8; + t->c_cc[VMIN] = 1; /* read returns when one char is available. */ + t->c_cc[VTIME] = 0; +} diff --git a/src/zos-pty.c b/src/zos-pty.c new file mode 100644 index 0000000..639868b --- /dev/null +++ b/src/zos-pty.c @@ -0,0 +1,108 @@ +///////////////////////////////////////////////////////////////////////////////// +//// Licensed Materials - Property of IBM +//// ZOSLIB +//// (C) Copyright IBM Corp. 2022. All Rights Reserved. +//// US Government Users Restricted Rights - Use, duplication +//// or disclosure restricted by GSA ADP Schedule Contract with IBM Corp. +///////////////////////////////////////////////////////////////////////////////// + +#define _AE_BIMODAL 1 + +#include "zos-base.h" +#include +#include +#include +#include + +int openpty(int *master, int *slave, char *name, const struct termios *termp, const struct winsize *winp) { + int fd; + char master_dev[20], slave_dev[20]; + + *master = -1; + *slave = -1; + + // Iterating through the 4-digit ptyp and ttyp devices + int i=0; + for (; i < 10000; i++) { // Maximum 9999 for 4-digit format + snprintf(master_dev, sizeof(master_dev), "/dev/ptyp%04d", i); + snprintf(slave_dev, sizeof(slave_dev), "/dev/ttyp%04d", i); + + if ((*master = open(master_dev, O_RDWR | O_NOCTTY)) != -1) { + if (grantpt(*master) != 0) + goto error; + + if (unlockpt(*master) != 0) + goto error; + + if ((*slave = open(slave_dev, O_RDWR | O_NOCTTY)) != -1) { + break; // Found an available pair + } + + close(*master); + *master = -1; + } + } + + if(i == 10000) + goto error; + + if (termp && tcsetattr(*slave, TCSAFLUSH, termp) == -1) { + perror("tcsetattr() error"); + goto error; + } + if (winp && ioctl(*slave, TIOCSWINSZ, winp) == -1) { + goto error; + } + + // ignoring name, not passed and size is unknown in the API + + return 0; + +error: + if (*slave != -1) { + close(*slave); + *slave = -1; + } + if (*master != -1) { + close(*master); + *master = -1; + } + return -1; +} + + +static int login_tty(int fd) { + setsid(); + + dup2(fd, STDIN_FILENO); + dup2(fd, STDOUT_FILENO); + dup2(fd, STDERR_FILENO); + if (fd > STDERR_FILENO) { + close(fd); + } + + return 0; +} + +pid_t forkpty(int *amaster, char *name, const struct termios *termp, const struct winsize *winp) { + int master, slave; + if (openpty(&master, &slave, name, termp, winp) == -1) { + return -1; + } + + pid_t pid = fork(); + switch (pid) { + case -1: + close(master); + close(slave); + return -1; + case 0: + close(master); + login_tty(slave); + return 0; + default: + close(slave); + *amaster = master; + return pid; + } +} From bac16750d96b64d0a4487bb65d525f839a9707da Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Fri, 3 Jan 2025 13:47:25 -0500 Subject: [PATCH 41/44] Add vasprintf and strcasestr - needed by htop --- include/stdio.h | 1 + include/string.h | 1 + include/zos-v2r5-symbolfixes.h | 2 +- src/zos-io.cc | 11 +++++++++++ src/zos-string.c | 10 ++++++++-- 5 files changed, 22 insertions(+), 3 deletions(-) diff --git a/include/stdio.h b/include/stdio.h index 310f18e..1dd803c 100644 --- a/include/stdio.h +++ b/include/stdio.h @@ -42,6 +42,7 @@ extern "C" { __Z_EXPORT extern FILE *fopen(const char *filename, const char *mode) __asm("__fopen_ascii"); __Z_EXPORT ssize_t getline(char **lineptr, size_t *n, FILE *stream); __Z_EXPORT ssize_t getdelim(char **lineptr, size_t *n, int delimiter, FILE *stream); +__Z_EXPORT int vasprintf(char **strp, const char *fmt, va_list ap); #if defined(__cplusplus) } diff --git a/include/string.h b/include/string.h index 25f0145..e555461 100644 --- a/include/string.h +++ b/include/string.h @@ -26,6 +26,7 @@ __Z_EXPORT const char *sigdescr_np(int); __Z_EXPORT const char *sigabbrev_np(int); __Z_EXPORT void *memmem(const void *haystack, size_t haystacklen, const void *needle, size_t needlelen); __Z_EXPORT int strverscmp(const char *l0, const char *r0); +__Z_EXPORT char *strcasestr(const char *haystack, const char *needle); // Linux includes strings.h in string.h, this avoids the // warning - implicitly declaring library function 'strcasecmp' diff --git a/include/zos-v2r5-symbolfixes.h b/include/zos-v2r5-symbolfixes.h index a5faf17..29c5673 100644 --- a/include/zos-v2r5-symbolfixes.h +++ b/include/zos-v2r5-symbolfixes.h @@ -48,7 +48,7 @@ #pragma redefine_extname fdopendir fdopendir_undefined #pragma redefine_extname pthread_condattr_setclock pthread_condattr_setclock_undefined #pragma redefine_extname asprintf asprintf_undefined -#pragma redefine_extname vasprintf vasprintf_undefined +#pragma redefine_extname eventfd eventfd_undefined #endif #endif // ZOS_V2R5_SYMBOLFIXES_H diff --git a/src/zos-io.cc b/src/zos-io.cc index 29687e9..0300394 100644 --- a/src/zos-io.cc +++ b/src/zos-io.cc @@ -1202,6 +1202,17 @@ int mkostemp(char *tmpl, int flags) { return mkostemps(tmpl, 0, flags); } +// Adapted from Musl C (MIT License) +int vasprintf(char **s, const char *fmt, va_list ap) { + va_list ap2; + va_copy(ap2, ap); + int l = vsnprintf(0, 0, fmt, ap2); + va_end(ap2); + + if (l<0 || !(*s=(char*)malloc(l+1U))) return -1; + return vsnprintf(*s, l+1U, fmt, ap); +} + #ifdef __cplusplus } diff --git a/src/zos-string.c b/src/zos-string.c index d301458..cd025f9 100644 --- a/src/zos-string.c +++ b/src/zos-string.c @@ -158,8 +158,7 @@ void *memmem(const void *l, size_t l_len, const void *s, size_t s_len) { } // Adapted from Musl c's implementation (MIT license) -int strverscmp(const char *l0, const char *r0) -{ +int strverscmp(const char *l0, const char *r0) { const unsigned char *l = (const void *)l0; const unsigned char *r = (const void *)r0; size_t i, dp, j; @@ -186,6 +185,13 @@ int strverscmp(const char *l0, const char *r0) return l[i] - r[i]; } +/* Taken from Musl C: https://git.musl-libc.org/cgit/musl/tree/src/string/strcasestr.c (MIT license)*/ +char *strcasestr(const char *h, const char *n) { + size_t l = strlen(n); + for (; *h; h++) if (!strncasecmp(h, n, l)) return (char *)h; + return 0; +} + #if TEST int main() { printf("values: segv=%d total=%d\n", SIGSEGV, sigTotal); From a6f5da144da08c94230b4027e2bbb5c0c2c692c7 Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Mon, 27 Jan 2025 15:18:16 -0500 Subject: [PATCH 42/44] Add a fix for symlinks with $ literals in the beginning --- include/sys/param.h | 7 ++++ src/zos-bpx.cc | 29 ++++++++++------- test/test-readlink.cc | 74 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 12 deletions(-) create mode 100644 test/test-readlink.cc diff --git a/include/sys/param.h b/include/sys/param.h index 8220c3a..0c80bec 100644 --- a/include/sys/param.h +++ b/include/sys/param.h @@ -9,5 +9,12 @@ #ifndef ZOS_SYS_PARAM_H_ #define ZOS_SYS_PARAM_H_ +// Could be defined in resolv.h +#ifndef MAXHOSTNAMELEN + #define MAXHOSTNAMELEN 64 +#endif + +#define MAXNAMLEN 255 +#define MAXPATHLEN 4096 #endif diff --git a/src/zos-bpx.cc b/src/zos-bpx.cc index bbbdfc2..a42ae89 100644 --- a/src/zos-bpx.cc +++ b/src/zos-bpx.cc @@ -95,26 +95,31 @@ ssize_t __readlink_orig(const char *path, char *buf, size_t bufsiz) asm("@@A0020 __Z_EXPORT ssize_t __readlink(const char *path, char *buf, size_t bufsiz) { ssize_t len = __readlink_orig(path, buf, bufsiz); if (len < 0) { - return -1; + return -1; } if ((len > 0 && buf[0] == '$') || (len > 1 && buf[0] == '/' && buf[1] == '$')) { - // Not sure if this is possible, but double check in case: - if (len < bufsiz) { - buf[len] = '\0'; - } else { - buf[bufsiz - 1] = '\0'; - } - char resolved_path[PATH_MAX]; - if (realpath(path, resolved_path) == NULL) { - return -1; + // Ensure the buffer is null-terminated + if (len < bufsiz) { + buf[len] = '\0'; + } else { + buf[bufsiz - 1] = '\0'; + } + + char resolved_path[PATH_MAX]; + if (realpath(path, resolved_path) != NULL) { + // Check if the resolved path still contains a '$' (indicating unresolved substitution) + if (strchr(resolved_path, '$') != NULL) { + // If '$' is still present, treat it as literal and return the original symlink + return len; } len = snprintf(buf, bufsiz, "%s", resolved_path); if (len >= bufsiz) { - errno = ENAMETOOLONG; - return -1; + errno = ENAMETOOLONG; + return -1; } + } } return len; diff --git a/test/test-readlink.cc b/test/test-readlink.cc new file mode 100644 index 0000000..3d2a2cb --- /dev/null +++ b/test/test-readlink.cc @@ -0,0 +1,74 @@ +#include +#include +#include +#include +#include +#include +#include + +// Forward declaration for the wrapper implementation +extern "C" ssize_t __readlink(const char *path, char *buf, size_t bufsiz); + +class ReadlinkTest : public ::testing::Test { +protected: + void SetUp() override { + test_dir = "/tmp/readlink_test_dir"; + mkdir(test_dir, 0755); + + target_file = std::string(test_dir) + "/$target"; + symlink_with_dollar = std::string(test_dir) + "/$dollar"; + normal_symlink = std::string(test_dir) + "/normal_symlink"; + unresolved_symlink = std::string(test_dir) + "/unresolved_symlink"; + + int fd = open(target_file.c_str(), O_CREAT | O_WRONLY, 0644); + ASSERT_NE(fd, -1); + write(fd, "test content", 12); + close(fd); + + ASSERT_EQ(symlink(target_file.c_str(), symlink_with_dollar.c_str()), 0); + ASSERT_EQ(symlink(target_file.c_str(), normal_symlink.c_str()), 0); + ASSERT_EQ(symlink("$VERSION/", unresolved_symlink.c_str()), -1); + } + + void TearDown() override { + unlink(symlink_with_dollar.c_str()); + unlink(normal_symlink.c_str()); + unlink(target_file.c_str()); + unresolved_symlink = std::string(test_dir) + "/unresolved_symlink"; + rmdir(test_dir); + } + + const char *test_dir; + std::string target_file; + std::string symlink_with_dollar; + std::string normal_symlink; + std::string unresolved_symlink; +}; + +TEST_F(ReadlinkTest, SymlinkWithDollarInName) { + // Test a symlink with "$" in its name + char buf[PATH_MAX]; + ssize_t len = __readlink(symlink_with_dollar.c_str(), buf, sizeof(buf)); + ASSERT_GT(len, 0); + buf[len] = '\0'; + EXPECT_STREQ(buf, target_file.c_str()); +} + +TEST_F(ReadlinkTest, NormalSymlink) { + // Test a normal symlink (no "$" in its name) + char buf[PATH_MAX]; + ssize_t len = __readlink(normal_symlink.c_str(), buf, sizeof(buf)); + ASSERT_GT(len, 0); + buf[len] = '\0'; + EXPECT_STREQ(buf, target_file.c_str()); +} + +TEST_F(ReadlinkTest, UnresolvedSymlinkWithDollarTarget) { + // Test a symlink where the target starts with "$VERSION/" + char buf[PATH_MAX]; + ssize_t len = __readlink(unresolved_symlink.c_str(), buf, sizeof(buf)); + ASSERT_GT(len, 0); + buf[len] = '\0'; + + EXPECT_STREQ(buf, "/"); +} From 68f947f0a4b0d994198195865cbf2741e3a368d1 Mon Sep 17 00:00:00 2001 From: Igor Todorovski Date: Tue, 28 Jan 2025 12:28:06 -0500 Subject: [PATCH 43/44] TODO one readlink test that fails on sysplex only --- test/test-readlink.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/test-readlink.cc b/test/test-readlink.cc index 3d2a2cb..a02cccb 100644 --- a/test/test-readlink.cc +++ b/test/test-readlink.cc @@ -27,7 +27,9 @@ class ReadlinkTest : public ::testing::Test { ASSERT_EQ(symlink(target_file.c_str(), symlink_with_dollar.c_str()), 0); ASSERT_EQ(symlink(target_file.c_str(), normal_symlink.c_str()), 0); +#if 0 // TODO: add a test for sysplex ASSERT_EQ(symlink("$VERSION/", unresolved_symlink.c_str()), -1); +#endif } void TearDown() override { @@ -63,6 +65,7 @@ TEST_F(ReadlinkTest, NormalSymlink) { EXPECT_STREQ(buf, target_file.c_str()); } +#if 0 TEST_F(ReadlinkTest, UnresolvedSymlinkWithDollarTarget) { // Test a symlink where the target starts with "$VERSION/" char buf[PATH_MAX]; @@ -72,3 +75,4 @@ TEST_F(ReadlinkTest, UnresolvedSymlinkWithDollarTarget) { EXPECT_STREQ(buf, "/"); } +#endif From ab82da463396736f602c484709dbe7fe40f7b535 Mon Sep 17 00:00:00 2001 From: Igor Todorovski <39890068+IgorTodorovskiIBM@users.noreply.github.com> Date: Thu, 30 Jan 2025 16:17:05 -0500 Subject: [PATCH 44/44] Update zos-v2r5-symbolfixes.h --- include/zos-v2r5-symbolfixes.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/zos-v2r5-symbolfixes.h b/include/zos-v2r5-symbolfixes.h index 29c5673..ce97999 100644 --- a/include/zos-v2r5-symbolfixes.h +++ b/include/zos-v2r5-symbolfixes.h @@ -49,6 +49,7 @@ #pragma redefine_extname pthread_condattr_setclock pthread_condattr_setclock_undefined #pragma redefine_extname asprintf asprintf_undefined #pragma redefine_extname eventfd eventfd_undefined +#pragma redefine_extname wait4 wait4_undefined #endif #endif // ZOS_V2R5_SYMBOLFIXES_H