Working build on Cetus, had to replace realloc with malloc+memset, don't know why

$ make main-mpi LIBCPATH=$HOME/bgqsys/lib
$ qsub -t 5 --cwd /gpfs/veas-fs0/jedbrown -n 1024 --mode c64 /home/jedbrown/collfs/main-mpi 0

NumProcs 65536  Min 2.17616@57053  Max 2.17648@31166  Ratio 1.00015  Ave 2.17631

Author: jedbrown

Makefile

@@ -8,9 +8,10 @@ CFLAGS = -std=c99 -fPIC -Wall -Wextra ${CFLAGS_DEBUG} -D_LARGEFILE64_SOURCE
 CFLAGS_DEBUG = -g3 -DDEBUG=1
 LIBCPATH ?= /lib
 LIBDL ?= -ldl
+LDSO ?= ld-2.12.2.so
 #LDFLAGS =  -Wl,-Bdynamic -Wl,-rpath,/home/aron/sys/lib -L/home/aron/sys/lib ${LIBDL} 
 #LDFLAGS = -dynamic -Wl,-Bdynamic -Wl,-rpath,/home/aron/bgpsys/lib -L/home/aron/bgpsys/lib ${LIBDL} 
-LDFLAGS = -dynamic -Wl,-Bdynamic -Wl,-rpath,${LIBCPATH} -L${LIBCPATH} ${LIBDL}
+LDFLAGS = -dynamic -Wl,-Bdynamic -Wl,-rpath,${LIBCPATH} -L${LIBCPATH} ${LIBDL} ${LIBCPATH}/${LDSO} -Wl,-rpath,${PWD} -Wl,--dynamic-linker=${LIBCPATH}/ld-2.12.2.so -Wl,-rpath,/bgsys/drivers/V1R1M0/ppc64/comm/sys/lib
 LIBDL = -ldl 
 COLLFS_SRC_C = collfs.c
 COLLFS_SRC_O = $(COLLFS_SRC_C:.c=.o)

collfs.c

@@ -404,14 +404,20 @@ static void *collfs_mmap(void *addr, size_t len, int prot, int flags, int fildes
       if (len > link->totallen) {
         if (link->refct==1) { // no clients have mmaped this file, safe to remap/realloc
           size_t totallen;
-          debug_printf(2, "reallocating to size %zd on offset %lld", len, (long long) off);
+          debug_printf(2, "reallocating to size %zd (was %zd) on offset %lld", len, link->totallen, (long long) off);
           totallen = extend_to_page(len);
           MPI_Comm_rank(CommStack->comm, &rank);
           if (!rank) {
             ((collfs_munmap_fp) unwrap.munmap)(link->mem,link->totallen);
             mem = ((collfs_mmap_fp) unwrap.mmap)(addr, totallen, prot, flags, link->fd, off);
           } else {
-            mem = realloc(link->mem, totallen);
+	    mem = malloc(totallen);
+	    debug_printf(2, "Allocated %zd bytes", totallen);
+	    memcpy(mem,link->mem,link->totallen);
+	    debug_printf(2, "Copied %zd bytes", link->totallen);
+	    free(link->mem);
+	    link->mem = 0;
+	    // mem = realloc(link->mem, totallen);
           }
           gotmem = !!mem;
           MPI_Allreduce(MPI_IN_PLACE, &gotmem, 1, MPI_INT, MPI_LAND, CommStack->comm);

main-mpi.c

@@ -9,24 +9,24 @@
 #include "collfs.h"
 #include "errmacros.h"
 
-int run_tests(const char *soname)
+int run_tests(int verbosity,const char *soname)
 {
   void *handle;
   int err, rank;
-  int (*func)(void);
+  int (*func)(int);
 
   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
 
-  printf("[%d] & attempting to open %s\n", rank, soname);
+  if (verbosity > 0) printf("[%d] & attempting to open %s\n", rank, soname);
   handle = dlopen(soname,RTLD_GLOBAL | RTLD_LAZY);
   if (!handle) ERR("failed to dlopen(\"%s\",RTLD_GLOBAL | RTLD_LAZY) due to: %s", soname, dlerror());
 
-  printf("[%d] & looking for thefunc in handle %p\n", rank, handle);
+  if (verbosity > 0) printf("[%d] & looking for thefunc in handle %p\n", rank, handle);
   func = dlsym(handle,"thefunc");
   if (!func) ERR("dlsym could not find symbol \"thefunc\" due to: %s", dlerror());
 
-  printf("[%d] & found thefunc at address %p\n", rank, func);
-  err = (*func)();CHK(err);
+  if (verbosity > 0) printf("[%d] & found thefunc at address %p\n", rank, func);
+  err = (*func)(verbosity);CHK(err);
 
   err = dlclose(handle);
   if (err) ERR("dlclose failed due to: %s", dlerror());
@@ -35,18 +35,38 @@ int run_tests(const char *soname)
 
 int main(int argc, char *argv[])
 {
-  int err;
+  int err,verbosity,rank,size;
+  double tstart,tend,elapsed;
   char path[MAXPATHLEN];
 
   MPI_Init(&argc,&argv);
-  collfs_initialize(2, NULL);
+  MPI_Comm_rank(MPI_COMM_WORLD,&rank);
+  MPI_Comm_size(MPI_COMM_WORLD,&size);
+  if (argc == 2) {
+    verbosity = atoi(argv[1]);
+  }
+  collfs_initialize(verbosity, NULL);
 
   if (!getcwd(path,sizeof path)) ERR("getcwd failed");
   strcat(path,"/libminimal_thefunc.so");
 
+  tstart = MPI_Wtime();
   collfs_comm_push(MPI_COMM_WORLD);
-  err = run_tests(path);CHK(err);
+  err = run_tests(verbosity,path);CHK(err);
   collfs_comm_pop();
+  tend = MPI_Wtime();
+  elapsed = tend - tstart;
+  if (verbosity > 1) printf("[%d] elapsed = %g\n",rank,elapsed);
+  {
+    struct {double time; int rank; } loc,gmax,gmin;
+    double gsum;
+    loc.time = elapsed;
+    loc.rank = rank;
+    MPI_Reduce(&loc,&gmax,1,MPI_DOUBLE_INT,MPI_MAXLOC,0,MPI_COMM_WORLD);
+    MPI_Reduce(&loc,&gmin,1,MPI_DOUBLE_INT,MPI_MINLOC,0,MPI_COMM_WORLD);
+    MPI_Reduce(&elapsed,&gsum,1,MPI_DOUBLE,MPI_SUM,0,MPI_COMM_WORLD);
+    if (!rank) printf("NumProcs %d  Min %g@%d  Max %g@%d  Ratio %g  Ave %g\n",size,gmin.time,gmin.rank,gmax.time,gmax.rank,gmax.time/gmin.time,gsum/size);
+  }
 
   collfs_finalize();
   MPI_Finalize();

thefunc.c

@@ -4,8 +4,9 @@
 #include <fcntl.h>
 #include <stdio.h>
 
-int thefunc(void) {
-  return printf("called %s\n", __func__) <= 0;
+int thefunc(int verbosity) {
+  if (verbosity >= 0) return printf("called %s\n", __func__) <= 0;
+  return 0;
 }
 
 int thetest_fxstat64(const char *path) {

thefunc.h

@@ -1,7 +1,7 @@
 #ifndef _thefunc_h
 #define _thefunc_h
 
-int thefunc(void);
+int thefunc(int);
 int thetest(const char *path);
 
 #endif