Fedora Core 5 interception working as well

Author: ahmadia

Makefile

@@ -7,7 +7,8 @@ MPICC = mpicc
 CFLAGS = -std=c99 -fPIC -Wall -Wextra ${CFLAGS_DEBUG} -D_LARGEFILE64_SOURCE
 CFLAGS_DEBUG = -g3 -DDEBUG=1
 #LDFLAGS =  -Wl,-Bdynamic -Wl,-rpath,/home/aron/sys/lib -L/home/aron/sys/lib ${LIBDL} 
-LDFLAGS = -dynamic -Wl,-Bdynamic -Wl,-rpath,/home/aron/bgpsys/lib -L/home/aron/bgpsys/lib ${LIBDL} 
+#LDFLAGS = -dynamic -Wl,-Bdynamic -Wl,-rpath,/home/aron/bgpsys/lib -L/home/aron/bgpsys/lib ${LIBDL} 
+LDFLAGS = -dynamic -Wl,-Bdynamic -Wl,-rpath,${LIBCPATH} -L${LIBCPATH} ${LIBDL} 
 LIBDL = -ldl 
 COLLFS_SRC_C = collfs.c
 COLLFS_SRC_O = $(COLLFS_SRC_C:.c=.o)
@@ -26,7 +27,7 @@ thefunc.o : thefunc.c
 	${CC} ${CFLAGS} -c -fPIC $^
 
 minimal_main : minimal_main.o libminimal_thefunc.so
-	${MPICC} -g3 -o $@ $^ ${LDFLAGS} 
+	${MPICC} -g3 -o $@ minimal_main.o ${LDFLAGS} 
 
 main : main.o libfoo.so libcollfs.so 
 #main : main.o libfoo.so libcollfs.so libc-collfs.so

README.txt

@@ -1,12 +1,37 @@
 ***********************************************************************************************
-# Applying collfs patches to glibc-2.4
+# What is collfs?
 
-# The magic rtld rules go in elf/rtld-Rules
+collfs is a user library and set of patches to glibc that enables file system calls to be handled collectively over an
+MPI Communicator.  collfs was written with the specific goal of providing scalable dynamic loading on supercomputers
+with faster interconnect than i/o performance, but is a general-purpose tool that can be used either directly as a
+library or implicitly by wrapping file system calls with collective versions.
 
-# This here is the whole point of all the shenanigans.
-# CPPFLAGS-rtld := -DNOT_IN_libc=1 -DIS_IN_rtld=1
+The collfs patches are currently quite minimal.  They insert an externally visible struct object into the run-time
+dynamic linker containing void function hooks for collective MPI versions of the standard file system API.  These function pointers
+are then used (when non-NULL) to replace file system function calls within the standard dynamic loading routines in glibc.  Additionally,
+we provide a small static C library via LD_PRELOAD that initializes MPI (this is pre-main, so it passes NULL to the Init
+routines) and activates the function hooks to point to our collective versions of the standard file system API,
+shadowing the original file system functions (but utilizing them by default).  
 
+***********************************************************************************************
+# Installing collfs on a Fedora Core 5 system
+
+Fedora Core 5 is at this point, quite old, but it features a 2.4 version of glibc, which is very close to the 2.4 glibc
+installed on the IBM Blue Gene/P.
+
+This assumes you have an FC5 installation with the standard development RPMs installed.  If you are unable to build
+glibc-2.4 from the SRPM, you will probably need to install the missing RPMs from the DVD or a web repository.
+
+Tips for setting up a non-root installation environment for SRPMs can be found here:
+http://www.owlriver.com/tips/non-root//
+
+mkdir -p ~/sandbox/glibc
+cd ~/sandbox/glibc
+wget http://dl.dropbox.com/u/65439/glibc-2.4-11-collfs.src.rpm
+rpmbuild --nodeps --rebuild -bc glibc-2.4-11-collfs.src.rpm
 
+This should leave you with a built glibc (good for testing) in the RPM build directory, which
+was/var/tmp/glibc-2.4-root on my machine.
 
 ***********************************************************************************************
 # Building glibc-2.4 for the compute nodes:

glibc-2.4-bgp-patches/elf/rtld.c

@@ -86,8 +86,8 @@ struct libc_collfs_api {
 struct libc_collfs_api _dl_collfs_api = {
   .fxstat64 = NULL,
   .xstat64  = NULL,
-  .open     = __open,
-  .close    = __close,
+  .open     = NULL,
+  .close    = NULL,
   .read     = NULL,
   .lseek    = NULL,
   .mmap     = NULL,

minimal_main.c

@@ -59,15 +59,17 @@ int main(int argc, char *argv[])
   collfs_open_fp _open_fp;
   char path[MAXPATHLEN];
 
+  typedef void (*vfptr)(void); 
+
   MPI_Init(&argc,&argv);
 
-  _dl_collfs_api.open = (void *) minimal_open;
+  _dl_collfs_api.open = (vfptr) minimal_open;
   _open_fp = (collfs_open_fp) _dl_collfs_api.open;  
   int fid = _open_fp("test_open.txt",O_RDONLY,0);
   close(fid);
 
   if (!getcwd(path,sizeof path)) ERR("getcwd failed");
-  strcat(path,"/libthefunc.so");
+  strcat(path,"/libminimal_thefunc.so");
   err = run_tests(path, path);CHK(err);
 
   MPI_Finalize();