restored main-mpi functionality on Shaheen, fixed bugs in dl-dload.c and collfs.c

Author: ahmadia

collfs.c

@@ -300,7 +300,10 @@ static int collfs_close(int fd)
       int rank = 0, xerr = 0;
 
       debug_printf(2, "%s(%d) collective", __func__, fd);
-      if (--link->refct > 0) return 0;
+      if (--link->refct > 0) {
+        debug_printf(2, "%s(%d) nonzero refcount %d", __func__, link->refct);
+        return 0;
+      }
       err = MPI_Comm_rank(CommStack ? CommStack->comm : MPI_COMM_WORLD, &rank); if (err) return -1;
       if (!rank) {
         ((collfs_munmap_fp) unwrap.munmap)(link->mem, link->totallen);
@@ -310,7 +313,9 @@ static int collfs_close(int fd)
       }
       *linkp = link->next;
       free(link);
+      debug_printf(2, "%s(%d) entering Bcast", __func__, fd);
       err = MPI_Bcast(&xerr, 1, MPI_INT, 0, CommStack->comm);      
+      debug_printf(2, "%s(%d) exiting Bcast", __func__, fd);
       return err;
     }
   }
@@ -404,6 +409,7 @@ static void *collfs_mmap(void *addr, size_t len, int prot, int flags, int fildes
         /* if (addr >= (void*)((char*) link->mem) && addr+len <= (void*)((char*)link->mem+link->totallen)) { */
           MPI_Comm_rank(CommStack->comm, &rank);
           if (!rank) {
+            link->refct++;
             return ((collfs_mmap_fp) unwrap.mmap)(addr, len, prot, flags, link->fd, off);
           } else {
 	    /* debug_printf(2, "Allocated %zd bytes range [%p %p]", len, mem, (void*)(char*)addr+len); */

glibc-2.4-bgp-patches/elf/dl-load.c

@@ -856,7 +856,7 @@ lose (int code, int fd, const char *name, char *realname, struct link_map *l,
 {
   /* The file might already be closed.  */
   if (fd != -1)
-    (void) __close (fd);
+    (void) __collfs_close (fd);
   if (l != NULL)
     {
       /* Remove the stillborn object from the list and free it.  */
@@ -927,7 +927,7 @@ _dl_map_object_from_fd (const char *name, int fd, struct filebuf *fbp,
       {
 	/* The object is already loaded.
 	   Just bump its reference count and return it.  */
-	__close (fd);
+	__collfs_close (fd);
 
 	/* If the name is not in the list of names for this object add
 	   it.  */
@@ -956,7 +956,7 @@ _dl_map_object_from_fd (const char *name, int fd, struct filebuf *fbp,
 
       /* No need to bump the refcount of the real object, ld.so will
 	 never be unloaded.  */
-      __close (fd);
+      __collfs_close (fd);
 
       return l;
     }
@@ -981,7 +981,7 @@ _dl_map_object_from_fd (const char *name, int fd, struct filebuf *fbp,
       _dl_zerofd = _dl_sysdep_open_zero_fill ();
       if (_dl_zerofd == -1)
 	{
-	  __close (fd);
+	  __collfs_close (fd);
 	  _dl_signal_error (errno, NULL, NULL,
 			    N_("cannot open zero fill device"));
 	}
@@ -1495,7 +1495,7 @@ cannot enable executable stack as shared object requires");
 #endif
 
   /* We are done mapping in the file.  We no longer need the descriptor.  */
-  if (__builtin_expect (__close (fd) != 0, 0))
+  if (__builtin_expect (__collfs_close (fd) != 0, 0))
     {
       errstring = N_("cannot close file descriptor");
       goto call_lose_errno;

makeinc.shaheen

@@ -0,0 +1,9 @@
+CC = /bgsys/drivers/ppcfloor/gnu-linux/powerpc-bgp-linux/bin/gcc
+MPICC = /bgsys/drivers/ppcfloor/comm/bin/mpicc
+
+CFLAGS = -std=c99 -fPIC -Wall -Wextra ${CFLAGS_DEBUG} -D_LARGEFILE64_SOURCE
+CFLAGS_DEBUG = -g3 -DDEBUG=1
+LIBCPATH = /home/aron/bgpsys/lib
+LIBDL = -ldl
+LDSO = ld-2.4.so
+LDFLAGS = -dynamic -Wl,-Bdynamic -Wl,-rpath,${LIBCPATH} -L${LIBCPATH} -lc ${LIBDL} ${LIBCPATH}/${LDSO} -Wl,-rpath,${PWD} -Wl,--dynamic-linker=${LIBCPATH}/${LDSO}