starcluster instance changes

Author: ahmadia

Makefile

@@ -41,9 +41,8 @@ collfs.o : collfs.c collfs.h libc-collfs.h
 main-nompi : main-nompi.o libcollfs-easy.so libminimal_thefunc.so
 	${CC} -g3 -o $@ $< ${LDFLAGS}
 
-#main : main.o libfoo.so libcollfs.so 
-#main : main.o libfoo.so libcollfs.so libc-collfs.so
-#	${MPICC} -g3 -o $@ $^ ${LDFLAGS} 
+test_collfs : test_collfs.o libfoo.so libcollfs.so libminimal_thefunc.so
+	${MPICC} -g3 -o $@ $^ ${LDFLAGS} 
 .c.o :
 	${CC} ${CFLAGS} -fPIC -c $<
 
@@ -52,8 +51,13 @@ libfoo.so : foo.o collfs.o
 
 collfs.o : collfs.c collfs.h libc-collfs.h
 libc-collfs.o : libc-collfs.c libc-collfs.h libc-collfs-private.h
+
 foo.o : foo.c foo.h errmacros.h collfs.h libc-collfs-private.h
-main.o : main.c errmacros.h
+	${MPICC} -c -fPIC -g3  $<
+
+test_collfs.o : test_collfs.c errmacros.h
+	${MPICC} -c -fPIC -g3  $<
+
 thefunc.o : thefunc.h collfs.h
 
 clean :

foo.c

@@ -14,6 +14,7 @@
 #include "collfs.h"
 #include "libc-collfs-private.h" /* So that we can call the wrapped versions directly */
 
+
 extern int MPI_Comm_rank ( MPI_Comm comm, int *rank ) __attribute__ ((weak));
 
 int foo(const char *path)
@@ -39,7 +40,7 @@ int foo(const char *path)
   pa = __collfs_mmap((void *)0,pagesize,PROT_READ,MAP_PRIVATE,fd,(off_t)0);
   if (pa == MAP_FAILED) ERR("[%d] mmap(%d,pagesize) failed",rank,fd);
 
-  while (__collfs_read(fd,&value,sizeof value) == sizeof value) {
+  while (__collfs_libc_read(fd,&value,sizeof value) == sizeof value) {
     sum ^= value;
   }
 
@@ -63,21 +64,21 @@ int foo2_inner(const char *path)
   off = __collfs_lseek(fd, 0, SEEK_SET);
   if (off != 0) ERR("lseek SET");
 
-  nc = __collfs_read(fd, buf, 2);
+  nc = __collfs_libc_read(fd, buf, 2);
   if (nc != 2) ERR("read [%d] but expected 2\nbuf: %4.4s", nc, buf);
   if (strncmp(buf, "ab", 2)) ERR("wrong content");
 
-  if (__collfs_read(fd, buf, 3) != 3) ERR("read");
+  if (__collfs_libc_read(fd, buf, 3) != 3) ERR("read");
   if (strncmp(buf, "cde", 3)) ERR("wrong repeat read");
 
   off = __collfs_lseek(fd, 8, SEEK_CUR);
   if (off != 13) ERR("lseek CUR");
-  if (__collfs_read(fd, buf, 3) != 3) ERR("read");
+  if (__collfs_libc_read(fd, buf, 3) != 3) ERR("read");
   if (strncmp(buf, "nop", 3)) ERR("wrong content");
 
   off = __collfs_lseek(fd, 5, SEEK_SET);
   if (off != 5) ERR("lseek SET");
-  if (__collfs_read(fd, buf, 3) != 3) ERR("read");
+  if (__collfs_libc_read(fd, buf, 3) != 3) ERR("read");
   if (strncmp(buf, "fgh", 3)) ERR("wrong content");
 
   err = collfs_comm_pop();CHK(err);

libc-collfs-private.h

@@ -6,13 +6,58 @@
 #include <sys/types.h>
 #include <unistd.h>
 
-int __collfs_fxstat64(int vers, int fd, struct stat64 *buf);
-int __collfs_xstat64(int vers, const char *file, struct stat64 *buf);
-void *__collfs_mmap(void *addr, size_t len, int prot, int flags, int fildes, off_t off);
-int __collfs_munmap(__ptr_t addr, size_t len);
-off_t __collfs_lseek(int fildes, off_t offset, int whence);
-int __collfs_open(const char *pathname, int flags, ...);
-int __collfs_close(int fd);
-ssize_t __collfs_read(int fd, void *buf, size_t count);
+struct libc_collfs_api {
+  void (*fxstat64)(void);
+  void (*xstat64)(void);
+  void (*open)(void);
+  void (*close)(void);
+  void (*read)(void);
+  void (*lseek)(void);
+  void (*mmap)(void);
+  void (*munmap)(void);
+};
+
+extern struct libc_collfs_api _dl_collfs_api;
+
+typedef int (*collfs_fxstat64_fp)(int vers, int fd, struct stat64 *buf);
+typedef int (*collfs_xstat64_fp)(int vers, const char *file, struct stat64 *buf);
+typedef int (*collfs_open_fp)(const char *pathname, int flags, mode_t mode);
+typedef int (*collfs_close_fp)(int fd);
+typedef ssize_t (*collfs_read_fp)(int fd, void *buf, size_t count);
+typedef off_t (*collfs_lseek_fp)(int fildes, off_t offset, int whence);
+typedef void *(*collfs_mmap_fp)(void *addr, size_t len, int prot, int flags, int fildes, off_t off);
+typedef int (*collfs_munmap_fp)(__ptr_t addr, size_t len);
+
+#define __collfs_open(pathname,flags)                                 \
+  ((_dl_collfs_api.open != NULL) ?                                      \
+   ((collfs_open_fp) _dl_collfs_api.open)(pathname,flags,O_RDONLY) : __open(pathname, flags))
+
+#define __collfs_close(fd)                                    \
+  ((_dl_collfs_api.close != NULL) ?                           \
+   ((collfs_close_fp) _dl_collfs_api.close)(fd) : __close(fd))                              
+
+#define __collfs_lseek(fildes, offset, whence)                          \
+  ((_dl_collfs_api.lseek != NULL) ?                                     \
+   ((collfs_lseek_fp) _dl_collfs_api.lseek)(fildes, offset, whence) : __lseek(fildes, offset, whence))
+
+#define __collfs_fxstat64(vers, fd, buf)                                \
+  ((_dl_collfs_api.fxstat64 != NULL) ?                                  \
+   ((collfs_fxstat64_fp) _dl_collfs_api.fxstat64)(vers, fd, buf) : __fxstat64(vers, fd, buf))
+
+#define __collfs_xstat64(vers, file, buf)                               \
+  ((_dl_collfs_api.xstat64 != NULL) ?                                   \
+   ((collfs_xstat64_fp) _dl_collfs_api.xstat64)(vers, file, buf) : __xstat64(vers, file, buf))
+
+#define __collfs_libc_read(fd, buf, count)                              \
+  ((_dl_collfs_api.read != NULL) ?                                      \
+   ((collfs_read_fp) _dl_collfs_api.read)(fd, buf, count) : read(fd, buf, count))
+
+#define __collfs_mmap(addr, len, prot, flags, fildes, off)              \
+  ((_dl_collfs_api.mmap != NULL) ?                                      \
+   ((collfs_mmap_fp) _dl_collfs_api.mmap)(addr, len, prot, flags, fildes, off) : mmap(addr, len, prot, flags, fildes, off))   
+
+#define __collfs_munmap(addr, len)                                      \
+  ((_dl_collfs_api.munmap != NULL) ?                                    \
+   ((collfs_munmap_fp) _dl_collfs_api.munmap)(addr, len) : munmap(addr, len))       
 
 #endif

shaheen/extended_tests.ll

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+#
+# @ job_name            = kslrun_job
+# @ job_type            = bluegene
+# @ output              = ./$(job_name)_$(jobid).out
+# @ error               = ./$(job_name)_$(jobid).err
+# @ environment         = COPY_ALL; 
+# @ wall_clock_limit    = 4:00:00,4:00:00
+# @ notification        = always
+# @ bg_size             = 2048
+# @ account_no          = k01
+
+# @ queue
+
+projdir=/project/k01/pyclaw
+sandbox=${projdir}/sandbox
+builddir=${projdir}/opt/share
+srcdir=${builddir}/sources
+
+pythondir=${builddir}/python/2.7.2/bgp
+ldpath=${pythondir}/lib
+numpy_path=${builddir}/numpy/1.6.2/bgp/lib/python
+nose_path=${builddir}/nose/1.1.2/bgp/lib/python
+clawpack_path=${builddir}/clawpack/dev/bgp/lib/python
+petsc4py_path=${builddir}/petsc4py/1.2/bgp/lib/python
+mpi_python_path=${builddir}/mpi4py/1.3/bgp/lib/python
+
+bgp_python_path=${numpy_path}:${nose_path}:${clawpack_path}:${petsc4py_path}:${mpi_python_path}
+
+bgp_python=${pythondir}/bin/python
+mpi_python=${builddir}/mpi4py/1.3/bgp/lib/python/mpi4py/bin/python-mpi
+
+testdir=/gpfs/scratch/aron/sandbox/import/collfs/tests
+
+cd $testdir
+logdir=${testdir}/runs
+mkdir -p ${logdir}
+
+for np in 1 2 4 8 16 32 64 128 256 512 1024 2048 4096
+do
+    mpirun -env LD_LIBRARY_PATH=${ldpath} -env PYTHONPATH=${bgp_python_path} \
+        -mode VN -exp_env HOME -n 2 ${bgp_python} test_python_importer.py &> ${logdir}/python_${np}.txt
+    mpirun -env LD_LIBRARY_PATH=${ldpath} -env PYTHONPATH=${bgp_python_path} \
+        -mode VN -exp_env HOME -n 2 ${mpi_python} test_collfs_importer.py &> ${logdir}/collfs_${np}.txt
+    mpirun -env LD_LIBRARY_PATH=${ldpath} -env PYTHONPATH=${bgp_python_path} \
+        -mode VN -exp_env HOME -n 2 ${bgp_python} test_mpi4py_cached_importer.py &> ${logdir}/asher_${np}.txt
+    mpirun -env LD_LIBRARY_PATH=${ldpath} -env PYTHONPATH=${bgp_python_path} \
+        -mode VN -exp_env HOME -n 2 ${mpi_python} test_mpi4py_cached_importer.py &> ${logdir}/asher_collfs_${np}.txt
+done

shaheen/kslrun_job_submit.ll

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+#
+# @ job_name            = kslrun_job
+# @ job_type            = bluegene
+# @ output              = ./$(job_name)_$(jobid).out
+# @ error               = ./$(job_name)_$(jobid).err
+# @ environment         = COPY_ALL; 
+# @ wall_clock_limit    = 2:00:00,2:00:00
+# @ notification        = always
+# @ bg_size             = 128
+# @ account_no          = k01
+
+# @ queue
+
+llx

shaheen/test_pyclaw.sh

@@ -0,0 +1,20 @@
+projdir=/project/k01/pyclaw
+sandbox=${projdir}/sandbox
+builddir=${projdir}/opt/share
+logdir=${builddir}/logs
+srcdir=${builddir}/sources
+
+pythondir=${builddir}/python/2.7.2/bgp
+ldpath=${pythondir}/lib
+numpy_path=${builddir}/numpy/1.6.2/bgp/lib/python
+nose_path=${builddir}/nose/1.1.2/bgp/lib/python
+clawpack_path=${builddir}/clawpack/dev/bgp/lib/python
+petsc4py_path=${builddir}/petsc4py/1.2/bgp/lib/python
+mpi_python_path=${builddir}/mpi4py/1.3/bgp/lib/python
+
+bgp_python_path=${numpy_path}:${nose_path}:${clawpack_path}:${petsc4py_path}:${mpi_python_path}
+
+mpi_python=${builddir}/mpi4py/1.3/bgp/lib/python/mpi4py/bin/python-mpi
+
+mpirun -env LD_LIBRARY_PATH=${ldpath} -env PYTHONPATH=${bgp_python_path} \
+    -mode VN -exp_env HOME -n 2 ${mpi_python} -v test.py

main.c renamed to test_collfs.c

@@ -15,6 +15,19 @@
 extern int MPI_Init(int *argc, char ***argv) __attribute__ ((weak));
 extern int MPI_Finalize() __attribute__ ((weak));
 
+struct libc_collfs_api {
+  void (*fxstat64)(void);
+  void (*xstat64)(void);
+  void (*open)(void);
+  void (*close)(void);
+  void (*read)(void);
+  void (*lseek)(void);
+  void (*mmap)(void);
+  void (*munmap)(void);
+};
+
+struct libc_collfs_api _dl_collfs_api;
+
 int run_tests(const char *soname, const char *path)
 {
   void *handle;
@@ -29,10 +42,6 @@ int run_tests(const char *soname, const char *path)
   if (!func) ERR("dlsym could not find symbol \"thefunc\" due to: %s", dlerror());
   err = (*func)();CHK(err);
 
-  test_fxstat64 = dlsym(handle,"thetest_fxstat64");
-  if (!func) ERR("dlsym could not find symbol \"thetest_fxstat64\" due to: %s", dlerror());
-  err = (*test_fxstat64)(path);CHK(err);
-
   err = dlclose(handle);
   if (err) ERR("dlclose failed due to: %s", dlerror());
   return 0;
@@ -48,7 +57,7 @@ int main(int argc, char *argv[])
     MPI_Init(&argc,&argv);
   }
   if (!getcwd(path,sizeof path)) ERR("getcwd failed");
-  strcat(path,"/libthefunc.so");
+  strcat(path,"/libminimal_thefunc.so");
   err = run_tests(path, path);CHK(err);
   err = foo(path);CHK(err);
   err = foo2("alphabet.txt");CHK(err);