Print memusage+Allow non-square global length dismensions

D_psi.c

@@ -1237,10 +1237,10 @@ void D_psi_prec(spinor * const P, spinor * const Q){
   static _Complex double alpha = -1.0;
 
   alpha = -0.5;
-  spinorPrecondition(P,Q,ws,T,L,alpha,0,1);
+  spinorPrecondition(P,Q,ws,T,LX,LY,LZ,alpha,0,1);
   D_psi(g_spinor_field[DUM_MATRIX],P);
   alpha = -0.5;
-  spinorPrecondition(P,g_spinor_field[DUM_MATRIX],ws,T,L,alpha,0,1);
+  spinorPrecondition(P,g_spinor_field[DUM_MATRIX],ws,T,LX,LY,LZ,alpha,0,1);
 }
 
 /* apply the Dirac operator to the block local spinor field s */

Dov_psi.c

@@ -86,7 +86,7 @@ void Dov_psi_prec(spinor * const P, spinor * const S) {
   static _Complex double alpha;
   Dov_psi(P,S);
   alpha = -1.0;
-  spinorPrecondition(P,P,ws,T,L,alpha,0,1);
+  spinorPrecondition(P,P,ws,T,LX,LY,LZ,alpha,0,1);
 
 }
 
@@ -219,20 +219,20 @@ void Qov_sq_psi_prec(spinor * const P, spinor * const S) {
   static _Complex double alpha = 0.0;
 
   alpha = ws->precExpo[0];
-  spinorPrecondition(P,S,ws,T,L,alpha,0,1);
+  spinorPrecondition(P,S,ws,T,LX,LY,LZ,alpha,0,1);
 
 
   Dov_psi(g_spinor_field[DUM_MATRIX], P);
   gamma5(P, g_spinor_field[DUM_MATRIX], VOLUME);
 
   alpha = ws->precExpo[1];
-  spinorPrecondition(P,P,ws,T,L,alpha,0,1);
+  spinorPrecondition(P,P,ws,T,LX,LY,LZ,alpha,0,1);
 
   Dov_psi(g_spinor_field[DUM_MATRIX], P);
   gamma5(P, g_spinor_field[DUM_MATRIX], VOLUME);
 
   alpha = ws->precExpo[2];
-  spinorPrecondition(P,P,ws,T,L,alpha,0,1);
+  spinorPrecondition(P,P,ws,T,LX,LY,LZ,alpha,0,1);
 
   return;
 }

bgq/bgq_field.h

@@ -45,9 +45,9 @@ Logically, it is a torus, therefore (t+i*GLOBAL_LT)==t (mod GLOBAL_LT) for every
 */
 
 #define GLOBAL_LT ((ucoord)T_global)
-#define GLOBAL_LX ((ucoord)LX*(ucoord)N_PROC_X)
-#define GLOBAL_LY ((ucoord)LY*(ucoord)N_PROC_Y)
-#define GLOBAL_LZ ((ucoord)LZ*(ucoord)N_PROC_Z)
+#define GLOBAL_LX ((ucoord)LX_global)
+#define GLOBAL_LY ((ucoord)LY_global)
+#define GLOBAL_LZ ((ucoord)LZ_global)
 #define GLOBAL_VOLUME (GLOBAL_LT*GLOBAL_LX*GLOBAL_LY*GLOBAL_LZ)
 
 

bgq/memusage.c

@@ -10,23 +10,28 @@
 #include <spi/bgp_SPI.h>
 #endif
 
-#if BGQ
-#include <process.h>
-#include <location.h>
-#include <personality.h>
+#if 1
+#include <kernel/process.h>
+#include <kernel/location.h>
+#include <kernel/memory.h>
+//#include <kernel/personality.h>
 #endif
 
 #include "global.h"
 #include <string.h>
 
-#ifdef BGQ
+#include <stdio.h>
+#include <malloc.h>
+
+
+#if 1
 /* returns memory per core in MBytes */
 unsigned bg_coreMB()
 {
     unsigned procMB, coreMB;
-    static _BGP_Personality_t mybgp;
-    Kernel_GetPersonality(&mybgp, sizeof(_BGP_Personality_t));
-    procMB = BGP_Personality_DDRSizeMB(&mybgp);
+    static Personality_t mybgp;
+    Kernel_GetPersonality(&mybgp, sizeof(Personality_t));
+    //procMB = BGP_Personality_DDRSizeMB(&mybgp);
     coreMB = procMB/Kernel_ProcessCount();
     return coreMB;
 }
@@ -46,7 +51,7 @@ void print_memusage()
     if (g_proc_id != 0)
         return;
 
-#if BGQ
+#if 1
     {
         unsigned memPerCore = bg_coreMB();
         fprintf(stderr, "MK_Memory available per Core:  %10ld MB\n", memPerCore);
@@ -64,7 +69,8 @@ void print_memusage()
     {
         fprintf(stderr, "MK_User time used:             %10ld,%6ld secs\n", (long)(usage.ru_utime.tv_sec), (long)(usage.ru_utime.tv_usec));
         fprintf(stderr, "MK_System time used:           %10ld,%6ld secs\n", (long)(usage.ru_stime.tv_sec), (long)(usage.ru_stime.tv_usec));
-        fprintf(stderr, "MK_Maximum resident set size:  %10ld MB\n", (long)usage.ru_maxrss / 1024);
+        fprintf(stderr, "MK_Maximum resident set size:  %10ld KB\n", usage.ru_maxrss);
+        fprintf(stderr, "MK_Signals                     %dl\n", usage.ru_nsignals);
 #if 0
         fprintf(stderr, "MK_Integral shared memory size:%10ld MB\n", (long)(usage.ru_ixrss) / 1024);
         fprintf(stderr, "MK_Integral unshared data size:%10ld MB\n", (long)(usage.ru_idrss) / 1024);
@@ -82,34 +88,67 @@ void print_memusage()
 #endif
     }
 
-#ifdef BGQ
+#if 1
     {
-        unsigned int memory_size = 0;
-        Kernel_GetMemorySize(KERNEL_MEMSIZE_HEAP, &memory_size);
-        fprintf(stderr, "MK_Memory size HEAP:           %10ld MB\n", (long)memory_size / (1024 * 1024));
+        unsigned long memory_size = 0;
+        //Kernel_GetMemorySize(KERNEL_MEMSIZE_HEAP, &memory_size);
+        //fprintf(stderr, "MK_Memory size HEAP:           %10ld MB\n", (long)memory_size / (1024 * 1024));
 
-        Kernel_GetMemorySize(KERNEL_MEMSIZE_STACK, &memory_size);
-        fprintf(stderr, "MK_Memory size STACK:          %10ld MB\n", (long)memory_size / (1024 * 1024));
+        //Kernel_GetMemorySize(KERNEL_MEMSIZE_STACK, &memory_size);
+        //fprintf(stderr, "MK_Memory size STACK:          %10ld MB\n", (long)memory_size / (1024 * 1024));
 
-        Kernel_GetMemorySize(KERNEL_MEMSIZE_HEAPAVAIL, &memory_size);
-        fprintf(stderr, "MK_Memory available HEAP:      %10ld MB\n", (long)memory_size / (1024 * 1024));
+        //Kernel_GetMemorySize(KERNEL_MEMSIZE_HEAPAVAIL, &memory_size);
+        //fprintf(stderr, "MK_Memory available HEAP:      %10ld MB\n", (long)memory_size / (1024 * 1024));
 
-        Kernel_GetMemorySize(KERNEL_MEMSIZE_STACKAVAIL, &memory_size);
-        fprintf(stderr, "MK_Memory available STACK:     %10ld MB\n", (long)memory_size / (1024 * 1024));
+        //Kernel_GetMemorySize(KERNEL_MEMSIZE_STACKAVAIL, &memory_size);
+        //fprintf(stderr, "MK_Memory available STACK:     %10ld MB\n", (long)memory_size / (1024 * 1024));
 
         Kernel_GetMemorySize(KERNEL_MEMSIZE_HEAPMAX, &memory_size);
         fprintf(stderr, "MK_Maximum memory HEAP:        %10ld MB\n", (long)memory_size / (1024 * 1024));
 
-        Kernel_GetMemorySize(KERNEL_MEMSIZE_GUARD, &memory_size);
-        fprintf(stderr, "MK_Heap guardpage:             %10ld MB\n", (long)memory_size / (1024 * 1024));
+        //Kernel_GetMemorySize(KERNEL_MEMSIZE_GUARD, &memory_size);
+        //fprintf(stderr, "MK_Heap guardpage:             %10ld MB\n", (long)memory_size / (1024 * 1024));
 
-        Kernel_GetMemorySize(KERNEL_MEMSIZE_SHARED, &memory_size);
-        fprintf(stderr, "MK_Shared memory:              %10ld MB\n", (long)memory_size / (1024 * 1024));
+        //Kernel_GetMemorySize(KERNEL_MEMSIZE_SHARED, &memory_size);
+        //fprintf(stderr, "MK_Shared memory:              %10ld MB\n", (long)memory_size / (1024 * 1024));
 
 #if 0
         Kernel_GetMemorySize(KERNEL_MEMSIZE_PERSIST, &memory_size);
         fprintf(stderr, "MK_Persistent memory:          %10ld MB\n", (long)memory_size / (1024 * 1024));
 #endif
+
+
+        uint64_t shared, persist, heapavail, stackavail, stack, heap, guard, mmap;
+        Kernel_GetMemorySize(KERNEL_MEMSIZE_SHARED, &shared);
+        Kernel_GetMemorySize(KERNEL_MEMSIZE_PERSIST, &persist);
+        Kernel_GetMemorySize(KERNEL_MEMSIZE_HEAPAVAIL, &heapavail);
+        Kernel_GetMemorySize(KERNEL_MEMSIZE_STACKAVAIL, &stackavail);
+        Kernel_GetMemorySize(KERNEL_MEMSIZE_STACK, &stack);
+        Kernel_GetMemorySize(KERNEL_MEMSIZE_HEAP, &heap);
+        Kernel_GetMemorySize(KERNEL_MEMSIZE_GUARD, &guard);
+        Kernel_GetMemorySize(KERNEL_MEMSIZE_MMAP, &mmap);
+
+        fprintf(stderr, "MK_Allocated heap: %.2f MB, avail. heap: %.2f MB\n", (double)heap/(1024*1024), (double)heapavail/(1024*1024));
+        fprintf(stderr, "MK_Allocated stack: %.2f MB, avail. stack: %.2f MB\n", (double)stack/(1024*1024), (double)stackavail/(1024*1024));
+        fprintf(stderr, "MK_Memory: shared: %.2f MB, persist: %.2f MB, guard: %.2f MB, mmap: %.2f MB\n", (double)shared/(1024*1024), (double)persist/(1024*1024), (double)guard/(1024*1024), (double)mmap/(1024*1024));
+
+
+        struct mallinfo m;
+        m = mallinfo();
+
+        unsigned int arena = m.arena;          /* size to sbrk */
+        fprintf(stderr, "MK_arena = %d \n", arena);
+
+        unsigned int uordblks = m.uordblks;     /* chunks in use, in bytes */
+        fprintf(stderr, "MK_uordblks = %d \n", uordblks);
+
+        unsigned int hblkhd = m.hblkhd;         /* mmap memory in bytes */
+        fprintf(stderr, "MK_hblkhd = %d \n", hblkhd);
+
+        unsigned int total_heap = uordblks + hblkhd;
+        fprintf(stderr, "MK_total_heap = %d \n", total_heap);
+
+        fflush(stderr);
     }
 #endif
 

build-bgq.sh

@@ -54,16 +54,18 @@ LIBS="${LIBS} -lpthread"
 LIBS="${LIBS} -lSPI_l1p"
 LIBS="${LIBS} -lSPI"
 LIBS="${LIBS} -lbgpm"
+LIBS="${LIBS} -lxlopt"
 
 
 LDFLAGS=""
-#LDFLAGS="${LDFLAGS} -qipa=level=2"
+LDFLAGS="${LDFLAGS} -O5"
 LDFLAGS="${LDFLAGS} -L/opt/ibmcmp/xlf/bg/14.1/lib64"
 LDFLAGS="${LDFLAGS} -L/opt/ibmcmp/xlsmp/bg/3.1/bglib64"
 LDFLAGS="${LDFLAGS} -L/usr/local/bg_soft/lapack/3.3.0"
 LDFLAGS="${LDFLAGS} -L/bgsys/ibm_essl/prod/opt/ibmmath/lib64"
 LDFLAGS="${LDFLAGS} -L/bgsys/drivers/ppcfloor/spi/lib"
 LDFLAGS="${LDFLAGS} -L/bgsys/drivers/ppcfloor/bgpm/lib/"
+LDFLAGS="${LDFLAGS} -L/opt/ibmcmp/vac/bg/12.1/lib64"
 #LDFLAGS="${LDFLAGS} -qipa=level=2"
 #LDFLAGS="${LDFLAGS} -L/opt/ibmcmp/xlsmp/bg/3.1/bglib64"
 #LDFLAGS="${LDFLAGS} -L/bgsys/ibm_essl/prod/opt/ibmmath/lib64"
@@ -95,7 +97,7 @@ CONFIGURE=""
 #CONFIGURE="${CONFIGURE} --with-fixedvolume"
 CONFIGURE="${CONFIGURE} --without-bgldram"
 CONFIGURE="${CONFIGURE} --with-limedir=${HOME}/lime"
-#CONFIGURE="${CONFIGURE} --with-lemondir=${ROOTPATH}/lemon"
+CONFIGURE="${CONFIGURE} --with-lemondir=${HOME}/lemon"
 CONFIGURE="${CONFIGURE} --enable-mpi"
 CONFIGURE="${CONFIGURE} --with-mpidimension=XYZT"
 #CONFIGURE="${CONFIGURE} --enable-omp"

gen_sources.c

@@ -107,10 +107,10 @@ int main(int argc,char *argv[]) {
   while ((c = getopt(argc, argv, "h?NCpOEdao:L:T:n:t:s:S:P:")) != -1) {
     switch (c) {
     case 'L':
-      L = atoi(optarg);
-      LX = L;
-      LY = L;
-      LZ = L;
+      int L = atoi(optarg);
+      LX_global = LX = L;
+      LY_global = LY = L;
+      LZ_global = LZ = L;
       break;
     case 'T':
       T = atoi(optarg);

global.h

@@ -77,9 +77,9 @@ EXTERN int g_relative_precision_flag;
 EXTERN int g_debug_level;
 EXTERN int g_disable_IO_checks;
 
-EXTERN int T_global;
+EXTERN int T_global, LX_global, LY_global, LZ_global;
 #ifndef FIXEDVOLUME
-EXTERN int T, L, LX, LY, LZ, VOLUME, BODY, SURFACE;
+EXTERN int T, /*L,*/ LX, LY, LZ, VOLUME, BODY, SURFACE;
 EXTERN int N_PROC_T, N_PROC_X, N_PROC_Y, N_PROC_Z;
 EXTERN int RAND, EDGES, VOLUMEPLUSRAND;
 EXTERN int TEOSLICE;

hmc_tm.c

@@ -417,7 +417,7 @@ int main_hmc(int argc,char *argv[]) {
     fprintf(stderr, "MK_init memusage:\n");
     print_memusage(); // MK
   }
-
+#if BGQ_REPLACE
 	assert(even_odd_flag);
 	bgq_indices_init();
 	bgq_comm_mpi_init();
@@ -430,9 +430,9 @@ int main_hmc(int argc,char *argv[]) {
 		fprintf(stderr, "MK_BGQ memusage:\n");
 		print_memusage(); // MK
 	}
+#endif
 // END MK
 
-
 	if(g_running_phmc) {
 		init_phmc();
 	}

invert_eo.c

@@ -400,15 +400,15 @@ int invert_eo(spinor * const Even_new, spinor * const Odd_new,
 
 	if(g_prec_sequence_d_dagger_d[2] != 0.0){
 	  alpha = g_prec_sequence_d_dagger_d[2];
-	  spinorPrecondition(g_spinor_field[DUM_DERI+1],g_spinor_field[DUM_DERI+1],ws,T,L,alpha,0,1);
+	  spinorPrecondition(g_spinor_field[DUM_DERI+1],g_spinor_field[DUM_DERI+1],ws,T,LX,LY,LZ,alpha,0,1);
 	}
 
 	iter = cg_her(g_spinor_field[DUM_DERI], g_spinor_field[DUM_DERI+1], max_iter, precision, 
 		    rel_prec, VOLUME, &Q_pm_psi_prec);
 
 	if(g_prec_sequence_d_dagger_d[0] != 0.0){
 	  alpha = g_prec_sequence_d_dagger_d[0];
-	  spinorPrecondition(g_spinor_field[DUM_DERI],g_spinor_field[DUM_DERI],ws,T,L,alpha,0,1);
+	  spinorPrecondition(g_spinor_field[DUM_DERI],g_spinor_field[DUM_DERI],ws,T,LX,LY,LZ,alpha,0,1);
 	}
 
       } else {
@@ -425,7 +425,7 @@ int invert_eo(spinor * const Even_new, spinor * const Odd_new,
 	static _Complex double alpha = 0.0;
 	if(g_prec_sequence_d_dagger_d[1] != 0.0){
 	  alpha = g_prec_sequence_d_dagger_d[1];
-	  spinorPrecondition(g_spinor_field[DUM_DERI+1],g_spinor_field[DUM_DERI+1],ws,T,L,alpha,0,1);
+	  spinorPrecondition(g_spinor_field[DUM_DERI+1],g_spinor_field[DUM_DERI+1],ws,T,LX,LY,LZ,alpha,0,1);
 	}
       }
 #endif

invert_overlap.c

@@ -62,15 +62,15 @@ void invert_overlap(const int op_id, const int index_start) {
       printf("# Using preconditioning (which one?)!\n");
 
       alpha = ws->precExpo[2];
-      spinorPrecondition(g_spinor_field[DUM_DERI+1],g_spinor_field[DUM_DERI+1],ws,T,L,alpha,0,1);
+      spinorPrecondition(g_spinor_field[DUM_DERI+1],g_spinor_field[DUM_DERI+1],ws,T,LX,LY,LZ,alpha,0,1);
 
       /* 	iter = cg_her(g_spinor_field[DUM_DERI], g_spinor_field[DUM_DERI+1], max_iter, precision,  */
       /* 		    rel_prec, VOLUME, &Q_pm_psi_prec); */
       optr->iterations = cg_her(g_spinor_field[DUM_DERI], g_spinor_field[DUM_DERI+1], optr->maxiter, optr->eps_sq,
 				optr->rel_prec, VOLUME, &Qov_sq_psi_prec);
 
       alpha = ws->precExpo[0];
-      spinorPrecondition(g_spinor_field[DUM_DERI],g_spinor_field[DUM_DERI],ws,T,L,alpha,0,1);
+      spinorPrecondition(g_spinor_field[DUM_DERI],g_spinor_field[DUM_DERI],ws,T,LX,LY,LZ,alpha,0,1);
 
     } 
     else {
@@ -87,7 +87,7 @@ void invert_overlap(const int op_id, const int index_start) {
     if(use_preconditioning == 1 && g_precWS!=NULL){
       ws=(spinorPrecWS*)g_precWS;
       alpha = ws->precExpo[1];
-      spinorPrecondition(g_spinor_field[DUM_DERI+1],g_spinor_field[DUM_DERI+1],ws,T,L,alpha,0,1);
+      spinorPrecondition(g_spinor_field[DUM_DERI+1],g_spinor_field[DUM_DERI+1],ws,T,LX,LY,LZ,alpha,0,1);
     }
 
   }

io/gauge_write.c

@@ -27,7 +27,7 @@ int write_gauge_field(char * filename, const int prec, paramsXlfInfo const *xlfI
   DML_Checksum     checksum;
   paramsIldgFormat *ildg;
 
-  bytes = (uint64_t)L * L * L * T_global * sizeof(su3) * prec / 16;
+  bytes = (uint64_t)LX_global * LY_global * LZ_global * T_global * sizeof(su3) * prec / 16;
 
   /* all these functions, except for write_binary_gauge_data do their own error handling */
   construct_writer(&writer, filename, 0);/* the 0 is for not appending */

io/gauge_write_binary.c

@@ -59,7 +59,7 @@ int write_binary_gauge_data(LemonWriter * lemonwriter, const int prec, DML_Check
     for(z = 0; z < LZ; z++) {
       for(y = 0; y < LY; y++) {
         for(x = 0; x < LX; x++) {
-          rank = (DML_SiteRank) ((((tG + t)*L + zG + z)*L + yG + y)*L + xG + x);
+          rank = (DML_SiteRank) ((((tG + t)*LZ_global + zG + z)*LY_global + yG + y)*LX_global + xG + x);
           memcpy(&tmp3[0], &g_gauge_field[ g_ipt[t][x][y][z] ][1], sizeof(su3));
           memcpy(&tmp3[1], &g_gauge_field[ g_ipt[t][x][y][z] ][2], sizeof(su3));
           memcpy(&tmp3[2], &g_gauge_field[ g_ipt[t][x][y][z] ][3], sizeof(su3));

io/params_construct_ildgFormat.c

@@ -8,9 +8,9 @@ paramsIldgFormat *construct_paramsIldgFormat(int const prec)
     kill_with_error(NULL, g_cart_id, "Could not allocate paramsIldgFormat.");
 
   format->prec = prec;
-  format->lx = L;
-  format->ly = L;
-  format->lz = L;
+  format->lx = LX_global;
+  format->ly = LY_global;
+  format->lz = LZ_global;
   format->lt = T_global;
 
   return format;

io/spinor_write_binary.c

@@ -55,7 +55,7 @@ int write_binary_spinor_data(spinor * const s, spinor * const r,
     for(z = 0; z < LZ; z++) {
       for(y = 0; y < LY; y++) {
         for(x = 0; x < LX; x++) {
-          rank = (DML_SiteRank) ((((tG + t)*L + zG + z)*L + yG + y)*L + xG + x);
+          rank = (DML_SiteRank) ((((tG + t)*LZ_global + zG + z)*LY_global + yG + y)*LX_global + xG + x);
           i = g_lexic2eosub[g_ipt[t][x][y][z]];
           if ((z  + zG + y  + yG +
                x  + xG + t + tG) % 2 == 0)
@@ -288,7 +288,7 @@ int write_binary_spinor_data_l(spinor * const s,
     for(z = 0; z < LZ; z++) {
       for(y = 0; y < LY; y++) {
         for(x = 0; x < LX; x++) {
-          rank = (DML_SiteRank) ((((tG + t)*L + zG + z)*L + yG + y)*L + xG + x);
+          rank = (DML_SiteRank) ((((tG + t)*LZ_global + zG + z)*LY_global + yG + y)*LX_global	 + xG + x);
           i = g_ipt[t][x][y][z];
 
           if (prec == 32)

io/utils_construct_reader.c

@@ -5,7 +5,7 @@ void construct_reader(READER ** reader, char * filename)
   LIME_FILE *fh = NULL;
   int status = 0;
 
-  if(g_debug_level > 0 && g_cart_id == 0) {
+  if(g_debug_level > 0 && g_proc_id == 0) {
 #ifdef HAVE_LIBLEMON
     printf("# Constructing LEMON reader for file %s ...\n", filename);
 #else

io/utils_kill_with_error.c

@@ -4,6 +4,7 @@ void kill_with_error(LIME_FILE *fh, int const rank, char const *error)
 {
   if (error != NULL)
   {
+	fflush(stderr);
     fprintf(stderr, "KILL_WITH_ERROR on node %d: %s", rank, error);
     fflush(stderr);
   }