Print memusage

bgq/Makefile.in

@@ -31,7 +31,7 @@ LDADD =
 COMPILE = ${CC} $(DEFS) ${INCLUDES} ${CPPFLAGS} ${CFLAGS}
 
 LIBRARIES = libbgq
-libbgq_TARGETS = bgq_legacy bgq_workers.inc bgq_workers_double bgq_workers_float bgq_ReadWeyllayout.inc bgq_HoppingMatrixWorker.inc bgq_ComputeWeyl.inc bgq_stdreductions bgq_reduction.inc bgq_stdoperators bgq_operator.inc asm1 asm2 bgq_qpx bgq_gaugefield bgq_spinorfield bgq_comm bgq_utils bgq_field bgq_HoppingMatrix bgq_dispatch mypapi
+libbgq_TARGETS = memusage bgq_legacy bgq_workers.inc bgq_workers_double bgq_workers_float bgq_ReadWeyllayout.inc bgq_HoppingMatrixWorker.inc bgq_ComputeWeyl.inc bgq_stdreductions bgq_reduction.inc bgq_stdoperators bgq_operator.inc asm1 asm2 bgq_qpx bgq_gaugefield bgq_spinorfield bgq_comm bgq_utils bgq_field bgq_HoppingMatrix bgq_dispatch mypapi
 libbgq_STARGETS = 
 
 libbgq_OBJECTS = $(addsuffix .o, ${libbgq_TARGETS})

bgq/memusage.c

@@ -0,0 +1,118 @@
+#include "memusage.h"
+
+#include "config.h"
+#include <sys/resource.h>
+
+#ifdef BGP
+#include <common/bgp_personality.h>
+#include <common/bgp_personality_inlines.h>
+#include <spi/kernel_interface.h>
+#include <spi/bgp_SPI.h>
+#endif
+
+#if BGQ
+#include <process.h>
+#include <location.h>
+#include <personality.h>
+#endif
+
+#include "global.h"
+#include <string.h>
+
+#ifdef BGQ
+/* returns memory per core in MBytes */
+unsigned bg_coreMB()
+{
+    unsigned procMB, coreMB;
+    static _BGP_Personality_t mybgp;
+    Kernel_GetPersonality(&mybgp, sizeof(_BGP_Personality_t));
+    procMB = BGP_Personality_DDRSizeMB(&mybgp);
+    coreMB = procMB/Kernel_ProcessCount();
+    return coreMB;
+}
+#endif
+
+/* return maximum memory usage of process in kBytes */
+unsigned bg_usedKB()
+{
+    struct rusage usage;
+    if (getrusage(RUSAGE_SELF, &usage) != 0)
+        return 0;
+    return usage.ru_maxrss;
+}
+
+void print_memusage()
+{
+    if (g_proc_id != 0)
+        return;
+
+#if BGQ
+    {
+        unsigned memPerCore = bg_coreMB();
+        fprintf(stderr, "MK_Memory available per Core:  %10ld MB\n", memPerCore);
+    }
+#endif
+
+    struct rusage usage;
+    memset(&usage, 0, sizeof(usage));
+    int returncode = 0;
+    if (returncode = getrusage(RUSAGE_SELF, &usage))
+    {
+        fprintf(stderr, "MK_getrusage error; return code %d\n", returncode);
+    }
+    else
+    {
+        fprintf(stderr, "MK_User time used:             %10ld,%6ld secs\n", (long)(usage.ru_utime.tv_sec), (long)(usage.ru_utime.tv_usec));
+        fprintf(stderr, "MK_System time used:           %10ld,%6ld secs\n", (long)(usage.ru_stime.tv_sec), (long)(usage.ru_stime.tv_usec));
+        fprintf(stderr, "MK_Maximum resident set size:  %10ld MB\n", (long)usage.ru_maxrss / 1024);
+#if 0
+        fprintf(stderr, "MK_Integral shared memory size:%10ld MB\n", (long)(usage.ru_ixrss) / 1024);
+        fprintf(stderr, "MK_Integral unshared data size:%10ld MB\n", (long)(usage.ru_idrss) / 1024);
+        fprintf(stderr, "MK_Integral unshared stack s.: %10ld MB\n", (long)(usage.ru_isrss) / 1024);
+        fprintf(stderr, "MK_Page reclaims:              %10ld\n", (long)(usage.ru_minflt));
+        fprintf(stderr, "MK_Page faults:                %10ld\n", (long)(usage.ru_majflt));
+        fprintf(stderr, "MK_Swaps:                      %10ld\n", (long)(usage.ru_nswap));
+        fprintf(stderr, "MK_Block input operations:     %10ld\n", (long)(usage.ru_inblock));
+        fprintf(stderr, "MK_Block output operations:    %10ld\n", (long)(usage.ru_oublock));
+        fprintf(stderr, "MK_Messages sent:              %10ld\n", (long)(usage.ru_msgsnd));
+        fprintf(stderr, "MK_Messages received:          %10ld\n", (long)(usage.ru_msgrcv));
+        fprintf(stderr, "MK_Signals received:           %10ld\n", (long)(usage.ru_nsignals));
+        fprintf(stderr, "MK_Voluntary context switches: %10ld\n", (long)(usage.ru_nvcsw));
+        fprintf(stderr, "MK_Involuntary context sw.:    %10ld\n", (long)(usage.ru_nivcsw));
+#endif
+    }
+
+#ifdef BGQ
+    {
+        unsigned int memory_size = 0;
+        Kernel_GetMemorySize(KERNEL_MEMSIZE_HEAP, &memory_size);
+        fprintf(stderr, "MK_Memory size HEAP:           %10ld MB\n", (long)memory_size / (1024 * 1024));
+
+        Kernel_GetMemorySize(KERNEL_MEMSIZE_STACK, &memory_size);
+        fprintf(stderr, "MK_Memory size STACK:          %10ld MB\n", (long)memory_size / (1024 * 1024));
+
+        Kernel_GetMemorySize(KERNEL_MEMSIZE_HEAPAVAIL, &memory_size);
+        fprintf(stderr, "MK_Memory available HEAP:      %10ld MB\n", (long)memory_size / (1024 * 1024));
+
+        Kernel_GetMemorySize(KERNEL_MEMSIZE_STACKAVAIL, &memory_size);
+        fprintf(stderr, "MK_Memory available STACK:     %10ld MB\n", (long)memory_size / (1024 * 1024));
+
+        Kernel_GetMemorySize(KERNEL_MEMSIZE_HEAPMAX, &memory_size);
+        fprintf(stderr, "MK_Maximum memory HEAP:        %10ld MB\n", (long)memory_size / (1024 * 1024));
+
+        Kernel_GetMemorySize(KERNEL_MEMSIZE_GUARD, &memory_size);
+        fprintf(stderr, "MK_Heap guardpage:             %10ld MB\n", (long)memory_size / (1024 * 1024));
+
+        Kernel_GetMemorySize(KERNEL_MEMSIZE_SHARED, &memory_size);
+        fprintf(stderr, "MK_Shared memory:              %10ld MB\n", (long)memory_size / (1024 * 1024));
+
+#if 0
+        Kernel_GetMemorySize(KERNEL_MEMSIZE_PERSIST, &memory_size);
+        fprintf(stderr, "MK_Persistent memory:          %10ld MB\n", (long)memory_size / (1024 * 1024));
+#endif
+    }
+#endif
+
+}
+
+

bgq/memusage.h

@@ -0,0 +1,2 @@
+
+void print_memusage();

hmc_tm.c

@@ -165,6 +165,11 @@ int main_hmc(int argc,char *argv[]) {
   g_proc_id = 0;
 #endif
 
+  if (g_proc_id == 0) {
+    fprintf(stderr, "MK_Static memusage:\n");
+    print_memusage(); // MK
+  }
+
 
   while ((c = getopt(argc, argv, "h?vVf:o:")) != -1) {
     switch (c) {
@@ -408,13 +413,23 @@ int main_hmc(int argc,char *argv[]) {
 
 
 // BEGIN MK
+  if (g_proc_id == 0) {
+    fprintf(stderr, "MK_init memusage:\n");
+    print_memusage(); // MK
+  }
+
 	assert(even_odd_flag);
 	bgq_indices_init();
 	bgq_comm_mpi_init();
 	bgq_comm_spi_init();
 	bgq_initbgqref();
 	bgq_spinorfields_init();
 	bgq_gaugefield_init();
+
+	if (g_proc_id == 0) {
+		fprintf(stderr, "MK_BGQ memusage:\n");
+		print_memusage(); // MK
+	}
 // END MK
 
 
@@ -468,6 +483,11 @@ int main_hmc(int argc,char *argv[]) {
     fclose(countfile);
   }
 
+  if (g_proc_id == 0) {
+    fprintf(stderr, "MK_Loop memusage:\n");
+    print_memusage(); // MK
+  }
+
   /* Loop for measurements */
   for(j = 0; j < Nmeas; j++) {
     if(g_proc_id == 0) {
@@ -582,6 +602,11 @@ int main_hmc(int argc,char *argv[]) {
     trajectory_counter++;
   } /* end of loop over trajectories */
 
+  if (g_proc_id == 0) {
+    fprintf(stderr, "MK_Done memusage:\n");
+    print_memusage(); // MK
+  }
+
   if(g_proc_id == 0 && Nmeas != 0) {
     printf("# Acceptance rate was %3.2f percent, %d out of %d trajectories accepted.\n", 100.*(double)Rate/(double)Nmeas, Rate, Nmeas);
     fflush(stdout);
@@ -607,6 +632,11 @@ int main_hmc(int argc,char *argv[]) {
     free_chi_spinor_field();
   }
 
+  if (g_proc_id == 0) {
+    fprintf(stderr, "MK_Exit memusage:\n");
+    print_memusage(); // MK
+  }
+
   return(0);
 #ifdef _KOJAK_INST
 #pragma pomp inst end(main)

init_spinor_field.c

@@ -69,9 +69,11 @@ int init_spinor_field(const int V, const int nr) {
     g_spinor_field[i] = g_spinor_field[i-1]+V;
   }
 
+//#if BGQ_REPLACE
   assert(!g_bgq_spinorfields);
   bgq_weylfield_collection *collection = bgq_spinorfields_allocate(nr, g_spinor_field[0], V);
   g_bgq_spinorfields = collection->controlblocks;
+//#endif
   return(0);
 }
 

job.ll.template

@@ -8,8 +8,8 @@
 #@ notification = error
 #@ notify_user = [email protected]
 #@ job_type = bluegene
-##@ bg_size = 32
-#@ bg_shape = 1x1x1x1
+#@ bg_size = 32
+##@ bg_shape = 1x1x1x1
 #@ bg_connectivity = TORUS
 #@ queue
 
@@ -50,10 +50,10 @@ LZ=64
 
 INPUT_FILE_PHMC=hmc_tm.input_phmc.${LOADL_STEP_ID}
 cat > ${INPUT_FILE_PHMC} <<+EOF
-T=256
-LX=128
-LY=128
-LZ=128
+T=16
+LX=16
+LY=16
+LZ=16
 
 OMPNumThreads = 64
 DebugLevel = 1