Fixes on BG/Q

Meinersbur · Dec 13, 2012 · 5b8acff · 5b8acff
1 parent 833b18e
commit 5b8acff
Show file tree

Hide file tree

Showing 10 changed files with 158 additions and 124 deletions.
diff --git a/Makefile.in b/Makefile.in
@@ -91,7 +91,7 @@ SUBDIRS = ${USESUBDIRS}
 # delete the default suffix rules
 .SUFFIXES:
 
-all: Makefile all-recursive dep hmc_tm invert 
+all: Makefile all-recursive dep hmc_tm invert benchmark bgqbench 
 
 #ifneq (,$(findstring lapack,${LIBS}))
 #all: Makefile all-recursive dep hmc_tm invert

diff --git a/Ptilde_nd.c b/Ptilde_nd.c
@@ -37,6 +37,7 @@
 #include "phmc.h"
 #include "Ptilde_nd.h"
 
+#include "bgq/bgq_spinorfield.c"
 
 #define PI 3.141592653589793
 
@@ -117,7 +118,7 @@ void Poly_tilde_ND(spinor *R_s, spinor *R_c, double *dd, int n,
     *aux3c_=NULL, *aux3c=NULL;
 
 
-#if ( defined SSE || defined SSE2 )
+#if 0
   svs_  = calloc(VOLUMEPLUSRAND+1, sizeof(spinor));
   svs   = (spinor *)(((unsigned long int)(svs_)+ALIGN_BASE)&~ALIGN_BASE);
   ds_   = calloc(VOLUMEPLUSRAND+1, sizeof(spinor));
@@ -143,30 +144,20 @@ void Poly_tilde_ND(spinor *R_s, spinor *R_c, double *dd, int n,
   aux3c_= calloc(VOLUMEPLUSRAND+1, sizeof(spinor));
   aux3c = (spinor *)(((unsigned long int)(aux3c_)+ALIGN_BASE)&~ALIGN_BASE);
 #else
-  svs_=calloc(VOLUMEPLUSRAND, sizeof(spinor));
-  svs = svs_;
-  ds_=calloc(VOLUMEPLUSRAND, sizeof(spinor));
-  ds = ds_;
-  dds_=calloc(VOLUMEPLUSRAND, sizeof(spinor));
-  dds = dds_;
-  auxs_=calloc(VOLUMEPLUSRAND, sizeof(spinor));
-  auxs = auxs_;
-  aux2s_=calloc(VOLUMEPLUSRAND, sizeof(spinor));
-  aux2s = aux2s_;
-  aux3s_=calloc(VOLUMEPLUSRAND, sizeof(spinor));
-  aux3s = aux3s_;
-  svc_=calloc(VOLUMEPLUSRAND, sizeof(spinor));
-  svc = svc_;
-  dc_=calloc(VOLUMEPLUSRAND, sizeof(spinor));
-  dc = dc_;
-  ddc_=calloc(VOLUMEPLUSRAND, sizeof(spinor));
-  ddc = ddc_;
-  auxc_=calloc(VOLUMEPLUSRAND, sizeof(spinor));
-  auxc = auxc_;
-  aux2c_=calloc(VOLUMEPLUSRAND, sizeof(spinor));
-  aux2c = aux2c_;
-  aux3c_=calloc(VOLUMEPLUSRAND, sizeof(spinor));
-  aux3c = aux3c_;
+  spinor *mem = malloc_aligned(12 * VOLUMEPLUSRAND * sizeof(spinor), BGQ_ALIGNMENT_L2);
+  bgq_weylfield_collection *col = bgq_spinorfields_allocate(12, mem, VOLUMEPLUSRAND);
+  svs = &mem[0 * VOLUMEPLUSRAND];
+  ds = &mem[1 * VOLUMEPLUSRAND];
+  dds = &mem[2 * VOLUMEPLUSRAND];
+  auxs = &mem[3 * VOLUMEPLUSRAND];
+  aux2s = &mem[4 * VOLUMEPLUSRAND];
+  aux3s = &mem[5 * VOLUMEPLUSRAND];
+  svc = &mem[6 * VOLUMEPLUSRAND];
+  dc = &mem[7 * VOLUMEPLUSRAND];
+  ddc = &mem[8 * VOLUMEPLUSRAND];
+  auxc = &mem[9 * VOLUMEPLUSRAND];
+  aux2c = &mem[10 * VOLUMEPLUSRAND];
+  aux3c = &mem[11 * VOLUMEPLUSRAND];
 #endif
 
   fact1=4/(phmc_cheb_evmax-phmc_cheb_evmin);
@@ -219,18 +210,8 @@ void Poly_tilde_ND(spinor *R_s, spinor *R_c, double *dd, int n,
   assign(&R_s[0], &auxs[0],VOLUME/2);
   assign(&R_c[0], &auxc[0],VOLUME/2);
 
-  free(svs_);
-  free(ds_);
-  free(dds_);
-  free(auxs_);
-  free(aux2s_);
-  free(aux3s_);
-  free(svc_);
-  free(dc_);
-  free(ddc_);
-  free(auxc_);
-  free(aux2c_);
-  free(aux3c_);
+  free(mem);
+  bgq_spinorfields_free(col);
 }
 
 double chebtilde_eval(int M, double *dd, double s){
@@ -279,10 +260,11 @@ void degree_of_Ptilde() {
 
   if(ini==0){
     phmc_ptilde_cheby_coef = calloc(phmc_max_ptilde_degree, sizeof(double)); 
+
     ini=1;
   }   
 
-#if ( defined SSE || defined SSE2 || defined SSE3)
+#if 0
   ss_   = calloc(VOLUMEPLUSRAND/2+1, sizeof(spinor));
   auxs_ = calloc(VOLUMEPLUSRAND/2+1, sizeof(spinor));
   aux2s_= calloc(VOLUMEPLUSRAND/2+1, sizeof(spinor));
@@ -298,12 +280,14 @@ void degree_of_Ptilde() {
   aux2c = (spinor *)(((unsigned long int)(aux2c_)+ALIGN_BASE)&~ALIGN_BASE);
 
 #else
-  ss   =calloc(VOLUMEPLUSRAND/2, sizeof(spinor));
-  auxs =calloc(VOLUMEPLUSRAND/2, sizeof(spinor));
-  aux2s=calloc(VOLUMEPLUSRAND/2, sizeof(spinor));
-  sc   =calloc(VOLUMEPLUSRAND/2, sizeof(spinor));
-  auxc =calloc(VOLUMEPLUSRAND/2, sizeof(spinor));
-  aux2c=calloc(VOLUMEPLUSRAND/2, sizeof(spinor));
+  spinor *mem = malloc_aligned(6 * VOLUMEPLUSRAND/2 * sizeof(spinor), BGQ_ALIGNMENT_L2);
+  ss   = &mem[0 * VOLUMEPLUSRAND/2];
+  auxs = &mem[1 * VOLUMEPLUSRAND/2];
+  aux2s= &mem[2 * VOLUMEPLUSRAND/2];
+  sc   = &mem[3 * VOLUMEPLUSRAND/2];
+  auxc = &mem[4 * VOLUMEPLUSRAND/2];
+  aux2c= &mem[5 * VOLUMEPLUSRAND/2];
+  bgq_weylfield_collection *col = bgq_spinorfields_allocate(6, mem, VOLUMEPLUSRAND/2);
 #endif
 
   Ptilde_cheb_coefs(phmc_cheb_evmin, phmc_cheb_evmax, phmc_ptilde_cheby_coef, phmc_max_ptilde_degree, -1.0); 
@@ -386,20 +370,16 @@ void degree_of_Ptilde() {
     printf("# NDPOLY Acceptance Polynomial degree set to %d\n\n", phmc_ptilde_n_cheby);
   }
 
-#if ( defined SSE || defined SSE2 || defined SSE3)
+#if 0
   free(ss_);
   free(auxs_);
   free(aux2s_);
   free(sc_);
   free(auxc_);
   free(aux2c_);
 #else
-  free(ss);
-  free(auxs);
-  free(aux2s);
-  free(sc);
-  free(auxc);
-  free(aux2c);
+  bgq_spinorfields_free(col);
+  free(mem);
 #endif
   return;
 }
diff --git a/benchmark.input.x86 b/benchmark.input.x86
diff --git a/bgq/bgq_spinorfield.c b/bgq/bgq_spinorfield.c
@@ -1670,6 +1670,8 @@ bgq_weylfield_controlblock *bgq_translate_spinorfield(const spinor *legacy_field
 		assert(result!=NULL);
 		return result;
 	}
+
+	assert(false);
 	return NULL;
 }
 

diff --git a/chebyshev_polynomial_nd.c b/chebyshev_polynomial_nd.c
@@ -33,6 +33,8 @@
 #include "phmc.h"
 #include "chebyshev_polynomial_nd.h"
 
+#include "bgq/bgq_spinorfield.h"
+
 
 
 #define PI 3.141592653589793
@@ -99,7 +101,7 @@ void QdaggerQ_poly(spinor *R_s, spinor *R_c, double *c, int n,
          *aux3c_=NULL, *aux3c=NULL;
 
 
-#if ( defined SSE || defined SSE2 )
+#if 0
    svs_  = calloc(VOLUMEPLUSRAND+1, sizeof(spinor));
    svs   = (spinor *)(((unsigned long int)(svs_)+ALIGN_BASE)&~ALIGN_BASE);
    ds_   = calloc(VOLUMEPLUSRAND+1, sizeof(spinor));
@@ -125,30 +127,20 @@ void QdaggerQ_poly(spinor *R_s, spinor *R_c, double *c, int n,
    aux3c_= calloc(VOLUMEPLUSRAND+1, sizeof(spinor));
    aux3c = (spinor *)(((unsigned long int)(aux3c_)+ALIGN_BASE)&~ALIGN_BASE);
 #else
-   svs_=calloc(VOLUMEPLUSRAND, sizeof(spinor));
-   svs = svs_;
-   ds_=calloc(VOLUMEPLUSRAND, sizeof(spinor));
-   ds = ds_;
-   dds_=calloc(VOLUMEPLUSRAND, sizeof(spinor));
-   dds = dds_;
-   auxs_=calloc(VOLUMEPLUSRAND, sizeof(spinor));
-   auxs = auxs_;
-   aux2s_=calloc(VOLUMEPLUSRAND, sizeof(spinor));
-   aux2s = aux2s_;
-   aux3s_=calloc(VOLUMEPLUSRAND, sizeof(spinor));
-   aux3s = aux3s_;
-   svc_=calloc(VOLUMEPLUSRAND, sizeof(spinor));
-   svc = svc_;
-   dc_=calloc(VOLUMEPLUSRAND, sizeof(spinor));
-   dc = dc_;
-   ddc_=calloc(VOLUMEPLUSRAND, sizeof(spinor));
-   ddc = ddc_;
-   auxc_=calloc(VOLUMEPLUSRAND, sizeof(spinor));
-   auxc = auxc_;
-   aux2c_=calloc(VOLUMEPLUSRAND, sizeof(spinor));
-   aux2c = aux2c_;
-   aux3c_=calloc(VOLUMEPLUSRAND, sizeof(spinor));
-   aux3c = aux3c_;
+   spinor *mem = malloc_aligned(12 * VOLUMEPLUSRAND/2 * sizeof(spinor), BGQ_ALIGNMENT_L2);
+   svs = &mem[0 * VOLUMEPLUSRAND/2];
+   ds = &mem[1 * VOLUMEPLUSRAND/2];
+   dds = &mem[2 * VOLUMEPLUSRAND/2];
+   auxs = &mem[3 * VOLUMEPLUSRAND/2];
+   aux2s = &mem[4 * VOLUMEPLUSRAND/2];
+   aux3s = &mem[5 * VOLUMEPLUSRAND/2];
+   svc = &mem[6 * VOLUMEPLUSRAND/2];
+   dc = &mem[7 * VOLUMEPLUSRAND/2];
+   ddc = &mem[8 * VOLUMEPLUSRAND/2];
+   auxc = &mem[9 * VOLUMEPLUSRAND/2];
+   aux2c = &mem[10 * VOLUMEPLUSRAND/2];
+   aux3c = &mem[11 * VOLUMEPLUSRAND/2];
+   bgq_weylfield_collection *col = bgq_spinorfields_allocate(12, mem, VOLUMEPLUSRAND/2);
 #endif
 
 
@@ -217,19 +209,8 @@ void QdaggerQ_poly(spinor *R_s, spinor *R_c, double *c, int n,
    */
 
 
-   free(svs_);  
-   free(ds_);   
-   free(dds_);  
-   free(auxs_); 
-   free(aux2s_);
-   free(aux3s_);
-   free(svc_);  
-   free(dc_);   
-   free(ddc_);  
-   free(auxc_); 
-   free(aux2c_);
-   free(aux3c_);
-
+   bgq_spinorfields_free(col);
+   free(mem);
 }
 
 
@@ -283,7 +264,7 @@ void degree_of_polynomial_nd(const int degree_of_p){
   }
 
 
-#if ( defined SSE || defined SSE2 || defined SSE3)
+#if 0
   ss_   = calloc(VOLUMEPLUSRAND/2+1, sizeof(spinor));
   auxs_ = calloc(VOLUMEPLUSRAND/2+1, sizeof(spinor));
   aux2s_= calloc(VOLUMEPLUSRAND/2+1, sizeof(spinor));
@@ -299,12 +280,14 @@ void degree_of_polynomial_nd(const int degree_of_p){
   aux2c = (spinor *)(((unsigned long int)(aux2c_)+ALIGN_BASE)&~ALIGN_BASE);
 
 #else
-  ss   =calloc(VOLUMEPLUSRAND/2, sizeof(spinor));
-  auxs =calloc(VOLUMEPLUSRAND/2, sizeof(spinor));
-  aux2s=calloc(VOLUMEPLUSRAND/2, sizeof(spinor));
-  sc   =calloc(VOLUMEPLUSRAND/2, sizeof(spinor));
-  auxc =calloc(VOLUMEPLUSRAND/2, sizeof(spinor));
-  aux2c=calloc(VOLUMEPLUSRAND/2, sizeof(spinor));
+  spinor *mem = malloc_aligned(6 * VOLUMEPLUSRAND/2 * sizeof(spinor), BGQ_ALIGNMENT_L2);
+  ss   = &mem[0 * VOLUMEPLUSRAND/2];
+  auxs = &mem[1 * VOLUMEPLUSRAND/2];
+  aux2s= &mem[2 * VOLUMEPLUSRAND/2];
+  sc   = &mem[3 * VOLUMEPLUSRAND/2];
+  auxc = &mem[4 * VOLUMEPLUSRAND/2];
+  aux2c= &mem[5 * VOLUMEPLUSRAND/2];
+  bgq_weylfield_collection *col = bgq_spinorfields_allocate(6, mem, VOLUMEPLUSRAND/2);
 #endif
 
 
@@ -353,20 +336,16 @@ void degree_of_polynomial_nd(const int degree_of_p){
   }
   /* RECALL THAT WE NEED AN EVEN DEGREE !!!! */
 
-#if ( defined SSE || defined SSE2 || defined SSE3)
+#if 0
    free(ss_);   
    free(auxs_); 
    free(aux2s_);
    free(sc_);   
    free(auxc_); 
    free(aux2c_);
 #else
-   free(ss);   
-   free(auxs); 
-   free(aux2s);
-   free(sc);   
-   free(auxc); 
-   free(aux2c);
+   bgq_spinorfields_free(col);
+   free(mem);
 #endif
 
 }
diff --git a/corestack.py b/corestack.py
@@ -0,0 +1,69 @@
+#! /usr/bin/env python
+import os
+import sys
+import glob
+import os.path
+import re
+import operator
+import shutil
+
+prog = re.compile(r'^Program\s*:\s*(?P<program>[\w/.]+)$');
+reason = re.compile(r'^\*\*\*FAULT');
+failaddr = re.compile(r'^While executing instruction at[^0x]*(?P<addr>\w+)$');
+
+stackstart = re.compile(r'^\+\+\+STACK$');
+stackframe = re.compile(r'^\w+\s*(?P<pc>\w+)$');
+
+
+
+exe = None
+
+def addr2line(addr):
+	#print 'addr2line -e ' + exe + ' ' + addr
+	sys.stdout.flush()
+	os.system('addr2line -e ' + exe + ' ' + addr)
+	sys.stdout.flush()
+
+
+def process_file(filename):
+	global exe
+	f = open(filename, 'r')
+	folder = os.path.split(filename)[0]
+	for line in f:
+		m = prog.match(line)
+		if m:
+			exe = m.group('program')
+			exe = os.path.join(folder, exe)
+			exe = os.path.realpath(exe)
+			print 'Program is', exe
+
+		m = reason.match(line);
+		if m:
+			print line
+
+		m = failaddr.match(line);
+		if m:
+			addr = m.group('addr')
+			sys.stdout.write('Fault at ')
+			sys.stdout.flush()
+			addr2line(addr)
+			print ''
+
+		m = stackstart.match(line)
+		if m:
+			print ''
+			print 'Thread Stack: '
+
+		m = stackframe.match(line)
+		if m:
+			addr = m.group('pc')
+			if addr != '0000000000000000':
+				addr2line('0x' + addr)
+
+
+def main(argv):
+	process_file(argv[1])
+
+
+if __name__ == "__main__":
+    main(sys.argv)
diff --git a/hmc.input.bgq b/hmc.input.bgq
@@ -1,4 +1,4 @@
-T=12
+T=48
 LX=24
 LY=20
 LZ=12
-Original file line number
+Diff line change
@@ -1,4 +1,4 @@
-    T=12
+    T=48
     LX=24
     LY=20
     LZ=12
@@ Expand Down @@