Skip to content

Commit

Permalink
Adaptions for Blugene/Q
Browse files Browse the repository at this point in the history
  • Loading branch information
@Meinersbur
Meinersbur committed Nov 3, 2012
1 parent 5a7f928 commit 3db8c3a
Show file tree
Hide file tree
Showing 10 changed files with 147 additions and 177 deletions.
4 changes: 4 additions & 0 deletions .cproject
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,13 +24,17 @@
<option id="gnu.cpp.compiler.option.preprocessor.def.840568824" name="Defined symbols (-D)" superClass="gnu.cpp.compiler.option.preprocessor.def" valueType="definedSymbols">
<listOptionValue builtIn="false" value="PARALLELXYZT=1"/>
<listOptionValue builtIn="false" value="BGQ_COORDCHECK=1"/>
<listOptionValue builtIn="false" value="BGQ_QPX=1"/>
<listOptionValue builtIn="false" value="SPI=1"/>
</option>
<inputType id="cdt.managedbuild.tool.gnu.cpp.compiler.input.1665367589" superClass="cdt.managedbuild.tool.gnu.cpp.compiler.input"/>
</tool>
<tool id="cdt.managedbuild.tool.gnu.c.compiler.base.962439157" name="GCC C Compiler" superClass="cdt.managedbuild.tool.gnu.c.compiler.base">
<option id="gnu.c.compiler.option.preprocessor.def.symbols.394627344" name="Defined symbols (-D)" superClass="gnu.c.compiler.option.preprocessor.def.symbols" valueType="definedSymbols">
<listOptionValue builtIn="false" value="PARALLELXYZT=1"/>
<listOptionValue builtIn="false" value="BGQ_COORDCHECK=1"/>
<listOptionValue builtIn="false" value="BGQ_QPX=1"/>
<listOptionValue builtIn="false" value="SPI=1"/>
</option>
<inputType id="cdt.managedbuild.tool.gnu.c.compiler.input.797548390" superClass="cdt.managedbuild.tool.gnu.c.compiler.input"/>
</tool>
Expand Down
1 change: 1 addition & 0 deletions bgq/Makefile.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ libbgq.a: ${libbgq_OBJECTS} ${libbgq_SOBJECTS} Makefile
# rule to generate .d files

$(addsuffix .d, $(libbgq_TARGETS) ${libbgq_STARGETS}): %.d: ${srcdir}/%.c Makefile
echo $(CCDEP) ${DEPFLAGS} ${CPPFLAGS} ${INCLUDES} ${DEFS} $< > $@
@ $(CCDEP) ${DEPFLAGS} ${CPPFLAGS} ${INCLUDES} ${DEFS} $< > $@

# rule to make dependencies
Expand Down
82 changes: 38 additions & 44 deletions bgq/bgq_comm.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,45 +179,40 @@ void bgq_comm_init() {

#ifdef SPI
// here comes the SPI initialization
uint64_t messageSizes[PHYSICAL_LD];
uint64_t roffsets[PHYSICAL_LD];
uint64_t soffsets[PHYSICAL_LD];

spi_num_dirs = 2*(COMM_T+COMM_X+COMM_Y+COMM_Z);
size_t i = 0;
if (COMM_T) {
messageSizes[COMM_ORD_TSEND] = LOCAL_HALO_T * sizeof(bgq_weyl_vec) / PHYSICAL_LP;
messageSizes[COMM_ORD_TRECV] = LOCAL_HALO_T * sizeof(bgq_weyl_vec) / PHYSICAL_LP;
}
if (COMM_X) {
messageSizes[COMM_ORD_XSEND] = PHYSICAL_HALO_X * sizeof(bgq_weyl_vec);
messageSizes[COMM_ORD_XRECV] = PHYSICAL_HALO_X * sizeof(bgq_weyl_vec);
}
if (COMM_Y) {
messageSizes[COMM_ORD_YSEND] = PHYSICAL_HALO_Y * sizeof(bgq_weyl_vec);
messageSizes[COMM_ORD_YRECV] = PHYSICAL_HALO_Y * sizeof(bgq_weyl_vec);
}
if (COMM_Z) {
messageSizes[COMM_ORD_ZSEND] = PHYSICAL_HALO_Z * sizeof(bgq_weyl_vec);
messageSizes[COMM_ORD_ZRECV] = PHYSICAL_HALO_Z * sizeof(bgq_weyl_vec);
}

totalMessageSize = 0;
for(size_t i = 0; i < spi_num_dirs; i++) {
soffsets[i] = totalMessageSize;
totalMessageSize += messageSizes[i];
}

for(size_t i = 0; i < spi_num_dirs; i++) {
// forward here is backward on the right neighbour
// and the other way around...
if(i%2 == 0) {
roffsets[i] = soffsets[i] + messageSizes[i];
}
else {
roffsets[i] = soffsets[i] - messageSizes[i-1];
}
}
uint64_t messageSizes[COMMDIR_COUNT];
uint64_t roffsets[COMMDIR_COUNT];
uint64_t soffsets[COMMDIR_COUNT];

spi_num_dirs = 2*(COMM_T+COMM_X+COMM_Y+COMM_Z);
size_t totalMessageSize = 0;
for (ucoord d_src = 0; d_src < PHYSICAL_LD; d_src+=1) {
bgq_direction d_dst = bgq_direction_revert(d_src);
bgq_dimension dim = bgq_direction2dimension(d_src);
if (!bgq_direction_isDistributed(d_src))
continue;

ucoord commdir_src = bgq_direction2commdir(d_src);
bgq_weylfield_section sec_recv = bgq_direction2section(d_src, false);
size_t secsize = bgq_weyl_section_offset(sec_recv+1) - bgq_weyl_section_offset(sec_recv);

ucoord commdir_dst = bgq_direction2commdir(d_dst);
bgq_weylfield_section sec_send = bgq_direction2section(d_dst, true);
assert(secsize == bgq_weyl_section_offset(sec_send+1) - bgq_weyl_section_offset(sec_send));

messageSizes[commdir_src] = secsize;
soffsets[commdir_src] = totalMessageSize;
totalMessageSize += secsize;

// forward here is backward on the right neighbor
// and the other way around...
if(commdir_src%2 == 0)
roffsets[commdir_src] = soffsets[commdir_src] + messageSizes[commdir_src];
else
roffsets[commdir_src] = soffsets[commdir_src] - messageSizes[commdir_src-1];
}




Personality_t pers;
int rc = 0;
Expand All @@ -233,11 +228,11 @@ void bgq_comm_init() {
get_destinations(mypers);

// adjust the SPI pointers to the send and receive buffers
SPIrecvBuffers = (char *)(recvBuffer);
SPIsendBuffers = (char *)(sendBuffer);
SPIrecvBuffers = (char*)recvBuffer;
SPIsendBuffers = (char*)sendBuffer;

// Setup the FIFO handles
rc = msg_InjFifoInit ( &injFifoHandle,
rc = msg_InjFifoInit (&injFifoHandle,
0, /* startingSubgroupId */
0, /* startingFifoId */
spi_num_dirs, /* numFifos */
Expand All @@ -254,8 +249,7 @@ void bgq_comm_init() {

// Create descriptors
// Injection Direct Put Descriptor, one for each neighbour
SPIDescriptors =
( MUHWI_Descriptor_t *)(((uint64_t)SPIDescriptorsMemory+64)&~(64-1));
SPIDescriptors = (MUHWI_Descriptor_t*)(((uint64_t)SPIDescriptorsMemory+64)&~(64-1));
create_descriptors(SPIDescriptors, messageSizes, soffsets, roffsets, spi_num_dirs);

// test communication
Expand Down
27 changes: 14 additions & 13 deletions bgq/bgq_qpx.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ typedef struct {
_Complex float q[2];
} v2cf;

#ifdef XLC
#ifdef __IBMC__
#define BGQ_VECTOR4DOUBLE_SUBSCRIPT(addr,idx) ( (*((vector4double*)(addr)))[(idx)] ) /* allows subscript like an array */
#else
typedef v4d vector4double;
Expand Down Expand Up @@ -343,7 +343,7 @@ typedef struct {
name

#define bgq_params(name) \
double (name)
vector4double (name)

#define bgq_vector4double_decl(name) \
vector4double name
Expand Down Expand Up @@ -903,18 +903,18 @@ do {\
do {\
bgq_lda_double(NAME2(dst,c00), 0, addr); /* qvlfdxa */ \
void *ptr = (addr); \
bgq_qvlfduxa(NAME2(dst,c00), addr, 32); \
bgq_qvlfduxa(NAME2(dst,c01), addr, 32); \
bgq_qvlfduxa(NAME2(dst,c02), addr, 32); \
bgq_qvlfduxa(NAME2(dst,c10), addr, 32); \
bgq_qvlfduxa(NAME2(dst,c11), addr, 32); \
bgq_qvlfduxa(NAME2(dst,c12), addr, 32); \
bgq_qvlfduxa(NAME2(dst,c20), addr, 32); \
bgq_qvlfduxa(NAME2(dst,c21), addr, 32); \
bgq_qvlfduxa(NAME2(dst,c22), addr, 32); \
bgq_qvlfduxa(NAME2(dst,c00), ptr, 32); \
bgq_qvlfduxa(NAME2(dst,c01), ptr, 32); \
bgq_qvlfduxa(NAME2(dst,c02), ptr, 32); \
bgq_qvlfduxa(NAME2(dst,c10), ptr, 32); \
bgq_qvlfduxa(NAME2(dst,c11), ptr, 32); \
bgq_qvlfduxa(NAME2(dst,c12), ptr, 32); \
bgq_qvlfduxa(NAME2(dst,c20), ptr, 32); \
bgq_qvlfduxa(NAME2(dst,c21), ptr, 32); \
bgq_qvlfduxa(NAME2(dst,c22), ptr, 32); \
} while (0)
#else
#define bgq_su3_matrix_load_double(dest, addr) \
#define bgq_su3_matrix_load_double(dest, addr) \W
bgq_lda_double(NAME2(dest,c00), 0, addr); \
bgq_lda_double(NAME2(dest,c01), 32, addr); \
bgq_lda_double(NAME2(dest,c02), 64, addr); \
Expand Down Expand Up @@ -1288,7 +1288,7 @@ do {\




#if 0
EXTERN_INLINE complexdouble bgq_su3_spinor_getcomplex_double(bgq_su3_spinor_params(spinor), size_t v, size_t c, size_t k) {
assert(0 <= v && v < 4);
assert(0 <= c && c < 4);
Expand Down Expand Up @@ -1356,6 +1356,7 @@ EXTERN_INLINE complexdouble bgq_su3_spinor_getcomplex_double(bgq_su3_spinor_para
return bgq_elem2(result) + bgq_elem3(result) * _Complex_I;
}
}
#endif


#if !BGQ_QPX
Expand Down
4 changes: 2 additions & 2 deletions bgq/bgq_spinorfield.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

#include <mpi.h>
#include <sys/stat.h>

#include <stddef.h>



Expand Down Expand Up @@ -137,7 +137,7 @@ static bgq_weyl_nonvec bgq_weyl_coord_encode(ucoord t, ucoord x, ucoord y, ucoor

static void bgq_weylveck_write(bgq_weyl_vec *target, ucoord k, bgq_weyl_nonvec data) {
ptrdiff_t offset = (uint8_t*)target - g_bgq_sec_comm;
size_t offend = bgq_weyl_section_offset(sec_comm_end) ;
size_t offend = bgq_weyl_section_offset(sec_comm_end);
if (offset > 0 && offset < offend) {
offset += bgq_weyl_section_offset(sec_comm);
ucoord index = bgq_offset2index(offset);
Expand Down
27 changes: 9 additions & 18 deletions bgq/bgq_utils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,6 @@
#ifndef BGQ_UTILS_H_
#define BGQ_UTILS_H_

#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wunused-but-set-variable"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic error "-Wimplicit-int"
#pragma GCC diagnostic error "-Wimplicit-function-declaration"

#include <assert.h>
#include <string.h>
#include <stdio.h>
Expand All @@ -39,20 +33,17 @@
#endif


typedef _Complex double complexdouble;
typedef _Complex float complexfloat;
#ifdef __GNUC__
#pragma GCC diagnostic ignored "-Wunused-variable"
#pragma GCC diagnostic ignored "-Wunused-but-set-variable"
#pragma GCC diagnostic ignored "-Wunused-function"
#pragma GCC diagnostic error "-Wimplicit-int"
#pragma GCC diagnostic error "-Wimplicit-function-declaration"
#endif

#if 0
#define cs2c99(cs) \
((cs).re + _Complex_I*(cs).im)

EXTERN_INLINE complex c992cs(double _Complex c99) {
complex result = { creal(c99), cimag(c99) };
//result.re = __creal(c99);
//result.im = __cimag(c99);
return result;
}
#endif
typedef _Complex double complexdouble;
typedef _Complex float complexfloat;

#define _CONCAT(X,Y) X##Y
#define CONCAT(X,Y) _CONCAT(X,Y)
Expand Down
Loading

0 comments on commit 3db8c3a

Please sign in to comment.