Merge branch 'develop'

Author: lee218llnl

pynamic-pyMPI-2.6a1/so_generator.py

@@ -199,7 +199,7 @@ def generate_c_file(file_prefix_in, my_id, num_functions, call_depth, extern, ut
     f.close()
 
 # compile a .c file into a Python-usable .so file
-def compile_file(file_prefix, i, num_utility_files, include_dir, CC):
+def compile_file(file_prefix, num_module_files, num_utility_files, include_dir, CC):
     filename = file_prefix + '.c'
     cwd = os.getcwd()
     outfile = file_prefix + '.so'
@@ -208,6 +208,9 @@ def compile_file(file_prefix, i, num_utility_files, include_dir, CC):
     else:
         command = '%s -g -fPIC -shared' %(CC)
     if file_prefix.find('module') != -1:
+        if file_prefix.find('begin') != -1:
+            for i in range(num_module_files):
+                command += ' -lmodule' + str(i)
         command += ' -I%s' %(include_dir)
         command += ' -Wl,-rpath=' + cwd + ' -L' + cwd
         for i in range(num_utility_files):
@@ -260,6 +263,7 @@ def barrier(self):
 """
     f.write(text)
 
+    f.write('import libmodulebegin\n')
     for i in range(num_files):
         f.write('import libmodule' + str(i) + '\n')
     f.write('import libmodulefinal\n')
@@ -274,6 +278,7 @@ def barrier(self):
 """
     f.write(text)
 
+    f.write('libmodulebegin.begin_break_here()\n')
     for i in range(num_files):
         f.write('libmodule' + str(i) + '.libmodule' + str(i) + '_entry()\n')
     f.write('libmodulefinal.break_here()\n')
@@ -332,7 +337,7 @@ def run_so_generator(num_files, avg_num_functions, call_depth, extern, seed, see
     for p,d,f in os.walk('./'):
         if p == './':
             for file in f:
-                if (file.find('libmodule') != -1 or file.find('libutility') != -1 or file.find('pynamic.h') != -1) and file.find('libmodulefinal.c') == -1:
+                if (file.find('libmodule') != -1 or file.find('libutility') != -1 or file.find('pynamic.h') != -1) and file.find('libmodulefinal.c') == -1 and file.find('libmodulebegin.c') == -1:
                     os.remove(file)
 
     if extern:
@@ -372,6 +377,7 @@ def run_so_generator(num_files, avg_num_functions, call_depth, extern, seed, see
         for i in range(num_utility_files):
             command = 'ar cru libpynamic.a %s' %(file_prefix+str(i)+'.o')
             run_command(command)
+    pynamic_header_file.write('void initlibmodulebegin();\n')
     for i in range(num_files - num_utility_files):
         pynamic_header_file.write('void initlibmodule%d();\n' %(i))
     pynamic_header_file.write('void initlibmodulefinal();\n')
@@ -390,6 +396,10 @@ def run_so_generator(num_files, avg_num_functions, call_depth, extern, seed, see
     command = 'ranlib libpynamic.a'
     run_command(command)
 
+    compile_file("libmodulebegin", num_files - num_utility_files, 0, include_dir, CC)
+    command = 'ar cru libpynamic.a libmodulebegin.o'
+    run_command(command)
+
     f = open("pyMPI_initialize.c", "r")
     lines = f.readlines()
     f.close()
@@ -399,6 +409,7 @@ def run_so_generator(num_files, avg_num_functions, call_depth, extern, seed, see
         if line.find('pyMPI_Macros.h') != -1:
             f.write('#include "pynamic.h"\n')
         if line.find('PyImport_AppendInittab') != -1:
+            f.write('  PyImport_AppendInittab("libmodulebegin", initlibmodulebegin);\n')
             for i in range(num_files - num_utility_files):
                 f.write('  PyImport_AppendInittab("libmodule%d", initlibmodule%d);\n' %(i, i))
             f.write('  PyImport_AppendInittab("libmodulefinal", initlibmodulefinal);\n')

pynamic.README

@@ -2,6 +2,8 @@
 # FILE: pynamic.README
 #
 # Updates:
+#          Feb 26 2018 GLL: Allow parallel builds with -j option
+#                           updated config.guess (to work for ppc64le)
 #          Apr 29 2014 GLL: Renamed pynamic-bigexe to pynamic-bigexe-pyMPI
 #                           create pynamic-bigexe-sdb-pyMPI
 #                           added libmodulefinal with break_here function
@@ -96,56 +98,62 @@
     % cd pynamic-pyMPI-2.6a1
     % config_pynamic.py
 
-    It will print out options and arguments:
+      USAGE:
+              config_pynamic.py <num_files> <avg_num_functions> [options] [-c <configure_options>]
 
-    --------------------------------------------------------
-    USAGE:
-            config_pynamic.py <num_files> <avg_num_functions> [options] [-c <configure_options>]
+              <num_files> = total number of shared objects to produce
+              <avg_num_functions> = average number of functions per shared object
+
+      OPTIONS:
 
-            <num_files> = total number of shared objects to produce
-            <avg_num_functions> = average number of functions per shared object
+      -c <configure_options>
+              pass the whitespace separated list of <configure_options> to configure
+              when building pyMPI.  All args after -c are sent to configure and not
+              interpreted by Pynamic
 
-    OPTIONS:
+      -b
+              generate the pynamic-bigexe-pyMPI and pynamic-bigexe-sdb-pyMPI executables
 
-    -c <configure_options>
-            pass the whitespace separated list of <configure_options> to configure
-            when building pyMPI.  All args after -c are sent to configure and not
-            interpreted by Pynamic
+      -d <call_depth>
+              maximum Pynamic call stack depth, default = 10
 
-    -b
-            generate the pynamic-bigexe-pyMPI and pynamic-bigexe-sdb-pyMPI executables
+      -e
+              enables external functions to call across modules
 
-    -d <call_depth>
-            maximum Pynamic call stack depth, default = 10
+      -i <python_include_dir>
+              add <python_include_dir> when compiling modules
 
-    -e
-            enables external functions to call across modules
+      -j <num_processes>
+              build in parallel with a max of <num_processes> processes
 
-    -i <python_include_dir>
-            add <python_include_dir> when compiling modules
+      -n <length>
+              add <length> characters to the function names
 
-    -n <length>
-            add <length> characters to the function names
+      -p
+              add a print statement to every generated function
 
-    -p
-            add a print statement to every generated function
+      -s <random_seed>
+              seed to the random number generator
 
-    -s <random_seed>
-            seed to the random number generator
+      -u <num_utility_mods> <avg_num_u_functions>
+              create <num_utility_mods> math library-like utility modules
+              with an average of <avg_num_u_functions> functions
+              NOTE: Number of python modules = <num_files> - <avg_num_u_functions>
 
-    -u <num_utility_mods> <avg_num_u_functions>
-            create <num_utility_mods> math library-like utility modules
-            with an average of <avg_num_u_functions> functions
-            NOTE: Number of python modules = <num_files> - <avg_num_u_functions>
+      --with-cc=<command>
+              use the C compiler located at <command> to build Pynamic modules.
+
+      --with-python=<command>
+              use the python located at <command> to build Pynamic modules.  Will
+              also be passed to the pyMPI configure script.
 
-    --with-cc=<command>
-            use the C compiler located at <command> to build Pynamic modules.
 
-    --with-python=<command>
-            use the python located at <command> to build Pynamic modules.  Will
-            also be passed to the pyMPI configure script.
     --------------------------------------------------------
 
+    Building a large number of shared objects may take a long time, in
+    which case it is advised to compiler those files in parallel using
+    the -j option, setting the value to the number of cores on the node.
+
     Options and arguments are provided so that a tester can model certain
     static properties of a Python-based scientific applications.
     For example, if the tester wants to model a code that has