Skip to content

Commit

Permalink
Whitespace cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
wavexx authored and aragilar committed Jun 11, 2018
1 parent 0af70ce commit bfaa83a
Show file tree
Hide file tree
Showing 46 changed files with 428 additions and 558 deletions.
38 changes: 19 additions & 19 deletions api_gen.py
Original file line number Diff line number Diff line change
@@ -1,24 +1,24 @@

"""
Generate the lowest-level Cython bindings to HDF5.
In order to translate HDF5 errors to exceptions, the raw HDF5 API is
wrapped with Cython "error wrappers". These are cdef functions with
the same names and signatures as their HDF5 equivalents, but implemented
in the h5py.defs extension module.
The h5py.defs files (defs.pyx and defs.pxd), along with the "real" HDF5
function definitions (_hdf5.pxd), are auto-generated by this script from
api_functions.txt. This file also contains annotations which indicate
whether a function requires a certain minimum version of HDF5, an
MPI-aware build of h5py, or special error handling.
This script is called automatically by the h5py build system when the
output files are missing, or api_functions.txt has been updated.
See the Line class in this module for documentation of the format for
api_functions.txt.
h5py/_hdf5.pxd: Cython "extern" definitions for HDF5 functions
h5py/defs.pxd: Cython definitions for error wrappers
h5py/defs.pyx: Cython implementations of error wrappers
Expand All @@ -32,19 +32,19 @@ class Line(object):

"""
Represents one line from the api_functions.txt file.
Exists to provide the following attributes:
mpi: Bool indicating if MPI required
error: Bool indicating if special error handling required
version: None or a minimum-version tuple
code: String with function return type
fname: String with function name
sig: String with raw function signature
args: String with sequence of arguments to call function
Example: MPI ERROR 1.8.12 int foo(char* a, size_t b)
.mpi: True
.error: True
.version: (1, 8, 12)
Expand All @@ -53,7 +53,7 @@ class Line(object):
.sig: "char* a, size_t b"
.args: "a, b"
"""

PATTERN = re.compile("""(?P<mpi>(MPI)[ ]+)?
(?P<error>(ERROR)[ ]+)?
(?P<version>([0-9]+\.[0-9]+\.[0-9]+))?
Expand All @@ -69,19 +69,19 @@ class Line(object):
[ ]+[ *]*
(?P<param>[a-zA-Z_]+[a-zA-Z0-9_]*)
""", re.VERBOSE)

def __init__(self, text):
""" Break the line into pieces and populate object attributes.
text: A valid function line, with leading/trailing whitespace stripped.
"""

m = self.PATTERN.match(text)
if m is None:
raise ValueError("Invalid line encountered: {0}".format(text))

parts = m.groupdict()

self.mpi = parts['mpi'] is not None
self.error = parts['error'] is not None
self.version = parts['version']
Expand Down Expand Up @@ -141,14 +141,14 @@ def run(self):
self.cython_imp.write(imp_preamble)

for text in self.functions:

# Directive specifying a header file
if not text.startswith(' ') and not text.startswith('#') and \
len(text.strip()) > 0:
inc = text.split(':')[0]
self.raw_defs.write('cdef extern from "%s.h":\n' % inc)
continue

text = text.strip()

# Whitespace or comment line
Expand All @@ -160,15 +160,15 @@ def run(self):
self.write_raw_sig()
self.write_cython_sig()
self.write_cython_imp()

self.functions.close()
self.cython_imp.close()
self.cython_defs.close()
self.raw_defs.close()

def add_cython_if(self, block):
""" Wrap a block of code in the required "IF" checks """

def wrapif(condition, code):
code = code.replace('\n', '\n ', code.count('\n')-1) # Yes, -1.
code = "IF {0}:\n {1}".format(condition, code)
Expand Down
7 changes: 3 additions & 4 deletions docs_api/automod.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,11 +49,11 @@ def safe_replace(istr, expr, rpl):


try:
class_exprs = dict(
class_exprs = dict(
(re.compile(class_base % x.replace(" ",r"\s"), re.VERBOSE), y) \
for x, y in class_exprs.iteritems() )
except AttributeError:
class_exprs = dict(
class_exprs = dict(
(re.compile(class_base % x.replace(" ",r"\s"), re.VERBOSE), y) \
for x, y in class_exprs.items() )

Expand Down Expand Up @@ -98,7 +98,7 @@ def rpl(target, match):
(?P<post>
\W? # May have trailing punctuation
(?:$|\s+) # Must be followed by whitespace or end of string
)
)
""" % const_exclude, re.VERBOSE)

def replace_constant(istr, current_module):
Expand Down Expand Up @@ -266,4 +266,3 @@ def getsig(docstring):

spx.connect('autodoc-process-signature', proc_sig)
spx.connect('autodoc-process-docstring', proc_doc)

22 changes: 11 additions & 11 deletions examples/collective_io.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# This file is to test collective io in h5py

"""
Author: Jialin Liu, [email protected]
Date: Nov 17, 2015
Expand All @@ -14,14 +14,14 @@
import time
import sys

#"run as "mpirun -np 64 python-mpi collective_io.py 1 file.h5"
#"run as "mpirun -np 64 python-mpi collective_io.py 1 file.h5"
#(1 is for collective write, other numbers for non-collective write)"

colw=1 #default is collective write
filename="parallel_test.hdf5"
if len(sys.argv)>2:
colw = int(sys.argv[1])
filename=str(sys.argv[2])
colw = int(sys.argv[1])
filename=str(sys.argv[2])
comm =MPI.COMM_WORLD
nproc = comm.Get_size()
f = h5py.File(filename, 'w', driver='mpio', comm=MPI.COMM_WORLD)
Expand All @@ -42,17 +42,17 @@
end=start+length_last_rank
temp=np.random.random((end-start,length_y))
if colw==1:
with dset.collective:
dset[start:end,:] = temp
else :
dset[start:end,:] = temp
with dset.collective:
dset[start:end,:] = temp
else:
dset[start:end,:] = temp
comm.Barrier()
timeend=MPI.Wtime()
if rank==0:
if colw==1:
print "collective write time %f" %(timeend-timestart)
else :
print "independent write time %f" %(timeend-timestart)
print "collective write time %f" %(timeend-timestart)
else:
print "independent write time %f" %(timeend-timestart)
print "data size x: %d y: %d" %(length_x, length_y)
print "file size ~%d GB" % (length_x*length_y/1024.0/1024.0/1024.0*8.0)
print "number of processes %d" %nproc
Expand Down
5 changes: 1 addition & 4 deletions examples/multiprocessing_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ def run_calculation():
dset.attrs['YSTART'] = YSTART
dset.attrs['XEXTENT'] = XEXTENT
dset.attrs['YEXTENT'] = YEXTENT

result = pool.imap(compute_row, (x*xincr for x in xrange(NX)))

for idx, arr in enumerate(result):
Expand Down Expand Up @@ -126,6 +126,3 @@ def visualize_file():
else:
print('Fractal found in "mandelbrot.hdf5". Delete file to re-run calculation.')
visualize_file()



29 changes: 14 additions & 15 deletions examples/swmr_inotify_example.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,19 @@
"""
Demonstrate the use of h5py in SWMR mode to monitor the growth of a dataset
on nofication of file modifications.
This demo uses pyinotify as a wrapper of Linux inotify.
https://pypi.python.org/pypi/pyinotify
Usage:
swmr_inotify_example.py [FILENAME [DATASETNAME]]
FILENAME: name of file to monitor. Default: swmr.h5
DATASETNAME: name of dataset to monitor in DATAFILE. Default: data
This script will open the file in SWMR mode and monitor the shape of the
dataset on every write event (from inotify). If another application is
concurrently writing data to the file, the writer must have have switched
dataset on every write event (from inotify). If another application is
concurrently writing data to the file, the writer must have have switched
the file into SWMR mode before this script can open the file.
"""
import asyncore
Expand All @@ -34,25 +34,25 @@ def monitor_dataset(self, filename, datasetname):
self.dset = self.f[datasetname]

self.get_dset_shape()

def get_dset_shape(self):
logging.debug("Refreshing dataset")
self.dset.refresh()

logging.debug("Getting shape")
shape = self.dset.shape
logging.info("Read data shape: %s"%str(shape))
return shape
return shape

def read_dataset(self, latest):
logging.info("Reading out dataset [%d]"%latest)
self.dset[latest:]

def process_IN_MODIFY(self, event):
logging.debug("File modified!")
shape = self.get_dset_shape()
self.read_dataset(shape[0])

def process_IN_CLOSE_WRITE(self, event):
logging.info("File writer closed file")
self.get_dset_shape()
Expand All @@ -62,7 +62,7 @@ def process_IN_CLOSE_WRITE(self, event):

if __name__ == "__main__":
logging.basicConfig(format='%(asctime)s %(levelname)s\t%(message)s',level=logging.INFO)

file_name = "swmr.h5"
if len(sys.argv) > 1:
file_name = sys.argv[1]
Expand All @@ -72,7 +72,7 @@ def process_IN_CLOSE_WRITE(self, event):


wm = pyinotify.WatchManager() # Watch Manager
mask = pyinotify.IN_MODIFY | pyinotify.IN_CLOSE_WRITE
mask = pyinotify.IN_MODIFY | pyinotify.IN_CLOSE_WRITE
evh = EventHandler()
evh.monitor_dataset( file_name, dataset_name )

Expand All @@ -82,4 +82,3 @@ def process_IN_CLOSE_WRITE(self, event):
# Sit in this loop() until the file writer closes the file
# or the user hits ctrl-c
asyncore.loop()

30 changes: 14 additions & 16 deletions examples/swmr_multiprocess.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,18 @@
"""
Demonstrate the use of h5py in SWMR mode to write to a dataset (appending)
Demonstrate the use of h5py in SWMR mode to write to a dataset (appending)
from one process while monitoring the growing dataset from another process.
Usage:
swmr_multiprocess.py [FILENAME [DATASETNAME]]
FILENAME: name of file to monitor. Default: swmrmp.h5
DATASETNAME: name of dataset to monitor in DATAFILE. Default: data
This script will start up two processes: a writer and a reader. The writer
will open/create the file (FILENAME) in SWMR mode, create a dataset and start
appending data to it. After each append the dataset is flushed and an event
sent to the reader process. Meanwhile the reader process will wait for events
from the writer and when triggered it will refresh the dataset and read the
sent to the reader process. Meanwhile the reader process will wait for events
from the writer and when triggered it will refresh the dataset and read the
current shape of it.
"""

Expand All @@ -29,13 +29,13 @@ def __init__(self, event, fname, dsetname, timeout = 2.0):
self._fname = fname
self._dsetname = dsetname
self._timeout = timeout

def run(self):
self.log = logging.getLogger('reader')
self.log.info("Waiting for initial event")
assert self._event.wait( self._timeout )
self._event.clear()

self.log.info("Opening file %s", self._fname)
f = h5py.File(self._fname, 'r', libver='latest', swmr=True)
assert f.swmr_mode
Expand All @@ -58,7 +58,7 @@ def __init__(self, event, fname, dsetname):
self._event = event
self._fname = fname
self._dsetname = dsetname

def run(self):
self.log = logging.getLogger('writer')
self.log.info("Creating file %s", self._fname)
Expand All @@ -72,7 +72,7 @@ def run(self):
f.swmr_mode = True
assert f.swmr_mode
self.log.debug("Sending initial event")
self._event.set()
self._event.set()

# Write loop
for i in range(5):
Expand All @@ -85,7 +85,7 @@ def run(self):
self.log.debug("Flushing data")
dset.flush()
self.log.info("Sending event")
self._event.set()
self._event.set()
finally:
f.close()

Expand All @@ -98,19 +98,17 @@ def run(self):
fname = sys.argv[1]
if len(sys.argv) > 2:
dsetname = sys.argv[2]

event = Event()
reader = SwmrReader(event, fname, dsetname)
writer = SwmrWriter(event, fname, dsetname)

logging.info("Starting reader")
reader.start()
logging.info("Starting reader")
writer.start()

logging.info("Waiting for writer to finish")
writer.join()
logging.info("Waiting for reader to finish")
reader.join()


Loading

0 comments on commit bfaa83a

Please sign in to comment.