Skip to content

Commit 847297c

Browse files
yuanqj8191sbinet
yuanqj8191
authored and
sbinet
committed
hdf5: add support for chunk cache on a per-dataset basis
* Add api OpenDatasetWith * Support chunk cache * Rename variables' name * Fix typos * Remove unnecessary if condition * Catch returned error * Rename variables * Update comments * Fix bug to return error
1 parent c8645ba commit 847297c

File tree

4 files changed

+161
-17
lines changed

4 files changed

+161
-17
lines changed

‎h5d_public.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
// Copyright ©2019 The Gonum Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style
3+
// license that can be found in the LICENSE file.
4+
5+
package hdf5
6+
7+
// #include "hdf5.h"
8+
import "C"
9+
10+
// Used to unset chunk cache configuration parameter.
11+
// https://support.hdfgroup.org/HDF5/doc/RM/RM_H5P.html#Property-SetChunkCache
12+
const (
13+
D_CHUNK_CACHE_NSLOTS_DEFAULT int = -1 // The number of chunk slots in the raw data chunk cache for this dataset
14+
D_CHUNK_CACHE_NBYTES_DEFAULT int = -1 // The total size of the raw data chunk cache for this dataset
15+
D_CHUNK_CACHE_W0_DEFAULT float64 = -1 // The chunk preemption policy for this dataset
16+
)

‎h5g_group.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,19 @@ func (g *CommonFG) OpenDataset(name string) (*Dataset, error) {
108108
return newDataset(hid, nil), nil
109109
}
110110

111+
// OpenDatasetWith opens and returns a named Dataset with a user-defined PropList.
112+
// The returned dataset must be closed by the user when it is no longer needed.
113+
func (g *CommonFG) OpenDatasetWith(name string, dapl *PropList) (*Dataset, error) {
114+
c_name := C.CString(name)
115+
defer C.free(unsafe.Pointer(c_name))
116+
117+
hid := C.H5Dopen2(g.id, c_name, dapl.id)
118+
if err := checkID(hid); err != nil {
119+
return nil, err
120+
}
121+
return newDataset(hid, nil), nil
122+
}
123+
111124
// NumObjects returns the number of objects in the Group.
112125
func (g *CommonFG) NumObjects() (uint, error) {
113126
var info C.H5G_info_t

‎h5p_proplist.go

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ package hdf5
99
// #include <string.h>
1010
// static inline hid_t _go_hdf5_H5P_DEFAULT() { return H5P_DEFAULT; }
1111
// static inline hid_t _go_hdf5_H5P_DATASET_CREATE() { return H5P_DATASET_CREATE; }
12+
// static inline hid_t _go_hdf5_H5P_DATASET_ACCESS() { return H5P_DATASET_ACCESS; }
1213
import "C"
1314

1415
import (
@@ -32,6 +33,7 @@ type PropList struct {
3233
var (
3334
P_DEFAULT *PropList = newPropList(C._go_hdf5_H5P_DEFAULT())
3435
P_DATASET_CREATE PropType = PropType(C._go_hdf5_H5P_DATASET_CREATE()) // Properties for dataset creation
36+
P_DATASET_ACCESS PropType = PropType(C._go_hdf5_H5P_DATASET_ACCESS()) // Properties for dataset access
3537
)
3638

3739
func newPropList(id C.hid_t) *PropList {
@@ -95,6 +97,25 @@ func (p *PropList) SetDeflate(level int) error {
9597
return h5err(C.H5Pset_deflate(C.hid_t(p.id), C.uint(level)))
9698
}
9799

100+
// SetChunkCache sets the raw data chunk cache parameters.
101+
// To reset them as default, use `D_CHUNK_CACHE_NSLOTS_DEFAULT`, `D_CHUNK_CACHE_NBYTES_DEFAULT` and `D_CHUNK_CACHE_W0_DEFAULT`.
102+
// https://support.hdfgroup.org/HDF5/doc/RM/RM_H5P.html#Property-SetChunkCache
103+
func (p *PropList) SetChunkCache(nslots, nbytes int, w0 float64) error {
104+
return h5err(C.H5Pset_chunk_cache(C.hid_t(p.id), C.size_t(nslots), C.size_t(nbytes), C.double(w0)))
105+
}
106+
107+
// GetChunkCache retrieves the number of chunk slots in the raw data chunk cache hash table.
108+
// https://support.hdfgroup.org/HDF5/doc/RM/RM_H5P.html#Property-GetChunkCache
109+
func (p *PropList) GetChunkCache() (nslots, nbytes int, w0 float64, err error) {
110+
var (
111+
c_nslots C.size_t
112+
c_nbytes C.size_t
113+
c_w0 C.double
114+
)
115+
err = h5err(C.H5Pget_chunk_cache(C.hid_t(p.id), &c_nslots, &c_nbytes, &c_w0))
116+
return int(c_nslots), int(c_nbytes), float64(c_w0), err
117+
}
118+
98119
func h5pclose(id C.hid_t) C.herr_t {
99120
return C.H5Pclose(id)
100121
}

‎h5p_proplist_test.go

Lines changed: 111 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -27,32 +27,32 @@ func TestChunk(t *testing.T) {
2727
)
2828
defer os.Remove(fn)
2929

30-
dclp, err := NewPropList(P_DATASET_CREATE)
30+
dcpl, err := NewPropList(P_DATASET_CREATE)
3131
if err != nil {
3232
t.Fatal(err)
3333
}
34-
defer dclp.Close()
35-
err = dclp.SetChunk(cdims)
34+
defer dcpl.Close()
35+
err = dcpl.SetChunk(cdims)
3636
if err != nil {
3737
t.Fatal(err)
3838
}
3939

40-
cdims_, err := dclp.GetChunk(len(cdims))
40+
cdimsChunk, err := dcpl.GetChunk(len(cdims))
4141
if err != nil {
4242
t.Fatal(err)
4343
}
44-
for i, cdim := range cdims_ {
44+
for i, cdim := range cdimsChunk {
4545
if cdim != cdims[i] {
4646
t.Fatalf("chunked dimensions mismatch: %d != %d", cdims[i], cdim)
4747
}
4848
}
4949

50-
data0, err := save(fn, dsn, dims, dclp)
50+
data0, err := save(fn, dsn, dims, dcpl)
5151
if err != nil {
5252
t.Fatal(err)
5353
}
5454

55-
data1, err := load(fn, dsn)
55+
data1, err := load(fn, dsn, nil)
5656
if err != nil {
5757
t.Fatal(err)
5858
}
@@ -73,26 +73,26 @@ func TestDeflate(t *testing.T) {
7373
)
7474
defer os.Remove(fn)
7575

76-
dclp, err := NewPropList(P_DATASET_CREATE)
76+
dcpl, err := NewPropList(P_DATASET_CREATE)
7777
if err != nil {
7878
t.Fatal(err)
7979
}
80-
defer dclp.Close()
81-
err = dclp.SetChunk(cdims)
80+
defer dcpl.Close()
81+
err = dcpl.SetChunk(cdims)
8282
if err != nil {
8383
t.Fatal(err)
8484
}
85-
err = dclp.SetDeflate(DefaultCompression)
85+
err = dcpl.SetDeflate(DefaultCompression)
8686
if err != nil {
8787
t.Fatal(err)
8888
}
8989

90-
data0, err := save(fn, dsn, dims, dclp)
90+
data0, err := save(fn, dsn, dims, dcpl)
9191
if err != nil {
9292
t.Fatal(err)
9393
}
9494

95-
data1, err := load(fn, dsn)
95+
data1, err := load(fn, dsn, nil)
9696
if err != nil {
9797
t.Fatal(err)
9898
}
@@ -102,7 +102,78 @@ func TestDeflate(t *testing.T) {
102102
}
103103
}
104104

105-
func save(fn, dsn string, dims []uint, dclp *PropList) ([]float64, error) {
105+
func TestChunkCache(t *testing.T) {
106+
DisplayErrors(true)
107+
defer DisplayErrors(false)
108+
var (
109+
fn = "test_chunk_cache.h5"
110+
dsn = "dset_chunk_cache"
111+
dims = []uint{1000, 1000}
112+
cdims = []uint{100, 100}
113+
)
114+
defer os.Remove(fn)
115+
116+
dcpl, err := NewPropList(P_DATASET_CREATE)
117+
if err != nil {
118+
t.Fatal(err)
119+
}
120+
defer dcpl.Close()
121+
err = dcpl.SetChunk(cdims)
122+
if err != nil {
123+
t.Fatal(err)
124+
}
125+
126+
cdimsChunk, err := dcpl.GetChunk(len(cdims))
127+
if err != nil {
128+
t.Fatal(err)
129+
}
130+
for i, cdim := range cdimsChunk {
131+
if cdim != cdims[i] {
132+
t.Fatalf("chunked dimensions mismatch: %d != %d", cdims[i], cdim)
133+
}
134+
}
135+
136+
data0, err := save(fn, dsn, dims, dcpl)
137+
if err != nil {
138+
t.Fatal(err)
139+
}
140+
141+
dapl, err := NewPropList(P_DATASET_ACCESS)
142+
if err != nil {
143+
t.Fatal(err)
144+
}
145+
defer dapl.Close()
146+
147+
nslots, nbytes, w0, err := dapl.GetChunkCache()
148+
if err != nil {
149+
t.Fatal(err)
150+
}
151+
152+
nslotsNew, nbytesNew, w0New := nslots*4, nbytes*2, w0/3
153+
if err := dapl.SetChunkCache(nslotsNew, nbytesNew, w0New); err != nil {
154+
t.Fatal(err)
155+
}
156+
if err := checkChunkCache(nslotsNew, nbytesNew, w0New, dapl); err != nil {
157+
t.Fatal(err)
158+
}
159+
160+
data1, err := load(fn, dsn, dapl)
161+
if err != nil {
162+
t.Fatal(err)
163+
}
164+
if err := compare(data0, data1); err != nil {
165+
t.Fatal(err)
166+
}
167+
168+
if err := dapl.SetChunkCache(D_CHUNK_CACHE_NSLOTS_DEFAULT, D_CHUNK_CACHE_NBYTES_DEFAULT, D_CHUNK_CACHE_W0_DEFAULT); err != nil {
169+
t.Fatal(err)
170+
}
171+
if err := checkChunkCache(nslots, nbytes, w0, dapl); err != nil {
172+
t.Fatal(err)
173+
}
174+
}
175+
176+
func save(fn, dsn string, dims []uint, dcpl *PropList) ([]float64, error) {
106177
f, err := CreateFile(fn, F_ACC_TRUNC)
107178
if err != nil {
108179
return nil, err
@@ -114,7 +185,7 @@ func save(fn, dsn string, dims []uint, dclp *PropList) ([]float64, error) {
114185
return nil, err
115186
}
116187

117-
dset, err := f.CreateDatasetWith(dsn, T_NATIVE_DOUBLE, dspace, dclp)
188+
dset, err := f.CreateDatasetWith(dsn, T_NATIVE_DOUBLE, dspace, dcpl)
118189
if err != nil {
119190
return nil, err
120191
}
@@ -132,14 +203,19 @@ func save(fn, dsn string, dims []uint, dclp *PropList) ([]float64, error) {
132203
return data, nil
133204
}
134205

135-
func load(fn, dsn string) ([]float64, error) {
206+
func load(fn, dsn string, dapl *PropList) ([]float64, error) {
136207
f, err := OpenFile(fn, F_ACC_RDONLY)
137208
if err != nil {
138209
return nil, err
139210
}
140211
defer f.Close()
141212

142-
dset, _ := f.OpenDataset(dsn)
213+
var dset *Dataset
214+
if dapl == nil {
215+
dset, err = f.OpenDataset(dsn)
216+
} else {
217+
dset, err = f.OpenDatasetWith(dsn, dapl)
218+
}
143219
if err != nil {
144220
return nil, err
145221
}
@@ -168,3 +244,21 @@ func compare(ds0, ds1 []float64) error {
168244
}
169245
return nil
170246
}
247+
248+
func checkChunkCache(nslots, nbytes int, w0 float64, dapl *PropList) error {
249+
nslotsCache, nbytesCache, w0Cache, err := dapl.GetChunkCache()
250+
if err != nil {
251+
return err
252+
}
253+
254+
if nslotsCache != nslots {
255+
return fmt.Errorf("`nslots` mismatch: %d != %d", nslots, nslotsCache)
256+
}
257+
if nbytesCache != nbytes {
258+
return fmt.Errorf("`nbytes` mismatch: %d != %d", nbytes, nbytesCache)
259+
}
260+
if math.Abs(w0Cache-w0) > 1e-5 {
261+
return fmt.Errorf("`w0` mismatch: %.6f != %.6f", w0, w0Cache)
262+
}
263+
return nil
264+
}

0 commit comments

Comments
 (0)