diff --git a/CMakeLists.txt b/CMakeLists.txt index 0437a01..aa7e610 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,5 +1,5 @@ cmake_minimum_required(VERSION 3.14 FATAL_ERROR) -project(libpressio VERSION "0.5.0" LANGUAGES CXX C) +project(libpressio VERSION "0.6.0" LANGUAGES CXX C) enable_testing() set(CMAKE_EXPORT_COMPILE_COMMANDS ON) @@ -22,37 +22,46 @@ configure_file( ) add_library(libpressio - #public headers - ./include/pressio_data.h - ./include/pressio_compressor.h - ./include/pressio.h - ./include/pressio_options.h - ./include/pressio_dtype.h - ./include/pressio_option.h - ./include/libpressio_ext/cpp/compressor.h - - #implementations + #core implementation + ./src/pressio.cc + ./src/pressio_compressor.cc + ./src/pressio_data.cc + ./src/pressio_dtype.cc ./src/pressio_metrics.cc - ./src/pressio_options.cc ./src/pressio_option.cc + ./src/pressio_options.cc + ./src/pressio_options_iter.cc + + #plugins ./src/plugins/compressors/compressor_base.cc ./src/plugins/compressors/sz_plugin.cc ./src/plugins/compressors/zfp_plugin.cc + ./src/plugins/metrics/composite.cc ./src/plugins/metrics/metrics_base.cc - ./src/plugins/metrics/time.cc ./src/plugins/metrics/size.cc - ./src/plugins/metrics/composite.cc - ./src/pressio_data.cc - ./src/pressio_dtype.cc - ./src/pressio.cc - ./src/pressio_compressor.cc - ./src/pressio_options_iter.cc + ./src/plugins/metrics/time.cc + ./src/plugins/io/posix.cc + #public headers + include/libpressio.h + include/libpressio_ext/compressors/sz.h + include/libpressio_ext/compressors/zfp.h + include/libpressio_ext/cpp/compressor.h + include/libpressio_ext/cpp/metrics.h + include/libpressio_ext/cpp/plugins.h + include/libpressio_ext/io/posix.h + include/pressio.h + include/pressio_compressor.h + include/pressio_data.h + include/pressio_dtype.h + include/pressio_metrics.h + include/pressio_option.h + include/pressio_options.h + include/pressio_options_iter.h #private headers - ./src/plugins.h - ./src/pressio_compressor_impl.h - ./src/pressio_options_impl.h + src/pressio_compressor_impl.h + src/pressio_options_impl.h ) target_include_directories(libpressio diff --git a/include/libpressio_ext/io/posix.h b/include/libpressio_ext/io/posix.h new file mode 100644 index 0000000..a3fd155 --- /dev/null +++ b/include/libpressio_ext/io/posix.h @@ -0,0 +1,93 @@ +#include + +#ifdef __cplusplus +extern "C" { +#endif + + struct pressio_data; + + /** \file + * \brief IO functions for POSIX compatible systems + * + * NOTE attempting to read files written by different machines has undefined behavior!! + */ + +#ifndef PRESSIO_POSIX_IO +#define PRESSIO_POSIX_IO + + /** read in a file from a POSIX FILE pointer + * + * NOTE attempting to read files written by different machines has undefined behavior!! + * + * \param[in,out] dims description of the dimension of the data if it known or NULL. + * If dims is NULL, the remainder of the file will be read and the size of the resulting pointer will be a 1d pressio_byte_dtype of appropriate length + * If dims is not null, The user SHOULD assume that the memory pointed to by this pointer has been "moved" in a C++11 sense and the user MUST not rely on its contents. + * The implementation MAY return this pointer and reuse the underlying space if pressio_data_has_data(dims) returns true. + * \param[in,out] in_file an file open for reading seeked to the beginning of the data to read in. + * \returns a pointer to a (possibly new) pressio data structure. + * + */ + struct pressio_data* pressio_io_data_fread(struct pressio_data* dims, FILE* in_file); + + + /** read in a file from a POSIX file descriptor + * + * NOTE attempting to read files written by different machines has undefined behavior!! + * + * \param[in,out] dims description of the dimension of the data if it known or NULL. + * If dims is NULL, the remainder of the file will be read and the size of the resulting pointer will be a 1d pressio_byte_dtype of appropriate length + * If dims is not null, The user SHOULD assume that the memory pointed to by this pointer has been "moved" in a C++11 sense and the user MUST not rely on its contents. + * The implementation MAY return this pointer and reuse the underlying space if pressio_data_has_data(ptr) returns true. + * \param[in,out] in_file an file open for reading seeked to the beginning of the data to read in. If dims is not null, only pressio_data_get_bytes(dims) bytes are read. + * \returns a pointer to a (possibly new) pressio data structure. + * + */ + struct pressio_data* pressio_io_data_read(struct pressio_data* dims, int in_filedes); + + /** read in a file at a specifed location on the file-system + * + * NOTE attempting to read files written by different machines has undefined behavior!! + * + * \param[in,out] dims description of the dimension of the data if it known or NULL. + * If dims is NULL, the remainder of the file will be read and the size of the resulting pointer will be a 1d pressio_byte_dtype of appropriate length + * If dims is not null, The user SHOULD assume that the memory pointed to by this pointer has been "moved" in a C++11 sense and the user MUST not rely on its contents. + * The implementation MAY return this pointer and reuse the underlying space if pressio_data_has_data(dims) returns true. + * \param[in,out] in_file an file open for reading seeked to the beginning of the data to read in. + * \returns a pointer to a (possibly new) pressio data structure. + * + */ + struct pressio_data* pressio_io_data_path_read(struct pressio_data* dims, const char* out_file); + + /** write in a file to the specified POSIX FILE pointer + * + * \param[in] data the data to be written. + * \param[in,out] out_file + * \returns the number of bytes written + * + */ + size_t pressio_io_data_fwrite(struct pressio_data* data, FILE* out_file); + + /** write in a file to the specified POSIX file descriptor + * + * \param[in] data the data to be written. + * \param[in,out] out_filedes the file descriptor to write to + * \returns the number of bytes written + * + */ + size_t pressio_io_data_write(struct pressio_data* data, int out_filedes); + + /** write in a file to the specified path on the file-system + * + * \param[in] data the data to be written. + * \param[in,out] path the path to write to + * \returns the number of bytes written + * + */ + size_t pressio_io_data_path_write(struct pressio_data* data, const char* path); + + +#endif /*PRESSIO_POSIX_IO*/ + +#ifdef __cplusplus +} +#endif diff --git a/include/pressio_data.h b/include/pressio_data.h index f3f2a12..d02759b 100644 --- a/include/pressio_data.h +++ b/include/pressio_data.h @@ -137,6 +137,13 @@ size_t pressio_data_get_dimention(struct pressio_data const* data, size_t const */ size_t pressio_data_get_bytes(struct pressio_data const* data); +/** + * returns the total number of elements to represent the data + * \param[in] data the pressio data to query + * \returns the total number of elements to represent the data + */ +size_t pressio_data_num_elements(struct pressio_data const* data); + #endif #ifdef __cplusplus diff --git a/src/plugins/io/posix.cc b/src/plugins/io/posix.cc new file mode 100644 index 0000000..656caca --- /dev/null +++ b/src/plugins/io/posix.cc @@ -0,0 +1,90 @@ +#include +#include +#include +#include "pressio_data.h" +#include "libpressio_ext/io/posix.h" + +namespace { + std::vector get_all_dimentions(struct pressio_data const* data) { + std::vector dims; + if(data) { + for (size_t i = 0; i < pressio_data_num_dimentions(data); ++i) { + dims.emplace_back(pressio_data_get_dimention(data, i)); + } + } + return dims; + } +} + +extern "C" { + + struct pressio_data* pressio_io_data_read(struct pressio_data* dims, int in_filedes) { + pressio_data* ret; + if(dims != nullptr) { + if(pressio_data_has_data(dims)) { + //re-use the buffer provided by dims + ret = dims; + } else { + //create a new buffer of the appropriate size + auto dtype = pressio_data_dtype(dims); + auto dims_v = get_all_dimentions(dims); + pressio_data_free(dims); + ret = pressio_data_new_owning(dtype, dims_v.size(), dims_v.data()); + } + } else { + struct stat statbuf; + if(fstat(in_filedes, &statbuf)) { + return nullptr; + } + size_t size = static_cast(statbuf.st_size); + ret = pressio_data_new_owning(pressio_byte_dtype, 1, &size); + } + read(in_filedes, pressio_data_ptr(ret, nullptr), pressio_data_get_bytes(ret)); + return ret; + } + + struct pressio_data* pressio_io_data_fread(struct pressio_data* dims, FILE* in_file) { + return pressio_io_data_read(dims, fileno(in_file)); + } + + + struct pressio_data* pressio_io_data_path_read(struct pressio_data* dims, const char* path) { + FILE* in_file = fopen(path, "r"); + if(in_file != nullptr) { + auto ret = pressio_io_data_fread(dims, in_file); + fclose(in_file); + return ret; + } else { + return nullptr; + } + } + + + size_t pressio_io_data_fwrite(struct pressio_data* data, FILE* out_file) { + + return fwrite(pressio_data_ptr(data, nullptr), + pressio_dtype_size(pressio_data_dtype(data)), + pressio_data_num_elements(data), + out_file + ); + } + + size_t pressio_io_data_write(struct pressio_data* data, int out_filedes) { + return write(out_filedes, + pressio_data_ptr(data, nullptr), + pressio_data_get_bytes(data) + ); + } + + size_t pressio_io_path_path_write(struct pressio_data* data, const char* path) { + FILE* out_file = fopen(path, "w"); + if(out_file != nullptr) { + auto ret = pressio_io_data_fwrite(data, out_file); + fclose(out_file); + return ret; + } else { + return 0; + } + } + +} diff --git a/src/pressio_data.cc b/src/pressio_data.cc index f308dcc..636b637 100644 --- a/src/pressio_data.cc +++ b/src/pressio_data.cc @@ -11,12 +11,16 @@ void pressio_data_libc_free_fn(void* data, void*) { namespace { template - size_t data_size_in_bytes(pressio_dtype type, size_t dimentions, T const dims[]) { + size_t data_size_in_elements(size_t dimentions, T const dims[]) { size_t totalsize = 1; for (size_t i = 0; i < dimentions; ++i) { totalsize *= dims[i]; } - return totalsize * pressio_dtype_size(type); + return totalsize; + } + template + size_t data_size_in_bytes(pressio_dtype type, size_t dimentions, T const dims[]) { + return data_size_in_elements(dimentions, dims) * pressio_dtype_size(type); } } @@ -89,6 +93,10 @@ struct pressio_data { return data_size_in_bytes(data_dtype, dimentions(), dims.data()); } + size_t num_elements() const { + return data_size_in_elements(dimentions(), dims.data()); + } + private: pressio_data(const pressio_dtype dtype, void* data, @@ -158,7 +166,7 @@ pressio_dtype pressio_data_dtype(struct pressio_data const* data) { } bool pressio_data_has_data(struct pressio_data const* data) { - return data->data() == nullptr; + return data->data() != nullptr; } size_t pressio_data_num_dimentions(struct pressio_data const* data) { @@ -173,4 +181,9 @@ size_t pressio_data_get_bytes(struct pressio_data const* data) { return data->size_in_bytes(); } +size_t pressio_data_num_elements(struct pressio_data const* data) { + return data->num_elements(); +} + + } diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 568b6ef..7f2f1aa 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -37,6 +37,7 @@ endfunction() add_gtest(test_pressio_data.cc) add_gtest(test_pressio_options.cc) add_gtest(test_sz_plugin.cc) +add_gtest(test_io.cc) target_include_directories(test_sz_plugin PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/../include) add_executable(sz_basic sz_basic.c make_input_data.cc) diff --git a/test/test_io.cc b/test/test_io.cc new file mode 100644 index 0000000..b1e8600 --- /dev/null +++ b/test/test_io.cc @@ -0,0 +1,135 @@ +#include +#include +#include "libpressio_ext/io/posix.h" +#include "pressio_data.h" +#include "gtest/gtest.h" + +class PressioDataIOTests: public ::testing::Test { + protected: + void SetUp() { + data = std::vector(size); + std::iota(std::begin(data), std::end(data), 0); + tmp_name = std::string("test_io_readXXXXXX\0"); + tmp_fd = mkstemp(const_cast(tmp_name.data())); + write(tmp_fd, data.data(), sizeof(int)*data.size()); + lseek(tmp_fd, 0, SEEK_SET); + } + + void TearDown() { + close(tmp_fd); + unlink(tmp_name.data()); + } + + const size_t dims[2] = {2,3}; + std::vector data; + int tmp_fd; + std::string tmp_name; + const size_t size = 6; +}; + +TEST_F(PressioDataIOTests, TestReadNullptr) { + auto data = pressio_io_data_read(nullptr, tmp_fd); + EXPECT_EQ(pressio_data_get_bytes(data), sizeof(int)*6); + EXPECT_EQ(pressio_data_dtype(data), pressio_byte_dtype); + pressio_data_free(data); +} + +TEST_F(PressioDataIOTests, TestReadEmpty) { + size_t sizes[] = {2,3}; + auto size_info = pressio_data_new_empty(pressio_int32_dtype, 2, sizes); + auto data = pressio_io_data_read(size_info, tmp_fd); + EXPECT_EQ(pressio_data_get_bytes(data), sizeof(int)*6); + EXPECT_EQ(pressio_data_dtype(data), pressio_int32_dtype); + pressio_data_free(data); +} + +TEST_F(PressioDataIOTests, TestReadFull) { + size_t sizes[] = {2,3}; + auto buffer = pressio_data_new_owning(pressio_int32_dtype, 2, sizes); + auto buffer_ptr = pressio_data_ptr(buffer, nullptr); + auto data = pressio_io_data_read(buffer, tmp_fd); + auto data_ptr = pressio_data_ptr(data, nullptr); + EXPECT_EQ(buffer_ptr, data_ptr); + EXPECT_EQ(pressio_data_get_bytes(data), sizeof(int)*6); + EXPECT_EQ(pressio_data_dtype(data), pressio_int32_dtype); + pressio_data_free(data); +} + + +TEST_F(PressioDataIOTests, TestReadPathNullptr) { + auto data = pressio_io_data_path_read(nullptr, tmp_name.c_str()); + EXPECT_EQ(pressio_data_get_bytes(data), sizeof(int)*6); + EXPECT_EQ(pressio_data_dtype(data), pressio_byte_dtype); + pressio_data_free(data); +} + +TEST_F(PressioDataIOTests, TestReadPathEmpty) { + size_t sizes[] = {2,3}; + auto size_info = pressio_data_new_empty(pressio_int32_dtype, 2, sizes); + auto data = pressio_io_data_path_read(size_info, tmp_name.c_str()); + EXPECT_EQ(pressio_data_get_bytes(data), sizeof(int)*6); + EXPECT_EQ(pressio_data_dtype(data), pressio_int32_dtype); + pressio_data_free(data); +} + +TEST_F(PressioDataIOTests, TestReadPathFull) { + size_t sizes[] = {2,3}; + auto buffer = pressio_data_new_owning(pressio_int32_dtype, 2, sizes); + auto buffer_ptr = pressio_data_ptr(buffer, nullptr); + auto data = pressio_io_data_path_read(buffer, tmp_name.c_str()); + auto data_ptr = pressio_data_ptr(data, nullptr); + EXPECT_EQ(buffer_ptr, data_ptr); + EXPECT_EQ(pressio_data_get_bytes(data), sizeof(int)*6); + EXPECT_EQ(pressio_data_dtype(data), pressio_int32_dtype); + pressio_data_free(data); +} + +TEST_F(PressioDataIOTests, TestFReadNullptr) { + auto tmp = fdopen(tmp_fd, "r"); + auto data = pressio_io_data_fread(nullptr, tmp); + EXPECT_EQ(pressio_data_get_bytes(data), sizeof(int)*6); + EXPECT_EQ(pressio_data_dtype(data), pressio_byte_dtype); + pressio_data_free(data); + fclose(tmp); +} + +TEST_F(PressioDataIOTests, TestFReadEmpty) { + size_t sizes[] = {2,3}; + auto size_info = pressio_data_new_empty(pressio_int32_dtype, 2, sizes); + auto tmp = fdopen(tmp_fd, "r"); + auto data = pressio_io_data_fread(size_info, tmp); + EXPECT_EQ(pressio_data_get_bytes(data), sizeof(int)*6); + EXPECT_EQ(pressio_data_dtype(data), pressio_int32_dtype); + pressio_data_free(data); + fclose(tmp); +} + +TEST_F(PressioDataIOTests, TestFReadFull) { + size_t sizes[] = {2,3}; + auto buffer = pressio_data_new_owning(pressio_int32_dtype, 2, sizes); + auto buffer_ptr = pressio_data_ptr(buffer, nullptr); + auto tmp = fdopen(tmp_fd, "r"); + auto data = pressio_io_data_fread(buffer, tmp); + auto data_ptr = pressio_data_ptr(data, nullptr); + EXPECT_EQ(buffer_ptr, data_ptr); + EXPECT_EQ(pressio_data_get_bytes(data), sizeof(int)*6); + EXPECT_EQ(pressio_data_dtype(data), pressio_int32_dtype); + pressio_data_free(data); + fclose(tmp); +} + + + +TEST_F(PressioDataIOTests, TestWrite) { + size_t sizes[] = {2,3}; + auto data = pressio_data_new_owning(pressio_int32_dtype, 2, sizes); + size_t buffer_size; + int* buffer = static_cast(pressio_data_ptr(data, &buffer_size)); + std::iota(buffer, buffer + pressio_data_num_dimentions(data), 0); + auto tmpwrite_name = std::string("test_io_readXXXXXX\0"); + auto tmpwrite_fd = mkstemp(const_cast(tmpwrite_name.data())); + EXPECT_EQ(pressio_io_data_write(data, tmpwrite_fd), sizeof(int)*6); + pressio_data_free(data); + close(tmpwrite_fd); + unlink(tmpwrite_name.data()); +}