Skip to content

Commit 2326203

Browse files
robertu94robertu94
committed
liblossy verison 0.1.0
Major Features: - C compatible API - A generic way to set options for lossy compressors - A generic way to represent data of different types - A generic way to run different lossy decompressors and compressors - A plugin system to easily extend the library with other compressors - An implementation for a liblossy for the SZ error bounded lossy compressor
0 parents  commit 2326203

35 files changed

+2073
-0
lines changed

‎.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
CMakeFiles/
2+
CMakeCache.txt
3+
build

‎CMakeLists.txt

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
cmake_minimum_required(VERSION 3.14 FATAL_ERROR)
2+
project(liblossy VERSION "0.1.0" LANGUAGES CXX C)
3+
4+
enable_testing()
5+
set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
6+
7+
include(GNUInstallDirs)
8+
find_package(ZFP)
9+
find_package(PkgConfig)
10+
pkg_search_module(SZ IMPORTED_TARGET GLOBAL sz)
11+
12+
13+
set(LIBLOSSY_FEATURES "sz")
14+
configure_file(
15+
${CMAKE_CURRENT_SOURCE_DIR}/include/lossy_version.h.in
16+
${CMAKE_CURRENT_BINARY_DIR}/include/lossy_version.h
17+
)
18+
configure_file(
19+
${CMAKE_CURRENT_SOURCE_DIR}/liblossy.pc.in
20+
${CMAKE_CURRENT_BINARY_DIR}/liblossy.pc
21+
@ONLY
22+
)
23+
24+
add_library(liblossy
25+
#public headers
26+
./include/lossy_data.h
27+
./include/lossy_compressor.h
28+
./include/lossy.h
29+
./include/lossy_options.h
30+
./include/lossy_dtype.h
31+
./include/lossy_option.h
32+
33+
#implementations
34+
./src/lossy_options.cc
35+
./src/lossy_option.cc
36+
./src/plugins/liblossy_plugin.cc
37+
./src/plugins/sz_plugin.cc
38+
./src/lossy_data.cc
39+
./src/lossy_dtype.cc
40+
./src/lossy.cc
41+
./src/lossy_compressor.cc
42+
./src/lossy_options_iter.cc
43+
44+
45+
#private headers
46+
./src/plugins.h
47+
./src/plugins/liblossy_plugin.h
48+
./src/lossy_compressor_impl.h
49+
./src/lossy_options_impl.h
50+
)
51+
52+
target_include_directories(liblossy
53+
PUBLIC
54+
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
55+
$<INSTALL_INTERFACE:include>
56+
PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/include
57+
)
58+
target_compile_options(liblossy PRIVATE $<$<CONFIG:Debug>: -Wall -Werror -Wextra -Wpedantic>)
59+
target_link_libraries(liblossy PUBLIC zfp::zfp PkgConfig::SZ)
60+
set_target_properties(liblossy PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED ON)
61+
62+
install(TARGETS liblossy EXPORT LibLossyConfig
63+
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
64+
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
65+
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
66+
)
67+
install(EXPORT LibLossyConfig DESTINATION share/LibLossy/cmake)
68+
install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/liblossy)
69+
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/include/lossy_version.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/liblossy)
70+
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/liblossy.pc DESTINATION ${CMAKE_INSTALL_PREFIX}/share/pkgconfig)
71+
export(TARGETS liblossy FILE LibLossy.cmake)
72+
73+
option(BUILD_TESTS "build the test cases and examples" OFF)
74+
75+
if(BUILD_TESTS)
76+
add_subdirectory(test)
77+
endif()
78+
79+
option(BUILD_DOCS "build the documetation" OFF)
80+
if(BUILD_DOCS)
81+
find_package(Doxygen REQUIRED dot)
82+
set(DOXYGEN_GENERATE_HTML YES)
83+
set(DOXYGEN_GENERATE_MAN YES)
84+
set(DOXYGEN_EXTRACT_LOCAL_METHODS YES)
85+
set(DOXYGEN_EXTRACT_STATIC YES)
86+
set(DOXYGEN_USE_MDFILE_AS_MAINPAGE README.md)
87+
doxygen_add_docs(
88+
docs
89+
${PROJECT_SOURCE_DIR}/README.md
90+
${PROJECT_SOURCE_DIR}/include
91+
${PROJECT_SOURCE_DIR}/src/plugins/liblossy_plugin.h
92+
COMMENT "Generate Documenation"
93+
)
94+
endif()

‎README.md

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
# Liblossy
2+
3+
Liblossy is a library that abstracts differences between different lossless and lossy compressors.
4+
5+
## Configuring Liblossy
6+
7+
Liblossy uses cmake to configure build options. See CMake documentation to see how to configure options
8+
9+
+ `CMAKE_INSTALL_PREFIX` - install the library to a local directory prefix
10+
+ `BUILD_DOCS` - build the project documentation
11+
+ `BUILD_TESTS` - build the test cases
12+
13+
## Building and Installing Liblossy
14+
15+
To build and install the library only.
16+
17+
```bash
18+
BUILD_DIR=build
19+
mkdir $BUILD_DIR
20+
cd $BUILD_DIR
21+
cmake ..
22+
make
23+
make install
24+
```
25+
26+
To build the documentation:
27+
28+
29+
```bash
30+
BUILD_DIR=build
31+
mkdir $BUILD_DIR
32+
cd $BUILD_DIR
33+
cmake . -DBUILD_DOCS=ON
34+
make docs
35+
# the html docs can be found in $BUILD_DIR/html/index.html
36+
# the man pages can be found in $BUILD_DIR/man/
37+
```
38+
39+
## Using Liblossy
40+
41+
Here is a minimal example of how to use liblossy:
42+
43+
44+
~~~{.c}
45+
#include <liblossy.h>
46+
#include <liblossy_ext/compressor_sz.h>
47+
48+
#include "make_input_data.h"
49+
50+
int main(int argc, char *argv[])
51+
{
52+
struct lossy* library = lossy_instance();
53+
struct lossy_compressor* compressor = lossy_get_compressor(library, "sz");
54+
struct lossy_options* sz_options = lossy_compressor_get_options(compressor);
55+
56+
lossy_options_set_integer(sz_options, "sz:mode", ABS);
57+
lossy_options_set_double(sz_options, "sz:abs_error_bound", 0.5);
58+
lossy_compressor_set_options(compressor, sz_options);
59+
60+
//load a 300x300x300 dataset
61+
double* rawinput_data = make_input_data();
62+
size_t dims[] = {300,300,300};
63+
struct lossy_data* input_data = lossy_data_new(lossy_double_dtype, rawinput_data, 3, dims);
64+
65+
//creates an output dataset pointer
66+
struct lossy_data* compressed_data = lossy_data_new_empty(lossy_byte_dtype, 0, NULL);
67+
68+
//configure the decompressed output area
69+
struct lossy_data* decompressed_data = lossy_data_new_empty(lossy_double_dtype, 3, dims);
70+
71+
//compress the data
72+
if(lossy_compressor_compress(compressor, input_data, &compressed_data)) {
73+
printf("%s\n", lossy_compressor_error_msg(compressor));
74+
exit(lossy_compressor_error_code(compressor));
75+
}
76+
77+
//decompress the data
78+
if(lossy_compressor_decompress(compressor, compressed_data, &decompressed_data)) {
79+
printf("%s\n", lossy_compressor_error_msg(compressor));
80+
exit(lossy_compressor_error_code(compressor));
81+
}
82+
83+
//free the decompressed_data
84+
free(lossy_data_ptr(decompressed_data, NULL));
85+
lossy_data_free(decompressed_data);
86+
87+
//free the compressed_data
88+
free(lossy_data_ptr(compressed_data, NULL));
89+
lossy_data_free(compressed_data);
90+
91+
//free the raw input data
92+
free(rawinput_data);
93+
lossy_data_free(input_data);
94+
95+
//free options and the library
96+
lossy_options_free(sz_options);
97+
lossy_release(&library);
98+
return 0;
99+
}
100+
~~~
101+
102+
More examples can be found in `test/`

‎include/liblossy.h

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
/*! \file
2+
* \brief A convenience header for liblossy for use in downstream applications
3+
*/
4+
#include <lossy.h>
5+
#include <lossy_compressor.h>
6+
#include <lossy_options.h>
7+
#include <lossy_options_iter.h>
8+
#include <lossy_option.h>
9+
#include <lossy_data.h>

‎include/liblossy_ext/compressor_sz.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
/** \file
2+
* \brief Includes definitions needed to use the SZ lossy compressor
3+
*/
4+
#include <sz.h>

‎include/lossy.h

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,72 @@
1+
#ifdef __cplusplus
2+
extern "C" {
3+
#endif
4+
5+
#ifndef LIBLOSSY_H
6+
#define LIBLOSSY_H
7+
8+
/*! \file
9+
* \brief Lossy compressor loader
10+
*/
11+
12+
struct lossy;
13+
struct lossy_compressor;
14+
15+
/**
16+
* gets a reference to a possibly shared instance of liblossy; initializes the library if necessary
17+
* \returns a pointer to a library instance
18+
*/
19+
struct lossy* lossy_instance();
20+
21+
22+
/**
23+
* \param[in,out] library the pointer to the library
24+
* \returns informs the library that this instance is no longer required; the pointer passed becomes invalid
25+
*/
26+
void lossy_release(struct lossy** library);
27+
28+
/**
29+
* \param[in] library the pointer to the library
30+
* \returns non-owning pointer to the requested instantiated lossy compressor; it may return the same pointer on multiple calls
31+
*/
32+
struct lossy_compressor* lossy_get_compressor(struct lossy* library, const char* const compressor_id);
33+
34+
/**
35+
* \param[in] library the pointer to the library
36+
* \returns a machine-readable error code for the last error on the library object
37+
*/
38+
int lossy_error_code(struct lossy* library);
39+
40+
/**
41+
* \param[in] library the pointer to the library
42+
* \returns a human-readable error message for the last error on the library object
43+
*/
44+
const char* lossy_error_msg(struct lossy* library);
45+
46+
/**
47+
* \returns a string with version and feature information
48+
*/
49+
const char* lossy_version();
50+
/**
51+
* \returns a string containing all the compressor_ids supported by this version separated by a space
52+
* it will not return more information than the tailored functions below
53+
* \see lossy_get_compressor the compressor_ids may be passed to lossy_get_compressor
54+
*/
55+
const char* lossy_features();
56+
/**
57+
* \returns the major version of the library
58+
*/
59+
unsigned int lossy_major_version();
60+
/**
61+
* \returns the minor version of the library
62+
*/
63+
unsigned int lossy_minor_version();
64+
/**
65+
* \returns the patch version of the library
66+
*/
67+
unsigned int lossy_patch_version();
68+
#endif
69+
70+
#ifdef __cplusplus
71+
}
72+
#endif

‎include/lossy_compressor.h

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
#ifdef __cplusplus
2+
extern "C" {
3+
#endif
4+
5+
#ifndef LIBLOSSY_COMPRESSOR_H
6+
#define LIBLOSSY_COMPRESSOR_H
7+
8+
/*! \file
9+
* \brief Compress, decompress, and configure lossy and lossless compressors
10+
* */
11+
12+
struct lossy_compressor;
13+
struct lossy_data;
14+
struct lossy_options;
15+
16+
//option getting/setting functions
17+
/*!
18+
* \returns a lossy options struct that represents the current options of the compressor
19+
* \param[in] compressor which compressor to get options for
20+
* It may return a pointer to the same memory on multiple calls.
21+
*/
22+
struct lossy_options* lossy_compressor_get_options(struct lossy_compressor const* compressor);
23+
/*!
24+
* sets the options for the lossy_compressor
25+
* \param[in] compressor which compressor to get options for
26+
* \param[in] options the options to set
27+
* \returns 0 if successful, or non-zero code on error.
28+
*/
29+
int lossy_compressor_set_options(struct lossy_compressor* compressor, struct lossy_options const * options);
30+
31+
//compression/decompression functions
32+
/*!
33+
* compresses the data in data using the specified compressor
34+
* \param[in] compressor compressor to be used
35+
* \param[in] input data to be compressed and associated metadata
36+
* \param[in,out] output output data and metadata that may be used for how to output information provide this information if possible
37+
* \returns returns 0 if there is no error or a compressor specific code there is an error
38+
*/
39+
int lossy_compressor_compress(struct lossy_compressor* compressor, struct lossy_data * input, struct lossy_data** output);
40+
/*!
41+
* decompresses the compressed data using the specified compressor
42+
* calling this without calling liblossy_compressor_set_options() has undefined behavior
43+
* decompressing with a compressor with different settings than used for compression has undefined behavior
44+
*
45+
* \param[in] compressor compressor to be used
46+
* \param[in] input data to be decompressed and associated metadata
47+
* \param[in,out] output output data and metadata that may be used for how to output information provide this information if possible
48+
* \returns returns 0 if there is no error or a compressor specific code there is an error
49+
*/
50+
int lossy_compressor_decompress(struct lossy_compressor* compressor, struct lossy_data * data, struct lossy_data** output);
51+
52+
/**
53+
* \param[in] compressor the compressor to query
54+
* \returns last error code for the compressor
55+
*/
56+
int lossy_compressor_error_code(struct lossy_compressor const* compressor);
57+
58+
/**
59+
* \param[in] compressor the compressor to query
60+
* \returns last error message for the compressor
61+
*/
62+
const char* lossy_compressor_error_msg(struct lossy_compressor const* compressor);
63+
64+
65+
/*!
66+
* Get the version and feature information. The version string may include more information than major/minor/patch provide.
67+
* \param[in] compressor the compressor to query
68+
* \returns a implementation specific version string
69+
*/
70+
const char* lossy_compressor_version(struct lossy_compressor const* compressor);
71+
/*!
72+
* \param[in] compressor the compressor to query
73+
* \returns the major version number or 0 if there is none
74+
*/
75+
int lossy_compressor_major_version(struct lossy_compressor const* compressor);
76+
/*!
77+
* \param[in] compressor the compressor to query
78+
* \returns the major version number or 0 if there is none
79+
*/
80+
int lossy_compressor_minor_version(struct lossy_compressor const* compressor);
81+
/*!
82+
* \param[in] compressor the compressor to query
83+
* \returns the major version number or 0 if there is none
84+
*/
85+
int lossy_compressor_patch_version(struct lossy_compressor const* compressor);
86+
87+
88+
89+
90+
#endif
91+
92+
#ifdef __cplusplus
93+
}
94+
#endif

0 commit comments

Comments
 (0)