Skip to content

Commit

Permalink
Reformat: 100 -> 120 character lines
Browse files Browse the repository at this point in the history
  • Loading branch information
fknorr committed Oct 20, 2021
1 parent 9817c28 commit ef2a5f7
Show file tree
Hide file tree
Showing 24 changed files with 668 additions and 1,046 deletions.
2 changes: 1 addition & 1 deletion .clang-format
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ BreakConstructorInitializersBeforeComma: true
BreakConstructorInitializers: BeforeComma
BreakAfterJavaFieldAnnotations: false
BreakStringLiterals: true
ColumnLimit: 100
ColumnLimit: 120
CommentPragmas: '^ NOLINT'
CompactNamespaces: false
ConstructorInitializerAllOnOneLineOrOnePerLine: true
Expand Down
19 changes: 5 additions & 14 deletions include/ndzip/array.hh
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,7 @@ class extent {
constexpr extent() noexcept = default;

template<typename... Init,
std::enable_if_t<((sizeof...(Init) == Dims) && ...
&& std::is_convertible_v<Init, index_type>),
int> = 0>
std::enable_if_t<((sizeof...(Init) == Dims) && ... && std::is_convertible_v<Init, index_type>), int> = 0>
NDZIP_UNIVERSAL constexpr extent(const Init &...components) noexcept
: _components{static_cast<index_type>(components)...} {}

Expand Down Expand Up @@ -111,9 +109,7 @@ class extent {
return eq;
}

NDZIP_UNIVERSAL friend bool operator!=(const extent &left, const extent &right) {
return !operator==(left, right);
}
NDZIP_UNIVERSAL friend bool operator!=(const extent &left, const extent &right) { return !operator==(left, right); }

NDZIP_UNIVERSAL iterator begin() { return _components; }

Expand Down Expand Up @@ -156,8 +152,7 @@ NDZIP_UNIVERSAL index_type linear_offset(extent<Dims> position, extent<Dims> spa
namespace ndzip::detail {

template<unsigned Dims>
NDZIP_UNIVERSAL index_type linear_index(
const ndzip::extent<Dims> &size, const ndzip::extent<Dims> &pos) {
NDZIP_UNIVERSAL index_type linear_index(const ndzip::extent<Dims> &size, const ndzip::extent<Dims> &pos) {
index_type l = pos[0];
for (unsigned d = 1; d < Dims; ++d) {
l = l * size[d] + pos[d];
Expand All @@ -182,17 +177,13 @@ class slice {
return detail::linear_index(_size, pos);
}

NDZIP_UNIVERSAL T &operator[](const ndzip::extent<Dims> &pos) const {
return _data[linear_index(pos)];
}
NDZIP_UNIVERSAL T &operator[](const ndzip::extent<Dims> &pos) const { return _data[linear_index(pos)]; }

#ifdef __NVCC__
#pragma diag_suppress 554 // will not be called if T == const T
#endif
// NVCC breaks on a SFINAE'd constructor slice(slice<U, Dims>) with U == std::remove_const_t<T>
NDZIP_UNIVERSAL operator slice<const T, Dims>() const {
return slice<const T, Dims>{_data, _size};
}
NDZIP_UNIVERSAL operator slice<const T, Dims>() const { return slice<const T, Dims>{_data, _size}; }
#ifdef __NVCC__
#pragma diag_default 554
#endif
Expand Down
3 changes: 1 addition & 2 deletions include/ndzip/cpu_encoder.hh
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,7 @@ class cpu_encoder {

size_t compress(const slice<const data_type, dimensions> &data, void *stream) const;

size_t decompress(
const void *stream, size_t bytes, const slice<data_type, dimensions> &data) const;
size_t decompress(const void *stream, size_t bytes, const slice<data_type, dimensions> &data) const;

private:
struct impl;
Expand Down
3 changes: 1 addition & 2 deletions include/ndzip/cuda_encoder.hh
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,7 @@ class cuda_encoder {
size_t compress(const slice<const data_type, dimensions> &item, void *stream,
kernel_duration *out_kernel_duration = nullptr) const;

size_t decompress(
const void *raw_stream, size_t bytes, const slice<data_type, dimensions> &data,
size_t decompress(const void *raw_stream, size_t bytes, const slice<data_type, dimensions> &data,
kernel_duration *out_kernel_duration = nullptr) const;
};

Expand Down
3 changes: 1 addition & 2 deletions include/ndzip/mt_cpu_encoder.hh
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,7 @@ class mt_cpu_encoder {

size_t compress(const slice<const data_type, dimensions> &data, void *stream) const;

size_t decompress(
const void *stream, size_t bytes, const slice<data_type, dimensions> &data) const;
size_t decompress(const void *stream, size_t bytes, const slice<data_type, dimensions> &data) const;

private:
struct impl;
Expand Down
3 changes: 1 addition & 2 deletions include/ndzip/sycl_encoder.hh
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,7 @@ class sycl_encoder {
size_t compress(const slice<const data_type, dimensions> &item, void *stream,
kernel_duration *out_kernel_duration = nullptr) const;

size_t decompress(
const void *raw_stream, size_t bytes, const slice<data_type, dimensions> &data,
size_t decompress(const void *raw_stream, size_t bytes, const slice<data_type, dimensions> &data,
kernel_duration *out_kernel_duration = nullptr) const;

private:
Expand Down
364 changes: 138 additions & 226 deletions src/benchmark/benchmark.cc

Large diffs are not rendered by default.

59 changes: 24 additions & 35 deletions src/compress/compress.cc
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,8 @@ namespace ndzip::detail {
using duration = std::chrono::system_clock::duration;

template<typename Encoder>
void compress_stream(const std::string &in, const std::string &out,
const ndzip::extent<Encoder::dimensions> &size, const Encoder &encoder,
const ndzip::detail::io_factory &io) {
void compress_stream(const std::string &in, const std::string &out, const ndzip::extent<Encoder::dimensions> &size,
const Encoder &encoder, const ndzip::detail::io_factory &io) {
using data_type = typename Encoder::data_type;

const auto array_chunk_length = static_cast<size_t>(num_elements(size) * sizeof(data_type));
Expand All @@ -32,9 +31,9 @@ void compress_stream(const std::string &in, const std::string &out,

while (auto *chunk = in_stream->read_exact()) {
auto input_buffer = static_cast<const data_type *>(chunk);
auto compressed_chunk_length = encoder.compress(
ndzip::slice<const data_type, Encoder::dimensions>(input_buffer, size),
out_stream->get_write_buffer());
auto compressed_chunk_length
= encoder.compress(ndzip::slice<const data_type, Encoder::dimensions>(input_buffer, size),
out_stream->get_write_buffer());
assert(compressed_chunk_length <= max_compressed_chunk_length);
out_stream->commit_chunk(compressed_chunk_length);
compressed_length += compressed_chunk_length;
Expand All @@ -45,21 +44,17 @@ void compress_stream(const std::string &in, const std::string &out,

const auto in_file_size = n_chunks * array_chunk_length;
std::cerr << "raw = " << n_chunks * in_file_size << " bytes";
if (n_chunks > 1) {
std::cerr << " (" << n_chunks << " chunks à " << array_chunk_length << " bytes)";
}
if (n_chunks > 1) { std::cerr << " (" << n_chunks << " chunks à " << array_chunk_length << " bytes)"; }
std::cerr << ", compressed = " << compressed_length << " bytes";
std::cerr << ", ratio = " << std::fixed << std::setprecision(4)
<< (static_cast<double>(compressed_length) / in_file_size);
std::cerr << ", time = " << std::setprecision(3) << std::fixed
<< std::chrono::duration_cast<std::chrono::duration<double>>(duration).count()
<< "s\n";
<< std::chrono::duration_cast<std::chrono::duration<double>>(duration).count() << "s\n";
}

template<typename Encoder>
void decompress_stream(const std::string &in, const std::string &out,
const ndzip::extent<Encoder::dimensions> &size, const Encoder &encoder,
const ndzip::detail::io_factory &io) {
void decompress_stream(const std::string &in, const std::string &out, const ndzip::extent<Encoder::dimensions> &size,
const Encoder &encoder, const ndzip::detail::io_factory &io) {
using data_type = typename Encoder::data_type;
const auto max_compressed_chunk_length = ndzip::compressed_size_bound<data_type>(size);
const auto array_chunk_length = static_cast<size_t>(num_elements(size) * sizeof(data_type));
Expand All @@ -73,8 +68,8 @@ void decompress_stream(const std::string &in, const std::string &out,
if (bytes_in_chunk == 0) { break; }

auto output_buffer = static_cast<data_type *>(out_stream->get_write_buffer());
auto compressed_size = encoder.decompress(chunk, bytes_in_chunk,
ndzip::slice<data_type, Encoder::dimensions>(output_buffer, size));
auto compressed_size = encoder.decompress(
chunk, bytes_in_chunk, ndzip::slice<data_type, Encoder::dimensions>(output_buffer, size));
assert(compressed_size <= bytes_in_chunk);
out_stream->commit_chunk(array_chunk_length);
compressed_bytes_left = bytes_in_chunk - compressed_size;
Expand All @@ -83,8 +78,7 @@ void decompress_stream(const std::string &in, const std::string &out,

template<typename Encoder>
void process_stream(bool decompress, const std::string &in, const std::string &out,
const ndzip::extent<Encoder::dimensions> &size, const Encoder &encoder,
const ndzip::detail::io_factory &io) {
const ndzip::extent<Encoder::dimensions> &size, const Encoder &encoder, const ndzip::detail::io_factory &io) {
if (decompress) {
decompress_stream(in, out, size, encoder, io);
} else {
Expand All @@ -93,19 +87,16 @@ void process_stream(bool decompress, const std::string &in, const std::string &o
}

template<template<typename, unsigned> typename Encoder, typename Data>
void process_stream(bool decompress, const std::vector<size_t> &size_components,
const std::string &in, const std::string &out, const ndzip::detail::io_factory &io) {
void process_stream(bool decompress, const std::vector<size_t> &size_components, const std::string &in,
const std::string &out, const ndzip::detail::io_factory &io) {
switch (size_components.size()) {
case 1:
return process_stream(
decompress, in, out, ndzip::extent{size_components[0]}, Encoder<Data, 1>{}, io);
case 1: return process_stream(decompress, in, out, ndzip::extent{size_components[0]}, Encoder<Data, 1>{}, io);
case 2:
return process_stream(decompress, in, out,
ndzip::extent{size_components[0], size_components[1]}, Encoder<Data, 2>{}, io);
return process_stream(
decompress, in, out, ndzip::extent{size_components[0], size_components[1]}, Encoder<Data, 2>{}, io);
case 3:
return process_stream(decompress, in, out,
ndzip::extent{size_components[0], size_components[1], size_components[2]},
Encoder<Data, 3>{}, io);
ndzip::extent{size_components[0], size_components[1], size_components[2]}, Encoder<Data, 3>{}, io);
// case 4:
// return process_stream(decompress, in, out, ndzip::extent{size_components[0],
// size_components[1],
Expand All @@ -115,9 +106,8 @@ void process_stream(bool decompress, const std::vector<size_t> &size_components,
}

template<typename Data>
void process_stream(bool decompress, const std::vector<size_t> &size_components,
const std::string &encoder, const std::string &in, const std::string &out,
const ndzip::detail::io_factory &io) {
void process_stream(bool decompress, const std::vector<size_t> &size_components, const std::string &encoder,
const std::string &in, const std::string &out, const ndzip::detail::io_factory &io) {
if (encoder == "cpu") {
process_stream<ndzip::cpu_encoder, Data>(decompress, size_components, in, out, io);
#if NDZIP_OPENMP_SUPPORT
Expand All @@ -138,9 +128,9 @@ void process_stream(bool decompress, const std::vector<size_t> &size_components,
}


void process_stream(bool decompress, const std::vector<size_t> &size_components,
const std::string &encoder, const std::string &data_type, const std::string &in,
const std::string &out, const ndzip::detail::io_factory &io) {
void process_stream(bool decompress, const std::vector<size_t> &size_components, const std::string &encoder,
const std::string &data_type, const std::string &in, const std::string &out,
const ndzip::detail::io_factory &io) {
if (data_type == "float") {
process_stream<float>(decompress, size_components, encoder, in, out, io);
} else if (data_type == "double") {
Expand Down Expand Up @@ -212,8 +202,7 @@ int main(int argc, char **argv) {
if (!io_factory) { io_factory = std::make_unique<ndzip::detail::stdio_io_factory>(); }

try {
ndzip::detail::process_stream(
decompress, size_components, encoder, data_type, input, output, *io_factory);
ndzip::detail::process_stream(decompress, size_components, encoder, data_type, input, output, *io_factory);
return EXIT_SUCCESS;
} catch (opts::error &e) {
std::cerr << e.what() << "\n\n" << usage << desc;
Expand Down
24 changes: 7 additions & 17 deletions src/io/io.cc
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,9 @@ class stdio_input_stream final : public input_stream {
assert(_n_chunks > 0 || remainder_from_last_chunk == 0);
assert(remainder_from_last_chunk <= _chunk_size);
size_t bytes_to_read = _chunk_size - remainder_from_last_chunk;
memmove(_chunk, static_cast<std::byte *>(_chunk) + remainder_from_last_chunk,
remainder_from_last_chunk);
auto bytes_read = fread(static_cast<std::byte *>(_chunk) + remainder_from_last_chunk, 1,
bytes_to_read, _file);
if (bytes_read < bytes_to_read && ferror(_file)) {
throw io_error("fread: "s + strerror(errno));
}
memmove(_chunk, static_cast<std::byte *>(_chunk) + remainder_from_last_chunk, remainder_from_last_chunk);
auto bytes_read = fread(static_cast<std::byte *>(_chunk) + remainder_from_last_chunk, 1, bytes_to_read, _file);
if (bytes_read < bytes_to_read && ferror(_file)) { throw io_error("fread: "s + strerror(errno)); }
auto bytes_in_chunk = remainder_from_last_chunk + bytes_read;
if (bytes_in_chunk > 0) { ++_n_chunks; }
return {_chunk, bytes_in_chunk};
Expand Down Expand Up @@ -107,9 +103,7 @@ class stdio_output_stream final : public output_stream {

void commit_chunk(size_t length) override {
_should_zero_buffer = true;
if (fwrite(_buffer, length, 1, _file) < 1) {
throw io_error("fwrite: "s + strerror(errno));
}
if (fwrite(_buffer, length, 1, _file) < 1) { throw io_error("fwrite: "s + strerror(errno)); }
}

private:
Expand All @@ -123,8 +117,7 @@ class stdio_output_stream final : public output_stream {

class mmap_input_stream final : public input_stream {
public:
explicit mmap_input_stream(const std::string &file_name, size_t max_chunk_size)
: _max_chunk_size(max_chunk_size) {
explicit mmap_input_stream(const std::string &file_name, size_t max_chunk_size) : _max_chunk_size(max_chunk_size) {
if (!file_name.empty() && file_name != "-") {
_fd = open(file_name.c_str(), O_RDONLY);
if (_fd == -1) { throw io_error("open: " + file_name + ": " + strerror(errno)); }
Expand Down Expand Up @@ -183,8 +176,7 @@ class mmap_input_stream final : public input_stream {

class mmap_output_stream final : public output_stream {
public:
explicit mmap_output_stream(const std::string &file_name, size_t max_chunk_size)
: _max_chunk_size(max_chunk_size) {
explicit mmap_output_stream(const std::string &file_name, size_t max_chunk_size) : _max_chunk_size(max_chunk_size) {
if (!file_name.empty() && file_name != "-") {
_fd = open(file_name.c_str(), O_RDWR | O_TRUNC | O_CREAT, (mode_t) 0666);
if (_fd == -1) { throw io_error("open: " + file_name + ": " + strerror(errno)); }
Expand Down Expand Up @@ -234,9 +226,7 @@ class mmap_output_stream final : public output_stream {

void truncate(size_t new_capacity) {
unmap_if_mapped();
if (ftruncate(_fd, new_capacity) == -1) {
throw io_error("ftruncate: "s + strerror(errno));
}
if (ftruncate(_fd, new_capacity) == -1) { throw io_error("ftruncate: "s + strerror(errno)); }
_capacity = new_capacity;
}

Expand Down
6 changes: 2 additions & 4 deletions src/io/io.hh
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,7 @@ class io_factory {

class stdio_io_factory : public io_factory {
public:
std::unique_ptr<input_stream> create_input_stream(
const std::string &file_name, size_t chunk_length) const override;
std::unique_ptr<input_stream> create_input_stream(const std::string &file_name, size_t chunk_length) const override;

std::unique_ptr<output_stream> create_output_stream(
const std::string &file_name, size_t max_chunk_length) const override;
Expand All @@ -59,8 +58,7 @@ class stdio_io_factory : public io_factory {

class mmap_io_factory : public io_factory {
public:
std::unique_ptr<input_stream> create_input_stream(
const std::string &file_name, size_t chunk_length) const override;
std::unique_ptr<input_stream> create_input_stream(const std::string &file_name, size_t chunk_length) const override;

std::unique_ptr<output_stream> create_output_stream(
const std::string &file_name, size_t max_chunk_length) const override;
Expand Down
Loading

0 comments on commit ef2a5f7

Please sign in to comment.