From f6bb0742addfc0ed4272066825a92e46c676c60f Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Fri, 24 Jan 2025 10:16:02 +0800 Subject: [PATCH 01/16] tmp save Signed-off-by: guo-shaoge --- dbms/src/Columns/ColumnAggregateFunction.h | 10 +- dbms/src/Columns/ColumnArray.cpp | 49 ++-- dbms/src/Columns/ColumnArray.h | 20 +- dbms/src/Columns/ColumnConst.h | 10 +- dbms/src/Columns/ColumnDecimal.cpp | 74 ++++-- dbms/src/Columns/ColumnDecimal.h | 64 ++--- dbms/src/Columns/ColumnFixedString.cpp | 75 +++++- dbms/src/Columns/ColumnFixedString.h | 56 +++-- dbms/src/Columns/ColumnFunction.h | 10 +- dbms/src/Columns/ColumnNullable.cpp | 31 ++- dbms/src/Columns/ColumnNullable.h | 12 +- dbms/src/Columns/ColumnString.cpp | 233 +++++++++++++----- dbms/src/Columns/ColumnString.h | 53 ++-- dbms/src/Columns/ColumnTuple.h | 17 +- dbms/src/Columns/ColumnVector.cpp | 47 ++-- dbms/src/Columns/ColumnVector.h | 28 ++- dbms/src/Columns/IColumn.h | 13 +- dbms/src/Columns/IColumnDummy.h | 10 +- .../gtest_column_serialize_deserialize.cpp | 62 ++--- 19 files changed, 577 insertions(+), 297 deletions(-) diff --git a/dbms/src/Columns/ColumnAggregateFunction.h b/dbms/src/Columns/ColumnAggregateFunction.h index f5e7963ef4d..1c1cd619d67 100644 --- a/dbms/src/Columns/ColumnAggregateFunction.h +++ b/dbms/src/Columns/ColumnAggregateFunction.h @@ -167,7 +167,8 @@ class ColumnAggregateFunction final : public COWPtrHelper & /* byte_size */, - const TiDB::TiDBCollatorPtr & /* collator */) const override + const TiDB::TiDBCollatorPtr & /* collator */, + const NullMap * /* nullmap */) const override { throw Exception( "Method countSerializeByteSizeForCmp is not supported for " + getName(), @@ -180,7 +181,8 @@ class ColumnAggregateFunction final : public COWPtrHelper & /* byte_size */, const IColumn::Offsets & /* offsets */, - const TiDB::TiDBCollatorPtr & /* collator */) const override + const TiDB::TiDBCollatorPtr & /* collator */, + const NullMap * /* nullmap */) const override { throw Exception( "Method countSerializeByteSizeForCmpColumnArray is not supported for " + getName(), @@ -199,7 +201,7 @@ class ColumnAggregateFunction final : public COWPtrHelper & /* pos */, size_t /* start */, size_t /* length */, - bool /* has_null */, + const NullMap * /* nullmap */, const TiDB::TiDBCollatorPtr & /* collator */, String * /* sort_key_container */) const override { @@ -217,7 +219,7 @@ class ColumnAggregateFunction final : public COWPtrHelper & /* pos */, size_t /* start */, size_t /* length */, - bool /* has_null */, + const NullMap * /* nullmap */, const IColumn::Offsets & /* offsets */, const TiDB::TiDBCollatorPtr & /* collator */, String * /* sort_key_container */) const override diff --git a/dbms/src/Columns/ColumnArray.cpp b/dbms/src/Columns/ColumnArray.cpp index 0ba742e092c..d661461b62f 100644 --- a/dbms/src/Columns/ColumnArray.cpp +++ b/dbms/src/Columns/ColumnArray.cpp @@ -220,18 +220,19 @@ const char * ColumnArray::deserializeAndInsertFromArena(const char * pos, const void ColumnArray::countSerializeByteSizeForCmp( PaddedPODArray & byte_size, - const TiDB::TiDBCollatorPtr & collator) const + const TiDB::TiDBCollatorPtr & collator, + const NullMap * nullmap) const { - countSerializeByteSizeImpl(byte_size, collator); + countSerializeByteSizeImpl(byte_size, collator, nullmap); } void ColumnArray::countSerializeByteSize(PaddedPODArray & byte_size) const { - countSerializeByteSizeImpl(byte_size, nullptr); + countSerializeByteSizeImpl(byte_size, nullptr, nullptr); } -template -void ColumnArray::countSerializeByteSizeImpl(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator) +template +void ColumnArray::countSerializeByteSizeImpl(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator, const NullMap * nullmap) const { RUNTIME_CHECK_MSG(byte_size.size() == size(), "size of byte_size({}) != column size({})", byte_size.size(), size()); @@ -251,8 +252,8 @@ void ColumnArray::countSerializeByteSizeImpl(PaddedPODArray & byte_size, for (size_t i = 0; i < size; ++i) byte_size[i] += sizeof(UInt32); - if constexpr (for_compare) - getData().countSerializeByteSizeForCmpColumnArray(byte_size, getOffsets(), collator); + if constexpr (compare_semantics) + getData().countSerializeByteSizeForCmpColumnArray(byte_size, getOffsets(), collator, nullmap); else getData().countSerializeByteSizeForColumnArray(byte_size, getOffsets()); } @@ -261,35 +262,39 @@ void ColumnArray::serializeToPosForCmp( PaddedPODArray & pos, size_t start, size_t length, - bool has_null, + const NullMap * nullmap, const TiDB::TiDBCollatorPtr & collator, String * sort_key_container) const { - if (has_null) - serializeToPosImpl(pos, start, length, collator, sort_key_container); + if (nullmap != nullptr) + serializeToPosImpl(pos, start, length, collator, sort_key_container, nullmap); else - serializeToPosImpl(pos, start, length, collator, sort_key_container); + serializeToPosImpl(pos, start, length, collator, sort_key_container, nullmap); } void ColumnArray::serializeToPos(PaddedPODArray & pos, size_t start, size_t length, bool has_null) const { if (has_null) - serializeToPosImpl(pos, start, length, nullptr, nullptr); + serializeToPosImpl(pos, start, length, nullptr, nullptr, nullptr); else - serializeToPosImpl(pos, start, length, nullptr, nullptr); + serializeToPosImpl(pos, start, length, nullptr, nullptr, nullptr); } -template +template void ColumnArray::serializeToPosImpl( PaddedPODArray & pos, size_t start, size_t length, const TiDB::TiDBCollatorPtr & collator, - String * sort_key_container) const + String * sort_key_container, + const NullMap * nullmap) const { RUNTIME_CHECK_MSG(length <= pos.size(), "length({}) > size of pos({})", length, pos.size()); RUNTIME_CHECK_MSG(start + length <= size(), "start({}) + length({}) > size of column({})", start, length, size()); + static_assert(!(has_null && has_nullmap)); + assert(!has_nullmap || (nullmap && nullmap->size() == size())); + /// countSerializeByteSize has already checked that the size of one element is not greater than UINT32_MAX for (size_t i = 0; i < length; ++i) { @@ -298,14 +303,20 @@ void ColumnArray::serializeToPosImpl( if (pos[i] == nullptr) continue; } + UInt32 len = sizeAt(start + i); + if constexpr (has_nullmap) + { + if ((*nullmap)[i] != 0) + len = 0; + } tiflash_compiler_builtin_memcpy(pos[i], &len, sizeof(UInt32)); pos[i] += sizeof(UInt32); } - if constexpr (for_compare) + if constexpr (compare_semantics) getData() - .serializeToPosForCmpColumnArray(pos, start, length, has_null, getOffsets(), collator, sort_key_container); + .serializeToPosForCmpColumnArray(pos, start, length, nullmap, getOffsets(), collator, sort_key_container); else getData().serializeToPosForColumnArray(pos, start, length, has_null, getOffsets()); } @@ -320,7 +331,7 @@ void ColumnArray::deserializeAndInsertFromPos(PaddedPODArray & pos, bool deserializeAndInsertFromPosImpl(pos, use_nt_align_buffer); } -template +template void ColumnArray::deserializeAndInsertFromPosImpl(PaddedPODArray & pos, bool use_nt_align_buffer) { auto & offsets = getOffsets(); @@ -336,7 +347,7 @@ void ColumnArray::deserializeAndInsertFromPosImpl(PaddedPODArray & pos, pos[i] += sizeof(UInt32); } - if constexpr (for_compare) + if constexpr (compare_semantics) getData().deserializeForCmpAndInsertFromPosColumnArray(pos, offsets, use_nt_align_buffer); else getData().deserializeAndInsertFromPosForColumnArray(pos, offsets, use_nt_align_buffer); diff --git a/dbms/src/Columns/ColumnArray.h b/dbms/src/Columns/ColumnArray.h index 7eb91feb028..64b890a354e 100644 --- a/dbms/src/Columns/ColumnArray.h +++ b/dbms/src/Columns/ColumnArray.h @@ -44,18 +44,19 @@ class ColumnArray final : public COWPtrHelper ColumnArray(const ColumnArray &) = default; - template - void countSerializeByteSizeImpl(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator) const; + template + void countSerializeByteSizeImpl(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator, const NullMap * nullmap) const; - template + template void serializeToPosImpl( PaddedPODArray & pos, size_t start, size_t length, const TiDB::TiDBCollatorPtr & collator, - String * sort_key_container) const; + String * sort_key_container, + const NullMap * nullmap) const; - template + template void deserializeAndInsertFromPosImpl(PaddedPODArray & pos, bool use_nt_align_buffer); public: @@ -96,14 +97,15 @@ class ColumnArray final : public COWPtrHelper String &) const override; const char * deserializeAndInsertFromArena(const char * pos, const TiDB::TiDBCollatorPtr &) override; - void countSerializeByteSizeForCmp(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator) + void countSerializeByteSizeForCmp(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator, const NullMap * nullmap) const override; void countSerializeByteSize(PaddedPODArray & byte_size) const override; void countSerializeByteSizeForCmpColumnArray( PaddedPODArray & /* byte_size */, const IColumn::Offsets & /* array_offsets */, - const TiDB::TiDBCollatorPtr & /* collator */) const override + const TiDB::TiDBCollatorPtr & /* collator */, + const NullMap * /* nullmap */) const override { throw Exception( "Method countSerializeByteSizeForCmpColumnArray is not supported for " + getName(), @@ -122,7 +124,7 @@ class ColumnArray final : public COWPtrHelper PaddedPODArray & pos, size_t start, size_t length, - bool has_null, + const NullMap * nullmap, const TiDB::TiDBCollatorPtr & collator, String * sort_key_container) const override; void serializeToPos(PaddedPODArray & pos, size_t start, size_t length, bool has_null) const override; @@ -131,7 +133,7 @@ class ColumnArray final : public COWPtrHelper PaddedPODArray & /* pos */, size_t /* start */, size_t /* length */, - bool /* has_null */, + const NullMap * /* nullmap */, const IColumn::Offsets & /* array_offsets */, const TiDB::TiDBCollatorPtr & /* collator */, String * /* sort_key_container */) const override diff --git a/dbms/src/Columns/ColumnConst.h b/dbms/src/Columns/ColumnConst.h index c17ba694ead..6d506236116 100644 --- a/dbms/src/Columns/ColumnConst.h +++ b/dbms/src/Columns/ColumnConst.h @@ -114,7 +114,8 @@ class ColumnConst final : public COWPtrHelper void countSerializeByteSizeForCmp( PaddedPODArray & /* byte_size */, - const TiDB::TiDBCollatorPtr & /* collator */) const override + const TiDB::TiDBCollatorPtr & /* collator */, + const NullMap * /* nullmap */) const override { throw Exception( "Method countSerializeByteSizeForCmp is not supported for " + getName(), @@ -128,7 +129,8 @@ class ColumnConst final : public COWPtrHelper void countSerializeByteSizeForCmpColumnArray( PaddedPODArray & /* byte_size */, const IColumn::Offsets & /* array_offsets */, - const TiDB::TiDBCollatorPtr & /* collaotr */) const override + const TiDB::TiDBCollatorPtr & /* collaotr */, + const NullMap * /* nullmap */) const override { throw Exception( "Method countSerializeByteSizeForCmpColumnArray is not supported for " + getName(), @@ -147,7 +149,7 @@ class ColumnConst final : public COWPtrHelper PaddedPODArray & /* pos */, size_t /* start */, size_t /* length */, - bool /* has_null */, + const NullMap * /* nullmap */, const TiDB::TiDBCollatorPtr & /* collator */, String * /* sort_key_container */) const override { @@ -166,7 +168,7 @@ class ColumnConst final : public COWPtrHelper PaddedPODArray & /* pos */, size_t /* start */, size_t /* length */, - bool /* has_null */, + const NullMap * /* nullmap */, const IColumn::Offsets & /* array_offsets */, const TiDB::TiDBCollatorPtr & /* collator */, String * /* sort_key_container */) const override diff --git a/dbms/src/Columns/ColumnDecimal.cpp b/dbms/src/Columns/ColumnDecimal.cpp index 36a774975ca..d2a3b2dc735 100644 --- a/dbms/src/Columns/ColumnDecimal.cpp +++ b/dbms/src/Columns/ColumnDecimal.cpp @@ -140,13 +140,13 @@ const char * ColumnDecimal::deserializeAndInsertFromArena(const char * pos, c } template -template -void ColumnDecimal::countSerializeByteSizeImpl(PaddedPODArray & byte_size) const +template +void ColumnDecimal::countSerializeByteSizeImpl(PaddedPODArray & byte_size, const NullMap *) const { RUNTIME_CHECK_MSG(byte_size.size() == size(), "size of byte_size({}) != column size({})", byte_size.size(), size()); size_t size = byte_size.size(); - if constexpr (for_compare && is_Decimal256) + if constexpr (compare_semantics && is_Decimal256) { for (size_t i = 0; i < size; ++i) { @@ -162,7 +162,7 @@ void ColumnDecimal::countSerializeByteSizeImpl(PaddedPODArray & byte_ // TODO add unit test template -template +template void ColumnDecimal::countSerializeByteSizeForColumnArrayImpl( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets) const @@ -173,7 +173,7 @@ void ColumnDecimal::countSerializeByteSizeForColumnArrayImpl( byte_size.size(), array_offsets.size()); - if constexpr (for_compare && is_Decimal256) + if constexpr (compare_semantics && is_Decimal256) { size_t size = array_offsets.size(); for (size_t i = 0; i < size; ++i) @@ -194,12 +194,16 @@ void ColumnDecimal::countSerializeByteSizeForColumnArrayImpl( } template -template -void ColumnDecimal::serializeToPosImpl(PaddedPODArray & pos, size_t start, size_t length) const +template +void ColumnDecimal::serializeToPosImpl(PaddedPODArray & pos, size_t start, size_t length, const NullMap * nullmap) const { RUNTIME_CHECK_MSG(length <= pos.size(), "length({}) > size of pos({})", length, pos.size()); RUNTIME_CHECK_MSG(start + length <= size(), "start({}) + length({}) > size of column({})", start, length, size()); + static_assert(!(has_null && has_nullmap)); + assert(!has_nullmap || (nullmap && nullmap->size() == size())); + + T def_val{}; for (size_t i = 0; i < length; ++i) { if constexpr (has_null) @@ -208,12 +212,29 @@ void ColumnDecimal::serializeToPosImpl(PaddedPODArray & pos, size_t s continue; } - if constexpr (for_compare && is_Decimal256) + if constexpr (compare_semantics && is_Decimal256) { + if constexpr (has_nullmap) + { + if ((*nullmap)[i] != 0) + { + pos[i] = serializeDecimal256Helper(pos[i], def_val); + continue; + } + } pos[i] = serializeDecimal256Helper(pos[i], data[start + i]); } else { + if constexpr (has_nullmap) + { + if ((*nullmap)[i] != 0) + { + tiflash_compiler_builtin_memcpy(pos[i], &def_val, sizeof(T)); + pos[i] += sizeof(T); + continue; + } + } tiflash_compiler_builtin_memcpy(pos[i], &data[start + i], sizeof(T)); pos[i] += sizeof(T); } @@ -221,12 +242,13 @@ void ColumnDecimal::serializeToPosImpl(PaddedPODArray & pos, size_t s } template -template +template void ColumnDecimal::serializeToPosForColumnArrayImpl( PaddedPODArray & pos, size_t start, size_t length, - const IColumn::Offsets & array_offsets) const + const IColumn::Offsets & array_offsets, + const NullMap * nullmap) const { RUNTIME_CHECK_MSG(length <= pos.size(), "length({}) > size of pos({})", length, pos.size()); RUNTIME_CHECK_MSG( @@ -241,6 +263,9 @@ void ColumnDecimal::serializeToPosForColumnArrayImpl( array_offsets.back(), size()); + static_assert(!(has_null && has_nullmap)); + assert(!has_nullmap || (nullmap && nullmap->size() == array_offsets.size())); + for (size_t i = 0; i < length; ++i) { if constexpr (has_null) @@ -250,32 +275,31 @@ void ColumnDecimal::serializeToPosForColumnArrayImpl( } size_t len = array_offsets[start + i] - array_offsets[start + i - 1]; - if constexpr (for_compare && is_Decimal256) + if constexpr (compare_semantics && is_Decimal256) { + if constexpr (has_nullmap) + { + if ((*nullmap)[i] != 0) + continue; + } for (size_t j = 0; j < len; ++j) pos[i] = serializeDecimal256Helper(pos[i], data[array_offsets[start + i - 1] + j]); } else { - if (len <= 4) - { - for (size_t j = 0; j < len; ++j) - tiflash_compiler_builtin_memcpy( - pos[i] + j * sizeof(T), - &data[array_offsets[start + i - 1] + j], - sizeof(T)); - } - else + if constexpr (has_nullmap) { - inline_memcpy(pos[i], &data[array_offsets[start + i - 1]], len * sizeof(T)); + if ((*nullmap)[i] != 0) + continue; } + inline_memcpy(pos[i], &data[array_offsets[start + i - 1]], len * sizeof(T)); pos[i] += len * sizeof(T); } } } template -template +template void ColumnDecimal::deserializeAndInsertFromPosImpl( PaddedPODArray & pos, bool use_nt_align_buffer [[maybe_unused]]) @@ -285,7 +309,7 @@ void ColumnDecimal::deserializeAndInsertFromPosImpl( // is_complex_decimal256 is true means Decimal256 is serialized by [bool, limb_count, n * limb]. // NT optimization is not implemented for simplicity. - static const bool is_complex_decimal256 = (for_compare && is_Decimal256); + static const bool is_complex_decimal256 = (compare_semantics && is_Decimal256); #ifdef TIFLASH_ENABLE_AVX_SUPPORT if (use_nt_align_buffer) @@ -383,7 +407,7 @@ void ColumnDecimal::deserializeAndInsertFromPosImpl( } template -template +template void ColumnDecimal::deserializeAndInsertFromPosForColumnArrayImpl( PaddedPODArray & pos, const IColumn::Offsets & array_offsets, @@ -410,7 +434,7 @@ void ColumnDecimal::deserializeAndInsertFromPosForColumnArrayImpl( for (size_t i = 0; i < size; ++i) { size_t len = array_offsets[start_point + i] - array_offsets[start_point + i - 1]; - if constexpr (for_compare && is_Decimal256) + if constexpr (compare_semantics && is_Decimal256) { for (size_t j = 0; j < len; ++j) pos[i] = const_cast( diff --git a/dbms/src/Columns/ColumnDecimal.h b/dbms/src/Columns/ColumnDecimal.h index 18211b13177..1f02419df8b 100644 --- a/dbms/src/Columns/ColumnDecimal.h +++ b/dbms/src/Columns/ColumnDecimal.h @@ -101,26 +101,27 @@ class ColumnDecimal final : public COWPtrHelper - void countSerializeByteSizeImpl(PaddedPODArray & byte_size) const; - template + template + void countSerializeByteSizeImpl(PaddedPODArray & byte_size, const NullMap *) const; + template void countSerializeByteSizeForColumnArrayImpl( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets) const; - template - void serializeToPosImpl(PaddedPODArray & pos, size_t start, size_t length) const; - template + template + void serializeToPosImpl(PaddedPODArray & pos, size_t start, size_t length, const NullMap * nullmap) const; + template void serializeToPosForColumnArrayImpl( PaddedPODArray & pos, size_t start, size_t length, - const IColumn::Offsets & array_offsets) const; + const IColumn::Offsets & array_offsets, + const NullMap * nullmap) const; - template + template void deserializeAndInsertFromPosImpl(PaddedPODArray & pos, bool use_nt_align_buffer [[maybe_unused]]); - template + template void deserializeAndInsertFromPosForColumnArrayImpl( PaddedPODArray & pos, const IColumn::Offsets & array_offsets, @@ -173,19 +174,20 @@ class ColumnDecimal final : public COWPtrHelper & byte_size, const TiDB::TiDBCollatorPtr &) const override + void countSerializeByteSizeForCmp(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr &, const NullMap *) const override { - countSerializeByteSizeImpl(byte_size); + countSerializeByteSizeImpl(byte_size, nullptr); } void countSerializeByteSize(PaddedPODArray & byte_size) const override { - countSerializeByteSizeImpl(byte_size); + countSerializeByteSizeImpl(byte_size, nullptr); } void countSerializeByteSizeForCmpColumnArray( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets, - const TiDB::TiDBCollatorPtr &) const override + const TiDB::TiDBCollatorPtr &, + const NullMap *) const override { countSerializeByteSizeForColumnArrayImpl(byte_size, array_offsets); } @@ -200,44 +202,46 @@ class ColumnDecimal final : public COWPtrHelper & pos, size_t start, size_t length, - bool has_null, + const NullMap * nullmap, const TiDB::TiDBCollatorPtr &, String *) const override { - if (has_null) - serializeToPosImpl(pos, start, length); + if (nullmap != nullptr) + serializeToPosImpl(pos, start, length, nullmap); else - serializeToPosImpl(pos, start, length); + serializeToPosImpl(pos, start, length, nullptr); } void serializeToPos(PaddedPODArray & pos, size_t start, size_t length, bool has_null) const override { if (has_null) - serializeToPosImpl(pos, start, length); + serializeToPosImpl(pos, start, length, nullptr); else - serializeToPosImpl(pos, start, length); + serializeToPosImpl(pos, start, length, nullptr); } void serializeToPosForCmpColumnArray( PaddedPODArray & pos, size_t start, size_t length, - bool has_null, + const NullMap * nullmap, const IColumn::Offsets & array_offsets, const TiDB::TiDBCollatorPtr &, String *) const override { - if (has_null) - serializeToPosForColumnArrayImpl( + if (nullmap != nullptr) + serializeToPosForColumnArrayImpl( pos, start, length, - array_offsets); + array_offsets, + nullmap); else - serializeToPosForColumnArrayImpl( + serializeToPosForColumnArrayImpl( pos, start, length, - array_offsets); + array_offsets, + nullptr); } void serializeToPosForColumnArray( PaddedPODArray & pos, @@ -247,17 +251,19 @@ class ColumnDecimal final : public COWPtrHelper( + serializeToPosForColumnArrayImpl( pos, start, length, - array_offsets); + array_offsets, + nullptr); else - serializeToPosForColumnArrayImpl( + serializeToPosForColumnArrayImpl( pos, start, length, - array_offsets); + array_offsets, + nullptr); } void deserializeForCmpAndInsertFromPos(PaddedPODArray & pos, bool use_nt_align_buffer) override diff --git a/dbms/src/Columns/ColumnFixedString.cpp b/dbms/src/Columns/ColumnFixedString.cpp index b15e420a1ce..dde147faef5 100644 --- a/dbms/src/Columns/ColumnFixedString.cpp +++ b/dbms/src/Columns/ColumnFixedString.cpp @@ -134,18 +134,33 @@ const char * ColumnFixedString::deserializeAndInsertFromArena(const char * pos, return pos + n; } -void ColumnFixedString::countSerializeByteSize(PaddedPODArray & byte_size) const +template +void ColumnFixedString::countSerializeByteSizeImpl(PaddedPODArray & byte_size, const NullMap * nullmap) const { RUNTIME_CHECK_MSG(byte_size.size() == size(), "size of byte_size({}) != column size({})", byte_size.size(), size()); + assert(!nullmap || nullmap->size() == size()); + size_t size = byte_size.size(); for (size_t i = 0; i < size; ++i) + { + if constexpr (has_nullmap) + { + if ((*nullmap)[i] != 0) + { + byte_size[i] += 1; + continue; + } + } byte_size[i] += n; + } } -void ColumnFixedString::countSerializeByteSizeForColumnArray( +template +void ColumnFixedString::countSerializeByteSizeForColumnArrayImpl( PaddedPODArray & byte_size, - const IColumn::Offsets & array_offsets) const + const IColumn::Offsets & array_offsets, + const NullMap * nullmap) const { RUNTIME_CHECK_MSG( byte_size.size() == array_offsets.size(), @@ -153,25 +168,40 @@ void ColumnFixedString::countSerializeByteSizeForColumnArray( byte_size.size(), array_offsets.size()); + assert(!nullmap || nullmap->size() == array_offsets.size()); + size_t size = array_offsets.size(); for (size_t i = 0; i < size; ++i) + { + if constexpr (has_nullmap) + { + if ((*nullmap)[i] != 0) + { + byte_size[i] += array_offsets[i] - array_offsets[i - 1]; + continue; + } + } byte_size[i] += n * (array_offsets[i] - array_offsets[i - 1]); + } } void ColumnFixedString::serializeToPos(PaddedPODArray & pos, size_t start, size_t length, bool has_null) const { if (has_null) - serializeToPosImpl(pos, start, length); + serializeToPosImpl(pos, start, length, nullptr); else - serializeToPosImpl(pos, start, length); + serializeToPosImpl(pos, start, length, nullptr); } -template -void ColumnFixedString::serializeToPosImpl(PaddedPODArray & pos, size_t start, size_t length) const +template +void ColumnFixedString::serializeToPosImpl(PaddedPODArray & pos, size_t start, size_t length, const NullMap * nullmap) const { RUNTIME_CHECK_MSG(length <= pos.size(), "length({}) > size of pos({})", length, pos.size()); RUNTIME_CHECK_MSG(start + length <= size(), "start({}) + length({}) > size of column({})", start, length, size()); + static_assert(!(has_null && has_nullmap)); + assert(!has_nullmap || (nullmap && nullmap->size() == size())); + for (size_t i = 0; i < length; ++i) { if constexpr (has_null) @@ -179,6 +209,18 @@ void ColumnFixedString::serializeToPosImpl(PaddedPODArray & pos, size_t if (pos[i] == nullptr) continue; } + if constexpr (has_nullmap) + { + if ((*nullmap)[i] != 0) + { + for (size_t j = 0; j < n; ++j) + { + *(pos[i]) = '\0'; + pos[i] += 1; + } + continue; + } + } inline_memcpy(pos[i], &chars[n * (start + i)], n); pos[i] += n; } @@ -192,17 +234,18 @@ void ColumnFixedString::serializeToPosForColumnArray( const IColumn::Offsets & array_offsets) const { if (has_null) - serializeToPosForColumnArrayImpl(pos, start, length, array_offsets); + serializeToPosForColumnArrayImpl(pos, start, length, array_offsets, nullptr); else - serializeToPosForColumnArrayImpl(pos, start, length, array_offsets); + serializeToPosForColumnArrayImpl(pos, start, length, array_offsets, nullptr); } -template +template void ColumnFixedString::serializeToPosForColumnArrayImpl( PaddedPODArray & pos, size_t start, size_t length, - const IColumn::Offsets & array_offsets) const + const IColumn::Offsets & array_offsets, + const NullMap * nullmap) const { RUNTIME_CHECK_MSG(length <= pos.size(), "length({}) > size of pos({})", length, pos.size()); RUNTIME_CHECK_MSG( @@ -217,6 +260,9 @@ void ColumnFixedString::serializeToPosForColumnArrayImpl( array_offsets.back(), size()); + static_assert(!(has_null && has_nullmap)); + assert(!has_nullmap || (nullmap && nullmap->size() == array_offsets.size())); + for (size_t i = 0; i < length; ++i) { if constexpr (has_null) @@ -224,8 +270,13 @@ void ColumnFixedString::serializeToPosForColumnArrayImpl( if (pos[i] == nullptr) continue; } - size_t len = array_offsets[start + i] - array_offsets[start + i - 1]; + if constexpr (has_nullmap) + { + if ((*nullmap)[i] != 0) + continue; + } + inline_memcpy(pos[i], &chars[n * array_offsets[start + i - 1]], n * len); pos[i] += n * len; } diff --git a/dbms/src/Columns/ColumnFixedString.h b/dbms/src/Columns/ColumnFixedString.h index 161cef5cf8a..0ac2ff89aa4 100644 --- a/dbms/src/Columns/ColumnFixedString.h +++ b/dbms/src/Columns/ColumnFixedString.h @@ -54,16 +54,25 @@ class ColumnFixedString final : public COWPtrHelper , chars(src.chars.begin(), src.chars.end()) , n(src.n){}; + template + void countSerializeByteSizeImpl(PaddedPODArray & byte_size, const NullMap * nullmap) const; - template - void serializeToPosImpl(PaddedPODArray & pos, size_t start, size_t length) const; + template + void countSerializeByteSizeForColumnArrayImpl( + PaddedPODArray & byte_size, + const IColumn::Offsets & array_offsets, + const NullMap * nullmap) const; - template + template + void serializeToPosImpl(PaddedPODArray & pos, size_t start, size_t length, const NullMap * nullmap) const; + + template void serializeToPosForColumnArrayImpl( PaddedPODArray & pos, size_t start, size_t length, - const IColumn::Offsets & array_offsets) const; + const IColumn::Offsets & array_offsets, + const NullMap * nullmap) const; public: std::string getName() const override { return "FixedString(" + std::to_string(n) + ")"; } @@ -115,7 +124,7 @@ class ColumnFixedString final : public COWPtrHelper const char * deserializeAndInsertFromArena(const char * pos, const TiDB::TiDBCollatorPtr &) override; - void countSerializeByteSizeForCmp(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator) + void countSerializeByteSizeForCmp(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator, const NullMap * nullmap) const override { // collator->sortKey() will change the string length, which may exceeds n. @@ -123,35 +132,51 @@ class ColumnFixedString final : public COWPtrHelper !collator, "{} doesn't support countSerializeByteSizeForCmp when collator is not null", getName()); - countSerializeByteSize(byte_size); + if (nullmap != nullptr) + countSerializeByteSizeImpl(byte_size, nullmap); + else + countSerializeByteSizeImpl(byte_size, nullptr); + } + void countSerializeByteSize(PaddedPODArray & byte_size) const override + { + countSerializeByteSizeImpl(byte_size, nullptr); } - void countSerializeByteSize(PaddedPODArray & byte_size) const override; void countSerializeByteSizeForCmpColumnArray( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets, - const TiDB::TiDBCollatorPtr & collator) const override + const TiDB::TiDBCollatorPtr & collator, + const NullMap * nullmap) const override { RUNTIME_CHECK_MSG( !collator, "{} doesn't support countSerializeByteSizeForCmpColumnArray when collator is not null", getName()); - countSerializeByteSizeForColumnArray(byte_size, array_offsets); + if (nullmap != nullptr) + countSerializeByteSizeForColumnArrayImpl(byte_size, array_offsets, nullmap); + else + countSerializeByteSizeForColumnArrayImpl(byte_size, array_offsets, nullptr); } void countSerializeByteSizeForColumnArray( PaddedPODArray & byte_size, - const IColumn::Offsets & array_offsets) const override; + const IColumn::Offsets & array_offsets) const override + { + countSerializeByteSizeForColumnArrayImpl(byte_size, array_offsets, nullptr); + } void serializeToPosForCmp( PaddedPODArray & pos, size_t start, size_t length, - bool has_null, + const NullMap * nullmap, const TiDB::TiDBCollatorPtr & collator, String *) const override { RUNTIME_CHECK_MSG(!collator, "{} doesn't support serializeToPosForCmp when collator is not null", getName()); - serializeToPos(pos, start, length, has_null); + if (nullmap != nullptr) + serializeToPosImpl(pos, start, length, nullmap); + else + serializeToPosImpl(pos, start, length, nullptr); } void serializeToPos(PaddedPODArray & pos, size_t start, size_t length, bool has_null) const override; @@ -159,7 +184,7 @@ class ColumnFixedString final : public COWPtrHelper PaddedPODArray & pos, size_t start, size_t length, - bool has_null, + const NullMap * nullmap, const IColumn::Offsets & array_offsets, const TiDB::TiDBCollatorPtr & collator, String *) const override @@ -168,7 +193,10 @@ class ColumnFixedString final : public COWPtrHelper !collator, "{} doesn't support serializeToPosForCmpColumnArray when collator is not null", getName()); - serializeToPosForColumnArray(pos, start, length, has_null, array_offsets); + if (nullmap != nullptr) + serializeToPosForColumnArrayImpl(pos, start, length, array_offsets, nullmap); + else + serializeToPosForColumnArrayImpl(pos, start, length, array_offsets, nullptr); } void serializeToPosForColumnArray( PaddedPODArray & pos, diff --git a/dbms/src/Columns/ColumnFunction.h b/dbms/src/Columns/ColumnFunction.h index fd43650d255..24fdcb3f1e7 100644 --- a/dbms/src/Columns/ColumnFunction.h +++ b/dbms/src/Columns/ColumnFunction.h @@ -122,7 +122,8 @@ class ColumnFunction final : public COWPtrHelper void countSerializeByteSizeForCmp( PaddedPODArray & /* byte_size */, - const TiDB::TiDBCollatorPtr & /* collator */) const override + const TiDB::TiDBCollatorPtr & /* collator */, + const NullMap * /* nullmap */) const override { throw Exception( "Method countSerializeByteSizeForCmp is not supported for " + getName(), @@ -136,7 +137,8 @@ class ColumnFunction final : public COWPtrHelper void countSerializeByteSizeForCmpColumnArray( PaddedPODArray & /* byte_size */, const IColumn::Offsets & /* offsets */, - const TiDB::TiDBCollatorPtr & /* collator */) const override + const TiDB::TiDBCollatorPtr & /* collator */, + const NullMap * /* nullmap */) const override { throw Exception( "Method countSerializeByteSizeForCmpColumnArray is not supported for " + getName(), @@ -155,7 +157,7 @@ class ColumnFunction final : public COWPtrHelper PaddedPODArray & /* pos */, size_t /* start */, size_t /* length */, - bool /* has_null */, + const NullMap * /* nullmap */, const TiDB::TiDBCollatorPtr & /* collator */, String * /*sort_key_container */) const override { @@ -174,7 +176,7 @@ class ColumnFunction final : public COWPtrHelper PaddedPODArray & /* pos */, size_t /* start */, size_t /* length */, - bool /* has_null */, + const NullMap * /* nullmap */, const IColumn::Offsets & /* array_offsets */, const TiDB::TiDBCollatorPtr & /* collator */, String * /* sort_key_container */) const override diff --git a/dbms/src/Columns/ColumnNullable.cpp b/dbms/src/Columns/ColumnNullable.cpp index acfa920ca7d..fa175995cd4 100644 --- a/dbms/src/Columns/ColumnNullable.cpp +++ b/dbms/src/Columns/ColumnNullable.cpp @@ -284,10 +284,12 @@ const char * ColumnNullable::deserializeAndInsertFromArena(const char * pos, con void ColumnNullable::countSerializeByteSizeForCmp( PaddedPODArray & byte_size, - const TiDB::TiDBCollatorPtr & collator) const + const TiDB::TiDBCollatorPtr & collator, + const NullMap * nullmap) const { - getNullMapColumn().countSerializeByteSizeForCmp(byte_size, collator); - getNestedColumn().countSerializeByteSizeForCmp(byte_size, collator); + assert(!nullmap); + getNullMapColumn().countSerializeByteSizeForCmp(byte_size, collator, nullptr); + getNestedColumn().countSerializeByteSizeForCmp(byte_size, collator, &getNullMapData()); } void ColumnNullable::countSerializeByteSize(PaddedPODArray & byte_size) const { @@ -298,10 +300,12 @@ void ColumnNullable::countSerializeByteSize(PaddedPODArray & byte_size) void ColumnNullable::countSerializeByteSizeForCmpColumnArray( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets, - const TiDB::TiDBCollatorPtr & collator) const + const TiDB::TiDBCollatorPtr & collator, + const NullMap * nullmap) const { - getNullMapColumn().countSerializeByteSizeForCmpColumnArray(byte_size, array_offsets, collator); - getNestedColumn().countSerializeByteSizeForCmpColumnArray(byte_size, array_offsets, collator); + assert(!nullmap); + getNullMapColumn().countSerializeByteSizeForCmpColumnArray(byte_size, array_offsets, collator, nullptr); + getNestedColumn().countSerializeByteSizeForCmpColumnArray(byte_size, array_offsets, collator, &getNullMapData()); } void ColumnNullable::countSerializeByteSizeForColumnArray( PaddedPODArray & byte_size, @@ -315,13 +319,15 @@ void ColumnNullable::serializeToPosForCmp( PaddedPODArray & pos, size_t start, size_t length, - bool has_null, + const NullMap * nullmap, const TiDB::TiDBCollatorPtr & collator, String * sort_key_container) const { - getNullMapColumn().serializeToPosForCmp(pos, start, length, has_null, collator, sort_key_container); - getNestedColumn().serializeToPosForCmp(pos, start, length, has_null, collator, sort_key_container); + assert(!nullmap); + getNullMapColumn().serializeToPosForCmp(pos, start, length, nullptr, collator, sort_key_container); + getNestedColumn().serializeToPosForCmp(pos, start, length, &getNullMapData(), collator, sort_key_container); } + void ColumnNullable::serializeToPos(PaddedPODArray & pos, size_t start, size_t length, bool has_null) const { getNullMapColumn().serializeToPos(pos, start, length, has_null); @@ -332,15 +338,16 @@ void ColumnNullable::serializeToPosForCmpColumnArray( PaddedPODArray & pos, size_t start, size_t length, - bool has_null, + const NullMap * nullmap, const IColumn::Offsets & array_offsets, const TiDB::TiDBCollatorPtr & collator, String * sort_key_container) const { + assert(!nullmap); getNullMapColumn() - .serializeToPosForCmpColumnArray(pos, start, length, has_null, array_offsets, collator, sort_key_container); + .serializeToPosForCmpColumnArray(pos, start, length, nullptr, array_offsets, collator, sort_key_container); getNestedColumn() - .serializeToPosForCmpColumnArray(pos, start, length, has_null, array_offsets, collator, sort_key_container); + .serializeToPosForCmpColumnArray(pos, start, length, &getNullMapData(), array_offsets, collator, sort_key_container); } void ColumnNullable::serializeToPosForColumnArray( PaddedPODArray & pos, diff --git a/dbms/src/Columns/ColumnNullable.h b/dbms/src/Columns/ColumnNullable.h index 76b5a6708b5..6f79c5b7b81 100644 --- a/dbms/src/Columns/ColumnNullable.h +++ b/dbms/src/Columns/ColumnNullable.h @@ -21,8 +21,7 @@ namespace DB { -using NullMap = ColumnUInt8::Container; -using ConstNullMapPtr = const NullMap *; +static_assert(std::is_same_v); /// Class that specifies nullable columns. A nullable column represents /// a column, which may have any type, provided with the possibility of @@ -78,14 +77,15 @@ class ColumnNullable final : public COWPtrHelper String &) const override; const char * deserializeAndInsertFromArena(const char * pos, const TiDB::TiDBCollatorPtr &) override; - void countSerializeByteSizeForCmp(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator) + void countSerializeByteSizeForCmp(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator, const NullMap * nullmap) const override; void countSerializeByteSize(PaddedPODArray & byte_size) const override; void countSerializeByteSizeForCmpColumnArray( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets, - const TiDB::TiDBCollatorPtr & collator) const override; + const TiDB::TiDBCollatorPtr & collator, + const NullMap * nullmap) const override; void countSerializeByteSizeForColumnArray( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets) const override; @@ -94,7 +94,7 @@ class ColumnNullable final : public COWPtrHelper PaddedPODArray & pos, size_t start, size_t length, - bool has_null, + const NullMap * nullmap, const TiDB::TiDBCollatorPtr & collator, String * sort_key_container) const override; void serializeToPos(PaddedPODArray & pos, size_t start, size_t length, bool has_null) const override; @@ -103,7 +103,7 @@ class ColumnNullable final : public COWPtrHelper PaddedPODArray & pos, size_t start, size_t length, - bool has_null, + const NullMap * nullmap, const IColumn::Offsets & array_offsets, const TiDB::TiDBCollatorPtr & collator, String * sort_key_container) const override; diff --git a/dbms/src/Columns/ColumnString.cpp b/dbms/src/Columns/ColumnString.cpp index 537ea145a59..50a65d26836 100644 --- a/dbms/src/Columns/ColumnString.cpp +++ b/dbms/src/Columns/ColumnString.cpp @@ -484,30 +484,44 @@ void ColumnString::getPermutationWithCollationImpl( void ColumnString::countSerializeByteSizeForCmp( PaddedPODArray & byte_size, - const TiDB::TiDBCollatorPtr & collator) const + const TiDB::TiDBCollatorPtr & collator, + const NullMap * nullmap) const { if likely (collator != nullptr) { if (collator->maxBytesForOneChar() > 1) - countSerializeByteSizeImpl(byte_size, collator); + countSerializeByteSizeNullMap(byte_size, collator, nullmap); else - countSerializeByteSizeImpl(byte_size, collator); + countSerializeByteSizeNullMap(byte_size, collator, nullmap); } else { - countSerializeByteSizeImpl(byte_size, nullptr); + countSerializeByteSizeNullMap(byte_size, nullptr, nullmap); } } void ColumnString::countSerializeByteSize(PaddedPODArray & byte_size) const { - countSerializeByteSizeImpl(byte_size, nullptr); + countSerializeByteSizeNullMap(byte_size, nullptr, nullptr); +} + +template +ALWAYS_INLINE inline void ColumnString::countSerializeByteSizeNullMap( + PaddedPODArray & byte_size, + const TiDB::TiDBCollatorPtr & collator, + const NullMap * nullmap) const +{ + if (nullmap != nullptr) + countSerializeByteSizeImpl(byte_size, collator, nullmap); + else + countSerializeByteSizeImpl(byte_size, collator, nullptr); } -template +template void ColumnString::countSerializeByteSizeImpl( PaddedPODArray & byte_size, - const TiDB::TiDBCollatorPtr & collator) const + const TiDB::TiDBCollatorPtr & collator, + const NullMap * nullmap) const { RUNTIME_CHECK_MSG(byte_size.size() == size(), "size of byte_size({}) != column size({})", byte_size.size(), size()); @@ -522,15 +536,25 @@ void ColumnString::countSerializeByteSizeImpl( sizeAt(i)); } - if constexpr (has_collator) + if constexpr (compare_semantics) { RUNTIME_CHECK(collator); + assert(!has_nullmap || (nullmap && nullmap->size() == size())); const size_t size = byte_size.size(); const size_t max_bytes_one_char = collator->maxBytesForOneChar(); for (size_t i = 0; i < size; ++i) { assert(sizeAt(i) > 0); + if constexpr (has_nullmap) + { + if ((*nullmap)[i] != 0) + { + byte_size[i] += sizeof(UInt32) + 1; + continue; + } + } + if constexpr (count_code_points) { const auto num_char = UTF8::countCodePoints(&chars[offsetAt(i)], sizeAt(i) - 1); @@ -545,6 +569,7 @@ void ColumnString::countSerializeByteSizeImpl( } else { + assert(!has_nullmap); size_t size = byte_size.size(); for (size_t i = 0; i < size; ++i) byte_size[i] += sizeof(UInt32) + sizeAt(i); @@ -554,26 +579,30 @@ void ColumnString::countSerializeByteSizeImpl( void ColumnString::countSerializeByteSizeForCmpColumnArray( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets, - const TiDB::TiDBCollatorPtr & collator) const + const TiDB::TiDBCollatorPtr & collator, + const NullMap * nullmap) const { if likely (collator != nullptr) { if (collator->maxBytesForOneChar() > 1) - countSerializeByteSizeForColumnArrayImpl( + countSerializeByteSizeForColumnArrayNullMap( byte_size, array_offsets, - collator); + collator, + nullmap); else - countSerializeByteSizeForColumnArrayImpl( + countSerializeByteSizeForColumnArrayNullMap( byte_size, array_offsets, - collator); + collator, + nullmap); } else { - countSerializeByteSizeForColumnArrayImpl( + countSerializeByteSizeForColumnArrayNullMap( byte_size, array_offsets, + nullptr, nullptr); } } @@ -582,17 +611,32 @@ void ColumnString::countSerializeByteSizeForColumnArray( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets) const { - countSerializeByteSizeForColumnArrayImpl( + countSerializeByteSizeForColumnArrayNullMap( byte_size, array_offsets, + nullptr, nullptr); } -template +template +void ColumnString::countSerializeByteSizeForColumnArrayNullMap( + PaddedPODArray & byte_size, + const IColumn::Offsets & array_offsets, + const TiDB::TiDBCollatorPtr & collator, + const NullMap * nullmap) const +{ + if (nullmap != nullptr) + countSerializeByteSizeForColumnArrayImpl(byte_size, array_offsets, collator, nullmap); + else + countSerializeByteSizeForColumnArrayImpl(byte_size, array_offsets, collator, nullptr); +} + +template void ColumnString::countSerializeByteSizeForColumnArrayImpl( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets, - const TiDB::TiDBCollatorPtr & collator) const + const TiDB::TiDBCollatorPtr & collator, + const NullMap * nullmap) const { RUNTIME_CHECK_MSG( byte_size.size() == array_offsets.size(), @@ -616,9 +660,10 @@ void ColumnString::countSerializeByteSizeForColumnArrayImpl( sizeAt(i)); } - if constexpr (has_collator) + if constexpr (compare_semantics) { RUNTIME_CHECK(collator); + assert(!has_nullmap || (nullmap && nullmap->size() == array_offsets.size())); size_t size = array_offsets.size(); const auto max_bytes_one_char = collator->maxBytesForOneChar(); @@ -626,6 +671,15 @@ void ColumnString::countSerializeByteSizeForColumnArrayImpl( { const size_t ele_count = array_offsets[i] - array_offsets[i - 1]; assert(offsetAt(array_offsets[i]) - offsetAt(array_offsets[i - 1]) >= ele_count); + if constexpr (has_nullmap) + { + if ((*nullmap)[i] != 0) + { + byte_size[i] += (sizeof(UInt32) + 1) * ele_count; + continue; + } + } + if constexpr (count_code_points) { size_t cur_row_bytes = 0; @@ -641,6 +695,7 @@ void ColumnString::countSerializeByteSizeForColumnArrayImpl( } else { + // NOTE: didn't check nullmap because we have to iterate through all rows, it's slow. byte_size[i] += sizeof(UInt32) * ele_count + offsetAt(array_offsets[i]) - offsetAt(array_offsets[i - 1]); } @@ -648,6 +703,7 @@ void ColumnString::countSerializeByteSizeForColumnArrayImpl( } else { + assert(!has_nullmap); size_t size = array_offsets.size(); for (size_t i = 0; i < size; ++i) byte_size[i] += sizeof(UInt32) * (array_offsets[i] - array_offsets[i - 1]) + offsetAt(array_offsets[i]) @@ -659,60 +715,63 @@ void ColumnString::serializeToPosForCmp( PaddedPODArray & pos, size_t start, size_t length, - bool has_null, + const NullMap * nullmap, const TiDB::TiDBCollatorPtr & collator, String * sort_key_container) const { - if (has_null) + if (nullmap != nullptr) { if likely (collator != nullptr) - serializeToPosImplType( + serializeToPosImplType( pos, start, length, collator, - sort_key_container); + sort_key_container, + nullmap); else - serializeToPosImplType(pos, start, length, nullptr, nullptr); + serializeToPosImplType(pos, start, length, nullptr, nullptr, nullmap); } else { if likely (collator != nullptr) - serializeToPosImplType( + serializeToPosImplType( pos, start, length, collator, - sort_key_container); + sort_key_container, + nullptr); else - serializeToPosImplType(pos, start, length, nullptr, nullptr); + serializeToPosImplType(pos, start, length, nullptr, nullptr, nullptr); } } void ColumnString::serializeToPos(PaddedPODArray & pos, size_t start, size_t length, bool has_null) const { if (has_null) - serializeToPosImplType(pos, start, length, nullptr, nullptr); + serializeToPosImplType(pos, start, length, nullptr, nullptr, nullptr); else - serializeToPosImplType(pos, start, length, nullptr, nullptr); + serializeToPosImplType(pos, start, length, nullptr, nullptr, nullptr); } -template +template void ColumnString::serializeToPosImplType( PaddedPODArray & pos, size_t start, size_t length, const TiDB::TiDBCollatorPtr & collator, - String * sort_key_container) const + String * sort_key_container, + const NullMap * nullmap) const { - if constexpr (has_collator) + if constexpr (compare_semantics) { RUNTIME_CHECK(collator && sort_key_container); #define M(VAR_PREFIX, COLLATOR_NAME, IMPL_TYPE, COLLATOR_ID) \ case (COLLATOR_ID): \ { \ - serializeToPosImpl(pos, start, length, collator, sort_key_container); \ + serializeToPosImpl(pos, start, length, collator, sort_key_container, nullmap); \ break; \ } @@ -728,41 +787,52 @@ void ColumnString::serializeToPosImplType( } else { - serializeToPosImpl( + assert(!nullmap); + serializeToPosImpl( pos, start, length, collator, - sort_key_container); + sort_key_container, + nullptr); } } -template +template void ColumnString::serializeToPosImpl( PaddedPODArray & pos, size_t start, size_t length, const TiDB::TiDBCollatorPtr & collator, - String * sort_key_container) const + String * sort_key_container, + const NullMap * nullmap) const { RUNTIME_CHECK_MSG(length <= pos.size(), "length({}) > size of pos({})", length, pos.size()); RUNTIME_CHECK_MSG(start + length <= size(), "start({}) + length({}) > size of column({})", start, length, size()); + static_assert(!(has_null && has_nullmap)); + assert(!has_nullmap || (nullmap && nullmap->size() == size())); + /// To avoid virtual function call of sortKey(). const auto * derived_collator = static_cast(collator); /// countSerializeByteSizeImpl has already checked that the size of one element is not greater than UINT32_MAX for (size_t i = 0; i < length; ++i) { - if constexpr (has_null) - { - if (pos[i] == nullptr) - continue; - } - - UInt32 str_size = sizeAt(start + i); - const void * src = &chars[offsetAt(start + i)]; - if constexpr (has_collator) + if constexpr (compare_semantics) { + UInt32 str_size = sizeAt(start + i); + const void * src = &chars[offsetAt(start + i)]; + if constexpr (has_nullmap) + { + if ((*nullmap)[i] != 0) + { + UInt32 str_size = 1; + tiflash_compiler_builtin_memcpy(pos[i], &str_size, sizeof(UInt32)); + *(pos[i]) = '\0'; + pos[i] += 1; + continue; + } + } auto sort_key = derived_collator->sortKey(reinterpret_cast(src), str_size - 1, *sort_key_container); // For terminating zero. @@ -777,6 +847,17 @@ void ColumnString::serializeToPosImpl( } else { + assert(!has_nullmap); + if constexpr (has_null) + { + if (pos[i] == nullptr) + continue; + } + + UInt32 str_size = sizeAt(start + i); + const void * src = &chars[offsetAt(start + i)]; + + assert(!nullmap); tiflash_compiler_builtin_memcpy(pos[i], &str_size, sizeof(UInt32)); pos[i] += sizeof(UInt32); inline_memcpy(pos[i], src, str_size); @@ -789,47 +870,51 @@ void ColumnString::serializeToPosForCmpColumnArray( PaddedPODArray & pos, size_t start, size_t length, - bool has_null, + const NullMap * nullmap, const IColumn::Offsets & array_offsets, const TiDB::TiDBCollatorPtr & collator, String * sort_key_container) const { - if (has_null) + if (nullmap != nullptr) { if likely (collator != nullptr) - serializeToPosForColumnArrayImplType( + serializeToPosForColumnArrayImplType( pos, start, length, array_offsets, collator, - sort_key_container); + sort_key_container, + nullmap); else - serializeToPosForColumnArrayImplType( + serializeToPosForColumnArrayImplType( pos, start, length, array_offsets, nullptr, - nullptr); + nullptr, + nullmap); } else { if likely (collator != nullptr) - serializeToPosForColumnArrayImplType( + serializeToPosForColumnArrayImplType( pos, start, length, array_offsets, collator, - sort_key_container); + sort_key_container, + nullptr); else - serializeToPosForColumnArrayImplType( + serializeToPosForColumnArrayImplType( pos, start, length, array_offsets, nullptr, + nullptr, nullptr); } } @@ -842,46 +927,50 @@ void ColumnString::serializeToPosForColumnArray( const IColumn::Offsets & array_offsets) const { if (has_null) - serializeToPosForColumnArrayImplType( + serializeToPosForColumnArrayImplType( pos, start, length, array_offsets, nullptr, + nullptr, nullptr); else - serializeToPosForColumnArrayImplType( + serializeToPosForColumnArrayImplType( pos, start, length, array_offsets, nullptr, + nullptr, nullptr); } -template +template void ColumnString::serializeToPosForColumnArrayImplType( PaddedPODArray & pos, size_t start, size_t length, const IColumn::Offsets & array_offsets, const TiDB::TiDBCollatorPtr & collator, - String * sort_key_container) const + String * sort_key_container, + const NullMap * nullmap) const { - if constexpr (has_collator) + if constexpr (compare_semantics) { RUNTIME_CHECK(collator && sort_key_container); #define M(VAR_PREFIX, COLLATOR_NAME, IMPL_TYPE, COLLATOR_ID) \ case (COLLATOR_ID): \ { \ - serializeToPosForColumnArrayImpl( \ + serializeToPosForColumnArrayImpl( \ pos, \ start, \ length, \ array_offsets, \ collator, \ - sort_key_container); \ + sort_key_container, \ + nullmap); \ break; \ } @@ -897,24 +986,27 @@ void ColumnString::serializeToPosForColumnArrayImplType( } else { - serializeToPosForColumnArrayImpl( + assert(!nullmap); + serializeToPosForColumnArrayImpl( pos, start, length, array_offsets, collator, - sort_key_container); + sort_key_container, + nullptr); } } -template +template void ColumnString::serializeToPosForColumnArrayImpl( PaddedPODArray & pos, size_t start, size_t length, const IColumn::Offsets & array_offsets, const TiDB::TiDBCollatorPtr & collator, - String * sort_key_container) const + String * sort_key_container, + const NullMap * nullmap) const { RUNTIME_CHECK_MSG(length <= pos.size(), "length({}) > size of pos({})", length, pos.size()); RUNTIME_CHECK_MSG( @@ -929,18 +1021,22 @@ void ColumnString::serializeToPosForColumnArrayImpl( array_offsets.back(), size()); + static_assert(!(has_null && has_nullmap)); + assert(!has_nullmap || (nullmap && nullmap->size() == array_offsets.size())); + /// countSerializeByteSizeForCmpColumnArray has already checked that the size of one element is not greater than UINT32_MAX - if constexpr (has_collator) + if constexpr (compare_semantics) { /// To avoid virtual function call of sortKey(). const auto * derived_collator = static_cast(collator); for (size_t i = 0; i < length; ++i) { - if constexpr (has_null) + if constexpr (has_nullmap) { - if (pos[i] == nullptr) + if ((*nullmap)[i] != 0) continue; } + for (size_t j = array_offsets[start + i - 1]; j < array_offsets[start + i]; ++j) { UInt32 str_size = sizeAt(j); @@ -961,6 +1057,7 @@ void ColumnString::serializeToPosForColumnArrayImpl( } else { + assert(!has_nullmap); for (size_t i = 0; i < length; ++i) { if constexpr (has_null) diff --git a/dbms/src/Columns/ColumnString.h b/dbms/src/Columns/ColumnString.h index da9eec75cb8..e187bc30a10 100644 --- a/dbms/src/Columns/ColumnString.h +++ b/dbms/src/Columns/ColumnString.h @@ -107,45 +107,65 @@ class ColumnString final : public COWPtrHelper } } - template - void countSerializeByteSizeImpl(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator) const; - template + template + ALWAYS_INLINE inline void countSerializeByteSizeNullMap( + PaddedPODArray & byte_size, + const TiDB::TiDBCollatorPtr & collator, + const NullMap * nullmap) const; + template + void countSerializeByteSizeImpl( + PaddedPODArray & byte_size, + const TiDB::TiDBCollatorPtr & collator, + const NullMap * nullmap) const; + + template + void countSerializeByteSizeForColumnArrayNullMap( + PaddedPODArray & byte_size, + const IColumn::Offsets & array_offsets, + const TiDB::TiDBCollatorPtr & collator, + const NullMap * nullmap) const; + template void countSerializeByteSizeForColumnArrayImpl( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets, - const TiDB::TiDBCollatorPtr & collator) const; + const TiDB::TiDBCollatorPtr & collator, + const NullMap * nullmap) const; - template + template void serializeToPosImplType( PaddedPODArray & pos, size_t start, size_t length, const TiDB::TiDBCollatorPtr & collator, - String * sort_key_container) const; - template + String * sort_key_container, + const NullMap * nullmap) const; + template void serializeToPosImpl( PaddedPODArray & pos, size_t start, size_t length, const TiDB::TiDBCollatorPtr & collator, - String * sort_key_container) const; + String * sort_key_container, + const NullMap * nullmap) const; - template + template void serializeToPosForColumnArrayImplType( PaddedPODArray & pos, size_t start, size_t length, const IColumn::Offsets & array_offsets, const TiDB::TiDBCollatorPtr & collator, - String * sort_key_container) const; - template + String * sort_key_container, + const NullMap * nullmap) const; + template void serializeToPosForColumnArrayImpl( PaddedPODArray & pos, size_t start, size_t length, const IColumn::Offsets & array_offsets, const TiDB::TiDBCollatorPtr & collator, - String * sort_key_container) const; + String * sort_key_container, + const NullMap * nullmap) const; void deserializeAndInsertFromPosImpl(PaddedPODArray & pos, bool use_nt_align_buffer); template @@ -297,14 +317,15 @@ class ColumnString final : public COWPtrHelper return pos + string_size; } - void countSerializeByteSizeForCmp(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator) + void countSerializeByteSizeForCmp(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator, const NullMap * nullmap) const override; void countSerializeByteSize(PaddedPODArray & byte_size) const override; void countSerializeByteSizeForCmpColumnArray( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets, - const TiDB::TiDBCollatorPtr & collator) const override; + const TiDB::TiDBCollatorPtr & collator, + const NullMap * nullmap) const override; void countSerializeByteSizeForColumnArray( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets) const override; @@ -313,7 +334,7 @@ class ColumnString final : public COWPtrHelper PaddedPODArray & pos, size_t start, size_t length, - bool has_null, + const NullMap * nullmap, const TiDB::TiDBCollatorPtr & collator, String * sort_key_container) const override; void serializeToPos(PaddedPODArray & pos, size_t start, size_t length, bool has_null) const override; @@ -322,7 +343,7 @@ class ColumnString final : public COWPtrHelper PaddedPODArray & pos, size_t start, size_t length, - bool has_null, + const NullMap * nullmap, const IColumn::Offsets & array_offsets, const TiDB::TiDBCollatorPtr & collator, String * sort_key_container) const override; diff --git a/dbms/src/Columns/ColumnTuple.h b/dbms/src/Columns/ColumnTuple.h index 9e89ff7b937..6629f3390db 100644 --- a/dbms/src/Columns/ColumnTuple.h +++ b/dbms/src/Columns/ColumnTuple.h @@ -95,11 +95,11 @@ class ColumnTuple final : public COWPtrHelper String &) const override; const char * deserializeAndInsertFromArena(const char * pos, const TiDB::TiDBCollatorPtr &) override; - void countSerializeByteSizeForCmp(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator) + void countSerializeByteSizeForCmp(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator, const NullMap * nullmap) const override { for (const auto & column : columns) - column->countSerializeByteSizeForCmp(byte_size, collator); + column->countSerializeByteSizeForCmp(byte_size, collator, nullmap); } void countSerializeByteSize(PaddedPODArray & byte_size) const override { @@ -110,10 +110,11 @@ class ColumnTuple final : public COWPtrHelper void countSerializeByteSizeForCmpColumnArray( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets, - const TiDB::TiDBCollatorPtr & collator) const override + const TiDB::TiDBCollatorPtr & collator, + const NullMap * nullmap) const override { for (const auto & column : columns) - column->countSerializeByteSizeForCmpColumnArray(byte_size, array_offsets, collator); + column->countSerializeByteSizeForCmpColumnArray(byte_size, array_offsets, collator, nullmap); } void countSerializeByteSizeForColumnArray( PaddedPODArray & byte_size, @@ -127,12 +128,12 @@ class ColumnTuple final : public COWPtrHelper PaddedPODArray & pos, size_t start, size_t length, - bool has_null, + const NullMap * nullmap, const TiDB::TiDBCollatorPtr & collator, String * sort_key_container) const override { for (const auto & column : columns) - column->serializeToPosForCmp(pos, start, length, has_null, collator, sort_key_container); + column->serializeToPosForCmp(pos, start, length, nullmap, collator, sort_key_container); } void serializeToPos(PaddedPODArray & pos, size_t start, size_t length, bool has_null) const override { @@ -144,7 +145,7 @@ class ColumnTuple final : public COWPtrHelper PaddedPODArray & pos, size_t start, size_t length, - bool has_null, + const NullMap * nullmap, const IColumn::Offsets & array_offsets, const TiDB::TiDBCollatorPtr & collator, String * sort_key_container) const override @@ -154,7 +155,7 @@ class ColumnTuple final : public COWPtrHelper pos, start, length, - has_null, + nullmap, array_offsets, collator, sort_key_container); diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp index 20a823124ed..f7d7692b965 100644 --- a/dbms/src/Columns/ColumnVector.cpp +++ b/dbms/src/Columns/ColumnVector.cpp @@ -88,18 +88,22 @@ template void ColumnVector::serializeToPos(PaddedPODArray & pos, size_t start, size_t length, bool has_null) const { if (has_null) - serializeToPosImpl(pos, start, length); + serializeToPosImpl(pos, start, length, nullptr); else - serializeToPosImpl(pos, start, length); + serializeToPosImpl(pos, start, length, nullptr); } template -template -void ColumnVector::serializeToPosImpl(PaddedPODArray & pos, size_t start, size_t length) const +template +void ColumnVector::serializeToPosImpl(PaddedPODArray & pos, size_t start, size_t length, const NullMap * nullmap) const { RUNTIME_CHECK_MSG(length <= pos.size(), "length({}) > size of pos({})", length, pos.size()); RUNTIME_CHECK_MSG(start + length <= size(), "start({}) + length({}) > size of column({})", start, length, size()); + static_assert(!(has_null && has_nullmap)); + assert(!has_nullmap || (nullmap && nullmap->size() == size())); + + T val{}; for (size_t i = 0; i < length; ++i) { if constexpr (has_null) @@ -107,6 +111,15 @@ void ColumnVector::serializeToPosImpl(PaddedPODArray & pos, size_t st if (pos[i] == nullptr) continue; } + if constexpr (has_nullmap) + { + if ((*nullmap)[start + i] != 0) + { + tiflash_compiler_builtin_memcpy(pos[i], &val, sizeof(T)); + pos[i] += sizeof(T); + continue; + } + } tiflash_compiler_builtin_memcpy(pos[i], &data[start + i], sizeof(T)); pos[i] += sizeof(T); } @@ -121,18 +134,19 @@ void ColumnVector::serializeToPosForColumnArray( const IColumn::Offsets & array_offsets) const { if (has_null) - serializeToPosForColumnArrayImpl(pos, start, length, array_offsets); + serializeToPosForColumnArrayImpl(pos, start, length, array_offsets, nullptr); else - serializeToPosForColumnArrayImpl(pos, start, length, array_offsets); + serializeToPosForColumnArrayImpl(pos, start, length, array_offsets, nullptr); } template -template +template void ColumnVector::serializeToPosForColumnArrayImpl( PaddedPODArray & pos, size_t start, size_t length, - const IColumn::Offsets & array_offsets) const + const IColumn::Offsets & array_offsets, + const NullMap * nullmap) const { RUNTIME_CHECK_MSG(length <= pos.size(), "length({}) > size of pos({})", length, pos.size()); RUNTIME_CHECK_MSG( @@ -147,6 +161,9 @@ void ColumnVector::serializeToPosForColumnArrayImpl( array_offsets.back(), size()); + static_assert(!(has_null && has_nullmap)); + assert(!has_nullmap || (nullmap && nullmap->size() == array_offsets.size())); + for (size_t i = 0; i < length; ++i) { if constexpr (has_null) @@ -155,18 +172,12 @@ void ColumnVector::serializeToPosForColumnArrayImpl( continue; } size_t len = array_offsets[start + i] - array_offsets[start + i - 1]; - if (len <= 4) + if constexpr (has_nullmap) { - for (size_t j = 0; j < len; ++j) - tiflash_compiler_builtin_memcpy( - pos[i] + j * sizeof(T), - &data[array_offsets[start + i - 1] + j], - sizeof(T)); - } - else - { - inline_memcpy(pos[i], &data[array_offsets[start + i - 1]], len * sizeof(T)); + if ((*nullmap)[i] != 0) + continue; } + inline_memcpy(pos[i], &data[array_offsets[start + i - 1]], len * sizeof(T)); pos[i] += len * sizeof(T); } } diff --git a/dbms/src/Columns/ColumnVector.h b/dbms/src/Columns/ColumnVector.h index 57275b6905a..212ca33f3b5 100644 --- a/dbms/src/Columns/ColumnVector.h +++ b/dbms/src/Columns/ColumnVector.h @@ -198,15 +198,16 @@ class ColumnVector final : public COWPtrHelper - void serializeToPosImpl(PaddedPODArray & pos, size_t start, size_t length) const; + template + void serializeToPosImpl(PaddedPODArray & pos, size_t start, size_t length, const NullMap * nullmap) const; - template + template void serializeToPosForColumnArrayImpl( PaddedPODArray & pos, size_t start, size_t length, - const IColumn::Offsets & array_offsets) const; + const IColumn::Offsets & array_offsets, + const NullMap * nullmap) const; public: bool isNumeric() const override { return is_arithmetic_v; } @@ -327,7 +328,7 @@ class ColumnVector final : public COWPtrHelper & byte_size, const TiDB::TiDBCollatorPtr &) const override + void countSerializeByteSizeForCmp(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr &, const NullMap *) const override { countSerializeByteSize(byte_size); } @@ -336,7 +337,8 @@ class ColumnVector final : public COWPtrHelper & byte_size, const IColumn::Offsets & array_offsets, - const TiDB::TiDBCollatorPtr &) const override + const TiDB::TiDBCollatorPtr &, + const NullMap *) const override { countSerializeByteSizeForColumnArray(byte_size, array_offsets); } @@ -348,11 +350,14 @@ class ColumnVector final : public COWPtrHelper & pos, size_t start, size_t length, - bool has_null, + const NullMap * nullmap, const TiDB::TiDBCollatorPtr &, String *) const override { - serializeToPos(pos, start, length, has_null); + if (nullmap != nullptr) + serializeToPosImpl(pos, start, length, nullmap); + else + serializeToPosImpl(pos, start, length, nullptr); } void serializeToPos(PaddedPODArray & pos, size_t start, size_t length, bool has_null) const override; @@ -360,12 +365,15 @@ class ColumnVector final : public COWPtrHelper & pos, size_t start, size_t length, - bool has_null, + const NullMap * nullmap, const IColumn::Offsets & array_offsets, const TiDB::TiDBCollatorPtr &, String *) const override { - serializeToPosForColumnArray(pos, start, length, has_null, array_offsets); + if (nullmap != nullptr) + serializeToPosForColumnArrayImpl(pos, start, length, array_offsets, nullmap); + else + serializeToPosForColumnArrayImpl(pos, start, length, array_offsets, nullptr); } void serializeToPosForColumnArray( PaddedPODArray & pos, diff --git a/dbms/src/Columns/IColumn.h b/dbms/src/Columns/IColumn.h index 4321ed0755a..0011d8454e0 100644 --- a/dbms/src/Columns/IColumn.h +++ b/dbms/src/Columns/IColumn.h @@ -38,6 +38,9 @@ extern const int SIZES_OF_COLUMNS_DOESNT_MATCH; class Arena; class ColumnGathererStream; +using NullMap = PaddedPODArray; +using ConstNullMapPtr = const NullMap *; + /// Declares interface to store columns in memory. class IColumn : public COWPtr { @@ -240,7 +243,8 @@ class IColumn : public COWPtr /// The byte_size.size() must be equal to the column size. virtual void countSerializeByteSizeForCmp( PaddedPODArray & /* byte_size */, - const TiDB::TiDBCollatorPtr & /* collator */) const + const TiDB::TiDBCollatorPtr & /* collator */, + const NullMap * /* nullmap */) const = 0; virtual void countSerializeByteSize(PaddedPODArray & /* byte_size */) const = 0; @@ -250,7 +254,8 @@ class IColumn : public COWPtr virtual void countSerializeByteSizeForCmpColumnArray( PaddedPODArray & /* byte_size */, const Offsets & /* array_offsets */, - const TiDB::TiDBCollatorPtr & /* collator */) const + const TiDB::TiDBCollatorPtr & /* collator */, + const NullMap * /* nullmap */) const = 0; virtual void countSerializeByteSizeForColumnArray( PaddedPODArray & /* byte_size */, @@ -266,7 +271,7 @@ class IColumn : public COWPtr PaddedPODArray & /* pos */, size_t /* start */, size_t /* length */, - bool /* has_null */, + const NullMap * /*nullmap*/, const TiDB::TiDBCollatorPtr & /* collator */, String * /* sort_key_container */) const = 0; @@ -284,7 +289,7 @@ class IColumn : public COWPtr PaddedPODArray & /* pos */, size_t /* start */, size_t /* length */, - bool /* has_null */, + const NullMap * /*nullmap*/, const Offsets & /* array_offsets */, const TiDB::TiDBCollatorPtr & /* collator */, String * /* sort_key_container */) const diff --git a/dbms/src/Columns/IColumnDummy.h b/dbms/src/Columns/IColumnDummy.h index 3882cb4080f..3b0550a9651 100644 --- a/dbms/src/Columns/IColumnDummy.h +++ b/dbms/src/Columns/IColumnDummy.h @@ -90,7 +90,8 @@ class IColumnDummy : public IColumn void countSerializeByteSizeForCmp( PaddedPODArray & /* byte_size */, - const TiDB::TiDBCollatorPtr & /* collator */) const override + const TiDB::TiDBCollatorPtr & /* collator */, + const NullMap * /* nullmap */) const override { throw Exception( "Method countSerializeByteSizeForCmp is not supported for " + getName(), @@ -104,7 +105,8 @@ class IColumnDummy : public IColumn void countSerializeByteSizeForCmpColumnArray( PaddedPODArray & /* byte_size */, const IColumn::Offsets & /* array_offsets */, - const TiDB::TiDBCollatorPtr & /* collator */) const override + const TiDB::TiDBCollatorPtr & /* collator */, + const NullMap * /* nullmap */) const override { throw Exception( "Method countSerializeByteSizeForCmpColumnArray is not supported for " + getName(), @@ -123,7 +125,7 @@ class IColumnDummy : public IColumn PaddedPODArray & /* pos */, size_t /* start */, size_t /* length */, - bool /* has_null */, + const NullMap * /* nullmap */, const TiDB::TiDBCollatorPtr & /* collator */, String * /* sort_key_container */) const override { @@ -142,7 +144,7 @@ class IColumnDummy : public IColumn PaddedPODArray & /* pos */, size_t /* start */, size_t /* length */, - bool /* has_null */, + const NullMap * /* nullmap */, const IColumn::Offsets & /* array_offsets */, const TiDB::TiDBCollatorPtr & /* collator */, String * /* sort_key_container */) const override diff --git a/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp b/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp index 6cf5742aba3..fd414d8c5e1 100644 --- a/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp +++ b/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp @@ -31,14 +31,14 @@ class TestColumnSerializeDeserialize : public ::testing::Test static void testCountSerializeByteSize( const ColumnPtr & column_ptr, const PaddedPODArray & result_byte_size, - bool for_compare = false, + bool compare_semantics = false, const TiDB::TiDBCollatorPtr & collator = nullptr) { PaddedPODArray byte_size; byte_size.resize_fill_zero(column_ptr->size()); for (size_t i = 0; i < column_ptr->size(); ++i) byte_size[i] = i; - if (!for_compare) + if (!compare_semantics) column_ptr->countSerializeByteSize(byte_size); else column_ptr->countSerializeByteSizeForCmp(byte_size, collator); @@ -51,7 +51,7 @@ class TestColumnSerializeDeserialize : public ::testing::Test const ColumnPtr & column_ptr, const ColumnPtr & offsets, const PaddedPODArray & result_byte_size, - bool for_compare = false, + bool compare_semantics = false, const TiDB::TiDBCollatorPtr & collator = nullptr) { auto column_array = ColumnArray::create(column_ptr->cloneFullColumn(), offsets->cloneFullColumn()); @@ -59,7 +59,7 @@ class TestColumnSerializeDeserialize : public ::testing::Test byte_size.resize_fill_zero(column_array->size()); for (size_t i = 0; i < column_array->size(); ++i) byte_size[i] = i; - if (!for_compare) + if (!compare_semantics) column_array->countSerializeByteSize(byte_size); else column_array->countSerializeByteSizeForCmp(byte_size, collator); @@ -164,26 +164,26 @@ class TestColumnSerializeDeserialize : public ::testing::Test static void testSerializeAndDeserialize( const ColumnPtr & column_ptr, - bool for_compare = false, + bool compare_semantics = false, const TiDB::TiDBCollatorPtr & collator = nullptr, String * sort_key_container = nullptr) { - doTestSerializeAndDeserialize(column_ptr, false, for_compare, collator, sort_key_container); - doTestSerializeAndDeserialize2(column_ptr, false, for_compare, collator, sort_key_container); - doTestSerializeAndDeserialize(column_ptr, true, for_compare, collator, sort_key_container); - doTestSerializeAndDeserialize2(column_ptr, true, for_compare, collator, sort_key_container); + doTestSerializeAndDeserialize(column_ptr, false, compare_semantics, collator, sort_key_container); + doTestSerializeAndDeserialize2(column_ptr, false, compare_semantics, collator, sort_key_container); + doTestSerializeAndDeserialize(column_ptr, true, compare_semantics, collator, sort_key_container); + doTestSerializeAndDeserialize2(column_ptr, true, compare_semantics, collator, sort_key_container); } static void doTestSerializeAndDeserialize( const ColumnPtr & column_ptr, bool use_nt_align_buffer, - bool for_compare = false, + bool compare_semantics = false, const TiDB::TiDBCollatorPtr & collator = nullptr, String * sort_key_container = nullptr) { PaddedPODArray byte_size; byte_size.resize_fill_zero(column_ptr->size()); - if (!for_compare) + if (!compare_semantics) column_ptr->countSerializeByteSize(byte_size); else column_ptr->countSerializeByteSizeForCmp(byte_size, collator); @@ -201,15 +201,15 @@ class TestColumnSerializeDeserialize : public ::testing::Test PaddedPODArray ori_pos; for (auto * ptr : pos) ori_pos.push_back(ptr); - if (!for_compare) + if (!compare_semantics) column_ptr->serializeToPos(pos, 0, byte_size.size() / 2, false); else - column_ptr->serializeToPosForCmp(pos, 0, byte_size.size() / 2, false, collator, sort_key_container); + column_ptr->serializeToPosForCmp(pos, 0, byte_size.size() / 2, nullptr, collator, sort_key_container); auto new_col_ptr = column_ptr->cloneEmpty(); if (use_nt_align_buffer) new_col_ptr->reserveAlign(byte_size.size(), FULL_VECTOR_SIZE_AVX2); - if (!for_compare) + if (!compare_semantics) new_col_ptr->deserializeAndInsertFromPos(ori_pos, use_nt_align_buffer); else new_col_ptr->deserializeForCmpAndInsertFromPos(ori_pos, use_nt_align_buffer); @@ -225,20 +225,20 @@ class TestColumnSerializeDeserialize : public ::testing::Test pos.push_back(nullptr); for (auto * ptr : pos) ori_pos.push_back(ptr); - if (!for_compare) + if (!compare_semantics) column_ptr->serializeToPos(pos, byte_size.size() / 2, byte_size.size() - byte_size.size() / 2, true); else column_ptr->serializeToPosForCmp( pos, byte_size.size() / 2, byte_size.size() - byte_size.size() / 2, - true, + nullptr, collator, sort_key_container); pos.resize(pos.size() - 1); ori_pos.resize(ori_pos.size() - 1); - if (!for_compare) + if (!compare_semantics) new_col_ptr->deserializeAndInsertFromPos(ori_pos, use_nt_align_buffer); else new_col_ptr->deserializeForCmpAndInsertFromPos(ori_pos, use_nt_align_buffer); @@ -253,12 +253,12 @@ class TestColumnSerializeDeserialize : public ::testing::Test } for (auto * ptr : pos) ori_pos.push_back(ptr); - if (!for_compare) + if (!compare_semantics) column_ptr->serializeToPos(pos, 0, byte_size.size(), true); else - column_ptr->serializeToPosForCmp(pos, 0, byte_size.size(), true, collator, sort_key_container); + column_ptr->serializeToPosForCmp(pos, 0, byte_size.size(), nullptr, collator, sort_key_container); - if (!for_compare) + if (!compare_semantics) new_col_ptr->deserializeAndInsertFromPos(ori_pos, use_nt_align_buffer); else new_col_ptr->deserializeForCmpAndInsertFromPos(ori_pos, use_nt_align_buffer); @@ -279,7 +279,7 @@ class TestColumnSerializeDeserialize : public ::testing::Test static void doTestSerializeAndDeserialize2( const ColumnPtr & column_ptr, bool use_nt_align_buffer, - bool for_compare = false, + bool compare_semantics = false, const TiDB::TiDBCollatorPtr & collator = nullptr, String * sort_key_container = nullptr) { @@ -287,7 +287,7 @@ class TestColumnSerializeDeserialize : public ::testing::Test return; PaddedPODArray byte_size; byte_size.resize_fill_zero(column_ptr->size()); - if (!for_compare) + if (!compare_semantics) column_ptr->countSerializeByteSize(byte_size); else column_ptr->countSerializeByteSizeForCmp(byte_size, collator); @@ -306,17 +306,17 @@ class TestColumnSerializeDeserialize : public ::testing::Test pos.push_back(nullptr); for (auto * ptr : pos) ori_pos.push_back(ptr); - if (!for_compare) + if (!compare_semantics) column_ptr->serializeToPos(pos, 0, byte_size.size() / 2, true); else - column_ptr->serializeToPosForCmp(pos, 0, byte_size.size() / 2, true, collator, sort_key_container); + column_ptr->serializeToPosForCmp(pos, 0, byte_size.size() / 2, nullptr, collator, sort_key_container); pos.resize(pos.size() - 1); ori_pos.resize(ori_pos.size() - 1); auto new_col_ptr = column_ptr->cloneEmpty(); if (use_nt_align_buffer) new_col_ptr->reserveAlign(byte_size.size(), FULL_VECTOR_SIZE_AVX2); - if (!for_compare) + if (!compare_semantics) new_col_ptr->deserializeAndInsertFromPos(ori_pos, use_nt_align_buffer); else new_col_ptr->deserializeForCmpAndInsertFromPos(ori_pos, use_nt_align_buffer); @@ -331,7 +331,7 @@ class TestColumnSerializeDeserialize : public ::testing::Test } for (auto * ptr : pos) ori_pos.push_back(ptr); - if (!for_compare) + if (!compare_semantics) column_ptr ->serializeToPos(pos, byte_size.size() / 2 - 1, byte_size.size() - byte_size.size() / 2 + 1, false); else @@ -339,10 +339,10 @@ class TestColumnSerializeDeserialize : public ::testing::Test pos, byte_size.size() / 2 - 1, byte_size.size() - byte_size.size() / 2 + 1, - false, + nullptr, collator, sort_key_container); - if (!for_compare) + if (!compare_semantics) new_col_ptr->deserializeAndInsertFromPos(ori_pos, use_nt_align_buffer); else new_col_ptr->deserializeForCmpAndInsertFromPos(ori_pos, use_nt_align_buffer); @@ -357,12 +357,12 @@ class TestColumnSerializeDeserialize : public ::testing::Test } for (auto * ptr : pos) ori_pos.push_back(ptr); - if (!for_compare) + if (!compare_semantics) column_ptr->serializeToPos(pos, 0, byte_size.size(), true); else - column_ptr->serializeToPosForCmp(pos, 0, byte_size.size(), true, collator, sort_key_container); + column_ptr->serializeToPosForCmp(pos, 0, byte_size.size(), nullptr, collator, sort_key_container); - if (!for_compare) + if (!compare_semantics) new_col_ptr->deserializeAndInsertFromPos(ori_pos, use_nt_align_buffer); else new_col_ptr->deserializeForCmpAndInsertFromPos(ori_pos, use_nt_align_buffer); From 51e24d86ae7367996e0c4acaf5308553d5626670 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Fri, 24 Jan 2025 10:49:19 +0800 Subject: [PATCH 02/16] refine Signed-off-by: guo-shaoge --- dbms/src/Columns/ColumnVector.cpp | 4 ++-- .../Columns/tests/gtest_column_serialize_deserialize.cpp | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp index f7d7692b965..51dc2869183 100644 --- a/dbms/src/Columns/ColumnVector.cpp +++ b/dbms/src/Columns/ColumnVector.cpp @@ -103,7 +103,7 @@ void ColumnVector::serializeToPosImpl(PaddedPODArray & pos, size_t st static_assert(!(has_null && has_nullmap)); assert(!has_nullmap || (nullmap && nullmap->size() == size())); - T val{}; + T def_val{}; for (size_t i = 0; i < length; ++i) { if constexpr (has_null) @@ -115,7 +115,7 @@ void ColumnVector::serializeToPosImpl(PaddedPODArray & pos, size_t st { if ((*nullmap)[start + i] != 0) { - tiflash_compiler_builtin_memcpy(pos[i], &val, sizeof(T)); + tiflash_compiler_builtin_memcpy(pos[i], &def_val, sizeof(T)); pos[i] += sizeof(T); continue; } diff --git a/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp b/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp index fd414d8c5e1..035da450b7e 100644 --- a/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp +++ b/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp @@ -41,7 +41,7 @@ class TestColumnSerializeDeserialize : public ::testing::Test if (!compare_semantics) column_ptr->countSerializeByteSize(byte_size); else - column_ptr->countSerializeByteSizeForCmp(byte_size, collator); + column_ptr->countSerializeByteSizeForCmp(byte_size, collator, nullptr); ASSERT_EQ(byte_size.size(), result_byte_size.size()); for (size_t i = 0; i < byte_size.size(); ++i) ASSERT_EQ(byte_size[i], i + result_byte_size[i]); @@ -62,7 +62,7 @@ class TestColumnSerializeDeserialize : public ::testing::Test if (!compare_semantics) column_array->countSerializeByteSize(byte_size); else - column_array->countSerializeByteSizeForCmp(byte_size, collator); + column_array->countSerializeByteSizeForCmp(byte_size, collator, nullptr); ASSERT_EQ(byte_size.size(), result_byte_size.size()); for (size_t i = 0; i < byte_size.size(); ++i) ASSERT_EQ(byte_size[i], sizeof(UInt32) + i + result_byte_size[i]); @@ -186,7 +186,7 @@ class TestColumnSerializeDeserialize : public ::testing::Test if (!compare_semantics) column_ptr->countSerializeByteSize(byte_size); else - column_ptr->countSerializeByteSizeForCmp(byte_size, collator); + column_ptr->countSerializeByteSizeForCmp(byte_size, collator, nullptr); size_t total_size = 0; for (const auto size : byte_size) total_size += size; @@ -290,7 +290,7 @@ class TestColumnSerializeDeserialize : public ::testing::Test if (!compare_semantics) column_ptr->countSerializeByteSize(byte_size); else - column_ptr->countSerializeByteSizeForCmp(byte_size, collator); + column_ptr->countSerializeByteSizeForCmp(byte_size, collator, nullptr); size_t total_size = 0; for (const auto size : byte_size) total_size += size; From 543f1b826b201a97e3783475de18dc8d2dc94864 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Fri, 24 Jan 2025 10:50:46 +0800 Subject: [PATCH 03/16] plan 1 code Signed-off-by: guo-shaoge --- dbms/src/Columns/ColumnArray.cpp | 6 +- dbms/src/Columns/ColumnArray.h | 11 ++- dbms/src/Columns/ColumnDecimal.cpp | 6 +- dbms/src/Columns/ColumnDecimal.h | 29 +++++- dbms/src/Columns/ColumnFixedString.cpp | 6 +- dbms/src/Columns/ColumnFixedString.h | 12 ++- dbms/src/Columns/ColumnNullable.cpp | 10 +- dbms/src/Columns/ColumnNullable.h | 6 +- dbms/src/Columns/ColumnString.cpp | 124 ++++++++++++++++++------- dbms/src/Columns/ColumnString.h | 6 +- dbms/src/Columns/ColumnTuple.h | 6 +- dbms/src/Columns/ColumnVector.cpp | 6 +- dbms/src/Columns/ColumnVector.h | 5 +- 13 files changed, 170 insertions(+), 63 deletions(-) diff --git a/dbms/src/Columns/ColumnArray.cpp b/dbms/src/Columns/ColumnArray.cpp index d661461b62f..8223cc5ced1 100644 --- a/dbms/src/Columns/ColumnArray.cpp +++ b/dbms/src/Columns/ColumnArray.cpp @@ -232,8 +232,10 @@ void ColumnArray::countSerializeByteSize(PaddedPODArray & byte_size) con } template -void ColumnArray::countSerializeByteSizeImpl(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator, const NullMap * nullmap) - const +void ColumnArray::countSerializeByteSizeImpl( + PaddedPODArray & byte_size, + const TiDB::TiDBCollatorPtr & collator, + const NullMap * nullmap) const { RUNTIME_CHECK_MSG(byte_size.size() == size(), "size of byte_size({}) != column size({})", byte_size.size(), size()); diff --git a/dbms/src/Columns/ColumnArray.h b/dbms/src/Columns/ColumnArray.h index 64b890a354e..48c3c044ea6 100644 --- a/dbms/src/Columns/ColumnArray.h +++ b/dbms/src/Columns/ColumnArray.h @@ -45,7 +45,10 @@ class ColumnArray final : public COWPtrHelper ColumnArray(const ColumnArray &) = default; template - void countSerializeByteSizeImpl(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator, const NullMap * nullmap) const; + void countSerializeByteSizeImpl( + PaddedPODArray & byte_size, + const TiDB::TiDBCollatorPtr & collator, + const NullMap * nullmap) const; template void serializeToPosImpl( @@ -97,8 +100,10 @@ class ColumnArray final : public COWPtrHelper String &) const override; const char * deserializeAndInsertFromArena(const char * pos, const TiDB::TiDBCollatorPtr &) override; - void countSerializeByteSizeForCmp(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator, const NullMap * nullmap) - const override; + void countSerializeByteSizeForCmp( + PaddedPODArray & byte_size, + const TiDB::TiDBCollatorPtr & collator, + const NullMap * nullmap) const override; void countSerializeByteSize(PaddedPODArray & byte_size) const override; void countSerializeByteSizeForCmpColumnArray( diff --git a/dbms/src/Columns/ColumnDecimal.cpp b/dbms/src/Columns/ColumnDecimal.cpp index d2a3b2dc735..cd1ea095b66 100644 --- a/dbms/src/Columns/ColumnDecimal.cpp +++ b/dbms/src/Columns/ColumnDecimal.cpp @@ -195,7 +195,11 @@ void ColumnDecimal::countSerializeByteSizeForColumnArrayImpl( template template -void ColumnDecimal::serializeToPosImpl(PaddedPODArray & pos, size_t start, size_t length, const NullMap * nullmap) const +void ColumnDecimal::serializeToPosImpl( + PaddedPODArray & pos, + size_t start, + size_t length, + const NullMap * nullmap) const { RUNTIME_CHECK_MSG(length <= pos.size(), "length({}) > size of pos({})", length, pos.size()); RUNTIME_CHECK_MSG(start + length <= size(), "start({}) + length({}) > size of column({})", start, length, size()); diff --git a/dbms/src/Columns/ColumnDecimal.h b/dbms/src/Columns/ColumnDecimal.h index 1f02419df8b..91ea2ede0db 100644 --- a/dbms/src/Columns/ColumnDecimal.h +++ b/dbms/src/Columns/ColumnDecimal.h @@ -174,7 +174,10 @@ class ColumnDecimal final : public COWPtrHelper & byte_size, const TiDB::TiDBCollatorPtr &, const NullMap *) const override + void countSerializeByteSizeForCmp( + PaddedPODArray & byte_size, + const TiDB::TiDBCollatorPtr &, + const NullMap *) const override { countSerializeByteSizeImpl(byte_size, nullptr); } @@ -207,16 +210,32 @@ class ColumnDecimal final : public COWPtrHelper(pos, start, length, nullmap); + serializeToPosImpl( + pos, + start, + length, + nullmap); else - serializeToPosImpl(pos, start, length, nullptr); + serializeToPosImpl( + pos, + start, + length, + nullptr); } void serializeToPos(PaddedPODArray & pos, size_t start, size_t length, bool has_null) const override { if (has_null) - serializeToPosImpl(pos, start, length, nullptr); + serializeToPosImpl( + pos, + start, + length, + nullptr); else - serializeToPosImpl(pos, start, length, nullptr); + serializeToPosImpl( + pos, + start, + length, + nullptr); } void serializeToPosForCmpColumnArray( diff --git a/dbms/src/Columns/ColumnFixedString.cpp b/dbms/src/Columns/ColumnFixedString.cpp index dde147faef5..3176fab6783 100644 --- a/dbms/src/Columns/ColumnFixedString.cpp +++ b/dbms/src/Columns/ColumnFixedString.cpp @@ -194,7 +194,11 @@ void ColumnFixedString::serializeToPos(PaddedPODArray & pos, size_t star } template -void ColumnFixedString::serializeToPosImpl(PaddedPODArray & pos, size_t start, size_t length, const NullMap * nullmap) const +void ColumnFixedString::serializeToPosImpl( + PaddedPODArray & pos, + size_t start, + size_t length, + const NullMap * nullmap) const { RUNTIME_CHECK_MSG(length <= pos.size(), "length({}) > size of pos({})", length, pos.size()); RUNTIME_CHECK_MSG(start + length <= size(), "start({}) + length({}) > size of column({})", start, length, size()); diff --git a/dbms/src/Columns/ColumnFixedString.h b/dbms/src/Columns/ColumnFixedString.h index 0ac2ff89aa4..1cd2713542b 100644 --- a/dbms/src/Columns/ColumnFixedString.h +++ b/dbms/src/Columns/ColumnFixedString.h @@ -59,9 +59,9 @@ class ColumnFixedString final : public COWPtrHelper template void countSerializeByteSizeForColumnArrayImpl( - PaddedPODArray & byte_size, - const IColumn::Offsets & array_offsets, - const NullMap * nullmap) const; + PaddedPODArray & byte_size, + const IColumn::Offsets & array_offsets, + const NullMap * nullmap) const; template void serializeToPosImpl(PaddedPODArray & pos, size_t start, size_t length, const NullMap * nullmap) const; @@ -124,8 +124,10 @@ class ColumnFixedString final : public COWPtrHelper const char * deserializeAndInsertFromArena(const char * pos, const TiDB::TiDBCollatorPtr &) override; - void countSerializeByteSizeForCmp(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator, const NullMap * nullmap) - const override + void countSerializeByteSizeForCmp( + PaddedPODArray & byte_size, + const TiDB::TiDBCollatorPtr & collator, + const NullMap * nullmap) const override { // collator->sortKey() will change the string length, which may exceeds n. RUNTIME_CHECK_MSG( diff --git a/dbms/src/Columns/ColumnNullable.cpp b/dbms/src/Columns/ColumnNullable.cpp index fa175995cd4..40a8c1a339b 100644 --- a/dbms/src/Columns/ColumnNullable.cpp +++ b/dbms/src/Columns/ColumnNullable.cpp @@ -346,8 +346,14 @@ void ColumnNullable::serializeToPosForCmpColumnArray( assert(!nullmap); getNullMapColumn() .serializeToPosForCmpColumnArray(pos, start, length, nullptr, array_offsets, collator, sort_key_container); - getNestedColumn() - .serializeToPosForCmpColumnArray(pos, start, length, &getNullMapData(), array_offsets, collator, sort_key_container); + getNestedColumn().serializeToPosForCmpColumnArray( + pos, + start, + length, + &getNullMapData(), + array_offsets, + collator, + sort_key_container); } void ColumnNullable::serializeToPosForColumnArray( PaddedPODArray & pos, diff --git a/dbms/src/Columns/ColumnNullable.h b/dbms/src/Columns/ColumnNullable.h index 6f79c5b7b81..ba747fe8a8e 100644 --- a/dbms/src/Columns/ColumnNullable.h +++ b/dbms/src/Columns/ColumnNullable.h @@ -77,8 +77,10 @@ class ColumnNullable final : public COWPtrHelper String &) const override; const char * deserializeAndInsertFromArena(const char * pos, const TiDB::TiDBCollatorPtr &) override; - void countSerializeByteSizeForCmp(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator, const NullMap * nullmap) - const override; + void countSerializeByteSizeForCmp( + PaddedPODArray & byte_size, + const TiDB::TiDBCollatorPtr & collator, + const NullMap * nullmap) const override; void countSerializeByteSize(PaddedPODArray & byte_size) const override; void countSerializeByteSizeForCmpColumnArray( diff --git a/dbms/src/Columns/ColumnString.cpp b/dbms/src/Columns/ColumnString.cpp index 50a65d26836..0789a6ac1fb 100644 --- a/dbms/src/Columns/ColumnString.cpp +++ b/dbms/src/Columns/ColumnString.cpp @@ -490,19 +490,31 @@ void ColumnString::countSerializeByteSizeForCmp( if likely (collator != nullptr) { if (collator->maxBytesForOneChar() > 1) - countSerializeByteSizeNullMap(byte_size, collator, nullmap); + countSerializeByteSizeNullMap( + byte_size, + collator, + nullmap); else - countSerializeByteSizeNullMap(byte_size, collator, nullmap); + countSerializeByteSizeNullMap( + byte_size, + collator, + nullmap); } else { - countSerializeByteSizeNullMap(byte_size, nullptr, nullmap); + countSerializeByteSizeNullMap( + byte_size, + nullptr, + nullmap); } } void ColumnString::countSerializeByteSize(PaddedPODArray & byte_size) const { - countSerializeByteSizeNullMap(byte_size, nullptr, nullptr); + countSerializeByteSizeNullMap( + byte_size, + nullptr, + nullptr); } template @@ -512,9 +524,15 @@ ALWAYS_INLINE inline void ColumnString::countSerializeByteSizeNullMap( const NullMap * nullmap) const { if (nullmap != nullptr) - countSerializeByteSizeImpl(byte_size, collator, nullmap); + countSerializeByteSizeImpl( + byte_size, + collator, + nullmap); else - countSerializeByteSizeImpl(byte_size, collator, nullptr); + countSerializeByteSizeImpl( + byte_size, + collator, + nullptr); } template @@ -626,9 +644,17 @@ void ColumnString::countSerializeByteSizeForColumnArrayNullMap( const NullMap * nullmap) const { if (nullmap != nullptr) - countSerializeByteSizeForColumnArrayImpl(byte_size, array_offsets, collator, nullmap); + countSerializeByteSizeForColumnArrayImpl( + byte_size, + array_offsets, + collator, + nullmap); else - countSerializeByteSizeForColumnArrayImpl(byte_size, array_offsets, collator, nullptr); + countSerializeByteSizeForColumnArrayImpl( + byte_size, + array_offsets, + collator, + nullptr); } template @@ -730,7 +756,13 @@ void ColumnString::serializeToPosForCmp( sort_key_container, nullmap); else - serializeToPosImplType(pos, start, length, nullptr, nullptr, nullmap); + serializeToPosImplType( + pos, + start, + length, + nullptr, + nullptr, + nullmap); } else { @@ -743,16 +775,34 @@ void ColumnString::serializeToPosForCmp( sort_key_container, nullptr); else - serializeToPosImplType(pos, start, length, nullptr, nullptr, nullptr); + serializeToPosImplType( + pos, + start, + length, + nullptr, + nullptr, + nullptr); } } void ColumnString::serializeToPos(PaddedPODArray & pos, size_t start, size_t length, bool has_null) const { if (has_null) - serializeToPosImplType(pos, start, length, nullptr, nullptr, nullptr); + serializeToPosImplType( + pos, + start, + length, + nullptr, + nullptr, + nullptr); else - serializeToPosImplType(pos, start, length, nullptr, nullptr, nullptr); + serializeToPosImplType( + pos, + start, + length, + nullptr, + nullptr, + nullptr); } template @@ -768,11 +818,17 @@ void ColumnString::serializeToPosImplType( { RUNTIME_CHECK(collator && sort_key_container); -#define M(VAR_PREFIX, COLLATOR_NAME, IMPL_TYPE, COLLATOR_ID) \ - case (COLLATOR_ID): \ - { \ - serializeToPosImpl(pos, start, length, collator, sort_key_container, nullmap); \ - break; \ +#define M(VAR_PREFIX, COLLATOR_NAME, IMPL_TYPE, COLLATOR_ID) \ + case (COLLATOR_ID): \ + { \ + serializeToPosImpl( \ + pos, \ + start, \ + length, \ + collator, \ + sort_key_container, \ + nullmap); \ + break; \ } switch (collator->getCollatorId()) @@ -908,14 +964,10 @@ void ColumnString::serializeToPosForCmpColumnArray( sort_key_container, nullptr); else - serializeToPosForColumnArrayImplType( - pos, - start, - length, - array_offsets, - nullptr, - nullptr, - nullptr); + serializeToPosForColumnArrayImplType< + /*has_null=*/false, + /*compare_semantics=*/false, + /*has_nullmap=*/false>(pos, start, length, array_offsets, nullptr, nullptr, nullptr); } } @@ -960,18 +1012,18 @@ void ColumnString::serializeToPosForColumnArrayImplType( { RUNTIME_CHECK(collator && sort_key_container); -#define M(VAR_PREFIX, COLLATOR_NAME, IMPL_TYPE, COLLATOR_ID) \ - case (COLLATOR_ID): \ - { \ +#define M(VAR_PREFIX, COLLATOR_NAME, IMPL_TYPE, COLLATOR_ID) \ + case (COLLATOR_ID): \ + { \ serializeToPosForColumnArrayImpl( \ - pos, \ - start, \ - length, \ - array_offsets, \ - collator, \ - sort_key_container, \ - nullmap); \ - break; \ + pos, \ + start, \ + length, \ + array_offsets, \ + collator, \ + sort_key_container, \ + nullmap); \ + break; \ } switch (collator->getCollatorId()) diff --git a/dbms/src/Columns/ColumnString.h b/dbms/src/Columns/ColumnString.h index e187bc30a10..8d9ee722c14 100644 --- a/dbms/src/Columns/ColumnString.h +++ b/dbms/src/Columns/ColumnString.h @@ -317,8 +317,10 @@ class ColumnString final : public COWPtrHelper return pos + string_size; } - void countSerializeByteSizeForCmp(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator, const NullMap * nullmap) - const override; + void countSerializeByteSizeForCmp( + PaddedPODArray & byte_size, + const TiDB::TiDBCollatorPtr & collator, + const NullMap * nullmap) const override; void countSerializeByteSize(PaddedPODArray & byte_size) const override; void countSerializeByteSizeForCmpColumnArray( diff --git a/dbms/src/Columns/ColumnTuple.h b/dbms/src/Columns/ColumnTuple.h index 6629f3390db..651a7bfca12 100644 --- a/dbms/src/Columns/ColumnTuple.h +++ b/dbms/src/Columns/ColumnTuple.h @@ -95,8 +95,10 @@ class ColumnTuple final : public COWPtrHelper String &) const override; const char * deserializeAndInsertFromArena(const char * pos, const TiDB::TiDBCollatorPtr &) override; - void countSerializeByteSizeForCmp(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator, const NullMap * nullmap) - const override + void countSerializeByteSizeForCmp( + PaddedPODArray & byte_size, + const TiDB::TiDBCollatorPtr & collator, + const NullMap * nullmap) const override { for (const auto & column : columns) column->countSerializeByteSizeForCmp(byte_size, collator, nullmap); diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp index 51dc2869183..48db5882a7a 100644 --- a/dbms/src/Columns/ColumnVector.cpp +++ b/dbms/src/Columns/ColumnVector.cpp @@ -95,7 +95,11 @@ void ColumnVector::serializeToPos(PaddedPODArray & pos, size_t start, template template -void ColumnVector::serializeToPosImpl(PaddedPODArray & pos, size_t start, size_t length, const NullMap * nullmap) const +void ColumnVector::serializeToPosImpl( + PaddedPODArray & pos, + size_t start, + size_t length, + const NullMap * nullmap) const { RUNTIME_CHECK_MSG(length <= pos.size(), "length({}) > size of pos({})", length, pos.size()); RUNTIME_CHECK_MSG(start + length <= size(), "start({}) + length({}) > size of column({})", start, length, size()); diff --git a/dbms/src/Columns/ColumnVector.h b/dbms/src/Columns/ColumnVector.h index 212ca33f3b5..344a3d10fbe 100644 --- a/dbms/src/Columns/ColumnVector.h +++ b/dbms/src/Columns/ColumnVector.h @@ -328,7 +328,10 @@ class ColumnVector final : public COWPtrHelper & byte_size, const TiDB::TiDBCollatorPtr &, const NullMap *) const override + void countSerializeByteSizeForCmp( + PaddedPODArray & byte_size, + const TiDB::TiDBCollatorPtr &, + const NullMap *) const override { countSerializeByteSize(byte_size); } From 370bf0ae1346af260b4c7dc061e94ae0d3d9b572 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Fri, 24 Jan 2025 15:56:55 +0800 Subject: [PATCH 04/16] refine Signed-off-by: guo-shaoge --- dbms/src/Columns/ColumnArray.cpp | 3 +- dbms/src/Columns/ColumnDecimal.cpp | 22 ++- dbms/src/Columns/ColumnFixedString.cpp | 8 +- dbms/src/Columns/ColumnNullable.cpp | 111 +++++++++-- dbms/src/Columns/ColumnString.cpp | 23 +-- dbms/src/Columns/ColumnVector.cpp | 19 +- dbms/src/Columns/IColumn.h | 16 ++ .../gtest_column_serialize_deserialize.cpp | 186 ++++++++++-------- 8 files changed, 255 insertions(+), 133 deletions(-) diff --git a/dbms/src/Columns/ColumnArray.cpp b/dbms/src/Columns/ColumnArray.cpp index 8223cc5ced1..d6d2657c402 100644 --- a/dbms/src/Columns/ColumnArray.cpp +++ b/dbms/src/Columns/ColumnArray.cpp @@ -238,6 +238,7 @@ void ColumnArray::countSerializeByteSizeImpl( const NullMap * nullmap) const { RUNTIME_CHECK_MSG(byte_size.size() == size(), "size of byte_size({}) != column size({})", byte_size.size(), size()); + assert(!nullmap || (nullmap->size() == size())); if unlikely (!getOffsets().empty() && getOffsets().back() > UINT32_MAX) { @@ -309,7 +310,7 @@ void ColumnArray::serializeToPosImpl( UInt32 len = sizeAt(start + i); if constexpr (has_nullmap) { - if ((*nullmap)[i] != 0) + if (DB::isNullAt(*nullmap, i)) len = 0; } tiflash_compiler_builtin_memcpy(pos[i], &len, sizeof(UInt32)); diff --git a/dbms/src/Columns/ColumnDecimal.cpp b/dbms/src/Columns/ColumnDecimal.cpp index cd1ea095b66..867b86e1b31 100644 --- a/dbms/src/Columns/ColumnDecimal.cpp +++ b/dbms/src/Columns/ColumnDecimal.cpp @@ -160,7 +160,6 @@ void ColumnDecimal::countSerializeByteSizeImpl(PaddedPODArray & byte_ } } -// TODO add unit test template template void ColumnDecimal::countSerializeByteSizeForColumnArrayImpl( @@ -220,7 +219,7 @@ void ColumnDecimal::serializeToPosImpl( { if constexpr (has_nullmap) { - if ((*nullmap)[i] != 0) + if (DB::isNullAt(*nullmap, i)) { pos[i] = serializeDecimal256Helper(pos[i], def_val); continue; @@ -232,7 +231,7 @@ void ColumnDecimal::serializeToPosImpl( { if constexpr (has_nullmap) { - if ((*nullmap)[i] != 0) + if (DB::isNullAt(*nullmap, i)) { tiflash_compiler_builtin_memcpy(pos[i], &def_val, sizeof(T)); pos[i] += sizeof(T); @@ -283,7 +282,7 @@ void ColumnDecimal::serializeToPosForColumnArrayImpl( { if constexpr (has_nullmap) { - if ((*nullmap)[i] != 0) + if (DB::isNullAt(*nullmap, i)) continue; } for (size_t j = 0; j < len; ++j) @@ -293,10 +292,21 @@ void ColumnDecimal::serializeToPosForColumnArrayImpl( { if constexpr (has_nullmap) { - if ((*nullmap)[i] != 0) + if (DB::isNullAt(*nullmap, i)) continue; } - inline_memcpy(pos[i], &data[array_offsets[start + i - 1]], len * sizeof(T)); + if (len <= 4) + { + for (size_t j = 0; j < len; ++j) + tiflash_compiler_builtin_memcpy( + pos[i] + j * sizeof(T), + &data[array_offsets[start + i - 1] + j], + sizeof(T)); + } + else + { + inline_memcpy(pos[i], &data[array_offsets[start + i - 1]], len * sizeof(T)); + } pos[i] += len * sizeof(T); } } diff --git a/dbms/src/Columns/ColumnFixedString.cpp b/dbms/src/Columns/ColumnFixedString.cpp index 3176fab6783..615d5b68b55 100644 --- a/dbms/src/Columns/ColumnFixedString.cpp +++ b/dbms/src/Columns/ColumnFixedString.cpp @@ -146,7 +146,7 @@ void ColumnFixedString::countSerializeByteSizeImpl(PaddedPODArray & byte { if constexpr (has_nullmap) { - if ((*nullmap)[i] != 0) + if (DB::isNullAt(*nullmap, i)) { byte_size[i] += 1; continue; @@ -175,7 +175,7 @@ void ColumnFixedString::countSerializeByteSizeForColumnArrayImpl( { if constexpr (has_nullmap) { - if ((*nullmap)[i] != 0) + if (DB::isNullAt(*nullmap, i)) { byte_size[i] += array_offsets[i] - array_offsets[i - 1]; continue; @@ -215,7 +215,7 @@ void ColumnFixedString::serializeToPosImpl( } if constexpr (has_nullmap) { - if ((*nullmap)[i] != 0) + if (DB::isNullAt(*nullmap, i)) { for (size_t j = 0; j < n; ++j) { @@ -277,7 +277,7 @@ void ColumnFixedString::serializeToPosForColumnArrayImpl( size_t len = array_offsets[start + i] - array_offsets[start + i - 1]; if constexpr (has_nullmap) { - if ((*nullmap)[i] != 0) + if (DB::isNullAt(*nullmap, i)) continue; } diff --git a/dbms/src/Columns/ColumnNullable.cpp b/dbms/src/Columns/ColumnNullable.cpp index 40a8c1a339b..b5e098cd38e 100644 --- a/dbms/src/Columns/ColumnNullable.cpp +++ b/dbms/src/Columns/ColumnNullable.cpp @@ -287,9 +287,18 @@ void ColumnNullable::countSerializeByteSizeForCmp( const TiDB::TiDBCollatorPtr & collator, const NullMap * nullmap) const { - assert(!nullmap); - getNullMapColumn().countSerializeByteSizeForCmp(byte_size, collator, nullptr); - getNestedColumn().countSerializeByteSizeForCmp(byte_size, collator, &getNullMapData()); + if unlikely (nullmap != nullptr) + { + auto new_nullmap_col = ColumnUInt8::create(); + DB::mergeNullMap(*nullmap, getNullMapData(), new_nullmap_col->getData()); + new_nullmap_col->countSerializeByteSizeForCmp(byte_size, collator, nullptr); + getNestedColumn().countSerializeByteSizeForCmp(byte_size, collator, &(new_nullmap_col->getData())); + } + else + { + getNullMapColumn().countSerializeByteSizeForCmp(byte_size, collator, nullptr); + getNestedColumn().countSerializeByteSizeForCmp(byte_size, collator, &getNullMapData()); + } } void ColumnNullable::countSerializeByteSize(PaddedPODArray & byte_size) const { @@ -303,9 +312,32 @@ void ColumnNullable::countSerializeByteSizeForCmpColumnArray( const TiDB::TiDBCollatorPtr & collator, const NullMap * nullmap) const { - assert(!nullmap); - getNullMapColumn().countSerializeByteSizeForCmpColumnArray(byte_size, array_offsets, collator, nullptr); - getNestedColumn().countSerializeByteSizeForCmpColumnArray(byte_size, array_offsets, collator, &getNullMapData()); + const auto & nested_nullmap = getNullMapData(); + assert(nested_nullmap.size() == array_offsets.back()); + if unlikely (nullmap != nullptr) + { + assert(nullmap->size() == array_offsets.size()); + auto new_nullmap_col = ColumnUInt8::create(); + auto & new_nullmap_data = new_nullmap_col->getData(); + new_nullmap_data.assign(nested_nullmap); + for (size_t i = 0; i < array_offsets.size(); ++i) + { + if (DB::isNullAt(*nullmap, i)) + { + const auto row_size = array_offsets[i] - array_offsets[i - 1]; + const auto row_offset = array_offsets[i - 1]; + for (size_t j = row_offset; j < row_offset + row_size; ++j) + setNullAt(new_nullmap_data, j); + } + } + new_nullmap_col->countSerializeByteSizeForCmpColumnArray(byte_size, array_offsets, collator, nullptr); + getNestedColumn().countSerializeByteSizeForCmpColumnArray(byte_size, array_offsets, collator, &new_nullmap_data); + } + else + { + getNullMapColumn().countSerializeByteSizeForCmpColumnArray(byte_size, array_offsets, collator, nullptr); + getNestedColumn().countSerializeByteSizeForCmpColumnArray(byte_size, array_offsets, collator, &nested_nullmap); + } } void ColumnNullable::countSerializeByteSizeForColumnArray( PaddedPODArray & byte_size, @@ -323,9 +355,18 @@ void ColumnNullable::serializeToPosForCmp( const TiDB::TiDBCollatorPtr & collator, String * sort_key_container) const { - assert(!nullmap); - getNullMapColumn().serializeToPosForCmp(pos, start, length, nullptr, collator, sort_key_container); - getNestedColumn().serializeToPosForCmp(pos, start, length, &getNullMapData(), collator, sort_key_container); + if unlikely (nullmap != nullptr) + { + auto new_nullmap_col = ColumnUInt8::create(); + DB::mergeNullMap(*nullmap, getNullMapData(), new_nullmap_col->getData()); + getNullMapColumn().serializeToPosForCmp(pos, start, length, nullptr, collator, sort_key_container); + getNestedColumn().serializeToPosForCmp(pos, start, length, &(new_nullmap_col->getData()), collator, sort_key_container); + } + else + { + getNullMapColumn().serializeToPosForCmp(pos, start, length, nullptr, collator, sort_key_container); + getNestedColumn().serializeToPosForCmp(pos, start, length, &getNullMapData(), collator, sort_key_container); + } } void ColumnNullable::serializeToPos(PaddedPODArray & pos, size_t start, size_t length, bool has_null) const @@ -343,17 +384,47 @@ void ColumnNullable::serializeToPosForCmpColumnArray( const TiDB::TiDBCollatorPtr & collator, String * sort_key_container) const { - assert(!nullmap); - getNullMapColumn() - .serializeToPosForCmpColumnArray(pos, start, length, nullptr, array_offsets, collator, sort_key_container); - getNestedColumn().serializeToPosForCmpColumnArray( - pos, - start, - length, - &getNullMapData(), - array_offsets, - collator, - sort_key_container); + const auto & nested_nullmap = getNullMapData(); + assert(nested_nullmap.size() == array_offsets.back()); + if unlikely (nullmap != nullptr) + { + assert(nullmap->size() == array_offsets.size()); + auto new_nullmap_col = ColumnUInt8::create(); + auto & new_nullmap_data = new_nullmap_col->getData(); + new_nullmap_data.assign(nested_nullmap); + for (size_t i = start; i < start + length; ++i) + { + if (DB::isNullAt(*nullmap, i)) + { + const auto row_size = array_offsets[i] - array_offsets[i - 1]; + const auto row_offset = array_offsets[i - 1]; + for (size_t j = row_offset; j < row_offset + row_size; ++j) + setNullAt(new_nullmap_data, j); + } + } + new_nullmap_col->serializeToPosForCmpColumnArray(pos, start, length, nullptr, array_offsets, collator, sort_key_container); + getNestedColumn().serializeToPosForCmpColumnArray( + pos, + start, + length, + &new_nullmap_data, + array_offsets, + collator, + sort_key_container); + } + else + { + getNullMapColumn() + .serializeToPosForCmpColumnArray(pos, start, length, nullptr, array_offsets, collator, sort_key_container); + getNestedColumn().serializeToPosForCmpColumnArray( + pos, + start, + length, + &getNullMapData(), + array_offsets, + collator, + sort_key_container); + } } void ColumnNullable::serializeToPosForColumnArray( PaddedPODArray & pos, diff --git a/dbms/src/Columns/ColumnString.cpp b/dbms/src/Columns/ColumnString.cpp index 0789a6ac1fb..8d54b16ae65 100644 --- a/dbms/src/Columns/ColumnString.cpp +++ b/dbms/src/Columns/ColumnString.cpp @@ -566,7 +566,7 @@ void ColumnString::countSerializeByteSizeImpl( assert(sizeAt(i) > 0); if constexpr (has_nullmap) { - if ((*nullmap)[i] != 0) + if (DB::isNullAt(*nullmap, i)) { byte_size[i] += sizeof(UInt32) + 1; continue; @@ -699,11 +699,8 @@ void ColumnString::countSerializeByteSizeForColumnArrayImpl( assert(offsetAt(array_offsets[i]) - offsetAt(array_offsets[i - 1]) >= ele_count); if constexpr (has_nullmap) { - if ((*nullmap)[i] != 0) - { - byte_size[i] += (sizeof(UInt32) + 1) * ele_count; + if (DB::isNullAt(*nullmap, i)) continue; - } } if constexpr (count_code_points) @@ -721,7 +718,6 @@ void ColumnString::countSerializeByteSizeForColumnArrayImpl( } else { - // NOTE: didn't check nullmap because we have to iterate through all rows, it's slow. byte_size[i] += sizeof(UInt32) * ele_count + offsetAt(array_offsets[i]) - offsetAt(array_offsets[i - 1]); } @@ -866,7 +862,6 @@ void ColumnString::serializeToPosImpl( RUNTIME_CHECK_MSG(length <= pos.size(), "length({}) > size of pos({})", length, pos.size()); RUNTIME_CHECK_MSG(start + length <= size(), "start({}) + length({}) > size of column({})", start, length, size()); - static_assert(!(has_null && has_nullmap)); assert(!has_nullmap || (nullmap && nullmap->size() == size())); /// To avoid virtual function call of sortKey(). @@ -876,14 +871,16 @@ void ColumnString::serializeToPosImpl( { if constexpr (compare_semantics) { + static_assert(!has_null); UInt32 str_size = sizeAt(start + i); const void * src = &chars[offsetAt(start + i)]; if constexpr (has_nullmap) { - if ((*nullmap)[i] != 0) + if (DB::isNullAt(*nullmap, i)) { UInt32 str_size = 1; tiflash_compiler_builtin_memcpy(pos[i], &str_size, sizeof(UInt32)); + pos[i] += sizeof(UInt32); *(pos[i]) = '\0'; pos[i] += 1; continue; @@ -903,7 +900,7 @@ void ColumnString::serializeToPosImpl( } else { - assert(!has_nullmap); + static_assert(!has_nullmap); if constexpr (has_null) { if (pos[i] == nullptr) @@ -913,7 +910,6 @@ void ColumnString::serializeToPosImpl( UInt32 str_size = sizeAt(start + i); const void * src = &chars[offsetAt(start + i)]; - assert(!nullmap); tiflash_compiler_builtin_memcpy(pos[i], &str_size, sizeof(UInt32)); pos[i] += sizeof(UInt32); inline_memcpy(pos[i], src, str_size); @@ -1038,7 +1034,6 @@ void ColumnString::serializeToPosForColumnArrayImplType( } else { - assert(!nullmap); serializeToPosForColumnArrayImpl( pos, start, @@ -1073,19 +1068,19 @@ void ColumnString::serializeToPosForColumnArrayImpl( array_offsets.back(), size()); - static_assert(!(has_null && has_nullmap)); assert(!has_nullmap || (nullmap && nullmap->size() == array_offsets.size())); /// countSerializeByteSizeForCmpColumnArray has already checked that the size of one element is not greater than UINT32_MAX if constexpr (compare_semantics) { + static_assert(!has_null); /// To avoid virtual function call of sortKey(). const auto * derived_collator = static_cast(collator); for (size_t i = 0; i < length; ++i) { if constexpr (has_nullmap) { - if ((*nullmap)[i] != 0) + if (DB::isNullAt(*nullmap, i)) continue; } @@ -1109,7 +1104,7 @@ void ColumnString::serializeToPosForColumnArrayImpl( } else { - assert(!has_nullmap); + static_assert(!has_nullmap); for (size_t i = 0; i < length; ++i) { if constexpr (has_null) diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp index 48db5882a7a..7a8a159e217 100644 --- a/dbms/src/Columns/ColumnVector.cpp +++ b/dbms/src/Columns/ColumnVector.cpp @@ -117,7 +117,7 @@ void ColumnVector::serializeToPosImpl( } if constexpr (has_nullmap) { - if ((*nullmap)[start + i] != 0) + if (DB::isNullAt(*nullmap, start + i)) { tiflash_compiler_builtin_memcpy(pos[i], &def_val, sizeof(T)); pos[i] += sizeof(T); @@ -175,13 +175,24 @@ void ColumnVector::serializeToPosForColumnArrayImpl( if (pos[i] == nullptr) continue; } - size_t len = array_offsets[start + i] - array_offsets[start + i - 1]; if constexpr (has_nullmap) { - if ((*nullmap)[i] != 0) + if (DB::isNullAt(*nullmap, i)) continue; } - inline_memcpy(pos[i], &data[array_offsets[start + i - 1]], len * sizeof(T)); + size_t len = array_offsets[start + i] - array_offsets[start + i - 1]; + if (len <= 4) + { + for (size_t j = 0; j < len; ++j) + tiflash_compiler_builtin_memcpy( + pos[i] + j * sizeof(T), + &data[array_offsets[start + i - 1] + j], + sizeof(T)); + } + else + { + inline_memcpy(pos[i], &data[array_offsets[start + i - 1]], len * sizeof(T)); + } pos[i] += len * sizeof(T); } } diff --git a/dbms/src/Columns/IColumn.h b/dbms/src/Columns/IColumn.h index 0011d8454e0..e668841edf2 100644 --- a/dbms/src/Columns/IColumn.h +++ b/dbms/src/Columns/IColumn.h @@ -41,6 +41,22 @@ class ColumnGathererStream; using NullMap = PaddedPODArray; using ConstNullMapPtr = const NullMap *; +inline bool isNullAt(const NullMap & nullmap, size_t n) +{ + return nullmap[n] != 0; +} +inline void mergeNullMap(const NullMap & m1, const NullMap & m2, NullMap & m3) +{ + RUNTIME_CHECK(m1.size() == m2.size()); + m3.resize_fill_zero(m1.size()); + for (size_t i = 0; i < m1.size(); ++i) + m3[i] = (DB::isNullAt(m1, i) || DB::isNullAt(m2, i)); +} +inline void setNullAt(NullMap & nullmap, size_t n) +{ + nullmap[n] = 1; +} + /// Declares interface to store columns in memory. class IColumn : public COWPtr { diff --git a/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp b/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp index 035da450b7e..d2b0e629aaa 100644 --- a/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp +++ b/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp @@ -168,25 +168,26 @@ class TestColumnSerializeDeserialize : public ::testing::Test const TiDB::TiDBCollatorPtr & collator = nullptr, String * sort_key_container = nullptr) { - doTestSerializeAndDeserialize(column_ptr, false, compare_semantics, collator, sort_key_container); - doTestSerializeAndDeserialize2(column_ptr, false, compare_semantics, collator, sort_key_container); - doTestSerializeAndDeserialize(column_ptr, true, compare_semantics, collator, sort_key_container); - doTestSerializeAndDeserialize2(column_ptr, true, compare_semantics, collator, sort_key_container); + if (compare_semantics) + { + doTestSerializeAndDeserializeForCmp(column_ptr, compare_semantics, collator, sort_key_container); + } + else + { + doTestSerializeAndDeserialize(column_ptr, false); + doTestSerializeAndDeserialize2(column_ptr, false); + doTestSerializeAndDeserialize(column_ptr, true); + doTestSerializeAndDeserialize2(column_ptr, true); + } } static void doTestSerializeAndDeserialize( const ColumnPtr & column_ptr, - bool use_nt_align_buffer, - bool compare_semantics = false, - const TiDB::TiDBCollatorPtr & collator = nullptr, - String * sort_key_container = nullptr) + bool use_nt_align_buffer) { PaddedPODArray byte_size; byte_size.resize_fill_zero(column_ptr->size()); - if (!compare_semantics) - column_ptr->countSerializeByteSize(byte_size); - else - column_ptr->countSerializeByteSizeForCmp(byte_size, collator, nullptr); + column_ptr->countSerializeByteSize(byte_size); size_t total_size = 0; for (const auto size : byte_size) total_size += size; @@ -201,18 +202,12 @@ class TestColumnSerializeDeserialize : public ::testing::Test PaddedPODArray ori_pos; for (auto * ptr : pos) ori_pos.push_back(ptr); - if (!compare_semantics) - column_ptr->serializeToPos(pos, 0, byte_size.size() / 2, false); - else - column_ptr->serializeToPosForCmp(pos, 0, byte_size.size() / 2, nullptr, collator, sort_key_container); + column_ptr->serializeToPos(pos, 0, byte_size.size() / 2, false); auto new_col_ptr = column_ptr->cloneEmpty(); if (use_nt_align_buffer) new_col_ptr->reserveAlign(byte_size.size(), FULL_VECTOR_SIZE_AVX2); - if (!compare_semantics) - new_col_ptr->deserializeAndInsertFromPos(ori_pos, use_nt_align_buffer); - else - new_col_ptr->deserializeForCmpAndInsertFromPos(ori_pos, use_nt_align_buffer); + new_col_ptr->deserializeAndInsertFromPos(ori_pos, use_nt_align_buffer); current_size = 0; pos.clear(); @@ -225,23 +220,11 @@ class TestColumnSerializeDeserialize : public ::testing::Test pos.push_back(nullptr); for (auto * ptr : pos) ori_pos.push_back(ptr); - if (!compare_semantics) - column_ptr->serializeToPos(pos, byte_size.size() / 2, byte_size.size() - byte_size.size() / 2, true); - else - column_ptr->serializeToPosForCmp( - pos, - byte_size.size() / 2, - byte_size.size() - byte_size.size() / 2, - nullptr, - collator, - sort_key_container); + column_ptr->serializeToPos(pos, byte_size.size() / 2, byte_size.size() - byte_size.size() / 2, true); pos.resize(pos.size() - 1); ori_pos.resize(ori_pos.size() - 1); - if (!compare_semantics) - new_col_ptr->deserializeAndInsertFromPos(ori_pos, use_nt_align_buffer); - else - new_col_ptr->deserializeForCmpAndInsertFromPos(ori_pos, use_nt_align_buffer); + new_col_ptr->deserializeAndInsertFromPos(ori_pos, use_nt_align_buffer); current_size = 0; pos.clear(); @@ -253,15 +236,9 @@ class TestColumnSerializeDeserialize : public ::testing::Test } for (auto * ptr : pos) ori_pos.push_back(ptr); - if (!compare_semantics) - column_ptr->serializeToPos(pos, 0, byte_size.size(), true); - else - column_ptr->serializeToPosForCmp(pos, 0, byte_size.size(), nullptr, collator, sort_key_container); + column_ptr->serializeToPos(pos, 0, byte_size.size(), true); - if (!compare_semantics) - new_col_ptr->deserializeAndInsertFromPos(ori_pos, use_nt_align_buffer); - else - new_col_ptr->deserializeForCmpAndInsertFromPos(ori_pos, use_nt_align_buffer); + new_col_ptr->deserializeAndInsertFromPos(ori_pos, use_nt_align_buffer); if (use_nt_align_buffer) new_col_ptr->flushNTAlignBuffer(); @@ -270,27 +247,18 @@ class TestColumnSerializeDeserialize : public ::testing::Test for (size_t i = 0; i < column_ptr->size(); ++i) result_col_ptr->insertFrom(*column_ptr, i); - if (collator != nullptr) - checkForColumnWithCollator(std::move(result_col_ptr), std::move(new_col_ptr), collator); - else - ASSERT_COLUMN_EQ(std::move(result_col_ptr), std::move(new_col_ptr)); + ASSERT_COLUMN_EQ(std::move(result_col_ptr), std::move(new_col_ptr)); } static void doTestSerializeAndDeserialize2( const ColumnPtr & column_ptr, - bool use_nt_align_buffer, - bool compare_semantics = false, - const TiDB::TiDBCollatorPtr & collator = nullptr, - String * sort_key_container = nullptr) + bool use_nt_align_buffer) { if (column_ptr->size() < 2) return; PaddedPODArray byte_size; byte_size.resize_fill_zero(column_ptr->size()); - if (!compare_semantics) - column_ptr->countSerializeByteSize(byte_size); - else - column_ptr->countSerializeByteSizeForCmp(byte_size, collator, nullptr); + column_ptr->countSerializeByteSize(byte_size); size_t total_size = 0; for (const auto size : byte_size) total_size += size; @@ -306,20 +274,14 @@ class TestColumnSerializeDeserialize : public ::testing::Test pos.push_back(nullptr); for (auto * ptr : pos) ori_pos.push_back(ptr); - if (!compare_semantics) - column_ptr->serializeToPos(pos, 0, byte_size.size() / 2, true); - else - column_ptr->serializeToPosForCmp(pos, 0, byte_size.size() / 2, nullptr, collator, sort_key_container); + column_ptr->serializeToPos(pos, 0, byte_size.size() / 2, true); pos.resize(pos.size() - 1); ori_pos.resize(ori_pos.size() - 1); auto new_col_ptr = column_ptr->cloneEmpty(); if (use_nt_align_buffer) new_col_ptr->reserveAlign(byte_size.size(), FULL_VECTOR_SIZE_AVX2); - if (!compare_semantics) - new_col_ptr->deserializeAndInsertFromPos(ori_pos, use_nt_align_buffer); - else - new_col_ptr->deserializeForCmpAndInsertFromPos(ori_pos, use_nt_align_buffer); + new_col_ptr->deserializeAndInsertFromPos(ori_pos, use_nt_align_buffer); current_size = 0; pos.clear(); @@ -331,21 +293,9 @@ class TestColumnSerializeDeserialize : public ::testing::Test } for (auto * ptr : pos) ori_pos.push_back(ptr); - if (!compare_semantics) - column_ptr - ->serializeToPos(pos, byte_size.size() / 2 - 1, byte_size.size() - byte_size.size() / 2 + 1, false); - else - column_ptr->serializeToPosForCmp( - pos, - byte_size.size() / 2 - 1, - byte_size.size() - byte_size.size() / 2 + 1, - nullptr, - collator, - sort_key_container); - if (!compare_semantics) - new_col_ptr->deserializeAndInsertFromPos(ori_pos, use_nt_align_buffer); - else - new_col_ptr->deserializeForCmpAndInsertFromPos(ori_pos, use_nt_align_buffer); + column_ptr + ->serializeToPos(pos, byte_size.size() / 2 - 1, byte_size.size() - byte_size.size() / 2 + 1, false); + new_col_ptr->deserializeAndInsertFromPos(ori_pos, use_nt_align_buffer); current_size = 0; pos.clear(); @@ -357,15 +307,83 @@ class TestColumnSerializeDeserialize : public ::testing::Test } for (auto * ptr : pos) ori_pos.push_back(ptr); - if (!compare_semantics) - column_ptr->serializeToPos(pos, 0, byte_size.size(), true); - else - column_ptr->serializeToPosForCmp(pos, 0, byte_size.size(), nullptr, collator, sort_key_container); + column_ptr->serializeToPos(pos, 0, byte_size.size(), true); - if (!compare_semantics) new_col_ptr->deserializeAndInsertFromPos(ori_pos, use_nt_align_buffer); - else - new_col_ptr->deserializeForCmpAndInsertFromPos(ori_pos, use_nt_align_buffer); + if (use_nt_align_buffer) + new_col_ptr->flushNTAlignBuffer(); + + auto result_col_ptr = column_ptr->cloneFullColumn(); + for (size_t i = 0; i < column_ptr->size(); ++i) + result_col_ptr->insertFrom(*column_ptr, i); + + ASSERT_COLUMN_EQ(std::move(result_col_ptr), std::move(new_col_ptr)); + } + + static void doTestSerializeAndDeserializeForCmp( + const ColumnPtr & column_ptr, + bool use_nt_align_buffer, + const TiDB::TiDBCollatorPtr & collator = nullptr, + String * sort_key_container = nullptr) + { + PaddedPODArray byte_size; + byte_size.resize_fill_zero(column_ptr->size()); + column_ptr->countSerializeByteSizeForCmp(byte_size, collator, nullptr); + size_t total_size = 0; + for (const auto size : byte_size) + total_size += size; + PaddedPODArray memory(total_size); + PaddedPODArray pos; + size_t current_size = 0; + for (size_t i = 0; i < byte_size.size() / 2; ++i) + { + pos.push_back(memory.data() + current_size); + current_size += byte_size[i]; + } + PaddedPODArray ori_pos; + for (auto * ptr : pos) + ori_pos.push_back(ptr); + column_ptr->serializeToPosForCmp(pos, 0, byte_size.size() / 2, nullptr, collator, sort_key_container); + + auto new_col_ptr = column_ptr->cloneEmpty(); + if (use_nt_align_buffer) + new_col_ptr->reserveAlign(byte_size.size(), FULL_VECTOR_SIZE_AVX2); + new_col_ptr->deserializeForCmpAndInsertFromPos(ori_pos, use_nt_align_buffer); + + current_size = 0; + pos.clear(); + ori_pos.clear(); + for (size_t i = byte_size.size() / 2; i < byte_size.size(); ++i) + { + pos.push_back(memory.data() + current_size); + current_size += byte_size[i]; + } + for (auto * ptr : pos) + ori_pos.push_back(ptr); + column_ptr->serializeToPosForCmp( + pos, + byte_size.size() / 2, + byte_size.size() - byte_size.size() / 2, + nullptr, + collator, + sort_key_container); + + new_col_ptr->deserializeForCmpAndInsertFromPos(ori_pos, use_nt_align_buffer); + + current_size = 0; + pos.clear(); + ori_pos.clear(); + for (const auto size : byte_size) + { + pos.push_back(memory.data() + current_size); + current_size += size; + } + for (auto * ptr : pos) + ori_pos.push_back(ptr); + + column_ptr->serializeToPosForCmp(pos, 0, byte_size.size(), nullptr, collator, sort_key_container); + new_col_ptr->deserializeForCmpAndInsertFromPos(ori_pos, use_nt_align_buffer); + if (use_nt_align_buffer) new_col_ptr->flushNTAlignBuffer(); From a13008f5065fbd93058270df8aff5016d87a34a8 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Sat, 25 Jan 2025 22:00:43 +0800 Subject: [PATCH 05/16] remove changes in count Signed-off-by: guo-shaoge --- dbms/src/Columns/ColumnAggregateFunction.h | 6 +- dbms/src/Columns/ColumnArray.cpp | 16 +-- dbms/src/Columns/ColumnArray.h | 14 +-- dbms/src/Columns/ColumnConst.h | 6 +- dbms/src/Columns/ColumnDecimal.h | 8 +- dbms/src/Columns/ColumnFixedString.cpp | 31 +---- dbms/src/Columns/ColumnFixedString.h | 30 ++--- dbms/src/Columns/ColumnFunction.h | 6 +- dbms/src/Columns/ColumnNullable.cpp | 54 ++------- dbms/src/Columns/ColumnNullable.h | 9 +- dbms/src/Columns/ColumnString.cpp | 113 +++--------------- dbms/src/Columns/ColumnString.h | 27 ++--- dbms/src/Columns/ColumnTuple.h | 13 +- dbms/src/Columns/ColumnVector.h | 8 +- dbms/src/Columns/IColumn.h | 6 +- dbms/src/Columns/IColumnDummy.h | 6 +- .../gtest_column_serialize_deserialize.cpp | 13 +- 17 files changed, 83 insertions(+), 283 deletions(-) diff --git a/dbms/src/Columns/ColumnAggregateFunction.h b/dbms/src/Columns/ColumnAggregateFunction.h index 1c1cd619d67..61b2d8d515a 100644 --- a/dbms/src/Columns/ColumnAggregateFunction.h +++ b/dbms/src/Columns/ColumnAggregateFunction.h @@ -167,8 +167,7 @@ class ColumnAggregateFunction final : public COWPtrHelper & /* byte_size */, - const TiDB::TiDBCollatorPtr & /* collator */, - const NullMap * /* nullmap */) const override + const TiDB::TiDBCollatorPtr & /* collator */) const override { throw Exception( "Method countSerializeByteSizeForCmp is not supported for " + getName(), @@ -181,8 +180,7 @@ class ColumnAggregateFunction final : public COWPtrHelper & /* byte_size */, const IColumn::Offsets & /* offsets */, - const TiDB::TiDBCollatorPtr & /* collator */, - const NullMap * /* nullmap */) const override + const TiDB::TiDBCollatorPtr & /* collator */) const override { throw Exception( "Method countSerializeByteSizeForCmpColumnArray is not supported for " + getName(), diff --git a/dbms/src/Columns/ColumnArray.cpp b/dbms/src/Columns/ColumnArray.cpp index d6d2657c402..6fb2d039111 100644 --- a/dbms/src/Columns/ColumnArray.cpp +++ b/dbms/src/Columns/ColumnArray.cpp @@ -220,25 +220,21 @@ const char * ColumnArray::deserializeAndInsertFromArena(const char * pos, const void ColumnArray::countSerializeByteSizeForCmp( PaddedPODArray & byte_size, - const TiDB::TiDBCollatorPtr & collator, - const NullMap * nullmap) const + const TiDB::TiDBCollatorPtr & collator) const { - countSerializeByteSizeImpl(byte_size, collator, nullmap); + countSerializeByteSizeImpl(byte_size, collator); } void ColumnArray::countSerializeByteSize(PaddedPODArray & byte_size) const { - countSerializeByteSizeImpl(byte_size, nullptr, nullptr); + countSerializeByteSizeImpl(byte_size, nullptr); } template -void ColumnArray::countSerializeByteSizeImpl( - PaddedPODArray & byte_size, - const TiDB::TiDBCollatorPtr & collator, - const NullMap * nullmap) const +void ColumnArray::countSerializeByteSizeImpl(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator) + const { RUNTIME_CHECK_MSG(byte_size.size() == size(), "size of byte_size({}) != column size({})", byte_size.size(), size()); - assert(!nullmap || (nullmap->size() == size())); if unlikely (!getOffsets().empty() && getOffsets().back() > UINT32_MAX) { @@ -256,7 +252,7 @@ void ColumnArray::countSerializeByteSizeImpl( byte_size[i] += sizeof(UInt32); if constexpr (compare_semantics) - getData().countSerializeByteSizeForCmpColumnArray(byte_size, getOffsets(), collator, nullmap); + getData().countSerializeByteSizeForCmpColumnArray(byte_size, getOffsets(), collator); else getData().countSerializeByteSizeForColumnArray(byte_size, getOffsets()); } diff --git a/dbms/src/Columns/ColumnArray.h b/dbms/src/Columns/ColumnArray.h index 48c3c044ea6..9052890d1fe 100644 --- a/dbms/src/Columns/ColumnArray.h +++ b/dbms/src/Columns/ColumnArray.h @@ -45,10 +45,7 @@ class ColumnArray final : public COWPtrHelper ColumnArray(const ColumnArray &) = default; template - void countSerializeByteSizeImpl( - PaddedPODArray & byte_size, - const TiDB::TiDBCollatorPtr & collator, - const NullMap * nullmap) const; + void countSerializeByteSizeImpl(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator) const; template void serializeToPosImpl( @@ -100,17 +97,14 @@ class ColumnArray final : public COWPtrHelper String &) const override; const char * deserializeAndInsertFromArena(const char * pos, const TiDB::TiDBCollatorPtr &) override; - void countSerializeByteSizeForCmp( - PaddedPODArray & byte_size, - const TiDB::TiDBCollatorPtr & collator, - const NullMap * nullmap) const override; + void countSerializeByteSizeForCmp(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator) + const override; void countSerializeByteSize(PaddedPODArray & byte_size) const override; void countSerializeByteSizeForCmpColumnArray( PaddedPODArray & /* byte_size */, const IColumn::Offsets & /* array_offsets */, - const TiDB::TiDBCollatorPtr & /* collator */, - const NullMap * /* nullmap */) const override + const TiDB::TiDBCollatorPtr & /* collator */) const override { throw Exception( "Method countSerializeByteSizeForCmpColumnArray is not supported for " + getName(), diff --git a/dbms/src/Columns/ColumnConst.h b/dbms/src/Columns/ColumnConst.h index 6d506236116..a2a46c04754 100644 --- a/dbms/src/Columns/ColumnConst.h +++ b/dbms/src/Columns/ColumnConst.h @@ -114,8 +114,7 @@ class ColumnConst final : public COWPtrHelper void countSerializeByteSizeForCmp( PaddedPODArray & /* byte_size */, - const TiDB::TiDBCollatorPtr & /* collator */, - const NullMap * /* nullmap */) const override + const TiDB::TiDBCollatorPtr & /* collator */) const override { throw Exception( "Method countSerializeByteSizeForCmp is not supported for " + getName(), @@ -129,8 +128,7 @@ class ColumnConst final : public COWPtrHelper void countSerializeByteSizeForCmpColumnArray( PaddedPODArray & /* byte_size */, const IColumn::Offsets & /* array_offsets */, - const TiDB::TiDBCollatorPtr & /* collaotr */, - const NullMap * /* nullmap */) const override + const TiDB::TiDBCollatorPtr & /* collaotr */) const override { throw Exception( "Method countSerializeByteSizeForCmpColumnArray is not supported for " + getName(), diff --git a/dbms/src/Columns/ColumnDecimal.h b/dbms/src/Columns/ColumnDecimal.h index 91ea2ede0db..ddfacb98bae 100644 --- a/dbms/src/Columns/ColumnDecimal.h +++ b/dbms/src/Columns/ColumnDecimal.h @@ -174,10 +174,7 @@ class ColumnDecimal final : public COWPtrHelper & byte_size, - const TiDB::TiDBCollatorPtr &, - const NullMap *) const override + void countSerializeByteSizeForCmp(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr &) const override { countSerializeByteSizeImpl(byte_size, nullptr); } @@ -189,8 +186,7 @@ class ColumnDecimal final : public COWPtrHelper & byte_size, const IColumn::Offsets & array_offsets, - const TiDB::TiDBCollatorPtr &, - const NullMap *) const override + const TiDB::TiDBCollatorPtr &) const override { countSerializeByteSizeForColumnArrayImpl(byte_size, array_offsets); } diff --git a/dbms/src/Columns/ColumnFixedString.cpp b/dbms/src/Columns/ColumnFixedString.cpp index 615d5b68b55..8c808726b68 100644 --- a/dbms/src/Columns/ColumnFixedString.cpp +++ b/dbms/src/Columns/ColumnFixedString.cpp @@ -134,33 +134,18 @@ const char * ColumnFixedString::deserializeAndInsertFromArena(const char * pos, return pos + n; } -template -void ColumnFixedString::countSerializeByteSizeImpl(PaddedPODArray & byte_size, const NullMap * nullmap) const +void ColumnFixedString::countSerializeByteSizeImpl(PaddedPODArray & byte_size) const { RUNTIME_CHECK_MSG(byte_size.size() == size(), "size of byte_size({}) != column size({})", byte_size.size(), size()); - assert(!nullmap || nullmap->size() == size()); - size_t size = byte_size.size(); for (size_t i = 0; i < size; ++i) - { - if constexpr (has_nullmap) - { - if (DB::isNullAt(*nullmap, i)) - { - byte_size[i] += 1; - continue; - } - } byte_size[i] += n; - } } -template void ColumnFixedString::countSerializeByteSizeForColumnArrayImpl( PaddedPODArray & byte_size, - const IColumn::Offsets & array_offsets, - const NullMap * nullmap) const + const IColumn::Offsets & array_offsets) const { RUNTIME_CHECK_MSG( byte_size.size() == array_offsets.size(), @@ -168,21 +153,9 @@ void ColumnFixedString::countSerializeByteSizeForColumnArrayImpl( byte_size.size(), array_offsets.size()); - assert(!nullmap || nullmap->size() == array_offsets.size()); - size_t size = array_offsets.size(); for (size_t i = 0; i < size; ++i) - { - if constexpr (has_nullmap) - { - if (DB::isNullAt(*nullmap, i)) - { - byte_size[i] += array_offsets[i] - array_offsets[i - 1]; - continue; - } - } byte_size[i] += n * (array_offsets[i] - array_offsets[i - 1]); - } } void ColumnFixedString::serializeToPos(PaddedPODArray & pos, size_t start, size_t length, bool has_null) const diff --git a/dbms/src/Columns/ColumnFixedString.h b/dbms/src/Columns/ColumnFixedString.h index 1cd2713542b..b909cf37927 100644 --- a/dbms/src/Columns/ColumnFixedString.h +++ b/dbms/src/Columns/ColumnFixedString.h @@ -54,14 +54,11 @@ class ColumnFixedString final : public COWPtrHelper , chars(src.chars.begin(), src.chars.end()) , n(src.n){}; - template - void countSerializeByteSizeImpl(PaddedPODArray & byte_size, const NullMap * nullmap) const; + void countSerializeByteSizeImpl(PaddedPODArray & byte_size) const; - template void countSerializeByteSizeForColumnArrayImpl( PaddedPODArray & byte_size, - const IColumn::Offsets & array_offsets, - const NullMap * nullmap) const; + const IColumn::Offsets & array_offsets) const; template void serializeToPosImpl(PaddedPODArray & pos, size_t start, size_t length, const NullMap * nullmap) const; @@ -124,46 +121,37 @@ class ColumnFixedString final : public COWPtrHelper const char * deserializeAndInsertFromArena(const char * pos, const TiDB::TiDBCollatorPtr &) override; - void countSerializeByteSizeForCmp( - PaddedPODArray & byte_size, - const TiDB::TiDBCollatorPtr & collator, - const NullMap * nullmap) const override + void countSerializeByteSizeForCmp(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator) + const override { // collator->sortKey() will change the string length, which may exceeds n. RUNTIME_CHECK_MSG( !collator, "{} doesn't support countSerializeByteSizeForCmp when collator is not null", getName()); - if (nullmap != nullptr) - countSerializeByteSizeImpl(byte_size, nullmap); - else - countSerializeByteSizeImpl(byte_size, nullptr); + countSerializeByteSizeImpl(byte_size); } void countSerializeByteSize(PaddedPODArray & byte_size) const override { - countSerializeByteSizeImpl(byte_size, nullptr); + countSerializeByteSizeImpl(byte_size); } void countSerializeByteSizeForCmpColumnArray( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets, - const TiDB::TiDBCollatorPtr & collator, - const NullMap * nullmap) const override + const TiDB::TiDBCollatorPtr & collator) const override { RUNTIME_CHECK_MSG( !collator, "{} doesn't support countSerializeByteSizeForCmpColumnArray when collator is not null", getName()); - if (nullmap != nullptr) - countSerializeByteSizeForColumnArrayImpl(byte_size, array_offsets, nullmap); - else - countSerializeByteSizeForColumnArrayImpl(byte_size, array_offsets, nullptr); + countSerializeByteSizeForColumnArrayImpl(byte_size, array_offsets); } void countSerializeByteSizeForColumnArray( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets) const override { - countSerializeByteSizeForColumnArrayImpl(byte_size, array_offsets, nullptr); + countSerializeByteSizeForColumnArrayImpl(byte_size, array_offsets); } void serializeToPosForCmp( diff --git a/dbms/src/Columns/ColumnFunction.h b/dbms/src/Columns/ColumnFunction.h index 24fdcb3f1e7..8343ea4d6c5 100644 --- a/dbms/src/Columns/ColumnFunction.h +++ b/dbms/src/Columns/ColumnFunction.h @@ -122,8 +122,7 @@ class ColumnFunction final : public COWPtrHelper void countSerializeByteSizeForCmp( PaddedPODArray & /* byte_size */, - const TiDB::TiDBCollatorPtr & /* collator */, - const NullMap * /* nullmap */) const override + const TiDB::TiDBCollatorPtr & /* collator */) const override { throw Exception( "Method countSerializeByteSizeForCmp is not supported for " + getName(), @@ -137,8 +136,7 @@ class ColumnFunction final : public COWPtrHelper void countSerializeByteSizeForCmpColumnArray( PaddedPODArray & /* byte_size */, const IColumn::Offsets & /* offsets */, - const TiDB::TiDBCollatorPtr & /* collator */, - const NullMap * /* nullmap */) const override + const TiDB::TiDBCollatorPtr & /* collator */) const override { throw Exception( "Method countSerializeByteSizeForCmpColumnArray is not supported for " + getName(), diff --git a/dbms/src/Columns/ColumnNullable.cpp b/dbms/src/Columns/ColumnNullable.cpp index b5e098cd38e..7ae5e4c6ba4 100644 --- a/dbms/src/Columns/ColumnNullable.cpp +++ b/dbms/src/Columns/ColumnNullable.cpp @@ -284,21 +284,10 @@ const char * ColumnNullable::deserializeAndInsertFromArena(const char * pos, con void ColumnNullable::countSerializeByteSizeForCmp( PaddedPODArray & byte_size, - const TiDB::TiDBCollatorPtr & collator, - const NullMap * nullmap) const + const TiDB::TiDBCollatorPtr & collator) const { - if unlikely (nullmap != nullptr) - { - auto new_nullmap_col = ColumnUInt8::create(); - DB::mergeNullMap(*nullmap, getNullMapData(), new_nullmap_col->getData()); - new_nullmap_col->countSerializeByteSizeForCmp(byte_size, collator, nullptr); - getNestedColumn().countSerializeByteSizeForCmp(byte_size, collator, &(new_nullmap_col->getData())); - } - else - { - getNullMapColumn().countSerializeByteSizeForCmp(byte_size, collator, nullptr); - getNestedColumn().countSerializeByteSizeForCmp(byte_size, collator, &getNullMapData()); - } + getNullMapColumn().countSerializeByteSizeForCmp(byte_size, collator); + getNestedColumn().countSerializeByteSizeForCmp(byte_size, collator); } void ColumnNullable::countSerializeByteSize(PaddedPODArray & byte_size) const { @@ -309,35 +298,10 @@ void ColumnNullable::countSerializeByteSize(PaddedPODArray & byte_size) void ColumnNullable::countSerializeByteSizeForCmpColumnArray( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets, - const TiDB::TiDBCollatorPtr & collator, - const NullMap * nullmap) const + const TiDB::TiDBCollatorPtr & collator) const { - const auto & nested_nullmap = getNullMapData(); - assert(nested_nullmap.size() == array_offsets.back()); - if unlikely (nullmap != nullptr) - { - assert(nullmap->size() == array_offsets.size()); - auto new_nullmap_col = ColumnUInt8::create(); - auto & new_nullmap_data = new_nullmap_col->getData(); - new_nullmap_data.assign(nested_nullmap); - for (size_t i = 0; i < array_offsets.size(); ++i) - { - if (DB::isNullAt(*nullmap, i)) - { - const auto row_size = array_offsets[i] - array_offsets[i - 1]; - const auto row_offset = array_offsets[i - 1]; - for (size_t j = row_offset; j < row_offset + row_size; ++j) - setNullAt(new_nullmap_data, j); - } - } - new_nullmap_col->countSerializeByteSizeForCmpColumnArray(byte_size, array_offsets, collator, nullptr); - getNestedColumn().countSerializeByteSizeForCmpColumnArray(byte_size, array_offsets, collator, &new_nullmap_data); - } - else - { - getNullMapColumn().countSerializeByteSizeForCmpColumnArray(byte_size, array_offsets, collator, nullptr); - getNestedColumn().countSerializeByteSizeForCmpColumnArray(byte_size, array_offsets, collator, &nested_nullmap); - } + getNullMapColumn().countSerializeByteSizeForCmpColumnArray(byte_size, array_offsets, collator); + getNestedColumn().countSerializeByteSizeForCmpColumnArray(byte_size, array_offsets, collator); } void ColumnNullable::countSerializeByteSizeForColumnArray( PaddedPODArray & byte_size, @@ -360,7 +324,8 @@ void ColumnNullable::serializeToPosForCmp( auto new_nullmap_col = ColumnUInt8::create(); DB::mergeNullMap(*nullmap, getNullMapData(), new_nullmap_col->getData()); getNullMapColumn().serializeToPosForCmp(pos, start, length, nullptr, collator, sort_key_container); - getNestedColumn().serializeToPosForCmp(pos, start, length, &(new_nullmap_col->getData()), collator, sort_key_container); + getNestedColumn() + .serializeToPosForCmp(pos, start, length, &(new_nullmap_col->getData()), collator, sort_key_container); } else { @@ -402,7 +367,8 @@ void ColumnNullable::serializeToPosForCmpColumnArray( setNullAt(new_nullmap_data, j); } } - new_nullmap_col->serializeToPosForCmpColumnArray(pos, start, length, nullptr, array_offsets, collator, sort_key_container); + new_nullmap_col + ->serializeToPosForCmpColumnArray(pos, start, length, nullptr, array_offsets, collator, sort_key_container); getNestedColumn().serializeToPosForCmpColumnArray( pos, start, diff --git a/dbms/src/Columns/ColumnNullable.h b/dbms/src/Columns/ColumnNullable.h index ba747fe8a8e..ad4abce3916 100644 --- a/dbms/src/Columns/ColumnNullable.h +++ b/dbms/src/Columns/ColumnNullable.h @@ -77,17 +77,14 @@ class ColumnNullable final : public COWPtrHelper String &) const override; const char * deserializeAndInsertFromArena(const char * pos, const TiDB::TiDBCollatorPtr &) override; - void countSerializeByteSizeForCmp( - PaddedPODArray & byte_size, - const TiDB::TiDBCollatorPtr & collator, - const NullMap * nullmap) const override; + void countSerializeByteSizeForCmp(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator) + const override; void countSerializeByteSize(PaddedPODArray & byte_size) const override; void countSerializeByteSizeForCmpColumnArray( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets, - const TiDB::TiDBCollatorPtr & collator, - const NullMap * nullmap) const override; + const TiDB::TiDBCollatorPtr & collator) const override; void countSerializeByteSizeForColumnArray( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets) const override; diff --git a/dbms/src/Columns/ColumnString.cpp b/dbms/src/Columns/ColumnString.cpp index 8d54b16ae65..b2ef0e5187e 100644 --- a/dbms/src/Columns/ColumnString.cpp +++ b/dbms/src/Columns/ColumnString.cpp @@ -484,62 +484,31 @@ void ColumnString::getPermutationWithCollationImpl( void ColumnString::countSerializeByteSizeForCmp( PaddedPODArray & byte_size, - const TiDB::TiDBCollatorPtr & collator, - const NullMap * nullmap) const + const TiDB::TiDBCollatorPtr & collator) const { if likely (collator != nullptr) { if (collator->maxBytesForOneChar() > 1) - countSerializeByteSizeNullMap( - byte_size, - collator, - nullmap); + countSerializeByteSizeImpl(byte_size, collator); else - countSerializeByteSizeNullMap( - byte_size, - collator, - nullmap); + countSerializeByteSizeImpl(byte_size, collator); } else { - countSerializeByteSizeNullMap( - byte_size, - nullptr, - nullmap); + countSerializeByteSizeImpl(byte_size, nullptr); } } -void ColumnString::countSerializeByteSize(PaddedPODArray & byte_size) const -{ - countSerializeByteSizeNullMap( - byte_size, - nullptr, - nullptr); -} -template -ALWAYS_INLINE inline void ColumnString::countSerializeByteSizeNullMap( - PaddedPODArray & byte_size, - const TiDB::TiDBCollatorPtr & collator, - const NullMap * nullmap) const +void ColumnString::countSerializeByteSize(PaddedPODArray & byte_size) const { - if (nullmap != nullptr) - countSerializeByteSizeImpl( - byte_size, - collator, - nullmap); - else - countSerializeByteSizeImpl( - byte_size, - collator, - nullptr); + countSerializeByteSizeImpl(byte_size, nullptr); } -template +template void ColumnString::countSerializeByteSizeImpl( PaddedPODArray & byte_size, - const TiDB::TiDBCollatorPtr & collator, - const NullMap * nullmap) const + const TiDB::TiDBCollatorPtr & collator) const { RUNTIME_CHECK_MSG(byte_size.size() == size(), "size of byte_size({}) != column size({})", byte_size.size(), size()); @@ -554,24 +523,15 @@ void ColumnString::countSerializeByteSizeImpl( sizeAt(i)); } - if constexpr (compare_semantics) + if constexpr (has_collator) { RUNTIME_CHECK(collator); - assert(!has_nullmap || (nullmap && nullmap->size() == size())); const size_t size = byte_size.size(); const size_t max_bytes_one_char = collator->maxBytesForOneChar(); for (size_t i = 0; i < size; ++i) { assert(sizeAt(i) > 0); - if constexpr (has_nullmap) - { - if (DB::isNullAt(*nullmap, i)) - { - byte_size[i] += sizeof(UInt32) + 1; - continue; - } - } if constexpr (count_code_points) { @@ -587,7 +547,6 @@ void ColumnString::countSerializeByteSizeImpl( } else { - assert(!has_nullmap); size_t size = byte_size.size(); for (size_t i = 0; i < size; ++i) byte_size[i] += sizeof(UInt32) + sizeAt(i); @@ -597,30 +556,26 @@ void ColumnString::countSerializeByteSizeImpl( void ColumnString::countSerializeByteSizeForCmpColumnArray( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets, - const TiDB::TiDBCollatorPtr & collator, - const NullMap * nullmap) const + const TiDB::TiDBCollatorPtr & collator) const { if likely (collator != nullptr) { if (collator->maxBytesForOneChar() > 1) - countSerializeByteSizeForColumnArrayNullMap( + countSerializeByteSizeForColumnArrayImpl( byte_size, array_offsets, - collator, - nullmap); + collator); else - countSerializeByteSizeForColumnArrayNullMap( + countSerializeByteSizeForColumnArrayImpl( byte_size, array_offsets, - collator, - nullmap); + collator); } else { - countSerializeByteSizeForColumnArrayNullMap( + countSerializeByteSizeForColumnArrayImpl( byte_size, array_offsets, - nullptr, nullptr); } } @@ -629,40 +584,17 @@ void ColumnString::countSerializeByteSizeForColumnArray( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets) const { - countSerializeByteSizeForColumnArrayNullMap( + countSerializeByteSizeForColumnArrayImpl( byte_size, array_offsets, - nullptr, nullptr); } -template -void ColumnString::countSerializeByteSizeForColumnArrayNullMap( - PaddedPODArray & byte_size, - const IColumn::Offsets & array_offsets, - const TiDB::TiDBCollatorPtr & collator, - const NullMap * nullmap) const -{ - if (nullmap != nullptr) - countSerializeByteSizeForColumnArrayImpl( - byte_size, - array_offsets, - collator, - nullmap); - else - countSerializeByteSizeForColumnArrayImpl( - byte_size, - array_offsets, - collator, - nullptr); -} - -template +template void ColumnString::countSerializeByteSizeForColumnArrayImpl( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets, - const TiDB::TiDBCollatorPtr & collator, - const NullMap * nullmap) const + const TiDB::TiDBCollatorPtr & collator) const { RUNTIME_CHECK_MSG( byte_size.size() == array_offsets.size(), @@ -686,10 +618,9 @@ void ColumnString::countSerializeByteSizeForColumnArrayImpl( sizeAt(i)); } - if constexpr (compare_semantics) + if constexpr (has_collator) { RUNTIME_CHECK(collator); - assert(!has_nullmap || (nullmap && nullmap->size() == array_offsets.size())); size_t size = array_offsets.size(); const auto max_bytes_one_char = collator->maxBytesForOneChar(); @@ -697,11 +628,6 @@ void ColumnString::countSerializeByteSizeForColumnArrayImpl( { const size_t ele_count = array_offsets[i] - array_offsets[i - 1]; assert(offsetAt(array_offsets[i]) - offsetAt(array_offsets[i - 1]) >= ele_count); - if constexpr (has_nullmap) - { - if (DB::isNullAt(*nullmap, i)) - continue; - } if constexpr (count_code_points) { @@ -725,7 +651,6 @@ void ColumnString::countSerializeByteSizeForColumnArrayImpl( } else { - assert(!has_nullmap); size_t size = array_offsets.size(); for (size_t i = 0; i < size; ++i) byte_size[i] += sizeof(UInt32) * (array_offsets[i] - array_offsets[i - 1]) + offsetAt(array_offsets[i]) diff --git a/dbms/src/Columns/ColumnString.h b/dbms/src/Columns/ColumnString.h index 8d9ee722c14..97de1b9cf73 100644 --- a/dbms/src/Columns/ColumnString.h +++ b/dbms/src/Columns/ColumnString.h @@ -112,24 +112,14 @@ class ColumnString final : public COWPtrHelper PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator, const NullMap * nullmap) const; - template - void countSerializeByteSizeImpl( - PaddedPODArray & byte_size, - const TiDB::TiDBCollatorPtr & collator, - const NullMap * nullmap) const; - template - void countSerializeByteSizeForColumnArrayNullMap( - PaddedPODArray & byte_size, - const IColumn::Offsets & array_offsets, - const TiDB::TiDBCollatorPtr & collator, - const NullMap * nullmap) const; - template + void countSerializeByteSizeImpl(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator) const; + + template void countSerializeByteSizeForColumnArrayImpl( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets, - const TiDB::TiDBCollatorPtr & collator, - const NullMap * nullmap) const; + const TiDB::TiDBCollatorPtr & collator) const; template void serializeToPosImplType( @@ -317,17 +307,14 @@ class ColumnString final : public COWPtrHelper return pos + string_size; } - void countSerializeByteSizeForCmp( - PaddedPODArray & byte_size, - const TiDB::TiDBCollatorPtr & collator, - const NullMap * nullmap) const override; + void countSerializeByteSizeForCmp(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator) + const override; void countSerializeByteSize(PaddedPODArray & byte_size) const override; void countSerializeByteSizeForCmpColumnArray( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets, - const TiDB::TiDBCollatorPtr & collator, - const NullMap * nullmap) const override; + const TiDB::TiDBCollatorPtr & collator) const override; void countSerializeByteSizeForColumnArray( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets) const override; diff --git a/dbms/src/Columns/ColumnTuple.h b/dbms/src/Columns/ColumnTuple.h index 651a7bfca12..d049fc74ebf 100644 --- a/dbms/src/Columns/ColumnTuple.h +++ b/dbms/src/Columns/ColumnTuple.h @@ -95,13 +95,11 @@ class ColumnTuple final : public COWPtrHelper String &) const override; const char * deserializeAndInsertFromArena(const char * pos, const TiDB::TiDBCollatorPtr &) override; - void countSerializeByteSizeForCmp( - PaddedPODArray & byte_size, - const TiDB::TiDBCollatorPtr & collator, - const NullMap * nullmap) const override + void countSerializeByteSizeForCmp(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator) + const override { for (const auto & column : columns) - column->countSerializeByteSizeForCmp(byte_size, collator, nullmap); + column->countSerializeByteSizeForCmp(byte_size, collator); } void countSerializeByteSize(PaddedPODArray & byte_size) const override { @@ -112,11 +110,10 @@ class ColumnTuple final : public COWPtrHelper void countSerializeByteSizeForCmpColumnArray( PaddedPODArray & byte_size, const IColumn::Offsets & array_offsets, - const TiDB::TiDBCollatorPtr & collator, - const NullMap * nullmap) const override + const TiDB::TiDBCollatorPtr & collator) const override { for (const auto & column : columns) - column->countSerializeByteSizeForCmpColumnArray(byte_size, array_offsets, collator, nullmap); + column->countSerializeByteSizeForCmpColumnArray(byte_size, array_offsets, collator); } void countSerializeByteSizeForColumnArray( PaddedPODArray & byte_size, diff --git a/dbms/src/Columns/ColumnVector.h b/dbms/src/Columns/ColumnVector.h index 344a3d10fbe..307eba1c40c 100644 --- a/dbms/src/Columns/ColumnVector.h +++ b/dbms/src/Columns/ColumnVector.h @@ -328,10 +328,7 @@ class ColumnVector final : public COWPtrHelper & byte_size, - const TiDB::TiDBCollatorPtr &, - const NullMap *) const override + void countSerializeByteSizeForCmp(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr &) const override { countSerializeByteSize(byte_size); } @@ -340,8 +337,7 @@ class ColumnVector final : public COWPtrHelper & byte_size, const IColumn::Offsets & array_offsets, - const TiDB::TiDBCollatorPtr &, - const NullMap *) const override + const TiDB::TiDBCollatorPtr &) const override { countSerializeByteSizeForColumnArray(byte_size, array_offsets); } diff --git a/dbms/src/Columns/IColumn.h b/dbms/src/Columns/IColumn.h index e668841edf2..5286e581808 100644 --- a/dbms/src/Columns/IColumn.h +++ b/dbms/src/Columns/IColumn.h @@ -259,8 +259,7 @@ class IColumn : public COWPtr /// The byte_size.size() must be equal to the column size. virtual void countSerializeByteSizeForCmp( PaddedPODArray & /* byte_size */, - const TiDB::TiDBCollatorPtr & /* collator */, - const NullMap * /* nullmap */) const + const TiDB::TiDBCollatorPtr & /* collator */) const = 0; virtual void countSerializeByteSize(PaddedPODArray & /* byte_size */) const = 0; @@ -270,8 +269,7 @@ class IColumn : public COWPtr virtual void countSerializeByteSizeForCmpColumnArray( PaddedPODArray & /* byte_size */, const Offsets & /* array_offsets */, - const TiDB::TiDBCollatorPtr & /* collator */, - const NullMap * /* nullmap */) const + const TiDB::TiDBCollatorPtr & /* collator */) const = 0; virtual void countSerializeByteSizeForColumnArray( PaddedPODArray & /* byte_size */, diff --git a/dbms/src/Columns/IColumnDummy.h b/dbms/src/Columns/IColumnDummy.h index 3b0550a9651..250f4bedc14 100644 --- a/dbms/src/Columns/IColumnDummy.h +++ b/dbms/src/Columns/IColumnDummy.h @@ -90,8 +90,7 @@ class IColumnDummy : public IColumn void countSerializeByteSizeForCmp( PaddedPODArray & /* byte_size */, - const TiDB::TiDBCollatorPtr & /* collator */, - const NullMap * /* nullmap */) const override + const TiDB::TiDBCollatorPtr & /* collator */) const override { throw Exception( "Method countSerializeByteSizeForCmp is not supported for " + getName(), @@ -105,8 +104,7 @@ class IColumnDummy : public IColumn void countSerializeByteSizeForCmpColumnArray( PaddedPODArray & /* byte_size */, const IColumn::Offsets & /* array_offsets */, - const TiDB::TiDBCollatorPtr & /* collator */, - const NullMap * /* nullmap */) const override + const TiDB::TiDBCollatorPtr & /* collator */) const override { throw Exception( "Method countSerializeByteSizeForCmpColumnArray is not supported for " + getName(), diff --git a/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp b/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp index d2b0e629aaa..de6f0c5e49d 100644 --- a/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp +++ b/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp @@ -181,9 +181,7 @@ class TestColumnSerializeDeserialize : public ::testing::Test } } - static void doTestSerializeAndDeserialize( - const ColumnPtr & column_ptr, - bool use_nt_align_buffer) + static void doTestSerializeAndDeserialize(const ColumnPtr & column_ptr, bool use_nt_align_buffer) { PaddedPODArray byte_size; byte_size.resize_fill_zero(column_ptr->size()); @@ -250,9 +248,7 @@ class TestColumnSerializeDeserialize : public ::testing::Test ASSERT_COLUMN_EQ(std::move(result_col_ptr), std::move(new_col_ptr)); } - static void doTestSerializeAndDeserialize2( - const ColumnPtr & column_ptr, - bool use_nt_align_buffer) + static void doTestSerializeAndDeserialize2(const ColumnPtr & column_ptr, bool use_nt_align_buffer) { if (column_ptr->size() < 2) return; @@ -293,8 +289,7 @@ class TestColumnSerializeDeserialize : public ::testing::Test } for (auto * ptr : pos) ori_pos.push_back(ptr); - column_ptr - ->serializeToPos(pos, byte_size.size() / 2 - 1, byte_size.size() - byte_size.size() / 2 + 1, false); + column_ptr->serializeToPos(pos, byte_size.size() / 2 - 1, byte_size.size() - byte_size.size() / 2 + 1, false); new_col_ptr->deserializeAndInsertFromPos(ori_pos, use_nt_align_buffer); current_size = 0; @@ -309,7 +304,7 @@ class TestColumnSerializeDeserialize : public ::testing::Test ori_pos.push_back(ptr); column_ptr->serializeToPos(pos, 0, byte_size.size(), true); - new_col_ptr->deserializeAndInsertFromPos(ori_pos, use_nt_align_buffer); + new_col_ptr->deserializeAndInsertFromPos(ori_pos, use_nt_align_buffer); if (use_nt_align_buffer) new_col_ptr->flushNTAlignBuffer(); From 817854b7a9418c3b3be90ab9529fc505ca4b51ab Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Sun, 26 Jan 2025 11:45:04 +0800 Subject: [PATCH 06/16] refine Signed-off-by: guo-shaoge --- dbms/src/Columns/ColumnNullable.cpp | 14 +++++++++----- dbms/src/Columns/ColumnString.h | 8 +------- dbms/src/Columns/IColumn.h | 5 +++-- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/dbms/src/Columns/ColumnNullable.cpp b/dbms/src/Columns/ColumnNullable.cpp index 7ae5e4c6ba4..f2de4b51bb8 100644 --- a/dbms/src/Columns/ColumnNullable.cpp +++ b/dbms/src/Columns/ColumnNullable.cpp @@ -321,9 +321,12 @@ void ColumnNullable::serializeToPosForCmp( { if unlikely (nullmap != nullptr) { + // This code path is not efficient, because of the temporary `new_nullmap_col`. + // But only got this code path when the column is like ColumnNullable(ColumnTuple(ColumnNullable)), + // which is rare for TiFlash, because ColumnTuple is not used for now. auto new_nullmap_col = ColumnUInt8::create(); - DB::mergeNullMap(*nullmap, getNullMapData(), new_nullmap_col->getData()); - getNullMapColumn().serializeToPosForCmp(pos, start, length, nullptr, collator, sort_key_container); + DB::mergeNullMap(start, length, *nullmap, getNullMapData(), new_nullmap_col->getData()); + new_nullmap_col->serializeToPosForCmp(pos, start, length, nullptr, collator, sort_key_container); getNestedColumn() .serializeToPosForCmp(pos, start, length, &(new_nullmap_col->getData()), collator, sort_key_container); } @@ -350,10 +353,11 @@ void ColumnNullable::serializeToPosForCmpColumnArray( String * sort_key_container) const { const auto & nested_nullmap = getNullMapData(); - assert(nested_nullmap.size() == array_offsets.back()); - if unlikely (nullmap != nullptr) + RUNTIME_CHECK(nested_nullmap.size() == array_offsets.back()); + if (nullmap != nullptr) { - assert(nullmap->size() == array_offsets.size()); + // Got this code path when the column is like ColumnNullable(ColumnArray(ColumnNullable)), + RUNTIME_CHECK(nullmap->size() == array_offsets.size()); auto new_nullmap_col = ColumnUInt8::create(); auto & new_nullmap_data = new_nullmap_col->getData(); new_nullmap_data.assign(nested_nullmap); diff --git a/dbms/src/Columns/ColumnString.h b/dbms/src/Columns/ColumnString.h index 97de1b9cf73..dd4f3c1f70f 100644 --- a/dbms/src/Columns/ColumnString.h +++ b/dbms/src/Columns/ColumnString.h @@ -107,14 +107,8 @@ class ColumnString final : public COWPtrHelper } } - template - ALWAYS_INLINE inline void countSerializeByteSizeNullMap( - PaddedPODArray & byte_size, - const TiDB::TiDBCollatorPtr & collator, - const NullMap * nullmap) const; - template + template void countSerializeByteSizeImpl(PaddedPODArray & byte_size, const TiDB::TiDBCollatorPtr & collator) const; - template void countSerializeByteSizeForColumnArrayImpl( PaddedPODArray & byte_size, diff --git a/dbms/src/Columns/IColumn.h b/dbms/src/Columns/IColumn.h index 5286e581808..be2171a0f4b 100644 --- a/dbms/src/Columns/IColumn.h +++ b/dbms/src/Columns/IColumn.h @@ -45,11 +45,12 @@ inline bool isNullAt(const NullMap & nullmap, size_t n) { return nullmap[n] != 0; } -inline void mergeNullMap(const NullMap & m1, const NullMap & m2, NullMap & m3) +inline void mergeNullMap(size_t start, size_t length, const NullMap & m1, const NullMap & m2, NullMap & m3) { RUNTIME_CHECK(m1.size() == m2.size()); + RUNTIME_CHECK(start + length < m1.size()); m3.resize_fill_zero(m1.size()); - for (size_t i = 0; i < m1.size(); ++i) + for (size_t i = start; i < start + length; ++i) m3[i] = (DB::isNullAt(m1, i) || DB::isNullAt(m2, i)); } inline void setNullAt(NullMap & nullmap, size_t n) From d9aeae7e502010f45dddb63b38e10454b1983dd9 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Sun, 26 Jan 2025 13:08:22 +0800 Subject: [PATCH 07/16] refine Signed-off-by: guo-shaoge --- dbms/src/Columns/ColumnArray.cpp | 10 ++-- dbms/src/Columns/ColumnDecimal.cpp | 46 ++++++------------- dbms/src/Columns/ColumnFixedString.cpp | 7 ++- dbms/src/Columns/ColumnNullable.cpp | 41 +++++++++++++---- dbms/src/Columns/ColumnString.cpp | 8 +--- dbms/src/Columns/ColumnVector.cpp | 41 +++++++++++++++-- dbms/src/Columns/ColumnVector.h | 16 +------ dbms/src/Columns/IColumn.h | 29 ++++++------ .../gtest_column_serialize_deserialize.cpp | 39 ++++++++++++++-- 9 files changed, 144 insertions(+), 93 deletions(-) diff --git a/dbms/src/Columns/ColumnArray.cpp b/dbms/src/Columns/ColumnArray.cpp index 6fb2d039111..6f5f68b9a7b 100644 --- a/dbms/src/Columns/ColumnArray.cpp +++ b/dbms/src/Columns/ColumnArray.cpp @@ -266,17 +266,17 @@ void ColumnArray::serializeToPosForCmp( String * sort_key_container) const { if (nullmap != nullptr) - serializeToPosImpl(pos, start, length, collator, sort_key_container, nullmap); + serializeToPosImpl(pos, start, length, collator, sort_key_container, nullmap); else - serializeToPosImpl(pos, start, length, collator, sort_key_container, nullmap); + serializeToPosImpl(pos, start, length, collator, sort_key_container, nullptr); } void ColumnArray::serializeToPos(PaddedPODArray & pos, size_t start, size_t length, bool has_null) const { if (has_null) - serializeToPosImpl(pos, start, length, nullptr, nullptr, nullptr); + serializeToPosImpl(pos, start, length, nullptr, nullptr, nullptr); else - serializeToPosImpl(pos, start, length, nullptr, nullptr, nullptr); + serializeToPosImpl(pos, start, length, nullptr, nullptr, nullptr); } template @@ -306,7 +306,7 @@ void ColumnArray::serializeToPosImpl( UInt32 len = sizeAt(start + i); if constexpr (has_nullmap) { - if (DB::isNullAt(*nullmap, i)) + if (DB::isNullAt(*nullmap, start + i)) len = 0; } tiflash_compiler_builtin_memcpy(pos[i], &len, sizeof(UInt32)); diff --git a/dbms/src/Columns/ColumnDecimal.cpp b/dbms/src/Columns/ColumnDecimal.cpp index 867b86e1b31..a15e4734396 100644 --- a/dbms/src/Columns/ColumnDecimal.cpp +++ b/dbms/src/Columns/ColumnDecimal.cpp @@ -214,30 +214,19 @@ void ColumnDecimal::serializeToPosImpl( if (pos[i] == nullptr) continue; } + if constexpr (has_nullmap) + { + if (DB::isNullAt(*nullmap, start + i)) + pos[i] = serializeDecimal256Helper(pos[i], def_val); + continue; + } if constexpr (compare_semantics && is_Decimal256) { - if constexpr (has_nullmap) - { - if (DB::isNullAt(*nullmap, i)) - { - pos[i] = serializeDecimal256Helper(pos[i], def_val); - continue; - } - } pos[i] = serializeDecimal256Helper(pos[i], data[start + i]); } else { - if constexpr (has_nullmap) - { - if (DB::isNullAt(*nullmap, i)) - { - tiflash_compiler_builtin_memcpy(pos[i], &def_val, sizeof(T)); - pos[i] += sizeof(T); - continue; - } - } tiflash_compiler_builtin_memcpy(pos[i], &data[start + i], sizeof(T)); pos[i] += sizeof(T); } @@ -267,7 +256,7 @@ void ColumnDecimal::serializeToPosForColumnArrayImpl( size()); static_assert(!(has_null && has_nullmap)); - assert(!has_nullmap || (nullmap && nullmap->size() == array_offsets.size())); + assert(!has_nullmap || (nullmap && nullmap->size() == array_offsets.back())); for (size_t i = 0; i < length; ++i) { @@ -276,32 +265,27 @@ void ColumnDecimal::serializeToPosForColumnArrayImpl( if (pos[i] == nullptr) continue; } + if constexpr (has_nullmap) + { + if (DB::isNullAt(*nullmap, start + i)) + continue; + } size_t len = array_offsets[start + i] - array_offsets[start + i - 1]; if constexpr (compare_semantics && is_Decimal256) { - if constexpr (has_nullmap) - { - if (DB::isNullAt(*nullmap, i)) - continue; - } for (size_t j = 0; j < len; ++j) pos[i] = serializeDecimal256Helper(pos[i], data[array_offsets[start + i - 1] + j]); } else { - if constexpr (has_nullmap) - { - if (DB::isNullAt(*nullmap, i)) - continue; - } if (len <= 4) { for (size_t j = 0; j < len; ++j) tiflash_compiler_builtin_memcpy( - pos[i] + j * sizeof(T), - &data[array_offsets[start + i - 1] + j], - sizeof(T)); + pos[i] + j * sizeof(T), + &data[array_offsets[start + i - 1] + j], + sizeof(T)); } else { diff --git a/dbms/src/Columns/ColumnFixedString.cpp b/dbms/src/Columns/ColumnFixedString.cpp index 8c808726b68..244f21bbc54 100644 --- a/dbms/src/Columns/ColumnFixedString.cpp +++ b/dbms/src/Columns/ColumnFixedString.cpp @@ -188,7 +188,7 @@ void ColumnFixedString::serializeToPosImpl( } if constexpr (has_nullmap) { - if (DB::isNullAt(*nullmap, i)) + if (DB::isNullAt(*nullmap, start + i)) { for (size_t j = 0; j < n; ++j) { @@ -247,13 +247,12 @@ void ColumnFixedString::serializeToPosForColumnArrayImpl( if (pos[i] == nullptr) continue; } - size_t len = array_offsets[start + i] - array_offsets[start + i - 1]; if constexpr (has_nullmap) { - if (DB::isNullAt(*nullmap, i)) + if (DB::isNullAt(*nullmap, start + i)) continue; } - + size_t len = array_offsets[start + i] - array_offsets[start + i - 1]; inline_memcpy(pos[i], &chars[n * array_offsets[start + i - 1]], n * len); pos[i] += n * len; } diff --git a/dbms/src/Columns/ColumnNullable.cpp b/dbms/src/Columns/ColumnNullable.cpp index f2de4b51bb8..5cd04385a57 100644 --- a/dbms/src/Columns/ColumnNullable.cpp +++ b/dbms/src/Columns/ColumnNullable.cpp @@ -354,6 +354,9 @@ void ColumnNullable::serializeToPosForCmpColumnArray( { const auto & nested_nullmap = getNullMapData(); RUNTIME_CHECK(nested_nullmap.size() == array_offsets.back()); + const auto nested_start = array_offsets[start - 1]; + const auto nested_length = array_offsets[start + length - 1] - array_offsets[start - 1]; + if (nullmap != nullptr) { // Got this code path when the column is like ColumnNullable(ColumnArray(ColumnNullable)), @@ -371,27 +374,45 @@ void ColumnNullable::serializeToPosForCmpColumnArray( setNullAt(new_nullmap_data, j); } } + // new_nullmap_col + // ->serializeToPosForCmpColumnArray(pos, start, length, nullptr, array_offsets, collator, sort_key_container); + // getNestedColumn().serializeToPosForCmpColumnArray( + // pos, + // start, + // length, + // &new_nullmap_data, + // array_offsets, + // collator, + // sort_key_container); new_nullmap_col - ->serializeToPosForCmpColumnArray(pos, start, length, nullptr, array_offsets, collator, sort_key_container); - getNestedColumn().serializeToPosForCmpColumnArray( + ->serializeToPosForCmp(pos, nested_start, nested_length, nullptr, collator, sort_key_container); + getNestedColumn().serializeToPosForCmp( pos, - start, - length, + nested_start, + nested_length, &new_nullmap_data, - array_offsets, collator, sort_key_container); } else { + // getNullMapColumn() + // .serializeToPosForCmpColumnArray(pos, start, length, nullptr, array_offsets, collator, sort_key_container); + // getNestedColumn().serializeToPosForCmpColumnArray( + // pos, + // start, + // length, + // &getNullMapData(), + // array_offsets, + // collator, + // sort_key_container); getNullMapColumn() - .serializeToPosForCmpColumnArray(pos, start, length, nullptr, array_offsets, collator, sort_key_container); - getNestedColumn().serializeToPosForCmpColumnArray( + .serializeToPosForCmp(pos, nested_start, nested_length, nullptr, collator, sort_key_container); + getNestedColumn().serializeToPosForCmp( pos, - start, - length, + nested_start, + nested_length, &getNullMapData(), - array_offsets, collator, sort_key_container); } diff --git a/dbms/src/Columns/ColumnString.cpp b/dbms/src/Columns/ColumnString.cpp index b2ef0e5187e..8af794c664e 100644 --- a/dbms/src/Columns/ColumnString.cpp +++ b/dbms/src/Columns/ColumnString.cpp @@ -499,7 +499,6 @@ void ColumnString::countSerializeByteSizeForCmp( } } - void ColumnString::countSerializeByteSize(PaddedPODArray & byte_size) const { countSerializeByteSizeImpl(byte_size, nullptr); @@ -532,7 +531,6 @@ void ColumnString::countSerializeByteSizeImpl( for (size_t i = 0; i < size; ++i) { assert(sizeAt(i) > 0); - if constexpr (count_code_points) { const auto num_char = UTF8::countCodePoints(&chars[offsetAt(i)], sizeAt(i) - 1); @@ -628,7 +626,6 @@ void ColumnString::countSerializeByteSizeForColumnArrayImpl( { const size_t ele_count = array_offsets[i] - array_offsets[i - 1]; assert(offsetAt(array_offsets[i]) - offsetAt(array_offsets[i - 1]) >= ele_count); - if constexpr (count_code_points) { size_t cur_row_bytes = 0; @@ -801,7 +798,7 @@ void ColumnString::serializeToPosImpl( const void * src = &chars[offsetAt(start + i)]; if constexpr (has_nullmap) { - if (DB::isNullAt(*nullmap, i)) + if (DB::isNullAt(*nullmap, start + i)) { UInt32 str_size = 1; tiflash_compiler_builtin_memcpy(pos[i], &str_size, sizeof(UInt32)); @@ -1005,10 +1002,9 @@ void ColumnString::serializeToPosForColumnArrayImpl( { if constexpr (has_nullmap) { - if (DB::isNullAt(*nullmap, i)) + if (DB::isNullAt(*nullmap, start + i)) continue; } - for (size_t j = array_offsets[start + i - 1]; j < array_offsets[start + i]; ++j) { UInt32 str_size = sizeAt(j); diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp index 7a8a159e217..f1e07bef6bc 100644 --- a/dbms/src/Columns/ColumnVector.cpp +++ b/dbms/src/Columns/ColumnVector.cpp @@ -93,6 +93,21 @@ void ColumnVector::serializeToPos(PaddedPODArray & pos, size_t start, serializeToPosImpl(pos, start, length, nullptr); } +template +void ColumnVector::serializeToPosForCmp( + PaddedPODArray & pos, + size_t start, + size_t length, + const NullMap * nullmap, + const TiDB::TiDBCollatorPtr &, + String *) const +{ + if (nullmap != nullptr) + serializeToPosImpl(pos, start, length, nullmap); + else + serializeToPosImpl(pos, start, length, nullptr); +} + template template void ColumnVector::serializeToPosImpl( @@ -143,6 +158,22 @@ void ColumnVector::serializeToPosForColumnArray( serializeToPosForColumnArrayImpl(pos, start, length, array_offsets, nullptr); } +template +void ColumnVector::serializeToPosForCmpColumnArray( + PaddedPODArray & pos, + size_t start, + size_t length, + const NullMap * nullmap, + const IColumn::Offsets & array_offsets, + const TiDB::TiDBCollatorPtr &, + String *) const +{ + if (nullmap != nullptr) + serializeToPosForColumnArrayImpl(pos, start, length, array_offsets, nullmap); + else + serializeToPosForColumnArrayImpl(pos, start, length, array_offsets, nullptr); +} + template template void ColumnVector::serializeToPosForColumnArrayImpl( @@ -175,19 +206,19 @@ void ColumnVector::serializeToPosForColumnArrayImpl( if (pos[i] == nullptr) continue; } + size_t len = array_offsets[start + i] - array_offsets[start + i - 1]; if constexpr (has_nullmap) { - if (DB::isNullAt(*nullmap, i)) + if (DB::isNullAt(*nullmap, start + i)) continue; } - size_t len = array_offsets[start + i] - array_offsets[start + i - 1]; if (len <= 4) { for (size_t j = 0; j < len; ++j) tiflash_compiler_builtin_memcpy( - pos[i] + j * sizeof(T), - &data[array_offsets[start + i - 1] + j], - sizeof(T)); + pos[i] + j * sizeof(T), + &data[array_offsets[start + i - 1] + j], + sizeof(T)); } else { diff --git a/dbms/src/Columns/ColumnVector.h b/dbms/src/Columns/ColumnVector.h index 307eba1c40c..fd49028bf70 100644 --- a/dbms/src/Columns/ColumnVector.h +++ b/dbms/src/Columns/ColumnVector.h @@ -351,13 +351,7 @@ class ColumnVector final : public COWPtrHelper(pos, start, length, nullmap); - else - serializeToPosImpl(pos, start, length, nullptr); - } + String *) const override; void serializeToPos(PaddedPODArray & pos, size_t start, size_t length, bool has_null) const override; void serializeToPosForCmpColumnArray( @@ -367,13 +361,7 @@ class ColumnVector final : public COWPtrHelper(pos, start, length, array_offsets, nullmap); - else - serializeToPosForColumnArrayImpl(pos, start, length, array_offsets, nullptr); - } + String *) const override; void serializeToPosForColumnArray( PaddedPODArray & pos, size_t start, diff --git a/dbms/src/Columns/IColumn.h b/dbms/src/Columns/IColumn.h index be2171a0f4b..7912208e514 100644 --- a/dbms/src/Columns/IColumn.h +++ b/dbms/src/Columns/IColumn.h @@ -282,24 +282,34 @@ class IColumn : public COWPtr /// Note: /// 1. The pos.size() must be greater than or equal to length. /// 2. If has_null is true, then the pos[i] could be nullptr, which means the i-th element does not need to be serialized. - virtual void serializeToPosForCmp( + virtual void serializeToPos( PaddedPODArray & /* pos */, size_t /* start */, size_t /* length */, - const NullMap * /*nullmap*/, - const TiDB::TiDBCollatorPtr & /* collator */, - String * /* sort_key_container */) const + bool /* has_null */) const = 0; - virtual void serializeToPos( + // Similar to serializeToPos, but there are two changes to make sure compare semantics is kept: + // 1. For ColumnString with collator, this method decode using collator first and then serialize to pos. + // 2. For ColumnNullable, a default value of nested column will be serialized if this row is null. + virtual void serializeToPosForCmp( PaddedPODArray & /* pos */, size_t /* start */, size_t /* length */, - bool /* has_null */) const + const NullMap * /*nullmap*/, + const TiDB::TiDBCollatorPtr & /* collator */, + String * /* sort_key_container */) const = 0; /// Serialize data of column from start to start + length into pointer of pos and forward each pos[i] to the end of /// serialized data. /// Only called by ColumnArray. + virtual void serializeToPosForColumnArray( + PaddedPODArray & /* pos */, + size_t /* start */, + size_t /* length */, + bool /* has_null */, + const Offsets & /* array_offsets */) const + = 0; virtual void serializeToPosForCmpColumnArray( PaddedPODArray & /* pos */, size_t /* start */, @@ -309,13 +319,6 @@ class IColumn : public COWPtr const TiDB::TiDBCollatorPtr & /* collator */, String * /* sort_key_container */) const = 0; - virtual void serializeToPosForColumnArray( - PaddedPODArray & /* pos */, - size_t /* start */, - size_t /* length */, - bool /* has_null */, - const Offsets & /* array_offsets */) const - = 0; /// Deserialize and insert data from pos and forward each pos[i] to the end of serialized data. /// Note: diff --git a/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp b/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp index de6f0c5e49d..53aaea3e972 100644 --- a/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp +++ b/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp @@ -41,7 +41,7 @@ class TestColumnSerializeDeserialize : public ::testing::Test if (!compare_semantics) column_ptr->countSerializeByteSize(byte_size); else - column_ptr->countSerializeByteSizeForCmp(byte_size, collator, nullptr); + column_ptr->countSerializeByteSizeForCmp(byte_size, collator); ASSERT_EQ(byte_size.size(), result_byte_size.size()); for (size_t i = 0; i < byte_size.size(); ++i) ASSERT_EQ(byte_size[i], i + result_byte_size[i]); @@ -62,7 +62,7 @@ class TestColumnSerializeDeserialize : public ::testing::Test if (!compare_semantics) column_array->countSerializeByteSize(byte_size); else - column_array->countSerializeByteSizeForCmp(byte_size, collator, nullptr); + column_array->countSerializeByteSizeForCmp(byte_size, collator); ASSERT_EQ(byte_size.size(), result_byte_size.size()); for (size_t i = 0; i < byte_size.size(); ++i) ASSERT_EQ(byte_size[i], sizeof(UInt32) + i + result_byte_size[i]); @@ -148,6 +148,19 @@ class TestColumnSerializeDeserialize : public ::testing::Test } } } + // else if (result_col_ptr->getFamilyName() == String("Nullable")) + // { + // for (size_t i = 0; i < result_col_ptr->size(); ++i) + // { + // ASSERT_EQ(result_col_ptr->isNullAt(i), new_col_ptr->isNullAt(i)); + // } + // const auto & nested_result_col_ptr = checkAndGetColumn(result_col_ptr.get())->getNestedColumnPtr(); + // const auto & nested_new_col_ptr = checkAndGetColumn(new_col_ptr.get())->getNestedColumnPtr(); + // checkForColumnWithCollator( + // nested_result_col_ptr, + // nested_new_col_ptr, + // collator); + // } else { for (size_t i = 0; i < result_col_ptr->size(); ++i) @@ -156,8 +169,9 @@ class TestColumnSerializeDeserialize : public ::testing::Test if (result_col_ptr->isNullAt(i)) continue; auto res = result_col_ptr->getDataAt(i); - auto sort_key = collator->sortKey(res.data, res.size, sort_key_container); - ASSERT_TRUE(sort_key == new_col_ptr->getDataAt(i)); + auto res_sort_key = collator->sortKey(res.data, res.size, sort_key_container); + auto act = new_col_ptr->getDataAt(i); + ASSERT_TRUE(res_sort_key == act); } } } @@ -323,7 +337,7 @@ class TestColumnSerializeDeserialize : public ::testing::Test { PaddedPODArray byte_size; byte_size.resize_fill_zero(column_ptr->size()); - column_ptr->countSerializeByteSizeForCmp(byte_size, collator, nullptr); + column_ptr->countSerializeByteSizeForCmp(byte_size, collator); size_t total_size = 0; for (const auto size : byte_size) total_size += size; @@ -603,6 +617,21 @@ try testCountSerializeByteSize(col_nullable_array_vec, {1 + 4 + 4, 1 + 4 + 8, 1 + 4 + 12}); testSerializeAndDeserialize(col_nullable_array_vec); testSerializeAndDeserialize(col_nullable_array_vec, true, nullptr, nullptr); + + // ColumnNullable(ColumnArray(ColumnNullable(ColumnString))) + auto col_offsets_1 = createColumn({1, 3, 6}).column; + auto col_array_string = ColumnArray::create(col_nullable_string, col_offsets_1); + auto col_nullable_array_string = ColumnNullable::create(col_array_string, createColumn({0, 1, 0}).column); + testCountSerializeByteSize(col_nullable_array_string, + {1 + 4 + 1 + 4 + 4, + 1 + 4 + 2 + 8 + 4, + 1 + 4 + 3 + 12 + 7}, true, nullptr); + testSerializeAndDeserialize(col_nullable_array_string); + testSerializeAndDeserialize(col_nullable_array_string, true, collator_utf8_bin, &sort_key_container); + testSerializeAndDeserialize(col_nullable_array_string, true, collator_utf8_general_ci, &sort_key_container); + testSerializeAndDeserialize(col_nullable_array_string, true, collator_utf8_unicode_ci, &sort_key_container); + + // ColumnNullable(ColumnTuple(ColumnNullable(ColumnString))) } CATCH From eb01d9eb3ca6d1653a3f9107d5690f228171c9ac Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Sun, 26 Jan 2025 18:16:57 +0800 Subject: [PATCH 08/16] refine Signed-off-by: guo-shaoge --- dbms/src/Columns/ColumnArray.cpp | 32 +++++- dbms/src/Columns/ColumnDecimal.cpp | 8 +- dbms/src/Columns/ColumnFixedString.cpp | 2 +- dbms/src/Columns/ColumnNullable.cpp | 104 +++--------------- dbms/src/Columns/ColumnString.cpp | 2 +- dbms/src/Columns/ColumnVector.cpp | 8 +- .../gtest_column_serialize_deserialize.cpp | 41 ++++--- 7 files changed, 78 insertions(+), 119 deletions(-) diff --git a/dbms/src/Columns/ColumnArray.cpp b/dbms/src/Columns/ColumnArray.cpp index 6f5f68b9a7b..c3dbbcf5cac 100644 --- a/dbms/src/Columns/ColumnArray.cpp +++ b/dbms/src/Columns/ColumnArray.cpp @@ -266,17 +266,41 @@ void ColumnArray::serializeToPosForCmp( String * sort_key_container) const { if (nullmap != nullptr) - serializeToPosImpl(pos, start, length, collator, sort_key_container, nullmap); + serializeToPosImpl( + pos, + start, + length, + collator, + sort_key_container, + nullmap); else - serializeToPosImpl(pos, start, length, collator, sort_key_container, nullptr); + serializeToPosImpl( + pos, + start, + length, + collator, + sort_key_container, + nullptr); } void ColumnArray::serializeToPos(PaddedPODArray & pos, size_t start, size_t length, bool has_null) const { if (has_null) - serializeToPosImpl(pos, start, length, nullptr, nullptr, nullptr); + serializeToPosImpl( + pos, + start, + length, + nullptr, + nullptr, + nullptr); else - serializeToPosImpl(pos, start, length, nullptr, nullptr, nullptr); + serializeToPosImpl( + pos, + start, + length, + nullptr, + nullptr, + nullptr); } template diff --git a/dbms/src/Columns/ColumnDecimal.cpp b/dbms/src/Columns/ColumnDecimal.cpp index a15e4734396..fe57e19cba8 100644 --- a/dbms/src/Columns/ColumnDecimal.cpp +++ b/dbms/src/Columns/ColumnDecimal.cpp @@ -256,7 +256,7 @@ void ColumnDecimal::serializeToPosForColumnArrayImpl( size()); static_assert(!(has_null && has_nullmap)); - assert(!has_nullmap || (nullmap && nullmap->size() == array_offsets.back())); + RUNTIME_CHECK(!has_nullmap || (nullmap && nullmap->size() == array_offsets.size())); for (size_t i = 0; i < length; ++i) { @@ -283,9 +283,9 @@ void ColumnDecimal::serializeToPosForColumnArrayImpl( { for (size_t j = 0; j < len; ++j) tiflash_compiler_builtin_memcpy( - pos[i] + j * sizeof(T), - &data[array_offsets[start + i - 1] + j], - sizeof(T)); + pos[i] + j * sizeof(T), + &data[array_offsets[start + i - 1] + j], + sizeof(T)); } else { diff --git a/dbms/src/Columns/ColumnFixedString.cpp b/dbms/src/Columns/ColumnFixedString.cpp index 244f21bbc54..de75137a705 100644 --- a/dbms/src/Columns/ColumnFixedString.cpp +++ b/dbms/src/Columns/ColumnFixedString.cpp @@ -238,7 +238,7 @@ void ColumnFixedString::serializeToPosForColumnArrayImpl( size()); static_assert(!(has_null && has_nullmap)); - assert(!has_nullmap || (nullmap && nullmap->size() == array_offsets.size())); + RUNTIME_CHECK(!has_nullmap || (nullmap && nullmap->size() == array_offsets.size())); for (size_t i = 0; i < length; ++i) { diff --git a/dbms/src/Columns/ColumnNullable.cpp b/dbms/src/Columns/ColumnNullable.cpp index 5cd04385a57..d93a4d6c983 100644 --- a/dbms/src/Columns/ColumnNullable.cpp +++ b/dbms/src/Columns/ColumnNullable.cpp @@ -319,22 +319,10 @@ void ColumnNullable::serializeToPosForCmp( const TiDB::TiDBCollatorPtr & collator, String * sort_key_container) const { - if unlikely (nullmap != nullptr) - { - // This code path is not efficient, because of the temporary `new_nullmap_col`. - // But only got this code path when the column is like ColumnNullable(ColumnTuple(ColumnNullable)), - // which is rare for TiFlash, because ColumnTuple is not used for now. - auto new_nullmap_col = ColumnUInt8::create(); - DB::mergeNullMap(start, length, *nullmap, getNullMapData(), new_nullmap_col->getData()); - new_nullmap_col->serializeToPosForCmp(pos, start, length, nullptr, collator, sort_key_container); - getNestedColumn() - .serializeToPosForCmp(pos, start, length, &(new_nullmap_col->getData()), collator, sort_key_container); - } - else - { - getNullMapColumn().serializeToPosForCmp(pos, start, length, nullptr, collator, sort_key_container); - getNestedColumn().serializeToPosForCmp(pos, start, length, &getNullMapData(), collator, sort_key_container); - } + // Nested ColumnNullable like ColumnNullable(ColumnArray(ColumnNullable(ColumnXXX))) not support. + RUNTIME_CHECK_MSG(!nullmap, "serializeToPosForCmp cannot handle nested nullable"); + getNullMapColumn().serializeToPosForCmp(pos, start, length, nullptr, collator, sort_key_container); + getNestedColumn().serializeToPosForCmp(pos, start, length, &getNullMapData(), collator, sort_key_container); } void ColumnNullable::serializeToPos(PaddedPODArray & pos, size_t start, size_t length, bool has_null) const @@ -344,78 +332,18 @@ void ColumnNullable::serializeToPos(PaddedPODArray & pos, size_t start, } void ColumnNullable::serializeToPosForCmpColumnArray( - PaddedPODArray & pos, - size_t start, - size_t length, - const NullMap * nullmap, - const IColumn::Offsets & array_offsets, - const TiDB::TiDBCollatorPtr & collator, - String * sort_key_container) const -{ - const auto & nested_nullmap = getNullMapData(); - RUNTIME_CHECK(nested_nullmap.size() == array_offsets.back()); - const auto nested_start = array_offsets[start - 1]; - const auto nested_length = array_offsets[start + length - 1] - array_offsets[start - 1]; - - if (nullmap != nullptr) - { - // Got this code path when the column is like ColumnNullable(ColumnArray(ColumnNullable)), - RUNTIME_CHECK(nullmap->size() == array_offsets.size()); - auto new_nullmap_col = ColumnUInt8::create(); - auto & new_nullmap_data = new_nullmap_col->getData(); - new_nullmap_data.assign(nested_nullmap); - for (size_t i = start; i < start + length; ++i) - { - if (DB::isNullAt(*nullmap, i)) - { - const auto row_size = array_offsets[i] - array_offsets[i - 1]; - const auto row_offset = array_offsets[i - 1]; - for (size_t j = row_offset; j < row_offset + row_size; ++j) - setNullAt(new_nullmap_data, j); - } - } - // new_nullmap_col - // ->serializeToPosForCmpColumnArray(pos, start, length, nullptr, array_offsets, collator, sort_key_container); - // getNestedColumn().serializeToPosForCmpColumnArray( - // pos, - // start, - // length, - // &new_nullmap_data, - // array_offsets, - // collator, - // sort_key_container); - new_nullmap_col - ->serializeToPosForCmp(pos, nested_start, nested_length, nullptr, collator, sort_key_container); - getNestedColumn().serializeToPosForCmp( - pos, - nested_start, - nested_length, - &new_nullmap_data, - collator, - sort_key_container); - } - else - { - // getNullMapColumn() - // .serializeToPosForCmpColumnArray(pos, start, length, nullptr, array_offsets, collator, sort_key_container); - // getNestedColumn().serializeToPosForCmpColumnArray( - // pos, - // start, - // length, - // &getNullMapData(), - // array_offsets, - // collator, - // sort_key_container); - getNullMapColumn() - .serializeToPosForCmp(pos, nested_start, nested_length, nullptr, collator, sort_key_container); - getNestedColumn().serializeToPosForCmp( - pos, - nested_start, - nested_length, - &getNullMapData(), - collator, - sort_key_container); - } + PaddedPODArray & /* pos */, + size_t /* start */, + size_t /* length */, + const NullMap * /* nullmap */, + const IColumn::Offsets & /* array_offsets */, + const TiDB::TiDBCollatorPtr & /* collator */, + String * /* sort_key_container */) const +{ + // Doesn't support ColumnArray(ColumnNullable(ColumnXXX)) + throw Exception( + "Method serializeToPosForCmpColumnArray is not supported for " + getName(), + ErrorCodes::NOT_IMPLEMENTED); } void ColumnNullable::serializeToPosForColumnArray( PaddedPODArray & pos, diff --git a/dbms/src/Columns/ColumnString.cpp b/dbms/src/Columns/ColumnString.cpp index 8af794c664e..6d498491638 100644 --- a/dbms/src/Columns/ColumnString.cpp +++ b/dbms/src/Columns/ColumnString.cpp @@ -990,7 +990,7 @@ void ColumnString::serializeToPosForColumnArrayImpl( array_offsets.back(), size()); - assert(!has_nullmap || (nullmap && nullmap->size() == array_offsets.size())); + RUNTIME_CHECK(!has_nullmap || (nullmap && nullmap->size() == array_offsets.size())); /// countSerializeByteSizeForCmpColumnArray has already checked that the size of one element is not greater than UINT32_MAX if constexpr (compare_semantics) diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp index f1e07bef6bc..2c449b06915 100644 --- a/dbms/src/Columns/ColumnVector.cpp +++ b/dbms/src/Columns/ColumnVector.cpp @@ -197,7 +197,7 @@ void ColumnVector::serializeToPosForColumnArrayImpl( size()); static_assert(!(has_null && has_nullmap)); - assert(!has_nullmap || (nullmap && nullmap->size() == array_offsets.size())); + RUNTIME_CHECK(!has_nullmap || (nullmap && nullmap->size() == array_offsets.size())); for (size_t i = 0; i < length; ++i) { @@ -216,9 +216,9 @@ void ColumnVector::serializeToPosForColumnArrayImpl( { for (size_t j = 0; j < len; ++j) tiflash_compiler_builtin_memcpy( - pos[i] + j * sizeof(T), - &data[array_offsets[start + i - 1] + j], - sizeof(T)); + pos[i] + j * sizeof(T), + &data[array_offsets[start + i - 1] + j], + sizeof(T)); } else { diff --git a/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp b/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp index 53aaea3e972..a6d5419030e 100644 --- a/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp +++ b/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp @@ -618,20 +618,25 @@ try testSerializeAndDeserialize(col_nullable_array_vec); testSerializeAndDeserialize(col_nullable_array_vec, true, nullptr, nullptr); - // ColumnNullable(ColumnArray(ColumnNullable(ColumnString))) - auto col_offsets_1 = createColumn({1, 3, 6}).column; - auto col_array_string = ColumnArray::create(col_nullable_string, col_offsets_1); - auto col_nullable_array_string = ColumnNullable::create(col_array_string, createColumn({0, 1, 0}).column); - testCountSerializeByteSize(col_nullable_array_string, - {1 + 4 + 1 + 4 + 4, - 1 + 4 + 2 + 8 + 4, - 1 + 4 + 3 + 12 + 7}, true, nullptr); - testSerializeAndDeserialize(col_nullable_array_string); - testSerializeAndDeserialize(col_nullable_array_string, true, collator_utf8_bin, &sort_key_container); - testSerializeAndDeserialize(col_nullable_array_string, true, collator_utf8_general_ci, &sort_key_container); - testSerializeAndDeserialize(col_nullable_array_string, true, collator_utf8_unicode_ci, &sort_key_container); - - // ColumnNullable(ColumnTuple(ColumnNullable(ColumnString))) + // ColumnNullable(ColumnArray(ColumnString)) + auto col_string = createColumn({"123", "2", "34", "456", "5678", "6"}).column; + auto col_array_string = ColumnArray::create(col_vector, col_offsets); + auto col_nullable_array_string = ColumnNullable::create(col_array_vec, createColumn({1, 0, 1}).column); + testSerializeAndDeserialize(col_nullable_array_vec); + testSerializeAndDeserialize(col_nullable_array_vec, true, nullptr, nullptr); + + // Nested ColumnNullable like ColumnNullable(ColumnArray(ColumnNullable(ColumnString))) not support. + // auto col_offsets_1 = createColumn({1, 3, 6}).column; + // auto col_array_string = ColumnArray::create(col_nullable_string, col_offsets_1); + // auto col_nullable_array_string = ColumnNullable::create(col_array_string, createColumn({0, 1, 0}).column); + // testCountSerializeByteSize(col_nullable_array_string, + // {1 + 4 + 1 + 4 + 4, + // 1 + 4 + 2 + 8 + 4, + // 1 + 4 + 3 + 12 + 7}, true, nullptr); + // testSerializeAndDeserialize(col_nullable_array_string); + // testSerializeAndDeserialize(col_nullable_array_string, true, collator_utf8_bin, &sort_key_container); + // testSerializeAndDeserialize(col_nullable_array_string, true, collator_utf8_general_ci, &sort_key_container); + // testSerializeAndDeserialize(col_nullable_array_string, true, collator_utf8_unicode_ci, &sort_key_container); } CATCH @@ -667,9 +672,10 @@ try auto col_array_nullable_string = ColumnArray::create(col_nullable_string, col_offsets); testCountSerializeByteSize(col_array_nullable_string, {4 + 5 + 4, 4 + 10 + 4, 4 + 15 + 7}); testSerializeAndDeserialize(col_array_nullable_string); - testSerializeAndDeserialize(col_array_nullable_string, true, collator_utf8_bin, &sort_key_container); - testSerializeAndDeserialize(col_array_nullable_string, true, collator_utf8_general_ci, &sort_key_container); - testSerializeAndDeserialize(col_array_nullable_string, true, collator_utf8_unicode_ci, &sort_key_container); + // compare semantics not support ColumnArray(ColumnNullable(ColumnString)). + // testSerializeAndDeserialize(col_array_nullable_string, true, collator_utf8_bin, &sort_key_container); + // testSerializeAndDeserialize(col_array_nullable_string, true, collator_utf8_general_ci, &sort_key_container); + // testSerializeAndDeserialize(col_array_nullable_string, true, collator_utf8_unicode_ci, &sort_key_container); // ColumnArray(ColumnDecimal) auto col_decimal_256 = createColumn( @@ -820,6 +826,7 @@ try true, collator_utf8_unicode_ci); + // ColumnString String sort_key_container; testSerializeAndDeserialize(col_string, true, collator_utf8_bin, &sort_key_container); testSerializeAndDeserialize(col_string, true, collator_utf8_general_ci, &sort_key_container); From 62fbbede009c9a949ca677d1eb4055c550730283 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Sun, 26 Jan 2025 18:20:56 +0800 Subject: [PATCH 09/16] refine Signed-off-by: guo-shaoge --- dbms/src/Columns/ColumnArray.cpp | 2 +- dbms/src/Columns/ColumnDecimal.cpp | 2 +- dbms/src/Columns/ColumnFixedString.cpp | 2 +- dbms/src/Columns/ColumnString.cpp | 2 +- dbms/src/Columns/ColumnVector.cpp | 2 +- .../tests/gtest_column_serialize_deserialize.cpp | 13 ------------- 6 files changed, 5 insertions(+), 18 deletions(-) diff --git a/dbms/src/Columns/ColumnArray.cpp b/dbms/src/Columns/ColumnArray.cpp index c3dbbcf5cac..ad20ed2a43b 100644 --- a/dbms/src/Columns/ColumnArray.cpp +++ b/dbms/src/Columns/ColumnArray.cpp @@ -316,7 +316,7 @@ void ColumnArray::serializeToPosImpl( RUNTIME_CHECK_MSG(start + length <= size(), "start({}) + length({}) > size of column({})", start, length, size()); static_assert(!(has_null && has_nullmap)); - assert(!has_nullmap || (nullmap && nullmap->size() == size())); + RUNTIME_CHECK(!has_nullmap || (nullmap && nullmap->size() == size())); /// countSerializeByteSize has already checked that the size of one element is not greater than UINT32_MAX for (size_t i = 0; i < length; ++i) diff --git a/dbms/src/Columns/ColumnDecimal.cpp b/dbms/src/Columns/ColumnDecimal.cpp index fe57e19cba8..0ae87c6ab66 100644 --- a/dbms/src/Columns/ColumnDecimal.cpp +++ b/dbms/src/Columns/ColumnDecimal.cpp @@ -204,7 +204,7 @@ void ColumnDecimal::serializeToPosImpl( RUNTIME_CHECK_MSG(start + length <= size(), "start({}) + length({}) > size of column({})", start, length, size()); static_assert(!(has_null && has_nullmap)); - assert(!has_nullmap || (nullmap && nullmap->size() == size())); + RUNTIME_CHECK(!has_nullmap || (nullmap && nullmap->size() == size())); T def_val{}; for (size_t i = 0; i < length; ++i) diff --git a/dbms/src/Columns/ColumnFixedString.cpp b/dbms/src/Columns/ColumnFixedString.cpp index de75137a705..b945fba690e 100644 --- a/dbms/src/Columns/ColumnFixedString.cpp +++ b/dbms/src/Columns/ColumnFixedString.cpp @@ -177,7 +177,7 @@ void ColumnFixedString::serializeToPosImpl( RUNTIME_CHECK_MSG(start + length <= size(), "start({}) + length({}) > size of column({})", start, length, size()); static_assert(!(has_null && has_nullmap)); - assert(!has_nullmap || (nullmap && nullmap->size() == size())); + RUNTIME_CHECK(!has_nullmap || (nullmap && nullmap->size() == size())); for (size_t i = 0; i < length; ++i) { diff --git a/dbms/src/Columns/ColumnString.cpp b/dbms/src/Columns/ColumnString.cpp index 6d498491638..2fcd6d83c5b 100644 --- a/dbms/src/Columns/ColumnString.cpp +++ b/dbms/src/Columns/ColumnString.cpp @@ -784,7 +784,7 @@ void ColumnString::serializeToPosImpl( RUNTIME_CHECK_MSG(length <= pos.size(), "length({}) > size of pos({})", length, pos.size()); RUNTIME_CHECK_MSG(start + length <= size(), "start({}) + length({}) > size of column({})", start, length, size()); - assert(!has_nullmap || (nullmap && nullmap->size() == size())); + RUNTIME_CHECK(!has_nullmap || (nullmap && nullmap->size() == size())); /// To avoid virtual function call of sortKey(). const auto * derived_collator = static_cast(collator); diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp index 2c449b06915..9b0769436f8 100644 --- a/dbms/src/Columns/ColumnVector.cpp +++ b/dbms/src/Columns/ColumnVector.cpp @@ -120,7 +120,7 @@ void ColumnVector::serializeToPosImpl( RUNTIME_CHECK_MSG(start + length <= size(), "start({}) + length({}) > size of column({})", start, length, size()); static_assert(!(has_null && has_nullmap)); - assert(!has_nullmap || (nullmap && nullmap->size() == size())); + RUNTIME_CHECK(!has_nullmap || (nullmap && nullmap->size() == size())); T def_val{}; for (size_t i = 0; i < length; ++i) diff --git a/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp b/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp index a6d5419030e..69052507f65 100644 --- a/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp +++ b/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp @@ -148,19 +148,6 @@ class TestColumnSerializeDeserialize : public ::testing::Test } } } - // else if (result_col_ptr->getFamilyName() == String("Nullable")) - // { - // for (size_t i = 0; i < result_col_ptr->size(); ++i) - // { - // ASSERT_EQ(result_col_ptr->isNullAt(i), new_col_ptr->isNullAt(i)); - // } - // const auto & nested_result_col_ptr = checkAndGetColumn(result_col_ptr.get())->getNestedColumnPtr(); - // const auto & nested_new_col_ptr = checkAndGetColumn(new_col_ptr.get())->getNestedColumnPtr(); - // checkForColumnWithCollator( - // nested_result_col_ptr, - // nested_new_col_ptr, - // collator); - // } else { for (size_t i = 0; i < result_col_ptr->size(); ++i) From 0ce2f534078ce84c3e373c5f725ba8e0786f4508 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Mon, 27 Jan 2025 13:52:48 +0800 Subject: [PATCH 10/16] refine Signed-off-by: guo-shaoge --- dbms/src/Columns/ColumnNullable.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dbms/src/Columns/ColumnNullable.cpp b/dbms/src/Columns/ColumnNullable.cpp index d93a4d6c983..c9c7e6e82ee 100644 --- a/dbms/src/Columns/ColumnNullable.cpp +++ b/dbms/src/Columns/ColumnNullable.cpp @@ -340,9 +340,11 @@ void ColumnNullable::serializeToPosForCmpColumnArray( const TiDB::TiDBCollatorPtr & /* collator */, String * /* sort_key_container */) const { - // Doesn't support ColumnArray(ColumnNullable(ColumnXXX)) + // Unable to handle ColumnArray(ColumnNullable(ColumnXXX)). Because the pos vector corresponds to the rows of ColumnArray, + // while ColumnNullable::nullmap corresponds to the rows of ColumnNullable. + // This means it's not easy to correctly serialize the row in ColumnNullable to the corresponding position in pos. throw Exception( - "Method serializeToPosForCmpColumnArray is not supported for " + getName(), + "serializeToPosForCmpColumnArray cannot handle ColumnArray(" + getName() + ")", ErrorCodes::NOT_IMPLEMENTED); } void ColumnNullable::serializeToPosForColumnArray( From c4c0eb9d3c10eda4d9ecb89287165af88715820b Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Mon, 27 Jan 2025 16:54:02 +0800 Subject: [PATCH 11/16] refine Signed-off-by: guo-shaoge --- dbms/src/Columns/ColumnDecimal.cpp | 2 +- dbms/src/Columns/ColumnFixedString.cpp | 7 ++--- dbms/src/Columns/ColumnNullable.h | 5 +++- dbms/src/Columns/ColumnString.cpp | 29 ++++++++++++------- dbms/src/Columns/ColumnVector.cpp | 4 +-- dbms/src/Columns/IColumn.h | 19 +++--------- .../gtest_column_serialize_deserialize.cpp | 2 ++ 7 files changed, 33 insertions(+), 35 deletions(-) diff --git a/dbms/src/Columns/ColumnDecimal.cpp b/dbms/src/Columns/ColumnDecimal.cpp index 0ae87c6ab66..610739d15c9 100644 --- a/dbms/src/Columns/ColumnDecimal.cpp +++ b/dbms/src/Columns/ColumnDecimal.cpp @@ -206,7 +206,7 @@ void ColumnDecimal::serializeToPosImpl( static_assert(!(has_null && has_nullmap)); RUNTIME_CHECK(!has_nullmap || (nullmap && nullmap->size() == size())); - T def_val{}; + static constexpr T def_val{}; for (size_t i = 0; i < length; ++i) { if constexpr (has_null) diff --git a/dbms/src/Columns/ColumnFixedString.cpp b/dbms/src/Columns/ColumnFixedString.cpp index b945fba690e..217b19cf913 100644 --- a/dbms/src/Columns/ColumnFixedString.cpp +++ b/dbms/src/Columns/ColumnFixedString.cpp @@ -190,11 +190,8 @@ void ColumnFixedString::serializeToPosImpl( { if (DB::isNullAt(*nullmap, start + i)) { - for (size_t j = 0; j < n; ++j) - { - *(pos[i]) = '\0'; - pos[i] += 1; - } + memset(pos[i], '\0', n); + pos[i] += n; continue; } } diff --git a/dbms/src/Columns/ColumnNullable.h b/dbms/src/Columns/ColumnNullable.h index ad4abce3916..5b216572e66 100644 --- a/dbms/src/Columns/ColumnNullable.h +++ b/dbms/src/Columns/ColumnNullable.h @@ -60,7 +60,10 @@ class ColumnNullable final : public COWPtrHelper std::string getName() const override { return "Nullable(" + nested_column->getName() + ")"; } MutableColumnPtr cloneResized(size_t size) const override; size_t size() const override { return static_cast(*null_map).size(); } - bool isNullAt(size_t n) const override { return static_cast(*null_map).getData()[n] != 0; } + bool isNullAt(size_t n) const override + { + return DB::isNullAt(static_cast(*null_map).getData(), n); + } Field operator[](size_t n) const override; void get(size_t n, Field & res) const override; UInt64 get64(size_t n) const override { return nested_column->get64(n); } diff --git a/dbms/src/Columns/ColumnString.cpp b/dbms/src/Columns/ColumnString.cpp index 2fcd6d83c5b..d2e1ab47bec 100644 --- a/dbms/src/Columns/ColumnString.cpp +++ b/dbms/src/Columns/ColumnString.cpp @@ -35,6 +35,18 @@ extern const int PARAMETER_OUT_OF_BOUND; extern const int SIZES_OF_COLUMNS_DOESNT_MATCH; } // namespace ErrorCodes +struct ColumnStringDefaultValue +{ + char mem[sizeof(UInt32) + 1] = {0}; + ColumnStringDefaultValue() + { + UInt32 str_size = 1; + tiflash_compiler_builtin_memcpy(&mem[0], &str_size, sizeof(str_size)); + } +}; + +static ColumnStringDefaultValue col_str_def_val; + MutableColumnPtr ColumnString::cloneResized(size_t to_size) const { auto res = ColumnString::create(); @@ -784,6 +796,7 @@ void ColumnString::serializeToPosImpl( RUNTIME_CHECK_MSG(length <= pos.size(), "length({}) > size of pos({})", length, pos.size()); RUNTIME_CHECK_MSG(start + length <= size(), "start({}) + length({}) > size of column({})", start, length, size()); + static_assert(!(has_null && has_nullmap)); RUNTIME_CHECK(!has_nullmap || (nullmap && nullmap->size() == size())); /// To avoid virtual function call of sortKey(). @@ -793,21 +806,17 @@ void ColumnString::serializeToPosImpl( { if constexpr (compare_semantics) { - static_assert(!has_null); - UInt32 str_size = sizeAt(start + i); - const void * src = &chars[offsetAt(start + i)]; if constexpr (has_nullmap) { if (DB::isNullAt(*nullmap, start + i)) { - UInt32 str_size = 1; - tiflash_compiler_builtin_memcpy(pos[i], &str_size, sizeof(UInt32)); - pos[i] += sizeof(UInt32); - *(pos[i]) = '\0'; - pos[i] += 1; + tiflash_compiler_builtin_memcpy(pos[i], &col_str_def_val.mem[0], sizeof(col_str_def_val.mem)); + pos[i] += sizeof(col_str_def_val.mem); continue; } } + UInt32 str_size = sizeAt(start + i); + const void * src = &chars[offsetAt(start + i)]; auto sort_key = derived_collator->sortKey(reinterpret_cast(src), str_size - 1, *sort_key_container); // For terminating zero. @@ -822,7 +831,6 @@ void ColumnString::serializeToPosImpl( } else { - static_assert(!has_nullmap); if constexpr (has_null) { if (pos[i] == nullptr) @@ -990,12 +998,12 @@ void ColumnString::serializeToPosForColumnArrayImpl( array_offsets.back(), size()); + static_assert(!(has_null && has_nullmap)); RUNTIME_CHECK(!has_nullmap || (nullmap && nullmap->size() == array_offsets.size())); /// countSerializeByteSizeForCmpColumnArray has already checked that the size of one element is not greater than UINT32_MAX if constexpr (compare_semantics) { - static_assert(!has_null); /// To avoid virtual function call of sortKey(). const auto * derived_collator = static_cast(collator); for (size_t i = 0; i < length; ++i) @@ -1025,7 +1033,6 @@ void ColumnString::serializeToPosForColumnArrayImpl( } else { - static_assert(!has_nullmap); for (size_t i = 0; i < length; ++i) { if constexpr (has_null) diff --git a/dbms/src/Columns/ColumnVector.cpp b/dbms/src/Columns/ColumnVector.cpp index 9b0769436f8..be92ec62c5f 100644 --- a/dbms/src/Columns/ColumnVector.cpp +++ b/dbms/src/Columns/ColumnVector.cpp @@ -122,7 +122,7 @@ void ColumnVector::serializeToPosImpl( static_assert(!(has_null && has_nullmap)); RUNTIME_CHECK(!has_nullmap || (nullmap && nullmap->size() == size())); - T def_val{}; + static constexpr T def_val{}; for (size_t i = 0; i < length; ++i) { if constexpr (has_null) @@ -206,12 +206,12 @@ void ColumnVector::serializeToPosForColumnArrayImpl( if (pos[i] == nullptr) continue; } - size_t len = array_offsets[start + i] - array_offsets[start + i - 1]; if constexpr (has_nullmap) { if (DB::isNullAt(*nullmap, start + i)) continue; } + size_t len = array_offsets[start + i] - array_offsets[start + i - 1]; if (len <= 4) { for (size_t j = 0; j < len; ++j) diff --git a/dbms/src/Columns/IColumn.h b/dbms/src/Columns/IColumn.h index 7912208e514..309c0c5dcac 100644 --- a/dbms/src/Columns/IColumn.h +++ b/dbms/src/Columns/IColumn.h @@ -45,18 +45,6 @@ inline bool isNullAt(const NullMap & nullmap, size_t n) { return nullmap[n] != 0; } -inline void mergeNullMap(size_t start, size_t length, const NullMap & m1, const NullMap & m2, NullMap & m3) -{ - RUNTIME_CHECK(m1.size() == m2.size()); - RUNTIME_CHECK(start + length < m1.size()); - m3.resize_fill_zero(m1.size()); - for (size_t i = start; i < start + length; ++i) - m3[i] = (DB::isNullAt(m1, i) || DB::isNullAt(m2, i)); -} -inline void setNullAt(NullMap & nullmap, size_t n) -{ - nullmap[n] = 1; -} /// Declares interface to store columns in memory. class IColumn : public COWPtr @@ -288,9 +276,9 @@ class IColumn : public COWPtr size_t /* length */, bool /* has_null */) const = 0; - // Similar to serializeToPos, but there are two changes to make sure compare semantics is kept: - // 1. For ColumnString with collator, this method decode using collator first and then serialize to pos. - // 2. For ColumnNullable, a default value of nested column will be serialized if this row is null. + /// Similar to serializeToPos, but there are two changes to make sure compare semantics is kept: + /// 1. For ColumnString with collator, this method first decode collator and then serialize to pos. + /// 2. For ColumnNullable(ColumnXXX), a default value of the nested column will be serialized if this row is null. virtual void serializeToPosForCmp( PaddedPODArray & /* pos */, size_t /* start */, @@ -310,6 +298,7 @@ class IColumn : public COWPtr bool /* has_null */, const Offsets & /* array_offsets */) const = 0; + /// Similary to serializeToPosForCmp, but only called by ColumnArray. virtual void serializeToPosForCmpColumnArray( PaddedPODArray & /* pos */, size_t /* start */, diff --git a/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp b/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp index 69052507f65..26eabc0b616 100644 --- a/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp +++ b/dbms/src/Columns/tests/gtest_column_serialize_deserialize.cpp @@ -612,6 +612,8 @@ try testSerializeAndDeserialize(col_nullable_array_vec); testSerializeAndDeserialize(col_nullable_array_vec, true, nullptr, nullptr); + // ColumnArray(ColumnNullable(ColumnVector)) not support. + // Nested ColumnNullable like ColumnNullable(ColumnArray(ColumnNullable(ColumnString))) not support. // auto col_offsets_1 = createColumn({1, 3, 6}).column; // auto col_array_string = ColumnArray::create(col_nullable_string, col_offsets_1); From 7b8c19fc64e2cf19c95178f7d37d371db6dafbd4 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Mon, 27 Jan 2025 19:28:44 +0800 Subject: [PATCH 12/16] agg batch serialize Signed-off-by: guo-shaoge --- dbms/src/Common/ColumnsHashing.h | 219 ++++++++++++++++++++++++++- dbms/src/Interpreters/Aggregator.cpp | 85 ++++++++--- dbms/src/Interpreters/Aggregator.h | 6 +- 3 files changed, 279 insertions(+), 31 deletions(-) diff --git a/dbms/src/Common/ColumnsHashing.h b/dbms/src/Common/ColumnsHashing.h index a03136bbed8..426cbe22d3b 100644 --- a/dbms/src/Common/ColumnsHashing.h +++ b/dbms/src/Common/ColumnsHashing.h @@ -50,6 +50,7 @@ struct HashMethodOneNumber using KeyHolderType = FieldType; static constexpr bool is_serialized_key = false; + static constexpr bool batch_get_key_holder = false; const FieldType * vec; @@ -87,17 +88,105 @@ struct HashMethodOneNumber const FieldType * getKeyData() const { return vec; } }; +class KeyStringBatchHandlerBase +{ +private: + size_t batch_row_idx = 0; + std::vector sort_key_containers{}; + std::vector batch_rows{}; + + template + size_t prepareNextBatchType( + const UInt8 * chars, + const IColumn::Offsets & offsets, + size_t batch_size, + const TiDB::TiDBCollatorPtr & collator) + { + const auto * derived_collator = static_cast(collator); + for (size_t i = 0; i < batch_size; ++i) + { + const auto row = batch_row_idx + i; + const auto last_offset = offsets[row - 1]; + // Remove last zero byte. + StringRef key(chars + last_offset, offsets[row] - offsets[row -1 ] - 1); + if constexpr (has_collator) + key = derived_collator->sortKey(key.data, key.size, sort_key_containers[i]); + + batch_rows[i] = key; + } + return 0; + } + + void santityCheck() const + { + // Make sure init() has called. + assert(sort_key_containers.size() == batch_rows.size() && !sort_key_containers.empty()); + } + +protected: + void init(size_t start_row, size_t batch_size) + { + batch_row_idx = start_row; + sort_key_containers.resize(batch_size); + batch_rows.resize(batch_size); + } + +public: + size_t prepareNextBatch( + const UInt8 * chars, + const IColumn::Offsets & offsets, + size_t batch_size, + const TiDB::TiDBCollatorPtr & collator) + { + if likely (collator) + { +#define M(VAR_PREFIX, COLLATOR_NAME, IMPL_TYPE, COLLATOR_ID) \ + case (COLLATOR_ID): \ + { \ + return prepareNextBatchType(chars, offsets, batch_size, collator); \ + break; \ + } + + switch (collator->getCollatorId()) + { + APPLY_FOR_COLLATOR_TYPES(M) + default: + { + throw Exception(fmt::format("unexpected collator: {}", collator->getCollatorId())); + } + }; +#undef M + } + else + { + return prepareNextBatchType(chars, offsets, batch_size, collator); + } + } + + // NOTE: i is the index of mini batch, it's not the row index of Column. + ALWAYS_INLINE inline ArenaKeyHolder getKeyHolderBatch(size_t i, Arena * pool) const + { + santityCheck(); + assert(i < batch_rows.size()); + return ArenaKeyHolder{batch_rows[i], pool}; + } +}; /// For the case when there is one string key. -template +template struct HashMethodString - : public columns_hashing_impl::HashMethodBase, Value, Mapped, use_cache> + : public columns_hashing_impl::HashMethodBase, Value, Mapped, use_cache> + , KeyStringBatchHandlerBase { using Self = HashMethodString; using Base = columns_hashing_impl::HashMethodBase; using KeyHolderType = ArenaKeyHolder; + static_assert(get_key_holder_batch_size == 0 || get_key_holder_batch_size >= 256); + using BatchHandlerBase = KeyStringBatchHandlerBase; + static constexpr bool is_serialized_key = false; + static constexpr bool batch_get_key_holder = (get_key_holder_batch_size > 0); const IColumn::Offset * offsets; const UInt8 * chars; @@ -116,11 +205,25 @@ struct HashMethodString collator = collators[0]; } + void initBatchHandler(size_t start_row) + { + assert(batch_get_key_holder); + BatchHandlerBase::init(start_row, get_key_holder_batch_size); + } + + size_t prepareNextBatch(Arena *) + { + assert(batch_get_key_holder); + return BatchHandlerBase::prepareNextBatch(chars, *offsets, get_key_holder_batch_size, collator); + } + ALWAYS_INLINE inline KeyHolderType getKeyHolder( ssize_t row, [[maybe_unused]] Arena * pool, std::vector & sort_key_containers) const { + assert(!batch_get_key_holder); + auto last_offset = row == 0 ? 0 : offsets[row - 1]; // Remove last zero byte. StringRef key(chars + last_offset, offsets[row] - last_offset - 1); @@ -143,6 +246,7 @@ struct HashMethodStringBin using KeyHolderType = ArenaKeyHolder; static constexpr bool is_serialized_key = false; + static constexpr bool batch_get_key_holder = false; const IColumn::Offset * offsets; const UInt8 * chars; @@ -343,6 +447,7 @@ struct HashMethodFastPathTwoKeysSerialized using KeyHolderType = SerializedKeyHolder; static constexpr bool is_serialized_key = true; + static constexpr bool batch_get_key_holder = false; Key1Desc key_1_desc; Key2Desc key_2_desc; @@ -380,6 +485,7 @@ struct HashMethodFixedString using KeyHolderType = ArenaKeyHolder; static constexpr bool is_serialized_key = false; + static constexpr bool batch_get_key_holder = false; size_t n; const ColumnFixedString::Chars_t * chars; @@ -428,6 +534,7 @@ struct HashMethodKeysFixed using KeyHolderType = Key; static constexpr bool is_serialized_key = false; + static constexpr bool batch_get_key_holder = false; static constexpr bool has_nullable_keys = has_nullable_keys_; Sizes key_sizes; @@ -570,21 +677,107 @@ struct HashMethodKeysFixed } }; +class KeySerializedBatchHandlerBase +{ +private: + size_t batch_row_idx = 0; + String sort_key_container{}; + PaddedPODArray byte_size{}; + PaddedPODArray pos{}; + PaddedPODArray ori_pos{}; + PaddedPODArray real_byte_size{}; + + ALWAYS_INLINE inline void santityCheck() const + { + assert(ori_pos.size() == pos.size() && real_byte_size.size() == pos.size()); + } + + ALWAYS_INLINE inline void resize(size_t batch_size) + { + pos.resize(batch_size); + ori_pos.resize(batch_size); + real_byte_size.resize(batch_size); + } + +protected: + void init(size_t start_row, const ColumnRawPtrs & key_columns, const TiDB::TiDBCollators & collators) + { + batch_row_idx = start_row; + byte_size.resize_fill_zero(key_columns[0]->size()); + for (size_t i = 0; i < key_columns.size(); ++i) + key_columns[i]->countSerializeByteSizeForCmp(byte_size, collators.empty() ? nullptr : collators[i]); + } + +public: + size_t prepareNextBatch( + const ColumnRawPtrs & key_columns, + Arena * pool, + size_t batch_size, + const TiDB::TiDBCollators & collators) + { + santityCheck(); + resize(batch_size); + + if unlikely (batch_size <= 0) + return 0; + + size_t mem_size = 0; + for (size_t i = batch_row_idx; i < batch_row_idx + batch_size; ++i) + mem_size += byte_size[i]; + + auto * ptr = static_cast(pool->alignedAlloc(mem_size, 16)); + for (size_t i = 0; i < batch_size; ++i) + { + pos[i] = ptr; + ori_pos[i] = ptr; + ptr += byte_size[i + batch_row_idx]; + } + + for (size_t i = 0; i < key_columns.size(); ++i) + key_columns[i]->serializeToPosForCmp( + pos, + batch_row_idx, + batch_size, + false, + collators.empty() ? nullptr : collators[i], + &sort_key_container); + + for (size_t i = 0; i < batch_size; ++i) + real_byte_size[i] = pos[i] - ori_pos[i]; + + batch_row_idx += batch_size; + + return mem_size; + } + + // NOTE: i is the index of mini batch, it's not the row index of Column. + ALWAYS_INLINE inline ArenaKeyHolder getKeyHolderBatch(size_t i, Arena * pool) const + { + santityCheck(); + assert(i < ori_pos.size()); + return ArenaKeyHolder{StringRef{ori_pos[i], real_byte_size[i]}, pool}; + } +}; /** Hash by concatenating serialized key values. * The serialized value differs in that it uniquely allows to deserialize it, having only the position with which it starts. * That is, for example, for strings, it contains first the serialized length of the string, and then the bytes. * Therefore, when aggregating by several strings, there is no ambiguity. */ -template +template struct HashMethodSerialized - : public columns_hashing_impl::HashMethodBase, Value, Mapped, false> + : public columns_hashing_impl::HashMethodBase, Value, Mapped, false> + , KeySerializedBatchHandlerBase { - using Self = HashMethodSerialized; + using Self = HashMethodSerialized; using Base = columns_hashing_impl::HashMethodBase; - using KeyHolderType = SerializedKeyHolder; + + static_assert(get_key_holder_batch_size == 0 || get_key_holder_batch_size >= 256); + using BatchHandlerBase = KeySerializedBatchHandlerBase; static constexpr bool is_serialized_key = true; + static constexpr bool batch_get_key_holder = (get_key_holder_batch_size > 0); + using KeyHolderType = typename std::conditional::type; ColumnRawPtrs key_columns; size_t keys_size; @@ -599,9 +792,22 @@ struct HashMethodSerialized , collators(collators_) {} + void initBatchHandler(size_t start_row) + { + assert(batch_get_key_holder); + BatchHandlerBase::init(start_row, key_columns, collators); + } + + size_t prepareNextBatch(Arena * pool) + { + assert(batch_get_key_holder); + return BatchHandlerBase::prepareNextBatch(key_columns, pool, get_key_holder_batch_size, collators); + } + ALWAYS_INLINE inline KeyHolderType getKeyHolder(size_t row, Arena * pool, std::vector & sort_key_containers) const { + assert(!batch_get_key_holder); return SerializedKeyHolder{ serializeKeysToPoolContiguous(row, keys_size, key_columns, collators, sort_key_containers, *pool), pool}; @@ -622,6 +828,7 @@ struct HashMethodHashed using KeyHolderType = Key; static constexpr bool is_serialized_key = false; + static constexpr bool batch_get_key_holder = false; ColumnRawPtrs key_columns; TiDB::TiDBCollators collators; diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 9af4ed8d466..0f43e94d362 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -46,9 +46,6 @@ extern const char random_fail_in_resize_callback[]; extern const char force_agg_prefetch[]; } // namespace FailPoints -static constexpr size_t agg_prefetch_step = 16; -static constexpr size_t agg_mini_batch = 256; - #define AggregationMethodName(NAME) AggregatedDataVariants::AggregationMethod_##NAME #define AggregationMethodNameTwoLevel(NAME) AggregatedDataVariants::AggregationMethod_##NAME##_two_level #define AggregationMethodType(NAME) AggregatedDataVariants::Type::NAME @@ -678,11 +675,12 @@ void NO_INLINE Aggregator::executeImpl( const bool disable_prefetch = (method.data.getBufferSizeInBytes() < prefetch_threshold); #endif - if constexpr (Method::State::is_serialized_key) - { - executeImplBatch(method, state, aggregates_pool, agg_process_info); - } - else if constexpr (Method::Data::is_string_hash_map) + // For key_serialized, memory allocation and key serialization will be batch-wise. + // For key_string, collation decode will be batch-wise. + if constexpr (Method::State::batch_get_key_holder) + state.initBatchHandler(agg_process_info.start_row); + + if constexpr (Method::Data::is_string_hash_map) { // StringHashMap doesn't support prefetch. executeImplBatch(method, state, aggregates_pool, agg_process_info); @@ -738,9 +736,9 @@ std::optional::Res } template -ALWAYS_INLINE inline void prepareBatch( +ALWAYS_INLINE inline void setupHashVals( size_t row_idx, - size_t end_row, + size_t batch_size, std::vector & hashvals, std::vector & key_holders, Arena * aggregates_pool, @@ -748,14 +746,19 @@ ALWAYS_INLINE inline void prepareBatch( Method & method, typename Method::State & state) { - assert(hashvals.size() == key_holders.size()); + assert(hashvals.size() == key_holders.size() && hashvals.size() == batch_size); - for (size_t i = row_idx, j = 0; i < row_idx + hashvals.size() && i < end_row; ++i, ++j) + for (size_t i = row_idx, j = 0; i < row_idx + batch_size; ++i, ++j) { - key_holders[j] = static_cast(&state)->getKeyHolder( - i, - aggregates_pool, - sort_key_containers); + if constexpr (Method::State::batch_get_key_holder) + key_holders[j] = static_cast(&state)->getKeyHolderBatch( + j, + aggregates_pool); + else + key_holders[j] = static_cast(&state)->getKeyHolder( + i, + aggregates_pool, + sort_key_containers); hashvals[j] = method.data.hash(keyHolderGetKey(key_holders[j])); } } @@ -851,6 +854,7 @@ void Aggregator::handleOneBatch( size_t i = agg_process_info.start_row; const size_t end = agg_process_info.start_row + rows; + Arena temp_batch_pool; size_t mini_batch_size = rows; std::vector hashvals; @@ -864,15 +868,20 @@ void Aggregator::handleOneBatch( key_holders.resize(agg_mini_batch); } + // i is the begin row index of each mini batch. while (i < end) { + size_t batch_mem_size = 0; + if constexpr (Method::State::batch_get_key_holder) + batch_mem_size = state.prepareNextBatch(mini_batch_size, &temp_batch_pool); + if constexpr (enable_prefetch) { if unlikely (i + mini_batch_size > end) mini_batch_size = end - i; - prepareBatch(i, end, hashvals, key_holders, aggregates_pool, sort_key_containers, method, state); + setupHashVals(i, mini_batch_size, hashvals, key_holders, aggregates_pool, sort_key_containers, method, state); } const auto cur_batch_end = i + mini_batch_size; @@ -890,6 +899,11 @@ void Aggregator::handleOneBatch( emplace_result_holder = emplaceOrFindKey(method, state, std::move(key_holders[k]), hashvals[k]); } + else if constexpr (Method::State::batch_get_key_holder) + { + emplace_result_holder + = emplaceOrFindKey(method, state, std::move(key_holders[k]), hashvals[k]); + } else { emplace_result_holder @@ -908,13 +922,9 @@ void Aggregator::handleOneBatch( if constexpr (compute_agg_data) { if (emplace_result.isFound()) - { aggregate_data = emplace_result.getMapped(); - } else - { agg_process_info.not_found_rows.push_back(j); - } } else { @@ -958,6 +968,9 @@ void Aggregator::handleOneBatch( processed_rows = j; } + if constexpr (Method::State::batch_get_key_holder) + temp_batch_pool.rollback(batch_mem_size); + if unlikely (!processed_rows.has_value()) break; @@ -1823,6 +1836,12 @@ void NO_INLINE Aggregator::convertToBlocksImplFinal( size_t data_index = 0; const auto rows = data.size(); std::unique_ptr places(new AggregateDataPtr[rows]); + std::unique_ptr key_places; + + static constexpr bool batch_deserialize_key = (Method::State::get_key_holder_batch_size && + Method::State::is_serialized_key); + if constexpr (batch_deserialize_key) + key_places = std::make_unique(rows); size_t current_bound = params.max_block_size; size_t key_columns_vec_index = 0; @@ -1830,8 +1849,16 @@ void NO_INLINE Aggregator::convertToBlocksImplFinal( data.forEachValue([&](const auto & key [[maybe_unused]], auto & mapped) { if constexpr (!skip_convert_key) { - agg_keys_helpers[key_columns_vec_index] - ->insertKeyIntoColumns(key, key_columns_vec[key_columns_vec_index], key_sizes_ref, params.collators); + if constexpr (batch_deserialize_key) + { + // Assume key is StringRef. + key_places[data_index] = key.data; + } + else + { + agg_keys_helpers[key_columns_vec_index] + ->insertKeyIntoColumns(key, key_columns_vec[key_columns_vec_index], key_sizes_ref, params.collators); + } } places[data_index] = mapped; ++data_index; @@ -1843,6 +1870,18 @@ void NO_INLINE Aggregator::convertToBlocksImplFinal( } }); + if constexpr (!skip_convert_key && batch_deserialize_key) + { + size_t batch_row_idx = 0; + size_t block_idx = 0; + while (batch_row_idx < rows) + { + state.insertKeyIntoColumnsBatch(key_places, key_columns_vec[block_idx]); + batch_row_idx += key_columns_vec[block_idx][0]->size(); + ++block_idx; + } + } + data_index = 0; current_bound = params.max_block_size; key_columns_vec_index = 0; diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index b9c1c18f484..95a4795a0a9 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -52,6 +52,8 @@ class IBlockOutputStream; template class AggHashTableToBlocksBlockInputStream; +static constexpr size_t agg_prefetch_step = 16; +static constexpr size_t agg_mini_batch = 256; /** Different data structures that can be used for aggregation * For efficiency, the aggregation data itself is put into the pool. @@ -231,7 +233,7 @@ struct AggregationMethodStringNoCache : data(other.data) {} - using State = ColumnsHashing::HashMethodString; + using State = ColumnsHashing::HashMethodString; template struct EmplaceOrFindKeyResult { @@ -683,7 +685,7 @@ struct AggregationMethodSerialized : data(other.data) {} - using State = ColumnsHashing::HashMethodSerialized; + using State = ColumnsHashing::HashMethodSerialized; template struct EmplaceOrFindKeyResult { From 241537d72143d3a0b902b6bd9bceb812c6c43aba Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Wed, 29 Jan 2025 11:57:10 +0800 Subject: [PATCH 13/16] refine Signed-off-by: guo-shaoge --- dbms/src/Common/ColumnsHashing.h | 108 +++++++++++++++------------ dbms/src/Interpreters/Aggregator.cpp | 57 ++++++++++---- dbms/src/Interpreters/Aggregator.h | 8 ++ 3 files changed, 109 insertions(+), 64 deletions(-) diff --git a/dbms/src/Common/ColumnsHashing.h b/dbms/src/Common/ColumnsHashing.h index 426cbe22d3b..49b55cc2abc 100644 --- a/dbms/src/Common/ColumnsHashing.h +++ b/dbms/src/Common/ColumnsHashing.h @@ -50,7 +50,6 @@ struct HashMethodOneNumber using KeyHolderType = FieldType; static constexpr bool is_serialized_key = false; - static constexpr bool batch_get_key_holder = false; const FieldType * vec; @@ -99,11 +98,14 @@ class KeyStringBatchHandlerBase size_t prepareNextBatchType( const UInt8 * chars, const IColumn::Offsets & offsets, - size_t batch_size, + size_t cur_batch_size, const TiDB::TiDBCollatorPtr & collator) { + if (cur_batch_size <= 0) + return 0; + const auto * derived_collator = static_cast(collator); - for (size_t i = 0; i < batch_size; ++i) + for (size_t i = 0; i < cur_batch_size; ++i) { const auto row = batch_row_idx + i; const auto last_offset = offsets[row - 1]; @@ -124,26 +126,33 @@ class KeyStringBatchHandlerBase } protected: - void init(size_t start_row, size_t batch_size) + bool inited() const { + return !sort_key_containers.empty(); + } + + void init(size_t start_row, size_t max_batch_size) + { + RUNTIME_CHECK(max_batch_size >= 256); batch_row_idx = start_row; - sort_key_containers.resize(batch_size); - batch_rows.resize(batch_size); + sort_key_containers.resize(max_batch_size); + batch_rows.reserve(max_batch_size); } -public: size_t prepareNextBatch( const UInt8 * chars, const IColumn::Offsets & offsets, - size_t batch_size, + size_t cur_batch_size, const TiDB::TiDBCollatorPtr & collator) { + batch_rows.resize(cur_batch_size); + if likely (collator) { #define M(VAR_PREFIX, COLLATOR_NAME, IMPL_TYPE, COLLATOR_ID) \ case (COLLATOR_ID): \ { \ - return prepareNextBatchType(chars, offsets, batch_size, collator); \ + return prepareNextBatchType(chars, offsets, cur_batch_size, collator); \ break; \ } @@ -159,7 +168,7 @@ class KeyStringBatchHandlerBase } else { - return prepareNextBatchType(chars, offsets, batch_size, collator); + return prepareNextBatchType(chars, offsets, cur_batch_size, collator); } } @@ -173,20 +182,19 @@ class KeyStringBatchHandlerBase }; /// For the case when there is one string key. -template +template struct HashMethodString - : public columns_hashing_impl::HashMethodBase, Value, Mapped, use_cache> + : public columns_hashing_impl::HashMethodBase, Value, Mapped, use_cache> , KeyStringBatchHandlerBase { using Self = HashMethodString; using Base = columns_hashing_impl::HashMethodBase; using KeyHolderType = ArenaKeyHolder; + using BatchKeyHolderType = KeyHolderType; - static_assert(get_key_holder_batch_size == 0 || get_key_holder_batch_size >= 256); using BatchHandlerBase = KeyStringBatchHandlerBase; static constexpr bool is_serialized_key = false; - static constexpr bool batch_get_key_holder = (get_key_holder_batch_size > 0); const IColumn::Offset * offsets; const UInt8 * chars; @@ -205,16 +213,16 @@ struct HashMethodString collator = collators[0]; } - void initBatchHandler(size_t start_row) + void initBatchHandler(size_t start_row, size_t max_batch_size) { - assert(batch_get_key_holder); - BatchHandlerBase::init(start_row, get_key_holder_batch_size); + assert(!BatchHandlerBase::inited()); + BatchHandlerBase::init(start_row, max_batch_size); } - size_t prepareNextBatch(Arena *) + size_t prepareNextBatch(Arena *, size_t cur_batch_size) { - assert(batch_get_key_holder); - return BatchHandlerBase::prepareNextBatch(chars, *offsets, get_key_holder_batch_size, collator); + assert(BatchHandlerBase::inited()); + return BatchHandlerBase::prepareNextBatch(chars, *offsets, cur_batch_size, collator); } ALWAYS_INLINE inline KeyHolderType getKeyHolder( @@ -222,7 +230,7 @@ struct HashMethodString [[maybe_unused]] Arena * pool, std::vector & sort_key_containers) const { - assert(!batch_get_key_holder); + assert(!BatchHandlerBase::inited()); auto last_offset = row == 0 ? 0 : offsets[row - 1]; // Remove last zero byte. @@ -244,9 +252,9 @@ struct HashMethodStringBin using Self = HashMethodStringBin; using Base = columns_hashing_impl::HashMethodBase; using KeyHolderType = ArenaKeyHolder; + using BatchKeyHolderType = KeyHolderType; static constexpr bool is_serialized_key = false; - static constexpr bool batch_get_key_holder = false; const IColumn::Offset * offsets; const UInt8 * chars; @@ -445,9 +453,9 @@ struct HashMethodFastPathTwoKeysSerialized using Self = HashMethodFastPathTwoKeysSerialized; using Base = columns_hashing_impl::HashMethodBase; using KeyHolderType = SerializedKeyHolder; + using BatchKeyHolderType = KeyHolderType; static constexpr bool is_serialized_key = true; - static constexpr bool batch_get_key_holder = false; Key1Desc key_1_desc; Key2Desc key_2_desc; @@ -483,9 +491,9 @@ struct HashMethodFixedString using Self = HashMethodFixedString; using Base = columns_hashing_impl::HashMethodBase; using KeyHolderType = ArenaKeyHolder; + using BatchKeyHolderType = KeyHolderType; static constexpr bool is_serialized_key = false; - static constexpr bool batch_get_key_holder = false; size_t n; const ColumnFixedString::Chars_t * chars; @@ -532,9 +540,9 @@ struct HashMethodKeysFixed using BaseHashed = columns_hashing_impl::HashMethodBase; using Base = columns_hashing_impl::BaseStateKeysFixed; using KeyHolderType = Key; + using BatchKeyHolderType = KeyHolderType; static constexpr bool is_serialized_key = false; - static constexpr bool batch_get_key_holder = false; static constexpr bool has_nullable_keys = has_nullable_keys_; Sizes key_sizes; @@ -700,33 +708,38 @@ class KeySerializedBatchHandlerBase } protected: + bool inited() const + { + return !byte_size.empty(); + } + void init(size_t start_row, const ColumnRawPtrs & key_columns, const TiDB::TiDBCollators & collators) { batch_row_idx = start_row; byte_size.resize_fill_zero(key_columns[0]->size()); + RUNTIME_CHECK(!byte_size.empty()); for (size_t i = 0; i < key_columns.size(); ++i) key_columns[i]->countSerializeByteSizeForCmp(byte_size, collators.empty() ? nullptr : collators[i]); } -public: size_t prepareNextBatch( const ColumnRawPtrs & key_columns, Arena * pool, - size_t batch_size, + size_t cur_batch_size, const TiDB::TiDBCollators & collators) { santityCheck(); - resize(batch_size); + resize(cur_batch_size); - if unlikely (batch_size <= 0) + if unlikely (cur_batch_size <= 0) return 0; size_t mem_size = 0; - for (size_t i = batch_row_idx; i < batch_row_idx + batch_size; ++i) + for (size_t i = batch_row_idx; i < batch_row_idx + cur_batch_size; ++i) mem_size += byte_size[i]; auto * ptr = static_cast(pool->alignedAlloc(mem_size, 16)); - for (size_t i = 0; i < batch_size; ++i) + for (size_t i = 0; i < cur_batch_size; ++i) { pos[i] = ptr; ori_pos[i] = ptr; @@ -737,15 +750,15 @@ class KeySerializedBatchHandlerBase key_columns[i]->serializeToPosForCmp( pos, batch_row_idx, - batch_size, + cur_batch_size, false, collators.empty() ? nullptr : collators[i], &sort_key_container); - for (size_t i = 0; i < batch_size; ++i) + for (size_t i = 0; i < cur_batch_size; ++i) real_byte_size[i] = pos[i] - ori_pos[i]; - batch_row_idx += batch_size; + batch_row_idx += cur_batch_size; return mem_size; } @@ -764,20 +777,17 @@ class KeySerializedBatchHandlerBase * That is, for example, for strings, it contains first the serialized length of the string, and then the bytes. * Therefore, when aggregating by several strings, there is no ambiguity. */ -template +template struct HashMethodSerialized - : public columns_hashing_impl::HashMethodBase, Value, Mapped, false> + : public columns_hashing_impl::HashMethodBase, Value, Mapped, false> , KeySerializedBatchHandlerBase { - using Self = HashMethodSerialized; + using Self = HashMethodSerialized; using Base = columns_hashing_impl::HashMethodBase; - - static_assert(get_key_holder_batch_size == 0 || get_key_holder_batch_size >= 256); using BatchHandlerBase = KeySerializedBatchHandlerBase; - static constexpr bool is_serialized_key = true; - static constexpr bool batch_get_key_holder = (get_key_holder_batch_size > 0); - using KeyHolderType = typename std::conditional::type; + using KeyHolderType = SerializedKeyHolder; + using BatchKeyHolderType = ArenaKeyHolder; ColumnRawPtrs key_columns; size_t keys_size; @@ -792,22 +802,22 @@ struct HashMethodSerialized , collators(collators_) {} - void initBatchHandler(size_t start_row) + void initBatchHandler(size_t start_row, size_t /* max_batch_size */) { - assert(batch_get_key_holder); + assert(!BatchHandlerBase::inited()); BatchHandlerBase::init(start_row, key_columns, collators); } - size_t prepareNextBatch(Arena * pool) + size_t prepareNextBatch(Arena * pool, size_t cur_batch_size) { - assert(batch_get_key_holder); - return BatchHandlerBase::prepareNextBatch(key_columns, pool, get_key_holder_batch_size, collators); + assert(BatchHandlerBase::inited()); + return BatchHandlerBase::prepareNextBatch(key_columns, pool, cur_batch_size, collators); } ALWAYS_INLINE inline KeyHolderType getKeyHolder(size_t row, Arena * pool, std::vector & sort_key_containers) const { - assert(!batch_get_key_holder); + assert(!BatchHandlerBase::inited()); return SerializedKeyHolder{ serializeKeysToPoolContiguous(row, keys_size, key_columns, collators, sort_key_containers, *pool), pool}; @@ -826,9 +836,9 @@ struct HashMethodHashed using Self = HashMethodHashed; using Base = columns_hashing_impl::HashMethodBase; using KeyHolderType = Key; + using BatchKeyHolderType = KeyHolderType; static constexpr bool is_serialized_key = false; - static constexpr bool batch_get_key_holder = false; ColumnRawPtrs key_columns; TiDB::TiDBCollators collators; diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 0f43e94d362..1a3ab2c3649 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -652,6 +652,12 @@ void Aggregator::createAggregateStates(AggregateDataPtr & aggregate_data) const } } +template +concept HasBatchGetKeyHolderMemberFunc = requires +{ + // todo also require initBatchHandler() + {std::declval().getKeyHolderBatch(std::declval(), std::declval())} -> std::same_as; +}; /** It's interesting - if you remove `noinline`, then gcc for some reason will inline this function, and the performance decreases (~ 10%). * (Probably because after the inline of this function, more internal functions no longer be inlined.) @@ -677,7 +683,7 @@ void NO_INLINE Aggregator::executeImpl( // For key_serialized, memory allocation and key serialization will be batch-wise. // For key_string, collation decode will be batch-wise. - if constexpr (Method::State::batch_get_key_holder) + if constexpr (HasBatchGetKeyHolderMemberFunc) state.initBatchHandler(agg_process_info.start_row); if constexpr (Method::Data::is_string_hash_map) @@ -1836,12 +1842,13 @@ void NO_INLINE Aggregator::convertToBlocksImplFinal( size_t data_index = 0; const auto rows = data.size(); std::unique_ptr places(new AggregateDataPtr[rows]); - std::unique_ptr key_places; + PaddedPODArray key_places; - static constexpr bool batch_deserialize_key = (Method::State::get_key_holder_batch_size && - Method::State::is_serialized_key); + // For key_serialized, deserialize will be batch-wise if it's serialized batch-wise. + static constexpr bool batch_deserialize_key = + (Method::State::get_key_holder_batch_size && Method::State::is_serialized_key); if constexpr (batch_deserialize_key) - key_places = std::make_unique(rows); + key_places.reserve(params.max_block_size); size_t current_bound = params.max_block_size; size_t key_columns_vec_index = 0; @@ -1852,7 +1859,12 @@ void NO_INLINE Aggregator::convertToBlocksImplFinal( if constexpr (batch_deserialize_key) { // Assume key is StringRef. - key_places[data_index] = key.data; + key_places.push_back(key.data); + if unlikely(key_places.size() >= params.max_block_size) + { + method.insertKeyIntoColumnsBatch(key_places, key_columns_vec[++key_columns_vec_index]); + key_places.clear(); + } } else { @@ -1872,14 +1884,8 @@ void NO_INLINE Aggregator::convertToBlocksImplFinal( if constexpr (!skip_convert_key && batch_deserialize_key) { - size_t batch_row_idx = 0; - size_t block_idx = 0; - while (batch_row_idx < rows) - { - state.insertKeyIntoColumnsBatch(key_places, key_columns_vec[block_idx]); - batch_row_idx += key_columns_vec[block_idx][0]->size(); - ++block_idx; - } + if (!key_places.empty()) + method.insertKeyIntoColumnsBatch(key_places, key_columns_vec[++key_columns_vec_index]); } data_index = 0; @@ -1922,11 +1928,26 @@ void NO_INLINE Aggregator::convertToBlockImplNotFinal( } agg_keys_helper.initAggKeys(data.size(), key_columns); } + + PaddedPODArray key_places; + // For key_serialized, deserialize will be batch-wise if it's serialized batch-wise. + static constexpr bool batch_deserialize_key = + (Method::State::get_key_holder_batch_size && Method::State::is_serialized_key); + if constexpr (batch_deserialize_key) + key_places.reserve(params.max_block_size); data.forEachValue([&](const auto & key [[maybe_unused]], auto & mapped) { if constexpr (!skip_convert_key) { - agg_keys_helper.insertKeyIntoColumns(key, key_columns, key_sizes_ref, params.collators); + if constexpr (batch_deserialize_key) + { + // Assume key is StringRef. + key_places.push_back(key.data); + } + else + { + agg_keys_helper.insertKeyIntoColumns(key, key_columns, key_sizes_ref, params.collators); + } } /// reserved, so push_back does not throw exceptions @@ -1935,6 +1956,12 @@ void NO_INLINE Aggregator::convertToBlockImplNotFinal( mapped = nullptr; }); + + if constexpr (!skip_convert_key && batch_deserialize_key) + { + if (!key_places.empty()) + method.insertKeyIntoColumnsBatch(key_places, key_columns); + } } template diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index 95a4795a0a9..7d89a8974bf 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -716,6 +716,14 @@ struct AggregationMethodSerialized for (size_t i = 0; i < key_columns.size(); ++i) pos = key_columns[i]->deserializeAndInsertFromArena(pos, collators.empty() ? nullptr : collators[i]); } + + static void insertKeyIntoColumnsBatch( + PaddedPODArray & key_places, + std::vector & key_columns) + { + for (auto * key_column : key_columns) + key_column->deserializeForCmpAndInsertFromPos(key_places, true); + } }; From e2bdc813e8dcf70df5a4ba29bd24800b0de2cb2f Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Thu, 30 Jan 2025 11:13:45 +0800 Subject: [PATCH 14/16] batch deserialize Signed-off-by: guo-shaoge --- dbms/src/Common/ColumnsHashing.h | 10 ++ dbms/src/Common/ColumnsHashingImpl.h | 5 + dbms/src/Interpreters/Aggregator.cpp | 242 +++++++++++++++++++-------- dbms/src/Interpreters/Aggregator.h | 22 +-- 4 files changed, 201 insertions(+), 78 deletions(-) diff --git a/dbms/src/Common/ColumnsHashing.h b/dbms/src/Common/ColumnsHashing.h index 49b55cc2abc..442cb153cce 100644 --- a/dbms/src/Common/ColumnsHashing.h +++ b/dbms/src/Common/ColumnsHashing.h @@ -213,6 +213,11 @@ struct HashMethodString collator = collators[0]; } + bool batchGetkeyHolder() override + { + return BatchHandlerBase::inited(); + } + void initBatchHandler(size_t start_row, size_t max_batch_size) { assert(!BatchHandlerBase::inited()); @@ -802,6 +807,11 @@ struct HashMethodSerialized , collators(collators_) {} + bool batchGetkeyHolder() override + { + return BatchHandlerBase::inited(); + } + void initBatchHandler(size_t start_row, size_t /* max_batch_size */) { assert(!BatchHandlerBase::inited()); diff --git a/dbms/src/Common/ColumnsHashingImpl.h b/dbms/src/Common/ColumnsHashingImpl.h index bf130d2bd29..a9a23db14f5 100644 --- a/dbms/src/Common/ColumnsHashingImpl.h +++ b/dbms/src/Common/ColumnsHashingImpl.h @@ -129,6 +129,11 @@ class HashMethodBase using Cache = LastElementCache; using Derived = TDerived; + bool batchGetKeyHolder() const override + { + return false; + } + template ALWAYS_INLINE inline EmplaceResult emplaceKey( Data & data, diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 1a3ab2c3649..9bf9402a7ed 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -666,10 +666,11 @@ concept HasBatchGetKeyHolderMemberFunc = requires template void NO_INLINE Aggregator::executeImpl( Method & method, - Arena * aggregates_pool, + AggregatedDataVariants & result, AggProcessInfo & agg_process_info, TiDB::TiDBCollators & collators) const { + auto * aggregates_pool = result.aggregates_pool; typename Method::State state(agg_process_info.key_columns, key_sizes, collators); // 2MB as prefetch threshold, because normally server L2 cache is 1MB. @@ -684,7 +685,10 @@ void NO_INLINE Aggregator::executeImpl( // For key_serialized, memory allocation and key serialization will be batch-wise. // For key_string, collation decode will be batch-wise. if constexpr (HasBatchGetKeyHolderMemberFunc) + { state.initBatchHandler(agg_process_info.start_row); + result.batch_get_key_holder = true; + } if constexpr (Method::Data::is_string_hash_map) { @@ -694,9 +698,19 @@ void NO_INLINE Aggregator::executeImpl( else { if (disable_prefetch) - executeImplBatch(method, state, aggregates_pool, agg_process_info); + { + if constexpr (state.batchGetKeyHolder()) + executeImplBatch(method, state, aggregates_pool, agg_process_info); + else + executeImplBatch(method, state, aggregates_pool, agg_process_info); + } else - executeImplBatch(method, state, aggregates_pool, agg_process_info); + { + if constexpr (state.batchGetKeyHolder()) + executeImplBatch(method, state, aggregates_pool, agg_process_info); + else + executeImplBatch(method, state, aggregates_pool, agg_process_info); + } } } @@ -741,12 +755,12 @@ std::optional::Res } } -template +template ALWAYS_INLINE inline void setupHashVals( size_t row_idx, size_t batch_size, std::vector & hashvals, - std::vector & key_holders, + std::vector & key_holders, Arena * aggregates_pool, std::vector & sort_key_containers, Method & method, @@ -769,7 +783,7 @@ ALWAYS_INLINE inline void setupHashVals( } } -template +template ALWAYS_INLINE void Aggregator::executeImplBatch( Method & method, typename Method::State & state, @@ -834,7 +848,7 @@ ALWAYS_INLINE void Aggregator::executeImplBatch( aggregates_pool); } -template +template void Aggregator::handleOneBatch( Method & method, typename Method::State & state, @@ -864,7 +878,7 @@ void Aggregator::handleOneBatch( size_t mini_batch_size = rows; std::vector hashvals; - std::vector key_holders; + std::vector key_holders; if constexpr (enable_prefetch) { // mini batch will only be used when HashTable is big(a.k.a enable_prefetch is true), @@ -874,7 +888,6 @@ void Aggregator::handleOneBatch( key_holders.resize(agg_mini_batch); } - // i is the begin row index of each mini batch. while (i < end) { @@ -1180,7 +1193,7 @@ bool Aggregator::executeOnBlockImpl( { \ executeImpl( \ *ToAggregationMethodPtr(NAME, result.aggregation_method_impl), \ - result.aggregates_pool, \ + result, \ agg_process_info, \ params.collators); \ break; \ @@ -1287,8 +1300,9 @@ Block Aggregator::convertOneBucketToBlock( bool final, size_t bucket) const { + const bool batch_get_key_holder = data_variants.batch_get_key_holder; #define FILLER_DEFINE(name, skip_convert_key) \ - auto filler_##name = [bucket, &method, arena, this]( \ + auto filler_##name = [bucket, &method, arena, this, batch_get_key_holder]( \ const Sizes & key_sizes, \ MutableColumns & key_columns, \ AggregateColumnsData & aggregate_columns, \ @@ -1296,15 +1310,30 @@ Block Aggregator::convertOneBucketToBlock( bool final_) { \ using METHOD_TYPE = std::decay_t; \ using DATA_TYPE = std::decay_t; \ - convertToBlockImpl( \ - method, \ - method.data.impls[bucket], \ - key_sizes, \ - key_columns, \ - aggregate_columns, \ - final_aggregate_columns, \ - arena, \ - final_); \ + if (METHOD_TYPE::State::is_serialized_key && batch_get_key_holder) \ + { \ + convertToBlockImpl( \ + method, \ + method.data.impls[bucket], \ + key_sizes, \ + key_columns, \ + aggregate_columns, \ + final_aggregate_columns, \ + arena, \ + final_); \ + } \ + else \ + { \ + convertToBlockImpl( \ + method, \ + method.data.impls[bucket], \ + key_sizes, \ + key_columns, \ + aggregate_columns, \ + final_aggregate_columns, \ + arena, \ + final_); \ + } \ } FILLER_DEFINE(convert_key, false); @@ -1347,22 +1376,38 @@ BlocksList Aggregator::convertOneBucketToBlocks( bool final, size_t bucket) const { + const auto batch_get_key_holder = data_variants.batch_get_key_holder; #define FILLER_DEFINE(name, skip_convert_key) \ - auto filler_##name = [bucket, &method, arena, this]( \ + auto filler_##name = [bucket, &method, arena, this, batch_deserialize_key]( \ const Sizes & key_sizes, \ std::vector & key_columns_vec, \ std::vector & aggregate_columns_vec, \ std::vector & final_aggregate_columns_vec, \ bool final_) { \ - convertToBlocksImpl( \ - method, \ - method.data.impls[bucket], \ - key_sizes, \ - key_columns_vec, \ - aggregate_columns_vec, \ - final_aggregate_columns_vec, \ - arena, \ - final_); \ + if (Method::State::is_serialized_key && batch_get_key_holder) \ + { \ + convertToBlocksImpl( \ + method, \ + method.data.impls[bucket], \ + key_sizes, \ + key_columns_vec, \ + aggregate_columns_vec, \ + final_aggregate_columns_vec, \ + arena, \ + final_); \ + } \ + else \ + { \ + convertToBlocksImpl( \ + method, \ + method.data.impls[bucket], \ + key_sizes, \ + key_columns_vec, \ + aggregate_columns_vec, \ + final_aggregate_columns_vec, \ + arena, \ + final_); \ + } \ }; FILLER_DEFINE(convert_key, false); @@ -1488,7 +1533,7 @@ void Aggregator::execute(const BlockInputStreamPtr & stream, AggregatedDataVaria src_bytes / elapsed_seconds / 1048576.0); } -template +template void Aggregator::convertToBlockImpl( Method & method, Table & data, @@ -1508,7 +1553,7 @@ void Aggregator::convertToBlockImpl( raw_key_columns.push_back(column.get()); if (final) - convertToBlockImplFinal( + convertToBlockImplFinal( method, data, key_sizes, @@ -1516,7 +1561,7 @@ void Aggregator::convertToBlockImpl( final_aggregate_columns, arena); else - convertToBlockImplNotFinal( + convertToBlockImplNotFinal( method, data, key_sizes, @@ -1527,7 +1572,7 @@ void Aggregator::convertToBlockImpl( data.clearAndShrink(); } -template +template void Aggregator::convertToBlocksImpl( Method & method, Table & data, @@ -1556,7 +1601,7 @@ void Aggregator::convertToBlocksImpl( } if (final) - convertToBlocksImplFinal( + convertToBlocksImplFinal( method, data, key_sizes, @@ -1564,7 +1609,7 @@ void Aggregator::convertToBlocksImpl( final_aggregate_columns_vec, arena); else - convertToBlocksImplNotFinal( + convertToBlocksImplNotFinal( method, data, key_sizes, @@ -1737,7 +1782,7 @@ struct AggregatorMethodInitKeyColumnHelper +template void NO_INLINE Aggregator::convertToBlockImplFinal( Method & method, Table & data, @@ -1762,15 +1807,34 @@ void NO_INLINE Aggregator::convertToBlockImplFinal( agg_keys_helper.initAggKeys(data.size(), key_columns); } + // For key_serialized, deserialize will be batch-wise if it's serialized batch-wise. + PaddedPODArray key_places; + if constexpr (batch_deserialize_key) + key_places.reserve(params.max_block_size); + // Doesn't prefetch agg data, because places[data.size()] is needed, which can be very large. data.forEachValue([&](const auto & key [[maybe_unused]], auto & mapped) { if constexpr (!skip_convert_key) { - agg_keys_helper.insertKeyIntoColumns(key, key_columns, key_sizes_ref, params.collators); + if constexpr (batch_deserialize_key) + { + // Assume key is StringRef, because only key_string and key_serialize can be batch-wise. + key_places.push_back(key.data); + } + else + { + agg_keys_helper.insertKeyIntoColumns(key, key_columns, key_sizes_ref, params.collators); + } } insertAggregatesIntoColumns(mapped, final_aggregate_columns, arena); }); + + if constexpr (!skip_convert_key && batch_deserialize_key) + { + if (!key_places.empty()) + method.insertKeyIntoColumnsBatch(key_places, key_columns); + } } namespace @@ -1810,7 +1874,7 @@ std::vector>> initAg } } // namespace -template +template void NO_INLINE Aggregator::convertToBlocksImplFinal( Method & method, Table & data, @@ -1842,11 +1906,9 @@ void NO_INLINE Aggregator::convertToBlocksImplFinal( size_t data_index = 0; const auto rows = data.size(); std::unique_ptr places(new AggregateDataPtr[rows]); - PaddedPODArray key_places; + PaddedPODArray key_places; // For key_serialized, deserialize will be batch-wise if it's serialized batch-wise. - static constexpr bool batch_deserialize_key = - (Method::State::get_key_holder_batch_size && Method::State::is_serialized_key); if constexpr (batch_deserialize_key) key_places.reserve(params.max_block_size); @@ -1858,13 +1920,8 @@ void NO_INLINE Aggregator::convertToBlocksImplFinal( { if constexpr (batch_deserialize_key) { - // Assume key is StringRef. + // Assume key is StringRef, because only key_string and key_serialize can be batch-wise. key_places.push_back(key.data); - if unlikely(key_places.size() >= params.max_block_size) - { - method.insertKeyIntoColumnsBatch(key_places, key_columns_vec[++key_columns_vec_index]); - key_places.clear(); - } } else { @@ -1877,6 +1934,9 @@ void NO_INLINE Aggregator::convertToBlocksImplFinal( if unlikely (data_index == current_bound) { + method.insertKeyIntoColumnsBatch(key_places, key_columns_vec[key_columns_vec_index]); + key_places.clear(); + ++key_columns_vec_index; current_bound += params.max_block_size; } @@ -1885,7 +1945,7 @@ void NO_INLINE Aggregator::convertToBlocksImplFinal( if constexpr (!skip_convert_key && batch_deserialize_key) { if (!key_places.empty()) - method.insertKeyIntoColumnsBatch(key_places, key_columns_vec[++key_columns_vec_index]); + method.insertKeyIntoColumnsBatch(key_places, key_columns_vec[key_columns_vec_index]); } data_index = 0; @@ -1907,7 +1967,7 @@ void NO_INLINE Aggregator::convertToBlocksImplFinal( } } -template +template void NO_INLINE Aggregator::convertToBlockImplNotFinal( Method & method, Table & data, @@ -1929,10 +1989,8 @@ void NO_INLINE Aggregator::convertToBlockImplNotFinal( agg_keys_helper.initAggKeys(data.size(), key_columns); } - PaddedPODArray key_places; // For key_serialized, deserialize will be batch-wise if it's serialized batch-wise. - static constexpr bool batch_deserialize_key = - (Method::State::get_key_holder_batch_size && Method::State::is_serialized_key); + PaddedPODArray key_places; if constexpr (batch_deserialize_key) key_places.reserve(params.max_block_size); @@ -1964,7 +2022,7 @@ void NO_INLINE Aggregator::convertToBlockImplNotFinal( } } -template +template void NO_INLINE Aggregator::convertToBlocksImplNotFinal( Method & method, Table & data, @@ -1991,13 +2049,27 @@ void NO_INLINE Aggregator::convertToBlocksImplNotFinal( agg_keys_helpers = initAggKeysForKeyColumnsVec(method, key_columns_vec, params.max_block_size, data.size()); } + PaddedPODArray key_places; + // For key_serialized, deserialize will be batch-wise if it's serialized batch-wise. + if constexpr (batch_deserialize_key) + key_places.reserve(params.max_block_size); + size_t data_index = 0; + size_t current_bound = params.max_block_size; + size_t key_columns_vec_index = 0; data.forEachValue([&](const auto & key [[maybe_unused]], auto & mapped) { - size_t key_columns_vec_index = data_index / params.max_block_size; if constexpr (!skip_convert_key) { - agg_keys_helpers[key_columns_vec_index] - ->insertKeyIntoColumns(key, key_columns_vec[key_columns_vec_index], key_sizes_ref, params.collators); + if constexpr (batch_deserialize_key) + { + // Assume key is StringRef, because only key_string and key_serialize can be batch-wise. + key_places.push_back(key.data); + } + else + { + agg_keys_helpers[key_columns_vec_index] + ->insertKeyIntoColumns(key, key_columns_vec[key_columns_vec_index], key_sizes_ref, params.collators); + } } /// reserved, so push_back does not throw exceptions @@ -2006,7 +2078,22 @@ void NO_INLINE Aggregator::convertToBlocksImplNotFinal( ++data_index; mapped = nullptr; + + if unlikely (data_index == current_bound) + { + method.insertKeyIntoColumnsBatch(key_places, key_columns_vec[++key_columns_vec_index]); + key_places.clear(); + + ++key_columns_vec_index; + current_bound += params.max_block_size; + } }); + + if constexpr (!skip_convert_key && batch_deserialize_key) + { + if (!key_places.empty()) + method.insertKeyIntoColumnsBatch(key_places, key_columns_vec[++key_columns_vec_index]); + } } template @@ -2214,7 +2301,7 @@ BlocksList Aggregator::prepareBlocksAndFill( return res_list; } - +// todo check if ok BlocksList Aggregator::prepareBlocksAndFillWithoutKey(AggregatedDataVariants & data_variants, bool final) const { size_t rows = 1; @@ -2256,20 +2343,36 @@ BlocksList Aggregator::prepareBlocksAndFillWithoutKey(AggregatedDataVariants & d BlocksList Aggregator::prepareBlocksAndFillSingleLevel(AggregatedDataVariants & data_variants, bool final) const { size_t rows = data_variants.size(); + const bool batch_get_key_holder = data_variants.batch_get_key_holder; #define M(NAME, skip_convert_key) \ case AggregationMethodType(NAME): \ { \ auto & tmp_method = *ToAggregationMethodPtr(NAME, data_variants.aggregation_method_impl); \ auto & tmp_data = ToAggregationMethodPtr(NAME, data_variants.aggregation_method_impl) -> data; \ - convertToBlocksImpl( \ - tmp_method, \ - tmp_data, \ - key_sizes, \ - key_columns_vec, \ - aggregate_columns_vec, \ - final_aggregate_columns_vec, \ - data_variants.aggregates_pool, \ - final_); \ + if (decltype(tmp_method)::State::is_serialized_key && batch_get_key_holder) \ + { \ + convertToBlocksImpl( \ + tmp_method, \ + tmp_data, \ + key_sizes, \ + key_columns_vec, \ + aggregate_columns_vec, \ + final_aggregate_columns_vec, \ + data_variants.aggregates_pool, \ + final_); \ + } \ + else \ + { \ + convertToBlocksImpl( \ + tmp_method, \ + tmp_data, \ + key_sizes, \ + key_columns_vec, \ + aggregate_columns_vec, \ + final_aggregate_columns_vec, \ + data_variants.aggregates_pool, \ + final_); \ + } \ break; \ } @@ -2277,7 +2380,7 @@ BlocksList Aggregator::prepareBlocksAndFillSingleLevel(AggregatedDataVariants & #define M_convert_key(NAME) M(NAME, false) #define FILLER_DEFINE(name, M_tmp) \ - auto filler_##name = [&data_variants, this]( \ + auto filler_##name = [&data_variants, this, batch_get_key_holder]( \ const Sizes & key_sizes, \ std::vector & key_columns_vec, \ std::vector & aggregate_columns_vec, \ @@ -2472,6 +2575,9 @@ MergingBucketsPtr Aggregator::mergeAndConvertToBlocks( non_empty_data[i]->aggregates_pools.end()); } + for (auto & data : non_empty_data) + RUNTIME_CHECK(non_empty_data[0]->batch_get_key_holder == data.batch_get_key_holder); + // for single level merge, concurrency must be 1. size_t merge_concurrency = has_at_least_one_two_level ? std::max(max_threads, 1) : 1; return std::make_shared(*this, non_empty_data, final, merge_concurrency); diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index 7d89a8974bf..5220ed70916 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -233,7 +233,7 @@ struct AggregationMethodStringNoCache : data(other.data) {} - using State = ColumnsHashing::HashMethodString; + using State = ColumnsHashing::HashMethodString; template struct EmplaceOrFindKeyResult { @@ -685,7 +685,7 @@ struct AggregationMethodSerialized : data(other.data) {} - using State = ColumnsHashing::HashMethodSerialized; + using State = ColumnsHashing::HashMethodSerialized; template struct EmplaceOrFindKeyResult { @@ -765,6 +765,8 @@ struct AggregatedDataVariants : private boost::noncopyable */ AggregatedDataWithoutKey without_key = nullptr; + bool batch_get_key_holder = false; + using AggregationMethod_key8 = AggregationMethodOneNumber; using AggregationMethod_key16 = AggregationMethodOneNumber; using AggregationMethod_key32 = AggregationMethodOneNumber; @@ -1463,14 +1465,14 @@ class Aggregator AggProcessInfo & agg_process_info, TiDB::TiDBCollators & collators) const; - template + template void executeImplBatch( Method & method, typename Method::State & state, Arena * aggregates_pool, AggProcessInfo & agg_process_info) const; - template + template void handleOneBatch( Method & method, typename Method::State & state, @@ -1508,7 +1510,7 @@ class Aggregator template void mergeSingleLevelDataImpl(ManyAggregatedDataVariants & non_empty_data) const; - template + template void convertToBlockImpl( Method & method, Table & data, @@ -1522,7 +1524,7 @@ class Aggregator // The template parameter skip_convert_key indicates whether we can skip deserializing the keys in the HashMap. // For example, select first_row(c1) from t group by c1, where c1 is a string column with collator, // only the result of first_row(c1) needs to be constructed. The key c1 only needs to reference to first_row(c1). - template + template void convertToBlocksImpl( Method & method, Table & data, @@ -1533,7 +1535,7 @@ class Aggregator Arena * arena, bool final) const; - template + template void convertToBlockImplFinal( Method & method, Table & data, @@ -1542,7 +1544,7 @@ class Aggregator MutableColumns & final_aggregate_columns, Arena * arena) const; - template + template void convertToBlocksImplFinal( Method & method, Table & data, @@ -1551,7 +1553,7 @@ class Aggregator std::vector & final_aggregate_columns_vec, Arena * arena) const; - template + template void convertToBlockImplNotFinal( Method & method, Table & data, @@ -1559,7 +1561,7 @@ class Aggregator std::vector key_columns, AggregateColumnsData & aggregate_columns) const; - template + template void convertToBlocksImplNotFinal( Method & method, Table & data, From 3354b7ff21d063dee0a6b4f930c72f94796274bf Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Thu, 30 Jan 2025 11:51:24 +0800 Subject: [PATCH 15/16] case ok Signed-off-by: guo-shaoge --- dbms/src/Common/ColumnsHashing.h | 61 ++-- dbms/src/Common/ColumnsHashingImpl.h | 5 - dbms/src/Interpreters/Aggregator.cpp | 402 ++++++++++++++++----------- dbms/src/Interpreters/Aggregator.h | 33 ++- 4 files changed, 293 insertions(+), 208 deletions(-) diff --git a/dbms/src/Common/ColumnsHashing.h b/dbms/src/Common/ColumnsHashing.h index 442cb153cce..30f9bddea3f 100644 --- a/dbms/src/Common/ColumnsHashing.h +++ b/dbms/src/Common/ColumnsHashing.h @@ -48,8 +48,10 @@ struct HashMethodOneNumber using Self = HashMethodOneNumber; using Base = columns_hashing_impl::HashMethodBase; using KeyHolderType = FieldType; + using BatchKeyHolderType = KeyHolderType; static constexpr bool is_serialized_key = false; + static constexpr bool can_batch_get_key_holder = false; const FieldType * vec; @@ -110,7 +112,7 @@ class KeyStringBatchHandlerBase const auto row = batch_row_idx + i; const auto last_offset = offsets[row - 1]; // Remove last zero byte. - StringRef key(chars + last_offset, offsets[row] - offsets[row -1 ] - 1); + StringRef key(chars + last_offset, offsets[row] - last_offset - 1); if constexpr (has_collator) key = derived_collator->sortKey(key.data, key.size, sort_key_containers[i]); @@ -121,15 +123,12 @@ class KeyStringBatchHandlerBase void santityCheck() const { - // Make sure init() has called. + // Make sure init() has been called. assert(sort_key_containers.size() == batch_rows.size() && !sort_key_containers.empty()); } protected: - bool inited() const - { - return !sort_key_containers.empty(); - } + bool inited() const { return !sort_key_containers.empty(); } void init(size_t start_row, size_t max_batch_size) { @@ -149,20 +148,20 @@ class KeyStringBatchHandlerBase if likely (collator) { -#define M(VAR_PREFIX, COLLATOR_NAME, IMPL_TYPE, COLLATOR_ID) \ - case (COLLATOR_ID): \ - { \ - return prepareNextBatchType(chars, offsets, cur_batch_size, collator); \ - break; \ - } +#define M(VAR_PREFIX, COLLATOR_NAME, IMPL_TYPE, COLLATOR_ID) \ + case (COLLATOR_ID): \ + { \ + return prepareNextBatchType(chars, offsets, cur_batch_size, collator); \ + break; \ + } switch (collator->getCollatorId()) { APPLY_FOR_COLLATOR_TYPES(M) - default: - { - throw Exception(fmt::format("unexpected collator: {}", collator->getCollatorId())); - } + default: + { + throw Exception(fmt::format("unexpected collator: {}", collator->getCollatorId())); + } }; #undef M } @@ -172,6 +171,7 @@ class KeyStringBatchHandlerBase } } +public: // NOTE: i is the index of mini batch, it's not the row index of Column. ALWAYS_INLINE inline ArenaKeyHolder getKeyHolderBatch(size_t i, Arena * pool) const { @@ -195,6 +195,8 @@ struct HashMethodString using BatchHandlerBase = KeyStringBatchHandlerBase; static constexpr bool is_serialized_key = false; + // todo + static constexpr bool can_batch_get_key_holder = false; const IColumn::Offset * offsets; const UInt8 * chars; @@ -213,11 +215,6 @@ struct HashMethodString collator = collators[0]; } - bool batchGetkeyHolder() override - { - return BatchHandlerBase::inited(); - } - void initBatchHandler(size_t start_row, size_t max_batch_size) { assert(!BatchHandlerBase::inited()); @@ -260,6 +257,7 @@ struct HashMethodStringBin using BatchKeyHolderType = KeyHolderType; static constexpr bool is_serialized_key = false; + static constexpr bool can_batch_get_key_holder = false; const IColumn::Offset * offsets; const UInt8 * chars; @@ -461,6 +459,7 @@ struct HashMethodFastPathTwoKeysSerialized using BatchKeyHolderType = KeyHolderType; static constexpr bool is_serialized_key = true; + static constexpr bool can_batch_get_key_holder = false; Key1Desc key_1_desc; Key2Desc key_2_desc; @@ -499,6 +498,7 @@ struct HashMethodFixedString using BatchKeyHolderType = KeyHolderType; static constexpr bool is_serialized_key = false; + static constexpr bool can_batch_get_key_holder = false; size_t n; const ColumnFixedString::Chars_t * chars; @@ -548,6 +548,7 @@ struct HashMethodKeysFixed using BatchKeyHolderType = KeyHolderType; static constexpr bool is_serialized_key = false; + static constexpr bool can_batch_get_key_holder = false; static constexpr bool has_nullable_keys = has_nullable_keys_; Sizes key_sizes; @@ -713,10 +714,7 @@ class KeySerializedBatchHandlerBase } protected: - bool inited() const - { - return !byte_size.empty(); - } + bool inited() const { return !byte_size.empty(); } void init(size_t start_row, const ColumnRawPtrs & key_columns, const TiDB::TiDBCollators & collators) { @@ -756,7 +754,7 @@ class KeySerializedBatchHandlerBase pos, batch_row_idx, cur_batch_size, - false, + nullptr, collators.empty() ? nullptr : collators[i], &sort_key_container); @@ -768,6 +766,7 @@ class KeySerializedBatchHandlerBase return mem_size; } +public: // NOTE: i is the index of mini batch, it's not the row index of Column. ALWAYS_INLINE inline ArenaKeyHolder getKeyHolderBatch(size_t i, Arena * pool) const { @@ -790,10 +789,12 @@ struct HashMethodSerialized using Self = HashMethodSerialized; using Base = columns_hashing_impl::HashMethodBase; using BatchHandlerBase = KeySerializedBatchHandlerBase; - static constexpr bool is_serialized_key = true; using KeyHolderType = SerializedKeyHolder; using BatchKeyHolderType = ArenaKeyHolder; + static constexpr bool is_serialized_key = true; + static constexpr bool can_batch_get_key_holder = true; + ColumnRawPtrs key_columns; size_t keys_size; TiDB::TiDBCollators collators; @@ -807,11 +808,6 @@ struct HashMethodSerialized , collators(collators_) {} - bool batchGetkeyHolder() override - { - return BatchHandlerBase::inited(); - } - void initBatchHandler(size_t start_row, size_t /* max_batch_size */) { assert(!BatchHandlerBase::inited()); @@ -849,6 +845,7 @@ struct HashMethodHashed using BatchKeyHolderType = KeyHolderType; static constexpr bool is_serialized_key = false; + static constexpr bool can_batch_get_key_holder = false; ColumnRawPtrs key_columns; TiDB::TiDBCollators collators; diff --git a/dbms/src/Common/ColumnsHashingImpl.h b/dbms/src/Common/ColumnsHashingImpl.h index a9a23db14f5..bf130d2bd29 100644 --- a/dbms/src/Common/ColumnsHashingImpl.h +++ b/dbms/src/Common/ColumnsHashingImpl.h @@ -129,11 +129,6 @@ class HashMethodBase using Cache = LastElementCache; using Derived = TDerived; - bool batchGetKeyHolder() const override - { - return false; - } - template ALWAYS_INLINE inline EmplaceResult emplaceKey( Data & data, diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index 9bf9402a7ed..c27921c6d99 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -46,6 +46,9 @@ extern const char random_fail_in_resize_callback[]; extern const char force_agg_prefetch[]; } // namespace FailPoints +static constexpr size_t agg_prefetch_step = 16; +static constexpr size_t agg_mini_batch = 256; + #define AggregationMethodName(NAME) AggregatedDataVariants::AggregationMethod_##NAME #define AggregationMethodNameTwoLevel(NAME) AggregatedDataVariants::AggregationMethod_##NAME##_two_level #define AggregationMethodType(NAME) AggregatedDataVariants::Type::NAME @@ -625,7 +628,8 @@ AggregatedDataVariants::Type Aggregator::chooseAggregationMethod() if (params.keys_size == 1 && types_not_null[0]->isFixedString()) return AggregatedDataVariants::Type::key_fixed_string; - return ChooseAggregationMethodFastPath(params.keys_size, types_not_null, params.collators); + // return ChooseAggregationMethodFastPath(params.keys_size, types_not_null, params.collators); + return AggregatedDataVariants::Type::serialized; } @@ -652,13 +656,6 @@ void Aggregator::createAggregateStates(AggregateDataPtr & aggregate_data) const } } -template -concept HasBatchGetKeyHolderMemberFunc = requires -{ - // todo also require initBatchHandler() - {std::declval().getKeyHolderBatch(std::declval(), std::declval())} -> std::same_as; -}; - /** It's interesting - if you remove `noinline`, then gcc for some reason will inline this function, and the performance decreases (~ 10%). * (Probably because after the inline of this function, more internal functions no longer be inlined.) * Inline does not make sense, since the inner loop is entirely inside this function. @@ -684,49 +681,63 @@ void NO_INLINE Aggregator::executeImpl( // For key_serialized, memory allocation and key serialization will be batch-wise. // For key_string, collation decode will be batch-wise. - if constexpr (HasBatchGetKeyHolderMemberFunc) + static constexpr bool batch_get_key_holder = Method::State::can_batch_get_key_holder; + if constexpr (batch_get_key_holder) { - state.initBatchHandler(agg_process_info.start_row); + state.initBatchHandler(agg_process_info.start_row, agg_mini_batch); result.batch_get_key_holder = true; } + using KeyHolderType = typename std::conditional< + batch_get_key_holder, + typename Method::State::BatchKeyHolderType, + typename Method::State::KeyHolderType>::type; if constexpr (Method::Data::is_string_hash_map) { // StringHashMap doesn't support prefetch. - executeImplBatch(method, state, aggregates_pool, agg_process_info); + executeImplBatch< + collect_hit_rate, + only_lookup, + /*enable_prefetch=*/false, + batch_get_key_holder, + KeyHolderType>(method, state, aggregates_pool, agg_process_info); } else { if (disable_prefetch) { - if constexpr (state.batchGetKeyHolder()) - executeImplBatch(method, state, aggregates_pool, agg_process_info); - else - executeImplBatch(method, state, aggregates_pool, agg_process_info); + executeImplBatch< + collect_hit_rate, + only_lookup, + /*enable_prefetch=*/false, + batch_get_key_holder, + KeyHolderType>(method, state, aggregates_pool, agg_process_info); } else { - if constexpr (state.batchGetKeyHolder()) - executeImplBatch(method, state, aggregates_pool, agg_process_info); - else - executeImplBatch(method, state, aggregates_pool, agg_process_info); + executeImplBatch< + collect_hit_rate, + only_lookup, + /*enable_prefetch=*/true, + batch_get_key_holder, + KeyHolderType>(method, state, aggregates_pool, agg_process_info); } } } -template +template std::optional::ResultType> Aggregator::emplaceOrFindKey( Method & method, typename Method::State & state, - typename Method::State::Derived::KeyHolderType && key_holder, + KeyHolderType & key_holder, size_t hashval) const { try { if constexpr (only_lookup) - return state.template findKey(method.data, std::move(key_holder), hashval); + return state.template findKey(method.data, key_holder, hashval); else - return state.template emplaceKey(method.data, std::move(key_holder), hashval); + return state.template emplaceKey(method.data, key_holder, hashval); } catch (ResizeException &) { @@ -755,7 +766,7 @@ std::optional::Res } } -template +template ALWAYS_INLINE inline void setupHashVals( size_t row_idx, size_t batch_size, @@ -766,14 +777,13 @@ ALWAYS_INLINE inline void setupHashVals( Method & method, typename Method::State & state) { - assert(hashvals.size() == key_holders.size() && hashvals.size() == batch_size); + assert(hashvals.size() == key_holders.size() && hashvals.size() >= batch_size); for (size_t i = row_idx, j = 0; i < row_idx + batch_size; ++i, ++j) { - if constexpr (Method::State::batch_get_key_holder) - key_holders[j] = static_cast(&state)->getKeyHolderBatch( - j, - aggregates_pool); + if constexpr (batch_get_key_holder) + key_holders[j] + = static_cast(&state)->getKeyHolderBatch(j, aggregates_pool); else key_holders[j] = static_cast(&state)->getKeyHolder( i, @@ -783,7 +793,13 @@ ALWAYS_INLINE inline void setupHashVals( } } -template +template < + bool collect_hit_rate, + bool only_lookup, + bool enable_prefetch, + bool batch_get_key_holder, + typename KeyHolderType, + typename Method> ALWAYS_INLINE void Aggregator::executeImplBatch( Method & method, typename Method::State & state, @@ -795,11 +811,13 @@ ALWAYS_INLINE void Aggregator::executeImplBatch( /// Optimization for special case when there are no aggregate functions. if (params.aggregates_size == 0) - return handleOneBatch( - method, - state, - agg_process_info, - aggregates_pool); + return handleOneBatch< + collect_hit_rate, + only_lookup, + enable_prefetch, + batch_get_key_holder, + /*compute_agg_data=*/false, + KeyHolderType>(method, state, agg_process_info, aggregates_pool); /// Optimization for special case when aggregating by 8bit key. if constexpr (std::is_same_v) @@ -841,14 +859,23 @@ ALWAYS_INLINE void Aggregator::executeImplBatch( } /// Generic case. - return handleOneBatch( - method, - state, - agg_process_info, - aggregates_pool); -} - -template + return handleOneBatch< + collect_hit_rate, + only_lookup, + enable_prefetch, + batch_get_key_holder, + /*compute_agg_data=*/true, + KeyHolderType>(method, state, agg_process_info, aggregates_pool); +} + +template < + bool collect_hit_rate, + bool only_lookup, + bool enable_prefetch, + bool batch_get_key_holder, + bool compute_agg_data, + typename KeyHolderType, + typename Method> void Aggregator::handleOneBatch( Method & method, typename Method::State & state, @@ -884,23 +911,32 @@ void Aggregator::handleOneBatch( // mini batch will only be used when HashTable is big(a.k.a enable_prefetch is true), // which can reduce cache miss of agg data. mini_batch_size = agg_mini_batch; - hashvals.resize(agg_mini_batch); - key_holders.resize(agg_mini_batch); } // i is the begin row index of each mini batch. while (i < end) { + if unlikely (i + mini_batch_size > end) + mini_batch_size = end - i; + size_t batch_mem_size = 0; - if constexpr (Method::State::batch_get_key_holder) - batch_mem_size = state.prepareNextBatch(mini_batch_size, &temp_batch_pool); + if constexpr (batch_get_key_holder) + batch_mem_size = state.prepareNextBatch(&temp_batch_pool, mini_batch_size); - if constexpr (enable_prefetch) + if constexpr (enable_prefetch || batch_get_key_holder) { - if unlikely (i + mini_batch_size > end) - mini_batch_size = end - i; + hashvals.resize(mini_batch_size); + key_holders.resize(mini_batch_size); - setupHashVals(i, mini_batch_size, hashvals, key_holders, aggregates_pool, sort_key_containers, method, state); + setupHashVals( + i, + mini_batch_size, + hashvals, + key_holders, + aggregates_pool, + sort_key_containers, + method, + state); } const auto cur_batch_end = i + mini_batch_size; @@ -915,13 +951,11 @@ void Aggregator::handleOneBatch( if likely (k + agg_prefetch_step < hashvals.size()) method.data.prefetch(hashvals[k + agg_prefetch_step]); - emplace_result_holder - = emplaceOrFindKey(method, state, std::move(key_holders[k]), hashvals[k]); + emplace_result_holder = emplaceOrFindKey(method, state, key_holders[k], hashvals[k]); } - else if constexpr (Method::State::batch_get_key_holder) + else if constexpr (batch_get_key_holder) { - emplace_result_holder - = emplaceOrFindKey(method, state, std::move(key_holders[k]), hashvals[k]); + emplace_result_holder = emplaceOrFindKey(method, state, key_holders[k], hashvals[k]); } else { @@ -987,7 +1021,8 @@ void Aggregator::handleOneBatch( processed_rows = j; } - if constexpr (Method::State::batch_get_key_holder) + // todo rollback all to avoid wasting align. + if constexpr (batch_get_key_holder) temp_batch_pool.rollback(batch_mem_size); if unlikely (!processed_rows.has_value()) @@ -1193,7 +1228,7 @@ bool Aggregator::executeOnBlockImpl( { \ executeImpl( \ *ToAggregationMethodPtr(NAME, result.aggregation_method_impl), \ - result, \ + result, \ agg_process_info, \ params.collators); \ break; \ @@ -1301,39 +1336,51 @@ Block Aggregator::convertOneBucketToBlock( size_t bucket) const { const bool batch_get_key_holder = data_variants.batch_get_key_holder; -#define FILLER_DEFINE(name, skip_convert_key) \ - auto filler_##name = [bucket, &method, arena, this, batch_get_key_holder]( \ - const Sizes & key_sizes, \ - MutableColumns & key_columns, \ - AggregateColumnsData & aggregate_columns, \ - MutableColumns & final_aggregate_columns, \ - bool final_) { \ - using METHOD_TYPE = std::decay_t; \ - using DATA_TYPE = std::decay_t; \ - if (METHOD_TYPE::State::is_serialized_key && batch_get_key_holder) \ - { \ - convertToBlockImpl( \ - method, \ - method.data.impls[bucket], \ - key_sizes, \ - key_columns, \ - aggregate_columns, \ - final_aggregate_columns, \ - arena, \ - final_); \ - } \ - else \ - { \ - convertToBlockImpl( \ - method, \ - method.data.impls[bucket], \ - key_sizes, \ - key_columns, \ - aggregate_columns, \ - final_aggregate_columns, \ - arena, \ - final_); \ - } \ +#define FILLER_DEFINE(name, skip_convert_key) \ + auto filler_##name = [bucket, &method, arena, this, batch_get_key_holder]( \ + const Sizes & key_sizes, \ + MutableColumns & key_columns, \ + AggregateColumnsData & aggregate_columns, \ + MutableColumns & final_aggregate_columns, \ + bool final_) { \ + (void)batch_get_key_holder; \ + using METHOD_TYPE = std::decay_t; \ + using DATA_TYPE = std::decay_t; \ + if constexpr (METHOD_TYPE::State::is_serialized_key && METHOD_TYPE::State::can_batch_get_key_holder) \ + { \ + if (batch_get_key_holder) \ + convertToBlockImpl( \ + method, \ + method.data.impls[bucket], \ + key_sizes, \ + key_columns, \ + aggregate_columns, \ + final_aggregate_columns, \ + arena, \ + final_); \ + else \ + convertToBlockImpl( \ + method, \ + method.data.impls[bucket], \ + key_sizes, \ + key_columns, \ + aggregate_columns, \ + final_aggregate_columns, \ + arena, \ + final_); \ + } \ + else \ + { \ + convertToBlockImpl( \ + method, \ + method.data.impls[bucket], \ + key_sizes, \ + key_columns, \ + aggregate_columns, \ + final_aggregate_columns, \ + arena, \ + final_); \ + } \ } FILLER_DEFINE(convert_key, false); @@ -1377,37 +1424,49 @@ BlocksList Aggregator::convertOneBucketToBlocks( size_t bucket) const { const auto batch_get_key_holder = data_variants.batch_get_key_holder; -#define FILLER_DEFINE(name, skip_convert_key) \ - auto filler_##name = [bucket, &method, arena, this, batch_deserialize_key]( \ - const Sizes & key_sizes, \ - std::vector & key_columns_vec, \ - std::vector & aggregate_columns_vec, \ - std::vector & final_aggregate_columns_vec, \ - bool final_) { \ - if (Method::State::is_serialized_key && batch_get_key_holder) \ - { \ - convertToBlocksImpl( \ - method, \ - method.data.impls[bucket], \ - key_sizes, \ - key_columns_vec, \ - aggregate_columns_vec, \ - final_aggregate_columns_vec, \ - arena, \ - final_); \ - } \ - else \ - { \ - convertToBlocksImpl( \ - method, \ - method.data.impls[bucket], \ - key_sizes, \ - key_columns_vec, \ - aggregate_columns_vec, \ - final_aggregate_columns_vec, \ - arena, \ - final_); \ - } \ +#define FILLER_DEFINE(name, skip_convert_key) \ + auto filler_##name = [bucket, &method, arena, this, batch_get_key_holder]( \ + const Sizes & key_sizes, \ + std::vector & key_columns_vec, \ + std::vector & aggregate_columns_vec, \ + std::vector & final_aggregate_columns_vec, \ + bool final_) { \ + (void)batch_get_key_holder; \ + if constexpr (Method::State::is_serialized_key && Method::State::can_batch_get_key_holder) \ + { \ + if (batch_get_key_holder) \ + convertToBlocksImpl( \ + method, \ + method.data.impls[bucket], \ + key_sizes, \ + key_columns_vec, \ + aggregate_columns_vec, \ + final_aggregate_columns_vec, \ + arena, \ + final_); \ + else \ + convertToBlocksImpl( \ + method, \ + method.data.impls[bucket], \ + key_sizes, \ + key_columns_vec, \ + aggregate_columns_vec, \ + final_aggregate_columns_vec, \ + arena, \ + final_); \ + } \ + else \ + { \ + convertToBlocksImpl( \ + method, \ + method.data.impls[bucket], \ + key_sizes, \ + key_columns_vec, \ + aggregate_columns_vec, \ + final_aggregate_columns_vec, \ + arena, \ + final_); \ + } \ }; FILLER_DEFINE(convert_key, false); @@ -1819,7 +1878,7 @@ void NO_INLINE Aggregator::convertToBlockImplFinal( if constexpr (batch_deserialize_key) { // Assume key is StringRef, because only key_string and key_serialize can be batch-wise. - key_places.push_back(key.data); + key_places.push_back(const_cast(key.data)); } else { @@ -1921,12 +1980,15 @@ void NO_INLINE Aggregator::convertToBlocksImplFinal( if constexpr (batch_deserialize_key) { // Assume key is StringRef, because only key_string and key_serialize can be batch-wise. - key_places.push_back(key.data); + key_places.push_back(const_cast(key.data)); } else { - agg_keys_helpers[key_columns_vec_index] - ->insertKeyIntoColumns(key, key_columns_vec[key_columns_vec_index], key_sizes_ref, params.collators); + agg_keys_helpers[key_columns_vec_index]->insertKeyIntoColumns( + key, + key_columns_vec[key_columns_vec_index], + key_sizes_ref, + params.collators); } } places[data_index] = mapped; @@ -1934,8 +1996,11 @@ void NO_INLINE Aggregator::convertToBlocksImplFinal( if unlikely (data_index == current_bound) { - method.insertKeyIntoColumnsBatch(key_places, key_columns_vec[key_columns_vec_index]); - key_places.clear(); + if constexpr (!skip_convert_key && batch_deserialize_key) + { + method.insertKeyIntoColumnsBatch(key_places, key_columns_vec[key_columns_vec_index]); + key_places.clear(); + } ++key_columns_vec_index; current_bound += params.max_block_size; @@ -1988,7 +2053,7 @@ void NO_INLINE Aggregator::convertToBlockImplNotFinal( } agg_keys_helper.initAggKeys(data.size(), key_columns); } - + // For key_serialized, deserialize will be batch-wise if it's serialized batch-wise. PaddedPODArray key_places; if constexpr (batch_deserialize_key) @@ -2000,7 +2065,7 @@ void NO_INLINE Aggregator::convertToBlockImplNotFinal( if constexpr (batch_deserialize_key) { // Assume key is StringRef. - key_places.push_back(key.data); + key_places.push_back(const_cast(key.data)); } else { @@ -2063,12 +2128,15 @@ void NO_INLINE Aggregator::convertToBlocksImplNotFinal( if constexpr (batch_deserialize_key) { // Assume key is StringRef, because only key_string and key_serialize can be batch-wise. - key_places.push_back(key.data); + key_places.push_back(const_cast(key.data)); } else { - agg_keys_helpers[key_columns_vec_index] - ->insertKeyIntoColumns(key, key_columns_vec[key_columns_vec_index], key_sizes_ref, params.collators); + agg_keys_helpers[key_columns_vec_index]->insertKeyIntoColumns( + key, + key_columns_vec[key_columns_vec_index], + key_sizes_ref, + params.collators); } } @@ -2081,8 +2149,11 @@ void NO_INLINE Aggregator::convertToBlocksImplNotFinal( if unlikely (data_index == current_bound) { - method.insertKeyIntoColumnsBatch(key_places, key_columns_vec[++key_columns_vec_index]); - key_places.clear(); + if constexpr (!skip_convert_key && batch_deserialize_key) + { + method.insertKeyIntoColumnsBatch(key_places, key_columns_vec[++key_columns_vec_index]); + key_places.clear(); + } ++key_columns_vec_index; current_bound += params.max_block_size; @@ -2344,26 +2415,38 @@ BlocksList Aggregator::prepareBlocksAndFillSingleLevel(AggregatedDataVariants & { size_t rows = data_variants.size(); const bool batch_get_key_holder = data_variants.batch_get_key_holder; -#define M(NAME, skip_convert_key) \ - case AggregationMethodType(NAME): \ - { \ - auto & tmp_method = *ToAggregationMethodPtr(NAME, data_variants.aggregation_method_impl); \ - auto & tmp_data = ToAggregationMethodPtr(NAME, data_variants.aggregation_method_impl) -> data; \ - if (decltype(tmp_method)::State::is_serialized_key && batch_get_key_holder) \ - { \ - convertToBlocksImpl( \ - tmp_method, \ - tmp_data, \ - key_sizes, \ - key_columns_vec, \ - aggregate_columns_vec, \ - final_aggregate_columns_vec, \ - data_variants.aggregates_pool, \ - final_); \ - } \ - else \ - { \ - convertToBlocksImpl( \ +#define M(NAME, skip_convert_key) \ + case AggregationMethodType(NAME): \ + { \ + auto & tmp_method = *ToAggregationMethodPtr(NAME, data_variants.aggregation_method_impl); \ + auto & tmp_data = ToAggregationMethodPtr(NAME, data_variants.aggregation_method_impl) -> data; \ + using MethodType = std::decay_t; \ + if constexpr (MethodType::State::is_serialized_key && MethodType::State::can_batch_get_key_holder) \ + { \ + if (batch_get_key_holder) \ + convertToBlocksImpl( \ + tmp_method, \ + tmp_data, \ + key_sizes, \ + key_columns_vec, \ + aggregate_columns_vec, \ + final_aggregate_columns_vec, \ + data_variants.aggregates_pool, \ + final_); \ + else \ + convertToBlocksImpl( \ + tmp_method, \ + tmp_data, \ + key_sizes, \ + key_columns_vec, \ + aggregate_columns_vec, \ + final_aggregate_columns_vec, \ + data_variants.aggregates_pool, \ + final_); \ + } \ + else \ + { \ + convertToBlocksImpl( \ tmp_method, \ tmp_data, \ key_sizes, \ @@ -2372,20 +2455,21 @@ BlocksList Aggregator::prepareBlocksAndFillSingleLevel(AggregatedDataVariants & final_aggregate_columns_vec, \ data_variants.aggregates_pool, \ final_); \ - } \ - break; \ + } \ + break; \ } #define M_skip_convert_key(NAME) M(NAME, true) #define M_convert_key(NAME) M(NAME, false) #define FILLER_DEFINE(name, M_tmp) \ - auto filler_##name = [&data_variants, this, batch_get_key_holder]( \ + auto filler_##name = [&data_variants, this, batch_get_key_holder]( \ const Sizes & key_sizes, \ std::vector & key_columns_vec, \ std::vector & aggregate_columns_vec, \ std::vector & final_aggregate_columns_vec, \ bool final_) { \ + (void)batch_get_key_holder; \ switch (data_variants.type) \ { \ APPLY_FOR_VARIANTS_SINGLE_LEVEL(M_tmp) \ @@ -2576,7 +2660,7 @@ MergingBucketsPtr Aggregator::mergeAndConvertToBlocks( } for (auto & data : non_empty_data) - RUNTIME_CHECK(non_empty_data[0]->batch_get_key_holder == data.batch_get_key_holder); + RUNTIME_CHECK(non_empty_data[0]->batch_get_key_holder == data->batch_get_key_holder); // for single level merge, concurrency must be 1. size_t merge_concurrency = has_at_least_one_two_level ? std::max(max_threads, 1) : 1; diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index 5220ed70916..272ee342dfe 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -52,9 +52,6 @@ class IBlockOutputStream; template class AggHashTableToBlocksBlockInputStream; -static constexpr size_t agg_prefetch_step = 16; -static constexpr size_t agg_mini_batch = 256; - /** Different data structures that can be used for aggregation * For efficiency, the aggregation data itself is put into the pool. * Data and pool ownership (states of aggregate functions) @@ -717,12 +714,11 @@ struct AggregationMethodSerialized pos = key_columns[i]->deserializeAndInsertFromArena(pos, collators.empty() ? nullptr : collators[i]); } - static void insertKeyIntoColumnsBatch( - PaddedPODArray & key_places, - std::vector & key_columns) + static void insertKeyIntoColumnsBatch(PaddedPODArray & key_places, std::vector & key_columns) { + // todo: nt optimization for (auto * key_column : key_columns) - key_column->deserializeForCmpAndInsertFromPos(key_places, true); + key_column->deserializeForCmpAndInsertFromPos(key_places, false); } }; @@ -1461,29 +1457,42 @@ class Aggregator template void executeImpl( Method & method, - Arena * aggregates_pool, + AggregatedDataVariants & result, AggProcessInfo & agg_process_info, TiDB::TiDBCollators & collators) const; - template + template < + bool collect_hit_rate, + bool only_loopup, + bool enable_prefetch, + bool batch_get_key_holder, + typename KeyHolderType, + typename Method> void executeImplBatch( Method & method, typename Method::State & state, Arena * aggregates_pool, AggProcessInfo & agg_process_info) const; - template + template < + bool collect_hit_rate, + bool only_lookup, + bool enable_prefetch, + bool batch_get_key_holder, + bool compute_agg_data, + typename KeyHolderType, + typename Method> void handleOneBatch( Method & method, typename Method::State & state, AggProcessInfo & agg_process_info, Arena * aggregates_pool) const; - template + template std::optional::ResultType> emplaceOrFindKey( Method & method, typename Method::State & state, - typename Method::State::Derived::KeyHolderType && key_holder, + KeyHolderType & key_holder, size_t hashval) const; template From 3bb7660585fb3d88bf50791c8dcc4483a5e2d431 Mon Sep 17 00:00:00 2001 From: guo-shaoge Date: Tue, 4 Feb 2025 23:34:53 +0800 Subject: [PATCH 16/16] key_string batch Signed-off-by: guo-shaoge --- dbms/src/Common/ColumnsHashing.h | 18 ++++++------- dbms/src/Common/ColumnsHashingImpl.h | 12 +++++++++ dbms/src/Interpreters/Aggregator.cpp | 40 +++++++++++++++++++++------- dbms/src/Interpreters/Aggregator.h | 6 +++++ 4 files changed, 57 insertions(+), 19 deletions(-) diff --git a/dbms/src/Common/ColumnsHashing.h b/dbms/src/Common/ColumnsHashing.h index 30f9bddea3f..d353a5d8be9 100644 --- a/dbms/src/Common/ColumnsHashing.h +++ b/dbms/src/Common/ColumnsHashing.h @@ -106,6 +106,8 @@ class KeyStringBatchHandlerBase if (cur_batch_size <= 0) return 0; + batch_rows.resize(cur_batch_size); + const auto * derived_collator = static_cast(collator); for (size_t i = 0; i < cur_batch_size; ++i) { @@ -118,13 +120,14 @@ class KeyStringBatchHandlerBase batch_rows[i] = key; } + batch_row_idx += cur_batch_size; return 0; } void santityCheck() const { // Make sure init() has been called. - assert(sort_key_containers.size() == batch_rows.size() && !sort_key_containers.empty()); + assert(!sort_key_containers.empty()); } protected: @@ -144,8 +147,6 @@ class KeyStringBatchHandlerBase size_t cur_batch_size, const TiDB::TiDBCollatorPtr & collator) { - batch_rows.resize(cur_batch_size); - if likely (collator) { #define M(VAR_PREFIX, COLLATOR_NAME, IMPL_TYPE, COLLATOR_ID) \ @@ -195,10 +196,9 @@ struct HashMethodString using BatchHandlerBase = KeyStringBatchHandlerBase; static constexpr bool is_serialized_key = false; - // todo - static constexpr bool can_batch_get_key_holder = false; + static constexpr bool can_batch_get_key_holder = true; - const IColumn::Offset * offsets; + const IColumn::Offsets * offsets; const UInt8 * chars; TiDB::TiDBCollatorPtr collator = nullptr; @@ -209,7 +209,7 @@ struct HashMethodString { const IColumn & column = *key_columns[0]; const auto & column_string = assert_cast(column); - offsets = column_string.getOffsets().data(); + offsets = &column_string.getOffsets(); chars = column_string.getChars().data(); if (!collators.empty()) collator = collators[0]; @@ -234,9 +234,9 @@ struct HashMethodString { assert(!BatchHandlerBase::inited()); - auto last_offset = row == 0 ? 0 : offsets[row - 1]; + auto last_offset = (*offsets)[row - 1]; // Remove last zero byte. - StringRef key(chars + last_offset, offsets[row] - last_offset - 1); + StringRef key(chars + last_offset, (*offsets)[row] - last_offset - 1); if (likely(collator)) key = collator->sortKey(key.data, key.size, sort_key_containers[0]); diff --git a/dbms/src/Common/ColumnsHashingImpl.h b/dbms/src/Common/ColumnsHashingImpl.h index bf130d2bd29..4f6c56be6d4 100644 --- a/dbms/src/Common/ColumnsHashingImpl.h +++ b/dbms/src/Common/ColumnsHashingImpl.h @@ -140,6 +140,12 @@ class HashMethodBase return emplaceImpl(key_holder, data); } + template + ALWAYS_INLINE inline EmplaceResult emplaceKey(Data & data, KeyHolder && key_holder) + { + return emplaceImpl(key_holder, data); + } + template ALWAYS_INLINE inline EmplaceResult emplaceKey(Data & data, KeyHolder && key_holder, size_t hashval) { @@ -157,6 +163,12 @@ class HashMethodBase return findKeyImpl(keyHolderGetKey(key_holder), data); } + template + ALWAYS_INLINE inline FindResult findKey(Data & data, KeyHolder && key_holder) + { + return findKeyImpl(keyHolderGetKey(key_holder), data); + } + template ALWAYS_INLINE inline FindResult findKey(Data & data, KeyHolder && key_holder, size_t hashval) { diff --git a/dbms/src/Interpreters/Aggregator.cpp b/dbms/src/Interpreters/Aggregator.cpp index c27921c6d99..9f0fd12efe9 100644 --- a/dbms/src/Interpreters/Aggregator.cpp +++ b/dbms/src/Interpreters/Aggregator.cpp @@ -745,6 +745,25 @@ std::optional::Res } } +template +std::optional::ResultType> Aggregator::emplaceOrFindKey( + Method & method, + typename Method::State & state, + KeyHolderType & key_holder) const +{ + try + { + if constexpr (only_lookup) + return state.template findKey(method.data, key_holder); + else + return state.template emplaceKey(method.data, key_holder); + } + catch (ResizeException &) + { + return {}; + } +} + template std::optional::ResultType> Aggregator::emplaceOrFindKey( Method & method, @@ -766,8 +785,8 @@ std::optional::Res } } -template -ALWAYS_INLINE inline void setupHashVals( +template +ALWAYS_INLINE inline void setupKeyHolderAndHashVal( size_t row_idx, size_t batch_size, std::vector & hashvals, @@ -777,7 +796,9 @@ ALWAYS_INLINE inline void setupHashVals( Method & method, typename Method::State & state) { - assert(hashvals.size() == key_holders.size() && hashvals.size() >= batch_size); + key_holders.resize(batch_size); + if constexpr (enable_prefetch) + hashvals.resize(batch_size); for (size_t i = row_idx, j = 0; i < row_idx + batch_size; ++i, ++j) { @@ -789,7 +810,9 @@ ALWAYS_INLINE inline void setupHashVals( i, aggregates_pool, sort_key_containers); - hashvals[j] = method.data.hash(keyHolderGetKey(key_holders[j])); + + if constexpr (enable_prefetch) + hashvals[j] = method.data.hash(keyHolderGetKey(key_holders[j])); } } @@ -906,7 +929,7 @@ void Aggregator::handleOneBatch( size_t mini_batch_size = rows; std::vector hashvals; std::vector key_holders; - if constexpr (enable_prefetch) + if constexpr (enable_prefetch || batch_get_key_holder) { // mini batch will only be used when HashTable is big(a.k.a enable_prefetch is true), // which can reduce cache miss of agg data. @@ -925,10 +948,7 @@ void Aggregator::handleOneBatch( if constexpr (enable_prefetch || batch_get_key_holder) { - hashvals.resize(mini_batch_size); - key_holders.resize(mini_batch_size); - - setupHashVals( + setupKeyHolderAndHashVal( i, mini_batch_size, hashvals, @@ -955,7 +975,7 @@ void Aggregator::handleOneBatch( } else if constexpr (batch_get_key_holder) { - emplace_result_holder = emplaceOrFindKey(method, state, key_holders[k], hashvals[k]); + emplace_result_holder = emplaceOrFindKey(method, state, key_holders[k]); } else { diff --git a/dbms/src/Interpreters/Aggregator.h b/dbms/src/Interpreters/Aggregator.h index 272ee342dfe..e4c7d33af13 100644 --- a/dbms/src/Interpreters/Aggregator.h +++ b/dbms/src/Interpreters/Aggregator.h @@ -1495,6 +1495,12 @@ class Aggregator KeyHolderType & key_holder, size_t hashval) const; + template + std::optional::ResultType> emplaceOrFindKey( + Method & method, + typename Method::State & state, + KeyHolderType & key_holder) const; + template std::optional::ResultType> emplaceOrFindKey( Method & method,