[DYOD] Add variable string segment

src/benchmark/operators/table_scan_sorted_benchmark.cpp

@@ -208,6 +208,7 @@ void registerTableScanSortedBenchmarks() {
  {"None", EncodingAndSupportedDataTypes(EncodingType::Unencoded, {"Int", "String"})},
  {"Dictionary", EncodingAndSupportedDataTypes(EncodingType::Dictionary, {"Int", "String"})},
  {"FixedStringDictionary", EncodingAndSupportedDataTypes(EncodingType::FixedStringDictionary, {"String"})},
+ {"VariableStringDictionary", EncodingAndSupportedDataTypes(EncodingType::VariableStringDictionary, {"String"})},
  {"FrameOfReference", EncodingAndSupportedDataTypes(EncodingType::FrameOfReference, {"Int"})},
  {"RunLength", EncodingAndSupportedDataTypes(EncodingType::RunLength, {"Int", "String"})},
  {"LZ4", EncodingAndSupportedDataTypes(EncodingType::LZ4, {"Int", "String"})}};

src/lib/CMakeLists.txt

@@ -209,12 +209,12 @@ set(
  operators/index_scan.hpp
  operators/insert.cpp
  operators/insert.hpp
- operators/join_helper/join_output_writing.cpp
- operators/join_helper/join_output_writing.hpp
  operators/join_hash.cpp
  operators/join_hash.hpp
  operators/join_hash/join_hash_steps.hpp
  operators/join_hash/join_hash_traits.hpp
+ operators/join_helper/join_output_writing.cpp
+ operators/join_helper/join_output_writing.hpp
  operators/join_index.cpp
  operators/join_index.hpp
  operators/join_nested_loop.cpp
@@ -484,6 +484,9 @@ set(
  storage/index/adaptive_radix_tree/adaptive_radix_tree_index.hpp
  storage/index/adaptive_radix_tree/adaptive_radix_tree_nodes.cpp
  storage/index/adaptive_radix_tree/adaptive_radix_tree_nodes.hpp
+ storage/index/chunk_index_statistics.cpp
+ storage/index/chunk_index_statistics.hpp
+ storage/index/chunk_index_type.hpp
  storage/index/group_key/composite_group_key_index.cpp
  storage/index/group_key/composite_group_key_index.hpp
  storage/index/group_key/group_key_index.cpp
@@ -496,10 +499,6 @@ set(
  storage/index/group_key/variable_length_key_proxy.hpp
  storage/index/group_key/variable_length_key_store.cpp
  storage/index/group_key/variable_length_key_store.hpp
- storage/index/chunk_index_statistics.cpp
- storage/index/chunk_index_statistics.hpp
- storage/index/table_index_statistics.cpp
- storage/index/table_index_statistics.hpp
  storage/index/partial_hash/flat_map_iterator.cpp
  storage/index/partial_hash/flat_map_iterator.hpp
  storage/index/partial_hash/flat_map_iterator_impl.cpp
@@ -508,7 +507,8 @@ set(
  storage/index/partial_hash/partial_hash_index.hpp
  storage/index/partial_hash/partial_hash_index_impl.cpp
  storage/index/partial_hash/partial_hash_index_impl.hpp
- storage/index/chunk_index_type.hpp
+ storage/index/table_index_statistics.cpp
+ storage/index/table_index_statistics.hpp
  storage/lqp_view.cpp
  storage/lqp_view.hpp
  storage/lz4_segment.cpp
@@ -560,9 +560,23 @@ set(
  storage/value_segment.hpp
  storage/value_segment/null_value_vector_iterable.hpp
  storage/value_segment/value_segment_iterable.hpp
+ storage/variable_string_dictionary/variable_string_dictionary_encoder.hpp
+ storage/variable_string_dictionary/variable_string_dictionary_iterable.hpp
+ storage/variable_string_dictionary/variable_string_vector.cpp
+ storage/variable_string_dictionary/variable_string_vector.hpp
+ storage/variable_string_dictionary/variable_string_vector_iterator.hpp
+ storage/variable_string_dictionary_segment.cpp
+ storage/variable_string_dictionary_segment.hpp
  storage/vector_compression/base_compressed_vector.hpp
  storage/vector_compression/base_vector_compressor.hpp
  storage/vector_compression/base_vector_decompressor.hpp
+ storage/vector_compression/bitpacking/bitpacking_compressor.cpp
+ storage/vector_compression/bitpacking/bitpacking_compressor.hpp
+ storage/vector_compression/bitpacking/bitpacking_decompressor.hpp
+ storage/vector_compression/bitpacking/bitpacking_iterator.hpp
+ storage/vector_compression/bitpacking/bitpacking_vector.cpp
+ storage/vector_compression/bitpacking/bitpacking_vector.hpp
+ storage/vector_compression/bitpacking/bitpacking_vector_type.hpp
  storage/vector_compression/compressed_vector_type.cpp
  storage/vector_compression/compressed_vector_type.hpp
  storage/vector_compression/fixed_width_integer/fixed_width_integer_compressor.cpp
@@ -571,13 +585,6 @@ set(
  storage/vector_compression/fixed_width_integer/fixed_width_integer_utils.hpp
  storage/vector_compression/fixed_width_integer/fixed_width_integer_vector.hpp
  storage/vector_compression/resolve_compressed_vector_type.hpp
- storage/vector_compression/bitpacking/bitpacking_compressor.cpp
- storage/vector_compression/bitpacking/bitpacking_compressor.hpp
- storage/vector_compression/bitpacking/bitpacking_iterator.hpp
- storage/vector_compression/bitpacking/bitpacking_decompressor.hpp
- storage/vector_compression/bitpacking/bitpacking_vector.hpp
- storage/vector_compression/bitpacking/bitpacking_vector.cpp
- storage/vector_compression/bitpacking/bitpacking_vector_type.hpp
  storage/vector_compression/vector_compression.cpp
  storage/vector_compression/vector_compression.hpp
  strong_typedef.hpp

src/lib/import_export/binary/binary_parser.cpp

@@ -26,6 +26,7 @@
 #include "storage/table.hpp"
 #include "storage/table_column_definition.hpp"
 #include "storage/value_segment.hpp"
+#include "storage/variable_string_dictionary_segment.hpp"
 #include "storage/vector_compression/bitpacking/bitpacking_vector.hpp"
 #include "storage/vector_compression/bitpacking/bitpacking_vector_type.hpp"
 #include "storage/vector_compression/compressed_vector_type.hpp"
@@ -175,6 +176,8 @@ std::shared_ptr<AbstractSegment> BinaryParser::_import_segment(std::ifstream& fi
  } else {
  Fail("Unsupported data type for FixedStringDictionary encoding");
  }
+ case EncodingType::VariableStringDictionary:
+ return _import_variable_string_length_segment<pmr_string>(file, row_count);
  case EncodingType::RunLength:
  return _import_run_length_segment<ColumnDataType>(file, row_count);
  case EncodingType::FrameOfReference:
@@ -219,6 +222,24 @@ std::shared_ptr<DictionarySegment<T>> BinaryParser::_import_dictionary_segment(s
  return std::make_shared<DictionarySegment<T>>(dictionary, attribute_vector);
 }
 
+template <typename T>
+std::shared_ptr<VariableStringDictionarySegment<T>> BinaryParser::_import_variable_string_length_segment(
+ std::ifstream& file, ChunkOffset row_count) {
+ // Read attribute vector compression type and use it to decompress.
+ const auto compressed_vector_type_id = _read_value<CompressedVectorTypeID>(file);
+ const auto attribute_vector = _import_attribute_vector(file, row_count, compressed_vector_type_id);
+
+ // Read offset vector.
+ const auto offset_vector_size = _read_value<uint32_t>(file);
+ const auto offset_vector = std::make_shared<pmr_vector<uint32_t>>(_read_values<uint32_t>(file, offset_vector_size));
+
+ // Read dictionary.
+ const auto dictionary_size = _read_value<uint32_t>(file);
+ const auto dictionary = std::make_shared<pmr_vector<char>>(_read_values<char>(file, dictionary_size));
+
+ return std::make_shared<VariableStringDictionarySegment<pmr_string>>(dictionary, attribute_vector, offset_vector);
+}
+
 std::shared_ptr<FixedStringDictionarySegment<pmr_string>> BinaryParser::_import_fixed_string_dictionary_segment(
  std::ifstream& file, ChunkOffset row_count) {
  const auto compressed_vector_type_id = _read_value<CompressedVectorTypeID>(file);

src/lib/import_export/binary/binary_parser.hpp

@@ -16,6 +16,7 @@
 #include "storage/run_length_segment.hpp"
 #include "storage/table.hpp"
 #include "storage/value_segment.hpp"
+#include "storage/variable_string_dictionary_segment.hpp"
 #include "storage/vector_compression/bitpacking/bitpacking_vector_type.hpp"
 
 namespace hyrise {
@@ -76,6 +77,10 @@ class BinaryParser {
  template <typename T>
  static std::shared_ptr<DictionarySegment<T>> _import_dictionary_segment(std::ifstream& file, ChunkOffset row_count);
 
+ template <typename T>
+ static std::shared_ptr<VariableStringDictionarySegment<T>> _import_variable_string_length_segment(
+ std::ifstream& file, ChunkOffset row_count);
+
  static std::shared_ptr<FixedStringDictionarySegment<pmr_string>> _import_fixed_string_dictionary_segment(
  std::ifstream& file, ChunkOffset row_count);
 

src/lib/import_export/binary/binary_writer.cpp

@@ -23,6 +23,7 @@
 #include "storage/segment_iterate.hpp"
 #include "storage/table.hpp"
 #include "storage/value_segment.hpp"
+#include "storage/variable_string_dictionary_segment.hpp"
 #include "storage/vector_compression/bitpacking/bitpacking_vector.hpp"
 #include "storage/vector_compression/bitpacking/bitpacking_vector_type.hpp"
 #include "storage/vector_compression/compressed_vector_type.hpp"
@@ -350,6 +351,26 @@ void BinaryWriter::_write_segment(const LZ4Segment<T>& lz4_segment, bool /*colum
  }
 }
 
+template <typename T>
+void BinaryWriter::_write_segment(const VariableStringDictionarySegment<T>& dictionary_segment,
+ bool /*column_is_nullable*/, std::ofstream& ofstream) {
+ export_value(ofstream, EncodingType::VariableStringDictionary);
+
+ // Write attribute vector compression type and data.
+ const auto compressed_vector_type_id = _compressed_vector_type_id<T>(dictionary_segment);
+ export_value(ofstream, compressed_vector_type_id);
+ _export_compressed_vector(ofstream, *dictionary_segment.compressed_vector_type(),
+ *dictionary_segment.attribute_vector());
+
+ // Write offset vector.
+ export_value(ofstream, static_cast<uint32_t>(dictionary_segment.offset_vector()->size()));
+ export_values(ofstream, *dictionary_segment.offset_vector());
+
+ // Write the dictionary size and dictionary
+ export_value(ofstream, static_cast<uint32_t>(dictionary_segment.dictionary()->size()));
+ export_values(ofstream, *dictionary_segment.dictionary());
+}
+
 template <typename T>
 CompressedVectorTypeID BinaryWriter::_compressed_vector_type_id(
  const AbstractEncodedSegment& abstract_encoded_segment) {

src/lib/import_export/binary/binary_writer.hpp

@@ -11,6 +11,7 @@
 #include "storage/reference_segment.hpp"
 #include "storage/run_length_segment.hpp"
 #include "storage/value_segment.hpp"
+#include "storage/variable_string_dictionary_segment.hpp"
 
 namespace hyrise {
 
@@ -222,6 +223,10 @@ class BinaryWriter {
  template <typename T>
  static void _write_segment(const LZ4Segment<T>& lz4_segment, bool /*column_is_nullable*/, std::ofstream& ofstream);
 
+ template <typename T>
+ static void _write_segment(const VariableStringDictionarySegment<T>& dictionary_segment, bool /*column_is_nullable*/,
+ std::ofstream& ofstream);
+
  template <typename T>
  static CompressedVectorTypeID _compressed_vector_type_id(const AbstractEncodedSegment& abstract_encoded_segment);
 

src/lib/operators/aggregate_hash.cpp

@@ -15,6 +15,7 @@
 #include <vector>
 
 #include <boost/container/pmr/monotonic_buffer_resource.hpp>
+#include <boost/unordered/unordered_flat_map.hpp>
 
 #include "aggregate/window_function_traits.hpp"
 #include "all_type_variant.hpp"

src/lib/operators/print.cpp

@@ -249,6 +249,10 @@ std::string Print::_segment_type(const std::shared_ptr<AbstractSegment>& segment
  segment_type += "FSD";
  break;
  }
+ case EncodingType::VariableStringDictionary: {
+ segment_type += "VSD";
+ break;
+ }
  case EncodingType::FrameOfReference: {
  segment_type += "FoR";
  break;

src/lib/operators/table_scan/column_like_table_scan_impl.cpp

@@ -15,6 +15,10 @@
 #include "storage/pos_lists/row_id_pos_list.hpp"
 #include "storage/segment_iterables/create_iterable_from_attribute_vector.hpp"
 #include "storage/segment_iterate.hpp"
+// NOLINTBEGIN(misc-include-cleaner): VariableStringDictionary is accessed in _find_matches_in_dictionary.
+#include "storage/variable_string_dictionary/variable_string_vector.hpp"
+#include "storage/variable_string_dictionary/variable_string_vector_iterator.hpp"
+// NOLINTEND(misc-include-cleaner)
 #include "types.hpp"
 #include "utils/assert.hpp"
 
@@ -49,8 +53,8 @@ void ColumnLikeTableScanImpl::_scan_generic_segment(
  const AbstractSegment& segment, const ChunkID chunk_id, RowIDPosList& matches,
  const std::shared_ptr<const AbstractPosList>& position_filter) const {
  segment_with_iterators_filtered(segment, position_filter, [&](auto iter, [[maybe_unused]] const auto end) {
- // Don't instantiate this for ReferenceSegments to save compile time as ReferenceSegments are handled
- // via position_filter
+ // Do not instantiate this for ReferenceSegments to save compile time as ReferenceSegments are handled via
+ // position_filter.
  if constexpr (!is_reference_segment_iterable_v<typename decltype(iter)::IterableType>) {
  using ColumnDataType = typename decltype(iter)::ValueType;
 
@@ -62,10 +66,10 @@ void ColumnLikeTableScanImpl::_scan_generic_segment(
  _scan_with_iterators<true>(functor, iter, end, chunk_id, matches);
  });
  } else {
- Fail("Can only handle strings");
+ Fail("Can only handle strings.");
  }
  } else {
- Fail("ReferenceSegments have their own code paths and should be handled there");
+ Fail("ReferenceSegments have their own code paths and should be handled there.");
  }
  });
 }
@@ -76,14 +80,27 @@ void ColumnLikeTableScanImpl::_scan_dictionary_segment(const BaseDictionarySegme
  // First, build a bitmap containing 1s/0s for matching/non-matching dictionary values. Second, iterate over the
  // attribute vector and check against the bitmap. If too many input rows have already been removed (are not part of
  // position_filter), this optimization is detrimental. See caller for that case.
- std::pair<size_t, std::vector<bool>> result;
+ auto result = std::pair<size_t, std::vector<bool>>{};
 
- if (segment.encoding_type() == EncodingType::Dictionary) {
- const auto& typed_segment = static_cast<const DictionarySegment<pmr_string>&>(segment);
- result = _find_matches_in_dictionary(*typed_segment.dictionary());
- } else {
- const auto& typed_segment = static_cast<const FixedStringDictionarySegment<pmr_string>&>(segment);
- result = _find_matches_in_dictionary(*typed_segment.fixed_string_dictionary());
+ switch (segment.encoding_type()) {
+ case EncodingType::Dictionary: {
+ const auto& typed_segment = static_cast<const DictionarySegment<pmr_string>&>(segment);
+ result = _find_matches_in_dictionary(*typed_segment.dictionary());
+ break;
+ }
+ case EncodingType::FixedStringDictionary: {
+ const auto& typed_segment = static_cast<const FixedStringDictionarySegment<pmr_string>&>(segment);
+ result = _find_matches_in_dictionary(*typed_segment.fixed_string_dictionary());
+ break;
+ }
+ case EncodingType::VariableStringDictionary: {
+ const auto& typed_segment = static_cast<const VariableStringDictionarySegment<pmr_string>&>(segment);
+ result = _find_matches_in_dictionary(*typed_segment.variable_string_dictionary());
+ break;
+ }
+ default: {
+ Fail("Segment is either not dictionary-encoded or encoding specialization is not implemented.");
+ }
  }
 
  const auto& match_count = result.first;
@@ -103,8 +120,8 @@ void ColumnLikeTableScanImpl::_scan_dictionary_segment(const BaseDictionarySegme
  return;
  }
 
- // LIKE matches no rows
- if (match_count == 0u) {
+ // LIKE matches no rows.
+ if (match_count == 0) {
  ++num_chunks_with_early_out;
  return;
  }
@@ -125,21 +142,23 @@ std::pair<size_t, std::vector<bool>> ColumnLikeTableScanImpl::_find_matches_in_d
  auto& count = result.first;
  auto& dictionary_matches = result.second;
 
- count = 0u;
- dictionary_matches.reserve(dictionary.size());
+ count = 0;
+ dictionary_matches.resize(dictionary.size());
 
  _matcher.resolve(_invert_results, [&](const auto& matcher) {
 #ifdef __clang__
-// For the loop through the dictionary, we want to use const auto& for DictionarySegments. However,
-// FixedStringVector iterators return an std::string_view value. Thus, we disable clang's -Wrange-loop-analysis
-// error about a potential copy for the loop value.
+// For the loop through the dictionary, we want to use const auto& for DictionarySegments. However, FixedStringVector
+// iterators return an std::string_view value. Thus, we disable clang's -Wrange-loop-analysis error about a potential
+// copy for the loop value.
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wrange-loop-analysis"
 #endif
+ auto index = size_t{0};
  for (const auto& value : dictionary) {
- const auto matches = matcher(value);
- count += static_cast<size_t>(matches);
- dictionary_matches.push_back(matches);
+ const auto match_result = matcher(value);
+ count += static_cast<size_t>(match_result);
+ dictionary_matches[index] = match_result;
+ ++index;
  }
 
 #ifdef __clang__

src/lib/operators/table_scan/column_like_table_scan_impl.hpp

@@ -7,8 +7,6 @@
 #include <utility>
 #include <vector>
 
-#include <boost/variant.hpp>
-
 #include "abstract_dereferenced_column_table_scan_impl.hpp"
 #include "expression/evaluation/like_matcher.hpp"
 #include "types.hpp"

src/lib/storage/create_iterable_from_segment.hpp

@@ -26,20 +26,22 @@ class ReferenceSegment;
 template <typename T, EraseReferencedSegmentType>
 class ReferenceSegmentIterable;
 
+template <typename T>
+ requires(std::is_same_v<T, pmr_string>)
+class VariableStringDictionarySegment;
+
 /**
  * @defgroup Uniform interface to create an iterable from a segment
  *
- * These methods cannot be part of the segments' interfaces because
- * reference segment are not templated and thus don’t know their type.
+ * These methods cannot be part of the segments' interfaces because reference segment are not templated and thus do not
+ * know their type.
  *
- * All iterables implement the same interface using static polymorphism
- * (i.e. the CRTP pattern, see segment_iterables/.hpp).
+ * All iterables implement the same interface using static polymorphism (i.e. the CRTP pattern, see
+ * segment_iterables/.hpp).
  *
- * In debug mode, create_iterable_from_segment returns a type erased
- * iterable, i.e., all iterators have the same type
+ * In debug mode, create_iterable_from_segment returns a type erased iterable, i.e., all iterators have the same type.
  *
- * Functions must be forward-declared because otherwise, we run into
- * circular include dependencies.
+ * Functions must be forward-declared because otherwise, we run into circular include dependencies.
  *
  * @{
  */
@@ -73,10 +75,13 @@ template <typename T, bool EraseSegmentType = HYRISE_DEBUG,
  : EraseReferencedSegmentType::No)>
 auto create_iterable_from_segment(const ReferenceSegment& segment);
 
+template <typename T, bool EraseSegmentType = HYRISE_DEBUG>
+auto create_iterable_from_segment(const VariableStringDictionarySegment<T>& segment);
+
 /**@}*/
 
 } // namespace hyrise
 
-// Include these only now to break up include dependencies
+// Include these only now to break up include dependencies.
 #include "create_iterable_from_reference_segment.ipp"
 #include "create_iterable_from_segment.ipp"