From 43ef9b7bda813c10f845fe55f7fc6113bbd47b07 Mon Sep 17 00:00:00 2001 From: Pallavi Sontakke Date: Tue, 8 Oct 2024 17:03:53 +0530 Subject: [PATCH] Release 2.17.0 This release adds support for PostgreSQL 17, significantly improves the performance of continuous aggregate refreshes, and contains performance improvements for analytical queries and delete operations over compressed hypertables. We recommend that you upgrade at the next available opportunity. **Highlighted features in TimescaleDB v2.17.0** * Full PostgreSQL 17 support for all existing features. TimescaleDB v2.17 is available for PostgreSQL 14, 15, 16, and 17. * Significant performance improvements for continuous aggregate policies: continuous aggregate refresh is now using `merge` instead of deleting old materialized data and re-inserting. This update can decrease dramatically the amount of data that must be written on the continuous aggregate in the presence of a small number of changes, reduce the `i/o` cost of refreshing a continuous aggregate, and generate fewer Write-Ahead Logs (`WAL`). Overall, continuous aggregate policies will be more lightweight, use less system resources, and complete faster. * Increased performance for real-time analytical queries over compressed hypertables: we are excited to introduce additional Single Instruction, Multiple Data (`SIMD`) vectorization optimization to our engine by supporting vectorized execution for queries that group by using the `segment_by` column(s) and aggregate using the basic aggregate functions (`sum`, `count`, `avg`, `min`, `max`). Stay tuned for more to come in follow-up releases! Support for grouping on additional columns, filtered aggregation, vectorized expressions, and `time_bucket` is coming soon. * Improved performance of deletes on compressed hypertables when a large amount of data is affected. This improvement speeds up operations that delete whole segments by skipping the decompression step. It is enabled for all deletes that filter by the `segment_by` column(s). **PostgreSQL 14 deprecation announcement** We will continue supporting PostgreSQL 14 until April 2025. Closer to that time, we will announce the specific version of TimescaleDB in which PostgreSQL 14 support will not be included going forward. **Features** * #6882: Allow delete of full segments on compressed chunks without decompression. * #7033: Use `merge` statement on continuous aggregates refresh. * #7126: Add functions to show the compression information. * #7147: Vectorize partial aggregation for `sum(int4)` with grouping on `segment by` columns. * #7204: Track additional extensions in telemetry. * #7207: Refactor the `decompress_batches_scan` functions for easier maintenance. * #7209: Add a function to drop the `osm` chunk. * #7275: Add support for the `returning` clause for `merge`. * #7200: Vectorize common aggregate functions like `min`, `max`, `sum`, `avg`, `stddev`, `variance` for compressed columns of arithmetic types, when there is grouping on `segment by` columns or no grouping. **Bug fixes** * #7187: Fix the string literal length for the `compressed_data_info` function. * #7191: Fix creating default indexes on chunks when migrating the data. * #7195: Fix the `segment by` and `order by` checks when dropping a column from a compressed hypertable. * #7201: Use the generic extension description when building `apt` and `rpm` loader packages. * #7227: Add an index to the `compression_chunk_size` catalog table. * #7229: Fix the foreign key constraints where the index and the constraint column order are different. * #7230: Do not propagate the foreign key constraints to the `osm` chunk. * #7234: Release the cache after accessing the cache entry. * #7258: Force English in the `pg_config` command executed by `cmake` to avoid the unexpected building errors. * #7270: Fix the memory leak in compressed DML batch filtering. * #7286: Fix the index column check while searching for the index. * #7290: Add check for null offset for continuous aggregates built on top of continuous aggregates. * #7301: Make foreign key behavior for hypertables consistent. * #7318: Fix chunk skipping range filtering. * #7320: Set the license specific extension comment in the install script. **Thanks** * @MiguelTubio for reporting and fixing the Windows build error. * @posuch for reporting the misleading extension description in the generic loader packages. * @snyrkill for discovering and reporting the issue with continuous aggregates built on top of continuous aggregates. --- .unreleased/group_by_segmentby | 1 - .unreleased/pr_6882 | 1 - .unreleased/pr_7033 | 1 - .unreleased/pr_7126 | 1 - .unreleased/pr_7187 | 1 - .unreleased/pr_7191 | 1 - .unreleased/pr_7195 | 1 - .unreleased/pr_7201 | 2 - .unreleased/pr_7204 | 1 - .unreleased/pr_7207 | 1 - .unreleased/pr_7209 | 1 - .unreleased/pr_7227 | 1 - .unreleased/pr_7229 | 1 - .unreleased/pr_7230 | 1 - .unreleased/pr_7234 | 1 - .unreleased/pr_7258 | 2 - .unreleased/pr_7270 | 1 - .unreleased/pr_7275 | 1 - .unreleased/pr_7286 | 1 - .unreleased/pr_7290 | 2 - .unreleased/pr_7295 | 1 - .unreleased/pr_7301 | 1 - .unreleased/pr_7318 | 1 - .unreleased/pr_7320 | 1 - .unreleased/vectorize-agg-functions | 1 - CHANGELOG.md | 71 +++++++++++++++++ sql/CMakeLists.txt | 8 +- sql/updates/2.16.1--2.17.0.sql | 11 +++ sql/updates/2.17.0--2.16.1.sql | 119 ++++++++++++++++++++++++++++ sql/updates/latest-dev.sql | 11 --- sql/updates/reverse-dev.sql | 119 ---------------------------- version.config | 2 +- 32 files changed, 207 insertions(+), 162 deletions(-) delete mode 100644 .unreleased/group_by_segmentby delete mode 100644 .unreleased/pr_6882 delete mode 100644 .unreleased/pr_7033 delete mode 100644 .unreleased/pr_7126 delete mode 100644 .unreleased/pr_7187 delete mode 100644 .unreleased/pr_7191 delete mode 100644 .unreleased/pr_7195 delete mode 100644 .unreleased/pr_7201 delete mode 100644 .unreleased/pr_7204 delete mode 100644 .unreleased/pr_7207 delete mode 100644 .unreleased/pr_7209 delete mode 100644 .unreleased/pr_7227 delete mode 100644 .unreleased/pr_7229 delete mode 100644 .unreleased/pr_7230 delete mode 100644 .unreleased/pr_7234 delete mode 100644 .unreleased/pr_7258 delete mode 100644 .unreleased/pr_7270 delete mode 100644 .unreleased/pr_7275 delete mode 100644 .unreleased/pr_7286 delete mode 100644 .unreleased/pr_7290 delete mode 100644 .unreleased/pr_7295 delete mode 100644 .unreleased/pr_7301 delete mode 100644 .unreleased/pr_7318 delete mode 100644 .unreleased/pr_7320 delete mode 100644 .unreleased/vectorize-agg-functions create mode 100644 sql/updates/2.16.1--2.17.0.sql create mode 100644 sql/updates/2.17.0--2.16.1.sql diff --git a/.unreleased/group_by_segmentby b/.unreleased/group_by_segmentby deleted file mode 100644 index 3858940d167..00000000000 --- a/.unreleased/group_by_segmentby +++ /dev/null @@ -1 +0,0 @@ -Implements: #7147: Vectorize partial aggregation for `sum(int4)` with grouping on `segment by` columns. diff --git a/.unreleased/pr_6882 b/.unreleased/pr_6882 deleted file mode 100644 index 1fb4222d433..00000000000 --- a/.unreleased/pr_6882 +++ /dev/null @@ -1 +0,0 @@ -Implements: #6882: Allow DELETE on the compressed chunks without decompression. diff --git a/.unreleased/pr_7033 b/.unreleased/pr_7033 deleted file mode 100644 index 4aae7a76d9c..00000000000 --- a/.unreleased/pr_7033 +++ /dev/null @@ -1 +0,0 @@ -Implements: #7033 Use MERGE statement on CAgg Refresh diff --git a/.unreleased/pr_7126 b/.unreleased/pr_7126 deleted file mode 100644 index 9b6d69ecdf6..00000000000 --- a/.unreleased/pr_7126 +++ /dev/null @@ -1 +0,0 @@ -Implements: #7126: Add functions to show the compression information. diff --git a/.unreleased/pr_7187 b/.unreleased/pr_7187 deleted file mode 100644 index 96052757d86..00000000000 --- a/.unreleased/pr_7187 +++ /dev/null @@ -1 +0,0 @@ -Fixes: #7187: Fix the string literal length for the `compressed_data_info` function. diff --git a/.unreleased/pr_7191 b/.unreleased/pr_7191 deleted file mode 100644 index 87c5e23c952..00000000000 --- a/.unreleased/pr_7191 +++ /dev/null @@ -1 +0,0 @@ -Fixes: #7191: Fix creating default indexes on chunks when migrating the data. diff --git a/.unreleased/pr_7195 b/.unreleased/pr_7195 deleted file mode 100644 index 752fc195b9b..00000000000 --- a/.unreleased/pr_7195 +++ /dev/null @@ -1 +0,0 @@ -Fixes: #7195: Fix the `segment by` and `order by` checks when dropping a column from a compressed hypertable. diff --git a/.unreleased/pr_7201 b/.unreleased/pr_7201 deleted file mode 100644 index b6dcd3599e4..00000000000 --- a/.unreleased/pr_7201 +++ /dev/null @@ -1,2 +0,0 @@ -Fixes: #7201: Use the generic extension description when building `apt` and `rpm` loader packages. -Thanks: @posuch for reporting the misleading extension description in the generic loader packages. diff --git a/.unreleased/pr_7204 b/.unreleased/pr_7204 deleted file mode 100644 index 373de172a55..00000000000 --- a/.unreleased/pr_7204 +++ /dev/null @@ -1 +0,0 @@ -Implements: #7204: Track additional extensions in telemetry. diff --git a/.unreleased/pr_7207 b/.unreleased/pr_7207 deleted file mode 100644 index 151b8f4b524..00000000000 --- a/.unreleased/pr_7207 +++ /dev/null @@ -1 +0,0 @@ -Implements: #7207: Refactor the `decompress_batches_scan` functions for easier maintenance. diff --git a/.unreleased/pr_7209 b/.unreleased/pr_7209 deleted file mode 100644 index 2cffbee100d..00000000000 --- a/.unreleased/pr_7209 +++ /dev/null @@ -1 +0,0 @@ -Implements: #7209: Add a function to drop the `osm` chunk. diff --git a/.unreleased/pr_7227 b/.unreleased/pr_7227 deleted file mode 100644 index aa016c06fca..00000000000 --- a/.unreleased/pr_7227 +++ /dev/null @@ -1 +0,0 @@ -Fixes: #7227: Add an index to the `compression_chunk_size` catalog table. diff --git a/.unreleased/pr_7229 b/.unreleased/pr_7229 deleted file mode 100644 index f9801b07d0e..00000000000 --- a/.unreleased/pr_7229 +++ /dev/null @@ -1 +0,0 @@ -Fixes: #7229: Fix the foreign key constraints where the index and the constraint column order are different. diff --git a/.unreleased/pr_7230 b/.unreleased/pr_7230 deleted file mode 100644 index e6908018f3b..00000000000 --- a/.unreleased/pr_7230 +++ /dev/null @@ -1 +0,0 @@ -Fixes: #7230: Do not propagate the foreign key constraints to the `osm` chunk. diff --git a/.unreleased/pr_7234 b/.unreleased/pr_7234 deleted file mode 100644 index 2b50803977e..00000000000 --- a/.unreleased/pr_7234 +++ /dev/null @@ -1 +0,0 @@ -Fixes: #7234: Release the cache after accessing the cache entry. diff --git a/.unreleased/pr_7258 b/.unreleased/pr_7258 deleted file mode 100644 index 79030683c4d..00000000000 --- a/.unreleased/pr_7258 +++ /dev/null @@ -1,2 +0,0 @@ -Fixes: #7258 Force English in the pg_config command executed by cmake to avoid unexpected building errors -Thanks: @MiguelTubio for reporting and fixing a Windows build error diff --git a/.unreleased/pr_7270 b/.unreleased/pr_7270 deleted file mode 100644 index ca437d53bf6..00000000000 --- a/.unreleased/pr_7270 +++ /dev/null @@ -1 +0,0 @@ -Fixes: #7270 Fix memory leak in compressed DML batch filtering diff --git a/.unreleased/pr_7275 b/.unreleased/pr_7275 deleted file mode 100644 index 71eff7e5c8f..00000000000 --- a/.unreleased/pr_7275 +++ /dev/null @@ -1 +0,0 @@ -Implements: #7275: Add support for RETURNING clause for MERGE diff --git a/.unreleased/pr_7286 b/.unreleased/pr_7286 deleted file mode 100644 index eb6320932c2..00000000000 --- a/.unreleased/pr_7286 +++ /dev/null @@ -1 +0,0 @@ -Fixes: #7286: Fix index column check while searching for index diff --git a/.unreleased/pr_7290 b/.unreleased/pr_7290 deleted file mode 100644 index 94781f51090..00000000000 --- a/.unreleased/pr_7290 +++ /dev/null @@ -1,2 +0,0 @@ -Fixes: #7290 Add check for NULL offset for caggs built on top of caggs -Thanks: @snyrkill for discovering and reporting the issue diff --git a/.unreleased/pr_7295 b/.unreleased/pr_7295 deleted file mode 100644 index a2dc9541625..00000000000 --- a/.unreleased/pr_7295 +++ /dev/null @@ -1 +0,0 @@ -Implements: #7295 Support ALTER TABLE SET ACCESS METHOD on hypertable diff --git a/.unreleased/pr_7301 b/.unreleased/pr_7301 deleted file mode 100644 index f8e502e7dfb..00000000000 --- a/.unreleased/pr_7301 +++ /dev/null @@ -1 +0,0 @@ -Fixes: #7301 Make foreign key behaviour for hypertables consistent diff --git a/.unreleased/pr_7318 b/.unreleased/pr_7318 deleted file mode 100644 index bf74089f938..00000000000 --- a/.unreleased/pr_7318 +++ /dev/null @@ -1 +0,0 @@ -Fixes: #7318: Fix chunk skipping range filtering diff --git a/.unreleased/pr_7320 b/.unreleased/pr_7320 deleted file mode 100644 index 059a6f1cfd6..00000000000 --- a/.unreleased/pr_7320 +++ /dev/null @@ -1 +0,0 @@ -Fixes: #7320 Set license specific extension comment in install script diff --git a/.unreleased/vectorize-agg-functions b/.unreleased/vectorize-agg-functions deleted file mode 100644 index 69b758eed28..00000000000 --- a/.unreleased/vectorize-agg-functions +++ /dev/null @@ -1 +0,0 @@ -Implements: #7200: Vectorize common aggregate functions like `min`, `max`, `sum`, `avg`, `stddev`, `variance` for compressed columns of arithmetic types, when there is grouping on segmentby columns or no grouping. diff --git a/CHANGELOG.md b/CHANGELOG.md index b7fe0e2c590..0fc7c13b6ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,77 @@ `psql` with the `-X` flag to prevent any `.psqlrc` commands from accidentally triggering the load of a previous DB version.** + +## 2.17.0 (2024-10-08) + +This release adds support for PostgreSQL 17, significantly improves the performance of continuous aggregate refreshes, +and contains performance improvements for analytical queries and delete operations over compressed hypertables. +We recommend that you upgrade at the next available opportunity. + +**Highlighted features in TimescaleDB v2.17.0** + +* Full PostgreSQL 17 support for all existing features. TimescaleDB v2.17 is available for PostgreSQL 14, 15, 16, and 17. + +* Significant performance improvements for continuous aggregate policies: continuous aggregate refresh is now using +`merge` instead of deleting old materialized data and re-inserting. + + This update can decrease dramatically the amount of data that must be written on the continuous aggregate in the + presence of a small number of changes, reduce the `i/o` cost of refreshing a continuous aggregate, and generate fewer + Write-Ahead Logs (`WAL`). + Overall, continuous aggregate policies will be more lightweight, use less system resources, and complete faster. + +* Increased performance for real-time analytical queries over compressed hypertables: + we are excited to introduce additional Single Instruction, Multiple Data (`SIMD`) vectorization optimization to our + engine by supporting vectorized execution for queries that group by using the `segment_by` column(s) and + aggregate using the basic aggregate functions (`sum`, `count`, `avg`, `min`, `max`). + + Stay tuned for more to come in follow-up releases! Support for grouping on additional columns, filtered aggregation, + vectorized expressions, and `time_bucket` is coming soon. + +* Improved performance of deletes on compressed hypertables when a large amount of data is affected. + + This improvement speeds up operations that delete whole segments by skipping the decompression step. + It is enabled for all deletes that filter by the `segment_by` column(s). + +**PostgreSQL 14 deprecation announcement** + + We will continue supporting PostgreSQL 14 until April 2025. Closer to that time, we will announce the specific + version of TimescaleDB in which PostgreSQL 14 support will not be included going forward. + +**Features** +* #6882: Allow delete of full segments on compressed chunks without decompression. +* #7033: Use `merge` statement on continuous aggregates refresh. +* #7126: Add functions to show the compression information. +* #7147: Vectorize partial aggregation for `sum(int4)` with grouping on `segment by` columns. +* #7204: Track additional extensions in telemetry. +* #7207: Refactor the `decompress_batches_scan` functions for easier maintenance. +* #7209: Add a function to drop the `osm` chunk. +* #7275: Add support for the `returning` clause for `merge`. +* #7200: Vectorize common aggregate functions like `min`, `max`, `sum`, `avg`, `stddev`, `variance` for compressed columns + of arithmetic types, when there is grouping on `segment by` columns or no grouping. + +**Bug fixes** +* #7187: Fix the string literal length for the `compressed_data_info` function. +* #7191: Fix creating default indexes on chunks when migrating the data. +* #7195: Fix the `segment by` and `order by` checks when dropping a column from a compressed hypertable. +* #7201: Use the generic extension description when building `apt` and `rpm` loader packages. +* #7227: Add an index to the `compression_chunk_size` catalog table. +* #7229: Fix the foreign key constraints where the index and the constraint column order are different. +* #7230: Do not propagate the foreign key constraints to the `osm` chunk. +* #7234: Release the cache after accessing the cache entry. +* #7258: Force English in the `pg_config` command executed by `cmake` to avoid the unexpected building errors. +* #7270: Fix the memory leak in compressed DML batch filtering. +* #7286: Fix the index column check while searching for the index. +* #7290: Add check for null offset for continuous aggregates built on top of continuous aggregates. +* #7301: Make foreign key behavior for hypertables consistent. +* #7318: Fix chunk skipping range filtering. +* #7320: Set the license specific extension comment in the install script. + +**Thanks** +* @MiguelTubio for reporting and fixing the Windows build error. +* @posuch for reporting the misleading extension description in the generic loader packages. +* @snyrkill for discovering and reporting the issue with continuous aggregates built on top of continuous aggregates. + ## 2.16.1 (2024-08-06) This release contains bug fixes since the 2.16.0 release. We recommend diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 93f79054a7a..baad98465c6 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -42,11 +42,12 @@ set(MOD_FILES updates/2.15.1--2.15.2.sql updates/2.15.2--2.15.3.sql updates/2.15.3--2.16.0.sql - updates/2.16.0--2.16.1.sql) + updates/2.16.0--2.16.1.sql + updates/2.16.1--2.17.0.sql) # The downgrade file to generate a downgrade script for the current version, as # specified in version.config -set(CURRENT_REV_FILE reverse-dev.sql) +set(CURRENT_REV_FILE 2.17.0--2.16.1.sql) # Files for generating old downgrade scripts. This should only include files for # downgrade from one version to its previous version since we do not support # skipping versions when downgrading. @@ -84,7 +85,8 @@ set(OLD_REV_FILES 2.15.2--2.15.1.sql 2.15.3--2.15.2.sql 2.16.0--2.15.3.sql - 2.16.1--2.16.0.sql) + 2.16.1--2.16.0.sql + 2.17.0--2.16.1.sql) set(MODULE_PATHNAME "$libdir/timescaledb-${PROJECT_VERSION_MOD}") set(LOADER_PATHNAME "$libdir/timescaledb") diff --git a/sql/updates/2.16.1--2.17.0.sql b/sql/updates/2.16.1--2.17.0.sql new file mode 100644 index 00000000000..7a086c6cd1a --- /dev/null +++ b/sql/updates/2.16.1--2.17.0.sql @@ -0,0 +1,11 @@ +CREATE FUNCTION _timescaledb_functions.compressed_data_info(_timescaledb_internal.compressed_data) + RETURNS TABLE (algorithm name, has_nulls bool) + AS '@MODULE_PATHNAME@', 'ts_update_placeholder' + LANGUAGE C STRICT IMMUTABLE SET search_path = pg_catalog, pg_temp; + +CREATE INDEX compression_chunk_size_idx ON _timescaledb_catalog.compression_chunk_size (compressed_chunk_id); + +CREATE FUNCTION _timescaledb_functions.drop_osm_chunk(hypertable REGCLASS) + RETURNS BOOL + AS '@MODULE_PATHNAME@', 'ts_update_placeholder' + LANGUAGE C VOLATILE; diff --git a/sql/updates/2.17.0--2.16.1.sql b/sql/updates/2.17.0--2.16.1.sql new file mode 100644 index 00000000000..a7287b3d8d1 --- /dev/null +++ b/sql/updates/2.17.0--2.16.1.sql @@ -0,0 +1,119 @@ +-- check whether we can safely downgrade the existing compression setup +CREATE OR REPLACE FUNCTION _timescaledb_functions.add_sequence_number_metadata_column( + comp_ch_schema_name text, + comp_ch_table_name text +) + RETURNS BOOL LANGUAGE PLPGSQL AS +$BODY$ +DECLARE + chunk_schema_name text; + chunk_table_name text; + index_name text; + segmentby_columns text; +BEGIN + SELECT ch.schema_name, ch.table_name INTO STRICT chunk_schema_name, chunk_table_name + FROM _timescaledb_catalog.chunk ch + INNER JOIN _timescaledb_catalog.chunk comp_ch + ON ch.compressed_chunk_id = comp_ch.id + WHERE comp_ch.schema_name = comp_ch_schema_name + AND comp_ch.table_name = comp_ch_table_name; + + IF NOT FOUND THEN + RAISE USING + ERRCODE = 'feature_not_supported', + MESSAGE = 'Cannot migrate compressed chunk to version 2.16.1, chunk not found'; + END IF; + + -- Add sequence number column to compressed chunk + EXECUTE format('ALTER TABLE %s.%s ADD COLUMN _ts_meta_sequence_num INT DEFAULT NULL', comp_ch_schema_name, comp_ch_table_name); + + -- Remove all indexes from compressed chunk + FOR index_name IN + SELECT format('%s.%s', i.schemaname, i.indexname) + FROM pg_indexes i + WHERE i.schemaname = comp_ch_schema_name + AND i.tablename = comp_ch_table_name + LOOP + EXECUTE format('DROP INDEX %s;', index_name); + END LOOP; + + -- Fetch the segmentby columns from compression settings + SELECT string_agg(cs.segmentby_column, ',') INTO segmentby_columns + FROM ( + SELECT unnest(segmentby) + FROM _timescaledb_catalog.compression_settings + WHERE relid = format('%s.%s', comp_ch_schema_name, comp_ch_table_name)::regclass::oid + AND segmentby IS NOT NULL + ) AS cs(segmentby_column); + + -- Create compressed chunk index based on sequence num metadata column + -- If there is no segmentby columns, we can skip creating the index + IF FOUND AND segmentby_columns IS NOT NULL THEN + EXECUTE format('CREATE INDEX ON %s.%s (%s, _ts_meta_sequence_num);', comp_ch_schema_name, comp_ch_table_name, segmentby_columns); + END IF; + + -- Mark compressed chunk as unordered + -- Marking the chunk status bit (2) makes it unordered + -- and disables some optimizations. In order to re-enable + -- them, you need to recompress these chunks. + UPDATE _timescaledb_catalog.chunk + SET status = status | 2 -- set unordered bit + WHERE schema_name = chunk_schema_name + AND table_name = chunk_table_name; + + RETURN true; +END +$BODY$ SET search_path TO pg_catalog, pg_temp; + +DO $$ +DECLARE + chunk_count int; + chunk_record record; +BEGIN + -- if we find chunks which don't have sequence number metadata column in + -- compressed chunk, we need to stop downgrade and have the user run + -- a migration script to re-add the missing columns + SELECT count(*) INTO STRICT chunk_count + FROM _timescaledb_catalog.chunk ch + INNER JOIN _timescaledb_catalog.chunk uncomp_ch + ON uncomp_ch.compressed_chunk_id = ch.id + WHERE not exists ( + SELECT + FROM pg_attribute att + WHERE attrelid=format('%I.%I',ch.schema_name,ch.table_name)::regclass + AND attname='_ts_meta_sequence_num') + AND NOT uncomp_ch.dropped; + + -- Doing the migration if we find 10 or less chunks that need to be migrated + IF chunk_count > 10 THEN + RAISE USING + ERRCODE = 'feature_not_supported', + MESSAGE = 'Cannot downgrade compressed hypertables with chunks that do not contain sequence numbers. Run timescaledb--2.17-2.16.1.sql migration script before downgrading.', + DETAIL = 'Number of chunks that need to be migrated: '|| chunk_count::text; + ELSIF chunk_count > 0 THEN + FOR chunk_record IN + SELECT comp_ch.* + FROM _timescaledb_catalog.chunk ch + INNER JOIN _timescaledb_catalog.chunk comp_ch + ON ch.compressed_chunk_id = comp_ch.id + WHERE not exists ( + SELECT + FROM pg_attribute att + WHERE attrelid=format('%I.%I',comp_ch.schema_name,comp_ch.table_name)::regclass + AND attname='_ts_meta_sequence_num') + AND NOT ch.dropped + LOOP + PERFORM _timescaledb_functions.add_sequence_number_metadata_column(chunk_record.schema_name, chunk_record.table_name); + RAISE LOG 'Migrated compressed chunk %s.%s to version 2.16.1', chunk_record.schema_name, chunk_record.table_name; + END LOOP; + + RAISE LOG 'Migration successful!'; + END IF; +END +$$; + +DROP FUNCTION _timescaledb_functions.add_sequence_number_metadata_column(text, text); + +DROP FUNCTION _timescaledb_functions.compressed_data_info(_timescaledb_internal.compressed_data); +DROP INDEX _timescaledb_catalog.compression_chunk_size_idx; +DROP FUNCTION IF EXISTS _timescaledb_functions.drop_osm_chunk(REGCLASS); diff --git a/sql/updates/latest-dev.sql b/sql/updates/latest-dev.sql index 7a086c6cd1a..e69de29bb2d 100644 --- a/sql/updates/latest-dev.sql +++ b/sql/updates/latest-dev.sql @@ -1,11 +0,0 @@ -CREATE FUNCTION _timescaledb_functions.compressed_data_info(_timescaledb_internal.compressed_data) - RETURNS TABLE (algorithm name, has_nulls bool) - AS '@MODULE_PATHNAME@', 'ts_update_placeholder' - LANGUAGE C STRICT IMMUTABLE SET search_path = pg_catalog, pg_temp; - -CREATE INDEX compression_chunk_size_idx ON _timescaledb_catalog.compression_chunk_size (compressed_chunk_id); - -CREATE FUNCTION _timescaledb_functions.drop_osm_chunk(hypertable REGCLASS) - RETURNS BOOL - AS '@MODULE_PATHNAME@', 'ts_update_placeholder' - LANGUAGE C VOLATILE; diff --git a/sql/updates/reverse-dev.sql b/sql/updates/reverse-dev.sql index a7287b3d8d1..e69de29bb2d 100644 --- a/sql/updates/reverse-dev.sql +++ b/sql/updates/reverse-dev.sql @@ -1,119 +0,0 @@ --- check whether we can safely downgrade the existing compression setup -CREATE OR REPLACE FUNCTION _timescaledb_functions.add_sequence_number_metadata_column( - comp_ch_schema_name text, - comp_ch_table_name text -) - RETURNS BOOL LANGUAGE PLPGSQL AS -$BODY$ -DECLARE - chunk_schema_name text; - chunk_table_name text; - index_name text; - segmentby_columns text; -BEGIN - SELECT ch.schema_name, ch.table_name INTO STRICT chunk_schema_name, chunk_table_name - FROM _timescaledb_catalog.chunk ch - INNER JOIN _timescaledb_catalog.chunk comp_ch - ON ch.compressed_chunk_id = comp_ch.id - WHERE comp_ch.schema_name = comp_ch_schema_name - AND comp_ch.table_name = comp_ch_table_name; - - IF NOT FOUND THEN - RAISE USING - ERRCODE = 'feature_not_supported', - MESSAGE = 'Cannot migrate compressed chunk to version 2.16.1, chunk not found'; - END IF; - - -- Add sequence number column to compressed chunk - EXECUTE format('ALTER TABLE %s.%s ADD COLUMN _ts_meta_sequence_num INT DEFAULT NULL', comp_ch_schema_name, comp_ch_table_name); - - -- Remove all indexes from compressed chunk - FOR index_name IN - SELECT format('%s.%s', i.schemaname, i.indexname) - FROM pg_indexes i - WHERE i.schemaname = comp_ch_schema_name - AND i.tablename = comp_ch_table_name - LOOP - EXECUTE format('DROP INDEX %s;', index_name); - END LOOP; - - -- Fetch the segmentby columns from compression settings - SELECT string_agg(cs.segmentby_column, ',') INTO segmentby_columns - FROM ( - SELECT unnest(segmentby) - FROM _timescaledb_catalog.compression_settings - WHERE relid = format('%s.%s', comp_ch_schema_name, comp_ch_table_name)::regclass::oid - AND segmentby IS NOT NULL - ) AS cs(segmentby_column); - - -- Create compressed chunk index based on sequence num metadata column - -- If there is no segmentby columns, we can skip creating the index - IF FOUND AND segmentby_columns IS NOT NULL THEN - EXECUTE format('CREATE INDEX ON %s.%s (%s, _ts_meta_sequence_num);', comp_ch_schema_name, comp_ch_table_name, segmentby_columns); - END IF; - - -- Mark compressed chunk as unordered - -- Marking the chunk status bit (2) makes it unordered - -- and disables some optimizations. In order to re-enable - -- them, you need to recompress these chunks. - UPDATE _timescaledb_catalog.chunk - SET status = status | 2 -- set unordered bit - WHERE schema_name = chunk_schema_name - AND table_name = chunk_table_name; - - RETURN true; -END -$BODY$ SET search_path TO pg_catalog, pg_temp; - -DO $$ -DECLARE - chunk_count int; - chunk_record record; -BEGIN - -- if we find chunks which don't have sequence number metadata column in - -- compressed chunk, we need to stop downgrade and have the user run - -- a migration script to re-add the missing columns - SELECT count(*) INTO STRICT chunk_count - FROM _timescaledb_catalog.chunk ch - INNER JOIN _timescaledb_catalog.chunk uncomp_ch - ON uncomp_ch.compressed_chunk_id = ch.id - WHERE not exists ( - SELECT - FROM pg_attribute att - WHERE attrelid=format('%I.%I',ch.schema_name,ch.table_name)::regclass - AND attname='_ts_meta_sequence_num') - AND NOT uncomp_ch.dropped; - - -- Doing the migration if we find 10 or less chunks that need to be migrated - IF chunk_count > 10 THEN - RAISE USING - ERRCODE = 'feature_not_supported', - MESSAGE = 'Cannot downgrade compressed hypertables with chunks that do not contain sequence numbers. Run timescaledb--2.17-2.16.1.sql migration script before downgrading.', - DETAIL = 'Number of chunks that need to be migrated: '|| chunk_count::text; - ELSIF chunk_count > 0 THEN - FOR chunk_record IN - SELECT comp_ch.* - FROM _timescaledb_catalog.chunk ch - INNER JOIN _timescaledb_catalog.chunk comp_ch - ON ch.compressed_chunk_id = comp_ch.id - WHERE not exists ( - SELECT - FROM pg_attribute att - WHERE attrelid=format('%I.%I',comp_ch.schema_name,comp_ch.table_name)::regclass - AND attname='_ts_meta_sequence_num') - AND NOT ch.dropped - LOOP - PERFORM _timescaledb_functions.add_sequence_number_metadata_column(chunk_record.schema_name, chunk_record.table_name); - RAISE LOG 'Migrated compressed chunk %s.%s to version 2.16.1', chunk_record.schema_name, chunk_record.table_name; - END LOOP; - - RAISE LOG 'Migration successful!'; - END IF; -END -$$; - -DROP FUNCTION _timescaledb_functions.add_sequence_number_metadata_column(text, text); - -DROP FUNCTION _timescaledb_functions.compressed_data_info(_timescaledb_internal.compressed_data); -DROP INDEX _timescaledb_catalog.compression_chunk_size_idx; -DROP FUNCTION IF EXISTS _timescaledb_functions.drop_osm_chunk(REGCLASS); diff --git a/version.config b/version.config index 29576ea3810..f6ed4469910 100644 --- a/version.config +++ b/version.config @@ -1,3 +1,3 @@ -version = 2.17.0-dev +version = 2.17.0 update_from_version = 2.16.1 downgrade_to_version = 2.16.1