Skip to content

Commit 2112fee

Browse files
macobomacobo
authored
CLICKHOUSE_DEFAULT_STORAGE_POLICY env (#4851)
In production we use `storage_policy = 'tiered'` by default but this is not reflected in any migrations. This change fixes that by introducing a new environment variable and plumbing to be used in new (and old) migrations Tested via setting env, doing `mix ecto.drop; mix ecto.create; mix ecto.migrate` and checking resulting table schemas.
1 parent 95471c0 commit 2112fee

12 files changed

Lines changed: 95 additions & 30 deletions

‎config/runtime.exs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -560,6 +560,9 @@ config :plausible, Plausible.IngestRepo,
560560
pool_size: ingest_pool_size,
561561
settings: [
562562
materialized_views_ignore_errors: 1
563+
],
564+
table_settings: [
565+
storage_policy: get_var_from_path_or_env(config_dir, "CLICKHOUSE_DEFAULT_STORAGE_POLICY")
563566
]
564567

565568
config :plausible, Plausible.AsyncInsertRepo,

‎lib/plausible/data_migration/acquisition_channel.ex‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ defmodule Plausible.DataMigration.AcquisitionChannel do
2525
"acquisition_channel_functions",
2626
[
2727
on_cluster_statement: on_cluster_statement,
28+
table_settings: Plausible.MigrationUtils.table_settings_expr(),
2829
dictionary_connection_params: Plausible.MigrationUtils.dictionary_connection_params(),
2930
insert_quorum: insert_quorum
3031
],

‎lib/plausible/data_migration/locations_sync.ex‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,12 @@ defmodule Plausible.DataMigration.LocationsSync do
9191
cluster? = Plausible.IngestRepo.clustered_table?("sessions_v2")
9292

9393
{:ok, _} = run_sql("truncate-location-data-table", cluster?: cluster?)
94-
{:ok, _} = run_sql("create-location-data-table", cluster?: cluster?)
94+
95+
{:ok, _} =
96+
run_sql("create-location-data-table",
97+
cluster?: cluster?,
98+
table_settings: Plausible.MigrationUtils.table_settings_expr(:suffix)
99+
)
95100

96101
countries =
97102
Location.Country.all()

‎lib/plausible/data_migration/numeric_ids.ex‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,6 @@ defmodule Plausible.DataMigration.NumericIDs do
1818
end
1919
end
2020

21-
@table_settings "SETTINGS index_granularity = 8192"
22-
2321
def run(opts \\ []) do
2422
interactive? = Keyword.get(opts, :interactive?, true)
2523

@@ -34,7 +32,7 @@ defmodule Plausible.DataMigration.NumericIDs do
3432

3533
table_settings =
3634
Keyword.get(opts, :table_settings) || System.get_env("NUMERIC_IDS_TABLE_SETTINGS") ||
37-
@table_settings
35+
Plausible.MigrationUtils.table_settings_expr()
3836

3937
start_from =
4038
Keyword.get(opts, :start_from) || System.get_env("NUMERIC_IDS_PARTITION_START_FROM")

‎lib/plausible/migration_utils.ex‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,4 +21,23 @@ defmodule Plausible.MigrationUtils do
2121
|> Enum.reject(&is_nil/1)
2222
|> Enum.join(" ")
2323
end
24+
25+
def table_settings() do
26+
IngestRepo.config()
27+
|> Keyword.get(:table_settings)
28+
|> Enum.reject(fn {_, v} -> is_nil(v) end)
29+
end
30+
31+
def table_settings_expr(type \\ :prefix) do
32+
expr = Enum.map_join(table_settings(), ", ", fn {k, v} -> "#{k} = #{encode(v)}" end)
33+
34+
case {table_settings(), type} do
35+
{[], _} -> ""
36+
{_, :prefix} -> "SETTINGS #{expr}"
37+
{_, :suffix} -> ", #{expr}"
38+
end
39+
end
40+
41+
defp encode(value) when is_number(value), do: value
42+
defp encode(value) when is_binary(value), do: "'#{value}'"
2443
end

‎priv/data_migrations/AcquisitionChannel/sql/acquisition_channel_functions.sql.eex‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,8 @@ ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_pro
99
<% else %>
1010
ENGINE = MergeTree()
1111
<% end %>
12-
ORDER BY referrer_source;
12+
ORDER BY referrer_source
13+
<%= @table_settings %>;
1314

1415
TRUNCATE TABLE acquisition_channel_source_category SETTINGS alter_sync=2;
1516

@@ -41,7 +42,8 @@ ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_pro
4142
<% else %>
4243
ENGINE = MergeTree()
4344
<% end %>
44-
ORDER BY referrer_source;
45+
ORDER BY referrer_source
46+
<%= @table_settings %>;
4547

4648
TRUNCATE TABLE acquisition_channel_paid_sources SETTINGS alter_sync=2;
4749

‎priv/data_migrations/LocationsSync/sql/create-location-data-table.sql.eex‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,4 @@ ENGINE = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{shard}/plausible_pro
1010
ENGINE = MergeTree()
1111
<% end %>
1212
ORDER BY (type, id)
13-
SETTINGS index_granularity = 128
13+
SETTINGS index_granularity = 128 <%= @table_settings %>

‎priv/ingest_repo/migrations/20200915070607_create_events_and_sessions.exs‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,11 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateEventsAndSessions do
1010
create_if_not_exists table(:events,
1111
primary_key: false,
1212
engine: "MergeTree",
13-
options:
14-
"PARTITION BY toYYYYMM(timestamp) ORDER BY (domain, toDate(timestamp), user_id) SETTINGS index_granularity = 8192"
13+
options: """
14+
PARTITION BY toYYYYMM(timestamp)
15+
ORDER BY (domain, toDate(timestamp), user_id)
16+
#{Plausible.MigrationUtils.table_settings_expr()}
17+
"""
1518
) do
1619
add(:name, :string)
1720
add(:domain, :string)
@@ -34,8 +37,11 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateEventsAndSessions do
3437
create_if_not_exists table(:sessions,
3538
primary_key: false,
3639
engine: "CollapsingMergeTree(sign)",
37-
options:
38-
"PARTITION BY toYYYYMM(start) ORDER BY (domain, toDate(start), user_id, session_id) SETTINGS index_granularity = 8192"
40+
options: """
41+
PARTITION BY toYYYYMM(start)
42+
ORDER BY (domain, toDate(start), user_id, session_id)
43+
#{Plausible.MigrationUtils.table_settings_expr()}
44+
"""
3945
) do
4046
add(:session_id, :UInt64)
4147
add(:sign, :Int8)

‎priv/ingest_repo/migrations/20211112130238_create_imported_tables.exs‎

Lines changed: 36 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
55
create_if_not_exists table(:imported_visitors,
66
primary_key: false,
77
engine: "MergeTree",
8-
options: "ORDER BY (site_id, date)"
8+
options: """
9+
ORDER BY (site_id, date)
10+
#{Plausible.MigrationUtils.table_settings_expr()}
11+
"""
912
) do
1013
add(:site_id, :UInt64)
1114
add(:date, :date)
@@ -19,7 +22,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
1922
create_if_not_exists table(:imported_sources,
2023
primary_key: false,
2124
engine: "MergeTree",
22-
options: "ORDER BY (site_id, date, source)"
25+
options: """
26+
ORDER BY (site_id, date, source)
27+
#{Plausible.MigrationUtils.table_settings_expr()}
28+
"""
2329
) do
2430
add(:site_id, :UInt64)
2531
add(:date, :date)
@@ -37,7 +43,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
3743
create_if_not_exists table(:imported_pages,
3844
primary_key: false,
3945
engine: "MergeTree",
40-
options: "ORDER BY (site_id, date, hostname, page)"
46+
options: """
47+
ORDER BY (site_id, date, hostname, page)
48+
#{Plausible.MigrationUtils.table_settings_expr()}
49+
"""
4150
) do
4251
add(:site_id, :UInt64)
4352
add(:date, :date)
@@ -52,7 +61,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
5261
create_if_not_exists table(:imported_entry_pages,
5362
primary_key: false,
5463
engine: "MergeTree",
55-
options: "ORDER BY (site_id, date, entry_page)"
64+
options: """
65+
ORDER BY (site_id, date, entry_page)
66+
#{Plausible.MigrationUtils.table_settings_expr()}
67+
"""
5668
) do
5769
add(:site_id, :UInt64)
5870
add(:date, :date)
@@ -66,7 +78,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
6678
create_if_not_exists table(:imported_exit_pages,
6779
primary_key: false,
6880
engine: "MergeTree",
69-
options: "ORDER BY (site_id, date, exit_page)"
81+
options: """
82+
ORDER BY (site_id, date, exit_page)
83+
#{Plausible.MigrationUtils.table_settings_expr()}
84+
"""
7085
) do
7186
add(:site_id, :UInt64)
7287
add(:date, :date)
@@ -78,7 +93,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
7893
create_if_not_exists table(:imported_locations,
7994
primary_key: false,
8095
engine: "MergeTree",
81-
options: "ORDER BY (site_id, date, country, region, city)"
96+
options: """
97+
ORDER BY (site_id, date, country, region, city)
98+
#{Plausible.MigrationUtils.table_settings_expr()}
99+
"""
82100
) do
83101
add(:site_id, :UInt64)
84102
add(:date, :date)
@@ -94,7 +112,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
94112
create_if_not_exists table(:imported_devices,
95113
primary_key: false,
96114
engine: "MergeTree",
97-
options: "ORDER BY (site_id, date, device)"
115+
options: """
116+
ORDER BY (site_id, date, device)
117+
#{Plausible.MigrationUtils.table_settings_expr()}
118+
"""
98119
) do
99120
add(:site_id, :UInt64)
100121
add(:date, :date)
@@ -108,7 +129,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
108129
create_if_not_exists table(:imported_browsers,
109130
primary_key: false,
110131
engine: "MergeTree",
111-
options: "ORDER BY (site_id, date, browser)"
132+
options: """
133+
ORDER BY (site_id, date, browser)
134+
#{Plausible.MigrationUtils.table_settings_expr()}
135+
"""
112136
) do
113137
add(:site_id, :UInt64)
114138
add(:date, :date)
@@ -122,7 +146,10 @@ defmodule Plausible.ClickhouseRepo.Migrations.CreateImportedVisitors do
122146
create_if_not_exists table(:imported_operating_systems,
123147
primary_key: false,
124148
engine: "MergeTree",
125-
options: "ORDER BY (site_id, date, operating_system)"
149+
options: """
150+
ORDER BY (site_id, date, operating_system)
151+
#{Plausible.MigrationUtils.table_settings_expr()}
152+
"""
126153
) do
127154
add(:site_id, :UInt64)
128155
add(:date, :date)

‎priv/ingest_repo/migrations/20230214114402_create_ingest_counters_table.exs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,10 @@ defmodule Plausible.IngestRepo.Migrations.CreateIngestCountersTable do
55
create_if_not_exists table(:ingest_counters,
66
primary_key: false,
77
engine: "SummingMergeTree(value)",
8-
options:
9-
"ORDER BY (domain, toDate(event_timebucket), metric, toStartOfMinute(event_timebucket))"
8+
options: """
9+
ORDER BY (domain, toDate(event_timebucket), metric, toStartOfMinute(event_timebucket))
10+
#{Plausible.MigrationUtils.table_settings_expr()}
11+
"""
1012
) do
1113
add(:event_timebucket, :utc_datetime)
1214
add(:domain, :"LowCardinality(String)")

0 commit comments

Comments
 (0)