Skip to content

ClickHouse: Made tags optional and adjusted TABLE creation for newer version #237

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MoellJ wants to merge 2 commits into
base: master
Choose a base branch
from
Open

ClickHouse: Made tags optional and adjusted TABLE creation for newer version #237

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bcc6473
ClickHouse: Made tags optional and adjusted TABLE creation for newer …
MoellJ Aug 24, 2023
dddc2a0
Update creator_test.go
MoellJ Aug 31, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cmd/tsbs_load_clickhouse/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ func init() {
Password: viper.GetString("password"),
LogBatches: viper.GetBool("log-batches"),
Debug: viper.GetInt("debug"),
InTableTag: viper.GetBool("in-table-partition-tag"),
DbName: loaderConf.DBName,
}

Expand Down
14 changes: 10 additions & 4 deletions pkg/targets/clickhouse/creator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,12 +127,17 @@ func createMetricsTable(conf *ClickhouseConfig, db *sqlx.DB, tableName string, f

// columnsWithType - column specifications with type. Ex.: "cpu_usage Float64"
var columnsWithType []string
for _, column := range columnNames {
for idx, column := range columnNames {
if len(column) == 0 {
// Skip nameless columns
continue
}
columnsWithType = append(columnsWithType, fmt.Sprintf("%s Nullable(Float64)", column))
if conf.InTableTag && idx == 0 {
columnsWithType = append(columnsWithType, fmt.Sprintf("%s Nullable(String)", column))
} else {
columnsWithType = append(columnsWithType, fmt.Sprintf("%s Nullable(Float64)", column))
}

}

sql := fmt.Sprintf(`
Expand All @@ -143,7 +148,7 @@ func createMetricsTable(conf *ClickhouseConfig, db *sqlx.DB, tableName string, f
tags_id UInt32,
%s,
additional_tags String DEFAULT ''
) ENGINE = MergeTree(created_date, (tags_id, created_at), 8192)
) ENGINE = MergeTree() PARTITION BY toYYYYMM(created_date) PRIMARY KEY (tags_id, created_at)
`,
tableName,
strings.Join(columnsWithType, ","))
Expand Down Expand Up @@ -180,7 +185,8 @@ func generateTagsTableQuery(tagNames, tagTypes []string) string {
"created_at DateTime DEFAULT now(),\n"+
"id UInt32,\n"+
"%s"+
") ENGINE = MergeTree(created_date, (%s), 8192)",
") ENGINE = MergeTree()"+
"PARTITION BY toYYYYMM(created_date) PRIMARY KEY %s",
cols,
index)
}
Expand Down
4 changes: 2 additions & 2 deletions pkg/targets/clickhouse/creator_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ func TestGenerateTagsTableQuery(t *testing.T) {
"created_at DateTime DEFAULT now(),\n" +
"id UInt32,\n" +
"tag1 Nullable(String)" +
") ENGINE = MergeTree(created_date, (id), 8192)"}, {
") ENGINE = MergeTree()PARTITION BY toYYYYMM(created_date) PRIMARY KEY id"}, {
inTagNames: []string{"tag1", "tag2", "tag3", "tag4"},
inTagTypes: []string{"int32", "int64", "float32", "float64"},
out: "CREATE TABLE tags(\n" +
Expand All @@ -29,7 +29,7 @@ func TestGenerateTagsTableQuery(t *testing.T) {
"tag2 Nullable(Int64),\n" +
"tag3 Nullable(Float32),\n" +
"tag4 Nullable(Float64)" +
") ENGINE = MergeTree(created_date, (id), 8192)"},
") ENGINE = MergeTree()PARTITION BY toYYYYMM(created_date) PRIMARY KEY id"},
}
for _, tc := range testCases {
t.Run(fmt.Sprintf("tags table for %v", tc.inTagNames), func(t *testing.T) {
Expand Down
1 change: 1 addition & 0 deletions pkg/targets/clickhouse/implemented_target.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ func (c clickhouseTarget) TargetSpecificFlags(flagPrefix string, flagSet *pflag.
flagSet.String(flagPrefix+"password", "", "Password to connect to ClickHouse")
flagSet.Bool(flagPrefix+"log-batches", false, "Whether to time individual batches.")
flagSet.Int(flagPrefix+"debug", 0, "Debug printing (choices: 0, 1, 2). (default 0)")
flagSet.Bool(flagPrefix+"in-table-partition-tag", false, "Whether the partition key (e.g. hostname) should also be in the metrics hypertable")
}

func (c clickhouseTarget) TargetName() string {
Expand Down