Add table default encodings/charsets to all migrations missing them, …

…retrofit existing DB schema (#26670)

For #25353.

This both fixes new installs going forward (in which case the final
migration is a no-op) and cleans up existing installs that have the
wrong collations (in the new migration).

# Checklist for submitter

If some of the following don't apply, delete the relevant line.

<!-- Note that API documentation changes are now addressed by the
product design team. -->

- [x] Changes file added for user-visible changes in `changes/`,
`orbit/changes/` or `ee/fleetd-chrome/changes`.
See [Changes
files](https://github.com/fleetdm/fleet/blob/main/docs/Contributing/Committing-Changes.md#changes-files)
for more information.
- [x] Input data is properly validated, `SELECT *` is avoided, SQL
injection is prevented (using placeholders for values in statements)
- [x] If database migrations are included, checked table schema to
confirm autoupdate
- For database migrations:
- [x] Checked schema for all modified table for columns that will
auto-update timestamps during migration.
- [x] Confirmed that updating the timestamps is acceptable, and will not
cause unwanted side effects.
- [x] Ensured the correct collation is explicitly set for character
columns (`COLLATE utf8mb4_unicode_ci`).
- [x] Manual QA for all new/changed functionality

changes/25353-collation

@@ -0,0 +1 @@
+* Set collation and character set explicitly on database tables that were missing explicit values.

server/datastore/mysql/migrations/tables/20230405232025_AddBootstrapPackagesTable.go

@@ -19,7 +19,7 @@ func Up_20230405232025(tx *sql.Tx) error {
 
 	    PRIMARY KEY (team_id),
 	    UNIQUE KEY idx_token (token)
-          )`)
+          ) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci`)
 	if err != nil {
 		return err
 	}

server/datastore/mysql/migrations/tables/20230411102858_CreateHostBootstrapPackagesTable.go

@@ -18,7 +18,7 @@ func Up_20230411102858(tx *sql.Tx) error {
 
             PRIMARY KEY (host_uuid),
             FOREIGN KEY (command_uuid) REFERENCES nano_commands (command_uuid) ON DELETE CASCADE
-          )`)
+          ) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci`)
 	if err != nil {
 		return fmt.Errorf("create host_mdm_apple_bootstrap_packages: %w", err)
 	}

server/datastore/mysql/migrations/tables/20230425082126_AddMDMAppleSetupAssistants.go

@@ -28,7 +28,7 @@ CREATE TABLE mdm_apple_setup_assistants (
     PRIMARY KEY (id),
     UNIQUE KEY idx_mdm_setup_assistant_global_or_team_id (global_or_team_id),
     FOREIGN KEY fk_mdm_setup_assistant_team_id (team_id) REFERENCES teams (id) ON DELETE CASCADE ON UPDATE CASCADE
-);
+) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci;
 `)
 	if err != nil {
 		return fmt.Errorf("failed to create mdm_apple_setup_assistants table: %w", err)

server/datastore/mysql/migrations/tables/20230425105727_AddEulasTable.go

@@ -19,7 +19,7 @@ func Up_20230425105727(tx *sql.Tx) error {
             created_at   datetime NOT NULL DEFAULT CURRENT_TIMESTAMP,
 
             PRIMARY KEY (id)
-          )`)
+          ) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci`)
 	if err != nil {
 		return fmt.Errorf("creating eulas table: %w", err)
 	}

server/datastore/mysql/migrations/tables/20230515144206_AddMDMAppleDefaultSetupAssistants.go

@@ -29,7 +29,7 @@ CREATE TABLE mdm_apple_default_setup_assistants (
     PRIMARY KEY (id),
     UNIQUE KEY idx_mdm_default_setup_assistant_global_or_team_id (global_or_team_id),
     FOREIGN KEY fk_mdm_default_setup_assistant_team_id (team_id) REFERENCES teams (id) ON DELETE CASCADE ON UPDATE CASCADE
-);
+) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci;
 `)
 	if err != nil {
 		return fmt.Errorf("failed to create mdm_apple_default_setup_assistants table: %w", err)

server/datastore/mysql/migrations/tables/20230517140952_AddHostSoftwareInstalledPathsTable.go

@@ -18,7 +18,7 @@ func Up_20230517140952(tx *sql.Tx) error {
 			installed_path TEXT            NOT NULL,
 			PRIMARY KEY (id),
 			KEY host_id_software_id_idx (host_id, software_id)
-		)
+		) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci
 	`)
 	if err != nil {
 		return err

server/datastore/mysql/migrations/tables/20230629140529_AddWstepTables.go

@@ -40,7 +40,7 @@ CREATE TABLE wstep_certificates (
 
 	PRIMARY KEY (serial),
 	FOREIGN KEY (serial) REFERENCES wstep_serials (serial)
-)`)
+) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci;`)
 	if err != nil {
 		return err
 	}
@@ -53,7 +53,7 @@ CREATE TABLE wstep_cert_auth_associations (
 	updated_at     timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
 
 	PRIMARY KEY (id, sha256)
-)`)
+) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci;`)
 	if err != nil {
 		return err
 	}

server/datastore/mysql/migrations/tables/20230629140530_AddMDMWindowsEnrollmentsTable.go

@@ -27,7 +27,7 @@ func Up_20230629140530(tx *sql.Tx) error {
 			updated_at timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
 			PRIMARY KEY (id),
 			UNIQUE KEY idx_type (mdm_hardware_id)
-        )`)
+        ) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci;`)
 	if err != nil {
 		return fmt.Errorf("failed to create mdm_windows_enrollments table: %w", err)
 	}

server/datastore/mysql/migrations/tables/20230814150442_AddHostScriptResultsTable.go

@@ -54,7 +54,7 @@ CREATE TABLE host_script_results (
     -- was offline and never sent results, we should eventually start accepting a new
     -- script execution).
     KEY idx_host_script_results_host_exit_created (host_id, exit_code, created_at)
-)`)
+) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci;`)
 	if err != nil {
 		return fmt.Errorf("failed to create host_script_results table: %w", err)
 	}

server/datastore/mysql/migrations/tables/20230906152143_AddScriptsTable.go

@@ -37,7 +37,7 @@ CREATE TABLE scripts (
 
     UNIQUE KEY idx_scripts_global_or_team_id_name (global_or_team_id, name),
     FOREIGN KEY fk_scripts_team_id (team_id) REFERENCES teams (id) ON DELETE CASCADE ON UPDATE CASCADE
-)`)
+) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci;`)
 	if err != nil {
 		return fmt.Errorf("failed to create scripts table: %w", err)
 	}

server/datastore/mysql/migrations/tables/20231009094544_CreateTableQueryReports.go

@@ -20,7 +20,7 @@ func Up_20231009094544(tx *sql.Tx) error {
 			last_fetched TIMESTAMP NOT NULL,
 			data JSON,
 			FOREIGN KEY (query_id) REFERENCES queries(id) ON DELETE CASCADE
-		);
+		) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci;
     `)
 	if err != nil {
 		return fmt.Errorf("failed to create table query_results: %w", err)

server/datastore/mysql/migrations/tables/20231031165350_AddWindowsMDMTables.go

@@ -24,7 +24,7 @@ CREATE TABLE windows_mdm_commands (
 	updated_at       TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
 
 	PRIMARY KEY (command_uuid)
-)`)
+) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci;`)
 	if err != nil {
 		return fmt.Errorf("failed to create windows_mdm_commands table: %w", err)
 	}
@@ -46,7 +46,7 @@ CREATE TABLE windows_mdm_command_queue (
 	FOREIGN KEY (command_uuid)
 		REFERENCES windows_mdm_commands (command_uuid)
 		ON DELETE CASCADE ON UPDATE CASCADE
-)`)
+) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci;`)
 	if err != nil {
 		return fmt.Errorf("failed to create windows_mdm_command_queue table: %w", err)
 	}
@@ -67,7 +67,7 @@ CREATE TABLE windows_mdm_responses (
 	FOREIGN KEY (enrollment_id)
 		REFERENCES mdm_windows_enrollments (id)
 		ON DELETE CASCADE ON UPDATE CASCADE
-)`)
+) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci;`)
 	if err != nil {
 		return fmt.Errorf("failed to create windows_mdm_responses table: %w", err)
 	}
@@ -105,7 +105,7 @@ CREATE TABLE windows_mdm_command_results (
 	FOREIGN KEY (response_id)
 		REFERENCES windows_mdm_responses (id)
 		ON DELETE CASCADE ON UPDATE CASCADE
-)`)
+) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci;`)
 	if err != nil {
 		return fmt.Errorf("failed to create windows_mdm_command_results table: %w", err)
 	}

server/datastore/mysql/migrations/tables/20240205121956_CreateTableHostLockWipe.go

@@ -23,7 +23,7 @@ func Up_20240205121956(tx *sql.Tx) error {
 			wipe_ref VARCHAR(36) NULL,
 			suspended TINYINT(1) NOT NULL DEFAULT FALSE,
 			PRIMARY KEY (host_id)
-		)
+		) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci
 	`
 
 	if _, err := tx.Exec(stmt); err != nil {

server/datastore/mysql/migrations/tables/20240314085226_AddCalendarEventTables.go

@@ -24,7 +24,7 @@ func Up_20240314085226(tx *sql.Tx) error {
 		updated_at TIMESTAMP NOT NULL NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
 
 		UNIQUE KEY idx_one_calendar_event_per_email (email)
-	);
+	) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci;
 `); err != nil {
 		return fmt.Errorf("create calendar_events table: %w", err)
 	}

server/datastore/mysql/migrations/tables/20240327115617_CreateTableNanoDDMRequests.go

@@ -21,7 +21,7 @@ CREATE TABLE mdm_apple_declarative_requests (
   raw_json TEXT,
   PRIMARY KEY (id),
   CONSTRAINT mdm_apple_declarative_requests_enrollment_id FOREIGN KEY (enrollment_id) REFERENCES nano_enrollments (id) ON DELETE CASCADE
-)
+) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci
 `)
 	if err != nil {
 		return fmt.Errorf("creating mdm_apple_declarative_requests: %w", err)

server/datastore/mysql/migrations/tables/20240515200020_AddSoftwareInstallerTables.go

@@ -138,7 +138,7 @@ CREATE TABLE IF NOT EXISTS host_software_installs (
   -- this index can be used to lookup results for a specific
   -- execution (execution ids, e.g. when updating the row for results)
   UNIQUE KEY idx_host_software_installs_execution_id (execution_id)
-)
+) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci
   `)
 	if err != nil {
 		return fmt.Errorf("creating host_software_installs table: %w", err)

server/datastore/mysql/migrations/tables/20240723102712_VPPDBUpdates.go

@@ -38,7 +38,7 @@ CREATE TABLE vpp_apps (
 	  REFERENCES software_titles (id)
 	  ON DELETE SET NULL
 	  ON UPDATE CASCADE
-)`)
+) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci`)
 	if err != nil {
 		return fmt.Errorf("failed to create table vpp_apps: %w", err)
 	}
@@ -58,7 +58,7 @@ CREATE TABLE vpp_apps_teams (
 	FOREIGN KEY (adam_id) REFERENCES vpp_apps (adam_id) ON DELETE CASCADE,
 	FOREIGN KEY (team_id) REFERENCES teams (id) ON DELETE CASCADE,
 	UNIQUE KEY idx_global_or_team_id_adam_id (global_or_team_id, adam_id)
-)`)
+) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci`)
 	if err != nil {
 		return fmt.Errorf("failed to create table vpp_apps_teams: %w", err)
 	}
@@ -91,7 +91,7 @@ CREATE TABLE host_vpp_software_installs (
 	FOREIGN KEY (user_id) REFERENCES users (id) ON DELETE SET NULL,
 	FOREIGN KEY (adam_id) REFERENCES vpp_apps (adam_id) ON DELETE CASCADE,
 	UNIQUE INDEX idx_host_vpp_software_installs_command_uuid (command_uuid)
-)`)
+) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci`)
 	if err != nil {
 		return fmt.Errorf("failed to create table host_vpp_software_installs: %w", err)
 	}

server/datastore/mysql/migrations/tables/20241025112748_AddSetupExperienceResultsTable.go

@@ -83,7 +83,7 @@ CREATE TABLE setup_experience_status_results (
 	CONSTRAINT fk_setup_experience_status_results_si_id FOREIGN KEY (software_installer_id) REFERENCES software_installers(id) ON DELETE CASCADE,
 	CONSTRAINT fk_setup_experience_status_results_va_id FOREIGN KEY (vpp_app_team_id) REFERENCES vpp_apps_teams(id) ON DELETE CASCADE,
 	CONSTRAINT fk_setup_experience_status_results_ses_id FOREIGN KEY (setup_experience_script_id) REFERENCES setup_experience_scripts(id) ON DELETE CASCADE
-)
+) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci;
 `)
 	// Service layer state machine like SetupExperienceNestStep()?
 	// Called from each of the three endpoints (software install, vpp

server/datastore/mysql/migrations/tables/20241025141855_CreateTableHostMDMAppleAwaitingConfiguration.go

@@ -14,7 +14,7 @@ func Up_20241025141855(tx *sql.Tx) error {
 CREATE TABLE host_mdm_apple_awaiting_configuration (
 	host_uuid           VARCHAR(255) NOT NULL PRIMARY KEY,
 	awaiting_configuration TINYINT(1) NOT NULL DEFAULT FALSE
-)`)
+) DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci`)
 	if err != nil {
 		return fmt.Errorf("creating host_mdm_apple_awaiting_configuration  table: %w", err)
 	}

server/datastore/mysql/migrations/tables/20250127162751_AddUnifiedQueueTable.go

@@ -3,14 +3,66 @@ package tables
 import (
 	"database/sql"
 	"fmt"
+
+	"github.com/jmoiron/sqlx"
 )
 
 func init() {
 	MigrationClient.AddMigration(Up_20250127162751, Down_20250127162751)
 }
 
+// changeCollation changes the default collation set of the database and all
+// table to the provided collation
+//
+// This is based on the changeCollation function that's included in this
+// module and part of the 20230315104937_EnsureUniformCollation migration.
+func changeCollation2025(tx *sql.Tx, charset string, collation string) (err error) {
+	_, err = tx.Exec(fmt.Sprintf("ALTER DATABASE DEFAULT CHARACTER SET `%s` COLLATE `%s`", charset, collation))
+	if err != nil {
+		return fmt.Errorf("alter database: %w", err)
+	}
+
+	txx := sqlx.Tx{Tx: tx}
+	var names []string
+	err = txx.Select(&names, `
+          SELECT table_name
+          FROM information_schema.TABLES AS T, information_schema.COLLATION_CHARACTER_SET_APPLICABILITY AS C
+          WHERE C.collation_name = T.table_collation
+          AND T.table_schema = (SELECT database())
+          AND (C.CHARACTER_SET_NAME != ? OR C.COLLATION_NAME != ?)
+	  -- exclude tables that have columns with specific collations
+	  AND table_name NOT IN ('hosts', 'enroll_secrets')`, charset, collation)
+	if err != nil {
+		return fmt.Errorf("selecting tables: %w", err)
+	}
+
+	// disable foreign checks before changing the collations, otherwise the
+	// migration might fail. These are re-enabled after we're done.
+	defer func() {
+		if _, execErr := tx.Exec("SET FOREIGN_KEY_CHECKS = 1"); execErr != nil {
+			err = fmt.Errorf("re-enabling foreign key checks: %w", err)
+		}
+	}()
+	if _, err := tx.Exec("SET FOREIGN_KEY_CHECKS = 0"); err != nil {
+		return fmt.Errorf("disabling foreign key checks: %w", err)
+	}
+	for _, name := range names {
+		_, err = tx.Exec(fmt.Sprintf("ALTER TABLE `%s` CONVERT TO CHARACTER SET `%s` COLLATE `%s`", name, charset, collation))
+		if err != nil {
+			return fmt.Errorf("alter table %s: %w", name, err)
+		}
+	}
+
+	return err
+}
+
 func Up_20250127162751(tx *sql.Tx) error {
-	_, err := tx.Exec(`
+	err := changeCollation2025(tx, "utf8mb4", "utf8mb4_unicode_ci")
+	if err != nil {
+		return fmt.Errorf("failed to fix collation: %w", err)
+	}
+
+	_, err = tx.Exec(`
 CREATE TABLE upcoming_activities (
 	id              BIGINT UNSIGNED NOT NULL AUTO_INCREMENT,
 	host_id         INT UNSIGNED NOT NULL,

server/datastore/mysql/migrations/tables/20250127162751_AddUnifiedQueueTable_test.go

@@ -0,0 +1,49 @@
+package tables
+
+import (
+	"testing"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/stretchr/testify/require"
+)
+
+// Test is for collation fix; uniQ migration didn't have a test before
+func TestUp_20250127162751(t *testing.T) {
+	db := applyUpToPrev(t)
+	execNoErr(t, db, "SET FOREIGN_KEY_CHECKS = 0")
+	execNoErr(t, db, "DROP TABLE mdm_apple_bootstrap_packages")
+	execNoErr(t, db, "CREATE TABLE mdm_apple_bootstrap_packages (team_id int(10) unsigned NOT NULL PRIMARY KEY, name varchar(255)) CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci")
+	execNoErr(t, db, "INSERT INTO mdm_apple_bootstrap_packages (team_id, name) VALUES (1, 'Care Package')")
+	execNoErr(t, db, "DROP TABLE host_mdm_apple_bootstrap_packages")
+	execNoErr(t, db, "CREATE TABLE host_mdm_apple_bootstrap_packages (host_uuid VARCHAR(127) NOT NULL PRIMARY KEY) CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci")
+	execNoErr(t, db, "INSERT INTO host_mdm_apple_bootstrap_packages (host_uuid) VALUES ('a123b123')")
+	execNoErr(t, db, "SET FOREIGN_KEY_CHECKS = 1")
+
+	// force a query with an error
+	var c int
+	err := sqlx.Get(db, &c, "SELECT COUNT(*) FROM host_mdm_apple_bootstrap_packages hmabp JOIN hosts h WHERE h.uuid = hmabp.host_uuid")
+	require.ErrorContains(t, err, "Error 1267")
+
+	applyNext(t, db)
+
+	err = sqlx.Get(db, &c, "SELECT COUNT(*) FROM host_mdm_apple_bootstrap_packages hmabp JOIN hosts h WHERE h.uuid = hmabp.host_uuid")
+	require.NoError(t, err)
+
+	// verify that there are no tables with the wrong collation
+	var names []string
+	err = sqlx.Select(db, &names, `
+          SELECT table_name
+          FROM information_schema.TABLES
+	  WHERE table_collation != "utf8mb4_unicode_ci" AND table_schema = (SELECT database())`)
+	require.NoError(t, err)
+	require.Empty(t, names)
+
+	// verify that the collation was maintained for certain columns
+	var columns []string
+	err = sqlx.Select(db, &columns, `
+	  SELECT column_name
+	  FROM information_schema.COLUMNS
+	  WHERE collation_name != "utf8mb4_unicode_ci" AND table_schema = (SELECT database())`)
+	require.NoError(t, err)
+	require.Equal(t, []string{"secret", "node_key", "orbit_node_key", "name_bin"}, columns)
+}