diff --git a/src/query/ast/src/ast/statements/statement.rs b/src/query/ast/src/ast/statements/statement.rs index 2696fa779db..92d8a51f326 100644 --- a/src/query/ast/src/ast/statements/statement.rs +++ b/src/query/ast/src/ast/statements/statement.rs @@ -204,6 +204,8 @@ pub enum Statement { TruncateTable(TruncateTableStmt), OptimizeTable(OptimizeTableStmt), VacuumTable(VacuumTableStmt), + VacuumTables(VacuumTablesStmt), + VacuumAll(VacuumAllStmt), VacuumDropTable(VacuumDropTableStmt), VacuumTemporaryFiles(VacuumTemporaryFiles), VacuumVirtualColumn(VacuumVirtualColumnStmt), @@ -565,6 +567,8 @@ impl Statement { | Statement::ShowDropTables(..) | Statement::OptimizeTable(..) | Statement::VacuumTable(..) + | Statement::VacuumTables(..) + | Statement::VacuumAll(..) | Statement::VacuumDropTable(..) | Statement::VacuumTemporaryFiles(..) | Statement::VacuumVirtualColumn(..) @@ -956,6 +960,8 @@ impl Display for Statement { Statement::TruncateTable(stmt) => write!(f, "{stmt}")?, Statement::OptimizeTable(stmt) => write!(f, "{stmt}")?, Statement::VacuumTable(stmt) => write!(f, "{stmt}")?, + Statement::VacuumTables(stmt) => write!(f, "{stmt}")?, + Statement::VacuumAll(stmt) => write!(f, "{stmt}")?, Statement::VacuumDropTable(stmt) => write!(f, "{stmt}")?, Statement::VacuumTemporaryFiles(stmt) => write!(f, "{stmt}")?, Statement::VacuumVirtualColumn(stmt) => write!(f, "{stmt}")?, diff --git a/src/query/ast/src/ast/statements/table.rs b/src/query/ast/src/ast/statements/table.rs index 7f94f1c6bfd..837572bc599 100644 --- a/src/query/ast/src/ast/statements/table.rs +++ b/src/query/ast/src/ast/statements/table.rs @@ -736,45 +736,52 @@ impl Display for TruncateTableStmt { #[derive(Debug, Clone, PartialEq, Drive, DriveMut, Walk, WalkMut)] pub struct VacuumTableStmt { - pub catalog: Option, pub database: Option, pub table: Identifier, - pub option: VacuumTableOption, } impl Display for VacuumTableStmt { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { write!(f, "VACUUM TABLE ")?; - write_dot_separated_list( - f, - self.catalog - .iter() - .chain(&self.database) - .chain(Some(&self.table)), - )?; - write!(f, " {}", &self.option)?; + write_dot_separated_list(f, self.database.iter().chain(Some(&self.table))) + } +} +#[derive(Debug, Clone, PartialEq, Drive, DriveMut, Walk, WalkMut)] +pub struct VacuumTablesStmt { + pub database: Option, +} + +impl Display for VacuumTablesStmt { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + write!(f, "VACUUM TABLES")?; + if let Some(database) = &self.database { + write!(f, " FROM {database}")?; + } Ok(()) } } +#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] +pub struct VacuumAllStmt; + +impl Display for VacuumAllStmt { + fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { + write!(f, "VACUUM ALL") + } +} + #[derive(Debug, Clone, PartialEq, Drive, DriveMut, Walk, WalkMut)] pub struct VacuumDropTableStmt { - pub catalog: Option, pub database: Option, - pub option: VacuumDropTableOption, } impl Display for VacuumDropTableStmt { fn fmt(&self, f: &mut Formatter) -> std::fmt::Result { - write!(f, "VACUUM DROP TABLE ")?; - if self.catalog.is_some() || self.database.is_some() { - write!(f, "FROM ")?; - write_dot_separated_list(f, self.catalog.iter().chain(&self.database))?; - write!(f, " ")?; + write!(f, "VACUUM DROP TABLE")?; + if let Some(database) = &self.database { + write!(f, " FROM {database}")?; } - write!(f, "{}", &self.option)?; - Ok(()) } } @@ -962,75 +969,18 @@ pub enum CompactTarget { Segment, } -#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] -pub struct VacuumTableOption { - // Some(true) means dry run with summary option - pub dry_run: Option, -} - -impl Display for VacuumTableOption { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - if let Some(summary) = self.dry_run { - write!(f, "DRY RUN")?; - if summary { - write!(f, " SUMMARY")?; - } - } - Ok(()) - } -} - -#[derive(Debug, Clone, PartialEq, Drive, DriveMut)] -pub struct VacuumDropTableOption { - // Some(true) means dry run with summary option - pub dry_run: Option, - pub limit: Option, -} - -impl Display for VacuumDropTableOption { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - if let Some(summary) = self.dry_run { - write!(f, "DRY RUN")?; - if summary { - write!(f, " SUMMARY")?; - } - } - if let Some(limit) = self.limit { - write!(f, " LIMIT {}", limit)?; - } - Ok(()) - } -} - #[derive(Debug, Clone, PartialEq, Drive, DriveMut, Walk, WalkMut)] pub enum OptimizeTableAction { - All, - Purge { before: Option }, Compact { target: CompactTarget }, } impl Display for OptimizeTableAction { fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { match self { - OptimizeTableAction::All => write!(f, "ALL"), - OptimizeTableAction::Purge { before } => { - write!(f, "PURGE")?; - if let Some(point) = before { - write!(f, " BEFORE {}", point)?; - } - Ok(()) - } - OptimizeTableAction::Compact { target } => { - match target { - CompactTarget::Block => { - write!(f, "COMPACT")?; - } - CompactTarget::Segment => { - write!(f, "COMPACT SEGMENT")?; - } - } - Ok(()) - } + OptimizeTableAction::Compact { target } => match target { + CompactTarget::Block => write!(f, "COMPACT"), + CompactTarget::Segment => write!(f, "COMPACT SEGMENT"), + }, } } } diff --git a/src/query/ast/src/parser/error_suggestion.rs b/src/query/ast/src/parser/error_suggestion.rs index 4c450e8f958..8d497f0b914 100644 --- a/src/query/ast/src/parser/error_suggestion.rs +++ b/src/query/ast/src/parser/error_suggestion.rs @@ -34,9 +34,12 @@ const PATTERNS: &[&str] = &[ "SHOW STATISTICS", "SHOW WORKLOAD GROUPS", "SHOW ONLINE NODES", + "VACUUM TABLE", + "VACUUM TABLES", + "VACUUM ALL", "VACUUM DROP TABLE", + "VACUUM DROPPED OBJECTS", "VACUUM TEMPORARY FILES", - "VACUUM TEMPORARY TABLES", "VACUUM VIRTUAL COLUMN", ]; @@ -264,10 +267,9 @@ mod tests { Some("Did you mean `SHOW TABLE_FUNCTIONS` or `SHOW TABLES`?".to_string()) ); - // Multiple suggestions when scores are very close assert_eq!( suggest_correction("vacuum temp"), - Some("Did you mean `VACUUM TEMPORARY FILES` or `VACUUM TEMPORARY TABLES`?".to_string()) + Some("Did you mean `VACUUM TEMPORARY FILES`?".to_string()) ); } @@ -276,10 +278,7 @@ mod tests { // Single word prefixes should get context help assert_eq!( suggest_correction("vacuum"), - Some( - "Try: `VACUUM DROP TABLE`, `VACUUM TEMPORARY FILES`, or `VACUUM TEMPORARY TABLES`" - .to_string() - ) + Some("Try: `VACUUM TABLE`, `VACUUM TABLES`, or `VACUUM ALL`".to_string()) ); let result = suggest_correction("show").unwrap(); @@ -366,7 +365,7 @@ mod tests { ); assert_eq!( suggest_correction("vacuum temp"), - Some("Did you mean `VACUUM TEMPORARY FILES` or `VACUUM TEMPORARY TABLES`?".to_string()) + Some("Did you mean `VACUUM TEMPORARY FILES`?".to_string()) ); // Should not recognize invalid starts diff --git a/src/query/ast/src/parser/statement.rs b/src/query/ast/src/parser/statement.rs index 121c535c237..81d1e3bbeed 100644 --- a/src/query/ast/src/parser/statement.rs +++ b/src/query/ast/src/parser/statement.rs @@ -1413,20 +1413,30 @@ pub fn statement_body(i: Input) -> IResult { }) }, ); - let optimize_table = map( - rule! { - OPTIMIZE ~ TABLE ~ #dot_separated_idents_1_to_3 ~ #optimize_table_action ~ ( LIMIT ~ #literal_u64 )? - }, - |(_, _, (catalog, database, table), action, opt_limit)| { - Statement::OptimizeTable(OptimizeTableStmt { - catalog, - database, - table, - action, - limit: opt_limit.map(|(_, limit)| limit), - }) - }, - ); + let optimize_table = alt(( + map( + rule! { + OPTIMIZE ~ TABLE ~ #dot_separated_idents_1_to_2 ~ PURGE + }, + |(_, _, (database, table), _)| { + Statement::VacuumTable(VacuumTableStmt { database, table }) + }, + ), + map( + rule! { + OPTIMIZE ~ TABLE ~ #dot_separated_idents_1_to_2 ~ #optimize_table_action ~ ( LIMIT ~ #literal_u64 )? + }, + |(_, _, (database, table), action, opt_limit)| { + Statement::OptimizeTable(OptimizeTableStmt { + catalog: None, + database, + table, + action, + limit: opt_limit.map(|(_, limit)| limit), + }) + }, + ), + )); let vacuum_temp_files = map( rule! { VACUUM ~ TEMPORARY ~ FILES ~ (RETAIN ~ #literal_duration)? ~ (LIMIT ~ #literal_u64)? @@ -1440,30 +1450,40 @@ pub fn statement_body(i: Input) -> IResult { ); let vacuum_table = map( rule! { - VACUUM ~ TABLE ~ #dot_separated_idents_1_to_3 ~ #vacuum_table_option + VACUUM ~ TABLE ~ #dot_separated_idents_1_to_2 }, - |(_, _, (catalog, database, table), option)| { - Statement::VacuumTable(VacuumTableStmt { - catalog, - database, - table, - option, + |(_, _, (database, table))| Statement::VacuumTable(VacuumTableStmt { database, table }), + ); + let vacuum_tables = map( + rule! { + VACUUM ~ TABLES ~ (FROM ~ ^#ident)? + }, + |(_, _, database)| { + Statement::VacuumTables(VacuumTablesStmt { + database: database.map(|(_, database)| database), }) }, ); + let vacuum_all = value(Statement::VacuumAll(VacuumAllStmt), rule! { VACUUM ~ ALL }); let vacuum_drop_table = map( rule! { - VACUUM ~ DROP ~ TABLE ~ (FROM ~ ^#dot_separated_idents_1_to_2)? ~ #vacuum_drop_table_option + VACUUM ~ DROP ~ TABLE ~ (FROM ~ ^#ident)? }, - |(_, _, _, database_option, option)| { - let (catalog, database) = database_option.map_or_else( - || (None, None), - |(_, catalog_database)| (catalog_database.0, Some(catalog_database.1)), - ); + |(_, _, _, database_option)| { Statement::VacuumDropTable(VacuumDropTableStmt { - catalog, - database, - option, + database: database_option.map(|(_, database)| database), + }) + }, + ); + let dropped_keyword = match_ident_text("DROPPED"); + let objects_keyword = match_ident_text("OBJECTS"); + let vacuum_dropped_objects = map( + rule! { + VACUUM ~ #dropped_keyword ~ #objects_keyword ~ (FROM ~ ^#ident)? + }, + |(_, _, _, database_option)| { + Statement::VacuumDropTable(VacuumDropTableStmt { + database: database_option.map(|(_, database)| database), }) }, ); @@ -2547,18 +2567,6 @@ pub fn statement_body(i: Input) -> IResult { |(_, name, _, args, _)| Statement::Call(CallStmt { name, args }), ); - let vacuum_temporary_tables = map( - rule! { - VACUUM ~ TEMPORARY ~ TABLES ~ ( LIMIT ~ ^#literal_u64 )? - }, - |(_, _, _, opt_limit)| { - Statement::Call(CallStmt { - name: Identifier::from_name(None, "fuse_vacuum_temporary_table"), - args: opt_limit.map(|v| v.1.to_string()).into_iter().collect(), - }) - }, - ); - let presign = map( rule! { PRESIGN ~ ( #presign_action )? @@ -3234,14 +3242,16 @@ pub fn statement_body(i: Input) -> IResult { ABORT | ROLLBACK => rule!(#abort).parse(i), TRUNCATE => rule!(#truncate_table : "`TRUNCATE TABLE [.]`" ).parse(i), - OPTIMIZE => rule!(#optimize_table : "`OPTIMIZE TABLE [.]
(ALL | PURGE | COMPACT [SEGMENT])`" + OPTIMIZE => rule!(#optimize_table : "`OPTIMIZE TABLE [.]
(PURGE | COMPACT [SEGMENT])`" ).parse(i), VACUUM => rule!( - #vacuum_temp_files : "VACUUM TEMPORARY FILES [RETAIN number SECONDS|DAYS] [LIMIT number]" - | #vacuum_table : "`VACUUM TABLE [.]
[RETAIN number HOURS] [DRY RUN | DRY RUN SUMMARY]`" - | #vacuum_drop_table : "`VACUUM DROP TABLE [FROM [.]] [RETAIN number HOURS] [DRY RUN | DRY RUN SUMMARY]`" + #vacuum_all : "`VACUUM ALL`" + | #vacuum_temp_files : "VACUUM TEMPORARY FILES [RETAIN number SECONDS|DAYS] [LIMIT number]" + | #vacuum_tables : "`VACUUM TABLES [FROM ]`" + | #vacuum_table : "`VACUUM TABLE [.]
`" + | #vacuum_drop_table : "`VACUUM DROP TABLE [FROM ]`" + | #vacuum_dropped_objects : "`VACUUM DROPPED OBJECTS [FROM ]`" | #vacuum_virtual_column : "`VACUUM VIRTUAL COLUMN FROM [.]
`" - | #vacuum_temporary_tables ).parse(i), ANALYZE => rule!(#analyze_table : "`ANALYZE TABLE [.]
`" ).parse(i), @@ -5517,20 +5527,11 @@ pub fn add_column_option(i: Input) -> IResult { } pub fn optimize_table_action(i: Input) -> IResult { - alt(( - value(OptimizeTableAction::All, rule! { ALL }), - map( - rule! { PURGE ~ (BEFORE ~ ^#travel_point)? }, - |(_, opt_travel_point)| OptimizeTableAction::Purge { - before: opt_travel_point.map(|(_, p)| p), - }, - ), - map(rule! { COMPACT ~ SEGMENT? }, |(_, opt_segment)| { - OptimizeTableAction::Compact { - target: opt_segment.map_or(CompactTarget::Block, |_| CompactTarget::Segment), - } - }), - )) + map(rule! { COMPACT ~ SEGMENT? }, |(_, opt_segment)| { + OptimizeTableAction::Compact { + target: opt_segment.map_or(CompactTarget::Block, |_| CompactTarget::Segment), + } + }) .parse(i) } @@ -5556,31 +5557,6 @@ pub fn literal_duration(i: Input) -> IResult { .parse(i) } -pub fn vacuum_drop_table_option(i: Input) -> IResult { - alt((map( - rule! { - (DRY ~ ^RUN ~ SUMMARY?)? ~ (LIMIT ~ #literal_u64)? - }, - |(opt_dry_run, opt_limit)| VacuumDropTableOption { - dry_run: opt_dry_run.map(|dry_run| dry_run.2.is_some()), - limit: opt_limit.map(|(_, limit)| limit as usize), - }, - ),)) - .parse(i) -} - -pub fn vacuum_table_option(i: Input) -> IResult { - alt((map( - rule! { - (DRY ~ ^RUN ~ SUMMARY?)? - }, - |opt_dry_run| VacuumTableOption { - dry_run: opt_dry_run.map(|dry_run| dry_run.2.is_some()), - }, - ),)) - .parse(i) -} - pub fn task_sql_block(i: Input) -> IResult { let single_statement = map( rule! { diff --git a/src/query/ast/src/visit/statement.rs b/src/query/ast/src/visit/statement.rs index 7b99b2fd99d..8aca9d4e90b 100644 --- a/src/query/ast/src/visit/statement.rs +++ b/src/query/ast/src/visit/statement.rs @@ -161,6 +161,8 @@ impl Walk for Statement { Statement::RenameTable(stmt) => try_walk!(stmt.walk(visitor)), Statement::OptimizeTable(stmt) => try_walk!(stmt.walk(visitor)), Statement::VacuumTable(stmt) => try_walk!(stmt.walk(visitor)), + Statement::VacuumTables(stmt) => try_walk!(stmt.walk(visitor)), + Statement::VacuumAll(_) => {} Statement::VacuumDropTable(stmt) => try_walk!(stmt.walk(visitor)), Statement::VacuumTemporaryFiles(_) => {} Statement::VacuumVirtualColumn(stmt) => try_walk!(stmt.walk(visitor)), @@ -390,6 +392,8 @@ impl WalkMut for Statement { Statement::RenameTable(stmt) => try_walk!(stmt.walk_mut(visitor)), Statement::OptimizeTable(stmt) => try_walk!(stmt.walk_mut(visitor)), Statement::VacuumTable(stmt) => try_walk!(stmt.walk_mut(visitor)), + Statement::VacuumTables(stmt) => try_walk!(stmt.walk_mut(visitor)), + Statement::VacuumAll(_) => {} Statement::VacuumDropTable(stmt) => try_walk!(stmt.walk_mut(visitor)), Statement::VacuumTemporaryFiles(_) => {} Statement::VacuumVirtualColumn(stmt) => try_walk!(stmt.walk_mut(visitor)), diff --git a/src/query/ast/src/visit/statement_table.rs b/src/query/ast/src/visit/statement_table.rs index 0b606d49c67..20490cb5b43 100644 --- a/src/query/ast/src/visit/statement_table.rs +++ b/src/query/ast/src/visit/statement_table.rs @@ -624,42 +624,6 @@ impl WalkMut for OptimizeTableStmt { } } -impl Walk for VacuumTableOption { - fn walk( - &self, - _visitor: &mut V, - ) -> Result, V::Error> { - Ok(VisitControl::Continue) - } -} - -impl WalkMut for VacuumTableOption { - fn walk_mut( - &mut self, - _visitor: &mut V, - ) -> Result, V::Error> { - Ok(VisitControl::Continue) - } -} - -impl Walk for VacuumDropTableOption { - fn walk( - &self, - _visitor: &mut V, - ) -> Result, V::Error> { - Ok(VisitControl::Continue) - } -} - -impl WalkMut for VacuumDropTableOption { - fn walk_mut( - &mut self, - _visitor: &mut V, - ) -> Result, V::Error> { - Ok(VisitControl::Continue) - } -} - impl Walk for RefreshVirtualColumnStmt { fn walk( &self, diff --git a/src/query/ast/tests/it/parser.rs b/src/query/ast/tests/it/parser.rs index 2d133fb935c..744c4be0a4a 100644 --- a/src/query/ast/tests/it/parser.rs +++ b/src/query/ast/tests/it/parser.rs @@ -366,8 +366,8 @@ SELECT * from s;"#, r#"drop role if exists 'test'"#, r#"OPTIMIZE TABLE t COMPACT SEGMENT LIMIT 10;"#, r#"OPTIMIZE TABLE t COMPACT LIMIT 10;"#, - r#"OPTIMIZE TABLE t PURGE BEFORE (SNAPSHOT => '9828b23f74664ff3806f44bbc1925ea5') LIMIT 10;"#, - r#"OPTIMIZE TABLE t PURGE BEFORE (TIMESTAMP => '2023-06-26 09:49:02.038483'::TIMESTAMP) LIMIT 10;"#, + r#"OPTIMIZE TABLE t PURGE;"#, + r#"OPTIMIZE TABLE db.t PURGE;"#, r#"ALTER TABLE t CLUSTER BY(c1);"#, r#"ALTER TABLE t PARTITION BY (date_trunc(day, c1), c2);"#, r#"ALTER TABLE t1 swap with t2;"#, @@ -426,13 +426,14 @@ SELECT * from s;"#, r#"ALTER DATABASE ctl.c RENAME TO a;"#, r#"ALTER DATABASE ctl.c refresh cache;"#, r#"VACUUM TABLE t;"#, - r#"VACUUM TABLE t DRY RUN;"#, - r#"VACUUM TABLE t DRY RUN SUMMARY;"#, + r#"VACUUM TABLE db.t;"#, + r#"VACUUM TABLES;"#, + r#"VACUUM TABLES FROM db;"#, + r#"VACUUM ALL;"#, r#"VACUUM DROP TABLE;"#, - r#"VACUUM DROP TABLE DRY RUN;"#, - r#"VACUUM DROP TABLE DRY RUN SUMMARY;"#, r#"VACUUM DROP TABLE FROM db;"#, - r#"VACUUM DROP TABLE FROM db LIMIT 10;"#, + r#"VACUUM DROPPED OBJECTS;"#, + r#"VACUUM DROPPED OBJECTS FROM db;"#, r#"VACUUM TEMPORARY FILES RETAIN 7 DAYS LIMIT 10;"#, r#"ATTACH TABLE db.attached (c1, c2) 's3://testbucket/data/' CONNECTION=(aws_key_id='minioadmin' aws_secret_key='minioadmin' endpoint_url='http://127.0.0.1:9900');"#, r#"CREATE DICTIONARY IF NOT EXISTS db.dict1 (id int, name string) PRIMARY KEY id SOURCE(mysql(host='127.0.0.1' port='3306')) COMMENT 'test dictionary';"#, @@ -1376,6 +1377,38 @@ fn test_statement_error() { } } +#[test] +fn test_removed_vacuum_syntax() { + let cases = [ + "VACUUM TABLE t DRY RUN", + "VACUUM TABLE t DRY RUN SUMMARY", + "VACUUM TABLE catalog.db.t", + "VACUUM TABLES FROM catalog.db", + "VACUUM TABLES FROM db LIMIT 10", + "VACUUM ALL FROM db", + "VACUUM ALL LIMIT 10", + "VACUUM DROP TABLE DRY RUN", + "VACUUM DROP TABLE DRY RUN SUMMARY", + "VACUUM DROP TABLE FROM db LIMIT 10", + "VACUUM DROP TABLE FROM catalog.db", + "VACUUM DROPPED OBJECTS FROM db LIMIT 10", + "VACUUM DROPPED OBJECTS FROM catalog.db", + "VACUUM TEMPORARY TABLES", + "OPTIMIZE TABLE t ALL", + "OPTIMIZE TABLE t PURGE LIMIT 10", + "OPTIMIZE TABLE catalog.db.t PURGE", + "OPTIMIZE TABLE t PURGE BEFORE (SNAPSHOT => '9828b23f74664ff3806f44bbc1925ea5')", + ]; + + for case in cases { + let tokens = tokenize_sql(case).unwrap(); + assert!( + parse_sql(&tokens, Dialect::PostgreSQL).is_err(), + "removed syntax should fail to parse: {case}" + ); + } +} + #[test] fn test_file_format_trim_space_option() { let sql = r#" diff --git a/src/query/ast/tests/it/testdata/stmt.txt b/src/query/ast/tests/it/testdata/stmt.txt index e4015a61908..7447e5b6e8d 100644 --- a/src/query/ast/tests/it/testdata/stmt.txt +++ b/src/query/ast/tests/it/testdata/stmt.txt @@ -16806,13 +16806,12 @@ OptimizeTable( ---------- Input ---------- -OPTIMIZE TABLE t PURGE BEFORE (SNAPSHOT => '9828b23f74664ff3806f44bbc1925ea5') LIMIT 10; +OPTIMIZE TABLE t PURGE; ---------- Output --------- -OPTIMIZE TABLE t PURGE BEFORE (SNAPSHOT => '9828b23f74664ff3806f44bbc1925ea5') LIMIT 10 +VACUUM TABLE t ---------- AST ------------ -OptimizeTable( - OptimizeTableStmt { - catalog: None, +VacuumTable( + VacuumTableStmt { database: None, table: Identifier { span: Some( @@ -16822,68 +16821,35 @@ OptimizeTable( quote: None, ident_type: None, }, - action: Purge { - before: Some( - Snapshot( - Literal { - span: Some( - 43..77, - ), - value: String( - "9828b23f74664ff3806f44bbc1925ea5", - ), - }, - ), - ), - }, - limit: Some( - 10, - ), }, ) ---------- Input ---------- -OPTIMIZE TABLE t PURGE BEFORE (TIMESTAMP => '2023-06-26 09:49:02.038483'::TIMESTAMP) LIMIT 10; +OPTIMIZE TABLE db.t PURGE; ---------- Output --------- -OPTIMIZE TABLE t PURGE BEFORE (TIMESTAMP => '2023-06-26 09:49:02.038483'::TIMESTAMP) LIMIT 10 +VACUUM TABLE db.t ---------- AST ------------ -OptimizeTable( - OptimizeTableStmt { - catalog: None, - database: None, +VacuumTable( + VacuumTableStmt { + database: Some( + Identifier { + span: Some( + 15..17, + ), + name: "db", + quote: None, + ident_type: None, + }, + ), table: Identifier { span: Some( - 15..16, + 18..19, ), name: "t", quote: None, ident_type: None, }, - action: Purge { - before: Some( - Timestamp( - Cast { - span: Some( - 72..83, - ), - expr: Literal { - span: Some( - 44..72, - ), - value: String( - "2023-06-26 09:49:02.038483", - ), - }, - target_type: Timestamp, - pg_style: true, - }, - ), - ), - }, - limit: Some( - 10, - ), }, ) @@ -19882,11 +19848,10 @@ AlterDatabase( ---------- Input ---------- VACUUM TABLE t; ---------- Output --------- -VACUUM TABLE t +VACUUM TABLE t ---------- AST ------------ VacuumTable( VacuumTableStmt { - catalog: None, database: None, table: Identifier { span: Some( @@ -19896,116 +19861,90 @@ VacuumTable( quote: None, ident_type: None, }, - option: VacuumTableOption { - dry_run: None, - }, }, ) ---------- Input ---------- -VACUUM TABLE t DRY RUN; +VACUUM TABLE db.t; ---------- Output --------- -VACUUM TABLE t DRY RUN +VACUUM TABLE db.t ---------- AST ------------ VacuumTable( VacuumTableStmt { - catalog: None, - database: None, + database: Some( + Identifier { + span: Some( + 13..15, + ), + name: "db", + quote: None, + ident_type: None, + }, + ), table: Identifier { span: Some( - 13..14, + 16..17, ), name: "t", quote: None, ident_type: None, }, - option: VacuumTableOption { - dry_run: Some( - false, - ), - }, }, ) ---------- Input ---------- -VACUUM TABLE t DRY RUN SUMMARY; +VACUUM TABLES; ---------- Output --------- -VACUUM TABLE t DRY RUN SUMMARY +VACUUM TABLES ---------- AST ------------ -VacuumTable( - VacuumTableStmt { - catalog: None, +VacuumTables( + VacuumTablesStmt { database: None, - table: Identifier { - span: Some( - 13..14, - ), - name: "t", - quote: None, - ident_type: None, - }, - option: VacuumTableOption { - dry_run: Some( - true, - ), - }, }, ) ---------- Input ---------- -VACUUM DROP TABLE; +VACUUM TABLES FROM db; ---------- Output --------- -VACUUM DROP TABLE +VACUUM TABLES FROM db ---------- AST ------------ -VacuumDropTable( - VacuumDropTableStmt { - catalog: None, - database: None, - option: VacuumDropTableOption { - dry_run: None, - limit: None, - }, +VacuumTables( + VacuumTablesStmt { + database: Some( + Identifier { + span: Some( + 19..21, + ), + name: "db", + quote: None, + ident_type: None, + }, + ), }, ) ---------- Input ---------- -VACUUM DROP TABLE DRY RUN; +VACUUM ALL; ---------- Output --------- -VACUUM DROP TABLE DRY RUN +VACUUM ALL ---------- AST ------------ -VacuumDropTable( - VacuumDropTableStmt { - catalog: None, - database: None, - option: VacuumDropTableOption { - dry_run: Some( - false, - ), - limit: None, - }, - }, +VacuumAll( + VacuumAllStmt, ) ---------- Input ---------- -VACUUM DROP TABLE DRY RUN SUMMARY; +VACUUM DROP TABLE; ---------- Output --------- -VACUUM DROP TABLE DRY RUN SUMMARY +VACUUM DROP TABLE ---------- AST ------------ VacuumDropTable( VacuumDropTableStmt { - catalog: None, database: None, - option: VacuumDropTableOption { - dry_run: Some( - true, - ), - limit: None, - }, }, ) @@ -20013,11 +19952,10 @@ VacuumDropTable( ---------- Input ---------- VACUUM DROP TABLE FROM db; ---------- Output --------- -VACUUM DROP TABLE FROM db +VACUUM DROP TABLE FROM db ---------- AST ------------ VacuumDropTable( VacuumDropTableStmt { - catalog: None, database: Some( Identifier { span: Some( @@ -20028,38 +19966,39 @@ VacuumDropTable( ident_type: None, }, ), - option: VacuumDropTableOption { - dry_run: None, - limit: None, - }, }, ) ---------- Input ---------- -VACUUM DROP TABLE FROM db LIMIT 10; +VACUUM DROPPED OBJECTS; ---------- Output --------- -VACUUM DROP TABLE FROM db LIMIT 10 +VACUUM DROP TABLE +---------- AST ------------ +VacuumDropTable( + VacuumDropTableStmt { + database: None, + }, +) + + +---------- Input ---------- +VACUUM DROPPED OBJECTS FROM db; +---------- Output --------- +VACUUM DROP TABLE FROM db ---------- AST ------------ VacuumDropTable( VacuumDropTableStmt { - catalog: None, database: Some( Identifier { span: Some( - 23..25, + 28..30, ), name: "db", quote: None, ident_type: None, }, ), - option: VacuumDropTableOption { - dry_run: None, - limit: Some( - 10, - ), - }, }, ) @@ -30019,7 +29958,7 @@ CopyIntoLocation( ---------- Input ---------- CREATE TASK IF NOT EXISTS MyTask1 AFTER 'task2', 'task3' WHEN SYSTEM$GET_PREDECESSOR_RETURN_VALUE('task_name') != 'VALIDATION' AS VACUUM TABLE t ---------- Output --------- -CREATE TASK IF NOT EXISTS MyTask1 AFTER 'task2', 'task3' WHEN SYSTEM$GET_PREDECESSOR_RETURN_VALUE('task_name') <> 'VALIDATION' AS VACUUM TABLE t +CREATE TASK IF NOT EXISTS MyTask1 AFTER 'task2', 'task3' WHEN SYSTEM$GET_PREDECESSOR_RETURN_VALUE('task_name') <> 'VALIDATION' AS VACUUM TABLE t ---------- AST ------------ CreateTask( CreateTaskStmt { @@ -30083,7 +30022,7 @@ CreateTask( }, ), sql: SingleStatement( - "VACUUM TABLE t ", + "VACUUM TABLE t", ), }, ) @@ -30092,7 +30031,7 @@ CreateTask( ---------- Input ---------- CREATE TASK IF NOT EXISTS MyTask1 DATABASE = 'target', TIMEZONE = 'America/Los Angeles' AS VACUUM TABLE t ---------- Output --------- -CREATE TASK IF NOT EXISTS MyTask1 database = 'target', timezone = 'America/Los Angeles' AS VACUUM TABLE t +CREATE TASK IF NOT EXISTS MyTask1 database = 'target', timezone = 'America/Los Angeles' AS VACUUM TABLE t ---------- AST ------------ CreateTask( CreateTaskStmt { @@ -30110,7 +30049,7 @@ CreateTask( after: [], when_condition: None, sql: SingleStatement( - "VACUUM TABLE t ", + "VACUUM TABLE t", ), }, ) @@ -30131,7 +30070,7 @@ CREATE TASK IF NOT EXISTS MyTask1 database = 'target', timezone = 'America/Los A BEGIN; INSERT INTO t VALUES ('a;'); DELETE FROM t WHERE c = ';'; -VACUUM TABLE t ; +VACUUM TABLE t; MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN UPDATE *; COMMIT; END; @@ -30156,7 +30095,7 @@ CreateTask( "BEGIN", "INSERT INTO t VALUES ('a;')", "DELETE FROM t WHERE c = ';'", - "VACUUM TABLE t ", + "VACUUM TABLE t", "MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN UPDATE *", "COMMIT", ], @@ -30909,7 +30848,7 @@ ALTER TASK MyTask2 MODIFY AS BEGIN BEGIN; INSERT INTO t VALUES ('a;'); DELETE FROM t WHERE c = ';'; -VACUUM TABLE t ; +VACUUM TABLE t; MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN UPDATE *; COMMIT; END; @@ -30924,7 +30863,7 @@ AlterTask( "BEGIN", "INSERT INTO t VALUES ('a;')", "DELETE FROM t WHERE c = ';'", - "VACUUM TABLE t ", + "VACUUM TABLE t", "MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN UPDATE *", "COMMIT", ], diff --git a/src/query/catalog/src/table.rs b/src/query/catalog/src/table.rs index 57f0ddf9daf..9e44578a54b 100644 --- a/src/query/catalog/src/table.rs +++ b/src/query/catalog/src/table.rs @@ -311,19 +311,6 @@ pub trait Table: Sync + Send { Ok(()) } - #[async_backtrace::framed] - async fn purge( - &self, - ctx: Arc, - instant: Option, - num_snapshot_limit: Option, - dry_run: bool, - ) -> Result>> { - let (_, _, _, _) = (ctx, instant, num_snapshot_limit, dry_run); - - Ok(None) - } - async fn table_statistics( &self, ctx: Arc, diff --git a/src/query/ee/src/storages/fuse/mod.rs b/src/query/ee/src/storages/fuse/mod.rs index d0f4f39be64..7b58993845a 100644 --- a/src/query/ee/src/storages/fuse/mod.rs +++ b/src/query/ee/src/storages/fuse/mod.rs @@ -15,5 +15,4 @@ pub mod operations; pub use operations::vacuum_drop_tables::vacuum_drop_tables; -pub use operations::vacuum_table::do_vacuum; pub use operations::vacuum_table_v2::do_vacuum2; diff --git a/src/query/ee/src/storages/fuse/operations/handler.rs b/src/query/ee/src/storages/fuse/operations/handler.rs index 9f26ccaf221..f61c0ab3415 100644 --- a/src/query/ee/src/storages/fuse/operations/handler.rs +++ b/src/query/ee/src/storages/fuse/operations/handler.rs @@ -24,7 +24,6 @@ use databend_enterprise_vacuum_handler::VacuumHandlerWrapper; use databend_enterprise_vacuum_handler::vacuum_handler::VacuumDropTablesResult; use databend_enterprise_vacuum_handler::vacuum_handler::VacuumTempOptions; -use crate::storages::fuse::do_vacuum; use crate::storages::fuse::operations::vacuum_table_v2::do_vacuum2; use crate::storages::fuse::operations::vacuum_temporary_files::do_vacuum_temporary_files; use crate::storages::fuse::vacuum_drop_tables; @@ -32,15 +31,6 @@ pub struct RealVacuumHandler {} #[async_trait::async_trait] impl VacuumHandler for RealVacuumHandler { - async fn do_vacuum( - &self, - table: &dyn Table, - ctx: Arc, - dry_run: bool, - ) -> Result>> { - do_vacuum(table, ctx, dry_run).await - } - async fn do_vacuum2( &self, table: &dyn Table, diff --git a/src/query/ee/src/storages/fuse/operations/mod.rs b/src/query/ee/src/storages/fuse/operations/mod.rs index e30bc3e699c..dcf818c4fd7 100644 --- a/src/query/ee/src/storages/fuse/operations/mod.rs +++ b/src/query/ee/src/storages/fuse/operations/mod.rs @@ -14,7 +14,6 @@ pub mod handler; pub mod vacuum_drop_tables; -pub mod vacuum_table; pub mod vacuum_table_v2; pub mod vacuum_temporary_files; pub use handler::RealVacuumHandler; diff --git a/src/query/ee/src/storages/fuse/operations/vacuum_table.rs b/src/query/ee/src/storages/fuse/operations/vacuum_table.rs deleted file mode 100644 index ed8104e122e..00000000000 --- a/src/query/ee/src/storages/fuse/operations/vacuum_table.rs +++ /dev/null @@ -1,420 +0,0 @@ -// Copyright 2023 Databend Cloud -// -// Licensed under the Elastic License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.elastic.co/licensing/elastic-license -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::HashSet; -use std::sync::Arc; -use std::time::Instant; - -use chrono::DateTime; -use chrono::TimeDelta; -use chrono::Utc; -use databend_common_catalog::table::Table; -use databend_common_catalog::table::TableExt; -use databend_common_catalog::table_context::TableContext; -use databend_common_exception::Result; -use databend_common_meta_app::schema::least_visible_time_ident::LeastVisibleTimeIdent; -use databend_common_storages_fuse::FuseTable; -use databend_storages_common_table_meta::meta::SegmentInfo; - -const DRY_RUN_LIMIT: usize = 1000; - -#[derive(Debug, PartialEq, Eq)] -pub struct SnapshotReferencedFiles { - pub segments: HashSet, - pub blocks: HashSet, - pub blocks_index: HashSet, - pub segments_stats: HashSet, -} - -impl SnapshotReferencedFiles { - pub fn all_files(&self) -> Vec { - let mut files = vec![]; - for file in &self.segments { - files.push(file.clone()); - } - for file in &self.blocks { - files.push(file.clone()); - } - for file in &self.blocks_index { - files.push(file.clone()); - } - for file in &self.segments_stats { - files.push(file.clone()); - } - files - } -} - -// return all the segment\block\index files referenced by current snapshot. -#[async_backtrace::framed] -pub async fn get_snapshot_referenced_files( - fuse_table: &FuseTable, - ctx: &Arc, -) -> Result> { - // 1. Find all segments referenced by the current snapshots (including branches and tags) - let segments_opt = fuse_table - .get_snapshot_referenced_segments(ctx.clone(), |status| { - ctx.set_status_info(&status); - }) - .await?; - - let Some(segments) = segments_opt else { - return Ok(None); - }; - - let segment_refs: Vec<&_> = segments.iter().collect(); - let locations_referenced = fuse_table - .get_block_locations(ctx.clone(), &segment_refs, false, false) - .await?; - - Ok(Some(SnapshotReferencedFiles { - segments: segments.into_iter().map(|(location, _)| location).collect(), - blocks: locations_referenced.block_location, - blocks_index: locations_referenced.bloom_location, - segments_stats: locations_referenced.hll_location, - })) -} - -// return orphan files to be purged -#[async_backtrace::framed] -async fn get_orphan_files_to_be_purged( - fuse_table: &FuseTable, - prefix: &str, - referenced_files: HashSet, - retention_time: DateTime, -) -> Result> { - let prefix = prefix.to_string(); - fuse_table - .list_files(prefix, |location, modified| { - modified <= retention_time && !referenced_files.contains(&location) - }) - .await -} - -#[async_backtrace::framed] -pub async fn do_gc_orphan_files( - fuse_table: &FuseTable, - ctx: &Arc, - retention_time: DateTime, - start: Instant, -) -> Result<()> { - // 1. Get all the files referenced by the current snapshot - let Some(referenced_files) = get_snapshot_referenced_files(fuse_table, ctx).await? else { - return Ok(()); - }; - let status = format!( - "gc orphan: read referenced files:{},{},{},{}, cost:{:?}", - referenced_files.segments.len(), - referenced_files.blocks.len(), - referenced_files.blocks_index.len(), - referenced_files.segments_stats.len(), - start.elapsed() - ); - ctx.set_status_info(&status); - - // 2. Purge orphan segment files. - // 2.1 Get orphan segment files to be purged - let location_gen = fuse_table.meta_location_generator(); - let segment_locations_to_be_purged = get_orphan_files_to_be_purged( - fuse_table, - location_gen.segment_location_prefix(), - referenced_files.segments, - retention_time, - ) - .await?; - let status = format!( - "gc orphan: read segment_locations_to_be_purged:{}, cost:{:?}, retention_time: {}", - segment_locations_to_be_purged.len(), - start.elapsed(), - retention_time - ); - ctx.set_status_info(&status); - - // 2.2 Delete all the orphan segment files to be purged - let purged_file_num = segment_locations_to_be_purged.len(); - fuse_table - .try_purge_location_files_and_cache::( - ctx.clone(), - HashSet::from_iter(segment_locations_to_be_purged.into_iter()), - ) - .await?; - - let status = format!( - "gc orphan: purged segment files:{}, cost:{:?}", - purged_file_num, - start.elapsed() - ); - ctx.set_status_info(&status); - - // 3. Purge orphan block files. - // 3.1 Get orphan block files to be purged - let block_locations_to_be_purged = get_orphan_files_to_be_purged( - fuse_table, - location_gen.block_location_prefix(), - referenced_files.blocks, - retention_time, - ) - .await?; - let status = format!( - "gc orphan: read block_locations_to_be_purged:{}, cost:{:?}", - block_locations_to_be_purged.len(), - start.elapsed() - ); - ctx.set_status_info(&status); - - // 3.2 Delete all the orphan block files to be purged - let purged_file_num = block_locations_to_be_purged.len(); - fuse_table - .try_purge_location_files( - ctx.clone(), - HashSet::from_iter(block_locations_to_be_purged.into_iter()), - ) - .await?; - let status = format!( - "gc orphan: purged block files:{}, cost:{:?}", - purged_file_num, - start.elapsed() - ); - ctx.set_status_info(&status); - - // 4. Purge orphan block index files. - // 4.1 Get orphan block index files to be purged - let index_locations_to_be_purged = get_orphan_files_to_be_purged( - fuse_table, - location_gen.block_bloom_index_prefix(), - referenced_files.blocks_index, - retention_time, - ) - .await?; - let status = format!( - "gc orphan: read index_locations_to_be_purged:{}, cost:{:?}", - index_locations_to_be_purged.len(), - start.elapsed() - ); - ctx.set_status_info(&status); - - // 4.2 Delete all the orphan block index files to be purged - let purged_file_num = index_locations_to_be_purged.len(); - fuse_table - .try_purge_location_files( - ctx.clone(), - HashSet::from_iter(index_locations_to_be_purged.into_iter()), - ) - .await?; - let status = format!( - "gc orphan: purged block index files:{}, cost:{:?}", - purged_file_num, - start.elapsed() - ); - ctx.set_status_info(&status); - - // 5. Purge orphan segment stats files. - // 5.1 Get orphan segment stats files to be purged - let stats_locations_to_be_purged = get_orphan_files_to_be_purged( - fuse_table, - location_gen.segment_statistics_location_prefix(), - referenced_files.segments_stats, - retention_time, - ) - .await?; - let status = format!( - "gc orphan: read stats_locations_to_be_purged:{}, cost:{:?}", - stats_locations_to_be_purged.len(), - start.elapsed() - ); - ctx.set_status_info(&status); - - // 5.2 Delete all the orphan segment stats files to be purged - let purged_file_num = stats_locations_to_be_purged.len(); - fuse_table - .try_purge_location_files( - ctx.clone(), - HashSet::from_iter(stats_locations_to_be_purged.into_iter()), - ) - .await?; - let status = format!( - "gc orphan: purged segment stats files:{}, cost:{:?}", - purged_file_num, - start.elapsed() - ); - ctx.set_status_info(&status); - Ok(()) -} - -#[async_backtrace::framed] -pub async fn do_dry_run_orphan_files( - fuse_table: &FuseTable, - ctx: &Arc, - retention_time: DateTime, - start: Instant, - purge_files: &mut Vec, - dry_run_limit: usize, -) -> Result<()> { - // 1. Get all the files referenced by the current snapshot - let Some(referenced_files) = get_snapshot_referenced_files(fuse_table, ctx).await? else { - return Ok(()); - }; - let status = format!( - "dry_run orphan: read referenced files:{},{},{},{}, cost:{:?}", - referenced_files.segments.len(), - referenced_files.blocks.len(), - referenced_files.blocks_index.len(), - referenced_files.segments_stats.len(), - start.elapsed() - ); - ctx.set_status_info(&status); - - let location_gen = fuse_table.meta_location_generator(); - // 2. Get purge orphan segment files. - let segment_locations_to_be_purged = get_orphan_files_to_be_purged( - fuse_table, - location_gen.segment_location_prefix(), - referenced_files.segments, - retention_time, - ) - .await?; - let status = format!( - "dry_run orphan: read segment_locations_to_be_purged:{}, cost:{:?}", - segment_locations_to_be_purged.len(), - start.elapsed() - ); - ctx.set_status_info(&status); - - purge_files.extend(segment_locations_to_be_purged); - if purge_files.len() >= dry_run_limit { - return Ok(()); - } - - // 3. Get purge orphan block files. - let block_locations_to_be_purged = get_orphan_files_to_be_purged( - fuse_table, - location_gen.block_location_prefix(), - referenced_files.blocks, - retention_time, - ) - .await?; - let status = format!( - "dry_run orphan: read block_locations_to_be_purged:{}, cost:{:?}", - block_locations_to_be_purged.len(), - start.elapsed() - ); - ctx.set_status_info(&status); - purge_files.extend(block_locations_to_be_purged); - if purge_files.len() >= dry_run_limit { - return Ok(()); - } - - // 4. Get purge orphan block index files. - let index_locations_to_be_purged = get_orphan_files_to_be_purged( - fuse_table, - location_gen.block_bloom_index_prefix(), - referenced_files.blocks_index, - retention_time, - ) - .await?; - let status = format!( - "dry_run orphan: read index_locations_to_be_purged:{}, cost:{:?}", - index_locations_to_be_purged.len(), - start.elapsed() - ); - ctx.set_status_info(&status); - - purge_files.extend(index_locations_to_be_purged); - - // 5. Get purge orphan segment stats files. - let stats_locations_to_be_purged = get_orphan_files_to_be_purged( - fuse_table, - location_gen.segment_statistics_location_prefix(), - referenced_files.segments_stats, - retention_time, - ) - .await?; - let status = format!( - "dry_run orphan: read stats_locations_to_be_purged:{}, cost:{:?}", - stats_locations_to_be_purged.len(), - start.elapsed() - ); - ctx.set_status_info(&status); - - purge_files.extend(stats_locations_to_be_purged); - - Ok(()) -} - -#[async_backtrace::framed] -pub async fn do_vacuum( - table: &dyn Table, - ctx: Arc, - dry_run: bool, -) -> Result>> { - let fuse_table = FuseTable::try_from_table(table)?; - let start = Instant::now(); - // First, do purge - let dry_run_limit = if dry_run { Some(DRY_RUN_LIMIT) } else { None }; - // Let the table navigate to the point according to the table's retention policy. - let navigation_point = None; - let purge_files_opt = fuse_table - .purge(ctx.clone(), navigation_point, dry_run_limit, dry_run) - .await?; - let status = format!("do_vacuum: purged table, cost:{:?}", start.elapsed()); - ctx.set_status_info(&status); - - let catalog = ctx.get_default_catalog()?; - let table_lvt = catalog - .get_table_lvt(&LeastVisibleTimeIdent::new( - ctx.get_tenant(), - fuse_table.get_table_info().ident.table_id, - )) - .await?; - let table = fuse_table.refresh(ctx.as_ref()).await?; - let fuse_table = FuseTable::try_from_table(table.as_ref())?; - // Technically, we should derive a reasonable retention period from the ByNumOfSnapshotsToKeep policy, - // but it's not worth the effort since VACUUM2 will replace legacy purge and vacuum soon. - // Use the table retention period for now. - let retention_period = if fuse_table.is_transient() { - // For transient table, keep no history data - TimeDelta::zero() - } else { - fuse_table.get_data_retention_period(ctx.as_ref())? - }; - let candidate_time = Utc::now() - retention_period; - let retention_time = if let Some(lvt) = table_lvt { - std::cmp::min(lvt.time, candidate_time) - } else { - candidate_time - }; - if let Some(mut purge_files) = purge_files_opt { - let dry_run_limit = dry_run_limit.unwrap(); - if purge_files.len() < dry_run_limit { - do_dry_run_orphan_files( - fuse_table, - &ctx, - retention_time, - start, - &mut purge_files, - dry_run_limit, - ) - .await?; - } - - if purge_files.len() > dry_run_limit { - purge_files = purge_files.into_iter().take(dry_run_limit).collect(); - } - Ok(Some(purge_files)) - } else { - debug_assert!(dry_run_limit.is_none()); - do_gc_orphan_files(fuse_table, &ctx, retention_time, start).await?; - Ok(None) - } -} diff --git a/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs b/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs index dc18cdde41a..81978460bae 100644 --- a/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs +++ b/src/query/ee/src/storages/fuse/operations/vacuum_table_v2.rs @@ -580,13 +580,12 @@ async fn vacuum_base_snapshot_phase( .iter() .cloned() .collect::>(); - let _ = fuse_table - .process_tags_for_purge( + fuse_table + .protect_table_tag_references( &catalog, &selection.gc_root_path, &mut selection.snapshots_to_gc, &mut protected_segments, - false, ) .await?; diff --git a/src/query/ee/tests/it/storages/fuse/operations/vacuum.rs b/src/query/ee/tests/it/storages/fuse/operations/vacuum.rs index 897687bdf4e..2a3058411b4 100644 --- a/src/query/ee/tests/it/storages/fuse/operations/vacuum.rs +++ b/src/query/ee/tests/it/storages/fuse/operations/vacuum.rs @@ -832,8 +832,8 @@ async fn test_vacuum_dropped_table_clean_autoincrement() -> anyhow::Result<()> { .execute_command(format!("drop table {db_name}.{tbl_name}").as_str()) .await?; - // 8. Vacuum dropped tables - fixture.execute_command("vacuum drop table").await?; + // 8. Vacuum dropped tables through the compatibility alias. + fixture.execute_command("vacuum dropped objects").await?; // 9. Ensure that table auto increment sequence is cleaned up let v = meta.get_pb(&sequence_storage_ident_0).await?; diff --git a/src/query/ee/tests/it/storages/fuse/operations/vacuum2.rs b/src/query/ee/tests/it/storages/fuse/operations/vacuum2.rs index afa571778e8..7dc42a85dd7 100644 --- a/src/query/ee/tests/it/storages/fuse/operations/vacuum2.rs +++ b/src/query/ee/tests/it/storages/fuse/operations/vacuum2.rs @@ -22,9 +22,13 @@ use databend_common_catalog::table::Table; use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataBlock; +use databend_common_sql::plans::VacuumTablesPlan; use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::io::SegmentsIO; +use databend_enterprise_query::table_ref::RealTableRefHandler; use databend_enterprise_query::test_kits::context::EESetup; +use databend_query::interpreters::Interpreter; +use databend_query::interpreters::VacuumTablesInterpreter; use databend_query::sessions::QueryContext; use databend_query::sessions::TableContextTableAccess; use databend_query::test_kits::TestFixture; @@ -33,27 +37,140 @@ use databend_storages_common_io::dedup_file_locations; use databend_storages_common_table_meta::meta::CompactSegmentInfo; use futures::TryStreamExt; -// TODO investigate this +async fn table_storage_files( + ctx: &QueryContext, + storage_root: &str, + db_name: &str, + table_name: &str, +) -> Result> { + let tenant = ctx.get_tenant(); + let catalog = ctx.get_default_catalog()?; + let table = catalog.get_table(&tenant, db_name, table_name).await?; + let database = catalog.get_database(&tenant, db_name).await?; + let table_path = Path::new(storage_root) + .join(database.get_db_info().database_id.db_id.to_string()) + .join(table.get_id().to_string()); + + Ok(walkdir::WalkDir::new(table_path) + .into_iter() + .map(|entry| entry.unwrap()) + .filter(|entry| entry.file_type().is_file()) + .map(|entry| entry.into_path()) + .collect()) +} + +async fn assert_only_current_snapshot_files( + ctx: &QueryContext, + storage_root: &str, + db_name: &str, + table_name: &str, +) -> Result<()> { + let files = table_storage_files(ctx, storage_root, db_name, table_name).await?; + + // Vacuum keeps the current snapshot and its location hint. + assert_eq!(files.len(), 2); + assert!( + files + .iter() + .any(|path| path.to_string_lossy().contains("/_ss/")) + ); + assert!(files.iter().any(|path| { + path.to_string_lossy() + .contains("last_snapshot_location_hint_v2") + })); + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_vacuum_table_command() -> anyhow::Result<()> { + let fixture = TestFixture::setup_with_custom(EESetup::new()).await?; + fixture + .default_session() + .get_settings() + .set_data_retention_time_in_days(0)?; + + let database = "vacuum_table_db"; + let table = "t"; + for statement in [ + format!("create database {database}"), + format!("create table {database}.{table} (c int) as select 1"), + format!("insert into {database}.{table} values (2)"), + format!("truncate table {database}.{table}"), + ] { + fixture.execute_command(&statement).await?; + } + + let ctx = fixture.new_query_ctx().await?; + let storage_root = fixture.storage_root(); + assert!( + table_storage_files(&ctx, storage_root, database, table) + .await? + .len() + > 2 + ); + + fixture + .execute_command(&format!("vacuum table {database}.{table}")) + .await?; + assert_only_current_snapshot_files(&ctx, storage_root, database, table).await?; + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_optimize_table_purge_uses_vacuum2() -> anyhow::Result<()> { + let fixture = TestFixture::setup_with_custom(EESetup::new()).await?; + fixture + .default_session() + .get_settings() + .set_data_retention_time_in_days(0)?; + + let database = "optimize_purge_db"; + let table = "t"; + for statement in [ + format!("create database {database}"), + format!("create table {database}.{table} (c int) as select 1"), + format!("insert into {database}.{table} values (2)"), + format!("truncate table {database}.{table}"), + ] { + fixture.execute_command(&statement).await?; + } + + let ctx = fixture.new_query_ctx().await?; + let storage_root = fixture.storage_root(); + assert!( + table_storage_files(&ctx, storage_root, database, table) + .await? + .len() + > 2 + ); + + fixture + .execute_command(&format!("optimize table {database}.{table} purge")) + .await?; + assert_only_current_snapshot_files(&ctx, storage_root, database, table).await?; + + Ok(()) +} + // NOTE: SHOULD specify flavor = "multi_thread", otherwise query execution might be hanged #[tokio::test(flavor = "multi_thread")] -async fn test_vacuum2_all() -> anyhow::Result<()> { +async fn test_vacuum_tables_commands() -> anyhow::Result<()> { let ee_setup = EESetup::new(); let fixture = TestFixture::setup_with_custom(ee_setup).await?; - // Adjust retention period to 0, so that dropped tables will be vacuumed immediately let session = fixture.default_session(); session.get_settings().set_data_retention_time_in_days(0)?; let ctx = fixture.new_query_ctx().await?; - - let setup_statements = vec![ - // create non-system db1, create fuse and non-fuse table in it. + let setup_statements = [ + // Create Fuse and non-Fuse tables in a named database. "create database db1", "create table db1.t1 (c int) as select 1", "insert into db1.t1 values (1)", "truncate table db1.t1", "create table db1.t2 (c int) engine = memory as select 1", "truncate table db1.t2", - // create fuse and non-fuse tables in default db + // Create Fuse and non-Fuse tables in the default database. "create table default.t1 (c int) as select 1", "insert into default.t1 values (1)", "truncate table default.t1", @@ -61,70 +178,211 @@ async fn test_vacuum2_all() -> anyhow::Result<()> { "truncate table default.t2", ]; - for stmt in setup_statements { - fixture.execute_command(stmt).await?; + for statement in setup_statements { + fixture.execute_command(statement).await?; } - // vacuum them all - let res = fixture.execute_command("call system$fuse_vacuum2()").await; + let storage_root = fixture.storage_root(); + assert!( + table_storage_files(&ctx, storage_root, "db1", "t1") + .await? + .len() + > 2 + ); + assert!( + table_storage_files(&ctx, storage_root, "default", "t1") + .await? + .len() + > 2 + ); + + // A scoped command vacuums only the selected database and skips non-Fuse tables. + fixture.execute_command("vacuum tables from db1").await?; + assert_only_current_snapshot_files(&ctx, storage_root, "db1", "t1").await?; + assert!( + table_storage_files(&ctx, storage_root, "default", "t1") + .await? + .len() + > 2 + ); - // Check that: + // The unscoped command processes all non-system databases. + fixture.execute_command("vacuum tables").await?; + assert_only_current_snapshot_files(&ctx, storage_root, "default", "t1").await?; - // 1. non-fuse tables should not stop us + Ok(()) +} - assert!(res.is_ok()); +#[tokio::test(flavor = "multi_thread")] +async fn test_vacuum_tables_propagates_query_abort() -> anyhow::Result<()> { + let fixture = TestFixture::setup_with_custom(EESetup::new()).await?; + fixture + .default_session() + .get_settings() + .set_data_retention_time_in_days(0)?; - // 2. fuse table data should be vacuumed + let database = "vacuum_abort_db"; + for statement in [ + format!("create database {database}"), + format!("create table {database}.t1 (c int) as select 1"), + format!("insert into {database}.t1 values (2)"), + format!("truncate table {database}.t1"), + format!("create table {database}.t2 (c int) as select 1"), + format!("insert into {database}.t2 values (2)"), + format!("truncate table {database}.t2"), + ] { + fixture.execute_command(&statement).await?; + } + let ctx = fixture.new_query_ctx().await?; let storage_root = fixture.storage_root(); + for table in ["t1", "t2"] { + assert!( + table_storage_files(&ctx, storage_root, database, table) + .await? + .len() + > 2 + ); + } - async fn check_files_left( - ctx: &QueryContext, - storage_root: &str, - db_name: &str, - tbl_name: &str, - ) -> Result<()> { - let tenant = ctx.get_tenant(); - let table = ctx - .get_default_catalog()? - .get_table(&tenant, db_name, tbl_name) - .await?; + ctx.get_current_session() + .force_kill_query(ErrorCode::AbortedQuery("cancel batch vacuum")); + let result = VacuumTablesInterpreter::try_create(ctx.clone(), VacuumTablesPlan { + catalog: "default".to_string(), + database: Some(database.to_string()), + })? + .execute2() + .await; + + match result { + Err(error) => assert_eq!(error.code(), ErrorCode::ABORTED_QUERY), + Ok(_) => panic!("batch vacuum must propagate query cancellation"), + } + for table in ["t1", "t2"] { + assert!( + table_storage_files(&ctx, storage_root, database, table) + .await? + .len() + > 2, + "batch vacuum must stop without processing remaining tables after cancellation" + ); + } - let db = ctx - .get_default_catalog()? - .get_database(&tenant, db_name) - .await?; + Ok(()) +} - let path = Path::new(storage_root) - .join(db.get_db_info().database_id.db_id.to_string()) - .join(table.get_id().to_string()); +#[tokio::test(flavor = "multi_thread")] +async fn test_vacuum_table_preserves_tagged_snapshot() -> anyhow::Result<()> { + let fixture = TestFixture::setup_with_custom(EESetup::new()).await?; + RealTableRefHandler::init()?; + fixture + .default_session() + .get_settings() + .set_data_retention_time_in_days(0)?; - let walker = walkdir::WalkDir::new(path).into_iter(); + let database = "vacuum_tag_db"; + let table = "t"; + for statement in [ + "set enable_experimental_table_ref=1".to_string(), + format!("create database {database}"), + format!("create table {database}.{table} (c int)"), + format!("insert into {database}.{table} values (1), (2)"), + format!("alter table {database}.{table} create tag before_vacuum"), + format!("truncate table {database}.{table}"), + format!("insert into {database}.{table} values (3)"), + format!("vacuum table {database}.{table}"), + ] { + fixture.execute_command(&statement).await?; + } - let mut files_left = Vec::new(); - for entry in walker { - let entry = entry.unwrap(); - if entry.file_type().is_file() { - files_left.push(entry); - } - } + let tagged_stream = fixture + .execute_query(&format!( + "select c from {database}.{table} at (tag => \"before_vacuum\")" + )) + .await?; + let tagged_blocks: Vec = tagged_stream.try_collect().await?; + assert_eq!( + tagged_blocks.iter().map(DataBlock::num_rows).sum::(), + 2, + "vacuum must preserve data referenced only by a live table tag" + ); - // There should be one snapshot file and one snapshot hint file left - assert_eq!(files_left.len(), 2); - - files_left.sort_by(|a, b| a.file_name().cmp(b.file_name())); - // First is the only snapshot left - files_left[0].path().to_string_lossy().contains("/_ss/"); - // Second one is the last snapshot location hint - files_left[1] - .path() - .to_string_lossy() - .contains("last_snapshot_location_hint_v2"); - Ok::<(), ErrorCode>(()) - } + let current_stream = fixture + .execute_query(&format!("select c from {database}.{table}")) + .await?; + let current_blocks: Vec = current_stream.try_collect().await?; + assert_eq!( + current_blocks + .iter() + .map(DataBlock::num_rows) + .sum::(), + 1 + ); + + Ok(()) +} - check_files_left(&ctx, storage_root, "db1", "t1").await?; - check_files_left(&ctx, storage_root, "default", "t1").await?; +#[tokio::test(flavor = "multi_thread")] +async fn test_vacuum_all_command() -> anyhow::Result<()> { + let fixture = TestFixture::setup_with_custom(EESetup::new()).await?; + fixture + .default_session() + .get_settings() + .set_data_retention_time_in_days(0)?; + + let database = "vacuum_all_db"; + fixture + .execute_command(&format!("create database {database}")) + .await?; + fixture + .execute_command(&format!( + "create table {database}.active (c int) as select 1" + )) + .await?; + fixture + .execute_command(&format!("insert into {database}.active values (2)")) + .await?; + fixture + .execute_command(&format!("truncate table {database}.active")) + .await?; + fixture + .execute_command(&format!( + "create table {database}.dropped (c int) as select 1" + )) + .await?; + + let ctx = fixture.new_query_ctx().await?; + let tenant = ctx.get_tenant(); + let dropped = ctx + .get_default_catalog()? + .get_table(&tenant, database, "dropped") + .await?; + let dropped = FuseTable::try_from_table(dropped.as_ref())?; + let dropped_operator = dropped.get_operator(); + let mut dropped_prefix = + FuseTable::parse_storage_prefix_from_table_info(dropped.get_table_info())?; + dropped_prefix.push('/'); + assert!( + !dropped_operator + .list_with(&dropped_prefix) + .recursive(true) + .await? + .is_empty() + ); + + fixture + .execute_command(&format!("drop table {database}.dropped")) + .await?; + fixture.execute_command("vacuum all").await?; + + assert_only_current_snapshot_files(&ctx, fixture.storage_root(), database, "active").await?; + assert!( + dropped_operator + .list_with(&dropped_prefix) + .recursive(true) + .await? + .is_empty() + ); Ok(()) } diff --git a/src/query/ee_features/vacuum_handler/src/vacuum_handler.rs b/src/query/ee_features/vacuum_handler/src/vacuum_handler.rs index 2dec82feaf6..3a7e9fe5d1c 100644 --- a/src/query/ee_features/vacuum_handler/src/vacuum_handler.rs +++ b/src/query/ee_features/vacuum_handler/src/vacuum_handler.rs @@ -29,13 +29,6 @@ pub type VacuumDropTablesResult = Result<(Option>, HashS #[async_trait::async_trait] pub trait VacuumHandler: Sync + Send { - async fn do_vacuum( - &self, - table: &dyn Table, - ctx: Arc, - dry_run: bool, - ) -> Result>>; - async fn do_vacuum2( &self, table: &dyn Table, @@ -75,16 +68,6 @@ impl VacuumHandlerWrapper { Self { handler } } - #[async_backtrace::framed] - pub async fn do_vacuum( - &self, - table: &dyn Table, - ctx: Arc, - dry_run: bool, - ) -> Result>> { - self.handler.do_vacuum(table, ctx, dry_run).await - } - #[async_backtrace::framed] pub async fn do_vacuum2( &self, diff --git a/src/query/service/src/interpreters/access/privilege_access.rs b/src/query/service/src/interpreters/access/privilege_access.rs index e193811ab5e..d34ba78f33e 100644 --- a/src/query/service/src/interpreters/access/privilege_access.rs +++ b/src/query/service/src/interpreters/access/privilege_access.rs @@ -2143,9 +2143,6 @@ impl AccessChecker for PrivilegeAccess { Plan::TruncateTable(plan) => { self.validate_table_access(&plan.catalog, &plan.database, &plan.table, UserPrivilegeType::Delete, false, false).await? } - Plan::OptimizePurge(plan) => { - self.validate_table_access(&plan.catalog, &plan.database, &plan.table, UserPrivilegeType::Super, false, false).await? - } Plan::OptimizeCompactSegment(plan) => { self.validate_table_access(&plan.catalog, &plan.database, &plan.table, UserPrivilegeType::Super, false, false).await? } @@ -2156,8 +2153,22 @@ impl AccessChecker for PrivilegeAccess { Plan::VacuumTable(plan) => { self.validate_table_access(&plan.catalog, &plan.database, &plan.table, UserPrivilegeType::Super, false, false).await? } + Plan::VacuumTables(plan) => { + if let Some(database) = &plan.database { + self.validate_db_access(&plan.catalog, database, UserPrivilegeType::Super, false).await? + } else { + self.validate_access(&GrantObject::Global, UserPrivilegeType::Super, false, false).await? + } + } + Plan::VacuumAll(_) => { + self.validate_access(&GrantObject::Global, UserPrivilegeType::Super, false, false).await? + } Plan::VacuumDropTable(plan) => { - self.validate_db_access(&plan.catalog, &plan.database, UserPrivilegeType::Super, false).await? + if plan.database.is_empty() { + self.validate_access(&GrantObject::Global, UserPrivilegeType::Super, false, false).await? + } else { + self.validate_db_access(&plan.catalog, &plan.database, UserPrivilegeType::Super, false).await? + } } Plan::VacuumTemporaryFiles(_) => { self.validate_access(&GrantObject::Global, UserPrivilegeType::Super, false, false).await? diff --git a/src/query/service/src/interpreters/hook/compact_hook.rs b/src/query/service/src/interpreters/hook/compact_hook.rs index b75d5a4fb3e..e3e212f257f 100644 --- a/src/query/service/src/interpreters/hook/compact_hook.rs +++ b/src/query/service/src/interpreters/hook/compact_hook.rs @@ -219,12 +219,8 @@ pub(crate) async fn compact_table( limit: compaction_limits.clone(), }); let s_expr = SExpr::create_leaf(Arc::new(compact_block)); - let compact_interpreter = OptimizeCompactBlockInterpreter::try_create( - ctx.clone(), - s_expr, - lock_opt.clone(), - false, - )?; + let compact_interpreter = + OptimizeCompactBlockInterpreter::try_create(ctx.clone(), s_expr, lock_opt.clone())?; let mut build_res = compact_interpreter.execute2().await?; // execute the compact pipeline if build_res.main_pipeline.is_complete_pipeline()? { diff --git a/src/query/service/src/interpreters/interpreter.rs b/src/query/service/src/interpreters/interpreter.rs index ab25fca144d..52bd5382699 100644 --- a/src/query/service/src/interpreters/interpreter.rs +++ b/src/query/service/src/interpreters/interpreter.rs @@ -21,8 +21,6 @@ use databend_common_ast::ast::AlterTableAction; use databend_common_ast::ast::AlterTableStmt; use databend_common_ast::ast::Literal; use databend_common_ast::ast::ModifyColumnAction; -use databend_common_ast::ast::OptimizeTableAction; -use databend_common_ast::ast::OptimizeTableStmt; use databend_common_ast::ast::Statement; use databend_common_base::base::short_sql; use databend_common_catalog::query_kind::QueryKind; @@ -395,11 +393,8 @@ fn need_acquire_lock(ctx: Arc, stmt: &Statement) -> bool { | Statement::MergeInto(_) | Statement::Update(_) | Statement::Delete(_) - | Statement::TruncateTable(_) => true, - Statement::OptimizeTable(OptimizeTableStmt { action, .. }) => matches!( - action, - OptimizeTableAction::All | OptimizeTableAction::Compact { .. } - ), + | Statement::TruncateTable(_) + | Statement::OptimizeTable(_) => true, Statement::AlterTable(AlterTableStmt { action, .. }) => matches!( action, AlterTableAction::ReclusterTable { .. } diff --git a/src/query/service/src/interpreters/interpreter_factory.rs b/src/query/service/src/interpreters/interpreter_factory.rs index f68563df2f9..03e6fabccf3 100644 --- a/src/query/service/src/interpreters/interpreter_factory.rs +++ b/src/query/service/src/interpreters/interpreter_factory.rs @@ -491,25 +491,28 @@ impl InterpreterFactory { Plan::TruncateTable(truncate_table) => Ok(Arc::new( TruncateTableInterpreter::try_create(ctx, *truncate_table.clone())?, )), - Plan::OptimizePurge(purge) => Ok(Arc::new(OptimizePurgeInterpreter::try_create( - ctx, - *purge.clone(), - )?)), Plan::OptimizeCompactSegment(compact_segment) => Ok(Arc::new( OptimizeCompactSegmentInterpreter::try_create(ctx, *compact_segment.clone())?, )), - Plan::OptimizeCompactBlock { s_expr, need_purge } => { + Plan::OptimizeCompactBlock { s_expr } => { Ok(Arc::new(OptimizeCompactBlockInterpreter::try_create( ctx, *s_expr.clone(), LockTableOption::LockWithRetry, - *need_purge, )?)) } Plan::VacuumTable(vacuum_table) => Ok(Arc::new(VacuumTableInterpreter::try_create( ctx, *vacuum_table.clone(), )?)), + Plan::VacuumTables(vacuum_tables) => Ok(Arc::new(VacuumTablesInterpreter::try_create( + ctx, + *vacuum_tables.clone(), + )?)), + Plan::VacuumAll(vacuum_all) => Ok(Arc::new(VacuumAllInterpreter::try_create( + ctx, + *vacuum_all.clone(), + )?)), Plan::VacuumDropTable(vacuum_drop_table) => Ok(Arc::new( VacuumDropTablesInterpreter::try_create(ctx, *vacuum_drop_table.clone())?, )), diff --git a/src/query/service/src/interpreters/interpreter_optimize_compact_block.rs b/src/query/service/src/interpreters/interpreter_optimize_compact_block.rs index 83a571cf994..a1a91fcb204 100644 --- a/src/query/service/src/interpreters/interpreter_optimize_compact_block.rs +++ b/src/query/service/src/interpreters/interpreter_optimize_compact_block.rs @@ -14,18 +14,15 @@ use std::sync::Arc; -use databend_common_base::runtime::GlobalIORuntime; use databend_common_catalog::lock::LockTableOption; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_pipeline::core::ExecutionInfo; use databend_common_sql::ColumnSet; use databend_common_sql::MetadataRef; use databend_common_sql::optimizer::ir::SExpr; use databend_common_sql::plans::OptimizeCompactBlock; use crate::interpreters::Interpreter; -use crate::interpreters::interpreter_optimize_purge::purge; use crate::physical_plans::PhysicalPlanBuilder; use crate::pipelines::PipelineBuildResult; use crate::schedulers::build_query_pipeline_without_render_result_set; @@ -36,7 +33,6 @@ pub struct OptimizeCompactBlockInterpreter { ctx: Arc, s_expr: SExpr, lock_opt: LockTableOption, - need_purge: bool, } impl OptimizeCompactBlockInterpreter { @@ -44,13 +40,11 @@ impl OptimizeCompactBlockInterpreter { ctx: Arc, s_expr: SExpr, lock_opt: LockTableOption, - need_purge: bool, ) -> Result { Ok(OptimizeCompactBlockInterpreter { ctx, s_expr, lock_opt, - need_purge, }) } } @@ -71,7 +65,6 @@ impl Interpreter for OptimizeCompactBlockInterpreter { catalog, database, table, - limit, .. } = self.s_expr.plan().clone().try_into()?; @@ -98,22 +91,6 @@ impl Interpreter for OptimizeCompactBlockInterpreter { } } - if self.need_purge { - let ctx = self.ctx.clone(); - let num_snapshot_limit = limit.segment_limit; - if build_res.main_pipeline.is_empty() { - purge(ctx, &catalog, &database, &table, num_snapshot_limit, None).await?; - } else { - build_res - .main_pipeline - .set_on_finished(move |info: &ExecutionInfo| match &info.res { - Ok(_) => GlobalIORuntime::instance().block_on(async move { - purge(ctx, &catalog, &database, &table, num_snapshot_limit, None).await - }), - Err(error_code) => Err(error_code.clone()), - }); - } - } Ok(build_res) } } diff --git a/src/query/service/src/interpreters/interpreter_optimize_purge.rs b/src/query/service/src/interpreters/interpreter_optimize_purge.rs deleted file mode 100644 index 2ab52e64790..00000000000 --- a/src/query/service/src/interpreters/interpreter_optimize_purge.rs +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use databend_common_catalog::table::TableExt; -use databend_common_exception::Result; -use databend_common_sql::plans::OptimizePurgePlan; -use databend_common_storages_factory::NavigationPoint; - -use crate::interpreters::Interpreter; -use crate::pipelines::PipelineBuildResult; -use crate::sessions::QueryContext; -use crate::sessions::TableContextTableAccess; - -pub struct OptimizePurgeInterpreter { - ctx: Arc, - plan: OptimizePurgePlan, -} - -impl OptimizePurgeInterpreter { - pub fn try_create(ctx: Arc, plan: OptimizePurgePlan) -> Result { - Ok(OptimizePurgeInterpreter { ctx, plan }) - } -} - -#[async_trait::async_trait] -impl Interpreter for OptimizePurgeInterpreter { - fn name(&self) -> &str { - "OptimizePurgeInterpreter" - } - - fn is_ddl(&self) -> bool { - true - } - - #[async_backtrace::framed] - async fn execute2(&self) -> Result { - purge( - self.ctx.clone(), - &self.plan.catalog, - &self.plan.database, - &self.plan.table, - self.plan.num_snapshot_limit, - self.plan.instant.clone(), - ) - .await?; - Ok(PipelineBuildResult::create()) - } -} - -pub(crate) async fn purge( - ctx: Arc, - catalog: &str, - database: &str, - table: &str, - num_snapshot_limit: Option, - instant: Option, -) -> Result<()> { - let catalog = ctx.get_catalog(catalog).await?; - // currently, context caches the table, we have to "refresh" - // the table by using the catalog API directly - let table = catalog - .get_table(&ctx.get_tenant(), database, table) - .await?; - // check mutability - table.check_mutable()?; - - let res = table.purge(ctx, instant, num_snapshot_limit, false).await?; - assert!(res.is_none()); - Ok(()) -} diff --git a/src/query/service/src/interpreters/interpreter_table_vacuum.rs b/src/query/service/src/interpreters/interpreter_table_vacuum.rs index d65cd26b6b6..6f87709ac22 100644 --- a/src/query/service/src/interpreters/interpreter_table_vacuum.rs +++ b/src/query/service/src/interpreters/interpreter_table_vacuum.rs @@ -14,88 +14,28 @@ use std::sync::Arc; -use databend_common_catalog::table::TableExt; use databend_common_exception::Result; -use databend_common_expression::DataBlock; -use databend_common_expression::FromData; -use databend_common_expression::types::StringType; -use databend_common_expression::types::UInt64Type; use databend_common_license::license::Feature::Vacuum; use databend_common_license::license_manager::LicenseManagerSwitch; use databend_common_sql::plans::VacuumTablePlan; -use databend_common_storages_fuse::FUSE_TBL_BLOCK_PREFIX; -use databend_common_storages_fuse::FUSE_TBL_SEGMENT_PREFIX; -use databend_common_storages_fuse::FUSE_TBL_SNAPSHOT_PREFIX; -use databend_common_storages_fuse::FUSE_TBL_XOR_BLOOM_INDEX_PREFIX; -use databend_common_storages_fuse::FuseTable; -use databend_enterprise_vacuum_handler::get_vacuum_handler; use crate::interpreters::Interpreter; use crate::pipelines::PipelineBuildResult; use crate::sessions::QueryContext; +use crate::sessions::TableContext; use crate::sessions::TableContextLicense; use crate::sessions::TableContextTableAccess; +use crate::table_functions::fuse_vacuum2::vacuum_table; pub struct VacuumTableInterpreter { ctx: Arc, plan: VacuumTablePlan, } -type FileStat = (u64, u64); - -#[derive(Debug, Default)] -struct Statistics { - pub snapshot_files: FileStat, - pub segment_files: FileStat, - pub block_files: FileStat, - pub index_files: FileStat, -} - impl VacuumTableInterpreter { pub fn try_create(ctx: Arc, plan: VacuumTablePlan) -> Result { Ok(VacuumTableInterpreter { ctx, plan }) } - - async fn get_statistics(&self, fuse_table: &FuseTable) -> Result { - let operator = fuse_table.get_operator(); - let table_data_prefix = format!("/{}", fuse_table.meta_location_generator().prefix()); - - let mut snapshot_files = (0, 0); - let mut segment_files = (0, 0); - let mut block_files = (0, 0); - let mut index_files = (0, 0); - - let prefix_with_stats = vec![ - (FUSE_TBL_SNAPSHOT_PREFIX, &mut snapshot_files), - (FUSE_TBL_SEGMENT_PREFIX, &mut segment_files), - (FUSE_TBL_BLOCK_PREFIX, &mut block_files), - (FUSE_TBL_XOR_BLOOM_INDEX_PREFIX, &mut index_files), - ]; - - for (dir_prefix, stat) in prefix_with_stats { - for entry in operator - .list_with(&format!("{}/{}/", table_data_prefix, dir_prefix)) - .await? - { - if entry.metadata().is_file() { - let mut content_length = entry.metadata().content_length(); - if content_length == 0 { - content_length = operator.stat(entry.path()).await?.content_length(); - } - - stat.0 += 1; - stat.1 += content_length; - } - } - } - - Ok(Statistics { - snapshot_files, - segment_files, - block_files, - index_files, - }) - } } #[async_trait::async_trait] @@ -113,74 +53,17 @@ impl Interpreter for VacuumTableInterpreter { LicenseManagerSwitch::instance() .check_enterprise_enabled(self.ctx.get_license_key(), Vacuum)?; - let catalog_name = self.plan.catalog.clone(); - let db_name = self.plan.database.clone(); - let tbl_name = self.plan.table.clone(); - let table = self - .ctx - .get_table(&catalog_name, &db_name, &tbl_name) - .await?; - - // check mutability - table.check_mutable()?; - - let fuse_table = FuseTable::try_from_table(table.as_ref())?; - - let handler = get_vacuum_handler(); - let purge_files_opt = handler - .do_vacuum( - fuse_table, - self.ctx.clone(), - self.plan.option.dry_run.is_some(), - ) - .await?; - - match purge_files_opt { - None => { - return { - let stat = self.get_statistics(fuse_table).await?; - let total_files = stat.snapshot_files.0 - + stat.segment_files.0 - + stat.block_files.0 - + stat.index_files.0; - let total_size = stat.snapshot_files.1 - + stat.segment_files.1 - + stat.block_files.1 - + stat.index_files.1; - PipelineBuildResult::from_blocks(vec![DataBlock::new_from_columns(vec![ - UInt64Type::from_data(vec![stat.snapshot_files.0]), - UInt64Type::from_data(vec![stat.snapshot_files.1]), - UInt64Type::from_data(vec![stat.segment_files.0]), - UInt64Type::from_data(vec![stat.segment_files.1]), - UInt64Type::from_data(vec![stat.block_files.0]), - UInt64Type::from_data(vec![stat.block_files.1]), - UInt64Type::from_data(vec![stat.index_files.0]), - UInt64Type::from_data(vec![stat.index_files.1]), - UInt64Type::from_data(vec![total_files]), - UInt64Type::from_data(vec![total_size]), - ])]) - }; - } - Some(purge_files) => { - let mut file_sizes = vec![]; - let operator = fuse_table.get_operator(); - for file in &purge_files { - file_sizes.push(operator.stat(file).await?.content_length()); - } - - // when `purge_files_opt` is some, it means `dry_run` is some, so safe to unwrap() - if self.plan.option.dry_run.unwrap() { - PipelineBuildResult::from_blocks(vec![DataBlock::new_from_columns(vec![ - UInt64Type::from_data(vec![purge_files.len() as u64]), - UInt64Type::from_data(vec![file_sizes.into_iter().sum()]), - ])]) - } else { - PipelineBuildResult::from_blocks(vec![DataBlock::new_from_columns(vec![ - StringType::from_data(purge_files), - UInt64Type::from_data(file_sizes), - ])]) - } - } - } + let catalog = self.ctx.get_catalog(&self.plan.catalog).await?; + let table_ctx: Arc = self.ctx.clone(); + vacuum_table( + &table_ctx, + catalog.as_ref(), + &self.plan.database, + &self.plan.table, + false, + ) + .await?; + + Ok(PipelineBuildResult::create()) } } diff --git a/src/query/service/src/interpreters/interpreter_vacuum_all.rs b/src/query/service/src/interpreters/interpreter_vacuum_all.rs new file mode 100644 index 00000000000..1130eeb25f0 --- /dev/null +++ b/src/query/service/src/interpreters/interpreter_vacuum_all.rs @@ -0,0 +1,82 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use databend_common_exception::Result; +use databend_common_license::license::Feature::Vacuum; +use databend_common_license::license_manager::LicenseManagerSwitch; +use databend_common_sql::plans::VacuumAllPlan; +use databend_common_sql::plans::VacuumDropTablePlan; +use databend_common_sql::plans::VacuumTablesPlan; +use databend_common_sql::plans::VacuumTemporaryFilesPlan; + +use crate::interpreters::Interpreter; +use crate::interpreters::VacuumDropTablesInterpreter; +use crate::interpreters::VacuumTablesInterpreter; +use crate::interpreters::VacuumTemporaryFilesInterpreter; +use crate::pipelines::PipelineBuildResult; +use crate::sessions::QueryContext; +use crate::sessions::TableContextLicense; + +pub struct VacuumAllInterpreter { + ctx: Arc, + plan: VacuumAllPlan, +} + +impl VacuumAllInterpreter { + pub fn try_create(ctx: Arc, plan: VacuumAllPlan) -> Result { + Ok(Self { ctx, plan }) + } +} + +#[async_trait::async_trait] +impl Interpreter for VacuumAllInterpreter { + fn name(&self) -> &str { + "VacuumAllInterpreter" + } + + fn is_ddl(&self) -> bool { + true + } + + #[async_backtrace::framed] + async fn execute2(&self) -> Result { + LicenseManagerSwitch::instance() + .check_enterprise_enabled(self.ctx.get_license_key(), Vacuum)?; + + VacuumTablesInterpreter::try_create(self.ctx.clone(), VacuumTablesPlan { + catalog: self.plan.catalog.clone(), + database: None, + })? + .execute2() + .await?; + + VacuumDropTablesInterpreter::try_create(self.ctx.clone(), VacuumDropTablePlan { + catalog: self.plan.catalog.clone(), + database: String::new(), + })? + .execute2() + .await?; + + VacuumTemporaryFilesInterpreter::try_create(self.ctx.clone(), VacuumTemporaryFilesPlan { + limit: None, + retain: None, + })? + .execute2() + .await?; + + Ok(PipelineBuildResult::create()) + } +} diff --git a/src/query/service/src/interpreters/interpreter_vacuum_drop_tables.rs b/src/query/service/src/interpreters/interpreter_vacuum_drop_tables.rs index d875158ec9a..f47de85ba73 100644 --- a/src/query/service/src/interpreters/interpreter_vacuum_drop_tables.rs +++ b/src/query/service/src/interpreters/interpreter_vacuum_drop_tables.rs @@ -12,9 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::cmp::min; use std::collections::BTreeMap; -use std::collections::HashMap; use std::collections::HashSet; use std::sync::Arc; @@ -22,10 +20,6 @@ use chrono::Duration; use databend_common_catalog::catalog::Catalog; use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use databend_common_expression::DataBlock; -use databend_common_expression::FromData; -use databend_common_expression::types::StringType; -use databend_common_expression::types::UInt64Type; use databend_common_license::license::Feature::Vacuum; use databend_common_license::license_manager::LicenseManagerSwitch; use databend_common_meta_api::GarbageCollectionApi; @@ -47,8 +41,6 @@ use crate::sessions::TableContextLicense; use crate::sessions::TableContextSettings; use crate::sessions::TableContextTableAccess; -const DRY_RUN_LIMIT: usize = 1000; - pub struct VacuumDropTablesInterpreter { ctx: Arc, plan: VacuumDropTablePlan, @@ -150,24 +142,18 @@ impl Interpreter for VacuumDropTablesInterpreter { let retention_time = chrono::Utc::now() - duration; - // Set vacuum timestamp before starting the vacuum operation (only in non-dry-run mode) - // This ensures undrop operations after this point will be blocked - if self.plan.option.dry_run.is_none() { - let tenant = ctx.get_tenant(); - let meta_api = UserApiProvider::instance().get_meta_store_client(); - - // CRITICAL: Must succeed in setting vacuum timestamp before proceeding - // If this fails, vacuum operation should not proceed to prevent data loss - meta_api - .fetch_set_vacuum_timestamp(&tenant, retention_time) - .await - .map_err(|e| { - ErrorCode::MetaStorageError(format!( - "Failed to set vacuum timestamp before vacuum operation: {}. Vacuum aborted to prevent data inconsistency.", - e - )) - })?; - } + // Set the vacuum timestamp before cleanup so later undrop operations are blocked. + let tenant = ctx.get_tenant(); + let meta_api = UserApiProvider::instance().get_meta_store_client(); + meta_api + .fetch_set_vacuum_timestamp(&tenant, retention_time) + .await + .map_err(|e| { + ErrorCode::MetaStorageError(format!( + "Failed to set vacuum timestamp before vacuum operation: {}. Vacuum aborted to prevent data inconsistency.", + e + )) + })?; let catalog = self.ctx.get_catalog(self.plan.catalog.as_str()).await?; info!( "=== VACUUM DROP TABLE STARTED === db: {:?}, retention_days: {}, retention_time: {:?}", @@ -188,7 +174,7 @@ impl Interpreter for VacuumDropTablesInterpreter { &tenant, database_name, Some(retention_time), - self.plan.option.limit, + None, )) .await?; @@ -248,16 +234,8 @@ impl Interpreter for VacuumDropTablesInterpreter { let handler = get_vacuum_handler(); let threads_nums = self.ctx.get_settings().get_max_vacuum_threads()? as usize; - let (files_opt, failed_tables) = handler - .do_vacuum_drop_tables( - threads_nums, - tables, - if self.plan.option.dry_run.is_some() { - Some(DRY_RUN_LIMIT) - } else { - None - }, - ) + let (_, failed_tables) = handler + .do_vacuum_drop_tables(threads_nums, tables, None) .await?; let failed_db_ids = failed_tables @@ -266,39 +244,34 @@ impl Interpreter for VacuumDropTablesInterpreter { .map(|id| *containing_db.get(id).unwrap()) .collect::>(); - let mut num_meta_keys_removed = 0; - // gc metadata only when not dry run - if self.plan.option.dry_run.is_none() { - let mut success_dropped_ids = vec![]; - // Since drop_ids contains view IDs, any views (if present) will be added to - // the success_dropped_id list, with removal from the meta-server attempted later. - for drop_id in drop_ids { - match &drop_id { - DroppedId::Db { db_id, db_name: _ } => { - if !failed_db_ids.contains(db_id) { - success_dropped_ids.push(drop_id); - } + let mut success_dropped_ids = vec![]; + // Since drop_ids contains view IDs, any views (if present) will be added to + // the success_dropped_id list, with removal from the meta-server attempted later. + for drop_id in drop_ids { + match &drop_id { + DroppedId::Db { db_id, db_name: _ } => { + if !failed_db_ids.contains(db_id) { + success_dropped_ids.push(drop_id); } - DroppedId::Table { name: _, id } => { - if !failed_tables.contains(&id.table_id) { - success_dropped_ids.push(drop_id); - } + } + DroppedId::Table { name: _, id } => { + if !failed_tables.contains(&id.table_id) { + success_dropped_ids.push(drop_id); } } } - info!( - "vacuum drop table summary - failed dbs: {}, failed tables: {}, successfully cleaned: {} items", - failed_db_ids.len(), - failed_tables.len(), - success_dropped_ids.len() - ); - if !failed_tables.is_empty() { - info!("failed table ids: {:?}", failed_tables); - } - - num_meta_keys_removed = self.gc_drop_tables(catalog, success_dropped_ids).await?; + } + info!( + "vacuum drop table summary - failed dbs: {}, failed tables: {}, successfully cleaned: {} items", + failed_db_ids.len(), + failed_tables.len(), + success_dropped_ids.len() + ); + if !failed_tables.is_empty() { + info!("failed table ids: {:?}", failed_tables); } + let num_meta_keys_removed = self.gc_drop_tables(catalog, success_dropped_ids).await?; let success_count = tables_count as u64 - failed_tables.len() as u64; let failed_count = failed_tables.len() as u64; @@ -307,65 +280,6 @@ impl Interpreter for VacuumDropTablesInterpreter { success_count, failed_count, tables_count, num_meta_keys_removed ); - match files_opt { - None => PipelineBuildResult::from_blocks(vec![DataBlock::new_from_columns(vec![ - UInt64Type::from_data(vec![success_count]), - UInt64Type::from_data(vec![failed_count]), - ])]), - Some(purge_files) => { - let mut len = min(purge_files.len(), DRY_RUN_LIMIT); - if let Some(limit) = self.plan.option.limit { - len = min(len, limit); - } - let purge_files = &purge_files[0..len]; - let mut table_file_sizes = HashMap::new(); - for (table_name, file, file_size) in purge_files { - table_file_sizes - .entry(table_name) - .and_modify(|file_sizes: &mut Vec<(String, u64)>| { - file_sizes.push((file.to_string(), *file_size)) - }) - .or_insert(vec![(file.to_string(), *file_size)]); - } - - if let Some(summary) = self.plan.option.dry_run { - if summary { - let mut tables = vec![]; - let mut total_files = vec![]; - let mut total_size = vec![]; - for (table, file_sizes) in table_file_sizes { - tables.push(table.to_string()); - total_files.push(file_sizes.len() as u64); - total_size.push(file_sizes.into_iter().map(|(_, num)| num).sum()); - } - - PipelineBuildResult::from_blocks(vec![DataBlock::new_from_columns(vec![ - StringType::from_data(tables), - UInt64Type::from_data(total_files), - UInt64Type::from_data(total_size), - ])]) - } else { - let mut tables = Vec::with_capacity(len); - let mut files = Vec::with_capacity(len); - let mut file_size = Vec::with_capacity(len); - for (table, file_sizes) in table_file_sizes { - for (file, size) in file_sizes { - tables.push(table.to_string()); - files.push(file); - file_size.push(size); - } - } - - PipelineBuildResult::from_blocks(vec![DataBlock::new_from_columns(vec![ - StringType::from_data(tables), - StringType::from_data(files), - UInt64Type::from_data(file_size), - ])]) - } - } else { - unreachable!(); - } - } - } + Ok(PipelineBuildResult::create()) } } diff --git a/src/query/service/src/interpreters/interpreter_vacuum_tables.rs b/src/query/service/src/interpreters/interpreter_vacuum_tables.rs new file mode 100644 index 00000000000..bb8020580de --- /dev/null +++ b/src/query/service/src/interpreters/interpreter_vacuum_tables.rs @@ -0,0 +1,62 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use databend_common_exception::Result; +use databend_common_license::license::Feature::Vacuum; +use databend_common_license::license_manager::LicenseManagerSwitch; +use databend_common_sql::plans::VacuumTablesPlan; + +use crate::interpreters::Interpreter; +use crate::pipelines::PipelineBuildResult; +use crate::sessions::QueryContext; +use crate::sessions::TableContext; +use crate::sessions::TableContextLicense; +use crate::sessions::TableContextTableAccess; +use crate::table_functions::fuse_vacuum2::vacuum_tables; + +pub struct VacuumTablesInterpreter { + ctx: Arc, + plan: VacuumTablesPlan, +} + +impl VacuumTablesInterpreter { + pub fn try_create(ctx: Arc, plan: VacuumTablesPlan) -> Result { + Ok(Self { ctx, plan }) + } +} + +#[async_trait::async_trait] +impl Interpreter for VacuumTablesInterpreter { + fn name(&self) -> &str { + "VacuumTablesInterpreter" + } + + fn is_ddl(&self) -> bool { + true + } + + #[async_backtrace::framed] + async fn execute2(&self) -> Result { + LicenseManagerSwitch::instance() + .check_enterprise_enabled(self.ctx.get_license_key(), Vacuum)?; + + let catalog = self.ctx.get_catalog(&self.plan.catalog).await?; + let table_ctx: Arc = self.ctx.clone(); + vacuum_tables(&table_ctx, catalog.as_ref(), self.plan.database.as_deref()).await?; + + Ok(PipelineBuildResult::create()) + } +} diff --git a/src/query/service/src/interpreters/interpreter_vacuum_temporary_files.rs b/src/query/service/src/interpreters/interpreter_vacuum_temporary_files.rs index 3c16b88cc76..bbbab43cd53 100644 --- a/src/query/service/src/interpreters/interpreter_vacuum_temporary_files.rs +++ b/src/query/service/src/interpreters/interpreter_vacuum_temporary_files.rs @@ -15,9 +15,6 @@ use std::sync::Arc; use databend_common_exception::Result; -use databend_common_expression::DataBlock; -use databend_common_expression::FromData; -use databend_common_expression::types::UInt64Type; use databend_common_license::license::Feature::Vacuum; use databend_common_license::license_manager::LicenseManagerSwitch; use databend_common_sql::plans::VacuumTemporaryFilesPlan; @@ -75,12 +72,8 @@ impl Interpreter for VacuumTemporaryFilesInterpreter { .plan .limit .map(|limit| limit.saturating_sub(removed_files as u64)); - let cleaned_temp_table_sessions = - vacuum_inactive_temp_tables(&table_ctx, session_limit).await? as u64; + vacuum_inactive_temp_tables(&table_ctx, session_limit).await?; - PipelineBuildResult::from_blocks(vec![DataBlock::new_from_columns(vec![ - UInt64Type::from_data(vec![removed_files as u64]), - UInt64Type::from_data(vec![cleaned_temp_table_sessions]), - ])]) + Ok(PipelineBuildResult::create()) } } diff --git a/src/query/service/src/interpreters/mod.rs b/src/query/service/src/interpreters/mod.rs index a21bf9ea110..04f865f2bf2 100644 --- a/src/query/service/src/interpreters/mod.rs +++ b/src/query/service/src/interpreters/mod.rs @@ -83,7 +83,6 @@ mod interpreter_notification_desc; mod interpreter_notification_drop; mod interpreter_optimize_compact_block; mod interpreter_optimize_compact_segment; -mod interpreter_optimize_purge; mod interpreter_password_policy_alter; mod interpreter_password_policy_create; mod interpreter_password_policy_desc; @@ -220,7 +219,9 @@ mod interpreter_user_stage_remove; mod interpreter_user_udf_alter; mod interpreter_user_udf_create; mod interpreter_user_udf_drop; +mod interpreter_vacuum_all; mod interpreter_vacuum_drop_tables; +mod interpreter_vacuum_tables; mod interpreter_vacuum_temporary_files; mod interpreter_view_alter; mod interpreter_view_create; @@ -277,7 +278,6 @@ pub use interpreter_object_tag::UnsetObjectTagsInterpreter; pub use interpreter_object_tag::cleanup_object_tags; pub use interpreter_optimize_compact_block::OptimizeCompactBlockInterpreter; pub use interpreter_optimize_compact_segment::OptimizeCompactSegmentInterpreter; -pub use interpreter_optimize_purge::OptimizePurgeInterpreter; pub use interpreter_password_policy_alter::AlterPasswordPolicyInterpreter; pub use interpreter_password_policy_create::CreatePasswordPolicyInterpreter; pub use interpreter_password_policy_desc::DescPasswordPolicyInterpreter; @@ -353,7 +353,9 @@ pub use interpreter_user_stage_remove::RemoveUserStageInterpreter; pub use interpreter_user_udf_alter::AlterUserUDFScript; pub use interpreter_user_udf_create::CreateUserUDFScript; pub use interpreter_user_udf_drop::DropUserUDFScript; +pub use interpreter_vacuum_all::VacuumAllInterpreter; pub use interpreter_vacuum_drop_tables::VacuumDropTablesInterpreter; +pub use interpreter_vacuum_tables::VacuumTablesInterpreter; pub use interpreter_vacuum_temporary_files::VacuumTemporaryFilesInterpreter; pub use interpreter_view_alter::AlterViewInterpreter; pub use interpreter_view_create::CreateViewInterpreter; diff --git a/src/query/service/src/sessions/queue_mgr.rs b/src/query/service/src/sessions/queue_mgr.rs index 3ade6d40a67..acaffa78756 100644 --- a/src/query/service/src/sessions/queue_mgr.rs +++ b/src/query/service/src/sessions/queue_mgr.rs @@ -666,10 +666,12 @@ impl QueryEntry { } // DDL: Heavy actions. - Plan::OptimizePurge(_) - | Plan::OptimizeCompactSegment(_) + Plan::OptimizeCompactSegment(_) | Plan::OptimizeCompactBlock { .. } | Plan::VacuumTable(_) + | Plan::VacuumTables(_) + | Plan::VacuumAll(_) + | Plan::VacuumDropTable(_) | Plan::VacuumTemporaryFiles(_) | Plan::RefreshIndex(_) | Plan::ReclusterTable(_) diff --git a/src/query/service/src/table_functions/fuse_vacuum2/fuse_vacuum2_table.rs b/src/query/service/src/table_functions/fuse_vacuum2/fuse_vacuum2_table.rs index 3547bea37b4..f270edff422 100644 --- a/src/query/service/src/table_functions/fuse_vacuum2/fuse_vacuum2_table.rs +++ b/src/query/service/src/table_functions/fuse_vacuum2/fuse_vacuum2_table.rs @@ -14,11 +14,8 @@ use std::sync::Arc; -use databend_common_catalog::catalog::Catalog; use databend_common_catalog::catalog_kind::CATALOG_DEFAULT; use databend_common_catalog::plan::DataSourcePlan; -use databend_common_catalog::table::Table; -use databend_common_catalog::table::TableExt; use databend_common_catalog::table_args::TableArgs; use databend_common_exception::ErrorCode; use databend_common_exception::Result; @@ -31,19 +28,16 @@ use databend_common_expression::TableSchemaRefExt; use databend_common_expression::types::StringType; use databend_common_license::license::Feature::Vacuum; use databend_common_license::license_manager::LicenseManagerSwitch; -use databend_common_storages_fuse::FuseTable; use databend_common_storages_fuse::table_functions::SimpleTableFunc; use databend_common_storages_fuse::table_functions::bool_literal; use databend_common_storages_fuse::table_functions::bool_value; use databend_common_storages_fuse::table_functions::parse_db_tb_args; use databend_common_storages_fuse::table_functions::string_literal; use databend_common_storages_fuse::table_functions::string_value; -use databend_enterprise_vacuum_handler::VacuumHandlerWrapper; -use databend_enterprise_vacuum_handler::get_vacuum_handler; -use log::info; -use log::warn; use crate::sessions::TableContext; +use crate::table_functions::fuse_vacuum2::vacuum_table; +use crate::table_functions::fuse_vacuum2::vacuum_tables; enum Vacuum2TableArgs { SingleTable { @@ -73,7 +67,6 @@ impl From<&Vacuum2TableArgs> for TableArgs { pub struct FuseVacuum2Table { args: Vacuum2TableArgs, - handler: Arc, } #[async_trait::async_trait] @@ -94,13 +87,13 @@ impl SimpleTableFunc for FuseVacuum2Table { LicenseManagerSwitch::instance().check_enterprise_enabled(ctx.get_license_key(), Vacuum)?; let catalog = ctx.get_catalog(CATALOG_DEFAULT).await?; - let res = match &self.args { + let result = match &self.args { Vacuum2TableArgs::SingleTable { arg_database_name, arg_table_name, respect_flash_back, } => { - self.apply_single_table( + vacuum_table( ctx, catalog.as_ref(), arg_database_name, @@ -109,10 +102,13 @@ impl SimpleTableFunc for FuseVacuum2Table { ) .await? } - Vacuum2TableArgs::All => self.apply_all_tables(ctx, catalog.as_ref()).await?, + Vacuum2TableArgs::All => { + vacuum_tables(ctx, catalog.as_ref(), None).await?; + vec![] + } }; Ok(Some(DataBlock::new_from_columns(vec![ - StringType::from_data(res), + StringType::from_data(result), ]))) } @@ -146,102 +142,6 @@ impl SimpleTableFunc for FuseVacuum2Table { )); } }; - Ok(Self { - args, - handler: get_vacuum_handler(), - }) - } -} - -impl FuseVacuum2Table { - async fn apply_single_table( - &self, - ctx: &Arc, - catalog: &dyn Catalog, - database_name: &str, - table_name: &str, - respect_flash_back: bool, - ) -> Result> { - let tbl = catalog - .get_table(&ctx.get_tenant(), database_name, table_name) - .await?; - - let tbl = FuseTable::try_from_table(tbl.as_ref()).map_err(|_| { - ErrorCode::StorageOther("Invalid table engine, only fuse table is supported") - })?; - - tbl.check_mutable()?; - - self.handler - .do_vacuum2(tbl, ctx.clone(), respect_flash_back) - .await - } - - async fn apply_all_tables( - &self, - ctx: &Arc, - catalog: &dyn Catalog, - ) -> Result> { - let tenant_id = ctx.get_tenant(); - let dbs = catalog.list_databases(&tenant_id).await?; - let num_db = dbs.len(); - - for (idx_db, db) in dbs.iter().enumerate() { - if db.engine().to_uppercase() == "SYSTEM" { - info!("Bypass system database [{}]", db.name()); - continue; - } - - info!( - "Processing db {}, progress: {}/{}", - db.name(), - idx_db + 1, - num_db - ); - let tables = catalog.list_tables(&tenant_id, db.name()).await?; - info!("Found {} tables in db {}", tables.len(), db.name()); - - let num_tbl = tables.len(); - for (idx_tbl, table) in tables.iter().enumerate() { - info!( - "Processing table {}.{}, db level progress: {}/{}", - db.name(), - table.get_table_info().name, - idx_tbl + 1, - num_tbl - ); - - let Ok(tbl) = FuseTable::try_from_table(table.as_ref()) else { - info!( - "Bypass non-fuse table {}.{}", - db.name(), - table.get_table_info().name - ); - continue; - }; - - if tbl.is_read_only() { - info!( - "Bypass read only table {}.{}", - db.name(), - table.get_table_info().name - ); - continue; - } - - let res = self.handler.do_vacuum2(tbl, ctx.clone(), false).await; - - if let Err(e) = res { - warn!( - "vacuum2 table {}.{} failed: {}", - db.name(), - table.get_table_info().name, - e - ); - }; - } - } - - Ok(vec![]) + Ok(Self { args }) } } diff --git a/src/query/service/src/table_functions/fuse_vacuum2/mod.rs b/src/query/service/src/table_functions/fuse_vacuum2/mod.rs index 6560a4acec7..20a17ac4fc3 100644 --- a/src/query/service/src/table_functions/fuse_vacuum2/mod.rs +++ b/src/query/service/src/table_functions/fuse_vacuum2/mod.rs @@ -13,5 +13,8 @@ // limitations under the License. mod fuse_vacuum2_table; +mod vacuum2; pub use fuse_vacuum2_table::FuseVacuum2Table; +pub(crate) use vacuum2::vacuum_table; +pub(crate) use vacuum2::vacuum_tables; diff --git a/src/query/service/src/table_functions/fuse_vacuum2/vacuum2.rs b/src/query/service/src/table_functions/fuse_vacuum2/vacuum2.rs new file mode 100644 index 00000000000..654749acf50 --- /dev/null +++ b/src/query/service/src/table_functions/fuse_vacuum2/vacuum2.rs @@ -0,0 +1,124 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use databend_common_catalog::catalog::Catalog; +use databend_common_catalog::table::Table; +use databend_common_catalog::table::TableExt; +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; +use databend_common_storages_fuse::FuseTable; +use databend_enterprise_vacuum_handler::get_vacuum_handler; +use log::info; +use log::warn; + +use crate::sessions::TableContext; + +pub(crate) async fn vacuum_table( + ctx: &Arc, + catalog: &dyn Catalog, + database_name: &str, + table_name: &str, + respect_flash_back: bool, +) -> Result> { + let table = catalog + .get_table(&ctx.get_tenant(), database_name, table_name) + .await?; + let table = FuseTable::try_from_table(table.as_ref()).map_err(|_| { + ErrorCode::StorageOther("Invalid table engine, only fuse table is supported") + })?; + + table.check_mutable()?; + get_vacuum_handler() + .do_vacuum2(table, ctx.clone(), respect_flash_back) + .await +} + +pub(crate) async fn vacuum_tables( + ctx: &Arc, + catalog: &dyn Catalog, + database_name: Option<&str>, +) -> Result<()> { + if let Some(database_name) = database_name { + vacuum_database(ctx, catalog, database_name).await?; + return Ok(()); + } + + let tenant = ctx.get_tenant(); + let databases = catalog.list_databases(&tenant).await?; + let num_databases = databases.len(); + + for (index, database) in databases.iter().enumerate() { + if database.engine().eq_ignore_ascii_case("SYSTEM") { + info!("Bypass system database [{}]", database.name()); + continue; + } + + info!( + "Processing db {}, progress: {}/{}", + database.name(), + index + 1, + num_databases + ); + vacuum_database(ctx, catalog, database.name()).await?; + } + + Ok(()) +} + +async fn vacuum_database( + ctx: &Arc, + catalog: &dyn Catalog, + database_name: &str, +) -> Result<()> { + let tenant = ctx.get_tenant(); + let tables = catalog.list_tables(&tenant, database_name).await?; + info!("Found {} tables in db {}", tables.len(), database_name); + + let num_tables = tables.len(); + let handler = get_vacuum_handler(); + for (index, table) in tables.iter().enumerate() { + let table_name = &table.get_table_info().name; + info!( + "Processing table {}.{}, db level progress: {}/{}", + database_name, + table_name, + index + 1, + num_tables + ); + + let Ok(table) = FuseTable::try_from_table(table.as_ref()) else { + info!("Bypass non-fuse table {}.{}", database_name, table_name); + continue; + }; + + if table.is_read_only() { + info!("Bypass read only table {}.{}", database_name, table_name); + continue; + } + + if let Err(error) = handler.do_vacuum2(table, ctx.clone(), false).await { + if error.code() == ErrorCode::ABORTED_QUERY { + return Err(error); + } + warn!( + "vacuum2 table {}.{} failed: {}", + database_name, table_name, error + ); + } + } + + Ok(()) +} diff --git a/src/query/service/src/table_functions/mod.rs b/src/query/service/src/table_functions/mod.rs index be43e2f76cf..a2fc7aa3d12 100644 --- a/src/query/service/src/table_functions/mod.rs +++ b/src/query/service/src/table_functions/mod.rs @@ -15,7 +15,7 @@ mod async_crash_me; mod billing_usage_daily; mod copy_history; -mod fuse_vacuum2; +pub(crate) mod fuse_vacuum2; mod get_lineage; #[cfg(feature = "storage-stage")] pub(crate) mod infer_schema; diff --git a/src/query/service/tests/it/sessions/queue_mgr.rs b/src/query/service/tests/it/sessions/queue_mgr.rs index 274d0dfc78a..a379768dfca 100644 --- a/src/query/service/tests/it/sessions/queue_mgr.rs +++ b/src/query/service/tests/it/sessions/queue_mgr.rs @@ -412,6 +412,18 @@ async fn test_heavy_actions() -> anyhow::Result<()> { sql: "vacuum table t", add_to_queue: true, }, + Query { + sql: "vacuum tables", + add_to_queue: true, + }, + Query { + sql: "vacuum all", + add_to_queue: true, + }, + Query { + sql: "vacuum drop table", + add_to_queue: true, + }, Query { sql: "vacuum temporary files", add_to_queue: true, diff --git a/src/query/service/tests/it/sql/exec/mod.rs b/src/query/service/tests/it/sql/exec/mod.rs index 996d2956f1d..871e3fc26b7 100644 --- a/src/query/service/tests/it/sql/exec/mod.rs +++ b/src/query/service/tests/it/sql/exec/mod.rs @@ -242,12 +242,11 @@ pub async fn test_snapshot_consistency() -> anyhow::Result<()> { let compact_task = async move { let compact_sql = format!("optimize table {}.{} compact", db2, tbl2); let (compact_plan, _) = planner2.plan_sql(&compact_sql).await?; - if let Plan::OptimizeCompactBlock { s_expr, need_purge } = compact_plan { + if let Plan::OptimizeCompactBlock { s_expr } = compact_plan { let optimize_interpreter = OptimizeCompactBlockInterpreter::try_create( ctx.clone(), *s_expr.clone(), LockTableOption::LockWithRetry, - need_purge, )?; let _ = optimize_interpreter.execute(ctx).await?; } diff --git a/src/query/service/tests/it/storages/fuse/operations/analyze.rs b/src/query/service/tests/it/storages/fuse/operations/analyze.rs index b5f5be3dc9f..bc3b2745bc5 100644 --- a/src/query/service/tests/it/storages/fuse/operations/analyze.rs +++ b/src/query/service/tests/it/storages/fuse/operations/analyze.rs @@ -12,10 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::Arc; - use databend_common_storages_fuse::FuseTable; -use databend_query::sessions::TableContext; use databend_query::sessions::TableContextTableAccess; use databend_query::test_kits::*; @@ -26,23 +23,12 @@ async fn test_fuse_snapshot_analyze() -> anyhow::Result<()> { let fixture = TestFixture::setup().await?; fixture.create_default_database().await?; - let ctx = fixture.new_query_ctx().await?; let case_name = "analyze_statistic_optimize"; do_insertions(&fixture).await?; fixture.analyze_table().await?; check_data_dir(&fixture, case_name, 3, 1, 2, 2, 2, 2, Some(()), None).await?; - // Purge will keep at least two snapshots. - let table = fixture.latest_default_table().await?; - let fuse_table = FuseTable::try_from_table(table.as_ref())?; - let snapshot_files = fuse_table.list_snapshot_files().await?; - let table_ctx: Arc = ctx.clone(); - fuse_table - .do_purge(&table_ctx, snapshot_files, None, false) - .await?; - check_data_dir(&fixture, case_name, 1, 1, 1, 1, 1, 1, Some(()), Some(())).await?; - Ok(()) } @@ -86,32 +72,3 @@ async fn test_fuse_snapshot_analyze_and_truncate() -> anyhow::Result<()> { Ok(()) } - -#[tokio::test(flavor = "multi_thread")] -async fn test_fuse_snapshot_analyze_purge() -> anyhow::Result<()> { - let fixture = TestFixture::setup().await?; - fixture.create_default_database().await?; - - let ctx = fixture.new_query_ctx().await?; - let case_name = "analyze_statistic_purge"; - do_insertions(&fixture).await?; - - fixture.analyze_table().await?; - check_data_dir(&fixture, case_name, 3, 1, 2, 2, 2, 2, Some(()), None).await?; - - append_sample_data(1, &fixture).await?; - fixture.analyze_table().await?; - check_data_dir(&fixture, case_name, 5, 2, 3, 3, 3, 3, Some(()), None).await?; - - // Purge will keep at least two snapshots. - let table = fixture.latest_default_table().await?; - let fuse_table = FuseTable::try_from_table(table.as_ref())?; - let snapshot_files = fuse_table.list_snapshot_files().await?; - let table_ctx: Arc = ctx.clone(); - fuse_table - .do_purge(&table_ctx, snapshot_files, None, false) - .await?; - check_data_dir(&fixture, case_name, 1, 1, 2, 2, 2, 2, Some(()), Some(())).await?; - - Ok(()) -} diff --git a/src/query/service/tests/it/storages/fuse/operations/gc.rs b/src/query/service/tests/it/storages/fuse/operations/gc.rs deleted file mode 100644 index 478014156f8..00000000000 --- a/src/query/service/tests/it/storages/fuse/operations/gc.rs +++ /dev/null @@ -1,354 +0,0 @@ -// Copyright 2022 Datafuse Labs. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::sync::Arc; - -use chrono::Duration; -use chrono::Utc; -use databend_common_storages_fuse::FuseTable; -use databend_common_storages_fuse::io::MetaWriter; -use databend_query::sessions::TableContext; -use databend_query::test_kits::*; -use databend_storages_common_table_meta::meta::Location; -use databend_storages_common_table_meta::meta::TableSnapshot; -use databend_storages_common_table_meta::meta::Versioned; -use uuid::Uuid; - -use crate::storages::fuse::operations::mutation::compact_segment; - -#[tokio::test(flavor = "multi_thread")] -async fn test_fuse_purge_normal_case() -> anyhow::Result<()> { - let fixture = TestFixture::setup().await?; - fixture.create_default_database().await?; - fixture.create_default_table().await?; - - let ctx = fixture.new_query_ctx().await?; - - // ingests some test data - append_sample_data(1, &fixture).await?; - - // do_gc - let table = fixture.latest_default_table().await?; - let fuse_table = FuseTable::try_from_table(table.as_ref())?; - let snapshot_files = fuse_table.list_snapshot_files().await?; - let table_ctx: Arc = ctx.clone(); - fuse_table - .do_purge(&table_ctx, snapshot_files, None, false) - .await?; - - let expected_num_of_snapshot = 1; - check_data_dir( - &fixture, - "do_gc: there should be 1 snapshot, 0 segment/block", - expected_num_of_snapshot, - 0, // 0 snapshot statistic - 1, // 1 segments - 1, // 1 blocks - 1, // 1 index - 1, // 1 block statistic - Some(()), - None, - ) - .await?; - - Ok(()) -} - -#[tokio::test(flavor = "multi_thread")] -async fn test_fuse_purge_normal_orphan_snapshot() -> anyhow::Result<()> { - let fixture = TestFixture::setup().await?; - fixture.create_default_database().await?; - fixture.create_default_table().await?; - - let ctx = fixture.new_query_ctx().await?; - - // ingests some test data - append_sample_data(1, &fixture).await?; - - let table = fixture.latest_default_table().await?; - let fuse_table = FuseTable::try_from_table(table.as_ref())?; - - // create orphan snapshot, its timestamp is larger than the current one - { - let current_snapshot = fuse_table.read_table_snapshot().await?.unwrap(); - let operator = fuse_table.get_operator(); - let location_gen = fuse_table.meta_location_generator(); - let orphan_snapshot_id = Uuid::new_v4(); - let orphan_snapshot_location = - location_gen.gen_snapshot_location(&orphan_snapshot_id, TableSnapshot::VERSION)?; - // orphan_snapshot is created by using `from_previous`, which guarantees - // that the timestamp of snapshot returned is larger than `current_snapshot`'s. - let orphan_snapshot = TableSnapshot::try_from_previous( - current_snapshot.clone(), - fuse_table.cluster_key_info(), - None, - TestFixture::default_table_meta_timestamps(), - )?; - orphan_snapshot - .write_meta(&operator, &orphan_snapshot_location) - .await?; - } - - // do_gc - let table_ctx: Arc = ctx.clone(); - let snapshot_files = fuse_table.list_snapshot_files().await?; - fuse_table - .do_purge(&table_ctx, snapshot_files, None, false) - .await?; - - // expects two snapshot there - // - one snapshot of the latest version - // - one orphan snapshot which timestamp is larger then the latest snapshot's - let expected_num_of_snapshot = 2; - check_data_dir( - &fixture, - "do_gc: there should be 1 snapshot, 0 segment/block", - expected_num_of_snapshot, - 0, // 0 snapshot statistic - 1, // 1 segments - 1, // 1 blocks - 1, // 1 index - 1, // 1 block statistic - Some(()), - None, - ) - .await?; - - Ok(()) -} - -#[tokio::test(flavor = "multi_thread")] -async fn test_fuse_purge_orphan_retention() -> anyhow::Result<()> { - // verifies that: - // - // - snapshots that beyond retention period shall be collected, but - // - if segments are referenced by snapshot within retention period, - // they shall not be collected during purge. - // the blocks referenced by those segments, shall not be collected as well. - // - // for example : - // - // ──┬── - // │ - // within retention - // │ - // │ S_2 ────────────────► seg_2 ──────────────► block_2 - // ──┴── - // beyond retention S_current───────────► seg_1 seg_c ──────────────► block_1 block_c - // │ - // │ S_1 ────────────────► seg_1 ──────────────► block_1 - // │ - // │ S_0 ────────────────► seg_0 ──────────────► block_0 - // - // - S_current is the gc root - // - S_1 is S_current's precedent - // - S_2, S_0 are orphan snapshots in S_current's point of view - // each of them is not a number of S_current's precedents - // - // - S_2 should NOT be purged - // since it is within the retention period - // - seg_2 shall NOT be purged, since it is referenced by S_2. - // - block_2 shall NOT be purged , since it is referenced by seg_2 - // - // - S_current, seg_1, seg_c, block_1 and block_c shall NOT be purged - // since they are referenced by the current table snapshot - // - // - S_1 should be purged - // - // - S_1 should be purged, since it is beyond the retention period - // - seg_1 and block_1 shall be purged - - // - S_0 should be purged, since it is beyond the retention period - // - seg_0 and block_0 shall be purged - // - // put them together, after GC, there will be - // - 2 snapshots left: s_current, s_2 - // - 3 segments left: seg_c, seg_2, seg_1 - // - 3 blocks left: block_c, block_2, block_1 - - let fixture = TestFixture::setup().await?; - fixture.create_default_database().await?; - fixture.create_default_table().await?; - - let ctx = fixture.new_query_ctx().await?; - - // 1. prepare `S_1` - let number_of_block = 1; - append_sample_data(number_of_block, &fixture).await?; - // no we have 1 snapshot, 1 segment, 1 blocks - - // 2. prepare `s_current` - append_sample_data(1, &fixture).await?; - // no we have 2 snapshot, 2 segment, 2 blocks - let table = fixture.latest_default_table().await?; - let fuse_table = FuseTable::try_from_table(table.as_ref())?; - let base_snapshot = fuse_table.read_table_snapshot().await?.unwrap(); - let base_timestamp = base_snapshot.timestamp.unwrap(); - - // 2. prepare `seg_2` - let num_of_segments = 1; - let blocks_per_segment = 1; - let segments = generate_segments( - fuse_table, - num_of_segments, - blocks_per_segment, - false, - TestFixture::default_table_meta_timestamps(), - ) - .await?; - let (segment_locations, _segment_info): (Vec<_>, Vec<_>) = segments.into_iter().unzip(); - - // 2. prepare S_2 - let new_timestamp = base_timestamp + Duration::minutes(1); - let _snapshot_location = - generate_snapshot_with_segments(fuse_table, segment_locations.clone(), Some(new_timestamp)) - .await?; - - // 2. prepare S_0 - { - let num_of_segments = 1; - let blocks_per_segment = 1; - let segments = generate_segments( - fuse_table, - num_of_segments, - blocks_per_segment, - false, - TestFixture::default_table_meta_timestamps(), - ) - .await?; - let segment_locations: Vec = segments.into_iter().map(|(l, _)| l).collect(); - let new_timestamp = base_timestamp - Duration::days(1); - let _snapshot_location = generate_snapshot_with_segments( - fuse_table, - segment_locations.clone(), - Some(new_timestamp), - ) - .await?; - } - - // do_gc - let table_ctx: Arc = ctx.clone(); - let snapshot_files = fuse_table.list_snapshot_files().await?; - fuse_table - .do_purge(&table_ctx, snapshot_files, None, false) - .await?; - - let expected_num_of_snapshot = 2; - let expected_num_of_segment = 3; - let expected_num_of_blocks = 3; - let expected_num_of_index = expected_num_of_blocks; - let expected_num_of_segment_stats = expected_num_of_segment; - check_data_dir( - &fixture, - "do_gc: verify retention period", - expected_num_of_snapshot, - 0, - expected_num_of_segment, - expected_num_of_blocks, - expected_num_of_index, - expected_num_of_segment_stats, - Some(()), - None, - ) - .await?; - - Ok(()) -} - -#[tokio::test(flavor = "multi_thread")] -async fn test_fuse_purge_older_version() -> anyhow::Result<()> { - let fixture = TestFixture::setup().await?; - fixture.create_default_database().await?; - fixture.create_normal_table().await?; - - generate_snapshots(&fixture).await?; - let ctx = fixture.new_query_ctx().await?; - let table_ctx: Arc = ctx.clone(); - let now = Utc::now(); - - // navigate to time point, snapshot 0 is purged. - { - let latest_table = fixture.latest_default_table().await?; - let fuse_table = FuseTable::try_from_table(latest_table.as_ref())?; - let snapshot_files = fuse_table.list_snapshot_files().await?; - let time_point = now - Duration::hours(12); - let snapshot_loc = fuse_table.snapshot_loc().unwrap(); - let table = fuse_table - .navigate_to_time_point(&table_ctx, snapshot_loc, time_point) - .await?; - table - .do_purge(&table_ctx, snapshot_files, None, false) - .await?; - - let expected_num_of_snapshot = 2; - let expected_num_of_segment = 3; - let expected_num_of_blocks = 6; - let expected_num_of_index = expected_num_of_blocks; - let expected_num_of_segment_stats = expected_num_of_segment; - check_data_dir( - &fixture, - "do_gc: navigate to time point", - expected_num_of_snapshot, - 0, - expected_num_of_segment, - expected_num_of_blocks, - expected_num_of_index, - expected_num_of_segment_stats, - Some(()), - None, - ) - .await?; - } - - // ingests some test data - append_sample_data(1, &fixture).await?; - - // Do compact segment, generate a new snapshot. - { - let table = fixture.latest_default_table().await?; - compact_segment(ctx.clone(), &table).await?; - check_data_dir(&fixture, "", 4, 0, 5, 7, 7, 5, Some(()), None).await?; - } - - let table = fixture.latest_default_table().await?; - let fuse_table = FuseTable::try_from_table(table.as_ref())?; - // base snapshot is root. do purge. - { - let snapshot_files = fuse_table.list_snapshot_files().await?; - fuse_table - .do_purge(&table_ctx, snapshot_files, None, false) - .await?; - - let expected_num_of_snapshot = 1; - let expected_num_of_segment = 1; - let expected_num_of_blocks = 7; - let expected_num_of_index = expected_num_of_blocks; - let expected_num_of_segment_stats = expected_num_of_segment; - check_data_dir( - &fixture, - "do_gc: with older version", - expected_num_of_snapshot, - 0, - expected_num_of_segment, - expected_num_of_blocks, - expected_num_of_index, - expected_num_of_segment_stats, - Some(()), - None, - ) - .await?; - } - - Ok(()) -} diff --git a/src/query/service/tests/it/storages/fuse/operations/mod.rs b/src/query/service/tests/it/storages/fuse/operations/mod.rs index 76e726b2f89..bdbebdf647f 100644 --- a/src/query/service/tests/it/storages/fuse/operations/mod.rs +++ b/src/query/service/tests/it/storages/fuse/operations/mod.rs @@ -19,7 +19,6 @@ mod clustering; mod commit; mod create_or_replace_ownership_object; -mod gc; mod internal_column; mod mutation; mod navigate; diff --git a/src/query/service/tests/it/storages/fuse/operations/mutation/mod.rs b/src/query/service/tests/it/storages/fuse/operations/mutation/mod.rs index cdeb474f525..58547d2d2af 100644 --- a/src/query/service/tests/it/storages/fuse/operations/mutation/mod.rs +++ b/src/query/service/tests/it/storages/fuse/operations/mutation/mod.rs @@ -18,4 +18,3 @@ mod recluster_mutator; mod segments_compact_mutator; pub use segments_compact_mutator::CompactSegmentTestFixture; -pub use segments_compact_mutator::compact_segment; diff --git a/src/query/service/tests/it/storages/fuse/operations/navigate.rs b/src/query/service/tests/it/storages/fuse/operations/navigate.rs index a359d24747b..112c1b47673 100644 --- a/src/query/service/tests/it/storages/fuse/operations/navigate.rs +++ b/src/query/service/tests/it/storages/fuse/operations/navigate.rs @@ -14,7 +14,6 @@ use std::time::Duration; -use chrono::Utc; use databend_common_catalog::table::NavigationPoint; use databend_common_exception::ErrorCode; use databend_common_expression::DataBlock; @@ -134,100 +133,6 @@ async fn test_fuse_navigate() -> anyhow::Result<()> { Ok(()) } -#[tokio::test(flavor = "multi_thread")] -async fn test_navigate_for_purge() -> anyhow::Result<()> { - // 1. Setup - let fixture = TestFixture::setup().await?; - let db = fixture.default_db_name(); - let tbl = fixture.default_table_name(); - - fixture.create_default_database().await?; - fixture.create_default_table().await?; - - // 1.1 first commit - let qry = format!( - "insert into {}.{} values (1, (2, 3)), (2, (4, 6)) ", - db, tbl - ); - let strm = fixture.execute_query(qry.as_str()).await?; - strm.try_collect::>().await?; - - // keep the first snapshot of the insertion - let table = fixture.latest_default_table().await?; - let _first_snapshot = FuseTable::try_from_table(table.as_ref())? - .snapshot_loc() - .unwrap(); - - // take a nap - tokio::time::sleep(Duration::from_millis(2)).await; - - // 1.2 second commit - let qry = format!("insert into {}.{} values (3, (6, 9)) ", db, tbl); - let strm = fixture.execute_query(qry.as_str()).await?; - strm.try_collect::>().await?; - // keep the snapshot of the second insertion - let table = fixture.latest_default_table().await?; - let second_snapshot = FuseTable::try_from_table(table.as_ref())? - .snapshot_loc() - .unwrap(); - - // take a nap - tokio::time::sleep(Duration::from_millis(2)).await; - - // 1.3 third commit - let qry = format!("insert into {}.{} values (4, (8, 12)) ", db, tbl); - let strm = fixture.execute_query(qry.as_str()).await?; - strm.try_collect::>().await?; - let table = fixture.latest_default_table().await?; - let third_snapshot = FuseTable::try_from_table(table.as_ref())? - .snapshot_loc() - .unwrap(); - - // 2. grab the history - let table = fixture.latest_default_table().await?; - let fuse_table = FuseTable::try_from_table(table.as_ref())?; - let reader = MetaReaders::table_snapshot_reader(fuse_table.get_operator()); - let loc = fuse_table.snapshot_loc().unwrap(); - assert_eq!(third_snapshot, loc); - let version = TableMetaLocationGenerator::snapshot_version(loc.as_str()); - let snapshots: Vec<_> = reader - .snapshot_history( - loc.clone(), - version, - fuse_table.meta_location_generator().clone(), - ) - .try_collect() - .await?; - - // 3. there should be three snapshots - assert_eq!(3, snapshots.len()); - - // 4. navigate by the time point - let meta = fuse_table.get_operator().stat(&loc).await?; - let modified = meta.last_modified(); - assert!(modified.is_some()); - let millis = modified.unwrap().timestamp_millis(); - let seconds = millis / 1000; - let nanos = ((millis % 1000) * 1_000_000) as u32; - let base_time = chrono::DateTime::::from_timestamp(seconds as i64, nanos) - .expect("valid timestamp from operator metadata"); - let time_point = base_time - chrono::Duration::milliseconds(1); - // navigate from the instant that is just one ms before the timestamp of the latest snapshot. - let (navigate, files) = fuse_table.list_by_time_point(time_point).await?; - assert_eq!(2, files.len()); - assert_eq!(navigate, third_snapshot); - - // 5. navigate by snapshot id. - let snapshot_id = snapshots[1].0.snapshot_id.simple().to_string(); - let (navigate, files) = fuse_table - .list_by_snapshot_id(&snapshot_id, time_point) - .await?; - assert_eq!(2, files.len()); - assert_eq!(navigate, second_snapshot); - - Ok(()) -} - #[tokio::test(flavor = "multi_thread")] async fn test_no_check_timestamp_maps_missing_prev_snapshot() -> anyhow::Result<()> { // When NO_CHECK TIMESTAMP finds a later snapshot whose predecessor object is gone diff --git a/src/query/service/tests/it/storages/fuse/operations/optimize.rs b/src/query/service/tests/it/storages/fuse/operations/optimize.rs index 07f9fb4ef9c..7515cae4532 100644 --- a/src/query/service/tests/it/storages/fuse/operations/optimize.rs +++ b/src/query/service/tests/it/storages/fuse/operations/optimize.rs @@ -18,20 +18,6 @@ use databend_query::sessions::TableContextSettings; use databend_query::test_kits::*; use futures_util::TryStreamExt; -use crate::storages::fuse::utils::do_purge_test; - -#[tokio::test(flavor = "multi_thread")] -async fn test_fuse_snapshot_optimize_purge() -> anyhow::Result<()> { - do_purge_test("test_fuse_snapshot_optimize_purge", 1, 0, 1, 1, 1, 1).await?; - Ok(()) -} - -#[tokio::test(flavor = "multi_thread")] -async fn test_fuse_snapshot_optimize_all() -> anyhow::Result<()> { - do_purge_test("test_fuse_snapshot_optimize_all", 1, 0, 1, 1, 1, 1).await?; - Ok(()) -} - #[tokio::test(flavor = "multi_thread")] async fn test_fuse_table_optimize() -> anyhow::Result<()> { let fixture = TestFixture::setup().await?; diff --git a/src/query/service/tests/it/storages/fuse/utils.rs b/src/query/service/tests/it/storages/fuse/utils.rs index 92b9564ec39..6b08804bb20 100644 --- a/src/query/service/tests/it/storages/fuse/utils.rs +++ b/src/query/service/tests/it/storages/fuse/utils.rs @@ -12,14 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::str; -use std::sync::Arc; - use chrono::Duration; use databend_common_exception::Result; use databend_common_expression::TableSchema; -use databend_common_storages_fuse::FuseTable; -use databend_query::sessions::TableContext; use databend_query::test_kits::*; use databend_storages_common_table_meta::meta::Statistics; use databend_storages_common_table_meta::meta::TableMetaTimestamps; @@ -47,48 +42,3 @@ pub async fn do_insertions(fixture: &TestFixture) -> Result<()> { append_sample_data_overwrite(1, true, fixture).await?; Ok(()) } - -pub async fn do_purge_test( - case_name: &str, - snapshot_count: u32, - table_statistic_count: u32, - segment_count: u32, - block_count: u32, - index_count: u32, - segment_stat_count: u32, -) -> Result<()> { - let fixture = TestFixture::setup().await?; - fixture.create_default_database().await?; - - // insert, and then insert overwrite (1 snapshot, 1 segment, 1 data block, 1 index block for each insertion); - do_insertions(&fixture).await?; - - // overwrite the table again, new data set: 1 block, 1 segment, 1 snapshot - append_sample_data_overwrite(1, true, &fixture).await?; - - // execute the query - let table = fixture.latest_default_table().await?; - let fuse_table = FuseTable::try_from_table(table.as_ref())?; - let snapshot_files = fuse_table.list_snapshot_files().await?; - let table_ctx: Arc = fixture.new_query_ctx().await?; - fuse_table - .do_purge(&table_ctx, snapshot_files, None, false) - .await?; - - check_data_dir( - &fixture, - case_name, - snapshot_count, - table_statistic_count, - segment_count, - block_count, - index_count, - segment_stat_count, - Some(()), - None, - ) - .await?; - history_should_have_item(&fixture, case_name, snapshot_count).await?; - - Ok(()) -} diff --git a/src/query/sql/src/planner/binder/binder.rs b/src/query/sql/src/planner/binder/binder.rs index c9a0fa332b3..7469dc044b3 100644 --- a/src/query/sql/src/planner/binder/binder.rs +++ b/src/query/sql/src/planner/binder/binder.rs @@ -354,6 +354,8 @@ impl Binder { Statement::TruncateTable(stmt) => self.bind_truncate_table(stmt).await?, Statement::OptimizeTable(stmt) => self.bind_optimize_table(bind_context, stmt).await?, Statement::VacuumTable(stmt) => self.bind_vacuum_table(bind_context, stmt).await?, + Statement::VacuumTables(stmt) => self.bind_vacuum_tables(bind_context, stmt).await?, + Statement::VacuumAll(stmt) => self.bind_vacuum_all(bind_context, stmt).await?, Statement::VacuumDropTable(stmt) => { self.bind_vacuum_drop_table(bind_context, stmt).await? } diff --git a/src/query/sql/src/planner/binder/ddl/table.rs b/src/query/sql/src/planner/binder/ddl/table.rs index 67cfe0a4550..135b61cceb9 100644 --- a/src/query/sql/src/planner/binder/ddl/table.rs +++ b/src/query/sql/src/planner/binder/ddl/table.rs @@ -59,8 +59,10 @@ use databend_common_ast::ast::TableType; use databend_common_ast::ast::TruncateTableStmt; use databend_common_ast::ast::UndropTableStmt; use databend_common_ast::ast::UriLocation; +use databend_common_ast::ast::VacuumAllStmt; use databend_common_ast::ast::VacuumDropTableStmt; use databend_common_ast::ast::VacuumTableStmt; +use databend_common_ast::ast::VacuumTablesStmt; use databend_common_ast::ast::VacuumTemporaryFiles; use databend_common_ast::ast::quote::QuotedIdent; use databend_common_ast::ast::quote::QuotedString; @@ -92,7 +94,6 @@ use databend_common_meta_app::schema::Constraint; use databend_common_meta_app::schema::CreateOption; use databend_common_meta_app::schema::TableIndex; use databend_common_meta_app::schema::TableIndexType; -use databend_common_meta_app::schema::is_materialized_view_engine; use databend_common_meta_app::storage::StorageParams; use databend_common_pipeline::core::SharedLockGuard; use databend_common_storage::EndpointPolicyScope; @@ -167,7 +168,6 @@ use crate::plans::ModifyTableCommentPlan; use crate::plans::ModifyTableConnectionPlan; use crate::plans::OptimizeCompactBlock; use crate::plans::OptimizeCompactSegmentPlan; -use crate::plans::OptimizePurgePlan; use crate::plans::Plan; use crate::plans::ReclusterPlan; use crate::plans::RefreshTableCachePlan; @@ -182,10 +182,10 @@ use crate::plans::SwapTablePlan; use crate::plans::TruncateTablePlan; use crate::plans::UndropTablePlan; use crate::plans::UnsetOptionsPlan; -use crate::plans::VacuumDropTableOption; +use crate::plans::VacuumAllPlan; use crate::plans::VacuumDropTablePlan; -use crate::plans::VacuumTableOption; use crate::plans::VacuumTablePlan; +use crate::plans::VacuumTablesPlan; use crate::plans::VacuumTemporaryFilesPlan; #[derive(Visitor)] @@ -1795,7 +1795,7 @@ impl Binder { #[async_backtrace::framed] pub(in crate::planner::binder) async fn bind_optimize_table( &mut self, - bind_context: &mut BindContext, + _bind_context: &mut BindContext, stmt: &OptimizeTableStmt, ) -> Result { let OptimizeTableStmt { @@ -1808,51 +1808,8 @@ impl Binder { let (catalog, database, table) = self.normalize_object_identifier_triple(catalog, database, table); - let table_meta = self.ctx.get_table(&catalog, &database, &table).await?; - let is_materialized_view = is_materialized_view_engine(table_meta.engine()); let limit = limit.map(|v| v as usize); let plan = match ast_action { - AstOptimizeTableAction::All if is_materialized_view => { - return Err(ErrorCode::InvalidOperation(format!( - "OPTIMIZE TABLE ALL is not supported on materialized view '{catalog}.{database}.{table}'; use OPTIMIZE TABLE ... COMPACT instead" - ))); - } - AstOptimizeTableAction::All => { - let compact_block = RelOperator::CompactBlock(OptimizeCompactBlock { - catalog, - database, - table, - limit: CompactionLimits { - segment_limit: limit, - block_limit: None, - }, - }); - let s_expr = SExpr::create_leaf(Arc::new(compact_block)); - Plan::OptimizeCompactBlock { - s_expr: Box::new(s_expr), - need_purge: true, - } - } - AstOptimizeTableAction::Purge { before } if is_materialized_view => { - return Err(ErrorCode::InvalidOperation(format!( - "OPTIMIZE TABLE PURGE is not supported on materialized view '{catalog}.{database}.{table}'" - ))); - } - AstOptimizeTableAction::Purge { before } => { - let instant = if let Some(point) = before { - let point = self.resolve_data_travel_point(bind_context, point)?; - Some(point) - } else { - None - }; - Plan::OptimizePurge(Box::new(OptimizePurgePlan { - catalog, - database, - table, - instant, - num_snapshot_limit: limit, - })) - } AstOptimizeTableAction::Compact { target } => match target { CompactTarget::Block => { let compact_block = RelOperator::CompactBlock(OptimizeCompactBlock { @@ -1867,7 +1824,6 @@ impl Binder { let s_expr = SExpr::create_leaf(Arc::new(compact_block)); Plan::OptimizeCompactBlock { s_expr: Box::new(s_expr), - need_purge: false, } } CompactTarget::Segment => { @@ -1890,24 +1846,45 @@ impl Binder { _bind_context: &mut BindContext, stmt: &VacuumTableStmt, ) -> Result { - let VacuumTableStmt { - catalog, + let database = stmt + .database + .as_ref() + .map(|database| normalize_identifier(database, &self.name_resolution_ctx).name) + .unwrap_or_else(|| self.ctx.get_current_database()); + let table = normalize_identifier(&stmt.table, &self.name_resolution_ctx).name; + + Ok(Plan::VacuumTable(Box::new(VacuumTablePlan { + catalog: self.ctx.get_current_catalog(), database, table, - option, - } = stmt; + }))) + } - let (catalog, database, table) = - self.normalize_object_identifier_triple(catalog, database, table); + #[async_backtrace::framed] + pub(in crate::planner::binder) async fn bind_vacuum_tables( + &mut self, + _bind_context: &mut BindContext, + stmt: &VacuumTablesStmt, + ) -> Result { + let database = stmt + .database + .as_ref() + .map(|database| normalize_identifier(database, &self.name_resolution_ctx).name); - let option = VacuumTableOption { - dry_run: option.dry_run, - }; - Ok(Plan::VacuumTable(Box::new(VacuumTablePlan { - catalog, + Ok(Plan::VacuumTables(Box::new(VacuumTablesPlan { + catalog: self.ctx.get_current_catalog(), database, - table, - option, + }))) + } + + #[async_backtrace::framed] + pub(in crate::planner::binder) async fn bind_vacuum_all( + &mut self, + _bind_context: &mut BindContext, + _stmt: &VacuumAllStmt, + ) -> Result { + Ok(Plan::VacuumAll(Box::new(VacuumAllPlan { + catalog: self.ctx.get_current_catalog(), }))) } @@ -1917,31 +1894,15 @@ impl Binder { _bind_context: &mut BindContext, stmt: &VacuumDropTableStmt, ) -> Result { - let VacuumDropTableStmt { - catalog, - database, - option, - } = stmt; - - let catalog = catalog - .as_ref() - .map(|ident| normalize_identifier(ident, &self.name_resolution_ctx).name) - .unwrap_or_else(|| self.ctx.get_current_catalog()); - let database = database + let database = stmt + .database .as_ref() - .map(|ident| normalize_identifier(ident, &self.name_resolution_ctx).name) - .unwrap_or_else(|| "".to_string()); + .map(|database| normalize_identifier(database, &self.name_resolution_ctx).name) + .unwrap_or_default(); - let option = { - VacuumDropTableOption { - dry_run: option.dry_run, - limit: option.limit, - } - }; Ok(Plan::VacuumDropTable(Box::new(VacuumDropTablePlan { - catalog, + catalog: self.ctx.get_current_catalog(), database, - option, }))) } diff --git a/src/query/sql/src/planner/format/display_plan.rs b/src/query/sql/src/planner/format/display_plan.rs index faee8da90a3..13cccc5fc63 100644 --- a/src/query/sql/src/planner/format/display_plan.rs +++ b/src/query/sql/src/planner/format/display_plan.rs @@ -106,10 +106,11 @@ impl Plan { Plan::RefreshTableCache(_) => Ok("RefreshTableCache".to_string()), Plan::ReclusterTable(_) => Ok("ReclusterTable".to_string()), Plan::TruncateTable(_) => Ok("TruncateTable".to_string()), - Plan::OptimizePurge(_) => Ok("OptimizePurge".to_string()), Plan::OptimizeCompactSegment(_) => Ok("OptimizeCompactSegment".to_string()), Plan::OptimizeCompactBlock { .. } => Ok("OptimizeCompactBlock".to_string()), Plan::VacuumTable(_) => Ok("VacuumTable".to_string()), + Plan::VacuumTables(_) => Ok("VacuumTables".to_string()), + Plan::VacuumAll(_) => Ok("VacuumAll".to_string()), Plan::VacuumDropTable(_) => Ok("VacuumDropTable".to_string()), Plan::VacuumTemporaryFiles(_) => Ok("VacuumTemporaryFiles".to_string()), Plan::AnalyzeTable(_) => Ok("AnalyzeTable".to_string()), diff --git a/src/query/sql/src/planner/plans/ddl/table.rs b/src/query/sql/src/planner/plans/ddl/table.rs index 182b65bdcad..d6ba9bf4499 100644 --- a/src/query/sql/src/planner/plans/ddl/table.rs +++ b/src/query/sql/src/planner/plans/ddl/table.rs @@ -115,37 +115,36 @@ pub struct VacuumTablePlan { pub catalog: String, pub database: String, pub table: String, - pub option: VacuumTableOption, } impl VacuumTablePlan { pub fn schema(&self) -> DataSchemaRef { - if let Some(summary) = self.option.dry_run { - if summary { - Arc::new(DataSchema::new(vec![ - DataField::new("total_files", DataType::Number(NumberDataType::UInt64)), - DataField::new("total_size", DataType::Number(NumberDataType::UInt64)), - ])) - } else { - Arc::new(DataSchema::new(vec![ - DataField::new("file", DataType::String), - DataField::new("file_size", DataType::Number(NumberDataType::UInt64)), - ])) - } - } else { - Arc::new(DataSchema::new(vec![ - DataField::new("snapshot_files", DataType::Number(NumberDataType::UInt64)), - DataField::new("snapshot_size", DataType::Number(NumberDataType::UInt64)), - DataField::new("segments_files", DataType::Number(NumberDataType::UInt64)), - DataField::new("segments_size", DataType::Number(NumberDataType::UInt64)), - DataField::new("block_files", DataType::Number(NumberDataType::UInt64)), - DataField::new("block_size", DataType::Number(NumberDataType::UInt64)), - DataField::new("index_files", DataType::Number(NumberDataType::UInt64)), - DataField::new("index_size", DataType::Number(NumberDataType::UInt64)), - DataField::new("total_files", DataType::Number(NumberDataType::UInt64)), - DataField::new("total_size", DataType::Number(NumberDataType::UInt64)), - ])) - } + Arc::new(DataSchema::empty()) + } +} + +/// Vacuum tables +#[derive(Clone, Debug)] +pub struct VacuumTablesPlan { + pub catalog: String, + pub database: Option, +} + +impl VacuumTablesPlan { + pub fn schema(&self) -> DataSchemaRef { + Arc::new(DataSchema::empty()) + } +} + +/// Vacuum all +#[derive(Clone, Debug)] +pub struct VacuumAllPlan { + pub catalog: String, +} + +impl VacuumAllPlan { + pub fn schema(&self) -> DataSchemaRef { + Arc::new(DataSchema::empty()) } } @@ -154,37 +153,11 @@ impl VacuumTablePlan { pub struct VacuumDropTablePlan { pub catalog: String, pub database: String, - pub option: VacuumDropTableOption, } impl VacuumDropTablePlan { pub fn schema(&self) -> DataSchemaRef { - if let Some(summary) = self.option.dry_run { - if summary { - Arc::new(DataSchema::new(vec![ - DataField::new("table", DataType::String), - DataField::new("total_files", DataType::Number(NumberDataType::UInt64)), - DataField::new("total_size", DataType::Number(NumberDataType::UInt64)), - ])) - } else { - Arc::new(DataSchema::new(vec![ - DataField::new("table", DataType::String), - DataField::new("file", DataType::String), - DataField::new("file_size", DataType::Number(NumberDataType::UInt64)), - ])) - } - } else { - Arc::new(DataSchema::new(vec![ - DataField::new( - "success_tables_count", - DataType::Number(NumberDataType::UInt64), - ), - DataField::new( - "failed_tables_count", - DataType::Number(NumberDataType::UInt64), - ), - ])) - } + Arc::new(DataSchema::empty()) } } @@ -196,28 +169,10 @@ pub struct VacuumTemporaryFilesPlan { impl crate::plans::VacuumTemporaryFilesPlan { pub fn schema(&self) -> DataSchemaRef { - Arc::new(DataSchema::new(vec![ - DataField::new("spill_files", DataType::Number(NumberDataType::UInt64)), - DataField::new( - "temp_table_sessions", - DataType::Number(NumberDataType::UInt64), - ), - ])) + Arc::new(DataSchema::empty()) } } -#[derive(Debug, Clone)] -pub struct VacuumDropTableOption { - // Some(true) means dry run with summary option - pub dry_run: Option, - pub limit: Option, -} - -#[derive(Debug, Clone)] -pub struct VacuumTableOption { - pub dry_run: Option, -} - #[derive(Clone, Debug)] pub struct AnalyzeTablePlan { pub catalog: String, diff --git a/src/query/sql/src/planner/plans/optimize.rs b/src/query/sql/src/planner/plans/optimize.rs index e3604868069..40412036a1d 100644 --- a/src/query/sql/src/planner/plans/optimize.rs +++ b/src/query/sql/src/planner/plans/optimize.rs @@ -13,20 +13,10 @@ // limitations under the License. use databend_common_catalog::table::CompactionLimits; -use databend_common_catalog::table::NavigationPoint; use crate::plans::Operator; use crate::plans::RelOp; -#[derive(Clone, Debug)] -pub struct OptimizePurgePlan { - pub catalog: String, - pub database: String, - pub table: String, - pub instant: Option, - pub num_snapshot_limit: Option, -} - #[derive(Clone, Debug)] pub struct OptimizeCompactSegmentPlan { pub catalog: String, diff --git a/src/query/sql/src/planner/plans/plan.rs b/src/query/sql/src/planner/plans/plan.rs index 58573a18025..d5b43d89830 100644 --- a/src/query/sql/src/planner/plans/plan.rs +++ b/src/query/sql/src/planner/plans/plan.rs @@ -149,7 +149,6 @@ use crate::plans::KillPlan; use crate::plans::ModifyTableColumnPlan; use crate::plans::ModifyTableCommentPlan; use crate::plans::OptimizeCompactSegmentPlan; -use crate::plans::OptimizePurgePlan; use crate::plans::PresignPlan; use crate::plans::ReclusterPlan; use crate::plans::RefreshDatabaseCachePlan; @@ -204,8 +203,10 @@ use crate::plans::UnsetWorkloadGroupQuotasPlan; use crate::plans::UseCatalogPlan; use crate::plans::UseDatabasePlan; use crate::plans::UseWarehousePlan; +use crate::plans::VacuumAllPlan; use crate::plans::VacuumDropTablePlan; use crate::plans::VacuumTablePlan; +use crate::plans::VacuumTablesPlan; use crate::plans::VacuumTemporaryFilesPlan; use crate::plans::VacuumVirtualColumnPlan; use crate::plans::copy_into_location::CopyIntoLocationPlan; @@ -332,6 +333,8 @@ pub enum Plan { RevertTable(Box), TruncateTable(Box), VacuumTable(Box), + VacuumTables(Box), + VacuumAll(Box), VacuumDropTable(Box), VacuumTemporaryFiles(Box), AnalyzeTable(Box), @@ -349,11 +352,9 @@ pub enum Plan { DropTableTag(Box), // Optimize - OptimizePurge(Box), OptimizeCompactSegment(Box), OptimizeCompactBlock { s_expr: Box, - need_purge: bool, }, // Insert @@ -575,7 +576,6 @@ impl Plan { Plan::Insert(_) => QueryKind::Insert, Plan::Replace(_) | Plan::DataMutation { .. } - | Plan::OptimizePurge(_) | Plan::OptimizeCompactSegment(_) | Plan::OptimizeCompactBlock { .. } => QueryKind::Update, _ => QueryKind::Other, @@ -615,6 +615,8 @@ impl Plan { Plan::ShowCreateMaterializedView(plan) => plan.schema(), Plan::DescribeTable(plan) => plan.schema(), Plan::VacuumTable(plan) => plan.schema(), + Plan::VacuumTables(plan) => plan.schema(), + Plan::VacuumAll(plan) => plan.schema(), Plan::VacuumDropTable(plan) => plan.schema(), Plan::VacuumTemporaryFiles(plan) => plan.schema(), Plan::ExistsTable(plan) => plan.schema(), @@ -747,3 +749,52 @@ impl Plan { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn vacuum_plans_have_no_result_set() { + let plans = [ + Plan::VacuumTable(Box::new(VacuumTablePlan { + catalog: "default".to_string(), + database: "default".to_string(), + table: "t".to_string(), + })), + Plan::VacuumTables(Box::new(VacuumTablesPlan { + catalog: "default".to_string(), + database: None, + })), + Plan::VacuumAll(Box::new(VacuumAllPlan { + catalog: "default".to_string(), + })), + Plan::VacuumDropTable(Box::new(VacuumDropTablePlan { + catalog: "default".to_string(), + database: String::new(), + })), + Plan::VacuumTemporaryFiles(Box::new(VacuumTemporaryFilesPlan { + limit: None, + retain: None, + })), + ]; + + for plan in plans { + assert!(plan.schema().fields().is_empty()); + assert!(!plan.has_result_set()); + } + } + + #[test] + fn virtual_column_vacuum_has_removed_files_result() { + let plan = Plan::VacuumVirtualColumn(Box::new(VacuumVirtualColumnPlan { + catalog: "default".to_string(), + database: "default".to_string(), + table: "t".to_string(), + })); + + assert_eq!(plan.schema().fields().len(), 1); + assert_eq!(plan.schema().field(0).name(), "removed_files"); + assert!(plan.has_result_set()); + } +} diff --git a/src/query/storages/fuse/src/fuse_table.rs b/src/query/storages/fuse/src/fuse_table.rs index e0c725964b2..45dd1194d3c 100644 --- a/src/query/storages/fuse/src/fuse_table.rs +++ b/src/query/storages/fuse/src/fuse_table.rs @@ -126,7 +126,6 @@ use databend_storages_common_table_meta::table::analyze_top_n_size_from_options; use futures_util::TryStreamExt; use itertools::Itertools; use log::info; -use log::warn; use opendal::Operator; use parking_lot::Mutex; use sha2::Digest; @@ -144,7 +143,6 @@ use crate::FUSE_OPT_KEY_FILE_SIZE; use crate::FUSE_OPT_KEY_ROW_PER_BLOCK; use crate::FuseSegmentFormat; use crate::FuseStorageFormat; -use crate::NavigationPoint; use crate::Table; use crate::TableStatistics; use crate::fuse_column::FuseTableColumnStatisticsProvider; @@ -1175,29 +1173,6 @@ impl Table for FuseTable { self.do_truncate(ctx, pipeline, TruncateMode::Normal).await } - #[fastrace::trace] - #[async_backtrace::framed] - async fn purge( - &self, - ctx: Arc, - instant: Option, - num_snapshot_limit: Option, - dry_run: bool, - ) -> Result>> { - match self.navigate_for_purge(&ctx, instant).await { - Ok((table, files)) => { - table - .do_purge(&ctx, files, num_snapshot_limit, dry_run) - .await - } - Err(e) if e.code() == ErrorCode::TABLE_HISTORICAL_DATA_NOT_FOUND => { - warn!("navigate failed: {:?}", e); - if dry_run { Ok(Some(vec![])) } else { Ok(None) } - } - Err(e) => Err(e), - } - } - async fn table_statistics( &self, ctx: Arc, diff --git a/src/query/storages/fuse/src/lib.rs b/src/query/storages/fuse/src/lib.rs index 04e9fd84c5e..3ea07900aca 100644 --- a/src/query/storages/fuse/src/lib.rs +++ b/src/query/storages/fuse/src/lib.rs @@ -67,7 +67,6 @@ pub mod statistics; pub mod table_functions; pub use constants::*; -use databend_common_catalog::table::NavigationPoint; use databend_common_catalog::table::Table; use databend_common_catalog::table::TableStatistics; pub use databend_common_catalog::table_context::TableContext; diff --git a/src/query/storages/fuse/src/operations/gc.rs b/src/query/storages/fuse/src/operations/gc.rs deleted file mode 100644 index 3629d9d2973..00000000000 --- a/src/query/storages/fuse/src/operations/gc.rs +++ /dev/null @@ -1,884 +0,0 @@ -// Copyright 2021 Datafuse Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::BTreeMap; -use std::collections::HashSet; -use std::sync::Arc; -use std::time::Instant; - -use chrono::Utc; -use databend_common_catalog::catalog::Catalog; -use databend_common_catalog::table::Table; -use databend_common_catalog::table_context::TableContext; -use databend_common_exception::ErrorCode; -use databend_common_exception::Result; -use databend_common_expression::ScalarRef; -use databend_common_meta_app::schema::DropTableTagReq; -use databend_common_meta_app::schema::LeastVisibleTime; -use databend_common_meta_app::schema::ListIndexesByIdReq; -use databend_common_meta_app::schema::ListTableTagsReq; -use databend_common_meta_app::schema::TableIndex; -use databend_common_meta_app::schema::least_visible_time_ident::LeastVisibleTimeIdent; -use databend_storages_common_cache::CacheAccessor; -use databend_storages_common_cache::CachedObject; -use databend_storages_common_index::BloomIndexMeta; -use databend_storages_common_index::InvertedIndexMeta; -use databend_storages_common_io::Files; -use databend_storages_common_table_meta::meta::CompactSegmentInfo; -use databend_storages_common_table_meta::meta::Location; -use databend_storages_common_table_meta::meta::SegmentInfo; -use databend_storages_common_table_meta::meta::TableSnapshot; -use databend_storages_common_table_meta::meta::TableSnapshotStatistics; -use databend_storages_common_table_meta::meta::column_oriented_segment::BLOOM_FILTER_INDEX_LOCATION; -use databend_storages_common_table_meta::meta::column_oriented_segment::ColumnOrientedSegment; -use databend_storages_common_table_meta::meta::column_oriented_segment::LOCATION; -use log::error; -use log::info; -use log::warn; - -use crate::FUSE_TBL_SNAPSHOT_PREFIX; -use crate::FuseTable; -use crate::index::InvertedIndexFile; -use crate::io::InvertedIndexReader; -use crate::io::SegmentsIO; -use crate::io::SnapshotLiteExtended; -use crate::io::SnapshotsIO; -use crate::io::TableMetaLocationGenerator; -use crate::io::read::ColumnOrientedSegmentReader; -use crate::io::read::RowOrientedSegmentReader; - -impl FuseTable { - pub async fn do_purge( - &self, - ctx: &Arc, - snapshot_files: Vec, - num_snapshot_limit: Option, - dry_run: bool, - ) -> Result>> { - let mut counter = PurgeCounter::new(); - - let res = self - .execute_purge( - ctx, - snapshot_files, - num_snapshot_limit, - &mut counter, - dry_run, - ) - .await; - info!("purge counter {:?}", counter); - res - } - - #[async_backtrace::framed] - async fn execute_purge( - &self, - ctx: &Arc, - snapshot_files: Vec, - num_snapshot_limit: Option, - counter: &mut PurgeCounter, - dry_run: bool, - ) -> Result>> { - // 1. Read the root snapshot. - let root_snapshot_location_op = self.snapshot_loc(); - if root_snapshot_location_op.is_none() { - return if dry_run { Ok(Some(vec![])) } else { Ok(None) }; - } - - let root_snapshot_location = root_snapshot_location_op.unwrap(); - let Some(root_snapshot) = - SnapshotsIO::read_snapshot_for_vacuum(self.get_operator(), &root_snapshot_location) - .await? - else { - return if dry_run { Ok(Some(vec![])) } else { Ok(None) }; - }; - - let catalog = ctx.get_catalog(&ctx.get_current_catalog()).await?; - if !dry_run { - // Persist the current snapshot timestamp as LVT so that concurrent - // writers/committers will only reference data newer than the gc root. - catalog - .set_table_lvt( - &LeastVisibleTimeIdent::new(ctx.get_tenant(), self.get_id()), - &LeastVisibleTime::new(root_snapshot.timestamp.unwrap()), - ) - .await?; - } - - let mut snapshot_files = snapshot_files; - // protected segments. - let mut segments = HashSet::from_iter(root_snapshot.segments.iter().cloned()); - let protected_table_stats_locs = self - .process_tags_for_purge( - &catalog, - &root_snapshot_location, - &mut snapshot_files, - &mut segments, - dry_run, - ) - .await?; - let segment_refs = segments.iter().collect::>(); - let referenced_locations = self - .get_block_locations(ctx.clone(), &segment_refs, true, false) - .await?; - let root_snapshot_lite = Arc::new(SnapshotLiteExtended { - format_version: root_snapshot.format_version, - snapshot_id: root_snapshot.snapshot_id, - timestamp: root_snapshot.timestamp, - segments, - table_statistics_location: root_snapshot.table_statistics_location(), - }); - - let snapshots_io = SnapshotsIO::create(ctx.clone(), self.operator.clone()); - let location_gen = self.meta_location_generator(); - let purged_snapshot_limit = num_snapshot_limit.unwrap_or(snapshot_files.len()); - - let mut read_snapshot_count = 0; - let mut remain_snapshots = Vec::::new(); - let mut dry_run_purge_files = vec![]; - let mut purged_snapshot_count = 0; - - let table_agg_index_ids = catalog - .list_index_ids_by_table_id(ListIndexesByIdReq::new(ctx.get_tenant(), self.get_id())) - .await?; - - let inverted_indexes = &self.table_info.meta.indexes; - - // 2. Read snapshot fields by chunk size. - let chunk_size = ctx.get_settings().get_max_threads()? as usize * 4; - for chunk in snapshot_files.chunks(chunk_size).rev() { - if let Err(err) = ctx.check_aborting() { - error!( - "gc: aborted query, because the server is shutting down or the query was killed. table: {}, ident {}", - self.table_info.desc, self.table_info.ident, - ); - return Err(err.with_context("failed to read snapshot")); - } - - let results = snapshots_io - .read_snapshot_lite_extends(chunk, root_snapshot_lite.clone(), false) - .await?; - let mut snapshots: Vec<_> = results.into_iter().flatten().collect(); - if snapshots.is_empty() { - break; - } - // Gather the remain snapshots. - snapshots.extend(std::mem::take(&mut remain_snapshots)); - // Sort snapshot by timestamp. - snapshots.sort_by(|a, b| a.timestamp.cmp(&b.timestamp)); - - // Set the last snapshot as base snapshot, extend the base snapshot. - let base_snapshot = snapshots.pop().unwrap(); - let base_segments = base_snapshot.segments.clone(); - let base_timestamp = base_snapshot.timestamp; - let base_ts_location_opt = base_snapshot.table_statistics_location.clone(); - remain_snapshots.push(base_snapshot); - - let mut snapshots_to_be_purged = HashSet::new(); - let mut segments_to_be_purged = HashSet::new(); - let mut ts_to_be_purged = HashSet::new(); - for s in snapshots.into_iter() { - if s.timestamp.is_some() && s.timestamp >= base_timestamp { - remain_snapshots.push(s); - continue; - } - - if let Ok(loc) = - location_gen.gen_snapshot_location(&s.snapshot_id, s.format_version) - { - if purged_snapshot_count >= purged_snapshot_limit { - break; - } - snapshots_to_be_purged.insert(loc); - purged_snapshot_count += 1; - } - - let diff: HashSet<_> = s.segments.difference(&base_segments).cloned().collect(); - segments_to_be_purged.extend(diff); - - if s.table_statistics_location.is_some() - && s.table_statistics_location != base_ts_location_opt - { - let stats_loc = s.table_statistics_location.unwrap(); - if !protected_table_stats_locs.contains(&stats_loc) { - ts_to_be_purged.insert(stats_loc); - } - } - } - - // Refresh status. - { - read_snapshot_count += chunk.len(); - let status = format!( - "gc: read snapshot files:{}/{}, cost:{:?}", - read_snapshot_count, - snapshot_files.len(), - counter.start.elapsed() - ); - ctx.set_status_info(&status); - } - - if !snapshots_to_be_purged.is_empty() { - if dry_run { - debug_assert!(num_snapshot_limit.is_some()); - self.dry_run_purge( - ctx, - &mut dry_run_purge_files, - &referenced_locations, - segments_to_be_purged, - ts_to_be_purged, - snapshots_to_be_purged, - &table_agg_index_ids, - ) - .await?; - - if dry_run_purge_files.len() >= num_snapshot_limit.unwrap() { - return Ok(Some(dry_run_purge_files)); - } - } else { - self.partial_purge( - ctx, - counter, - &referenced_locations, - segments_to_be_purged, - ts_to_be_purged, - snapshots_to_be_purged, - &table_agg_index_ids, - inverted_indexes, - ) - .await?; - - if purged_snapshot_count >= purged_snapshot_limit { - return Ok(None); - } - } - } - } - - if !remain_snapshots.is_empty() { - let mut snapshots_to_be_purged = HashSet::new(); - let mut segments_to_be_purged = HashSet::new(); - let mut ts_to_be_purged = HashSet::new(); - for s in remain_snapshots { - if let Ok(loc) = - location_gen.gen_snapshot_location(&s.snapshot_id, s.format_version) - { - if purged_snapshot_count >= purged_snapshot_limit { - break; - } - snapshots_to_be_purged.insert(loc); - purged_snapshot_count += 1; - } - - segments_to_be_purged.extend(s.segments); - - if s.table_statistics_location.is_some() { - ts_to_be_purged.insert(s.table_statistics_location.unwrap()); - } - } - if dry_run { - self.dry_run_purge( - ctx, - &mut dry_run_purge_files, - &referenced_locations, - segments_to_be_purged, - ts_to_be_purged, - snapshots_to_be_purged, - &table_agg_index_ids, - ) - .await?; - } else { - self.partial_purge( - ctx, - counter, - &referenced_locations, - segments_to_be_purged, - ts_to_be_purged, - snapshots_to_be_purged, - &table_agg_index_ids, - inverted_indexes, - ) - .await?; - } - } - - if dry_run { - return Ok(Some(dry_run_purge_files)); - } - - Ok(None) - } - - #[allow(clippy::too_many_arguments)] - async fn dry_run_purge( - &self, - ctx: &Arc, - purge_files: &mut Vec, - locations_referenced_by_root: &LocationTuple, - segments_to_be_purged: HashSet, - ts_to_be_purged: HashSet, - snapshots_to_be_purged: HashSet, - table_agg_index_ids: &[u64], - ) -> Result<()> { - let chunk_size = ctx.get_settings().get_max_threads()? as usize * 4; - // Purge segments&blocks by chunk size - let segment_locations = Vec::from_iter(segments_to_be_purged); - for chunk in segment_locations.chunks(chunk_size) { - // since we are purging files, the ErrorCode::STORAGE_NOT_FOUND error can be safely ignored. - let chunk_refs: Vec<&Location> = chunk.iter().collect(); - let locations = self - .get_block_locations(ctx.clone(), &chunk_refs, false, true) - .await?; - - for loc in &locations.block_location { - if locations_referenced_by_root.block_location.contains(loc) { - continue; - } - purge_files.push(loc.to_string()); - for index_id in table_agg_index_ids { - purge_files.push( - TableMetaLocationGenerator::gen_agg_index_location_from_block_location( - loc, *index_id, - ), - ) - } - } - - for loc in &locations.bloom_location { - if locations_referenced_by_root.bloom_location.contains(loc) { - continue; - } - purge_files.push(loc.to_string()) - } - - purge_files.extend(chunk.iter().map(|loc| loc.0.clone())); - } - purge_files.extend(ts_to_be_purged.iter().map(|loc| loc.to_string())); - purge_files.extend(snapshots_to_be_purged.iter().map(|loc| loc.to_string())); - - Ok(()) - } - - #[allow(clippy::too_many_arguments)] - async fn partial_purge( - &self, - ctx: &Arc, - counter: &mut PurgeCounter, - locations_referenced_by_root: &LocationTuple, - segments_to_be_purged: HashSet, - ts_to_be_purged: HashSet, - snapshots_to_be_purged: HashSet, - table_agg_index_ids: &[u64], - inverted_indexes: &BTreeMap, - ) -> Result<()> { - let chunk_size = ctx.get_settings().get_max_threads()? as usize * 4; - // Purge segments&blocks by chunk size - let mut count = 0; - let segment_locations = Vec::from_iter(segments_to_be_purged); - for chunk in segment_locations.chunks(chunk_size) { - // since we are purging files, the ErrorCode::STORAGE_NOT_FOUND error can be safely ignored. - let chunk_refs: Vec<&Location> = chunk.iter().collect(); - let locations = self - .get_block_locations(ctx.clone(), &chunk_refs, false, true) - .await?; - - let mut blocks_to_be_purged = HashSet::new(); - let mut agg_indexes_to_be_purged = HashSet::new(); - let mut inverted_indexes_to_be_purged = HashSet::new(); - for loc in &locations.block_location { - if locations_referenced_by_root.block_location.contains(loc) { - continue; - } - blocks_to_be_purged.insert(loc.to_string()); - for index_id in table_agg_index_ids { - agg_indexes_to_be_purged.insert( - TableMetaLocationGenerator::gen_agg_index_location_from_block_location( - loc, *index_id, - ), - ); - } - - for idx in inverted_indexes.values() { - inverted_indexes_to_be_purged.insert( - TableMetaLocationGenerator::gen_inverted_index_location_from_block_location( - loc, - idx.name.as_str(), - idx.version.as_str(), - ), - ); - } - } - - let mut blooms_to_be_purged = HashSet::new(); - for loc in &locations.bloom_location { - if locations_referenced_by_root.bloom_location.contains(loc) { - continue; - } - blooms_to_be_purged.insert(loc.to_string()); - } - - let mut stats_to_be_purged = HashSet::new(); - for loc in &locations.hll_location { - if locations_referenced_by_root.hll_location.contains(loc) { - continue; - } - stats_to_be_purged.insert(loc.to_string()); - } - - let segment_locations_to_be_purged = HashSet::from_iter( - chunk - .iter() - .map(|loc| loc.0.clone()) - .collect::>(), - ); - - // Refresh status. - { - count += chunk.len(); - let status = format!( - "gc: read purged segment files:{}/{}, cost:{:?}", - count, - segment_locations.len(), - counter.start.elapsed() - ); - ctx.set_status_info(&status); - } - - self.purge_block_segments( - ctx, - counter, - blocks_to_be_purged, - agg_indexes_to_be_purged, - inverted_indexes_to_be_purged, - blooms_to_be_purged, - stats_to_be_purged, - segment_locations_to_be_purged, - ) - .await?; - } - - self.purge_ts_snapshots(ctx, counter, ts_to_be_purged, snapshots_to_be_purged) - .await - } - - async fn purge_block_segments( - &self, - ctx: &Arc, - counter: &mut PurgeCounter, - blocks_to_be_purged: HashSet, - agg_indexes_to_be_purged: HashSet, - inverted_indexes_to_be_purged: HashSet, - blooms_to_be_purged: HashSet, - stats_to_be_purged: HashSet, - segments_to_be_purged: HashSet, - ) -> Result<()> { - // 1. Try to purge block file chunks. - let blocks_count = blocks_to_be_purged.len(); - if blocks_count > 0 { - counter.blocks += blocks_count; - self.try_purge_location_files(ctx.clone(), blocks_to_be_purged) - .await?; - } - - let agg_index_count = agg_indexes_to_be_purged.len(); - if agg_index_count > 0 { - counter.agg_indexes += agg_index_count; - self.try_purge_location_files(ctx.clone(), agg_indexes_to_be_purged) - .await?; - } - - let inverted_index_count = inverted_indexes_to_be_purged.len(); - if inverted_index_count > 0 { - counter.inverted_indexes += inverted_index_count; - - // if there is inverted index file cache, evict the cached items - if let Some(inverted_index_cache) = InvertedIndexFile::cache() { - for index_path in &inverted_indexes_to_be_purged { - InvertedIndexReader::cache_key_of_index_columns(index_path) - .iter() - .for_each(|cache_key| { - inverted_index_cache.evict(cache_key); - }) - } - } - - self.try_purge_location_files_and_cache::( - ctx.clone(), - inverted_indexes_to_be_purged, - ) - .await?; - } - - // 2. Try to purge bloom index file chunks. - let blooms_count = blooms_to_be_purged.len(); - if blooms_count > 0 { - counter.blooms += blooms_count; - self.try_purge_location_files_and_cache::( - ctx.clone(), - blooms_to_be_purged, - ) - .await?; - } - - // 3. Try to purge segment statistic file chunks. - let stats_count = stats_to_be_purged.len(); - if stats_count > 0 { - counter.hlls += stats_count; - self.try_purge_location_files(ctx.clone(), stats_to_be_purged) - .await?; - } - - // 4. Try to purge segment file chunks. - let segments_count = segments_to_be_purged.len(); - if segments_count > 0 { - counter.segments += segments_count; - self.try_purge_location_files_and_cache::( - ctx.clone(), - segments_to_be_purged, - ) - .await?; - } - Ok(()) - } - - async fn purge_ts_snapshots( - &self, - ctx: &Arc, - counter: &mut PurgeCounter, - ts_to_be_purged: HashSet, - snapshots_to_be_purged: HashSet, - ) -> Result<()> { - // 3. Purge table statistic files - let ts_count = ts_to_be_purged.len(); - if ts_count > 0 { - counter.table_statistics += ts_count; - self.try_purge_location_files_and_cache::( - ctx.clone(), - ts_to_be_purged, - ) - .await?; - } - - // 4. Purge snapshots. - let snapshots_count = snapshots_to_be_purged.len(); - if snapshots_count > 0 { - counter.snapshots += snapshots_count; - self.try_purge_location_files_and_cache::( - ctx.clone(), - snapshots_to_be_purged, - ) - .await?; - } - - // 5. Refresh status. - { - let status = format!( - "gc: block files purged:{}, bloom files purged:{}, segment stats files purged:{}, segment files purged:{}, table statistic files purged:{}, snapshots purged:{}, take:{:?}", - counter.blocks, - counter.blooms, - counter.hlls, - counter.segments, - counter.table_statistics, - counter.snapshots, - counter.start.elapsed() - ); - ctx.set_status_info(&status); - } - Ok(()) - } - - // Purge file by location chunks. - #[async_backtrace::framed] - pub async fn try_purge_location_files( - &self, - ctx: Arc, - locations_to_be_purged: HashSet, - ) -> Result<()> { - let fuse_file = Files::create(ctx.clone(), self.operator.clone()); - fuse_file.remove_file_in_batch(locations_to_be_purged).await - } - - // Purge file by location chunks. - #[async_backtrace::framed] - pub async fn try_purge_location_files_and_cache( - &self, - ctx: Arc, - locations_to_be_purged: HashSet, - ) -> Result<()> - where - T: CachedObject, - { - if let Some(cache) = T::cache() { - for loc in locations_to_be_purged.iter() { - cache.evict(loc); - } - } - self.try_purge_location_files(ctx, locations_to_be_purged) - .await - } - - /// Protect base segments referenced by tags and remove tagged snapshots from gc candidates. - pub async fn process_tags_for_purge( - &self, - catalog: &Arc, - root_snapshot_location: &String, - snapshot_files_to_gc: &mut Vec, - protected_segments: &mut HashSet, - dry_run: bool, - ) -> Result> { - let now = Utc::now(); - let tags = catalog - .list_table_tags(ListTableTagsReq { - table_id: self.get_id(), - include_expired: true, - }) - .await?; - - let mut protected_snapshot_locs = HashSet::new(); - let mut protected_table_stats_locs = HashSet::new(); - for (tag_name, seq_tag) in tags { - if seq_tag - .data - .expire_at - .is_some_and(|expire_at| expire_at <= now) - { - if !dry_run { - if let Err(e) = catalog - .drop_table_tag(DropTableTagReq { - table_id: self.get_id(), - tag_name, - seq: Some(seq_tag.seq), - }) - .await - { - warn!( - "drop expired tag failed, ignored, table: {}, err: {}", - self.table_info.desc, e - ); - } - } - continue; - } - - let tag_snapshot_loc = seq_tag.data.snapshot_loc; - if &tag_snapshot_loc < root_snapshot_location { - if let Some(snapshot) = - SnapshotsIO::read_snapshot_for_vacuum(self.get_operator(), &tag_snapshot_loc) - .await? - { - protected_segments.extend(snapshot.segments.iter().cloned()); - if let Some(stats_loc) = &snapshot.table_statistics_location { - protected_table_stats_locs.insert(stats_loc.clone()); - } - } - protected_snapshot_locs.insert(tag_snapshot_loc); - } - } - - snapshot_files_to_gc.retain(|path| !protected_snapshot_locs.contains(path)); - Ok(protected_table_stats_locs) - } - - #[async_backtrace::framed] - pub async fn get_block_locations( - &self, - ctx: Arc, - segment_locations: &[&Location], - put_cache: bool, - ignore_err: bool, - ) -> Result { - let mut blocks = HashSet::new(); - let mut blooms = HashSet::new(); - let mut hlls = HashSet::new(); - - let fuse_segments = SegmentsIO::create(ctx.clone(), self.operator.clone(), self.schema()); - let chunk_size = ctx.get_settings().get_max_threads()? as usize * 4; - let projection = HashSet::from([ - LOCATION.to_string(), - BLOOM_FILTER_INDEX_LOCATION.to_string(), - ]); - for chunk in segment_locations.chunks(chunk_size) { - let results = match self.is_column_oriented() { - true => { - let segments = fuse_segments - .generic_read_compact_segments::( - chunk, - put_cache, - &projection, - ) - .await?; - let mut results = Vec::new(); - for segment in segments { - match segment { - Ok(segment) => match LocationTuple::try_from(segment) { - Ok(location_tuple) => results.push(Ok(location_tuple)), - Err(e) => results.push(Err(e)), - }, - Err(e) => results.push(Err(e)), - } - } - results - } - false => { - let segments = fuse_segments - .generic_read_compact_segments::( - chunk, - put_cache, - &projection, - ) - .await?; - let mut results = Vec::new(); - for segment in segments { - match segment { - Ok(segment) => match LocationTuple::try_from(segment) { - Ok(location_tuple) => results.push(Ok(location_tuple)), - Err(e) => results.push(Err(e)), - }, - Err(e) => results.push(Err(e)), - } - } - results - } - }; - for (idx, location_tuple) in results.into_iter().enumerate() { - let location_tuple = match location_tuple { - Err(e) if e.code() == ErrorCode::STORAGE_NOT_FOUND && ignore_err => { - let location = chunk[idx]; - // concurrent gc: someone else has already collected this segment, ignore it - warn!( - "concurrent gc: segment of location {} already collected. table: {}, ident {}", - location.0, self.table_info.desc, self.table_info.ident, - ); - continue; - } - Err(e) => return Err(e), - Ok(v) => v, - }; - blocks.extend(location_tuple.block_location.into_iter()); - blooms.extend(location_tuple.bloom_location.into_iter()); - hlls.extend(location_tuple.hll_location.into_iter()); - } - } - - Ok(LocationTuple { - block_location: blocks, - bloom_location: blooms, - hll_location: hlls, - }) - } - - pub async fn list_snapshot_files(&self) -> Result> { - let prefix = format!( - "{}/{}/", - self.meta_location_generator().prefix(), - FUSE_TBL_SNAPSHOT_PREFIX, - ); - SnapshotsIO::list_files(self.get_operator(), &prefix, None).await - } -} - -#[derive(Default)] -pub struct LocationTuple { - pub block_location: HashSet, - pub bloom_location: HashSet, - pub hll_location: HashSet, -} - -impl TryFrom> for LocationTuple { - type Error = ErrorCode; - fn try_from(value: Arc) -> Result { - let mut block_location = HashSet::new(); - let mut bloom_location = HashSet::new(); - let mut hll_location = HashSet::new(); - let block_metas = value.block_metas()?; - for block_meta in block_metas.into_iter() { - block_location.insert(block_meta.location.0.clone()); - if let Some(bloom_loc) = &block_meta.bloom_filter_index_location { - bloom_location.insert(bloom_loc.0.clone()); - } - } - if let Some(loc) = value.as_ref().summary.additional_stats_loc() { - hll_location.insert(loc.0); - } - Ok(Self { - block_location, - bloom_location, - hll_location, - }) - } -} - -impl TryFrom> for LocationTuple { - type Error = ErrorCode; - fn try_from(value: Arc) -> Result { - let mut block_location = HashSet::new(); - let mut bloom_location = HashSet::new(); - let mut hll_location = HashSet::new(); - - let location_path = value.location_path_col(); - for path in location_path.iter() { - block_location.insert(path.to_string()); - } - - let (index, _) = value - .segment_schema - .column_with_name(BLOOM_FILTER_INDEX_LOCATION) - .unwrap(); - let column = value.block_metas.get_by_offset(index).to_column(); - for value in column.iter() { - if let ScalarRef::Tuple(values) = value { - let path = values[0].as_string().unwrap(); - bloom_location.insert(path.to_string()); - } - } - - if let Some(loc) = value.as_ref().summary.additional_stats_loc() { - hll_location.insert(loc.0); - } - Ok(Self { - block_location, - bloom_location, - hll_location, - }) - } -} - -#[derive(Debug)] -struct PurgeCounter { - start: Instant, - blocks: usize, - agg_indexes: usize, - inverted_indexes: usize, - blooms: usize, - hlls: usize, - segments: usize, - table_statistics: usize, - snapshots: usize, -} - -impl PurgeCounter { - fn new() -> Self { - Self { - start: Instant::now(), - blocks: 0, - agg_indexes: 0, - inverted_indexes: 0, - blooms: 0, - hlls: 0, - segments: 0, - table_statistics: 0, - snapshots: 0, - } - } -} diff --git a/src/query/storages/fuse/src/operations/mod.rs b/src/query/storages/fuse/src/operations/mod.rs index 6a392c6f895..40dc8090757 100644 --- a/src/query/storages/fuse/src/operations/mod.rs +++ b/src/query/storages/fuse/src/operations/mod.rs @@ -19,7 +19,6 @@ mod changes; mod commit; mod common; mod compact; -mod gc; mod inverted_index; mod merge; mod merge_into; diff --git a/src/query/storages/fuse/src/operations/navigate.rs b/src/query/storages/fuse/src/operations/navigate.rs index 9419226ed41..8e803c863ad 100644 --- a/src/query/storages/fuse/src/operations/navigate.rs +++ b/src/query/storages/fuse/src/operations/navigate.rs @@ -39,12 +39,10 @@ use databend_storages_common_table_meta::table::OPT_KEY_CLUSTER_TYPE; use databend_storages_common_table_meta::table::OPT_KEY_SNAPSHOT_LOCATION; use databend_storages_common_table_meta::table::OPT_KEY_SOURCE_TABLE_ID; use futures::TryStreamExt; -use log::info; use opendal::EntryMode; use crate::FUSE_TBL_SNAPSHOT_PREFIX; use crate::FuseTable; -use crate::fuse_table::RetentionPolicy; use crate::io::MetaReaders; use crate::io::SnapshotHistoryReader; use crate::io::SnapshotsIO; @@ -131,20 +129,6 @@ impl FuseTable { .await } - pub async fn navigate_back_with_limit( - &self, - ctx: &Arc, - location: String, - limit: usize, - ) -> Result> { - let mut counter = 0; - self.find(ctx, location, |_snapshot| { - counter += 1; - counter >= limit - }) - .await - } - #[async_backtrace::framed] async fn navigate_to_snapshot( &self, @@ -411,198 +395,6 @@ impl FuseTable { } } - #[async_backtrace::framed] - pub async fn navigate_for_purge( - &self, - ctx: &Arc, - navigation_point: Option, - ) -> Result<(Arc, Vec)> { - let retention_policy = self.get_data_retention_policy(ctx.as_ref())?; - let root_snapshot = if let Some(snapshot) = self.read_table_snapshot().await? { - snapshot - } else { - return Err(ErrorCode::TableHistoricalDataNotFound( - "No historical data found at given point", - )); - }; - - assert!(root_snapshot.timestamp.is_some()); - - match retention_policy { - RetentionPolicy::ByTimePeriod(time_delta) => { - info!("navigate by time period, {:?}", time_delta); - let mut time_point = root_snapshot.timestamp.unwrap() - time_delta; - let (candidate_snapshot_path, files) = match navigation_point { - Some(NavigationPoint::TimePoint(point)) => { - time_point = std::cmp::min(point, time_point); - self.list_by_time_point(time_point).await - } - Some(NavigationPoint::SnapshotID(snapshot_id)) => { - self.list_by_snapshot_id(snapshot_id.as_str(), time_point) - .await - } - Some(NavigationPoint::StreamInfo(info)) => { - self.list_by_stream(info, time_point).await - } - Some(NavigationPoint::TableTag(tag_name)) => { - let snapshot_loc = self.get_tag_snapshot_location(ctx, &tag_name).await?; - self.list_by_location(snapshot_loc, time_point).await - } - None => self.list_by_time_point(time_point).await, - }?; - - let table = self - .navigate_to_time_point(ctx, candidate_snapshot_path, time_point) - .await?; - - Ok((table, files)) - } - RetentionPolicy::ByNumOfSnapshotsToKeep(num) => { - assert!(num > 0); - info!("navigate by number of snapshots, {:?}", num); - let table = self - .navigate_back_with_limit(ctx, self.snapshot_loc().unwrap(), num) - .await?; - - // Safe to unwrap: table snapshot and snapshot timestamp exist, otherwise we should not be here - let timestamp = table - .read_table_snapshot() - .await? - .unwrap() - .timestamp - .unwrap(); - - let (_candidate_snapshot_path, files) = self.list_by_time_point(timestamp).await?; - - Ok((table, files)) - } - } - } - - #[async_backtrace::framed] - pub async fn list_by_time_point( - &self, - time_point: DateTime, - ) -> Result<(String, Vec)> { - let Some(location) = self.snapshot_loc() else { - return Err(ErrorCode::TableHistoricalDataNotFound("No historical data")); - }; - - let prefix = self.snapshot_prefix(); - - let files = self - .list_files(prefix, |_, modified| modified <= time_point) - .await?; - if files.is_empty() { - return Err(ErrorCode::TableHistoricalDataNotFound( - "No historical data found at given point", - )); - } - - Ok((location, files)) - } - - #[async_backtrace::framed] - pub async fn list_by_snapshot_id( - &self, - snapshot_id: &str, - retention_point: DateTime, - ) -> Result<(String, Vec)> { - // TODO(Sky): unify location related logic into a single place - let mut location = None; - let prefix = self.snapshot_prefix(); - let prefix_loc = format!("{}{}", prefix, snapshot_id); - let prefix_loc_v5 = format!("{}{}{}", prefix, VACUUM2_OBJECT_KEY_PREFIX, snapshot_id); - - let files = self - .list_files(prefix, |loc, modified| { - if loc.starts_with(&prefix_loc) || loc.starts_with(&prefix_loc_v5) { - location = Some(loc); - } - modified <= retention_point - }) - .await?; - let location = location.ok_or_else(|| { - ErrorCode::TableHistoricalDataNotFound("No historical data found at given point") - })?; - Ok((location, files)) - } - - #[async_backtrace::framed] - async fn list_by_stream( - &self, - stream_info: TableInfo, - retention_point: DateTime, - ) -> Result<(String, Vec)> { - let snapshot_loc = self - .stream_snapshot_location(&stream_info)? - .ok_or_else(|| { - ErrorCode::TableHistoricalDataNotFound("No historical data found at given point") - })?; - self.list_by_location(snapshot_loc, retention_point).await - } - - #[async_backtrace::framed] - async fn list_by_location( - &self, - snapshot_loc: String, - retention_point: DateTime, - ) -> Result<(String, Vec)> { - let mut found = false; - let prefix = self.snapshot_prefix(); - - let files = self - .list_files(prefix, |loc, modified| { - if loc == snapshot_loc { - found = true; - } - modified <= retention_point - }) - .await?; - - if !found { - return Err(ErrorCode::TableHistoricalDataNotFound( - "No historical data found at given point", - )); - } - Ok((snapshot_loc, files)) - } - - #[async_backtrace::framed] - pub async fn list_files(&self, prefix: String, mut f: F) -> Result> - where F: FnMut(String, DateTime) -> bool { - let mut file_list = vec![]; - let op = self.operator.clone(); - let mut ds = op.lister_with(&prefix).await?; - while let Some(de) = ds.try_next().await? { - let meta = de.metadata(); - match meta.mode() { - EntryMode::FILE => { - let modified = if let Some(v) = meta.last_modified() { - Some(v) - } else { - let meta = op.stat(de.path()).await?; - meta.last_modified() - }; - - let location = de.path().to_string(); - if let Some(modified) = modified { - if f(location.clone(), modified) { - file_list.push((location, modified)); - } - } - } - _ => { - continue; - } - } - } - - file_list.sort_by(|(_, m1), (_, m2)| m2.cmp(m1)); - - Ok(file_list.into_iter().map(|v| v.0).collect()) - } - #[fastrace::trace] #[async_backtrace::framed] pub async fn navigate_to_location( diff --git a/src/query/storages/fuse/src/operations/vacuum.rs b/src/query/storages/fuse/src/operations/vacuum.rs index da5042957df..f0d1c01e56e 100644 --- a/src/query/storages/fuse/src/operations/vacuum.rs +++ b/src/query/storages/fuse/src/operations/vacuum.rs @@ -22,10 +22,12 @@ use chrono::DateTime; use chrono::Duration; use chrono::TimeDelta; use chrono::Utc; +use databend_common_catalog::catalog::Catalog; use databend_common_catalog::table::TableExt; use databend_common_catalog::table_context::TableContext; use databend_common_exception::ErrorCode; use databend_common_exception::Result; +use databend_common_meta_app::schema::DropTableTagReq; use databend_common_meta_app::schema::LeastVisibleTime; use databend_common_meta_app::schema::ListTableTagsReq; use databend_common_meta_app::schema::TableInfo; @@ -192,6 +194,61 @@ pub async fn is_gc_candidate_segment_block( } impl FuseTable { + /// Protect live table-tag references from VACUUM2 and remove expired tags. + pub async fn protect_table_tag_references( + &self, + catalog: &Arc, + root_snapshot_location: &str, + snapshot_files_to_gc: &mut Vec, + protected_segments: &mut HashSet, + ) -> Result<()> { + let now = Utc::now(); + let tags = catalog + .list_table_tags(ListTableTagsReq { + table_id: self.get_id(), + include_expired: true, + }) + .await?; + + let mut protected_snapshot_locs = HashSet::new(); + for (tag_name, seq_tag) in tags { + if seq_tag + .data + .expire_at + .is_some_and(|expire_at| expire_at <= now) + { + if let Err(error) = catalog + .drop_table_tag(DropTableTagReq { + table_id: self.get_id(), + tag_name, + seq: Some(seq_tag.seq), + }) + .await + { + warn!( + "drop expired tag failed, ignored, table: {}, err: {}", + self.table_info.desc, error + ); + } + continue; + } + + let tag_snapshot_loc = seq_tag.data.snapshot_loc; + if tag_snapshot_loc.as_str() < root_snapshot_location { + if let Some(snapshot) = + SnapshotsIO::read_snapshot_for_vacuum(self.get_operator(), &tag_snapshot_loc) + .await? + { + protected_segments.extend(snapshot.segments.iter().cloned()); + } + protected_snapshot_locs.insert(tag_snapshot_loc); + } + } + + snapshot_files_to_gc.retain(|path| !protected_snapshot_locs.contains(path)); + Ok(()) + } + pub async fn vacuum_table( &self, ctx: Arc, @@ -229,7 +286,7 @@ impl FuseTable { /// List files until a specific timestamp /// /// This implementation uses UUID v7 timestamp extraction for precise filtering. - /// Used by both do_vacuum and do_vacuum2. + /// Used by vacuum2. pub async fn list_files_until_timestamp( &self, path: &str, diff --git a/src/query/storages/hive/hive/src/hive_table.rs b/src/query/storages/hive/hive/src/hive_table.rs index bb2af5d8ab2..32a1059b003 100644 --- a/src/query/storages/hive/hive/src/hive_table.rs +++ b/src/query/storages/hive/hive/src/hive_table.rs @@ -25,7 +25,6 @@ use databend_common_catalog::plan::Partitions; use databend_common_catalog::plan::PartitionsShuffleKind; use databend_common_catalog::plan::PushDownInfo; use databend_common_catalog::table::DistributionLevel; -use databend_common_catalog::table::NavigationPoint; use databend_common_catalog::table::Table; use databend_common_catalog::table::TableStatistics; use databend_common_catalog::table_args::TableArgs; @@ -469,17 +468,6 @@ impl Table for HiveTable { ))) } - #[async_backtrace::framed] - async fn purge( - &self, - _ctx: Arc, - _instant: Option, - _limit: Option, - _dry_run: bool, - ) -> Result>> { - Ok(None) - } - async fn table_statistics( &self, _ctx: Arc, diff --git a/tests/longrun/example/background.sh b/tests/longrun/example/background.sh index 147505b137a..4531fc42349 100644 --- a/tests/longrun/example/background.sh +++ b/tests/longrun/example/background.sh @@ -7,6 +7,6 @@ for ((i=0; i<=3; i++)); do sleep 5 log_command bendsql --dsn \"$DSN\" -q \"OPTIMIZE TABLE example COMPACT LIMIT 10\" sleep 5 - log_command bendsql --dsn \"$DSN\" -q \"OPTIMIZE TABLE example purge \" + log_command bendsql --dsn \"$DSN\" -q \"VACUUM TABLE example\" sleep 5 -done \ No newline at end of file +done diff --git a/tests/sqllogictests/suites/base/09_fuse_engine/09_0008_fuse_optimize_table.test b/tests/sqllogictests/suites/base/09_fuse_engine/09_0008_fuse_optimize_table.test index c0e89fe329d..596456e47b4 100644 --- a/tests/sqllogictests/suites/base/09_fuse_engine/09_0008_fuse_optimize_table.test +++ b/tests/sqllogictests/suites/base/09_fuse_engine/09_0008_fuse_optimize_table.test @@ -49,26 +49,9 @@ select column_name, column_type, row_count from fuse_column('db_09_0008', 't') ---- a UInt64 3 -statement ok -optimize table `t` purge - -query I -select count(*) from fuse_snapshot('db_09_0008', 't') ----- -5 - statement ok set data_retention_time_in_days = 0 -statement ok -optimize table `t` purge - -# Flaky Tests -# query B -# select count(*)<5 from fuse_snapshot('db_09_0008', 't') -# ---- -# 1 - query I select * from t order by a ---- @@ -96,10 +79,10 @@ select * from t order by a 10 statement ok -optimize table `t` all +optimize table `t` compact query II -select segment_count,block_count from fuse_snapshot('db_09_0008', 't') limit 2 +select segment_count,block_count from fuse_snapshot('db_09_0008', 't') limit 1 ---- 1 1 @@ -543,13 +526,10 @@ select count() from fuse_snapshot('db_09_0008', 't10') ---- 6 -statement ok -optimize table t10 purge limit 2 - query I -select count() from fuse_snapshot('db_09_0008', 't10') +select count() from t10 ---- -4 +9 diff --git a/tests/sqllogictests/suites/ee/05_ee_ddl/05_0017_ddl_materialized_view.test b/tests/sqllogictests/suites/ee/05_ee_ddl/05_0017_ddl_materialized_view.test index 3ae7f81d6cc..a95cd46a623 100644 --- a/tests/sqllogictests/suites/ee/05_ee_ddl/05_0017_ddl_materialized_view.test +++ b/tests/sqllogictests/suites/ee/05_ee_ddl/05_0017_ddl_materialized_view.test @@ -313,9 +313,6 @@ OPTIMIZE TABLE test_mv COMPACT SEGMENT statement error 3905 OPTIMIZE TABLE test_mv PURGE -statement error 3905 -OPTIMIZE TABLE test_mv ALL - # Dedicated MV maintenance reuses Fuse plans without opening TABLE syntax on MVs. statement ok ALTER MATERIALIZED VIEW test_mv CLUSTER BY (id) diff --git a/tests/sqllogictests/suites/task/task_ddl_transaction_test.test b/tests/sqllogictests/suites/task/task_ddl_transaction_test.test index 5dfbb6597be..caf0c0c75bb 100644 --- a/tests/sqllogictests/suites/task/task_ddl_transaction_test.test +++ b/tests/sqllogictests/suites/task/task_ddl_transaction_test.test @@ -24,7 +24,7 @@ CREATE TASK transactionTask query TTT? select name, warehouse, schedule, SPLIT(definition, '\n') from system.tasks where name = 'transactionTask' ---- -transactionTask mywh CRON 0 0 0 1 1 ? 2100 ["BEGIN","SELECT 1;","BEGIN;","DELETE FROM t WHERE c = ';';","VACUUM TABLE t ;","MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN UPDATE *;","COMMIT;","END;"] +transactionTask mywh CRON 0 0 0 1 1 ? 2100 ["BEGIN","SELECT 1;","BEGIN;","DELETE FROM t WHERE c = ';';","VACUUM TABLE t;","MERGE INTO t USING s ON t.id = s.id WHEN MATCHED THEN UPDATE *;","COMMIT;","END;"] query T select state from system.tasks where name = 'transactionTask' diff --git a/tests/suites/0_stateless/17_altertable/17_0002_alter_table_purge_before.result b/tests/suites/0_stateless/17_altertable/17_0002_alter_table_purge_before.result deleted file mode 100644 index ebd3f748ca9..00000000000 --- a/tests/suites/0_stateless/17_altertable/17_0002_alter_table_purge_before.result +++ /dev/null @@ -1,31 +0,0 @@ -2 -1 -1 -checking that there should are 3 snapshots before purge -true -alter table add a column -checking that after purge (by snapshot id) there should be 3 snapshots left -true -checking that after purge (by snapshot id) there should be 4 rows left -true -alter table drop a column -checking that after purge (by snapshot id) there should be 4 snapshots left -true -checking that after purge (by snapshot id) there should be 4 rows left -true -2 -1 -1 -1 -checking that there should are 4 snapshots before purge -true -alter table add a column -checking that after purge (by timestamp) there should be at least 2 snapshots left -true -checking that after purge (by timestamp) there should be 5 rows left -true -alter table drop a column -checking that after purge (by timestamp) there should be at least 2 snapshots left -true -checking that after purge (by timestamp) there should be 5 rows left -true diff --git a/tests/suites/0_stateless/17_altertable/17_0002_alter_table_purge_before.sh b/tests/suites/0_stateless/17_altertable/17_0002_alter_table_purge_before.sh deleted file mode 100755 index e849a7819cf..00000000000 --- a/tests/suites/0_stateless/17_altertable/17_0002_alter_table_purge_before.sh +++ /dev/null @@ -1,95 +0,0 @@ -#!/usr/bin/env bash - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. "$CURDIR"/../../../shell_env.sh - - - -# PURGE BEFORE SNAPSHOT - -## Setup -echo "create table t17_0002(c int not null)" | bendsql_connect_root -## - 1st snapshot contains 2 rows, 1 block, 1 segment -echo "insert into t17_0002 values(1),(2)" | bendsql_connect_root -## - 2nd snapshot contains 3 rows, 2 blocks, 2 segments -echo "insert into t17_0002 values(3)" | bendsql_connect_root -## - 3rd snapshot contains 4 rows, 3 blocks, 3 segments -echo "insert into t17_0002 values(4)" | bendsql_connect_root - -echo "checking that there should are 3 snapshots before purge" -echo "select count(*)=3 from fuse_snapshot('default', 't17_0002')" | bendsql_connect_root - -## location the id of 2nd snapshot -SNAPSHOT_ID=$(echo "select snapshot_id from fuse_snapshot('default','t17_0002') where row_count=3 limit 1" | bendsql_connect_root) -#TIMEPOINT=$(echo "select timestamp from fuse_snapshot('default', 't17_0002') where row_count=3" | bendsql_connect_root) - -# alter table add a column -echo "alter table add a column" -# alter table will generate a new snapshot -echo "alter table t17_0002 add column a float default 1.01" | bendsql_connect_root - -## verify -echo "set data_retention_time_in_days=0; optimize table t17_0002 purge before (snapshot => '$SNAPSHOT_ID')" | bendsql_connect_root -echo "checking that after purge (by snapshot id) there should be 3 snapshots left" -echo "select count(*)=3 from fuse_snapshot('default', 't17_0002')" | bendsql_connect_root -echo "checking that after purge (by snapshot id) there should be 4 rows left" -echo "select count(*)=4 from t17_0002" | bendsql_connect_root - -# alter table drop a column -echo "alter table drop a column" -# alter table will generate a new snapshot -echo "alter table t17_0002 drop column c" | bendsql_connect_root - -## verify -echo "set data_retention_time_in_days=0; optimize table t17_0002 purge before (snapshot => '$SNAPSHOT_ID')" | bendsql_connect_root -echo "checking that after purge (by snapshot id) there should be 4 snapshots left" -echo "select count(*)=4 from fuse_snapshot('default', 't17_0002')" | bendsql_connect_root -echo "checking that after purge (by snapshot id) there should be 4 rows left" -echo "select count(*)=4 from t17_0002" | bendsql_connect_root - -## Drop table. -echo "drop table t17_0002 all" | bendsql_connect_root - -# PURGE BEFORE TIMESTAMP - -## Setup -echo "create table t17_0002(c int not null)" | bendsql_connect_root -## - 1st snapshot contains 2 rows, 1 block, 1 segment -echo "insert into t17_0002 values(1),(2)" | bendsql_connect_root -## - 2nd snapshot contains 3 rows, 2 blocks, 2 segments -echo "insert into t17_0002 values(3)" | bendsql_connect_root -## - 3rd snapshot contains 4 rows, 3 blocks, 3 segments -echo "insert into t17_0002 values(4)" | bendsql_connect_root -## - 4rd snapshot contains 5 rows, 4 blocks, 4 segments -echo "insert into t17_0002 values(5)" | bendsql_connect_root - -echo "checking that there should are 4 snapshots before purge" -echo "select count(*)=4 from fuse_snapshot('default', 't17_0002')" | bendsql_connect_root - -## location the timestamp of latest snapshot -TIMEPOINT=$(echo "select timestamp from fuse_snapshot('default', 't17_0002') where row_count=5 limit 1" | bendsql_connect_root) - -# alter table add a column -echo "alter table add a column" -echo "alter table t17_0002 add column a float default 1.01" | bendsql_connect_root - -## verify -echo "set data_retention_time_in_days=0; optimize table t17_0002 purge before (TIMESTAMP => '$TIMEPOINT'::TIMESTAMP)" | bendsql_connect_root -echo "checking that after purge (by timestamp) there should be at least 2 snapshots left" -echo "select count(*)>=2 from fuse_snapshot('default', 't17_0002')" | bendsql_connect_root -echo "checking that after purge (by timestamp) there should be 5 rows left" -echo "select count(*)=5 from t17_0002" | bendsql_connect_root - -# alter table drop a column -echo "alter table drop a column" -echo "alter table t17_0002 drop column a" | bendsql_connect_root - -## verify -echo "set data_retention_time_in_days=0; optimize table t17_0002 purge before (TIMESTAMP => '$TIMEPOINT'::TIMESTAMP)" | bendsql_connect_root -echo "checking that after purge (by timestamp) there should be at least 2 snapshots left" -echo "select count(*)>=2 from fuse_snapshot('default', 't17_0002')" | bendsql_connect_root -echo "checking that after purge (by timestamp) there should be 5 rows left" -echo "select count(*)=5 from t17_0002" | bendsql_connect_root - -## Drop table. -echo "drop table t17_0002 all" | bendsql_connect_root diff --git a/tests/suites/0_stateless/18_rbac/18_0007_privilege_access.result b/tests/suites/0_stateless/18_rbac/18_0007_privilege_access.result index c6929040605..92460efa195 100644 --- a/tests/suites/0_stateless/18_rbac/18_0007_privilege_access.result +++ b/tests/suites/0_stateless/18_rbac/18_0007_privilege_access.result @@ -28,6 +28,8 @@ test -- insert overwrite 3 test -- optimize table Error: APIError: QueryFailed: [1063]Permission denied: privilege [Super] is required on 'default'.'default'.'t20_0012' for user 'test-user'@'%' with roles [public,test-role1,test-role2] +test -- vacuum drop table +Error: APIError: QueryFailed: [1063]Permission denied: privilege [Super] is required on *.* for user 'test-user'@'%' with roles [public,test-role1,test-role2]. Note: Please ensure that your current role have the appropriate permissions to create a new Object true === NETWORK_POLICY SETTING === Error: APIError: QueryFailed: [1063]Permission Denied: Setting of network_policy is restricted to account_admin role diff --git a/tests/suites/0_stateless/18_rbac/18_0007_privilege_access.sh b/tests/suites/0_stateless/18_rbac/18_0007_privilege_access.sh index 1e8d68577cf..3c107f12a38 100755 --- a/tests/suites/0_stateless/18_rbac/18_0007_privilege_access.sh +++ b/tests/suites/0_stateless/18_rbac/18_0007_privilege_access.sh @@ -105,12 +105,17 @@ select * from t20_0012 order by c; ## optimize table - test permission denied echo "select 'test -- optimize table'" | $TEST_USER_CONNECT -echo "optimize table t20_0012 all" | $TEST_USER_CONNECT +echo "optimize table t20_0012 compact" | $TEST_USER_CONNECT + +## global vacuum drop table - test permission denied +## This must require global SUPER instead of trying to resolve an empty database name. +echo "select 'test -- vacuum drop table'" | $TEST_USER_CONNECT +echo "vacuum drop table" | $TEST_USER_CONNECT ## grant user privilege and test optimize run_root_sql "GRANT Super ON *.* TO 'test-user';" run_test_user " -set data_retention_time_in_days=0; optimize table t20_0012 all; +optimize table t20_0012 compact; select count(*)>=1 from fuse_snapshot('default', 't20_0012'); " diff --git a/tests/suites/0_stateless/20+_others/20_0011_purge_before.result b/tests/suites/0_stateless/20+_others/20_0011_purge_before.result deleted file mode 100644 index c8bc057b7f5..00000000000 --- a/tests/suites/0_stateless/20+_others/20_0011_purge_before.result +++ /dev/null @@ -1,18 +0,0 @@ -2 -1 -1 -checking that there should are 3 snapshots before purge -true -checking that after purge (by snapshot id) there should be 2 snapshots left -true -checking that after purge (by snapshot id) there should be 4 rows left -true -2 -1 -1 -checking that there should are 3 snapshots before purge -true -checking that after purge (by timestamp) there should be 1 snapshot left -true -checking that after purge (by timestamp) there should be 4 rows left -true diff --git a/tests/suites/0_stateless/20+_others/20_0011_purge_before.sh b/tests/suites/0_stateless/20+_others/20_0011_purge_before.sh deleted file mode 100755 index 33445060126..00000000000 --- a/tests/suites/0_stateless/20+_others/20_0011_purge_before.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env bash - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. "$CURDIR"/../../../shell_env.sh - - - -# PURGE BEFORE SNAPSHOT - -## Setup -echo "create table t20_0011(c int not null)" | bendsql_connect_root -## - 1st snapshot contains 2 rows, 1 block, 1 segment -echo "insert into t20_0011 values(1),(2)" | bendsql_connect_root -## - 2nd snapshot contains 3 rows, 2 blocks, 2 segments -echo "insert into t20_0011 values(3)" | bendsql_connect_root -## - 3rd snapshot contains 4 rows, 3 blocks, 3 segments -echo "insert into t20_0011 values(4)" | bendsql_connect_root - -echo "checking that there should are 3 snapshots before purge" -echo "select count(*)=3 from fuse_snapshot('default', 't20_0011')" | bendsql_connect_root - -## location the id of 2nd snapshot -SNAPSHOT_ID=$(echo "select snapshot_id from fuse_snapshot('default','t20_0011') where row_count=3" | bendsql_connect_root) -#TIMEPOINT=$(echo "select timestamp from fuse_snapshot('default', 't20_0011') where row_count=3" | bendsql_connect_root) - -## verify -echo "set data_retention_time_in_days=0; optimize table t20_0011 purge before (snapshot => '$SNAPSHOT_ID')" | bendsql_connect_root -echo "checking that after purge (by snapshot id) there should be 2 snapshots left" -echo "select count(*)=2 from fuse_snapshot('default', 't20_0011')" | bendsql_connect_root -echo "checking that after purge (by snapshot id) there should be 4 rows left" -echo "select count(*)=4 from t20_0011" | bendsql_connect_root - -## Drop table. -echo "drop table t20_0011 all" | bendsql_connect_root - -# PURGE BEFORE TIMESTAMP - -## Setup -echo "create table t20_0011(c int not null)" | bendsql_connect_root -## - 1st snapshot contains 2 rows, 1 block, 1 segment -echo "insert into t20_0011 values(1),(2)" | bendsql_connect_root -## - 2nd snapshot contains 3 rows, 2 blocks, 2 segments -echo "insert into t20_0011 values(3)" | bendsql_connect_root -## - 3rd snapshot contains 4 rows, 3 blocks, 3 segments -echo "insert into t20_0011 values(4)" | bendsql_connect_root - -echo "checking that there should are 3 snapshots before purge" -echo "select count(*)=3 from fuse_snapshot('default', 't20_0011')" | bendsql_connect_root - -## location the timestamp of latest snapshot -TIMEPOINT=$(echo "select timestamp from fuse_snapshot('default', 't20_0011') where row_count=4" | bendsql_connect_root) - -## verify -echo "set data_retention_time_in_days=0; optimize table t20_0011 purge before (TIMESTAMP => '$TIMEPOINT'::TIMESTAMP)" | bendsql_connect_root -echo "checking that after purge (by timestamp) there should be 1 snapshot left" -echo "select count(*)=1 from fuse_snapshot('default', 't20_0011')" | bendsql_connect_root -echo "checking that after purge (by timestamp) there should be 4 rows left" -echo "select count(*)=4 from t20_0011" | bendsql_connect_root - -## Drop table. -echo "drop table t20_0011 all" | bendsql_connect_root diff --git a/tests/suites/5_ee/01_vacuum/01_0000_ee_vacuum.py b/tests/suites/5_ee/01_vacuum/01_0000_ee_vacuum.py index 0ed3b191b2c..9f7d989c130 100755 --- a/tests/suites/5_ee/01_vacuum/01_0000_ee_vacuum.py +++ b/tests/suites/5_ee/01_vacuum/01_0000_ee_vacuum.py @@ -37,7 +37,7 @@ def get_license(): def compact_data(name): mycursor = mydb.cursor() - mycursor.execute("optimize table gc_test all;") + mycursor.execute("optimize table gc_test compact;") if __name__ == "__main__": @@ -66,16 +66,6 @@ def compact_data(name): mycursor.execute("select a from gc_test order by a;") old_datas = mycursor.fetchall() - mycursor.execute("vacuum table gc_test dry run;") - datas = mycursor.fetchall() - print(datas) - - mycursor.execute("select a from gc_test order by a;") - datas = mycursor.fetchall() - - if old_datas != datas: - print("vacuum dry run lose data: %s : %s" % (old_datas, datas)) - client1.send("vacuum table gc_test;") client1.expect(prompt) diff --git a/tests/suites/5_ee/01_vacuum/01_0000_ee_vacuum.result b/tests/suites/5_ee/01_vacuum/01_0000_ee_vacuum.result index 98c6615c1ad..2fd4fba6a8f 100644 --- a/tests/suites/5_ee/01_vacuum/01_0000_ee_vacuum.result +++ b/tests/suites/5_ee/01_vacuum/01_0000_ee_vacuum.result @@ -1,2 +1 @@ -[] vacuum success diff --git a/tests/suites/5_ee/01_vacuum/01_0002_ee_vacuum_drop_table.result b/tests/suites/5_ee/01_vacuum/01_0002_ee_vacuum_drop_table.result index 138ca96ac53..1d9fd47b613 100644 --- a/tests/suites/5_ee/01_vacuum/01_0002_ee_vacuum_drop_table.result +++ b/tests/suites/5_ee/01_vacuum/01_0002_ee_vacuum_drop_table.result @@ -4,7 +4,5 @@ 4 888 1024 -1 -1 2 2 diff --git a/tests/suites/5_ee/01_vacuum/01_0002_ee_vacuum_drop_table.sh b/tests/suites/5_ee/01_vacuum/01_0002_ee_vacuum_drop_table.sh index 5765b19e72c..6171f676bae 100755 --- a/tests/suites/5_ee/01_vacuum/01_0002_ee_vacuum_drop_table.sh +++ b/tests/suites/5_ee/01_vacuum/01_0002_ee_vacuum_drop_table.sh @@ -3,23 +3,23 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../../../shell_env.sh -## test vacuum drop table dry run output -echo "drop database if exists test_vacuum_drop_dry_run" | bendsql_connect_root -echo "CREATE DATABASE test_vacuum_drop_dry_run" | bendsql_connect_root -echo "create table test_vacuum_drop_dry_run.a(c int)" | bendsql_connect_root -echo "INSERT INTO test_vacuum_drop_dry_run.a VALUES (1)" | bendsql_connect_root_null -echo "drop table test_vacuum_drop_dry_run.a" | bendsql_connect_root -count=$(echo "set data_retention_time_in_days=0; vacuum drop table dry run" | bendsql_connect_root | wc -l) -if [[ ! "$count" ]]; then - echo "vacuum drop table dry run, count:$count" +## test vacuum drop table empty output + +echo "drop database if exists test_vacuum_drop_empty_output" | bendsql_connect_root +echo "CREATE DATABASE test_vacuum_drop_empty_output" | bendsql_connect_root +echo "create table test_vacuum_drop_empty_output.a(c int)" | bendsql_connect_root +echo "INSERT INTO test_vacuum_drop_empty_output.a VALUES (1)" | bendsql_connect_root_null +echo "drop table test_vacuum_drop_empty_output.a" | bendsql_connect_root +output=$(echo "set data_retention_time_in_days=0; vacuum drop table from test_vacuum_drop_empty_output" | bendsql_connect_root) +if [[ -n "$output" ]]; then + echo "vacuum drop table returned unexpected output: $output" exit 1 fi -count=$(echo "set data_retention_time_in_days=0; vacuum drop table dry run summary" | bendsql_connect_root | wc -l) -if [[ ! "$count" ]]; then - echo "vacuum drop table dry run summary, count:$count" +if echo "undrop table test_vacuum_drop_empty_output.a" | bendsql_connect_root > /dev/null 2>&1; then + echo "vacuumed table should not be recoverable" exit 1 fi -echo "drop database if exists test_vacuum_drop_dry_run" | bendsql_connect_root +echo "drop database if exists test_vacuum_drop_empty_output" | bendsql_connect_root ## Setup echo "drop database if exists test_vacuum_drop" | bendsql_connect_root @@ -97,15 +97,7 @@ echo "drop table table_drop_external_location;" | bendsql_connect_root echo "set data_retention_time_in_days=0;vacuum drop table" | bendsql_connect_root > /dev/null -## dry run echo "CREATE DATABASE test_vacuum_drop_4" | bendsql_connect_root -echo "create table test_vacuum_drop_4.a(c int)" | bendsql_connect_root -echo "INSERT INTO test_vacuum_drop_4.a VALUES (1)" | bendsql_connect_root_null -echo "select * from test_vacuum_drop_4.a" | bendsql_connect_root -echo "drop table test_vacuum_drop_4.a" | bendsql_connect_root -echo "set data_retention_time_in_days=0;vacuum drop table dry run" | bendsql_connect_root > /dev/null -echo "undrop table test_vacuum_drop_4.a" | bendsql_connect_root -echo "select * from test_vacuum_drop_4.a" | bendsql_connect_root # check vacuum drop table with the same name echo "create table test_vacuum_drop_4.b(c int)" | bendsql_connect_root @@ -117,17 +109,17 @@ echo "select * from test_vacuum_drop_4.b" | bendsql_connect_root echo "set data_retention_time_in_days=0; vacuum drop table" | bendsql_connect_root > /dev/null echo "select * from test_vacuum_drop_4.b" | bendsql_connect_root -## test vacuum table output +## test vacuum table empty output echo "create table test_vacuum_drop_4.c(c int)" | bendsql_connect_root echo "INSERT INTO test_vacuum_drop_4.c VALUES (1),(2)" | bendsql_connect_root_null -count=$(echo "set data_retention_time_in_days=0; vacuum table test_vacuum_drop_4.c" | bendsql_connect_root | awk '{print $9}') -if [[ "$count" != "4" ]]; then - echo "vacuum table, count:$count" +output=$(echo "set data_retention_time_in_days=0; vacuum table test_vacuum_drop_4.c" | bendsql_connect_root) +if [[ -n "$output" ]]; then + echo "vacuum table returned unexpected output: $output" exit 1 fi -count=$(echo "set data_retention_time_in_days=0; vacuum table test_vacuum_drop_4.c dry run summary" | bendsql_connect_root | wc -l) -if [[ "$count" != "1" ]]; then - echo "vacuum table dry run summary, count:$count" +row_count=$(echo "select count(*) from test_vacuum_drop_4.c" | bendsql_connect_root) +if [[ "$row_count" != "2" ]]; then + echo "vacuum table changed live rows, count:$row_count" exit 1 fi diff --git a/tests/suites/5_ee/01_vacuum/01_003_vacuum_table_only_orphans.result b/tests/suites/5_ee/01_vacuum/01_003_vacuum_table_only_orphans.result deleted file mode 100644 index 6d6a0335784..00000000000 --- a/tests/suites/5_ee/01_vacuum/01_003_vacuum_table_only_orphans.result +++ /dev/null @@ -1,32 +0,0 @@ ->>>> create or replace database test_vacuum_table_only_orphans ->>>> create or replace table test_vacuum_table_only_orphans.a(c int) 'fs:///tmp/test_vacuum_table_only_orphans/' ->>>> insert into test_vacuum_table_only_orphans.a values (1),(2) -2 ->>>> insert into test_vacuum_table_only_orphans.a values (2),(3) -2 ->>>> insert into test_vacuum_table_only_orphans.a values (3),(4) -2 -before purge -4 -4 -4 ->>>> truncate table test_vacuum_table_only_orphans.a ->>>> set data_retention_time_in_days=0; optimize table test_vacuum_table_only_orphans.a purge -after purge -1 -1 -1 -after add pure orphan files -4 -4 -4 -after vacuum -4 -4 -4 ->>>> insert into test_vacuum_table_only_orphans.a values (1) -1 -after vacuum -2 -2 -1 diff --git a/tests/suites/5_ee/01_vacuum/01_003_vacuum_table_only_orphans.sh b/tests/suites/5_ee/01_vacuum/01_003_vacuum_table_only_orphans.sh deleted file mode 100755 index 23813d498d9..00000000000 --- a/tests/suites/5_ee/01_vacuum/01_003_vacuum_table_only_orphans.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env bash - -CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -. "$CURDIR"/../../../shell_env.sh - -stmt "create or replace database test_vacuum_table_only_orphans" - -mkdir -p /tmp/test_vacuum_table_only_orphans/ - -stmt "create or replace table test_vacuum_table_only_orphans.a(c int) 'fs:///tmp/test_vacuum_table_only_orphans/'" - - -stmt "insert into test_vacuum_table_only_orphans.a values (1),(2)" -stmt "insert into test_vacuum_table_only_orphans.a values (2),(3)" -stmt "insert into test_vacuum_table_only_orphans.a values (3),(4)" - -SNAPSHOT_LOCATION=$(echo "select snapshot_location from fuse_snapshot('test_vacuum_table_only_orphans','a') limit 1" | bendsql_connect_root) -PREFIX=$(echo "$SNAPSHOT_LOCATION" | cut -d'/' -f1-2) - -echo "before purge" - -ls -l /tmp/test_vacuum_table_only_orphans/"$PREFIX"/_b/ | wc -l -ls -l /tmp/test_vacuum_table_only_orphans/"$PREFIX"/_sg/ | wc -l -ls -l /tmp/test_vacuum_table_only_orphans/"$PREFIX"/_i_b_v2/ | wc -l - - -stmt "truncate table test_vacuum_table_only_orphans.a" - -stmt "set data_retention_time_in_days=0; optimize table test_vacuum_table_only_orphans.a purge" - -echo "after purge" - -ls -l /tmp/test_vacuum_table_only_orphans/"$PREFIX"/_b/ | wc -l -ls -l /tmp/test_vacuum_table_only_orphans/"$PREFIX"/_sg/ | wc -l -ls -l /tmp/test_vacuum_table_only_orphans/"$PREFIX"/_i_b_v2/ | wc -l - - -# simulates orphans -touch /tmp/test_vacuum_table_only_orphans/"$PREFIX"/_b/o1 -touch /tmp/test_vacuum_table_only_orphans/"$PREFIX"/_b/o2 -touch /tmp/test_vacuum_table_only_orphans/"$PREFIX"/_b/o3 - -touch /tmp/test_vacuum_table_only_orphans/"$PREFIX"/_sg/sg1 -touch /tmp/test_vacuum_table_only_orphans/"$PREFIX"/_sg/sg2 -touch /tmp/test_vacuum_table_only_orphans/"$PREFIX"/_sg/sg3 - -touch /tmp/test_vacuum_table_only_orphans/"$PREFIX"/_i_b_v2/bf1 -touch /tmp/test_vacuum_table_only_orphans/"$PREFIX"/_i_b_v2/bf2 -touch /tmp/test_vacuum_table_only_orphans/"$PREFIX"/_i_b_v2/bf3 - -echo "after add pure orphan files" - -ls -l /tmp/test_vacuum_table_only_orphans/"$PREFIX"/_b/ | wc -l -ls -l /tmp/test_vacuum_table_only_orphans/"$PREFIX"/_sg/ | wc -l -ls -l /tmp/test_vacuum_table_only_orphans/"$PREFIX"/_i_b_v2/ | wc -l - - -stmt "set data_retention_time_in_days=0; vacuum table test_vacuum_table_only_orphans.a" > /dev/null - -echo "after vacuum" - -ls -l /tmp/test_vacuum_table_only_orphans/"$PREFIX"/_b/ | wc -l -ls -l /tmp/test_vacuum_table_only_orphans/"$PREFIX"/_sg/ | wc -l -ls -l /tmp/test_vacuum_table_only_orphans/"$PREFIX"/_i_b_v2/ | wc -l - -stmt "insert into test_vacuum_table_only_orphans.a values (1)" -stmt "set data_retention_time_in_days=0; vacuum table test_vacuum_table_only_orphans.a" > /dev/null - -echo "after vacuum" - -ls -l /tmp/test_vacuum_table_only_orphans/"$PREFIX"/_b/ | wc -l -ls -l /tmp/test_vacuum_table_only_orphans/"$PREFIX"/_sg/ | wc -l -ls -l /tmp/test_vacuum_table_only_orphans/"$PREFIX"/_i_b_v2/ | wc -l diff --git a/tests/suites/5_ee/05_stream/05_0002_ee_stream_create.result b/tests/suites/5_ee/05_stream/05_0002_ee_stream_create.result index 0f45b47a080..dad96731107 100644 --- a/tests/suites/5_ee/05_stream/05_0002_ee_stream_create.result +++ b/tests/suites/5_ee/05_stream/05_0002_ee_stream_create.result @@ -7,5 +7,3 @@ Error: APIError: QueryFailed: [2733]Change tracking has been missing for the tim Error: APIError: QueryFailed: [2733]Change tracking has been missing for the time range requested on table 'db_stream'.'base' Error: APIError: QueryFailed: [2733]Change tracking has been missing for the time range requested on table 'db_stream'.'base' Error: APIError: QueryFailed: [2733]Change tracking has been missing for the time range requested on table 'db_stream'.'base' -3 -2 diff --git a/tests/suites/5_ee/05_stream/05_0002_ee_stream_create.sh b/tests/suites/5_ee/05_stream/05_0002_ee_stream_create.sh index fb36f08be25..bfde7b592de 100755 --- a/tests/suites/5_ee/05_stream/05_0002_ee_stream_create.sh +++ b/tests/suites/5_ee/05_stream/05_0002_ee_stream_create.sh @@ -35,10 +35,6 @@ echo "alter table db_stream.base set options(change_tracking = true)" | bendsql_ echo "create stream db_stream.s4 on table db_stream.base at (stream => db_stream.s2)" | bendsql_connect_root echo "select a from db_stream.s2" | bendsql_connect_root -echo "select count(*) from fuse_snapshot('db_stream', 'base')" | bendsql_connect_root -echo "set data_retention_time_in_days=0; optimize table db_stream.base purge before (stream => db_stream.s2)" | bendsql_connect_root -echo "select count(*) from fuse_snapshot('db_stream', 'base')" | bendsql_connect_root - echo "drop stream if exists db_stream.s2" | bendsql_connect_root echo "drop stream if exists db_stream.t2" | bendsql_connect_root echo "drop table if exists db_stream.base all" | bendsql_connect_root