From 44a8bf74f324ea6858cdc713bd96f232cefc63ad Mon Sep 17 00:00:00 2001 From: ddeleo Date: Thu, 2 Sep 2021 11:26:29 -0400 Subject: [PATCH 1/2] Add cancel_jobs.sql and delete_datasets.sql stored procedures --- stored_procedures/README.md | 28 +++++++++++++++- stored_procedures/cancel_jobs.sql | 47 +++++++++++++++++++++++++++ stored_procedures/delete_datasets.sql | 44 +++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 stored_procedures/cancel_jobs.sql create mode 100644 stored_procedures/delete_datasets.sql diff --git a/stored_procedures/README.md b/stored_procedures/README.md index 29d86e64b..6b66efc15 100644 --- a/stored_procedures/README.md +++ b/stored_procedures/README.md @@ -14,12 +14,38 @@ CALL bqutil.procedure.GetNextIds(10, next_ids); ## Stored Procedures -* [GetNextIds](#GetNextIds) +* [cancel_jobs](#cancel_jobscancel_jobssql) + +* [delete_datasets](#delete_datasetsproject_id-string-dataset_like_filter-stringdelete_datasetssql) + +* [GetNextIds](#getnextidsid_count-int64-out-next_ids-arrayint64get_next_idsql) ## Documentation +### [cancel_jobs(project_id STRING, job_id_like_filter STRING)](cancel_jobs.sql) + +Cancels all running or pending jobs which match the input argument, job_id_like_filter. + +```sql +# Cancel jobs which match a job_id prefix +CALL bqutil.procedure.cancel_jobs("YOUR_PROJECT_ID", "job_id_prefix%"); +# Cancel jobs which match a job_id suffix +CALL bqutil.procedure.cancel_jobs("YOUR_PROJECT_ID", "%job_id_suffix"); +``` + +### [delete_datasets(project_id STRING, dataset_like_filter STRING)](delete_datasets.sql) + +Deletes all BigQuery datasets which match the input argument, dataset_like_filter. +```sql +# Delete datasets which match a dataset_id prefix +CALL bqutil.procedure.delete_datasets("YOUR_PROJECT_ID", "dataset_id_prefix%"); +# Delete datasets which match a dataset_id suffix +CALL bqutil.procedure.delete_datasets("YOUR_PROJECT_ID", "%dataset_id_suffix"); +``` + ### [GetNextIds(id_count INT64, OUT next_ids ARRAY)](get_next_id.sql) Generates next ids and inserts them into a sample table. This implementation prevents against race condition. + ```sql BEGIN DECLARE next_ids ARRAY DEFAULT []; diff --git a/stored_procedures/cancel_jobs.sql b/stored_procedures/cancel_jobs.sql new file mode 100644 index 000000000..4f5ffe669 --- /dev/null +++ b/stored_procedures/cancel_jobs.sql @@ -0,0 +1,47 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +CREATE OR REPLACE PROCEDURE bqutil.procedure.cancel_jobs (project_id STRING, job_id_like_filter STRING) +BEGIN + DECLARE job_ids ARRAY; + DECLARE i INT64; + + EXECUTE IMMEDIATE + FORMAT(""" + (SELECT + ARRAY_AGG(job_id) + FROM + `%s`.`region-us`.INFORMATION_SCHEMA.JOBS_BY_PROJECT + WHERE + state IN ("RUNNING","PENDING") + AND job_id LIKE "%s" + AND job_id <> @@script.job_id); + """, project_id, job_id_like_filter) + INTO job_ids; + + SET i = ARRAY_LENGTH(job_ids); + WHILE i > 0 DO + BEGIN + CALL BQ.JOBS.CANCEL(project_id || '.' || job_ids[ORDINAL(i)]); + SET i = i - 1; + EXCEPTION WHEN ERROR THEN + SET i = i - 1; + SELECT @@error.message; + CONTINUE; + END; + END WHILE; +END; \ No newline at end of file diff --git a/stored_procedures/delete_datasets.sql b/stored_procedures/delete_datasets.sql new file mode 100644 index 000000000..7946e6c52 --- /dev/null +++ b/stored_procedures/delete_datasets.sql @@ -0,0 +1,44 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +CREATE OR REPLACE PROCEDURE bqutil.procedure.delete_datasets (project_id STRING, dataset_like_filter STRING) +BEGIN + DECLARE datasets ARRAY; + DECLARE i INT64 DEFAULT 0; + + EXECUTE IMMEDIATE + FORMAT(""" + (SELECT + ARRAY_AGG(SCHEMA_NAME) + FROM region-us.INFORMATION_SCHEMA.SCHEMATA + WHERE SCHEMA_NAME LIKE "%s") + """, dataset_like_filter) + INTO datasets; + + SET i = ARRAY_LENGTH(datasets); + WHILE i > 0 DO + BEGIN + EXECUTE IMMEDIATE + FORMAT("DROP SCHEMA IF EXISTS `%s`.%s CASCADE", + project_id, datasets[ORDINAL(i)]); + SET i = i - 1; + EXCEPTION WHEN ERROR THEN + SET i = i - 1; + SELECT @@error.message; + CONTINUE; + END; + END WHILE; +END; \ No newline at end of file From 00f6be9e11d2801e102adc87d2dd8797e07a088a Mon Sep 17 00:00:00 2001 From: ddeleo Date: Wed, 15 Sep 2021 13:57:34 -0400 Subject: [PATCH 2/2] Trailing newlines --- stored_procedures/cancel_jobs.sql | 2 +- stored_procedures/delete_datasets.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stored_procedures/cancel_jobs.sql b/stored_procedures/cancel_jobs.sql index 4f5ffe669..99c1d9eab 100644 --- a/stored_procedures/cancel_jobs.sql +++ b/stored_procedures/cancel_jobs.sql @@ -44,4 +44,4 @@ BEGIN CONTINUE; END; END WHILE; -END; \ No newline at end of file +END; diff --git a/stored_procedures/delete_datasets.sql b/stored_procedures/delete_datasets.sql index 7946e6c52..d3990e15f 100644 --- a/stored_procedures/delete_datasets.sql +++ b/stored_procedures/delete_datasets.sql @@ -41,4 +41,4 @@ BEGIN CONTINUE; END; END WHILE; -END; \ No newline at end of file +END;