diff --git a/common/utils/src/main/resources/error/error-conditions.json b/common/utils/src/main/resources/error/error-conditions.json index 2017f1fc331e..b61d18825230 100644 --- a/common/utils/src/main/resources/error/error-conditions.json +++ b/common/utils/src/main/resources/error/error-conditions.json @@ -997,6 +997,12 @@ ], "sqlState" : "0A000" }, + "CREATE_OR_REPLACE_WITH_IF_NOT_EXISTS_IS_NOT_ALLOWED" : { + "message" : [ + "CREATE OR REPLACE with IF NOT EXISTS is not allowed." + ], + "sqlState" : "42000" + }, "CREATE_PERMANENT_VIEW_WITHOUT_ALIAS" : { "message" : [ "Not allowed to create the permanent view without explicitly assigning an alias for the expression ." @@ -6019,7 +6025,7 @@ "TEMP_TABLE_OR_VIEW_ALREADY_EXISTS" : { "message" : [ "Cannot create the temporary view because it already exists.", - "Choose a different name, drop or replace the existing view." + "Choose a different name, drop or replace the existing view, or add the IF NOT EXISTS clause to tolerate pre-existing views." ], "sqlState" : "42P07" }, @@ -7572,16 +7578,6 @@ "Empty set in grouping sets is not supported." ] }, - "_LEGACY_ERROR_TEMP_0052" : { - "message" : [ - "CREATE VIEW with both IF NOT EXISTS and REPLACE is not allowed." - ] - }, - "_LEGACY_ERROR_TEMP_0053" : { - "message" : [ - "It is not allowed to define a TEMPORARY view with IF NOT EXISTS." - ] - }, "_LEGACY_ERROR_TEMP_0056" : { "message" : [ "Invalid time travel spec: ." diff --git a/docs/sql-ref-syntax-ddl-create-view.md b/docs/sql-ref-syntax-ddl-create-view.md index 2d832636b38f..21174f12300e 100644 --- a/docs/sql-ref-syntax-ddl-create-view.md +++ b/docs/sql-ref-syntax-ddl-create-view.md @@ -47,7 +47,6 @@ CREATE [ OR REPLACE ] [ [ GLOBAL ] TEMPORARY ] VIEW [ IF NOT EXISTS ] view_ident * **IF NOT EXISTS** Creates a view if it does not exist. - This clause is not supported for `TEMPORARY` views yet. * **view_identifier** @@ -87,8 +86,8 @@ CREATE OR REPLACE VIEW experienced_employee AS SELECT id, name FROM all_employee WHERE working_years > 5; --- Create a global temporary view `subscribed_movies`. -CREATE GLOBAL TEMPORARY VIEW subscribed_movies +-- Create a global temporary view `subscribed_movies` if it does not exist. +CREATE GLOBAL TEMPORARY VIEW IF NOT EXISTS subscribed_movies AS SELECT mo.member_id, mb.full_name, mo.movie_title FROM movies AS mo INNER JOIN members AS mb ON mo.member_id = mb.id; diff --git a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 index 132ced820e9a..5e7b29b47e88 100644 --- a/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 +++ b/sql/api/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBaseParser.g4 @@ -323,7 +323,7 @@ statement (PARTITIONED ON identifierList) | (TBLPROPERTIES propertyList))* AS query #createView - | CREATE (OR REPLACE)? GLOBAL? TEMPORARY VIEW + | CREATE (OR REPLACE)? GLOBAL? TEMPORARY VIEW (IF errorCapturingNot EXISTS)? tableIdentifier (LEFT_PAREN colTypeList RIGHT_PAREN)? tableProvider (OPTIONS propertyList)? #createTempViewUsing | ALTER VIEW identifierReference AS? query #alterViewQuery diff --git a/sql/api/src/main/scala/org/apache/spark/sql/errors/CompilationErrors.scala b/sql/api/src/main/scala/org/apache/spark/sql/errors/CompilationErrors.scala index 6a275b9ad0c1..5009ab4d479a 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/errors/CompilationErrors.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/errors/CompilationErrors.scala @@ -143,6 +143,12 @@ private[sql] trait CompilationErrors extends DataTypeErrorsBase { errorClass = "CANNOT_MODIFY_CONFIG", messageParameters = Map("key" -> toSQLConf(key), "docroot" -> docroot)) } + + def createViewWithBothIfNotExistsAndReplaceError(): Throwable = { + new AnalysisException( + errorClass = "CREATE_OR_REPLACE_WITH_IF_NOT_EXISTS_IS_NOT_ALLOWED", + messageParameters = Map("resourceType" -> "VIEW")) + } } private[sql] object CompilationErrors extends CompilationErrors diff --git a/sql/api/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala b/sql/api/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala index 553161ea2db0..4451b1f1e17b 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/errors/QueryParsingErrors.scala @@ -619,7 +619,10 @@ private[sql] object QueryParsingErrors extends DataTypeErrorsBase { } def createViewWithBothIfNotExistsAndReplaceError(ctx: CreateViewContext): Throwable = { - new ParseException(errorClass = "_LEGACY_ERROR_TEMP_0052", ctx) + new ParseException( + errorClass = "CREATE_OR_REPLACE_WITH_IF_NOT_EXISTS_IS_NOT_ALLOWED", + messageParameters = Map("resourceType" -> "VIEW"), + ctx) } def temporaryViewWithSchemaBindingMode(ctx: StatementContext): Throwable = { @@ -637,10 +640,6 @@ private[sql] object QueryParsingErrors extends DataTypeErrorsBase { messageParameters = Map("statement" -> statement)) } - def defineTempViewWithIfNotExistsError(ctx: CreateViewContext): Throwable = { - new ParseException(errorClass = "_LEGACY_ERROR_TEMP_0053", ctx) - } - def notAllowedToAddDBPrefixForTempViewError( nameParts: Seq[String], ctx: CreateViewContext): Throwable = { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/GlobalTempViewManager.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/GlobalTempViewManager.scala index aeeedebe330d..8fd74f221042 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/GlobalTempViewManager.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/GlobalTempViewManager.scala @@ -49,15 +49,28 @@ class GlobalTempViewManager(database: String) { } /** - * Creates a global temp view, or issue an exception if the view already exists and - * `overrideIfExists` is false. + * Creates a global temp view. + * + * Issue an `AnalysisException` when `ignoreIfExists` and `overrideIfExists` both are true + * because they are mutually exclusive. + * + * If the view already exists: (1) when `ignoreIfExists` is ture, performs no-op; + * (2) otherwise do replacement or issue an exception according to `overrideIfExists`. */ def create( name: String, viewDefinition: TemporaryViewRelation, + ignoreIfExists: Boolean, overrideIfExists: Boolean): Unit = synchronized { - if (!overrideIfExists && viewDefinitions.contains(name)) { - throw new TempTableAlreadyExistsException(name) + if (ignoreIfExists && overrideIfExists) { + throw QueryCompilationErrors.createViewWithBothIfNotExistsAndReplaceError() + } + if (viewDefinitions.contains(name)) { + if (ignoreIfExists) { + return + } else if (!overrideIfExists) { + throw new TempTableAlreadyExistsException(name) + } } viewDefinitions.put(name, viewDefinition) } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala index be90c7ad3656..ef8c85f1cc4d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalog.scala @@ -669,10 +669,18 @@ class SessionCatalog( def createTempView( name: String, viewDefinition: TemporaryViewRelation, + ignoreIfExists: Boolean, overrideIfExists: Boolean): Unit = synchronized { + if (ignoreIfExists && overrideIfExists) { + throw QueryCompilationErrors.createViewWithBothIfNotExistsAndReplaceError() + } val normalized = format(name) - if (tempViews.contains(normalized) && !overrideIfExists) { - throw new TempTableAlreadyExistsException(name) + if (tempViews.contains(normalized)) { + if (ignoreIfExists) { + return + } else if (!overrideIfExists) { + throw new TempTableAlreadyExistsException(name) + } } tempViews.put(normalized, viewDefinition) } @@ -683,8 +691,10 @@ class SessionCatalog( def createGlobalTempView( name: String, viewDefinition: TemporaryViewRelation, + ignoreIfExists: Boolean, overrideIfExists: Boolean): Unit = { - globalTempViewManager.create(format(name), viewDefinition, overrideIfExists) + globalTempViewManager.create( + format(name), viewDefinition, ignoreIfExists, overrideIfExists) } /** @@ -697,7 +707,8 @@ class SessionCatalog( val viewName = format(name.table) if (name.database.isEmpty) { if (tempViews.contains(viewName)) { - createTempView(viewName, viewDefinition, overrideIfExists = true) + createTempView( + viewName, viewDefinition, ignoreIfExists = false, overrideIfExists = true) true } else { false diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala index f65523c844f3..044635212a42 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala @@ -43,23 +43,25 @@ trait AnalysisTest extends PlanTest { catalog: SessionCatalog, name: String, plan: LogicalPlan, + ignoreIfExists: Boolean, overrideIfExists: Boolean): Unit = { val identifier = TableIdentifier(name) val metadata = createTempViewMetadata(identifier, plan.schema) val viewDefinition = TemporaryViewRelation(metadata, Some(plan)) - catalog.createTempView(name, viewDefinition, overrideIfExists) + catalog.createTempView(name, viewDefinition, ignoreIfExists, overrideIfExists) } protected def createGlobalTempView( catalog: SessionCatalog, name: String, plan: LogicalPlan, + ignoreIfExists: Boolean, overrideIfExists: Boolean): Unit = { val globalDb = Some(SQLConf.get.getConf(StaticSQLConf.GLOBAL_TEMP_DATABASE)) val identifier = TableIdentifier(name, globalDb) val metadata = createTempViewMetadata(identifier, plan.schema) val viewDefinition = TemporaryViewRelation(metadata, Some(plan)) - catalog.createGlobalTempView(name, viewDefinition, overrideIfExists) + catalog.createGlobalTempView(name, viewDefinition, ignoreIfExists, overrideIfExists) } private def createTempViewMetadata( @@ -79,13 +81,18 @@ trait AnalysisTest extends PlanTest { catalog.createDatabase( CatalogDatabase("default", "", new URI("loc"), Map.empty), ignoreIfExists = false) - createTempView(catalog, "TaBlE", TestRelations.testRelation, overrideIfExists = true) - createTempView(catalog, "TaBlE2", TestRelations.testRelation2, overrideIfExists = true) - createTempView(catalog, "TaBlE3", TestRelations.testRelation3, overrideIfExists = true) - createGlobalTempView(catalog, "TaBlE4", TestRelations.testRelation4, overrideIfExists = true) - createGlobalTempView(catalog, "TaBlE5", TestRelations.testRelation5, overrideIfExists = true) + createTempView(catalog, "TaBlE", TestRelations.testRelation, + ignoreIfExists = false, overrideIfExists = true) + createTempView(catalog, "TaBlE2", TestRelations.testRelation2, + ignoreIfExists = false, overrideIfExists = true) + createTempView(catalog, "TaBlE3", TestRelations.testRelation3, + ignoreIfExists = false, overrideIfExists = true) + createGlobalTempView(catalog, "TaBlE4", TestRelations.testRelation4, + ignoreIfExists = false, overrideIfExists = true) + createGlobalTempView(catalog, "TaBlE5", TestRelations.testRelation5, + ignoreIfExists = false, overrideIfExists = true) createTempView(catalog, "streamingTable", TestRelations.streamingRelation, - overrideIfExists = true) + ignoreIfExists = false, overrideIfExists = true) new Analyzer(catalog) { catalogManager.tempVariableManager.create( Seq("testA", "testVarA"), diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala index a1b7113d7947..fb3d2826a5be 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/catalog/SessionCatalogSuite.scala @@ -389,17 +389,18 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually { withBasicCatalog { catalog => val tempTable1 = Range(1, 10, 1, 10) val tempTable2 = Range(1, 20, 2, 10) - createTempView(catalog, "tbl1", tempTable1, overrideIfExists = false) - createTempView(catalog, "tbl2", tempTable2, overrideIfExists = false) + createTempView(catalog, "tbl1", tempTable1, ignoreIfExists = false, overrideIfExists = false) + createTempView(catalog, "tbl2", tempTable2, ignoreIfExists = false, overrideIfExists = false) assert(getTempViewRawPlan(catalog.getTempView("tbl1")) == Option(tempTable1)) assert(getTempViewRawPlan(catalog.getTempView("tbl2")) == Option(tempTable2)) assert(getTempViewRawPlan(catalog.getTempView("tbl3")).isEmpty) // Temporary view already exists intercept[TempTableAlreadyExistsException] { - createTempView(catalog, "tbl1", tempTable1, overrideIfExists = false) + createTempView(catalog, "tbl1", tempTable1, + ignoreIfExists = false, overrideIfExists = false) } // Temporary view already exists but we override it - createTempView(catalog, "tbl1", tempTable2, overrideIfExists = true) + createTempView(catalog, "tbl1", tempTable2, ignoreIfExists = false, overrideIfExists = true) assert(getTempViewRawPlan(catalog.getTempView("tbl1")) == Option(tempTable2)) } } @@ -440,7 +441,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually { test("drop temp table") { withBasicCatalog { catalog => val tempTable = Range(1, 10, 2, 10) - createTempView(catalog, "tbl1", tempTable, overrideIfExists = false) + createTempView(catalog, "tbl1", tempTable, ignoreIfExists = false, overrideIfExists = false) catalog.setCurrentDatabase("db2") assert(getTempViewRawPlan(catalog.getTempView("tbl1")) == Some(tempTable)) assert(catalog.externalCatalog.listTables("db2").toSet == Set("tbl1", "tbl2")) @@ -452,7 +453,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually { catalog.dropTable(TableIdentifier("tbl1"), ignoreIfNotExists = false, purge = false) assert(catalog.externalCatalog.listTables("db2").toSet == Set("tbl2")) // If database is specified, temp tables are never dropped - createTempView(catalog, "tbl1", tempTable, overrideIfExists = false) + createTempView(catalog, "tbl1", tempTable, ignoreIfExists = false, overrideIfExists = false) catalog.createTable(newTable("tbl1", "db2"), ignoreIfExists = false) catalog.dropTable(TableIdentifier("tbl1", Some("db2")), ignoreIfNotExists = false, purge = false) @@ -507,7 +508,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually { test("rename temp table") { withBasicCatalog { catalog => val tempTable = Range(1, 10, 2, 10) - createTempView(catalog, "tbl1", tempTable, overrideIfExists = false) + createTempView(catalog, "tbl1", tempTable, ignoreIfExists = false, overrideIfExists = false) catalog.setCurrentDatabase("db2") assert(getTempViewRawPlan(catalog.getTempView("tbl1")) == Option(tempTable)) assert(catalog.externalCatalog.listTables("db2").toSet == Set("tbl1", "tbl2")) @@ -740,7 +741,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually { withBasicCatalog { catalog => val tempTable1 = Range(1, 10, 1, 10) val metastoreTable1 = catalog.externalCatalog.getTable("db2", "tbl1") - createTempView(catalog, "tbl1", tempTable1, overrideIfExists = false) + createTempView(catalog, "tbl1", tempTable1, ignoreIfExists = false, overrideIfExists = false) catalog.setCurrentDatabase("db2") // If we explicitly specify the database, we'll look up the relation in that database assert(catalog.lookupRelation(TableIdentifier("tbl1", Some("db2"))).children.head @@ -834,7 +835,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually { assert(catalog.tableExists(TableIdentifier("tbl1"))) assert(catalog.tableExists(TableIdentifier("tbl2"))) - createTempView(catalog, "tbl3", tempTable, overrideIfExists = false) + createTempView(catalog, "tbl3", tempTable, ignoreIfExists = false, overrideIfExists = false) // tableExists should not check temp view. assert(!catalog.tableExists(TableIdentifier("tbl3"))) @@ -863,7 +864,7 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually { s"`${CatalogManager.SESSION_CATALOG_NAME}`.`default`.`view1`") ) - createTempView(catalog, "view1", tempTable, overrideIfExists = false) + createTempView(catalog, "view1", tempTable, ignoreIfExists = false, overrideIfExists = false) assert(catalog.getTempViewOrPermanentTableMetadata( TableIdentifier("view1")).identifier.table == "view1") assert(catalog.getTempViewOrPermanentTableMetadata( @@ -883,8 +884,8 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually { test("list tables without pattern") { withBasicCatalog { catalog => val tempTable = Range(1, 10, 2, 10) - createTempView(catalog, "tbl1", tempTable, overrideIfExists = false) - createTempView(catalog, "tbl4", tempTable, overrideIfExists = false) + createTempView(catalog, "tbl1", tempTable, ignoreIfExists = false, overrideIfExists = false) + createTempView(catalog, "tbl4", tempTable, ignoreIfExists = false, overrideIfExists = false) assert(catalog.listTables("db1").toSet == Set(TableIdentifier("tbl1"), TableIdentifier("tbl4"))) assert(catalog.listTables("db2").toSet == @@ -901,8 +902,8 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually { test("list tables with pattern") { withBasicCatalog { catalog => val tempTable = Range(1, 10, 2, 10) - createTempView(catalog, "tbl1", tempTable, overrideIfExists = false) - createTempView(catalog, "tbl4", tempTable, overrideIfExists = false) + createTempView(catalog, "tbl1", tempTable, ignoreIfExists = false, overrideIfExists = false) + createTempView(catalog, "tbl4", tempTable, ignoreIfExists = false, overrideIfExists = false) assert(catalog.listTables("db1", "*").toSet == catalog.listTables("db1").toSet) assert(catalog.listTables("db2", "*").toSet == catalog.listTables("db2").toSet) assert(catalog.listTables("db2", "tbl*").toSet == @@ -924,8 +925,10 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually { catalog.createTable(newTable("tbl1", "mydb"), ignoreIfExists = false) catalog.createTable(newTable("tbl2", "mydb"), ignoreIfExists = false) val tempTable = Range(1, 10, 2, 10) - createTempView(catalog, "temp_view1", tempTable, overrideIfExists = false) - createTempView(catalog, "temp_view4", tempTable, overrideIfExists = false) + createTempView(catalog, "temp_view1", tempTable, + ignoreIfExists = false, overrideIfExists = false) + createTempView(catalog, "temp_view4", tempTable, + ignoreIfExists = false, overrideIfExists = false) assert(catalog.listTables("mydb").toSet == catalog.listTables("mydb", "*").toSet) assert(catalog.listTables("mydb").toSet == catalog.listTables("mydb", "*", true).toSet) @@ -951,8 +954,10 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually { test("list temporary view with pattern") { withBasicCatalog { catalog => val tempTable = Range(1, 10, 2, 10) - createTempView(catalog, "temp_view1", tempTable, overrideIfExists = false) - createTempView(catalog, "temp_view4", tempTable, overrideIfExists = false) + createTempView(catalog, "temp_view1", tempTable, + ignoreIfExists = false, overrideIfExists = false) + createTempView(catalog, "temp_view4", tempTable, + ignoreIfExists = false, overrideIfExists = false) assert(catalog.listLocalTempViews("*").toSet == Set(TableIdentifier("temp_view1"), TableIdentifier("temp_view4"))) assert(catalog.listLocalTempViews("temp_view*").toSet == @@ -965,10 +970,14 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually { test("list global temporary view and local temporary view with pattern") { withBasicCatalog { catalog => val tempTable = Range(1, 10, 2, 10) - createTempView(catalog, "temp_view1", tempTable, overrideIfExists = false) - createTempView(catalog, "temp_view4", tempTable, overrideIfExists = false) - createGlobalTempView(catalog, "global_temp_view1", tempTable, overrideIfExists = false) - createGlobalTempView(catalog, "global_temp_view2", tempTable, overrideIfExists = false) + createTempView(catalog, "temp_view1", tempTable, + ignoreIfExists = false, overrideIfExists = false) + createTempView(catalog, "temp_view4", tempTable, + ignoreIfExists = false, overrideIfExists = false) + createGlobalTempView(catalog, "global_temp_view1", tempTable, + ignoreIfExists = false, overrideIfExists = false) + createGlobalTempView(catalog, "global_temp_view2", tempTable, + ignoreIfExists = false, overrideIfExists = false) assert(catalog.listTables(catalog.globalTempDatabase, "*").toSet == Set(TableIdentifier("temp_view1"), TableIdentifier("temp_view4"), @@ -1866,7 +1875,8 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually { test("copy SessionCatalog state - temp views") { withEmptyCatalog { original => val tempTable1 = Range(1, 10, 1, 10) - createTempView(original, "copytest1", tempTable1, overrideIfExists = false) + createTempView(original, "copytest1", tempTable1, + ignoreIfExists = false, overrideIfExists = false) // check if tables copied over val clone = new SessionCatalog(original.externalCatalog) @@ -1880,7 +1890,8 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually { assert(getTempViewRawPlan(original.getTempView("copytest1")) == Some(tempTable1)) val tempTable2 = Range(1, 20, 2, 10) - createTempView(original, "copytest2", tempTable2, overrideIfExists = false) + createTempView(original, "copytest2", tempTable2, + ignoreIfExists = false, overrideIfExists = false) assert(clone.getTempView("copytest2").isEmpty) } } @@ -1938,13 +1949,15 @@ abstract class SessionCatalogSuite extends AnalysisTest with Eventually { test("SPARK-34197: refreshTable should not invalidate the relation cache for temporary views") { withBasicCatalog { catalog => - createTempView(catalog, "tbl1", Range(1, 10, 1, 10), false) + createTempView(catalog, "tbl1", Range(1, 10, 1, 10), + ignoreIfExists = false, overrideIfExists = false) val qualifiedName1 = QualifiedTableName(SESSION_CATALOG_NAME, "default", "tbl1") catalog.cacheTable(qualifiedName1, Range(1, 10, 1, 10)) catalog.refreshTable(TableIdentifier("tbl1")) assert(catalog.getCachedTable(qualifiedName1) != null) - createGlobalTempView(catalog, "tbl2", Range(2, 10, 1, 10), false) + createGlobalTempView(catalog, "tbl2", Range(2, 10, 1, 10), + ignoreIfExists = false, overrideIfExists = false) val qualifiedName2 = QualifiedTableName(SESSION_CATALOG_NAME, catalog.globalTempDatabase, "tbl2") catalog.cacheTable(qualifiedName2, Range(2, 10, 1, 10)) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index f8f6e31be1bc..07030d83eed5 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -501,8 +501,8 @@ class SparkSqlAstBuilder extends AstBuilder { }.toMap val optionsWithLocation = location.map(l => optionsList + ("path" -> l)).getOrElse(optionsList) - CreateTempViewUsing(table, schema, replace = false, global = false, provider, - optionsWithLocation) + CreateTempViewUsing(table, schema, ignoreIfExists = false, replace = false, + global = false, provider, optionsWithLocation) }) } } @@ -515,6 +515,7 @@ class SparkSqlAstBuilder extends AstBuilder { CreateTempViewUsing( tableIdent = visitTableIdentifier(ctx.tableIdentifier()), userSpecifiedSchema = Option(ctx.colTypeList()).map(createSchema), + ignoreIfExists = ctx.EXISTS != null, replace = ctx.REPLACE != null, global = ctx.GLOBAL != null, provider = ctx.tableProvider.multipartIdentifier.getText, @@ -695,12 +696,6 @@ class SparkSqlAstBuilder extends AstBuilder { ctx.REPLACE != null, finalSchemaBinding) } else { - // Disallows 'CREATE TEMPORARY VIEW IF NOT EXISTS' to be consistent with - // 'CREATE TEMPORARY TABLE' - if (ctx.EXISTS != null) { - throw QueryParsingErrors.defineTempViewWithIfNotExistsError(ctx) - } - withIdentClause(ctx.identifierReference(), Seq(qPlan), (ident, otherPlans) => { val tableIdentifier = ident.asTableIdentifier if (tableIdentifier.database.isDefined) { diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala index 11ec17ca57fd..3d6090e4a9d6 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/views.scala @@ -36,7 +36,7 @@ import org.apache.spark.sql.classic.ClassicConversions.castToImpl import org.apache.spark.sql.connector.catalog.CatalogV2Implicits.NamespaceHelper import org.apache.spark.sql.errors.QueryCompilationErrors import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Relation -import org.apache.spark.sql.internal.StaticSQLConf +import org.apache.spark.sql.internal.{SessionStateHelper, StaticSQLConf} import org.apache.spark.sql.types.{MetadataBuilder, StructType} import org.apache.spark.sql.util.SchemaUtils import org.apache.spark.util.ArrayImplicits._ @@ -81,7 +81,8 @@ case class CreateViewCommand( extends RunnableCommand with AnalysisOnlyCommand with CTEInChildren - with CreateTempView { + with CreateTempView + with SessionStateHelper { import ViewHelper._ @@ -123,7 +124,7 @@ case class CreateViewCommand( } } - val catalog = sparkSession.sessionState.catalog + val catalog = sessionState(sparkSession).catalog // When creating a permanent view, not allowed to reference temporary objects. // This should be called after `qe.assertAnalyzed()` (i.e., `child` can be resolved) @@ -144,9 +145,10 @@ case class CreateViewCommand( aliasedPlan, referredTempFunctions, collation) - catalog.createTempView(name.table, tableDefinition, overrideIfExists = replace) + catalog.createTempView(name.table, tableDefinition, + ignoreIfExists = allowExisting, overrideIfExists = replace) } else if (viewType == GlobalTempView) { - val db = sparkSession.sessionState.conf.getConf(StaticSQLConf.GLOBAL_TEMP_DATABASE) + val db = getSqlConf(sparkSession).getConf(StaticSQLConf.GLOBAL_TEMP_DATABASE) val viewIdent = TableIdentifier(name.table, Option(db)) val aliasedPlan = aliasPlan(sparkSession, analyzedPlan) val tableDefinition = createTemporaryViewRelation( @@ -158,7 +160,8 @@ case class CreateViewCommand( analyzedPlan, aliasedPlan, referredTempFunctions) - catalog.createGlobalTempView(name.table, tableDefinition, overrideIfExists = replace) + catalog.createGlobalTempView(name.table, tableDefinition, + ignoreIfExists = allowExisting, overrideIfExists = replace) } else if (catalog.tableExists(name)) { val tableMetadata = catalog.getTableMetadata(name) if (allowExisting) { @@ -205,7 +208,7 @@ case class CreateViewCommand( val meta = new MetadataBuilder().putString("comment", colComment).build() Alias(attr, colName)(explicitMetadata = Some(meta)) } - session.sessionState.executePlan(Project(projectList, analyzedPlan)).analyzed + sessionState(session).executePlan(Project(projectList, analyzedPlan)).analyzed } } @@ -219,7 +222,7 @@ case class CreateViewCommand( throw QueryCompilationErrors.createPersistedViewFromDatasetAPINotAllowedError() } val aliasedSchema = CharVarcharUtils.getRawSchema( - aliasPlan(session, analyzedPlan).schema, session.sessionState.conf) + aliasPlan(session, analyzedPlan).schema, getSqlConf(session)) val newProperties = generateViewProperties( properties, session, analyzedPlan.schema.fieldNames, aliasedSchema.fieldNames, viewSchemaMode) @@ -259,7 +262,7 @@ case class AlterViewAsCommand( query: LogicalPlan, isAnalyzed: Boolean = false, referredTempFunctions: Seq[String] = Seq.empty) - extends RunnableCommand with AnalysisOnlyCommand with CTEInChildren { + extends RunnableCommand with AnalysisOnlyCommand with CTEInChildren with SessionStateHelper { import ViewHelper._ @@ -279,7 +282,7 @@ case class AlterViewAsCommand( } override def run(session: SparkSession): Seq[Row] = { - val isTemporary = session.sessionState.catalog.isTempView(name) + val isTemporary = sessionState(session).catalog.isTempView(name) verifyTemporaryObjectsNotExists(isTemporary, name, query, referredTempFunctions) verifyAutoGeneratedAliasesNotExists(query, isTemporary, name) SchemaUtils.checkIndeterminateCollationInSchema(query.schema) @@ -292,7 +295,7 @@ case class AlterViewAsCommand( } private def alterTemporaryView(session: SparkSession, analyzedPlan: LogicalPlan): Unit = { - val catalog = session.sessionState.catalog + val catalog = sessionState(session).catalog val getRawTempView: String => Option[TemporaryViewRelation] = if (name.database.isEmpty) { catalog.getRawTempView } else { @@ -307,11 +310,11 @@ case class AlterViewAsCommand( analyzedPlan, aliasedPlan = analyzedPlan, referredTempFunctions) - session.sessionState.catalog.alterTempViewDefinition(name, tableDefinition) + sessionState(session).catalog.alterTempViewDefinition(name, tableDefinition) } private def alterPermanentView(session: SparkSession, analyzedPlan: LogicalPlan): Unit = { - val viewMeta = session.sessionState.catalog.getTableMetadata(name) + val viewMeta = sessionState(session).catalog.getTableMetadata(name) // Detect cyclic view reference on ALTER VIEW. val viewIdent = viewMeta.identifier @@ -334,7 +337,7 @@ case class AlterViewAsCommand( viewOriginalText = Some(originalText), viewText = Some(originalText)) - session.sessionState.catalog.alterTable(updatedViewMeta) + sessionState(session).catalog.alterTable(updatedViewMeta) } override def withCTEDefs(cteDefs: Seq[CTERelationDef]): LogicalPlan = { @@ -352,12 +355,12 @@ case class AlterViewAsCommand( * @param viewSchemaMode The new schema binding mode. */ case class AlterViewSchemaBindingCommand(name: TableIdentifier, viewSchemaMode: ViewSchemaMode) - extends LeafRunnableCommand { + extends LeafRunnableCommand with SessionStateHelper { import ViewHelper._ override def run(session: SparkSession): Seq[Row] = { - val isTemporary = session.sessionState.catalog.isTempView(name) + val isTemporary = sessionState(session).catalog.isTempView(name) if (isTemporary) { throw QueryCompilationErrors.cannotAlterTempViewWithSchemaBindingError() } @@ -366,7 +369,7 @@ case class AlterViewSchemaBindingCommand(name: TableIdentifier, viewSchemaMode: } private def alterPermanentView(session: SparkSession, viewSchemaMode: ViewSchemaMode): Unit = { - val viewMeta = session.sessionState.catalog.getTableMetadata(name) + val viewMeta = sessionState(session).catalog.getTableMetadata(name) val viewIdent = viewMeta.identifier @@ -382,7 +385,7 @@ case class AlterViewSchemaBindingCommand(name: TableIdentifier, viewSchemaMode: val updatedViewMeta = viewMeta.copy(properties = newProperties) - session.sessionState.catalog.alterTable(updatedViewMeta) + sessionState(session).catalog.alterTable(updatedViewMeta) } } @@ -397,10 +400,10 @@ case class AlterViewSchemaBindingCommand(name: TableIdentifier, viewSchemaMode: case class ShowViewsCommand( databaseName: String, tableIdentifierPattern: Option[String], - override val output: Seq[Attribute]) extends LeafRunnableCommand { + override val output: Seq[Attribute]) extends LeafRunnableCommand with SessionStateHelper { override def run(sparkSession: SparkSession): Seq[Row] = { - val catalog = sparkSession.sessionState.catalog + val catalog = sessionState(sparkSession).catalog // Show the information of views. val views = tableIdentifierPattern.map(catalog.listViews(databaseName, _)) @@ -415,7 +418,7 @@ case class ShowViewsCommand( } } -object ViewHelper extends SQLConfHelper with Logging with CapturesConfig { +object ViewHelper extends SQLConfHelper with Logging with CapturesConfig with SessionStateHelper { import CatalogTable._ /** @@ -524,7 +527,7 @@ object ViewHelper extends SQLConfHelper with Logging with CapturesConfig { tempFunctionNames: Seq[String] = Seq.empty, tempVariableNames: Seq[Seq[String]] = Seq.empty): Map[String, String] = { - val conf = session.sessionState.conf + val conf = getSqlConf(session) // Generate the query column names, throw an AnalysisException if there exists duplicate column // names. @@ -539,7 +542,7 @@ object ViewHelper extends SQLConfHelper with Logging with CapturesConfig { } // Generate the view default catalog and namespace, as well as captured SQL configs. - val manager = session.sessionState.catalogManager + val manager = sessionState(session).catalogManager removeReferredTempNames(removeSQLConfigs(removeQueryColumnNames(properties))) ++ catalogAndNamespaceToProps( manager.currentCatalog.name, manager.currentNamespace.toImmutableArraySeq) ++ @@ -698,10 +701,8 @@ object ViewHelper extends SQLConfHelper with Logging with CapturesConfig { referredTempFunctions: Seq[String], collation: Option[String] = None): TemporaryViewRelation = { val rawTempView = getRawTempView(name.table) - val uncache = rawTempView.map { r => - needsToUncache(r, aliasedPlan) - }.getOrElse(false) - val storeAnalyzedPlanForView = session.sessionState.conf.storeAnalyzedPlanForView || + val uncache = rawTempView.exists(needsToUncache(_, aliasedPlan)) + val storeAnalyzedPlanForView = getSqlConf(session).storeAnalyzedPlanForView || originalText.isEmpty if (replace && uncache) { logDebug(s"Try to uncache ${name.quotedString} before replacing.") diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ddl.scala index f22aa9282ed5..1330f0bc33bc 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ddl.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ddl.scala @@ -29,7 +29,7 @@ import org.apache.spark.sql.errors.QueryCompilationErrors import org.apache.spark.sql.execution.command.{DDLUtils, LeafRunnableCommand} import org.apache.spark.sql.execution.command.ViewHelper.createTemporaryViewRelation import org.apache.spark.sql.execution.datasources.v2.DataSourceV2Utils -import org.apache.spark.sql.internal.StaticSQLConf +import org.apache.spark.sql.internal.{SessionStateHelper, StaticSQLConf} import org.apache.spark.sql.types._ /** @@ -76,10 +76,11 @@ case class CreateTable( case class CreateTempViewUsing( tableIdent: TableIdentifier, userSpecifiedSchema: Option[StructType], + ignoreIfExists: Boolean, replace: Boolean, global: Boolean, provider: String, - options: Map[String, String]) extends LeafRunnableCommand { + options: Map[String, String]) extends LeafRunnableCommand with SessionStateHelper { if (tableIdent.database.isDefined) { throw QueryCompilationErrors.cannotSpecifyDatabaseForTempViewError(tableIdent) @@ -88,6 +89,7 @@ case class CreateTempViewUsing( override def argString(maxFields: Int): String = { s"[tableIdent:$tableIdent " + userSpecifiedSchema.map(_.toString() + " ").getOrElse("") + + s"ignoreIfExists:$ignoreIfExists " + s"replace:$replace " + s"provider:$provider " + conf.redactOptions(options) @@ -98,8 +100,9 @@ case class CreateTempViewUsing( throw QueryCompilationErrors.cannotCreateTempViewUsingHiveDataSourceError() } - val catalog = sparkSession.sessionState.catalog - val unresolvedPlan = DataSource.lookupDataSourceV2(provider, sparkSession.sessionState.conf) + val conf = getSqlConf(sparkSession) + val catalog = sessionState(sparkSession).catalog + val unresolvedPlan = DataSource.lookupDataSourceV2(provider, conf) .flatMap { tblProvider => DataSourceV2Utils.loadV2Source(sparkSession, tblProvider, userSpecifiedSchema, CaseInsensitiveMap(options), provider) @@ -111,10 +114,10 @@ case class CreateTempViewUsing( options = options) LogicalRelation(dataSource.resolveRelation()) } - val analyzedPlan = sparkSession.sessionState.analyzer.execute(unresolvedPlan) + val analyzedPlan = sessionState(sparkSession).analyzer.execute(unresolvedPlan) if (global) { - val db = sparkSession.sessionState.conf.getConf(StaticSQLConf.GLOBAL_TEMP_DATABASE) + val db = conf.getConf(StaticSQLConf.GLOBAL_TEMP_DATABASE) val viewIdent = TableIdentifier(tableIdent.table, Option(db)) val viewDefinition = createTemporaryViewRelation( viewIdent, @@ -125,7 +128,7 @@ case class CreateTempViewUsing( analyzedPlan, aliasedPlan = analyzedPlan, referredTempFunctions = Seq.empty) - catalog.createGlobalTempView(tableIdent.table, viewDefinition, replace) + catalog.createGlobalTempView(tableIdent.table, viewDefinition, ignoreIfExists, replace) } else { val viewDefinition = createTemporaryViewRelation( tableIdent, @@ -136,7 +139,7 @@ case class CreateTempViewUsing( analyzedPlan, aliasedPlan = analyzedPlan, referredTempFunctions = Seq.empty) - catalog.createTempView(tableIdent.table, viewDefinition, replace) + catalog.createTempView(tableIdent.table, viewDefinition, ignoreIfExists, replace) } Seq.empty[Row] diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/describe.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/describe.sql.out index e49673d33943..e005ce3f6b2c 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/describe.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/describe.sql.out @@ -26,7 +26,7 @@ CREATE TEMPORARY VIEW temp_Data_Source_View To '10', Table 'test1') -- !query analysis -CreateTempViewUsing [tableIdent:`temp_Data_Source_View` replace:false provider:org.apache.spark.sql.sources.DDLScanSource Map(From -> 1, To -> 10, Table -> test1) +CreateTempViewUsing [tableIdent:`temp_Data_Source_View` ignoreIfExists:false replace:false provider:org.apache.spark.sql.sources.DDLScanSource Map(From -> 1, To -> 10, Table -> test1) -- !query diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/postgreSQL/join.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/postgreSQL/join.sql.out index 439094c11286..22692b157ef8 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/postgreSQL/join.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/postgreSQL/join.sql.out @@ -1936,7 +1936,7 @@ CreateViewCommand `zt2`, select * from -- !query create or replace temporary view zt3(f3 int) using parquet -- !query analysis -CreateTempViewUsing [tableIdent:`zt3` StructType(StructField(f3,IntegerType,true)) replace:true provider:parquet Map() +CreateTempViewUsing [tableIdent:`zt3` StructType(StructField(f3,IntegerType,true)) ignoreIfExists:false replace:true provider:parquet Map() -- !query @@ -2035,13 +2035,13 @@ Project [unique2#x, ten#x, tenthous#x, unique2#x, hundred#x] -- !query create or replace temporary view a (i integer) using parquet -- !query analysis -CreateTempViewUsing [tableIdent:`a` StructType(StructField(i,IntegerType,true)) replace:true provider:parquet Map() +CreateTempViewUsing [tableIdent:`a` StructType(StructField(i,IntegerType,true)) ignoreIfExists:false replace:true provider:parquet Map() -- !query create or replace temporary view b (x integer, y integer) using parquet -- !query analysis -CreateTempViewUsing [tableIdent:`b` StructType(StructField(x,IntegerType,true),StructField(y,IntegerType,true)) replace:true provider:parquet Map() +CreateTempViewUsing [tableIdent:`b` StructType(StructField(x,IntegerType,true),StructField(y,IntegerType,true)) ignoreIfExists:false replace:true provider:parquet Map() -- !query diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/show-tables.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/show-tables.sql.out index 8ae12b928d72..b49ad79c5ea6 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/show-tables.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/show-tables.sql.out @@ -34,7 +34,7 @@ CreateDataSourceTableCommand `spark_catalog`.`showdb`.`show_t2`, false -- !query CREATE TEMPORARY VIEW show_t3(e int) USING parquet -- !query analysis -CreateTempViewUsing [tableIdent:`show_t3` StructType(StructField(e,IntegerType,true)) replace:false provider:parquet Map() +CreateTempViewUsing [tableIdent:`show_t3` StructType(StructField(e,IntegerType,true)) ignoreIfExists:false replace:false provider:parquet Map() -- !query diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/show-views.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/show-views.sql.out index d092590b143b..324fb76f91ac 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/show-views.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/show-views.sql.out @@ -49,7 +49,7 @@ CreateViewCommand `view_3`, SELECT 1 as col1, false, false, GlobalTempView, UNSU -- !query CREATE TEMPORARY VIEW view_4(e INT) USING parquet -- !query analysis -CreateTempViewUsing [tableIdent:`view_4` StructType(StructField(e,IntegerType,true)) replace:false provider:parquet Map() +CreateTempViewUsing [tableIdent:`view_4` StructType(StructField(e,IntegerType,true)) ignoreIfExists:false replace:false provider:parquet Map() -- !query diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/show_columns.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/show_columns.sql.out index 76c3b88a3ce6..4a16b0316a57 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/show_columns.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/show_columns.sql.out @@ -28,7 +28,7 @@ CreateDataSourceTableCommand `spark_catalog`.`showdb`.`showcolumn2`, false -- !query CREATE TEMPORARY VIEW showColumn3 (col3 int, `col 4` int) USING json -- !query analysis -CreateTempViewUsing [tableIdent:`showColumn3` StructType(StructField(col3,IntegerType,true),StructField(col 4,IntegerType,true)) replace:false provider:json Map() +CreateTempViewUsing [tableIdent:`showColumn3` StructType(StructField(col3,IntegerType,true),StructField(col 4,IntegerType,true)) ignoreIfExists:false replace:false provider:json Map() -- !query diff --git a/sql/core/src/test/resources/sql-tests/analyzer-results/udf/postgreSQL/udf-join.sql.out b/sql/core/src/test/resources/sql-tests/analyzer-results/udf/postgreSQL/udf-join.sql.out index 09ef51fcf231..629819a2e3b8 100644 --- a/sql/core/src/test/resources/sql-tests/analyzer-results/udf/postgreSQL/udf-join.sql.out +++ b/sql/core/src/test/resources/sql-tests/analyzer-results/udf/postgreSQL/udf-join.sql.out @@ -1935,7 +1935,7 @@ CreateViewCommand `zt2`, select * from -- !query create or replace temporary view zt3(f3 int) using parquet -- !query analysis -CreateTempViewUsing [tableIdent:`zt3` StructType(StructField(f3,IntegerType,true)) replace:true provider:parquet Map() +CreateTempViewUsing [tableIdent:`zt3` StructType(StructField(f3,IntegerType,true)) ignoreIfExists:false replace:true provider:parquet Map() -- !query @@ -2034,13 +2034,13 @@ Project [cast(udf(cast(unique2#x as string)) as int) AS udf(unique2)#x, cast(udf -- !query create or replace temporary view a (i integer) using parquet -- !query analysis -CreateTempViewUsing [tableIdent:`a` StructType(StructField(i,IntegerType,true)) replace:true provider:parquet Map() +CreateTempViewUsing [tableIdent:`a` StructType(StructField(i,IntegerType,true)) ignoreIfExists:false replace:true provider:parquet Map() -- !query create or replace temporary view b (x integer, y integer) using parquet -- !query analysis -CreateTempViewUsing [tableIdent:`b` StructType(StructField(x,IntegerType,true),StructField(y,IntegerType,true)) replace:true provider:parquet Map() +CreateTempViewUsing [tableIdent:`b` StructType(StructField(x,IntegerType,true),StructField(y,IntegerType,true)) ignoreIfExists:false replace:true provider:parquet Map() -- !query diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala index a09b7e0827c4..78706138a108 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2Suite.scala @@ -633,6 +633,7 @@ class DataSourceV2Suite extends QueryTest with SharedSparkSession with AdaptiveS Seq(classOf[SimpleDataSourceV2], classOf[JavaSimpleDataSourceV2]).foreach { cls => withClue(cls.getName) { sql(s"CREATE or REPLACE TEMPORARY VIEW s1 USING ${cls.getName}") + sql(s"CREATE TEMPORARY VIEW IF NOT EXISTS s1 USING ${cls.getName}") checkAnswer(sql("select * from s1"), (0 until 10).map(i => Row(i, -i))) checkAnswer(sql("select j from s1"), (0 until 10).map(i => Row(-i))) checkAnswer(sql("select * from s1 where i > 5"), diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala index 31d8dd0740e1..054fd69b9b15 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/GlobalTempViewSuite.scala @@ -108,6 +108,9 @@ class GlobalTempViewSuite extends QueryTest with SharedSparkSession { checkAnswer(spark.table(s"$globalTempDB.src"), Row(1, "a")) sql(s"INSERT INTO $globalTempDB.src SELECT 2, 'b'") checkAnswer(spark.table(s"$globalTempDB.src"), Row(1, "a") :: Row(2, "b") :: Nil) + // perform noop with IF NOT EXISTS if view already exists + sql(s"CREATE GLOBAL TEMP VIEW IF NOT EXISTS src USING csv OPTIONS (PATH '${path.toURI}')") + checkAnswer(spark.table(s"$globalTempDB.src"), Row(1, "a") :: Row(2, "b") :: Nil) } } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala index 050a004a9353..d7ea62a5efa0 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewSuite.scala @@ -26,6 +26,7 @@ import org.apache.spark.sql.catalyst.plans.logical.Project import org.apache.spark.sql.catalyst.trees.Origin import org.apache.spark.sql.connector.catalog.CatalogManager.SESSION_CATALOG_NAME import org.apache.spark.sql.internal.SQLConf._ +import org.apache.spark.sql.internal.StaticSQLConf.GLOBAL_TEMP_DATABASE import org.apache.spark.sql.test.{SharedSparkSession, SQLTestUtils} class SimpleSQLViewSuite extends SQLViewSuite with SharedSparkSession @@ -471,15 +472,6 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils { "CREATE TEMPORARY VIEW or the corresponding Dataset APIs only accept single-part view names")) } - test("error handling: disallow IF NOT EXISTS for CREATE TEMPORARY VIEW") { - withTempView("myabcdview") { - val e = intercept[ParseException] { - sql("CREATE TEMPORARY VIEW IF NOT EXISTS myabcdview AS SELECT * FROM jt") - } - assert(e.message.contains("It is not allowed to define a TEMPORARY view with IF NOT EXISTS")) - } - } - test("error handling: fail if the temp view sql itself is invalid") { // A database that does not exist assertAnalysisErrorCondition( @@ -586,17 +578,27 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils { } } - test("correctly handle CREATE VIEW IF NOT EXISTS") { - withTable("jt2") { - withView("testView") { - sql("CREATE VIEW testView AS SELECT id FROM jt") - - val df = (1 until 10).map(i => i -> i).toDF("i", "j") - df.write.format("json").saveAsTable("jt2") - sql("CREATE VIEW IF NOT EXISTS testView AS SELECT * FROM jt2") - - // make sure our view doesn't change. - checkAnswer(sql("SELECT * FROM testView ORDER BY id"), (1 to 9).map(i => Row(i))) + test("correctly handle CREATE VIEW / TEMPORARY VIEW / GLOBAL TEMPORARY VIEW IF NOT EXISTS") { + Seq("VIEW", "TEMPORARY VIEW", "GLOBAL TEMPORARY VIEW").foreach { viewType => + withTable("jt2") { + withView("testView") { + sql(s"CREATE $viewType testView AS SELECT id FROM jt") + + val df = (1 until 10).map(i => i -> i).toDF("i", "j") + df.write.format("json").saveAsTable("jt2") + sql(s"CREATE $viewType IF NOT EXISTS testView AS SELECT * FROM jt2") + + // make sure our view doesn't change. + val viewIdentifier = viewType match { + case "GLOBAL TEMPORARY VIEW" => + TableIdentifier("testView", Some(conf.getConf(GLOBAL_TEMP_DATABASE))) + case _ => + TableIdentifier("testView") + } + checkAnswer( + sql(s"SELECT * FROM ${viewIdentifier.quotedString} ORDER BY id"), + (1 to 9).map(i => Row(i))) + } } } } @@ -627,11 +629,15 @@ abstract class SQLViewSuite extends QueryTest with SQLTestUtils { sql("DROP VIEW testView") - val e = intercept[ParseException] { - sql("CREATE OR REPLACE VIEW IF NOT EXISTS testView AS SELECT id FROM jt") - } - assert(e.message.contains( - "CREATE VIEW with both IF NOT EXISTS and REPLACE is not allowed")) + checkError( + intercept[ParseException] { + sql("CREATE OR REPLACE VIEW IF NOT EXISTS testView AS SELECT id FROM jt") + }, + condition = "CREATE_OR_REPLACE_WITH_IF_NOT_EXISTS_IS_NOT_ALLOWED", + sqlState = "42000", + parameters = Map("resourceType" -> "VIEW"), + context = ExpectedContext( + "CREATE OR REPLACE VIEW IF NOT EXISTS testView AS SELECT id FROM jt", 0, 65)) } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala index 94e60db67ac7..d09b4e2005e7 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala @@ -502,10 +502,10 @@ class SparkSqlParserSuite extends AnalysisTest with SharedSparkSession { test("SPARK-33118 CREATE TEMPORARY TABLE with LOCATION") { assertEqual("CREATE TEMPORARY TABLE t USING parquet OPTIONS (path '/data/tmp/testspark1')", - CreateTempViewUsing(TableIdentifier("t", None), None, false, false, "parquet", + CreateTempViewUsing(TableIdentifier("t", None), None, false, false, false, "parquet", Map("path" -> "/data/tmp/testspark1"))) assertEqual("CREATE TEMPORARY TABLE t USING parquet LOCATION '/data/tmp/testspark1'", - CreateTempViewUsing(TableIdentifier("t", None), None, false, false, "parquet", + CreateTempViewUsing(TableIdentifier("t", None), None, false, false, false, "parquet", Map("path" -> "/data/tmp/testspark1"))) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index 3d4da878869e..dfb31122d846 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -782,8 +782,17 @@ abstract class DDLSuite extends QueryTest with DDLSuiteBase { // to a temp file by withResourceTempPath withResourceTempPath("test-data/cars.csv") { tmpFile => withTempView("testview") { - sql(s"CREATE OR REPLACE TEMPORARY VIEW testview (c1 String, c2 String) USING " + - "org.apache.spark.sql.execution.datasources.csv.CSVFileFormat " + + sql(s"CREATE OR REPLACE TEMPORARY VIEW testview (c1 String, c2 String) " + + s"USING org.apache.spark.sql.execution.datasources.csv.CSVFileFormat " + + s"OPTIONS (PATH '${tmpFile.toURI}')") + + checkAnswer( + sql("select c1, c2 from testview order by c1 limit 1"), + Row("1997", "Ford") :: Nil) + + // column names are changed, perform noop with IF NOT EXISTS when view exists + sql(s"CREATE TEMPORARY VIEW IF NOT EXISTS testview (d1 String, d2 String) " + + s"USING org.apache.spark.sql.execution.datasources.csv.CSVFileFormat " + s"OPTIONS (PATH '${tmpFile.toURI}')") checkAnswer( diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala index 18042bf73adf..6062a17a2003 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala @@ -201,7 +201,8 @@ class PlanResolutionSuite extends SharedSparkSession with AnalysisTest { new InMemoryCatalog, EmptyFunctionRegistry, new SQLConf().copy(SQLConf.CASE_SENSITIVE -> true)) - createTempView(v1SessionCatalog, "v", LocalRelation(Nil), false) + createTempView(v1SessionCatalog, "v", LocalRelation(Nil), + ignoreIfExists = false, overrideIfExists = false) private val tempVariableManager = new TempVariableManager diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/StreamRelationSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/StreamRelationSuite.scala index d38dd821efda..8980224df9ec 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/StreamRelationSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/StreamRelationSuite.scala @@ -41,12 +41,14 @@ class StreamRelationSuite extends SharedSparkSession with AnalysisTest { catalog.createDatabase( CatalogDatabase("default", "", new URI("loc"), Map.empty), ignoreIfExists = false) - createTempView(catalog, "table", TestRelations.testRelation, overrideIfExists = true) - createTempView(catalog, "table2", TestRelations.testRelation2, overrideIfExists = true) + createTempView(catalog, "table", TestRelations.testRelation, + ignoreIfExists = false, overrideIfExists = true) + createTempView(catalog, "table2", TestRelations.testRelation2, + ignoreIfExists = false, overrideIfExists = true) createTempView(catalog, "streamingTable", TestRelations.streamingRelation, - overrideIfExists = true) + ignoreIfExists = false, overrideIfExists = true) createTempView(catalog, "streamingTable2", TestRelations.streamingRelation, - overrideIfExists = true) + ignoreIfExists = false, overrideIfExists = true) new Analyzer(catalog) { override val extendedResolutionRules = extendedAnalysisRules } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala index ebfffc14b014..28ddd58b7983 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala @@ -79,7 +79,8 @@ class CatalogSuite extends SharedSparkSession with AnalysisTest with BeforeAndAf } private def createTempTable(name: String): Unit = { - createTempView(sessionCatalog, name, Range(1, 2, 3, 4), overrideIfExists = true) + createTempView(sessionCatalog, name, Range(1, 2, 3, 4), + ignoreIfExists = false, overrideIfExists = true) } private def dropTable(name: String, db: Option[String] = None): Unit = { @@ -774,7 +775,8 @@ class CatalogSuite extends SharedSparkSession with AnalysisTest with BeforeAndAf assert(spark.catalog.listTables().collect().map(_.name).toSet == Set()) assert(forkedSession.catalog.listTables().collect().map(_.name).toSet == Set("my_temp_table")) createTempView( - forkedSession.sessionState.catalog, "fork_table", Range(1, 2, 3, 4), overrideIfExists = true) + forkedSession.sessionState.catalog, "fork_table", Range(1, 2, 3, 4), + ignoreIfExists = false, overrideIfExists = true) assert(spark.catalog.listTables().collect().map(_.name).toSet == Set()) } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/ListTablesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/ListTablesSuite.scala index 9de5c6aab9cc..14120cb7c0ef 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/ListTablesSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/ListTablesSuite.scala @@ -38,7 +38,8 @@ class ListTablesSuite extends QueryTest super.beforeAll() // The catalog in HiveContext is a case insensitive one. createTempView( - sessionState.catalog, "ListTablesSuiteTable", df.logicalPlan, overrideIfExists = true) + sessionState.catalog, "ListTablesSuiteTable", df.logicalPlan, + ignoreIfExists = false, overrideIfExists = true) sql("CREATE TABLE HiveListTablesSuiteTable (key int, value string)") sql("CREATE DATABASE IF NOT EXISTS ListTablesSuiteDB") sql("CREATE TABLE ListTablesSuiteDB.HiveInDBListTablesSuiteTable (key int, value string)")