[CALCITE-3323] Add mode to SqlValidator that treats statements as valid if they contain unknown functions (Ryan Fu)

The mode is controlled by new method SqlValidator.isLenientOperatorLookup().

If it encounters an unknown function, or a known function with invalid
number or types of arguments, the validator now creates a
SqlUnresolvedFunction. The return type is inferred to be UNKNOWN, so
that an enclosing function call can be validated.

Removed .factorypath files and updated gitignore files to reject these.

Close apache/calcite#1471

Author: fib-seq

.gitignore

@@ -25,6 +25,7 @@ settings.xml
 .project
 .buildpath
 .classpath
+.factorypath
 .settings
 .checkstyle
 

core/src/main/java/org/apache/calcite/sql/SqlFunction.java

@@ -18,6 +18,8 @@
 
 import org.apache.calcite.linq4j.function.Functions;
 import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.sql.parser.SqlParserPos;
+import org.apache.calcite.sql.type.OperandTypes;
 import org.apache.calcite.sql.type.SqlOperandTypeChecker;
 import org.apache.calcite.sql.type.SqlOperandTypeInference;
 import org.apache.calcite.sql.type.SqlReturnTypeInference;
@@ -271,25 +273,37 @@ SqlSyntax.FUNCTION, getKind(),
         return validator.deriveConstructorType(scope, call, this, function,
             argTypes);
       }
+
+      validCoercionType:
       if (function == null) {
-        boolean changed = false;
         if (validator.isTypeCoercionEnabled()) {
           // try again if implicit type coercion is allowed.
-          function = (SqlFunction) SqlUtil.lookupRoutine(validator.getOperatorTable(),
-              getNameAsId(), argTypes, argNames, getFunctionType(), SqlSyntax.FUNCTION, getKind(),
-              validator.getCatalogReader().nameMatcher(),
-              true);
+          function = (SqlFunction)
+              SqlUtil.lookupRoutine(validator.getOperatorTable(), getNameAsId(),
+                  argTypes, argNames, getFunctionType(), SqlSyntax.FUNCTION,
+                  getKind(), validator.getCatalogReader().nameMatcher(), true);
           // try to coerce the function arguments to the declared sql type name.
           // if we succeed, the arguments would be wrapped with CAST operator.
           if (function != null) {
             TypeCoercion typeCoercion = validator.getTypeCoercion();
-            changed = typeCoercion.userDefinedFunctionCoercion(scope, call, function);
+            if (typeCoercion.userDefinedFunctionCoercion(scope, call, function)) {
+              break validCoercionType;
+            }
+          }
+          // if function doesn't exist within operator table and known function
+          // handling is turned off then create a more permissive function
+          if (function == null && validator.isLenientOperatorLookup()) {
+            final SqlFunction x = (SqlFunction) call.getOperator();
+            final SqlIdentifier identifier =
+                Util.first(x.getSqlIdentifier(),
+                    new SqlIdentifier(x.getName(), SqlParserPos.ZERO));
+            function = new SqlUnresolvedFunction(identifier, null,
+                null, OperandTypes.VARIADIC, null, x.getFunctionType());
+            break validCoercionType;
           }
         }
-        if (!changed) {
-          throw validator.handleUnresolvedFunction(call, this, argTypes,
-              argNames);
-        }
+        throw validator.handleUnresolvedFunction(call, this, argTypes,
+            argNames);
       }
 
       // REVIEW jvs 25-Mar-2005:  This is, in a sense, expanding

core/src/main/java/org/apache/calcite/sql/validate/SqlValidator.java

@@ -770,6 +770,27 @@ boolean validateModality(SqlSelect select, SqlModality modality,
 
   SqlValidatorScope getWithScope(SqlNode withItem);
 
+  /**
+   * Sets whether this validator should be lenient upon encountering an unknown
+   * function.
+   *
+   * @param lenient Whether to be lenient when encountering an unknown function
+   */
+  SqlValidator setLenientOperatorLookup(boolean lenient);
+
+  /** Returns whether this validator should be lenient upon encountering an
+   * unknown function.
+   *
+   * <p>If true, if a statement contains a call to a function that is not
+   * present in the operator table, or if the call does not have the required
+   * number or types of operands, the validator nevertheless regards the
+   * statement as valid. The type of the function call will be
+   * {@link #getUnknownType() UNKNOWN}.
+   *
+   * <p>If false (the default behavior), an unknown function call causes a
+   * validation error to be thrown. */
+  boolean isLenientOperatorLookup();
+
   /**
    * Set if implicit type coercion is allowed when the validator does validation.
    * See {@link org.apache.calcite.sql.validate.implicit.TypeCoercionImpl} for the details.

core/src/main/java/org/apache/calcite/sql/validate/SqlValidatorImpl.java

@@ -273,6 +273,8 @@ public class SqlValidatorImpl implements SqlValidatorWithHints {
 
   protected boolean expandColumnReferences;
 
+  protected boolean lenientOperatorLookup;
+
   private boolean rewriteCalls;
 
   private NullCollation nullCollation = NullCollation.HIGH;
@@ -326,6 +328,7 @@ protected SqlValidatorImpl(
     groupFinder = new AggFinder(opTab, false, false, true, null, nameMatcher);
     aggOrOverOrGroupFinder = new AggFinder(opTab, true, true, true, null,
         nameMatcher);
+    this.lenientOperatorLookup = false;
     this.enableTypeCoercion = catalogReader.getConfig() == null
         || catalogReader.getConfig().typeCoercion();
     this.typeCoercion = TypeCoercions.getTypeCoercion(this, conformance);
@@ -3832,6 +3835,15 @@ public SqlValidatorScope getWithScope(SqlNode withItem) {
     return scopes.get(withItem);
   }
 
+  public SqlValidator setLenientOperatorLookup(boolean lenient) {
+    this.lenientOperatorLookup = lenient;
+    return this;
+  }
+
+  public boolean isLenientOperatorLookup() {
+    return this.lenientOperatorLookup;
+  }
+
   public SqlValidator setEnableTypeCoercion(boolean enabled) {
     this.enableTypeCoercion = enabled;
     return this;

core/src/test/java/org/apache/calcite/sql/test/AbstractSqlTester.java

@@ -294,6 +294,10 @@ public SqlTester withCaseSensitive(boolean sensitive) {
     return with("caseSensitive", sensitive);
   }
 
+  public SqlTester withLenientOperatorLookup(boolean lenient) {
+    return with("lenientOperatorLookup", lenient);
+  }
+
   public SqlTester withLex(Lex lex) {
     return withQuoting(lex.quoting)
         .withCaseSensitive(lex.caseSensitive)

core/src/test/java/org/apache/calcite/sql/test/SqlTestFactory.java

@@ -60,6 +60,7 @@ public class SqlTestFactory {
           .put("quotedCasing", Casing.UNCHANGED)
           .put("unquotedCasing", Casing.TO_UPPER)
           .put("caseSensitive", true)
+          .put("lenientOperatorLookup", false)
           .put("enableTypeCoercion", true)
           .put("conformance", SqlConformanceEnum.DEFAULT)
           .put("operatorTable", SqlStdOperatorTable.instance())
@@ -130,12 +131,15 @@ public static SqlParser.Config createParserConfig(ImmutableMap<String, Object> o
   public SqlValidator getValidator() {
     final SqlConformance conformance =
         (SqlConformance) options.get("conformance");
+    final boolean lenientOperatorLookup =
+        (boolean) options.get("lenientOperatorLookup");
     final boolean enableTypeCoercion = (boolean) options.get("enableTypeCoercion");
     return validatorFactory.create(operatorTable.get(),
         catalogReader.get(),
         typeFactory.get(),
         conformance)
-        .setEnableTypeCoercion(enableTypeCoercion);
+        .setEnableTypeCoercion(enableTypeCoercion)
+        .setLenientOperatorLookup(lenientOperatorLookup);
   }
 
   public SqlAdvisor createAdvisor() {

core/src/test/java/org/apache/calcite/sql/test/SqlTester.java

@@ -83,6 +83,10 @@ enum VmName {
   /** Returns a tester that tests with implicit type coercion on/off. */
   SqlTester enableTypeCoercion(boolean enabled);
 
+  /** Returns a tester that does not fail validation if it encounters an
+   * unknown function. */
+  SqlTester withLenientOperatorLookup(boolean lenient);
+
   /** Returns a tester that gets connections from a given factory. */
   SqlTester withConnectionFactory(
       CalciteAssert.ConnectionFactory connectionFactory);

core/src/test/java/org/apache/calcite/test/SqlValidatorTest.java

@@ -1480,6 +1480,25 @@ public void _testLikeAndSimilarFails() {
         .fails("No match found for function signature FOO..");
   }
 
+  @Test public void testUnknownFunctionHandling() {
+    final Sql s = sql("?").withTester(t -> t.withLenientOperatorLookup(true));
+    s.expr("concat('a', 2)").ok();
+    s.expr("foo('2001-12-21')").ok();
+    s.expr("\"foo\"('b')").ok();
+    s.expr("foo()").ok();
+    s.expr("'a' || foo(bar('2001-12-21'))").ok();
+    s.expr("cast(foo(5, 2) as DECIMAL)").ok();
+    s.expr("select ascii('xyz')").ok();
+    s.expr("select get_bit(CAST('FFFF' as BINARY), 1)").ok();
+    s.expr("select now()").ok();
+    s.expr("^TIMESTAMP_CMP_TIMESTAMPTZ^").fails("(?s).*");
+    s.expr("atan(0)").ok();
+    s.expr("select row_number() over () from emp").ok();
+    s.expr("select coalesce(1, 2, 3)").ok();
+    s.sql("select count() from emp").ok(); // too few args
+    s.sql("select sum(1, 2) from emp").ok(); // too many args
+  }
+
   @Test public void testJdbcFunctionCall() {
     expr("{fn log10(1)}").ok();
     expr("{fn locate('','')}").ok();
@@ -9893,6 +9912,8 @@ private static int prec(SqlOperator op) {
 
   @Test public void testDummy() {
     // (To debug individual statements, paste them into this method.)
+    final Sql s = sql("?").withTester(t -> t.withLenientOperatorLookup(true));
+    s.sql("select count() from emp").ok();
   }
 
   @Test public void testCustomColumnResolving() {