[Feature][scaleph-ui-react] upgrade seatunnel connectors to 2.3.3 (#609)

* feature: upgrade seatunnel fakesource connectors to 2.3.3

* feature: upgrade seatunnel file connectors to 2.3.3

* feature: upgrade seatunnel http connectors to 2.3.3

* feature: upgrade seatunnel jdbc connectors to 2.3.3

* feature: upgrade seatunnel kafka source connector to 2.3.3

* feature: upgrade seatunnel kafka sink connector to 2.3.3

* temp commit: upgrade seatunnel mongodb connector to 2.3.3

* resolve conflicts

scaleph-common/src/main/java/cn/sliew/scaleph/common/dict/seatunnel/SeaTunnelPluginMapping.java

@@ -86,8 +86,8 @@ public enum SeaTunnelPluginMapping {
     SINK_REDIS(SEATUNNEL, SINK, REDIS, "connector-redis", BETA),
     SOURCE_ELASTICSEARCH(SEATUNNEL, SOURCE, ELASTICSEARCH, "connector-elasticsearch", UNKNOWN, BATCH, COLUMN_PROJECTION),
     SINK_ELASTICSEARCH(SEATUNNEL, SINK, ELASTICSEARCH, "connector-elasticsearch", GA, CDC),
-    SOURCE_MONGODB(SEATUNNEL, SOURCE, MONGODB, "connector-mongodb", BETA, BATCH, COLUMN_PROJECTION),
-    SINK_MONGODB(SEATUNNEL, SINK, MONGODB, "connector-mongodb", BETA, BATCH, STREAM),
+    SOURCE_MONGODB(SEATUNNEL, SOURCE, MONGODB, "connector-mongodb", BETA, BATCH, EXACTLY_ONCE, COLUMN_PROJECTION, PARALLELISM, SUPPORT_USER_DEFINED_SPLIT),
+    SINK_MONGODB(SEATUNNEL, SINK, MONGODB, "connector-mongodb", BETA, EXACTLY_ONCE, CDC),
     SOURCE_AMAZON_DYNAMODB(SEATUNNEL, SOURCE, AMAZON_DYNAMODB, "connector-amazondynamodb", BETA, BATCH, COLUMN_PROJECTION),
     SINK_AMAZON_DYNAMODB(SEATUNNEL, SINK, AMAZON_DYNAMODB, "connector-amazondynamodb", BETA),
     SOURCE_CASSANDRA(SEATUNNEL, SOURCE, CASSANDRA, "connector-cassandra", BETA, BATCH, COLUMN_PROJECTION),

scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/elasticsearch/sink/ElasticsearchSinkPlugin.java

@@ -54,12 +54,12 @@ public ElasticsearchSinkPlugin() {
         props.add(KEY_DELIMITER);
         props.add(MAX_RETRY_COUNT);
         props.add(MAX_BATCH_SIZE);
-//        props.add(TLS_VERIFY_CERTIFICATE);
-//        props.add(TLS_VERIFY_HOSTNAMES);
-//        props.add(TLS_KEYSTORE_PATH);
-//        props.add(TLS_KEYSTORE_PASSWORD);
-//        props.add(TLS_TRUSTSTORE_PATH);
-//        props.add(TLS_TRUSTSTORE_PASSWORD);
+        props.add(TLS_VERIFY_CERTIFICATE);
+        props.add(TLS_VERIFY_HOSTNAMES);
+        props.add(TLS_KEYSTORE_PATH);
+        props.add(TLS_KEYSTORE_PASSWORD);
+        props.add(TLS_TRUSTSTORE_PATH);
+        props.add(TLS_TRUSTSTORE_PASSWORD);
         props.add(CommonProperties.PARALLELISM);
         props.add(CommonProperties.SOURCE_TABLE_NAME);
         this.supportedProperties = props;

scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/email/sink/EmailSinkProperties.java

@@ -23,7 +23,7 @@
 public enum EmailSinkProperties {
     ;
 
-    public static final PropertyDescriptor<String> EMAIL_HOST = new PropertyDescriptor.Builder<String>()
+    public static final PropertyDescriptor<String> EMAIL_HOST = new PropertyDescriptor.Builder()
             .name("email_host")
             .description("SMTP server host")
             .type(PropertyType.STRING)
@@ -32,7 +32,7 @@ public enum EmailSinkProperties {
             .addValidator(Validators.NON_BLANK_VALIDATOR)
             .validateAndBuild();
 
-    public static final PropertyDescriptor<String> EMAIL_TRANSPORT_PROTOCOL = new PropertyDescriptor.Builder<String>()
+    public static final PropertyDescriptor<String> EMAIL_TRANSPORT_PROTOCOL = new PropertyDescriptor.Builder()
             .name("email_transport_protocol")
             .description("The protocol to load the session.")
             .type(PropertyType.STRING)
@@ -41,7 +41,7 @@ public enum EmailSinkProperties {
             .addValidator(Validators.NON_BLANK_VALIDATOR)
             .validateAndBuild();
 
-    public static final PropertyDescriptor<String> EMAIL_FROM_ADDRESS = new PropertyDescriptor.Builder<String>()
+    public static final PropertyDescriptor<String> EMAIL_FROM_ADDRESS = new PropertyDescriptor.Builder()
             .name("email_from_address")
             .description("Sender Email Address")
             .type(PropertyType.STRING)
@@ -50,7 +50,7 @@ public enum EmailSinkProperties {
             .addValidator(Validators.NON_BLANK_VALIDATOR)
             .validateAndBuild();
 
-    public static final PropertyDescriptor<String> EMAIL_SMTP_AUTH = new PropertyDescriptor.Builder<String>()
+    public static final PropertyDescriptor<String> EMAIL_SMTP_AUTH = new PropertyDescriptor.Builder()
             .name("email_smtp_auth")
             .description("Whether to authenticate the customer")
             .type(PropertyType.STRING)
@@ -59,7 +59,7 @@ public enum EmailSinkProperties {
             .addValidator(Validators.NON_BLANK_VALIDATOR)
             .validateAndBuild();
 
-    public static final PropertyDescriptor<String> EMAIL_AUTHORIZATION_CODE = new PropertyDescriptor.Builder<String>()
+    public static final PropertyDescriptor<String> EMAIL_AUTHORIZATION_CODE = new PropertyDescriptor.Builder()
             .name("email_authorization_code")
             .description("authorization code,You can obtain the authorization code from the mailbox Settings.")
             .type(PropertyType.STRING)
@@ -68,7 +68,7 @@ public enum EmailSinkProperties {
             .addValidator(Validators.NON_BLANK_VALIDATOR)
             .validateAndBuild();
 
-    public static final PropertyDescriptor<String> EMAIL_TO_ADDRESS = new PropertyDescriptor.Builder<String>()
+    public static final PropertyDescriptor<String> EMAIL_TO_ADDRESS = new PropertyDescriptor.Builder()
             .name("email_to_address")
             .description("Receiver Email Address")
             .type(PropertyType.STRING)
@@ -77,7 +77,7 @@ public enum EmailSinkProperties {
             .addValidator(Validators.NON_BLANK_VALIDATOR)
             .validateAndBuild();
 
-    public static final PropertyDescriptor<String> EMAIL_MESSAGE_HEADLINE = new PropertyDescriptor.Builder<String>()
+    public static final PropertyDescriptor<String> EMAIL_MESSAGE_HEADLINE = new PropertyDescriptor.Builder()
             .name("email_message_headline")
             .description("The subject line of the entire message.")
             .type(PropertyType.STRING)
@@ -86,7 +86,7 @@ public enum EmailSinkProperties {
             .addValidator(Validators.NON_BLANK_VALIDATOR)
             .validateAndBuild();
 
-    public static final PropertyDescriptor<String> EMAIL_MESSAGE_CONTENT = new PropertyDescriptor.Builder<String>()
+    public static final PropertyDescriptor<String> EMAIL_MESSAGE_CONTENT = new PropertyDescriptor.Builder()
             .name("email_message_content")
             .description("The body of the entire message.")
             .type(PropertyType.STRING)

scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/fake/source/FakeProperties.java

@@ -341,4 +341,53 @@ public enum FakeProperties {
             .addValidator(Validators.NON_BLANK_VALIDATOR)
             .validateAndBuild();
 
+
+    public static final PropertyDescriptor<JsonNode> DATE_YEAR_TEMPLATE = new PropertyDescriptor.Builder()
+            .name("date.year.template")
+            .description("The template list of year of date like 'yyyy', if user configured it, connector will randomly select an item from the template list")
+            .type(PropertyType.OBJECT)
+            .parser(Parsers.JSON_PARSER)
+            .addValidator(Validators.NON_BLANK_VALIDATOR)
+            .validateAndBuild();
+
+    public static final PropertyDescriptor<JsonNode> DATE_MONTH_TEMPLATE = new PropertyDescriptor.Builder()
+            .name("date.month.template")
+            .description("The template list of month of date like 'MM', if user configured it, connector will randomly select an item from the template list")
+            .type(PropertyType.OBJECT)
+            .parser(Parsers.JSON_PARSER)
+            .addValidator(Validators.NON_BLANK_VALIDATOR)
+            .validateAndBuild();
+
+    public static final PropertyDescriptor<JsonNode> DATE_DAY_TEMPLATE = new PropertyDescriptor.Builder()
+            .name("date.day.template")
+            .description("The template list of day of date like 'dd', if user configured it, connector will randomly select an item from the template list")
+            .type(PropertyType.OBJECT)
+            .parser(Parsers.JSON_PARSER)
+            .addValidator(Validators.NON_BLANK_VALIDATOR)
+            .validateAndBuild();
+
+    public static final PropertyDescriptor<JsonNode> TIME_HOUR_TEMPLATE = new PropertyDescriptor.Builder()
+            .name("time.hour.template")
+            .description("The template list of hour of time like 'HH', if user configured it, connector will randomly select an item from the template list")
+            .type(PropertyType.OBJECT)
+            .parser(Parsers.JSON_PARSER)
+            .addValidator(Validators.NON_BLANK_VALIDATOR)
+            .validateAndBuild();
+
+    public static final PropertyDescriptor<JsonNode> TIME_MINUTE_TEMPLATE = new PropertyDescriptor.Builder()
+            .name("time.minute.template")
+            .description("The template list of minute of time like 'mm', if user configured it, connector will randomly select an item from the template list")
+            .type(PropertyType.OBJECT)
+            .parser(Parsers.JSON_PARSER)
+            .addValidator(Validators.NON_BLANK_VALIDATOR)
+            .validateAndBuild();
+
+    public static final PropertyDescriptor<JsonNode> TIME_SECOND_TEMPLATE = new PropertyDescriptor.Builder()
+            .name("time.second.template")
+            .description("The template list of second of time like 'ss', if user configured it, connector will randomly select an item from the template list")
+            .type(PropertyType.OBJECT)
+            .parser(Parsers.JSON_PARSER)
+            .addValidator(Validators.NON_BLANK_VALIDATOR)
+            .validateAndBuild();
+
 }

scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/fake/source/FakeSourcePlugin.java

@@ -76,6 +76,13 @@ public FakeSourcePlugin() {
         props.add(DOUBLE_MIN);
         props.add(DOUBLE_MAX);
         props.add(DOUBLE_TEMPLATE);
+
+        props.add(DATE_YEAR_TEMPLATE);
+        props.add(DATE_MONTH_TEMPLATE);
+        props.add(DATE_DAY_TEMPLATE);
+        props.add(TIME_HOUR_TEMPLATE);
+        props.add(TIME_MINUTE_TEMPLATE);
+        props.add(TIME_SECOND_TEMPLATE);
         props.add(CommonProperties.PARALLELISM);
         props.add(CommonProperties.RESULT_TABLE_NAME);
         supportedProperties = Collections.unmodifiableList(props);

scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/FileSinkProperties.java

@@ -154,4 +154,12 @@ public enum FileSinkProperties {
             .addValidator(Validators.POSITIVE_INTEGER_VALIDATOR)
             .validateAndBuild();
 
+    public static final PropertyDescriptor<Integer> TMP_PATH = new PropertyDescriptor.Builder()
+            .name("tmp_path")
+            .description("Data write temporary path")
+            .type(PropertyType.STRING)
+            .parser(Parsers.STRING_PARSER)
+            .addValidator(Validators.NON_BLANK_VALIDATOR)
+            .validateAndBuild();
+
 }

scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/FileSourceProperties.java

@@ -23,6 +23,13 @@
 
 public enum FileSourceProperties {
     ;
+    public static final PropertyDescriptor<String> FILE_FILTER_PATTERN = new PropertyDescriptor.Builder<String>()
+            .name("file_filter_pattern")
+            .description("File pattern. The connector will filter some files base on the pattern.")
+            .type(PropertyType.STRING)
+            .parser(Parsers.STRING_PARSER)
+            .addValidator(Validators.NON_BLANK_VALIDATOR)
+            .validateAndBuild();
 
     public static final PropertyDescriptor<String> FILE_FORMAT_TYPE = new PropertyDescriptor.Builder<String>()
             .name("file_format_type")
@@ -50,6 +57,14 @@ public enum FileSourceProperties {
             .addValidator(Validators.NON_BLANK_VALIDATOR)
             .validateAndBuild();
 
+    public static final PropertyDescriptor<JsonNode> READ_PARTITIONS = new PropertyDescriptor.Builder()
+            .name("read_partitions")
+            .description("The partitions that the user want to read")
+            .type(PropertyType.OBJECT)
+            .parser(Parsers.JSON_PARSER)
+            .addValidator(Validators.NON_BLANK_VALIDATOR)
+            .validateAndBuild();
+
     public static final PropertyDescriptor<JsonNode> SCHEMA = new PropertyDescriptor.Builder()
             .name("schema")
             .description("The schema information of upstream data.")

scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/ftp/source/FtpFileSourcePlugin.java

@@ -52,6 +52,7 @@ public FtpFileSourcePlugin() {
         props.add(PATH);
         props.add(SKIP_HEADER_ROW_NUMBER);
         props.add(SHEET_NAME);
+        props.add(FILE_FILTER_PATTERN);
         props.add(FILE_FORMAT_TYPE);
         props.add(READ_COLUMNS);
         props.add(SCHEMA);

scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/hdfs/source/HDFSFileSourcePlugin.java

@@ -54,6 +54,7 @@ public HDFSFileSourcePlugin() {
         props.add(PATH);
         props.add(SKIP_HEADER_ROW_NUMBER);
         props.add(SHEET_NAME);
+        props.add(FILE_FILTER_PATTERN);
         props.add(FILE_FORMAT_TYPE);
         props.add(READ_COLUMNS);
         props.add(SCHEMA);

scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/local/source/LocalFileSourcePlugin.java

@@ -45,6 +45,7 @@ public LocalFileSourcePlugin() {
         props.add(PATH);
         props.add(SKIP_HEADER_ROW_NUMBER);
         props.add(SHEET_NAME);
+        props.add(FILE_FILTER_PATTERN);
         props.add(FILE_FORMAT_TYPE);
         props.add(READ_COLUMNS);
         props.add(SCHEMA);

scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/oss/source/OSSSourcePlugin.java

@@ -52,6 +52,7 @@ public OSSSourcePlugin() {
         props.add(PATH);
         props.add(SKIP_HEADER_ROW_NUMBER);
         props.add(SHEET_NAME);
+        props.add(FILE_FILTER_PATTERN);
         props.add(FILE_FORMAT_TYPE);
         props.add(READ_COLUMNS);
         props.add(SCHEMA);

scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/ossjindo/source/OSSJindoSourcePlugin.java

@@ -52,6 +52,7 @@ public OSSJindoSourcePlugin() {
         props.add(PATH);
         props.add(SKIP_HEADER_ROW_NUMBER);
         props.add(SHEET_NAME);
+        props.add(FILE_FILTER_PATTERN);
         props.add(FILE_FORMAT_TYPE);
         props.add(READ_COLUMNS);
         props.add(SCHEMA);

scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/s3/source/S3SourcePlugin.java

@@ -53,6 +53,7 @@ public S3SourcePlugin() {
         props.add(PATH);
         props.add(SKIP_HEADER_ROW_NUMBER);
         props.add(SHEET_NAME);
+        props.add(FILE_FILTER_PATTERN);
         props.add(FILE_FORMAT_TYPE);
         props.add(READ_COLUMNS);
         props.add(SCHEMA);

scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/sftp/source/SftpFileSourcePlugin.java

@@ -52,6 +52,7 @@ public SftpFileSourcePlugin() {
         props.add(PATH);
         props.add(SKIP_HEADER_ROW_NUMBER);
         props.add(SHEET_NAME);
+        props.add(FILE_FILTER_PATTERN);
         props.add(FILE_FORMAT_TYPE);
         props.add(READ_COLUMNS);
         props.add(SCHEMA);

scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hbase/sink/HbaseSinkProperties.java

@@ -61,6 +61,7 @@ public enum HbaseSinkProperties {
             .description("The mode of writing null value")
             .type(PropertyType.STRING)
             .parser(Parsers.STRING_PARSER)
+            .allowableValues("skip", "empty")
             .addValidator(Validators.NON_BLANK_VALIDATOR)
             .validateAndBuild();
 
@@ -86,6 +87,7 @@ public enum HbaseSinkProperties {
             .description("The encoding of string field")
             .type(PropertyType.STRING)
             .parser(Parsers.STRING_PARSER)
+            .allowableValues("utf8", "gbk")
             .addValidator(Validators.NON_BLANK_VALIDATOR)
             .validateAndBuild();
 

scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/hive/source/HiveSourcePlugin.java

@@ -24,6 +24,7 @@
 import cn.sliew.scaleph.plugin.framework.core.PluginInfo;
 import cn.sliew.scaleph.plugin.framework.property.PropertyDescriptor;
 import cn.sliew.scaleph.plugin.seatunnel.flink.SeaTunnelConnectorPlugin;
+import cn.sliew.scaleph.plugin.seatunnel.flink.connectors.file.FileSourceProperties;
 import cn.sliew.scaleph.plugin.seatunnel.flink.env.CommonProperties;
 import cn.sliew.scaleph.plugin.seatunnel.flink.resource.ResourceProperties;
 import cn.sliew.scaleph.plugin.seatunnel.flink.resource.ResourceProperty;
@@ -37,8 +38,6 @@
 import java.util.List;
 
 import static cn.sliew.scaleph.plugin.seatunnel.flink.connectors.hive.HiveProperties.*;
-import static cn.sliew.scaleph.plugin.seatunnel.flink.connectors.hive.source.HiveSourceProperties.READ_COLUMNS;
-import static cn.sliew.scaleph.plugin.seatunnel.flink.connectors.hive.source.HiveSourceProperties.READ_PARTITIONS;
 
 @AutoService(SeaTunnelConnectorPlugin.class)
 public class HiveSourcePlugin extends SeaTunnelConnectorPlugin {
@@ -50,8 +49,8 @@ public HiveSourcePlugin() {
 
         final List<PropertyDescriptor> props = new ArrayList<>();
         props.add(TABLE_NAME);
-        props.add(READ_PARTITIONS);
-        props.add(READ_COLUMNS);
+        props.add(FileSourceProperties.READ_COLUMNS);
+        props.add(FileSourceProperties.READ_PARTITIONS);
         props.add(CommonProperties.PARALLELISM);
         props.add(CommonProperties.RESULT_TABLE_NAME);
         supportedProperties = Collections.unmodifiableList(props);

scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/iotdb/IoTDBProperties.java

@@ -51,4 +51,11 @@ public enum IoTDBProperties {
             .addValidator(Validators.NON_BLANK_VALIDATOR)
             .validateAndBuild();
 
+    public static final PropertyDescriptor<Integer> CONNECTION_TIMEOUT_IN_MS = new PropertyDescriptor.Builder()
+            .name("connection_timeout_in_ms")
+            .description("The maximum time (in ms) to wait when connect IoTDB")
+            .type(PropertyType.INT)
+            .parser(Parsers.INTEGER_PARSER)
+            .addValidator(Validators.POSITIVE_INTEGER_VALIDATOR)
+            .validateAndBuild();
 }

scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/iotdb/sink/IoTDBSinkProperties.java

@@ -129,13 +129,5 @@ public enum IoTDBSinkProperties {
             .parser(Parsers.BOOLEAN_PARSER)
             .addValidator(Validators.BOOLEAN_VALIDATOR)
             .validateAndBuild();
-
-    public static final PropertyDescriptor<Integer> CONNECTION_TIMEOUT_IN_MS = new PropertyDescriptor.Builder()
-            .name("connection_timeout_in_ms")
-            .description("The maximum time (in ms) to wait when connect IoTDB")
-            .type(PropertyType.INT)
-            .parser(Parsers.INTEGER_PARSER)
-            .addValidator(Validators.POSITIVE_INTEGER_VALIDATOR)
-            .validateAndBuild();
-
+
 }

scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/jdbc/JdbcProperties.java

@@ -73,4 +73,12 @@ public enum JdbcProperties {
             .parser(Parsers.INTEGER_PARSER)
             .addValidator(Validators.POSITIVE_INTEGER_VALIDATOR)
             .validateAndBuild();
+
+    public static final PropertyDescriptor<String> COMPATIBLE_MODE = new PropertyDescriptor.Builder<String>()
+            .name("compatible_mode")
+            .description("The compatible mode of database")
+            .type(PropertyType.STRING)
+            .parser(Parsers.STRING_PARSER)
+            .addValidator(Validators.NON_BLANK_VALIDATOR)
+            .validateAndBuild();
 }