Skip to content

Commit 19df560

Browse files
kalencayakalencaya
authored
[Feature][scaleph-ui-react] upgrade seatunnel connectors to 2.3.3 (#609)
* feature: upgrade seatunnel fakesource connectors to 2.3.3 * feature: upgrade seatunnel file connectors to 2.3.3 * feature: upgrade seatunnel http connectors to 2.3.3 * feature: upgrade seatunnel jdbc connectors to 2.3.3 * feature: upgrade seatunnel kafka source connector to 2.3.3 * feature: upgrade seatunnel kafka sink connector to 2.3.3 * temp commit: upgrade seatunnel mongodb connector to 2.3.3 * resolve conflicts
1 parent eb5a764 commit 19df560

File tree

50 files changed

+415
-289
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

50 files changed

+415
-289
lines changed

‎scaleph-common/src/main/java/cn/sliew/scaleph/common/dict/seatunnel/SeaTunnelPluginMapping.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,8 +86,8 @@ public enum SeaTunnelPluginMapping {
8686
SINK_REDIS(SEATUNNEL, SINK, REDIS, "connector-redis", BETA),
8787
SOURCE_ELASTICSEARCH(SEATUNNEL, SOURCE, ELASTICSEARCH, "connector-elasticsearch", UNKNOWN, BATCH, COLUMN_PROJECTION),
8888
SINK_ELASTICSEARCH(SEATUNNEL, SINK, ELASTICSEARCH, "connector-elasticsearch", GA, CDC),
89-
SOURCE_MONGODB(SEATUNNEL, SOURCE, MONGODB, "connector-mongodb", BETA, BATCH, COLUMN_PROJECTION),
90-
SINK_MONGODB(SEATUNNEL, SINK, MONGODB, "connector-mongodb", BETA, BATCH, STREAM),
89+
SOURCE_MONGODB(SEATUNNEL, SOURCE, MONGODB, "connector-mongodb", BETA, BATCH, EXACTLY_ONCE, COLUMN_PROJECTION, PARALLELISM, SUPPORT_USER_DEFINED_SPLIT),
90+
SINK_MONGODB(SEATUNNEL, SINK, MONGODB, "connector-mongodb", BETA, EXACTLY_ONCE, CDC),
9191
SOURCE_AMAZON_DYNAMODB(SEATUNNEL, SOURCE, AMAZON_DYNAMODB, "connector-amazondynamodb", BETA, BATCH, COLUMN_PROJECTION),
9292
SINK_AMAZON_DYNAMODB(SEATUNNEL, SINK, AMAZON_DYNAMODB, "connector-amazondynamodb", BETA),
9393
SOURCE_CASSANDRA(SEATUNNEL, SOURCE, CASSANDRA, "connector-cassandra", BETA, BATCH, COLUMN_PROJECTION),

‎scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/elasticsearch/sink/ElasticsearchSinkPlugin.java

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -54,12 +54,12 @@ public ElasticsearchSinkPlugin() {
5454
props.add(KEY_DELIMITER);
5555
props.add(MAX_RETRY_COUNT);
5656
props.add(MAX_BATCH_SIZE);
57-
// props.add(TLS_VERIFY_CERTIFICATE);
58-
// props.add(TLS_VERIFY_HOSTNAMES);
59-
// props.add(TLS_KEYSTORE_PATH);
60-
// props.add(TLS_KEYSTORE_PASSWORD);
61-
// props.add(TLS_TRUSTSTORE_PATH);
62-
// props.add(TLS_TRUSTSTORE_PASSWORD);
57+
props.add(TLS_VERIFY_CERTIFICATE);
58+
props.add(TLS_VERIFY_HOSTNAMES);
59+
props.add(TLS_KEYSTORE_PATH);
60+
props.add(TLS_KEYSTORE_PASSWORD);
61+
props.add(TLS_TRUSTSTORE_PATH);
62+
props.add(TLS_TRUSTSTORE_PASSWORD);
6363
props.add(CommonProperties.PARALLELISM);
6464
props.add(CommonProperties.SOURCE_TABLE_NAME);
6565
this.supportedProperties = props;

‎scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/email/sink/EmailSinkProperties.java

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
public enum EmailSinkProperties {
2424
;
2525

26-
public static final PropertyDescriptor<String> EMAIL_HOST = new PropertyDescriptor.Builder<String>()
26+
public static final PropertyDescriptor<String> EMAIL_HOST = new PropertyDescriptor.Builder()
2727
.name("email_host")
2828
.description("SMTP server host")
2929
.type(PropertyType.STRING)
@@ -32,7 +32,7 @@ public enum EmailSinkProperties {
3232
.addValidator(Validators.NON_BLANK_VALIDATOR)
3333
.validateAndBuild();
3434

35-
public static final PropertyDescriptor<String> EMAIL_TRANSPORT_PROTOCOL = new PropertyDescriptor.Builder<String>()
35+
public static final PropertyDescriptor<String> EMAIL_TRANSPORT_PROTOCOL = new PropertyDescriptor.Builder()
3636
.name("email_transport_protocol")
3737
.description("The protocol to load the session.")
3838
.type(PropertyType.STRING)
@@ -41,7 +41,7 @@ public enum EmailSinkProperties {
4141
.addValidator(Validators.NON_BLANK_VALIDATOR)
4242
.validateAndBuild();
4343

44-
public static final PropertyDescriptor<String> EMAIL_FROM_ADDRESS = new PropertyDescriptor.Builder<String>()
44+
public static final PropertyDescriptor<String> EMAIL_FROM_ADDRESS = new PropertyDescriptor.Builder()
4545
.name("email_from_address")
4646
.description("Sender Email Address")
4747
.type(PropertyType.STRING)
@@ -50,7 +50,7 @@ public enum EmailSinkProperties {
5050
.addValidator(Validators.NON_BLANK_VALIDATOR)
5151
.validateAndBuild();
5252

53-
public static final PropertyDescriptor<String> EMAIL_SMTP_AUTH = new PropertyDescriptor.Builder<String>()
53+
public static final PropertyDescriptor<String> EMAIL_SMTP_AUTH = new PropertyDescriptor.Builder()
5454
.name("email_smtp_auth")
5555
.description("Whether to authenticate the customer")
5656
.type(PropertyType.STRING)
@@ -59,7 +59,7 @@ public enum EmailSinkProperties {
5959
.addValidator(Validators.NON_BLANK_VALIDATOR)
6060
.validateAndBuild();
6161

62-
public static final PropertyDescriptor<String> EMAIL_AUTHORIZATION_CODE = new PropertyDescriptor.Builder<String>()
62+
public static final PropertyDescriptor<String> EMAIL_AUTHORIZATION_CODE = new PropertyDescriptor.Builder()
6363
.name("email_authorization_code")
6464
.description("authorization code,You can obtain the authorization code from the mailbox Settings.")
6565
.type(PropertyType.STRING)
@@ -68,7 +68,7 @@ public enum EmailSinkProperties {
6868
.addValidator(Validators.NON_BLANK_VALIDATOR)
6969
.validateAndBuild();
7070

71-
public static final PropertyDescriptor<String> EMAIL_TO_ADDRESS = new PropertyDescriptor.Builder<String>()
71+
public static final PropertyDescriptor<String> EMAIL_TO_ADDRESS = new PropertyDescriptor.Builder()
7272
.name("email_to_address")
7373
.description("Receiver Email Address")
7474
.type(PropertyType.STRING)
@@ -77,7 +77,7 @@ public enum EmailSinkProperties {
7777
.addValidator(Validators.NON_BLANK_VALIDATOR)
7878
.validateAndBuild();
7979

80-
public static final PropertyDescriptor<String> EMAIL_MESSAGE_HEADLINE = new PropertyDescriptor.Builder<String>()
80+
public static final PropertyDescriptor<String> EMAIL_MESSAGE_HEADLINE = new PropertyDescriptor.Builder()
8181
.name("email_message_headline")
8282
.description("The subject line of the entire message.")
8383
.type(PropertyType.STRING)
@@ -86,7 +86,7 @@ public enum EmailSinkProperties {
8686
.addValidator(Validators.NON_BLANK_VALIDATOR)
8787
.validateAndBuild();
8888

89-
public static final PropertyDescriptor<String> EMAIL_MESSAGE_CONTENT = new PropertyDescriptor.Builder<String>()
89+
public static final PropertyDescriptor<String> EMAIL_MESSAGE_CONTENT = new PropertyDescriptor.Builder()
9090
.name("email_message_content")
9191
.description("The body of the entire message.")
9292
.type(PropertyType.STRING)

‎scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/fake/source/FakeProperties.java

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -341,4 +341,53 @@ public enum FakeProperties {
341341
.addValidator(Validators.NON_BLANK_VALIDATOR)
342342
.validateAndBuild();
343343

344+
345+
public static final PropertyDescriptor<JsonNode> DATE_YEAR_TEMPLATE = new PropertyDescriptor.Builder()
346+
.name("date.year.template")
347+
.description("The template list of year of date like 'yyyy', if user configured it, connector will randomly select an item from the template list")
348+
.type(PropertyType.OBJECT)
349+
.parser(Parsers.JSON_PARSER)
350+
.addValidator(Validators.NON_BLANK_VALIDATOR)
351+
.validateAndBuild();
352+
353+
public static final PropertyDescriptor<JsonNode> DATE_MONTH_TEMPLATE = new PropertyDescriptor.Builder()
354+
.name("date.month.template")
355+
.description("The template list of month of date like 'MM', if user configured it, connector will randomly select an item from the template list")
356+
.type(PropertyType.OBJECT)
357+
.parser(Parsers.JSON_PARSER)
358+
.addValidator(Validators.NON_BLANK_VALIDATOR)
359+
.validateAndBuild();
360+
361+
public static final PropertyDescriptor<JsonNode> DATE_DAY_TEMPLATE = new PropertyDescriptor.Builder()
362+
.name("date.day.template")
363+
.description("The template list of day of date like 'dd', if user configured it, connector will randomly select an item from the template list")
364+
.type(PropertyType.OBJECT)
365+
.parser(Parsers.JSON_PARSER)
366+
.addValidator(Validators.NON_BLANK_VALIDATOR)
367+
.validateAndBuild();
368+
369+
public static final PropertyDescriptor<JsonNode> TIME_HOUR_TEMPLATE = new PropertyDescriptor.Builder()
370+
.name("time.hour.template")
371+
.description("The template list of hour of time like 'HH', if user configured it, connector will randomly select an item from the template list")
372+
.type(PropertyType.OBJECT)
373+
.parser(Parsers.JSON_PARSER)
374+
.addValidator(Validators.NON_BLANK_VALIDATOR)
375+
.validateAndBuild();
376+
377+
public static final PropertyDescriptor<JsonNode> TIME_MINUTE_TEMPLATE = new PropertyDescriptor.Builder()
378+
.name("time.minute.template")
379+
.description("The template list of minute of time like 'mm', if user configured it, connector will randomly select an item from the template list")
380+
.type(PropertyType.OBJECT)
381+
.parser(Parsers.JSON_PARSER)
382+
.addValidator(Validators.NON_BLANK_VALIDATOR)
383+
.validateAndBuild();
384+
385+
public static final PropertyDescriptor<JsonNode> TIME_SECOND_TEMPLATE = new PropertyDescriptor.Builder()
386+
.name("time.second.template")
387+
.description("The template list of second of time like 'ss', if user configured it, connector will randomly select an item from the template list")
388+
.type(PropertyType.OBJECT)
389+
.parser(Parsers.JSON_PARSER)
390+
.addValidator(Validators.NON_BLANK_VALIDATOR)
391+
.validateAndBuild();
392+
344393
}

‎scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/fake/source/FakeSourcePlugin.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,13 @@ public FakeSourcePlugin() {
7676
props.add(DOUBLE_MIN);
7777
props.add(DOUBLE_MAX);
7878
props.add(DOUBLE_TEMPLATE);
79+
80+
props.add(DATE_YEAR_TEMPLATE);
81+
props.add(DATE_MONTH_TEMPLATE);
82+
props.add(DATE_DAY_TEMPLATE);
83+
props.add(TIME_HOUR_TEMPLATE);
84+
props.add(TIME_MINUTE_TEMPLATE);
85+
props.add(TIME_SECOND_TEMPLATE);
7986
props.add(CommonProperties.PARALLELISM);
8087
props.add(CommonProperties.RESULT_TABLE_NAME);
8188
supportedProperties = Collections.unmodifiableList(props);

‎scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/FileSinkProperties.java

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,4 +154,12 @@ public enum FileSinkProperties {
154154
.addValidator(Validators.POSITIVE_INTEGER_VALIDATOR)
155155
.validateAndBuild();
156156

157+
public static final PropertyDescriptor<Integer> TMP_PATH = new PropertyDescriptor.Builder()
158+
.name("tmp_path")
159+
.description("Data write temporary path")
160+
.type(PropertyType.STRING)
161+
.parser(Parsers.STRING_PARSER)
162+
.addValidator(Validators.NON_BLANK_VALIDATOR)
163+
.validateAndBuild();
164+
157165
}

‎scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/FileSourceProperties.java

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,13 @@
2323

2424
public enum FileSourceProperties {
2525
;
26+
public static final PropertyDescriptor<String> FILE_FILTER_PATTERN = new PropertyDescriptor.Builder<String>()
27+
.name("file_filter_pattern")
28+
.description("File pattern. The connector will filter some files base on the pattern.")
29+
.type(PropertyType.STRING)
30+
.parser(Parsers.STRING_PARSER)
31+
.addValidator(Validators.NON_BLANK_VALIDATOR)
32+
.validateAndBuild();
2633

2734
public static final PropertyDescriptor<String> FILE_FORMAT_TYPE = new PropertyDescriptor.Builder<String>()
2835
.name("file_format_type")
@@ -50,6 +57,14 @@ public enum FileSourceProperties {
5057
.addValidator(Validators.NON_BLANK_VALIDATOR)
5158
.validateAndBuild();
5259

60+
public static final PropertyDescriptor<JsonNode> READ_PARTITIONS = new PropertyDescriptor.Builder()
61+
.name("read_partitions")
62+
.description("The partitions that the user want to read")
63+
.type(PropertyType.OBJECT)
64+
.parser(Parsers.JSON_PARSER)
65+
.addValidator(Validators.NON_BLANK_VALIDATOR)
66+
.validateAndBuild();
67+
5368
public static final PropertyDescriptor<JsonNode> SCHEMA = new PropertyDescriptor.Builder()
5469
.name("schema")
5570
.description("The schema information of upstream data.")

‎scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/ftp/source/FtpFileSourcePlugin.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ public FtpFileSourcePlugin() {
5252
props.add(PATH);
5353
props.add(SKIP_HEADER_ROW_NUMBER);
5454
props.add(SHEET_NAME);
55+
props.add(FILE_FILTER_PATTERN);
5556
props.add(FILE_FORMAT_TYPE);
5657
props.add(READ_COLUMNS);
5758
props.add(SCHEMA);

‎scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/hdfs/source/HDFSFileSourcePlugin.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ public HDFSFileSourcePlugin() {
5454
props.add(PATH);
5555
props.add(SKIP_HEADER_ROW_NUMBER);
5656
props.add(SHEET_NAME);
57+
props.add(FILE_FILTER_PATTERN);
5758
props.add(FILE_FORMAT_TYPE);
5859
props.add(READ_COLUMNS);
5960
props.add(SCHEMA);

‎scaleph-plugins/scaleph-plugin-seatunnel-connectors/src/main/java/cn/sliew/scaleph/plugin/seatunnel/flink/connectors/file/local/source/LocalFileSourcePlugin.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ public LocalFileSourcePlugin() {
4545
props.add(PATH);
4646
props.add(SKIP_HEADER_ROW_NUMBER);
4747
props.add(SHEET_NAME);
48+
props.add(FILE_FILTER_PATTERN);
4849
props.add(FILE_FORMAT_TYPE);
4950
props.add(READ_COLUMNS);
5051
props.add(SCHEMA);

0 commit comments

Comments
 (0)