diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b4b1d8e39d..508014de35 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -21,18 +21,23 @@ updates: directory: "/" open-pull-requests-limit: 15 schedule: - interval: "weekly" + interval: "monthly" ignore: - dependency-name: "*" - update-types: [ "version-update:semver-major" ] + update-types: [ "version-update:semver-major", "version-update:semver-patch" ] + - dependency-name: "software.amazon.awssdk:s3" + update-types: [ "version-update:semver-patch" ] + - dependency-name: "com.aliyun:dingtalk" + update-types: [ "version-update:semver-patch" ] - package-ecosystem: "gomod" directory: "eventmesh-sdks/eventmesh-sdk-go" + # Disabled temporarily since the Go SDK is not integrated with CI + open-pull-requests-limit: 0 schedule: interval: "monthly" ignore: - dependency-name: "*" - # Disabled temporarily since the Go SDK is not integrated with CI - update-types: [ "version-update:semver-major", "version-update:semver-minor", "version-update:semver-patch" ] + update-types: [ "version-update:semver-major", "version-update:semver-patch" ] - package-ecosystem: "github-actions" directory: "/" schedule: diff --git a/.github/workflows/auto-dependabot.yml b/.github/workflows/auto-dependabot.yml index 0d43a05866..512a43f6ec 100644 --- a/.github/workflows/auto-dependabot.yml +++ b/.github/workflows/auto-dependabot.yml @@ -27,7 +27,7 @@ jobs: # Pull request Auto merge is not enabled for this repository dependabot: runs-on: ubuntu-latest - if: github.actor == 'dependabot[bot]' + if: github.event.pull_request.user.login == 'dependabot[bot]' steps: - name: Dependabot metadata id: metadata diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ce5eae95b7..d63c381107 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,16 +39,15 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + submodules: true - - if: matrix.language == 'cpp' || matrix.language == 'csharp' - name: Build C - run: | - git submodule init - git submodule update - make -C ./eventmesh-sdks/eventmesh-sdk-c + - name: Build C SDK + if: matrix.language == 'cpp' + run: make -C ./eventmesh-sdks/eventmesh-sdk-c - name: Setup Gradle - uses: gradle/actions/setup-gradle@v3 + uses: gradle/actions/setup-gradle@v4 - name: Set up JDK 11 uses: actions/setup-java@v4 @@ -59,7 +58,7 @@ jobs: - name: GenerateGrammarSource run: ./gradlew clean generateGrammarSource --parallel --daemon --scan env: - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - name: Set up JDK ${{ matrix.java }} uses: actions/setup-java@v4 @@ -69,14 +68,16 @@ jobs: # https://docs.gradle.org/current/userguide/performance.html - name: Build - run: ./gradlew clean build dist jacocoTestReport -x spotlessJava -x generateGrammarSource --parallel --daemon --scan + run: > + ./gradlew clean build dist jacocoTestReport --parallel --daemon --scan + -x spotlessJava -x generateGrammarSource -x generateDistLicense -x checkDeniedLicense env: - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - name: Install plugin run: ./gradlew installPlugin --scan env: - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - name: Upload coverage report to codecov.io run: bash <(curl -s https://codecov.io/bash) || echo 'Failed to upload coverage report!' diff --git a/.github/workflows/code-scanning.yml b/.github/workflows/code-scanning.yml index 0a7b2ecb28..5476923b0a 100644 --- a/.github/workflows/code-scanning.yml +++ b/.github/workflows/code-scanning.yml @@ -60,7 +60,7 @@ jobs: - name: Setup Gradle if: matrix.language == 'java' - uses: gradle/actions/setup-gradle@v3 + uses: gradle/actions/setup-gradle@v4 with: cache-disabled: true @@ -68,7 +68,7 @@ jobs: if: matrix.language == 'java' run: ./gradlew clean assemble compileTestJava --parallel --daemon --scan env: - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} - name: Perform CodeQL analysis uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index dfb36970d0..171c787369 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -41,10 +41,10 @@ jobs: apache/eventmesh - name: Build and push - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} file: ./docker/Dockerfile_jdk8 - context: ./ + context: ./ \ No newline at end of file diff --git a/.github/workflows/license.yml b/.github/workflows/license.yml index ca0e0936a6..9172e85414 100644 --- a/.github/workflows/license.yml +++ b/.github/workflows/license.yml @@ -38,7 +38,7 @@ jobs: java-version: 11 - name: Setup Gradle - uses: gradle/actions/setup-gradle@v3 + uses: gradle/actions/setup-gradle@v4 - name: Check license compatibility run: ./gradlew clean checkDeniedLicense diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index c1e4f4ab15..8b5b166215 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -42,7 +42,7 @@ jobs: It has been 60 days since the last activity on this pull request. I am reaching out here to gently remind you that the Apache EventMesh community values every pull request, and please feel free to get in touch with the reviewers at any time. They are available to assist you in advancing the progress of your pull request and offering the latest feedback. If you encounter any challenges during development, seeking support within the community is encouraged. We sincerely appreciate your contributions to Apache EventMesh. - exempt-issue-labels: 'pinned,discussion,help wanted,WIP,weopen-star,GLCC,summer of code' + exempt-issue-labels: 'pinned,discussion,help wanted,WIP,weopen-star,GLCC,GSoC' exempt-pr-labels: 'help wanted,dependencies' exempt-all-milestones: true # Exempt all issues/PRs with milestones from stale operations-per-run: 300 diff --git a/.idea/icon.png b/.idea/icon.png deleted file mode 100644 index 59aa06dac9..0000000000 Binary files a/.idea/icon.png and /dev/null differ diff --git a/README.md b/README.md index 67944cbb51..b9975018dd 100644 --- a/README.md +++ b/README.md @@ -180,26 +180,21 @@ eventmesh-runtime-0-a-0 1/1 Running 0 15s ## Contributing +[![GitHub repo Good Issues for newbies](https://img.shields.io/github/issues/apache/eventmesh/good%20first%20issue?style=flat&logo=github&logoColor=green&label=Good%20First%20issues)](https://github.com/apache/eventmesh/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) [![GitHub Help Wanted issues](https://img.shields.io/github/issues/apache/eventmesh/help%20wanted?style=flat&logo=github&logoColor=b545d1&label=%22Help%20Wanted%22%20issues)](https://github.com/apache/eventmesh/issues?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22) [![GitHub Help Wanted PRs](https://img.shields.io/github/issues-pr/apache/eventmesh/help%20wanted?style=flat&logo=github&logoColor=b545d1&label=%22Help%20Wanted%22%20PRs)](https://github.com/apache/eventmesh/pulls?q=is%3Aopen+is%3Aissue+label%3A%22help+wanted%22) [![GitHub repo Issues](https://img.shields.io/github/issues/apache/eventmesh?style=flat&logo=github&logoColor=red&label=Issues)](https://github.com/apache/eventmesh/issues?q=is%3Aopen) + Each contributor has played an important role in promoting the robust development of Apache EventMesh. We sincerely appreciate all contributors who have contributed code and documents. - [Contributing Guideline](https://eventmesh.apache.org/community/contribute/contribute) - [Good First Issues](https://github.com/apache/eventmesh/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22) -Here is the [List of Contributors](https://github.com/apache/eventmesh/graphs/contributors), thank you all! :) - - - - - ## CNCF Landscape
- - + -Apache EventMesh enriches the CNCF Cloud Native Landscape. +Apache EventMesh enriches the CNCF Cloud Native Landscape.
diff --git a/build.gradle b/build.gradle index 3fc27890d1..2e50eb3891 100644 --- a/build.gradle +++ b/build.gradle @@ -42,17 +42,17 @@ buildscript { dependencies { classpath "com.github.spotbugs.snom:spotbugs-gradle-plugin:5.2.5" - classpath "io.spring.gradle:dependency-management-plugin:1.0.11.RELEASE" + classpath "io.spring.gradle:dependency-management-plugin:1.1.6" classpath "com.diffplug.spotless:spotless-plugin-gradle:6.13.0" classpath "org.apache.httpcomponents:httpclient:4.5.14" - classpath "commons-io:commons-io:2.11.0" + classpath "commons-io:commons-io:2.18.0" } } plugins { id 'org.cyclonedx.bom' version '1.8.2' - id 'com.github.jk1.dependency-license-report' version '2.7' + id 'com.github.jk1.dependency-license-report' version '2.9' } allprojects { @@ -95,10 +95,12 @@ allprojects { checkstyleMain.exclude('**/org/apache/eventmesh/client/grpc/protos**') .exclude('**/org/apache/eventmesh/common/protocol/grpc/cloudevents**') - .exclude('**/org/apache/eventmesh/common/protocol/grpc/protos/**') + .exclude('**/org/apache/eventmesh/common/protocol/grpc/proto**') + .exclude('**/org/apache/eventmesh/common/protocol/grpc/adminserver/**') .exclude('**/org/apache/eventmesh/connector/openfunction/client/EventMeshGrpcService**') .exclude('**/org/apache/eventmesh/connector/openfunction/client/CallbackServiceGrpc**') .exclude('**/org/apache/eventmesh/connector/jdbc/antlr**') + .exclude('**/org/apache/eventmesh/meta/raft/rpc/**') dependencies { repositories { @@ -107,7 +109,7 @@ allprojects { url "https://maven.aliyun.com/repository/public" } } - testImplementation "org.junit.jupiter:junit-jupiter:5.6.0" + testImplementation "org.junit.jupiter:junit-jupiter:5.11.0" } spotless { @@ -161,9 +163,13 @@ tasks.register('dist') { ["eventmesh-common", "eventmesh-meta:eventmesh-meta-api", "eventmesh-metrics-plugin:eventmesh-metrics-api", + "eventmesh-openconnect:eventmesh-openconnect-java", + "eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-api", "eventmesh-protocol-plugin:eventmesh-protocol-api", + "eventmesh-registry:eventmesh-registry-api", "eventmesh-retry:eventmesh-retry-api", "eventmesh-runtime", + "eventmesh-runtime-v2", "eventmesh-security-plugin:eventmesh-security-api", "eventmesh-spi", "eventmesh-starter", @@ -204,6 +210,48 @@ tasks.register('dist') { } } +tasks.register('dist-admin') { + subprojects.forEach { subProject -> + dependsOn("${subProject.path}:jar") + } + def includedProjects = + [ + "eventmesh-admin-server", + "eventmesh-common", + "eventmesh-spi", + "eventmesh-registry:eventmesh-registry-api", + "eventmesh-registry:eventmesh-registry-nacos", + "eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-api" + ] + doLast { + includedProjects.each { + def subProject = findProject(it) + copy { + from subProject.jar.archivePath + into rootProject.file('dist/apps') + } + copy { + from subProject.configurations.runtimeClasspath + into rootProject.file('dist/lib') + exclude 'eventmesh-*' + } + copy { + from subProject.file('bin') + into rootProject.file('dist/bin') + } + copy { + from subProject.file('conf') + from subProject.sourceSets.main.resources.srcDirs + into rootProject.file('dist/conf') + duplicatesStrategy = DuplicatesStrategy.EXCLUDE + exclude 'META-INF' + } + + } + } + +} + tasks.register('installPlugin') { var pluginProjects = subprojects.findAll { it.file('gradle.properties').exists() @@ -402,10 +450,11 @@ tasks.register('checkDeniedLicense') { "GPL-1.0", "GPL-2.0", "GPL-3.0", "AGPL-3.0", "LGPL-2.0", "LGPL-2.1", "LGPL-3.0", "GPL-1.0-only", "GPL-2.0-only", "GPL-3.0-only", "AGPL-3.0-only", "LGPL-2.0-only", "LGPL-2.1-only", "LGPL-3.0-only", "QPL-1.0", "Sleepycat", "SSPL-1.0", "CPOL-1.02", - "BSD-4-Clause", "BSD-4-Clause-UC", "NPL-1.0", "NPL-1.1", "JSON" + "BSD-4-Clause", "BSD-4-Clause-UC", "NPL-1.0", "NPL-1.1", "JSON", + "The GNU General Public License, v2 with Universal FOSS Exception, v1.0" ] // Update exemptions according to https://github.com/apache/eventmesh/issues/4842 - def allowedArtifacts = ["amqp-client", "stax-api", "javassist", "ST4", "xsdlib"] + def allowedArtifacts = ["amqp-client", "stax-api", "javassist", "ST4", "xsdlib", "jsqlparser"] def licenseFile = file('tools/dist-license/LICENSE') def lines = licenseFile.readLines() @@ -648,33 +697,30 @@ subprojects { sign publishing.publications.mavenJava } - def grpcVersion = '1.64.0' - def log4jVersion = '2.23.1' - def jacksonVersion = '2.17.1' - def dropwizardMetricsVersion = '4.2.25' + def grpcVersion = '1.68.0' + def log4jVersion = '2.24.1' + def jacksonVersion = '2.18.0' + def dropwizardMetricsVersion = '4.2.26' def opentelemetryVersion = '1.36.0' - def cloudeventsVersion = '2.5.0' - def curatorVersion = '5.6.0' - def mockitoVersion = '3.12.4' + def cloudeventsVersion = '3.0.0' + def curatorVersion = '5.7.0' + def mockitoVersion = '4.11.0' dependencyManagement { dependencies { - dependency "org.apache.commons:commons-lang3:3.6" + dependency "org.apache.commons:commons-lang3:3.17.0" dependency "org.apache.commons:commons-collections4:4.4" - dependency "org.apache.commons:commons-text:1.9" - - dependency "commons-io:commons-io:2.11.0" - - dependency "commons-validator:commons-validator:1.7" - - dependency "com.google.guava:guava:31.1-jre" + dependency "org.apache.commons:commons-text:1.12.0" + dependency "commons-io:commons-io:2.18.0" + dependency "commons-validator:commons-validator:1.9.0" + dependency "com.google.guava:guava:33.3.0-jre" dependency "org.slf4j:slf4j-api:2.0.13" dependency "org.apache.logging.log4j:log4j-api:${log4jVersion}" dependency "org.apache.logging.log4j:log4j-core:${log4jVersion}" dependency "org.apache.logging.log4j:log4j-slf4j2-impl:${log4jVersion}" - dependency "com.lmax:disruptor:3.4.2" + dependency "com.lmax:disruptor:3.4.4" dependency "com.fasterxml.jackson.core:jackson-databind:${jacksonVersion}" dependency "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}" @@ -684,10 +730,10 @@ subprojects { dependency "com.squareup.okhttp3:okhttp:3.14.9" - dependency "org.asynchttpclient:async-http-client:2.12.0" + dependency "org.asynchttpclient:async-http-client:2.12.3" dependency "org.apache.httpcomponents:httpclient:4.5.14" - dependency "io.netty:netty-all:4.1.79.Final" + dependency "io.netty:netty-all:4.1.112.Final" dependency "io.dropwizard.metrics:metrics-core:${dropwizardMetricsVersion}" dependency "io.dropwizard.metrics:metrics-healthchecks:${dropwizardMetricsVersion}" @@ -704,16 +750,16 @@ subprojects { dependency "io.openmessaging:openmessaging-api:2.2.1-pubsub" - dependency "com.h3xstream.findsecbugs:findsecbugs-plugin:1.11.0" - dependency "com.mebigfatguy.fb-contrib:fb-contrib:7.6.0" + dependency "com.h3xstream.findsecbugs:findsecbugs-plugin:1.13.0" + dependency "com.mebigfatguy.fb-contrib:fb-contrib:7.6.4" dependency "com.jayway.jsonpath:json-path:2.9.0" dependency "org.springframework.boot:spring-boot-starter-web:2.7.18" dependency "io.openmessaging:registry-server:0.0.1" - dependency "org.junit.jupiter:junit-jupiter:5.6.0" + dependency "org.junit.jupiter:junit-jupiter:5.11.0" dependency "org.junit-pioneer:junit-pioneer:1.9.1" - dependency "org.assertj:assertj-core:2.6.0" + dependency "org.assertj:assertj-core:3.26.3" dependency "org.mockito:mockito-core:${mockitoVersion}" dependency "org.mockito:mockito-inline:${mockitoVersion}" @@ -731,12 +777,12 @@ subprojects { dependency "javax.annotation:javax.annotation-api:1.3.2" - dependency "com.github.seancfoley:ipaddress:5.3.3" + dependency "com.github.seancfoley:ipaddress:5.5.0" dependency "com.google.code.gson:gson:2.11.0" dependency "org.javassist:javassist:3.30.2-GA" - dependency "com.alibaba.nacos:nacos-client:2.3.2" + dependency "com.alibaba.nacos:nacos-client:2.4.1" dependency 'org.apache.zookeeper:zookeeper:3.9.2' dependency "org.apache.curator:curator-client:${curatorVersion}" @@ -744,13 +790,18 @@ subprojects { dependency "org.apache.curator:curator-recipes:${curatorVersion}" dependency "org.apache.curator:curator-test:${curatorVersion}" - dependency "org.projectlombok:lombok:1.18.32" - dependency "com.github.seancfoley:ipaddress:5.3.3" + dependency "org.projectlombok:lombok:1.18.34" dependency "javax.annotation:javax.annotation-api:1.3.2" - dependency "com.alibaba.fastjson2:fastjson2:2.0.48" + dependency "com.alibaba.fastjson2:fastjson2:2.0.52" - dependency "software.amazon.awssdk:s3:2.25.55" + dependency "software.amazon.awssdk:s3:2.29.5" dependency "com.github.rholder:guava-retrying:2.0.0" + + dependency "com.alibaba:druid-spring-boot-starter:1.2.23" + dependency "com.baomidou:mybatis-plus-boot-starter:3.5.7" + dependency "com.mysql:mysql-connector-j:8.4.0" + dependency "org.springframework.boot:spring-boot-starter-jetty:2.7.18" + dependency "org.locationtech.jts:jts-core:1.20.0" } } } diff --git a/eventmesh-admin-server/.gitignore b/eventmesh-admin-server/.gitignore new file mode 100644 index 0000000000..b63da4551b --- /dev/null +++ b/eventmesh-admin-server/.gitignore @@ -0,0 +1,42 @@ +.gradle +build/ +!gradle/wrapper/gradle-wrapper.jar +!**/src/main/**/build/ +!**/src/test/**/build/ + +### IntelliJ IDEA ### +.idea/modules.xml +.idea/jarRepositories.xml +.idea/compiler.xml +.idea/libraries/ +*.iws +*.iml +*.ipr +out/ +!**/src/main/**/out/ +!**/src/test/**/out/ + +### Eclipse ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache +bin/ +!**/src/main/**/bin/ +!**/src/test/**/bin/ + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ + +### VS Code ### +.vscode/ + +### Mac OS ### +.DS_Store \ No newline at end of file diff --git a/eventmesh-admin-server/bin/start-admin.sh b/eventmesh-admin-server/bin/start-admin.sh new file mode 100644 index 0000000000..1633036617 --- /dev/null +++ b/eventmesh-admin-server/bin/start-admin.sh @@ -0,0 +1,200 @@ +#!/bin/bash +# +# Licensed to Apache Software Foundation (ASF) under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Apache Software Foundation (ASF) licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +#=========================================================================================== +# Java Environment Setting +#=========================================================================================== +set -e +# Server configuration may be inconsistent, add these configurations to avoid garbled code problems +export LANG=en_US.UTF-8 +export LC_CTYPE=en_US.UTF-8 +export LC_ALL=en_US.UTF-8 + +TMP_JAVA_HOME="/customize/your/java/home/here" + +# Detect operating system. +OS=$(uname) + +function is_java8_or_11 { + local _java="$1" + [[ -x "$_java" ]] || return 1 + [[ "$("$_java" -version 2>&1)" =~ 'java version "1.8' || "$("$_java" -version 2>&1)" =~ 'openjdk version "1.8' || "$("$_java" -version 2>&1)" =~ 'java version "11' || "$("$_java" -version 2>&1)" =~ 'openjdk version "11' ]] || return 2 + return 0 +} + +function extract_java_version { + local _java="$1" + local version=$("$_java" -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F '.' '{if ($1 == 1 && $2 == 8) print "8"; else if ($1 == 11) print "11"; else print "unknown"}') + echo "$version" +} + +# 0(not running), 1(is running) +#function is_proxyRunning { +# local _pid="$1" +# local pid=`ps ax | grep -i 'org.apache.eventmesh.runtime.boot.EventMeshStartup' |grep java | grep -v grep | awk '{print $1}'|grep $_pid` +# if [ -z "$pid" ] ; then +# return 0 +# else +# return 1 +# fi +#} + +function get_pid { + local ppid="" + if [ -f ${EVENTMESH_ADMIN_HOME}/bin/pid-admin.file ]; then + ppid=$(cat ${EVENTMESH_ADMIN_HOME}/bin/pid-admin.file) + # If the process does not exist, it indicates that the previous process terminated abnormally. + if [ ! -d /proc/$ppid ]; then + # Remove the residual file. + rm ${EVENTMESH_ADMIN_HOME}/bin/pid-admin.file + echo -e "ERROR\t EventMesh process had already terminated unexpectedly before, please check log output." + ppid="" + fi + else + if [[ $OS =~ Msys ]]; then + # There is a Bug on Msys that may not be able to kill the identified process + ppid=`jps -v | grep -i "org.apache.eventmesh.admin.server.ExampleAdminServer" | grep java | grep -v grep | awk -F ' ' {'print $1'}` + elif [[ $OS =~ Darwin ]]; then + # Known problem: grep Java may not be able to accurately identify Java processes + ppid=$(/bin/ps -o user,pid,command | grep "java" | grep -i "org.apache.eventmesh.admin.server.ExampleAdminServer" | grep -Ev "^root" |awk -F ' ' {'print $2'}) + else + if [ $DOCKER ]; then + # No need to exclude root user in Docker containers. + ppid=$(ps -C java -o user,pid,command --cols 99999 | grep -w $EVENTMESH_ADMIN_HOME | grep -i "org.apache.eventmesh.admin.server.ExampleAdminServer" | awk -F ' ' {'print $2'}) + else + # It is required to identify the process as accurately as possible on Linux. + ppid=$(ps -C java -o user,pid,command --cols 99999 | grep -w $EVENTMESH_ADMIN_HOME | grep -i "org.apache.eventmesh.admin.server.ExampleAdminServer" | grep -Ev "^root" | awk -F ' ' {'print $2'}) + fi + fi + fi + echo "$ppid"; +} + +#=========================================================================================== +# Locate Java Executable +#=========================================================================================== + +if [[ -d "$TMP_JAVA_HOME" ]] && is_java8_or_11 "$TMP_JAVA_HOME/bin/java"; then + JAVA="$TMP_JAVA_HOME/bin/java" + JAVA_VERSION=$(extract_java_version "$TMP_JAVA_HOME/bin/java") +elif [[ -d "$JAVA_HOME" ]] && is_java8_or_11 "$JAVA_HOME/bin/java"; then + JAVA="$JAVA_HOME/bin/java" + JAVA_VERSION=$(extract_java_version "$JAVA_HOME/bin/java") +elif is_java8_or_11 "$(which java)"; then + JAVA="$(which java)" + JAVA_VERSION=$(extract_java_version "$(which java)") +else + echo -e "ERROR\t Java 8 or 11 not found, operation abort." + exit 9; +fi + +echo "EventMesh using Java version: $JAVA_VERSION, path: $JAVA" + +EVENTMESH_ADMIN_HOME=$(cd "$(dirname "$0")/.." && pwd) +export EVENTMESH_ADMIN_HOME + +EVENTMESH_ADMIN_LOG_HOME="${EVENTMESH_ADMIN_HOME}/logs" +export EVENTMESH_ADMIN_LOG_HOME + +echo -e "EVENTMESH_ADMIN_HOME : ${EVENTMESH_ADMIN_HOME}\nEVENTMESH_ADMIN_LOG_HOME : ${EVENTMESH_ADMIN_LOG_HOME}" + +function make_logs_dir { + if [ ! -e "${EVENTMESH_ADMIN_LOG_HOME}" ]; then mkdir -p "${EVENTMESH_ADMIN_LOG_HOME}"; fi +} + +error_exit () +{ + echo -e "ERROR\t $1 !!" + exit 1 +} + +export JAVA_HOME + +#=========================================================================================== +# JVM Configuration +#=========================================================================================== +#if [ $1 = "prd" -o $1 = "benchmark" ]; then JAVA_OPT="${JAVA_OPT} -server -Xms2048M -Xmx4096M -Xmn2048m -XX:SurvivorRatio=4" +#elif [ $1 = "sit" ]; then JAVA_OPT="${JAVA_OPT} -server -Xms256M -Xmx512M -Xmn256m -XX:SurvivorRatio=4" +#elif [ $1 = "dev" ]; then JAVA_OPT="${JAVA_OPT} -server -Xms128M -Xmx256M -Xmn128m -XX:SurvivorRatio=4" +#fi + +GC_LOG_FILE="${EVENTMESH_ADMIN_LOG_HOME}/eventmesh_admin_gc_%p.log" + +JAVA_OPT="${JAVA_OPT} -server -Xms1g -Xmx1g" +JAVA_OPT="${JAVA_OPT} -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0 -XX:SurvivorRatio=8 -XX:MaxGCPauseMillis=50" +JAVA_OPT="${JAVA_OPT} -verbose:gc" +if [[ "$JAVA_VERSION" == "8" ]]; then + # Set JAVA_OPT for Java 8 + JAVA_OPT="${JAVA_OPT} -Xloggc:${GC_LOG_FILE} -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m" + JAVA_OPT="${JAVA_OPT} -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintAdaptiveSizePolicy" +elif [[ "$JAVA_VERSION" == "11" ]]; then + # Set JAVA_OPT for Java 11 + XLOG_PARAM="time,level,tags:filecount=5,filesize=30m" + JAVA_OPT="${JAVA_OPT} -Xlog:gc*:${GC_LOG_FILE}:${XLOG_PARAM}" + JAVA_OPT="${JAVA_OPT} -Xlog:safepoint:${GC_LOG_FILE}:${XLOG_PARAM} -Xlog:ergo*=debug:${GC_LOG_FILE}:${XLOG_PARAM}" +fi +JAVA_OPT="${JAVA_OPT} -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${EVENTMESH_ADMIN_LOG_HOME} -XX:ErrorFile=${EVENTMESH_ADMIN_LOG_HOME}/hs_err_%p.log" +JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow" +JAVA_OPT="${JAVA_OPT} -XX:+AlwaysPreTouch" +JAVA_OPT="${JAVA_OPT} -XX:MaxDirectMemorySize=8G" +JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages -XX:-UseBiasedLocking" +JAVA_OPT="${JAVA_OPT} -Dio.netty.leakDetectionLevel=advanced" +JAVA_OPT="${JAVA_OPT} -Dio.netty.allocator.type=pooled" +JAVA_OPT="${JAVA_OPT} -Djava.security.egd=file:/dev/./urandom" +JAVA_OPT="${JAVA_OPT} -Dlog4j.configurationFile=${EVENTMESH_ADMIN_HOME}/conf/log4j2.xml" +JAVA_OPT="${JAVA_OPT} -Deventmesh.log.home=${EVENTMESH_ADMIN_LOG_HOME}" +JAVA_OPT="${JAVA_OPT} -DconfPath=${EVENTMESH_ADMIN_HOME}/conf" +JAVA_OPT="${JAVA_OPT} -DconfigurationPath=${EVENTMESH_ADMIN_HOME}/conf" +JAVA_OPT="${JAVA_OPT} -Dlog4j2.AsyncQueueFullPolicy=Discard" +JAVA_OPT="${JAVA_OPT} -Drocketmq.client.logUseSlf4j=true" +JAVA_OPT="${JAVA_OPT} -DeventMeshPluginDir=${EVENTMESH_ADMIN_HOME}/plugin" + +#if [ -f "pid.file" ]; then +# pid=`cat pid.file` +# if ! is_proxyRunning "$pid"; then +# echo "proxy is running already" +# exit 9; +# else +# echo "err pid$pid, rm pid.file" +# rm pid.file +# fi +#fi + +pid=$(get_pid) +if [[ $pid == "ERROR"* ]]; then + echo -e "${pid}" + exit 9 +fi +if [ -n "$pid" ]; then + echo -e "ERROR\t The server is already running (pid=$pid), there is no need to execute start.sh again." + exit 9 +fi + +make_logs_dir + +echo "Using Java version: $JAVA_VERSION, path: $JAVA" >> ${EVENTMESH_ADMIN_LOG_HOME}/eventmesh-admin.out + +EVENTMESH_ADMIN_MAIN=org.apache.eventmesh.admin.server.ExampleAdminServer +if [ $DOCKER ]; then + $JAVA $JAVA_OPT -classpath ${EVENTMESH_ADMIN_HOME}/conf:${EVENTMESH_ADMIN_HOME}/apps/*:${EVENTMESH_ADMIN_HOME}/lib/* $EVENTMESH_ADMIN_MAIN >> ${EVENTMESH_ADMIN_LOG_HOME}/eventmesh-admin.out +else + $JAVA $JAVA_OPT -classpath ${EVENTMESH_ADMIN_HOME}/conf:${EVENTMESH_ADMIN_HOME}/apps/*:${EVENTMESH_ADMIN_HOME}/lib/* $EVENTMESH_ADMIN_MAIN >> ${EVENTMESH_ADMIN_LOG_HOME}/eventmesh-admin.out 2>&1 & +echo $!>${EVENTMESH_ADMIN_HOME}/bin/pid-admin.file +fi +exit 0 diff --git a/eventmesh-admin-server/bin/stop-admin.sh b/eventmesh-admin-server/bin/stop-admin.sh new file mode 100644 index 0000000000..207531d7fa --- /dev/null +++ b/eventmesh-admin-server/bin/stop-admin.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# +# Licensed to Apache Software Foundation (ASF) under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Apache Software Foundation (ASF) licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Detect operating system +OS=$(uname) + +EVENTMESH_ADMIN_HOME=`cd $(dirname $0)/.. && pwd` + +export EVENTMESH_ADMIN_HOME + +function get_pid { + local ppid="" + if [ -f ${EVENTMESH_ADMIN_HOME}/bin/pid-admin.file ]; then + ppid=$(cat ${EVENTMESH_ADMIN_HOME}/bin/pid-admin.file) + # If the process does not exist, it indicates that the previous process terminated abnormally. + if [ ! -d /proc/$ppid ]; then + # Remove the residual file and return an error status. + rm ${EVENTMESH_ADMIN_HOME}/bin/pid-admin.file + echo -e "ERROR\t EventMesh admin process had already terminated unexpectedly before, please check log output." + ppid="" + fi + else + if [[ $OS =~ Msys ]]; then + # There is a Bug on Msys that may not be able to kill the identified process + ppid=`jps -v | grep -i "org.apache.eventmesh.admin.server.ExampleAdminServer" | grep java | grep -v grep | awk -F ' ' {'print $1'}` + elif [[ $OS =~ Darwin ]]; then + # Known problem: grep Java may not be able to accurately identify Java processes + ppid=$(/bin/ps -o user,pid,command | grep "java" | grep -i "org.apache.eventmesh.admin.server.ExampleAdminServer" | grep -Ev "^root" |awk -F ' ' {'print $2'}) + else + # It is required to identify the process as accurately as possible on Linux + ppid=$(ps -C java -o user,pid,command --cols 99999 | grep -w $EVENTMESH_ADMIN_HOME | grep -i "org.apache.eventmesh.admin.server.ExampleAdminServer" | grep -Ev "^root" |awk -F ' ' {'print $2'}) + fi + fi + echo "$ppid"; +} + +pid=$(get_pid) +if [[ $pid == "ERROR"* ]]; then + echo -e "${pid}" + exit 9 +fi +if [ -z "$pid" ];then + echo -e "ERROR\t No EventMesh admin server running." + exit 9 +fi + +kill ${pid} +echo "Send shutdown request to EventMesh admin(${pid}) OK" + +[[ $OS =~ Msys ]] && PS_PARAM=" -W " +stop_timeout=60 +for no in $(seq 1 $stop_timeout); do + if ps $PS_PARAM -p "$pid" 2>&1 > /dev/null; then + if [ $no -lt $stop_timeout ]; then + echo "[$no] server shutting down ..." + sleep 1 + continue + fi + + echo "shutdown server timeout, kill process: $pid" + kill -9 $pid; sleep 1; break; + echo "`date +'%Y-%m-%-d %H:%M:%S'` , pid : [$pid] , error message : abnormal shutdown which can not be closed within 60s" > ../logs/shutdown.error + else + echo "shutdown server ok!"; break; + fi +done + +if [ -f "pid-admin.file" ]; then + rm pid-admin.file +fi + + diff --git a/eventmesh-admin-server/build.gradle b/eventmesh-admin-server/build.gradle new file mode 100644 index 0000000000..fdfe1bffe8 --- /dev/null +++ b/eventmesh-admin-server/build.gradle @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dependencies { + implementation project(":eventmesh-spi") + implementation project(":eventmesh-common") + implementation project(":eventmesh-registry:eventmesh-registry-api") + implementation project(":eventmesh-registry:eventmesh-registry-nacos") + implementation project(":eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-api") + implementation "com.alibaba.nacos:nacos-client" + implementation("org.springframework.boot:spring-boot-starter-web") { + exclude group: "org.springframework.boot", module: "spring-boot-starter-tomcat" + } + implementation 'org.springframework.boot:spring-boot-starter-jetty' + implementation "io.grpc:grpc-core" + implementation "io.grpc:grpc-protobuf" + implementation "io.grpc:grpc-stub" + implementation "io.grpc:grpc-netty-shaded" + + // https://mvnrepository.com/artifact/com.baomidou/mybatis-plus-boot-starter + implementation "com.baomidou:mybatis-plus-boot-starter" + + // https://mvnrepository.com/artifact/com.alibaba/druid-spring-boot-starter + implementation "com.alibaba:druid-spring-boot-starter" + compileOnly 'com.mysql:mysql-connector-j' + compileOnly 'org.projectlombok:lombok' + testImplementation 'junit:junit:4.13.2' + testImplementation 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} + +configurations.implementation { + exclude group: "org.springframework.boot", module: "spring-boot-starter-logging" +} + +sourceSets { + main { + resources { + srcDirs = ['src/main/resources', 'conf'] + } + } +} + diff --git a/eventmesh-admin-server/conf/application.yaml b/eventmesh-admin-server/conf/application.yaml new file mode 100644 index 0000000000..7765d90ce8 --- /dev/null +++ b/eventmesh-admin-server/conf/application.yaml @@ -0,0 +1,58 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +spring: + datasource: + url: jdbc:mysql://localhost:3306/eventmesh?serverTimezone=GMT%2B8&characterEncoding=utf-8&useSSL=false&allowPublicKeyRetrieval=true + username: //db_username + password: //db_password + driver-class-name: com.mysql.cj.jdbc.Driver + initialSize: 1 + minIdle: 1 + maxActive: 20 + maxWait: 10000 + timeBetweenEvictionRunsMillis: 60000 + minEvictableIdleTimeMillis: 300000 + validationQuery: SELECT 1 FROM DUAL + testWhileIdle: true + testOnBorrow: false + testOnReturn: false + poolPreparedStatements: false + maxPoolPreparedStatementPerConnectionSize: 20 + filters: stat + connectionProperties: "druid.stat.mergeSql=true;druid.stat.slowSqlMillis=5000" +# secret keys +sysPubKey: +appPrivKey: + +mybatis-plus: + mapper-locations: classpath:mapper/*.xml + configuration: + map-underscore-to-camel-case: false + log-impl: org.apache.ibatis.logging.stdout.StdOutImpl +# http server port +server: + port: 8082 +event-mesh: + admin-server: + serviceName: DEFAULT_GROUP@@em_adm_server + # grpc server port + port: 8081 + adminServerList: + R1: http://localhost:8082;http://localhost:8082 + R2: http://localhost:8092;http://localhost:8092 + region: R1 \ No newline at end of file diff --git a/eventmesh-admin-server/conf/eventmesh-admin.properties b/eventmesh-admin-server/conf/eventmesh-admin.properties new file mode 100644 index 0000000000..30507ec02c --- /dev/null +++ b/eventmesh-admin-server/conf/eventmesh-admin.properties @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +eventMesh.registry.plugin.type=nacos +eventMesh.registry.plugin.server-addr=localhost:8848 diff --git a/eventmesh-admin-server/conf/eventmesh.sql b/eventmesh-admin-server/conf/eventmesh.sql new file mode 100644 index 0000000000..4d11ab1585 --- /dev/null +++ b/eventmesh-admin-server/conf/eventmesh.sql @@ -0,0 +1,185 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET NAMES utf8 */; +/*!50503 SET NAMES utf8 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + + +-- export eventmesh database +CREATE DATABASE IF NOT EXISTS `eventmesh` /*!40100 DEFAULT CHARACTER SET utf8 COLLATE utf8_bin */ /*!80016 DEFAULT ENCRYPTION='N' */; +USE `eventmesh`; + +-- export table eventmesh.event_mesh_data_source structure +CREATE TABLE IF NOT EXISTS `event_mesh_data_source` ( + `id` int unsigned NOT NULL AUTO_INCREMENT, + `dataType` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT '', + `description` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL, + `configuration` text CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `configurationClass` varchar(200) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '', + `region` varchar(50) COLLATE utf8_bin DEFAULT NULL, + `createUid` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT '', + `updateUid` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT '', + `createTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updateTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) USING BTREE +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + +-- export table eventmesh.event_mesh_job_info structure +CREATE TABLE IF NOT EXISTS `event_mesh_job_info` ( + `id` int unsigned NOT NULL AUTO_INCREMENT, + `jobID` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `jobDesc` varchar(50) COLLATE utf8_bin NOT NULL, + `taskID` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT '', + `transportType` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT '', + `sourceData` int NOT NULL DEFAULT '0', + `targetData` int NOT NULL DEFAULT '0', + `jobState` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '', + `jobType` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '', + `fromRegion` varchar(50) COLLATE utf8_bin DEFAULT NULL, + `runningRegion` varchar(50) COLLATE utf8_bin DEFAULT NULL, + `createUid` varchar(50) COLLATE utf8_bin DEFAULT NULL, + `updateUid` varchar(50) COLLATE utf8_bin DEFAULT NULL, + `createTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updateTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) USING BTREE, + UNIQUE KEY `jobID` (`jobID`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + +-- export table eventmesh.event_mesh_mysql_position structure +CREATE TABLE IF NOT EXISTS `event_mesh_mysql_position` ( + `id` int unsigned NOT NULL AUTO_INCREMENT, + `jobID` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT '', + `serverUUID` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL, + `address` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `position` bigint DEFAULT NULL, + `gtid` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL, + `currentGtid` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL, + `timestamp` bigint DEFAULT NULL, + `journalName` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL, + `createTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updateTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `jobID` (`jobID`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin ROW_FORMAT=DYNAMIC; + +-- export table eventmesh.event_mesh_position_reporter_history structure +CREATE TABLE IF NOT EXISTS `event_mesh_position_reporter_history` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `job` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT '', + `record` text CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `address` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '', + `createTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + KEY `job` (`job`), + KEY `address` (`address`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin COMMENT='record position reporter changes'; + +-- export table eventmesh.event_mesh_runtime_heartbeat structure +CREATE TABLE IF NOT EXISTS `event_mesh_runtime_heartbeat` ( + `id` bigint unsigned NOT NULL AUTO_INCREMENT, + `adminAddr` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `runtimeAddr` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `jobID` varchar(50) COLLATE utf8_bin DEFAULT NULL, + `reportTime` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL COMMENT 'runtime local report time', + `updateTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `createTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + KEY `jobID` (`jobID`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + +-- export table eventmesh.event_mesh_runtime_history structure +CREATE TABLE IF NOT EXISTS `event_mesh_runtime_history` ( + `id` bigint NOT NULL AUTO_INCREMENT, + `job` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT '', + `address` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT '', + `createTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + KEY `address` (`address`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin ROW_FORMAT=DYNAMIC COMMENT='record runtime task change history'; + +-- export table eventmesh.event_mesh_task_info structure +CREATE TABLE IF NOT EXISTS `event_mesh_task_info` ( + `id` int unsigned NOT NULL AUTO_INCREMENT, + `taskID` varchar(50) COLLATE utf8_bin NOT NULL, + `taskName` varchar(50) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `taskDesc` varchar(50) COLLATE utf8_bin NOT NULL, + `taskState` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT '' COMMENT 'taskstate', + `sourceRegion` varchar(50) COLLATE utf8_bin DEFAULT NULL, + `targetRegion` varchar(50) COLLATE utf8_bin DEFAULT NULL, + `createUid` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT '', + `updateUid` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT '', + `createTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updateTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) USING BTREE, + UNIQUE KEY `taskID` (`taskID`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + +-- export table eventmesh.event_mesh_verify structure +CREATE TABLE IF NOT EXISTS `event_mesh_verify` ( + `id` int unsigned NOT NULL AUTO_INCREMENT, + `taskID` varchar(50) COLLATE utf8_bin DEFAULT NULL, + `jobID` varchar(50) COLLATE utf8_bin DEFAULT NULL, + `recordID` varchar(50) COLLATE utf8_bin DEFAULT NULL, + `recordSig` varchar(50) COLLATE utf8_bin DEFAULT NULL, + `connectorName` varchar(200) COLLATE utf8_bin DEFAULT NULL, + `connectorStage` varchar(50) COLLATE utf8_bin DEFAULT NULL, + `position` text COLLATE utf8_bin DEFAULT NULL, + `createTime` timestamp NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + +-- eventmesh.event_mesh_weredis_position definition +CREATE TABLE `event_mesh_weredis_position` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `jobID` varchar(50) COLLATE utf8_bin NOT NULL DEFAULT '', + `address` varchar(50) COLLATE utf8_bin DEFAULT NULL, + `clusterName` varchar(50) COLLATE utf8_bin DEFAULT NULL, + `partitionName` varchar(50) COLLATE utf8_bin DEFAULT NULL, + `masterReplid` varchar(50) COLLATE utf8_bin DEFAULT NULL, + `host` varchar(50) COLLATE utf8_bin DEFAULT NULL, + `replOffset` bigint(20) NOT NULL DEFAULT '-1', + `createTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, + `updateTime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`), + UNIQUE KEY `jobID` (`jobID`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_bin ROW_FORMAT=DYNAMIC; + + +CREATE TABLE `event_mesh_monitor` ( + `id` int(10) unsigned NOT NULL AUTO_INCREMENT, + `taskID` varchar(50) COLLATE utf8_bin DEFAULT NULL, + `jobID` varchar(50) COLLATE utf8_bin DEFAULT NULL, + `address` varchar(50) COLLATE utf8_bin DEFAULT NULL, + `transportType` varchar(50) COLLATE utf8_bin DEFAULT NULL, + `connectorStage` varchar(50) COLLATE utf8_bin DEFAULT NULL, + `totalReqNum` bigint DEFAULT NULL, + `totalTimeCost` bigint DEFAULT NULL, + `maxTimeCost` bigint DEFAULT NULL, + `avgTimeCost` bigint DEFAULT NULL, + `tps` double DEFAULT NULL, + `createTime` timestamp NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8 COLLATE=utf8_bin; + +/*!40101 SET SQL_MODE=IFNULL(@OLD_SQL_MODE, '') */; +/*!40014 SET FOREIGN_KEY_CHECKS=IFNULL(@OLD_FOREIGN_KEY_CHECKS, 1) */; +/*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; +/*!40111 SET SQL_NOTES=IFNULL(@OLD_SQL_NOTES, 1) */; diff --git a/eventmesh-admin-server/conf/log4j2.xml b/eventmesh-admin-server/conf/log4j2.xml new file mode 100644 index 0000000000..acc6acb8ba --- /dev/null +++ b/eventmesh-admin-server/conf/log4j2.xml @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/eventmesh-admin-server/conf/mapper/EventMeshDataSourceMapper.xml b/eventmesh-admin-server/conf/mapper/EventMeshDataSourceMapper.xml new file mode 100644 index 0000000000..50e6ad82cc --- /dev/null +++ b/eventmesh-admin-server/conf/mapper/EventMeshDataSourceMapper.xml @@ -0,0 +1,44 @@ + + + + + + + + + + + + + + + + + + + + + + id,dataType,description, + configuration,configurationClass,region, + createUid,updateUid,createTime,updateTime + + diff --git a/eventmesh-admin-server/conf/mapper/EventMeshJobInfoMapper.xml b/eventmesh-admin-server/conf/mapper/EventMeshJobInfoMapper.xml new file mode 100644 index 0000000000..a053d1c838 --- /dev/null +++ b/eventmesh-admin-server/conf/mapper/EventMeshJobInfoMapper.xml @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + id,jobID,jobDesc, + taskID,transportType,sourceData, + targetData,jobState,jobType, + fromRegion,runningRegion,createUid, + updateUid,createTime,updateTime + + diff --git a/eventmesh-admin-server/conf/mapper/EventMeshMonitorMapper.xml b/eventmesh-admin-server/conf/mapper/EventMeshMonitorMapper.xml new file mode 100644 index 0000000000..f77fb8ba77 --- /dev/null +++ b/eventmesh-admin-server/conf/mapper/EventMeshMonitorMapper.xml @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + id,taskID,jobID,address,transportType,connectorStage, + totalReqNum,totalTimeCost,maxTimeCost,avgTimeCost, + tps,createTime + + diff --git a/eventmesh-admin-server/conf/mapper/EventMeshMysqlPositionMapper.xml b/eventmesh-admin-server/conf/mapper/EventMeshMysqlPositionMapper.xml new file mode 100644 index 0000000000..9bcc7f42bb --- /dev/null +++ b/eventmesh-admin-server/conf/mapper/EventMeshMysqlPositionMapper.xml @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + id,jobID,serverUUID, + address,position,gtid, + currentGtid,timestamp,journalName, + createTime,updateTime + + diff --git a/eventmesh-admin-server/conf/mapper/EventMeshPositionReporterHistoryMapper.xml b/eventmesh-admin-server/conf/mapper/EventMeshPositionReporterHistoryMapper.xml new file mode 100644 index 0000000000..a9e4fe6f1b --- /dev/null +++ b/eventmesh-admin-server/conf/mapper/EventMeshPositionReporterHistoryMapper.xml @@ -0,0 +1,38 @@ + + + + + + + + + + + + + + + + + id,job,record, + address,createTime + + diff --git a/eventmesh-admin-server/conf/mapper/EventMeshRuntimeHeartbeatMapper.xml b/eventmesh-admin-server/conf/mapper/EventMeshRuntimeHeartbeatMapper.xml new file mode 100644 index 0000000000..200b1bf54a --- /dev/null +++ b/eventmesh-admin-server/conf/mapper/EventMeshRuntimeHeartbeatMapper.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + id,adminAddr,runtimeAddr, + jobID,reportTime,updateTime, + createTime + + diff --git a/eventmesh-admin-server/conf/mapper/EventMeshRuntimeHistoryMapper.xml b/eventmesh-admin-server/conf/mapper/EventMeshRuntimeHistoryMapper.xml new file mode 100644 index 0000000000..281cce30f9 --- /dev/null +++ b/eventmesh-admin-server/conf/mapper/EventMeshRuntimeHistoryMapper.xml @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + id,job,address, + createTime + + diff --git a/eventmesh-admin-server/conf/mapper/EventMeshTaskInfoMapper.xml b/eventmesh-admin-server/conf/mapper/EventMeshTaskInfoMapper.xml new file mode 100644 index 0000000000..c3514fd945 --- /dev/null +++ b/eventmesh-admin-server/conf/mapper/EventMeshTaskInfoMapper.xml @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + id,taskID,taskName, + taskDesc,taskState,sourceRegion,targetRegion, + createUid,updateUid,createTime, + updateTime + + diff --git a/eventmesh-admin-server/conf/mapper/EventMeshVerifyMapper.xml b/eventmesh-admin-server/conf/mapper/EventMeshVerifyMapper.xml new file mode 100644 index 0000000000..45727498cc --- /dev/null +++ b/eventmesh-admin-server/conf/mapper/EventMeshVerifyMapper.xml @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + id,taskID,jobID,recordID, + recordSig,connectorName,connectorStage, + position,createTime + + diff --git a/eventmesh-admin-server/gradle.properties b/eventmesh-admin-server/gradle.properties new file mode 100644 index 0000000000..a9fd83fea0 --- /dev/null +++ b/eventmesh-admin-server/gradle.properties @@ -0,0 +1,16 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/AdminServerProperties.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/AdminServerProperties.java new file mode 100644 index 0000000000..2e6d3c018a --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/AdminServerProperties.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server; + +import java.util.Map; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +import lombok.Getter; +import lombok.Setter; + +@ConfigurationProperties("event-mesh.admin-server") +@Getter +@Setter +public class AdminServerProperties { + + private int port; + private boolean enableSSL; + private String configurationPath; + private String configurationFile; + private String serviceName; + private Map adminServerList; + private String region; +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/AdminServerRuntimeException.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/AdminServerRuntimeException.java new file mode 100644 index 0000000000..e68d05100f --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/AdminServerRuntimeException.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server; + +import lombok.Getter; + +@Getter +public class AdminServerRuntimeException extends RuntimeException { + private final int code; + + public AdminServerRuntimeException(int code, String message) { + super(message); + this.code = code; + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/ExampleAdminServer.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/ExampleAdminServer.java new file mode 100644 index 0000000000..d0f2111041 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/ExampleAdminServer.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server; + +import org.apache.eventmesh.admin.server.constants.AdminServerConstants; +import org.apache.eventmesh.common.config.ConfigService; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@SpringBootApplication(scanBasePackages = "org.apache.eventmesh.admin.server", exclude = {DataSourceAutoConfiguration.class}) +public class ExampleAdminServer { + + public static void main(String[] args) throws Exception { + ConfigService.getInstance().setConfigPath(AdminServerConstants.EVENTMESH_CONF_HOME).setRootConfig(AdminServerConstants.EVENTMESH_CONF_FILE); + SpringApplication.run(ExampleAdminServer.class); + log.info("admin start success."); + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/constants/AdminServerConstants.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/constants/AdminServerConstants.java new file mode 100644 index 0000000000..8ed079fd31 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/constants/AdminServerConstants.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.constants; + +public class AdminServerConstants { + public static final String CONF_ENV = "configurationPath"; + + public static final String EVENTMESH_CONF_HOME = System.getProperty(CONF_ENV, System.getenv(CONF_ENV)); + + public static final String EVENTMESH_CONF_FILE = "eventmesh-admin.properties"; +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/BaseServer.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/BaseServer.java new file mode 100644 index 0000000000..9bbe4ce305 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/BaseServer.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web; + +import org.apache.eventmesh.common.ComponentLifeCycle; +import org.apache.eventmesh.common.remote.payload.PayloadFactory; + +import javax.annotation.PostConstruct; +import javax.annotation.PreDestroy; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public abstract class BaseServer implements ComponentLifeCycle { + + static { + PayloadFactory.getInstance().init(); + } + + @PostConstruct + public void init() throws Exception { + log.info("[{}] server starting at port [{}]", this.getClass().getSimpleName(), getPort()); + start(); + log.info("[{}] server started at port [{}]", this.getClass().getSimpleName(), getPort()); + } + + @PreDestroy + public void shutdown() throws Exception { + log.info("[{}] server will destroy", this.getClass().getSimpleName()); + stop(); + log.info("[{}] server has be destroy", this.getClass().getSimpleName()); + } + + public abstract int getPort(); +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/GrpcServer.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/GrpcServer.java new file mode 100644 index 0000000000..d2a0330355 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/GrpcServer.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web; + +import org.apache.eventmesh.admin.server.AdminServerProperties; +import org.apache.eventmesh.admin.server.web.service.AdminGrpcServer; + +import java.util.concurrent.TimeUnit; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Controller; + +import io.grpc.Server; +import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; + +import lombok.extern.slf4j.Slf4j; + +@Controller +@Slf4j +public class GrpcServer extends BaseServer { + + @Autowired + AdminGrpcServer adminGrpcServer; + + @Autowired + AdminServerProperties properties; + + private Server server; + + @Override + public void start() throws Exception { + NettyServerBuilder serverBuilder = NettyServerBuilder.forPort(getPort()).addService(adminGrpcServer); + if (properties.isEnableSSL()) { + serverBuilder.sslContext(null); + } + server = serverBuilder.build(); + server.start(); + } + + @Override + public void stop() { + try { + if (server != null) { + server.shutdown(); + if (!server.awaitTermination(30, TimeUnit.SECONDS)) { + log.warn("[{}] server don't graceful stop in 30s, it will shutdown now", this.getClass().getSimpleName()); + server.shutdownNow(); + } + } + } catch (InterruptedException e) { + log.warn("destroy [{}] server fail", this.getClass().getSimpleName(), e); + } + } + + @Override + public int getPort() { + return properties.getPort(); + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/HttpServer.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/HttpServer.java new file mode 100644 index 0000000000..0a20d8645e --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/HttpServer.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web; + +import org.apache.eventmesh.admin.server.web.db.service.EventMeshTaskInfoService; +import org.apache.eventmesh.admin.server.web.service.monitor.MonitorBizService; +import org.apache.eventmesh.admin.server.web.service.task.TaskBizService; +import org.apache.eventmesh.admin.server.web.service.verify.VerifyBizService; +import org.apache.eventmesh.common.remote.request.CreateTaskRequest; +import org.apache.eventmesh.common.remote.request.QueryTaskInfoRequest; +import org.apache.eventmesh.common.remote.request.QueryTaskMonitorRequest; +import org.apache.eventmesh.common.remote.request.ReportMonitorRequest; +import org.apache.eventmesh.common.remote.request.ReportVerifyRequest; +import org.apache.eventmesh.common.remote.request.TaskBachRequest; +import org.apache.eventmesh.common.remote.request.TaskIDRequest; +import org.apache.eventmesh.common.remote.response.CreateTaskResponse; +import org.apache.eventmesh.common.remote.response.HttpResponseResult; +import org.apache.eventmesh.common.remote.response.QueryTaskInfoResponse; +import org.apache.eventmesh.common.remote.response.QueryTaskMonitorResponse; +import org.apache.eventmesh.common.remote.response.SimpleResponse; +import org.apache.eventmesh.common.utils.JsonUtils; + +import java.util.ArrayList; +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RestController; + +import lombok.extern.slf4j.Slf4j; + +@RestController +@RequestMapping("/eventmesh/admin") +@Slf4j +public class HttpServer { + + @Autowired + private TaskBizService taskService; + + @Autowired + private VerifyBizService verifyService; + + @Autowired + private MonitorBizService monitorService; + + @Autowired + private EventMeshTaskInfoService taskInfoService; + + @RequestMapping(value = "/createTask", method = RequestMethod.POST) + public String createOrUpdateTask(@RequestBody CreateTaskRequest task) { + log.info("receive http proto create task:{}", task); + CreateTaskResponse createTaskResponse = taskService.createTask(task); + log.info("receive http proto create task result:{}", createTaskResponse); + SimpleResponse simpleResponse = new SimpleResponse(); + simpleResponse.setData(createTaskResponse); + return JsonUtils.toJSONString(simpleResponse); + } + + + @RequestMapping(value = "/reportVerify", method = RequestMethod.POST) + public String reportVerify(@RequestBody ReportVerifyRequest request) { + log.info("receive http proto report verify request:{}", request); + boolean result = verifyService.reportVerifyRecord(request); + log.info("receive http proto report verify result:{}", result); + SimpleResponse simpleResponse = new SimpleResponse(); + simpleResponse.setData(result); + return JsonUtils.toJSONString(simpleResponse); + } + + @RequestMapping(value = "/reportMonitor", method = RequestMethod.POST) + public String reportMonitor(@RequestBody ReportMonitorRequest request) { + log.info("receive http proto report monitor request:{}", request); + boolean result = monitorService.reportMonitorRecord(request); + log.info("receive http proto report monitor result:{}", result); + SimpleResponse simpleResponse = new SimpleResponse(); + simpleResponse.setData(result); + return JsonUtils.toJSONString(simpleResponse); + } + + @RequestMapping(value = "/queryTaskMonitor", method = RequestMethod.POST) + public String queryTaskMonitor(@RequestBody QueryTaskMonitorRequest request) { + log.info("receive http proto query task monitor request:{}", request); + QueryTaskMonitorResponse result = monitorService.queryTaskMonitors(request); + log.info("receive http proto query task monitor result:{}", result); + SimpleResponse simpleResponse = new SimpleResponse(); + simpleResponse.setData(result); + return JsonUtils.toJSONString(simpleResponse); + } + + @RequestMapping(value = "/queryTaskInfo", method = RequestMethod.POST) + public HttpResponseResult queryTaskInfo(@RequestBody QueryTaskInfoRequest taskInfoRequest) { + log.info("receive http query task info request:{}", taskInfoRequest); + List taskInfosResponse = taskService.queryTaskInfo(taskInfoRequest); + log.info("receive http query task info taskInfosResponse:{}", taskInfoRequest); + if (taskInfosResponse.isEmpty()) { + return HttpResponseResult.failed("NOT FOUND"); + } + return HttpResponseResult.success(taskInfosResponse); + } + + @RequestMapping(value = "/deleteTask", method = RequestMethod.DELETE) + public HttpResponseResult deleteTask(@RequestBody TaskIDRequest taskIDRequest) { + log.info("receive need to delete taskID:{}", taskIDRequest.getTaskID()); + boolean result = taskService.deleteTaskByTaskID(taskIDRequest); + if (result) { + return HttpResponseResult.success(); + } else { + return HttpResponseResult.failed(); + } + } + + @RequestMapping(value = "/startTask", method = RequestMethod.POST) + public HttpResponseResult startTask(@RequestBody TaskIDRequest taskIDRequest) { + log.info("receive start task ID:{}", taskIDRequest.getTaskID()); + taskService.startTask(taskIDRequest); + return HttpResponseResult.success(); + } + + @RequestMapping(value = "/restartTask", method = RequestMethod.POST) + public HttpResponseResult restartTask(@RequestBody TaskIDRequest taskIDRequest) { + log.info("receive restart task ID:{}", taskIDRequest.getTaskID()); + taskService.restartTask(taskIDRequest); + return HttpResponseResult.success(); + } + + @RequestMapping(value = "/stopTask", method = RequestMethod.POST) + public HttpResponseResult stopTask(@RequestBody TaskIDRequest taskIDRequest) { + log.info("receive stop task ID:{}", taskIDRequest.getTaskID()); + taskService.stopTask(taskIDRequest); + return HttpResponseResult.success(); + } + + @RequestMapping(value = "/restartBatch", method = RequestMethod.POST) + public HttpResponseResult restartBatch(@RequestBody List taskBachRequestList) { + log.info("receive restart batch task IDs:{}", taskBachRequestList); + List errorNames = new ArrayList<>(); + taskService.restartBatchTask(taskBachRequestList, errorNames); + if (!errorNames.isEmpty()) { + return HttpResponseResult.failed(errorNames); + } + return HttpResponseResult.success(); + } + + @RequestMapping(value = "stopBatch", method = RequestMethod.POST) + public HttpResponseResult stopBatch(@RequestBody List taskBachRequestList) { + log.info("receive stop batch task IDs:{}", taskBachRequestList); + List errorNames = new ArrayList<>(); + taskService.stopBatchTask(taskBachRequestList, errorNames); + if (!errorNames.isEmpty()) { + return HttpResponseResult.failed(errorNames); + } + return HttpResponseResult.success(); + } + + @RequestMapping(value = "/startBatch", method = RequestMethod.POST) + public HttpResponseResult startBatch(@RequestBody List taskBachRequestList) { + log.info("receive start batch task IDs:{}", taskBachRequestList); + List errorNames = new ArrayList<>(); + taskService.startBatchTask(taskBachRequestList, errorNames); + if (!errorNames.isEmpty()) { + return HttpResponseResult.failed(errorNames); + } + return HttpResponseResult.success(); + } + +} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/Request.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/Request.java new file mode 100644 index 0000000000..9484e986f1 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/Request.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web; + +public class Request { + private String uid; + private T data; + + public String getUid() { + return uid; + } + + public void setUid(String uid) { + this.uid = uid; + } + + public T getData() { + return data; + } + + public void setData(T data) { + this.data = data; + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/Response.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/Response.java new file mode 100644 index 0000000000..d573c3bac4 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/Response.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web; + +import org.apache.eventmesh.common.remote.exception.ErrorCode; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class Response { + + private int code; + + private boolean success; + + private String desc; + + private T data; + + public static Response success() { + Response response = new Response<>(); + response.success = true; + response.code = ErrorCode.SUCCESS; + return response; + } + + public static Response success(T data) { + Response response = new Response<>(); + response.success = true; + response.data = data; + return response; + } + + public static Response fail(int code, String desc) { + Response response = new Response<>(); + response.success = false; + response.code = code; + response.desc = desc; + return response; + } + + public static Response fail(int code, String desc, T data) { + Response response = new Response<>(); + response.success = false; + response.code = code; + response.desc = desc; + response.data = data; + return response; + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/ServerController.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/ServerController.java new file mode 100644 index 0000000000..5623cbad33 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/ServerController.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web; + +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/eventmesh/admin") +public class ServerController { + +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/config/MybatisPlusConfig.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/config/MybatisPlusConfig.java new file mode 100644 index 0000000000..15d362bcd0 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/config/MybatisPlusConfig.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import com.baomidou.mybatisplus.annotation.DbType; +import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor; +import com.baomidou.mybatisplus.extension.plugins.inner.PaginationInnerInterceptor; + +@Configuration +public class MybatisPlusConfig { + + @Bean + public MybatisPlusInterceptor paginationInterceptor() { + MybatisPlusInterceptor interceptor = new MybatisPlusInterceptor(); + PaginationInnerInterceptor pageInterceptor = new PaginationInnerInterceptor(DbType.MYSQL); + pageInterceptor.setMaxLimit(500L); + interceptor.addInnerInterceptor(pageInterceptor); + return interceptor; + } + +} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/DBThreadPool.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/DBThreadPool.java new file mode 100644 index 0000000000..277ea66656 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/DBThreadPool.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db; + +import org.apache.eventmesh.common.EventMeshThreadFactory; + +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import javax.annotation.PreDestroy; + +import org.springframework.stereotype.Component; + +import lombok.extern.slf4j.Slf4j; + +@Component +@Slf4j +public class DBThreadPool { + + private final ThreadPoolExecutor executor = + new ThreadPoolExecutor(Runtime.getRuntime().availableProcessors() * 2, + Runtime.getRuntime().availableProcessors() * 2, 0L, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(1000), new EventMeshThreadFactory("admin-server-db"), + new ThreadPoolExecutor.DiscardOldestPolicy()); + + + private final ScheduledThreadPoolExecutor checkScheduledExecutor = + new ScheduledThreadPoolExecutor(Runtime.getRuntime().availableProcessors(), new EventMeshThreadFactory("admin-server-check-scheduled"), + new ThreadPoolExecutor.DiscardOldestPolicy()); + + @PreDestroy + private void destroy() { + if (!executor.isShutdown()) { + try { + executor.shutdown(); + if (!executor.awaitTermination(30, TimeUnit.SECONDS)) { + log.info("wait handler thread pool shutdown timeout, it will shutdown immediately"); + executor.shutdownNow(); + } + } catch (InterruptedException e) { + log.warn("wait handler thread pool shutdown fail"); + } + } + + if (!checkScheduledExecutor.isShutdown()) { + try { + checkScheduledExecutor.shutdown(); + if (!checkScheduledExecutor.awaitTermination(30, TimeUnit.SECONDS)) { + log.info("wait scheduled thread pool shutdown timeout, it will shutdown immediately"); + checkScheduledExecutor.shutdownNow(); + } + } catch (InterruptedException e) { + log.warn("wait scheduled thread pool shutdown fail"); + } + } + } + + public ThreadPoolExecutor getExecutors() { + return executor; + } + + public ScheduledThreadPoolExecutor getCheckExecutor() { + return checkScheduledExecutor; + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/DruidDataSource.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/DruidDataSource.java new file mode 100644 index 0000000000..fb26d44d30 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/DruidDataSource.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db; + + +import org.apache.eventmesh.admin.server.web.utils.EncryptUtil; +import org.apache.eventmesh.admin.server.web.utils.ParamType; + +import org.apache.commons.lang3.StringUtils; + +import java.io.IOException; +import java.sql.SQLException; + +import javax.sql.DataSource; + +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; + +import lombok.extern.slf4j.Slf4j; + +@Configuration +@ComponentScan +@Slf4j +public class DruidDataSource { + + @Value("${spring.datasource.url}") + private String dbUrl; + + @Value("${spring.datasource.username}") + private String username; + + @Value("${spring.datasource.password}") + private String password; + + @Value("${spring.datasource.driver-class-name}") + private String driverClassName; + + @Value("${spring.datasource.initialSize}") + private int initialSize; + + @Value("${spring.datasource.minIdle}") + private int minIdle; + + @Value("${spring.datasource.maxActive}") + private int maxActive; + + @Value("${spring.datasource.maxWait}") + private int maxWait; + + @Value("${spring.datasource.timeBetweenEvictionRunsMillis}") + private int timeBetweenEvictionRunsMillis; + + @Value("${spring.datasource.minEvictableIdleTimeMillis}") + private int minEvictableIdleTimeMillis; + + @Value("${spring.datasource.validationQuery}") + private String validationQuery; + + @Value("${spring.datasource.testWhileIdle}") + private boolean testWhileIdle; + + @Value("${spring.datasource.testOnBorrow}") + private boolean testOnBorrow; + + @Value("${spring.datasource.testOnReturn}") + private boolean testOnReturn; + + @Value("${spring.datasource.poolPreparedStatements}") + private boolean poolPreparedStatements; + + @Value("${spring.datasource.maxPoolPreparedStatementPerConnectionSize}") + private int maxPoolPreparedStatementPerConnectionSize; + + @Value("${spring.datasource.filters}") + private String filters; + + @Value("{spring.datasource.connectionProperties}") + private String connectionProperties; + + @Value("${sysPubKey}") + private String sysPubKeyStr; + + @Value("${appPrivKey}") + private String appPrivKeyStr; + + + @Bean + @Primary + public DataSource dataSource() throws Exception { + try (com.alibaba.druid.pool.DruidDataSource datasource = new com.alibaba.druid.pool.DruidDataSource()) { + datasource.setUrl(this.dbUrl); + datasource.setUsername(username); + datasource.setPassword(rsaDecrypt(sysPubKeyStr, appPrivKeyStr, password)); + datasource.setDriverClassName(driverClassName); + datasource.setInitialSize(initialSize); + datasource.setMinIdle(minIdle); + datasource.setMaxActive(maxActive); + datasource.setMaxWait(maxWait); + datasource.setTimeBetweenEvictionRunsMillis(timeBetweenEvictionRunsMillis); + datasource.setMinEvictableIdleTimeMillis(minEvictableIdleTimeMillis); + datasource.setValidationQuery(validationQuery); + datasource.setTestWhileIdle(testWhileIdle); + datasource.setTestOnBorrow(testOnBorrow); + datasource.setTestOnReturn(testOnReturn); + datasource.setPoolPreparedStatements(poolPreparedStatements); + datasource.setMaxPoolPreparedStatementPerConnectionSize(maxPoolPreparedStatementPerConnectionSize); + try { + datasource.setFilters(filters); + } catch (SQLException e) { + log.error("druid configuration initialization filter", e); + } + datasource.setConnectionProperties(connectionProperties); + + return datasource; + } + } + + public static String rsaDecrypt(String sysPubKeyStr, String appPrivKeyStr, String encrtyptText) throws IOException { + if (StringUtils.isNotBlank(encrtyptText) && encrtyptText.length() > "{RSA}".length() && encrtyptText.startsWith("{RSA}")) { + String text = encrtyptText.startsWith("{RSA}") ? encrtyptText.substring("{RSA}".length()) : encrtyptText; + + try { + return EncryptUtil.decrypt(ParamType.STRING, sysPubKeyStr, ParamType.STRING, appPrivKeyStr, ParamType.STRING, text); + } catch (Exception e) { + throw new RuntimeException("decrypt error", e); + } + } else { + return encrtyptText; + } + } + +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshDataSource.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshDataSource.java new file mode 100644 index 0000000000..e6e328984c --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshDataSource.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.entity; + +import java.io.Serializable; +import java.util.Date; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; + +import lombok.Data; + +/** + * TableName event_mesh_data_source + */ +@TableName(value = "event_mesh_data_source") +@Data +public class EventMeshDataSource implements Serializable { + @TableId(type = IdType.AUTO) + private Integer id; + + private String dataType; + + private String description; + + private String configuration; + + private String configurationClass; + + private String region; + + private String createUid; + + private String updateUid; + + private Date createTime; + + private Date updateTime; + + private static final long serialVersionUID = 1L; +} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshJobInfo.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshJobInfo.java new file mode 100644 index 0000000000..a77eaaaca2 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshJobInfo.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.entity; + +import java.io.Serializable; +import java.util.Date; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; + +import lombok.Data; + +/** + * TableName event_mesh_job_info + */ +@TableName(value = "event_mesh_job_info") +@Data +public class EventMeshJobInfo implements Serializable { + @TableId(type = IdType.AUTO) + private Integer id; + + private String jobID; + + private String jobDesc; + + private String taskID; + + private String transportType; + + private Integer sourceData; + + private Integer targetData; + + private String jobState; + + private String jobType; + + // job request from region + private String fromRegion; + + // job actually running region + private String runningRegion; + + private String createUid; + + private String updateUid; + + private Date createTime; + + private Date updateTime; + + private static final long serialVersionUID = 1L; +} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshMonitor.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshMonitor.java new file mode 100644 index 0000000000..0507464b5b --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshMonitor.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.entity; + +import java.io.Serializable; +import java.util.Date; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; + +import lombok.Data; + +/** + * TableName event_mesh_monitor + */ +@TableName(value = "event_mesh_monitor") +@Data +public class EventMeshMonitor implements Serializable { + + @TableId(type = IdType.AUTO) + private Integer id; + + private String taskID; + private String jobID; + private String address; + private String transportType; + private String connectorStage; + private Long totalReqNum; + private Long totalTimeCost; + private Long maxTimeCost; + private Long avgTimeCost; + private Double tps; + private Date createTime; + + private static final long serialVersionUID = 1L; +} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshMysqlPosition.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshMysqlPosition.java new file mode 100644 index 0000000000..5e5d5745c1 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshMysqlPosition.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.entity; + +import java.io.Serializable; +import java.util.Date; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; + +import lombok.Data; + +/** + * TableName event_mesh_mysql_position + */ +@TableName(value = "event_mesh_mysql_position") +@Data +public class EventMeshMysqlPosition implements Serializable { + @TableId(type = IdType.AUTO) + private Integer id; + + private String jobID; + + private String serverUUID; + + private String address; + + private Long position; + + private String gtid; + + private String currentGtid; + + private Long timestamp; + + private String journalName; + + private Date createTime; + + private Date updateTime; + + private static final long serialVersionUID = 1L; +} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshPositionReporterHistory.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshPositionReporterHistory.java new file mode 100644 index 0000000000..8518c38918 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshPositionReporterHistory.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.entity; + +import java.io.Serializable; +import java.util.Date; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; + +import lombok.Data; + +/** + * TableName event_mesh_position_reporter_history + */ +@TableName(value = "event_mesh_position_reporter_history") +@Data +public class EventMeshPositionReporterHistory implements Serializable { + @TableId(type = IdType.AUTO) + private Long id; + + private String job; + + private String record; + + private String address; + + private Date createTime; + + private static final long serialVersionUID = 1L; +} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshRuntimeHeartbeat.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshRuntimeHeartbeat.java new file mode 100644 index 0000000000..95e6c5e261 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshRuntimeHeartbeat.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.entity; + +import java.io.Serializable; +import java.util.Date; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; + +import lombok.Data; + +/** + * TableName event_mesh_runtime_heartbeat + */ +@TableName(value = "event_mesh_runtime_heartbeat") +@Data +public class EventMeshRuntimeHeartbeat implements Serializable { + @TableId(type = IdType.AUTO) + private Long id; + + private String adminAddr; + + private String runtimeAddr; + + private String jobID; + + private String reportTime; + + private Date updateTime; + + private Date createTime; + + private static final long serialVersionUID = 1L; +} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshRuntimeHistory.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshRuntimeHistory.java new file mode 100644 index 0000000000..ea7e10cbad --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshRuntimeHistory.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.entity; + +import java.io.Serializable; +import java.util.Date; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; + +import lombok.Data; + +/** + * TableName event_mesh_runtime_history + */ +@TableName(value = "event_mesh_runtime_history") +@Data +public class EventMeshRuntimeHistory implements Serializable { + @TableId(type = IdType.AUTO) + private Long id; + + private String job; + + private String address; + + private Date createTime; + + private static final long serialVersionUID = 1L; +} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshTaskInfo.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshTaskInfo.java new file mode 100644 index 0000000000..2d40f4a082 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshTaskInfo.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.entity; + +import java.io.Serializable; +import java.util.Date; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; + +import lombok.Data; + +/** + * TableName event_mesh_task_info + */ +@TableName(value = "event_mesh_task_info") +@Data +public class EventMeshTaskInfo implements Serializable { + @TableId(type = IdType.AUTO) + private Integer id; + + private String taskID; + + private String taskName; + + private String taskDesc; + + private String taskState; + + private String sourceRegion; + + private String targetRegion; + + private String createUid; + + private String updateUid; + + private Date createTime; + + private Date updateTime; + + private static final long serialVersionUID = 1L; +} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshVerify.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshVerify.java new file mode 100644 index 0000000000..c5a6c35f8d --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshVerify.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.entity; + +import java.io.Serializable; +import java.util.Date; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; + +import lombok.Data; + +/** + * TableName event_mesh_verify + */ +@TableName(value = "event_mesh_verify") +@Data +public class EventMeshVerify implements Serializable { + + @TableId(type = IdType.AUTO) + private Integer id; + + private String taskID; + + private String jobID; + + private String recordID; + + private String recordSig; + + private String connectorName; + + private String connectorStage; + + private String position; + + private Date createTime; + + private static final long serialVersionUID = 1L; +} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshWeredisPosition.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshWeredisPosition.java new file mode 100644 index 0000000000..2117230826 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/entity/EventMeshWeredisPosition.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.entity; + +import java.io.Serializable; +import java.util.Date; + +import com.baomidou.mybatisplus.annotation.IdType; +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; + +import lombok.Data; + +/** + * TableName event_mesh_weredis_position + */ +@TableName(value = "event_mesh_weredis_position") +@Data +public class EventMeshWeredisPosition implements Serializable { + @TableId(type = IdType.AUTO) + private Integer id; + + private String jobID; + + // connection run address + private String address; + + private String clusterName; + + private String partitionName; + + private String masterReplid; + + //weredis run host + private String host; + + private Long replOffset = -1L; + + private Date createTime; + + private Date updateTime; + + private static final long serialVersionUID = 1L; +} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshDataSourceMapper.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshDataSourceMapper.java new file mode 100644 index 0000000000..c59e28428f --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshDataSourceMapper.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.mapper; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshDataSource; + +import org.apache.ibatis.annotations.Mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + +/** + * for table 'event_mesh_data_source' db operation + * 2024-05-09 15:52:49 + * entity.db.web.server.admin.eventmesh.apache.org.EventMeshDataSource + */ +@Mapper +public interface EventMeshDataSourceMapper extends BaseMapper { + +} + + + + diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshJobInfoExtMapper.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshJobInfoExtMapper.java new file mode 100644 index 0000000000..c04c4e3748 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshJobInfoExtMapper.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.mapper; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; + +import org.apache.ibatis.annotations.Insert; +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Param; + +import java.util.List; + +import org.springframework.transaction.annotation.Transactional; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + +/** + * etx operator for table event_mesh_job_info + */ +@Mapper +public interface EventMeshJobInfoExtMapper extends BaseMapper { + + @Insert("") + @Transactional(rollbackFor = Exception.class) + int saveBatch(@Param("jobs") List jobInfoList); +} + + + + diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshJobInfoMapper.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshJobInfoMapper.java new file mode 100644 index 0000000000..39f8a4aed6 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshJobInfoMapper.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.mapper; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; + +import org.apache.ibatis.annotations.Mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + +/** + * for table 'event_mesh_job_info' db operation + * entity org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo + */ +@Mapper +public interface EventMeshJobInfoMapper extends BaseMapper { + +} + + + + diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshMonitorMapper.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshMonitorMapper.java new file mode 100644 index 0000000000..db77224637 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshMonitorMapper.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.mapper; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshMonitor; + +import org.apache.ibatis.annotations.Mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + +/** + * event_mesh_monitor + * Entity org.apache.eventmesh.admin.server.web.db.entity.EventMeshMonitor + */ +@Mapper +public interface EventMeshMonitorMapper extends BaseMapper { + +} + + + + diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshMysqlPositionMapper.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshMysqlPositionMapper.java new file mode 100644 index 0000000000..f0a0467d76 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshMysqlPositionMapper.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.mapper; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshMysqlPosition; + +import org.apache.ibatis.annotations.Mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + +/** + * for table `event_mesh_mysql_position` db operation + * 2024-05-14 17:15:03 + * entity.db.web.server.admin.eventmesh.apache.org.EventMeshMysqlPosition + */ +@Mapper +public interface EventMeshMysqlPositionMapper extends BaseMapper { + +} + + + + diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshPositionReporterHistoryMapper.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshPositionReporterHistoryMapper.java new file mode 100644 index 0000000000..adc6723b79 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshPositionReporterHistoryMapper.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.mapper; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshPositionReporterHistory; + +import org.apache.ibatis.annotations.Mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + +/** + * for table 'event_mesh_position_reporter_history' db operation + * 2024-05-14 17:15:03 + * entity.db.web.server.admin.eventmesh.apache.org.EventMeshPositionReporterHistory + */ +@Mapper +public interface EventMeshPositionReporterHistoryMapper extends BaseMapper { + +} + + + + diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshRuntimeHeartbeatMapper.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshRuntimeHeartbeatMapper.java new file mode 100644 index 0000000000..813769d2ab --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshRuntimeHeartbeatMapper.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.mapper; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshRuntimeHeartbeat; + +import org.apache.ibatis.annotations.Mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + +/** + * for table 'event_mesh_runtime_heartbeat' db operation + * 2024-05-14 17:15:03 + * entity.db.web.server.admin.eventmesh.apache.org.EventMeshRuntimeHeartbeat + */ +@Mapper +public interface EventMeshRuntimeHeartbeatMapper extends BaseMapper { + +} + + + + diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshRuntimeHistoryMapper.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshRuntimeHistoryMapper.java new file mode 100644 index 0000000000..5fc3a21f54 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshRuntimeHistoryMapper.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.mapper; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshRuntimeHistory; + +import org.apache.ibatis.annotations.Mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + +/** + * for table 'event_mesh_runtime_history' db operation + * 2024-05-14 17:15:03 + * entity.db.web.server.admin.eventmesh.apache.org.EventMeshRuntimeHistory + */ +@Mapper +public interface EventMeshRuntimeHistoryMapper extends BaseMapper { + +} + + + + diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshTaskInfoMapper.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshTaskInfoMapper.java new file mode 100644 index 0000000000..d1d472b8c4 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshTaskInfoMapper.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.mapper; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshTaskInfo; + +import org.apache.ibatis.annotations.Mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + +/** + * event_mesh_task_info + * Entity org.apache.eventmesh.admin.server.web.db.entity.EventMeshTaskInfo + */ +@Mapper +public interface EventMeshTaskInfoMapper extends BaseMapper { + +} + + + + diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshVerifyMapper.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshVerifyMapper.java new file mode 100644 index 0000000000..b444d1e4b4 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/mapper/EventMeshVerifyMapper.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.mapper; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshVerify; + +import org.apache.ibatis.annotations.Mapper; + +import com.baomidou.mybatisplus.core.mapper.BaseMapper; + +/** + * event_mesh_verify + * Entity org.apache.eventmesh.admin.server.web.db.entity.EventMeshVerify + */ +@Mapper +public interface EventMeshVerifyMapper extends BaseMapper { + +} + + + + diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshDataSourceService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshDataSourceService.java new file mode 100644 index 0000000000..29e2b8122e --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshDataSourceService.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.service; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshDataSource; + +import com.baomidou.mybatisplus.extension.service.IService; + +/** +* +* for table 'event_mesh_data_source' db operation +* 2024-05-09 15:52:49 +*/ +public interface EventMeshDataSourceService extends IService { + +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshJobInfoExtService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshJobInfoExtService.java new file mode 100644 index 0000000000..22fc5ae299 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshJobInfoExtService.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.service; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; + +import java.util.List; + +/** + * ext operator for table event_mesh_job + */ +public interface EventMeshJobInfoExtService { + int batchSave(List jobs); +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshJobInfoService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshJobInfoService.java new file mode 100644 index 0000000000..572e451ceb --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshJobInfoService.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.service; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; + +import com.baomidou.mybatisplus.extension.service.IService; + +/** + * event_mesh_job_info + */ +public interface EventMeshJobInfoService extends IService { + +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshMonitorService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshMonitorService.java new file mode 100644 index 0000000000..4180f82a97 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshMonitorService.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.service; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshMonitor; + +import com.baomidou.mybatisplus.extension.service.IService; + +/** + * event_mesh_monitor + */ +public interface EventMeshMonitorService extends IService { + +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshMysqlPositionService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshMysqlPositionService.java new file mode 100644 index 0000000000..2bf34ab922 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshMysqlPositionService.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.service; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshMysqlPosition; + +import com.baomidou.mybatisplus.extension.service.IService; + +/** +* for table 'event_mesh_mysql_position' db operation +* 2024-05-14 17:15:03 +*/ +public interface EventMeshMysqlPositionService extends IService { +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshPositionReporterHistoryService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshPositionReporterHistoryService.java new file mode 100644 index 0000000000..976eec94ea --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshPositionReporterHistoryService.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.service; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshPositionReporterHistory; + +import com.baomidou.mybatisplus.extension.service.IService; + +/** +* for table 'event_mesh_position_reporter_history' db operation +* 2024-05-14 17:15:03 +*/ +public interface EventMeshPositionReporterHistoryService extends IService { + +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshRuntimeHeartbeatService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshRuntimeHeartbeatService.java new file mode 100644 index 0000000000..f0e5d1c61b --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshRuntimeHeartbeatService.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.service; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshRuntimeHeartbeat; + +import com.baomidou.mybatisplus.extension.service.IService; + +/** + * for table 'event_mesh_runtime_heartbeat' db operation + * 2024-05-14 17:15:03 + */ +public interface EventMeshRuntimeHeartbeatService extends IService { + +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshRuntimeHistoryService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshRuntimeHistoryService.java new file mode 100644 index 0000000000..1f6db0e12e --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshRuntimeHistoryService.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.service; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshRuntimeHistory; + +import com.baomidou.mybatisplus.extension.service.IService; + +/** + * for table 'event_mesh_runtime_history' db operation + * 2024-05-14 17:15:03 + */ +public interface EventMeshRuntimeHistoryService extends IService { + +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshTaskInfoService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshTaskInfoService.java new file mode 100644 index 0000000000..04da6a7952 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshTaskInfoService.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.service; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshTaskInfo; +import org.apache.eventmesh.common.remote.request.QueryTaskInfoRequest; +import org.apache.eventmesh.common.remote.response.QueryTaskInfoResponse; + +import java.util.List; + +import com.baomidou.mybatisplus.extension.service.IService; + +/** + * event_mesh_task_info + */ +public interface EventMeshTaskInfoService extends IService { + + List queryTaskInfo(QueryTaskInfoRequest taskInfoRequest); + + // boolean deleteTaskByTaskID(String taskID); + +} diff --git a/eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/Transformer.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshVerifyService.java similarity index 71% rename from eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/Transformer.java rename to eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshVerifyService.java index 8239dfcb6e..97f2d7268e 100644 --- a/eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/Transformer.java +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/EventMeshVerifyService.java @@ -1,32 +1,29 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.transformer; - -import com.fasterxml.jackson.core.JsonProcessingException; - -/** - * EventMesh transformer interface, specified transformer implementation includes: - * 1. Constant - * 2. Original - * 3. Template - */ -public interface Transformer { - - String transform(String json) throws JsonProcessingException; - -} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.service; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshVerify; + +import com.baomidou.mybatisplus.extension.service.IService; + +/** + * event_mesh_verify + */ +public interface EventMeshVerifyService extends IService { + +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshDataSourceServiceImpl.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshDataSourceServiceImpl.java new file mode 100644 index 0000000000..f703425c94 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshDataSourceServiceImpl.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.service.impl; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshDataSource; +import org.apache.eventmesh.admin.server.web.db.mapper.EventMeshDataSourceMapper; +import org.apache.eventmesh.admin.server.web.db.service.EventMeshDataSourceService; + +import org.springframework.stereotype.Service; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; + +/** + * for table 'event_mesh_data_source' db operation + * 2024-05-09 15:52:49 + */ +@Service +public class EventMeshDataSourceServiceImpl extends ServiceImpl + implements EventMeshDataSourceService { + +} + + + + diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshJobInfoExtServiceImpl.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshJobInfoExtServiceImpl.java new file mode 100644 index 0000000000..6cf0ebf6b2 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshJobInfoExtServiceImpl.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.service.impl; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; +import org.apache.eventmesh.admin.server.web.db.mapper.EventMeshJobInfoExtMapper; +import org.apache.eventmesh.admin.server.web.db.service.EventMeshJobInfoExtService; + +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class EventMeshJobInfoExtServiceImpl implements EventMeshJobInfoExtService { + @Autowired + EventMeshJobInfoExtMapper mapper; + + @Override + public int batchSave(List jobs) { + return mapper.saveBatch(jobs); + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshJobInfoServiceImpl.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshJobInfoServiceImpl.java new file mode 100644 index 0000000000..4613e0809d --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshJobInfoServiceImpl.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.service.impl; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; +import org.apache.eventmesh.admin.server.web.db.mapper.EventMeshJobInfoMapper; +import org.apache.eventmesh.admin.server.web.db.service.EventMeshJobInfoService; + +import org.springframework.stereotype.Service; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; + +/** + * event_mesh_job_info + */ +@Service +public class EventMeshJobInfoServiceImpl extends ServiceImpl + implements EventMeshJobInfoService { + +} + + + + diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshMonitorServiceImpl.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshMonitorServiceImpl.java new file mode 100644 index 0000000000..ebb4220000 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshMonitorServiceImpl.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.service.impl; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshMonitor; +import org.apache.eventmesh.admin.server.web.db.mapper.EventMeshMonitorMapper; +import org.apache.eventmesh.admin.server.web.db.service.EventMeshMonitorService; + +import org.springframework.stereotype.Service; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; + +/** + * event_mesh_monitor + */ +@Service +public class EventMeshMonitorServiceImpl extends ServiceImpl + implements EventMeshMonitorService { + +} + + + + diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshMysqlPositionServiceImpl.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshMysqlPositionServiceImpl.java new file mode 100644 index 0000000000..353443b78d --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshMysqlPositionServiceImpl.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.service.impl; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshMysqlPosition; +import org.apache.eventmesh.admin.server.web.db.mapper.EventMeshMysqlPositionMapper; +import org.apache.eventmesh.admin.server.web.db.service.EventMeshMysqlPositionService; + +import org.springframework.stereotype.Service; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; + +import lombok.extern.slf4j.Slf4j; + +/** + * for table 'event_mesh_mysql_position' db operation + * 2024-05-14 17:15:03 + */ +@Service +@Slf4j +public class EventMeshMysqlPositionServiceImpl extends ServiceImpl + implements EventMeshMysqlPositionService { + +} + + + + diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshPositionReporterHistoryServiceImpl.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshPositionReporterHistoryServiceImpl.java new file mode 100644 index 0000000000..d546e412ba --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshPositionReporterHistoryServiceImpl.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.service.impl; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshPositionReporterHistory; +import org.apache.eventmesh.admin.server.web.db.mapper.EventMeshPositionReporterHistoryMapper; +import org.apache.eventmesh.admin.server.web.db.service.EventMeshPositionReporterHistoryService; + +import org.springframework.stereotype.Service; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; + +/** + * for table 'event_mesh_position_reporter_history' db operation + * 2024-05-14 17:15:03 + */ +@Service +public class EventMeshPositionReporterHistoryServiceImpl extends ServiceImpl + implements EventMeshPositionReporterHistoryService { + +} + + + + diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshRuntimeHeartbeatServiceImpl.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshRuntimeHeartbeatServiceImpl.java new file mode 100644 index 0000000000..452569e3a6 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshRuntimeHeartbeatServiceImpl.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.service.impl; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshRuntimeHeartbeat; +import org.apache.eventmesh.admin.server.web.db.mapper.EventMeshRuntimeHeartbeatMapper; +import org.apache.eventmesh.admin.server.web.db.service.EventMeshRuntimeHeartbeatService; + +import org.springframework.stereotype.Service; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; + +import lombok.extern.slf4j.Slf4j; + +/** + * for table 'event_mesh_runtime_heartbeat' db operation + * 2024-05-14 17:15:03 + */ +@Service +@Slf4j +public class EventMeshRuntimeHeartbeatServiceImpl extends ServiceImpl + implements EventMeshRuntimeHeartbeatService { + +} + + + + diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshRuntimeHistoryServiceImpl.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshRuntimeHistoryServiceImpl.java new file mode 100644 index 0000000000..d39e868ce5 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshRuntimeHistoryServiceImpl.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.service.impl; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshRuntimeHistory; +import org.apache.eventmesh.admin.server.web.db.mapper.EventMeshRuntimeHistoryMapper; +import org.apache.eventmesh.admin.server.web.db.service.EventMeshRuntimeHistoryService; + +import org.springframework.stereotype.Service; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; + +/** + * for table 'event_mesh_runtime_history' db operation + * 2024-05-14 17:15:03 + */ +@Service +public class EventMeshRuntimeHistoryServiceImpl extends ServiceImpl + implements EventMeshRuntimeHistoryService { + +} + + + + diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshTaskInfoServiceImpl.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshTaskInfoServiceImpl.java new file mode 100644 index 0000000000..91acb51a76 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshTaskInfoServiceImpl.java @@ -0,0 +1,323 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.service.impl; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshDataSource; +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshMysqlPosition; +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshTaskInfo; +import org.apache.eventmesh.admin.server.web.db.mapper.EventMeshTaskInfoMapper; +import org.apache.eventmesh.admin.server.web.db.service.EventMeshDataSourceService; +import org.apache.eventmesh.admin.server.web.db.service.EventMeshJobInfoService; +import org.apache.eventmesh.admin.server.web.db.service.EventMeshMysqlPositionService; +import org.apache.eventmesh.admin.server.web.db.service.EventMeshTaskInfoService; +import org.apache.eventmesh.common.remote.JobState; +import org.apache.eventmesh.common.remote.TaskState; +import org.apache.eventmesh.common.remote.request.QueryTaskInfoRequest; +import org.apache.eventmesh.common.remote.response.QueryTaskInfoResponse; + +import org.apache.commons.lang3.ObjectUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import com.baomidou.mybatisplus.core.toolkit.Wrappers; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; + +import lombok.extern.slf4j.Slf4j; + +/** + * event_mesh_task_info + */ +@Slf4j +@Service +public class EventMeshTaskInfoServiceImpl extends ServiceImpl + implements EventMeshTaskInfoService { + + @Autowired + private EventMeshTaskInfoMapper taskInfoMapper; + + @Autowired + private EventMeshJobInfoService jobInfoService; + + @Autowired + private EventMeshDataSourceService dataSourceService; + + @Autowired + private EventMeshMysqlPositionService mysqlPositionService; + + @Override + public List queryTaskInfo(QueryTaskInfoRequest taskInfoRequest) { + + log.info("receive query task info request:{}", taskInfoRequest); + + List queryTaskInfoResponseList = new ArrayList<>(); + + Integer currentPage = taskInfoRequest.getCurrentPage(); + Integer pageSize = taskInfoRequest.getPageSize(); + + // query by page + if (StringUtils.isEmpty(taskInfoRequest.getTaskID()) + && currentPage != null + && pageSize != null) { + + Page page = new Page<>(); + page.setCurrent(currentPage); + page.setSize(pageSize); + List eventMeshTaskInfoList = taskInfoMapper.selectPage(page, Wrappers.query() + .ne("taskState", TaskState.DELETE.name())).getRecords(); + queryTaskInfoResponseList = getQueryTaskInfoResponses(eventMeshTaskInfoList, queryTaskInfoResponseList); + + } + + if (StringUtils.isNotEmpty(taskInfoRequest.getTaskID()) || StringUtils.isNotEmpty(taskInfoRequest.getTaskID())) { + queryTaskInfoResponseList = eventMeshTaskInfoList(taskInfoRequest); + } + + // if (StringUtils.isNotEmpty(taskInfoRequest.getJobType())) { + // + // } + // + // if (StringUtils.isNotEmpty(taskInfoRequest.getSourceDataID())) { + // + // } + // + // if (StringUtils.isNotEmpty(taskInfoRequest.getTargetDataID())) { + // + // } + // + // if (StringUtils.isNotEmpty(taskInfoRequest.getIp())) { + // + // } + // + // if (StringUtils.isNotEmpty(taskInfoRequest.getSourceTableName())) { + // + // } + // + // if (StringUtils.isNotEmpty(taskInfoRequest.getTaskMathID())) { + // + // } + + log.info("query event mesh task info response result:{}", queryTaskInfoResponseList); + + return queryTaskInfoResponseList; + } + + @Transactional + private List eventMeshTaskInfoList(QueryTaskInfoRequest taskInfoRequest) { + + List eventMeshTaskInfoList = new ArrayList<>(); + + Page page = new Page<>(); + page.setCurrent(taskInfoRequest.getCurrentPage()); + page.setSize(taskInfoRequest.getPageSize()); + + if (StringUtils.isNotEmpty(taskInfoRequest.getTaskID())) { + eventMeshTaskInfoList = taskInfoMapper.selectPage(page, Wrappers.query() + .eq("taskID", taskInfoRequest.getTaskID()) + .ne("taskState", TaskState.DELETE.name())) + .getRecords(); + } + + if (StringUtils.isNotEmpty(taskInfoRequest.getTaskDesc())) { + eventMeshTaskInfoList = taskInfoMapper.selectPage(page, Wrappers.query() + .like("taskDesc", taskInfoRequest.getTaskDesc()) + .ne("jobState", JobState.DELETE.name())) + .getRecords(); + } + + List eventMeshTaskInfos = new ArrayList<>(); + + List queryTaskInfoResponse = getQueryTaskInfoResponses(eventMeshTaskInfoList, eventMeshTaskInfos); + log.info("query task info result queryTaskInfoResponse:{}", queryTaskInfoResponse); + + return queryTaskInfoResponse; + } + + private List getQueryTaskInfoResponses(List eventMeshTaskInfoList, + List eventMeshTaskInfos) { + + for (EventMeshTaskInfo meshTaskInfo : eventMeshTaskInfoList) { + QueryTaskInfoResponse eventMeshTaskInfo = initEventMeshTaskInfo(meshTaskInfo); + eventMeshTaskInfos.add(eventMeshTaskInfo); + } + + if (!eventMeshTaskInfoList.isEmpty()) { + List eventMeshJobInfoList = new ArrayList<>(); + for (QueryTaskInfoResponse eventMeshTaskInfo : eventMeshTaskInfos) { + List eventMeshJobInfos = jobInfoService.list(Wrappers.query() + .eq("taskID", eventMeshTaskInfo.getTaskID()) + .ne("jobState", JobState.DELETE.name())); + + for (EventMeshJobInfo eventMeshJobInfo : eventMeshJobInfos) { + QueryTaskInfoResponse.EventMeshJobInfo eventMeshJobInfoCovert = initEventMeshJobInfo(eventMeshJobInfo); + eventMeshJobInfoList.add(eventMeshJobInfoCovert); + } + + if (!eventMeshJobInfoList.isEmpty()) { + for (QueryTaskInfoResponse.EventMeshJobInfo eventMeshJobInfo : eventMeshJobInfoList) { + QueryTaskInfoResponse.EventMeshDataSource dataSource = covertEventMeshDataSource( + querySourceOrSinkData(eventMeshJobInfo.getSourceData())); + QueryTaskInfoResponse.EventMeshDataSource dataSink = covertEventMeshDataSource( + querySourceOrSinkData(eventMeshJobInfo.getTargetData())); + + EventMeshMysqlPosition eventMeshMysqlPosition = mysqlPositionService.getOne(Wrappers.query().eq( + "jobID", + eventMeshJobInfo.getJobID() + )); + + + QueryTaskInfoResponse.EventMeshMysqlPosition mysqlPosition = covertEventMeshMysqlPosition(eventMeshMysqlPosition); + + eventMeshJobInfo.setEventMeshMysqlPosition(mysqlPosition); + eventMeshJobInfo.setDataSource(dataSource); + eventMeshJobInfo.setDataSink(dataSink); + } + } + + // set job info to same taskID + eventMeshTaskInfo.setEventMeshJobInfoList(eventMeshJobInfoList); + } + } + + List queryTaskInfoResponse = new ArrayList<>(); + if (!eventMeshTaskInfos.isEmpty()) { + queryTaskInfoResponse.addAll(eventMeshTaskInfos); + } + + return queryTaskInfoResponse; + } + + /** + * QueryTaskInfoResponse.EventMeshDataSource covert + * + * @param eventMeshData EventMeshDataSource + * @return meshData + */ + private static QueryTaskInfoResponse.EventMeshDataSource covertEventMeshDataSource(EventMeshDataSource eventMeshData) { + QueryTaskInfoResponse.EventMeshDataSource meshData = new QueryTaskInfoResponse.EventMeshDataSource(); + if (ObjectUtils.isEmpty(eventMeshData)) { + return null; + } + meshData.setId(eventMeshData.getId()); + meshData.setDataType(eventMeshData.getDataType()); + meshData.setConfiguration(eventMeshData.getConfiguration()); + meshData.setConfigurationClass(eventMeshData.getConfigurationClass()); + meshData.setDescription(eventMeshData.getDescription()); + meshData.setRegion(eventMeshData.getRegion()); + meshData.setCreateUid(eventMeshData.getCreateUid()); + meshData.setUpdateUid(eventMeshData.getUpdateUid()); + meshData.setCreateTime(eventMeshData.getCreateTime()); + meshData.setUpdateTime(eventMeshData.getUpdateTime()); + return meshData; + } + + /** + * getSourceOrSinkData + * + * @param id id + * @return EventMeshDataSource + */ + private EventMeshDataSource querySourceOrSinkData(Integer id) { + return dataSourceService.getOne(Wrappers.query().eq( + "id", + id)); + } + + /** + * QueryTaskInfoResponse.EventMeshMysqlPosition + * + * @param mysqlPosition EventMeshMysqlPosition + * @return position + */ + private static QueryTaskInfoResponse.EventMeshMysqlPosition covertEventMeshMysqlPosition(EventMeshMysqlPosition mysqlPosition) { + QueryTaskInfoResponse.EventMeshMysqlPosition position = new QueryTaskInfoResponse.EventMeshMysqlPosition(); + if (ObjectUtils.isEmpty(mysqlPosition)) { + return null; + } + position.setId(mysqlPosition.getId()); + position.setJobID(mysqlPosition.getJobID()); + position.setServerUUID(mysqlPosition.getServerUUID()); + position.setAddress(mysqlPosition.getAddress()); + position.setPosition(mysqlPosition.getPosition()); + position.setGtid(mysqlPosition.getGtid()); + position.setCurrentGtid(mysqlPosition.getCurrentGtid()); + position.setTimestamp(mysqlPosition.getTimestamp()); + position.setJournalName(mysqlPosition.getJournalName()); + position.setCreateTime(mysqlPosition.getCreateTime()); + position.setUpdateTime(mysqlPosition.getUpdateTime()); + return position; + } + + /** + * EventMeshJobInfo covert + * + * @param eventMeshJobInfo EventMeshJobInfo + * @return QueryTaskInfoResponse.EventMeshJobInfo + */ + private static QueryTaskInfoResponse.EventMeshJobInfo initEventMeshJobInfo(EventMeshJobInfo eventMeshJobInfo) { + QueryTaskInfoResponse.EventMeshJobInfo eventMeshJobInfoCovert = new QueryTaskInfoResponse.EventMeshJobInfo(); + if (ObjectUtils.isEmpty(eventMeshJobInfo)) { + return null; + } + eventMeshJobInfoCovert.setId(eventMeshJobInfo.getId()); + eventMeshJobInfoCovert.setJobID(eventMeshJobInfo.getJobID()); + eventMeshJobInfoCovert.setJobDesc(eventMeshJobInfo.getJobDesc()); + eventMeshJobInfoCovert.setTaskID(eventMeshJobInfo.getTaskID()); + eventMeshJobInfoCovert.setTransportType(eventMeshJobInfo.getTransportType()); + eventMeshJobInfoCovert.setSourceData(eventMeshJobInfo.getSourceData()); + eventMeshJobInfoCovert.setTargetData(eventMeshJobInfo.getTargetData()); + eventMeshJobInfoCovert.setJobState(eventMeshJobInfo.getJobState()); + eventMeshJobInfoCovert.setJobType(eventMeshJobInfo.getJobType()); + eventMeshJobInfoCovert.setFromRegion(eventMeshJobInfo.getFromRegion()); + eventMeshJobInfoCovert.setRunningRegion(eventMeshJobInfo.getRunningRegion()); + eventMeshJobInfoCovert.setCreateUid(eventMeshJobInfo.getCreateUid()); + eventMeshJobInfoCovert.setUpdateUid(eventMeshJobInfo.getUpdateUid()); + eventMeshJobInfoCovert.setCreateTime(eventMeshJobInfo.getCreateTime()); + eventMeshJobInfoCovert.setUpdateTime(eventMeshJobInfo.getUpdateTime()); + return eventMeshJobInfoCovert; + } + + /** + * EventMeshTaskInfo covert + * + * @param meshTaskInfo EventMeshTaskInfo + * @return QueryTaskInfoResponse + */ + private static QueryTaskInfoResponse initEventMeshTaskInfo(EventMeshTaskInfo meshTaskInfo) { + QueryTaskInfoResponse eventMeshTaskInfo = new QueryTaskInfoResponse(); + eventMeshTaskInfo.setId(meshTaskInfo.getId()); + eventMeshTaskInfo.setTaskID(meshTaskInfo.getTaskID()); + eventMeshTaskInfo.setTaskDesc(meshTaskInfo.getTaskDesc()); + eventMeshTaskInfo.setTaskState(meshTaskInfo.getTaskState()); + eventMeshTaskInfo.setSourceRegion(meshTaskInfo.getSourceRegion()); + eventMeshTaskInfo.setTargetRegion(meshTaskInfo.getTargetRegion()); + eventMeshTaskInfo.setCreateUid(meshTaskInfo.getCreateUid()); + eventMeshTaskInfo.setUpdateUid(meshTaskInfo.getUpdateUid()); + eventMeshTaskInfo.setCreateTime(meshTaskInfo.getCreateTime()); + eventMeshTaskInfo.setUpdateTime(meshTaskInfo.getUpdateTime()); + return eventMeshTaskInfo; + } + +} \ No newline at end of file diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshVerifyServiceImpl.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshVerifyServiceImpl.java new file mode 100644 index 0000000000..5e49ba32ea --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/db/service/impl/EventMeshVerifyServiceImpl.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.db.service.impl; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshVerify; +import org.apache.eventmesh.admin.server.web.db.mapper.EventMeshVerifyMapper; +import org.apache.eventmesh.admin.server.web.db.service.EventMeshVerifyService; + +import org.springframework.stereotype.Service; + +import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl; + +/** + * event_mesh_verify + */ +@Service +public class EventMeshVerifyServiceImpl extends ServiceImpl + implements EventMeshVerifyService { + +} + + + + diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/BaseRequestHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/BaseRequestHandler.java new file mode 100644 index 0000000000..7f08f388c9 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/BaseRequestHandler.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.handler; + +import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; +import org.apache.eventmesh.common.remote.request.BaseRemoteRequest; +import org.apache.eventmesh.common.remote.response.BaseRemoteResponse; + +public abstract class BaseRequestHandler { + + public BaseRemoteResponse handlerRequest(T request, Metadata metadata) { + return handler(request, metadata); + } + + protected abstract S handler(T request, Metadata metadata); +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/RequestHandlerFactory.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/RequestHandlerFactory.java new file mode 100644 index 0000000000..9375fb537e --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/RequestHandlerFactory.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.handler; + +import org.apache.eventmesh.common.remote.request.BaseRemoteRequest; +import org.apache.eventmesh.common.remote.response.BaseRemoteResponse; + +import java.lang.reflect.ParameterizedType; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.springframework.context.ApplicationListener; +import org.springframework.context.event.ContextRefreshedEvent; +import org.springframework.stereotype.Component; + +@Component +public class RequestHandlerFactory implements ApplicationListener { + + private final Map> handlers = + new ConcurrentHashMap<>(); + + public BaseRequestHandler getHandler(String type) { + return handlers.get(type); + } + + @Override + @SuppressWarnings({"rawtypes", "unchecked"}) + public void onApplicationEvent(ContextRefreshedEvent event) { + Map beans = + event.getApplicationContext().getBeansOfType(BaseRequestHandler.class); + + for (BaseRequestHandler requestHandler : beans.values()) { + Class clazz = requestHandler.getClass(); + boolean skip = false; + while (!clazz.getSuperclass().equals(BaseRequestHandler.class)) { + if (clazz.getSuperclass().equals(Object.class)) { + skip = true; + break; + } + clazz = clazz.getSuperclass(); + } + if (skip) { + continue; + } + + Class c = (Class) ((ParameterizedType) clazz.getGenericSuperclass()).getActualTypeArguments()[0]; + handlers.putIfAbsent(c.getSimpleName(), requestHandler); + } + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/FetchJobRequestHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/FetchJobRequestHandler.java new file mode 100644 index 0000000000..3392084c28 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/FetchJobRequestHandler.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.handler.impl; + +import org.apache.eventmesh.admin.server.web.handler.BaseRequestHandler; +import org.apache.eventmesh.admin.server.web.pojo.JobDetail; +import org.apache.eventmesh.admin.server.web.service.job.JobInfoBizService; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; +import org.apache.eventmesh.common.remote.exception.ErrorCode; +import org.apache.eventmesh.common.remote.job.JobConnectorConfig; +import org.apache.eventmesh.common.remote.request.FetchJobRequest; +import org.apache.eventmesh.common.remote.response.FetchJobResponse; +import org.apache.eventmesh.common.utils.JsonUtils; + +import org.apache.commons.lang3.StringUtils; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import lombok.extern.slf4j.Slf4j; + +@Component +@Slf4j +public class FetchJobRequestHandler extends BaseRequestHandler { + + @Autowired + JobInfoBizService jobInfoBizService; + + @Override + public FetchJobResponse handler(FetchJobRequest request, Metadata metadata) { + if (StringUtils.isBlank(request.getJobID())) { + return FetchJobResponse.failResponse(ErrorCode.BAD_REQUEST, "job id is empty"); + } + FetchJobResponse response = FetchJobResponse.successResponse(); + JobDetail detail = jobInfoBizService.getJobDetail(request.getJobID()); + if (detail == null) { + return response; + } + response.setId(detail.getJobID()); + JobConnectorConfig config = new JobConnectorConfig(); + config.setSourceConnectorConfig(JsonUtils.objectToMap(detail.getSourceDataSource().getConf())); + config.setSourceConnectorDesc(detail.getSourceConnectorDesc()); + config.setSinkConnectorConfig(JsonUtils.objectToMap(detail.getSinkDataSource().getConf())); + config.setSinkConnectorDesc(detail.getSinkConnectorDesc()); + response.setConnectorConfig(config); + response.setTransportType(detail.getTransportType()); + response.setState(detail.getState()); + response.setPosition(detail.getPositions()); + response.setType(detail.getJobType()); + return response; + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/FetchPositionHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/FetchPositionHandler.java new file mode 100644 index 0000000000..85ef0e6113 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/FetchPositionHandler.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.handler.impl; + +import org.apache.eventmesh.admin.server.web.db.DBThreadPool; +import org.apache.eventmesh.admin.server.web.handler.BaseRequestHandler; +import org.apache.eventmesh.admin.server.web.service.position.PositionBizService; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; +import org.apache.eventmesh.common.remote.exception.ErrorCode; +import org.apache.eventmesh.common.remote.request.FetchPositionRequest; +import org.apache.eventmesh.common.remote.response.FetchPositionResponse; + +import org.apache.commons.lang3.StringUtils; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import lombok.extern.slf4j.Slf4j; + +@Component +@Slf4j +public class FetchPositionHandler extends BaseRequestHandler { + + @Autowired + DBThreadPool executor; + + @Autowired + PositionBizService positionBizService; + + @Override + protected FetchPositionResponse handler(FetchPositionRequest request, Metadata metadata) { + if (request.getDataSourceType() == null) { + return FetchPositionResponse.failResponse(ErrorCode.BAD_REQUEST, "illegal data type, it's empty"); + } + if (StringUtils.isBlank(request.getJobID())) { + return FetchPositionResponse.failResponse(ErrorCode.BAD_REQUEST, "illegal job id, it's empty"); + } + return FetchPositionResponse.successResponse(positionBizService.getPosition(request, metadata)); + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportHeartBeatHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportHeartBeatHandler.java new file mode 100644 index 0000000000..26a9b430b7 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportHeartBeatHandler.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.handler.impl; + +import org.apache.eventmesh.admin.server.web.db.DBThreadPool; +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshRuntimeHeartbeat; +import org.apache.eventmesh.admin.server.web.handler.BaseRequestHandler; +import org.apache.eventmesh.admin.server.web.service.heatbeat.RuntimeHeartbeatBizService; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; +import org.apache.eventmesh.common.remote.exception.ErrorCode; +import org.apache.eventmesh.common.remote.request.ReportHeartBeatRequest; +import org.apache.eventmesh.common.remote.response.SimpleResponse; +import org.apache.eventmesh.common.utils.IPUtils; + +import org.apache.commons.lang3.StringUtils; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import lombok.extern.slf4j.Slf4j; + +@Component +@Slf4j +public class ReportHeartBeatHandler extends BaseRequestHandler { + + @Autowired + RuntimeHeartbeatBizService heartbeatBizService; + + @Autowired + DBThreadPool executor; + + @Override + protected SimpleResponse handler(ReportHeartBeatRequest request, Metadata metadata) { + if (StringUtils.isBlank(request.getJobID()) || StringUtils.isBlank(request.getAddress())) { + log.info("request [{}] id or reporter address is empty", request); + return SimpleResponse.fail(ErrorCode.BAD_REQUEST, "request id or reporter address is empty"); + } + executor.getExecutors().execute(() -> { + EventMeshRuntimeHeartbeat heartbeat = new EventMeshRuntimeHeartbeat(); + heartbeat.setJobID(request.getJobID()); + heartbeat.setReportTime(request.getReportedTimeStamp()); + heartbeat.setAdminAddr(IPUtils.getLocalAddress()); + heartbeat.setRuntimeAddr(request.getAddress()); + try { + if (!heartbeatBizService.saveOrUpdateByRuntimeAddress(heartbeat)) { + log.warn("save or update heartbeat request [{}] fail", request); + } + } catch (Exception e) { + log.warn("save or update heartbeat request [{}] fail", request, e); + } + }); + + return SimpleResponse.success(); + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportJobRequestHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportJobRequestHandler.java new file mode 100644 index 0000000000..c876014f63 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportJobRequestHandler.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.handler.impl; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; +import org.apache.eventmesh.admin.server.web.handler.BaseRequestHandler; +import org.apache.eventmesh.admin.server.web.pojo.TaskDetail; +import org.apache.eventmesh.admin.server.web.service.job.JobInfoBizService; +import org.apache.eventmesh.admin.server.web.service.position.PositionBizService; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; +import org.apache.eventmesh.common.remote.JobState; +import org.apache.eventmesh.common.remote.TransportType; +import org.apache.eventmesh.common.remote.datasource.DataSourceType; +import org.apache.eventmesh.common.remote.exception.ErrorCode; +import org.apache.eventmesh.common.remote.offset.RecordPosition; +import org.apache.eventmesh.common.remote.request.RecordPositionRequest; +import org.apache.eventmesh.common.remote.request.ReportJobRequest; +import org.apache.eventmesh.common.remote.response.SimpleResponse; + +import org.apache.commons.lang3.StringUtils; + +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import lombok.extern.slf4j.Slf4j; + + +@Component +@Slf4j +public class ReportJobRequestHandler extends BaseRequestHandler { + + @Autowired + JobInfoBizService jobInfoBizService; + + @Autowired + PositionBizService positionBizService; + + @Override + public SimpleResponse handler(ReportJobRequest request, Metadata metadata) { + log.info("receive report job request:{}", request); + if (StringUtils.isBlank(request.getJobID())) { + return SimpleResponse.fail(ErrorCode.BAD_REQUEST, "illegal job id, it's empty"); + } + EventMeshJobInfo jobInfo = jobInfoBizService.getJobInfo(request.getJobID()); + if (jobInfo == null) { + return SimpleResponse.fail(ErrorCode.BAD_REQUEST, "illegal job id, not exist target job,jobID:" + request.getJobID()); + } + boolean recordResult = recordPosition(request, metadata, jobInfo); + boolean result = recordResult && jobInfoBizService.updateJobState(jobInfo.getJobID(), request.getState()); + if (result) { + return SimpleResponse.success(); + } else { + return SimpleResponse.fail(ErrorCode.INTERNAL_ERR, "update job failed."); + } + } + + private boolean recordPosition(ReportJobRequest request, Metadata metadata, EventMeshJobInfo jobInfo) { + if (!jobInfo.getJobState().equalsIgnoreCase(JobState.INIT.name()) || !JobState.RUNNING.name().equalsIgnoreCase(request.getState().name())) { + log.info("skip record position because of job state not from init change to running.jobID:{}", jobInfo.getJobID()); + return true; + } + + TaskDetail taskDetail = jobInfoBizService.getTaskDetail(jobInfo.getTaskID(), DataSourceType.MYSQL); + if (taskDetail.getFullTask() == null || taskDetail.getIncreaseTask() == null) { + log.info("skip record position because of not exist full and increase job.jobID:{}", jobInfo.getJobID()); + return true; + } + + List recordPositionList = + positionBizService.getPositionByJobID(taskDetail.getIncreaseTask().getJobID(), DataSourceType.MYSQL); + if (!recordPositionList.isEmpty()) { + log.info("skip record position because of increase job has exist position.jobID:{},position list size:{}", jobInfo.getJobID(), + recordPositionList.size()); + return true; + } + + RecordPositionRequest recordPositionRequest = new RecordPositionRequest(); + recordPositionRequest.setFullJobID(taskDetail.getFullTask().getJobID()); + recordPositionRequest.setIncreaseJobID(taskDetail.getIncreaseTask().getJobID()); + recordPositionRequest.setUpdateState(request.getState()); + recordPositionRequest.setAddress(request.getAddress()); + TransportType currentTransportType = TransportType.getTransportType(jobInfo.getTransportType()); + recordPositionRequest.setDataSourceType(currentTransportType.getSrc()); + return positionBizService.recordPosition(recordPositionRequest, metadata); + } + +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportMonitorHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportMonitorHandler.java new file mode 100644 index 0000000000..a36939bb88 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportMonitorHandler.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.handler.impl; + +import org.apache.eventmesh.admin.server.AdminServerProperties; +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; +import org.apache.eventmesh.admin.server.web.handler.BaseRequestHandler; +import org.apache.eventmesh.admin.server.web.service.job.JobInfoBizService; +import org.apache.eventmesh.admin.server.web.service.monitor.MonitorBizService; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; +import org.apache.eventmesh.common.remote.exception.ErrorCode; +import org.apache.eventmesh.common.remote.request.ReportMonitorRequest; +import org.apache.eventmesh.common.remote.response.SimpleResponse; + +import org.apache.commons.lang3.StringUtils; + +import java.util.Arrays; +import java.util.List; +import java.util.Random; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; + +import lombok.extern.slf4j.Slf4j; + +@Component +@Slf4j +public class ReportMonitorHandler extends BaseRequestHandler { + + @Autowired + private MonitorBizService monitorService; + + @Autowired + JobInfoBizService jobInfoBizService; + + @Autowired + private AdminServerProperties properties; + + @Override + protected SimpleResponse handler(ReportMonitorRequest request, Metadata metadata) { + if (StringUtils.isAnyBlank(request.getTaskID(), request.getJobID(), request.getAddress())) { + log.info("report monitor request [{}] illegal", request); + return SimpleResponse.fail(ErrorCode.BAD_REQUEST, "request task id,job id or address is none"); + } + + String jobID = request.getJobID(); + EventMeshJobInfo jobInfo = jobInfoBizService.getJobInfo(jobID); + if (jobInfo == null || StringUtils.isBlank(jobInfo.getFromRegion())) { + log.info("report monitor job info [{}] illegal", request); + return SimpleResponse.fail(ErrorCode.BAD_REQUEST, "job info is null or fromRegion is blank,job id:" + jobID); + } + String fromRegion = jobInfo.getFromRegion(); + String transportType = jobInfo.getTransportType(); + if (StringUtils.isEmpty(request.getTransportType())) { + request.setTransportType(transportType); + } + String localRegion = properties.getRegion(); + log.info("report monitor request from region:{},localRegion:{},request:{}", fromRegion, localRegion, request); + if (fromRegion.equalsIgnoreCase(localRegion)) { + return monitorService.reportMonitorRecord(request) ? SimpleResponse.success() : + SimpleResponse.fail(ErrorCode.INTERNAL_ERR, "save monitor " + + "request fail"); + } else { + List adminServerList = Arrays.asList(properties.getAdminServerList().get(fromRegion).split(";")); + if (adminServerList == null || adminServerList.isEmpty()) { + throw new RuntimeException("No admin server available for region: " + fromRegion); + } + String targetUrl = adminServerList.get(new Random().nextInt(adminServerList.size())) + "/eventmesh/admin/reportMonitor"; + log.info("start transfer monitor request to from region admin server. from region:{}, targetUrl:{}", fromRegion, targetUrl); + RestTemplate restTemplate = new RestTemplate(); + ResponseEntity response = restTemplate.postForEntity(targetUrl, request, String.class); + if (!response.getStatusCode().is2xxSuccessful()) { + log.error("transfer monitor request to from region admin server error. from region:{}, targetUrl:{}", fromRegion, targetUrl); + return SimpleResponse.fail(ErrorCode.INTERNAL_ERR, + "save monitor request fail,code:" + response.getStatusCode() + ",msg:" + response.getBody()); + } + return SimpleResponse.success(); + } + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportPositionHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportPositionHandler.java new file mode 100644 index 0000000000..7a30bef80a --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportPositionHandler.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.handler.impl; + +import org.apache.eventmesh.admin.server.web.db.DBThreadPool; +import org.apache.eventmesh.admin.server.web.handler.BaseRequestHandler; +import org.apache.eventmesh.admin.server.web.pojo.JobDetail; +import org.apache.eventmesh.admin.server.web.service.job.JobInfoBizService; +import org.apache.eventmesh.admin.server.web.service.position.PositionBizService; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; +import org.apache.eventmesh.common.remote.exception.ErrorCode; +import org.apache.eventmesh.common.remote.request.ReportPositionRequest; +import org.apache.eventmesh.common.remote.response.SimpleResponse; + +import org.apache.commons.lang3.StringUtils; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import lombok.extern.slf4j.Slf4j; + +@Component +@Slf4j +public class ReportPositionHandler extends BaseRequestHandler { + + @Autowired + private JobInfoBizService jobInfoBizService; + + @Autowired + private DBThreadPool executor; + + @Autowired + private PositionBizService positionBizService; + + @Override + protected SimpleResponse handler(ReportPositionRequest request, Metadata metadata) { + log.info("receive report position request:{}", request); + if (StringUtils.isBlank(request.getJobID())) { + log.info("request [{}] illegal job id", request); + return SimpleResponse.fail(ErrorCode.BAD_REQUEST, "illegal job id, it's empty"); + } + if (request.getDataSourceType() == null) { + log.info("request [{}] illegal data type", request); + return SimpleResponse.fail(ErrorCode.BAD_REQUEST, "illegal data type, it's empty"); + } + if (StringUtils.isBlank(request.getJobID())) { + log.info("request [{}] illegal job id", request); + return SimpleResponse.fail(ErrorCode.BAD_REQUEST, "illegal job id, it's empty"); + } + if (request.getRecordPositionList() == null || request.getRecordPositionList().isEmpty()) { + log.info("request [{}] illegal record position", request); + return SimpleResponse.fail(ErrorCode.BAD_REQUEST, "illegal record position list, it's empty"); + } + + positionBizService.isValidatePositionRequest(request.getDataSourceType()); + + executor.getExecutors().execute(() -> { + try { + boolean reported = positionBizService.reportPosition(request, metadata); + if (reported) { + if (log.isDebugEnabled()) { + log.debug("handle runtime [{}] report data type [{}] job [{}] position [{}] success", + request.getAddress(), request.getDataSourceType(), request.getJobID(), + request.getRecordPositionList()); + } + } else { + log.warn("handle runtime [{}] report data type [{}] job [{}] position [{}] fail", + request.getAddress(), request.getDataSourceType(), request.getJobID(), + request.getRecordPositionList()); + } + } catch (Exception e) { + log.warn("handle position request fail, request [{}]", request, e); + } finally { + try { + JobDetail detail = jobInfoBizService.getJobDetail(request.getJobID()); + if (detail != null && !detail.getState().equals(request.getState()) && !jobInfoBizService.updateJobState(request.getJobID(), + request.getState())) { + log.warn("update job [{}] old state [{}] to [{}] fail", request.getJobID(), detail.getState(), request.getState()); + } + } catch (Exception e) { + log.warn("update job id [{}] type [{}] state [{}] fail", request.getJobID(), + request.getDataSourceType(), request.getState(), e); + } + } + }); + return SimpleResponse.success(); + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportVerifyHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportVerifyHandler.java new file mode 100644 index 0000000000..e7f1d1257f --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/handler/impl/ReportVerifyHandler.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.handler.impl; + +import org.apache.eventmesh.admin.server.AdminServerProperties; +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; +import org.apache.eventmesh.admin.server.web.handler.BaseRequestHandler; +import org.apache.eventmesh.admin.server.web.service.job.JobInfoBizService; +import org.apache.eventmesh.admin.server.web.service.verify.VerifyBizService; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; +import org.apache.eventmesh.common.remote.exception.ErrorCode; +import org.apache.eventmesh.common.remote.request.ReportVerifyRequest; +import org.apache.eventmesh.common.remote.response.SimpleResponse; + +import org.apache.commons.lang3.StringUtils; + +import java.util.Arrays; +import java.util.List; +import java.util.Random; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Component; +import org.springframework.web.client.RestTemplate; + +import lombok.extern.slf4j.Slf4j; + +@Component +@Slf4j +public class ReportVerifyHandler extends BaseRequestHandler { + + @Autowired + private VerifyBizService verifyService; + + @Autowired + JobInfoBizService jobInfoBizService; + + @Autowired + private AdminServerProperties properties; + + @Override + protected SimpleResponse handler(ReportVerifyRequest request, Metadata metadata) { + if (StringUtils.isAnyBlank(request.getTaskID(), request.getJobID(), request.getRecordSig(), request.getRecordID(), + request.getConnectorStage())) { + log.info("report verify request [{}] illegal", request); + return SimpleResponse.fail(ErrorCode.BAD_REQUEST, "request task id,job id, sign, record id or stage is none"); + } + + String jobID = request.getJobID(); + EventMeshJobInfo jobInfo = jobInfoBizService.getJobInfo(jobID); + if (jobInfo == null || StringUtils.isBlank(jobInfo.getFromRegion())) { + log.info("report verify job info [{}] illegal", request); + return SimpleResponse.fail(ErrorCode.BAD_REQUEST, "job info is null or fromRegion is blank,job id:" + jobID); + } + + String fromRegion = jobInfo.getFromRegion(); + String localRegion = properties.getRegion(); + log.info("report verify request from region:{},localRegion:{},request:{}", fromRegion, localRegion, request); + if (fromRegion.equalsIgnoreCase(localRegion)) { + return verifyService.reportVerifyRecord(request) ? SimpleResponse.success() : SimpleResponse.fail(ErrorCode.INTERNAL_ERR, "save verify " + + "request fail"); + } else { + log.info("start transfer report verify to from region admin server. from region:{}", fromRegion); + List adminServerList = Arrays.asList(properties.getAdminServerList().get(fromRegion).split(";")); + if (adminServerList == null || adminServerList.isEmpty()) { + throw new RuntimeException("No admin server available for region: " + fromRegion); + } + String targetUrl = adminServerList.get(new Random().nextInt(adminServerList.size())) + "/eventmesh/admin/reportVerify"; + RestTemplate restTemplate = new RestTemplate(); + ResponseEntity response = restTemplate.postForEntity(targetUrl, request, String.class); + if (!response.getStatusCode().is2xxSuccessful()) { + return SimpleResponse.fail(ErrorCode.INTERNAL_ERR, + "save verify request fail,code:" + response.getStatusCode() + ",msg:" + response.getBody()); + } + return SimpleResponse.success(); + } + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/pojo/BinlogPosition.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/pojo/BinlogPosition.java new file mode 100644 index 0000000000..5bd8daab10 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/pojo/BinlogPosition.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.pojo; + + +import lombok.Data; + +@Data +public class BinlogPosition { + private String file; + private Long position; +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/pojo/JobDetail.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/pojo/JobDetail.java new file mode 100644 index 0000000000..0e2fa64878 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/pojo/JobDetail.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.pojo; + +import org.apache.eventmesh.common.remote.TaskState; +import org.apache.eventmesh.common.remote.TransportType; +import org.apache.eventmesh.common.remote.datasource.DataSource; +import org.apache.eventmesh.common.remote.job.JobType; +import org.apache.eventmesh.common.remote.offset.RecordPosition; + +import java.util.Date; +import java.util.List; + +import lombok.Data; + +@Data +public class JobDetail { + private Integer id; + + private String jobID; + + private String jobDesc; + + private String taskID; + + private TaskState state; + + private JobType jobType; + + private Date createTime; + + private Date updateTime; + + private String createUid; + + private String updateUid; + + // job request from region + private String fromRegion; + + // job actually running region + private String runningRegion; + + private DataSource sourceDataSource; + + private String sourceConnectorDesc; + + private DataSource sinkDataSource; + + private String sinkConnectorDesc; + + private TransportType transportType; + + private List positions; +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/pojo/TaskDetail.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/pojo/TaskDetail.java new file mode 100644 index 0000000000..2b174209e2 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/pojo/TaskDetail.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.pojo; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; + +import lombok.Data; + +/** + * Description: + */ +@Data +public class TaskDetail { + + private EventMeshJobInfo fullTask; + + private EventMeshJobInfo increaseTask; + +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/AdminGrpcServer.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/AdminGrpcServer.java new file mode 100644 index 0000000000..bc822ad6c3 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/AdminGrpcServer.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.service; + +import org.apache.eventmesh.admin.server.AdminServerRuntimeException; +import org.apache.eventmesh.admin.server.web.handler.BaseRequestHandler; +import org.apache.eventmesh.admin.server.web.handler.RequestHandlerFactory; +import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Payload; +import org.apache.eventmesh.common.remote.exception.ErrorCode; +import org.apache.eventmesh.common.remote.payload.PayloadUtil; +import org.apache.eventmesh.common.remote.request.BaseRemoteRequest; +import org.apache.eventmesh.common.remote.response.BaseRemoteResponse; +import org.apache.eventmesh.common.remote.response.SimpleResponse; + +import org.apache.commons.lang3.StringUtils; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import io.grpc.stub.ServerCallStreamObserver; +import io.grpc.stub.StreamObserver; + +import lombok.extern.slf4j.Slf4j; + +@Service +@Slf4j +public class AdminGrpcServer extends AdminServiceGrpc.AdminServiceImplBase { + + @Autowired + RequestHandlerFactory handlerFactory; + + private Payload process(Payload value) { + if (value == null || StringUtils.isBlank(value.getMetadata().getType())) { + return PayloadUtil.from(SimpleResponse.fail(ErrorCode.BAD_REQUEST, "bad request: type not exists")); + } + try { + BaseRequestHandler handler = handlerFactory.getHandler(value.getMetadata().getType()); + if (handler == null) { + return PayloadUtil.from(SimpleResponse.fail(ErrorCode.BAD_REQUEST, "not match any request handler")); + } + BaseRemoteResponse response = handler.handlerRequest((BaseRemoteRequest) PayloadUtil.parse(value), value.getMetadata()); + if (response == null) { + log.warn("received request type [{}] handler [{}], then replay empty response", value.getMetadata().getType(), + handler.getClass().getName()); + response = SimpleResponse.success(); + } + return PayloadUtil.from(response); + } catch (Exception e) { + log.warn("process payload {} fail", value.getMetadata().getType(), e); + if (e instanceof AdminServerRuntimeException) { + return PayloadUtil.from(SimpleResponse.fail(((AdminServerRuntimeException) e).getCode(), e.getMessage())); + } + return PayloadUtil.from(SimpleResponse.fail(ErrorCode.INTERNAL_ERR, "admin server internal err")); + } + } + + public StreamObserver invokeBiStream(StreamObserver responseObserver) { + return new StreamObserver() { + @Override + public void onNext(Payload value) { + Payload payload = process(value); + if (payload == null) { + return; + } + responseObserver.onNext(payload); + } + + @Override + public void onError(Throwable t) { + if (responseObserver instanceof ServerCallStreamObserver) { + if (!((ServerCallStreamObserver) responseObserver).isCancelled()) { + log.warn("admin gRPC server fail", t); + } + } + } + + @Override + public void onCompleted() { + responseObserver.onCompleted(); + } + }; + } + + public void invoke(Payload request, StreamObserver responseObserver) { + responseObserver.onNext(process(request)); + responseObserver.onCompleted(); + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/AdminServer.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/AdminServer.java new file mode 100644 index 0000000000..fd7582800d --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/AdminServer.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.service; + +import org.apache.eventmesh.admin.server.AdminServerProperties; +import org.apache.eventmesh.admin.server.AdminServerRuntimeException; +import org.apache.eventmesh.common.ComponentLifeCycle; +import org.apache.eventmesh.common.Constants; +import org.apache.eventmesh.common.config.CommonConfiguration; +import org.apache.eventmesh.common.config.ConfigService; +import org.apache.eventmesh.common.remote.exception.ErrorCode; +import org.apache.eventmesh.common.utils.IPUtils; +import org.apache.eventmesh.registry.RegisterServerInfo; +import org.apache.eventmesh.registry.RegistryFactory; +import org.apache.eventmesh.registry.RegistryService; + +import org.apache.commons.lang3.StringUtils; + +import javax.annotation.PostConstruct; + +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.stereotype.Service; + +import lombok.extern.slf4j.Slf4j; + +@Service +@Slf4j +public class AdminServer implements ComponentLifeCycle, ApplicationListener { + private final RegistryService registryService; + + private final RegisterServerInfo adminServeInfo; + + private final CommonConfiguration configuration; + + public AdminServer(AdminServerProperties properties) { + configuration = + ConfigService.getInstance().buildConfigInstance(CommonConfiguration.class); + if (configuration == null) { + throw new AdminServerRuntimeException(ErrorCode.STARTUP_CONFIG_MISS, "common configuration file miss"); + } + this.adminServeInfo = new RegisterServerInfo(); + + adminServeInfo.setHealth(true); + adminServeInfo.setAddress(IPUtils.getLocalAddress() + ":" + properties.getPort()); + String name = Constants.ADMIN_SERVER_REGISTRY_NAME; + if (StringUtils.isNotBlank(properties.getServiceName())) { + name = properties.getServiceName(); + } + adminServeInfo.setServiceName(name); + registryService = RegistryFactory.getInstance(configuration.getEventMeshRegistryPluginType()); + } + + @Override + @PostConstruct + public void start() { + if (configuration.isEventMeshRegistryPluginEnabled()) { + registryService.init(); + } + } + + @Override + public void stop() { + if (configuration.isEventMeshRegistryPluginEnabled()) { + registryService.unRegister(adminServeInfo); + try { + Thread.sleep(3000); + } catch (InterruptedException ignore) { + log.warn("interrupted when sleep"); + Thread.currentThread().interrupt(); + } + registryService.shutdown(); + } + } + + @Override + public void onApplicationEvent(ApplicationReadyEvent event) { + if (configuration.isEventMeshRegistryPluginEnabled()) { + log.info("application is started and registry plugin is enabled, it's will register admin self"); + registryService.register(adminServeInfo); + } + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/datasource/DataSourceBizService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/datasource/DataSourceBizService.java new file mode 100644 index 0000000000..4d2d670100 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/datasource/DataSourceBizService.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.service.datasource; + +import org.apache.eventmesh.admin.server.AdminServerRuntimeException; +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshDataSource; +import org.apache.eventmesh.admin.server.web.db.service.EventMeshDataSourceService; +import org.apache.eventmesh.common.remote.exception.ErrorCode; +import org.apache.eventmesh.common.remote.request.CreateOrUpdateDataSourceReq; +import org.apache.eventmesh.common.utils.JsonUtils; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class DataSourceBizService { + + @Autowired + private EventMeshDataSourceService dataSourceService; + + public EventMeshDataSource createDataSource(CreateOrUpdateDataSourceReq dataSource) { + EventMeshDataSource entity = new EventMeshDataSource(); + entity.setConfiguration(JsonUtils.toJSONString(dataSource.getConfig())); + entity.setConfigurationClass(dataSource.getConfigClass()); + entity.setDataType(dataSource.getType().name()); + entity.setCreateUid(dataSource.getOperator()); + entity.setUpdateUid(dataSource.getOperator()); + entity.setRegion(dataSource.getRegion()); + entity.setDescription(dataSource.getDesc()); + if (dataSourceService.save(entity)) { + return entity; + } + throw new AdminServerRuntimeException(ErrorCode.BAD_DB_DATA, "save data source fail"); + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/heatbeat/RuntimeHeartbeatBizService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/heatbeat/RuntimeHeartbeatBizService.java new file mode 100644 index 0000000000..95dff6e5b3 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/heatbeat/RuntimeHeartbeatBizService.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.service.heatbeat; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshRuntimeHeartbeat; +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshRuntimeHistory; +import org.apache.eventmesh.admin.server.web.db.service.EventMeshRuntimeHeartbeatService; +import org.apache.eventmesh.admin.server.web.db.service.EventMeshRuntimeHistoryService; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.baomidou.mybatisplus.core.toolkit.Wrappers; + +import lombok.extern.slf4j.Slf4j; + +/** + * for table 'event_mesh_runtime_heartbeat' db operation 2024-05-14 17:15:03 + */ +@Service +@Slf4j +public class RuntimeHeartbeatBizService { + + @Autowired + EventMeshRuntimeHistoryService historyService; + + @Autowired + EventMeshRuntimeHeartbeatService heartbeatService; + + public boolean saveOrUpdateByRuntimeAddress(EventMeshRuntimeHeartbeat entity) { + EventMeshRuntimeHeartbeat old = heartbeatService.getOne(Wrappers.query().eq( + "runtimeAddr", + entity.getRuntimeAddr())); + if (old == null) { + return heartbeatService.save(entity); + } else { + if (Long.parseLong(old.getReportTime()) >= Long.parseLong(entity.getReportTime())) { + log.info("update heartbeat record ignore, current report time late than db, job [{}], remote [{}]", entity.getJobID(), + entity.getRuntimeAddr()); + return true; + } + try { + return heartbeatService.update(entity, Wrappers.update().eq("updateTime", + old.getUpdateTime())); + } finally { + if (old.getJobID() != null && !old.getJobID().equals(entity.getJobID())) { + EventMeshRuntimeHistory history = new EventMeshRuntimeHistory(); + history.setAddress(old.getAdminAddr()); + history.setJob(old.getJobID()); + try { + historyService.save(history); + } catch (Exception e) { + log.warn("save runtime job changed history fail", e); + } + + log.info("runtime [{}] changed job, old job [{}], now [{}]", entity.getRuntimeAddr(), old.getJobID(), + entity.getJobID()); + } + } + } + } +} + + + + diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/job/JobInfoBizService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/job/JobInfoBizService.java new file mode 100644 index 0000000000..c200d9801a --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/job/JobInfoBizService.java @@ -0,0 +1,302 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.service.job; + +import org.apache.eventmesh.admin.server.AdminServerProperties; +import org.apache.eventmesh.admin.server.AdminServerRuntimeException; +import org.apache.eventmesh.admin.server.web.db.DBThreadPool; +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshDataSource; +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshRuntimeHeartbeat; +import org.apache.eventmesh.admin.server.web.db.service.EventMeshDataSourceService; +import org.apache.eventmesh.admin.server.web.db.service.EventMeshJobInfoExtService; +import org.apache.eventmesh.admin.server.web.db.service.EventMeshJobInfoService; +import org.apache.eventmesh.admin.server.web.db.service.EventMeshRuntimeHeartbeatService; +import org.apache.eventmesh.admin.server.web.pojo.JobDetail; +import org.apache.eventmesh.admin.server.web.pojo.TaskDetail; +import org.apache.eventmesh.admin.server.web.service.datasource.DataSourceBizService; +import org.apache.eventmesh.admin.server.web.service.position.PositionBizService; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.remote.JobState; +import org.apache.eventmesh.common.remote.TaskState; +import org.apache.eventmesh.common.remote.TransportType; +import org.apache.eventmesh.common.remote.datasource.DataSource; +import org.apache.eventmesh.common.remote.datasource.DataSourceType; +import org.apache.eventmesh.common.remote.exception.ErrorCode; +import org.apache.eventmesh.common.remote.job.JobType; +import org.apache.eventmesh.common.remote.request.CreateOrUpdateDataSourceReq; +import org.apache.eventmesh.common.utils.JsonUtils; + +import org.apache.commons.lang3.StringUtils; + +import java.time.Duration; +import java.util.LinkedList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import javax.annotation.PostConstruct; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import com.baomidou.mybatisplus.core.toolkit.Wrappers; + +import lombok.extern.slf4j.Slf4j; + +/** + * for table 'event_mesh_job_info' db operation + */ +@Service +@Slf4j +public class JobInfoBizService { + + @Autowired + private EventMeshJobInfoService jobInfoService; + + @Autowired + private EventMeshJobInfoExtService jobInfoExtService; + + @Autowired + private DataSourceBizService dataSourceBizService; + + @Autowired + private EventMeshDataSourceService dataSourceService; + + @Autowired + private PositionBizService positionBizService; + + @Autowired + private AdminServerProperties properties; + + @Autowired + EventMeshRuntimeHeartbeatService heartbeatService; + + private final long heatBeatPeriod = Duration.ofMillis(5000).toMillis(); + + @Autowired + DBThreadPool executor; + + @PostConstruct + public void init() { + log.info("init check job info scheduled task."); + executor.getCheckExecutor().scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + checkJobInfo(); + } + }, 10, 10, TimeUnit.SECONDS); + } + + public boolean updateJobState(String jobID, TaskState state) { + if (jobID == null || state == null) { + return false; + } + EventMeshJobInfo jobInfo = new EventMeshJobInfo(); + jobInfo.setJobState(state.name()); + return jobInfoService.update(jobInfo, Wrappers.update().eq("jobID", jobID).ne("jobState", JobState.DELETE.name())); + } + + public boolean updateJobState(String jobID, JobState state) { + if (jobID == null || state == null) { + return false; + } + EventMeshJobInfo jobInfo = new EventMeshJobInfo(); + jobInfo.setJobState(state.name()); + return jobInfoService.update(jobInfo, Wrappers.update().eq("jobID", jobID).ne("jobState", JobState.DELETE.name())); + } + + @Transactional + public List createJobs(List jobs) { + if (jobs == null || jobs.isEmpty() || jobs.stream().anyMatch(job -> StringUtils.isBlank(job.getTaskID()))) { + log.warn("when create jobs, task id is empty or jobs config is empty "); + return null; + } + List entityList = new LinkedList<>(); + + for (JobDetail job : jobs) { + // if running region not equal with admin region continue + if (!job.getRunningRegion().equals(properties.getRegion())) { + continue; + } + EventMeshJobInfo entity = new EventMeshJobInfo(); + entity.setJobState(TaskState.INIT.name()); + entity.setTaskID(job.getTaskID()); + entity.setJobType(job.getJobType().name()); + entity.setJobDesc(job.getJobDesc()); + String jobID = UUID.randomUUID().toString(); + entity.setJobID(jobID); + entity.setTransportType(job.getTransportType().name()); + entity.setCreateUid(job.getCreateUid()); + entity.setUpdateUid(job.getUpdateUid()); + entity.setFromRegion(job.getFromRegion()); + entity.setRunningRegion(job.getRunningRegion()); + CreateOrUpdateDataSourceReq source = new CreateOrUpdateDataSourceReq(); + source.setType(job.getTransportType().getSrc()); + source.setOperator(job.getCreateUid()); + source.setRegion(job.getSourceDataSource().getRegion()); + source.setDesc(job.getSourceConnectorDesc()); + Config sourceConfig = job.getSourceDataSource().getConf(); + source.setConfig(sourceConfig); + source.setConfigClass(job.getSourceDataSource().getConfClazz().getName()); + EventMeshDataSource createdSource = dataSourceBizService.createDataSource(source); + entity.setSourceData(createdSource.getId()); + + CreateOrUpdateDataSourceReq sink = new CreateOrUpdateDataSourceReq(); + sink.setType(job.getTransportType().getDst()); + sink.setOperator(job.getCreateUid()); + sink.setRegion(job.getSinkDataSource().getRegion()); + sink.setDesc(job.getSinkConnectorDesc()); + Config sinkConfig = job.getSinkDataSource().getConf(); + sink.setConfig(sinkConfig); + sink.setConfigClass(job.getSinkDataSource().getConfClazz().getName()); + EventMeshDataSource createdSink = dataSourceBizService.createDataSource(sink); + entity.setTargetData(createdSink.getId()); + + entityList.add(entity); + } + int changed = jobInfoExtService.batchSave(entityList); + if (changed != entityList.size()) { + throw new AdminServerRuntimeException(ErrorCode.INTERNAL_ERR, String.format("create [%d] jobs of not match expect [%d]", + changed, jobs.size())); + } + return entityList; + } + + + public JobDetail getJobDetail(String jobID) { + if (jobID == null) { + return null; + } + EventMeshJobInfo job = jobInfoService.getOne(Wrappers.query().eq("jobID", jobID)); + if (job == null) { + return null; + } + JobDetail detail = new JobDetail(); + detail.setTaskID(job.getTaskID()); + detail.setJobID(job.getJobID()); + EventMeshDataSource source = dataSourceService.getById(job.getSourceData()); + EventMeshDataSource target = dataSourceService.getById(job.getTargetData()); + if (source != null) { + if (!StringUtils.isBlank(source.getConfiguration())) { + try { + DataSource sourceDataSource = new DataSource(); + Class configClass = Class.forName(source.getConfigurationClass()); + sourceDataSource.setConf((Config) JsonUtils.parseObject(source.getConfiguration(), configClass)); + detail.setSourceDataSource(sourceDataSource); + } catch (Exception e) { + log.warn("parse source config id [{}] fail", job.getSourceData(), e); + throw new AdminServerRuntimeException(ErrorCode.BAD_DB_DATA, "illegal source data source config"); + } + } + detail.setSourceConnectorDesc(source.getDescription()); + if (source.getDataType() != null) { + detail.setPositions(positionBizService.getPositionByJobID(job.getJobID(), + DataSourceType.getDataSourceType(source.getDataType()))); + + } + } + if (target != null) { + if (!StringUtils.isBlank(target.getConfiguration())) { + try { + DataSource sinkDataSource = new DataSource(); + Class configClass = Class.forName(target.getConfigurationClass()); + sinkDataSource.setConf((Config) JsonUtils.parseObject(target.getConfiguration(), configClass)); + detail.setSinkDataSource(sinkDataSource); + } catch (Exception e) { + log.warn("parse sink config id [{}] fail", job.getSourceData(), e); + throw new AdminServerRuntimeException(ErrorCode.BAD_DB_DATA, "illegal target data sink config"); + } + } + detail.setSinkConnectorDesc(target.getDescription()); + } + + TaskState state = TaskState.fromIndex(job.getJobState()); + if (state == null) { + throw new AdminServerRuntimeException(ErrorCode.BAD_DB_DATA, "illegal job state in db"); + } + detail.setState(state); + detail.setTransportType(TransportType.getTransportType(job.getTransportType())); + detail.setJobType(JobType.fromIndex(job.getJobType())); + detail.setJobDesc(job.getJobDesc()); + return detail; + } + + public EventMeshJobInfo getJobInfo(String jobID) { + if (jobID == null) { + return null; + } + return jobInfoService.getOne(Wrappers.query().eq("jobID", jobID)); + } + + public List getJobsByTaskID(String taskID) { + if (taskID == null) { + return null; + } + return jobInfoService.list(Wrappers.query().eq("taskID", taskID)); + } + + public void checkJobInfo() { + List eventMeshJobInfoList = jobInfoService.list(Wrappers.query().eq("jobState", JobState.RUNNING.name())); + log.info("start check job info.to check job size:{}", eventMeshJobInfoList.size()); + for (EventMeshJobInfo jobInfo : eventMeshJobInfoList) { + String jobID = jobInfo.getJobID(); + if (StringUtils.isEmpty(jobID)) { + continue; + } + List heartbeatList = heartbeatService.list((Wrappers.query().eq("jobID", jobID))); + if (heartbeatList == null || heartbeatList.size() == 0) { + continue; + } + // if last heart beat update time have delay three period.print job heart beat delay warn + long currentTimeStamp = System.currentTimeMillis(); + if (currentTimeStamp - heartbeatList.get(0).getUpdateTime().getTime() > 3 * heatBeatPeriod) { + log.warn("current job heart heart has delay.jobID:{},currentTimeStamp:{},last update time:{}", jobID, currentTimeStamp, + heartbeatList.get(0).getUpdateTime()); + } + } + } + + public TaskDetail getTaskDetail(String taskID, DataSourceType dataSourceType) { + TaskDetail taskDetail = new TaskDetail(); + List jobInfoList = getJobsByTaskID(taskID); + if (jobInfoList == null || jobInfoList.size() == 0) { + return taskDetail; + } + for (EventMeshJobInfo jobInfo : jobInfoList) { + TransportType currentTransportType = TransportType.getTransportType(jobInfo.getTransportType()); + JobType jobType = JobType.fromIndex(jobInfo.getJobType()); + if (currentTransportType.getSrc().equals(dataSourceType)) { + if (jobType.name().equalsIgnoreCase(JobType.FULL.name())) { + taskDetail.setFullTask(jobInfo); + } + if (jobType.name().equalsIgnoreCase(JobType.INCREASE.name())) { + taskDetail.setIncreaseTask(jobInfo); + } + } + } + return taskDetail; + } + + +} + + + + diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/monitor/MonitorBizService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/monitor/MonitorBizService.java new file mode 100644 index 0000000000..3377334144 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/monitor/MonitorBizService.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.service.monitor; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshMonitor; +import org.apache.eventmesh.admin.server.web.db.service.EventMeshMonitorService; +import org.apache.eventmesh.common.remote.request.QueryTaskMonitorRequest; +import org.apache.eventmesh.common.remote.request.ReportMonitorRequest; +import org.apache.eventmesh.common.remote.response.QueryTaskMonitorResponse; +import org.apache.eventmesh.common.remote.task.TaskMonitor; +import org.apache.eventmesh.common.utils.JsonUtils; + +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import com.baomidou.mybatisplus.core.metadata.OrderItem; +import com.baomidou.mybatisplus.extension.plugins.pagination.Page; + +import lombok.extern.slf4j.Slf4j; + +@Service +@Slf4j +public class MonitorBizService { + + @Autowired + private EventMeshMonitorService monitorService; + + public boolean reportMonitorRecord(ReportMonitorRequest request) { + EventMeshMonitor monitor = new EventMeshMonitor(); + monitor.setTaskID(request.getTaskID()); + monitor.setJobID(request.getJobID()); + monitor.setAddress(request.getAddress()); + monitor.setTransportType(request.getTransportType()); + monitor.setConnectorStage(request.getConnectorStage()); + monitor.setTotalReqNum(request.getTotalReqNum()); + monitor.setTotalTimeCost(request.getTotalTimeCost()); + monitor.setMaxTimeCost(request.getMaxTimeCost()); + monitor.setAvgTimeCost(request.getAvgTimeCost()); + monitor.setTps(request.getTps()); + return monitorService.save(monitor); + } + + public QueryTaskMonitorResponse queryTaskMonitors(QueryTaskMonitorRequest request) { + if (StringUtils.isBlank(request.getTaskID())) { + throw new RuntimeException("task id is empty"); + } + long limit = request.getLimit(); + if (limit <= 0) { + log.info("query task monitor limit:{},use default value:{}", limit, 10); + limit = 10; + } + + Page queryPage = new Page<>(); + queryPage.setCurrent(1); + queryPage.setSize(limit); + queryPage.addOrder(OrderItem.desc("createTime")); + + QueryWrapper queryWrapper = new QueryWrapper(); + queryWrapper.eq("taskID", request.getTaskID()); + if (StringUtils.isNotEmpty(request.getJobID())) { + queryWrapper.eq("jobID", request.getJobID()); + } + List eventMeshMonitors = monitorService.list(queryPage, queryWrapper); + List taskMonitorList = new ArrayList<>(); + if (eventMeshMonitors != null) { + log.info("query event mesh monitor size:{}", eventMeshMonitors.size()); + if (log.isDebugEnabled()) { + log.debug("query event mesh monitor content:{}", JsonUtils.toJSONString(eventMeshMonitors)); + } + for (EventMeshMonitor eventMeshMonitor : eventMeshMonitors) { + TaskMonitor monitor = new TaskMonitor(); + monitor.setTaskID(eventMeshMonitor.getTaskID()); + monitor.setJobID(eventMeshMonitor.getJobID()); + monitor.setAddress(eventMeshMonitor.getAddress()); + monitor.setTransportType(eventMeshMonitor.getTransportType()); + monitor.setConnectorStage(eventMeshMonitor.getConnectorStage()); + monitor.setTotalReqNum(eventMeshMonitor.getTotalReqNum()); + monitor.setTotalTimeCost(eventMeshMonitor.getTotalTimeCost()); + monitor.setMaxTimeCost(eventMeshMonitor.getMaxTimeCost()); + monitor.setAvgTimeCost(eventMeshMonitor.getAvgTimeCost()); + monitor.setTps(eventMeshMonitor.getTps()); + monitor.setCreateTime(eventMeshMonitor.getCreateTime()); + taskMonitorList.add(monitor); + } + } + QueryTaskMonitorResponse queryTaskMonitorResponse = new QueryTaskMonitorResponse(); + queryTaskMonitorResponse.setTaskMonitors(taskMonitorList); + return queryTaskMonitorResponse; + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/IFetchPositionHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/IFetchPositionHandler.java new file mode 100644 index 0000000000..2c039062f3 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/IFetchPositionHandler.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.service.position; + +import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; +import org.apache.eventmesh.common.remote.offset.RecordPosition; +import org.apache.eventmesh.common.remote.request.FetchPositionRequest; + +import java.util.List; + +/** + * IFetchPositionHandler + */ +public interface IFetchPositionHandler { + + List handler(FetchPositionRequest request, Metadata metadata); +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/IRecordPositionHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/IRecordPositionHandler.java new file mode 100644 index 0000000000..fa38e14320 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/IRecordPositionHandler.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.service.position; + +import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; +import org.apache.eventmesh.common.remote.request.RecordPositionRequest; + +/** + * IRecordPositionHandler + */ +public interface IRecordPositionHandler { + + boolean handler(RecordPositionRequest request, Metadata metadata); + +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/IReportPositionHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/IReportPositionHandler.java new file mode 100644 index 0000000000..75f392e395 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/IReportPositionHandler.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.service.position; + +import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; +import org.apache.eventmesh.common.remote.request.ReportPositionRequest; + +/** + * IReportPositionHandler + */ +public interface IReportPositionHandler { + + boolean handler(ReportPositionRequest request, Metadata metadata); +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/PositionBizService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/PositionBizService.java new file mode 100644 index 0000000000..0c4cd7a423 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/PositionBizService.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.service.position; + +import org.apache.eventmesh.admin.server.AdminServerRuntimeException; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; +import org.apache.eventmesh.common.remote.datasource.DataSourceType; +import org.apache.eventmesh.common.remote.exception.ErrorCode; +import org.apache.eventmesh.common.remote.offset.RecordPosition; +import org.apache.eventmesh.common.remote.request.FetchPositionRequest; +import org.apache.eventmesh.common.remote.request.RecordPositionRequest; +import org.apache.eventmesh.common.remote.request.ReportPositionRequest; + +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import lombok.extern.slf4j.Slf4j; + +@Service +@Slf4j +public class PositionBizService { + + @Autowired + PositionHandlerFactory factory; + + // called isValidateReportRequest before call this + public List getPosition(FetchPositionRequest request, Metadata metadata) { + if (request == null) { + return null; + } + isValidatePositionRequest(request.getDataSourceType()); + IFetchPositionHandler handler = factory.getHandler(request.getDataSourceType()); + return handler.handler(request, metadata); + } + + public void isValidatePositionRequest(DataSourceType type) { + if (type == null) { + throw new AdminServerRuntimeException(ErrorCode.BAD_REQUEST, "data source type is null"); + } + IReportPositionHandler handler = factory.getHandler(type); + if (handler == null) { + throw new AdminServerRuntimeException(ErrorCode.BAD_REQUEST, + String.format("illegal data base type [%s], it not match any report position handler", type)); + } + } + + // called isValidateReportRequest before call this + public boolean reportPosition(ReportPositionRequest request, Metadata metadata) { + if (request == null) { + return false; + } + isValidatePositionRequest(request.getDataSourceType()); + IReportPositionHandler handler = factory.getHandler(request.getDataSourceType()); + return handler.handler(request, metadata); + } + + public List getPositionByJobID(String jobID, DataSourceType type) { + if (jobID == null || type == null) { + return null; + } + isValidatePositionRequest(type); + PositionHandler handler = factory.getHandler(type); + FetchPositionRequest request = new FetchPositionRequest(); + request.setJobID(jobID); + return handler.handler(request, null); + } + + public boolean recordPosition(RecordPositionRequest request, Metadata metadata) { + isValidatePositionRequest(request.getDataSourceType()); + IRecordPositionHandler handler = factory.getHandler(request.getDataSourceType()); + return handler.handler(request, metadata); + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/PositionHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/PositionHandler.java new file mode 100644 index 0000000000..9cbaf3fad6 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/PositionHandler.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.service.position; + +import org.apache.eventmesh.common.remote.datasource.DataSourceType; + +public abstract class PositionHandler implements IReportPositionHandler, IFetchPositionHandler, IRecordPositionHandler { + + protected abstract DataSourceType getSourceType(); +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/PositionHandlerFactory.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/PositionHandlerFactory.java new file mode 100644 index 0000000000..c2065f80f4 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/PositionHandlerFactory.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.service.position; + +import org.apache.eventmesh.common.remote.datasource.DataSourceType; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import org.springframework.context.ApplicationListener; +import org.springframework.context.event.ContextRefreshedEvent; +import org.springframework.stereotype.Component; + +import lombok.extern.slf4j.Slf4j; + +@Component +@Slf4j +public class PositionHandlerFactory implements ApplicationListener { + + private final Map handlers = + new ConcurrentHashMap<>(); + + public PositionHandler getHandler(DataSourceType type) { + return handlers.get(type); + } + + @Override + public void onApplicationEvent(ContextRefreshedEvent event) { + Map beans = + event.getApplicationContext().getBeansOfType(PositionHandler.class); + + for (PositionHandler handler : beans.values()) { + DataSourceType type = handler.getSourceType(); + if (handlers.containsKey(type)) { + log.warn("data source type [{}] handler already exists", type); + continue; + } + handlers.put(type, handler); + } + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/impl/HttpPositionHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/impl/HttpPositionHandler.java new file mode 100644 index 0000000000..a58fa31c07 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/impl/HttpPositionHandler.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.service.position.impl; + +import org.apache.eventmesh.admin.server.web.db.service.EventMeshPositionReporterHistoryService; +import org.apache.eventmesh.admin.server.web.service.position.PositionHandler; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; +import org.apache.eventmesh.common.remote.datasource.DataSourceType; +import org.apache.eventmesh.common.remote.offset.RecordPosition; +import org.apache.eventmesh.common.remote.request.FetchPositionRequest; +import org.apache.eventmesh.common.remote.request.RecordPositionRequest; +import org.apache.eventmesh.common.remote.request.ReportPositionRequest; + +import java.util.ArrayList; +import java.util.List; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import lombok.extern.slf4j.Slf4j; + +@Component +@Slf4j +public class HttpPositionHandler extends PositionHandler { + + @Autowired + EventMeshPositionReporterHistoryService historyService; + + @Override + protected DataSourceType getSourceType() { + return DataSourceType.HTTP; + } + + @Override + public boolean handler(ReportPositionRequest request, Metadata metadata) { + log.info("receive http position report request:{}", request); + // mock wemq postion report store + return true; + } + + @Override + public List handler(FetchPositionRequest request, Metadata metadata) { + // mock http position fetch request + List recordPositionList = new ArrayList<>(); + return recordPositionList; + } + + @Override + public boolean handler(RecordPositionRequest request, Metadata metadata) { + return true; + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/impl/MysqlPositionHandler.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/impl/MysqlPositionHandler.java new file mode 100644 index 0000000000..8545078d80 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/position/impl/MysqlPositionHandler.java @@ -0,0 +1,351 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.service.position.impl; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshMysqlPosition; +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshPositionReporterHistory; +import org.apache.eventmesh.admin.server.web.db.service.EventMeshMysqlPositionService; +import org.apache.eventmesh.admin.server.web.db.service.EventMeshPositionReporterHistoryService; +import org.apache.eventmesh.admin.server.web.pojo.BinlogPosition; +import org.apache.eventmesh.admin.server.web.pojo.JobDetail; +import org.apache.eventmesh.admin.server.web.service.job.JobInfoBizService; +import org.apache.eventmesh.admin.server.web.service.position.PositionHandler; +import org.apache.eventmesh.admin.server.web.utils.JdbcUtils; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceFullConfig; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; +import org.apache.eventmesh.common.remote.datasource.DataSourceType; +import org.apache.eventmesh.common.remote.offset.RecordPosition; +import org.apache.eventmesh.common.remote.offset.canal.CanalRecordOffset; +import org.apache.eventmesh.common.remote.offset.canal.CanalRecordPartition; +import org.apache.eventmesh.common.remote.request.FetchPositionRequest; +import org.apache.eventmesh.common.remote.request.RecordPositionRequest; +import org.apache.eventmesh.common.remote.request.ReportPositionRequest; +import org.apache.eventmesh.common.utils.JsonUtils; + +import org.apache.commons.lang3.StringUtils; + +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.locks.LockSupport; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.dao.DuplicateKeyException; +import org.springframework.stereotype.Component; + +import com.alibaba.druid.pool.DruidDataSource; +import com.baomidou.mybatisplus.core.toolkit.Wrappers; + +import lombok.extern.slf4j.Slf4j; + +@Component +@Slf4j +public class MysqlPositionHandler extends PositionHandler { + private static final int RETRY_TIMES = 3; + private static final String SQL_SELECT_RDB_VERSION = "select version() as rdb_version"; + private static final String SQL_SHOW_BINLOG_POSITION = "SHOW MASTER STATUS"; + private static final String SQL_SELECT_SERVER_UUID_IN_MARIADB = "SELECT @@global.server_id as server_uuid"; + private static final String SQL_SHOW_SERVER_UUID_IN_MYSQL = "SELECT @@server_uuid as server_uuid"; + private static final String SQL_SELECT_GTID_IN_MARIADB = "SELECT @@global.gtid_binlog_pos as gtid"; + private static final String SQL_SELECT_GTID_IN_MYSQL = "SELECT @@gtid_executed as gtid"; + + private final long retryPeriod = Duration.ofMillis(500).toNanos(); + + @Autowired + EventMeshMysqlPositionService positionService; + + @Autowired + EventMeshPositionReporterHistoryService historyService; + + @Autowired + JobInfoBizService jobInfoBizService; + + @Override + protected DataSourceType getSourceType() { + return DataSourceType.MYSQL; + } + + private boolean isNotForward(EventMeshMysqlPosition now, EventMeshMysqlPosition old) { + if (StringUtils.isNotBlank(old.getJournalName()) && old.getJournalName().equals(now.getJournalName()) + && old.getPosition() >= now.getPosition()) { + log.info("job [{}] report position [{}] by runtime [{}] less than db position [{}] journal name [{}] by [{}]", now.getJobID(), + now.getPosition(), now.getAddress(), now.getJournalName(), old.getPosition(), old.getAddress()); + return true; + } + return false; + } + + public boolean saveOrUpdateByJob(EventMeshMysqlPosition position) { + for (int i = 0; i < RETRY_TIMES; i++) { + EventMeshMysqlPosition old = positionService.getOne(Wrappers.query().eq("jobId", position.getJobID())); + if (old == null) { + try { + return positionService.save(position); + } catch (DuplicateKeyException e) { + log.warn("current insert position fail, it will retry in 500ms"); + LockSupport.parkNanos(retryPeriod); + continue; + } catch (Exception e) { + log.warn("insert position fail catch unknown exception", e); + return false; + } + } + + if (isNotForward(position, old)) { + return true; + } + try { + if (!positionService.update(position, + Wrappers.update().eq("updateTime", old.getUpdateTime()).eq("jobID", old.getJobID()))) { + log.warn("update position [{}] fail, maybe current update. it will retry in 500ms", position); + LockSupport.parkNanos(retryPeriod); + continue; + } + return true; + } finally { + if (old.getAddress() != null && !old.getAddress().equals(position.getAddress())) { + EventMeshPositionReporterHistory history = new EventMeshPositionReporterHistory(); + history.setRecord(JsonUtils.toJSONString(position)); + history.setJob(old.getJobID()); + history.setAddress(old.getAddress()); + log.info("job [{}] position reporter changed old [{}], now [{}]", position.getJobID(), old, position); + try { + historyService.save(history); + } catch (Exception e) { + log.warn("save job [{}] mysql position reporter changed history fail, now reporter [{}], old [{}]", position.getJobID(), + position.getAddress(), old.getAddress(), e); + } + } + } + } + return false; + } + + @Override + public boolean handler(ReportPositionRequest request, Metadata metadata) { + try { + List recordPositionList = request.getRecordPositionList(); + RecordPosition recordPosition = recordPositionList.get(0); + if (recordPosition == null || recordPosition.getRecordPartition() == null || recordPosition.getRecordOffset() == null) { + log.warn("report mysql position, but record-partition/partition/offset is null"); + return false; + } + if (!(recordPosition.getRecordPartition() instanceof CanalRecordPartition)) { + log.warn("report mysql position, but record partition class [{}] not match [{}]", + recordPosition.getRecordPartition().getRecordPartitionClass(), CanalRecordPartition.class); + return false; + } + if (!(recordPosition.getRecordOffset() instanceof CanalRecordOffset)) { + log.warn("report mysql position, but record offset class [{}] not match [{}]", + recordPosition.getRecordOffset().getRecordOffsetClass(), CanalRecordOffset.class); + return false; + } + EventMeshMysqlPosition position = new EventMeshMysqlPosition(); + position.setJobID(request.getJobID()); + position.setAddress(request.getAddress()); + CanalRecordOffset offset = (CanalRecordOffset) recordPosition.getRecordOffset(); + if (offset != null) { + position.setPosition(offset.getOffset()); + position.setGtid(offset.getGtid()); + position.setCurrentGtid(offset.getCurrentGtid()); + } + CanalRecordPartition partition = (CanalRecordPartition) recordPosition.getRecordPartition(); + if (partition != null) { + position.setServerUUID(partition.getServerUUID()); + position.setTimestamp(partition.getTimeStamp()); + position.setJournalName(partition.getJournalName()); + } + if (!saveOrUpdateByJob(position)) { + log.warn("update job position fail [{}]", request); + return false; + } + return true; + } catch (Exception e) { + log.warn("save position job [{}] fail", request.getJobID(), e); + } + + return false; + } + + @Override + public List handler(FetchPositionRequest request, Metadata metadata) { + List positionList = positionService.list(Wrappers.query().eq("jobID", request.getJobID())); + List recordPositionList = new ArrayList<>(); + for (EventMeshMysqlPosition position : positionList) { + CanalRecordPartition partition = new CanalRecordPartition(); + partition.setTimeStamp(position.getTimestamp()); + partition.setJournalName(position.getJournalName()); + partition.setServerUUID(position.getServerUUID()); + RecordPosition recordPosition = new RecordPosition(); + recordPosition.setRecordPartition(partition); + CanalRecordOffset offset = new CanalRecordOffset(); + offset.setOffset(position.getPosition()); + offset.setGtid(position.getGtid()); + offset.setCurrentGtid(position.getCurrentGtid()); + recordPosition.setRecordOffset(offset); + recordPositionList.add(recordPosition); + } + return recordPositionList; + } + + @Override + public boolean handler(RecordPositionRequest request, Metadata metadata) { + try { + String fullJobID = request.getFullJobID(); + String increaseJobID = request.getIncreaseJobID(); + log.info("start record full job position to increase job position,full jobID:{}, increase jobID:{}.", fullJobID, increaseJobID); + JobDetail fullJobDetail = jobInfoBizService.getJobDetail(fullJobID); + CanalSourceConfig canalSourceConfig = (CanalSourceConfig) fullJobDetail.getSourceDataSource().getConf(); + CanalSourceFullConfig canalSourceFullConfig = JsonUtils.mapToObject(canalSourceConfig.getSourceConfig(), CanalSourceFullConfig.class); + try (DruidDataSource druidDataSource = JdbcUtils.createDruidDataSource(canalSourceFullConfig.getSourceConnectorConfig().getUrl(), + canalSourceFullConfig.getSourceConnectorConfig().getUserName(), canalSourceFullConfig.getSourceConnectorConfig().getPassWord())) { + + DataSourceType dataSourceType = checkRDBDataSourceType(druidDataSource); + + ReportPositionRequest reportPositionRequest = new ReportPositionRequest(); + reportPositionRequest.setJobID(increaseJobID); + reportPositionRequest.setDataSourceType(DataSourceType.MYSQL); + reportPositionRequest.setAddress(request.getAddress()); + + RecordPosition recordPosition = new RecordPosition(); + CanalRecordOffset recordOffset = new CanalRecordOffset(); + BinlogPosition binlogPosition = queryBinlogPosition(druidDataSource); + String gtid = queryGTID(druidDataSource, dataSourceType); + recordOffset.setOffset(binlogPosition.getPosition()); + recordOffset.setGtid(gtid); + recordPosition.setRecordOffset(recordOffset); + + CanalRecordPartition recordPartition = new CanalRecordPartition(); + String serverUUID = queryServerUUID(druidDataSource, dataSourceType); + recordPartition.setJournalName(binlogPosition.getFile()); + recordPartition.setServerUUID(serverUUID); + recordPosition.setRecordPartition(recordPartition); + + List recordPositions = new ArrayList<>(); + recordPositions.add(recordPosition); + + reportPositionRequest.setRecordPositionList(recordPositions); + log.info("start store increase task position,jobID:{},request:{}", increaseJobID, reportPositionRequest); + handler(reportPositionRequest, metadata); + } + return true; + } catch (Exception e) { + log.error("record full job position to increase job position failed.", e); + return false; + } + } + + private DataSourceType checkRDBDataSourceType(DruidDataSource druidDataSource) { + try { + log.info("execute sql '{}' start.", SQL_SELECT_RDB_VERSION); + try (PreparedStatement preparedStatement = druidDataSource.getConnection().prepareStatement(SQL_SELECT_RDB_VERSION)) { + ResultSet resultSet = preparedStatement.executeQuery(); + if (resultSet.next()) { + log.info("execute sql '{}' result:{}", SQL_SELECT_RDB_VERSION, resultSet); + String rdbVersion = resultSet.getString("rdb_version"); + if (StringUtils.isNotBlank(rdbVersion)) { + if (rdbVersion.toLowerCase().contains(DataSourceType.MariaDB.getName().toLowerCase())) { + return DataSourceType.MariaDB; + } + } + } + } + } catch (Exception e) { + log.warn("select rdb version failed,data source:{}", druidDataSource, e); + throw new RuntimeException("select rdb version failed"); + } + return DataSourceType.MYSQL; + } + + private BinlogPosition queryBinlogPosition(DruidDataSource druidDataSource) { + BinlogPosition binlogPosition = new BinlogPosition(); + try { + log.info("execute sql '{}' start.", SQL_SHOW_BINLOG_POSITION); + try (PreparedStatement preparedStatement = druidDataSource.getConnection().prepareStatement(SQL_SHOW_BINLOG_POSITION)) { + ResultSet resultSet = preparedStatement.executeQuery(); + if (resultSet.next()) { + log.info("execute sql '{}' result:{}", SQL_SELECT_RDB_VERSION, resultSet); + String fileName = resultSet.getString("File"); + Long position = resultSet.getLong("Position"); + binlogPosition.setFile(fileName); + binlogPosition.setPosition(position); + } + } + } catch (Exception e) { + log.warn("show binlog position failed,data source:{}", druidDataSource, e); + throw new RuntimeException("show binlog position failed"); + } + return binlogPosition; + } + + private String queryServerUUID(DruidDataSource druidDataSource, DataSourceType dataSourceType) { + String serverUUID = ""; + try { + String queryServerUUIDSql; + if (DataSourceType.MariaDB.equals(dataSourceType)) { + queryServerUUIDSql = SQL_SELECT_SERVER_UUID_IN_MARIADB; + } else { + queryServerUUIDSql = SQL_SHOW_SERVER_UUID_IN_MYSQL; + } + log.info("execute sql '{}' start.", queryServerUUIDSql); + try (PreparedStatement preparedStatement = druidDataSource.getConnection().prepareStatement(queryServerUUIDSql)) { + ResultSet resultSet = preparedStatement.executeQuery(); + if (resultSet.next()) { + log.info("execute sql '{}' result:{}", queryServerUUIDSql, resultSet); + serverUUID = resultSet.getString("server_uuid"); + log.info("execute sql '{}',query server_uuid result:{}", queryServerUUIDSql, serverUUID); + return serverUUID; + } + } + } catch (Exception e) { + log.warn("select server_uuid failed,data source:{}", druidDataSource, e); + throw new RuntimeException("select server_uuid failed"); + } + return serverUUID; + } + + private String queryGTID(DruidDataSource druidDataSource, DataSourceType dataSourceType) { + String gitd = ""; + try { + String queryGTIDSql; + if (DataSourceType.MariaDB.equals(dataSourceType)) { + queryGTIDSql = SQL_SELECT_GTID_IN_MARIADB; + } else { + queryGTIDSql = SQL_SELECT_GTID_IN_MYSQL; + } + log.info("execute sql '{}' start.", queryGTIDSql); + try (PreparedStatement preparedStatement = druidDataSource.getConnection().prepareStatement(queryGTIDSql)) { + ResultSet resultSet = preparedStatement.executeQuery(); + if (resultSet.next()) { + log.info("execute sql '{}' result:{}", queryGTIDSql, resultSet); + gitd = resultSet.getString("gtid"); + log.info("execute sql '{}',select gitd result:{}", queryGTIDSql, gitd); + return gitd; + } + } + } catch (Exception e) { + log.warn("select gtid failed,data source:{}", druidDataSource, e); + // when db server not open gitd mode, ignore gtid query exception + //throw new RuntimeException("select gtid failed"); + } + return gitd; + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/task/TaskBizService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/task/TaskBizService.java new file mode 100644 index 0000000000..d3c7087d47 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/task/TaskBizService.java @@ -0,0 +1,360 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.service.task; + +import org.apache.eventmesh.admin.server.AdminServerProperties; +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshJobInfo; +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshTaskInfo; +import org.apache.eventmesh.admin.server.web.db.service.EventMeshTaskInfoService; +import org.apache.eventmesh.admin.server.web.pojo.JobDetail; +import org.apache.eventmesh.admin.server.web.service.job.JobInfoBizService; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.exception.EventMeshException; +import org.apache.eventmesh.common.remote.JobState; +import org.apache.eventmesh.common.remote.TaskState; +import org.apache.eventmesh.common.remote.datasource.DataSource; +import org.apache.eventmesh.common.remote.datasource.DataSourceType; +import org.apache.eventmesh.common.remote.request.CreateTaskRequest; +import org.apache.eventmesh.common.remote.request.QueryTaskInfoRequest; +import org.apache.eventmesh.common.remote.request.TaskBachRequest; +import org.apache.eventmesh.common.remote.request.TaskIDRequest; +import org.apache.eventmesh.common.remote.response.CreateTaskResponse; +import org.apache.eventmesh.common.remote.response.QueryTaskInfoResponse; +import org.apache.eventmesh.common.remote.response.SimpleResponse; +import org.apache.eventmesh.common.utils.JsonUtils; + +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Random; +import java.util.UUID; +import java.util.stream.Collectors; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.ResponseEntity; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; +import org.springframework.web.client.RestTemplate; + +import com.baomidou.mybatisplus.core.toolkit.Wrappers; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Service +public class TaskBizService { + + @Autowired + private EventMeshTaskInfoService taskInfoService; + + @Autowired + private JobInfoBizService jobInfoService; + + @Autowired + private AdminServerProperties properties; + + private static final String TYPE = "type"; + + private static final String DESC = "desc"; + + private static final String CONF_CLAZZ = "confClazz"; + + private static final String CONF = "conf"; + + private static final String REGION = "region"; + + @Transactional + public CreateTaskResponse createTask(CreateTaskRequest req) { + String taskID = req.getTaskId(); + if (StringUtils.isEmpty(taskID)) { + taskID = UUID.randomUUID().toString(); + req.setTaskId(taskID); + } + + String targetRegion = req.getTargetRegion(); + String remoteResponse = ""; + // not from other admin && target not equals with self region + if (!req.isFlag() && !properties.getRegion().equals(targetRegion)) { + List adminServerList = Arrays.asList(properties.getAdminServerList().get(targetRegion).split(";")); + if (adminServerList == null || adminServerList.isEmpty()) { + throw new RuntimeException("No admin server available for region: " + targetRegion); + } + String targetUrl = adminServerList.get(new Random().nextInt(adminServerList.size())) + "/eventmesh/admin/createTask"; + + RestTemplate restTemplate = new RestTemplate(); + req.setFlag(true); + ResponseEntity response = restTemplate.postForEntity(targetUrl, req, String.class); + if (!response.getStatusCode().is2xxSuccessful()) { + throw new RuntimeException("Failed to create task on admin server: " + targetUrl); + } + remoteResponse = response.getBody(); + } + + String finalTaskID = taskID; + List jobs = req.getJobs().stream().map(x -> { + JobDetail job = null; + try { + job = parse(x); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } + job.setTaskID(finalTaskID); + job.setCreateUid(req.getUid()); + job.setUpdateUid(req.getUid()); + return job; + }).collect(Collectors.toList()); + + EventMeshTaskInfo taskInfo = new EventMeshTaskInfo(); + taskInfo.setTaskID(finalTaskID); + taskInfo.setTaskName(req.getTaskName()); + taskInfo.setTaskDesc(req.getTaskDesc()); + taskInfo.setTaskState(TaskState.INIT.name()); + taskInfo.setCreateUid(req.getUid()); + taskInfo.setSourceRegion(req.getSourceRegion()); + taskInfo.setTargetRegion(req.getTargetRegion()); + List eventMeshJobInfoList = jobInfoService.createJobs(jobs); + taskInfoService.save(taskInfo); + return buildCreateTaskResponse(finalTaskID, eventMeshJobInfoList, remoteResponse); + } + + private JobDetail parse(CreateTaskRequest.JobDetail src) throws ClassNotFoundException { + JobDetail dst = new JobDetail(); + dst.setJobDesc(src.getJobDesc()); + dst.setTransportType(src.getTransportType()); + dst.setSourceConnectorDesc(src.getSourceConnectorDesc()); + try { + dst.setSourceDataSource(mapToDataSource(src.getSourceDataSource())); + dst.setSinkDataSource(mapToDataSource(src.getSinkDataSource())); + } catch (ClassNotFoundException e) { + throw new RuntimeException("Failed to map data source", e); + } + dst.setSinkConnectorDesc(src.getSinkConnectorDesc()); + // full/increase/check + dst.setJobType(src.getJobType()); + dst.setFromRegion(src.getFromRegion()); + dst.setRunningRegion(src.getRunningRegion()); + return dst; + } + + private DataSource mapToDataSource(Map dataMap) throws ClassNotFoundException { + DataSource dataSource = new DataSource(); + dataSource.setType(DataSourceType.fromString(dataMap.get(TYPE).toString())); + dataSource.setDesc((String) dataMap.get(DESC)); + dataSource.setConfClazz((Class) Class.forName(dataMap.get(CONF_CLAZZ).toString())); + dataSource.setConf(JsonUtils.parseObject(JsonUtils.toJSONString(dataMap.get(CONF)), dataSource.getConfClazz())); + dataSource.setRegion((String) dataMap.get(REGION)); + return dataSource; + } + + private CreateTaskResponse buildCreateTaskResponse(String taskId, List eventMeshJobInfoList, String remoteResponse) { + CreateTaskResponse createTaskResponse = new CreateTaskResponse(); + createTaskResponse.setTaskId(taskId); + List jobDetailList = new ArrayList<>(); + if (!eventMeshJobInfoList.isEmpty()) { + for (EventMeshJobInfo eventMeshJobInfo : eventMeshJobInfoList) { + CreateTaskRequest.JobDetail jobDetail = new CreateTaskRequest.JobDetail(); + jobDetail.setJobId(eventMeshJobInfo.getJobID()); + jobDetail.setRunningRegion(eventMeshJobInfo.getRunningRegion()); + jobDetailList.add(jobDetail); + } + } + if (!StringUtils.isEmpty(remoteResponse)) { + SimpleResponse response = JsonUtils.parseObject(remoteResponse, SimpleResponse.class); + CreateTaskResponse remoteCreateTaskResponse = JsonUtils.convertValue(response.getData(), CreateTaskResponse.class); + jobDetailList.addAll(remoteCreateTaskResponse.getJobIdList()); + } + createTaskResponse.setJobIdList(jobDetailList); + return createTaskResponse; + } + + /** + * start task + * @param taskIDRequest id + */ + @Transactional + public void startTask(TaskIDRequest taskIDRequest) { + try { + EventMeshTaskInfo taskInfoServiceOne = taskInfoService.getOne(Wrappers.query() + .eq("taskID", taskIDRequest.getTaskID())); + + if (Objects.isNull(taskInfoServiceOne)) { + throw new EventMeshException("task not found"); + } + + if (TaskState.DELETE.name().equals(taskInfoServiceOne.getTaskState())) { + throw new EventMeshException("task already deleted"); + } + + // update task state + taskInfoService.update(Wrappers.update() + .eq("id", taskInfoServiceOne.getId()) + .set("taskState", TaskState.RUNNING.name())); + + List eventMeshJobInfos = jobInfoService.getJobsByTaskID(taskIDRequest.getTaskID()); + + for (EventMeshJobInfo eventMeshJobInfo : eventMeshJobInfos) { + // update job state by jonID + jobInfoService.updateJobState(eventMeshJobInfo.getJobID(), JobState.RUNNING); + } + + // todo: start task job eventmesh-runtime-v2 schedule ? + + } catch (Exception e) { + log.info("start task exception:{}", e.getMessage()); + throw new EventMeshException("start task exception"); + } + } + + @Transactional + public boolean deleteTaskByTaskID(TaskIDRequest taskIDRequest) { + try { + EventMeshTaskInfo taskInfoServiceOne = taskInfoService.getOne(Wrappers.query() + .eq("taskID", taskIDRequest.getTaskID())); + + if (Objects.isNull(taskInfoServiceOne)) { + throw new EventMeshException("task not found"); + } + + if (!TaskState.DELETE.name().equals(taskInfoServiceOne.getTaskState())) { + // update task state to delete + taskInfoService.update(Wrappers.update() + .eq("id", taskInfoServiceOne.getId()) + .set("taskState", TaskState.DELETE.name())); + } + List eventMeshJobInfos = jobInfoService.getJobsByTaskID(taskInfoServiceOne.getTaskID()); + for (EventMeshJobInfo eventMeshJobInfo : eventMeshJobInfos) { + // update job state to delete + jobInfoService.updateJobState(eventMeshJobInfo.getJobID(), JobState.DELETE); + } + // todo: data source config need delete? + + } catch (RuntimeException e) { + log.error("delete task failed:{}", e.getMessage()); + throw new EventMeshException("delete task failed"); + } + return true; + } + + public List queryTaskInfo(QueryTaskInfoRequest taskInfoRequest) { + return taskInfoService.queryTaskInfo(taskInfoRequest); + } + + @Transactional + public void restartTask(TaskIDRequest taskIDRequest) { + try { + EventMeshTaskInfo taskInfoServiceOne = taskInfoService.getOne(Wrappers.query() + .eq("taskID", taskIDRequest.getTaskID()) + .ne("taskState", TaskState.DELETE.name())); + + if (Objects.isNull(taskInfoServiceOne)) { + throw new EventMeshException("task not found"); + } + if (!TaskState.RUNNING.name().equals(taskInfoServiceOne.getTaskState())) { + taskInfoService.update(Wrappers.update() + .eq("id", taskInfoServiceOne.getId()) + .set("taskState", TaskState.RUNNING.name())); + } + List eventMeshJobInfos = jobInfoService.getJobsByTaskID(taskInfoServiceOne.getTaskID()); + for (EventMeshJobInfo eventMeshJobInfo : eventMeshJobInfos) { + // update job state to restart + jobInfoService.updateJobState(eventMeshJobInfo.getJobID(), JobState.RUNNING); + } + // todo: start task job eventmesh-runtime-v2 schedule? + + } catch (RuntimeException e) { + log.error("restart task filed:{}", e.getMessage()); + throw new EventMeshException("restart task filed"); + } + } + + @Transactional + public void stopTask(TaskIDRequest taskIDRequest) { + try { + EventMeshTaskInfo taskInfoServiceOne = taskInfoService.getOne(Wrappers.query() + .eq("taskID", taskIDRequest.getTaskID())); + + if (Objects.isNull(taskInfoServiceOne)) { + throw new EventMeshException("task not found"); + } + if (!TaskState.PAUSE.name().equals(taskInfoServiceOne.getTaskState())) { + taskInfoService.update(Wrappers.update() + .eq("id", taskInfoServiceOne.getId()) + .set("taskState", TaskState.PAUSE.name())); + } + + List eventMeshJobInfos = jobInfoService.getJobsByTaskID(taskInfoServiceOne.getTaskID()); + for (EventMeshJobInfo eventMeshJobInfo : eventMeshJobInfos) { + // update job state to pause + jobInfoService.updateJobState(eventMeshJobInfo.getJobID(), JobState.PAUSE); + } + + // todo: stop task job eventmesh-runtime-v2 schedule? + + } catch (RuntimeException e) { + log.error("stop task filed:{}", e.getMessage()); + throw new EventMeshException("stop task filed"); + } + } + + @Transactional + public void restartBatchTask(List taskIDRequestList, List errorNames) { + for (TaskBachRequest task : taskIDRequestList) { + try { + TaskIDRequest taskIDRequest = new TaskIDRequest(); + taskIDRequest.setTaskID(task.getTaskID()); + startTask(taskIDRequest); + } catch (RuntimeException e) { + log.error("restart batch task failed:{}", e.getMessage()); + errorNames.add(task.getTaskName()); + } + } + } + + @Transactional + public void stopBatchTask(List taskIDRequestList, List errorNames) { + for (TaskBachRequest task : taskIDRequestList) { + try { + TaskIDRequest taskIDRequest = new TaskIDRequest(); + taskIDRequest.setTaskID(task.getTaskID()); + stopTask(taskIDRequest); + } catch (RuntimeException e) { + log.error("stop batch task failed:{}", e.getMessage()); + errorNames.add(task.getTaskName()); + } + } + } + + @Transactional + public void startBatchTask(List taskIDRequestList, List errorNames) { + for (TaskBachRequest task : taskIDRequestList) { + try { + TaskIDRequest taskIDRequest = new TaskIDRequest(); + taskIDRequest.setTaskID(task.getTaskID()); + restartTask(taskIDRequest); + } catch (RuntimeException e) { + log.error("start batch task failed:{}", e.getMessage()); + errorNames.add(task.getTaskName()); + } + } + } + +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/verify/VerifyBizService.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/verify/VerifyBizService.java new file mode 100644 index 0000000000..e4f08b30cc --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/service/verify/VerifyBizService.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.service.verify; + +import org.apache.eventmesh.admin.server.web.db.entity.EventMeshVerify; +import org.apache.eventmesh.admin.server.web.db.service.EventMeshVerifyService; +import org.apache.eventmesh.common.remote.request.ReportVerifyRequest; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class VerifyBizService { + + @Autowired + private EventMeshVerifyService verifyService; + + public boolean reportVerifyRecord(ReportVerifyRequest request) { + EventMeshVerify verify = new EventMeshVerify(); + verify.setRecordID(request.getRecordID()); + verify.setRecordSig(request.getRecordSig()); + verify.setPosition(request.getPosition()); + verify.setTaskID(request.getTaskID()); + verify.setJobID(request.getJobID()); + verify.setConnectorName(request.getConnectorName()); + verify.setConnectorStage(request.getConnectorStage()); + return verifyService.save(verify); + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/Base64.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/Base64.java new file mode 100644 index 0000000000..f85807b7f9 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/Base64.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.utils; + +public class Base64 { + private static char[] alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=".toCharArray(); + private static byte[] codes = new byte[256]; + + public Base64() { + } + + public static char[] encode(byte[] data) { + char[] out = new char[(data.length + 2) / 3 * 4]; + int i = 0; + + for (int index = 0; i < data.length; index += 4) { + boolean quad = false; + boolean trip = false; + int val = 255 & data[i]; + val <<= 8; + if (i + 1 < data.length) { + val |= 255 & data[i + 1]; + trip = true; + } + + val <<= 8; + if (i + 2 < data.length) { + val |= 255 & data[i + 2]; + quad = true; + } + + out[index + 3] = alphabet[quad ? val & 63 : 64]; + val >>= 6; + out[index + 2] = alphabet[trip ? val & 63 : 64]; + val >>= 6; + out[index + 1] = alphabet[val & 63]; + val >>= 6; + out[index + 0] = alphabet[val & 63]; + i += 3; + } + + return out; + } + + public static byte[] decode(char[] data) { + int tempLen = data.length; + + int len; + for (len = 0; len < data.length; ++len) { + if (data[len] > 255 || codes[data[len]] < 0) { + --tempLen; + } + } + + len = tempLen / 4 * 3; + if (tempLen % 4 == 3) { + len += 2; + } + + if (tempLen % 4 == 2) { + ++len; + } + + byte[] out = new byte[len]; + int shift = 0; + int accum = 0; + int index = 0; + + for (int ix = 0; ix < data.length; ++ix) { + int value = data[ix] > 255 ? -1 : codes[data[ix]]; + if (value >= 0) { + accum <<= 6; + shift += 6; + accum |= value; + if (shift >= 8) { + shift -= 8; + out[index++] = (byte) (accum >> shift & 255); + } + } + } + + if (index != out.length) { + throw new Error("Miscalculated data length (wrote " + index + " instead of " + out.length + ")"); + } else { + return out; + } + } + + static { + int i; + for (i = 0; i < 256; ++i) { + codes[i] = -1; + } + + for (i = 65; i <= 90; ++i) { + codes[i] = (byte) (i - 65); + } + + for (i = 97; i <= 122; ++i) { + codes[i] = (byte) (26 + i - 97); + } + + for (i = 48; i <= 57; ++i) { + codes[i] = (byte) (52 + i - 48); + } + + codes[43] = 62; + codes[47] = 63; + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/Base64Utils.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/Base64Utils.java new file mode 100644 index 0000000000..9c9a258671 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/Base64Utils.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.utils; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.OutputStream; + +public class Base64Utils { + private static final int CACHE_SIZE = 1024; + + public Base64Utils() { + } + + public static byte[] decode(String base64) throws Exception { + return Base64.decode(base64.toCharArray()); + } + + public static String encode(byte[] bytes) throws Exception { + return new String(Base64.encode(bytes)); + } + + public static String encodeFile(String filePath) throws Exception { + byte[] bytes = fileToByte(filePath); + return encode(bytes); + } + + public static void decodeToFile(String filePath, String base64) throws Exception { + byte[] bytes = decode(base64); + byteArrayToFile(bytes, filePath); + } + + public static byte[] fileToByte(String filePath) throws Exception { + byte[] data = new byte[0]; + File file = new File(filePath); + if (file.exists()) { + FileInputStream in = new FileInputStream(file); + ByteArrayOutputStream out = new ByteArrayOutputStream(2048); + byte[] cache = new byte[1024]; + int nread; + + while ((nread = in.read(cache)) != -1) { + out.write(cache, 0, nread); + out.flush(); + } + + out.close(); + in.close(); + data = out.toByteArray(); + } + + return data; + } + + public static void byteArrayToFile(byte[] bytes, String filePath) throws Exception { + InputStream in = new ByteArrayInputStream(bytes); + File destFile = new File(filePath); + if (!destFile.getParentFile().exists()) { + destFile.getParentFile().mkdirs(); + } + + destFile.createNewFile(); + OutputStream out = new FileOutputStream(destFile); + byte[] cache = new byte[1024]; + + int nread; + while ((nread = ((InputStream) in).read(cache)) != -1) { + ((OutputStream) out).write(cache, 0, nread); + ((OutputStream) out).flush(); + } + + ((OutputStream) out).close(); + ((InputStream) in).close(); + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/EncryptUtil.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/EncryptUtil.java new file mode 100644 index 0000000000..06c8bbc330 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/EncryptUtil.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.utils; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; + +public class EncryptUtil { + public EncryptUtil() { + } + + private static byte[] hexStringToBytes(String hexString) { + if (hexString != null && !hexString.equals("")) { + hexString = hexString.toUpperCase(); + int length = hexString.length() / 2; + char[] hexChars = hexString.toCharArray(); + byte[] d = new byte[length]; + + for (int i = 0; i < length; ++i) { + int pos = i * 2; + d[i] = (byte) (charToByte(hexChars[pos]) << 4 | charToByte(hexChars[pos + 1])); + } + + return d; + } else { + return null; + } + } + + public static String byteToHexString(byte[] b) { + String a = ""; + + for (int i = 0; i < b.length; ++i) { + String hex = Integer.toHexString(b[i] & 255); + if (hex.length() == 1) { + hex = '0' + hex; + } + + a = a + hex; + } + + return a; + } + + private static byte charToByte(char c) { + return (byte) "0123456789ABCDEF".indexOf(c); + } + + private static String readFileContent(String filePath) { + File file = new File(filePath); + BufferedReader reader = null; + StringBuffer key = new StringBuffer(); + + try { + IOException e; + try { + reader = new BufferedReader(new FileReader(file)); + e = null; + + String tempString; + while ((tempString = reader.readLine()) != null) { + if (!tempString.startsWith("--")) { + key.append(tempString); + } + } + + reader.close(); + } catch (IOException ioException) { + e = ioException; + e.printStackTrace(); + } + } finally { + if (reader != null) { + try { + reader.close(); + } catch (IOException ioException) { + ioException.printStackTrace(); + } + } + + } + + return key.toString(); + } + + public static String decrypt(String sysPubKeyFile, String appPrivKeyFile, String encStr) throws Exception { + String pubKeyBase64 = readFileContent(sysPubKeyFile); + String privKeyBase64 = readFileContent(appPrivKeyFile); + byte[] encBin = hexStringToBytes(encStr); + byte[] pubDecBin = RSAUtils.decryptByPublicKeyBlock(encBin, pubKeyBase64); + byte[] privDecBin = RSAUtils.decryptByPrivateKeyBlock(pubDecBin, privKeyBase64); + return new String(privDecBin); + } + + public static String decrypt(ParamType pubKeyType, String sysPubKey, ParamType privKeyType, String appPrivKey, ParamType passwdType, + String passwd) throws Exception { + String pubKeyBase64 = pubKeyType == ParamType.FILE ? readFileContent(sysPubKey) : sysPubKey; + String privKeyBase64 = privKeyType == ParamType.FILE ? readFileContent(appPrivKey) : appPrivKey; + String passwdContent = passwdType == ParamType.FILE ? readFileContent(passwd) : passwd; + byte[] encBin = hexStringToBytes(passwdContent); + byte[] pubDecBin = RSAUtils.decryptByPublicKeyBlock(encBin, pubKeyBase64); + byte[] privDecBin = RSAUtils.decryptByPrivateKeyBlock(pubDecBin, privKeyBase64); + return new String(privDecBin); + } + + public static String encrypt(String appPubKeyFile, String sysPrivKeyFile, String passwd) throws Exception { + String pubKeyBase64 = readFileContent(appPubKeyFile); + String privKeyBase64 = readFileContent(sysPrivKeyFile); + byte[] pubEncBin = RSAUtils.encryptByPublicKeyBlock(passwd.getBytes(), pubKeyBase64); + byte[] privEncBin = RSAUtils.encryptByPrivateKeyBlock(pubEncBin, privKeyBase64); + return byteToHexString(privEncBin); + } + + public static String encrypt(ParamType pubKeyType, String appPubKey, ParamType privKeyType, String sysPrivKey, String passwd) throws Exception { + String pubKeyBase64 = pubKeyType == ParamType.FILE ? readFileContent(appPubKey) : appPubKey; + String privKeyBase64 = privKeyType == ParamType.FILE ? readFileContent(sysPrivKey) : sysPrivKey; + byte[] pubEncBin = RSAUtils.encryptByPublicKeyBlock(passwd.getBytes(), pubKeyBase64); + byte[] privEncBin = RSAUtils.encryptByPrivateKeyBlock(pubEncBin, privKeyBase64); + return byteToHexString(privEncBin); + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/JdbcUtils.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/JdbcUtils.java new file mode 100644 index 0000000000..c012806e2e --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/JdbcUtils.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.utils; + +import com.alibaba.druid.pool.DruidDataSource; + +public class JdbcUtils { + + public static DruidDataSource createDruidDataSource(String url, String userName, String passWord) { + DruidDataSource dataSource = new DruidDataSource(); + dataSource.setUrl(url); + dataSource.setUsername(userName); + dataSource.setPassword(passWord); + dataSource.setInitialSize(5); + dataSource.setMinIdle(5); + dataSource.setMaxActive(20); + dataSource.setMaxWait(60000); + dataSource.setTimeBetweenEvictionRunsMillis(60000); + dataSource.setMinEvictableIdleTimeMillis(300000); + dataSource.setValidationQuery("SELECT 1"); + dataSource.setTestWhileIdle(true); + dataSource.setTestOnBorrow(false); + dataSource.setTestOnReturn(false); + dataSource.setPoolPreparedStatements(true); + dataSource.setMaxPoolPreparedStatementPerConnectionSize(20); + return dataSource; + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/ParamType.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/ParamType.java new file mode 100644 index 0000000000..ed58a49b89 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/ParamType.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.utils; + +public enum ParamType { + FILE, + STRING; + + private ParamType() { + } +} diff --git a/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/RSAUtils.java b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/RSAUtils.java new file mode 100644 index 0000000000..9353eb3f17 --- /dev/null +++ b/eventmesh-admin-server/src/main/java/org/apache/eventmesh/admin/server/web/utils/RSAUtils.java @@ -0,0 +1,255 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.admin.server.web.utils; + +import java.io.ByteArrayOutputStream; +import java.security.Key; +import java.security.KeyFactory; +import java.security.KeyPair; +import java.security.KeyPairGenerator; +import java.security.PrivateKey; +import java.security.PublicKey; +import java.security.Signature; +import java.security.interfaces.RSAPrivateKey; +import java.security.interfaces.RSAPublicKey; +import java.security.spec.PKCS8EncodedKeySpec; +import java.security.spec.X509EncodedKeySpec; +import java.util.HashMap; +import java.util.Map; + +import javax.crypto.Cipher; + +public class RSAUtils { + public static final String KEY_ALGORITHM = "RSA"; + public static final String SIGNATURE_ALGORITHM = "MD5withRSA"; + private static final String PUBLIC_KEY = "RSAPublicKey"; + private static final String PRIVATE_KEY = "RSAPrivateKey"; + private static final int MAX_ENCRYPT_BLOCK = 117; + private static final int MAX_DECRYPT_BLOCK = 128; + + public RSAUtils() { + } + + public static Map genKeyPair() throws Exception { + KeyPairGenerator keyPairGen = KeyPairGenerator.getInstance("RSA"); + keyPairGen.initialize(1024); + KeyPair keyPair = keyPairGen.generateKeyPair(); + RSAPublicKey publicKey = (RSAPublicKey) keyPair.getPublic(); + RSAPrivateKey privateKey = (RSAPrivateKey) keyPair.getPrivate(); + Map keyMap = new HashMap(2); + keyMap.put("RSAPublicKey", publicKey); + keyMap.put("RSAPrivateKey", privateKey); + return keyMap; + } + + public static String sign(byte[] data, String privateKey) throws Exception { + byte[] keyBytes = Base64Utils.decode(privateKey); + PKCS8EncodedKeySpec pkcs8KeySpec = new PKCS8EncodedKeySpec(keyBytes); + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + PrivateKey privateK = keyFactory.generatePrivate(pkcs8KeySpec); + Signature signature = Signature.getInstance("MD5withRSA"); + signature.initSign(privateK); + signature.update(data); + return Base64Utils.encode(signature.sign()); + } + + public static boolean verify(byte[] data, String publicKey, String sign) throws Exception { + byte[] keyBytes = Base64Utils.decode(publicKey); + X509EncodedKeySpec keySpec = new X509EncodedKeySpec(keyBytes); + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + PublicKey publicK = keyFactory.generatePublic(keySpec); + Signature signature = Signature.getInstance("MD5withRSA"); + signature.initVerify(publicK); + signature.update(data); + return signature.verify(Base64Utils.decode(sign)); + } + + public static byte[] decryptByPrivateKey(byte[] encryptedData, String privateKey) throws Exception { + byte[] keyBytes = Base64Utils.decode(privateKey); + PKCS8EncodedKeySpec pkcs8KeySpec = new PKCS8EncodedKeySpec(keyBytes); + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + Key privateK = keyFactory.generatePrivate(pkcs8KeySpec); + Cipher cipher = Cipher.getInstance(keyFactory.getAlgorithm()); + cipher.init(2, privateK); + int inputLen = encryptedData.length; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + int offSet = 0; + + for (int i = 0; inputLen - offSet > 0; offSet = i * 128) { + byte[] cache; + if (inputLen - offSet > 128) { + cache = cipher.doFinal(encryptedData, offSet, 128); + } else { + cache = cipher.doFinal(encryptedData, offSet, inputLen - offSet); + } + + out.write(cache, 0, cache.length); + ++i; + } + + byte[] decryptedData = out.toByteArray(); + out.close(); + return decryptedData; + } + + public static byte[] decryptByPrivateKeyBlock(byte[] encryptedData, String privateKey) throws Exception { + byte[] keyBytes = Base64Utils.decode(privateKey); + PKCS8EncodedKeySpec pkcs8KeySpec = new PKCS8EncodedKeySpec(keyBytes); + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + Key privateK = keyFactory.generatePrivate(pkcs8KeySpec); + Cipher cipher = Cipher.getInstance(keyFactory.getAlgorithm()); + cipher.init(2, privateK); + int inputLen = encryptedData.length; + int offSet = 0; + byte[] cache = cipher.doFinal(encryptedData, offSet, inputLen); + return cache; + } + + public static byte[] decryptByPublicKey(byte[] encryptedData, String publicKey) throws Exception { + byte[] keyBytes = Base64Utils.decode(publicKey); + X509EncodedKeySpec x509KeySpec = new X509EncodedKeySpec(keyBytes); + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + Key publicK = keyFactory.generatePublic(x509KeySpec); + Cipher cipher = Cipher.getInstance(keyFactory.getAlgorithm()); + cipher.init(2, publicK); + int inputLen = encryptedData.length; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + int offSet = 0; + + for (int i = 0; inputLen - offSet > 0; offSet = i * 128) { + byte[] cache; + if (inputLen - offSet > 128) { + cache = cipher.doFinal(encryptedData, offSet, 128); + } else { + cache = cipher.doFinal(encryptedData, offSet, inputLen - offSet); + } + + out.write(cache, 0, cache.length); + ++i; + } + + byte[] decryptedData = out.toByteArray(); + out.close(); + return decryptedData; + } + + public static byte[] decryptByPublicKeyBlock(byte[] encryptedData, String publicKey) throws Exception { + byte[] keyBytes = Base64Utils.decode(publicKey); + X509EncodedKeySpec x509KeySpec = new X509EncodedKeySpec(keyBytes); + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + Key publicK = keyFactory.generatePublic(x509KeySpec); + Cipher cipher = Cipher.getInstance(keyFactory.getAlgorithm()); + cipher.init(2, publicK); + int inputLen = encryptedData.length; + int offSet = 0; + byte[] cache = cipher.doFinal(encryptedData, offSet, inputLen); + return cache; + } + + public static byte[] encryptByPublicKey(byte[] data, String publicKey) throws Exception { + byte[] keyBytes = Base64Utils.decode(publicKey); + X509EncodedKeySpec x509KeySpec = new X509EncodedKeySpec(keyBytes); + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + Key publicK = keyFactory.generatePublic(x509KeySpec); + Cipher cipher = Cipher.getInstance(keyFactory.getAlgorithm()); + cipher.init(1, publicK); + int inputLen = data.length; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + int offSet = 0; + + for (int i = 0; inputLen - offSet > 0; offSet = i * 117) { + byte[] cache; + if (inputLen - offSet > 117) { + cache = cipher.doFinal(data, offSet, 117); + } else { + cache = cipher.doFinal(data, offSet, inputLen - offSet); + } + + out.write(cache, 0, cache.length); + ++i; + } + + byte[] encryptedData = out.toByteArray(); + out.close(); + return encryptedData; + } + + public static byte[] encryptByPublicKeyBlock(byte[] data, String publicKey) throws Exception { + byte[] keyBytes = Base64Utils.decode(publicKey); + X509EncodedKeySpec x509KeySpec = new X509EncodedKeySpec(keyBytes); + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + Key publicK = keyFactory.generatePublic(x509KeySpec); + Cipher cipher = Cipher.getInstance(keyFactory.getAlgorithm()); + cipher.init(1, publicK); + int inputLen = data.length; + int offSet = 0; + byte[] cache = cipher.doFinal(data, offSet, inputLen); + return cache; + } + + public static byte[] encryptByPrivateKey(byte[] data, String privateKey) throws Exception { + byte[] keyBytes = Base64Utils.decode(privateKey); + PKCS8EncodedKeySpec pkcs8KeySpec = new PKCS8EncodedKeySpec(keyBytes); + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + Key privateK = keyFactory.generatePrivate(pkcs8KeySpec); + Cipher cipher = Cipher.getInstance(keyFactory.getAlgorithm()); + cipher.init(1, privateK); + int inputLen = data.length; + ByteArrayOutputStream out = new ByteArrayOutputStream(); + int offSet = 0; + + for (int i = 0; inputLen - offSet > 0; offSet = i * 117) { + byte[] cache; + if (inputLen - offSet > 117) { + cache = cipher.doFinal(data, offSet, 117); + } else { + cache = cipher.doFinal(data, offSet, inputLen - offSet); + } + + out.write(cache, 0, cache.length); + ++i; + } + + byte[] encryptedData = out.toByteArray(); + out.close(); + return encryptedData; + } + + public static byte[] encryptByPrivateKeyBlock(byte[] data, String privateKey) throws Exception { + byte[] keyBytes = Base64Utils.decode(privateKey); + PKCS8EncodedKeySpec pkcs8KeySpec = new PKCS8EncodedKeySpec(keyBytes); + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + Key privateK = keyFactory.generatePrivate(pkcs8KeySpec); + Cipher cipher = Cipher.getInstance(keyFactory.getAlgorithm()); + cipher.init(1, privateK); + int inputLen = data.length; + int offSet = 0; + byte[] cache = cipher.doFinal(data, offSet, inputLen); + return cache; + } + + public static String getPrivateKey(Map keyMap) throws Exception { + Key key = (Key) keyMap.get("RSAPrivateKey"); + return Base64Utils.encode(key.getEncoded()); + } + + public static String getPublicKey(Map keyMap) throws Exception { + Key key = (Key) keyMap.get("RSAPublicKey"); + return Base64Utils.encode(key.getEncoded()); + } +} diff --git a/eventmesh-admin-server/src/main/resources/META-INF/spring.factories b/eventmesh-admin-server/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..0d9e9bae6e --- /dev/null +++ b/eventmesh-admin-server/src/main/resources/META-INF/spring.factories @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + org.apache.eventmesh.admin.server.AdminServerProperties \ No newline at end of file diff --git a/eventmesh-common/build.gradle b/eventmesh-common/build.gradle index 07068fe4c1..21b6e63d44 100644 --- a/eventmesh-common/build.gradle +++ b/eventmesh-common/build.gradle @@ -15,7 +15,7 @@ * limitations under the License. */ -def grpcVersion = '1.64.0' +def grpcVersion = '1.68.0' dependencies { api "com.google.guava:guava" @@ -48,6 +48,7 @@ dependencies { implementation "org.apache.httpcomponents:httpclient" implementation "io.netty:netty-all" + compileOnly 'com.mysql:mysql-connector-j' implementation "io.grpc:grpc-protobuf:${grpcVersion}" implementation "io.grpc:grpc-stub:${grpcVersion}" diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/AbstractComponent.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/AbstractComponent.java new file mode 100644 index 0000000000..375b6cb1d3 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/AbstractComponent.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common; + +import java.util.concurrent.atomic.AtomicBoolean; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public abstract class AbstractComponent implements ComponentLifeCycle { + private final AtomicBoolean started = new AtomicBoolean(false); + private final AtomicBoolean stopped = new AtomicBoolean(false); + + @Override + public void start() throws Exception { + if (!started.compareAndSet(false, true)) { + log.info("component [{}] has started", this.getClass()); + return; + } + log.info("component [{}] will start", this.getClass()); + run(); + log.info("component [{}] started successfully", this.getClass()); + } + + @Override + public void stop() throws Exception { + if (!stopped.compareAndSet(false, true)) { + log.info("component [{}] has stopped", this.getClass()); + return; + } + log.info("component [{}] will stop", this.getClass()); + shutdown(); + log.info("component [{}] stopped successfully", this.getClass()); + } + + protected abstract void run() throws Exception; + + protected abstract void shutdown() throws Exception; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/ComponentLifeCycle.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/ComponentLifeCycle.java new file mode 100644 index 0000000000..76fdd548d0 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/ComponentLifeCycle.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common; + +/** + * LifeCycle of EventMesh Component + */ +public interface ComponentLifeCycle { + + void start() throws Exception; + + void stop() throws Exception; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/Constants.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/Constants.java index 867d50b43b..2460129e75 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/Constants.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/Constants.java @@ -200,9 +200,13 @@ public class Constants { public static final String GRPC = "GRPC"; + public static final String ADMIN = "ADMIN"; + public static final String OS_NAME_KEY = "os.name"; public static final String OS_WIN_PREFIX = "win"; public static final String DEFAULT = "default"; + + public static final String ADMIN_SERVER_REGISTRY_NAME = "DEFAULT_GROUP@@em_adm_server"; } diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/CommonConfiguration.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/CommonConfiguration.java index 2f38a372ce..04c4ae60ed 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/CommonConfiguration.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/CommonConfiguration.java @@ -106,6 +106,21 @@ public class CommonConfiguration { @ConfigField(field = "server.retry.plugin.type") private String eventMeshRetryPluginType = Constants.DEFAULT; + @ConfigField(field = "registry.plugin.server-addr", notEmpty = true) + private String registryAddr = ""; + + @ConfigField(field = "registry.plugin.type", notEmpty = true) + private String eventMeshRegistryPluginType = "nacos"; + + @ConfigField(field = "registry.plugin.username") + private String eventMeshRegistryPluginUsername = ""; + + @ConfigField(field = "registry.plugin.password") + private String eventMeshRegistryPluginPassword = ""; + + @ConfigField(field = "registry.plugin.enabled") + private boolean eventMeshRegistryPluginEnabled = false; + public void reload() { this.eventMeshWebhookOrigin = "eventmesh." + eventMeshIDC; diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/ConfigService.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/ConfigService.java index 7c5b17d7e6..3f3f609a1f 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/ConfigService.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/ConfigService.java @@ -60,6 +60,9 @@ private ConfigService() { } public ConfigService setConfigPath(String configPath) { + if (StringUtils.isNotBlank(configPath) && !configPath.endsWith(File.separator)) { + configPath = configPath + File.separator; + } this.configPath = configPath; return this; } @@ -128,7 +131,7 @@ public T getConfig(ConfigInfo configInfo) throws IOException { } else { filePath = path.startsWith(FILE_PATH_PREFIX) ? path.substring(FILE_PATH_PREFIX.length()) : this.configPath + path; } - + filePath = normalizeFilePath(filePath); if (filePath.contains(".jar")) { try (final InputStream inputStream = getClass().getResourceAsStream(Objects.requireNonNull(resourceUrl))) { if (inputStream == null) { @@ -149,6 +152,15 @@ public T getConfig(ConfigInfo configInfo) throws IOException { return (T) object; } + private String normalizeFilePath(String filePath) { + if (System.getProperty("os.name").toLowerCase().contains("win")) { + if (filePath.startsWith("/")) { + filePath = filePath.substring(1); + } + } + return filePath; + } + private void populateConfig(Object object, Class clazz, Config config) throws NoSuchFieldException, IOException, IllegalAccessException { ConfigInfo configInfo = new ConfigInfo(); diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/FileLoad.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/FileLoad.java index a0c81ea481..4f8c6687b8 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/FileLoad.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/FileLoad.java @@ -68,6 +68,7 @@ class PropertiesFileLoad implements FileLoad { private final Convert convert = new Convert(); @SuppressWarnings("unchecked") + @Override public T getConfig(ConfigInfo configInfo) throws IOException { final Properties properties = new Properties(); if (StringUtils.isNotBlank(configInfo.getResourceUrl())) { diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/config/Config.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/Config.java similarity index 93% rename from eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/config/Config.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/Config.java index 330aa93932..d4bdee7778 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/config/Config.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/Config.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.openconnect.api.config; +package org.apache.eventmesh.common.config.connector; public abstract class Config { } diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/config/Constants.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/Constants.java similarity index 73% rename from eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/config/Constants.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/Constants.java index 59794e562a..817efb6d3a 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/config/Constants.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/Constants.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.openconnect.api.config; +package org.apache.eventmesh.common.config.connector; public class Constants { @@ -30,4 +30,21 @@ public class Constants { public static final int DEFAULT_ATTEMPT = 3; public static final int DEFAULT_PORT = 8080; + + // ======================== Source Constants ======================== + /** + * Default capacity + */ + public static final int DEFAULT_CAPACITY = 1024; + + /** + * Default poll batch size + */ + public static final int DEFAULT_POLL_BATCH_SIZE = 10; + + /** + * Default poll timeout (unit: ms) + */ + public static final long DEFAULT_POLL_TIMEOUT = 5000L; + } diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/PollConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/PollConfig.java new file mode 100644 index 0000000000..cf3f06be91 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/PollConfig.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.config.connector; + +import lombok.Data; + +/** + * Source Poll Config + */ +@Data +public class PollConfig { + + /** + * Capacity of the poll queue + */ + private int capacity = Constants.DEFAULT_CAPACITY; + + /** + * Max batch size of the poll + */ + private int maxBatchSize = Constants.DEFAULT_POLL_BATCH_SIZE; + + /** + * Max wait time of the poll + */ + private long maxWaitTime = Constants.DEFAULT_POLL_TIMEOUT; + +} diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/config/PubSubConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/PubSubConfig.java similarity index 95% rename from eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/config/PubSubConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/PubSubConfig.java index 6f5c9cd5f4..be83d51127 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/config/PubSubConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/PubSubConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.openconnect.api.config; +package org.apache.eventmesh.common.config.connector; import lombok.Data; diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/config/SinkConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/SinkConfig.java similarity index 94% rename from eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/config/SinkConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/SinkConfig.java index 527d02e04d..4ef68291d3 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/config/SinkConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/SinkConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.openconnect.api.config; +package org.apache.eventmesh.common.config.connector; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/config/SourceConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/SourceConfig.java similarity index 80% rename from eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/config/SourceConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/SourceConfig.java index 2942be936c..f7bc42970c 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/config/SourceConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/SourceConfig.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.openconnect.api.config; +package org.apache.eventmesh.common.config.connector; -import org.apache.eventmesh.openconnect.offsetmgmt.api.config.OffsetStorageConfig; +import org.apache.eventmesh.common.config.connector.offset.OffsetStorageConfig; import lombok.Data; import lombok.EqualsAndHashCode; @@ -30,4 +30,7 @@ public abstract class SourceConfig extends Config { private OffsetStorageConfig offsetStorageConfig; + // Polling configuration, e.g. capacity, batch size, wait time, etc. + private PollConfig pollConfig = new PollConfig(); + } diff --git a/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/sink/config/DingDingSinkConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/dingtalk/DingDingSinkConfig.java similarity index 88% rename from eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/sink/config/DingDingSinkConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/dingtalk/DingDingSinkConfig.java index aa4245219f..3482d5665c 100644 --- a/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/sink/config/DingDingSinkConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/dingtalk/DingDingSinkConfig.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.dingtalk.sink.config; +package org.apache.eventmesh.common.config.connector.dingtalk; -import org.apache.eventmesh.openconnect.api.config.SinkConfig; +import org.apache.eventmesh.common.config.connector.SinkConfig; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/sink/config/SinkConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/dingtalk/SinkConnectorConfig.java similarity index 94% rename from eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/sink/config/SinkConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/dingtalk/SinkConnectorConfig.java index 2211fd1adc..1dfcd76640 100644 --- a/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/sink/config/SinkConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/dingtalk/SinkConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.dingtalk.sink.config; +package org.apache.eventmesh.common.config.connector.dingtalk; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/sink/config/FileSinkConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/file/FileSinkConfig.java similarity index 89% rename from eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/sink/config/FileSinkConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/file/FileSinkConfig.java index abdae52770..7de6daa51e 100644 --- a/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/sink/config/FileSinkConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/file/FileSinkConfig.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.file.sink.config; +package org.apache.eventmesh.common.config.connector.file; -import org.apache.eventmesh.openconnect.api.config.SinkConfig; +import org.apache.eventmesh.common.config.connector.SinkConfig; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/source/config/FileSourceConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/file/FileSourceConfig.java similarity index 88% rename from eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/source/config/FileSourceConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/file/FileSourceConfig.java index b969544403..06bc4c1745 100644 --- a/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/source/config/FileSourceConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/file/FileSourceConfig.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.file.source.config; +package org.apache.eventmesh.common.config.connector.file; -import org.apache.eventmesh.openconnect.api.config.SourceConfig; +import org.apache.eventmesh.common.config.connector.SourceConfig; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/sink/config/SinkConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/file/SinkConnectorConfig.java similarity index 94% rename from eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/sink/config/SinkConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/file/SinkConnectorConfig.java index a8cf958f7f..ee42a17759 100644 --- a/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/sink/config/SinkConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/file/SinkConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.file.sink.config; +package org.apache.eventmesh.common.config.connector.file; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/source/config/SourceConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/file/SourceConnectorConfig.java similarity index 94% rename from eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/source/config/SourceConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/file/SourceConnectorConfig.java index 6376a7fb4a..786d8fa743 100644 --- a/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/source/config/SourceConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/file/SourceConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.file.source.config; +package org.apache.eventmesh.common.config.connector.file; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/config/HttpRetryConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/http/HttpRetryConfig.java similarity index 90% rename from eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/config/HttpRetryConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/http/HttpRetryConfig.java index 0bceac7d47..319732a875 100644 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/config/HttpRetryConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/http/HttpRetryConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.http.sink.config; +package org.apache.eventmesh.common.config.connector.http; import lombok.Data; @@ -24,8 +24,8 @@ public class HttpRetryConfig { // maximum number of retries, default 2, minimum 0 private int maxRetries = 2; - // retry interval, default 2000ms - private int interval = 2000; + // retry interval, default 1000ms + private int interval = 1000; // Default value is false, indicating that only requests with network-level errors will be retried. // If set to true, all failed requests will be retried, including network-level errors and non-2xx responses. diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/config/HttpSinkConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/http/HttpSinkConfig.java similarity index 89% rename from eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/config/HttpSinkConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/http/HttpSinkConfig.java index 3dd0c2b6a5..3c429f3355 100644 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/config/HttpSinkConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/http/HttpSinkConfig.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.http.sink.config; +package org.apache.eventmesh.common.config.connector.http; -import org.apache.eventmesh.openconnect.api.config.SinkConfig; +import org.apache.eventmesh.common.config.connector.SinkConfig; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/config/HttpSourceConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/http/HttpSourceConfig.java similarity index 88% rename from eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/config/HttpSourceConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/http/HttpSourceConfig.java index bee870cb1a..476dfb10de 100644 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/config/HttpSourceConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/http/HttpSourceConfig.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.http.source.config; +package org.apache.eventmesh.common.config.connector.http; -import org.apache.eventmesh.openconnect.api.config.SourceConfig; +import org.apache.eventmesh.common.config.connector.SourceConfig; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/config/HttpWebhookConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/http/HttpWebhookConfig.java similarity index 95% rename from eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/config/HttpWebhookConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/http/HttpWebhookConfig.java index f15bac4568..96b9e09826 100644 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/config/HttpWebhookConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/http/HttpWebhookConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.http.sink.config; +package org.apache.eventmesh.common.config.connector.http; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/config/SinkConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/http/SinkConnectorConfig.java similarity index 81% rename from eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/config/SinkConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/http/SinkConnectorConfig.java index 9bb338cceb..65fc8fe72d 100644 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/config/SinkConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/http/SinkConnectorConfig.java @@ -15,9 +15,8 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.http.sink.config; +package org.apache.eventmesh.common.config.connector.http; -import io.vertx.core.http.HttpClientOptions; import lombok.Data; @@ -29,19 +28,19 @@ public class SinkConnectorConfig { private String[] urls; // keepAlive, default true - private boolean keepAlive = HttpClientOptions.DEFAULT_KEEP_ALIVE; + private boolean keepAlive = true; // timeunit: ms, default 60000ms - private int keepAliveTimeout = HttpClientOptions.DEFAULT_KEEP_ALIVE_TIMEOUT * 1000; // Keep units consistent + private int keepAliveTimeout = 60 * 1000; // Keep units consistent // timeunit: ms, default 5000ms, recommended scope: 5000ms - 10000ms private int connectionTimeout = 5000; // timeunit: ms, default 5000ms - private int idleTimeout; + private int idleTimeout = 5000; - // maximum number of HTTP/1 connections a client will pool, default 5 - private int maxConnectionPoolSize = HttpClientOptions.DEFAULT_MAX_POOL_SIZE; + // maximum number of HTTP/1 connections a client will pool, default 50 + private int maxConnectionPoolSize = 50; // retry config private HttpRetryConfig retryConfig = new HttpRetryConfig(); @@ -49,6 +48,15 @@ public class SinkConnectorConfig { // webhook config private HttpWebhookConfig webhookConfig = new HttpWebhookConfig(); + private String deliveryStrategy = "ROUND_ROBIN"; + + private boolean skipDeliverException = false; + + // managed pipelining param, default true + private boolean isParallelized = true; + + private int parallelism = 2; + /** * Fill default values if absent (When there are multiple default values for a field) diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/http/SourceConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/http/SourceConnectorConfig.java new file mode 100644 index 0000000000..2c091e321a --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/http/SourceConnectorConfig.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.config.connector.http; + +import java.util.HashMap; +import java.util.Map; + +import lombok.Data; + +@Data +public class SourceConnectorConfig { + + private String connectorName; + + private String path = "/"; + + private int port; + + // timeunit: ms, default 5000ms + private int idleTimeout = 5000; + + /** + *
    + *
  • The maximum size allowed for form attributes when Content-Type is application/x-www-form-urlencoded or multipart/form-data
    • + *
  • Default is 1MB (1024 * 1024 bytes).
    • + *
  • If you receive a "size exceed allowed maximum capacity" error, you can increase this value.
    • + *
  • Note: This applies only when handling form data submissions.
    • + *
+ */ + private int maxFormAttributeSize = 1024 * 1024; + + // max size of the queue, default 1000 + private int maxStorageSize = 1000; + + // batch size, default 10 + private int batchSize = 10; + + // protocol, default CloudEvent + private String protocol = "Common"; + + // extra config, e.g. GitHub secret + private Map extraConfig = new HashMap<>(); + + // data consistency enabled, default true + private boolean dataConsistencyEnabled = true; +} diff --git a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/sink/config/KnativeSinkConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/knative/KnativeSinkConfig.java similarity index 88% rename from eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/sink/config/KnativeSinkConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/knative/KnativeSinkConfig.java index 85bb38a90b..aff7c275a5 100644 --- a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/sink/config/KnativeSinkConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/knative/KnativeSinkConfig.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.knative.sink.config; +package org.apache.eventmesh.common.config.connector.knative; -import org.apache.eventmesh.openconnect.api.config.SinkConfig; +import org.apache.eventmesh.common.config.connector.SinkConfig; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/source/config/KnativeSourceConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/knative/KnativeSourceConfig.java similarity index 88% rename from eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/source/config/KnativeSourceConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/knative/KnativeSourceConfig.java index 8ef36a8d78..644161d915 100644 --- a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/source/config/KnativeSourceConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/knative/KnativeSourceConfig.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.knative.source.config; +package org.apache.eventmesh.common.config.connector.knative; -import org.apache.eventmesh.openconnect.api.config.SourceConfig; +import org.apache.eventmesh.common.config.connector.SourceConfig; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/sink/config/SinkConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/knative/SinkConnectorConfig.java similarity index 93% rename from eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/sink/config/SinkConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/knative/SinkConnectorConfig.java index 5694c643f8..076bd68756 100644 --- a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/sink/config/SinkConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/knative/SinkConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.knative.sink.config; +package org.apache.eventmesh.common.config.connector.knative; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/source/config/SourceConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/knative/SourceConnectorConfig.java similarity index 93% rename from eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/source/config/SourceConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/knative/SourceConnectorConfig.java index 8d21c86b7b..98e7815520 100644 --- a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/source/config/SourceConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/knative/SourceConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.knative.source.config; +package org.apache.eventmesh.common.config.connector.knative; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/sink/config/LarkSinkConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/lark/LarkSinkConfig.java similarity index 89% rename from eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/sink/config/LarkSinkConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/lark/LarkSinkConfig.java index a97ece91b0..a9235129a0 100644 --- a/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/sink/config/LarkSinkConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/lark/LarkSinkConfig.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.lark.sink.config; +package org.apache.eventmesh.common.config.connector.lark; -import org.apache.eventmesh.openconnect.api.config.SinkConfig; +import org.apache.eventmesh.common.config.connector.SinkConfig; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/sink/config/SinkConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/lark/SinkConnectorConfig.java similarity index 56% rename from eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/sink/config/SinkConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/lark/SinkConnectorConfig.java index cde3aa6737..a4895b8b2e 100644 --- a/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/sink/config/SinkConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/lark/SinkConnectorConfig.java @@ -15,13 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.lark.sink.config; - -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import org.apache.commons.lang3.StringUtils; - -import com.lark.oapi.service.im.v1.enums.ReceiveIdTypeEnum; +package org.apache.eventmesh.common.config.connector.lark; import lombok.Data; @@ -53,8 +47,6 @@ public class SinkConnectorConfig { /** * When sinking CouldEvent to lark, choose to call - * {@link org.apache.eventmesh.connector.lark.sink.ImServiceHandler#sink(ConnectRecord)} - * or {@link org.apache.eventmesh.connector.lark.sink.ImServiceHandler#sinkAsync(ConnectRecord)} */ private String sinkAsync = "true"; @@ -62,20 +54,4 @@ public class SinkConnectorConfig { private String retryDelayInMills = "1000"; - public void validateSinkConfiguration() { - // validate blank - if (StringUtils.isAnyBlank(appId, appSecret, receiveId)) { - throw new IllegalArgumentException("appId or appSecret or receiveId is blank,please check it."); - } - - // validate receiveIdType - if (!StringUtils.containsAny(receiveIdType, ReceiveIdTypeEnum.CHAT_ID.getValue(), - ReceiveIdTypeEnum.EMAIL.getValue(), - ReceiveIdTypeEnum.OPEN_ID.getValue(), - ReceiveIdTypeEnum.USER_ID.getValue(), - ReceiveIdTypeEnum.UNION_ID.getValue())) { - throw new IllegalArgumentException( - String.format("sinkConnectorConfig.receiveIdType=[%s], Invalid.", receiveIdType)); - } - } } diff --git a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/sink/config/KafkaSinkConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/kafka/KafkaSinkConfig.java similarity index 82% rename from eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/sink/config/KafkaSinkConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/kafka/KafkaSinkConfig.java index 9bb79551db..973eed11ff 100644 --- a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/sink/config/KafkaSinkConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/kafka/KafkaSinkConfig.java @@ -15,12 +15,14 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.kafka.sink.config; +package org.apache.eventmesh.common.config.connector.mq.kafka; -import org.apache.eventmesh.openconnect.api.config.SinkConfig; +import org.apache.eventmesh.common.config.connector.SinkConfig; import lombok.Data; +import lombok.EqualsAndHashCode; +@EqualsAndHashCode(callSuper = true) @Data public class KafkaSinkConfig extends SinkConfig { diff --git a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/source/config/KafkaSourceConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/kafka/KafkaSourceConfig.java similarity index 82% rename from eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/source/config/KafkaSourceConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/kafka/KafkaSourceConfig.java index 4319ec96d7..bf44a82710 100644 --- a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/source/config/KafkaSourceConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/kafka/KafkaSourceConfig.java @@ -15,12 +15,14 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.kafka.source.config; +package org.apache.eventmesh.common.config.connector.mq.kafka; -import org.apache.eventmesh.openconnect.api.config.SourceConfig; +import org.apache.eventmesh.common.config.connector.SourceConfig; import lombok.Data; +import lombok.EqualsAndHashCode; +@EqualsAndHashCode(callSuper = true) @Data public class KafkaSourceConfig extends SourceConfig { diff --git a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/sink/config/SinkConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/kafka/SinkConnectorConfig.java similarity index 96% rename from eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/sink/config/SinkConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/kafka/SinkConnectorConfig.java index a240bf4f49..e7584319cb 100644 --- a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/sink/config/SinkConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/kafka/SinkConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.kafka.sink.config; +package org.apache.eventmesh.common.config.connector.mq.kafka; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/source/config/SourceConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/kafka/SourceConnectorConfig.java similarity index 93% rename from eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/source/config/SourceConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/kafka/SourceConnectorConfig.java index 2d2f52f85c..eb7406f664 100644 --- a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/source/config/SourceConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/kafka/SourceConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.kafka.source.config; +package org.apache.eventmesh.common.config.connector.mq.kafka; import lombok.Data; @@ -32,5 +32,4 @@ public class SourceConnectorConfig { private String enableAutoCommit = "false"; private String sessionTimeoutMS = "10000"; private String maxPollRecords = "1000"; - private int pollTimeOut = 100; } diff --git a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/sink/config/PulsarSinkConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/pulsar/PulsarSinkConfig.java similarity index 82% rename from eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/sink/config/PulsarSinkConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/pulsar/PulsarSinkConfig.java index 8d7bfe384b..8cbfd5fb2c 100644 --- a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/sink/config/PulsarSinkConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/pulsar/PulsarSinkConfig.java @@ -15,12 +15,14 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.pulsar.sink.config; +package org.apache.eventmesh.common.config.connector.mq.pulsar; -import org.apache.eventmesh.openconnect.api.config.SinkConfig; +import org.apache.eventmesh.common.config.connector.SinkConfig; import lombok.Data; +import lombok.EqualsAndHashCode; +@EqualsAndHashCode(callSuper = true) @Data public class PulsarSinkConfig extends SinkConfig { diff --git a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/source/config/PulsarSourceConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/pulsar/PulsarSourceConfig.java similarity index 82% rename from eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/source/config/PulsarSourceConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/pulsar/PulsarSourceConfig.java index a80c2a0e50..43eb2ca854 100644 --- a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/source/config/PulsarSourceConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/pulsar/PulsarSourceConfig.java @@ -15,12 +15,14 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.pulsar.source.config; +package org.apache.eventmesh.common.config.connector.mq.pulsar; -import org.apache.eventmesh.openconnect.api.config.SourceConfig; +import org.apache.eventmesh.common.config.connector.SourceConfig; import lombok.Data; +import lombok.EqualsAndHashCode; +@EqualsAndHashCode(callSuper = true) @Data public class PulsarSourceConfig extends SourceConfig { diff --git a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/sink/config/SinkConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/pulsar/SinkConnectorConfig.java similarity index 93% rename from eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/sink/config/SinkConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/pulsar/SinkConnectorConfig.java index e1ebba09cb..b66f1a5324 100644 --- a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/sink/config/SinkConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/pulsar/SinkConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.pulsar.sink.config; +package org.apache.eventmesh.common.config.connector.mq.pulsar; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/source/config/SourceConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/pulsar/SourceConnectorConfig.java similarity index 93% rename from eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/source/config/SourceConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/pulsar/SourceConnectorConfig.java index bb8ff46875..9f8fbce2d2 100644 --- a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/source/config/SourceConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/pulsar/SourceConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.pulsar.source.config; +package org.apache.eventmesh.common.config.connector.mq.pulsar; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/sink/config/RabbitMQSinkConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/rabbitmq/RabbitMQSinkConfig.java similarity index 88% rename from eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/sink/config/RabbitMQSinkConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/rabbitmq/RabbitMQSinkConfig.java index 1c4ec66d47..8dcb8ad50c 100644 --- a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/sink/config/RabbitMQSinkConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/rabbitmq/RabbitMQSinkConfig.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.rabbitmq.sink.config; +package org.apache.eventmesh.common.config.connector.mq.rabbitmq; -import org.apache.eventmesh.openconnect.api.config.SinkConfig; +import org.apache.eventmesh.common.config.connector.SinkConfig; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/source/config/RabbitMQSourceConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/rabbitmq/RabbitMQSourceConfig.java similarity index 88% rename from eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/source/config/RabbitMQSourceConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/rabbitmq/RabbitMQSourceConfig.java index 55ce402ae7..c268c1005a 100644 --- a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/source/config/RabbitMQSourceConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/rabbitmq/RabbitMQSourceConfig.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.rabbitmq.source.config; +package org.apache.eventmesh.common.config.connector.mq.rabbitmq; -import org.apache.eventmesh.openconnect.api.config.SourceConfig; +import org.apache.eventmesh.common.config.connector.SourceConfig; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/sink/config/SinkConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/rabbitmq/SinkConnectorConfig.java similarity index 88% rename from eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/sink/config/SinkConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/rabbitmq/SinkConnectorConfig.java index 358964bdc0..a6f633f943 100644 --- a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/sink/config/SinkConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/rabbitmq/SinkConnectorConfig.java @@ -15,9 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.rabbitmq.sink.config; - -import com.rabbitmq.client.BuiltinExchangeType; +package org.apache.eventmesh.common.config.connector.mq.rabbitmq; import lombok.Data; @@ -36,7 +34,7 @@ public class SinkConnectorConfig { private String virtualHost; - private BuiltinExchangeType exchangeType; + private String exchangeType; private String exchangeName; diff --git a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/source/config/SourceConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/rabbitmq/SourceConnectorConfig.java similarity index 88% rename from eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/source/config/SourceConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/rabbitmq/SourceConnectorConfig.java index d83149f9ca..29a041338f 100644 --- a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/source/config/SourceConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/rabbitmq/SourceConnectorConfig.java @@ -15,9 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.rabbitmq.source.config; - -import com.rabbitmq.client.BuiltinExchangeType; +package org.apache.eventmesh.common.config.connector.mq.rabbitmq; import lombok.Data; @@ -36,7 +34,7 @@ public class SourceConnectorConfig { private String virtualHost; - private BuiltinExchangeType exchangeType; + private String exchangeType; private String exchangeName; diff --git a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/sink/config/RocketMQSinkConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/rocketmq/RocketMQSinkConfig.java similarity index 88% rename from eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/sink/config/RocketMQSinkConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/rocketmq/RocketMQSinkConfig.java index bc4191c9bf..d7c08e8b79 100644 --- a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/sink/config/RocketMQSinkConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/rocketmq/RocketMQSinkConfig.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.rocketmq.sink.config; +package org.apache.eventmesh.common.config.connector.mq.rocketmq; -import org.apache.eventmesh.openconnect.api.config.SinkConfig; +import org.apache.eventmesh.common.config.connector.SinkConfig; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/source/config/RocketMQSourceConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/rocketmq/RocketMQSourceConfig.java similarity index 88% rename from eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/source/config/RocketMQSourceConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/rocketmq/RocketMQSourceConfig.java index 43a74fabd4..5c531d91c7 100644 --- a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/source/config/RocketMQSourceConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/rocketmq/RocketMQSourceConfig.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.rocketmq.source.config; +package org.apache.eventmesh.common.config.connector.mq.rocketmq; -import org.apache.eventmesh.openconnect.api.config.SourceConfig; +import org.apache.eventmesh.common.config.connector.SourceConfig; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/sink/config/SinkConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/rocketmq/SinkConnectorConfig.java similarity index 93% rename from eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/sink/config/SinkConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/rocketmq/SinkConnectorConfig.java index 034dd7ce0c..93472f7a4c 100644 --- a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/sink/config/SinkConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/rocketmq/SinkConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.rocketmq.sink.config; +package org.apache.eventmesh.common.config.connector.mq.rocketmq; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/source/config/SourceConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/rocketmq/SourceConnectorConfig.java similarity index 93% rename from eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/source/config/SourceConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/rocketmq/SourceConnectorConfig.java index 1dc8a0d5c9..641d1873bb 100644 --- a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/source/config/SourceConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/mq/rocketmq/SourceConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.rocketmq.source.config; +package org.apache.eventmesh.common.config.connector.mq.rocketmq; import lombok.Data; diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/config/OffsetStorageConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/offset/OffsetStorageConfig.java similarity index 82% rename from eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/config/OffsetStorageConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/offset/OffsetStorageConfig.java index 30a56a3d2f..60448d3691 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/config/OffsetStorageConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/offset/OffsetStorageConfig.java @@ -15,7 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.openconnect.offsetmgmt.api.config; +package org.apache.eventmesh.common.config.connector.offset; + +import org.apache.eventmesh.common.remote.datasource.DataSourceType; import java.util.Map; @@ -29,4 +31,8 @@ public class OffsetStorageConfig { private String offsetStorageAddr; private Map extensions; + + private DataSourceType dataSourceType; + + private DataSourceType dataSinkType; } diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/sink/config/OpenFunctionSinkConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/openfunction/OpenFunctionSinkConfig.java similarity index 88% rename from eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/sink/config/OpenFunctionSinkConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/openfunction/OpenFunctionSinkConfig.java index 6e4fbe3dc4..151ff68834 100644 --- a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/sink/config/OpenFunctionSinkConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/openfunction/OpenFunctionSinkConfig.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.openfunction.sink.config; +package org.apache.eventmesh.common.config.connector.openfunction; -import org.apache.eventmesh.openconnect.api.config.SinkConfig; +import org.apache.eventmesh.common.config.connector.SinkConfig; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/source/config/OpenFunctionSourceConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/openfunction/OpenFunctionSourceConfig.java similarity index 88% rename from eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/source/config/OpenFunctionSourceConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/openfunction/OpenFunctionSourceConfig.java index f3f5b52756..e3a6123ed9 100644 --- a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/source/config/OpenFunctionSourceConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/openfunction/OpenFunctionSourceConfig.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.openfunction.source.config; +package org.apache.eventmesh.common.config.connector.openfunction; -import org.apache.eventmesh.openconnect.api.config.SourceConfig; +import org.apache.eventmesh.common.config.connector.SourceConfig; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/sink/config/SinkConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/openfunction/SinkConnectorConfig.java similarity index 93% rename from eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/sink/config/SinkConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/openfunction/SinkConnectorConfig.java index 77c1b7c701..b8bc937390 100644 --- a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/sink/config/SinkConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/openfunction/SinkConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.openfunction.sink.config; +package org.apache.eventmesh.common.config.connector.openfunction; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/source/config/SourceConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/openfunction/SourceConnectorConfig.java similarity index 93% rename from eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/source/config/SourceConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/openfunction/SourceConnectorConfig.java index 93fc9739fc..09dbdea14c 100644 --- a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/source/config/SourceConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/openfunction/SourceConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.openfunction.source.config; +package org.apache.eventmesh.common.config.connector.openfunction; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/sink/config/PravegaSinkConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/pravega/PravegaSinkConfig.java similarity index 88% rename from eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/sink/config/PravegaSinkConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/pravega/PravegaSinkConfig.java index 739c6539b5..a94766c20c 100644 --- a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/sink/config/PravegaSinkConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/pravega/PravegaSinkConfig.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.pravega.sink.config; +package org.apache.eventmesh.common.config.connector.pravega; -import org.apache.eventmesh.openconnect.api.config.SinkConfig; +import org.apache.eventmesh.common.config.connector.SinkConfig; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/source/config/PravegaSourceConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/pravega/PravegaSourceConfig.java similarity index 88% rename from eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/source/config/PravegaSourceConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/pravega/PravegaSourceConfig.java index 87fbd3d573..105d3474ee 100644 --- a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/source/config/PravegaSourceConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/pravega/PravegaSourceConfig.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.pravega.source.config; +package org.apache.eventmesh.common.config.connector.pravega; -import org.apache.eventmesh.openconnect.api.config.SourceConfig; +import org.apache.eventmesh.common.config.connector.SourceConfig; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/sink/config/SinkConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/pravega/SinkConnectorConfig.java similarity index 95% rename from eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/sink/config/SinkConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/pravega/SinkConnectorConfig.java index c254dee3b8..7f56ea57a8 100644 --- a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/sink/config/SinkConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/pravega/SinkConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.pravega.sink.config; +package org.apache.eventmesh.common.config.connector.pravega; import java.net.URI; diff --git a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/source/config/SourceConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/pravega/SourceConnectorConfig.java similarity index 95% rename from eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/source/config/SourceConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/pravega/SourceConnectorConfig.java index 1ff54d1c1c..da0f8c5f13 100644 --- a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/source/config/SourceConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/pravega/SourceConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.pravega.source.config; +package org.apache.eventmesh.common.config.connector.pravega; import java.net.URI; diff --git a/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/source/config/PrometheusSourceConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/prometheus/PrometheusSourceConfig.java similarity index 88% rename from eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/source/config/PrometheusSourceConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/prometheus/PrometheusSourceConfig.java index 292b7e62f1..3393e4a193 100644 --- a/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/source/config/PrometheusSourceConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/prometheus/PrometheusSourceConfig.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.prometheus.source.config; +package org.apache.eventmesh.common.config.connector.prometheus; -import org.apache.eventmesh.openconnect.api.config.SourceConfig; +import org.apache.eventmesh.common.config.connector.SourceConfig; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/source/config/SourceConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/prometheus/SourceConnectorConfig.java similarity index 94% rename from eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/source/config/SourceConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/prometheus/SourceConnectorConfig.java index 7e8b7ba93e..1df3fe18cb 100644 --- a/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/source/config/SourceConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/prometheus/SourceConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.prometheus.source.config; +package org.apache.eventmesh.common.config.connector.prometheus; import lombok.Data; diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/JdbcConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/JdbcConfig.java new file mode 100644 index 0000000000..fc784fc187 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/JdbcConfig.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.config.connector.rdb; + +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; + +import java.util.Set; + +import lombok.Data; + +@Data +public class JdbcConfig { + private String url; + + private String dbAddress; + + private int dbPort; + + private String userName; + + private String passWord; + + private Set databases; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalMySQLType.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalMySQLType.java new file mode 100644 index 0000000000..b5107ccbf3 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalMySQLType.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.config.connector.rdb.canal; + +import java.util.HashMap; +import java.util.Map; + +import com.mysql.cj.MysqlType; + +public enum CanalMySQLType { + BIT("BIT"), + TINYINT("TINYINT"), + SMALLINT("SMALLINT"), + MEDIUMINT("MEDIUMINT"), + INT("INT"), + BIGINT("BIGINT"), + DECIMAL("DECIMAL"), + FLOAT("FLOAT"), + DOUBLE("DOUBLE"), + DATE("DATE"), + DATETIME("DATETIME"), + TIMESTAMP("TIMESTAMP"), + TIME("TIME"), + YEAR("YEAR"), + CHAR("CHAR"), + VARCHAR("VARCHAR"), + BINARY("BINARY"), + VARBINARY("VARBINARY"), + TINYBLOB("TINYBLOB"), + BLOB("BLOB"), + MEDIUMBLOB("MEDIUMBLOB"), + LONGBLOB("LONGBLOB"), + TINYTEXT("TINYTEXT"), + TEXT("TEXT"), + MEDIUMTEXT("MEDIUMTEXT"), + LONGTEXT("LONGTEXT"), + ENUM("ENUM"), + SET("SET"), + JSON("JSON"), + GEOMETRY("GEOMETRY"), + // MysqlType not include the following type + POINT("POINT"), + LINESTRING("LINESTRING"), + POLYGON("POLYGON"), + MULTIPOINT("MULTIPOINT"), + GEOMETRY_COLLECTION("GEOMETRYCOLLECTION"), + GEOM_COLLECTION("GEOMCOLLECTION"), + MULTILINESTRING("MULTILINESTRING"), + MULTIPOLYGON("MULTIPOLYGON"); + + private final String codeKey; + private final MysqlType mysqlType; + + CanalMySQLType(String codeKey) { + this.codeKey = codeKey; + this.mysqlType = MysqlType.getByName(codeKey); + } + + private static final Map TYPES = new HashMap<>(); + + static { + CanalMySQLType[] values = values(); + for (CanalMySQLType tableType : values) { + TYPES.put(tableType.codeKey, tableType); + } + } + + public String genPrepareStatement4Insert() { + switch (this) { + case GEOMETRY: + case GEOM_COLLECTION: + case GEOMETRY_COLLECTION: + return "ST_GEOMFROMTEXT(?)"; + case POINT: + return "ST_PointFromText(?)"; + case LINESTRING: + return "ST_LineStringFromText(?)"; + case POLYGON: + return "ST_PolygonFromText(?)"; + case MULTIPOINT: + return "ST_MultiPointFromText(?)"; + case MULTILINESTRING: + return "ST_MultiLineStringFromText(?)"; + case MULTIPOLYGON: + return "ST_MultiPolygonFromText(?)"; + default: + return "?"; + } + } + + public static CanalMySQLType valueOfCode(String code) { + CanalMySQLType type = TYPES.get(code.toUpperCase()); + if (type != null) { + return type; + } + switch (MysqlType.getByName(code)) { + case BOOLEAN: + case TINYINT: + case TINYINT_UNSIGNED: + return TINYINT; + case SMALLINT: + case SMALLINT_UNSIGNED: + return SMALLINT; + case INT: + case INT_UNSIGNED: + return INT; + case BIGINT: + case BIGINT_UNSIGNED: + return BIGINT; + case MEDIUMINT: + case MEDIUMINT_UNSIGNED: + return MEDIUMINT; + case DECIMAL: + case DECIMAL_UNSIGNED: + return DECIMAL; + case FLOAT: + case FLOAT_UNSIGNED: + return FLOAT; + case DOUBLE: + case DOUBLE_UNSIGNED: + return DOUBLE; + case BIT: + return BIT; + case BINARY: + return BINARY; + case VARBINARY: + return VARBINARY; + case TINYBLOB: + return TINYBLOB; + case MEDIUMBLOB: + return MEDIUMBLOB; + case LONGBLOB: + return LONGBLOB; + case BLOB: + return BLOB; + case CHAR: + return CHAR; + case VARCHAR: + return VARCHAR; + case TINYTEXT: + return TINYTEXT; + case MEDIUMTEXT: + return MEDIUMTEXT; + case LONGTEXT: + return LONGTEXT; + case TEXT: + return TEXT; + case DATE: + return DATE; + case TIME: + return TIME; + case TIMESTAMP: + return TIMESTAMP; + case DATETIME: + return DATETIME; + case YEAR: + return YEAR; + case JSON: + return JSON; + case ENUM: + return ENUM; + case SET: + return SET; + case GEOMETRY: + return GEOMETRY; + case NULL: + case UNKNOWN: + default: + throw new UnsupportedOperationException("Unsupported mysql columnType " + code); + } + } + + public MysqlType getMysqlType() { + return mysqlType; + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkConfig.java new file mode 100644 index 0000000000..c535c7f52a --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkConfig.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.config.connector.rdb.canal; + +import org.apache.eventmesh.common.config.connector.SinkConfig; + +import java.util.Map; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class CanalSinkConfig extends SinkConfig { + + // used to convert canal full/increment/check connector config + private Map sinkConfig; + +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkFullConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkFullConfig.java new file mode 100644 index 0000000000..dca16b100c --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkFullConfig.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.config.connector.rdb.canal; + +import org.apache.eventmesh.common.config.connector.SinkConfig; + +import lombok.Data; +import lombok.EqualsAndHashCode; + + +@Data +@EqualsAndHashCode(callSuper = true) +public class CanalSinkFullConfig extends SinkConfig { + private SinkConnectorConfig sinkConnectorConfig; + private String zeroDate; + private int parallel = 2; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkIncrementConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkIncrementConfig.java new file mode 100644 index 0000000000..aeb9d5a0e2 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSinkIncrementConfig.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.config.connector.rdb.canal; + +import org.apache.eventmesh.common.remote.job.SyncMode; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class CanalSinkIncrementConfig extends CanalSinkConfig { + + // batchSize + private Integer batchSize = 50; + + // enable batch + private Boolean useBatch = true; + + // sink thread size for single channel + private Integer poolSize = 5; + + // sync mode: field/row + private SyncMode syncMode = SyncMode.ROW; + + private boolean isGTIDMode = false; + + private boolean isMariaDB = true; + + // skip sink process exception + private Boolean skipException = false; + + public SinkConnectorConfig sinkConnectorConfig; + +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceCheckConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceCheckConfig.java new file mode 100644 index 0000000000..f326301d7d --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceCheckConfig.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.config.connector.rdb.canal; + +import org.apache.eventmesh.common.config.connector.SourceConfig; +import org.apache.eventmesh.common.remote.offset.RecordPosition; + +import java.util.List; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class CanalSourceCheckConfig extends SourceConfig { + private SourceConnectorConfig sourceConnectorConfig; + private List startPosition; + private int parallel; + private int flushSize; + private int executePeriod = 3600; + private Integer pagePerSecond = 1; + private Integer recordPerSecond = 100; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceConfig.java new file mode 100644 index 0000000000..db17fbe75d --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceConfig.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.config.connector.rdb.canal; + +import org.apache.eventmesh.common.config.connector.SourceConfig; + +import java.util.Map; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class CanalSourceConfig extends SourceConfig { + + // used to convert canal full/increment/check connector config + private Map sourceConfig; + +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceFullConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceFullConfig.java new file mode 100644 index 0000000000..53988ca055 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceFullConfig.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.config.connector.rdb.canal; + +import org.apache.eventmesh.common.config.connector.SourceConfig; +import org.apache.eventmesh.common.remote.offset.RecordPosition; + +import java.util.List; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class CanalSourceFullConfig extends SourceConfig { + private SourceConnectorConfig sourceConnectorConfig; + private List startPosition; + private int parallel = 2; + private int flushSize = 20; + private Integer pagePerSecond = 1; + private Integer recordPerSecond = 100; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceIncrementConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceIncrementConfig.java new file mode 100644 index 0000000000..7f73727140 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/CanalSourceIncrementConfig.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.config.connector.rdb.canal; + +import org.apache.eventmesh.common.remote.job.SyncConsistency; +import org.apache.eventmesh.common.remote.job.SyncMode; +import org.apache.eventmesh.common.remote.offset.RecordPosition; + +import java.util.List; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class CanalSourceIncrementConfig extends CanalSourceConfig { + + private String destination; + + private Long canalInstanceId = 1L; + + private String desc = "canalSourceInstance"; + + private boolean ddlSync = false; + + private boolean filterTableError = false; + + private Long slaveId; + + private Short clientId = 1; + + private String serverUUID; + + private boolean isMariaDB = true; + + private boolean isGTIDMode = true; + + private Integer batchSize = 10000; + + private Long batchTimeout = -1L; + + private String tableFilter; + + private String fieldFilter; + + private List recordPositions; + + // ================================= channel parameter + // ================================ + + // enable remedy + private Boolean enableRemedy = false; + + // sync mode: field/row + private SyncMode syncMode = SyncMode.ROW; + + // sync consistency + private SyncConsistency syncConsistency = SyncConsistency.BASE; + + // ================================= system parameter + // ================================ + + // Column name of the bidirectional synchronization mark + private String needSyncMarkTableColumnName; + + // Column value of the bidirectional synchronization mark + private String needSyncMarkTableColumnValue; + + private SourceConnectorConfig sourceConnectorConfig; + +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/JobRdbFullPosition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/JobRdbFullPosition.java new file mode 100644 index 0000000000..42ba889bbd --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/JobRdbFullPosition.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.config.connector.rdb.canal; + +import java.math.BigDecimal; + +import lombok.Data; +import lombok.ToString; + +@Data +@ToString +public class JobRdbFullPosition { + private String jobId; + private String schema; + private String tableName; + private String primaryKeyRecords; + private long maxCount; + private long handledRecordCount = 0; + private boolean finished; + private BigDecimal percent; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbColumnDefinition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbColumnDefinition.java new file mode 100644 index 0000000000..94c0135c3e --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbColumnDefinition.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.config.connector.rdb.canal; + +import java.sql.JDBCType; + +import lombok.Data; + +@Data +public class RdbColumnDefinition { + protected String name; + protected JDBCType jdbcType; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbDBDefinition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbDBDefinition.java new file mode 100644 index 0000000000..ab3ed336f8 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbDBDefinition.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.config.connector.rdb.canal; + +import java.util.Set; + +import lombok.Data; + +/** + * Description: as class name + */ +@Data +public class RdbDBDefinition { + private String schemaName; + private Set tables; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableDefinition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableDefinition.java new file mode 100644 index 0000000000..c281035578 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/RdbTableDefinition.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.config.connector.rdb.canal; + +import lombok.Data; + +/** + * Description: as class name + */ +@Data +public class RdbTableDefinition { + protected String schemaName; + protected String tableName; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/SinkConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/SinkConnectorConfig.java new file mode 100644 index 0000000000..761cdba4bb --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/SinkConnectorConfig.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.config.connector.rdb.canal; + +import org.apache.eventmesh.common.config.connector.rdb.JdbcConfig; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * Configuration parameters for a sink connector. + */ +@Data +@EqualsAndHashCode(callSuper = true) +public class SinkConnectorConfig extends JdbcConfig { + private String connectorName; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/SourceConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/SourceConnectorConfig.java new file mode 100644 index 0000000000..9a95696a0d --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/SourceConnectorConfig.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.config.connector.rdb.canal; + +import org.apache.eventmesh.common.config.connector.rdb.JdbcConfig; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * Represents the configuration for a database connector. + */ +@Data +@EqualsAndHashCode(callSuper = true) +public class SourceConnectorConfig extends JdbcConfig { + private String connectorName; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/Constants.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/Constants.java new file mode 100644 index 0000000000..8c51c7255b --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/Constants.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.config.connector.rdb.canal.mysql; + +public class Constants { + public static final String MySQLQuot = "`"; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MySQLColumnDef.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MySQLColumnDef.java new file mode 100644 index 0000000000..cdc9adf33f --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MySQLColumnDef.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.config.connector.rdb.canal.mysql; + +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalMySQLType; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbColumnDefinition; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class MySQLColumnDef extends RdbColumnDefinition { + private CanalMySQLType type; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MySQLTableDef.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MySQLTableDef.java new file mode 100644 index 0000000000..4266a96060 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/canal/mysql/MySQLTableDef.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.config.connector.rdb.canal.mysql; + +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; + +import java.util.List; +import java.util.Map; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * Description: + */ +@Data +@EqualsAndHashCode(callSuper = true) +public class MySQLTableDef extends RdbTableDefinition { + private List primaryKeys; + private Map columnDefinitions; +} diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/config/JdbcConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/jdbc/JdbcConfig.java similarity index 96% rename from eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/config/JdbcConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/jdbc/JdbcConfig.java index d40801854c..1b46a76c99 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/config/JdbcConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/jdbc/JdbcConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.jdbc.config; +package org.apache.eventmesh.common.config.connector.rdb.jdbc; import java.util.Properties; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/config/JdbcSinkConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/jdbc/JdbcSinkConfig.java similarity index 89% rename from eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/config/JdbcSinkConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/jdbc/JdbcSinkConfig.java index 2a5af32cad..83711c68be 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/config/JdbcSinkConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/jdbc/JdbcSinkConfig.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.jdbc.sink.config; +package org.apache.eventmesh.common.config.connector.rdb.jdbc; -import org.apache.eventmesh.openconnect.api.config.SinkConfig; +import org.apache.eventmesh.common.config.connector.SinkConfig; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/config/JdbcSourceConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/jdbc/JdbcSourceConfig.java similarity index 88% rename from eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/config/JdbcSourceConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/jdbc/JdbcSourceConfig.java index b330c331bf..553a0581a2 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/config/JdbcSourceConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/jdbc/JdbcSourceConfig.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.jdbc.source.config; +package org.apache.eventmesh.common.config.connector.rdb.jdbc; -import org.apache.eventmesh.openconnect.api.config.SourceConfig; +import org.apache.eventmesh.common.config.connector.SourceConfig; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/config/MysqlConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/jdbc/MysqlConfig.java similarity index 95% rename from eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/config/MysqlConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/jdbc/MysqlConfig.java index 032921350f..ede507e1be 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/config/MysqlConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/jdbc/MysqlConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.jdbc.source.config; +package org.apache.eventmesh.common.config.connector.rdb.jdbc; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/config/SinkConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/jdbc/SinkConnectorConfig.java similarity index 90% rename from eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/config/SinkConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/jdbc/SinkConnectorConfig.java index e971b7b000..afbd51d143 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/config/SinkConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/jdbc/SinkConnectorConfig.java @@ -15,9 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.jdbc.sink.config; - -import org.apache.eventmesh.connector.jdbc.config.JdbcConfig; +package org.apache.eventmesh.common.config.connector.rdb.jdbc; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/config/SourceConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/jdbc/SourceConnectorConfig.java similarity index 95% rename from eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/config/SourceConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/jdbc/SourceConnectorConfig.java index a27dbfedb4..a10d8e09d0 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/config/SourceConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/jdbc/SourceConnectorConfig.java @@ -15,9 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.jdbc.source.config; - -import org.apache.eventmesh.connector.jdbc.config.JdbcConfig; +package org.apache.eventmesh.common.config.connector.rdb.jdbc; import java.util.List; diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/config/MongodbSinkConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/mongodb/MongodbSinkConfig.java similarity index 82% rename from eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/config/MongodbSinkConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/mongodb/MongodbSinkConfig.java index 6df203bd6a..7d019ba1ff 100644 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/config/MongodbSinkConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/mongodb/MongodbSinkConfig.java @@ -15,12 +15,14 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.mongodb.sink.config; +package org.apache.eventmesh.common.config.connector.rdb.mongodb; -import org.apache.eventmesh.openconnect.api.config.SinkConfig; +import org.apache.eventmesh.common.config.connector.SinkConfig; import lombok.Data; +import lombok.EqualsAndHashCode; +@EqualsAndHashCode(callSuper = true) @Data public class MongodbSinkConfig extends SinkConfig { diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/config/MongodbSourceConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/mongodb/MongodbSourceConfig.java similarity index 82% rename from eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/config/MongodbSourceConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/mongodb/MongodbSourceConfig.java index d67ad6c277..00dca10b29 100644 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/config/MongodbSourceConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/mongodb/MongodbSourceConfig.java @@ -15,12 +15,14 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.mongodb.source.config; +package org.apache.eventmesh.common.config.connector.rdb.mongodb; -import org.apache.eventmesh.openconnect.api.config.SourceConfig; +import org.apache.eventmesh.common.config.connector.SourceConfig; import lombok.Data; +import lombok.EqualsAndHashCode; +@EqualsAndHashCode(callSuper = true) @Data public class MongodbSourceConfig extends SourceConfig { diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/config/SinkConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/mongodb/SinkConnectorConfig.java similarity index 93% rename from eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/config/SinkConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/mongodb/SinkConnectorConfig.java index 087b2e48ce..58ae5ffc42 100644 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/config/SinkConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/mongodb/SinkConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.mongodb.sink.config; +package org.apache.eventmesh.common.config.connector.rdb.mongodb; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/config/SourceConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/mongodb/SourceConnectorConfig.java similarity index 93% rename from eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/config/SourceConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/mongodb/SourceConnectorConfig.java index 5c48645a43..8ace602f80 100644 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/config/SourceConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/rdb/mongodb/SourceConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.mongodb.source.config; +package org.apache.eventmesh.common.config.connector.rdb.mongodb; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/sink/config/RedisSinkConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/redis/RedisSinkConfig.java similarity index 82% rename from eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/sink/config/RedisSinkConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/redis/RedisSinkConfig.java index 62cddbf312..27070343d4 100644 --- a/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/sink/config/RedisSinkConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/redis/RedisSinkConfig.java @@ -15,12 +15,14 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.redis.sink.config; +package org.apache.eventmesh.common.config.connector.redis; -import org.apache.eventmesh.openconnect.api.config.SinkConfig; +import org.apache.eventmesh.common.config.connector.SinkConfig; import lombok.Data; +import lombok.EqualsAndHashCode; +@EqualsAndHashCode(callSuper = true) @Data public class RedisSinkConfig extends SinkConfig { diff --git a/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/source/config/RedisSourceConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/redis/RedisSourceConfig.java similarity index 82% rename from eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/source/config/RedisSourceConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/redis/RedisSourceConfig.java index f01a556c9f..5b04e6a820 100644 --- a/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/source/config/RedisSourceConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/redis/RedisSourceConfig.java @@ -15,12 +15,14 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.redis.source.config; +package org.apache.eventmesh.common.config.connector.redis; -import org.apache.eventmesh.openconnect.api.config.SourceConfig; +import org.apache.eventmesh.common.config.connector.SourceConfig; import lombok.Data; +import lombok.EqualsAndHashCode; +@EqualsAndHashCode(callSuper = true) @Data public class RedisSourceConfig extends SourceConfig { diff --git a/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/sink/config/SinkConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/redis/SinkConnectorConfig.java similarity index 94% rename from eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/sink/config/SinkConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/redis/SinkConnectorConfig.java index 34fb0326a4..ebf53adc3b 100644 --- a/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/sink/config/SinkConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/redis/SinkConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.redis.sink.config; +package org.apache.eventmesh.common.config.connector.redis; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/source/config/SourceConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/redis/SourceConnectorConfig.java similarity index 94% rename from eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/source/config/SourceConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/redis/SourceConnectorConfig.java index 9913e16320..b363924ecb 100644 --- a/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/source/config/SourceConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/redis/SourceConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.redis.source.config; +package org.apache.eventmesh.common.config.connector.redis; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/source/config/S3SourceConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/s3/S3SourceConfig.java similarity index 83% rename from eventmesh-connectors/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/source/config/S3SourceConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/s3/S3SourceConfig.java index 0f4bec5d56..7691b6e235 100644 --- a/eventmesh-connectors/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/source/config/S3SourceConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/s3/S3SourceConfig.java @@ -15,12 +15,14 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.s3.source.config; +package org.apache.eventmesh.common.config.connector.s3; -import org.apache.eventmesh.openconnect.api.config.SourceConfig; +import org.apache.eventmesh.common.config.connector.SourceConfig; import lombok.Data; +import lombok.EqualsAndHashCode; +@EqualsAndHashCode(callSuper = true) @Data public class S3SourceConfig extends SourceConfig { diff --git a/eventmesh-connectors/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/source/config/SourceConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/s3/SourceConnectorConfig.java similarity index 96% rename from eventmesh-connectors/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/source/config/SourceConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/s3/SourceConnectorConfig.java index 2c9cb5a80c..fdc4531255 100644 --- a/eventmesh-connectors/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/source/config/SourceConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/s3/SourceConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.s3.source.config; +package org.apache.eventmesh.common.config.connector.s3; import java.util.Map; diff --git a/eventmesh-connectors/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/sink/config/SinkConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/slack/SinkConnectorConfig.java similarity index 94% rename from eventmesh-connectors/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/sink/config/SinkConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/slack/SinkConnectorConfig.java index 41884a94a1..4267beddff 100644 --- a/eventmesh-connectors/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/sink/config/SinkConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/slack/SinkConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.slack.sink.config; +package org.apache.eventmesh.common.config.connector.slack; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/sink/config/SlackSinkConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/slack/SlackSinkConfig.java similarity index 89% rename from eventmesh-connectors/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/sink/config/SlackSinkConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/slack/SlackSinkConfig.java index 016cd9ae97..969306ae6e 100644 --- a/eventmesh-connectors/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/sink/config/SlackSinkConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/slack/SlackSinkConfig.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.slack.sink.config; +package org.apache.eventmesh.common.config.connector.slack; -import org.apache.eventmesh.openconnect.api.config.SinkConfig; +import org.apache.eventmesh.common.config.connector.SinkConfig; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/connector/SinkConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/spring/SinkConnectorConfig.java similarity index 93% rename from eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/connector/SinkConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/spring/SinkConnectorConfig.java index a71c4ab66c..0ded9886dc 100644 --- a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/connector/SinkConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/spring/SinkConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.spring.sink.connector; +package org.apache.eventmesh.common.config.connector.spring; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/connector/SourceConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/spring/SourceConnectorConfig.java similarity index 93% rename from eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/connector/SourceConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/spring/SourceConnectorConfig.java index 242869087f..c31f8c9885 100644 --- a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/connector/SourceConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/spring/SourceConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.spring.source.connector; +package org.apache.eventmesh.common.config.connector.spring; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/config/SpringSinkConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/spring/SpringSinkConfig.java similarity index 83% rename from eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/config/SpringSinkConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/spring/SpringSinkConfig.java index aff0d8a0f3..14f1f4bbe5 100644 --- a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/config/SpringSinkConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/spring/SpringSinkConfig.java @@ -15,10 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.spring.sink.config; +package org.apache.eventmesh.common.config.connector.spring; -import org.apache.eventmesh.connector.spring.sink.connector.SinkConnectorConfig; -import org.apache.eventmesh.openconnect.api.config.SinkConfig; +import org.apache.eventmesh.common.config.connector.SinkConfig; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/config/SpringSourceConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/spring/SpringSourceConfig.java similarity index 82% rename from eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/config/SpringSourceConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/spring/SpringSourceConfig.java index 191e9b1196..e99dfc594c 100644 --- a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/config/SpringSourceConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/spring/SpringSourceConfig.java @@ -15,10 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.spring.source.config; +package org.apache.eventmesh.common.config.connector.spring; -import org.apache.eventmesh.connector.spring.source.connector.SourceConnectorConfig; -import org.apache.eventmesh.openconnect.api.config.SourceConfig; +import org.apache.eventmesh.common.config.connector.SourceConfig; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/sink/config/SinkConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/wechat/SinkConnectorConfig.java similarity index 94% rename from eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/sink/config/SinkConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/wechat/SinkConnectorConfig.java index e575e65c67..95ca5e64c2 100644 --- a/eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/sink/config/SinkConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/wechat/SinkConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.wechat.sink.config; +package org.apache.eventmesh.common.config.connector.wechat; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/sink/config/WeChatSinkConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/wechat/WeChatSinkConfig.java similarity index 88% rename from eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/sink/config/WeChatSinkConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/wechat/WeChatSinkConfig.java index a77e1c77b5..e968202c4c 100644 --- a/eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/sink/config/WeChatSinkConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/wechat/WeChatSinkConfig.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.wechat.sink.config; +package org.apache.eventmesh.common.config.connector.wechat; -import org.apache.eventmesh.openconnect.api.config.SinkConfig; +import org.apache.eventmesh.common.config.connector.SinkConfig; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/sink/config/SinkConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/wecom/SinkConnectorConfig.java similarity index 94% rename from eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/sink/config/SinkConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/wecom/SinkConnectorConfig.java index 013d5a8bb7..b32b5b7d36 100644 --- a/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/sink/config/SinkConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/wecom/SinkConnectorConfig.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.wecom.sink.config; +package org.apache.eventmesh.common.config.connector.wecom; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/sink/config/WeComSinkConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/wecom/WeComSinkConfig.java similarity index 89% rename from eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/sink/config/WeComSinkConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/wecom/WeComSinkConfig.java index 8af43bdbe8..5177baa448 100644 --- a/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/sink/config/WeComSinkConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/config/connector/wecom/WeComSinkConfig.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.wecom.sink.config; +package org.apache.eventmesh.common.config.connector.wecom; -import org.apache.eventmesh.openconnect.api.config.SinkConfig; +import org.apache.eventmesh.common.config.connector.SinkConfig; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/enums/ComponentType.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/enums/ComponentType.java new file mode 100644 index 0000000000..a48e3fe013 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/enums/ComponentType.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.enums; + +public enum ComponentType { + CONNECTOR("connector"), + FUNCTION("function"), + MESH("mesh"); + + public String name; + + ComponentType(String name) { + this.name = name; + } + + public String componentTypeName() { + return this.name; + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/enums/ConnectorStage.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/enums/ConnectorStage.java new file mode 100644 index 0000000000..90265fba4a --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/enums/ConnectorStage.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.enums; + +public enum ConnectorStage { + SOURCE, + SINK +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/grpc/adminserver/AdminServiceGrpc.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/grpc/adminserver/AdminServiceGrpc.java new file mode 100644 index 0000000000..df5f3ef338 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/grpc/adminserver/AdminServiceGrpc.java @@ -0,0 +1,373 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.protocol.grpc.adminserver; + +import static io.grpc.MethodDescriptor.generateFullMethodName; + +/** + * AdminServiceGrpc + */ +@javax.annotation.Generated( + value = "by gRPC proto compiler (version 1.40.0)", + comments = "Source: event_mesh_admin_service.proto") +@io.grpc.stub.annotations.GrpcGenerated +public final class AdminServiceGrpc { + + private AdminServiceGrpc() { + } + + public static final String SERVICE_NAME = "AdminService"; + + // Static method descriptors that strictly reflect the proto. + private static volatile io.grpc.MethodDescriptor getInvokeBiStreamMethod; + + @io.grpc.stub.annotations.RpcMethod( + fullMethodName = SERVICE_NAME + '/' + "invokeBiStream", + requestType = Payload.class, + responseType = Payload.class, + methodType = io.grpc.MethodDescriptor.MethodType.BIDI_STREAMING) + public static io.grpc.MethodDescriptor getInvokeBiStreamMethod() { + io.grpc.MethodDescriptor getInvokeBiStreamMethod; + if ((getInvokeBiStreamMethod = AdminServiceGrpc.getInvokeBiStreamMethod) == null) { + synchronized (AdminServiceGrpc.class) { + if ((getInvokeBiStreamMethod = AdminServiceGrpc.getInvokeBiStreamMethod) == null) { + AdminServiceGrpc.getInvokeBiStreamMethod = getInvokeBiStreamMethod = + io.grpc.MethodDescriptor.newBuilder() + .setType(io.grpc.MethodDescriptor.MethodType.BIDI_STREAMING) + .setFullMethodName(generateFullMethodName(SERVICE_NAME, "invokeBiStream")) + .setSampledToLocalTracing(true) + .setRequestMarshaller(io.grpc.protobuf.ProtoUtils.marshaller( + Payload.getDefaultInstance())) + .setResponseMarshaller(io.grpc.protobuf.ProtoUtils.marshaller( + Payload.getDefaultInstance())) + .setSchemaDescriptor(new AdminServiceMethodDescriptorSupplier("invokeBiStream")) + .build(); + } + } + } + return getInvokeBiStreamMethod; + } + + private static volatile io.grpc.MethodDescriptor getInvokeMethod; + + @io.grpc.stub.annotations.RpcMethod( + fullMethodName = SERVICE_NAME + '/' + "invoke", + requestType = Payload.class, + responseType = Payload.class, + methodType = io.grpc.MethodDescriptor.MethodType.UNARY) + public static io.grpc.MethodDescriptor getInvokeMethod() { + io.grpc.MethodDescriptor getInvokeMethod; + if ((getInvokeMethod = AdminServiceGrpc.getInvokeMethod) == null) { + synchronized (AdminServiceGrpc.class) { + if ((getInvokeMethod = AdminServiceGrpc.getInvokeMethod) == null) { + AdminServiceGrpc.getInvokeMethod = getInvokeMethod = + io.grpc.MethodDescriptor.newBuilder() + .setType(io.grpc.MethodDescriptor.MethodType.UNARY) + .setFullMethodName(generateFullMethodName(SERVICE_NAME, "invoke")) + .setSampledToLocalTracing(true) + .setRequestMarshaller(io.grpc.protobuf.ProtoUtils.marshaller( + Payload.getDefaultInstance())) + .setResponseMarshaller(io.grpc.protobuf.ProtoUtils.marshaller( + Payload.getDefaultInstance())) + .setSchemaDescriptor(new AdminServiceMethodDescriptorSupplier("invoke")) + .build(); + } + } + } + return getInvokeMethod; + } + + /** + * Creates a new async stub that supports all call types for the service + */ + public static AdminServiceStub newStub(io.grpc.Channel channel) { + io.grpc.stub.AbstractStub.StubFactory factory = + new io.grpc.stub.AbstractStub.StubFactory() { + @Override + public AdminServiceStub newStub(io.grpc.Channel channel, io.grpc.CallOptions callOptions) { + return new AdminServiceStub(channel, callOptions); + } + }; + return AdminServiceStub.newStub(factory, channel); + } + + /** + * Creates a new blocking-style stub that supports unary and streaming output calls on the service + */ + public static AdminServiceBlockingStub newBlockingStub( + io.grpc.Channel channel) { + io.grpc.stub.AbstractStub.StubFactory factory = + new io.grpc.stub.AbstractStub.StubFactory() { + @Override + public AdminServiceBlockingStub newStub(io.grpc.Channel channel, io.grpc.CallOptions callOptions) { + return new AdminServiceBlockingStub(channel, callOptions); + } + }; + return AdminServiceBlockingStub.newStub(factory, channel); + } + + /** + * Creates a new ListenableFuture-style stub that supports unary calls on the service + */ + public static AdminServiceFutureStub newFutureStub( + io.grpc.Channel channel) { + io.grpc.stub.AbstractStub.StubFactory factory = + new io.grpc.stub.AbstractStub.StubFactory() { + @Override + public AdminServiceFutureStub newStub(io.grpc.Channel channel, io.grpc.CallOptions callOptions) { + return new AdminServiceFutureStub(channel, callOptions); + } + }; + return AdminServiceFutureStub.newStub(factory, channel); + } + + /** + * + */ + public static abstract class AdminServiceImplBase implements io.grpc.BindableService { + + /** + * + */ + public io.grpc.stub.StreamObserver invokeBiStream( + io.grpc.stub.StreamObserver responseObserver) { + return io.grpc.stub.ServerCalls.asyncUnimplementedStreamingCall(getInvokeBiStreamMethod(), responseObserver); + } + + /** + * + */ + public void invoke(Payload request, + io.grpc.stub.StreamObserver responseObserver) { + io.grpc.stub.ServerCalls.asyncUnimplementedUnaryCall(getInvokeMethod(), responseObserver); + } + + @Override + public final io.grpc.ServerServiceDefinition bindService() { + return io.grpc.ServerServiceDefinition.builder(getServiceDescriptor()) + .addMethod( + getInvokeBiStreamMethod(), + io.grpc.stub.ServerCalls.asyncBidiStreamingCall( + new MethodHandlers< + Payload, + Payload>( + this, METHODID_INVOKE_BI_STREAM))) + .addMethod( + getInvokeMethod(), + io.grpc.stub.ServerCalls.asyncUnaryCall( + new MethodHandlers< + Payload, + Payload>( + this, METHODID_INVOKE))) + .build(); + } + } + + /** + * + */ + public static final class AdminServiceStub extends io.grpc.stub.AbstractAsyncStub { + + private AdminServiceStub( + io.grpc.Channel channel, io.grpc.CallOptions callOptions) { + super(channel, callOptions); + } + + @Override + protected AdminServiceStub build( + io.grpc.Channel channel, io.grpc.CallOptions callOptions) { + return new AdminServiceStub(channel, callOptions); + } + + /** + * + */ + public io.grpc.stub.StreamObserver invokeBiStream( + io.grpc.stub.StreamObserver responseObserver) { + return io.grpc.stub.ClientCalls.asyncBidiStreamingCall( + getChannel().newCall(getInvokeBiStreamMethod(), getCallOptions()), responseObserver); + } + + /** + * + */ + public void invoke(Payload request, + io.grpc.stub.StreamObserver responseObserver) { + io.grpc.stub.ClientCalls.asyncUnaryCall( + getChannel().newCall(getInvokeMethod(), getCallOptions()), request, responseObserver); + } + } + + /** + * + */ + public static final class AdminServiceBlockingStub extends io.grpc.stub.AbstractBlockingStub { + + private AdminServiceBlockingStub( + io.grpc.Channel channel, io.grpc.CallOptions callOptions) { + super(channel, callOptions); + } + + @Override + protected AdminServiceBlockingStub build( + io.grpc.Channel channel, io.grpc.CallOptions callOptions) { + return new AdminServiceBlockingStub(channel, callOptions); + } + + /** + * + */ + public Payload invoke(Payload request) { + return io.grpc.stub.ClientCalls.blockingUnaryCall( + getChannel(), getInvokeMethod(), getCallOptions(), request); + } + } + + /** + * + */ + public static final class AdminServiceFutureStub extends io.grpc.stub.AbstractFutureStub { + + private AdminServiceFutureStub( + io.grpc.Channel channel, io.grpc.CallOptions callOptions) { + super(channel, callOptions); + } + + @Override + protected AdminServiceFutureStub build( + io.grpc.Channel channel, io.grpc.CallOptions callOptions) { + return new AdminServiceFutureStub(channel, callOptions); + } + + /** + * + */ + public com.google.common.util.concurrent.ListenableFuture invoke( + Payload request) { + return io.grpc.stub.ClientCalls.futureUnaryCall( + getChannel().newCall(getInvokeMethod(), getCallOptions()), request); + } + } + + private static final int METHODID_INVOKE = 0; + private static final int METHODID_INVOKE_BI_STREAM = 1; + + private static final class MethodHandlers implements + io.grpc.stub.ServerCalls.UnaryMethod, + io.grpc.stub.ServerCalls.ServerStreamingMethod, + io.grpc.stub.ServerCalls.ClientStreamingMethod, + io.grpc.stub.ServerCalls.BidiStreamingMethod { + + private final AdminServiceImplBase serviceImpl; + private final int methodId; + + MethodHandlers(AdminServiceImplBase serviceImpl, int methodId) { + this.serviceImpl = serviceImpl; + this.methodId = methodId; + } + + @Override + @SuppressWarnings("unchecked") + public void invoke(Req request, io.grpc.stub.StreamObserver responseObserver) { + switch (methodId) { + case METHODID_INVOKE: + serviceImpl.invoke((Payload) request, + (io.grpc.stub.StreamObserver) responseObserver); + break; + default: + throw new AssertionError(); + } + } + + @Override + @SuppressWarnings("unchecked") + public io.grpc.stub.StreamObserver invoke( + io.grpc.stub.StreamObserver responseObserver) { + switch (methodId) { + case METHODID_INVOKE_BI_STREAM: + return (io.grpc.stub.StreamObserver) serviceImpl.invokeBiStream( + (io.grpc.stub.StreamObserver) responseObserver); + default: + throw new AssertionError(); + } + } + } + + private static abstract class AdminServiceBaseDescriptorSupplier + implements io.grpc.protobuf.ProtoFileDescriptorSupplier, io.grpc.protobuf.ProtoServiceDescriptorSupplier { + + AdminServiceBaseDescriptorSupplier() { + } + + @Override + public com.google.protobuf.Descriptors.FileDescriptor getFileDescriptor() { + return EventMeshAdminService.getDescriptor(); + } + + @Override + public com.google.protobuf.Descriptors.ServiceDescriptor getServiceDescriptor() { + return getFileDescriptor().findServiceByName("AdminService"); + } + } + + private static final class AdminServiceFileDescriptorSupplier + extends AdminServiceBaseDescriptorSupplier { + + AdminServiceFileDescriptorSupplier() { + } + } + + private static final class AdminServiceMethodDescriptorSupplier + extends AdminServiceBaseDescriptorSupplier + implements io.grpc.protobuf.ProtoMethodDescriptorSupplier { + + private final String methodName; + + AdminServiceMethodDescriptorSupplier(String methodName) { + this.methodName = methodName; + } + + @Override + public com.google.protobuf.Descriptors.MethodDescriptor getMethodDescriptor() { + return getServiceDescriptor().findMethodByName(methodName); + } + } + + private static volatile io.grpc.ServiceDescriptor serviceDescriptor; + + public static io.grpc.ServiceDescriptor getServiceDescriptor() { + io.grpc.ServiceDescriptor result = serviceDescriptor; + if (result == null) { + synchronized (AdminServiceGrpc.class) { + result = serviceDescriptor; + if (result == null) { + serviceDescriptor = result = io.grpc.ServiceDescriptor.newBuilder(SERVICE_NAME) + .setSchemaDescriptor(new AdminServiceFileDescriptorSupplier()) + .addMethod(getInvokeBiStreamMethod()) + .addMethod(getInvokeMethod()) + .build(); + } + } + } + return result; + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/grpc/adminserver/EventMeshAdminService.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/grpc/adminserver/EventMeshAdminService.java new file mode 100644 index 0000000000..d67ed4159a --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/grpc/adminserver/EventMeshAdminService.java @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.protocol.grpc.adminserver;// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: event_mesh_admin_service.proto + +public final class EventMeshAdminService { + + private EventMeshAdminService() { + } + + public static void registerAllExtensions( + com.google.protobuf.ExtensionRegistryLite registry) { + } + + public static void registerAllExtensions( + com.google.protobuf.ExtensionRegistry registry) { + registerAllExtensions( + (com.google.protobuf.ExtensionRegistryLite) registry); + } + + static final com.google.protobuf.Descriptors.Descriptor + internal_static_Metadata_descriptor; + static final + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internal_static_Metadata_fieldAccessorTable; + static final com.google.protobuf.Descriptors.Descriptor + internal_static_Metadata_HeadersEntry_descriptor; + static final + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internal_static_Metadata_HeadersEntry_fieldAccessorTable; + static final com.google.protobuf.Descriptors.Descriptor + internal_static_Payload_descriptor; + static final + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internal_static_Payload_fieldAccessorTable; + + public static com.google.protobuf.Descriptors.FileDescriptor + getDescriptor() { + return descriptor; + } + + private static com.google.protobuf.Descriptors.FileDescriptor + descriptor; + + static { + String[] descriptorData = { + "\n\036event_mesh_admin_service.proto\032\031google" + + "/protobuf/any.proto\"q\n\010Metadata\022\014\n\004type\030" + + "\003 \001(\t\022\'\n\007headers\030\007 \003(\0132\026.Metadata.Header" + + "sEntry\032.\n\014HeadersEntry\022\013\n\003key\030\001 \001(\t\022\r\n\005v" + + "alue\030\002 \001(\t:\0028\001\"4\n\007Payload\022\033\n\010metadata\030\002 " + + "\001(\0132\t.Metadata\022\014\n\004body\030\003 \001(\0142Z\n\014AdminSer" + + "vice\022*\n\016invokeBiStream\022\010.Payload\032\010.Paylo" + + "ad\"\000(\0010\001\022\036\n\006invoke\022\010.Payload\032\010.Payload\"\000" + + "B\002P\001b\006proto3" + }; + descriptor = com.google.protobuf.Descriptors.FileDescriptor + .internalBuildGeneratedFileFrom(descriptorData, + new com.google.protobuf.Descriptors.FileDescriptor[] { + com.google.protobuf.AnyProto.getDescriptor(), + }); + internal_static_Metadata_descriptor = + getDescriptor().getMessageTypes().get(0); + internal_static_Metadata_fieldAccessorTable = new + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_Metadata_descriptor, + new String[] {"Type", "Headers",}); + internal_static_Metadata_HeadersEntry_descriptor = + internal_static_Metadata_descriptor.getNestedTypes().get(0); + internal_static_Metadata_HeadersEntry_fieldAccessorTable = new + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_Metadata_HeadersEntry_descriptor, + new String[] {"Key", "Value",}); + internal_static_Payload_descriptor = + getDescriptor().getMessageTypes().get(1); + internal_static_Payload_fieldAccessorTable = new + com.google.protobuf.GeneratedMessageV3.FieldAccessorTable( + internal_static_Payload_descriptor, + new String[] {"Metadata", "Body",}); + com.google.protobuf.AnyProto.getDescriptor(); + } + + // @@protoc_insertion_point(outer_class_scope) +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/grpc/adminserver/Metadata.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/grpc/adminserver/Metadata.java new file mode 100644 index 0000000000..f25754f288 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/grpc/adminserver/Metadata.java @@ -0,0 +1,962 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.protocol.grpc.adminserver;// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: event_mesh_admin_service.proto + +/** + * Protobuf type {@code Metadata} + */ +public final class Metadata extends + com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:Metadata) + MetadataOrBuilder { + + private static final long serialVersionUID = 0L; + + // Use Metadata.newBuilder() to construct. + private Metadata(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private Metadata() { + type_ = ""; + } + + @Override + @SuppressWarnings({"unused"}) + protected Object newInstance( + UnusedPrivateParameter unused) { + return new Metadata(); + } + + @Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + + private Metadata( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new NullPointerException(); + } + int mutable_bitField0_ = 0; + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + case 26: { + String s = input.readStringRequireUtf8(); + + type_ = s; + break; + } + case 58: { + if (!((mutable_bitField0_ & 0x00000001) != 0)) { + headers_ = com.google.protobuf.MapField.newMapField( + HeadersDefaultEntryHolder.defaultEntry); + mutable_bitField0_ |= 0x00000001; + } + com.google.protobuf.MapEntry + headers__ = input.readMessage( + HeadersDefaultEntryHolder.defaultEntry.getParserForType(), extensionRegistry); + headers_.getMutableMap().put( + headers__.getKey(), headers__.getValue()); + break; + } + default: { + if (!parseUnknownField( + input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return EventMeshAdminService.internal_static_Metadata_descriptor; + } + + @SuppressWarnings({"rawtypes"}) + @Override + protected com.google.protobuf.MapField internalGetMapField( + int number) { + switch (number) { + case 7: + return internalGetHeaders(); + default: + throw new RuntimeException( + "Invalid map field number: " + number); + } + } + + @Override + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internalGetFieldAccessorTable() { + return EventMeshAdminService.internal_static_Metadata_fieldAccessorTable + .ensureFieldAccessorsInitialized( + Metadata.class, Builder.class); + } + + public static final int TYPE_FIELD_NUMBER = 3; + private volatile Object type_; + + /** + * string type = 3; + * + * @return The type. + */ + @Override + public String getType() { + Object ref = type_; + if (ref instanceof String) { + return (String) ref; + } else { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + type_ = s; + return s; + } + } + + /** + * string type = 3; + * + * @return The bytes for type. + */ + @Override + public com.google.protobuf.ByteString + getTypeBytes() { + Object ref = type_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (String) ref); + type_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + public static final int HEADERS_FIELD_NUMBER = 7; + + private static final class HeadersDefaultEntryHolder { + + static final com.google.protobuf.MapEntry< + String, String> defaultEntry = + com.google.protobuf.MapEntry + .newDefaultInstance( + EventMeshAdminService.internal_static_Metadata_HeadersEntry_descriptor, + com.google.protobuf.WireFormat.FieldType.STRING, + "", + com.google.protobuf.WireFormat.FieldType.STRING, + ""); + } + + private com.google.protobuf.MapField< + String, String> headers_; + + private com.google.protobuf.MapField + internalGetHeaders() { + if (headers_ == null) { + return com.google.protobuf.MapField.emptyMapField( + HeadersDefaultEntryHolder.defaultEntry); + } + return headers_; + } + + public int getHeadersCount() { + return internalGetHeaders().getMap().size(); + } + + /** + * map<string, string> headers = 7; + */ + + @Override + public boolean containsHeaders( + String key) { + if (key == null) { + throw new NullPointerException(); + } + return internalGetHeaders().getMap().containsKey(key); + } + + /** + * Use {@link #getHeadersMap()} instead. + */ + @Override + @Deprecated + public java.util.Map getHeaders() { + return getHeadersMap(); + } + + /** + * map<string, string> headers = 7; + */ + @Override + + public java.util.Map getHeadersMap() { + return internalGetHeaders().getMap(); + } + + /** + * map<string, string> headers = 7; + */ + @Override + + public String getHeadersOrDefault( + String key, + String defaultValue) { + if (key == null) { + throw new NullPointerException(); + } + java.util.Map map = + internalGetHeaders().getMap(); + return map.containsKey(key) ? map.get(key) : defaultValue; + } + + /** + * map<string, string> headers = 7; + */ + @Override + + public String getHeadersOrThrow( + String key) { + if (key == null) { + throw new NullPointerException(); + } + java.util.Map map = + internalGetHeaders().getMap(); + if (!map.containsKey(key)) { + throw new IllegalArgumentException(); + } + return map.get(key); + } + + private byte memoizedIsInitialized = -1; + + @Override + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) { + return true; + } + if (isInitialized == 0) { + return false; + } + + memoizedIsInitialized = 1; + return true; + } + + @Override + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + if (!getTypeBytes().isEmpty()) { + com.google.protobuf.GeneratedMessageV3.writeString(output, 3, type_); + } + com.google.protobuf.GeneratedMessageV3 + .serializeStringMapTo( + output, + internalGetHeaders(), + HeadersDefaultEntryHolder.defaultEntry, + 7); + unknownFields.writeTo(output); + } + + @Override + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) { + return size; + } + + size = 0; + if (!getTypeBytes().isEmpty()) { + size += com.google.protobuf.GeneratedMessageV3.computeStringSize(3, type_); + } + for (java.util.Map.Entry entry + : internalGetHeaders().getMap().entrySet()) { + com.google.protobuf.MapEntry + headers__ = HeadersDefaultEntryHolder.defaultEntry.newBuilderForType() + .setKey(entry.getKey()) + .setValue(entry.getValue()) + .build(); + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(7, headers__); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @Override + public boolean equals(final Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof Metadata)) { + return super.equals(obj); + } + Metadata other = (Metadata) obj; + + if (!getType() + .equals(other.getType())) { + return false; + } + if (!internalGetHeaders().equals( + other.internalGetHeaders())) { + return false; + } + if (!unknownFields.equals(other.unknownFields)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + hash = (37 * hash) + TYPE_FIELD_NUMBER; + hash = (53 * hash) + getType().hashCode(); + if (!internalGetHeaders().getMap().isEmpty()) { + hash = (37 * hash) + HEADERS_FIELD_NUMBER; + hash = (53 * hash) + internalGetHeaders().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static Metadata parseFrom( + java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static Metadata parseFrom( + java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static Metadata parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static Metadata parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static Metadata parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static Metadata parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static Metadata parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input); + } + + public static Metadata parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input, extensionRegistry); + } + + public static Metadata parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseDelimitedWithIOException(PARSER, input); + } + + public static Metadata parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseDelimitedWithIOException(PARSER, input, extensionRegistry); + } + + public static Metadata parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input); + } + + public static Metadata parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input, extensionRegistry); + } + + @Override + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(Metadata prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + @Override + public Builder toBuilder() { + return this == DEFAULT_INSTANCE + ? new Builder() : new Builder().mergeFrom(this); + } + + @Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code Metadata} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessageV3.Builder implements + // @@protoc_insertion_point(builder_implements:Metadata) + MetadataOrBuilder { + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return EventMeshAdminService.internal_static_Metadata_descriptor; + } + + @SuppressWarnings({"rawtypes"}) + protected com.google.protobuf.MapField internalGetMapField( + int number) { + switch (number) { + case 7: + return internalGetHeaders(); + default: + throw new RuntimeException( + "Invalid map field number: " + number); + } + } + + @SuppressWarnings({"rawtypes"}) + protected com.google.protobuf.MapField internalGetMutableMapField( + int number) { + switch (number) { + case 7: + return internalGetMutableHeaders(); + default: + throw new RuntimeException( + "Invalid map field number: " + number); + } + } + + @Override + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internalGetFieldAccessorTable() { + return EventMeshAdminService.internal_static_Metadata_fieldAccessorTable + .ensureFieldAccessorsInitialized( + Metadata.class, Builder.class); + } + + // Construct using Metadata.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3 + .alwaysUseFieldBuilders) { + } + } + + @Override + public Builder clear() { + super.clear(); + type_ = ""; + + internalGetMutableHeaders().clear(); + return this; + } + + @Override + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return EventMeshAdminService.internal_static_Metadata_descriptor; + } + + @Override + public Metadata getDefaultInstanceForType() { + return Metadata.getDefaultInstance(); + } + + @Override + public Metadata build() { + Metadata result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + @Override + public Metadata buildPartial() { + Metadata result = new Metadata(this); + int from_bitField0_ = bitField0_; + result.type_ = type_; + result.headers_ = internalGetHeaders(); + result.headers_.makeImmutable(); + onBuilt(); + return result; + } + + @Override + public Builder clone() { + return super.clone(); + } + + @Override + public Builder setField( + com.google.protobuf.Descriptors.FieldDescriptor field, + Object value) { + return super.setField(field, value); + } + + @Override + public Builder clearField( + com.google.protobuf.Descriptors.FieldDescriptor field) { + return super.clearField(field); + } + + @Override + public Builder clearOneof( + com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return super.clearOneof(oneof); + } + + @Override + public Builder setRepeatedField( + com.google.protobuf.Descriptors.FieldDescriptor field, + int index, Object value) { + return super.setRepeatedField(field, index, value); + } + + @Override + public Builder addRepeatedField( + com.google.protobuf.Descriptors.FieldDescriptor field, + Object value) { + return super.addRepeatedField(field, value); + } + + @Override + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof Metadata) { + return mergeFrom((Metadata) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(Metadata other) { + if (other == Metadata.getDefaultInstance()) { + return this; + } + if (!other.getType().isEmpty()) { + type_ = other.type_; + onChanged(); + } + internalGetMutableHeaders().mergeFrom( + other.internalGetHeaders()); + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + @Override + public final boolean isInitialized() { + return true; + } + + @Override + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Metadata parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (Metadata) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private int bitField0_; + + private Object type_ = ""; + + /** + * string type = 3; + * + * @return The type. + */ + public String getType() { + Object ref = type_; + if (!(ref instanceof String)) { + com.google.protobuf.ByteString bs = + (com.google.protobuf.ByteString) ref; + String s = bs.toStringUtf8(); + type_ = s; + return s; + } else { + return (String) ref; + } + } + + /** + * string type = 3; + * + * @return The bytes for type. + */ + public com.google.protobuf.ByteString + getTypeBytes() { + Object ref = type_; + if (ref instanceof String) { + com.google.protobuf.ByteString b = + com.google.protobuf.ByteString.copyFromUtf8( + (String) ref); + type_ = b; + return b; + } else { + return (com.google.protobuf.ByteString) ref; + } + } + + /** + * string type = 3; + * + * @param value The type to set. + * @return This builder for chaining. + */ + public Builder setType( + String value) { + if (value == null) { + throw new NullPointerException(); + } + + type_ = value; + onChanged(); + return this; + } + + /** + * string type = 3; + * + * @return This builder for chaining. + */ + public Builder clearType() { + + type_ = getDefaultInstance().getType(); + onChanged(); + return this; + } + + /** + * string type = 3; + * + * @param value The bytes for type to set. + * @return This builder for chaining. + */ + public Builder setTypeBytes( + com.google.protobuf.ByteString value) { + if (value == null) { + throw new NullPointerException(); + } + checkByteStringIsUtf8(value); + + type_ = value; + onChanged(); + return this; + } + + private com.google.protobuf.MapField< + String, String> headers_; + + private com.google.protobuf.MapField + internalGetHeaders() { + if (headers_ == null) { + return com.google.protobuf.MapField.emptyMapField( + HeadersDefaultEntryHolder.defaultEntry); + } + return headers_; + } + + private com.google.protobuf.MapField + internalGetMutableHeaders() { + onChanged(); + ; + if (headers_ == null) { + headers_ = com.google.protobuf.MapField.newMapField( + HeadersDefaultEntryHolder.defaultEntry); + } + if (!headers_.isMutable()) { + headers_ = headers_.copy(); + } + return headers_; + } + + public int getHeadersCount() { + return internalGetHeaders().getMap().size(); + } + + /** + * map<string, string> headers = 7; + */ + + @Override + public boolean containsHeaders( + String key) { + if (key == null) { + throw new NullPointerException(); + } + return internalGetHeaders().getMap().containsKey(key); + } + + /** + * Use {@link #getHeadersMap()} instead. + */ + @Override + @Deprecated + public java.util.Map getHeaders() { + return getHeadersMap(); + } + + /** + * map<string, string> headers = 7; + */ + @Override + + public java.util.Map getHeadersMap() { + return internalGetHeaders().getMap(); + } + + /** + * map<string, string> headers = 7; + */ + @Override + + public String getHeadersOrDefault( + String key, + String defaultValue) { + if (key == null) { + throw new NullPointerException(); + } + java.util.Map map = + internalGetHeaders().getMap(); + return map.containsKey(key) ? map.get(key) : defaultValue; + } + + /** + * map<string, string> headers = 7; + */ + @Override + + public String getHeadersOrThrow( + String key) { + if (key == null) { + throw new NullPointerException(); + } + java.util.Map map = + internalGetHeaders().getMap(); + if (!map.containsKey(key)) { + throw new IllegalArgumentException(); + } + return map.get(key); + } + + public Builder clearHeaders() { + internalGetMutableHeaders().getMutableMap() + .clear(); + return this; + } + + /** + * map<string, string> headers = 7; + */ + + public Builder removeHeaders( + String key) { + if (key == null) { + throw new NullPointerException(); + } + internalGetMutableHeaders().getMutableMap() + .remove(key); + return this; + } + + /** + * Use alternate mutation accessors instead. + */ + @Deprecated + public java.util.Map + getMutableHeaders() { + return internalGetMutableHeaders().getMutableMap(); + } + + /** + * map<string, string> headers = 7; + */ + public Builder putHeaders( + String key, + String value) { + if (key == null) { + throw new NullPointerException(); + } + if (value == null) { + throw new NullPointerException(); + } + internalGetMutableHeaders().getMutableMap() + .put(key, value); + return this; + } + + /** + * map<string, string> headers = 7; + */ + + public Builder putAllHeaders( + java.util.Map values) { + internalGetMutableHeaders().getMutableMap() + .putAll(values); + return this; + } + + @Override + public final Builder setUnknownFields( + final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + @Override + public final Builder mergeUnknownFields( + final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:Metadata) + } + + // @@protoc_insertion_point(class_scope:Metadata) + private static final Metadata DEFAULT_INSTANCE; + + static { + DEFAULT_INSTANCE = new Metadata(); + } + + public static Metadata getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + private static final com.google.protobuf.Parser + PARSER = new com.google.protobuf.AbstractParser() { + @Override + public Metadata parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new Metadata(input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + @Override + public Metadata getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + +} + diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/grpc/adminserver/MetadataOrBuilder.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/grpc/adminserver/MetadataOrBuilder.java new file mode 100644 index 0000000000..7afef491f8 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/grpc/adminserver/MetadataOrBuilder.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.protocol.grpc.adminserver;// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: event_mesh_admin_service.proto + +public interface MetadataOrBuilder extends + // @@protoc_insertion_point(interface_extends:Metadata) + com.google.protobuf.MessageOrBuilder { + + /** + * string type = 3; + * + * @return The type. + */ + String getType(); + + /** + * string type = 3; + * + * @return The bytes for type. + */ + com.google.protobuf.ByteString + getTypeBytes(); + + /** + * map<string, string> headers = 7; + */ + int getHeadersCount(); + + /** + * map<string, string> headers = 7; + */ + boolean containsHeaders( + String key); + + /** + * Use {@link #getHeadersMap()} instead. + */ + @Deprecated + java.util.Map + getHeaders(); + + /** + * map<string, string> headers = 7; + */ + java.util.Map + getHeadersMap(); + + /** + * map<string, string> headers = 7; + */ + + String getHeadersOrDefault( + String key, + String defaultValue); + + /** + * map<string, string> headers = 7; + */ + + String getHeadersOrThrow( + String key); +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/grpc/adminserver/Payload.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/grpc/adminserver/Payload.java new file mode 100644 index 0000000000..a0067099a0 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/grpc/adminserver/Payload.java @@ -0,0 +1,893 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.protocol.grpc.adminserver;// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: event_mesh_admin_service.proto + +/** + * Protobuf type {@code Payload} + */ +public final class Payload extends + com.google.protobuf.GeneratedMessageV3 implements + // @@protoc_insertion_point(message_implements:Payload) + PayloadOrBuilder { + + private static final long serialVersionUID = 0L; + + // Use Payload.newBuilder() to construct. + private Payload(com.google.protobuf.GeneratedMessageV3.Builder builder) { + super(builder); + } + + private Payload() { + } + + @Override + @SuppressWarnings({"unused"}) + protected Object newInstance( + UnusedPrivateParameter unused) { + return new Payload(); + } + + @Override + public final com.google.protobuf.UnknownFieldSet + getUnknownFields() { + return this.unknownFields; + } + + private Payload( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + this(); + if (extensionRegistry == null) { + throw new NullPointerException(); + } + com.google.protobuf.UnknownFieldSet.Builder unknownFields = + com.google.protobuf.UnknownFieldSet.newBuilder(); + try { + boolean done = false; + while (!done) { + int tag = input.readTag(); + switch (tag) { + case 0: + done = true; + break; + case 18: { + Metadata.Builder subBuilder = null; + if (metadata_ != null) { + subBuilder = metadata_.toBuilder(); + } + metadata_ = input.readMessage(Metadata.parser(), extensionRegistry); + if (subBuilder != null) { + subBuilder.mergeFrom(metadata_); + metadata_ = subBuilder.buildPartial(); + } + + break; + } + case 26: { + com.google.protobuf.Any.Builder subBuilder = null; + if (body_ != null) { + subBuilder = body_.toBuilder(); + } + body_ = input.readMessage(com.google.protobuf.Any.parser(), extensionRegistry); + if (subBuilder != null) { + subBuilder.mergeFrom(body_); + body_ = subBuilder.buildPartial(); + } + + break; + } + default: { + if (!parseUnknownField( + input, unknownFields, extensionRegistry, tag)) { + done = true; + } + break; + } + } + } + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + throw e.setUnfinishedMessage(this); + } catch (java.io.IOException e) { + throw new com.google.protobuf.InvalidProtocolBufferException( + e).setUnfinishedMessage(this); + } finally { + this.unknownFields = unknownFields.build(); + makeExtensionsImmutable(); + } + } + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return EventMeshAdminService.internal_static_Payload_descriptor; + } + + @Override + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internalGetFieldAccessorTable() { + return EventMeshAdminService.internal_static_Payload_fieldAccessorTable + .ensureFieldAccessorsInitialized( + Payload.class, Builder.class); + } + + public static final int METADATA_FIELD_NUMBER = 2; + private Metadata metadata_; + + /** + * .Metadata metadata = 2; + * + * @return Whether the metadata field is set. + */ + @Override + public boolean hasMetadata() { + return metadata_ != null; + } + + /** + * .Metadata metadata = 2; + * + * @return The metadata. + */ + @Override + public Metadata getMetadata() { + return metadata_ == null ? Metadata.getDefaultInstance() : metadata_; + } + + /** + * .Metadata metadata = 2; + */ + @Override + public MetadataOrBuilder getMetadataOrBuilder() { + return getMetadata(); + } + + public static final int BODY_FIELD_NUMBER = 3; + private com.google.protobuf.Any body_; + + /** + * .google.protobuf.Any body = 3; + * + * @return Whether the body field is set. + */ + @Override + public boolean hasBody() { + return body_ != null; + } + + /** + * .google.protobuf.Any body = 3; + * + * @return The body. + */ + @Override + public com.google.protobuf.Any getBody() { + return body_ == null ? com.google.protobuf.Any.getDefaultInstance() : body_; + } + + /** + * .google.protobuf.Any body = 3; + */ + @Override + public com.google.protobuf.AnyOrBuilder getBodyOrBuilder() { + return getBody(); + } + + private byte memoizedIsInitialized = -1; + + @Override + public final boolean isInitialized() { + byte isInitialized = memoizedIsInitialized; + if (isInitialized == 1) { + return true; + } + if (isInitialized == 0) { + return false; + } + + memoizedIsInitialized = 1; + return true; + } + + @Override + public void writeTo(com.google.protobuf.CodedOutputStream output) + throws java.io.IOException { + if (metadata_ != null) { + output.writeMessage(2, getMetadata()); + } + if (body_ != null) { + output.writeMessage(3, getBody()); + } + unknownFields.writeTo(output); + } + + @Override + public int getSerializedSize() { + int size = memoizedSize; + if (size != -1) { + return size; + } + + size = 0; + if (metadata_ != null) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(2, getMetadata()); + } + if (body_ != null) { + size += com.google.protobuf.CodedOutputStream + .computeMessageSize(3, getBody()); + } + size += unknownFields.getSerializedSize(); + memoizedSize = size; + return size; + } + + @Override + public boolean equals(final Object obj) { + if (obj == this) { + return true; + } + if (!(obj instanceof Payload)) { + return super.equals(obj); + } + Payload other = (Payload) obj; + + if (hasMetadata() != other.hasMetadata()) { + return false; + } + if (hasMetadata()) { + if (!getMetadata() + .equals(other.getMetadata())) { + return false; + } + } + if (hasBody() != other.hasBody()) { + return false; + } + if (hasBody()) { + if (!getBody() + .equals(other.getBody())) { + return false; + } + } + if (!unknownFields.equals(other.unknownFields)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + if (memoizedHashCode != 0) { + return memoizedHashCode; + } + int hash = 41; + hash = (19 * hash) + getDescriptor().hashCode(); + if (hasMetadata()) { + hash = (37 * hash) + METADATA_FIELD_NUMBER; + hash = (53 * hash) + getMetadata().hashCode(); + } + if (hasBody()) { + hash = (37 * hash) + BODY_FIELD_NUMBER; + hash = (53 * hash) + getBody().hashCode(); + } + hash = (29 * hash) + unknownFields.hashCode(); + memoizedHashCode = hash; + return hash; + } + + public static Payload parseFrom( + java.nio.ByteBuffer data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static Payload parseFrom( + java.nio.ByteBuffer data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static Payload parseFrom( + com.google.protobuf.ByteString data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static Payload parseFrom( + com.google.protobuf.ByteString data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static Payload parseFrom(byte[] data) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data); + } + + public static Payload parseFrom( + byte[] data, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return PARSER.parseFrom(data, extensionRegistry); + } + + public static Payload parseFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input); + } + + public static Payload parseFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input, extensionRegistry); + } + + public static Payload parseDelimitedFrom(java.io.InputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseDelimitedWithIOException(PARSER, input); + } + + public static Payload parseDelimitedFrom( + java.io.InputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseDelimitedWithIOException(PARSER, input, extensionRegistry); + } + + public static Payload parseFrom( + com.google.protobuf.CodedInputStream input) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input); + } + + public static Payload parseFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + return com.google.protobuf.GeneratedMessageV3 + .parseWithIOException(PARSER, input, extensionRegistry); + } + + @Override + public Builder newBuilderForType() { + return newBuilder(); + } + + public static Builder newBuilder() { + return DEFAULT_INSTANCE.toBuilder(); + } + + public static Builder newBuilder(Payload prototype) { + return DEFAULT_INSTANCE.toBuilder().mergeFrom(prototype); + } + + @Override + public Builder toBuilder() { + return this == DEFAULT_INSTANCE + ? new Builder() : new Builder().mergeFrom(this); + } + + @Override + protected Builder newBuilderForType( + com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + Builder builder = new Builder(parent); + return builder; + } + + /** + * Protobuf type {@code Payload} + */ + public static final class Builder extends + com.google.protobuf.GeneratedMessageV3.Builder implements + // @@protoc_insertion_point(builder_implements:Payload) + PayloadOrBuilder { + + public static final com.google.protobuf.Descriptors.Descriptor + getDescriptor() { + return EventMeshAdminService.internal_static_Payload_descriptor; + } + + @Override + protected com.google.protobuf.GeneratedMessageV3.FieldAccessorTable + internalGetFieldAccessorTable() { + return EventMeshAdminService.internal_static_Payload_fieldAccessorTable + .ensureFieldAccessorsInitialized( + Payload.class, Builder.class); + } + + // Construct using Payload.newBuilder() + private Builder() { + maybeForceBuilderInitialization(); + } + + private Builder( + com.google.protobuf.GeneratedMessageV3.BuilderParent parent) { + super(parent); + maybeForceBuilderInitialization(); + } + + private void maybeForceBuilderInitialization() { + if (com.google.protobuf.GeneratedMessageV3 + .alwaysUseFieldBuilders) { + } + } + + @Override + public Builder clear() { + super.clear(); + if (metadataBuilder_ == null) { + metadata_ = null; + } else { + metadata_ = null; + metadataBuilder_ = null; + } + if (bodyBuilder_ == null) { + body_ = null; + } else { + body_ = null; + bodyBuilder_ = null; + } + return this; + } + + @Override + public com.google.protobuf.Descriptors.Descriptor + getDescriptorForType() { + return EventMeshAdminService.internal_static_Payload_descriptor; + } + + @Override + public Payload getDefaultInstanceForType() { + return Payload.getDefaultInstance(); + } + + @Override + public Payload build() { + Payload result = buildPartial(); + if (!result.isInitialized()) { + throw newUninitializedMessageException(result); + } + return result; + } + + @Override + public Payload buildPartial() { + Payload result = new Payload(this); + if (metadataBuilder_ == null) { + result.metadata_ = metadata_; + } else { + result.metadata_ = metadataBuilder_.build(); + } + if (bodyBuilder_ == null) { + result.body_ = body_; + } else { + result.body_ = bodyBuilder_.build(); + } + onBuilt(); + return result; + } + + @Override + public Builder clone() { + return super.clone(); + } + + @Override + public Builder setField( + com.google.protobuf.Descriptors.FieldDescriptor field, + Object value) { + return super.setField(field, value); + } + + @Override + public Builder clearField( + com.google.protobuf.Descriptors.FieldDescriptor field) { + return super.clearField(field); + } + + @Override + public Builder clearOneof( + com.google.protobuf.Descriptors.OneofDescriptor oneof) { + return super.clearOneof(oneof); + } + + @Override + public Builder setRepeatedField( + com.google.protobuf.Descriptors.FieldDescriptor field, + int index, Object value) { + return super.setRepeatedField(field, index, value); + } + + @Override + public Builder addRepeatedField( + com.google.protobuf.Descriptors.FieldDescriptor field, + Object value) { + return super.addRepeatedField(field, value); + } + + @Override + public Builder mergeFrom(com.google.protobuf.Message other) { + if (other instanceof Payload) { + return mergeFrom((Payload) other); + } else { + super.mergeFrom(other); + return this; + } + } + + public Builder mergeFrom(Payload other) { + if (other == Payload.getDefaultInstance()) { + return this; + } + if (other.hasMetadata()) { + mergeMetadata(other.getMetadata()); + } + if (other.hasBody()) { + mergeBody(other.getBody()); + } + this.mergeUnknownFields(other.unknownFields); + onChanged(); + return this; + } + + @Override + public final boolean isInitialized() { + return true; + } + + @Override + public Builder mergeFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws java.io.IOException { + Payload parsedMessage = null; + try { + parsedMessage = PARSER.parsePartialFrom(input, extensionRegistry); + } catch (com.google.protobuf.InvalidProtocolBufferException e) { + parsedMessage = (Payload) e.getUnfinishedMessage(); + throw e.unwrapIOException(); + } finally { + if (parsedMessage != null) { + mergeFrom(parsedMessage); + } + } + return this; + } + + private Metadata metadata_; + private com.google.protobuf.SingleFieldBuilderV3< + Metadata, Metadata.Builder, MetadataOrBuilder> metadataBuilder_; + + /** + * .Metadata metadata = 2; + * + * @return Whether the metadata field is set. + */ + public boolean hasMetadata() { + return metadataBuilder_ != null || metadata_ != null; + } + + /** + * .Metadata metadata = 2; + * + * @return The metadata. + */ + public Metadata getMetadata() { + if (metadataBuilder_ == null) { + return metadata_ == null ? Metadata.getDefaultInstance() : metadata_; + } else { + return metadataBuilder_.getMessage(); + } + } + + /** + * .Metadata metadata = 2; + */ + public Builder setMetadata(Metadata value) { + if (metadataBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + metadata_ = value; + onChanged(); + } else { + metadataBuilder_.setMessage(value); + } + + return this; + } + + /** + * .Metadata metadata = 2; + */ + public Builder setMetadata( + Metadata.Builder builderForValue) { + if (metadataBuilder_ == null) { + metadata_ = builderForValue.build(); + onChanged(); + } else { + metadataBuilder_.setMessage(builderForValue.build()); + } + + return this; + } + + /** + * .Metadata metadata = 2; + */ + public Builder mergeMetadata(Metadata value) { + if (metadataBuilder_ == null) { + if (metadata_ != null) { + metadata_ = + Metadata.newBuilder(metadata_).mergeFrom(value).buildPartial(); + } else { + metadata_ = value; + } + onChanged(); + } else { + metadataBuilder_.mergeFrom(value); + } + + return this; + } + + /** + * .Metadata metadata = 2; + */ + public Builder clearMetadata() { + if (metadataBuilder_ == null) { + metadata_ = null; + onChanged(); + } else { + metadata_ = null; + metadataBuilder_ = null; + } + + return this; + } + + /** + * .Metadata metadata = 2; + */ + public Metadata.Builder getMetadataBuilder() { + + onChanged(); + return getMetadataFieldBuilder().getBuilder(); + } + + /** + * .Metadata metadata = 2; + */ + public MetadataOrBuilder getMetadataOrBuilder() { + if (metadataBuilder_ != null) { + return metadataBuilder_.getMessageOrBuilder(); + } else { + return metadata_ == null ? + Metadata.getDefaultInstance() : metadata_; + } + } + + /** + * .Metadata metadata = 2; + */ + private com.google.protobuf.SingleFieldBuilderV3< + Metadata, Metadata.Builder, MetadataOrBuilder> + getMetadataFieldBuilder() { + if (metadataBuilder_ == null) { + metadataBuilder_ = new com.google.protobuf.SingleFieldBuilderV3< + Metadata, Metadata.Builder, MetadataOrBuilder>( + getMetadata(), + getParentForChildren(), + isClean()); + metadata_ = null; + } + return metadataBuilder_; + } + + private com.google.protobuf.Any body_; + private com.google.protobuf.SingleFieldBuilderV3< + com.google.protobuf.Any, com.google.protobuf.Any.Builder, com.google.protobuf.AnyOrBuilder> bodyBuilder_; + + /** + * .google.protobuf.Any body = 3; + * + * @return Whether the body field is set. + */ + public boolean hasBody() { + return bodyBuilder_ != null || body_ != null; + } + + /** + * .google.protobuf.Any body = 3; + * + * @return The body. + */ + public com.google.protobuf.Any getBody() { + if (bodyBuilder_ == null) { + return body_ == null ? com.google.protobuf.Any.getDefaultInstance() : body_; + } else { + return bodyBuilder_.getMessage(); + } + } + + /** + * .google.protobuf.Any body = 3; + */ + public Builder setBody(com.google.protobuf.Any value) { + if (bodyBuilder_ == null) { + if (value == null) { + throw new NullPointerException(); + } + body_ = value; + onChanged(); + } else { + bodyBuilder_.setMessage(value); + } + + return this; + } + + /** + * .google.protobuf.Any body = 3; + */ + public Builder setBody( + com.google.protobuf.Any.Builder builderForValue) { + if (bodyBuilder_ == null) { + body_ = builderForValue.build(); + onChanged(); + } else { + bodyBuilder_.setMessage(builderForValue.build()); + } + + return this; + } + + /** + * .google.protobuf.Any body = 3; + */ + public Builder mergeBody(com.google.protobuf.Any value) { + if (bodyBuilder_ == null) { + if (body_ != null) { + body_ = + com.google.protobuf.Any.newBuilder(body_).mergeFrom(value).buildPartial(); + } else { + body_ = value; + } + onChanged(); + } else { + bodyBuilder_.mergeFrom(value); + } + + return this; + } + + /** + * .google.protobuf.Any body = 3; + */ + public Builder clearBody() { + if (bodyBuilder_ == null) { + body_ = null; + onChanged(); + } else { + body_ = null; + bodyBuilder_ = null; + } + + return this; + } + + /** + * .google.protobuf.Any body = 3; + */ + public com.google.protobuf.Any.Builder getBodyBuilder() { + + onChanged(); + return getBodyFieldBuilder().getBuilder(); + } + + /** + * .google.protobuf.Any body = 3; + */ + public com.google.protobuf.AnyOrBuilder getBodyOrBuilder() { + if (bodyBuilder_ != null) { + return bodyBuilder_.getMessageOrBuilder(); + } else { + return body_ == null ? + com.google.protobuf.Any.getDefaultInstance() : body_; + } + } + + /** + * .google.protobuf.Any body = 3; + */ + private com.google.protobuf.SingleFieldBuilderV3< + com.google.protobuf.Any, com.google.protobuf.Any.Builder, com.google.protobuf.AnyOrBuilder> + getBodyFieldBuilder() { + if (bodyBuilder_ == null) { + bodyBuilder_ = new com.google.protobuf.SingleFieldBuilderV3< + com.google.protobuf.Any, com.google.protobuf.Any.Builder, com.google.protobuf.AnyOrBuilder>( + getBody(), + getParentForChildren(), + isClean()); + body_ = null; + } + return bodyBuilder_; + } + + @Override + public final Builder setUnknownFields( + final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.setUnknownFields(unknownFields); + } + + @Override + public final Builder mergeUnknownFields( + final com.google.protobuf.UnknownFieldSet unknownFields) { + return super.mergeUnknownFields(unknownFields); + } + + // @@protoc_insertion_point(builder_scope:Payload) + } + + // @@protoc_insertion_point(class_scope:Payload) + private static final Payload DEFAULT_INSTANCE; + + static { + DEFAULT_INSTANCE = new Payload(); + } + + public static Payload getDefaultInstance() { + return DEFAULT_INSTANCE; + } + + private static final com.google.protobuf.Parser + PARSER = new com.google.protobuf.AbstractParser() { + @Override + public Payload parsePartialFrom( + com.google.protobuf.CodedInputStream input, + com.google.protobuf.ExtensionRegistryLite extensionRegistry) + throws com.google.protobuf.InvalidProtocolBufferException { + return new Payload(input, extensionRegistry); + } + }; + + public static com.google.protobuf.Parser parser() { + return PARSER; + } + + @Override + public com.google.protobuf.Parser getParserForType() { + return PARSER; + } + + @Override + public Payload getDefaultInstanceForType() { + return DEFAULT_INSTANCE; + } + +} + diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/grpc/adminserver/PayloadOrBuilder.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/grpc/adminserver/PayloadOrBuilder.java new file mode 100644 index 0000000000..a50a340e1b --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/grpc/adminserver/PayloadOrBuilder.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.protocol.grpc.adminserver;// Generated by the protocol buffer compiler. DO NOT EDIT! +// source: event_mesh_admin_service.proto + +public interface PayloadOrBuilder extends + // @@protoc_insertion_point(interface_extends:Payload) + com.google.protobuf.MessageOrBuilder { + + /** + * .Metadata metadata = 2; + * + * @return Whether the metadata field is set. + */ + boolean hasMetadata(); + + /** + * .Metadata metadata = 2; + * + * @return The metadata. + */ + Metadata getMetadata(); + + /** + * .Metadata metadata = 2; + */ + MetadataOrBuilder getMetadataOrBuilder(); + + /** + * .google.protobuf.Any body = 3; + * + * @return Whether the body field is set. + */ + boolean hasBody(); + + /** + * .google.protobuf.Any body = 3; + * + * @return The body. + */ + com.google.protobuf.Any getBody(); + + /** + * .google.protobuf.Any body = 3; + */ + com.google.protobuf.AnyOrBuilder getBodyOrBuilder(); +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/grpc/proto/event_mesh_admin_service.proto b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/grpc/proto/event_mesh_admin_service.proto new file mode 100644 index 0000000000..fa7095840f --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/protocol/grpc/proto/event_mesh_admin_service.proto @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +syntax="proto3"; + +import "google/protobuf/any.proto"; + +option java_multiple_files=true; + +message Metadata { + string type = 3; + map headers = 7; +} + + +message Payload { + Metadata metadata = 2; + bytes body = 3; +} + +service AdminService { + rpc invokeBiStream(stream Payload) returns (stream Payload){} + rpc invoke(Payload) returns (Payload){} +} + diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/JobState.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/JobState.java new file mode 100644 index 0000000000..150a67e302 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/JobState.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote; + +import java.util.HashMap; +import java.util.Map; + +import lombok.ToString; + +@ToString +public enum JobState { + INIT, RUNNING, COMPLETE, DELETE, FAIL, PAUSE; + private static final JobState[] STATES_NUM_INDEX = JobState.values(); + private static final Map STATES_NAME_INDEX = new HashMap<>(); + + static { + for (JobState jobState : STATES_NUM_INDEX) { + STATES_NAME_INDEX.put(jobState.name(), jobState); + } + } + + public static JobState fromIndex(Integer index) { + if (index == null || index < 0 || index >= STATES_NUM_INDEX.length) { + return null; + } + + return STATES_NUM_INDEX[index]; + } + + public static JobState fromIndex(String index) { + if (index == null || index.isEmpty()) { + return null; + } + + return STATES_NAME_INDEX.get(index); + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/TaskState.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/TaskState.java new file mode 100644 index 0000000000..606339c443 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/TaskState.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote; + +import java.util.HashMap; +import java.util.Map; + +import lombok.ToString; + +@ToString +public enum TaskState { + INIT, STARTED, RUNNING, PAUSE, COMPLETE, DELETE, FAIL; + private static final TaskState[] STATES_NUM_INDEX = TaskState.values(); + private static final Map STATES_NAME_INDEX = new HashMap<>(); + static { + for (TaskState jobState : STATES_NUM_INDEX) { + STATES_NAME_INDEX.put(jobState.name(), jobState); + } + } + + public static TaskState fromIndex(Integer index) { + if (index == null || index < 0 || index >= STATES_NUM_INDEX.length) { + return null; + } + + return STATES_NUM_INDEX[index]; + } + + public static TaskState fromIndex(String index) { + if (index == null || index.isEmpty()) { + return null; + } + + return STATES_NAME_INDEX.get(index); + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/TransportType.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/TransportType.java new file mode 100644 index 0000000000..6b43598398 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/TransportType.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote; + +import org.apache.eventmesh.common.remote.datasource.DataSourceType; + +import java.util.HashMap; +import java.util.Map; + +import lombok.Getter; + +@Getter +public enum TransportType { + MYSQL_MYSQL(DataSourceType.MYSQL, DataSourceType.MYSQL), + REDIS_REDIS(DataSourceType.REDIS, DataSourceType.REDIS), + ROCKETMQ_ROCKETMQ(DataSourceType.ROCKETMQ, DataSourceType.ROCKETMQ), + MYSQL_HTTP(DataSourceType.MYSQL, DataSourceType.HTTP), + ROCKETMQ_HTTP(DataSourceType.ROCKETMQ, DataSourceType.HTTP), + HTTP_MYSQL(DataSourceType.HTTP, DataSourceType.MYSQL), + HTTP_REDIS(DataSourceType.HTTP, DataSourceType.REDIS), + HTTP_ROCKETMQ(DataSourceType.HTTP, DataSourceType.ROCKETMQ), + REDIS_MQ(DataSourceType.REDIS, DataSourceType.ROCKETMQ), + HTTP_HTTP(DataSourceType.HTTP, DataSourceType.HTTP), + ; + private static final Map INDEX_TYPES = new HashMap<>(); + private static final TransportType[] TYPES = TransportType.values(); + private static final String SEPARATOR = "@"; + + static { + for (TransportType type : TYPES) { + INDEX_TYPES.put(type.name(), type); + } + } + + private final DataSourceType src; + + private final DataSourceType dst; + + TransportType(DataSourceType src, DataSourceType dst) { + this.src = src; + this.dst = dst; + } + + + public static TransportType getTransportType(String index) { + if (index == null || index.isEmpty()) { + return null; + } + return INDEX_TYPES.get(index); + } + + public static TransportType getTransportType(Integer index) { + if (index == null || index < 0 || index >= TYPES.length) { + return null; + } + return TYPES[index]; + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSource.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSource.java new file mode 100644 index 0000000000..afda984805 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSource.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.datasource; + +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceConfig; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +import lombok.Data; + +@Data +public class DataSource { + + private DataSourceType type; + + private String desc; + @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS) + @JsonSubTypes({ + @JsonSubTypes.Type(value = CanalSourceConfig.class, name = "CanalSourceConfig"), + @JsonSubTypes.Type(value = CanalSinkConfig.class, name = "CanalSinkConfig") + }) + private Config conf; + + private Class confClazz; + + private String region; + +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSourceClassify.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSourceClassify.java new file mode 100644 index 0000000000..8cb01c9204 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSourceClassify.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.datasource; + +public enum DataSourceClassify { + // relationship db + RDB, + MQ, + CACHE, + TUNNEL; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSourceConf.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSourceConf.java new file mode 100644 index 0000000000..9701a9fa11 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSourceConf.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.datasource; + + +public abstract class DataSourceConf { + public abstract Class getConfClass(); +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSourceDriverType.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSourceDriverType.java new file mode 100644 index 0000000000..f1c0f54e5f --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSourceDriverType.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.datasource; + +public enum DataSourceDriverType { + MYSQL, + MariaDB, + REDIS, + ROCKETMQ, + HTTP; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSourceType.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSourceType.java new file mode 100644 index 0000000000..1c14239c3b --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/datasource/DataSourceType.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.datasource; + +import java.util.HashMap; +import java.util.Map; + +import lombok.Getter; +import lombok.ToString; + +@Getter +@ToString +public enum DataSourceType { + MYSQL("MySQL", DataSourceDriverType.MYSQL, DataSourceClassify.RDB), + MariaDB("MariaDB", DataSourceDriverType.MariaDB, DataSourceClassify.RDB), + REDIS("Redis", DataSourceDriverType.REDIS, DataSourceClassify.CACHE), + ROCKETMQ("RocketMQ", DataSourceDriverType.ROCKETMQ, DataSourceClassify.MQ), + HTTP("HTTP", DataSourceDriverType.HTTP, DataSourceClassify.TUNNEL); + private static final Map INDEX_TYPES = new HashMap<>(); + private static final DataSourceType[] TYPES = DataSourceType.values(); + static { + for (DataSourceType type : TYPES) { + INDEX_TYPES.put(type.name(), type); + } + } + + private final String name; + private final DataSourceDriverType driverType; + private final DataSourceClassify classify; + + DataSourceType(String name, DataSourceDriverType driverType, DataSourceClassify classify) { + this.name = name; + this.driverType = driverType; + this.classify = classify; + } + + public static DataSourceType getDataSourceType(String index) { + if (index == null || index.isEmpty()) { + return null; + } + return INDEX_TYPES.get(index); + } + + public static DataSourceType getDataSourceType(Integer index) { + if (index == null || index < 0 || index >= TYPES.length) { + return null; + } + return TYPES[index]; + } + + public static DataSourceType fromString(String type) { + for (DataSourceType dataSourceType : DataSourceType.values()) { + if (dataSourceType.name().equalsIgnoreCase(type)) { + return dataSourceType; + } + } + throw new IllegalArgumentException("No enum constant for type: " + type); + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/exception/ErrorCode.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/exception/ErrorCode.java new file mode 100644 index 0000000000..f24971acc7 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/exception/ErrorCode.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.exception; + +public class ErrorCode { + + public static final int SUCCESS = 0; + public static final int BAD_REQUEST = 4001; + public static final int BAD_DB_DATA = 4002; + + public static final int INTERNAL_ERR = 5000; + public static final int STARTUP_CONFIG_MISS = 5001; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/exception/PayloadFormatException.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/exception/PayloadFormatException.java new file mode 100644 index 0000000000..affa6b8bbf --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/exception/PayloadFormatException.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.exception; + +public class PayloadFormatException extends RemoteRuntimeException { + + public PayloadFormatException(int code, String desc) { + super(code, desc); + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/exception/RemoteRuntimeException.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/exception/RemoteRuntimeException.java new file mode 100644 index 0000000000..708a558d37 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/exception/RemoteRuntimeException.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.exception; + +public class RemoteRuntimeException extends RuntimeException { + + protected final int code; + protected final String message; + + public RemoteRuntimeException(int code, String message) { + this.code = code; + this.message = message; + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/job/JobConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/job/JobConnectorConfig.java new file mode 100644 index 0000000000..14e8178cf3 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/job/JobConnectorConfig.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.job; + +import java.util.Map; + +import lombok.Data; + +/** + * Description: + */ +@Data +public class JobConnectorConfig { + private Map sourceConnectorConfig; + + private String sourceConnectorDesc; + + private Map sinkConnectorConfig; + + private String sinkConnectorDesc; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/job/JobType.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/job/JobType.java new file mode 100644 index 0000000000..83d2f56964 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/job/JobType.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.job; + +import java.util.HashMap; +import java.util.Map; + +public enum JobType { + FULL, + INCREASE, + CHECK; + + private static final JobType[] STATES_NUM_INDEX = JobType.values(); + private static final Map STATES_NAME_INDEX = new HashMap<>(); + static { + for (JobType jobType : STATES_NUM_INDEX) { + STATES_NAME_INDEX.put(jobType.name(), jobType); + } + } + + public static JobType fromIndex(Integer index) { + if (index == null || index < 0 || index > STATES_NUM_INDEX.length) { + return null; + } + + return STATES_NUM_INDEX[index]; + } + + public static JobType fromIndex(String index) { + if (index == null || index.isEmpty()) { + return null; + } + + return STATES_NAME_INDEX.get(index); + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/job/SyncConsistency.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/job/SyncConsistency.java new file mode 100644 index 0000000000..a5aec2aa38 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/job/SyncConsistency.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.job; + +public enum SyncConsistency { + /** + * based with media + */ + MEDIA("M"), + /** + * based with store + */ + STORE("S"), + /** + * Based on the current change value, eventual consistency + */ + BASE("B"); + + private String value; + + SyncConsistency(String value) { + this.value = value; + } + + public static SyncConsistency valuesOf(String value) { + SyncConsistency[] modes = values(); + for (SyncConsistency mode : modes) { + if (mode.value.equalsIgnoreCase(value)) { + return mode; + } + } + return null; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public boolean isMedia() { + return this.equals(SyncConsistency.MEDIA); + } + + public boolean isStore() { + return this.equals(SyncConsistency.STORE); + } + + public boolean isBase() { + return this.equals(SyncConsistency.BASE); + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/job/SyncMode.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/job/SyncMode.java new file mode 100644 index 0000000000..0f2f9bdfcb --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/job/SyncMode.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.job; + +public enum SyncMode { + /** + * row + */ + ROW("R"), + /** + * field + */ + FIELD("F"); + + private String value; + + SyncMode(String value) { + this.value = value; + } + + public static SyncMode valuesOf(String value) { + SyncMode[] modes = values(); + for (SyncMode mode : modes) { + if (mode.value.equalsIgnoreCase(value)) { + return mode; + } + } + return null; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + + public boolean isRow() { + return this.equals(SyncMode.ROW); + } + + public boolean isField() { + return this.equals(SyncMode.FIELD); + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/RecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/RecordOffset.java new file mode 100644 index 0000000000..f78585ca15 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/RecordOffset.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.offset; + +public abstract class RecordOffset { + + public abstract Class getRecordOffsetClass(); + + public RecordOffset() { + + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/RecordPartition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/RecordPartition.java new file mode 100644 index 0000000000..00e4c30e48 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/RecordPartition.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.offset; + +public abstract class RecordPartition { + + public abstract Class getRecordPartitionClass(); + + public RecordPartition() { + + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/RecordPosition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/RecordPosition.java new file mode 100644 index 0000000000..5f45390b73 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/RecordPosition.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.offset; + +import org.apache.eventmesh.common.remote.offset.S3.S3RecordOffset; +import org.apache.eventmesh.common.remote.offset.S3.S3RecordPartition; +import org.apache.eventmesh.common.remote.offset.canal.CanalFullRecordOffset; +import org.apache.eventmesh.common.remote.offset.canal.CanalFullRecordPartition; +import org.apache.eventmesh.common.remote.offset.canal.CanalRecordOffset; +import org.apache.eventmesh.common.remote.offset.canal.CanalRecordPartition; +import org.apache.eventmesh.common.remote.offset.file.FileRecordOffset; +import org.apache.eventmesh.common.remote.offset.file.FileRecordPartition; +import org.apache.eventmesh.common.remote.offset.kafka.KafkaRecordOffset; +import org.apache.eventmesh.common.remote.offset.kafka.KafkaRecordPartition; +import org.apache.eventmesh.common.remote.offset.pulsar.PulsarRecordOffset; +import org.apache.eventmesh.common.remote.offset.pulsar.PulsarRecordPartition; +import org.apache.eventmesh.common.remote.offset.rocketmq.RocketMQRecordOffset; +import org.apache.eventmesh.common.remote.offset.rocketmq.RocketMQRecordPartition; + +import java.util.Objects; + +import com.fasterxml.jackson.annotation.JsonSubTypes; +import com.fasterxml.jackson.annotation.JsonTypeInfo; + +public class RecordPosition { + + @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS) + @JsonSubTypes({ + @JsonSubTypes.Type(value = CanalRecordPartition.class, name = "CanalRecordPartition"), + @JsonSubTypes.Type(value = CanalFullRecordPartition.class, name = "CanalFullRecordPartition"), + @JsonSubTypes.Type(value = FileRecordPartition.class, name = "FileRecordPartition"), + @JsonSubTypes.Type(value = S3RecordPartition.class, name = "S3RecordPartition"), + @JsonSubTypes.Type(value = KafkaRecordPartition.class, name = "KafkaRecordPartition"), + @JsonSubTypes.Type(value = PulsarRecordPartition.class, name = "PulsarRecordPartition"), + @JsonSubTypes.Type(value = RocketMQRecordPartition.class, name = "RocketMQRecordPartition"), + }) + private RecordPartition recordPartition; + + private Class recordPartitionClazz; + + @JsonTypeInfo(use = JsonTypeInfo.Id.CLASS) + @JsonSubTypes({ + @JsonSubTypes.Type(value = CanalRecordOffset.class, name = "CanalRecordOffset"), + @JsonSubTypes.Type(value = CanalFullRecordOffset.class, name = "CanalFullRecordOffset"), + @JsonSubTypes.Type(value = FileRecordOffset.class, name = "FileRecordOffset"), + @JsonSubTypes.Type(value = S3RecordOffset.class, name = "S3RecordOffset"), + @JsonSubTypes.Type(value = KafkaRecordOffset.class, name = "KafkaRecordOffset"), + @JsonSubTypes.Type(value = PulsarRecordOffset.class, name = "PulsarRecordOffset"), + @JsonSubTypes.Type(value = RocketMQRecordOffset.class, name = "RocketMQRecordOffset"), + }) + private RecordOffset recordOffset; + + private Class recordOffsetClazz; + + public RecordPosition() { + + } + + public RecordPosition( + RecordPartition recordPartition, RecordOffset recordOffset) { + this.recordPartition = recordPartition; + this.recordOffset = recordOffset; + this.recordPartitionClazz = recordPartition.getRecordPartitionClass(); + this.recordOffsetClazz = recordOffset.getRecordOffsetClass(); + } + + public RecordPartition getRecordPartition() { + return recordPartition; + } + + public void setRecordPartition(RecordPartition recordPartition) { + this.recordPartition = recordPartition; + if (recordPartition == null) { + this.recordPartitionClazz = null; + return; + } + this.recordPartitionClazz = recordPartition.getRecordPartitionClass(); + } + + public RecordOffset getRecordOffset() { + return recordOffset; + } + + public void setRecordOffset(RecordOffset recordOffset) { + this.recordOffset = recordOffset; + if (recordOffset == null) { + this.recordOffsetClazz = null; + return; + } + this.recordOffsetClazz = recordOffset.getRecordOffsetClass(); + } + + public Class getRecordPartitionClazz() { + return recordPartitionClazz; + } + + public Class getRecordOffsetClazz() { + return recordOffsetClazz; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof RecordPosition)) { + return false; + } + RecordPosition position = (RecordPosition) o; + return recordPartition.equals(position.recordPartition) && recordOffset.equals(position.recordOffset); + } + + @Override + public int hashCode() { + return Objects.hash(recordPartition, recordOffset); + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/S3/S3RecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/S3/S3RecordOffset.java new file mode 100644 index 0000000000..9f38e4b2c6 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/S3/S3RecordOffset.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.offset.S3; + +import org.apache.eventmesh.common.remote.offset.RecordOffset; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +@EqualsAndHashCode(callSuper = true) +@Data +@ToString +public class S3RecordOffset extends RecordOffset { + + private Long offset; + + @Override + public Class getRecordOffsetClass() { + return S3RecordOffset.class; + } + + public S3RecordOffset() { + + } +} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/RecordPartition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/S3/S3RecordPartition.java similarity index 58% rename from eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/RecordPartition.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/S3/S3RecordPartition.java index 959a6b9b3c..3e42a4d093 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/RecordPartition.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/S3/S3RecordPartition.java @@ -15,32 +15,33 @@ * limitations under the License. */ -package org.apache.eventmesh.openconnect.offsetmgmt.api.data; +package org.apache.eventmesh.common.remote.offset.S3; + +import org.apache.eventmesh.common.remote.offset.RecordPartition; -import java.util.HashMap; -import java.util.Map; import java.util.Objects; -public class RecordPartition { +import lombok.Data; +import lombok.ToString; - /** - * if pull message from mq key=topic, - * value=topicName key=brokerName, - * value=brokerName key=queueId, - * value=queueId - */ - private Map partition = new HashMap<>(); - public RecordPartition() { +@Data +@ToString +public class S3RecordPartition extends RecordPartition { - } + private String region; + + private String bucket; - public RecordPartition(Map partition) { - this.partition = partition; + private String fileName; + + @Override + public Class getRecordPartitionClass() { + return S3RecordPartition.class; } - public Map getPartition() { - return partition; + public S3RecordPartition() { + super(); } @Override @@ -48,15 +49,15 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (!(o instanceof RecordPartition)) { + if (o == null || getClass() != o.getClass()) { return false; } - RecordPartition partition1 = (RecordPartition) o; - return Objects.equals(partition, partition1.partition); + S3RecordPartition that = (S3RecordPartition) o; + return Objects.equals(fileName, that.fileName); } @Override public int hashCode() { - return Objects.hash(partition); + return Objects.hash(fileName); } } diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordOffset.java new file mode 100644 index 0000000000..a0a077b7f5 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordOffset.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.offset.canal; + +import org.apache.eventmesh.common.config.connector.rdb.canal.JobRdbFullPosition; +import org.apache.eventmesh.common.remote.offset.RecordOffset; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +@Data +@EqualsAndHashCode(callSuper = true) +@ToString +public class CanalFullRecordOffset extends RecordOffset { + private JobRdbFullPosition position; + + @Override + public Class getRecordOffsetClass() { + return CanalFullRecordOffset.class; + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordPartition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordPartition.java new file mode 100644 index 0000000000..73626fa78f --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalFullRecordPartition.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.offset.canal; + +import org.apache.eventmesh.common.remote.offset.RecordPartition; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + + +@Data +@ToString +@EqualsAndHashCode(callSuper = true) +public class CanalFullRecordPartition extends RecordPartition { + + @Override + public Class getRecordPartitionClass() { + return CanalFullRecordPartition.class; + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalRecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalRecordOffset.java new file mode 100644 index 0000000000..d0f2053f4d --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalRecordOffset.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.offset.canal; + +import org.apache.eventmesh.common.remote.offset.RecordOffset; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +@EqualsAndHashCode(callSuper = true) +@Data +@ToString +public class CanalRecordOffset extends RecordOffset { + + private Long offset; + + // mysql instance gtid range + private String gtid; + + private String currentGtid; + + @Override + public Class getRecordOffsetClass() { + return CanalRecordOffset.class; + } + + public CanalRecordOffset() { + + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalRecordPartition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalRecordPartition.java new file mode 100644 index 0000000000..ded82306e3 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/canal/CanalRecordPartition.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.offset.canal; + +import org.apache.eventmesh.common.remote.offset.RecordPartition; + +import java.util.Objects; + +import lombok.Data; +import lombok.ToString; + + +@Data +@ToString +public class CanalRecordPartition extends RecordPartition { + + private String serverUUID; + + private String journalName; + + private Long timeStamp; + + @Override + public Class getRecordPartitionClass() { + return CanalRecordPartition.class; + } + + public CanalRecordPartition() { + super(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + CanalRecordPartition that = (CanalRecordPartition) o; + return Objects.equals(journalName, that.journalName) && Objects.equals(timeStamp, that.timeStamp); + } + + @Override + public int hashCode() { + return Objects.hash(journalName, timeStamp); + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/file/FileRecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/file/FileRecordOffset.java new file mode 100644 index 0000000000..cda293cb8c --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/file/FileRecordOffset.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.offset.file; + +import org.apache.eventmesh.common.remote.offset.RecordOffset; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +@EqualsAndHashCode(callSuper = true) +@Data +@ToString +public class FileRecordOffset extends RecordOffset { + + private Long offset; + + @Override + public Class getRecordOffsetClass() { + return FileRecordOffset.class; + } + + public FileRecordOffset() { + + } +} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/RecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/file/FileRecordPartition.java similarity index 61% rename from eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/RecordOffset.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/file/FileRecordPartition.java index fee70de27e..1a6dddad44 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/RecordOffset.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/file/FileRecordPartition.java @@ -15,30 +15,29 @@ * limitations under the License. */ -package org.apache.eventmesh.openconnect.offsetmgmt.api.data; +package org.apache.eventmesh.common.remote.offset.file; + +import org.apache.eventmesh.common.remote.offset.RecordPartition; -import java.util.HashMap; -import java.util.Map; import java.util.Objects; -public class RecordOffset { +import lombok.Data; +import lombok.ToString; - /** - * if pull message from mq key=queueOffset, - * value=queueOffset value - */ - private Map offset = new HashMap<>(); - public RecordOffset() { +@Data +@ToString +public class FileRecordPartition extends RecordPartition { - } + private String fileName; - public RecordOffset(Map offset) { - this.offset = offset; + @Override + public Class getRecordPartitionClass() { + return FileRecordPartition.class; } - public Map getOffset() { - return offset; + public FileRecordPartition() { + super(); } @Override @@ -46,16 +45,15 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (!(o instanceof RecordOffset)) { + if (o == null || getClass() != o.getClass()) { return false; } - RecordOffset offset1 = (RecordOffset) o; - return Objects.equals(offset, offset1.offset); + FileRecordPartition that = (FileRecordPartition) o; + return Objects.equals(fileName, that.fileName); } @Override public int hashCode() { - return Objects.hash(offset); + return Objects.hash(fileName); } - } diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/http/HttpRecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/http/HttpRecordOffset.java new file mode 100644 index 0000000000..f5084c755f --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/http/HttpRecordOffset.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.offset.http; + +import org.apache.eventmesh.common.remote.offset.RecordOffset; + +import java.util.Map; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +@EqualsAndHashCode(callSuper = true) +@Data +@ToString +public class HttpRecordOffset extends RecordOffset { + + private Map offsetMap; + + @Override + public Class getRecordOffsetClass() { + return HttpRecordOffset.class; + } + + public HttpRecordOffset() { + + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/http/HttpRecordPartition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/http/HttpRecordPartition.java new file mode 100644 index 0000000000..453b3b501e --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/http/HttpRecordPartition.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.offset.http; + +import org.apache.eventmesh.common.remote.offset.RecordPartition; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + + +@EqualsAndHashCode(callSuper = true) +@Data +@ToString +public class HttpRecordPartition extends RecordPartition { + + @Override + public Class getRecordPartitionClass() { + return HttpRecordPartition.class; + } + + public HttpRecordPartition() { + super(); + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/jdbc/JdbcRecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/jdbc/JdbcRecordOffset.java new file mode 100644 index 0000000000..a97a90e658 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/jdbc/JdbcRecordOffset.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.offset.jdbc; + +import org.apache.eventmesh.common.remote.offset.RecordOffset; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +@EqualsAndHashCode(callSuper = true) +@Data +@ToString +public class JdbcRecordOffset extends RecordOffset { + + private Long offset; + + @Override + public Class getRecordOffsetClass() { + return JdbcRecordOffset.class; + } + + public JdbcRecordOffset() { + + } +} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/RecordPosition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/jdbc/JdbcRecordPartition.java similarity index 57% rename from eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/RecordPosition.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/jdbc/JdbcRecordPartition.java index 0b836ab607..1eb6937a87 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/RecordPosition.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/jdbc/JdbcRecordPartition.java @@ -15,28 +15,29 @@ * limitations under the License. */ -package org.apache.eventmesh.openconnect.offsetmgmt.api.data; +package org.apache.eventmesh.common.remote.offset.jdbc; + +import org.apache.eventmesh.common.remote.offset.RecordPartition; import java.util.Objects; -public class RecordPosition { +import lombok.Data; +import lombok.ToString; - private final RecordPartition recordPartition; - private final RecordOffset recordOffset; +@Data +@ToString +public class JdbcRecordPartition extends RecordPartition { - public RecordPosition( - RecordPartition recordPartition, RecordOffset recordOffset) { - this.recordPartition = recordPartition; - this.recordOffset = recordOffset; - } + private String fileName; - public RecordPartition getPartition() { - return recordPartition; + @Override + public Class getRecordPartitionClass() { + return JdbcRecordPartition.class; } - public RecordOffset getOffset() { - return recordOffset; + public JdbcRecordPartition() { + super(); } @Override @@ -44,15 +45,15 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (!(o instanceof RecordPosition)) { + if (o == null || getClass() != o.getClass()) { return false; } - RecordPosition position = (RecordPosition) o; - return recordPartition.equals(position.recordPartition) && recordOffset.equals(position.recordOffset); + JdbcRecordPartition that = (JdbcRecordPartition) o; + return Objects.equals(fileName, that.fileName); } @Override public int hashCode() { - return Objects.hash(recordPartition, recordOffset); + return Objects.hash(fileName); } } diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/kafka/KafkaRecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/kafka/KafkaRecordOffset.java new file mode 100644 index 0000000000..134ef06fe4 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/kafka/KafkaRecordOffset.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.offset.kafka; + +import org.apache.eventmesh.common.remote.offset.RecordOffset; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +@EqualsAndHashCode(callSuper = true) +@Data +@ToString +public class KafkaRecordOffset extends RecordOffset { + + private Long offset; + + @Override + public Class getRecordOffsetClass() { + return KafkaRecordOffset.class; + } + + public KafkaRecordOffset() { + + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/kafka/KafkaRecordPartition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/kafka/KafkaRecordPartition.java new file mode 100644 index 0000000000..77dec82267 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/kafka/KafkaRecordPartition.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.offset.kafka; + +import org.apache.eventmesh.common.remote.offset.RecordPartition; + +import java.util.Objects; + +import lombok.Data; +import lombok.ToString; + + +@Data +@ToString +public class KafkaRecordPartition extends RecordPartition { + + private String topic; + + private Integer partition; + + @Override + public Class getRecordPartitionClass() { + return KafkaRecordPartition.class; + } + + public KafkaRecordPartition() { + super(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + KafkaRecordPartition that = (KafkaRecordPartition) o; + return Objects.equals(topic, that.topic) && Objects.equals(partition, that.partition); + } + + @Override + public int hashCode() { + return Objects.hash(topic, partition); + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/prometheus/PrometheusRecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/prometheus/PrometheusRecordOffset.java new file mode 100644 index 0000000000..acb5b3ce02 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/prometheus/PrometheusRecordOffset.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.offset.prometheus; + +import org.apache.eventmesh.common.remote.offset.RecordOffset; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +@EqualsAndHashCode(callSuper = true) +@Data +@ToString +public class PrometheusRecordOffset extends RecordOffset { + + private Long offset; + + @Override + public Class getRecordOffsetClass() { + return PrometheusRecordOffset.class; + } + + public PrometheusRecordOffset() { + + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/prometheus/PrometheusRecordPartition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/prometheus/PrometheusRecordPartition.java new file mode 100644 index 0000000000..74302504c2 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/prometheus/PrometheusRecordPartition.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.offset.prometheus; + +import org.apache.eventmesh.common.remote.offset.RecordPartition; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + + +@EqualsAndHashCode(callSuper = true) +@Data +@ToString +public class PrometheusRecordPartition extends RecordPartition { + + @Override + public Class getRecordPartitionClass() { + return PrometheusRecordPartition.class; + } + + public PrometheusRecordPartition() { + super(); + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/pulsar/PulsarRecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/pulsar/PulsarRecordOffset.java new file mode 100644 index 0000000000..bbe3d43803 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/pulsar/PulsarRecordOffset.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.offset.pulsar; + +import org.apache.eventmesh.common.remote.offset.RecordOffset; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +@EqualsAndHashCode(callSuper = true) +@Data +@ToString +public class PulsarRecordOffset extends RecordOffset { + + /** + * if pull message from mq + * key=queueOffset, + * value=queueOffset value + */ + private Long queueOffset; + + @Override + public Class getRecordOffsetClass() { + return PulsarRecordOffset.class; + } + + public PulsarRecordOffset() { + + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/pulsar/PulsarRecordPartition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/pulsar/PulsarRecordPartition.java new file mode 100644 index 0000000000..0c152b50b3 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/pulsar/PulsarRecordPartition.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.offset.pulsar; + +import org.apache.eventmesh.common.remote.offset.RecordPartition; + +import java.util.Objects; + +import lombok.Data; +import lombok.ToString; + + +@Data +@ToString +public class PulsarRecordPartition extends RecordPartition { + + private String topic; + + private Long queueId; + + + @Override + public Class getRecordPartitionClass() { + return PulsarRecordPartition.class; + } + + public PulsarRecordPartition() { + super(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + PulsarRecordPartition that = (PulsarRecordPartition) o; + return Objects.equals(topic, that.topic) && Objects.equals(queueId, + that.queueId); + } + + @Override + public int hashCode() { + return Objects.hash(topic, queueId); + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/rocketmq/RocketMQRecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/rocketmq/RocketMQRecordOffset.java new file mode 100644 index 0000000000..56094c9e72 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/rocketmq/RocketMQRecordOffset.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.offset.rocketmq; + +import org.apache.eventmesh.common.remote.offset.RecordOffset; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +@EqualsAndHashCode(callSuper = true) +@Data +@ToString +public class RocketMQRecordOffset extends RecordOffset { + + /** + * if pull message from mq + * key=queueOffset, + * value=queueOffset value + */ + private Long queueOffset; + + @Override + public Class getRecordOffsetClass() { + return RocketMQRecordOffset.class; + } + + public RocketMQRecordOffset() { + + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/rocketmq/RocketMQRecordPartition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/rocketmq/RocketMQRecordPartition.java new file mode 100644 index 0000000000..0963af6f59 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/rocketmq/RocketMQRecordPartition.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.offset.rocketmq; + +import org.apache.eventmesh.common.remote.offset.RecordPartition; + +import java.util.Objects; + +import lombok.Data; +import lombok.ToString; + + +@Data +@ToString +public class RocketMQRecordPartition extends RecordPartition { + + /** + * key=topic,value=topicName key=brokerName,value=brokerName key=queueId,value=queueId + */ + + private String broker; + + private String topic; + + private String queueId; + + + @Override + public Class getRecordPartitionClass() { + return RocketMQRecordPartition.class; + } + + public RocketMQRecordPartition() { + super(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RocketMQRecordPartition that = (RocketMQRecordPartition) o; + return Objects.equals(broker, that.broker) && Objects.equals(topic, that.topic) && Objects.equals(queueId, + that.queueId); + } + + @Override + public int hashCode() { + return Objects.hash(broker, topic, queueId); + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/spring/SpringRecordOffset.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/spring/SpringRecordOffset.java new file mode 100644 index 0000000000..d0916c5175 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/spring/SpringRecordOffset.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.offset.spring; + +import org.apache.eventmesh.common.remote.offset.RecordOffset; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +@EqualsAndHashCode(callSuper = true) +@Data +@ToString +public class SpringRecordOffset extends RecordOffset { + + private Long offset; + + @Override + public Class getRecordOffsetClass() { + return SpringRecordOffset.class; + } + + public SpringRecordOffset() { + + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/spring/SpringRecordPartition.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/spring/SpringRecordPartition.java new file mode 100644 index 0000000000..4b536da139 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/offset/spring/SpringRecordPartition.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.offset.spring; + +import org.apache.eventmesh.common.remote.offset.RecordPartition; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + + +@EqualsAndHashCode(callSuper = true) +@Data +@ToString +public class SpringRecordPartition extends RecordPartition { + + @Override + public Class getRecordPartitionClass() { + return SpringRecordPartition.class; + } + + public SpringRecordPartition() { + super(); + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/payload/IPayload.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/payload/IPayload.java new file mode 100644 index 0000000000..aca27ffc21 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/payload/IPayload.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.payload; + +/** + * IPayload + */ +public interface IPayload { + +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/payload/PayloadFactory.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/payload/PayloadFactory.java new file mode 100644 index 0000000000..74e4880443 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/payload/PayloadFactory.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.payload; + +import java.lang.reflect.Modifier; +import java.util.Map; +import java.util.ServiceLoader; +import java.util.concurrent.ConcurrentHashMap; + +public class PayloadFactory { + + private PayloadFactory() { + } + + private static class PayloadFactoryHolder { + + private static final PayloadFactory INSTANCE = new PayloadFactory(); + } + + public static PayloadFactory getInstance() { + return PayloadFactoryHolder.INSTANCE; + } + + private final Map> registryPayload = new ConcurrentHashMap<>(); + + private boolean initialized = false; + + public void init() { + scan(); + } + + private synchronized void scan() { + if (initialized) { + return; + } + ServiceLoader payloads = ServiceLoader.load(IPayload.class); + for (IPayload payload : payloads) { + register(payload.getClass().getSimpleName(), payload.getClass()); + } + initialized = true; + } + + public void register(String type, Class clazz) { + if (Modifier.isAbstract(clazz.getModifiers())) { + return; + } + if (registryPayload.containsKey(type)) { + throw new RuntimeException(String.format("Fail to register, type:%s ,clazz:%s ", type, clazz.getName())); + } + registryPayload.put(type, clazz); + } + + public Class getClassByType(String type) { + return registryPayload.get(type); + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/payload/PayloadUtil.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/payload/PayloadUtil.java new file mode 100644 index 0000000000..6a21d5a825 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/payload/PayloadUtil.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.payload; + +import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Payload; +import org.apache.eventmesh.common.remote.exception.ErrorCode; +import org.apache.eventmesh.common.remote.exception.PayloadFormatException; +import org.apache.eventmesh.common.utils.JsonUtils; + +import com.fasterxml.jackson.databind.util.ByteBufferBackedInputStream; +import com.google.protobuf.Any; +import com.google.protobuf.UnsafeByteOperations; + +public class PayloadUtil { + + public static Payload from(IPayload payload) { + byte[] payloadBytes = JsonUtils.toJSONBytes(payload); + Metadata.Builder metadata = Metadata.newBuilder().setType(payload.getClass().getSimpleName()); + return Payload.newBuilder().setMetadata(metadata).setBody(Any.newBuilder().setValue(UnsafeByteOperations.unsafeWrap(payloadBytes))).build(); + } + + public static IPayload parse(Payload payload) { + Class targetClass = PayloadFactory.getInstance().getClassByType(payload.getMetadata().getType()); + if (targetClass == null) { + throw new PayloadFormatException(ErrorCode.BAD_REQUEST, + "unknown payload type:" + payload.getMetadata().getType()); + } + return (IPayload) JsonUtils.parseObject(new ByteBufferBackedInputStream(payload.getBody().getValue().asReadOnlyByteBuffer()), targetClass); + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/BaseRemoteRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/BaseRemoteRequest.java new file mode 100644 index 0000000000..b8c4c06207 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/BaseRemoteRequest.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.request; + +import org.apache.eventmesh.common.remote.payload.IPayload; + +import java.util.HashMap; +import java.util.Map; + +import lombok.Getter; + +@Getter +public abstract class BaseRemoteRequest implements IPayload { + + private final Map header = new HashMap<>(); + + public void addHeader(String key, String value) { + if (key == null || value == null) { + return; + } + header.put(key, value); + } + + public void addHeaders(Map map) { + if (map == null || map.isEmpty()) { + return; + } + map.forEach((k, v) -> { + if (k == null || v == null) { + return; + } + this.header.put(k, v); + }); + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/CreateOrUpdateDataSourceReq.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/CreateOrUpdateDataSourceReq.java new file mode 100644 index 0000000000..f78349703a --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/CreateOrUpdateDataSourceReq.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.request; + +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.remote.datasource.DataSourceType; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * create or update datasource with custom data source config + */ +@Data +@EqualsAndHashCode(callSuper = true) +public class CreateOrUpdateDataSourceReq extends BaseRemoteRequest { + + private Integer id; + private DataSourceType type; + private String desc; + private Config config; + private String configClass; + private String region; + private String operator; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/CreateTaskRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/CreateTaskRequest.java new file mode 100644 index 0000000000..b09a3e10ed --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/CreateTaskRequest.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.request; + +import org.apache.eventmesh.common.remote.TransportType; +import org.apache.eventmesh.common.remote.job.JobType; + +import java.util.List; +import java.util.Map; + +import lombok.Data; + +/** + * Description: create task without task id, otherwise update task + */ +@Data +public class CreateTaskRequest { + + private String taskId; + + // task name + private String taskName; + + // task description + private String taskDesc; + + // task owner or updater + private String uid; + + private List jobs; + + // task source region + private String sourceRegion; + + // task target region + private String targetRegion; + + // mark request send by other region admin, default is false + private boolean flag = false; + + @Data + public static class JobDetail { + + private String jobId; + + private String jobDesc; + + // full/increase/check + private JobType jobType; + + private Map sourceDataSource; + + private String sourceConnectorDesc; + + private Map sinkDataSource; + + private String sinkConnectorDesc; + + private TransportType transportType; + + // job request from region + private String fromRegion; + + // job actually running region + private String runningRegion; + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/FetchJobRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/FetchJobRequest.java new file mode 100644 index 0000000000..2693a4a3aa --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/FetchJobRequest.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.request; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class FetchJobRequest extends BaseRemoteRequest { + private String jobID; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/FetchPositionRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/FetchPositionRequest.java new file mode 100644 index 0000000000..90563251ab --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/FetchPositionRequest.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.request; + +import org.apache.eventmesh.common.remote.datasource.DataSourceType; +import org.apache.eventmesh.common.remote.offset.RecordPosition; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class FetchPositionRequest extends BaseRemoteRequest { + + private String jobID; + + private String address; + + private RecordPosition recordPosition; + + private DataSourceType dataSourceType; + +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/QueryTaskInfoRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/QueryTaskInfoRequest.java new file mode 100644 index 0000000000..c0973cf63d --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/QueryTaskInfoRequest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.request; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class QueryTaskInfoRequest { + + private String taskDesc; + + private String taskID; + + private String jobType; + + private String sourceDataID; + + private String targetDataID; + + private String ip; + + private String sourceTableName; + + private String taskMathID; + + private Integer currentPage; + + private Integer pageSize; + +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/QueryTaskMonitorRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/QueryTaskMonitorRequest.java new file mode 100644 index 0000000000..cd777d5019 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/QueryTaskMonitorRequest.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.request; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +@Data +@EqualsAndHashCode(callSuper = true) +@ToString +public class QueryTaskMonitorRequest extends BaseRemoteRequest { + private String taskID; + private String jobID; + private long limit; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/RecordPositionRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/RecordPositionRequest.java new file mode 100644 index 0000000000..b04a6f1041 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/RecordPositionRequest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.request; + +import org.apache.eventmesh.common.remote.JobState; +import org.apache.eventmesh.common.remote.datasource.DataSourceType; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +@Data +@EqualsAndHashCode(callSuper = true) +@ToString +public class RecordPositionRequest extends BaseRemoteRequest { + + private String fullJobID; + + private String increaseJobID; + // prepare to update job state to current state + private JobState updateState; + + private String address; + + private DataSourceType dataSourceType; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportHeartBeatRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportHeartBeatRequest.java new file mode 100644 index 0000000000..fb61ca3618 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportHeartBeatRequest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.request; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class ReportHeartBeatRequest extends BaseRemoteRequest { + + private String address; + + private String reportedTimeStamp; + + private String jobID; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportJobRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportJobRequest.java new file mode 100644 index 0000000000..aec33e4616 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportJobRequest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.request; + +import org.apache.eventmesh.common.remote.JobState; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +@Data +@EqualsAndHashCode(callSuper = true) +@ToString +public class ReportJobRequest extends BaseRemoteRequest { + + private String jobID; + + private JobState state; + + private String address; + +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportMonitorRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportMonitorRequest.java new file mode 100644 index 0000000000..12278df27f --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportMonitorRequest.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.request; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +@Data +@EqualsAndHashCode(callSuper = true) +@ToString +public class ReportMonitorRequest extends BaseRemoteRequest { + private String taskID; + private String jobID; + private String address; + private String connectorStage; + private String transportType; + private long totalReqNum; + private long totalTimeCost; + private long maxTimeCost; + private long avgTimeCost; + private double tps; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportPositionRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportPositionRequest.java new file mode 100644 index 0000000000..42694d5675 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportPositionRequest.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.request; + +import org.apache.eventmesh.common.remote.TaskState; +import org.apache.eventmesh.common.remote.datasource.DataSourceType; +import org.apache.eventmesh.common.remote.offset.RecordPosition; + +import java.util.List; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +@Data +@EqualsAndHashCode(callSuper = true) +@ToString +public class ReportPositionRequest extends BaseRemoteRequest { + + private String jobID; + + private List recordPositionList; + + private TaskState state; + + private String address; + + private DataSourceType dataSourceType; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportVerifyRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportVerifyRequest.java new file mode 100644 index 0000000000..bd38881c3d --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/ReportVerifyRequest.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.request; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.ToString; + +@Data +@EqualsAndHashCode(callSuper = true) +@ToString +public class ReportVerifyRequest extends BaseRemoteRequest { + + private String taskID; + + private String jobID; + + private String recordID; + + private String recordSig; + + private String connectorName; + + private String connectorStage; + + private String position; +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/TaskBachRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/TaskBachRequest.java new file mode 100644 index 0000000000..306badc156 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/TaskBachRequest.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.request; + +import lombok.Data; + +@Data +public class TaskBachRequest { + + private String taskID; + + private String taskName; + +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/TaskIDRequest.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/TaskIDRequest.java new file mode 100644 index 0000000000..37a72916ff --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/request/TaskIDRequest.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.request; + +import lombok.Data; + +@Data +public class TaskIDRequest { + + private String taskID; + +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/BaseRemoteResponse.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/BaseRemoteResponse.java new file mode 100644 index 0000000000..84ea3661bd --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/BaseRemoteResponse.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.response; + +import org.apache.eventmesh.common.remote.payload.IPayload; + +import java.util.HashMap; +import java.util.Map; + +import lombok.Getter; +import lombok.Setter; + +@Getter +public abstract class BaseRemoteResponse implements IPayload { + @Setter + private boolean success = true; + @Setter + private int errorCode; + @Setter + private String desc; + + private Map header = new HashMap<>(); + @Setter + private T data; + + public void addHeader(String key, String value) { + if (key == null || value == null) { + return; + } + header.put(key, value); + } + + public void addHeaders(Map map) { + if (map == null || map.isEmpty()) { + return; + } + map.forEach((k, v) -> { + if (k == null || v == null) { + return; + } + this.header.put(k, v); + }); + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/CreateTaskResponse.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/CreateTaskResponse.java new file mode 100644 index 0000000000..24e7871e04 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/CreateTaskResponse.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.response; + +import org.apache.eventmesh.common.remote.request.CreateTaskRequest; + +import java.util.List; + +import lombok.Data; + +@Data +public class CreateTaskResponse { + + private String taskId; + + private List jobIdList; + +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/FetchJobResponse.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/FetchJobResponse.java new file mode 100644 index 0000000000..95d2d157e0 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/FetchJobResponse.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.response; + +import org.apache.eventmesh.common.remote.TaskState; +import org.apache.eventmesh.common.remote.TransportType; +import org.apache.eventmesh.common.remote.exception.ErrorCode; +import org.apache.eventmesh.common.remote.job.JobConnectorConfig; +import org.apache.eventmesh.common.remote.job.JobType; +import org.apache.eventmesh.common.remote.offset.RecordPosition; + +import java.util.List; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class FetchJobResponse extends BaseRemoteResponse { + + private String id; + + private TransportType transportType; + + private JobConnectorConfig connectorConfig; + + private List position; + + private TaskState state; + + private JobType type; + + public static FetchJobResponse successResponse() { + FetchJobResponse response = new FetchJobResponse(); + response.setSuccess(true); + response.setErrorCode(ErrorCode.SUCCESS); + return response; + } + + public static FetchJobResponse failResponse(int code, String desc) { + FetchJobResponse response = new FetchJobResponse(); + response.setSuccess(false); + response.setErrorCode(code); + response.setDesc(desc); + return response; + } + +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/FetchPositionResponse.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/FetchPositionResponse.java new file mode 100644 index 0000000000..613623d654 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/FetchPositionResponse.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.response; + +import org.apache.eventmesh.common.remote.exception.ErrorCode; +import org.apache.eventmesh.common.remote.offset.RecordPosition; + +import java.util.List; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class FetchPositionResponse extends BaseRemoteResponse { + + private List recordPosition; + + public static FetchPositionResponse successResponse() { + FetchPositionResponse response = new FetchPositionResponse(); + response.setSuccess(true); + response.setErrorCode(ErrorCode.SUCCESS); + return response; + } + + public static FetchPositionResponse successResponse(List recordPosition) { + FetchPositionResponse response = successResponse(); + response.setRecordPosition(recordPosition); + return response; + } + + public static FetchPositionResponse failResponse(int code, String desc) { + FetchPositionResponse response = new FetchPositionResponse(); + response.setSuccess(false); + response.setErrorCode(code); + response.setDesc(desc); + return response; + } + +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/HttpResponseResult.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/HttpResponseResult.java new file mode 100644 index 0000000000..b6ca8cef0d --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/HttpResponseResult.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.response; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@AllArgsConstructor +public class HttpResponseResult { + + private Integer code; + + private String message; + + private T data; + + public HttpResponseResult(Integer code, String message) { + this.code = code; + this.message = message; + } + + public HttpResponseResult(Integer code, T data) { + this.code = code; + this.data = data; + } + + public static HttpResponseResult success() { + return new HttpResponseResult<>(200, "success"); + } + + public static HttpResponseResult success(T data) { + return new HttpResponseResult<>(200, "success", data); + } + + public static HttpResponseResult failed() { + return new HttpResponseResult<>(500, "failed"); + } + + public static HttpResponseResult failed(T data) { + return new HttpResponseResult<>(500, "failed", data); + } + + public static HttpResponseResult exception(T data) { + return new HttpResponseResult<>(300, data); + } + +} \ No newline at end of file diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/QueryTaskInfoResponse.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/QueryTaskInfoResponse.java new file mode 100644 index 0000000000..4c0c536eae --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/QueryTaskInfoResponse.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.response; + +import java.util.Date; +import java.util.List; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class QueryTaskInfoResponse { + + // event_mesh_task_info + private Integer id; + + private String taskID; + + private String taskName; + + private String taskDesc; + + private String taskState; + + private String sourceRegion; + + private String targetRegion; + + private String createUid; + + private String updateUid; + + private Date createTime; + + private Date updateTime; + + List eventMeshJobInfoList; + + @Data + public static class EventMeshJobInfo { + // event_mesh_job_info + private Integer id; + + private String jobID; + + private String jobDesc; + + private String taskID; + + private String transportType; + + private Integer sourceData; + + private Integer targetData; + + private String jobState; + + private String jobType; + + // job request from region + private String fromRegion; + + // job actually running region + private String runningRegion; + + private String createUid; + + private String updateUid; + + private Date createTime; + + private Date updateTime; + + // private List eventMeshDataSource; + + private EventMeshDataSource dataSource; + + private EventMeshDataSource dataSink; + + private EventMeshMysqlPosition eventMeshMysqlPosition; + + } + + @Data + public static class EventMeshDataSource { + + private Integer id; + + private String dataType; + + private String description; + + private String configuration; + + private String configurationClass; + + private String region; + + private String createUid; + + private String updateUid; + + private Date createTime; + + private Date updateTime; + } + + @Data + public static class EventMeshMysqlPosition { + + private Integer id; + + private String jobID; + + private String serverUUID; + + private String address; + + private Long position; + + private String gtid; + + private String currentGtid; + + private Long timestamp; + + private String journalName; + + private Date createTime; + + private Date updateTime; + } + +} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/config/SourceConnectorConfig.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/QueryTaskMonitorResponse.java similarity index 79% rename from eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/config/SourceConnectorConfig.java rename to eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/QueryTaskMonitorResponse.java index 873a0d1922..432729a995 100644 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/config/SourceConnectorConfig.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/QueryTaskMonitorResponse.java @@ -15,18 +15,17 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.http.source.config; +package org.apache.eventmesh.common.remote.response; -import lombok.Data; +import org.apache.eventmesh.common.remote.task.TaskMonitor; -@Data -public class SourceConnectorConfig { +import java.util.List; - private String connectorName; +import lombok.Data; - private String path; +@Data +public class QueryTaskMonitorResponse { - private int port; + private List taskMonitors; - private int idleTimeout; } diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/SimpleResponse.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/SimpleResponse.java new file mode 100644 index 0000000000..a4cdd52f99 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/response/SimpleResponse.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.response; + +import org.apache.eventmesh.common.remote.exception.ErrorCode; + +public class SimpleResponse extends BaseRemoteResponse { + /** + * just mean remote received or process success + */ + public static SimpleResponse success() { + return new SimpleResponse(); + } + + public static SimpleResponse fail(int errorCode, String msg) { + SimpleResponse response = new SimpleResponse(); + response.setErrorCode(errorCode); + response.setDesc(msg); + response.setSuccess(false); + return response; + } + + + /** + * build an error response. + * + * @param exception exception + * @return response + */ + public static SimpleResponse fail(Throwable exception) { + return fail(ErrorCode.INTERNAL_ERR, exception.getMessage()); + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/task/TaskMonitor.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/task/TaskMonitor.java new file mode 100644 index 0000000000..6d303eec3b --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/remote/task/TaskMonitor.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.remote.task; + +import java.io.Serializable; +import java.util.Date; + +import lombok.Data; + +@Data +public class TaskMonitor implements Serializable { + private String taskID; + private String jobID; + private String address; + private String transportType; + private String connectorStage; + private long totalReqNum; + private long totalTimeCost; + private long maxTimeCost; + private long avgTimeCost; + private double tps; + private Date createTime; + private static final long serialVersionUID = 1L; + +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/stubs/HeaderStub.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/stubs/HeaderStub.java new file mode 100644 index 0000000000..1782d46dd6 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/stubs/HeaderStub.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.stubs; + +import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; +import org.apache.eventmesh.common.protocol.http.header.Header; +import org.apache.eventmesh.common.utils.HttpConvertsUtils; + +import java.util.Map; + +public class HeaderStub extends Header { + + public String code; + public String eventmeshenv; + + @Override + public Map toMap() { + return new HttpConvertsUtils().httpMapConverts(this, new ProtocolKey(), new ProtocolKey.EventMeshInstanceKey()); + } +} diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/utils/ConfigurationContextUtil.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/utils/ConfigurationContextUtil.java index 8617888994..fede64d650 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/utils/ConfigurationContextUtil.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/utils/ConfigurationContextUtil.java @@ -17,6 +17,7 @@ package org.apache.eventmesh.common.utils; +import static org.apache.eventmesh.common.Constants.ADMIN; import static org.apache.eventmesh.common.Constants.GRPC; import static org.apache.eventmesh.common.Constants.HTTP; import static org.apache.eventmesh.common.Constants.TCP; @@ -36,7 +37,7 @@ public class ConfigurationContextUtil { private static final ConcurrentHashMap CONFIGURATION_MAP = new ConcurrentHashMap<>(); - public static final List KEYS = Lists.newArrayList(HTTP, TCP, GRPC); + public static final List KEYS = Lists.newArrayList(HTTP, TCP, GRPC, ADMIN); /** * Save http, tcp, grpc configuration at startup for global use. diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/utils/IPUtils.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/utils/IPUtils.java index 998735181e..dcef8f8243 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/utils/IPUtils.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/utils/IPUtils.java @@ -37,6 +37,7 @@ import io.netty.channel.Channel; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import inet.ipaddr.HostName; @@ -46,7 +47,10 @@ @Slf4j public class IPUtils { - public static String getLocalAddress() { + @Getter + public static String localAddress = init(); + + private static String init() { // if the progress works under docker environment // return the host ip about this docker located from environment value String dockerHostIp = System.getenv("docker_host_ip"); diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/utils/JsonUtils.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/utils/JsonUtils.java index cf07bdfbe7..f2328541c4 100644 --- a/eventmesh-common/src/main/java/org/apache/eventmesh/common/utils/JsonUtils.java +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/utils/JsonUtils.java @@ -24,6 +24,7 @@ import org.apache.commons.lang3.StringUtils; import java.io.IOException; +import java.io.InputStream; import java.lang.reflect.Type; import java.util.Map; import java.util.Objects; @@ -53,6 +54,14 @@ public class JsonUtils { OBJECT_MAPPER.registerModule(new JavaTimeModule()); } + public static T convertValue(Object fromValue, Class toValueType) { + return OBJECT_MAPPER.convertValue(fromValue, toValueType); + } + + public static T convertValue(Object fromValue, TypeReference toValueTypeRef) { + return OBJECT_MAPPER.convertValue(fromValue, toValueTypeRef); + } + public static T mapToObject(Map map, Class beanClass) { if (map == null) { return null; @@ -60,7 +69,15 @@ public static T mapToObject(Map map, Class beanClass) { Object obj = OBJECT_MAPPER.convertValue(map, beanClass); return beanClass.cast(obj); } - + + public static Map objectToMap(Object obj) { + if (obj == null) { + return null; + } + return OBJECT_MAPPER.convertValue(obj, new TypeReference>() { + }); + } + /** * Serialize object to json string. * @@ -108,6 +125,14 @@ public static T parseObject(String text, Class clazz) { } } + public static T parseObject(InputStream inputStream, Class clazz) { + try { + return OBJECT_MAPPER.readValue(inputStream, clazz); + } catch (IOException e) { + throw new JsonException("deserialize input stream to object error", e); + } + } + public static T parseObject(String text, Type type) { if (StringUtils.isEmpty(text)) { return null; @@ -156,6 +181,21 @@ public static T parseTypeReferenceObject(String text, TypeReference typeR } } + public static T parseTypeReferenceObject(Object object, TypeReference typeReference) { + if (object == null) { + return null; + } + return convertValue(object, typeReference); + } + + public static T parseTypeReferenceObject(byte[] text, TypeReference typeReference) { + try { + return OBJECT_MAPPER.readValue(text, typeReference); + } catch (IOException e) { + throw new JsonException("deserialize json string to typeReference error", e); + } + } + public static JsonNode getJsonNode(String text) { if (StringUtils.isEmpty(text)) { return null; diff --git a/eventmesh-common/src/main/java/org/apache/eventmesh/common/utils/PagedList.java b/eventmesh-common/src/main/java/org/apache/eventmesh/common/utils/PagedList.java new file mode 100644 index 0000000000..322c585f04 --- /dev/null +++ b/eventmesh-common/src/main/java/org/apache/eventmesh/common/utils/PagedList.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.utils; + +import java.util.List; + +public class PagedList { + private int totalSize; + private int totalPage; + private int size; + private int page; + + private List data; + + public int getTotalSize() { + return totalSize; + } + + public void setTotalSize(int totalSize) { + this.totalSize = totalSize; + } + + public int getTotalPage() { + return totalPage; + } + + public void setTotalPage(int totalPage) { + this.totalPage = totalPage; + } + + public int getSize() { + return size; + } + + public void setSize(int size) { + this.size = size; + } + + public int getPage() { + return page; + } + + public void setPage(int page) { + this.page = page; + } + + public List getData() { + return data; + } + + public void setData(List data) { + this.data = data; + } +} diff --git a/eventmesh-common/src/main/resources/META-INF/services/org.apache.eventmesh.common.remote.payload.IPayload b/eventmesh-common/src/main/resources/META-INF/services/org.apache.eventmesh.common.remote.payload.IPayload new file mode 100644 index 0000000000..433cf57ed1 --- /dev/null +++ b/eventmesh-common/src/main/resources/META-INF/services/org.apache.eventmesh.common.remote.payload.IPayload @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.eventmesh.common.remote.request.FetchJobRequest +org.apache.eventmesh.common.remote.response.FetchJobResponse +org.apache.eventmesh.common.remote.request.ReportPositionRequest +org.apache.eventmesh.common.remote.request.ReportJobRequest +org.apache.eventmesh.common.remote.request.ReportVerifyRequest +org.apache.eventmesh.common.remote.request.ReportHeartBeatRequest +org.apache.eventmesh.common.remote.request.FetchPositionRequest +org.apache.eventmesh.common.remote.response.FetchPositionResponse \ No newline at end of file diff --git a/eventmesh-common/src/test/java/org/apache/eventmesh/common/loadbalance/RandomLoadBalanceSelectorTest.java b/eventmesh-common/src/test/java/org/apache/eventmesh/common/loadbalance/RandomLoadBalanceSelectorTest.java index 434e19f3b2..8f5c457264 100644 --- a/eventmesh-common/src/test/java/org/apache/eventmesh/common/loadbalance/RandomLoadBalanceSelectorTest.java +++ b/eventmesh-common/src/test/java/org/apache/eventmesh/common/loadbalance/RandomLoadBalanceSelectorTest.java @@ -34,7 +34,7 @@ public class RandomLoadBalanceSelectorTest { private RandomLoadBalanceSelector randomLoadBalanceSelector; @BeforeEach - public void befor() { + public void before() { List address = new ArrayList<>(); address.add("A"); address.add("B"); diff --git a/eventmesh-common/src/test/java/org/apache/eventmesh/common/utils/HttpConvertsUtilsTest.java b/eventmesh-common/src/test/java/org/apache/eventmesh/common/utils/HttpConvertsUtilsTest.java new file mode 100644 index 0000000000..253b1de926 --- /dev/null +++ b/eventmesh-common/src/test/java/org/apache/eventmesh/common/utils/HttpConvertsUtilsTest.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.utils; + +import org.apache.eventmesh.common.protocol.http.common.ProtocolKey; +import org.apache.eventmesh.common.protocol.http.common.ProtocolKey.EventMeshInstanceKey; +import org.apache.eventmesh.common.protocol.http.header.Header; +import org.apache.eventmesh.common.stubs.HeaderStub; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class HttpConvertsUtilsTest { + + private final HeaderStub headerStub = new HeaderStub(); + private final ProtocolKey mockedProtocolKey = new ProtocolKey(); + private final EventMeshInstanceKey mockedEventMeshProtocolKey = new EventMeshInstanceKey(); + + @Test + void httpMapConverts() { + Map httpMapConverts = new HttpConvertsUtils().httpMapConverts(headerStub, mockedProtocolKey); + Assertions.assertEquals(httpMapConverts.get(headerStub.code), headerStub.code); + } + + @Test + void testHttpMapConverts() { + Map httpMapConverts = new HttpConvertsUtils().httpMapConverts(headerStub, mockedProtocolKey, mockedEventMeshProtocolKey); + Assertions.assertEquals(httpMapConverts.get(headerStub.code), headerStub.code); + Assertions.assertEquals(httpMapConverts.get(headerStub.eventmeshenv), headerStub.eventmeshenv); + } + + @Test + void httpHeaderConverts() { + HashMap headerParams = new HashMap<>(); + String code = "test"; + headerParams.put("code", code); + Header header = new HttpConvertsUtils().httpHeaderConverts(headerStub, headerParams); + Assertions.assertEquals(code, header.toMap().get("code")); + } + + @Test + void testHttpHeaderConverts() { + HashMap headerParams = new HashMap<>(); + String env = "test"; + headerParams.put("eventmeshenv", env); + Header header = new HttpConvertsUtils().httpHeaderConverts(headerStub, headerParams, mockedEventMeshProtocolKey); + Assertions.assertEquals(env, header.toMap().get("eventmeshenv")); + } +} diff --git a/eventmesh-common/src/test/java/org/apache/eventmesh/common/utils/IPUtilsTest.java b/eventmesh-common/src/test/java/org/apache/eventmesh/common/utils/IPUtilsTest.java index b3fa293e89..757486dd89 100644 --- a/eventmesh-common/src/test/java/org/apache/eventmesh/common/utils/IPUtilsTest.java +++ b/eventmesh-common/src/test/java/org/apache/eventmesh/common/utils/IPUtilsTest.java @@ -19,16 +19,9 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.junitpioneer.jupiter.SetEnvironmentVariable; public class IPUtilsTest { - @Test - @SetEnvironmentVariable(key = "docker_host_ip", value = "dockHostIP") - public void testDockerIP() { - Assertions.assertEquals("dockHostIP", IPUtils.getLocalAddress()); - } - @Test public void testLocalhostIP() { Assertions.assertNotNull(IPUtils.getLocalAddress()); diff --git a/eventmesh-common/src/test/java/org/apache/eventmesh/common/utils/JsonPathUtilsTest.java b/eventmesh-common/src/test/java/org/apache/eventmesh/common/utils/JsonPathUtilsTest.java new file mode 100644 index 0000000000..e66b8d711a --- /dev/null +++ b/eventmesh-common/src/test/java/org/apache/eventmesh/common/utils/JsonPathUtilsTest.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.common.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; + +public class JsonPathUtilsTest { + + @Test + public void tesTisEmptyJsonObject() { + String emptyJsonObject = "{}"; + assertTrue(JsonPathUtils.isEmptyJsonObject(emptyJsonObject)); + + String jsonObject = "{\"key\": \"value\"}"; + assertFalse(JsonPathUtils.isEmptyJsonObject(jsonObject)); + + String emptyJsonArray = "[]"; + assertFalse(JsonPathUtils.isEmptyJsonObject(emptyJsonArray)); + + String jsonArray = "[{\"key\": \"value\"}]"; + assertFalse(JsonPathUtils.isEmptyJsonObject(jsonArray)); + + String empty = ""; + assertFalse(JsonPathUtils.isEmptyJsonObject(empty)); + } + + @Test + public void testParseStrict() { + String json = "{\"key\": \"value\"}"; + JsonNode result = JsonPathUtils.parseStrict(json); + assertNotNull(result); + assertEquals("value", result.get("key").asText()); + + String emptyJsonObject = "{}"; + JsonNode result2 = JsonPathUtils.parseStrict(emptyJsonObject); + assertNotNull(result2); + assertTrue(result2.isEmpty()); + + } + + @Test + public void testBuildJsonString() { + Map person = new HashMap<>(); + person.put("name", "John"); + person.put("age", "30"); + String actual = JsonPathUtils.buildJsonString("person", person); + String excepted = "{\"person\":{\"name\":\"John\",\"age\":\"30\"}}"; + assertNotNull(actual); + assertEquals(excepted, actual); + } + + @Test + public void testIsValidAndDefinite() { + String jsonPath = "$.person[0].name"; + String jsonPath2 = "$.person[*].address.city"; + String jsonPath3 = "person.job[0].title"; + + assertTrue(JsonPathUtils.isValidAndDefinite(jsonPath)); + assertFalse(JsonPathUtils.isValidAndDefinite(jsonPath2)); + assertFalse(JsonPathUtils.isValidAndDefinite(jsonPath3)); + + String jsonPath4 = null; + String jsonPath5 = ""; + + assertFalse(JsonPathUtils.isValidAndDefinite(jsonPath4)); + assertFalse(JsonPathUtils.isValidAndDefinite(jsonPath5)); + } + + + @Test + public void testGetJsonPathValue() { + String jsonContent = "{ \"person\": { \"name\": \"John Doe\", \"age\": 30, \"address\": { \"city\": \"New York\" } } }"; + + String jsonPath1 = "$.person.name"; + String jsonPath2 = "$.person.address.city"; + String jsonPath3 = "$.person.age"; + + assertEquals("John Doe", JsonPathUtils.getJsonPathValue(jsonContent, jsonPath1)); + assertEquals("New York", JsonPathUtils.getJsonPathValue(jsonContent, jsonPath2)); + assertEquals("30", JsonPathUtils.getJsonPathValue(jsonContent, jsonPath3)); + + } + + @Test + public void testConvertToJsonNode() throws JsonProcessingException { + String jsonString1 = "{\"name\": \"John Doe\", \"age\": 30, \"address\": { \"city\": \"New York\" }}"; + + JsonNode node1 = JsonPathUtils.convertToJsonNode(jsonString1); + assertEquals("John Doe", node1.get("name").asText()); + assertEquals("New York", node1.get("address").get("city").asText()); + assertEquals("30", node1.get("age").asText()); + } + + @Test + public void testMatchJsonPathValueWithString() { + String jsonString = "{\"name\": \"John Doe\", \"age\": 30, \"address\": { \"city\": \"New York\" }}"; + + String jsonPath1 = "$.name"; + String result1 = JsonPathUtils.matchJsonPathValueWithString(jsonString, jsonPath1); + assertEquals("John Doe", result1); + + String jsonPath2 = "$.age"; + String result2 = JsonPathUtils.matchJsonPathValueWithString(jsonString, jsonPath2); + assertEquals("30", result2); // Age should be returned as a string + + String jsonPath3 = "$.address.city"; + String result3 = JsonPathUtils.matchJsonPathValueWithString(jsonString, jsonPath3); + assertEquals("New York", result3); + + String jsonPath4 = "$.job"; + String result4 = JsonPathUtils.matchJsonPathValueWithString(jsonString, jsonPath4); + assertEquals("null", result4); + } + + @Test + public void testJsonPathParse() { + String jsonString = "{\"name\": \"John Doe\", \"age\": 30, \"address\": { \"city\": \"New York\" }}"; + + String jsonPath1 = "$.name"; + Object result1 = JsonPathUtils.jsonPathParse(jsonString, jsonPath1); + assertNotNull(result1); + assertEquals("John Doe", result1); + + String jsonPath2 = "$.address.city"; + Object result2 = JsonPathUtils.jsonPathParse(jsonString, jsonPath2); + assertNotNull(result2); + assertEquals("New York", result2); + } + + @Test + public void testMatchJsonPathValue() throws JsonProcessingException { + String jsonString = "{\"name\": \"John Doe\", \"age\": 30, \"address\": { \"city\": \"New York\" }}"; + String jsonPath1 = "$.name"; + String result1 = JsonPathUtils.matchJsonPathValue(jsonString, jsonPath1); + assertEquals("\"John Doe\"", result1); + + String jsonPath2 = "$.address.city"; + String result2 = JsonPathUtils.matchJsonPathValue(jsonString, jsonPath2); + assertEquals("\"New York\"", result2); + + String jsonPath3 = "$.job"; + String result3 = JsonPathUtils.matchJsonPathValue(jsonString, jsonPath3); + assertEquals("null", result3); + } +} diff --git a/eventmesh-common/src/test/java/org/apache/eventmesh/common/utils/LogUtilTest.java b/eventmesh-common/src/test/java/org/apache/eventmesh/common/utils/LogUtilTest.java new file mode 100644 index 0000000000..03c52dadad --- /dev/null +++ b/eventmesh-common/src/test/java/org/apache/eventmesh/common/utils/LogUtilTest.java @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.eventmesh.common.utils; + +import static org.mockito.ArgumentMatchers.same; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import java.util.function.Supplier; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.junit.jupiter.MockitoExtension; +import org.slf4j.Logger; +import org.slf4j.spi.LoggingEventBuilder; + +@ExtendWith(MockitoExtension.class) +class LogUtilTest { + + private Logger mockLogger; + private LoggingEventBuilder mockEventBuilder; + private Supplier supplier; + private String logMessage; + + @BeforeEach + void setUp() { + + mockLogger = mock(Logger.class); + mockEventBuilder = mock(LoggingEventBuilder.class); + + supplier = () -> "{\"orderId\": 12345, \"amount\": 100}"; + logMessage = "Processing order with data: {}"; + } + + @Test + void testDebugLogsWithSupplier() { + + doReturn(mockEventBuilder).when(mockLogger).atDebug(); + doReturn(mockEventBuilder).when(mockEventBuilder).addArgument(same(supplier)); + + LogUtil.debug(mockLogger, logMessage, supplier); + + verify(mockLogger).atDebug(); + verify(mockEventBuilder).addArgument(same(supplier)); + verify(mockEventBuilder).log(logMessage); + + } + + @Test + void testDebugLogsWithSupplierAndException() { + Throwable throwable = new RuntimeException("Order processing failed"); + + + doReturn(mockEventBuilder).when(mockLogger).atDebug(); + doReturn(mockEventBuilder).when(mockEventBuilder).addArgument(same(supplier)); + doReturn(mockEventBuilder).when(mockEventBuilder).setCause(throwable); + + LogUtil.debug(mockLogger, logMessage, supplier, throwable); + + verify(mockLogger).atDebug(); + verify(mockEventBuilder).addArgument(same(supplier)); + verify(mockEventBuilder).setCause(throwable); + verify(mockEventBuilder).log(logMessage); + } + + @Test + void testDebugLogsWithSuppliers() { + + Supplier supplier2 = () -> "{\"orderId\": 67890, \"amount\": 200}"; + + doReturn(mockEventBuilder).when(mockLogger).atDebug(); + doReturn(mockEventBuilder).when(mockEventBuilder).addArgument(same(supplier)); + doReturn(mockEventBuilder).when(mockEventBuilder).addArgument(same(supplier2)); + + LogUtil.debug(mockLogger, logMessage, supplier, supplier2); + + verify(mockLogger).atDebug(); + verify(mockEventBuilder).addArgument(same(supplier)); + verify(mockEventBuilder).addArgument(same(supplier2)); + verify(mockEventBuilder).log(logMessage); + } + + @Test + void testInfoLogsWithSupplier() { + + doReturn(mockEventBuilder).when(mockLogger).atInfo(); + doReturn(mockEventBuilder).when(mockEventBuilder).addArgument(same(supplier)); + + LogUtil.info(mockLogger, logMessage, supplier); + + verify(mockLogger).atInfo(); + verify(mockEventBuilder).addArgument(same(supplier)); + verify(mockEventBuilder).log(logMessage); + + } + + @Test + void testInfoLogsWithSupplierAndException() { + + Throwable throwable = new RuntimeException("Order processing failed"); + + doReturn(mockEventBuilder).when(mockLogger).atInfo(); + doReturn(mockEventBuilder).when(mockEventBuilder).addArgument(same(supplier)); + doReturn(mockEventBuilder).when(mockEventBuilder).setCause(throwable); + + LogUtil.info(mockLogger, logMessage, supplier, throwable); + + verify(mockLogger).atInfo(); + verify(mockEventBuilder).addArgument(same(supplier)); + verify(mockEventBuilder).setCause(throwable); + verify(mockEventBuilder).log(logMessage); + + } + + @Test + void testInfoLogsWithSuppliers() { + + Supplier supplier2 = () -> "{\"orderId\": 67890, \"amount\": 200}"; + + doReturn(mockEventBuilder).when(mockLogger).atInfo(); + doReturn(mockEventBuilder).when(mockEventBuilder).addArgument(same(supplier)); + doReturn(mockEventBuilder).when(mockEventBuilder).addArgument(same(supplier2)); + + LogUtil.info(mockLogger, logMessage, supplier, supplier2); + + verify(mockLogger).atInfo(); + verify(mockEventBuilder).addArgument(same(supplier)); + verify(mockEventBuilder).addArgument(same(supplier2)); + verify(mockEventBuilder).log(logMessage); + } + + @Test + void testWarnLogsWithSupplier() { + + doReturn(mockEventBuilder).when(mockLogger).atWarn(); + doReturn(mockEventBuilder).when(mockEventBuilder).addArgument(same(supplier)); + + LogUtil.warn(mockLogger, logMessage, supplier); + + verify(mockLogger).atWarn(); + verify(mockEventBuilder).addArgument(same(supplier)); + verify(mockEventBuilder).log(logMessage); + + } + + @Test + void testWarnLogsWithSupplierAndException() { + + Throwable throwable = new RuntimeException("Order processing failed"); + + doReturn(mockEventBuilder).when(mockLogger).atWarn(); + doReturn(mockEventBuilder).when(mockEventBuilder).addArgument(same(supplier)); + doReturn(mockEventBuilder).when(mockEventBuilder).setCause(throwable); + + LogUtil.warn(mockLogger, logMessage, supplier, throwable); + + verify(mockLogger).atWarn(); + verify(mockEventBuilder).addArgument(same(supplier)); + verify(mockEventBuilder).setCause(throwable); + verify(mockEventBuilder).log(logMessage); + + } + + @Test + void testWarnLogsWithSuppliers() { + + Supplier supplier2 = () -> "{\"orderId\": 67890, \"amount\": 200}"; + + doReturn(mockEventBuilder).when(mockLogger).atWarn(); + doReturn(mockEventBuilder).when(mockEventBuilder).addArgument(same(supplier)); + doReturn(mockEventBuilder).when(mockEventBuilder).addArgument(same(supplier2)); + + LogUtil.warn(mockLogger, logMessage, supplier, supplier2); + + verify(mockLogger).atWarn(); + verify(mockEventBuilder).addArgument(same(supplier)); + verify(mockEventBuilder).addArgument(same(supplier2)); + verify(mockEventBuilder).log(logMessage); + } + +} \ No newline at end of file diff --git a/eventmesh-common/src/test/java/org/apache/eventmesh/common/utils/ThreadUtilsTest.java b/eventmesh-common/src/test/java/org/apache/eventmesh/common/utils/ThreadUtilsTest.java new file mode 100644 index 0000000000..0cba2a6ad9 --- /dev/null +++ b/eventmesh-common/src/test/java/org/apache/eventmesh/common/utils/ThreadUtilsTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.eventmesh.common.utils; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.Test; + +class ThreadUtilsTest { + + @Test + void testRandomPauseBetweenMinAndMax() { + + long min = 1000; + long max = 5000; + + long start = System.currentTimeMillis(); + ThreadUtils.randomPause(min, max, TimeUnit.MILLISECONDS); + long end = System.currentTimeMillis(); + + long pause = end - start; + + assertTrue(pause >= min && pause <= max, "Pause time should be between min and max"); + } + + @Test + void testRandomPauseWithInterruption() { + + Thread.currentThread().interrupt(); + ThreadUtils.randomPause(1000, 2000, TimeUnit.MILLISECONDS); + assertTrue(Thread.currentThread().isInterrupted()); + } + + @Test + void testDeprecatedSleep() { + + ThreadUtils.sleep(1000); + assertTrue(true, "Method should execute without any exception"); + } + + @Test + void testSleepWithTimeOutAndTimeUnit() throws InterruptedException { + + ThreadUtils.sleepWithThrowException(5000, TimeUnit.MILLISECONDS); + assertTrue(true, "Method should execute without any exception"); + } + + @Test + void testSleepWithNullTimeUnit() throws InterruptedException { + + ThreadUtils.sleepWithThrowException(5000, null); + assertTrue(true, "Method should not throw any exception with null TimeUnit"); + } + + @Test + void testSleepWithThrowExceptionInterruption() { + Thread.currentThread().interrupt(); + + assertThrows(InterruptedException.class, () -> { + ThreadUtils.sleepWithThrowException(5000, TimeUnit.MILLISECONDS); + }); + } + + @Test + void testGetPIDWithRealProcessId() { + + long pid = ThreadUtils.getPID(); + assertTrue(pid > 0); + + long cashedPId = ThreadUtils.getPID(); + assertEquals(pid, cashedPId); + } + + @Test + void testGetPIDWithMultiThread() throws InterruptedException { + + final long[] pid1 = new long[1]; + final long[] pid2 = new long[1]; + + Thread thread1 = new Thread(() -> { + pid1[0] = ThreadUtils.getPID(); + assertTrue(pid1[0] > 0); + }); + + Thread thread2 = new Thread(() -> { + pid2[0] = ThreadUtils.getPID(); + assertTrue(pid2[0] > 0); + }); + + thread1.start(); + thread2.start(); + + thread1.join(); + thread2.join(); + + assertEquals(pid1[0], pid2[0]); + } +} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-canal/build.gradle b/eventmesh-connectors/eventmesh-connector-canal/build.gradle new file mode 100644 index 0000000000..6beeac41eb --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/build.gradle @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +List canal = [ + "com.alibaba.otter:canal.instance.manager:$canal_version", + "com.alibaba.otter:canal.parse:$canal_version", + "com.alibaba.otter:canal.server:$canal_version" +] + +dependencies { + api project(":eventmesh-openconnect:eventmesh-openconnect-java") + implementation "org.locationtech.jts:jts-core" + implementation project(":eventmesh-common") + implementation canal + implementation "com.alibaba:druid" + compileOnly 'com.mysql:mysql-connector-j' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' + testImplementation "org.mockito:mockito-core" + testImplementation "org.mockito:mockito-junit-jupiter" +} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-canal/gradle.properties b/eventmesh-connectors/eventmesh-connector-canal/gradle.properties new file mode 100644 index 0000000000..a439bdacf9 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/gradle.properties @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +canal_version=1.1.7 +pluginType=connector +pluginName=MySQL \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/ByteArrayConverter.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/ByteArrayConverter.java new file mode 100644 index 0000000000..350b678856 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/ByteArrayConverter.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal; + +import org.apache.commons.beanutils.ConversionException; +import org.apache.commons.beanutils.Converter; +import org.apache.commons.beanutils.converters.ArrayConverter; +import org.apache.commons.beanutils.converters.ByteConverter; + +import java.nio.charset.StandardCharsets; + + +public class ByteArrayConverter implements Converter { + + public static final Converter SQL_BYTES = new ByteArrayConverter(null); + private static final Converter converter = new ArrayConverter(byte[].class, new ByteConverter()); + + protected final Object defaultValue; + protected final boolean useDefault; + + public ByteArrayConverter() { + this.defaultValue = null; + this.useDefault = false; + } + + public ByteArrayConverter(Object defaultValue) { + this.defaultValue = defaultValue; + this.useDefault = true; + } + + public Object convert(Class type, Object value) { + if (value == null) { + if (useDefault) { + return (defaultValue); + } else { + throw new ConversionException("No value specified"); + } + } + + if (value instanceof byte[]) { + return (value); + } + + if (value instanceof String) { + try { + return ((String) value).getBytes(StandardCharsets.ISO_8859_1); + } catch (Exception e) { + throw new ConversionException(e); + } + } + + return converter.convert(type, value); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/CanalConnectRecord.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/CanalConnectRecord.java new file mode 100644 index 0000000000..6f112081e8 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/CanalConnectRecord.java @@ -0,0 +1,284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal; + +import org.apache.eventmesh.common.remote.job.SyncConsistency; +import org.apache.eventmesh.common.remote.job.SyncMode; +import org.apache.eventmesh.connector.canal.model.EventColumn; +import org.apache.eventmesh.connector.canal.model.EventType; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; + +import lombok.Data; + +@Data +public class CanalConnectRecord implements Serializable { + + private static final long serialVersionUID = 1L; + + private String schemaName; + + private String tableName; + + // mysql instance gtid range + private String gtid; + + private String currentGtid; + + /** + * The business type of the changed data (I/U/D/C/A/E), consistent with the EventType defined in EntryProtocol in canal. + */ + private EventType eventType; + + /** + * The business time of the changed data. + */ + private long executeTime; + + /** + * The primary key value before the change, if it is insert/delete, the primary key value before and after the change is the same. + */ + private List oldKeys = new ArrayList(); + + /** + * The primary key value after the change, if it is insert/delete, the primary key value before and after the change is the same. + */ + private List keys = new ArrayList(); + + /** + * Other fields that are not primary keys + */ + private List columns = new ArrayList(); + + // ====================== Additional properties of the data during the running process ============================= + /** + * The expected size, based on the estimation of the binlog event + */ + private long size = 1024; + + /** + * The id of the synchronization mapping relationship + */ + private long pairId = -1; + + /** + * When eventType = CREATE/ALTER/ERASE, it is the corresponding SQL statement, other situations are dynamically generated INSERT/UPDATE/DELETE sql + */ + private String sql; + + /** + * The schemaName of ddl/query, there will be cross-database ddl, need to keep the current schemaName of executing ddl + */ + private String ddlSchemaName; + + /** + * Custom synchronization mode, allows to override the default pipeline parameter, such as for remedial data synchronization + */ + private SyncMode syncMode; + + /** + * Custom synchronization consistency, allows to override the default pipeline parameter, + * such as forcing the database to be queried for field groups + */ + private SyncConsistency syncConsistency; + + /** + * Whether it is remedy data, such as data automatically generated by loopback remedy, or manual correction data produced by freedom + */ + private boolean remedy = false; + + /** + * Generate the corresponding hint content + */ + private String hint; + + /** + * Whether to ignore the schema when generating SQL, such as for tddl/drds, need to ignore the schema + */ + private boolean withoutSchema = false; + + private String journalName; + + private long binLogOffset; + + public CanalConnectRecord() { + super(); + } + + // ======================== helper method ================= + + /** + * Return all fields to be changed + */ + public List getUpdatedColumns() { + List columns = new ArrayList(); + for (EventColumn column : this.columns) { + if (column.isUpdate()) { + columns.add(column); + } + } + + return columns; + } + + /** + * Return all changed primary key fields + */ + public List getUpdatedKeys() { + List columns = new ArrayList(); + for (EventColumn column : this.keys) { + if (column.isUpdate()) { + columns.add(column); + } + } + + return columns; + } + + private List cloneColumn(List columns) { + if (columns == null) { + return null; + } + + List cloneColumns = new ArrayList(); + for (EventColumn column : columns) { + cloneColumns.add(column.clone()); + } + + return cloneColumns; + } + + public CanalConnectRecord clone() { + CanalConnectRecord record = new CanalConnectRecord(); + record.setTableName(tableName); + record.setSchemaName(schemaName); + record.setDdlSchemaName(ddlSchemaName); + record.setEventType(eventType); + record.setExecuteTime(executeTime); + record.setKeys(cloneColumn(keys)); + record.setColumns(cloneColumn(columns)); + record.setOldKeys(cloneColumn(oldKeys)); + record.setSize(size); + record.setPairId(pairId); + record.setSql(sql); + record.setSyncMode(syncMode); + record.setSyncConsistency(syncConsistency); + record.setRemedy(remedy); + record.setHint(hint); + record.setWithoutSchema(withoutSchema); + return record; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((columns == null) ? 0 : columns.hashCode()); + result = prime * result + ((eventType == null) ? 0 : eventType.hashCode()); + result = prime * result + (int) (executeTime ^ (executeTime >>> 32)); + result = prime * result + ((keys == null) ? 0 : keys.hashCode()); + result = prime * result + ((oldKeys == null) ? 0 : oldKeys.hashCode()); + result = prime * result + (int) (pairId ^ (pairId >>> 32)); + result = prime * result + ((schemaName == null) ? 0 : schemaName.hashCode()); + result = prime * result + ((tableName == null) ? 0 : tableName.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + CanalConnectRecord other = (CanalConnectRecord) obj; + if (columns == null) { + if (other.columns != null) { + return false; + } + } else if (!columns.equals(other.columns)) { + return false; + } + if (eventType != other.eventType) { + return false; + } + if (executeTime != other.executeTime) { + return false; + } + if (keys == null) { + if (other.keys != null) { + return false; + } + } else if (!keys.equals(other.keys)) { + return false; + } + if (oldKeys == null) { + if (other.oldKeys != null) { + return false; + } + } else if (!oldKeys.equals(other.oldKeys)) { + return false; + } + if (pairId != other.pairId) { + return false; + } + if (schemaName == null) { + if (other.schemaName != null) { + return false; + } + } else if (!schemaName.equals(other.schemaName)) { + return false; + } + if (tableName == null) { + if (other.tableName != null) { + return false; + } + } else if (!tableName.equals(other.tableName)) { + return false; + } + return true; + } + + @Override + public String toString() { + return "CanalConnectRecord{" + + "tableName='" + tableName + '\'' + + ", schemaName='" + schemaName + '\'' + + ", eventType=" + eventType + + ", executeTime=" + executeTime + + ", oldKeys=" + oldKeys + + ", keys=" + keys + + ", columns=" + columns + + ", size=" + size + + ", pairId=" + pairId + + ", sql='" + sql + '\'' + + ", ddlSchemaName='" + ddlSchemaName + '\'' + + ", syncMode=" + syncMode + + ", syncConsistency=" + syncConsistency + + ", remedy=" + remedy + + ", hint='" + hint + '\'' + + ", withoutSchema=" + withoutSchema + + '}'; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/DatabaseConnection.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/DatabaseConnection.java new file mode 100644 index 0000000000..0310e5434c --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/DatabaseConnection.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal; + + +import org.apache.eventmesh.common.config.connector.rdb.canal.SinkConnectorConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.SourceConnectorConfig; + +import java.sql.Connection; +import java.sql.SQLException; + +import com.alibaba.druid.pool.DruidDataSource; + +public class DatabaseConnection { + + public static DruidDataSource sourceDataSource; + + public static DruidDataSource sinkDataSource; + + public static SourceConnectorConfig sourceConfig; + + public static SinkConnectorConfig sinkConfig; + + public static DruidDataSource createDruidDataSource(String url, String userName, String passWord) { + DruidDataSource dataSource = new DruidDataSource(); + dataSource.setUrl(url); + dataSource.setUsername(userName); + dataSource.setPassword(passWord); + dataSource.setInitialSize(5); + dataSource.setMinIdle(5); + dataSource.setMaxActive(20); + dataSource.setMaxWait(60000); + dataSource.setTimeBetweenEvictionRunsMillis(60000); + dataSource.setMinEvictableIdleTimeMillis(300000); + dataSource.setValidationQuery("SELECT 1"); + dataSource.setTestWhileIdle(true); + dataSource.setTestOnBorrow(false); + dataSource.setTestOnReturn(false); + dataSource.setPoolPreparedStatements(true); + dataSource.setMaxPoolPreparedStatementPerConnectionSize(20); + return dataSource; + } + + public static void initSourceConnection() { + sourceDataSource = createDruidDataSource(sourceConfig.getUrl(), + sourceConfig.getUserName(), + sourceConfig.getPassWord()); + } + + public static void initSinkConnection() { + sinkDataSource = createDruidDataSource(sinkConfig.getUrl(), + sinkConfig.getUserName(), + sinkConfig.getPassWord()); + } + + + public static Connection getSourceConnection() throws SQLException { + return sourceDataSource.getConnection(); + } + + public static Connection getSinkConnection() throws SQLException { + return sinkDataSource.getConnection(); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlTimestampConverter.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlTimestampConverter.java new file mode 100644 index 0000000000..8df0b1c097 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlTimestampConverter.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal; + +import org.apache.commons.beanutils.ConversionException; +import org.apache.commons.beanutils.Converter; +import org.apache.commons.lang.time.DateFormatUtils; + +import java.sql.Timestamp; +import java.text.ParseException; +import java.text.ParsePosition; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Locale; + +public class SqlTimestampConverter implements Converter { + + /** + * Field description + */ + public static final String[] DATE_FORMATS = new String[] {"yyyy-MM-dd", "HH:mm:ss", "yyyy-MM-dd HH:mm:ss", + "yyyy-MM-dd hh:mm:ss.fffffffff", "EEE MMM dd HH:mm:ss zzz yyyy", + DateFormatUtils.ISO_DATETIME_FORMAT.getPattern(), + DateFormatUtils.ISO_DATETIME_TIME_ZONE_FORMAT.getPattern(), + DateFormatUtils.SMTP_DATETIME_FORMAT.getPattern(), }; + + public static final Converter SQL_TIMESTAMP = new SqlTimestampConverter(null); + + /** + * The default value specified to our Constructor, if any. + */ + private final Object defaultValue; + + /** + * Should we return the default value on conversion errors? + */ + private final boolean useDefault; + + /** + * Create a {@link Converter} that will throw a {@link ConversionException} if a conversion error occurs. + */ + public SqlTimestampConverter() { + this.defaultValue = null; + this.useDefault = false; + } + + /** + * Create a {@link Converter} that will return the specified default value if a conversion error occurs. + * + * @param defaultValue The default value to be returned + */ + public SqlTimestampConverter(Object defaultValue) { + this.defaultValue = defaultValue; + this.useDefault = true; + } + + /** + * Convert the specified input object into an output object of the specified type. + * + * @param type Data type to which this value should be converted + * @param value The input value to be converted + * @throws ConversionException if conversion cannot be performed successfully + */ + public Object convert(Class type, Object value) { + if (value == null) { + if (useDefault) { + return (defaultValue); + } else { + throw new ConversionException("No value specified"); + } + } + + if (value instanceof java.sql.Date && java.sql.Date.class.equals(type)) { + return value; + } else if (value instanceof java.sql.Time && java.sql.Time.class.equals(type)) { + return value; + } else if (value instanceof Timestamp && Timestamp.class.equals(type)) { + return value; + } else { + try { + if (java.sql.Date.class.equals(type)) { + return new java.sql.Date(convertTimestamp2TimeMillis(value.toString())); + } else if (java.sql.Time.class.equals(type)) { + return new java.sql.Time(convertTimestamp2TimeMillis(value.toString())); + } else if (Timestamp.class.equals(type)) { + return new Timestamp(convertTimestamp2TimeMillis(value.toString())); + } else { + return new Timestamp(convertTimestamp2TimeMillis(value.toString())); + } + } catch (Exception e) { + throw new ConversionException("Value format invalid: " + e.getMessage(), e); + } + } + + } + + private Long convertTimestamp2TimeMillis(String input) { + if (input == null) { + return null; + } + + try { + return Timestamp.valueOf(input).getTime(); + } catch (Exception nfe) { + try { + try { + return parseDate(input, Locale.ENGLISH).getTime(); + } catch (Exception err) { + return parseDate(input, Locale.getDefault()).getTime(); + } + } catch (Exception err) { + return Long.parseLong(input); + } + } + } + + private Date parseDate(String str, Locale locale) throws ParseException { + if ((str == null) || (SqlTimestampConverter.DATE_FORMATS == null)) { + throw new IllegalArgumentException("Date and Patterns must not be null"); + } + + SimpleDateFormat parser = null; + ParsePosition pos = new ParsePosition(0); + + for (int i = 0; i < SqlTimestampConverter.DATE_FORMATS.length; i++) { + if (i == 0) { + parser = new SimpleDateFormat(SqlTimestampConverter.DATE_FORMATS[0], locale); + } else { + parser.applyPattern(SqlTimestampConverter.DATE_FORMATS[i]); + } + pos.setIndex(0); + Date date = parser.parse(str, pos); + if ((date != null) && (pos.getIndex() == str.length())) { + return date; + } + } + + throw new ParseException("Unable to parse the date: " + str, -1); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java new file mode 100644 index 0000000000..273f5cde4c --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/SqlUtils.java @@ -0,0 +1,921 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal; + +import static org.apache.eventmesh.connector.canal.ByteArrayConverter.SQL_BYTES; +import static org.apache.eventmesh.connector.canal.SqlTimestampConverter.SQL_TIMESTAMP; + +import org.apache.commons.beanutils.ConvertUtilsBean; +import org.apache.commons.lang.StringUtils; + +import java.io.UnsupportedEncodingException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.nio.charset.StandardCharsets; +import java.sql.Blob; +import java.sql.Clob; +import java.sql.Date; +import java.sql.JDBCType; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Time; +import java.sql.Timestamp; +import java.sql.Types; +import java.time.DateTimeException; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.ZoneId; +import java.time.temporal.Temporal; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.io.WKBReader; +import org.locationtech.jts.io.WKTReader; + +import com.mysql.cj.Constants; +import com.mysql.cj.MysqlType; +import com.taobao.tddl.dbsync.binlog.LogBuffer; + +public class SqlUtils { + + public static final String REQUIRED_FIELD_NULL_SUBSTITUTE = " "; + private static final Map> sqlTypeToJavaTypeMap = new HashMap>(); + private static final ConvertUtilsBean convertUtilsBean = new ConvertUtilsBean(); + private static final GeometryFactory GEOMETRY_FACTORY = new GeometryFactory(); + private static final WKBReader WKB_READER = new WKBReader(GEOMETRY_FACTORY); + private static final BigDecimal NANO_SEC = new BigDecimal(LogBuffer.DIG_BASE); + private static final LocalDateTime BASE = LocalDateTime.of(1970, 1, 1, 0, 0, 0, 0); + private static final long ONE_HOUR = 3600; + private static final long ONE_MINUTE = 60; + + static { + // regist Converter + convertUtilsBean.register(SQL_TIMESTAMP, Date.class); + convertUtilsBean.register(SQL_TIMESTAMP, Time.class); + convertUtilsBean.register(SQL_TIMESTAMP, Timestamp.class); + convertUtilsBean.register(SQL_BYTES, byte[].class); + + // bool + sqlTypeToJavaTypeMap.put(Types.BOOLEAN, Boolean.class); + + // int + sqlTypeToJavaTypeMap.put(Types.TINYINT, Integer.class); + sqlTypeToJavaTypeMap.put(Types.SMALLINT, Integer.class); + sqlTypeToJavaTypeMap.put(Types.INTEGER, Integer.class); + + // long + sqlTypeToJavaTypeMap.put(Types.BIGINT, Long.class); + // mysql bit + sqlTypeToJavaTypeMap.put(Types.BIT, BigInteger.class); + + // decimal + sqlTypeToJavaTypeMap.put(Types.REAL, Float.class); + sqlTypeToJavaTypeMap.put(Types.FLOAT, Float.class); + sqlTypeToJavaTypeMap.put(Types.DOUBLE, Double.class); + sqlTypeToJavaTypeMap.put(Types.NUMERIC, BigDecimal.class); + sqlTypeToJavaTypeMap.put(Types.DECIMAL, BigDecimal.class); + + // date + sqlTypeToJavaTypeMap.put(Types.DATE, Date.class); + sqlTypeToJavaTypeMap.put(Types.TIME, Time.class); + sqlTypeToJavaTypeMap.put(Types.TIMESTAMP, Timestamp.class); + + // blob + sqlTypeToJavaTypeMap.put(Types.BLOB, byte[].class); + + // byte[] + sqlTypeToJavaTypeMap.put(Types.REF, byte[].class); + sqlTypeToJavaTypeMap.put(Types.OTHER, byte[].class); + sqlTypeToJavaTypeMap.put(Types.ARRAY, byte[].class); + sqlTypeToJavaTypeMap.put(Types.STRUCT, byte[].class); + sqlTypeToJavaTypeMap.put(Types.SQLXML, byte[].class); + sqlTypeToJavaTypeMap.put(Types.BINARY, byte[].class); + sqlTypeToJavaTypeMap.put(Types.DATALINK, byte[].class); + sqlTypeToJavaTypeMap.put(Types.DISTINCT, byte[].class); + sqlTypeToJavaTypeMap.put(Types.VARBINARY, byte[].class); + sqlTypeToJavaTypeMap.put(Types.JAVA_OBJECT, byte[].class); + sqlTypeToJavaTypeMap.put(Types.LONGVARBINARY, byte[].class); + + // String + sqlTypeToJavaTypeMap.put(Types.CHAR, String.class); + sqlTypeToJavaTypeMap.put(Types.VARCHAR, String.class); + sqlTypeToJavaTypeMap.put(Types.LONGVARCHAR, String.class); + sqlTypeToJavaTypeMap.put(Types.LONGNVARCHAR, String.class); + sqlTypeToJavaTypeMap.put(Types.NCHAR, String.class); + sqlTypeToJavaTypeMap.put(Types.NVARCHAR, String.class); + sqlTypeToJavaTypeMap.put(Types.NCLOB, String.class); + sqlTypeToJavaTypeMap.put(Types.CLOB, String.class); + } + + public static String genPrepareSqlOfInClause(int size) { + StringBuilder sql = new StringBuilder(); + sql.append("("); + for (int i = 0; i < size; i++) { + sql.append("?"); + if (i < size - 1) { + sql.append(","); + } + } + sql.append(")"); + return sql.toString(); + } + + public static void setInClauseParameters(PreparedStatement preparedStatement, List params) throws SQLException { + setInClauseParameters(preparedStatement, 0, params); + } + + public static void setInClauseParameters(PreparedStatement preparedStatement, int paramIndexStart, List params) throws SQLException { + for (int i = 0; i < params.size(); i++) { + preparedStatement.setString(paramIndexStart + i, params.get(i)); + } + } + + public static String sqlValueToString(ResultSet rs, int index, int sqlType) throws SQLException { + Class requiredType = sqlTypeToJavaTypeMap.get(sqlType); + if (requiredType == null) { + throw new IllegalArgumentException("unknow java.sql.Types - " + sqlType); + } + + return getResultSetValue(rs, index, requiredType); + } + + public static Object stringToSqlValue(String value, int sqlType, boolean isRequired, boolean isEmptyStringNulled) { + if (SqlUtils.isTextType(sqlType)) { + if ((value == null) || (StringUtils.isEmpty(value) && isEmptyStringNulled)) { + return isRequired ? REQUIRED_FIELD_NULL_SUBSTITUTE : null; + } else { + return value; + } + } else { + if (StringUtils.isEmpty(value)) { + return isEmptyStringNulled ? null : value; + } else { + Class requiredType = sqlTypeToJavaTypeMap.get(sqlType); + if (requiredType == null) { + throw new IllegalArgumentException("unknow java.sql.Types - " + sqlType); + } else if (requiredType.equals(String.class)) { + return value; + } else if (isNumeric(sqlType)) { + return convertUtilsBean.convert(value.trim(), requiredType); + } else { + return convertUtilsBean.convert(value, requiredType); + } + } + } + } + + public static String encoding(String source, int sqlType, String sourceEncoding, String targetEncoding) { + switch (sqlType) { + case Types.CHAR: + case Types.VARCHAR: + case Types.LONGVARCHAR: + case Types.NCHAR: + case Types.NVARCHAR: + case Types.LONGNVARCHAR: + case Types.CLOB: + case Types.NCLOB: + if (!StringUtils.isEmpty(source)) { + String fromEncoding = StringUtils.isBlank(sourceEncoding) ? "UTF-8" : sourceEncoding; + String toEncoding = StringUtils.isBlank(targetEncoding) ? "UTF-8" : targetEncoding; + + // if (false == StringUtils.equalsIgnoreCase(fromEncoding, + // toEncoding)) { + try { + return new String(source.getBytes(fromEncoding), toEncoding); + } catch (UnsupportedEncodingException e) { + throw new IllegalArgumentException(e.getMessage(), e); + } + // } + } + break; + default: + throw new IllegalStateException("Unexpected value: " + sqlType); + } + + return source; + } + + /** + * Retrieve a JDBC column value from a ResultSet, using the specified value type. + *

+ * Uses the specifically typed ResultSet accessor methods, falling back to {@link #getResultSetValue(ResultSet, int)} for unknown types. + *

+ * Note that the returned value may not be assignable to the specified required type, in case of an unknown type. Calling code needs to deal with + * this case appropriately, e.g. throwing a corresponding exception. + * + * @param rs is the ResultSet holding the data + * @param index is the column index + * @param requiredType the required value type (may be null) + * @return the value object + * @throws SQLException if thrown by the JDBC API + */ + private static String getResultSetValue(ResultSet rs, int index, Class requiredType) throws SQLException { + if (requiredType == null) { + return getResultSetValue(rs, index); + } + + Object value = null; + boolean wasNullCheck = false; + + // Explicitly extract typed value, as far as possible. + if (String.class.equals(requiredType)) { + value = rs.getString(index); + } else if (boolean.class.equals(requiredType) || Boolean.class.equals(requiredType)) { + value = rs.getBoolean(index); + wasNullCheck = true; + } else if (byte.class.equals(requiredType) || Byte.class.equals(requiredType)) { + value = rs.getByte(index); + wasNullCheck = true; + } else if (short.class.equals(requiredType) || Short.class.equals(requiredType)) { + value = rs.getShort(index); + wasNullCheck = true; + } else if (int.class.equals(requiredType) || Integer.class.equals(requiredType)) { + value = rs.getLong(index); + wasNullCheck = true; + } else if (long.class.equals(requiredType) || Long.class.equals(requiredType)) { + value = rs.getBigDecimal(index); + wasNullCheck = true; + } else if (float.class.equals(requiredType) || Float.class.equals(requiredType)) { + value = rs.getFloat(index); + wasNullCheck = true; + } else if (double.class.equals(requiredType) || Double.class.equals(requiredType) || Number.class.equals(requiredType)) { + value = rs.getDouble(index); + wasNullCheck = true; + } else if (Time.class.equals(requiredType)) { + value = rs.getString(index); + } else if (Timestamp.class.equals(requiredType) || Date.class.equals(requiredType)) { + value = rs.getString(index); + } else if (BigDecimal.class.equals(requiredType)) { + value = rs.getBigDecimal(index); + } else if (BigInteger.class.equals(requiredType)) { + value = rs.getBigDecimal(index); + } else if (Blob.class.equals(requiredType)) { + value = rs.getBlob(index); + } else if (Clob.class.equals(requiredType)) { + value = rs.getClob(index); + } else if (byte[].class.equals(requiredType)) { + byte[] bytes = rs.getBytes(index); + if (bytes != null) { + value = new String(bytes, StandardCharsets.ISO_8859_1); + } + } else { + // Some unknown type desired -> rely on getObject. + value = getResultSetValue(rs, index); + } + + // Perform was-null check if demanded (for results that the + // JDBC driver returns as primitives). + if (wasNullCheck && (value != null) && rs.wasNull()) { + value = null; + } + + return (value == null) ? null : convertUtilsBean.convert(value); + } + + /** + * Retrieve a JDBC column value from a ResultSet, using the most appropriate value type. The returned value should be a detached value object, not + * having any ties to the active ResultSet: in particular, it should not be a Blob or Clob object but rather a byte array respectively String + * representation. + *

+ * Uses the getObject(index) method, but includes additional "hacks" to get around Oracle 10g returning a non-standard object for its + * TIMESTAMP datatype and a java.sql.Date for DATE columns leaving out the time portion: These columns will explicitly be extracted + * as standard java.sql.Timestamp object. + * + * @param rs is the ResultSet holding the data + * @param index is the column index + * @return the value object + * @throws SQLException if thrown by the JDBC API + * @see Blob + * @see Clob + * @see Timestamp + */ + private static String getResultSetValue(ResultSet rs, int index) throws SQLException { + Object obj = rs.getObject(index); + return (obj == null) ? null : convertUtilsBean.convert(obj); + } + + // private static Object convertTimestamp(Timestamp timestamp) { + // return (timestamp == null) ? null : timestamp.getTime(); + // } + + /** + * Check whether the given SQL type is numeric. + */ + public static boolean isNumeric(int sqlType) { + return (Types.BIT == sqlType) || (Types.BIGINT == sqlType) || (Types.DECIMAL == sqlType) || (Types.DOUBLE == sqlType) + || (Types.FLOAT == sqlType) || (Types.INTEGER == sqlType) || (Types.NUMERIC == sqlType) || (Types.REAL == sqlType) + || (Types.SMALLINT == sqlType) || (Types.TINYINT == sqlType); + } + + public static boolean isTextType(int sqlType) { + return sqlType == Types.CHAR || sqlType == Types.VARCHAR || sqlType == Types.CLOB || sqlType == Types.LONGVARCHAR || sqlType == Types.NCHAR + || sqlType == Types.NVARCHAR || sqlType == Types.NCLOB || sqlType == Types.LONGNVARCHAR; + } + + public static JDBCType toJDBCType(String connectorDataType) { + MysqlType mysqlType = MysqlType.getByName(connectorDataType); + return JDBCType.valueOf(mysqlType.getJdbcType()); + } + + public static BigDecimal toBigDecimal(Object value) { + if (value == null) { + return null; + } + if (value instanceof String) { + String strValue = (String) value; + if (!org.apache.commons.lang3.StringUtils.isNotBlank(strValue)) { + return null; + } + try { + return new BigDecimal(strValue); + } catch (Exception e) { + if ("true".equals(strValue)) { + return BigDecimal.ONE; + } + if ("false".equals(strValue)) { + return BigDecimal.ZERO; + } + return new BigDecimal(strValue); + } + } else if (value instanceof Number) { + if (value instanceof BigDecimal) { + return (BigDecimal) value; + } + if (value instanceof Integer) { + return BigDecimal.valueOf(((Integer) value).longValue()); + } + if (value instanceof Long) { + return BigDecimal.valueOf(((Long) value)); + } + if (value instanceof Double) { + return BigDecimal.valueOf(((Double) value)); + } + if (value instanceof Float) { + return BigDecimal.valueOf(((Float) value).doubleValue()); + } + if (value instanceof BigInteger) { + return new BigDecimal((BigInteger) value); + } + if (value instanceof Byte) { + return BigDecimal.valueOf(((Byte) value).longValue()); + } + if (value instanceof Short) { + return BigDecimal.valueOf(((Short) value).longValue()); + } + return null; + } else if (value instanceof Boolean) { + return Boolean.TRUE.equals(value) ? BigDecimal.ONE : BigDecimal.ZERO; + } else { + throw new UnsupportedOperationException("class " + value.getClass() + ", value '" + value + "' , parse to big decimal failed."); + } + } + + public static Double toDouble(Object value) { + if (value == null) { + return null; + } + if (value instanceof String) { + String strValue = (String) value; + if (org.apache.commons.lang3.StringUtils.isBlank(strValue)) { + return null; + } + try { + return Double.parseDouble(strValue); + } catch (Exception e) { + if ("true".equals(strValue)) { + return 1.0d; + } + if ("false".equals(strValue)) { + return 0.0d; + } + return new BigDecimal(strValue).doubleValue(); + } + } else if (value instanceof Number) { + return ((Number) value).doubleValue(); + } else { + if (value instanceof Boolean) { + return Boolean.TRUE.equals(value) ? 1.0d : 0.0d; + } + throw new UnsupportedOperationException("class " + value.getClass() + ", value '" + value + "' , parse to double failed."); + } + } + + public static Long toLong(Object value) { + if (value == null) { + return null; + } + if (value instanceof String) { + String strValue = (String) value; + if (org.apache.commons.lang3.StringUtils.isBlank(strValue)) { + return null; + } + try { + return Long.parseLong(strValue); + } catch (Exception e) { + try { + return Long.decode(strValue); + } catch (Exception e2) { + if ("true".equals(strValue)) { + return 1L; + } + if ("false".equals(strValue)) { + return 0L; + } + return new BigDecimal(strValue).longValue(); + } + } + } else if (value instanceof Number) { + return ((Number) value).longValue(); + } else { + if (value instanceof Boolean) { + return Boolean.TRUE.equals(value) ? 1L : 0L; + } + throw new UnsupportedOperationException(value.getClass() + ", value '" + value + "' , parse to long failed."); + } + } + + public static boolean isZeroTime(Object value) { + if (value == null || org.apache.commons.lang3.StringUtils.isBlank(value.toString())) { + return false; + } + return value.toString().startsWith("0000-00-00"); + } + + public static String removeZone(String datetime) { + if (datetime == null || datetime.length() == 0) { + return datetime; + } + int len = datetime.length(); + if (datetime.charAt(len - 1) == 'Z' || datetime.charAt(len - 1) == 'z') { + return datetime.substring(0, len - 1).trim(); + } + if (len >= 7) { + char checkCharAt1 = datetime.charAt(len - 2); + if ((checkCharAt1 == '+' || checkCharAt1 == '-') && len >= 10) { + return datetime.substring(0, len - 2).trim(); + } + char checkCharAt2 = datetime.charAt(len - 3); + if ((checkCharAt2 == '+' || checkCharAt2 == '-') && len >= 11) { + return datetime.substring(0, len - 3).trim(); + } + char checkCharAt3 = datetime.charAt(len - 6); + if ((checkCharAt3 == '+' || checkCharAt3 == '-') && checkCharAt2 == ':') { + return datetime.substring(0, len - 6).trim(); + } + char checkCharAt4 = datetime.charAt(len - 5); + if ((checkCharAt4 == '+' || checkCharAt4 == '-') && checkCharAt2 == ':') { + return datetime.substring(0, len - 5).trim(); + } + char checkCharAt5 = len >= 9 ? datetime.charAt(len - 9) : ' '; + if ((checkCharAt5 == '+' || checkCharAt5 == '-') && checkCharAt2 == ':' && checkCharAt3 == ':') { + return datetime.substring(0, len - 9).trim(); + } + char checkCharAt6 = datetime.charAt(len - 7); + if (checkCharAt6 == '+' || checkCharAt6 == '-') { + return datetime.substring(0, len - 7).trim(); + } + if (checkCharAt4 == '+' || checkCharAt4 == '-') { + return datetime.substring(0, len - 5).trim(); + } + } + return datetime; + } + + + + public static String bytes2hex(byte[] b) { + if (b == null) { + return null; + } + if (b.length == 0) { + return ""; + } + StringBuilder hs = new StringBuilder(); + for (byte element : b) { + String stmp = Integer.toHexString(element & 255).toUpperCase(); + if (stmp.length() == 1) { + hs.append(Constants.CJ_MINOR_VERSION); + hs.append(stmp); + } else { + hs.append(stmp); + } + } + return hs.toString(); + } + + public static String convertToString(Object value) { + if (value == null) { + return null; + } + if (value instanceof String) { + return (String) value; + } + if (value instanceof BigInteger) { + return value.toString(); + } + if (value instanceof BigDecimal) { + return ((BigDecimal) value).toPlainString(); + } + if (value instanceof Number) { + return new BigDecimal(value.toString()).toPlainString(); + } + if (value instanceof Boolean) { + return Boolean.TRUE.equals(value) ? "1" : "0"; + } + if (value instanceof byte[]) { + return "0x" + bytes2hex((byte[]) value); + } + if (value instanceof Timestamp) { + long nanos = ((Timestamp) value).getNanos(); + value = Instant.ofEpochMilli(((Timestamp) value).getTime() - (nanos / 1000000)).plusNanos(nanos).atZone(ZoneId.systemDefault()) + .toLocalDateTime(); + } else if (value instanceof Date) { + value = ((Date) value).toLocalDate().atTime(0, 0); + } else if (value instanceof Time) { + value = LocalDateTime.of(LocalDate.of(1970, 1, 1), + Instant.ofEpochMilli(((Time) value).getTime()).atZone(ZoneId.systemDefault()).toLocalTime()); + } else if (value instanceof java.util.Date) { + value = ((java.util.Date) value).toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); + } + if (value instanceof LocalDateTime) { + return coverLocalDateTime2String((LocalDateTime) value); + } else if (value instanceof OffsetDateTime) { + OffsetDateTime zone = (OffsetDateTime) value; + String datetimeStr = coverLocalDateTime2String(zone.toLocalDateTime()); + String zonedStr = zone.getOffset().toString(); + if ("Z".equals(zonedStr)) { + return datetimeStr + "+00:00"; + } + return datetimeStr + zonedStr; + } else if (!(value instanceof LocalTime)) { + return value.toString(); + } else { + LocalTime local3 = (LocalTime) value; + return String.format("%02d:%02d:%02d", local3.getHour(), local3.getMinute(), local3.getSecond()); + } + } + + + private static String coverLocalDateTime2String(LocalDateTime localDateTime) { + LocalDate localDate = localDateTime.toLocalDate(); + LocalTime localTime = localDateTime.toLocalTime(); + int year = localDate.getYear(); + int month = localDate.getMonthValue(); + int day = localDate.getDayOfMonth(); + int hour = localTime.getHour(); + int minute = localTime.getMinute(); + int second = localTime.getSecond(); + int nano = localTime.getNano(); + return nano == 0 ? String.format("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second) : + String.format("%04d-%02d-%02d %02d:%02d:%02d.%s", year, month, day, hour, minute, second, + new BigDecimal(nano).divide(NANO_SEC).toPlainString().substring(2)); + } + + public static String toMySqlTime(Object value) { + if (value == null || StringUtils.isBlank(value.toString())) { + return null; + } + if (value instanceof String) { + return value.toString(); + } + LocalDateTime localTime = toLocalDateTime(value); + if (BASE.isBefore(localTime) || BASE.isEqual(localTime)) { + long diffHours = Duration.between(BASE, localTime).toHours(); + if (localTime.getNano() == 0) { + return String.format("%02d:%02d:%02d", diffHours, localTime.getMinute(), localTime.getSecond()); + } + return String.format("%02d:%02d:%02d.%s", diffHours, localTime.getMinute(), localTime.getSecond(), + Integer.parseInt(trimEnd(String.valueOf(localTime.getNano()), '0'))); + } + Duration duration = Duration.between(localTime, BASE); + long totalSecond = duration.getSeconds(); + long hours = totalSecond / ONE_HOUR; + long remaining = totalSecond - (hours * ONE_HOUR); + long minutes = remaining / ONE_MINUTE; + remaining = remaining - (minutes * ONE_MINUTE); + if (duration.getNano() == 0) { + return String.format("-%02d:%02d:%02d", hours, minutes, remaining); + } + return String.format("-%02d:%02d:%02d.%s", hours, minutes, remaining, Integer.parseInt(trimEnd(String.valueOf(duration.getNano()), '0'))); + } + + public static String trimEnd(String str, char trimChar) { + if (str == null || str.isEmpty()) { + return str; + } + char[] val = str.toCharArray(); + int len = val.length; + while (0 < len && val[len - 1] == trimChar) { + len--; + } + return len < val.length ? str.substring(0, len) : str; + } + + public static byte[] numberToBinaryArray(Number number) { + BigInteger bigInt = BigInteger.valueOf(number.longValue()); + int size = (bigInt.bitLength() + 7) / 8; + byte[] result = new byte[size]; + byte[] bigIntBytes = bigInt.toByteArray(); + int start = bigInt.bitLength() % 8 == 0 ? 1 : 0; + int length = Math.min(bigIntBytes.length - start, size); + System.arraycopy(bigIntBytes, start, result, size - length, length); + return result; + } + + public static Integer toInt(Object value) { + if (value == null) { + return null; + } + if (value instanceof String) { + String strValue = ((String) value).toLowerCase(); + if (StringUtils.isBlank(strValue)) { + return null; + } + try { + return Integer.parseInt(strValue); + } catch (Exception e) { + try { + return Integer.decode(strValue); + } catch (Exception e2) { + if ("true".equals(strValue)) { + return 1; + } + if ("false".equals(strValue)) { + return 0; + } + return new BigDecimal(strValue).intValue(); + } + } + } else if (value instanceof Number) { + return ((Number) value).intValue(); + } else { + if (value instanceof Boolean) { + return Boolean.TRUE.equals(value) ? 1 : 0; + } + throw new UnsupportedOperationException("class " + value.getClass() + ", value '" + value + "' , parse to int failed."); + } + } + + private static LocalDateTime toLocalDateTime(String value) { + if (value.trim().length() >= 4) { + String dateStr2 = removeZone(value); + int len = dateStr2.length(); + if (len == 4) { + return LocalDateTime.of(Integer.parseInt(dateStr2), 1, 1, 0, 0, 0, 0); + } + if (dateStr2.charAt(4) == '-') { + switch (len) { + case 7: + String[] dataParts = dateStr2.split("-"); + return LocalDateTime.of(Integer.parseInt(dataParts[0]), Integer.parseInt(dataParts[1]), 1, 0, 0, 0, 0); + case 8: + case 9: + case 11: + case 12: + case 14: + case 15: + case 17: + case 18: + default: + String[] dataTime = dateStr2.split(" "); + String[] dataParts2 = dataTime[0].split("-"); + String[] timeParts = dataTime[1].split(":"); + String[] secondParts = timeParts[2].split("\\."); + secondParts[1] = StringUtils.rightPad(secondParts[1], 9, Constants.CJ_MINOR_VERSION); + return LocalDateTime.of(Integer.parseInt(dataParts2[0]), Integer.parseInt(dataParts2[1]), Integer.parseInt(dataParts2[2]), + Integer.parseInt(timeParts[0]), Integer.parseInt(timeParts[1]), Integer.parseInt(secondParts[0]), + Integer.parseInt(secondParts[1])); + case 10: + String[] dataParts3 = dateStr2.split("-"); + return LocalDateTime.of(Integer.parseInt(dataParts3[0]), Integer.parseInt(dataParts3[1]), Integer.parseInt(dataParts3[2]), 0, + 0, 0, 0); + case 13: + String[] dataTime2 = dateStr2.split(" "); + String[] dataParts4 = dataTime2[0].split("-"); + return LocalDateTime.of(Integer.parseInt(dataParts4[0]), Integer.parseInt(dataParts4[1]), Integer.parseInt(dataParts4[2]), + Integer.parseInt(dataTime2[1]), 0, 0, 0); + case 16: + String[] dataTime3 = dateStr2.split(" "); + String[] dataParts5 = dataTime3[0].split("-"); + String[] timeParts2 = dataTime3[1].split(":"); + return LocalDateTime.of(Integer.parseInt(dataParts5[0]), Integer.parseInt(dataParts5[1]), Integer.parseInt(dataParts5[2]), + Integer.parseInt(timeParts2[0]), Integer.parseInt(timeParts2[1]), 0, 0); + case 19: + String[] dataTime4 = dateStr2.split(" "); + String[] dataParts6 = dataTime4[0].split("-"); + String[] timeParts3 = dataTime4[1].split(":"); + return LocalDateTime.of(Integer.parseInt(dataParts6[0]), Integer.parseInt(dataParts6[1]), Integer.parseInt(dataParts6[2]), + Integer.parseInt(timeParts3[0]), Integer.parseInt(timeParts3[1]), Integer.parseInt(timeParts3[2]), 0); + } + } else if (dateStr2.charAt(2) == ':') { + switch (len) { + case 5: + String[] timeParts4 = dateStr2.split(":"); + return LocalDateTime.of(0, 1, 1, Integer.parseInt(timeParts4[0]), Integer.parseInt(timeParts4[1]), 0, 0); + case 8: + String[] timeParts5 = dateStr2.split(":"); + return LocalDateTime.of(0, 1, 1, Integer.parseInt(timeParts5[0]), Integer.parseInt(timeParts5[1]), + Integer.parseInt(timeParts5[2]), 0); + default: + String[] timeParts6 = dateStr2.split(":"); + String[] secondParts2 = timeParts6[2].split("\\."); + secondParts2[1] = StringUtils.rightPad(secondParts2[1], 9, Constants.CJ_MINOR_VERSION); + return LocalDateTime.of(0, 1, 1, Integer.parseInt(timeParts6[0]), Integer.parseInt(timeParts6[1]), + Integer.parseInt(secondParts2[0]), Integer.parseInt(secondParts2[1])); + } + } else { + throw new UnsupportedOperationException(value.getClass() + ", value '" + value + "' , parse to local date time failed."); + } + } else if (StringUtils.isNumeric(value)) { + return LocalDateTime.of(Integer.parseInt(value), 1, 1, 0, 0, 0, 0); + } else { + throw new DateTimeException(value + " format error."); + } + } + + public static LocalDateTime toLocalDateTime(Object value) { + if (value == null || StringUtils.isBlank(value.toString())) { + return null; + } + if (value instanceof Temporal) { + if (value instanceof LocalDateTime) { + return (LocalDateTime) value; + } + if (value instanceof OffsetDateTime) { + return ((OffsetDateTime) value).toLocalDateTime(); + } + if (value instanceof LocalTime) { + return LocalDateTime.of(LocalDate.of(1970, 1, 1), (LocalTime) value); + } else if (value instanceof LocalDate) { + return LocalDateTime.of((LocalDate) value, LocalTime.of(0, 0)); + } else { + throw new UnsupportedOperationException(value.getClass() + ", value '" + value + "' , parse local date time failed."); + } + } else if (!(value instanceof java.util.Date)) { + return toLocalDateTime(value.toString()); + } else { + if (value instanceof Timestamp) { + long nanos = ((Timestamp) value).getNanos(); + return Instant.ofEpochMilli(((Timestamp) value).getTime() - (nanos / 1000000)).plusNanos(nanos).atZone(ZoneId.systemDefault()) + .toLocalDateTime(); + } else if (value instanceof Date) { + return ((Date) value).toLocalDate().atTime(0, 0); + } else { + if (!(value instanceof Time)) { + return ((java.util.Date) value).toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); + } + return LocalDateTime.of(LocalDate.of(1970, 1, 1), + Instant.ofEpochMilli(((Time) value).getTime()).atZone(ZoneId.systemDefault()).toLocalTime()); + } + } + } + + public static boolean isHexNumber(String str) { + boolean flag = true; + if (str.startsWith("0x") || str.startsWith("0X")) { + str = str.substring(2); + } + int i = 0; + while (true) { + if (i < str.length()) { + char cc = str.charAt(i); + if (cc != '0' && cc != '1' && cc != '2' && cc != '3' && cc != '4' && cc != '5' && cc != '6' && cc != '7' && cc != '8' && cc != '9' + && cc != 'A' && cc != 'B' && cc != 'C' && cc != 'D' && cc != 'E' && cc != 'F' && cc != 'a' && cc != 'b' && cc != 'c' && cc != 'd' + && cc != 'e' && cc != 'f') { + flag = false; + break; + } + i++; + } else { + break; + } + } + return flag; + } + + public static byte[] toBytes(Object value) { + if (value == null) { + return null; + } + if (value instanceof String) { + String strVal = (String) value; + if ((strVal.startsWith("0x") || strVal.startsWith("0X")) && isHexNumber(strVal)) { + return hex2bytes(strVal.substring(2)); + } + return ((String) value).getBytes(StandardCharsets.ISO_8859_1); + } else if (value instanceof byte[]) { + return (byte[]) value; + } else { + throw new UnsupportedOperationException("class " + value.getClass() + ", value '" + value + "' , parse to bytes failed."); + } + } + + public static String toGeometry(Object value) throws Exception { + if (value == null) { + return null; + } + if (value instanceof String) { + String strVal = (String) value; + if (!strVal.startsWith("0x") && !strVal.startsWith("0X")) { + return (String) value; + } + return new WKTReader().read((String) value).toText(); + } else if (value instanceof byte[]) { + // mysql add 4 byte in header of geometry + byte[] bytes = (byte[]) value; + if (bytes.length > 4) { + byte[] dst = new byte[bytes.length - 4]; + System.arraycopy(bytes, 4, dst, 0, bytes.length - 4); + return new WKBReader().read(dst).toText(); + } + return new WKBReader().read(bytes).toText(); + } else { + throw new UnsupportedOperationException("class " + value.getClass() + ", value '" + value + "' , " + "parse to geometry failed."); + } + } + + public static byte[] hex2bytes(String hexStr) { + if (hexStr == null) { + return null; + } + if (org.apache.commons.lang3.StringUtils.isBlank(hexStr)) { + return new byte[0]; + } + + if (hexStr.length() % 2 == 1) { + hexStr = "0" + hexStr; + } + + int count = hexStr.length() / 2; + byte[] ret = new byte[count]; + for (int i = 0; i < count; i++) { + int index = i * 2; + char c1 = hexStr.charAt(index); + char c2 = hexStr.charAt(index + 1); + ret[i] = (byte) (toByte(c1) << 4); + ret[i] = (byte) (ret[i] | toByte(c2)); + } + return ret; + } + + private static byte toByte(char src) { + switch (Character.toUpperCase(src)) { + case '0': + return 0; + case '1': + return 1; + case '2': + return 2; + case '3': + return 3; + case '4': + return 4; + case '5': + return 5; + case '6': + return 6; + case '7': + return 7; + case '8': + return 8; + case '9': + return 9; + case 'A': + return 10; + case 'B': + return 11; + case 'C': + return 12; + case 'D': + return 13; + case 'E': + return 14; + case 'F': + return 15; + default: + throw new IllegalStateException("0-F"); + } + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/config/CanalServerConfig.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/config/CanalServerConfig.java new file mode 100644 index 0000000000..b28982f7b6 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/config/CanalServerConfig.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.config; + +import org.apache.eventmesh.common.config.connector.Config; + +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode(callSuper = true) +public class CanalServerConfig extends Config { + + private boolean sourceEnable; + + private boolean sinkEnable; + +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/dialect/AbstractDbDialect.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/dialect/AbstractDbDialect.java new file mode 100644 index 0000000000..4cf0f82ec9 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/dialect/AbstractDbDialect.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.dialect; + +import org.apache.eventmesh.connector.canal.template.SqlTemplate; + +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.jdbc.support.lob.LobHandler; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.support.TransactionTemplate; + +import lombok.extern.slf4j.Slf4j; + + +@Slf4j +public abstract class AbstractDbDialect implements DbDialect { + + protected int databaseMajorVersion; + protected int databaseMinorVersion; + protected String databaseName; + protected SqlTemplate sqlTemplate; + protected JdbcTemplate jdbcTemplate; + protected TransactionTemplate transactionTemplate; + protected LobHandler lobHandler; + + public AbstractDbDialect(final JdbcTemplate jdbcTemplate, LobHandler lobHandler) { + this.jdbcTemplate = jdbcTemplate; + this.lobHandler = lobHandler; + + this.transactionTemplate = new TransactionTemplate(); + transactionTemplate.setTransactionManager(new DataSourceTransactionManager(jdbcTemplate.getDataSource())); + transactionTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + + } + + public AbstractDbDialect(JdbcTemplate jdbcTemplate, LobHandler lobHandler, String name, int majorVersion, + int minorVersion) { + this.jdbcTemplate = jdbcTemplate; + this.lobHandler = lobHandler; + + this.transactionTemplate = new TransactionTemplate(); + transactionTemplate.setTransactionManager(new DataSourceTransactionManager(jdbcTemplate.getDataSource())); + transactionTemplate.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); + + this.databaseName = name; + this.databaseMajorVersion = majorVersion; + this.databaseMinorVersion = minorVersion; + + } + + public String getName() { + return databaseName; + } + + public int getMajorVersion() { + return databaseMajorVersion; + } + + @Override + public int getMinorVersion() { + return databaseMinorVersion; + } + + public String getVersion() { + return databaseMajorVersion + "." + databaseMinorVersion; + } + + public LobHandler getLobHandler() { + return lobHandler; + } + + public JdbcTemplate getJdbcTemplate() { + return jdbcTemplate; + } + + public TransactionTemplate getTransactionTemplate() { + return transactionTemplate; + } + + public SqlTemplate getSqlTemplate() { + return sqlTemplate; + } + + public String getShardColumns(String schema, String table) { + return null; + } + + public void destory() { + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/dialect/DbDialect.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/dialect/DbDialect.java new file mode 100644 index 0000000000..781c2fe954 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/dialect/DbDialect.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.dialect; + +import org.apache.eventmesh.connector.canal.template.SqlTemplate; + +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.support.lob.LobHandler; +import org.springframework.transaction.support.TransactionTemplate; + +/** + * DbDialect + */ +public interface DbDialect { + + public String getName(); + + public String getVersion(); + + public int getMajorVersion(); + + public int getMinorVersion(); + + public String getDefaultSchema(); + + public String getDefaultCatalog(); + + public boolean isCharSpacePadded(); + + public boolean isCharSpaceTrimmed(); + + public boolean isEmptyStringNulled(); + + public boolean isSupportMergeSql(); + + public LobHandler getLobHandler(); + + public JdbcTemplate getJdbcTemplate(); + + public TransactionTemplate getTransactionTemplate(); + + public SqlTemplate getSqlTemplate(); + + public String getShardColumns(String schema, String table); + + public void destory(); +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/dialect/MysqlDialect.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/dialect/MysqlDialect.java new file mode 100644 index 0000000000..bfe5628716 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/dialect/MysqlDialect.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.dialect; + +import org.apache.eventmesh.connector.canal.template.MysqlSqlTemplate; + +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.support.lob.LobHandler; + + +public class MysqlDialect extends AbstractDbDialect { + + public MysqlDialect(JdbcTemplate jdbcTemplate, LobHandler lobHandler) { + super(jdbcTemplate, lobHandler); + sqlTemplate = new MysqlSqlTemplate(); + } + + public boolean isCharSpacePadded() { + return false; + } + + public boolean isCharSpaceTrimmed() { + return true; + } + + public boolean isEmptyStringNulled() { + return false; + } + + public boolean isSupportMergeSql() { + return true; + } + + public String getDefaultSchema() { + return null; + } + + public String getDefaultCatalog() { + return jdbcTemplate.queryForObject("select database()", String.class); + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/interceptor/SqlBuilderLoadInterceptor.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/interceptor/SqlBuilderLoadInterceptor.java new file mode 100644 index 0000000000..1d7bd35b94 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/interceptor/SqlBuilderLoadInterceptor.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.interceptor; + +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkIncrementConfig; +import org.apache.eventmesh.connector.canal.CanalConnectRecord; +import org.apache.eventmesh.connector.canal.dialect.DbDialect; +import org.apache.eventmesh.connector.canal.model.EventColumn; +import org.apache.eventmesh.connector.canal.model.EventType; +import org.apache.eventmesh.connector.canal.template.SqlTemplate; + +import java.util.List; + +import org.springframework.util.CollectionUtils; + +import lombok.Getter; +import lombok.Setter; + +/** + * compute latest sql + */ +public class SqlBuilderLoadInterceptor { + + @Getter + @Setter + private DbDialect dbDialect; + + public boolean before(CanalSinkIncrementConfig sinkConfig, CanalConnectRecord record) { + // build sql + SqlTemplate sqlTemplate = dbDialect.getSqlTemplate(); + EventType type = record.getEventType(); + String sql = null; + + String schemaName = (record.isWithoutSchema() ? null : record.getSchemaName()); + + String shardColumns = null; + + if (type.isInsert()) { + sql = sqlTemplate.getMergeSql(schemaName, + record.getTableName(), + buildColumnNames(record.getKeys()), + buildColumnNames(record.getColumns()), + new String[] {}, + true, + shardColumns); + } else if (type.isUpdate()) { + boolean existOldKeys = !CollectionUtils.isEmpty(record.getOldKeys()); + boolean rowMode = sinkConfig.getSyncMode().isRow(); + String[] keyColumns = null; + String[] otherColumns = null; + if (existOldKeys) { + // update table xxx set pk = newPK where pk = oldPk + keyColumns = buildColumnNames(record.getOldKeys()); + otherColumns = buildColumnNames(record.getUpdatedColumns(), record.getKeys()); + } else { + keyColumns = buildColumnNames(record.getKeys()); + otherColumns = buildColumnNames(record.getUpdatedColumns()); + } + + // not support the column default not null for merge sql + // if (rowMode && !existOldKeys) { + // sql = sqlTemplate.getMergeSql(schemaName, + // record.getTableName(), + // keyColumns, + // otherColumns, + // new String[] {}, + // true, + // shardColumns); + // } else { + // sql = sqlTemplate.getUpdateSql(schemaName, record.getTableName(), keyColumns, otherColumns, true, shardColumns); + // } + sql = sqlTemplate.getUpdateSql(schemaName, record.getTableName(), keyColumns, otherColumns, true, shardColumns); + } else if (type.isDelete()) { + sql = sqlTemplate.getDeleteSql(schemaName, + record.getTableName(), + buildColumnNames(record.getKeys())); + } + + if (record.getHint() != null) { + record.setSql(record.getHint() + sql); + } else { + record.setSql(sql); + } + return false; + } + + private String[] buildColumnNames(List columns) { + String[] result = new String[columns.size()]; + for (int i = 0; i < columns.size(); i++) { + EventColumn column = columns.get(i); + result[i] = column.getColumnName(); + } + return result; + } + + private String[] buildColumnNames(List columns1, List columns2) { + String[] result = new String[columns1.size() + columns2.size()]; + int i = 0; + for (i = 0; i < columns1.size(); i++) { + EventColumn column = columns1.get(i); + result[i] = column.getColumnName(); + } + + for (; i < columns1.size() + columns2.size(); i++) { + EventColumn column = columns2.get(i - columns1.size()); + result[i] = column.getColumnName(); + } + return result; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/model/EventColumn.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/model/EventColumn.java new file mode 100644 index 0000000000..352fc060a0 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/model/EventColumn.java @@ -0,0 +1,163 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.model; + +import java.io.Serializable; + +import lombok.Getter; +import lombok.Setter; + +public class EventColumn implements Serializable { + + @Setter + @Getter + private int index; + + @Getter + @Setter + private int columnType; + + @Getter + @Setter + private String columnName; + + /** + * timestamp,Datetime is long + */ + @Setter + private String columnValue; + + private boolean isNull; + + private boolean isKey; + + private boolean isUpdate = true; + + public String getColumnValue() { + if (isNull) { + columnValue = null; + return null; + } else { + return columnValue; + } + } + + public boolean isNull() { + return isNull; + } + + public void setNull(boolean isNull) { + this.isNull = isNull; + } + + public boolean isKey() { + return isKey; + } + + public void setKey(boolean isKey) { + this.isKey = isKey; + } + + public boolean isUpdate() { + return isUpdate; + } + + public void setUpdate(boolean isUpdate) { + this.isUpdate = isUpdate; + } + + public EventColumn clone() { + EventColumn column = new EventColumn(); + column.setIndex(index); + column.setColumnName(columnName); + column.setColumnType(columnType); + column.setColumnValue(columnValue); + column.setKey(isKey); + column.setNull(isNull); + column.setUpdate(isUpdate); + return column; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((columnName == null) ? 0 : columnName.hashCode()); + result = prime * result + columnType; + result = prime * result + ((columnValue == null) ? 0 : columnValue.hashCode()); + result = prime * result + index; + result = prime * result + (isKey ? 1231 : 1237); + result = prime * result + (isNull ? 1231 : 1237); + result = prime * result + (isUpdate ? 1231 : 1237); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + EventColumn other = (EventColumn) obj; + if (columnName == null) { + if (other.columnName != null) { + return false; + } + } else if (!columnName.equals(other.columnName)) { + return false; + } + if (columnType != other.columnType) { + return false; + } + if (columnValue == null) { + if (other.columnValue != null) { + return false; + } + } else if (!columnValue.equals(other.columnValue)) { + return false; + } + if (index != other.index) { + return false; + } + if (isKey != other.isKey) { + return false; + } + if (isNull != other.isNull) { + return false; + } + return isUpdate == other.isUpdate; + } + + @Override + public String toString() { + return "EventColumn{" + + "index=" + index + + ", columnType=" + columnType + + ", columnName='" + columnName + '\'' + + ", columnValue='" + columnValue + '\'' + + ", isNull=" + isNull + + ", isKey=" + isKey + + ", isUpdate=" + isUpdate + + '}'; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/model/EventColumnIndexComparable.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/model/EventColumnIndexComparable.java new file mode 100644 index 0000000000..ca55f57292 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/model/EventColumnIndexComparable.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.model; + +import java.util.Comparator; + +public class EventColumnIndexComparable implements Comparator { + + public int compare(EventColumn o1, EventColumn o2) { + return Integer.compare(o1.getIndex(), o2.getIndex()); + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/model/EventType.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/model/EventType.java new file mode 100644 index 0000000000..a1537c9f58 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/model/EventType.java @@ -0,0 +1,156 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.model; + +/** + * chang the eventtype num to I/U/D/C/A/E. + */ +public enum EventType { + + /** + * Insert row. + */ + INSERT("I"), + + /** + * Update row. + */ + UPDATE("U"), + + /** + * Delete row. + */ + DELETE("D"), + + /** + * Create table. + */ + CREATE("C"), + + /** + * Alter table. + */ + ALTER("A"), + + /** + * Erase table. + */ + ERASE("E"), + + /** + * Query. + */ + QUERY("Q"), + + /** + * Truncate. + */ + TRUNCATE("T"), + + /** + * rename. + */ + RENAME("R"), + + /** + * create index. + */ + CINDEX("CI"), + + /** + * drop index. + */ + DINDEX("DI"); + + private String value; + + private EventType(String value) { + this.value = value; + } + + public boolean isInsert() { + return this.equals(EventType.INSERT); + } + + public boolean isUpdate() { + return this.equals(EventType.UPDATE); + } + + public boolean isDelete() { + return this.equals(EventType.DELETE); + } + + public boolean isCreate() { + return this.equals(EventType.CREATE); + } + + public boolean isAlter() { + return this.equals(EventType.ALTER); + } + + public boolean isErase() { + return this.equals(EventType.ERASE); + } + + public boolean isQuery() { + return this.equals(EventType.QUERY); + } + + public boolean isTruncate() { + return this.equals(EventType.TRUNCATE); + } + + public boolean isRename() { + return this.equals(EventType.RENAME); + } + + public boolean isCindex() { + return this.equals(EventType.CINDEX); + } + + public boolean isDindex() { + return this.equals(EventType.DINDEX); + } + + public boolean isDdl() { + return isCreate() || isAlter() || isErase() || isTruncate() || isRename() || isCindex() || isDindex(); + } + + public boolean isDml() { + return isInsert() || isUpdate() || isDelete(); + } + + public static EventType valuesOf(String value) { + EventType[] eventTypes = values(); + for (EventType eventType : eventTypes) { + if (eventType.value.equalsIgnoreCase(value)) { + return eventType; + } + } + return null; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/server/CanalConnectServer.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/server/CanalConnectServer.java new file mode 100644 index 0000000000..6cc3d013dd --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/server/CanalConnectServer.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.server; + +import org.apache.eventmesh.connector.canal.config.CanalServerConfig; +import org.apache.eventmesh.connector.canal.sink.connector.CanalSinkConnector; +import org.apache.eventmesh.connector.canal.source.connector.CanalSourceConnector; +import org.apache.eventmesh.openconnect.Application; +import org.apache.eventmesh.openconnect.util.ConfigUtil; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class CanalConnectServer { + + public static void main(String[] args) throws Exception { + + CanalServerConfig serverConfig = ConfigUtil.parse(CanalServerConfig.class, "server-config.yml"); + + if (serverConfig.isSourceEnable()) { + Application canalSourceApp = new Application(); + canalSourceApp.run(CanalSourceConnector.class); + } + + if (serverConfig.isSinkEnable()) { + Application canalSinkApp = new Application(); + canalSinkApp.run(CanalSinkConnector.class); + } + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/DbLoadContext.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/DbLoadContext.java new file mode 100644 index 0000000000..3498e87e7b --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/DbLoadContext.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.sink; + +import org.apache.eventmesh.connector.canal.CanalConnectRecord; + +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; + +import lombok.Data; + +@Data +public class DbLoadContext { + + private String gtid; + + private List lastProcessedRecords; + + private List prepareRecords; + + private List processedRecords; + + private List failedRecords; + + public DbLoadContext() { + lastProcessedRecords = Collections.synchronizedList(new LinkedList<>()); + prepareRecords = Collections.synchronizedList(new LinkedList<>()); + processedRecords = Collections.synchronizedList(new LinkedList<>()); + failedRecords = Collections.synchronizedList(new LinkedList<>()); + } + + +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/DbLoadData.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/DbLoadData.java new file mode 100644 index 0000000000..ea48de7749 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/DbLoadData.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.sink; + +import org.apache.eventmesh.connector.canal.CanalConnectRecord; +import org.apache.eventmesh.connector.canal.model.EventType; + +import java.util.ArrayList; +import java.util.List; + +/** + * Classify the data according to the table and insert/update/delete types. + * + * 

+ * Purpose of classification: to optimize the insert statement in batches.
+ * 1. Due to the restrictions of MySQL indexes, concurrent execution of insert statements needs to be avoided.
+ *
+ */ +public class DbLoadData { + + private List tables = new ArrayList(); + + public DbLoadData() { + // nothing + } + + public DbLoadData(List records) { + for (CanalConnectRecord record : records) { + merge(record); + } + } + + public void merge(CanalConnectRecord record) { + TableLoadData tableData = findTableData(record); + + EventType type = record.getEventType(); + if (type.isInsert()) { + tableData.getInsertDatas().add(record); + } else if (type.isUpdate()) { + tableData.getUpdateDatas().add(record); + } else if (type.isDelete()) { + tableData.getDeleteDatas().add(record); + } + } + + public List getTables() { + return tables; + } + + private synchronized TableLoadData findTableData(CanalConnectRecord record) { + for (TableLoadData table : tables) { + if (table.getSchemaName().equals(record.getSchemaName()) + && table.getTableName().equals(record.getTableName())) { + return table; + } + } + + TableLoadData data = new TableLoadData(record.getSchemaName(), record.getTableName()); + tables.add(data); + return data; + } + + /** + * classify by table + */ + public static class TableLoadData { + + private String schemaName; + + private String tableName; + private List insertDatas = new ArrayList<>(); + private List upadateDatas = new ArrayList<>(); + private List deleteDatas = new ArrayList<>(); + + public TableLoadData(String schemaName, String tableName) { + this.schemaName = schemaName; + this.tableName = tableName; + } + + public List getInsertDatas() { + return insertDatas; + } + + public void setInsertDatas(List insertDatas) { + this.insertDatas = insertDatas; + } + + public List getUpdateDatas() { + return upadateDatas; + } + + public void setUpdateDatas(List upadateDatas) { + this.upadateDatas = upadateDatas; + } + + public List getDeleteDatas() { + return deleteDatas; + } + + public void setDeleteDatas(List deleteDatas) { + this.deleteDatas = deleteDatas; + } + + public String getSchemaName() { + return schemaName; + } + + public void setSchemaName(String schemaName) { + this.schemaName = schemaName; + } + + public String getTableName() { + return tableName; + } + + public void setTableName(String tableName) { + this.tableName = tableName; + } + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/DbLoadMerger.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/DbLoadMerger.java new file mode 100644 index 0000000000..af53532dd8 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/DbLoadMerger.java @@ -0,0 +1,280 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.sink; + +import org.apache.eventmesh.connector.canal.CanalConnectRecord; +import org.apache.eventmesh.connector.canal.model.EventColumn; +import org.apache.eventmesh.connector.canal.model.EventColumnIndexComparable; +import org.apache.eventmesh.connector.canal.model.EventType; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import org.springframework.util.CollectionUtils; + +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + + +/** + * 
+ * merge the same schema-table change record.
+ * The result of merging multiple change data with the same primary key (pk) is:
+ * 1, I
+ * 2, U
+ * 3, D
+ * If there is one "I" (Insert) and multiple "U" (Update), merge into "I";
+ * If there are multiple "U" (Update), take the latest one;
+ *
+ */ +@Slf4j +public class DbLoadMerger { + + /** + * Merge a batch of data based on table and primary key information, + * ensuring that there is only one record for each primary key in a table + * + * @param eventDatas + * @return + */ + public static List merge(List eventDatas) { + Map result = new LinkedHashMap(); + for (CanalConnectRecord eventData : eventDatas) { + merge(eventData, result); + } + return new LinkedList<>(result.values()); + } + + public static void merge(CanalConnectRecord record, Map result) { + EventType eventType = record.getEventType(); + switch (eventType) { + case INSERT: + mergeInsert(record, result); + break; + case UPDATE: + mergeUpdate(record, result); + break; + case DELETE: + mergeDelete(record, result); + break; + default: + break; + } + } + + private static void mergeInsert(CanalConnectRecord record, Map result) { + RowKey rowKey = new RowKey(record.getSchemaName(), record.getTableName(), + record.getKeys()); + if (!result.containsKey(rowKey)) { + result.put(rowKey, record); + } else { + CanalConnectRecord oldRecord = result.get(rowKey); + record.setSize(oldRecord.getSize() + record.getSize()); + if (oldRecord.getEventType() == EventType.DELETE) { + result.put(rowKey, record); + } else if (record.getEventType() == EventType.UPDATE + || record.getEventType() == EventType.INSERT) { + log.warn("update-insert/insert-insert happend. before[{}] , after[{}]", oldRecord, record); + CanalConnectRecord mergeEventData = replaceColumnValue(record, oldRecord); + mergeEventData.getOldKeys().clear(); + result.put(rowKey, mergeEventData); + } + } + } + + private static void mergeUpdate(CanalConnectRecord record, Map result) { + RowKey rowKey = new RowKey(record.getSchemaName(), record.getTableName(), record.getKeys()); + if (!CollectionUtils.isEmpty(record.getOldKeys())) { + RowKey oldKey = new RowKey(record.getSchemaName(), record.getTableName(), + record.getOldKeys()); + if (!result.containsKey(oldKey)) { + result.put(rowKey, record); + } else { + CanalConnectRecord oldRecord = result.get(oldKey); + record.setSize(oldRecord.getSize() + record.getSize()); + if (oldRecord.getEventType() == EventType.INSERT) { + record.setEventType(EventType.INSERT); + result.remove(oldKey); + + CanalConnectRecord mergeEventData = replaceColumnValue(record, oldRecord); + mergeEventData.getOldKeys().clear(); + result.put(rowKey, mergeEventData); + } else if (oldRecord.getEventType() == EventType.UPDATE) { + result.remove(oldKey); + CanalConnectRecord mergeEventData = replaceColumnValue(record, oldRecord); + result.put(rowKey, mergeEventData); + } else { + throw new RuntimeException("delete(has old pks) + update impossible happed!"); + } + } + } else { + if (!result.containsKey(rowKey)) { + result.put(rowKey, record); + } else { + CanalConnectRecord oldRecord = result.get(rowKey); + if (oldRecord.getEventType() == EventType.INSERT) { + oldRecord.setEventType(EventType.INSERT); + + CanalConnectRecord mergeEventData = replaceColumnValue(record, oldRecord); + result.put(rowKey, mergeEventData); + } else if (oldRecord.getEventType() == EventType.UPDATE) { + CanalConnectRecord mergeEventData = replaceColumnValue(record, oldRecord); + result.put(rowKey, mergeEventData); + } else if (oldRecord.getEventType() == EventType.DELETE) { + result.put(rowKey, record); + } + } + } + } + + private static void mergeDelete(CanalConnectRecord record, Map result) { + RowKey rowKey = new RowKey(record.getSchemaName(), record.getTableName(), + record.getKeys()); + if (!result.containsKey(rowKey)) { + result.put(rowKey, record); + } else { + CanalConnectRecord oldRecord = result.get(rowKey); + record.setSize(oldRecord.getSize() + record.getSize()); + if (!CollectionUtils.isEmpty(oldRecord.getOldKeys())) { + record.setKeys(oldRecord.getOldKeys()); + record.getOldKeys().clear(); + + result.remove(rowKey); + result.put(new RowKey(record.getSchemaName(), record.getTableName(), + record.getKeys()), record); + } else { + record.getOldKeys().clear(); + result.put(rowKey, record); + } + + } + } + + /** + * Merge the old value that exists in the old record and does not exist in the new record into the new record, + * and save the old primary key of the last change to the old primary key of this change. + * + * @param newRecord + * @param oldRecord + * @return + */ + private static CanalConnectRecord replaceColumnValue(CanalConnectRecord newRecord, CanalConnectRecord oldRecord) { + List newColumns = newRecord.getColumns(); + List oldColumns = oldRecord.getColumns(); + List temp = new ArrayList<>(); + for (EventColumn oldColumn : oldColumns) { + boolean contain = false; + for (EventColumn newColumn : newColumns) { + if (oldColumn.getColumnName().equalsIgnoreCase(newColumn.getColumnName())) { + newColumn.setUpdate(newColumn.isUpdate() || oldColumn.isUpdate()); + contain = true; + } + } + + if (!contain) { + temp.add(oldColumn); + } + } + newColumns.addAll(temp); + Collections.sort(newColumns, new EventColumnIndexComparable()); + newRecord.setOldKeys(oldRecord.getOldKeys()); + if (oldRecord.getSyncConsistency() != null) { + newRecord.setSyncConsistency(oldRecord.getSyncConsistency()); + } + if (oldRecord.getSyncMode() != null) { + newRecord.setSyncMode(oldRecord.getSyncMode()); + } + + if (oldRecord.isRemedy()) { + newRecord.setRemedy(true); + } + newRecord.setSize(oldRecord.getSize() + newRecord.getSize()); + return newRecord; + } + + @Setter + @Getter + public static class RowKey implements Serializable { + + private String schemaName; + private String tableName; + + public RowKey(String schemaName, String tableName, List keys) { + this.schemaName = schemaName; + this.tableName = tableName; + this.keys = keys; + } + + public RowKey(List keys) { + this.keys = keys; + } + + private List keys = new ArrayList(); + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((keys == null) ? 0 : keys.hashCode()); + result = prime * result + ((schemaName == null) ? 0 : schemaName.hashCode()); + result = prime * result + ((tableName == null) ? 0 : tableName.hashCode()); + return result; + } + + @SuppressWarnings("checkstyle:NeedBraces") + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null) { + return false; + } + if (!(obj instanceof RowKey)) { + return false; + } + RowKey other = (RowKey) obj; + if (keys == null) { + if (other.keys != null) { + return false; + } + } else if (!keys.equals(other.keys)) { + return false; + } + if (schemaName == null) { + if (other.schemaName != null) { + return false; + } + } else if (!schemaName.equals(other.schemaName)) { + return false; + } + if (tableName == null) { + return other.tableName == null; + } else { + return tableName.equals(other.tableName); + } + } + + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/GtidBatch.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/GtidBatch.java new file mode 100644 index 0000000000..dd6559b832 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/GtidBatch.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.sink; + +import org.apache.eventmesh.connector.canal.CanalConnectRecord; + +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +public class GtidBatch { + private int totalBatches; + private List> batches; + private int receivedBatchCount; + + public GtidBatch(int totalBatches) { + this.totalBatches = totalBatches; + this.batches = new CopyOnWriteArrayList<>(new List[totalBatches]); + this.receivedBatchCount = 0; + } + + public void addBatch(int batchIndex, List batchRecords) { + batches.set(batchIndex, batchRecords); + receivedBatchCount++; + } + + public List> getBatches() { + return batches; + } + + public boolean isComplete() { + return receivedBatchCount == totalBatches; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/GtidBatchManager.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/GtidBatchManager.java new file mode 100644 index 0000000000..30060aa8f5 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/GtidBatchManager.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.sink; + +import org.apache.eventmesh.connector.canal.CanalConnectRecord; + +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; + +public class GtidBatchManager { + + private static ConcurrentHashMap gtidBatchMap = new ConcurrentHashMap<>(); + + public static void addBatch(String gtid, int batchIndex, int totalBatches, List batchRecords) { + gtidBatchMap.computeIfAbsent(gtid, k -> new GtidBatch(totalBatches)).addBatch(batchIndex, batchRecords); + } + + public static GtidBatch getGtidBatch(String gtid) { + return gtidBatchMap.get(gtid); + } + + public static boolean isComplete(String gtid) { + GtidBatch batch = gtidBatchMap.get(gtid); + return batch != null && batch.isComplete(); + } + + public static void removeGtidBatch(String gtid) { + gtidBatchMap.remove(gtid); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalCheckConsumer.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalCheckConsumer.java new file mode 100644 index 0000000000..fb9a33b49f --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalCheckConsumer.java @@ -0,0 +1,540 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.sink.connector; + +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalMySQLType; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkFullConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.Constants; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLColumnDef; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLTableDef; +import org.apache.eventmesh.common.remote.offset.canal.CanalFullRecordOffset; +import org.apache.eventmesh.common.utils.JsonUtils; +import org.apache.eventmesh.connector.canal.DatabaseConnection; +import org.apache.eventmesh.connector.canal.SqlUtils; +import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; +import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendExceptionContext; +import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendResult; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; + +import org.apache.commons.lang3.StringUtils; + +import java.math.BigDecimal; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Types; +import java.text.MessageFormat; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.LockSupport; + +import com.alibaba.druid.pool.DruidPooledConnection; +import com.fasterxml.jackson.core.type.TypeReference; + +import lombok.extern.slf4j.Slf4j; + + +@Slf4j +public class CanalCheckConsumer { + private BlockingQueue> queue; + private RdbTableMgr tableMgr; + private CanalSinkFullConfig config; + private final DateTimeFormatter dataTimePattern = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSSSSS"); + + + public CanalCheckConsumer(BlockingQueue> queue, RdbTableMgr tableMgr, CanalSinkFullConfig config) { + this.config = config; + this.queue = queue; + this.tableMgr = tableMgr; + } + + + public void start(AtomicBoolean flag) { + while (flag.get()) { + List sinkRecords = null; + try { + sinkRecords = queue.poll(2, TimeUnit.SECONDS); + if (sinkRecords == null || sinkRecords.isEmpty()) { + continue; + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + ConnectRecord record = sinkRecords.get(0); + Map dataMap = + JsonUtils.parseTypeReferenceObject((byte[]) record.getData(), new TypeReference>() { + }); + + List> sourceRows = JsonUtils.parseObject(dataMap.get("data").toString(), List.class); + + if (sourceRows == null || sourceRows.isEmpty()) { + if (log.isDebugEnabled()) { + log.debug("[{}] got rows data is none", this.getClass()); + } + return; + } + CanalFullRecordOffset offset = JsonUtils.parseObject(dataMap.get("offset").toString(), CanalFullRecordOffset.class); + if (offset == null || offset.getPosition() == null) { + if (log.isDebugEnabled()) { + log.debug("[{}] got canal full offset is none", this.getClass()); + } + return; + } + + MySQLTableDef tableDefinition = (MySQLTableDef) tableMgr.getTable(offset.getPosition().getSchema(), offset.getPosition().getTableName()); + if (tableDefinition == null) { + log.warn("target schema [{}] table [{}] is not exists", offset.getPosition().getSchema(), offset.getPosition().getTableName()); + return; + } + + String sql = genTargetPkInSql(tableDefinition, sourceRows.size(), Constants.MySQLQuot, Constants.MySQLQuot, "*"); + DruidPooledConnection connection = null; + PreparedStatement statement = null; + try { + connection = DatabaseConnection.sinkDataSource.getConnection(); + statement = + connection.prepareStatement(sql); + setPrepareParams(statement, sourceRows, tableDefinition); + log.debug("select sql {}", statement.toString()); + ResultSet resultSet = statement.executeQuery(); + List> targetRows = new LinkedList<>(); + while (resultSet.next()) { + Map columnValues = new LinkedHashMap<>(); + for (Map.Entry col : + tableDefinition.getColumnDefinitions().entrySet()) { + columnValues.put(col.getKey(), readColumn(resultSet, col.getKey(), + col.getValue().getType())); + } + targetRows.add(columnValues); + } + compareData(sourceRows, targetRows, tableDefinition); + record.getCallback().onSuccess(convertToSendResult(record)); + } catch (SQLException e) { + log.warn("check sink process schema [{}] table [{}] connector check fail", tableDefinition.getSchemaName(), + tableDefinition.getTableName(), + e); + LockSupport.parkNanos(3000 * 1000L); + record.getCallback().onException(buildSendExceptionContext(record, e)); + } catch (Exception e) { + log.error("check sink process schema [{}] table [{}] catch unknown exception", tableDefinition.getSchemaName(), + tableDefinition.getTableName(), e); + record.getCallback().onException(buildSendExceptionContext(record, e)); + } finally { + if (statement != null) { + try { + statement.close(); + } catch (SQLException e) { + log.error("close prepare statement fail", e); + } + } + + if (connection != null) { + try { + connection.close(); + } catch (SQLException e) { + log.error("close db connection fail", e); + } + } + } + } + } + + private void compareData(List> sourceRows, List> targetRows, MySQLTableDef tableDefinition) { + List> differenceSource = new ArrayList<>(sourceRows); + List> differenceTarget = new ArrayList<>(targetRows); + // Find common elements and remove from difference lists + for (Map source : sourceRows) { + for (Map target : targetRows) { + if (source.equals(target)) { + differenceSource.remove(source); + differenceTarget.remove(target); + break; + } + } + } + if (!differenceSource.isEmpty()) { + log.error("source rows is not equals target rows, source rows are [{}]", differenceSource); + } + + if (!differenceTarget.isEmpty()) { + log.error("source rows is not equals target rows, target rows are [{}]", differenceTarget); + } + } + + private void setPrepareParams(PreparedStatement preparedStatement, List> rows, MySQLTableDef tableDef) throws Exception { + List cols = new ArrayList<>(tableDef.getColumnDefinitions().values()); + int index = 0; + for (Map col : rows) { + for (MySQLColumnDef mySQLColumnDef : cols) { + if (tableDef.getPrimaryKeys().contains(mySQLColumnDef.getName())) { + index++; + writeColumn(preparedStatement, index, mySQLColumnDef, col.get(mySQLColumnDef.getName())); + } + } + } + } + + public Object readColumn(ResultSet rs, String colName, CanalMySQLType colType) throws Exception { + switch (colType) { + case TINYINT: + case SMALLINT: + case MEDIUMINT: + case INT: + Long valueLong = rs.getLong(colName); + if (rs.wasNull()) { + return null; + } + if (valueLong.compareTo((long) Integer.MAX_VALUE) > 0) { + return valueLong; + } + return valueLong.intValue(); + case BIGINT: + String v = rs.getString(colName); + if (v == null) { + return null; + } + BigDecimal valueBigInt = new BigDecimal(v); + if (valueBigInt.compareTo(BigDecimal.valueOf(Long.MAX_VALUE)) > 0) { + return valueBigInt; + } + return valueBigInt.longValue(); + case FLOAT: + case DOUBLE: + case DECIMAL: + return rs.getBigDecimal(colName); + case DATE: + return rs.getObject(colName, LocalDate.class).toString(); + case TIME: + return rs.getObject(colName, LocalTime.class).toString(); + case DATETIME: + case TIMESTAMP: + return rs.getObject(colName, LocalDateTime.class).toString(); + case YEAR: + int year = rs.getInt(colName); + if (rs.wasNull()) { + return null; + } + return year; + case CHAR: + case VARCHAR: + case TINYTEXT: + case TEXT: + case MEDIUMTEXT: + case LONGTEXT: + case ENUM: + case SET: + case JSON: + return rs.getString(colName); + case BIT: + case BINARY: + case VARBINARY: + case TINYBLOB: + case BLOB: + case MEDIUMBLOB: + case LONGBLOB: + return rs.getBytes(colName); + case GEOMETRY: + case GEOMETRY_COLLECTION: + case GEOM_COLLECTION: + case POINT: + case LINESTRING: + case POLYGON: + case MULTIPOINT: + case MULTILINESTRING: + case MULTIPOLYGON: + byte[] geo = rs.getBytes(colName); + if (geo == null) { + return null; + } + return SqlUtils.toGeometry(geo); + default: + return rs.getObject(colName); + } + } + + public void writeColumn(PreparedStatement ps, int index, MySQLColumnDef colType, Object value) throws Exception { + if (colType == null) { + String colVal = null; + if (value != null) { + colVal = value.toString(); + } + if (colVal == null) { + ps.setNull(index, Types.VARCHAR); + } else { + ps.setString(index, colVal); + } + } else if (value == null) { + ps.setNull(index, colType.getJdbcType().getVendorTypeNumber()); + } else { + switch (colType.getType()) { + case TINYINT: + case SMALLINT: + case MEDIUMINT: + case INT: + Long longValue = SqlUtils.toLong(value); + if (longValue == null) { + ps.setNull(index, 4); + return; + } else { + ps.setLong(index, longValue); + return; + } + case BIGINT: + case DECIMAL: + BigDecimal bigDecimalValue = SqlUtils.toBigDecimal(value); + if (bigDecimalValue == null) { + ps.setNull(index, 3); + return; + } else { + ps.setBigDecimal(index, bigDecimalValue); + return; + } + case FLOAT: + case DOUBLE: + Double doubleValue = SqlUtils.toDouble(value); + if (doubleValue == null) { + ps.setNull(index, 8); + } else { + ps.setDouble(index, doubleValue); + } + return; + case DATE: + case DATETIME: + case TIMESTAMP: + LocalDateTime dateValue = null; + if (!SqlUtils.isZeroTime(value)) { + try { + dateValue = SqlUtils.toLocalDateTime(value); + } catch (Exception e) { + ps.setString(index, SqlUtils.convertToString(value)); + return; + } + } else if (StringUtils.isNotBlank(config.getZeroDate())) { + dateValue = SqlUtils.toLocalDateTime(config.getZeroDate()); + } else { + ps.setObject(index, value); + return; + } + if (dateValue == null) { + ps.setNull(index, Types.TIMESTAMP); + } else { + ps.setString(index, dataTimePattern.format(dateValue)); + } + return; + case TIME: + String timeValue = SqlUtils.toMySqlTime(value); + if (StringUtils.isBlank(timeValue)) { + ps.setNull(index, 12); + return; + } else { + ps.setString(index, timeValue); + return; + } + case YEAR: + LocalDateTime yearValue = null; + if (!SqlUtils.isZeroTime(value)) { + yearValue = SqlUtils.toLocalDateTime(value); + } else if (StringUtils.isNotBlank(config.getZeroDate())) { + yearValue = SqlUtils.toLocalDateTime(config.getZeroDate()); + } else { + ps.setInt(index, 0); + return; + } + if (yearValue == null) { + ps.setNull(index, 4); + } else { + ps.setInt(index, yearValue.getYear()); + } + return; + case CHAR: + case VARCHAR: + case TINYTEXT: + case TEXT: + case MEDIUMTEXT: + case LONGTEXT: + case ENUM: + case SET: + String strValue = value.toString(); + if (strValue == null) { + ps.setNull(index, Types.VARCHAR); + return; + } else { + ps.setString(index, strValue); + return; + } + case JSON: + String jsonValue = value.toString(); + if (jsonValue == null) { + ps.setNull(index, Types.VARCHAR); + } else { + ps.setString(index, jsonValue); + } + return; + case BIT: + if (value instanceof Boolean) { + byte[] arrayBoolean = new byte[1]; + arrayBoolean[0] = (byte) (Boolean.TRUE.equals(value) ? 1 : 0); + ps.setBytes(index, arrayBoolean); + return; + } else if (value instanceof Number) { + ps.setBytes(index, SqlUtils.numberToBinaryArray((Number) value)); + return; + } else if ((value instanceof byte[]) || value.toString().startsWith("0x") || value.toString().startsWith("0X")) { + byte[] arrayBoolean = SqlUtils.toBytes(value); + if (arrayBoolean == null || arrayBoolean.length == 0) { + ps.setNull(index, Types.BIT); + return; + } else { + ps.setBytes(index, arrayBoolean); + return; + } + } else { + ps.setBytes(index, SqlUtils.numberToBinaryArray(SqlUtils.toInt(value))); + return; + } + case BINARY: + case VARBINARY: + case TINYBLOB: + case BLOB: + case MEDIUMBLOB: + case LONGBLOB: + byte[] binaryValue = SqlUtils.toBytes(value); + if (binaryValue == null) { + ps.setNull(index, Types.BINARY); + return; + } else { + ps.setBytes(index, binaryValue); + return; + } + case GEOMETRY: + case GEOMETRY_COLLECTION: + case GEOM_COLLECTION: + case POINT: + case LINESTRING: + case POLYGON: + case MULTIPOINT: + case MULTILINESTRING: + case MULTIPOLYGON: + String geoValue = SqlUtils.toGeometry(value); + if (geoValue == null) { + ps.setNull(index, Types.VARCHAR); + return; + } + ps.setString(index, geoValue); + return; + default: + throw new UnsupportedOperationException("columnType '" + colType + "' Unsupported."); + } + } + } + + public String genTargetPkInSql(MySQLTableDef def, int pkGroupSize, String leftQuote, String rightQuote, String selectEleStr) { + List pkCols = def.getPrimaryKeys(); + if (pkCols == null || pkCols.isEmpty()) { + throw new IllegalArgumentException("unsupported pk is empty table check."); + } else if (pkCols.size() == 1) { + return genSinglePkInSql(def, pkGroupSize, leftQuote, rightQuote, selectEleStr); + } else { + return genMultiPkInSql(def, pkGroupSize, leftQuote, rightQuote, selectEleStr); + } + } + + public String genSinglePkInSql(MySQLTableDef def, int pkGroupSize, String leftQuote, String rightQuote, String selectEleStr) { + return MessageFormat.format(genFetchSqlFormat(leftQuote, rightQuote, selectEleStr), def.getSchemaName(), def.getTableName(), + leftQuote + def.getPrimaryKeys().get(0) + rightQuote, genSinglePkPlaceHolderStr(pkGroupSize)); + } + + public String genMultiPkInSql(MySQLTableDef def, int pkGroupSize, String leftQuote, String rightQuote, String selectEleStr) { + String fetchSqlFormat = genFetchSqlFormat(leftQuote, rightQuote, selectEleStr); + List pkCols = def.getPrimaryKeys(); + StringBuilder pksBuilder = new StringBuilder("("); + for (int i = 0; i < pkCols.size(); i++) { + if (i != 0) { + pksBuilder.append(","); + } + pksBuilder.append(leftQuote).append(pkCols.get(i)).append(rightQuote); + } + pksBuilder.append(")"); + return MessageFormat.format(fetchSqlFormat, def.getSchemaName(), def.getTableName(), pksBuilder.toString(), + genMultiPkPlaceHolderStr(pkGroupSize, pkCols.size())); + } + + public String genFetchSqlFormat(String leftQuote, String rightQuote, String selectEleStr) { + return "select " + selectEleStr + " from " + leftQuote + "{0}" + rightQuote + "." + leftQuote + "{1}" + rightQuote + " where {2} in ({3})"; + } + + public String genSinglePkPlaceHolderStr(int valueSize) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < valueSize; i++) { + if (i != 0) { + sb.append(","); + } + sb.append("?"); + } + return sb.toString(); + } + + public String genMultiPkPlaceHolderStr(int valueSize, int sizePerGroup) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < valueSize; i++) { + if (i != 0) { + sb.append(","); + } + sb.append("("); + for (int j = 0; j < sizePerGroup; j++) { + if (j != 0) { + sb.append(","); + } + sb.append("?"); + } + sb.append(")"); + } + return sb.toString(); + } + + + + private SendExceptionContext buildSendExceptionContext(ConnectRecord record, Throwable e) { + SendExceptionContext sendExceptionContext = new SendExceptionContext(); + sendExceptionContext.setMessageId(record.getRecordId()); + sendExceptionContext.setCause(e); + if (StringUtils.isNotEmpty(record.getExtension("topic"))) { + sendExceptionContext.setTopic(record.getExtension("topic")); + } + return sendExceptionContext; + } + + private SendResult convertToSendResult(ConnectRecord record) { + SendResult result = new SendResult(); + result.setMessageId(record.getRecordId()); + if (StringUtils.isNotEmpty(record.getExtension("topic"))) { + result.setTopic(record.getExtension("topic")); + } + return result; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalFullConsumer.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalFullConsumer.java new file mode 100644 index 0000000000..939d1101aa --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalFullConsumer.java @@ -0,0 +1,391 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.sink.connector; + +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkFullConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.Constants; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLColumnDef; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLTableDef; +import org.apache.eventmesh.common.remote.offset.canal.CanalFullRecordOffset; +import org.apache.eventmesh.common.utils.JsonUtils; +import org.apache.eventmesh.connector.canal.DatabaseConnection; +import org.apache.eventmesh.connector.canal.SqlUtils; +import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; +import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendExceptionContext; +import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendResult; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; + +import org.apache.commons.lang3.StringUtils; + +import java.math.BigDecimal; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Types; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.LockSupport; + +import com.alibaba.druid.pool.DruidPooledConnection; +import com.fasterxml.jackson.core.type.TypeReference; + +import lombok.extern.slf4j.Slf4j; + + +@Slf4j +public class CanalFullConsumer { + private BlockingQueue> queue; + private RdbTableMgr tableMgr; + private CanalSinkFullConfig config; + private final DateTimeFormatter dataTimePattern = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSSSSSSS"); + + + public CanalFullConsumer(BlockingQueue> queue, RdbTableMgr tableMgr, CanalSinkFullConfig config) { + this.config = config; + this.queue = queue; + this.tableMgr = tableMgr; + } + + + public void start(AtomicBoolean flag) { + while (flag.get()) { + List sinkRecords = null; + try { + sinkRecords = queue.poll(2, TimeUnit.SECONDS); + if (sinkRecords == null || sinkRecords.isEmpty()) { + continue; + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + ConnectRecord record = sinkRecords.get(0); + Map dataMap = + JsonUtils.parseTypeReferenceObject((byte[]) record.getData(), new TypeReference>() { + }); + + List> rows = JsonUtils.parseObject(dataMap.get("data").toString(), List.class); + + if (rows == null || rows.isEmpty()) { + if (log.isDebugEnabled()) { + log.debug("[{}] got rows data is none", this.getClass()); + } + return; + } + CanalFullRecordOffset offset = JsonUtils.parseObject(dataMap.get("offset").toString(), CanalFullRecordOffset.class); + if (offset == null || offset.getPosition() == null) { + if (log.isDebugEnabled()) { + log.debug("[{}] got canal full offset is none", this.getClass()); + } + return; + } + + MySQLTableDef tableDefinition = (MySQLTableDef) tableMgr.getTable(offset.getPosition().getSchema(), offset.getPosition().getTableName()); + if (tableDefinition == null) { + log.warn("target schema [{}] table [{}] is not exists", offset.getPosition().getSchema(), offset.getPosition().getTableName()); + return; + } + List cols = new ArrayList<>(tableDefinition.getColumnDefinitions().values()); + String sql = generateInsertPrepareSql(offset.getPosition().getSchema(), offset.getPosition().getTableName(), + cols); + DruidPooledConnection connection = null; + PreparedStatement statement = null; + try { + connection = DatabaseConnection.sinkDataSource.getConnection(); + statement = + connection.prepareStatement(sql); + for (Map col : rows) { + setPrepareParams(statement, col, cols); + log.debug("insert sql {}", statement.toString()); + statement.addBatch(); + } + statement.executeBatch(); + connection.commit(); + log.info("execute batch insert sql size: {}", rows.size()); + record.getCallback().onSuccess(convertToSendResult(record)); + } catch (SQLException e) { + log.warn("full sink process schema [{}] table [{}] connector write fail", tableDefinition.getSchemaName(), + tableDefinition.getTableName(), + e); + LockSupport.parkNanos(3000 * 1000L); + record.getCallback().onException(buildSendExceptionContext(record, e)); + } catch (Exception e) { + log.error("full sink process schema [{}] table [{}] catch unknown exception", tableDefinition.getSchemaName(), + tableDefinition.getTableName(), e); + record.getCallback().onException(buildSendExceptionContext(record, e)); + try { + if (connection != null && !connection.isClosed()) { + connection.rollback(); + } + } catch (SQLException rollback) { + log.warn("full sink process schema [{}] table [{}] rollback fail", tableDefinition.getSchemaName(), + tableDefinition.getTableName(), e); + } + } finally { + if (statement != null) { + try { + statement.close(); + } catch (SQLException e) { + log.error("close prepare statement fail", e); + } + } + + if (connection != null) { + try { + connection.close(); + } catch (SQLException e) { + log.error("close db connection fail", e); + } + } + } + } + } + + + private SendExceptionContext buildSendExceptionContext(ConnectRecord record, Throwable e) { + SendExceptionContext sendExceptionContext = new SendExceptionContext(); + sendExceptionContext.setMessageId(record.getRecordId()); + sendExceptionContext.setCause(e); + if (StringUtils.isNotEmpty(record.getExtension("topic"))) { + sendExceptionContext.setTopic(record.getExtension("topic")); + } + return sendExceptionContext; + } + + private SendResult convertToSendResult(ConnectRecord record) { + SendResult result = new SendResult(); + result.setMessageId(record.getRecordId()); + if (StringUtils.isNotEmpty(record.getExtension("topic"))) { + result.setTopic(record.getExtension("topic")); + } + return result; + } + + private void setPrepareParams(PreparedStatement preparedStatement, Map col, List columnDefs) throws Exception { + for (int i = 0; i < columnDefs.size(); i++) { + writeColumn(preparedStatement, i + 1, columnDefs.get(i), col.get(columnDefs.get(i).getName())); + } + } + + public void writeColumn(PreparedStatement ps, int index, MySQLColumnDef colType, Object value) throws Exception { + if (colType == null) { + String colVal = null; + if (value != null) { + colVal = value.toString(); + } + if (colVal == null) { + ps.setNull(index, Types.VARCHAR); + } else { + ps.setString(index, colVal); + } + } else if (value == null) { + ps.setNull(index, colType.getJdbcType().getVendorTypeNumber()); + } else { + switch (colType.getType()) { + case TINYINT: + case SMALLINT: + case MEDIUMINT: + case INT: + Long longValue = SqlUtils.toLong(value); + if (longValue == null) { + ps.setNull(index, 4); + return; + } else { + ps.setLong(index, longValue); + return; + } + case BIGINT: + case DECIMAL: + BigDecimal bigDecimalValue = SqlUtils.toBigDecimal(value); + if (bigDecimalValue == null) { + ps.setNull(index, 3); + return; + } else { + ps.setBigDecimal(index, bigDecimalValue); + return; + } + case FLOAT: + case DOUBLE: + Double doubleValue = SqlUtils.toDouble(value); + if (doubleValue == null) { + ps.setNull(index, 8); + } else { + ps.setDouble(index, doubleValue); + } + return; + case DATE: + case DATETIME: + case TIMESTAMP: + LocalDateTime dateValue = null; + if (!SqlUtils.isZeroTime(value)) { + try { + dateValue = SqlUtils.toLocalDateTime(value); + } catch (Exception e) { + ps.setString(index, SqlUtils.convertToString(value)); + return; + } + } else if (StringUtils.isNotBlank(config.getZeroDate())) { + dateValue = SqlUtils.toLocalDateTime(config.getZeroDate()); + } else { + ps.setObject(index, value); + return; + } + if (dateValue == null) { + ps.setNull(index, Types.TIMESTAMP); + } else { + ps.setString(index, dataTimePattern.format(dateValue)); + } + return; + case TIME: + String timeValue = SqlUtils.toMySqlTime(value); + if (StringUtils.isBlank(timeValue)) { + ps.setNull(index, 12); + return; + } else { + ps.setString(index, timeValue); + return; + } + case YEAR: + LocalDateTime yearValue = null; + if (!SqlUtils.isZeroTime(value)) { + yearValue = SqlUtils.toLocalDateTime(value); + } else if (StringUtils.isNotBlank(config.getZeroDate())) { + yearValue = SqlUtils.toLocalDateTime(config.getZeroDate()); + } else { + ps.setInt(index, 0); + return; + } + if (yearValue == null) { + ps.setNull(index, 4); + } else { + ps.setInt(index, yearValue.getYear()); + } + return; + case CHAR: + case VARCHAR: + case TINYTEXT: + case TEXT: + case MEDIUMTEXT: + case LONGTEXT: + case ENUM: + case SET: + String strValue = value.toString(); + if (strValue == null) { + ps.setNull(index, Types.VARCHAR); + return; + } else { + ps.setString(index, strValue); + return; + } + case JSON: + String jsonValue = value.toString(); + if (jsonValue == null) { + ps.setNull(index, Types.VARCHAR); + } else { + ps.setString(index, jsonValue); + } + return; + case BIT: + if (value instanceof Boolean) { + byte[] arrayBoolean = new byte[1]; + arrayBoolean[0] = (byte) (Boolean.TRUE.equals(value) ? 1 : 0); + ps.setBytes(index, arrayBoolean); + return; + } else if (value instanceof Number) { + ps.setBytes(index, SqlUtils.numberToBinaryArray((Number) value)); + return; + } else if ((value instanceof byte[]) || value.toString().startsWith("0x") || value.toString().startsWith("0X")) { + byte[] arrayBoolean = SqlUtils.toBytes(value); + if (arrayBoolean == null || arrayBoolean.length == 0) { + ps.setNull(index, Types.BIT); + return; + } else { + ps.setBytes(index, arrayBoolean); + return; + } + } else { + ps.setBytes(index, SqlUtils.numberToBinaryArray(SqlUtils.toInt(value))); + return; + } + case BINARY: + case VARBINARY: + case TINYBLOB: + case BLOB: + case MEDIUMBLOB: + case LONGBLOB: + byte[] binaryValue = SqlUtils.toBytes(value); + if (binaryValue == null) { + ps.setNull(index, Types.BINARY); + return; + } else { + ps.setBytes(index, binaryValue); + return; + } + case GEOMETRY: + case GEOMETRY_COLLECTION: + case GEOM_COLLECTION: + case POINT: + case LINESTRING: + case POLYGON: + case MULTIPOINT: + case MULTILINESTRING: + case MULTIPOLYGON: + String geoValue = SqlUtils.toGeometry(value); + if (geoValue == null) { + ps.setNull(index, Types.VARCHAR); + return; + } + ps.setString(index, geoValue); + return; + default: + throw new UnsupportedOperationException("columnType '" + colType + "' Unsupported."); + } + } + } + + private String generateInsertPrepareSql(String schema, String table, List cols) { + StringBuilder builder = new StringBuilder(); + builder.append("INSERT IGNORE INTO "); + builder.append(Constants.MySQLQuot); + builder.append(schema); + builder.append(Constants.MySQLQuot); + builder.append("."); + builder.append(Constants.MySQLQuot); + builder.append(table); + builder.append(Constants.MySQLQuot); + StringBuilder columns = new StringBuilder(); + StringBuilder values = new StringBuilder(); + for (MySQLColumnDef colInfo : cols) { + if (columns.length() > 0) { + columns.append(", "); + values.append(", "); + } + String wrapName = Constants.MySQLQuot + colInfo.getName() + Constants.MySQLQuot; + columns.append(wrapName); + values.append(colInfo.getType() == null ? "?" : colInfo.getType().genPrepareStatement4Insert()); + } + builder.append("(").append(columns).append(")"); + builder.append(" VALUES "); + builder.append("(").append(values).append(")"); + return builder.toString(); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkCheckConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkCheckConnector.java new file mode 100644 index 0000000000..6819c936fd --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkCheckConnector.java @@ -0,0 +1,151 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.sink.connector; + +import org.apache.eventmesh.common.EventMeshThreadFactory; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkFullConfig; +import org.apache.eventmesh.common.exception.EventMeshException; +import org.apache.eventmesh.connector.canal.DatabaseConnection; +import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; +import org.apache.eventmesh.openconnect.api.ConnectorCreateService; +import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; +import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; +import org.apache.eventmesh.openconnect.api.sink.Sink; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; +import org.apache.eventmesh.openconnect.util.ConfigUtil; + +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class CanalSinkCheckConnector implements Sink, ConnectorCreateService { + + private CanalSinkFullConfig config; + private RdbTableMgr tableMgr; + private ThreadPoolExecutor executor; + private final BlockingQueue> queue = new LinkedBlockingQueue<>(10000); + private final AtomicBoolean flag = new AtomicBoolean(true); + + @Override + public void start() throws Exception { + tableMgr.start(); + } + + @Override + public void stop() throws Exception { + flag.set(false); + if (!executor.isShutdown()) { + executor.shutdown(); + try { + if (!executor.awaitTermination(5, TimeUnit.SECONDS)) { + log.warn("wait thread pool shutdown timeout, it will shutdown now"); + executor.shutdownNow(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.info("shutdown thread pool fail"); + } + } + if (DatabaseConnection.sinkDataSource != null) { + DatabaseConnection.sinkDataSource.close(); + log.info("data source has been closed"); + } + } + + @Override + public Sink create() { + return new CanalSinkCheckConnector(); + } + + @Override + public Class configClass() { + return CanalSinkFullConfig.class; + } + + @Override + public void init(Config config) throws Exception { + this.config = (CanalSinkFullConfig) config; + init(); + } + + @Override + public void init(ConnectorContext connectorContext) throws Exception { + CanalSinkConfig canalSinkConfig = (CanalSinkConfig) ((SinkConnectorContext) connectorContext).getSinkConfig(); + this.config = ConfigUtil.parse(canalSinkConfig.getSinkConfig(), CanalSinkFullConfig.class); + init(); + } + + private void init() { + if (config.getSinkConnectorConfig() == null) { + throw new EventMeshException(String.format("[%s] sink config is null", this.getClass())); + } + DatabaseConnection.sinkConfig = this.config.getSinkConnectorConfig(); + DatabaseConnection.initSinkConnection(); + DatabaseConnection.sinkDataSource.setDefaultAutoCommit(false); + + tableMgr = new RdbTableMgr(this.config.getSinkConnectorConfig(), DatabaseConnection.sinkDataSource); + executor = new ThreadPoolExecutor(config.getParallel(), config.getParallel(), 0L, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(), new EventMeshThreadFactory("canal-sink-check")); + List consumers = new LinkedList<>(); + for (int i = 0; i < config.getParallel(); i++) { + CanalCheckConsumer canalCheckConsumer = new CanalCheckConsumer(queue, tableMgr, config); + consumers.add(canalCheckConsumer); + } + consumers.forEach(c -> executor.execute(() -> c.start(flag))); + } + + @Override + public void commit(ConnectRecord record) { + + } + + @Override + public String name() { + return null; + } + + @Override + public void onException(ConnectRecord record) { + + } + + @Override + public void put(List sinkRecords) { + if (sinkRecords == null || sinkRecords.isEmpty() || sinkRecords.get(0) == null) { + if (log.isDebugEnabled()) { + log.debug("[{}] got sink records are none", this.getClass()); + } + return; + } + try { + queue.put(sinkRecords); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkConnector.java new file mode 100644 index 0000000000..b03df2dfff --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkConnector.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.sink.connector; + +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkConfig; +import org.apache.eventmesh.common.remote.job.JobType; +import org.apache.eventmesh.openconnect.api.ConnectorCreateService; +import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; +import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; +import org.apache.eventmesh.openconnect.api.sink.Sink; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; + +import java.util.List; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class CanalSinkConnector implements Sink, ConnectorCreateService { + + private CanalSinkConfig sinkConfig; + + private Sink sink; + + @Override + public Class configClass() { + return CanalSinkConfig.class; + } + + @Override + public void init(Config config) throws Exception { + // init config for canal source connector + this.sinkConfig = (CanalSinkConfig) config; + } + + @Override + public void init(ConnectorContext connectorContext) throws Exception { + // init config for canal source connector + SinkConnectorContext sinkConnectorContext = (SinkConnectorContext) connectorContext; + if (sinkConnectorContext.getJobType().equals(JobType.FULL)) { + this.sink = new CanalSinkFullConnector(); + } else if (sinkConnectorContext.getJobType().equals(JobType.INCREASE)) { + this.sink = new CanalSinkIncrementConnector(); + } else if (sinkConnectorContext.getJobType().equals(JobType.CHECK)) { + this.sink = new CanalSinkCheckConnector(); + } else { + throw new RuntimeException("unsupported job type " + sinkConnectorContext.getJobType()); + } + this.sink.init(sinkConnectorContext); + } + + @Override + public void start() throws Exception { + this.sink.start(); + } + + @Override + public void commit(ConnectRecord record) { + this.sink.commit(record); + } + + @Override + public String name() { + return this.sink.name(); + } + + @Override + public void onException(ConnectRecord record) { + this.sink.onException(record); + } + + @Override + public void stop() throws Exception { + this.sink.stop(); + } + + @Override + public void put(List sinkRecords) { + this.sink.put(sinkRecords); + } + + @Override + public Sink create() { + return new CanalSinkConnector(); + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java new file mode 100644 index 0000000000..cb50dc5648 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkFullConnector.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.sink.connector; + +import org.apache.eventmesh.common.EventMeshThreadFactory; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkFullConfig; +import org.apache.eventmesh.common.exception.EventMeshException; +import org.apache.eventmesh.connector.canal.DatabaseConnection; +import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; +import org.apache.eventmesh.openconnect.api.ConnectorCreateService; +import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; +import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; +import org.apache.eventmesh.openconnect.api.sink.Sink; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; +import org.apache.eventmesh.openconnect.util.ConfigUtil; + +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class CanalSinkFullConnector implements Sink, ConnectorCreateService { + + private CanalSinkFullConfig config; + private RdbTableMgr tableMgr; + private ThreadPoolExecutor executor; + private final BlockingQueue> queue = new LinkedBlockingQueue<>(10000); + private final AtomicBoolean flag = new AtomicBoolean(true); + + @Override + public void start() throws Exception { + tableMgr.start(); + } + + @Override + public void stop() throws Exception { + flag.set(false); + if (!executor.isShutdown()) { + executor.shutdown(); + try { + if (!executor.awaitTermination(5, TimeUnit.SECONDS)) { + log.warn("wait thread pool shutdown timeout, it will shutdown now"); + executor.shutdownNow(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.info("shutdown thread pool fail"); + } + } + if (DatabaseConnection.sinkDataSource != null) { + DatabaseConnection.sinkDataSource.close(); + log.info("data source has been closed"); + } + } + + @Override + public Sink create() { + return new CanalSinkFullConnector(); + } + + @Override + public Class configClass() { + return CanalSinkFullConfig.class; + } + + @Override + public void init(Config config) throws Exception { + this.config = (CanalSinkFullConfig) config; + init(); + } + + @Override + public void init(ConnectorContext connectorContext) throws Exception { + SinkConnectorContext sinkConnectorContext = (SinkConnectorContext) connectorContext; + CanalSinkConfig canalSinkConfig = (CanalSinkConfig) sinkConnectorContext.getSinkConfig(); + this.config = ConfigUtil.parse(canalSinkConfig.getSinkConfig(), CanalSinkFullConfig.class); + init(); + } + + private void init() { + if (config.getSinkConnectorConfig() == null) { + throw new EventMeshException(String.format("[%s] sink config is null", this.getClass())); + } + DatabaseConnection.sinkConfig = this.config.getSinkConnectorConfig(); + DatabaseConnection.initSinkConnection(); + DatabaseConnection.sinkDataSource.setDefaultAutoCommit(false); + + tableMgr = new RdbTableMgr(this.config.getSinkConnectorConfig(), DatabaseConnection.sinkDataSource); + executor = new ThreadPoolExecutor(config.getParallel(), config.getParallel(), 0L, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(), new EventMeshThreadFactory("canal-sink-full")); + List consumers = new LinkedList<>(); + for (int i = 0; i < config.getParallel(); i++) { + CanalFullConsumer canalFullConsumer = new CanalFullConsumer(queue, tableMgr, config); + consumers.add(canalFullConsumer); + } + consumers.forEach(c -> executor.execute(() -> c.start(flag))); + } + + @Override + public void commit(ConnectRecord record) { + + } + + @Override + public String name() { + return null; + } + + @Override + public void onException(ConnectRecord record) { + + } + + @Override + public void put(List sinkRecords) { + if (sinkRecords == null || sinkRecords.isEmpty() || sinkRecords.get(0) == null) { + if (log.isDebugEnabled()) { + log.debug("[{}] got sink records are none", this.getClass()); + } + return; + } + try { + queue.put(sinkRecords); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkIncrementConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkIncrementConnector.java new file mode 100644 index 0000000000..84373ae7a7 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/sink/connector/CanalSinkIncrementConnector.java @@ -0,0 +1,865 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.sink.connector; + +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSinkIncrementConfig; +import org.apache.eventmesh.common.utils.JsonUtils; +import org.apache.eventmesh.connector.canal.CanalConnectRecord; +import org.apache.eventmesh.connector.canal.DatabaseConnection; +import org.apache.eventmesh.connector.canal.SqlUtils; +import org.apache.eventmesh.connector.canal.dialect.DbDialect; +import org.apache.eventmesh.connector.canal.dialect.MysqlDialect; +import org.apache.eventmesh.connector.canal.interceptor.SqlBuilderLoadInterceptor; +import org.apache.eventmesh.connector.canal.model.EventColumn; +import org.apache.eventmesh.connector.canal.model.EventType; +import org.apache.eventmesh.connector.canal.sink.DbLoadContext; +import org.apache.eventmesh.connector.canal.sink.DbLoadData; +import org.apache.eventmesh.connector.canal.sink.DbLoadData.TableLoadData; +import org.apache.eventmesh.connector.canal.sink.DbLoadMerger; +import org.apache.eventmesh.connector.canal.sink.GtidBatch; +import org.apache.eventmesh.connector.canal.sink.GtidBatchManager; +import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; +import org.apache.eventmesh.openconnect.api.ConnectorCreateService; +import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; +import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; +import org.apache.eventmesh.openconnect.api.sink.Sink; +import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendExceptionContext; +import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendResult; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; +import org.apache.eventmesh.openconnect.util.ConfigUtil; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang.exception.ExceptionUtils; +import org.apache.commons.lang3.SerializationUtils; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Types; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +import org.springframework.dao.DataAccessException; +import org.springframework.dao.DeadlockLoserDataAccessException; +import org.springframework.jdbc.core.BatchPreparedStatementSetter; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.PreparedStatementSetter; +import org.springframework.jdbc.core.StatementCallback; +import org.springframework.jdbc.core.StatementCreatorUtils; +import org.springframework.jdbc.support.lob.DefaultLobHandler; +import org.springframework.jdbc.support.lob.LobCreator; +import org.springframework.transaction.support.TransactionCallback; +import org.springframework.util.CollectionUtils; + +import com.alibaba.otter.canal.common.utils.NamedThreadFactory; +import com.fasterxml.jackson.core.type.TypeReference; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class CanalSinkIncrementConnector implements Sink, ConnectorCreateService { + + private CanalSinkIncrementConfig sinkConfig; + + private JdbcTemplate jdbcTemplate; + + private SqlBuilderLoadInterceptor interceptor; + + private DbDialect dbDialect; + + private ExecutorService executor; + + private ExecutorService gtidSingleExecutor; + + private int batchSize = 50; + + private boolean useBatch = true; + + private RdbTableMgr tableMgr; + + @Override + public Class configClass() { + return CanalSinkIncrementConfig.class; + } + + @Override + public void init(Config config) throws Exception { + // init config for canal source connector + this.sinkConfig = (CanalSinkIncrementConfig) config; + } + + @Override + public void init(ConnectorContext connectorContext) throws Exception { + // init config for canal source connector + SinkConnectorContext sinkConnectorContext = (SinkConnectorContext) connectorContext; + CanalSinkConfig canalSinkConfig = (CanalSinkConfig) sinkConnectorContext.getSinkConfig(); + this.sinkConfig = ConfigUtil.parse(canalSinkConfig.getSinkConfig(), CanalSinkIncrementConfig.class); + this.batchSize = sinkConfig.getBatchSize(); + this.useBatch = sinkConfig.getUseBatch(); + DatabaseConnection.sinkConfig = this.sinkConfig.getSinkConnectorConfig(); + DatabaseConnection.initSinkConnection(); + jdbcTemplate = new JdbcTemplate(DatabaseConnection.sinkDataSource); + dbDialect = new MysqlDialect(jdbcTemplate, new DefaultLobHandler()); + interceptor = new SqlBuilderLoadInterceptor(); + interceptor.setDbDialect(dbDialect); + tableMgr = new RdbTableMgr(sinkConfig.getSinkConnectorConfig(), DatabaseConnection.sinkDataSource); + executor = new ThreadPoolExecutor(sinkConfig.getPoolSize(), + sinkConfig.getPoolSize(), + 0L, + TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(sinkConfig.getPoolSize() * 4), + new NamedThreadFactory("canalSink"), + new ThreadPoolExecutor.CallerRunsPolicy()); + gtidSingleExecutor = Executors.newSingleThreadExecutor(r -> new Thread(r, "gtidSingleExecutor")); + } + + @Override + public void start() throws Exception { + tableMgr.start(); + } + + @Override + public void commit(ConnectRecord record) { + + } + + @Override + public String name() { + return this.sinkConfig.getSinkConnectorConfig().getConnectorName(); + } + + @Override + public void onException(ConnectRecord record) { + + } + + @Override + public void stop() { + executor.shutdown(); + gtidSingleExecutor.shutdown(); + } + + @Override + public void put(List sinkRecords) { + DbLoadContext context = new DbLoadContext(); + for (ConnectRecord connectRecord : sinkRecords) { + List canalConnectRecordList = new ArrayList<>(); + + List canalConnectRecords = convertToCanalConnectRecord(connectRecord); + + // deep copy connectRecord data + for (CanalConnectRecord record : canalConnectRecords) { + canalConnectRecordList.add(SerializationUtils.clone(record)); + } + canalConnectRecordList = filterRecord(canalConnectRecordList); + if (isDdlDatas(canalConnectRecordList)) { + doDdl(context, canalConnectRecordList, connectRecord); + } else if (sinkConfig.isGTIDMode()) { + doLoadWithGtid(context, sinkConfig, connectRecord); + } else { + canalConnectRecordList = DbLoadMerger.merge(canalConnectRecordList); + + DbLoadData loadData = new DbLoadData(); + doBefore(canalConnectRecordList, loadData); + + doLoad(context, sinkConfig, loadData, connectRecord); + + } + + } + } + + @Override + public Sink create() { + return new CanalSinkIncrementConnector(); + } + + private boolean isDdlDatas(List canalConnectRecordList) { + boolean result = false; + for (CanalConnectRecord canalConnectRecord : canalConnectRecordList) { + result |= canalConnectRecord.getEventType().isDdl(); + if (result && !canalConnectRecord.getEventType().isDdl()) { + throw new RuntimeException("ddl/dml can't be in one batch, it's may be a bug , pls submit issues."); + } + } + return result; + } + + private List filterRecord(List canalConnectRecordList) { + return canalConnectRecordList.stream() + .filter(record -> tableMgr.getTable(record.getSchemaName(), record.getTableName()) != null) + .collect(Collectors.toList()); + } + + private void doDdl(DbLoadContext context, List canalConnectRecordList, ConnectRecord connectRecord) { + for (final CanalConnectRecord record : canalConnectRecordList) { + try { + Boolean result = jdbcTemplate.execute(new StatementCallback() { + + public Boolean doInStatement(Statement stmt) throws SQLException, DataAccessException { + boolean result = true; + if (StringUtils.isNotEmpty(record.getDdlSchemaName())) { + result &= stmt.execute("use `" + record.getDdlSchemaName() + "`"); + } + result &= stmt.execute(record.getSql()); + return result; + } + }); + if (Boolean.TRUE.equals(result)) { + context.getProcessedRecords().add(record); + } else { + context.getFailedRecords().add(record); + } + } catch (Throwable e) { + connectRecord.getCallback().onException(buildSendExceptionContext(connectRecord, e)); + throw new RuntimeException(e); + } + } + connectRecord.getCallback().onSuccess(convertToSendResult(connectRecord)); + } + + private SendExceptionContext buildSendExceptionContext(ConnectRecord record, Throwable e) { + SendExceptionContext sendExceptionContext = new SendExceptionContext(); + sendExceptionContext.setMessageId(record.getRecordId()); + sendExceptionContext.setCause(e); + if (org.apache.commons.lang3.StringUtils.isNotEmpty(record.getExtension("topic"))) { + sendExceptionContext.setTopic(record.getExtension("topic")); + } + return sendExceptionContext; + } + + private SendResult convertToSendResult(ConnectRecord record) { + SendResult result = new SendResult(); + result.setMessageId(record.getRecordId()); + if (org.apache.commons.lang3.StringUtils.isNotEmpty(record.getExtension("topic"))) { + result.setTopic(record.getExtension("topic")); + } + return result; + } + + private void doBefore(List canalConnectRecordList, final DbLoadData loadData) { + for (final CanalConnectRecord record : canalConnectRecordList) { + boolean filter = interceptor.before(sinkConfig, record); + if (!filter) { + loadData.merge(record); + } + } + } + + private void doLoad(DbLoadContext context, CanalSinkIncrementConfig sinkConfig, DbLoadData loadData, ConnectRecord connectRecord) { + List> batchDatas = new ArrayList<>(); + for (TableLoadData tableData : loadData.getTables()) { + if (useBatch) { + batchDatas.addAll(split(tableData.getDeleteDatas())); + } else { + for (CanalConnectRecord data : tableData.getDeleteDatas()) { + batchDatas.add(Arrays.asList(data)); + } + } + } + + doTwoPhase(context, sinkConfig, batchDatas, true, connectRecord); + + batchDatas.clear(); + + for (TableLoadData tableData : loadData.getTables()) { + if (useBatch) { + batchDatas.addAll(split(tableData.getInsertDatas())); + batchDatas.addAll(split(tableData.getUpdateDatas())); + } else { + for (CanalConnectRecord data : tableData.getInsertDatas()) { + batchDatas.add(Arrays.asList(data)); + } + for (CanalConnectRecord data : tableData.getUpdateDatas()) { + batchDatas.add(Arrays.asList(data)); + } + } + } + + doTwoPhase(context, sinkConfig, batchDatas, true, connectRecord); + + batchDatas.clear(); + } + + private void doLoadWithGtid(DbLoadContext context, CanalSinkIncrementConfig sinkConfig, ConnectRecord connectRecord) { + int batchIndex = connectRecord.getExtension("batchIndex", Integer.class); + int totalBatches = connectRecord.getExtension("totalBatches", Integer.class); + List canalConnectRecordList = convertToCanalConnectRecord(connectRecord); + + String gtid = canalConnectRecordList.get(0).getCurrentGtid(); + GtidBatchManager.addBatch(gtid, batchIndex, totalBatches, canalConnectRecordList); + // check whether the batch is complete + if (GtidBatchManager.isComplete(gtid)) { + GtidBatch batch = GtidBatchManager.getGtidBatch(gtid); + List> totalRows = batch.getBatches(); + List filteredRows = new ArrayList<>(); + for (List canalConnectRecords : totalRows) { + canalConnectRecords = filterRecord(canalConnectRecords); + if (!CollectionUtils.isEmpty(canalConnectRecords)) { + for (final CanalConnectRecord record : canalConnectRecords) { + boolean filter = interceptor.before(sinkConfig, record); + filteredRows.add(record); + } + } + } + context.setGtid(gtid); + Future result = gtidSingleExecutor.submit(new DbLoadWorker(context, filteredRows, dbDialect, false, sinkConfig)); + Exception ex = null; + try { + ex = result.get(); + if (ex == null) { + connectRecord.getCallback().onSuccess(convertToSendResult(connectRecord)); + } + } catch (Exception e) { + ex = e; + } + Boolean skipException = sinkConfig.getSkipException(); + if (skipException != null && skipException) { + if (ex != null) { + // do skip + log.warn("skip exception will ack data : {} , caused by {}", + filteredRows, + ExceptionUtils.getFullStackTrace(ex)); + GtidBatchManager.removeGtidBatch(gtid); + connectRecord.getCallback().onSuccess(convertToSendResult(connectRecord)); + } + } else { + if (ex != null) { + log.error("sink connector will shutdown by " + ex.getMessage(), ExceptionUtils.getFullStackTrace(ex)); + connectRecord.getCallback().onException(buildSendExceptionContext(connectRecord, ex)); + gtidSingleExecutor.shutdown(); + System.exit(1); + } else { + GtidBatchManager.removeGtidBatch(gtid); + } + } + } else { + log.info("Batch received, waiting for other batches."); + // ack this record + connectRecord.getCallback().onSuccess(convertToSendResult(connectRecord)); + } + } + + private List convertToCanalConnectRecord(ConnectRecord connectRecord) { + List canalConnectRecordList; + try { + canalConnectRecordList = + JsonUtils.parseTypeReferenceObject((byte[]) connectRecord.getData(), new TypeReference>() { + }); + } catch (Exception e) { + log.error("Failed to parse the canalConnectRecords.", e); + connectRecord.getCallback().onException(buildSendExceptionContext(connectRecord, e)); + throw new RuntimeException("Failed to parse the canalConnectRecords.", e); + } + return canalConnectRecordList; + } + + private List> split(List records) { + List> result = new ArrayList<>(); + if (records == null || records.isEmpty()) { + return result; + } else { + int[] bits = new int[records.size()]; + for (int i = 0; i < bits.length; i++) { + while (i < bits.length && bits[i] == 1) { + i++; + } + + if (i >= bits.length) { + break; + } + + List batch = new ArrayList<>(); + bits[i] = 1; + batch.add(records.get(i)); + for (int j = i + 1; j < bits.length && batch.size() < batchSize; j++) { + if (bits[j] == 0 && canBatch(records.get(i), records.get(j))) { + batch.add(records.get(j)); + bits[j] = 1; + } + } + result.add(batch); + } + + return result; + } + } + + private boolean canBatch(CanalConnectRecord source, CanalConnectRecord target) { + return StringUtils.equals(source.getSchemaName(), + target.getSchemaName()) + && StringUtils.equals(source.getTableName(), target.getTableName()) + && StringUtils.equals(source.getSql(), target.getSql()); + } + + private void doTwoPhase(DbLoadContext context, CanalSinkIncrementConfig sinkConfig, List> totalRows, boolean canBatch, + ConnectRecord connectRecord) { + List> results = new ArrayList<>(); + for (List rows : totalRows) { + if (CollectionUtils.isEmpty(rows)) { + continue; + } + results.add(executor.submit(new DbLoadWorker(context, rows, dbDialect, canBatch, sinkConfig))); + } + + boolean partFailed = false; + for (Future result : results) { + Exception ex = null; + try { + ex = result.get(); + if (ex == null) { + connectRecord.getCallback().onSuccess(convertToSendResult(connectRecord)); + } + } catch (Exception e) { + ex = e; + } + + if (ex != null) { + log.warn("##load phase one failed!", ex); + partFailed = true; + } + } + + if (partFailed) { + List retryRecords = new ArrayList<>(); + for (List rows : totalRows) { + retryRecords.addAll(rows); + } + + context.getFailedRecords().clear(); + + Boolean skipException = sinkConfig.getSkipException(); + if (skipException != null && skipException) { + for (CanalConnectRecord retryRecord : retryRecords) { + DbLoadWorker worker = new DbLoadWorker(context, Arrays.asList(retryRecord), dbDialect, false, sinkConfig); + try { + Exception ex = worker.call(); + if (ex != null) { + // do skip + log.warn("skip exception for data : {} , caused by {}", + retryRecord, + ExceptionUtils.getFullStackTrace(ex)); + connectRecord.getCallback().onSuccess(convertToSendResult(connectRecord)); + } + } catch (Exception ex) { + // do skip + log.warn("skip exception for data : {} , caused by {}", + retryRecord, + ExceptionUtils.getFullStackTrace(ex)); + connectRecord.getCallback().onSuccess(convertToSendResult(connectRecord)); + } + } + } else { + DbLoadWorker worker = new DbLoadWorker(context, retryRecords, dbDialect, false, sinkConfig); + try { + Exception ex = worker.call(); + if (ex != null) { + throw ex; + } + } catch (Exception ex) { + log.error("##load phase two failed!", ex); + log.error("sink connector will shutdown by " + ex.getMessage(), ex); + connectRecord.getCallback().onException(buildSendExceptionContext(connectRecord, ex)); + executor.shutdown(); + System.exit(1); + } + } + } + } + + enum ExecuteResult { + SUCCESS, ERROR, RETRY + } + + class DbLoadWorker implements Callable { + + private final DbLoadContext context; + private final DbDialect dbDialect; + private final List records; + private final boolean canBatch; + + private final CanalSinkIncrementConfig sinkConfig; + + private final List allFailedRecords = new ArrayList<>(); + private final List allProcessedRecords = new ArrayList<>(); + private final List processedRecords = new ArrayList<>(); + private final List failedRecords = new ArrayList<>(); + + public DbLoadWorker(DbLoadContext context, List records, DbDialect dbDialect, boolean canBatch, + CanalSinkIncrementConfig sinkConfig) { + this.context = context; + this.records = records; + this.canBatch = canBatch; + this.dbDialect = dbDialect; + this.sinkConfig = sinkConfig; + } + + public Exception call() throws Exception { + try { + return doCall(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + private Exception doCall() { + RuntimeException error = null; + ExecuteResult exeResult = null; + + if (sinkConfig.isGTIDMode()) { + int retryCount = 0; + final List toExecuteRecords = new ArrayList<>(); + try { + if (!CollectionUtils.isEmpty(failedRecords)) { + // if failedRecords not empty, make it retry + toExecuteRecords.addAll(failedRecords); + } else { + toExecuteRecords.addAll(records); + // add to failed record first, maybe get lob or datasource error + failedRecords.addAll(toExecuteRecords); + } + JdbcTemplate template = dbDialect.getJdbcTemplate(); + String sourceGtid = context.getGtid(); + if (StringUtils.isNotEmpty(sourceGtid) && !sinkConfig.isMariaDB()) { + String setMySQLGtid = "SET @@session.gtid_next = '" + sourceGtid + "';"; + template.execute(setMySQLGtid); + } else if (StringUtils.isNotEmpty(sourceGtid) && sinkConfig.isMariaDB()) { + throw new RuntimeException("unsupport gtid mode for mariaDB"); + } else { + log.error("gtid is empty in gtid mode"); + throw new RuntimeException("gtid is empty in gtid mode"); + } + + final LobCreator lobCreator = dbDialect.getLobHandler().getLobCreator(); + int affect = (Integer) dbDialect.getTransactionTemplate().execute((TransactionCallback) status -> { + try { + failedRecords.clear(); + processedRecords.clear(); + int affect1 = 0; + for (CanalConnectRecord record : toExecuteRecords) { + int affects = template.update(record.getSql(), new PreparedStatementSetter() { + public void setValues(PreparedStatement ps) throws SQLException { + doPreparedStatement(ps, dbDialect, lobCreator, record); + } + }); + affect1 = affect1 + affects; + processStat(record, affects, false); + } + return affect1; + } catch (Exception e) { + // rollback + status.setRollbackOnly(); + throw new RuntimeException("Failed to executed", e); + } finally { + lobCreator.close(); + } + }); + + // reset gtid + if (sinkConfig.isMariaDB()) { + throw new RuntimeException("unsupport gtid mode for mariaDB"); + } else { + String resetMySQLGtid = "SET @@session.gtid_next = 'AUTOMATIC';"; + dbDialect.getJdbcTemplate().execute(resetMySQLGtid); + } + + error = null; + exeResult = ExecuteResult.SUCCESS; + } catch (DeadlockLoserDataAccessException ex) { + error = new RuntimeException(ExceptionUtils.getFullStackTrace(ex)); + exeResult = ExecuteResult.RETRY; + } catch (Throwable ex) { + error = new RuntimeException(ExceptionUtils.getFullStackTrace(ex)); + exeResult = ExecuteResult.ERROR; + } + + if (ExecuteResult.SUCCESS == exeResult) { + allFailedRecords.addAll(failedRecords); + allProcessedRecords.addAll(processedRecords); + failedRecords.clear(); + processedRecords.clear(); + } else if (ExecuteResult.RETRY == exeResult) { + retryCount = retryCount + 1; + processedRecords.clear(); + failedRecords.clear(); + failedRecords.addAll(toExecuteRecords); + int retry = 3; + if (retryCount >= retry) { + processFailedDatas(toExecuteRecords.size()); + throw new RuntimeException(String.format("execute retry %s times failed", retryCount), error); + } else { + try { + int retryWait = 3000; + int wait = retryCount * retryWait; + wait = Math.max(wait, retryWait); + Thread.sleep(wait); + } catch (InterruptedException ex) { + Thread.interrupted(); + processFailedDatas(toExecuteRecords.size()); + throw new RuntimeException(ex); + } + } + } else { + processedRecords.clear(); + failedRecords.clear(); + failedRecords.addAll(toExecuteRecords); + processFailedDatas(toExecuteRecords.size()); + throw error; + } + } else { + int index = 0; + while (index < records.size()) { + final List toExecuteRecords = new ArrayList<>(); + if (useBatch && canBatch) { + int end = Math.min(index + batchSize, records.size()); + toExecuteRecords.addAll(records.subList(index, end)); + index = end; + } else { + toExecuteRecords.add(records.get(index)); + index = index + 1; + } + + int retryCount = 0; + while (true) { + try { + if (!CollectionUtils.isEmpty(failedRecords)) { + toExecuteRecords.clear(); + toExecuteRecords.addAll(failedRecords); + } else { + failedRecords.addAll(toExecuteRecords); + } + + final LobCreator lobCreator = dbDialect.getLobHandler().getLobCreator(); + if (useBatch && canBatch) { + JdbcTemplate template = dbDialect.getJdbcTemplate(); + final String sql = toExecuteRecords.get(0).getSql(); + + int[] affects = new int[toExecuteRecords.size()]; + + affects = (int[]) dbDialect.getTransactionTemplate().execute((TransactionCallback) status -> { + try { + failedRecords.clear(); + processedRecords.clear(); + int[] affects1 = template.batchUpdate(sql, new BatchPreparedStatementSetter() { + + public void setValues(PreparedStatement ps, int idx) throws SQLException { + doPreparedStatement(ps, dbDialect, lobCreator, toExecuteRecords.get(idx)); + } + + public int getBatchSize() { + return toExecuteRecords.size(); + } + }); + return affects1; + } catch (Exception e) { + // rollback + status.setRollbackOnly(); + throw new RuntimeException("Failed to execute batch ", e); + } finally { + lobCreator.close(); + } + }); + + for (int i = 0; i < toExecuteRecords.size(); i++) { + assert affects != null; + processStat(toExecuteRecords.get(i), affects[i], true); + } + } else { + final CanalConnectRecord record = toExecuteRecords.get(0); + JdbcTemplate template = dbDialect.getJdbcTemplate(); + int affect = 0; + affect = (Integer) dbDialect.getTransactionTemplate().execute((TransactionCallback) status -> { + try { + failedRecords.clear(); + processedRecords.clear(); + int affect1 = template.update(record.getSql(), new PreparedStatementSetter() { + + public void setValues(PreparedStatement ps) throws SQLException { + doPreparedStatement(ps, dbDialect, lobCreator, record); + } + }); + return affect1; + } catch (Exception e) { + // rollback + status.setRollbackOnly(); + throw new RuntimeException("Failed to executed", e); + } finally { + lobCreator.close(); + } + }); + processStat(record, affect, false); + } + + error = null; + exeResult = ExecuteResult.SUCCESS; + } catch (DeadlockLoserDataAccessException ex) { + error = new RuntimeException(ExceptionUtils.getFullStackTrace(ex)); + exeResult = ExecuteResult.RETRY; + } catch (Throwable ex) { + error = new RuntimeException(ExceptionUtils.getFullStackTrace(ex)); + exeResult = ExecuteResult.ERROR; + } + + if (ExecuteResult.SUCCESS == exeResult) { + allFailedRecords.addAll(failedRecords); + allProcessedRecords.addAll(processedRecords); + failedRecords.clear(); + processedRecords.clear(); + break; // do next eventData + } else if (ExecuteResult.RETRY == exeResult) { + retryCount = retryCount + 1; + processedRecords.clear(); + failedRecords.clear(); + failedRecords.addAll(toExecuteRecords); + int retry = 3; + if (retryCount >= retry) { + processFailedDatas(index); + throw new RuntimeException(String.format("execute retry %s times failed", retryCount), error); + } else { + try { + int retryWait = 3000; + int wait = retryCount * retryWait; + wait = Math.max(wait, retryWait); + Thread.sleep(wait); + } catch (InterruptedException ex) { + Thread.interrupted(); + processFailedDatas(index); + throw new RuntimeException(ex); + } + } + } else { + processedRecords.clear(); + failedRecords.clear(); + failedRecords.addAll(toExecuteRecords); + processFailedDatas(index); + throw error; + } + } + } + } + + context.getFailedRecords().addAll(allFailedRecords); + context.getProcessedRecords().addAll(allProcessedRecords); + return null; + } + + private void doPreparedStatement(PreparedStatement ps, DbDialect dbDialect, LobCreator lobCreator, + CanalConnectRecord record) throws SQLException { + EventType type = record.getEventType(); + List columns = new ArrayList(); + if (type.isInsert()) { + columns.addAll(record.getColumns()); + columns.addAll(record.getKeys()); + } else if (type.isDelete()) { + columns.addAll(record.getKeys()); + } else if (type.isUpdate()) { + boolean existOldKeys = !CollectionUtils.isEmpty(record.getOldKeys()); + columns.addAll(record.getUpdatedColumns()); + columns.addAll(record.getKeys()); + if (existOldKeys) { + columns.addAll(record.getOldKeys()); + } + } + + for (int i = 0; i < columns.size(); i++) { + int paramIndex = i + 1; + EventColumn column = columns.get(i); + int sqlType = column.getColumnType(); + + Object param = null; + if (dbDialect instanceof MysqlDialect + && (sqlType == Types.TIME || sqlType == Types.TIMESTAMP || sqlType == Types.DATE)) { + param = column.getColumnValue(); + } else { + param = SqlUtils.stringToSqlValue(column.getColumnValue(), + sqlType, + false, + dbDialect.isEmptyStringNulled()); + } + + try { + switch (sqlType) { + case Types.CLOB: + lobCreator.setClobAsString(ps, paramIndex, (String) param); + break; + + case Types.BLOB: + lobCreator.setBlobAsBytes(ps, paramIndex, (byte[]) param); + break; + case Types.TIME: + case Types.TIMESTAMP: + case Types.DATE: + if (dbDialect instanceof MysqlDialect) { + ps.setObject(paramIndex, param); + } else { + StatementCreatorUtils.setParameterValue(ps, paramIndex, sqlType, null, param); + } + break; + case Types.BIT: + if (dbDialect instanceof MysqlDialect) { + StatementCreatorUtils.setParameterValue(ps, paramIndex, Types.DECIMAL, null, param); + } else { + StatementCreatorUtils.setParameterValue(ps, paramIndex, sqlType, null, param); + } + break; + default: + StatementCreatorUtils.setParameterValue(ps, paramIndex, sqlType, null, param); + break; + } + } catch (SQLException ex) { + log.error("## SetParam error , [pairId={}, sqltype={}, value={}]", + record.getPairId(), sqlType, param); + throw ex; + } + } + } + + private void processStat(CanalConnectRecord record, int affect, boolean batch) { + if (batch && (affect < 1 && affect != Statement.SUCCESS_NO_INFO)) { + failedRecords.add(record); + } else if (!batch && affect < 1) { + failedRecords.add(record); + } else { + processedRecords.add(record); + // this.processStat(record, context); + } + } + + private void processFailedDatas(int index) { + allFailedRecords.addAll(failedRecords); + context.getFailedRecords().addAll(allFailedRecords); + for (; index < records.size(); index++) { + context.getFailedRecords().add(records.get(index)); + } + allProcessedRecords.addAll(processedRecords); + context.getProcessedRecords().addAll(allProcessedRecords); + } + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/EntryParser.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/EntryParser.java new file mode 100644 index 0000000000..d7388c628b --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/EntryParser.java @@ -0,0 +1,349 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.source; + +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceIncrementConfig; +import org.apache.eventmesh.connector.canal.CanalConnectRecord; +import org.apache.eventmesh.connector.canal.model.EventColumn; +import org.apache.eventmesh.connector.canal.model.EventColumnIndexComparable; +import org.apache.eventmesh.connector.canal.model.EventType; +import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; + +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.springframework.util.CollectionUtils; + +import com.alibaba.otter.canal.protocol.CanalEntry; +import com.alibaba.otter.canal.protocol.CanalEntry.Column; +import com.alibaba.otter.canal.protocol.CanalEntry.Entry; +import com.alibaba.otter.canal.protocol.CanalEntry.RowChange; +import com.alibaba.otter.canal.protocol.CanalEntry.RowData; + +import lombok.extern.slf4j.Slf4j; + +/** + * data object parse + */ +@Slf4j +public class EntryParser { + + public static Map> parse(CanalSourceIncrementConfig sourceConfig, List datas, + RdbTableMgr tables) { + List recordList = new ArrayList<>(); + List transactionDataBuffer = new ArrayList<>(); + // need check weather the entry is loopback + boolean needSync; + Map> recordMap = new HashMap<>(); + try { + for (Entry entry : datas) { + switch (entry.getEntryType()) { + case ROWDATA: + RowChange rowChange = RowChange.parseFrom(entry.getStoreValue()); + // don't support gtid for mariadb + if (sourceConfig.getServerUUID() != null && sourceConfig.isGTIDMode() && !sourceConfig.isMariaDB()) { + if (checkGtidForEntry(entry, sourceConfig)) { + transactionDataBuffer.add(entry); + } + } else { + // if not gtid mode, need check weather the entry is loopback by specified column value + needSync = checkNeedSync(sourceConfig, rowChange); + if (needSync) { + log.debug("entry evenType {}|rowChange {}", rowChange.getEventType(), rowChange); + transactionDataBuffer.add(entry); + } + } + break; + case TRANSACTIONEND: + parseRecordListWithEntryBuffer(sourceConfig, recordList, transactionDataBuffer, tables); + if (!recordList.isEmpty()) { + List transactionEndList = new ArrayList<>(recordList); + recordMap.put(entry.getHeader().getLogfileOffset(), transactionEndList); + } + recordList.clear(); + transactionDataBuffer.clear(); + break; + default: + break; + } + } + + // add last data in transactionDataBuffer, in case no TRANSACTIONEND + parseRecordListWithEntryBuffer(sourceConfig, recordList, transactionDataBuffer, tables); + if (!recordList.isEmpty()) { + List transactionEndList = new ArrayList<>(recordList); + CanalConnectRecord lastCanalConnectRecord = transactionEndList.get(transactionEndList.size() - 1); + recordMap.put(lastCanalConnectRecord.getBinLogOffset(), transactionEndList); + } + recordList.clear(); + transactionDataBuffer.clear(); + + } catch (Exception e) { + throw new RuntimeException(e); + } + return recordMap; + } + + private static boolean checkGtidForEntry(Entry entry, CanalSourceIncrementConfig sourceConfig) { + String currentGtid = entry.getHeader().getPropsList().get(0).getValue(); + return currentGtid.contains(sourceConfig.getServerUUID()); + } + + private static void parseRecordListWithEntryBuffer(CanalSourceIncrementConfig sourceConfig, + List recordList, + List transactionDataBuffer, RdbTableMgr tables) { + for (Entry bufferEntry : transactionDataBuffer) { + List recordParsedList = internParse(sourceConfig, bufferEntry, tables); + if (CollectionUtils.isEmpty(recordParsedList)) { + continue; + } + long totalSize = bufferEntry.getHeader().getEventLength(); + long eachSize = totalSize / recordParsedList.size(); + for (CanalConnectRecord record : recordParsedList) { + if (record == null) { + continue; + } + record.setSize(eachSize); + recordList.add(record); + } + } + } + + private static boolean checkNeedSync(CanalSourceIncrementConfig sourceConfig, RowChange rowChange) { + Column markedColumn = null; + CanalEntry.EventType eventType = rowChange.getEventType(); + if (StringUtils.isEmpty(sourceConfig.getNeedSyncMarkTableColumnName())) { + return true; + } + if (eventType.equals(CanalEntry.EventType.DELETE)) { + markedColumn = getColumnIgnoreCase(rowChange.getRowDatas(0).getBeforeColumnsList(), + sourceConfig.getNeedSyncMarkTableColumnName()); + } else if (eventType.equals(CanalEntry.EventType.INSERT) || eventType.equals(CanalEntry.EventType.UPDATE)) { + markedColumn = getColumnIgnoreCase(rowChange.getRowDatas(0).getAfterColumnsList(), + sourceConfig.getNeedSyncMarkTableColumnName()); + } + if (markedColumn != null) { + return StringUtils.equalsIgnoreCase(markedColumn.getValue(), + sourceConfig.getNeedSyncMarkTableColumnValue()); + } + return false; + } + + private static Column getColumnIgnoreCase(List columns, String columName) { + for (Column column : columns) { + if (column.getName().equalsIgnoreCase(columName)) { + return column; + } + } + return null; + } + + private static List internParse(CanalSourceIncrementConfig sourceConfig, Entry entry, + RdbTableMgr tableMgr) { + String schemaName = entry.getHeader().getSchemaName(); + String tableName = entry.getHeader().getTableName(); + if (tableMgr.getTable(schemaName, tableName) == null) { + return null; + } + + RowChange rowChange = null; + try { + rowChange = RowChange.parseFrom(entry.getStoreValue()); + } catch (Exception e) { + throw new RuntimeException("parser of canal-event has an error , data:" + entry.toString(), e); + } + + if (rowChange == null) { + return null; + } + + EventType eventType = EventType.valueOf(rowChange.getEventType().name()); + + if (eventType.isQuery()) { + return null; + } + + if (eventType.isDdl()) { + log.warn("unsupported ddl event type: {}", eventType); + return null; + } + + List recordList = new ArrayList<>(); + for (RowData rowData : rowChange.getRowDatasList()) { + CanalConnectRecord record = internParse(sourceConfig, entry, rowChange, rowData); + recordList.add(record); + } + + return recordList; + } + + private static CanalConnectRecord internParse(CanalSourceIncrementConfig canalSourceConfig, Entry entry, + RowChange rowChange, RowData rowData) { + CanalConnectRecord canalConnectRecord = new CanalConnectRecord(); + canalConnectRecord.setTableName(entry.getHeader().getTableName()); + canalConnectRecord.setSchemaName(entry.getHeader().getSchemaName()); + canalConnectRecord.setEventType(EventType.valueOf(rowChange.getEventType().name())); + canalConnectRecord.setExecuteTime(entry.getHeader().getExecuteTime()); + canalConnectRecord.setJournalName(entry.getHeader().getLogfileName()); + canalConnectRecord.setBinLogOffset(entry.getHeader().getLogfileOffset()); + // if enabled gtid mode, gtid not null + if (canalSourceConfig.isGTIDMode()) { + if (canalSourceConfig.isMariaDB()) { + String currentGtid = entry.getHeader().getGtid(); + canalConnectRecord.setGtid(currentGtid); + canalConnectRecord.setCurrentGtid(currentGtid); + } else { + String currentGtid = entry.getHeader().getPropsList().get(0).getValue(); + String gtidRange = replaceGtidRange(entry.getHeader().getGtid(), currentGtid, canalSourceConfig.getServerUUID()); + canalConnectRecord.setGtid(gtidRange); + canalConnectRecord.setCurrentGtid(currentGtid); + } + } + + EventType eventType = canalConnectRecord.getEventType(); + + List beforeColumns = rowData.getBeforeColumnsList(); + List afterColumns = rowData.getAfterColumnsList(); + + boolean isRowMode = canalSourceConfig.getSyncMode().isRow(); + + Map keyColumns = new LinkedHashMap<>(); + Map oldKeyColumns = new LinkedHashMap<>(); + Map notKeyColumns = new LinkedHashMap<>(); + + if (eventType.isInsert()) { + for (Column column : afterColumns) { + if (column.getIsKey()) { + keyColumns.put(column.getName(), copyEventColumn(column, true)); + } else { + notKeyColumns.put(column.getName(), copyEventColumn(column, true)); + } + } + } else if (eventType.isDelete()) { + for (Column column : beforeColumns) { + if (column.getIsKey()) { + keyColumns.put(column.getName(), copyEventColumn(column, true)); + } else { + notKeyColumns.put(column.getName(), copyEventColumn(column, true)); + } + } + } else if (eventType.isUpdate()) { + for (Column column : beforeColumns) { + if (column.getIsKey()) { + oldKeyColumns.put(column.getName(), copyEventColumn(column, true)); + keyColumns.put(column.getName(), copyEventColumn(column, true)); + } else { + if (isRowMode && entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE) { + notKeyColumns.put(column.getName(), copyEventColumn(column, true)); + } + } + } + for (Column column : afterColumns) { + if (column.getIsKey()) { + keyColumns.put(column.getName(), copyEventColumn(column, true)); + } else if (isRowMode || entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE + || column.getUpdated()) { + + boolean isUpdate = true; + if (entry.getHeader().getSourceType() == CanalEntry.Type.MYSQL) { + isUpdate = column.getUpdated(); + } + + notKeyColumns.put(column.getName(), copyEventColumn(column, isUpdate)); + } + } + + if (entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE) { + checkUpdateKeyColumns(oldKeyColumns, keyColumns); + } + } + + List keys = new ArrayList<>(keyColumns.values()); + List oldKeys = new ArrayList<>(oldKeyColumns.values()); + List columns = new ArrayList<>(notKeyColumns.values()); + + keys.sort(new EventColumnIndexComparable()); + oldKeys.sort(new EventColumnIndexComparable()); + columns.sort(new EventColumnIndexComparable()); + if (!keyColumns.isEmpty()) { + canalConnectRecord.setKeys(keys); + if (canalConnectRecord.getEventType().isUpdate() && !oldKeys.equals(keys)) { + canalConnectRecord.setOldKeys(oldKeys); + } + canalConnectRecord.setColumns(columns); + } else { + throw new RuntimeException("this row data has no pks , entry: " + entry + " and rowData: " + + rowData); + } + + return canalConnectRecord; + } + + public static String replaceGtidRange(String gtid, String currentGtid, String serverUUID) { + String[] gtidRangeArray = gtid.split(","); + for (int i = 0; i < gtidRangeArray.length; i++) { + String gtidRange = gtidRangeArray[i]; + if (gtidRange.startsWith(serverUUID)) { + gtidRangeArray[i] = gtidRange.replaceFirst("\\d+$", currentGtid.split(":")[1]); + } + } + return String.join(",", gtidRangeArray); + } + + private static void checkUpdateKeyColumns(Map oldKeyColumns, + Map keyColumns) { + if (oldKeyColumns.isEmpty()) { + return; + } + if (keyColumns.size() > oldKeyColumns.size()) { + return; + } + + if (keyColumns.isEmpty()) { + keyColumns.putAll(oldKeyColumns); + return; + } + + if (oldKeyColumns.size() != keyColumns.size()) { + for (String oldKey : oldKeyColumns.keySet()) { + if (keyColumns.get(oldKey) == null) { + keyColumns.put(oldKey, oldKeyColumns.get(oldKey)); + } + } + } + } + + private static EventColumn copyEventColumn(Column column, boolean isUpdate) { + EventColumn eventColumn = new EventColumn(); + eventColumn.setIndex(column.getIndex()); + eventColumn.setKey(column.getIsKey()); + eventColumn.setNull(column.getIsNull()); + eventColumn.setColumnName(column.getName()); + eventColumn.setColumnValue(column.getValue()); + eventColumn.setUpdate(isUpdate); + eventColumn.setColumnType(column.getSqlType()); + + return eventColumn; + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalFullProducer.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalFullProducer.java new file mode 100644 index 0000000000..644b77247d --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalFullProducer.java @@ -0,0 +1,443 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.source.connector; + +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalMySQLType; +import org.apache.eventmesh.common.config.connector.rdb.canal.JobRdbFullPosition; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbColumnDefinition; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.Constants; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLColumnDef; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLTableDef; +import org.apache.eventmesh.common.exception.EventMeshException; +import org.apache.eventmesh.common.remote.offset.canal.CanalFullRecordOffset; +import org.apache.eventmesh.common.remote.offset.canal.CanalFullRecordPartition; +import org.apache.eventmesh.common.utils.JsonUtils; +import org.apache.eventmesh.connector.canal.SqlUtils; +import org.apache.eventmesh.connector.canal.source.position.TableFullPosition; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; + +import java.math.BigDecimal; +import java.math.RoundingMode; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.LockSupport; + +import javax.sql.DataSource; + +import com.google.common.util.concurrent.RateLimiter; + +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + + +@Slf4j +public class CanalFullProducer { + private BlockingQueue> queue; + private final DataSource dataSource; + private final MySQLTableDef tableDefinition; + private final TableFullPosition tableFullPosition; + private final JobRdbFullPosition startPosition; + private static final int LIMIT = 2048; + private final int flushSize; + private final AtomicReference choosePrimaryKey = new AtomicReference<>(null); + private static final DateTimeFormatter DATE_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + private static final DateTimeFormatter DATE_STAMP_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"); + private AtomicLong scanCount = new AtomicLong(0); + private final RateLimiter pageLimiter; + @Setter + private RateLimiter recordLimiter; + + + public CanalFullProducer(BlockingQueue> queue, DataSource dataSource, + MySQLTableDef tableDefinition, JobRdbFullPosition startPosition, int flushSize, int pagePerSecond) { + this.queue = queue; + this.dataSource = dataSource; + this.tableDefinition = tableDefinition; + this.startPosition = startPosition; + this.tableFullPosition = JsonUtils.parseObject(startPosition.getPrimaryKeyRecords(), TableFullPosition.class); + this.scanCount.set(startPosition.getHandledRecordCount()); + this.flushSize = flushSize; + this.pageLimiter = RateLimiter.create(pagePerSecond); + } + + public void choosePrimaryKey() { + for (RdbColumnDefinition col : tableDefinition.getColumnDefinitions().values()) { + if (tableFullPosition.getCurPrimaryKeyCols().get(col.getName()) != null) { + // random choose the first primary key from the table + choosePrimaryKey.set(col.getName()); + log.info("schema [{}] table [{}] choose primary key [{}]", tableDefinition.getSchemaName(), tableDefinition.getTableName(), + col.getName()); + return; + } + } + throw new EventMeshException("illegal: can't pick any primary key"); + } + + + public void start(AtomicBoolean flag) { + choosePrimaryKey(); + // used to page query + boolean isFirstSelect = true; + List> rows = new LinkedList<>(); + while (flag.get()) { + // acquire a permit before each database read + pageLimiter.acquire(); + + String scanSql = generateScanSql(isFirstSelect); + log.info("scan sql is [{}] , cur position [{}]", scanSql, JsonUtils.toJSONString(tableFullPosition.getCurPrimaryKeyCols())); + + try (Connection connection = dataSource.getConnection(); PreparedStatement statement = + connection.prepareStatement(scanSql, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY)) { + statement.setFetchSize(Integer.MIN_VALUE); + setPrepareStatementValue(statement); + try (ResultSet resultSet = statement.executeQuery()) { + Map lastCol = null; + while (flag.get() && resultSet.next()) { + Map columnValues = new LinkedHashMap<>(); + for (Map.Entry col : + tableDefinition.getColumnDefinitions().entrySet()) { + columnValues.put(col.getKey(), readColumn(resultSet, col.getKey(), + col.getValue().getType())); + } + lastCol = columnValues; + rows.add(lastCol); + this.scanCount.incrementAndGet(); + if (rows.size() < flushSize) { + continue; + } + refreshPosition(lastCol); + // may be not reach + commitConnectRecord(rows, false, this.scanCount.get(), startPosition); + rows = new LinkedList<>(); + } + + if (lastCol == null || checkIsScanFinish(lastCol)) { + log.info("full scan db [{}] table [{}] finish", tableDefinition.getSchemaName(), + tableDefinition.getTableName()); + // commit the last record if rows.size() < flushSize + commitConnectRecord(rows, true, this.scanCount.get(), startPosition); + return; + } + refreshPosition(lastCol); + } catch (InterruptedException ignore) { + log.info("full scan db [{}] table [{}] interrupted", tableDefinition.getSchemaName(), + tableDefinition.getTableName()); + Thread.currentThread().interrupt(); + return; + } + } catch (SQLException e) { + log.error("full source process schema [{}] table [{}] catch SQLException fail", tableDefinition.getSchemaName(), + tableDefinition.getTableName(), e); + LockSupport.parkNanos(3000 * 1000L); + } catch (Exception e) { + log.error("full source process schema [{}] table [{}] catch unknown exception", tableDefinition.getSchemaName(), + tableDefinition.getTableName(), e); + return; + } + if (isFirstSelect) { + isFirstSelect = false; + } + } + } + + private void commitConnectRecord(List> rows, boolean isFinished, long migratedCount, JobRdbFullPosition position) + throws InterruptedException { + if (rows == null || rows.isEmpty()) { + return; + } + JobRdbFullPosition jobRdbFullPosition = new JobRdbFullPosition(); + jobRdbFullPosition.setPrimaryKeyRecords(JsonUtils.toJSONString(tableFullPosition)); + jobRdbFullPosition.setTableName(tableDefinition.getTableName()); + jobRdbFullPosition.setSchema(tableDefinition.getSchemaName()); + jobRdbFullPosition.setFinished(isFinished); + jobRdbFullPosition.setHandledRecordCount(migratedCount); + jobRdbFullPosition.setMaxCount(position.getMaxCount()); + if (isFinished) { + jobRdbFullPosition.setPercent(new BigDecimal("100")); + } else { + double num = 100.0d * ((double) migratedCount) * 1.0d / (double) position.getMaxCount(); + String number = Double.toString(num); + BigDecimal percent = new BigDecimal(number).setScale(2, RoundingMode.HALF_UP); + jobRdbFullPosition.setPercent(percent); + } + CanalFullRecordOffset offset = new CanalFullRecordOffset(); + offset.setPosition(jobRdbFullPosition); + CanalFullRecordPartition partition = new CanalFullRecordPartition(); + Map dataMap = new HashMap<>(); + dataMap.put("data", JsonUtils.toJSONString(rows)); + dataMap.put("partition", JsonUtils.toJSONString(partition)); + dataMap.put("offset", JsonUtils.toJSONString(offset)); + ArrayList records = new ArrayList<>(); + records.add( + new ConnectRecord(partition, offset, System.currentTimeMillis(), JsonUtils.toJSONString(dataMap).getBytes(StandardCharsets.UTF_8))); + // global limiter, 100 records per second default + recordLimiter.acquire(); + queue.put(records); + } + + private boolean checkIsScanFinish(Map lastCol) { + Object lastPrimaryValue = lastCol.get(choosePrimaryKey.get()); + Object maxPrimaryValue = tableFullPosition.getMaxPrimaryKeyCols().get(choosePrimaryKey.get()); + if (lastPrimaryValue instanceof Number) { + BigDecimal last = new BigDecimal(String.valueOf(lastPrimaryValue)); + BigDecimal max = + new BigDecimal(String.valueOf(maxPrimaryValue)); + return last.compareTo(max) > 0; + } + if (lastPrimaryValue instanceof Comparable) { + return ((Comparable) lastPrimaryValue).compareTo(maxPrimaryValue) > 0; + } + return false; + } + + public Object readColumn(ResultSet rs, String colName, CanalMySQLType colType) throws Exception { + switch (colType) { + case TINYINT: + case SMALLINT: + case MEDIUMINT: + case INT: + Long valueLong = rs.getLong(colName); + if (rs.wasNull()) { + return null; + } + if (valueLong.compareTo((long) Integer.MAX_VALUE) > 0) { + return valueLong; + } + return valueLong.intValue(); + case BIGINT: + String v = rs.getString(colName); + if (v == null) { + return null; + } + BigDecimal valueBigInt = new BigDecimal(v); + if (valueBigInt.compareTo(BigDecimal.valueOf(Long.MAX_VALUE)) > 0) { + return valueBigInt; + } + return valueBigInt.longValue(); + case FLOAT: + case DOUBLE: + case DECIMAL: + return rs.getBigDecimal(colName); + case DATE: + return rs.getObject(colName, LocalDate.class); + case TIME: + return rs.getObject(colName, LocalTime.class); + case DATETIME: + case TIMESTAMP: + return rs.getObject(colName, LocalDateTime.class); + case YEAR: + int year = rs.getInt(colName); + if (rs.wasNull()) { + return null; + } + return year; + case CHAR: + case VARCHAR: + case TINYTEXT: + case TEXT: + case MEDIUMTEXT: + case LONGTEXT: + case ENUM: + case SET: + case JSON: + return rs.getString(colName); + case BIT: + case BINARY: + case VARBINARY: + case TINYBLOB: + case BLOB: + case MEDIUMBLOB: + case LONGBLOB: + return rs.getBytes(colName); + case GEOMETRY: + case GEOMETRY_COLLECTION: + case GEOM_COLLECTION: + case POINT: + case LINESTRING: + case POLYGON: + case MULTIPOINT: + case MULTILINESTRING: + case MULTIPOLYGON: + byte[] geo = rs.getBytes(colName); + if (geo == null) { + return null; + } + return SqlUtils.toGeometry(geo); + default: + return rs.getObject(colName); + } + } + + + private void refreshPosition(Map lastCol) { + Map nextPosition = new LinkedHashMap<>(); + for (Map.Entry entry : tableFullPosition.getCurPrimaryKeyCols().entrySet()) { + nextPosition.put(entry.getKey(), lastCol.get(entry.getKey())); + } + tableFullPosition.setCurPrimaryKeyCols(nextPosition); + } + + private void setPrepareStatementValue(PreparedStatement statement) throws SQLException { + String colName = choosePrimaryKey.get(); + if (colName == null) { + return; + } + RdbColumnDefinition columnDefinition = tableDefinition.getColumnDefinitions().get(colName); + Object value = tableFullPosition.getCurPrimaryKeyCols().get(colName); + String str; + switch (columnDefinition.getJdbcType()) { + case BIT: + case TINYINT: + case SMALLINT: + case INTEGER: + case BIGINT: + statement.setBigDecimal(1, new BigDecimal(String.valueOf(value))); + break; + case DECIMAL: + case FLOAT: + case DOUBLE: + case NUMERIC: + statement.setDouble(1, new BigDecimal(String.valueOf(value)).doubleValue()); + break; + case CHAR: + case VARCHAR: + case LONGNVARCHAR: + case NCHAR: + case NVARCHAR: + case LONGVARCHAR: + case CLOB: + case NCLOB: + statement.setString(1, String.valueOf(value)); + break; + case BLOB: + case VARBINARY: + case BINARY: + str = String.valueOf(value); + String hexStr = str; + if (str.startsWith("0x")) { + hexStr = str.substring(str.indexOf("0x")); + } + byte[] bytes = SqlUtils.hex2bytes(hexStr); + statement.setBytes(1, bytes); + break; + case DATE: + Instant d; + if (value instanceof Long) { + Long val = (Long) value; + d = Instant.ofEpochMilli(val); + str = d.atZone(ZoneId.systemDefault()).toLocalDateTime().format(DATE_FORMATTER); + } else if (value instanceof Integer) { + Integer val = (Integer) value; + d = Instant.ofEpochMilli((long) val); + str = d.atZone(ZoneId.systemDefault()).toLocalDateTime().format(DATE_FORMATTER); + } else if (value instanceof String) { + str = (String) value; + } else { + if (!(value instanceof LocalDate)) { + throw new IllegalArgumentException("unsupported date class type:" + value.getClass().getSimpleName()); + } + str = ((LocalDate) value).format(DATE_FORMATTER); + } + statement.setString(1, str); + break; + case TIMESTAMP: + if (value instanceof String) { + str = (String) value; + } else { + if (!(value instanceof LocalDateTime)) { + throw new IllegalArgumentException("unsupported timestamp class type:" + value.getClass().getSimpleName()); + } + str = ((LocalDateTime) value).format(DATE_STAMP_FORMATTER); + } + statement.setString(1, str); + break; + default: + throw new EventMeshException(String.format("not support the primary key type [%s]", value.getClass())); + } + } + + + private void generateQueryColumnsSql(StringBuilder builder, Collection rdbColDefs) { + if (rdbColDefs == null || rdbColDefs.isEmpty()) { + builder.append("*"); + return; + } + boolean first = true; + for (RdbColumnDefinition colDef : rdbColDefs) { + if (first) { + first = false; + } else { + builder.append(","); + } + builder.append(Constants.MySQLQuot); + builder.append(colDef.getName()); + builder.append(Constants.MySQLQuot); + } + } + + private String generateScanSql(boolean isFirst) { + StringBuilder builder = new StringBuilder(); + builder.append("select "); + generateQueryColumnsSql(builder, tableDefinition.getColumnDefinitions().values()); + builder.append(" from "); + builder.append(Constants.MySQLQuot); + builder.append(tableDefinition.getSchemaName()); + builder.append(Constants.MySQLQuot); + builder.append("."); + builder.append(Constants.MySQLQuot); + builder.append(tableDefinition.getTableName()); + builder.append(Constants.MySQLQuot); + buildWhereSql(builder, isFirst); + builder.append(" limit " + LIMIT); + return builder.toString(); + } + + private void buildWhereSql(StringBuilder builder, boolean isEquals) { + builder.append(" where ") + .append(Constants.MySQLQuot) + .append(choosePrimaryKey.get()) + .append(Constants.MySQLQuot); + if (isEquals) { + builder.append(" >= ? "); + } else { + builder.append(" > ? "); + } + builder.append(" order by ").append(Constants.MySQLQuot).append(choosePrimaryKey.get()).append(Constants.MySQLQuot) + .append(" asc "); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceCheckConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceCheckConnector.java new file mode 100644 index 0000000000..4d3e569dcd --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceCheckConnector.java @@ -0,0 +1,220 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.source.connector; + +import org.apache.eventmesh.common.AbstractComponent; +import org.apache.eventmesh.common.EventMeshThreadFactory; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceCheckConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.JobRdbFullPosition; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLTableDef; +import org.apache.eventmesh.common.exception.EventMeshException; +import org.apache.eventmesh.connector.canal.DatabaseConnection; +import org.apache.eventmesh.connector.canal.source.position.CanalCheckPositionMgr; +import org.apache.eventmesh.connector.canal.source.table.RdbSimpleTable; +import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; +import org.apache.eventmesh.openconnect.api.ConnectorCreateService; +import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; +import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; +import org.apache.eventmesh.openconnect.api.source.Source; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; +import org.apache.eventmesh.openconnect.util.ConfigUtil; + +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import com.google.common.util.concurrent.RateLimiter; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class CanalSourceCheckConnector extends AbstractComponent implements Source, ConnectorCreateService { + + private CanalSourceCheckConfig config; + private CanalCheckPositionMgr positionMgr; + private RdbTableMgr tableMgr; + private ThreadPoolExecutor executor; + private final ScheduledExecutorService scheduledThreadPoolExecutor = Executors.newSingleThreadScheduledExecutor(); + private final BlockingQueue> queue = new LinkedBlockingQueue<>(10000); + private final AtomicBoolean flag = new AtomicBoolean(true); + private RateLimiter globalLimiter; + + @Override + protected void run() throws Exception { + scheduledThreadPoolExecutor.scheduleAtFixedRate(() -> { + try { + this.tableMgr.start(); + } catch (Exception e) { + log.error("start tableMgr fail", e); + throw new RuntimeException(e); + } + try { + this.positionMgr.start(); + } catch (Exception e) { + throw new RuntimeException(e); + } + // if (positionMgr.isFinished()) { + // log.info("connector [{}] has finished the job", config.getSourceConnectorConfig().getConnectorName()); + // return; + // } + executor = new ThreadPoolExecutor(config.getParallel(), config.getParallel(), 0L, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(), new EventMeshThreadFactory("canal-source-check")); + List producers = new LinkedList<>(); + if (config.getSourceConnectorConfig().getDatabases() != null) { + for (RdbDBDefinition db : config.getSourceConnectorConfig().getDatabases()) { + for (RdbTableDefinition table : db.getTables()) { + try { + log.info("it will create producer of db [{}] table [{}]", db.getSchemaName(), table.getTableName()); + RdbSimpleTable simpleTable = new RdbSimpleTable(db.getSchemaName(), table.getTableName()); + JobRdbFullPosition position = positionMgr.getPosition(simpleTable); + if (position == null) { + throw new EventMeshException(String.format("db [%s] table [%s] have none position info", + db.getSchemaName(), table.getTableName())); + } + RdbTableDefinition tableDefinition = tableMgr.getTable(simpleTable); + if (tableDefinition == null) { + throw new EventMeshException(String.format("db [%s] table [%s] have none table definition info", + db.getSchemaName(), table.getTableName())); + } + CanalFullProducer producer = + new CanalFullProducer(queue, DatabaseConnection.sourceDataSource, (MySQLTableDef) tableDefinition, + position, config.getFlushSize(), config.getPagePerSecond()); + producer.setRecordLimiter(globalLimiter); + producers.add(producer); + } catch (Exception e) { + log.error("create schema [{}] table [{}] producers fail", db.getSchemaName(), + table.getTableName(), e); + } + } + } + } + producers.forEach(p -> executor.execute(() -> p.start(flag))); + + }, 0, config.getExecutePeriod(), TimeUnit.SECONDS); + } + + @Override + protected void shutdown() throws Exception { + flag.set(false); + if (!executor.isShutdown()) { + executor.shutdown(); + try { + if (!executor.awaitTermination(5, TimeUnit.SECONDS)) { + log.warn("wait thread pool shutdown timeout, it will shutdown now"); + executor.shutdownNow(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.info("shutdown thread pool fail"); + } + } + if (!scheduledThreadPoolExecutor.isShutdown()) { + scheduledThreadPoolExecutor.shutdown(); + try { + if (!scheduledThreadPoolExecutor.awaitTermination(5, TimeUnit.SECONDS)) { + log.warn("wait scheduledThreadPoolExecutor shutdown timeout, it will shutdown now"); + scheduledThreadPoolExecutor.shutdownNow(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.info("shutdown scheduledThreadPoolExecutor fail"); + } + } + if (DatabaseConnection.sourceDataSource != null) { + DatabaseConnection.sourceDataSource.close(); + log.info("data source has been closed"); + } + } + + @Override + public Source create() { + return new CanalSourceCheckConnector(); + } + + @Override + public Class configClass() { + return CanalSourceCheckConfig.class; + } + + @Override + public void init(Config config) throws Exception { + this.config = (CanalSourceCheckConfig) config; + init(); + } + + private void init() { + DatabaseConnection.sourceConfig = this.config.getSourceConnectorConfig(); + DatabaseConnection.initSourceConnection(); + this.tableMgr = new RdbTableMgr(config.getSourceConnectorConfig(), DatabaseConnection.sourceDataSource); + this.positionMgr = new CanalCheckPositionMgr(config, tableMgr); + this.globalLimiter = RateLimiter.create(config.getRecordPerSecond()); + } + + @Override + public void init(ConnectorContext connectorContext) throws Exception { + SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; + CanalSourceConfig canalSourceConfig = (CanalSourceConfig) sourceConnectorContext.getSourceConfig(); + this.config = ConfigUtil.parse(canalSourceConfig.getSourceConfig(), CanalSourceCheckConfig.class); + init(); + } + + @Override + public void commit(ConnectRecord record) { + // nothing + } + + @Override + public String name() { + return this.config.getSourceConnectorConfig().getConnectorName(); + } + + @Override + public void onException(ConnectRecord record) { + + } + + @Override + public List poll() { + while (flag.get()) { + try { + List records = queue.poll(5, TimeUnit.SECONDS); + if (records == null || records.isEmpty()) { + continue; + } + return records; + } catch (InterruptedException ignore) { + Thread.currentThread().interrupt(); + log.info("[{}] thread interrupted", this.getClass()); + return null; + } + } + log.info("[{}] life flag is stop, so return null", this.getClass()); + return null; + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceConnector.java new file mode 100644 index 0000000000..e24301ae07 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceConnector.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.source.connector; + +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceConfig; +import org.apache.eventmesh.common.remote.job.JobType; +import org.apache.eventmesh.openconnect.api.ConnectorCreateService; +import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; +import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; +import org.apache.eventmesh.openconnect.api.source.Source; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; + +import java.util.List; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class CanalSourceConnector implements Source, ConnectorCreateService { + + private CanalSourceConfig sourceConfig; + + private Source source; + + @Override + public Class configClass() { + return CanalSourceConfig.class; + } + + @Override + public void init(Config config) throws Exception { + // init config for canal source connector + this.sourceConfig = (CanalSourceConfig) config; + } + + @Override + public void init(ConnectorContext connectorContext) throws Exception { + SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; + if (sourceConnectorContext.getJobType().equals(JobType.FULL)) { + this.source = new CanalSourceFullConnector(); + } else if (sourceConnectorContext.getJobType().equals(JobType.INCREASE)) { + this.source = new CanalSourceIncrementConnector(); + } else if (sourceConnectorContext.getJobType().equals(JobType.CHECK)) { + this.source = new CanalSourceCheckConnector(); + } else { + throw new RuntimeException("unsupported job type " + sourceConnectorContext.getJobType()); + } + this.source.init(sourceConnectorContext); + } + + + @Override + public void start() throws Exception { + this.source.start(); + } + + + @Override + public void commit(ConnectRecord record) { + this.source.commit(record); + } + + @Override + public String name() { + return this.source.name(); + } + + @Override + public void onException(ConnectRecord record) { + this.source.onException(record); + } + + @Override + public void stop() throws Exception { + this.source.stop(); + } + + @Override + public List poll() { + return this.source.poll(); + } + + @Override + public Source create() { + return new CanalSourceConnector(); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java new file mode 100644 index 0000000000..df28342c39 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceFullConnector.java @@ -0,0 +1,187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.source.connector; + +import org.apache.eventmesh.common.AbstractComponent; +import org.apache.eventmesh.common.EventMeshThreadFactory; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceFullConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.JobRdbFullPosition; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLTableDef; +import org.apache.eventmesh.common.exception.EventMeshException; +import org.apache.eventmesh.connector.canal.DatabaseConnection; +import org.apache.eventmesh.connector.canal.source.position.CanalFullPositionMgr; +import org.apache.eventmesh.connector.canal.source.table.RdbSimpleTable; +import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; +import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; +import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; +import org.apache.eventmesh.openconnect.api.source.Source; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; +import org.apache.eventmesh.openconnect.util.ConfigUtil; + +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import com.google.common.util.concurrent.RateLimiter; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class CanalSourceFullConnector extends AbstractComponent implements Source { + + private CanalSourceFullConfig config; + private CanalFullPositionMgr positionMgr; + private RdbTableMgr tableMgr; + private ThreadPoolExecutor executor; + private final BlockingQueue> queue = new LinkedBlockingQueue<>(10000); + private final AtomicBoolean flag = new AtomicBoolean(true); + private RateLimiter globalLimiter; + + @Override + protected void run() throws Exception { + this.tableMgr.start(); + this.positionMgr.start(); + if (positionMgr.isFinished()) { + log.info("connector [{}] has finished the job", config.getSourceConnectorConfig().getConnectorName()); + return; + } + executor = new ThreadPoolExecutor(config.getParallel(), config.getParallel(), 0L, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(), new EventMeshThreadFactory("canal-source-full")); + List producers = new LinkedList<>(); + if (config.getSourceConnectorConfig().getDatabases() != null) { + for (RdbDBDefinition db : config.getSourceConnectorConfig().getDatabases()) { + for (RdbTableDefinition table : db.getTables()) { + try { + log.info("it will create producer of db [{}] table [{}]", db.getSchemaName(), table.getTableName()); + RdbSimpleTable simpleTable = new RdbSimpleTable(db.getSchemaName(), table.getTableName()); + JobRdbFullPosition position = positionMgr.getPosition(simpleTable); + if (position == null) { + throw new EventMeshException(String.format("db [%s] table [%s] have none position info", + db.getSchemaName(), table.getTableName())); + } + RdbTableDefinition tableDefinition = tableMgr.getTable(simpleTable); + if (tableDefinition == null) { + throw new EventMeshException(String.format("db [%s] table [%s] have none table definition info", + db.getSchemaName(), table.getTableName())); + } + CanalFullProducer producer = + new CanalFullProducer(queue, DatabaseConnection.sourceDataSource, (MySQLTableDef) tableDefinition, + position, config.getFlushSize(), config.getPagePerSecond()); + producer.setRecordLimiter(globalLimiter); + producers.add(producer); + } catch (Exception e) { + log.error("create schema [{}] table [{}] producers fail", db.getSchemaName(), + table.getTableName(), e); + } + } + } + } + producers.forEach(p -> executor.execute(() -> p.start(flag))); + } + + @Override + protected void shutdown() throws Exception { + flag.set(false); + if (!executor.isShutdown()) { + executor.shutdown(); + try { + if (!executor.awaitTermination(5, TimeUnit.SECONDS)) { + log.warn("wait thread pool shutdown timeout, it will shutdown now"); + executor.shutdownNow(); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.info("shutdown thread pool fail"); + } + } + if (DatabaseConnection.sourceDataSource != null) { + DatabaseConnection.sourceDataSource.close(); + log.info("data source has been closed"); + } + } + + @Override + public Class configClass() { + return CanalSourceFullConfig.class; + } + + @Override + public void init(Config config) throws Exception { + this.config = (CanalSourceFullConfig) config; + init(); + } + + private void init() { + DatabaseConnection.sourceConfig = this.config.getSourceConnectorConfig(); + DatabaseConnection.initSourceConnection(); + this.tableMgr = new RdbTableMgr(config.getSourceConnectorConfig(), DatabaseConnection.sourceDataSource); + this.positionMgr = new CanalFullPositionMgr(config, tableMgr); + this.globalLimiter = RateLimiter.create(config.getRecordPerSecond()); + } + + @Override + public void init(ConnectorContext connectorContext) throws Exception { + SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; + CanalSourceConfig canalSourceConfig = (CanalSourceConfig) sourceConnectorContext.getSourceConfig(); + this.config = ConfigUtil.parse(canalSourceConfig.getSourceConfig(), CanalSourceFullConfig.class); + init(); + } + + @Override + public void commit(ConnectRecord record) { + // nothing + } + + @Override + public String name() { + return this.config.getSourceConnectorConfig().getConnectorName(); + } + + @Override + public void onException(ConnectRecord record) { + + } + + @Override + public List poll() { + while (flag.get()) { + try { + List records = queue.poll(2, TimeUnit.SECONDS); + if (records == null || records.isEmpty()) { + continue; + } + return records; + } catch (InterruptedException ignore) { + Thread.currentThread().interrupt(); + log.info("[{}] thread interrupted", this.getClass()); + return null; + } + } + log.info("[{}] life flag is stop, so return null", this.getClass()); + return null; + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceIncrementConnector.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceIncrementConnector.java new file mode 100644 index 0000000000..c6e7603805 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/connector/CanalSourceIncrementConnector.java @@ -0,0 +1,464 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.source.connector; + +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceIncrementConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; +import org.apache.eventmesh.common.remote.datasource.DataSourceType; +import org.apache.eventmesh.common.remote.offset.RecordPosition; +import org.apache.eventmesh.common.remote.offset.canal.CanalRecordOffset; +import org.apache.eventmesh.common.remote.offset.canal.CanalRecordPartition; +import org.apache.eventmesh.common.utils.JsonUtils; +import org.apache.eventmesh.connector.canal.CanalConnectRecord; +import org.apache.eventmesh.connector.canal.DatabaseConnection; +import org.apache.eventmesh.connector.canal.source.EntryParser; +import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; +import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; +import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; +import org.apache.eventmesh.openconnect.api.source.Source; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; +import org.apache.eventmesh.openconnect.util.ConfigUtil; + +import org.apache.commons.lang3.StringUtils; + +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.LockSupport; + +import com.alibaba.druid.pool.DruidDataSource; +import com.alibaba.otter.canal.instance.core.CanalInstance; +import com.alibaba.otter.canal.instance.core.CanalInstanceGenerator; +import com.alibaba.otter.canal.instance.manager.CanalInstanceWithManager; +import com.alibaba.otter.canal.instance.manager.model.Canal; +import com.alibaba.otter.canal.instance.manager.model.CanalParameter; +import com.alibaba.otter.canal.instance.manager.model.CanalParameter.ClusterMode; +import com.alibaba.otter.canal.instance.manager.model.CanalParameter.HAMode; +import com.alibaba.otter.canal.instance.manager.model.CanalParameter.IndexMode; +import com.alibaba.otter.canal.instance.manager.model.CanalParameter.MetaMode; +import com.alibaba.otter.canal.instance.manager.model.CanalParameter.RunMode; +import com.alibaba.otter.canal.instance.manager.model.CanalParameter.SourcingType; +import com.alibaba.otter.canal.instance.manager.model.CanalParameter.StorageMode; +import com.alibaba.otter.canal.parse.CanalEventParser; +import com.alibaba.otter.canal.parse.ha.CanalHAController; +import com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser; +import com.alibaba.otter.canal.protocol.CanalEntry.Entry; +import com.alibaba.otter.canal.protocol.ClientIdentity; +import com.alibaba.otter.canal.server.embedded.CanalServerWithEmbedded; +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class CanalSourceIncrementConnector implements Source { + + private CanalSourceIncrementConfig sourceConfig; + + private CanalServerWithEmbedded canalServer; + + private ClientIdentity clientIdentity; + + private String tableFilter = null; + + private String fieldFilter = null; + + private volatile boolean running = false; + + private static final int maxEmptyTimes = 10; + + private RdbTableMgr tableMgr; + + private static final String SQL_SELECT_RDB_VERSION = "select version() as rdb_version"; + + private static final String SQL_SELECT_SERVER_UUID_IN_MARIADB = "SELECT @@global.server_id as server_uuid"; + + private static final String SQL_SHOW_SERVER_UUID_IN_MYSQL = "SELECT @@server_uuid as server_uuid"; + + @Override + public Class configClass() { + return CanalSourceConfig.class; + } + + @Override + public void init(Config config) throws Exception { + // init config for canal source connector + this.sourceConfig = (CanalSourceIncrementConfig) config; + } + + @Override + public void init(ConnectorContext connectorContext) throws Exception { + SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; + CanalSourceConfig canalSourceConfig = (CanalSourceConfig) sourceConnectorContext.getSourceConfig(); + this.sourceConfig = ConfigUtil.parse(canalSourceConfig.getSourceConfig(), CanalSourceIncrementConfig.class); + if (sourceConnectorContext.getRecordPositionList() != null) { + this.sourceConfig.setRecordPositions(sourceConnectorContext.getRecordPositionList()); + } + // filter: your_database\\.your_table; .*\\..* (all database & table) + tableFilter = buildTableFilters(sourceConfig); + + if (StringUtils.isNotEmpty(sourceConfig.getFieldFilter())) { + fieldFilter = sourceConfig.getFieldFilter(); + } + DatabaseConnection.sourceConfig = sourceConfig.getSourceConnectorConfig(); + DatabaseConnection.initSourceConnection(); + + DataSourceType dataSourceType = checkRDBDataSourceType(DatabaseConnection.sourceDataSource); + String serverUUID = queryServerUUID(DatabaseConnection.sourceDataSource, dataSourceType); + if (StringUtils.isNotEmpty(serverUUID)) { + log.info("init source increment connector, serverUUID: {}", serverUUID); + sourceConfig.setServerUUID(serverUUID); + } else { + log.warn("get source data source serverUUID empty please check"); + } + tableMgr = new RdbTableMgr(sourceConfig.getSourceConnectorConfig(), DatabaseConnection.sourceDataSource); + + canalServer = CanalServerWithEmbedded.instance(); + + canalServer.setCanalInstanceGenerator(new CanalInstanceGenerator() { + @Override + public CanalInstance generate(String destination) { + Canal canal = buildCanal(sourceConfig); + + CanalInstanceWithManager instance = new CanalInstanceWithManager(canal, tableFilter) { + + protected CanalHAController initHaController() { + return super.initHaController(); + } + + protected void startEventParserInternal(CanalEventParser parser, boolean isGroup) { + super.startEventParserInternal(parser, isGroup); + + if (eventParser instanceof MysqlEventParser) { + // set eventParser support type + ((MysqlEventParser) eventParser).setSupportBinlogFormats("ROW"); + ((MysqlEventParser) eventParser).setSupportBinlogImages("FULL"); + MysqlEventParser mysqlEventParser = (MysqlEventParser) eventParser; + mysqlEventParser.setParallel(false); + if (StringUtils.isNotEmpty(fieldFilter)) { + mysqlEventParser.setFieldFilter(fieldFilter); + } + + CanalHAController haController = mysqlEventParser.getHaController(); + if (!haController.isStart()) { + haController.start(); + } + } + } + }; + return instance; + } + }); + } + + private String queryServerUUID(DruidDataSource sourceDataSource, DataSourceType dataSourceType) { + String serverUUID = ""; + try { + String queryServerUUIDSql; + if (DataSourceType.MariaDB.equals(dataSourceType)) { + queryServerUUIDSql = SQL_SELECT_SERVER_UUID_IN_MARIADB; + } else { + queryServerUUIDSql = SQL_SHOW_SERVER_UUID_IN_MYSQL; + } + log.info("execute sql '{}' start.", queryServerUUIDSql); + try (PreparedStatement preparedStatement = sourceDataSource.getConnection().prepareStatement(queryServerUUIDSql)) { + ResultSet resultSet = preparedStatement.executeQuery(); + if (resultSet.next()) { + log.info("execute sql '{}' result:{}", queryServerUUIDSql, resultSet); + serverUUID = resultSet.getString("server_uuid"); + log.info("execute sql '{}',query server_uuid result:{}", queryServerUUIDSql, serverUUID); + return serverUUID; + } + } + } catch (Exception e) { + log.warn("select server_uuid failed,data source:{}", sourceDataSource, e); + throw new RuntimeException("select server_uuid failed"); + } + return serverUUID; + } + + // check is mariadb or mysql + private DataSourceType checkRDBDataSourceType(DruidDataSource sourceDataSource) { + try { + log.info("execute sql '{}' start.", SQL_SELECT_RDB_VERSION); + try (PreparedStatement preparedStatement = sourceDataSource.getConnection().prepareStatement(SQL_SELECT_RDB_VERSION)) { + ResultSet resultSet = preparedStatement.executeQuery(); + if (resultSet.next()) { + log.info("execute sql '{}' result:{}", SQL_SELECT_RDB_VERSION, resultSet); + String rdbVersion = resultSet.getString("rdb_version"); + if (StringUtils.isNotBlank(rdbVersion)) { + if (rdbVersion.toLowerCase().contains(DataSourceType.MariaDB.getName().toLowerCase())) { + return DataSourceType.MariaDB; + } + } + } + } + } catch (Exception e) { + log.warn("select rdb version failed,data source:{}", sourceDataSource, e); + throw new RuntimeException("select rdb version failed"); + } + return DataSourceType.MYSQL; + } + + private String buildTableFilters(CanalSourceIncrementConfig sourceConfig) { + StringBuilder tableFilterBuilder = new StringBuilder(); + Set dbDefinitions = sourceConfig.getSourceConnectorConfig().getDatabases(); + for (RdbDBDefinition dbDefinition : dbDefinitions) { + Set tableDefinitions = dbDefinition.getTables(); + for (RdbTableDefinition rdbTableDefinition : tableDefinitions) { + if (tableFilterBuilder.length() > 0) { + tableFilterBuilder.append(","); + } + String dbName = rdbTableDefinition.getSchemaName(); + String tableName = rdbTableDefinition.getTableName(); + tableFilterBuilder.append(dbName); + tableFilterBuilder.append("\\."); + tableFilterBuilder.append(tableName); + } + } + return tableFilterBuilder.toString(); + } + + private Canal buildCanal(CanalSourceIncrementConfig sourceConfig) { + long slaveId = 10000; + if (sourceConfig.getSlaveId() != null) { + slaveId = sourceConfig.getSlaveId(); + } + + Canal canal = new Canal(); + canal.setId(sourceConfig.getCanalInstanceId()); + canal.setName(sourceConfig.getDestination()); + canal.setDesc(sourceConfig.getDesc()); + + CanalParameter parameter = new CanalParameter(); + + parameter.setRunMode(RunMode.EMBEDDED); + parameter.setClusterMode(ClusterMode.STANDALONE); + parameter.setMetaMode(MetaMode.MEMORY); + parameter.setHaMode(HAMode.HEARTBEAT); + parameter.setIndexMode(IndexMode.MEMORY); + parameter.setStorageMode(StorageMode.MEMORY); + parameter.setMemoryStorageBufferSize(32 * 1024); + + parameter.setSourcingType(SourcingType.MYSQL); + parameter.setDbAddresses(Collections.singletonList(new InetSocketAddress(sourceConfig.getSourceConnectorConfig().getDbAddress(), + sourceConfig.getSourceConnectorConfig().getDbPort()))); + parameter.setDbUsername(sourceConfig.getSourceConnectorConfig().getUserName()); + parameter.setDbPassword(sourceConfig.getSourceConnectorConfig().getPassWord()); + + // set if enabled gtid mode + parameter.setGtidEnable(sourceConfig.isGTIDMode()); + + // check positions + // example: Arrays.asList("{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}", + // "{\"journalName\":\"mysql-bin.000001\",\"position\":6163L,\"timestamp\":1322803601000L}") + if (sourceConfig.getRecordPositions() != null && !sourceConfig.getRecordPositions().isEmpty()) { + List recordPositions = sourceConfig.getRecordPositions(); + List positions = new ArrayList<>(); + recordPositions.forEach(recordPosition -> { + Map recordPositionMap = new HashMap<>(); + CanalRecordPartition canalRecordPartition = (CanalRecordPartition) (recordPosition.getRecordPartition()); + CanalRecordOffset canalRecordOffset = (CanalRecordOffset) (recordPosition.getRecordOffset()); + recordPositionMap.put("journalName", canalRecordPartition.getJournalName()); + recordPositionMap.put("timestamp", canalRecordPartition.getTimeStamp()); + recordPositionMap.put("position", canalRecordOffset.getOffset()); + // for mariaDB not support gtid mode + if (sourceConfig.isGTIDMode() && !sourceConfig.isMariaDB()) { + String gtidRange = canalRecordOffset.getGtid(); + if (gtidRange != null) { + if (canalRecordOffset.getCurrentGtid() != null) { + gtidRange = EntryParser.replaceGtidRange(canalRecordOffset.getGtid(), canalRecordOffset.getCurrentGtid(), + sourceConfig.getServerUUID()); + } + recordPositionMap.put("gtid", gtidRange); + } + } + positions.add(JsonUtils.toJSONString(recordPositionMap)); + }); + parameter.setPositions(positions); + } + + parameter.setSlaveId(slaveId); + + parameter.setDefaultConnectionTimeoutInSeconds(30); + parameter.setConnectionCharset("UTF-8"); + parameter.setConnectionCharsetNumber((byte) 33); + parameter.setReceiveBufferSize(8 * 1024); + parameter.setSendBufferSize(8 * 1024); + + // heartbeat detect + parameter.setDetectingEnable(false); + + parameter.setDdlIsolation(sourceConfig.isDdlSync()); + parameter.setFilterTableError(sourceConfig.isFilterTableError()); + parameter.setMemoryStorageRawEntry(false); + + canal.setCanalParameter(parameter); + return canal; + } + + + @Override + public void start() throws Exception { + if (running) { + return; + } + tableMgr.start(); + canalServer.start(); + + canalServer.start(sourceConfig.getDestination()); + this.clientIdentity = new ClientIdentity(sourceConfig.getDestination(), sourceConfig.getClientId(), tableFilter); + canalServer.subscribe(clientIdentity); + + running = true; + } + + + @Override + public void commit(ConnectRecord record) { + + } + + @Override + public String name() { + return this.sourceConfig.getSourceConnectorConfig().getConnectorName(); + } + + @Override + public void onException(ConnectRecord record) { + + } + + @Override + public void stop() { + if (!running) { + return; + } + running = false; + canalServer.stop(sourceConfig.getDestination()); + canalServer.stop(); + } + + @Override + public List poll() { + int emptyTimes = 0; + com.alibaba.otter.canal.protocol.Message message = null; + if (sourceConfig.getBatchTimeout() < 0) { + while (running) { + message = canalServer.getWithoutAck(clientIdentity, sourceConfig.getBatchSize()); + if (message == null || message.getId() == -1L) { // empty + applyWait(emptyTimes++); + } else { + break; + } + } + } else { // perform with timeout + while (running) { + message = + canalServer.getWithoutAck(clientIdentity, sourceConfig.getBatchSize(), sourceConfig.getBatchTimeout(), TimeUnit.MILLISECONDS); + if (message == null || message.getId() == -1L) { // empty + continue; + } + break; + } + } + + List entries; + assert message != null; + if (message.isRaw()) { + entries = new ArrayList<>(message.getRawEntries().size()); + for (ByteString entry : message.getRawEntries()) { + try { + entries.add(Entry.parseFrom(entry)); + } catch (InvalidProtocolBufferException e) { + throw new RuntimeException(e); + } + } + } else { + entries = message.getEntries(); + } + + List result = new ArrayList<>(); + // key: Xid offset + Map> connectorRecordMap = EntryParser.parse(sourceConfig, entries, tableMgr); + + if (!connectorRecordMap.isEmpty()) { + Set>> entrySet = connectorRecordMap.entrySet(); + for (Map.Entry> entry : entrySet) { + List connectRecordList = entry.getValue(); + CanalConnectRecord lastRecord = entry.getValue().get(connectRecordList.size() - 1); + CanalRecordPartition canalRecordPartition = new CanalRecordPartition(); + canalRecordPartition.setServerUUID(sourceConfig.getServerUUID()); + canalRecordPartition.setJournalName(lastRecord.getJournalName()); + canalRecordPartition.setTimeStamp(lastRecord.getExecuteTime()); + // Xid offset with gtid + Long binLogOffset = entry.getKey(); + CanalRecordOffset canalRecordOffset = new CanalRecordOffset(); + canalRecordOffset.setOffset(binLogOffset); + if (StringUtils.isNotEmpty(lastRecord.getGtid()) && StringUtils.isNotEmpty(lastRecord.getCurrentGtid())) { + canalRecordOffset.setGtid(lastRecord.getGtid()); + canalRecordOffset.setCurrentGtid(lastRecord.getCurrentGtid()); + } + + // split record list + List> splitLists = new ArrayList<>(); + for (int i = 0; i < connectRecordList.size(); i += sourceConfig.getBatchSize()) { + int end = Math.min(i + sourceConfig.getBatchSize(), connectRecordList.size()); + List subList = connectRecordList.subList(i, end); + splitLists.add(subList); + } + + for (int i = 0; i < splitLists.size(); i++) { + ConnectRecord connectRecord = new ConnectRecord(canalRecordPartition, canalRecordOffset, System.currentTimeMillis()); + connectRecord.addExtension("messageId", String.valueOf(message.getId())); + connectRecord.addExtension("batchIndex", i); + connectRecord.addExtension("totalBatches", splitLists.size()); + connectRecord.setData(JsonUtils.toJSONString(splitLists.get(i)).getBytes(StandardCharsets.UTF_8)); + result.add(connectRecord); + } + } + log.debug("message {} has been processed", message); + } + log.debug("ack message, messageId {}", message.getId()); + canalServer.ack(clientIdentity, message.getId()); + + return result; + } + + // Handle the situation of no data and avoid empty loop death + private void applyWait(int emptyTimes) { + int newEmptyTimes = Math.min(emptyTimes, maxEmptyTimes); + if (emptyTimes <= 3) { + Thread.yield(); + } else { + LockSupport.parkNanos(1000 * 1000L * newEmptyTimes); + } + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalCheckPositionMgr.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalCheckPositionMgr.java new file mode 100644 index 0000000000..149c62602c --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalCheckPositionMgr.java @@ -0,0 +1,250 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.source.position; + +import org.apache.eventmesh.common.AbstractComponent; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceCheckConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.JobRdbFullPosition; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbColumnDefinition; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.Constants; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLTableDef; +import org.apache.eventmesh.common.remote.offset.RecordPosition; +import org.apache.eventmesh.common.remote.offset.canal.CanalFullRecordOffset; +import org.apache.eventmesh.common.utils.JsonUtils; +import org.apache.eventmesh.connector.canal.DatabaseConnection; +import org.apache.eventmesh.connector.canal.source.table.RdbSimpleTable; +import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; + +import org.apache.commons.lang3.StringUtils; + +import java.sql.JDBCType; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.LinkedHashMap; +import java.util.Map; + +import javax.sql.DataSource; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class CanalCheckPositionMgr extends AbstractComponent { + + private final CanalSourceCheckConfig config; + private final Map positions = new LinkedHashMap<>(); + private final RdbTableMgr tableMgr; + + public CanalCheckPositionMgr(CanalSourceCheckConfig config, RdbTableMgr tableMgr) { + this.config = config; + this.tableMgr = tableMgr; + } + + @Override + protected void run() throws Exception { + if (config == null || config.getSourceConnectorConfig() == null || config.getSourceConnectorConfig().getDatabases() == null) { + log.info("config or database is null"); + return; + } + prepareRecordPosition(); + initPositions(); + } + + public void prepareRecordPosition() { + if (config.getStartPosition() != null && !config.getStartPosition().isEmpty()) { + for (RecordPosition record : config.getStartPosition()) { + CanalFullRecordOffset offset = (CanalFullRecordOffset) record.getRecordOffset(); + RdbSimpleTable table = new RdbSimpleTable(offset.getPosition().getSchema(), offset.getPosition().getTableName()); + positions.put(table, offset.getPosition()); + } + } + } + + public JobRdbFullPosition getPosition(RdbSimpleTable table) { + return positions.get(table); + } + + public boolean isFinished() { + for (JobRdbFullPosition position : positions.values()) { + if (!position.isFinished()) { + log.info("schema [{}] table [{}] is not finish", position.getSchema(), position.getTableName()); + return false; + } + } + return true; + } + + private void initPositions() { + for (RdbDBDefinition database : config.getSourceConnectorConfig().getDatabases()) { + for (RdbTableDefinition table : database.getTables()) { + try { + RdbSimpleTable simpleTable = new RdbSimpleTable(database.getSchemaName(), table.getTableName()); + RdbTableDefinition tableDefinition; + if ((tableDefinition = tableMgr.getTable(simpleTable)) == null) { + log.error("db [{}] table [{}] definition is null", database.getSchemaName(), table.getTableName()); + continue; + } + log.info("init position of data [{}] table [{}]", database.getSchemaName(), table.getTableName()); + + JobRdbFullPosition recordPosition = positions.get(simpleTable); + if (recordPosition == null || !recordPosition.isFinished()) { + positions.put(simpleTable, + fetchTableInfo(DatabaseConnection.sourceDataSource, (MySQLTableDef) tableDefinition, recordPosition)); + } + } catch (Exception e) { + log.error("process schema [{}] table [{}] position fail", database.getSchemaName(), table.getTableName(), e); + } + + } + } + } + + private JobRdbFullPosition fetchTableInfo(DataSource dataSource, MySQLTableDef tableDefinition, JobRdbFullPosition recordPosition) + throws SQLException { + TableFullPosition position = new TableFullPosition(); + Map preMinPrimaryKeys = new LinkedHashMap<>(); + Map preMaxPrimaryKeys = new LinkedHashMap<>(); + for (String pk : tableDefinition.getPrimaryKeys()) { + Object min = fetchMinPrimaryKey(dataSource, tableDefinition, preMinPrimaryKeys, pk); + Object max = fetchMaxPrimaryKey(dataSource, tableDefinition, preMaxPrimaryKeys, pk); + preMinPrimaryKeys.put(pk, min); + preMaxPrimaryKeys.put(pk, max); + position.getCurPrimaryKeyCols().put(pk, min); + position.getMinPrimaryKeyCols().put(pk, min); + position.getMaxPrimaryKeyCols().put(pk, max); + } + JobRdbFullPosition jobRdbFullPosition = new JobRdbFullPosition(); + if (recordPosition != null) { + if (StringUtils.isNotBlank(recordPosition.getPrimaryKeyRecords())) { + TableFullPosition record = JsonUtils.parseObject(recordPosition.getPrimaryKeyRecords(), TableFullPosition.class); + if (record != null && record.getCurPrimaryKeyCols() != null && !record.getCurPrimaryKeyCols().isEmpty()) { + position.setCurPrimaryKeyCols(record.getCurPrimaryKeyCols()); + } + } + jobRdbFullPosition.setPercent(recordPosition.getPercent()); + } + long rowCount = queryCurTableRowCount(dataSource, tableDefinition); + jobRdbFullPosition.setSchema(tableDefinition.getSchemaName()); + jobRdbFullPosition.setTableName(tableDefinition.getTableName()); + jobRdbFullPosition.setMaxCount(rowCount); + jobRdbFullPosition.setPrimaryKeyRecords(JsonUtils.toJSONString(position)); + return jobRdbFullPosition; + } + + + private long queryCurTableRowCount(DataSource datasource, MySQLTableDef tableDefinition) throws SQLException { + String sql = "select `AVG_ROW_LENGTH`,`DATA_LENGTH` from information_schema.TABLES where `TABLE_SCHEMA`='" + tableDefinition.getSchemaName() + + "' and `TABLE_NAME`='" + tableDefinition.getTableName() + "'"; + try (Statement statement = datasource.getConnection().createStatement(); ResultSet resultSet = statement.executeQuery(sql)) { + long result = 0L; + if (resultSet.next()) { + long avgRowLength = resultSet.getLong("AVG_ROW_LENGTH"); + long dataLength = resultSet.getLong("DATA_LENGTH"); + if (avgRowLength != 0L) { + result = dataLength / avgRowLength; + } + } + return result; + } + } + + private void appendPrePrimaryKey(Map preMap, StringBuilder sql) { + if (preMap != null && !preMap.isEmpty()) { + sql.append(" WHERE "); + boolean first = true; + for (Map.Entry entry : preMap.entrySet()) { + if (first) { + first = false; + } else { + sql.append(" AND "); + } + sql.append(Constants.MySQLQuot).append(entry.getKey()).append(Constants.MySQLQuot).append("=?"); + } + } + } + + private void setValue2Statement(PreparedStatement ps, Map preMap, MySQLTableDef tableDefinition) throws SQLException { + if (preMap != null && !preMap.isEmpty()) { + int index = 1; + for (Map.Entry entry : preMap.entrySet()) { + RdbColumnDefinition def = tableDefinition.getColumnDefinitions().get(entry.getKey()); + ps.setObject(index, entry.getValue(), def.getJdbcType().getVendorTypeNumber()); + ++index; + } + } + } + + private Object fetchMinPrimaryKey(DataSource dataSource, MySQLTableDef tableDefinition, Map prePrimary, String curPrimaryKeyCol) + throws SQLException { + StringBuilder builder = new StringBuilder(); + builder.append("SELECT MIN(").append(Constants.MySQLQuot).append(curPrimaryKeyCol).append(Constants.MySQLQuot) + .append(") min_primary_key FROM").append(Constants.MySQLQuot).append(tableDefinition.getSchemaName()).append(Constants.MySQLQuot) + .append(".").append(Constants.MySQLQuot).append(tableDefinition.getTableName()).append(Constants.MySQLQuot); + appendPrePrimaryKey(prePrimary, builder); + String sql = builder.toString(); + log.info("fetch min primary sql [{}]", sql); + try (PreparedStatement statement = dataSource.getConnection().prepareStatement(sql)) { + setValue2Statement(statement, prePrimary, tableDefinition); + try (ResultSet resultSet = statement.executeQuery()) { + if (resultSet.next()) { + RdbColumnDefinition columnDefinition = tableDefinition.getColumnDefinitions().get(curPrimaryKeyCol); + if (columnDefinition.getJdbcType() == JDBCType.TIMESTAMP) { + return resultSet.getString("min_primary_key"); + } else { + return resultSet.getObject("min_primary_key"); + } + } + } + } + return null; + } + + private Object fetchMaxPrimaryKey(DataSource dataSource, MySQLTableDef tableDefinition, Map prePrimary, String curPrimaryKeyCol) + throws SQLException { + StringBuilder builder = new StringBuilder(); + builder.append("SELECT MAX(").append(Constants.MySQLQuot).append(curPrimaryKeyCol).append(Constants.MySQLQuot) + .append(") max_primary_key FROM").append(Constants.MySQLQuot).append(tableDefinition.getSchemaName()).append(Constants.MySQLQuot) + .append(".").append(Constants.MySQLQuot).append(tableDefinition.getTableName()).append(Constants.MySQLQuot); + appendPrePrimaryKey(prePrimary, builder); + String sql = builder.toString(); + log.info("fetch max primary sql [{}]", sql); + try (PreparedStatement statement = dataSource.getConnection().prepareStatement(sql)) { + setValue2Statement(statement, prePrimary, tableDefinition); + try (ResultSet resultSet = statement.executeQuery()) { + if (resultSet.next()) { + RdbColumnDefinition columnDefinition = tableDefinition.getColumnDefinitions().get(curPrimaryKeyCol); + if (columnDefinition.getJdbcType() == JDBCType.TIMESTAMP) { + return resultSet.getString("max_primary_key"); + } else { + return resultSet.getObject("max_primary_key"); + } + } + } + } + return null; + } + + + @Override + protected void shutdown() throws Exception { + + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java new file mode 100644 index 0000000000..dad0ddbf3b --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/CanalFullPositionMgr.java @@ -0,0 +1,252 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.source.position; + +import org.apache.eventmesh.common.AbstractComponent; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalSourceFullConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.JobRdbFullPosition; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbColumnDefinition; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.Constants; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLTableDef; +import org.apache.eventmesh.common.remote.offset.RecordPosition; +import org.apache.eventmesh.common.remote.offset.canal.CanalFullRecordOffset; +import org.apache.eventmesh.common.utils.JsonUtils; +import org.apache.eventmesh.connector.canal.DatabaseConnection; +import org.apache.eventmesh.connector.canal.source.table.RdbSimpleTable; +import org.apache.eventmesh.connector.canal.source.table.RdbTableMgr; + +import org.apache.commons.lang3.StringUtils; + +import java.sql.Connection; +import java.sql.JDBCType; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.LinkedHashMap; +import java.util.Map; + +import javax.sql.DataSource; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class CanalFullPositionMgr extends AbstractComponent { + + private final CanalSourceFullConfig config; + private final Map positions = new LinkedHashMap<>(); + private final RdbTableMgr tableMgr; + + public CanalFullPositionMgr(CanalSourceFullConfig config, RdbTableMgr tableMgr) { + this.config = config; + this.tableMgr = tableMgr; + } + + @Override + protected void run() throws Exception { + if (config == null || config.getSourceConnectorConfig() == null || config.getSourceConnectorConfig().getDatabases() == null) { + log.info("config or database is null"); + return; + } + prepareRecordPosition(); + initPositions(); + } + + public void prepareRecordPosition() { + if (config.getStartPosition() != null && !config.getStartPosition().isEmpty()) { + for (RecordPosition record : config.getStartPosition()) { + CanalFullRecordOffset offset = (CanalFullRecordOffset) record.getRecordOffset(); + RdbSimpleTable table = new RdbSimpleTable(offset.getPosition().getSchema(), offset.getPosition().getTableName()); + positions.put(table, offset.getPosition()); + } + } + } + + public JobRdbFullPosition getPosition(RdbSimpleTable table) { + return positions.get(table); + } + + public boolean isFinished() { + for (JobRdbFullPosition position : positions.values()) { + if (!position.isFinished()) { + log.info("schema [{}] table [{}] is not finish", position.getSchema(), position.getTableName()); + return false; + } + } + return true; + } + + private void initPositions() { + for (RdbDBDefinition database : config.getSourceConnectorConfig().getDatabases()) { + for (RdbTableDefinition table : database.getTables()) { + try { + RdbSimpleTable simpleTable = new RdbSimpleTable(database.getSchemaName(), table.getTableName()); + RdbTableDefinition tableDefinition; + if ((tableDefinition = tableMgr.getTable(simpleTable)) == null) { + log.error("db [{}] table [{}] definition is null", database.getSchemaName(), table.getTableName()); + continue; + } + log.info("init position of data [{}] table [{}]", database.getSchemaName(), table.getTableName()); + + JobRdbFullPosition recordPosition = positions.get(simpleTable); + if (recordPosition == null || !recordPosition.isFinished()) { + positions.put(simpleTable, + fetchTableInfo(DatabaseConnection.sourceDataSource, (MySQLTableDef) tableDefinition, recordPosition)); + } + } catch (Exception e) { + log.error("process schema [{}] table [{}] position fail", database.getSchemaName(), table.getTableName(), e); + } + + } + } + } + + private JobRdbFullPosition fetchTableInfo(DataSource dataSource, MySQLTableDef tableDefinition, JobRdbFullPosition recordPosition) + throws SQLException { + TableFullPosition position = new TableFullPosition(); + Map preMinPrimaryKeys = new LinkedHashMap<>(); + Map preMaxPrimaryKeys = new LinkedHashMap<>(); + for (String pk : tableDefinition.getPrimaryKeys()) { + Object min = fetchMinPrimaryKey(dataSource, tableDefinition, preMinPrimaryKeys, pk); + Object max = fetchMaxPrimaryKey(dataSource, tableDefinition, preMaxPrimaryKeys, pk); + preMinPrimaryKeys.put(pk, min); + preMaxPrimaryKeys.put(pk, max); + position.getCurPrimaryKeyCols().put(pk, min); + position.getMinPrimaryKeyCols().put(pk, min); + position.getMaxPrimaryKeyCols().put(pk, max); + } + JobRdbFullPosition jobRdbFullPosition = new JobRdbFullPosition(); + if (recordPosition != null) { + if (StringUtils.isNotBlank(recordPosition.getPrimaryKeyRecords())) { + TableFullPosition record = JsonUtils.parseObject(recordPosition.getPrimaryKeyRecords(), TableFullPosition.class); + if (record != null && record.getCurPrimaryKeyCols() != null && !record.getCurPrimaryKeyCols().isEmpty()) { + position.setCurPrimaryKeyCols(record.getCurPrimaryKeyCols()); + } + } + jobRdbFullPosition.setPercent(recordPosition.getPercent()); + } + long rowCount = queryCurTableRowCount(dataSource, tableDefinition); + jobRdbFullPosition.setSchema(tableDefinition.getSchemaName()); + jobRdbFullPosition.setTableName(tableDefinition.getTableName()); + jobRdbFullPosition.setMaxCount(rowCount); + jobRdbFullPosition.setPrimaryKeyRecords(JsonUtils.toJSONString(position)); + return jobRdbFullPosition; + } + + + private long queryCurTableRowCount(DataSource datasource, MySQLTableDef tableDefinition) throws SQLException { + String sql = "select `AVG_ROW_LENGTH`,`DATA_LENGTH` from information_schema.TABLES where `TABLE_SCHEMA`='" + tableDefinition.getSchemaName() + + "' and `TABLE_NAME`='" + tableDefinition.getTableName() + "'"; + try (Connection conn = datasource.getConnection(); Statement statement = conn.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + long result = 0L; + if (resultSet.next()) { + long avgRowLength = resultSet.getLong("AVG_ROW_LENGTH"); + long dataLength = resultSet.getLong("DATA_LENGTH"); + if (avgRowLength != 0L) { + result = dataLength / avgRowLength; + } + } + return result; + } + } + + private void appendPrePrimaryKey(Map preMap, StringBuilder sql) { + if (preMap != null && !preMap.isEmpty()) { + sql.append(" WHERE "); + boolean first = true; + for (Map.Entry entry : preMap.entrySet()) { + if (first) { + first = false; + } else { + sql.append(" AND "); + } + sql.append(Constants.MySQLQuot).append(entry.getKey()).append(Constants.MySQLQuot).append("=?"); + } + } + } + + private void setValue2Statement(PreparedStatement ps, Map preMap, MySQLTableDef tableDefinition) throws SQLException { + if (preMap != null && !preMap.isEmpty()) { + int index = 1; + for (Map.Entry entry : preMap.entrySet()) { + RdbColumnDefinition def = tableDefinition.getColumnDefinitions().get(entry.getKey()); + ps.setObject(index, entry.getValue(), def.getJdbcType().getVendorTypeNumber()); + ++index; + } + } + } + + private Object fetchMinPrimaryKey(DataSource dataSource, MySQLTableDef tableDefinition, Map prePrimary, String curPrimaryKeyCol) + throws SQLException { + StringBuilder builder = new StringBuilder(); + builder.append("SELECT MIN(").append(Constants.MySQLQuot).append(curPrimaryKeyCol).append(Constants.MySQLQuot) + .append(") min_primary_key FROM").append(Constants.MySQLQuot).append(tableDefinition.getSchemaName()).append(Constants.MySQLQuot) + .append(".").append(Constants.MySQLQuot).append(tableDefinition.getTableName()).append(Constants.MySQLQuot); + appendPrePrimaryKey(prePrimary, builder); + String sql = builder.toString(); + log.info("fetch min primary sql [{}]", sql); + try (Connection conn = dataSource.getConnection(); PreparedStatement statement = conn.prepareStatement(sql)) { + setValue2Statement(statement, prePrimary, tableDefinition); + try (ResultSet resultSet = statement.executeQuery()) { + if (resultSet.next()) { + RdbColumnDefinition columnDefinition = tableDefinition.getColumnDefinitions().get(curPrimaryKeyCol); + if (columnDefinition.getJdbcType() == JDBCType.TIMESTAMP) { + return resultSet.getString("min_primary_key"); + } else { + return resultSet.getObject("min_primary_key"); + } + } + } + } + return null; + } + + private Object fetchMaxPrimaryKey(DataSource dataSource, MySQLTableDef tableDefinition, Map prePrimary, String curPrimaryKeyCol) + throws SQLException { + StringBuilder builder = new StringBuilder(); + builder.append("SELECT MAX(").append(Constants.MySQLQuot).append(curPrimaryKeyCol).append(Constants.MySQLQuot) + .append(") max_primary_key FROM").append(Constants.MySQLQuot).append(tableDefinition.getSchemaName()).append(Constants.MySQLQuot) + .append(".").append(Constants.MySQLQuot).append(tableDefinition.getTableName()).append(Constants.MySQLQuot); + appendPrePrimaryKey(prePrimary, builder); + String sql = builder.toString(); + log.info("fetch max primary sql [{}]", sql); + try (Connection conn = dataSource.getConnection(); PreparedStatement statement = conn.prepareStatement(sql)) { + setValue2Statement(statement, prePrimary, tableDefinition); + try (ResultSet resultSet = statement.executeQuery()) { + if (resultSet.next()) { + RdbColumnDefinition columnDefinition = tableDefinition.getColumnDefinitions().get(curPrimaryKeyCol); + if (columnDefinition.getJdbcType() == JDBCType.TIMESTAMP) { + return resultSet.getString("max_primary_key"); + } else { + return resultSet.getObject("max_primary_key"); + } + } + } + } + return null; + } + + + @Override + protected void shutdown() throws Exception { + + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/TableFullPosition.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/TableFullPosition.java new file mode 100644 index 0000000000..b1a8024ec5 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/position/TableFullPosition.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.source.position; + +import java.util.LinkedHashMap; +import java.util.Map; + +import lombok.Data; + +@Data +public class TableFullPosition { + private Map curPrimaryKeyCols = new LinkedHashMap<>(); + private Map minPrimaryKeyCols = new LinkedHashMap<>(); + private Map maxPrimaryKeyCols = new LinkedHashMap<>(); +} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/ConnectorRecordPartition.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbSimpleTable.java similarity index 56% rename from eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/ConnectorRecordPartition.java rename to eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbSimpleTable.java index 3308654385..5b9c35fff3 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/ConnectorRecordPartition.java +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbSimpleTable.java @@ -15,53 +15,45 @@ * limitations under the License. */ -package org.apache.eventmesh.openconnect.offsetmgmt.api.storage; +package org.apache.eventmesh.connector.canal.source.table; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; -import java.util.Map; import java.util.Objects; -/** - * extend record partition - */ -public class ConnectorRecordPartition extends RecordPartition { - - /** - * connect name - */ - private String connectorName; - - public ConnectorRecordPartition() { +import lombok.Data; +@Data +public class RdbSimpleTable extends RdbTableDefinition { + public RdbSimpleTable(String database, String schema, String tableName) { + this.schemaName = schema; + this.tableName = tableName; + this.database = database; } - public ConnectorRecordPartition(String connectorName, Map partition) { - super(partition); - this.connectorName = connectorName; + public RdbSimpleTable(String schema, String tableName) { + this(null, schema, tableName); } - public String getConnectorName() { - return connectorName; - } + private final String database; @Override public boolean equals(Object o) { if (this == o) { return true; } - if (!(o instanceof ConnectorRecordPartition)) { + if (o == null || getClass() != o.getClass()) { return false; } if (!super.equals(o)) { return false; } - ConnectorRecordPartition that = (ConnectorRecordPartition) o; - return this.connectorName.equals(that.connectorName); + RdbSimpleTable that = (RdbSimpleTable) o; + return Objects.equals(database, that.database); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), connectorName); + return Objects.hash(super.hashCode(), database); } } diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java new file mode 100644 index 0000000000..954b81ca70 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/source/table/RdbTableMgr.java @@ -0,0 +1,212 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.source.table; + +import org.apache.eventmesh.common.AbstractComponent; +import org.apache.eventmesh.common.config.connector.rdb.JdbcConfig; +import org.apache.eventmesh.common.config.connector.rdb.canal.CanalMySQLType; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbDBDefinition; +import org.apache.eventmesh.common.config.connector.rdb.canal.RdbTableDefinition; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLColumnDef; +import org.apache.eventmesh.common.config.connector.rdb.canal.mysql.MySQLTableDef; +import org.apache.eventmesh.common.exception.EventMeshException; +import org.apache.eventmesh.connector.canal.SqlUtils; + +import java.sql.Connection; +import java.sql.JDBCType; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import javax.sql.DataSource; + +import lombok.extern.slf4j.Slf4j; + +/** + * Description: + */ +@Slf4j + +public class RdbTableMgr extends AbstractComponent { + + private final JdbcConfig config; + private final Map tables = new HashMap<>(); + private final DataSource dataSource; + + public RdbTableMgr(JdbcConfig config, DataSource dataSource) { + this.config = config; + this.dataSource = dataSource; + } + + public RdbTableDefinition getTable(String schema, String tableName) { + return getTable(new RdbSimpleTable(schema, tableName)); + } + + public RdbTableDefinition getTable(RdbSimpleTable table) { + return tables.get(table); + } + + @Override + protected void run() { + if (config != null && config.getDatabases() != null) { + for (RdbDBDefinition db : config.getDatabases()) { + if (db.getTables() == null) { + log.warn("init db [{}] position, but it's tables are null", db.getSchemaName()); + continue; + } + for (RdbTableDefinition table : db.getTables()) { + try { + MySQLTableDef mysqlTable = new MySQLTableDef(); + mysqlTable.setSchemaName(db.getSchemaName()); + mysqlTable.setTableName(table.getTableName()); + List tables = Collections.singletonList(table.getTableName()); + Map> primaryKeys = queryTablePrimaryKey(db.getSchemaName(), tables); + Map> columns = queryColumns(db.getSchemaName(), tables); + if (primaryKeys == null || primaryKeys.isEmpty() || primaryKeys.get(table.getTableName()) == null) { + log.warn("init db [{}] table [{}] info, and primary keys are empty", db.getSchemaName(), table.getTableName()); + } else { + mysqlTable.setPrimaryKeys(primaryKeys.get(table.getTableName())); + } + if (columns == null || columns.isEmpty() || columns.get(table.getTableName()) == null) { + log.warn("init db [{}] table [{}] info, and columns are empty", db.getSchemaName(), table.getTableName()); + throw new EventMeshException("db [{}] table [{}] columns are empty"); + } else { + LinkedHashMap cols = new LinkedHashMap<>(); + columns.get(table.getTableName()).forEach(x -> cols.put(x.getName(), x)); + mysqlTable.setColumnDefinitions(cols); + } + + this.tables.put(new RdbSimpleTable(db.getSchemaName(), table.getTableName()), mysqlTable); + } catch (SQLException e) { + log.error("init rdb table schema [{}] table [{}] fail", db.getSchemaName(), table.getTableName(), e); + throw new EventMeshException(e); + } + } + + } + } + } + + private Map> queryTablePrimaryKey(String schema, List tables) throws SQLException { + Map> primaryKeys = new LinkedHashMap<>(); + String prepareTables = SqlUtils.genPrepareSqlOfInClause(tables.size()); + String sql = "select L.TABLE_NAME,L.COLUMN_NAME,R.CONSTRAINT_TYPE from " + + "INFORMATION_SCHEMA.KEY_COLUMN_USAGE L left join INFORMATION_SCHEMA.TABLE_CONSTRAINTS R on L" + + ".TABLE_SCHEMA = R.TABLE_SCHEMA and L.TABLE_NAME = R.TABLE_NAME and L.CONSTRAINT_CATALOG = R" + + ".CONSTRAINT_CATALOG and L.CONSTRAINT_SCHEMA = R.CONSTRAINT_SCHEMA and L.CONSTRAINT_NAME = R" + + ".CONSTRAINT_NAME where L.TABLE_SCHEMA = ? and L.TABLE_NAME in " + prepareTables + " and R" + + ".CONSTRAINT_TYPE IN ('PRIMARY KEY') order by L.ORDINAL_POSITION asc"; + try (Connection conn = dataSource.getConnection(); + PreparedStatement statement = conn.prepareStatement(sql)) { + statement.setString(1, schema); + SqlUtils.setInClauseParameters(statement, 2, tables); + try (ResultSet rs = statement.executeQuery()) { + if (rs == null) { + return null; + } + while (rs.next()) { + String tableName = rs.getString("TABLE_NAME"); + String colName = rs.getString("COLUMN_NAME"); + primaryKeys.compute(tableName, (k, v) -> { + if (v == null) { + v = new LinkedList<>(); + } + v.add(colName); + return v; + }); + } + } + } + return primaryKeys; + } + + private Map> queryColumns(String schema, List tables) throws SQLException { + String prepareTables = SqlUtils.genPrepareSqlOfInClause(tables.size()); + String sql = "select TABLE_SCHEMA,TABLE_NAME,COLUMN_NAME,IS_NULLABLE,DATA_TYPE,CHARACTER_MAXIMUM_LENGTH," + + "CHARACTER_OCTET_LENGTH,NUMERIC_SCALE,NUMERIC_PRECISION,DATETIME_PRECISION,CHARACTER_SET_NAME," + + "COLLATION_NAME,COLUMN_TYPE,COLUMN_DEFAULT,COLUMN_COMMENT,ORDINAL_POSITION,EXTRA from " + + "INFORMATION_SCHEMA.COLUMNS where TABLE_SCHEMA = ? and TABLE_NAME in " + prepareTables + " order by " + "ORDINAL_POSITION asc"; + Map> cols = new LinkedHashMap<>(); + Connection conn = null; + PreparedStatement statement = null; + ResultSet rs = null; + try { + conn = dataSource.getConnection(); + statement = conn.prepareStatement(sql); + statement.setString(1, schema); + SqlUtils.setInClauseParameters(statement, 2, tables); + rs = statement.executeQuery(); + if (rs == null) { + return null; + } + while (rs.next()) { + String dataType = rs.getString("DATA_TYPE"); + JDBCType jdbcType = SqlUtils.toJDBCType(dataType); + MySQLColumnDef col = new MySQLColumnDef(); + col.setJdbcType(jdbcType); + col.setType(CanalMySQLType.valueOfCode(dataType)); + String colName = rs.getString("COLUMN_NAME"); + col.setName(colName); + String tableName = rs.getString("TABLE_NAME"); + cols.compute(tableName, (k, v) -> { + if (v == null) { + v = new LinkedList<>(); + } + v.add(col); + return v; + }); + } + } catch (SQLException e) { + log.error("init rdb table schema [{}] tables fail", schema, e); + } finally { + if (rs != null) { + try { + rs.close(); + } catch (SQLException e) { + log.error("close result set fail", e); + } + } + if (statement != null) { + try { + statement.close(); + } catch (SQLException e) { + log.error("close prepare statement fail", e); + } + } + if (conn != null) { + try { + conn.close(); + } catch (SQLException e) { + log.error("close db connection fail", e); + } + } + } + return cols; + } + + @Override + protected void shutdown() throws Exception { + + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/template/AbstractSqlTemplate.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/template/AbstractSqlTemplate.java new file mode 100644 index 0000000000..10c647c8f1 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/template/AbstractSqlTemplate.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.template; + +/** + * implement SQL CRUD with standard SQL + */ +public abstract class AbstractSqlTemplate implements SqlTemplate { + + private static final String DOT = "."; + + public String getSelectSql(String schemaName, String tableName, String[] pkNames, String[] columnNames) { + StringBuilder sql = new StringBuilder("select "); + int size = columnNames.length; + for (int i = 0; i < size; i++) { + sql.append(appendEscape(columnNames[i])).append((i + 1 < size) ? " , " : ""); + } + + sql.append(" from ").append(getFullName(schemaName, tableName)).append(" where ( "); + appendColumnEquals(sql, pkNames, "and"); + sql.append(" ) "); + return sql.toString().intern(); + } + + public String getUpdateSql(String schemaName, String tableName, String[] pkNames, String[] columnNames, boolean updatePks, String shardColumn) { + StringBuilder sql = new StringBuilder("update " + getFullName(schemaName, tableName) + " set "); + appendExcludeSingleShardColumnEquals(sql, columnNames, ",", updatePks, shardColumn); + sql.append(" where ("); + appendColumnEquals(sql, pkNames, "and"); + sql.append(")"); + return sql.toString().intern(); + } + + public String getInsertSql(String schemaName, String tableName, String[] pkNames, String[] columnNames) { + StringBuilder sql = new StringBuilder("insert into " + getFullName(schemaName, tableName) + "("); + String[] allColumns = new String[pkNames.length + columnNames.length]; + System.arraycopy(columnNames, 0, allColumns, 0, columnNames.length); + System.arraycopy(pkNames, 0, allColumns, columnNames.length, pkNames.length); + + int size = allColumns.length; + for (int i = 0; i < size; i++) { + sql.append(appendEscape(allColumns[i])).append((i + 1 < size) ? "," : ""); + } + + sql.append(") values ("); + appendColumnQuestions(sql, allColumns); + sql.append(")"); + return sql.toString().intern(); + } + + public String getDeleteSql(String schemaName, String tableName, String[] pkNames) { + StringBuilder sql = new StringBuilder("delete from " + getFullName(schemaName, tableName) + " where "); + appendColumnEquals(sql, pkNames, "and"); + return sql.toString().intern(); + } + + protected String getFullName(String schemaName, String tableName) { + StringBuilder sb = new StringBuilder(); + if (schemaName != null) { + sb.append(appendEscape(schemaName)).append(DOT); + } + sb.append(appendEscape(tableName)); + return sb.toString().intern(); + } + + // ================ helper method ============ + + protected String appendEscape(String columnName) { + return columnName; + } + + protected void appendColumnQuestions(StringBuilder sql, String[] columns) { + int size = columns.length; + for (int i = 0; i < size; i++) { + sql.append("?").append((i + 1 < size) ? " , " : ""); + } + } + + protected void appendColumnEquals(StringBuilder sql, String[] columns, String separator) { + int size = columns.length; + for (int i = 0; i < size; i++) { + sql.append(" ").append(appendEscape(columns[i])).append(" = ").append("? "); + if (i != size - 1) { + sql.append(separator); + } + } + } + + protected void appendExcludeSingleShardColumnEquals(StringBuilder sql, String[] columns, String separator, boolean updatePks, + String excludeShardColumn) { + int size = columns.length; + for (int i = 0; i < size; i++) { + if (!updatePks && columns[i].equals(excludeShardColumn)) { + continue; + } + sql.append(" ").append(appendEscape(columns[i])).append(" = ").append("? "); + if (i != size - 1) { + sql.append(separator); + } + } + } +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/template/MysqlSqlTemplate.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/template/MysqlSqlTemplate.java new file mode 100644 index 0000000000..37b45c746f --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/template/MysqlSqlTemplate.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.template; + +public class MysqlSqlTemplate extends AbstractSqlTemplate { + + private static final String ESCAPE = "`"; + + public String getMergeSql(String schemaName, String tableName, String[] pkNames, String[] columnNames, + String[] viewColumnNames, boolean includePks, String shardColumn) { + StringBuilder sql = new StringBuilder("insert into " + getFullName(schemaName, tableName) + "("); + int size = columnNames.length; + for (int i = 0; i < size; i++) { + sql.append(appendEscape(columnNames[i])).append(" , "); + } + size = pkNames.length; + for (int i = 0; i < size; i++) { + sql.append(appendEscape(pkNames[i])).append((i + 1 < size) ? " , " : ""); + } + + sql.append(") values ("); + size = columnNames.length; + for (int i = 0; i < size; i++) { + sql.append("?").append(" , "); + } + size = pkNames.length; + for (int i = 0; i < size; i++) { + sql.append("?").append((i + 1 < size) ? " , " : ""); + } + sql.append(")"); + sql.append(" on duplicate key update "); + + size = columnNames.length; + for (int i = 0; i < size; i++) { + if (!includePks && columnNames[i].equals(shardColumn)) { + continue; + } + + sql.append(appendEscape(columnNames[i])) + .append("=values(") + .append(appendEscape(columnNames[i])) + .append(")"); + if (includePks) { + sql.append(" , "); + } else { + sql.append((i + 1 < size) ? " , " : ""); + } + } + + if (includePks) { + size = pkNames.length; + for (int i = 0; i < size; i++) { + sql.append(appendEscape(pkNames[i])).append("=values(").append(appendEscape(pkNames[i])).append(")"); + sql.append((i + 1 < size) ? " , " : ""); + } + } + + return sql.toString().intern(); + } + + protected String appendEscape(String columnName) { + return ESCAPE + columnName + ESCAPE; + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/template/SqlTemplate.java b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/template/SqlTemplate.java new file mode 100644 index 0000000000..5b92cac2eb --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/java/org/apache/eventmesh/connector/canal/template/SqlTemplate.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.canal.template; + +/** + * SqlTemplate + */ +public interface SqlTemplate { + + public String getSelectSql(String schemaName, String tableName, String[] pkNames, String[] columnNames); + + public String getUpdateSql(String schemaName, String tableName, String[] pkNames, String[] columnNames, boolean updatePks, String shardColumn); + + public String getDeleteSql(String schemaName, String tableName, String[] pkNames); + + public String getInsertSql(String schemaName, String tableName, String[] pkNames, String[] columnNames); + + public String getMergeSql(String schemaName, String tableName, String[] pkNames, String[] columnNames, + String[] viewColumnNames, boolean updatePks, String shardColumn); +} diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.openconnect.api.ConnectorCreateService b/eventmesh-connectors/eventmesh-connector-canal/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.openconnect.api.ConnectorCreateService new file mode 100644 index 0000000000..f55b34d852 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.openconnect.api.ConnectorCreateService @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +MySQL-Source=org.apache.eventmesh.connector.canal.source.connector.CanalSourceConnector +MySQL-Sink=org.apache.eventmesh.connector.canal.sink.connector.CanalSinkConnector diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/resources/server-config.yml b/eventmesh-connectors/eventmesh-connector-canal/src/main/resources/server-config.yml new file mode 100644 index 0000000000..5f66dd0f68 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/resources/server-config.yml @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +sourceEnable: true +sinkEnable: true diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/resources/sink-config.yml b/eventmesh-connectors/eventmesh-connector-canal/src/main/resources/sink-config.yml new file mode 100644 index 0000000000..210361dc28 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/resources/sink-config.yml @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +pubSubConfig: + meshAddress: 127.0.0.1:10000 + subject: TopicTest + idc: FT + env: PRD + group: rocketmqSink + appId: 5031 + userName: rocketmqSinkUser + passWord: rocketmqPassWord +connectorConfig: + connectorName: rocketmqSink + nameServer: 127.0.0.1:9876 + topic: TopicTest diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/main/resources/source-config.yml b/eventmesh-connectors/eventmesh-connector-canal/src/main/resources/source-config.yml new file mode 100644 index 0000000000..7a7880b877 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/main/resources/source-config.yml @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +pubSubConfig: + meshAddress: 127.0.0.1:10000 + subject: TopicTest + idc: FT + env: PRD + group: rocketmqSource + appId: 5032 + userName: rocketmqSourceUser + passWord: rocketmqPassWord +connectorConfig: + connectorName: rocketmqSource + nameserver: 127.0.0.1:9876 + topic: TopicTest + commitOffsetIntervalMs: 5000 +offsetStorageConfig: + offsetStorageType: nacos + offsetStorageAddr: 127.0.0.1:8848 + extensions: { + #same with topic + dataId: TopicTest, + #same with group + group: rocketmqSource + } diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/test/resources/sink-config.yml b/eventmesh-connectors/eventmesh-connector-canal/src/test/resources/sink-config.yml new file mode 100644 index 0000000000..210361dc28 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/test/resources/sink-config.yml @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +pubSubConfig: + meshAddress: 127.0.0.1:10000 + subject: TopicTest + idc: FT + env: PRD + group: rocketmqSink + appId: 5031 + userName: rocketmqSinkUser + passWord: rocketmqPassWord +connectorConfig: + connectorName: rocketmqSink + nameServer: 127.0.0.1:9876 + topic: TopicTest diff --git a/eventmesh-connectors/eventmesh-connector-canal/src/test/resources/source-config.yml b/eventmesh-connectors/eventmesh-connector-canal/src/test/resources/source-config.yml new file mode 100644 index 0000000000..7a7880b877 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-canal/src/test/resources/source-config.yml @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +pubSubConfig: + meshAddress: 127.0.0.1:10000 + subject: TopicTest + idc: FT + env: PRD + group: rocketmqSource + appId: 5032 + userName: rocketmqSourceUser + passWord: rocketmqPassWord +connectorConfig: + connectorName: rocketmqSource + nameserver: 127.0.0.1:9876 + topic: TopicTest + commitOffsetIntervalMs: 5000 +offsetStorageConfig: + offsetStorageType: nacos + offsetStorageAddr: 127.0.0.1:8848 + extensions: { + #same with topic + dataId: TopicTest, + #same with group + group: rocketmqSource + } diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/build.gradle b/eventmesh-connectors/eventmesh-connector-chatgpt/build.gradle index 7de61f3cbe..95b80d02e8 100644 --- a/eventmesh-connectors/eventmesh-connector-chatgpt/build.gradle +++ b/eventmesh-connectors/eventmesh-connector-chatgpt/build.gradle @@ -19,10 +19,10 @@ dependencies { api project(":eventmesh-openconnect:eventmesh-openconnect-java") implementation project(":eventmesh-common") implementation 'com.theokanning.openai-gpt3-java:service:0.18.2' - implementation 'io.cloudevents:cloudevents-http-vertx:2.3.0' + implementation 'io.cloudevents:cloudevents-http-vertx:3.0.0' implementation 'io.vertx:vertx-web:4.5.8' testImplementation "org.apache.httpcomponents:httpclient" compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' -} \ No newline at end of file +} diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/config/ChatGPTServerConfig.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/config/ChatGPTServerConfig.java index 7d162920d7..a8d026067e 100644 --- a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/config/ChatGPTServerConfig.java +++ b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/config/ChatGPTServerConfig.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.chatgpt.config; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/ChatGPTSourceConfig.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/ChatGPTSourceConfig.java index 9596866910..21ddd84dc6 100644 --- a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/ChatGPTSourceConfig.java +++ b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/config/ChatGPTSourceConfig.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.chatgpt.source.config; -import org.apache.eventmesh.openconnect.api.config.SourceConfig; +import org.apache.eventmesh.common.config.connector.SourceConfig; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/connector/ChatGPTSourceConnector.java b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/connector/ChatGPTSourceConnector.java index a947bc135d..1b6955feb2 100644 --- a/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/connector/ChatGPTSourceConnector.java +++ b/eventmesh-connectors/eventmesh-connector-chatgpt/src/main/java/org/apache/eventmesh/connector/chatgpt/source/connector/ChatGPTSourceConnector.java @@ -18,6 +18,7 @@ package org.apache.eventmesh.connector.chatgpt.source.connector; import org.apache.eventmesh.common.ThreadPoolFactory; +import org.apache.eventmesh.common.config.connector.Config; import org.apache.eventmesh.common.exception.EventMeshException; import org.apache.eventmesh.connector.chatgpt.source.config.ChatGPTSourceConfig; import org.apache.eventmesh.connector.chatgpt.source.dto.ChatGPTRequestDTO; @@ -25,7 +26,6 @@ import org.apache.eventmesh.connector.chatgpt.source.handlers.ChatHandler; import org.apache.eventmesh.connector.chatgpt.source.handlers.ParseHandler; import org.apache.eventmesh.connector.chatgpt.source.managers.OpenaiManager; -import org.apache.eventmesh.openconnect.api.config.Config; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; import org.apache.eventmesh.openconnect.api.source.Source; @@ -61,8 +61,6 @@ @Slf4j public class ChatGPTSourceConnector implements Source { - private static final int DEFAULT_BATCH_SIZE = 10; - private ChatGPTSourceConfig sourceConfig; private BlockingQueue queue; private HttpServer server; @@ -79,6 +77,9 @@ public class ChatGPTSourceConnector implements Source { private static final String APPLICATION_JSON = "application/json"; private static final String TEXT_PLAIN = "text/plain"; + private int maxBatchSize; + private long maxPollWaitTime; + @Override public Class configClass() { @@ -129,7 +130,9 @@ private void doInit() { if (StringUtils.isNotEmpty(parsePromptTemplateStr)) { this.parseHandler = new ParseHandler(openaiManager, parsePromptTemplateStr); } - this.queue = new LinkedBlockingQueue<>(1024); + this.maxBatchSize = sourceConfig.getPollConfig().getMaxBatchSize(); + this.maxPollWaitTime = sourceConfig.getPollConfig().getMaxWaitTime(); + this.queue = new LinkedBlockingQueue<>(sourceConfig.getPollConfig().getCapacity()); final Vertx vertx = Vertx.vertx(); final Router router = Router.router(vertx); router.route().path(this.sourceConfig.connectorConfig.getPath()).method(HttpMethod.POST).handler(BodyHandler.create()).handler(ctx -> { @@ -224,6 +227,11 @@ public String name() { return this.sourceConfig.getConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() { Throwable t = this.server.close().cause(); @@ -234,14 +242,21 @@ public void stop() { @Override public List poll() { - List connectRecords = new ArrayList<>(DEFAULT_BATCH_SIZE); - for (int i = 0; i < DEFAULT_BATCH_SIZE; i++) { + long startTime = System.currentTimeMillis(); + long remainingTime = maxPollWaitTime; + + List connectRecords = new ArrayList<>(maxBatchSize); + for (int i = 0; i < maxBatchSize; i++) { try { - CloudEvent event = queue.poll(3, TimeUnit.SECONDS); + CloudEvent event = queue.poll(remainingTime, TimeUnit.MILLISECONDS); if (event == null) { break; } connectRecords.add(CloudEventUtil.convertEventToRecord(event)); + + // calculate elapsed time and update remaining time for next poll + long elapsedTime = System.currentTimeMillis() - startTime; + remainingTime = maxPollWaitTime > elapsedTime ? maxPollWaitTime - elapsedTime : 0; } catch (InterruptedException e) { break; } diff --git a/eventmesh-connectors/eventmesh-connector-dingtalk/build.gradle b/eventmesh-connectors/eventmesh-connector-dingtalk/build.gradle index 46d16c3b35..dfe1a40e95 100644 --- a/eventmesh-connectors/eventmesh-connector-dingtalk/build.gradle +++ b/eventmesh-connectors/eventmesh-connector-dingtalk/build.gradle @@ -25,7 +25,7 @@ dependencies { implementation project(":eventmesh-common") implementation project(":eventmesh-sdks:eventmesh-sdk-java") implementation project(":eventmesh-openconnect:eventmesh-openconnect-java") - implementation "com.aliyun:dingtalk:2.0.61" + implementation "com.aliyun:dingtalk:2.1.27" implementation 'com.google.guava:guava' compileOnly 'org.projectlombok:lombok' diff --git a/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/config/DingDingConnectServerConfig.java b/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/config/DingDingConnectServerConfig.java index ec744922c2..d9657bc23b 100644 --- a/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/config/DingDingConnectServerConfig.java +++ b/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/config/DingDingConnectServerConfig.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.dingtalk.config; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/sink/connector/DingDingSinkConnector.java b/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/sink/connector/DingDingSinkConnector.java index 754461875f..8c5a1e6611 100644 --- a/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/sink/connector/DingDingSinkConnector.java +++ b/eventmesh-connectors/eventmesh-connector-dingtalk/src/main/java/org/apache/eventmesh/connector/dingtalk/sink/connector/DingDingSinkConnector.java @@ -17,11 +17,11 @@ package org.apache.eventmesh.connector.dingtalk.sink.connector; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.dingtalk.DingDingSinkConfig; import org.apache.eventmesh.common.utils.JsonUtils; import org.apache.eventmesh.connector.dingtalk.common.constants.ConnectRecordExtensionKeys; import org.apache.eventmesh.connector.dingtalk.config.DingDingMessageTemplateType; -import org.apache.eventmesh.connector.dingtalk.sink.config.DingDingSinkConfig; -import org.apache.eventmesh.openconnect.api.config.Config; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; import org.apache.eventmesh.openconnect.api.sink.Sink; @@ -103,6 +103,11 @@ public String name() { return this.sinkConfig.getSinkConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() { isRunning = false; diff --git a/eventmesh-connectors/eventmesh-connector-dingtalk/src/test/java/org/apache/eventmesh/connector/dingtalk/sink/connector/DingDingSinkConnectorTest.java b/eventmesh-connectors/eventmesh-connector-dingtalk/src/test/java/org/apache/eventmesh/connector/dingtalk/sink/connector/DingDingSinkConnectorTest.java index 38a136748c..736137629d 100644 --- a/eventmesh-connectors/eventmesh-connector-dingtalk/src/test/java/org/apache/eventmesh/connector/dingtalk/sink/connector/DingDingSinkConnectorTest.java +++ b/eventmesh-connectors/eventmesh-connector-dingtalk/src/test/java/org/apache/eventmesh/connector/dingtalk/sink/connector/DingDingSinkConnectorTest.java @@ -21,12 +21,10 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import org.apache.eventmesh.common.config.connector.dingtalk.DingDingSinkConfig; import org.apache.eventmesh.connector.dingtalk.common.constants.ConnectRecordExtensionKeys; import org.apache.eventmesh.connector.dingtalk.config.DingDingMessageTemplateType; -import org.apache.eventmesh.connector.dingtalk.sink.config.DingDingSinkConfig; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; import org.apache.eventmesh.openconnect.util.ConfigUtil; import java.lang.reflect.Field; @@ -90,9 +88,7 @@ public void testSendMessageToDingDing() throws Exception { final int times = 3; List records = new ArrayList<>(); for (int i = 0; i < times; i++) { - RecordPartition partition = new RecordPartition(); - RecordOffset offset = new RecordOffset(); - ConnectRecord connectRecord = new ConnectRecord(partition, offset, + ConnectRecord connectRecord = new ConnectRecord(null, null, System.currentTimeMillis(), "Hello, EventMesh!".getBytes(StandardCharsets.UTF_8)); connectRecord.addExtension(ConnectRecordExtensionKeys.DINGTALK_TEMPLATE_TYPE, DingDingMessageTemplateType.PLAIN_TEXT.getTemplateType()); diff --git a/eventmesh-connectors/eventmesh-connector-file/build.gradle b/eventmesh-connectors/eventmesh-connector-file/build.gradle index 11eaa5b7e0..ea441248b8 100644 --- a/eventmesh-connectors/eventmesh-connector-file/build.gradle +++ b/eventmesh-connectors/eventmesh-connector-file/build.gradle @@ -16,9 +16,10 @@ */ dependencies { + implementation project(":eventmesh-common") api project(":eventmesh-openconnect:eventmesh-openconnect-java") - testImplementation('org.junit.jupiter:junit-jupiter') - testImplementation('org.mockito:mockito-junit-jupiter') + testImplementation 'org.junit.jupiter:junit-jupiter' + testImplementation 'org.mockito:mockito-junit-jupiter' compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' } \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/config/FileServerConfig.java b/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/config/FileServerConfig.java index b6b506521e..ad25b8a410 100644 --- a/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/config/FileServerConfig.java +++ b/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/config/FileServerConfig.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.file.config; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/sink/connector/FileSinkConnector.java b/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/sink/connector/FileSinkConnector.java index f1d1ccc57b..fabae0d43a 100644 --- a/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/sink/connector/FileSinkConnector.java +++ b/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/sink/connector/FileSinkConnector.java @@ -17,8 +17,8 @@ package org.apache.eventmesh.connector.file.sink.connector; -import org.apache.eventmesh.connector.file.sink.config.FileSinkConfig; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.file.FileSinkConfig; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; import org.apache.eventmesh.openconnect.api.sink.Sink; @@ -103,6 +103,11 @@ public String name() { return this.sinkConfig.getConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() { outputStream.flush(); diff --git a/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/source/connector/FileSourceConnector.java b/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/source/connector/FileSourceConnector.java index 2b55284741..68b1a50989 100644 --- a/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/source/connector/FileSourceConnector.java +++ b/eventmesh-connectors/eventmesh-connector-file/src/main/java/org/apache/eventmesh/connector/file/source/connector/FileSourceConnector.java @@ -17,13 +17,14 @@ package org.apache.eventmesh.connector.file.source.connector; -import org.apache.eventmesh.connector.file.source.config.FileSourceConfig; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.file.FileSourceConfig; +import org.apache.eventmesh.common.remote.offset.RecordPartition; +import org.apache.eventmesh.common.remote.offset.file.FileRecordPartition; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; import org.apache.eventmesh.openconnect.api.source.Source; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; import java.io.BufferedReader; import java.io.File; @@ -34,9 +35,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import lombok.extern.slf4j.Slf4j; @@ -87,6 +86,11 @@ public String name() { return this.sourceConfig.getConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() { try { @@ -118,9 +122,9 @@ public List poll() { } public static RecordPartition convertToRecordPartition(String fileName) { - Map map = new HashMap<>(); - map.put("fileName", fileName); - return new RecordPartition(map); + FileRecordPartition fileRecordPartition = new FileRecordPartition(); + fileRecordPartition.setFileName(fileName); + return fileRecordPartition; } private static String getFileName(String filePath) throws NullPointerException { diff --git a/eventmesh-connectors/eventmesh-connector-file/src/test/java/org/apache/eventmesh/connector/file/FileSourceConnectorTest.java b/eventmesh-connectors/eventmesh-connector-file/src/test/java/org/apache/eventmesh/connector/file/FileSourceConnectorTest.java index 49421dd077..9cfea3cc59 100644 --- a/eventmesh-connectors/eventmesh-connector-file/src/test/java/org/apache/eventmesh/connector/file/FileSourceConnectorTest.java +++ b/eventmesh-connectors/eventmesh-connector-file/src/test/java/org/apache/eventmesh/connector/file/FileSourceConnectorTest.java @@ -20,8 +20,8 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import org.apache.eventmesh.connector.file.source.config.FileSourceConfig; -import org.apache.eventmesh.connector.file.source.config.SourceConnectorConfig; +import org.apache.eventmesh.common.config.connector.file.FileSourceConfig; +import org.apache.eventmesh.common.config.connector.file.SourceConnectorConfig; import org.apache.eventmesh.connector.file.source.connector.FileSourceConnector; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; diff --git a/eventmesh-connectors/eventmesh-connector-http/build.gradle b/eventmesh-connectors/eventmesh-connector-http/build.gradle index 11e0d78023..48c7aecd06 100644 --- a/eventmesh-connectors/eventmesh-connector-http/build.gradle +++ b/eventmesh-connectors/eventmesh-connector-http/build.gradle @@ -21,12 +21,13 @@ dependencies { implementation 'io.cloudevents:cloudevents-http-vertx:3.0.0' implementation 'io.vertx:vertx-web:4.5.8' - implementation 'io.vertx:vertx-web-client:4.5.8' + implementation 'io.vertx:vertx-web-client:4.5.9' implementation 'dev.failsafe:failsafe:3.3.2' - testImplementation "org.apache.httpcomponents:httpclient" + + testImplementation 'org.apache.httpcomponents.client5:httpclient5:5.4' + testImplementation 'org.apache.httpcomponents.client5:httpclient5-fluent:5.4' testImplementation 'org.mock-server:mockserver-netty:5.15.0' - testImplementation 'com.squareup.okhttp3:okhttp:4.12.0' compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' -} \ No newline at end of file +} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/common/SynchronizedCircularFifoQueue.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/common/SynchronizedCircularFifoQueue.java new file mode 100644 index 0000000000..0564e58734 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/common/SynchronizedCircularFifoQueue.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.http.common; + +import org.apache.commons.collections4.queue.CircularFifoQueue; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; + + +/** + * SynchronizedCircularFifoQueue is a synchronized version of CircularFifoQueue. + */ +public class SynchronizedCircularFifoQueue extends CircularFifoQueue { + + /** + *

Default constructor. capacity = 32

+ */ + public SynchronizedCircularFifoQueue() { + super(); + } + + public SynchronizedCircularFifoQueue(Collection coll) { + super(coll); + } + + public SynchronizedCircularFifoQueue(int size) { + super(size); + } + + @Override + public synchronized boolean add(E element) { + return super.add(element); + } + + @Override + public synchronized void clear() { + super.clear(); + } + + @Override + public synchronized E element() { + return super.element(); + } + + @Override + public synchronized E get(int index) { + return super.get(index); + } + + @Override + public synchronized boolean isAtFullCapacity() { + return super.isAtFullCapacity(); + } + + @Override + public synchronized boolean isEmpty() { + return super.isEmpty(); + } + + @Override + public synchronized boolean isFull() { + return super.isFull(); + } + + @Override + public synchronized int maxSize() { + return super.maxSize(); + } + + @Override + public synchronized boolean offer(E element) { + return super.offer(element); + } + + @Override + public synchronized E peek() { + return super.peek(); + } + + @Override + public synchronized E poll() { + return super.poll(); + } + + @Override + public synchronized E remove() { + return super.remove(); + } + + @Override + public synchronized int size() { + return super.size(); + } + + /** + *

Fetch a range of elements from the queue.

+ * + * @param start start index + * @param end end index + * @param removed whether to remove the elements from the queue + * @return list of elements + */ + public synchronized List fetchRange(int start, int end, boolean removed) { + + if (start < 0 || start > end) { + throw new IllegalArgumentException("Invalid range"); + } + end = Math.min(end, this.size()); + + Iterator iterator = this.iterator(); + List items = new ArrayList<>(end - start); + + int count = 0; + while (iterator.hasNext() && count < end) { + E item = iterator.next(); + if (item != null && count >= start) { + // Add the element to the list + items.add(item); + if (removed) { + // Remove the element from the queue + iterator.remove(); + } + } + count++; + } + return items; + + } + + +} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/config/HttpServerConfig.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/config/HttpServerConfig.java index 81a9f20923..8517b869fd 100644 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/config/HttpServerConfig.java +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/config/HttpServerConfig.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.http.config; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/server/HttpConnectServer.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/server/HttpConnectServer.java index 8d753d2815..dbe0838bc8 100644 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/server/HttpConnectServer.java +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/server/HttpConnectServer.java @@ -19,7 +19,7 @@ import org.apache.eventmesh.connector.http.config.HttpServerConfig; import org.apache.eventmesh.connector.http.sink.HttpSinkConnector; -import org.apache.eventmesh.connector.http.source.connector.HttpSourceConnector; +import org.apache.eventmesh.connector.http.source.HttpSourceConnector; import org.apache.eventmesh.openconnect.Application; import org.apache.eventmesh.openconnect.util.ConfigUtil; diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/HttpSinkConnector.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/HttpSinkConnector.java index 23d09fa141..8e808ccc93 100644 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/HttpSinkConnector.java +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/HttpSinkConnector.java @@ -17,13 +17,15 @@ package org.apache.eventmesh.connector.http.sink; -import org.apache.eventmesh.connector.http.sink.config.HttpSinkConfig; -import org.apache.eventmesh.connector.http.sink.config.SinkConnectorConfig; -import org.apache.eventmesh.connector.http.sink.handle.CommonHttpSinkHandler; -import org.apache.eventmesh.connector.http.sink.handle.HttpSinkHandler; -import org.apache.eventmesh.connector.http.sink.handle.RetryHttpSinkHandler; -import org.apache.eventmesh.connector.http.sink.handle.WebhookHttpSinkHandler; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.EventMeshThreadFactory; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.http.HttpSinkConfig; +import org.apache.eventmesh.common.config.connector.http.SinkConnectorConfig; +import org.apache.eventmesh.connector.http.sink.handler.HttpSinkHandler; +import org.apache.eventmesh.connector.http.sink.handler.impl.CommonHttpSinkHandler; +import org.apache.eventmesh.connector.http.sink.handler.impl.HttpSinkHandlerRetryWrapper; +import org.apache.eventmesh.connector.http.sink.handler.impl.WebhookHttpSinkHandler; +import org.apache.eventmesh.openconnect.api.ConnectorCreateService; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; import org.apache.eventmesh.openconnect.api.sink.Sink; @@ -31,24 +33,39 @@ import java.util.List; import java.util.Objects; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import lombok.Getter; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @Slf4j -public class HttpSinkConnector implements Sink { +public class HttpSinkConnector implements Sink, ConnectorCreateService { private HttpSinkConfig httpSinkConfig; @Getter private HttpSinkHandler sinkHandler; + private ThreadPoolExecutor executor; + + private final LinkedBlockingQueue queue = new LinkedBlockingQueue<>(10000); + + private final AtomicBoolean isStart = new AtomicBoolean(true); + @Override public Class configClass() { return HttpSinkConfig.class; } + @Override + public Sink create() { + return new HttpSinkConnector(); + } + @Override public void init(Config config) throws Exception { this.httpSinkConfig = (HttpSinkConfig) config; @@ -80,15 +97,34 @@ private void doInit() { this.sinkHandler = nonRetryHandler; } else if (maxRetries > 0) { // Wrap the sink handler with a retry handler - this.sinkHandler = new RetryHttpSinkHandler(this.httpSinkConfig.connectorConfig, nonRetryHandler); + this.sinkHandler = new HttpSinkHandlerRetryWrapper(this.httpSinkConfig.connectorConfig, nonRetryHandler); } else { throw new IllegalArgumentException("Max retries must be greater than or equal to 0."); } + boolean isParallelized = this.httpSinkConfig.connectorConfig.isParallelized(); + int parallelism = isParallelized ? this.httpSinkConfig.connectorConfig.getParallelism() : 1; + executor = new ThreadPoolExecutor(parallelism, parallelism, 0L, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>(), new EventMeshThreadFactory("http-sink-handler")); } @Override public void start() throws Exception { this.sinkHandler.start(); + for (int i = 0; i < this.httpSinkConfig.connectorConfig.getParallelism(); i++) { + executor.execute(() -> { + while (isStart.get()) { + ConnectRecord connectRecord = null; + try { + connectRecord = queue.poll(2, TimeUnit.SECONDS); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + if (connectRecord != null) { + sinkHandler.handle(connectRecord); + } + } + }); + } } @Override @@ -101,9 +137,25 @@ public String name() { return this.httpSinkConfig.connectorConfig.getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() throws Exception { + isStart.set(false); + while (!queue.isEmpty()) { + ConnectRecord record = queue.poll(); + this.sinkHandler.handle(record); + } + try { + Thread.sleep(50); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } this.sinkHandler.stop(); + log.info("All tasks completed, start shut down http sink connector"); } @Override @@ -114,8 +166,7 @@ public void put(List sinkRecords) { log.warn("ConnectRecord data is null, ignore."); continue; } - // Handle the ConnectRecord - this.sinkHandler.handle(sinkRecord); + queue.put(sinkRecord); } catch (Exception e) { log.error("Failed to sink message via HTTP. ", e); } diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpAttemptEvent.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpAttemptEvent.java new file mode 100644 index 0000000000..8163852f8f --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpAttemptEvent.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.http.sink.data; + +import java.util.concurrent.atomic.AtomicInteger; + +/** + * Single HTTP attempt event + */ +public class HttpAttemptEvent { + + public static final String PREFIX = "http-attempt-event-"; + + private final int maxAttempts; + + private final AtomicInteger attempts; + + private Throwable lastException; + + + public HttpAttemptEvent(int maxAttempts) { + this.maxAttempts = maxAttempts; + this.attempts = new AtomicInteger(0); + } + + /** + * Increment the attempts + */ + public void incrementAttempts() { + attempts.incrementAndGet(); + } + + /** + * Update the event, incrementing the attempts and setting the last exception + * + * @param exception the exception to update, can be null + */ + public void updateEvent(Throwable exception) { + // increment the attempts + incrementAttempts(); + + // update the last exception + lastException = exception; + } + + /** + * Check if the attempts are less than the maximum attempts + * + * @return true if the attempts are less than the maximum attempts, false otherwise + */ + public boolean canAttempt() { + return attempts.get() < maxAttempts; + } + + public boolean isComplete() { + if (attempts.get() == 0) { + // No start yet + return false; + } + + // If no attempt can be made or the last exception is null, the event completed + return !canAttempt() || lastException == null; + } + + + public int getMaxAttempts() { + return maxAttempts; + } + + public int getAttempts() { + return attempts.get(); + } + + public Throwable getLastException() { + return lastException; + } + + /** + * Get the limited exception message with the default limit of 256 + * + * @return the limited exception message + */ + public String getLimitedExceptionMessage() { + return getLimitedExceptionMessage(256); + } + + /** + * Get the limited exception message with the specified limit + * + * @param maxLimit the maximum limit of the exception message + * @return the limited exception message + */ + public String getLimitedExceptionMessage(int maxLimit) { + if (lastException == null) { + return ""; + } + String message = lastException.getMessage(); + if (message == null) { + return ""; + } + if (message.length() > maxLimit) { + return message.substring(0, maxLimit); + } + return message; + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpConnectRecord.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpConnectRecord.java index 1bfd223079..9c8b1ce673 100644 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpConnectRecord.java +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpConnectRecord.java @@ -17,31 +17,64 @@ package org.apache.eventmesh.connector.http.sink.data; +import org.apache.eventmesh.common.remote.offset.http.HttpRecordOffset; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.KeyValue; +import java.io.Serializable; import java.time.LocalDateTime; +import java.util.Base64; +import java.util.HashMap; import java.util.Map; import java.util.UUID; import lombok.Builder; -import lombok.Data; +import lombok.Getter; /** * a special ConnectRecord for HttpSinkConnector */ -@Data +@Getter @Builder -public class HttpConnectRecord { +public class HttpConnectRecord implements Serializable { - private String type; + private static final long serialVersionUID = 5271462532332251473L; + + /** + * The unique identifier for the HttpConnectRecord + */ + private final String httpRecordId = UUID.randomUUID().toString(); - private String time; + /** + * The time when the HttpConnectRecord was created + */ + private LocalDateTime createTime; - private String uuid; + /** + * The type of the HttpConnectRecord + */ + private String type; + /** + * The event id of the HttpConnectRecord + */ private String eventId; - private ConnectRecord data; + private Object data; + + private KeyValue extensions; + + @Override + public String toString() { + return "HttpConnectRecord{" + + "createTime=" + createTime + + ", httpRecordId='" + httpRecordId + + ", type='" + type + + ", eventId='" + eventId + + ", data=" + data + + ", extensions=" + extensions + + '}'; + } /** * Convert ConnectRecord to HttpConnectRecord @@ -50,18 +83,36 @@ public class HttpConnectRecord { * @return the converted HttpConnectRecord */ public static HttpConnectRecord convertConnectRecord(ConnectRecord record, String type) { - Map offsetMap = record.getPosition().getOffset().getOffset(); + Map offsetMap = new HashMap<>(); + if (record != null && record.getPosition() != null && record.getPosition().getRecordOffset() != null) { + if (HttpRecordOffset.class.equals(record.getPosition().getRecordOffsetClazz())) { + offsetMap = ((HttpRecordOffset) record.getPosition().getRecordOffset()).getOffsetMap(); + } + } String offset = "0"; if (!offsetMap.isEmpty()) { offset = offsetMap.values().iterator().next().toString(); } - return HttpConnectRecord.builder() - .type(type) - .time(LocalDateTime.now().toString()) - .uuid(UUID.randomUUID().toString()) - .eventId(type + "-" + offset) - .data(record) - .build(); + if (record.getData() instanceof byte[]) { + String data = Base64.getEncoder().encodeToString((byte[]) record.getData()); + record.addExtension("isBase64", true); + return HttpConnectRecord.builder() + .type(type) + .createTime(LocalDateTime.now()) + .eventId(type + "-" + offset) + .data(data) + .extensions(record.getExtensions()) + .build(); + } else { + record.addExtension("isBase64", false); + return HttpConnectRecord.builder() + .type(type) + .createTime(LocalDateTime.now()) + .eventId(type + "-" + offset) + .data(record.getData()) + .extensions(record.getExtensions()) + .build(); + } } } diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpExportMetadata.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpExportMetadata.java index 848012f152..111ee6b3e9 100644 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpExportMetadata.java +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpExportMetadata.java @@ -17,6 +17,7 @@ package org.apache.eventmesh.connector.http.sink.data; +import java.io.Serializable; import java.time.LocalDateTime; import lombok.Builder; @@ -27,7 +28,10 @@ */ @Data @Builder -public class HttpExportMetadata { +public class HttpExportMetadata implements Serializable { + + private static final long serialVersionUID = 1121010466793041920L; + private String url; private int code; @@ -36,7 +40,7 @@ public class HttpExportMetadata { private LocalDateTime receivedTime; - private String uuid; + private String recordId; private String retriedBy; diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpExportRecord.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpExportRecord.java index b6382aee7a..c6bdb02884 100644 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpExportRecord.java +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpExportRecord.java @@ -17,6 +17,8 @@ package org.apache.eventmesh.connector.http.sink.data; +import java.io.Serializable; + import lombok.AllArgsConstructor; import lombok.Data; @@ -25,7 +27,9 @@ */ @Data @AllArgsConstructor -public class HttpExportRecord { +public class HttpExportRecord implements Serializable { + + private static final long serialVersionUID = 6010283911452947157L; private HttpExportMetadata metadata; diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpExportRecordPage.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpExportRecordPage.java index 5c44eb3b7f..81e582c33a 100644 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpExportRecordPage.java +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/HttpExportRecordPage.java @@ -17,6 +17,7 @@ package org.apache.eventmesh.connector.http.sink.data; +import java.io.Serializable; import java.util.List; import lombok.AllArgsConstructor; @@ -27,7 +28,9 @@ */ @Data @AllArgsConstructor -public class HttpExportRecordPage { +public class HttpExportRecordPage implements Serializable { + + private static final long serialVersionUID = 1143791658357035990L; private int pageNum; diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/MultiHttpRequestContext.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/MultiHttpRequestContext.java new file mode 100644 index 0000000000..66f5d0e7ec --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/data/MultiHttpRequestContext.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.http.sink.data; + +import java.util.concurrent.atomic.AtomicInteger; + + +/** + * Multi HTTP request context + */ +public class MultiHttpRequestContext { + + public static final String NAME = "multi-http-request-context"; + + /** + * The remaining requests to be processed. + */ + private final AtomicInteger remainingRequests; + + /** + * The last failed event. + * If retries occur but still fail, it will be logged, and only the last one will be retained. + */ + private HttpAttemptEvent lastFailedEvent; + + public MultiHttpRequestContext(int remainingEvents) { + this.remainingRequests = new AtomicInteger(remainingEvents); + } + + /** + * Decrement the remaining requests by 1. + */ + public void decrementRemainingRequests() { + remainingRequests.decrementAndGet(); + } + + /** + * Check if all requests have been processed. + * + * @return true if all requests have been processed, false otherwise. + */ + public boolean isAllRequestsProcessed() { + return remainingRequests.get() == 0; + } + + public int getRemainingRequests() { + return remainingRequests.get(); + } + + public HttpAttemptEvent getLastFailedEvent() { + return lastFailedEvent; + } + + public void setLastFailedEvent(HttpAttemptEvent lastFailedEvent) { + this.lastFailedEvent = lastFailedEvent; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handle/CommonHttpSinkHandler.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handle/CommonHttpSinkHandler.java deleted file mode 100644 index e21046c4d2..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handle/CommonHttpSinkHandler.java +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.sink.handle; - -import org.apache.eventmesh.connector.http.sink.config.SinkConnectorConfig; -import org.apache.eventmesh.connector.http.sink.data.HttpConnectRecord; -import org.apache.eventmesh.connector.http.util.HttpUtils; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.net.URI; -import java.util.Arrays; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import io.netty.handler.codec.http.HttpHeaderNames; -import io.vertx.core.Future; -import io.vertx.core.MultiMap; -import io.vertx.core.Vertx; -import io.vertx.core.buffer.Buffer; -import io.vertx.core.http.HttpHeaders; -import io.vertx.ext.web.client.HttpResponse; -import io.vertx.ext.web.client.WebClient; -import io.vertx.ext.web.client.WebClientOptions; - -import lombok.Getter; -import lombok.extern.slf4j.Slf4j; - -/** - * Common HTTP/HTTPS Sink Handler implementation to handle ConnectRecords by sending them over HTTP or HTTPS to configured URLs. - * - *

This handler initializes a WebClient for making HTTP requests based on the provided SinkConnectorConfig. - * It handles processing ConnectRecords by converting them to HttpConnectRecord and sending them asynchronously to each configured URL using the - * WebClient.

- * - *

The handler uses Vert.x's WebClient to perform HTTP/HTTPS requests. It initializes the WebClient in the {@link #start()} - * method and closes it in the {@link #stop()} method to manage resources efficiently.

- * - *

Each ConnectRecord is processed and sent to all configured URLs concurrently using asynchronous HTTP requests.

- */ -@Slf4j -@Getter -public class CommonHttpSinkHandler implements HttpSinkHandler { - - private final SinkConnectorConfig connectorConfig; - - private final List urls; - - private WebClient webClient; - - - public CommonHttpSinkHandler(SinkConnectorConfig sinkConnectorConfig) { - this.connectorConfig = sinkConnectorConfig; - // Initialize URLs - String[] urlStrings = sinkConnectorConfig.getUrls(); - this.urls = Arrays.stream(urlStrings) - .map(URI::create) - .collect(Collectors.toList()); - } - - /** - * Initializes the WebClient for making HTTP requests based on the provided SinkConnectorConfig. - */ - @Override - public void start() { - // Create WebClient - doInitWebClient(); - } - - /** - * Initializes the WebClient with the provided configuration options. - */ - private void doInitWebClient() { - final Vertx vertx = Vertx.vertx(); - WebClientOptions options = new WebClientOptions() - .setKeepAlive(this.connectorConfig.isKeepAlive()) - .setKeepAliveTimeout(this.connectorConfig.getKeepAliveTimeout() / 1000) - .setIdleTimeout(this.connectorConfig.getIdleTimeout()) - .setIdleTimeoutUnit(TimeUnit.MILLISECONDS) - .setConnectTimeout(this.connectorConfig.getConnectionTimeout()) - .setMaxPoolSize(this.connectorConfig.getMaxConnectionPoolSize()); - this.webClient = WebClient.create(vertx, options); - } - - /** - * Processes a ConnectRecord by sending it over HTTP or HTTPS. This method should be called for each ConnectRecord that needs to be processed. - * - * @param record the ConnectRecord to process - */ - @Override - public void handle(ConnectRecord record) { - for (URI url : this.urls) { - // convert ConnectRecord to HttpConnectRecord - String type = String.format("%s.%s.%s", connectorConfig.getConnectorName(), url.getScheme(), "common"); - HttpConnectRecord httpConnectRecord = HttpConnectRecord.convertConnectRecord(record, type); - deliver(url, httpConnectRecord); - } - } - - - /** - * Processes HttpConnectRecord on specified URL while returning its own processing logic. - * This method sends the HttpConnectRecord to the specified URL using the WebClient. - * - * @param url URI to which the HttpConnectRecord should be sent - * @param httpConnectRecord HttpConnectRecord to process - * @return processing chain - */ - @Override - public Future> deliver(URI url, HttpConnectRecord httpConnectRecord) { - // create headers - MultiMap headers = HttpHeaders.headers() - .set(HttpHeaderNames.CONTENT_TYPE, "application/json; charset=utf-8") - .set(HttpHeaderNames.ACCEPT, "application/json; charset=utf-8"); - - // get timestamp and offset - Long timestamp = httpConnectRecord.getData().getTimestamp(); - Map offset = httpConnectRecord.getData().getPosition().getOffset().getOffset(); - - // send the request - return this.webClient.post(url.getPath()) - .host(url.getHost()) - .port(url.getPort() == -1 ? (Objects.equals(url.getScheme(), "https") ? 443 : 80) : url.getPort()) - .putHeaders(headers) - .ssl(Objects.equals(url.getScheme(), "https")) - .sendJson(httpConnectRecord) - .onSuccess(res -> { - log.info("Request sent successfully. Record: timestamp={}, offset={}", timestamp, offset); - // log the response - if (HttpUtils.is2xxSuccessful(res.statusCode())) { - if (log.isDebugEnabled()) { - log.debug("Received successful response: statusCode={}. Record: timestamp={}, offset={}, responseBody={}", - res.statusCode(), timestamp, offset, res.bodyAsString()); - } else { - log.info("Received successful response: statusCode={}. Record: timestamp={}, offset={}", res.statusCode(), timestamp, offset); - } - } else { - if (log.isDebugEnabled()) { - log.warn("Received non-2xx response: statusCode={}. Record: timestamp={}, offset={}, responseBody={}", - res.statusCode(), timestamp, offset, res.bodyAsString()); - } else { - log.warn("Received non-2xx response: statusCode={}. Record: timestamp={}, offset={}", res.statusCode(), timestamp, offset); - } - } - - }) - .onFailure(err -> log.error("Request failed to send. Record: timestamp={}, offset={}", timestamp, offset, err)); - } - - - /** - * Cleans up and releases resources used by the HTTP/HTTPS handler. - */ - @Override - public void stop() { - if (this.webClient != null) { - this.webClient.close(); - } else { - log.warn("WebClient is null, ignore."); - } - } - - -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handle/RetryHttpSinkHandler.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handle/RetryHttpSinkHandler.java deleted file mode 100644 index 06700261d5..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handle/RetryHttpSinkHandler.java +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.sink.handle; - -import org.apache.eventmesh.connector.http.sink.config.HttpRetryConfig; -import org.apache.eventmesh.connector.http.sink.config.SinkConnectorConfig; -import org.apache.eventmesh.connector.http.sink.data.HttpConnectRecord; -import org.apache.eventmesh.connector.http.sink.data.HttpExportMetadata; -import org.apache.eventmesh.connector.http.sink.data.HttpExportRecord; -import org.apache.eventmesh.connector.http.util.HttpUtils; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; - -import java.net.ConnectException; -import java.net.URI; -import java.time.Duration; -import java.time.LocalDateTime; -import java.util.Arrays; -import java.util.List; -import java.util.UUID; -import java.util.stream.Collectors; - -import io.vertx.core.Future; -import io.vertx.core.buffer.Buffer; -import io.vertx.ext.web.client.HttpResponse; - -import lombok.extern.slf4j.Slf4j; - -import dev.failsafe.Failsafe; -import dev.failsafe.RetryPolicy; -import dev.failsafe.RetryPolicyBuilder; -import dev.failsafe.event.ExecutionEvent; - - -@Slf4j -public class RetryHttpSinkHandler implements HttpSinkHandler { - - private final SinkConnectorConfig connectorConfig; - - // Retry policy builder - private RetryPolicyBuilder> retryPolicyBuilder; - - private final List urls; - - private final HttpSinkHandler sinkHandler; - - - public RetryHttpSinkHandler(SinkConnectorConfig connectorConfig, HttpSinkHandler sinkHandler) { - this.connectorConfig = connectorConfig; - this.sinkHandler = sinkHandler; - - // Initialize retry - initRetry(); - - // Initialize URLs - String[] urlStrings = connectorConfig.getUrls(); - this.urls = Arrays.stream(urlStrings) - .map(URI::create) - .collect(Collectors.toList()); - } - - private void initRetry() { - HttpRetryConfig httpRetryConfig = this.connectorConfig.getRetryConfig(); - - this.retryPolicyBuilder = RetryPolicy.>builder() - .handleIf(e -> e instanceof ConnectException) - .handleResultIf(response -> httpRetryConfig.isRetryOnNonSuccess() && !HttpUtils.is2xxSuccessful(response.statusCode())) - .withMaxRetries(httpRetryConfig.getMaxRetries()) - .withDelay(Duration.ofMillis(httpRetryConfig.getInterval())); - } - - - /** - * Initializes the WebClient for making HTTP requests based on the provided SinkConnectorConfig. - */ - @Override - public void start() { - sinkHandler.start(); - } - - - /** - * Processes a ConnectRecord by sending it over HTTP or HTTPS. This method should be called for each ConnectRecord that needs to be processed. - * - * @param record the ConnectRecord to process - */ - @Override - public void handle(ConnectRecord record) { - for (URI url : this.urls) { - // convert ConnectRecord to HttpConnectRecord - String type = String.format("%s.%s.%s", - this.connectorConfig.getConnectorName(), url.getScheme(), - this.connectorConfig.getWebhookConfig().isActivate() ? "webhook" : "common"); - HttpConnectRecord httpConnectRecord = HttpConnectRecord.convertConnectRecord(record, type); - // handle the HttpConnectRecord - deliver(url, httpConnectRecord); - } - } - - - /** - * Processes HttpConnectRecord on specified URL while returning its own processing logic This method provides the retry power to process the - * HttpConnectRecord - * - * @param url URI to which the HttpConnectRecord should be sent - * @param httpConnectRecord HttpConnectRecord to process - * @return processing chain - */ - @Override - public Future> deliver(URI url, HttpConnectRecord httpConnectRecord) { - // Only webhook mode needs to use the UUID to identify the request - String id = httpConnectRecord.getUuid(); - - // Build the retry policy - RetryPolicy> retryPolicy = retryPolicyBuilder - .onSuccess(event -> { - if (connectorConfig.getWebhookConfig().isActivate()) { - // convert the result to an HttpExportRecord - HttpExportRecord exportRecord = covertToExportRecord(httpConnectRecord, event, event.getResult(), event.getException(), url, id); - // add the data to the queue - ((WebhookHttpSinkHandler) sinkHandler).addDataToQueue(exportRecord); - } - }) - .onRetry(event -> { - if (log.isDebugEnabled()) { - log.warn("Retrying the request to {} for the {} time. HttpConnectRecord= {}", url, event.getAttemptCount(), httpConnectRecord); - } else { - log.warn("Retrying the request to {} for the {} time.", url, event.getAttemptCount()); - } - if (connectorConfig.getWebhookConfig().isActivate()) { - HttpExportRecord exportRecord = - covertToExportRecord(httpConnectRecord, event, event.getLastResult(), event.getLastException(), url, id); - ((WebhookHttpSinkHandler) sinkHandler).addDataToQueue(exportRecord); - } - // update the HttpConnectRecord - httpConnectRecord.setTime(LocalDateTime.now().toString()); - httpConnectRecord.setUuid(UUID.randomUUID().toString()); - }) - .onFailure(event -> { - if (log.isDebugEnabled()) { - log.error("Failed to send the request to {} after {} attempts. HttpConnectRecord= {}", url, event.getAttemptCount(), - httpConnectRecord, event.getException()); - } else { - log.error("Failed to send the request to {} after {} attempts.", url, event.getAttemptCount(), event.getException()); - } - if (connectorConfig.getWebhookConfig().isActivate()) { - HttpExportRecord exportRecord = covertToExportRecord(httpConnectRecord, event, event.getResult(), event.getException(), url, id); - ((WebhookHttpSinkHandler) sinkHandler).addDataToQueue(exportRecord); - } - }).build(); - - // Handle the HttpConnectRecord with retry - Failsafe.with(retryPolicy) - .getStageAsync(() -> sinkHandler.deliver(url, httpConnectRecord).toCompletionStage()); - - return null; - } - - /** - * Converts the ExecutionCompletedEvent to an HttpExportRecord. - * - * @param httpConnectRecord HttpConnectRecord - * @param event ExecutionEvent - * @param response the response of the request, may be null - * @param e the exception thrown during the request, may be null - * @param url the URL the request was sent to - * @param id UUID - * @return the converted HttpExportRecord - */ - private HttpExportRecord covertToExportRecord(HttpConnectRecord httpConnectRecord, ExecutionEvent event, HttpResponse response, - Throwable e, URI url, String id) { - - HttpExportMetadata httpExportMetadata = HttpExportMetadata.builder() - .url(url.toString()) - .code(response != null ? response.statusCode() : -1) - .message(response != null ? response.statusMessage() : e.getMessage()) - .receivedTime(LocalDateTime.now()) - .uuid(httpConnectRecord.getUuid()) - .retriedBy(event.getAttemptCount() > 1 ? id : null) - .retryNum(event.getAttemptCount() - 1).build(); - - return new HttpExportRecord(httpExportMetadata, response == null ? null : response.bodyAsString()); - } - - /** - * Cleans up and releases resources used by the HTTP/HTTPS handler. - */ - @Override - public void stop() { - sinkHandler.stop(); - } -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/AbstractHttpSinkHandler.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/AbstractHttpSinkHandler.java new file mode 100644 index 0000000000..9ef760617c --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/AbstractHttpSinkHandler.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.http.sink.handler; + +import org.apache.eventmesh.common.config.connector.http.SinkConnectorConfig; +import org.apache.eventmesh.connector.http.sink.data.HttpAttemptEvent; +import org.apache.eventmesh.connector.http.sink.data.HttpConnectRecord; +import org.apache.eventmesh.connector.http.sink.data.MultiHttpRequestContext; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; + +import java.net.URI; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +import lombok.Getter; + +/** + * AbstractHttpSinkHandler is an abstract class that provides a base implementation for HttpSinkHandler. + */ +public abstract class AbstractHttpSinkHandler implements HttpSinkHandler { + + @Getter + private final SinkConnectorConfig sinkConnectorConfig; + + @Getter + private final List urls; + + private final HttpDeliveryStrategy deliveryStrategy; + + private int roundRobinIndex = 0; + + protected AbstractHttpSinkHandler(SinkConnectorConfig sinkConnectorConfig) { + this.sinkConnectorConfig = sinkConnectorConfig; + this.deliveryStrategy = HttpDeliveryStrategy.valueOf(sinkConnectorConfig.getDeliveryStrategy()); + // Initialize URLs + String[] urlStrings = sinkConnectorConfig.getUrls(); + this.urls = Arrays.stream(urlStrings) + .map(URI::create) + .collect(Collectors.toList()); + } + + /** + * Processes a ConnectRecord by sending it over HTTP or HTTPS. This method should be called for each ConnectRecord that needs to be processed. + * + * @param record the ConnectRecord to process + */ + @Override + public void handle(ConnectRecord record) { + // build attributes + Map attributes = new ConcurrentHashMap<>(); + + switch (deliveryStrategy) { + case ROUND_ROBIN: + attributes.put(MultiHttpRequestContext.NAME, new MultiHttpRequestContext(1)); + URI url = urls.get(roundRobinIndex); + roundRobinIndex = (roundRobinIndex + 1) % urls.size(); + sendRecordToUrl(record, attributes, url); + break; + case BROADCAST: + for (URI broadcastUrl : urls) { + attributes.put(MultiHttpRequestContext.NAME, new MultiHttpRequestContext(urls.size())); + sendRecordToUrl(record, attributes, broadcastUrl); + } + break; + default: + throw new IllegalArgumentException("Unknown delivery strategy: " + deliveryStrategy); + } + } + + private void sendRecordToUrl(ConnectRecord record, Map attributes, URI url) { + // convert ConnectRecord to HttpConnectRecord + String type = String.format("%s.%s.%s", + this.sinkConnectorConfig.getConnectorName(), url.getScheme(), + this.sinkConnectorConfig.getWebhookConfig().isActivate() ? "webhook" : "common"); + HttpConnectRecord httpConnectRecord = HttpConnectRecord.convertConnectRecord(record, type); + + // add AttemptEvent to the attributes + HttpAttemptEvent attemptEvent = new HttpAttemptEvent(this.sinkConnectorConfig.getRetryConfig().getMaxRetries() + 1); + attributes.put(HttpAttemptEvent.PREFIX + httpConnectRecord.getHttpRecordId(), attemptEvent); + + // deliver the record + deliver(url, httpConnectRecord, attributes, record); + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/HttpDeliveryStrategy.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/HttpDeliveryStrategy.java new file mode 100644 index 0000000000..2e770eb120 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/HttpDeliveryStrategy.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.http.sink.handler; + +public enum HttpDeliveryStrategy { + ROUND_ROBIN, + BROADCAST +} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handle/HttpSinkHandler.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/HttpSinkHandler.java similarity index 82% rename from eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handle/HttpSinkHandler.java rename to eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/HttpSinkHandler.java index 09fd66a762..d5a27940e5 100644 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handle/HttpSinkHandler.java +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/HttpSinkHandler.java @@ -15,12 +15,13 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.http.sink.handle; +package org.apache.eventmesh.connector.http.sink.handler; import org.apache.eventmesh.connector.http.sink.data.HttpConnectRecord; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; import java.net.URI; +import java.util.Map; import io.vertx.core.Future; import io.vertx.core.buffer.Buffer; @@ -32,14 +33,14 @@ * *

Any class that needs to process ConnectRecords via HTTP or HTTPS should implement this interface. * Implementing classes must provide implementations for the {@link #start()}, {@link #handle(ConnectRecord)}, - * {@link #deliver(URI, HttpConnectRecord)}, and {@link #stop()} methods.

+ * {@link #deliver(URI, HttpConnectRecord, Map, ConnectRecord)}, and {@link #stop()} methods.

* *

Implementing classes should ensure thread safety and handle HTTP/HTTPS communication efficiently. * The {@link #start()} method initializes any necessary resources for HTTP/HTTPS communication. The {@link #handle(ConnectRecord)} method processes a - * ConnectRecord by sending it over HTTP or HTTPS. The {@link #deliver(URI, HttpConnectRecord)} method processes HttpConnectRecord on specified URL - * while returning its own processing logic {@link #stop()} method releases any resources used for HTTP/HTTPS communication.

+ * ConnectRecord by sending it over HTTP or HTTPS. The {@link #deliver(URI, HttpConnectRecord, Map, ConnectRecord)} method processes HttpConnectRecord + * on specified URL while returning its own processing logic {@link #stop()} method releases any resources used for HTTP/HTTPS communication.

* - *

It's recommended to handle exceptions gracefully within the {@link #deliver(URI, HttpConnectRecord)} method + *

It's recommended to handle exceptions gracefully within the {@link #deliver(URI, HttpConnectRecord, Map, ConnectRecord)} method * to prevent message loss or processing interruptions.

*/ public interface HttpSinkHandler { @@ -62,9 +63,10 @@ public interface HttpSinkHandler { * * @param url URI to which the HttpConnectRecord should be sent * @param httpConnectRecord HttpConnectRecord to process + * @param attributes additional attributes to be used in processing * @return processing chain */ - Future> deliver(URI url, HttpConnectRecord httpConnectRecord); + Future> deliver(URI url, HttpConnectRecord httpConnectRecord, Map attributes, ConnectRecord connectRecord); /** * Cleans up and releases resources used by the HTTP/HTTPS handler. This method should be called when the handler is no longer needed. diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/impl/CommonHttpSinkHandler.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/impl/CommonHttpSinkHandler.java new file mode 100644 index 0000000000..0b57cc06ef --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/impl/CommonHttpSinkHandler.java @@ -0,0 +1,270 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.http.sink.handler.impl; + +import org.apache.eventmesh.common.config.connector.http.SinkConnectorConfig; +import org.apache.eventmesh.common.utils.JsonUtils; +import org.apache.eventmesh.connector.http.sink.data.HttpAttemptEvent; +import org.apache.eventmesh.connector.http.sink.data.HttpConnectRecord; +import org.apache.eventmesh.connector.http.sink.data.MultiHttpRequestContext; +import org.apache.eventmesh.connector.http.sink.handler.AbstractHttpSinkHandler; +import org.apache.eventmesh.connector.http.util.HttpUtils; +import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendExceptionContext; +import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendResult; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; + +import java.net.URI; +import java.time.ZoneId; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.TimeUnit; + +import io.netty.handler.codec.http.HttpHeaderNames; +import io.vertx.core.Future; +import io.vertx.core.MultiMap; +import io.vertx.core.Vertx; +import io.vertx.core.buffer.Buffer; +import io.vertx.core.http.HttpHeaders; +import io.vertx.ext.web.client.HttpResponse; +import io.vertx.ext.web.client.WebClient; +import io.vertx.ext.web.client.WebClientOptions; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +/** + * Common HTTP/HTTPS Sink Handler implementation to handle ConnectRecords by sending them over HTTP or HTTPS to configured URLs. + * + *

This handler initializes a WebClient for making HTTP requests based on the provided SinkConnectorConfig. + * It handles processing ConnectRecords by converting them to HttpConnectRecord and sending them asynchronously to each configured URL using the + * WebClient.

+ * + *

The handler uses Vert.x's WebClient to perform HTTP/HTTPS requests. It initializes the WebClient in the {@link #start()} + * method and closes it in the {@link #stop()} method to manage resources efficiently.

+ * + *

Each ConnectRecord is processed and sent to all configured URLs concurrently using asynchronous HTTP requests.

+ */ +@Slf4j +@Getter +public class CommonHttpSinkHandler extends AbstractHttpSinkHandler { + + private WebClient webClient; + + + public CommonHttpSinkHandler(SinkConnectorConfig sinkConnectorConfig) { + super(sinkConnectorConfig); + } + + /** + * Initializes the WebClient for making HTTP requests based on the provided SinkConnectorConfig. + */ + @Override + public void start() { + // Create WebClient + doInitWebClient(); + } + + /** + * Initializes the WebClient with the provided configuration options. + */ + private void doInitWebClient() { + SinkConnectorConfig sinkConnectorConfig = getSinkConnectorConfig(); + final Vertx vertx = Vertx.vertx(); + WebClientOptions options = new WebClientOptions() + .setKeepAlive(sinkConnectorConfig.isKeepAlive()) + .setKeepAliveTimeout(sinkConnectorConfig.getKeepAliveTimeout() / 1000) + .setIdleTimeout(sinkConnectorConfig.getIdleTimeout()) + .setIdleTimeoutUnit(TimeUnit.MILLISECONDS) + .setConnectTimeout(sinkConnectorConfig.getConnectionTimeout()) + .setMaxPoolSize(sinkConnectorConfig.getMaxConnectionPoolSize()) + .setPipelining(sinkConnectorConfig.isParallelized()); + this.webClient = WebClient.create(vertx, options); + } + + /** + * Processes HttpConnectRecord on specified URL while returning its own processing logic. This method sends the HttpConnectRecord to the specified + * URL using the WebClient. + * + * @param url URI to which the HttpConnectRecord should be sent + * @param httpConnectRecord HttpConnectRecord to process + * @param attributes additional attributes to be used in processing + * @return processing chain + */ + @Override + public Future> deliver(URI url, HttpConnectRecord httpConnectRecord, Map attributes, + ConnectRecord connectRecord) { + // create headers + Map extensionMap = new HashMap<>(); + Set extensionKeySet = httpConnectRecord.getExtensions().keySet(); + for (String extensionKey : extensionKeySet) { + Object v = httpConnectRecord.getExtensions().getObject(extensionKey); + extensionMap.put(extensionKey, v); + } + + MultiMap headers = HttpHeaders.headers() + .set(HttpHeaderNames.CONTENT_TYPE, "application/json; charset=utf-8") + .set(HttpHeaderNames.ACCEPT, "application/json; charset=utf-8") + .set("extension", JsonUtils.toJSONString(extensionMap)); + // get timestamp and offset + Long timestamp = httpConnectRecord.getCreateTime() + .atZone(ZoneId.systemDefault()) + .toInstant() + .toEpochMilli(); + + // send the request + return this.webClient.post(url.getPath()) + .host(url.getHost()) + .port(url.getPort() == -1 ? (Objects.equals(url.getScheme(), "https") ? 443 : 80) : url.getPort()) + .putHeaders(headers) + .ssl(Objects.equals(url.getScheme(), "https")) + .sendJson(httpConnectRecord.getData()) + .onSuccess(res -> { + log.info("Request sent successfully. Record: timestamp={}", timestamp); + + Exception e = null; + + // log the response + if (HttpUtils.is2xxSuccessful(res.statusCode())) { + if (log.isDebugEnabled()) { + log.debug("Received successful response: statusCode={}. Record: timestamp={}, responseBody={}", + res.statusCode(), timestamp, res.bodyAsString()); + } else { + log.info("Received successful response: statusCode={}. Record: timestamp={}", res.statusCode(), timestamp); + } + } else { + if (log.isDebugEnabled()) { + log.warn("Received non-2xx response: statusCode={}. Record: timestamp={}, responseBody={}", + res.statusCode(), timestamp, res.bodyAsString()); + } else { + log.warn("Received non-2xx response: statusCode={}. Record: timestamp={}", res.statusCode(), timestamp); + } + + e = new RuntimeException("Unexpected HTTP response code: " + res.statusCode()); + } + + // try callback + tryCallback(httpConnectRecord, e, attributes, connectRecord); + }).onFailure(err -> { + log.error("Request failed to send. Record: timestamp={}", timestamp, err); + + // try callback + tryCallback(httpConnectRecord, err, attributes, connectRecord); + }); + } + + /** + * Tries to call the callback based on the result of the request. + * + * @param httpConnectRecord the HttpConnectRecord to use + * @param e the exception thrown during the request, may be null + * @param attributes additional attributes to be used in processing + */ + private void tryCallback(HttpConnectRecord httpConnectRecord, Throwable e, Map attributes, ConnectRecord record) { + // get and update the attempt event + HttpAttemptEvent attemptEvent = (HttpAttemptEvent) attributes.get(HttpAttemptEvent.PREFIX + httpConnectRecord.getHttpRecordId()); + attemptEvent.updateEvent(e); + + // get and update the multiHttpRequestContext + MultiHttpRequestContext multiHttpRequestContext = getAndUpdateMultiHttpRequestContext(attributes, attemptEvent); + + if (multiHttpRequestContext.isAllRequestsProcessed()) { + // do callback + if (record.getCallback() == null) { + if (log.isDebugEnabled()) { + log.warn("ConnectRecord callback is null. Ignoring callback. {}", record); + } else { + log.warn("ConnectRecord callback is null. Ignoring callback."); + } + return; + } + + // get the last failed event + HttpAttemptEvent lastFailedEvent = multiHttpRequestContext.getLastFailedEvent(); + if (lastFailedEvent == null) { + // success + record.getCallback().onSuccess(convertToSendResult(record)); + } else { + // failure + record.getCallback().onException(buildSendExceptionContext(record, lastFailedEvent.getLastException())); + } + } else { + log.warn("still have requests to process, size {}|attempt num {}", + multiHttpRequestContext.getRemainingRequests(), attemptEvent.getAttempts()); + } + } + + + /** + * Gets and updates the multi http request context based on the provided attributes and HttpConnectRecord. + * + * @param attributes the attributes to use + * @param attemptEvent the HttpAttemptEvent to use + * @return the updated multi http request context + */ + private MultiHttpRequestContext getAndUpdateMultiHttpRequestContext(Map attributes, HttpAttemptEvent attemptEvent) { + // get the multi http request context + MultiHttpRequestContext multiHttpRequestContext = (MultiHttpRequestContext) attributes.get(MultiHttpRequestContext.NAME); + + // Check if the current attempted event has completed + if (attemptEvent.isComplete()) { + // decrement the counter + multiHttpRequestContext.decrementRemainingRequests(); + + if (attemptEvent.getLastException() != null) { + // if all attempts are exhausted, set the last failed event + multiHttpRequestContext.setLastFailedEvent(attemptEvent); + } + } + + return multiHttpRequestContext; + } + + private SendResult convertToSendResult(ConnectRecord record) { + SendResult result = new SendResult(); + result.setMessageId(record.getRecordId()); + if (org.apache.commons.lang3.StringUtils.isNotEmpty(record.getExtension("topic"))) { + result.setTopic(record.getExtension("topic")); + } + return result; + } + + private SendExceptionContext buildSendExceptionContext(ConnectRecord record, Throwable e) { + SendExceptionContext sendExceptionContext = new SendExceptionContext(); + sendExceptionContext.setMessageId(record.getRecordId()); + sendExceptionContext.setCause(e); + if (org.apache.commons.lang3.StringUtils.isNotEmpty(record.getExtension("topic"))) { + sendExceptionContext.setTopic(record.getExtension("topic")); + } + return sendExceptionContext; + } + + + /** + * Cleans up and releases resources used by the HTTP/HTTPS handler. + */ + @Override + public void stop() { + if (this.webClient != null) { + this.webClient.close(); + } else { + log.warn("WebClient is null, ignore."); + } + } +} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/impl/HttpSinkHandlerRetryWrapper.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/impl/HttpSinkHandlerRetryWrapper.java new file mode 100644 index 0000000000..050839451a --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/impl/HttpSinkHandlerRetryWrapper.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.http.sink.handler.impl; + +import org.apache.eventmesh.common.config.connector.http.HttpRetryConfig; +import org.apache.eventmesh.common.config.connector.http.SinkConnectorConfig; +import org.apache.eventmesh.connector.http.sink.data.HttpConnectRecord; +import org.apache.eventmesh.connector.http.sink.handler.AbstractHttpSinkHandler; +import org.apache.eventmesh.connector.http.sink.handler.HttpSinkHandler; +import org.apache.eventmesh.connector.http.util.HttpUtils; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; + +import java.net.ConnectException; +import java.net.URI; +import java.time.Duration; +import java.util.Map; + +import io.vertx.core.Future; +import io.vertx.core.buffer.Buffer; +import io.vertx.ext.web.client.HttpResponse; + +import lombok.extern.slf4j.Slf4j; + +import dev.failsafe.Failsafe; +import dev.failsafe.RetryPolicy; + + +/** + * HttpSinkHandlerRetryWrapper is a wrapper class for the HttpSinkHandler that provides retry functionality for failed HTTP requests. + */ +@Slf4j +public class HttpSinkHandlerRetryWrapper extends AbstractHttpSinkHandler { + + private final HttpRetryConfig httpRetryConfig; + + private final HttpSinkHandler sinkHandler; + + private final RetryPolicy> retryPolicy; + + public HttpSinkHandlerRetryWrapper(SinkConnectorConfig sinkConnectorConfig, HttpSinkHandler sinkHandler) { + super(sinkConnectorConfig); + this.sinkHandler = sinkHandler; + this.httpRetryConfig = getSinkConnectorConfig().getRetryConfig(); + this.retryPolicy = buildRetryPolicy(); + } + + private RetryPolicy> buildRetryPolicy() { + return RetryPolicy.>builder() + .handleIf(e -> e instanceof ConnectException) + .handleResultIf(response -> httpRetryConfig.isRetryOnNonSuccess() && !HttpUtils.is2xxSuccessful(response.statusCode())) + .withMaxRetries(httpRetryConfig.getMaxRetries()) + .withDelay(Duration.ofMillis(httpRetryConfig.getInterval())) + .onRetry(event -> { + if (log.isDebugEnabled()) { + log.warn("Failed to deliver message after {} attempts. Retrying in {} ms. Error: {}", + event.getAttemptCount(), httpRetryConfig.getInterval(), event.getLastException()); + } else { + log.warn("Failed to deliver message after {} attempts. Retrying in {} ms.", + event.getAttemptCount(), httpRetryConfig.getInterval()); + } + }).onFailure(event -> { + if (log.isDebugEnabled()) { + log.error("Failed to deliver message after {} attempts. Error: {}", + event.getAttemptCount(), event.getException()); + } else { + log.error("Failed to deliver message after {} attempts.", + event.getAttemptCount()); + } + }).build(); + } + + /** + * Initializes the WebClient for making HTTP requests based on the provided SinkConnectorConfig. + */ + @Override + public void start() { + sinkHandler.start(); + } + + + /** + * Processes HttpConnectRecord on specified URL while returning its own processing logic This method provides the retry power to process the + * HttpConnectRecord + * + * @param url URI to which the HttpConnectRecord should be sent + * @param httpConnectRecord HttpConnectRecord to process + * @param attributes additional attributes to pass to the processing chain + * @return processing chain + */ + @Override + public Future> deliver(URI url, HttpConnectRecord httpConnectRecord, Map attributes, + ConnectRecord connectRecord) { + Failsafe.with(retryPolicy) + .getStageAsync(() -> sinkHandler.deliver(url, httpConnectRecord, attributes, connectRecord).toCompletionStage()); + return null; + } + + + /** + * Cleans up and releases resources used by the HTTP/HTTPS handler. + */ + @Override + public void stop() { + sinkHandler.stop(); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handle/WebhookHttpSinkHandler.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/impl/WebhookHttpSinkHandler.java similarity index 65% rename from eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handle/WebhookHttpSinkHandler.java rename to eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/impl/WebhookHttpSinkHandler.java index e07683fcfa..0751918ee7 100644 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handle/WebhookHttpSinkHandler.java +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/sink/handler/impl/WebhookHttpSinkHandler.java @@ -15,11 +15,13 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.http.sink.handle; +package org.apache.eventmesh.connector.http.sink.handler.impl; +import org.apache.eventmesh.common.config.connector.http.HttpWebhookConfig; +import org.apache.eventmesh.common.config.connector.http.SinkConnectorConfig; import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.connector.http.sink.config.HttpWebhookConfig; -import org.apache.eventmesh.connector.http.sink.config.SinkConnectorConfig; +import org.apache.eventmesh.connector.http.common.SynchronizedCircularFifoQueue; +import org.apache.eventmesh.connector.http.sink.data.HttpAttemptEvent; import org.apache.eventmesh.connector.http.sink.data.HttpConnectRecord; import org.apache.eventmesh.connector.http.sink.data.HttpExportMetadata; import org.apache.eventmesh.connector.http.sink.data.HttpExportRecord; @@ -30,13 +32,10 @@ import java.net.URI; import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Objects; -import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; import io.netty.handler.codec.http.HttpResponseStatus; import io.vertx.core.Future; @@ -64,8 +63,6 @@ @Slf4j public class WebhookHttpSinkHandler extends CommonHttpSinkHandler { - private final SinkConnectorConfig sinkConnectorConfig; - // the configuration for webhook private final HttpWebhookConfig webhookConfig; @@ -73,25 +70,31 @@ public class WebhookHttpSinkHandler extends CommonHttpSinkHandler { private HttpServer exportServer; // store the received data, when webhook is enabled - private final ConcurrentLinkedQueue receivedDataQueue; + private final SynchronizedCircularFifoQueue receivedDataQueue; + + private volatile boolean exportStarted = false; - // the maximum queue size - private final int maxQueueSize; + private volatile boolean exportDestroyed = false; - // the current queue size - private final AtomicInteger currentQueueSize; + public boolean isExportStarted() { + return exportStarted; + } + + public boolean isExportDestroyed() { + return exportDestroyed; + } public WebhookHttpSinkHandler(SinkConnectorConfig sinkConnectorConfig) { super(sinkConnectorConfig); - this.sinkConnectorConfig = sinkConnectorConfig; + this.webhookConfig = sinkConnectorConfig.getWebhookConfig(); - this.maxQueueSize = this.webhookConfig.getMaxStorageSize(); - this.currentQueueSize = new AtomicInteger(0); - this.receivedDataQueue = new ConcurrentLinkedQueue<>(); + int maxQueueSize = this.webhookConfig.getMaxStorageSize(); + this.receivedDataQueue = new SynchronizedCircularFifoQueue<>(maxQueueSize); // init the export server doInitExportServer(); } + /** * Initialize the server for exporting the received data */ @@ -135,7 +138,7 @@ private void doInitExportServer() { int pageNum = StringUtils.isBlank(pageNumStr) ? 1 : Integer.parseInt(pageNumStr); int pageSize = Integer.parseInt(pageSizeStr); - if (currentQueueSize.get() == 0) { + if (receivedDataQueue.isEmpty()) { ctx.response() .putHeader(HttpHeaders.CONTENT_TYPE, "application/json; charset=utf-8") .setStatusCode(HttpResponseStatus.NO_CONTENT.code()) @@ -148,12 +151,12 @@ private void doInitExportServer() { List exportRecords; if (Objects.equals(type, TypeEnum.POLL.getValue())) { // If the type is poll, only the first page of data is exported and removed - exportRecords = getDataFromQueue(0, pageSize, true); + exportRecords = receivedDataQueue.fetchRange(0, pageSize, true); } else { // If the type is peek, the specified page of data is exported without removing int startIndex = (pageNum - 1) * pageSize; int endIndex = startIndex + pageSize; - exportRecords = getDataFromQueue(startIndex, endIndex, false); + exportRecords = receivedDataQueue.fetchRange(startIndex, endIndex, false); } // Create HttpExportRecordPage @@ -185,26 +188,15 @@ public void start() { // start the webclient super.start(); // start the export server - Throwable t = this.exportServer.listen().cause(); - if (t != null) { - throw new EventMeshException("Failed to start Vertx server. ", t); - } - } - - /** - * Processes a ConnectRecord by sending it over HTTP or HTTPS. This method should be called for each ConnectRecord that needs to be processed. - * - * @param record the ConnectRecord to process - */ - @Override - public void handle(ConnectRecord record) { - for (URI url : super.getUrls()) { - // convert ConnectRecord to HttpConnectRecord - String type = String.format("%s.%s.%s", this.getConnectorConfig().getConnectorName(), url.getScheme(), "webhook"); - HttpConnectRecord httpConnectRecord = HttpConnectRecord.convertConnectRecord(record, type); - // handle the HttpConnectRecord - deliver(url, httpConnectRecord); - } + this.exportServer.listen(res -> { + if (res.succeeded()) { + this.exportStarted = true; + log.info("WebhookHttpExportServer started on port: {}", this.webhookConfig.getPort()); + } else { + log.error("WebhookHttpExportServer failed to start on port: {}", this.webhookConfig.getPort()); + throw new EventMeshException("Failed to start Vertx server. ", res.cause()); + } + }); } @@ -214,91 +206,63 @@ public void handle(ConnectRecord record) { * * @param url URI to which the HttpConnectRecord should be sent * @param httpConnectRecord HttpConnectRecord to process + * @param attributes additional attributes to be used in processing * @return processing chain */ @Override - public Future> deliver(URI url, HttpConnectRecord httpConnectRecord) { + public Future> deliver(URI url, HttpConnectRecord httpConnectRecord, Map attributes, + ConnectRecord connectRecord) { // send the request - Future> responseFuture = super.deliver(url, httpConnectRecord); + Future> responseFuture = super.deliver(url, httpConnectRecord, attributes, connectRecord); // store the received data return responseFuture.onComplete(arr -> { - // If open retry, return directly and handled by RetryHttpSinkHandler - if (sinkConnectorConfig.getRetryConfig().getMaxRetries() > 0) { - return; - } - // create ExportMetadataBuilder + // get HttpAttemptEvent + HttpAttemptEvent attemptEvent = (HttpAttemptEvent) attributes.get(HttpAttemptEvent.PREFIX + httpConnectRecord.getHttpRecordId()); + + // get the response HttpResponse response = arr.succeeded() ? arr.result() : null; - HttpExportMetadata httpExportMetadata = HttpExportMetadata.builder() - .url(url.toString()) - .code(response != null ? response.statusCode() : -1) - .message(response != null ? response.statusMessage() : arr.cause().getMessage()) - .receivedTime(LocalDateTime.now()) - .retriedBy(null) - .uuid(httpConnectRecord.getUuid()) - .retryNum(0) - .build(); + // create ExportMetadata + HttpExportMetadata httpExportMetadata = buildHttpExportMetadata(url, response, httpConnectRecord, attemptEvent); // create ExportRecord HttpExportRecord exportRecord = new HttpExportRecord(httpExportMetadata, arr.succeeded() ? arr.result().bodyAsString() : null); // add the data to the queue - addDataToQueue(exportRecord); + receivedDataQueue.offer(exportRecord); }); } - /** - * Adds the received data to the queue. + * Builds the HttpExportMetadata object based on the response, HttpConnectRecord, and HttpRetryEvent. * - * @param exportRecord the received data to add to the queue + * @param url the URI to which the HttpConnectRecord was sent + * @param response the response received from the URI + * @param httpConnectRecord the HttpConnectRecord that was sent + * @param attemptEvent the HttpAttemptEvent that was used to send the HttpConnectRecord + * @return the HttpExportMetadata object */ - public void addDataToQueue(HttpExportRecord exportRecord) { - // If the current queue size is greater than or equal to the maximum queue size, remove the oldest element - if (currentQueueSize.get() >= maxQueueSize) { - Object removedData = receivedDataQueue.poll(); - if (log.isDebugEnabled()) { - log.debug("The queue is full, remove the oldest element: {}", removedData); - } else { - log.info("The queue is full, remove the oldest element"); - } - currentQueueSize.decrementAndGet(); + private HttpExportMetadata buildHttpExportMetadata(URI url, HttpResponse response, HttpConnectRecord httpConnectRecord, + HttpAttemptEvent attemptEvent) { + + String msg = null; + // order of precedence: lastException > response > null + if (attemptEvent.getLastException() != null) { + msg = attemptEvent.getLimitedExceptionMessage(); + } else if (response != null) { + msg = response.statusMessage(); } - // Try to put the received data into the queue - if (receivedDataQueue.offer(exportRecord)) { - currentQueueSize.incrementAndGet(); - log.debug("Successfully put the received data into the queue: {}", exportRecord); - } else { - log.error("Failed to put the received data into the queue: {}", exportRecord); - } - } - /** - * Gets the received data from the queue. - * - * @param startIndex the start index of the data to get - * @param endIndex the end index of the data to get - * @param removed whether to remove the data from the queue - * @return the received data - */ - private List getDataFromQueue(int startIndex, int endIndex, boolean removed) { - Iterator iterator = receivedDataQueue.iterator(); - - List pageItems = new ArrayList<>(endIndex - startIndex); - int count = 0; - while (iterator.hasNext() && count < endIndex) { - HttpExportRecord item = iterator.next(); - if (count >= startIndex) { - pageItems.add(item); - if (removed) { - iterator.remove(); - currentQueueSize.decrementAndGet(); - } - } - count++; - } - return pageItems; + return HttpExportMetadata.builder() + .url(url.toString()) + .code(response != null ? response.statusCode() : -1) + .message(msg) + .receivedTime(LocalDateTime.now()) + .recordId(httpConnectRecord.getHttpRecordId()) + .retryNum(attemptEvent.getAttempts() - 1) + .build(); } + /** * Cleans up and releases resources used by the HTTP/HTTPS handler. */ @@ -308,10 +272,15 @@ public void stop() { super.stop(); // stop the export server if (this.exportServer != null) { - Throwable t = this.exportServer.close().cause(); - if (t != null) { - throw new EventMeshException("Failed to stop Vertx server. ", t); - } + this.exportServer.close(res -> { + if (res.succeeded()) { + this.exportDestroyed = true; + log.info("WebhookHttpExportServer stopped on port: {}", this.webhookConfig.getPort()); + } else { + log.error("WebhookHttpExportServer failed to stop on port: {}", this.webhookConfig.getPort()); + throw new EventMeshException("Failed to stop Vertx server. ", res.cause()); + } + }); } else { log.warn("Callback server is null, ignore."); } diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/HttpSourceConnector.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/HttpSourceConnector.java new file mode 100644 index 0000000000..5aa8c63dd9 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/HttpSourceConnector.java @@ -0,0 +1,219 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.http.source; + +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.http.HttpSourceConfig; +import org.apache.eventmesh.common.exception.EventMeshException; +import org.apache.eventmesh.connector.http.source.data.CommonResponse; +import org.apache.eventmesh.connector.http.source.protocol.Protocol; +import org.apache.eventmesh.connector.http.source.protocol.ProtocolFactory; +import org.apache.eventmesh.openconnect.api.ConnectorCreateService; +import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; +import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; +import org.apache.eventmesh.openconnect.api.source.Source; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import io.netty.handler.codec.http.HttpResponseStatus; +import io.vertx.core.Vertx; +import io.vertx.core.http.HttpServer; +import io.vertx.core.http.HttpServerOptions; +import io.vertx.ext.web.Route; +import io.vertx.ext.web.Router; +import io.vertx.ext.web.RoutingContext; +import io.vertx.ext.web.handler.LoggerHandler; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class HttpSourceConnector implements Source, ConnectorCreateService { + + private HttpSourceConfig sourceConfig; + + private BlockingQueue queue; + + private int batchSize; + + private Route route; + + private Protocol protocol; + + private HttpServer server; + + @Getter + private volatile boolean started = false; + + @Getter + private volatile boolean destroyed = false; + + + @Override + public Class configClass() { + return HttpSourceConfig.class; + } + + @Override + public Source create() { + return new HttpSourceConnector(); + } + + @Override + public void init(Config config) { + this.sourceConfig = (HttpSourceConfig) config; + doInit(); + } + + @Override + public void init(ConnectorContext connectorContext) { + SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; + this.sourceConfig = (HttpSourceConfig) sourceConnectorContext.getSourceConfig(); + doInit(); + } + + private void doInit() { + // init queue + int maxQueueSize = this.sourceConfig.getConnectorConfig().getMaxStorageSize(); + this.queue = new LinkedBlockingQueue<>(maxQueueSize); + + // init batch size + this.batchSize = this.sourceConfig.getConnectorConfig().getBatchSize(); + + // init protocol + String protocolName = this.sourceConfig.getConnectorConfig().getProtocol(); + this.protocol = ProtocolFactory.getInstance(this.sourceConfig.connectorConfig, protocolName); + + final Vertx vertx = Vertx.vertx(); + final Router router = Router.router(vertx); + route = router.route() + .path(this.sourceConfig.connectorConfig.getPath()) + .handler(LoggerHandler.create()); + + // set protocol handler + this.protocol.setHandler(route, queue); + + // create server + this.server = vertx.createHttpServer(new HttpServerOptions() + .setPort(this.sourceConfig.connectorConfig.getPort()) + .setMaxFormAttributeSize(this.sourceConfig.connectorConfig.getMaxFormAttributeSize()) + .setIdleTimeout(this.sourceConfig.connectorConfig.getIdleTimeout()) + .setIdleTimeoutUnit(TimeUnit.MILLISECONDS)).requestHandler(router); + } + + @Override + public void start() { + this.server.listen(res -> { + if (res.succeeded()) { + this.started = true; + log.info("HttpSourceConnector started on port: {}", this.sourceConfig.getConnectorConfig().getPort()); + } else { + log.error("HttpSourceConnector failed to start on port: {}", this.sourceConfig.getConnectorConfig().getPort()); + throw new EventMeshException("failed to start Vertx server", res.cause()); + } + }); + } + + @Override + public void commit(ConnectRecord record) { + if (sourceConfig.getConnectorConfig().isDataConsistencyEnabled()) { + log.debug("HttpSourceConnector commit record: {}", record.getRecordId()); + RoutingContext routingContext = (RoutingContext) record.getExtensionObj("routingContext"); + if (routingContext != null) { + routingContext.response() + .putHeader("content-type", "application/json") + .setStatusCode(HttpResponseStatus.OK.code()) + .end(CommonResponse.success().toJsonStr()); + } else { + log.error("Failed to commit the record, routingContext is null, recordId: {}", record.getRecordId()); + } + } + } + + @Override + public String name() { + return this.sourceConfig.getConnectorConfig().getConnectorName(); + } + + @Override + public void onException(ConnectRecord record) { + if (this.route != null) { + this.route.failureHandler(ctx -> { + log.error("Failed to handle the request, recordId {}. ", record.getRecordId(), ctx.failure()); + // Return Bad Response + ctx.response() + .setStatusCode(HttpResponseStatus.INTERNAL_SERVER_ERROR.code()) + .end("{\"status\":\"failed\",\"recordId\":\"" + record.getRecordId() + "\"}"); + }); + } + } + + @Override + public void stop() { + if (this.server != null) { + this.server.close(res -> { + if (res.succeeded()) { + this.destroyed = true; + log.info("HttpSourceConnector stopped on port: {}", this.sourceConfig.getConnectorConfig().getPort()); + } else { + log.error("HttpSourceConnector failed to stop on port: {}", this.sourceConfig.getConnectorConfig().getPort()); + throw new EventMeshException("failed to stop Vertx server", res.cause()); + } + } + ); + } else { + log.warn("HttpSourceConnector server is null, ignore."); + } + } + + @Override + public List poll() { + long startTime = System.currentTimeMillis(); + long maxPollWaitTime = 5000; + long remainingTime = maxPollWaitTime; + + // poll from queue + List connectRecords = new ArrayList<>(batchSize); + for (int i = 0; i < batchSize; i++) { + try { + Object obj = queue.poll(remainingTime, TimeUnit.MILLISECONDS); + if (obj == null) { + break; + } + // convert to ConnectRecord + ConnectRecord connectRecord = protocol.convertToConnectRecord(obj); + connectRecords.add(connectRecord); + + // calculate elapsed time and update remaining time for next poll + long elapsedTime = System.currentTimeMillis() - startTime; + remainingTime = maxPollWaitTime > elapsedTime ? maxPollWaitTime - elapsedTime : 0; + } catch (Exception e) { + log.error("Failed to poll from queue.", e); + throw new RuntimeException(e); + } + + } + return connectRecords; + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/connector/HttpSourceConnector.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/connector/HttpSourceConnector.java deleted file mode 100644 index f9a6c568e0..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/connector/HttpSourceConnector.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.source.connector; - -import org.apache.eventmesh.common.exception.EventMeshException; -import org.apache.eventmesh.connector.http.source.config.HttpSourceConfig; -import org.apache.eventmesh.openconnect.api.config.Config; -import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; -import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; -import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.CloudEventUtil; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - -import io.cloudevents.CloudEvent; -import io.cloudevents.http.vertx.VertxMessageFactory; -import io.netty.handler.codec.http.HttpResponseStatus; -import io.vertx.core.Vertx; -import io.vertx.core.http.HttpMethod; -import io.vertx.core.http.HttpServer; -import io.vertx.core.http.HttpServerOptions; -import io.vertx.ext.web.Router; -import io.vertx.ext.web.handler.LoggerHandler; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class HttpSourceConnector implements Source { - - private static final int DEFAULT_BATCH_SIZE = 10; - - private HttpSourceConfig sourceConfig; - private BlockingQueue queue; - private HttpServer server; - - @Override - public Class configClass() { - return HttpSourceConfig.class; - } - - @Override - public void init(Config config) { - this.sourceConfig = (HttpSourceConfig) config; - doInit(); - } - - @Override - public void init(ConnectorContext connectorContext) { - SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; - this.sourceConfig = (HttpSourceConfig) sourceConnectorContext.getSourceConfig(); - doInit(); - } - - private void doInit() { - this.queue = new LinkedBlockingQueue<>(1000); - - final Vertx vertx = Vertx.vertx(); - final Router router = Router.router(vertx); - router.route() - .path(this.sourceConfig.connectorConfig.getPath()) - .method(HttpMethod.POST) - .handler(LoggerHandler.create()) - .handler(ctx -> { - VertxMessageFactory.createReader(ctx.request()) - .map(reader -> { - CloudEvent event = reader.toEvent(); - if (event.getSubject() == null) { - throw new IllegalStateException("attribute 'subject' cannot be null"); - } - if (event.getDataContentType() == null) { - throw new IllegalStateException("attribute 'datacontenttype' cannot be null"); - } - if (event.getData() == null) { - throw new IllegalStateException("attribute 'data' cannot be null"); - } - return event; - }) - .onSuccess(event -> { - queue.add(event); - log.info("[HttpSourceConnector] Succeed to convert payload into CloudEvent. StatusCode={}", HttpResponseStatus.OK.code()); - ctx.response().setStatusCode(HttpResponseStatus.OK.code()).end(); - }) - .onFailure(t -> { - log.error("[HttpSourceConnector] Malformed request. StatusCode={}", HttpResponseStatus.BAD_REQUEST.code(), t); - ctx.response().setStatusCode(HttpResponseStatus.BAD_REQUEST.code()).end(); - }); - }); - this.server = vertx.createHttpServer(new HttpServerOptions() - .setPort(this.sourceConfig.connectorConfig.getPort()) - .setIdleTimeout(this.sourceConfig.connectorConfig.getIdleTimeout())).requestHandler(router); - } - - @Override - public void start() { - Throwable t = this.server.listen().cause(); - if (t != null) { - throw new EventMeshException("failed to start Vertx server", t); - } - } - - @Override - public void commit(ConnectRecord record) { - - } - - @Override - public String name() { - return this.sourceConfig.getConnectorConfig().getConnectorName(); - } - - @Override - public void stop() { - Throwable t = this.server.close().cause(); - if (t != null) { - throw new EventMeshException("failed to stop Vertx server", t); - } - } - - @Override - public List poll() { - List connectRecords = new ArrayList<>(DEFAULT_BATCH_SIZE); - for (int i = 0; i < DEFAULT_BATCH_SIZE; i++) { - try { - CloudEvent event = queue.poll(3, TimeUnit.SECONDS); - if (event == null) { - break; - } - connectRecords.add(CloudEventUtil.convertEventToRecord(event)); - } catch (InterruptedException e) { - break; - } - } - return connectRecords; - } - -} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/data/CommonResponse.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/data/CommonResponse.java new file mode 100644 index 0000000000..870f2afbe5 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/data/CommonResponse.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.http.source.data; + +import java.io.Serializable; +import java.time.LocalDateTime; + +import com.alibaba.fastjson2.JSON; +import com.alibaba.fastjson2.JSONWriter.Feature; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * Webhook response. + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class CommonResponse implements Serializable { + + private static final long serialVersionUID = 8616938575207104455L; + + private String msg; + + private LocalDateTime handleTime; + + /** + * Convert to json string. + * + * @return json string + */ + public String toJsonStr() { + return JSON.toJSONString(this, Feature.WriteMapNullValue); + } + + + /** + * Create a success response. + * + * @return response + */ + public static CommonResponse success() { + return base("success"); + } + + + /** + * Create a base response. + * + * @param msg message + * @return response + */ + public static CommonResponse base(String msg) { + return new CommonResponse(msg, LocalDateTime.now()); + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/data/WebhookRequest.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/data/WebhookRequest.java new file mode 100644 index 0000000000..9e1dcb7b4c --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/data/WebhookRequest.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.http.source.data; + +import java.io.Serializable; +import java.util.Map; + +import io.vertx.ext.web.RoutingContext; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * Webhook Protocol Request. + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class WebhookRequest implements Serializable { + + private static final long serialVersionUID = -483500600756490500L; + + private String protocolName; + + private String url; + + private Map headers; + + private Object payload; + + private RoutingContext routingContext; + +} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/Protocol.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/Protocol.java new file mode 100644 index 0000000000..c5a22139e0 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/Protocol.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.http.source.protocol; + +import org.apache.eventmesh.common.config.connector.http.SourceConnectorConfig; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; + +import java.util.concurrent.BlockingQueue; + +import io.vertx.ext.web.Route; + + +/** + * Protocol Interface. + * All protocols should implement this interface. + */ +public interface Protocol { + + + /** + * Initialize the protocol. + * + * @param sourceConnectorConfig source connector config + */ + void initialize(SourceConnectorConfig sourceConnectorConfig); + + + /** + * Handle the protocol message. + * + * @param route route + * @param queue queue info + */ + void setHandler(Route route, BlockingQueue queue); + + + /** + * Convert the message to ConnectRecord. + * + * @param message message + * @return ConnectRecord + */ + ConnectRecord convertToConnectRecord(Object message); +} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/ProtocolFactory.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/ProtocolFactory.java new file mode 100644 index 0000000000..6e6100e88b --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/ProtocolFactory.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.http.source.protocol; + +import org.apache.eventmesh.common.config.connector.http.SourceConnectorConfig; +import org.apache.eventmesh.connector.http.source.protocol.impl.CloudEventProtocol; +import org.apache.eventmesh.connector.http.source.protocol.impl.CommonProtocol; +import org.apache.eventmesh.connector.http.source.protocol.impl.GitHubProtocol; + +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Protocol factory. This class is responsible for storing and creating instances of {@link Protocol} classes. + */ +public class ProtocolFactory { + + // protocol name -> protocol class + private static final ConcurrentHashMap> protocols = new ConcurrentHashMap<>(); + + static { + // register all protocols + registerProtocol(CloudEventProtocol.PROTOCOL_NAME, CloudEventProtocol.class); + registerProtocol(GitHubProtocol.PROTOCOL_NAME, GitHubProtocol.class); + registerProtocol(CommonProtocol.PROTOCOL_NAME, CommonProtocol.class); + } + + + /** + * Register a protocol + * + * @param name name of the protocol + * @param clazz class of the protocol + */ + public static void registerProtocol(String name, Class clazz) { + if (Protocol.class.isAssignableFrom(clazz)) { + // put the class into the map(case insensitive) + protocols.put(name.toLowerCase(), clazz); + } else { + throw new IllegalArgumentException("Class " + clazz.getName() + " does not implement Protocol interface"); + } + } + + /** + * Get an instance of a protocol, if it is not already created, create a new instance + * + * @param name name of the protocol + * @return instance of the protocol + */ + public static Protocol getInstance(SourceConnectorConfig sourceConnectorConfig, String name) { + // get the class by name(case insensitive) + Class clazz = Optional.ofNullable(protocols.get(name.toLowerCase())) + .orElseThrow(() -> new IllegalArgumentException("Protocol " + name + " is not registered")); + try { + // create a new instance + Protocol protocol = (Protocol) clazz.newInstance(); + // initialize the protocol + protocol.initialize(sourceConnectorConfig); + return protocol; + } catch (InstantiationException | IllegalAccessException e) { + throw new IllegalArgumentException("Failed to instantiate protocol " + name, e); + } + } + +} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/WebhookConstants.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/WebhookConstants.java new file mode 100644 index 0000000000..b31637427b --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/WebhookConstants.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.http.source.protocol; + +public class WebhookConstants { + + /** + * -------------------------------------- About GitHub -------------------------------------- + */ + + // A globally unique identifier (GUID) to identify the delivery. + public static final String GITHUB_DELIVERY = "X-GitHub-Delivery"; + + // This header is sent if the webhook is configured with a secret. + // We recommend that you use the more secure X-Hub-Signature-256 instead + public static final String GITHUB_SIGNATURE = "X-Hub-Signature"; + + // This header is sent if the webhook is configured with a secret + public static final String GITHUB_SIGNATURE_256 = "X-Hub-Signature-256"; + + public static final String GITHUB_HASH_265_PREFIX = "sha256="; + + // The name of the event that triggered the delivery. + public static final String GITHUB_EVENT = "X-GitHub-Event"; + + // The unique identifier of the webhook. + public static final String GITHUB_HOOK_ID = "X-GitHub-Hook-ID"; + + // The unique identifier of the resource where the webhook was created. + public static final String GITHUB_HOOK_INSTALLATION_TARGET_ID = "X-GitHub-Hook-Installation-Target-ID"; + + // The type of resource where the webhook was created. + public static final String GITHUB_HOOK_INSTALLATION_TARGET_TYPE = "X-GitHub-Hook-Installation-Target-Type"; + +} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/impl/CloudEventProtocol.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/impl/CloudEventProtocol.java new file mode 100644 index 0000000000..10158f6eba --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/impl/CloudEventProtocol.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.http.source.protocol.impl; + +import org.apache.eventmesh.common.config.connector.http.SourceConnectorConfig; +import org.apache.eventmesh.connector.http.source.data.CommonResponse; +import org.apache.eventmesh.connector.http.source.protocol.Protocol; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; +import org.apache.eventmesh.openconnect.util.CloudEventUtil; + +import java.util.concurrent.BlockingQueue; + +import io.cloudevents.CloudEvent; +import io.cloudevents.http.vertx.VertxMessageFactory; +import io.netty.handler.codec.http.HttpResponseStatus; +import io.vertx.core.http.HttpMethod; +import io.vertx.ext.web.Route; + +import lombok.extern.slf4j.Slf4j; + +/** + * CloudEvent Protocol. + */ +@Slf4j +public class CloudEventProtocol implements Protocol { + + // Protocol name + public static final String PROTOCOL_NAME = "CloudEvent"; + + + /** + * Initialize the protocol. + * + * @param sourceConnectorConfig source connector config + */ + @Override + public void initialize(SourceConnectorConfig sourceConnectorConfig) { + + } + + + /** + * Handle the protocol message for CloudEvent. + * + * @param route route + * @param queue queue info + */ + @Override + public void setHandler(Route route, BlockingQueue queue) { + route.method(HttpMethod.POST) + .handler(ctx -> VertxMessageFactory.createReader(ctx.request()) + .map(reader -> { + CloudEvent event = reader.toEvent(); + if (event.getSubject() == null) { + throw new IllegalStateException("attribute 'subject' cannot be null"); + } + if (event.getDataContentType() == null) { + throw new IllegalStateException("attribute 'datacontenttype' cannot be null"); + } + if (event.getData() == null) { + throw new IllegalStateException("attribute 'data' cannot be null"); + } + return event; + }) + .onSuccess(event -> { + // Add the event to the queue, thread-safe + if (!queue.offer(event)) { + throw new IllegalStateException("Failed to store the request."); + } + log.info("[HttpSourceConnector] Succeed to convert payload into CloudEvent. StatusCode={}", HttpResponseStatus.OK.code()); + ctx.response() + .setStatusCode(HttpResponseStatus.OK.code()) + .end(CommonResponse.success().toJsonStr()); + }) + .onFailure(t -> { + log.error("[HttpSourceConnector] Malformed request. StatusCode={}", HttpResponseStatus.BAD_REQUEST.code(), t); + ctx.response() + .setStatusCode(HttpResponseStatus.BAD_REQUEST.code()) + .end(CommonResponse.base(t.getMessage()).toJsonStr()); + })); + } + + /** + * Convert the message to ConnectRecord. + * + * @param message message + * @return ConnectRecord + */ + @Override + public ConnectRecord convertToConnectRecord(Object message) { + return CloudEventUtil.convertEventToRecord((CloudEvent) message); + } +} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/impl/CommonProtocol.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/impl/CommonProtocol.java new file mode 100644 index 0000000000..e831dc9723 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/impl/CommonProtocol.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.http.source.protocol.impl; + +import org.apache.eventmesh.common.Constants; +import org.apache.eventmesh.common.config.connector.http.SourceConnectorConfig; +import org.apache.eventmesh.common.utils.JsonUtils; +import org.apache.eventmesh.connector.http.source.data.CommonResponse; +import org.apache.eventmesh.connector.http.source.data.WebhookRequest; +import org.apache.eventmesh.connector.http.source.protocol.Protocol; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; + +import java.util.Base64; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.stream.Collectors; + +import io.netty.handler.codec.http.HttpResponseStatus; +import io.vertx.core.http.HttpMethod; +import io.vertx.core.json.JsonObject; +import io.vertx.ext.web.Route; +import io.vertx.ext.web.handler.BodyHandler; + +import lombok.extern.slf4j.Slf4j; + +/** + * Common Protocol. This class represents the common webhook protocol. The processing method of this class does not perform any other operations + * except storing the request and returning a general response. + */ +@Slf4j +public class CommonProtocol implements Protocol { + + public static final String PROTOCOL_NAME = "Common"; + + private SourceConnectorConfig sourceConnectorConfig; + + /** + * Initialize the protocol + * + * @param sourceConnectorConfig source connector config + */ + @Override + public void initialize(SourceConnectorConfig sourceConnectorConfig) { + this.sourceConnectorConfig = sourceConnectorConfig; + } + + /** + * Set the handler for the route + * + * @param route route + * @param queue queue info + */ + @Override + public void setHandler(Route route, BlockingQueue queue) { + route.method(HttpMethod.POST) + .handler(BodyHandler.create()) + .handler(ctx -> { + // Get the payload + Object payload = ctx.body().asString(Constants.DEFAULT_CHARSET.toString()); + payload = JsonUtils.parseObject(payload.toString(), String.class); + + // Create and store the webhook request + Map headerMap = ctx.request().headers().entries().stream() + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + WebhookRequest webhookRequest = new WebhookRequest(PROTOCOL_NAME, ctx.request().absoluteURI(), headerMap, payload, ctx); + if (!queue.offer(webhookRequest)) { + throw new IllegalStateException("Failed to store the request."); + } + + if (!sourceConnectorConfig.isDataConsistencyEnabled()) { + // Return 200 OK + ctx.response() + .setStatusCode(HttpResponseStatus.OK.code()) + .end(CommonResponse.success().toJsonStr()); + } + + }) + .failureHandler(ctx -> { + log.error("Failed to handle the request. ", ctx.failure()); + + // Return Bad Response + ctx.response() + .setStatusCode(ctx.statusCode()) + .end(CommonResponse.base(ctx.failure().getMessage()).toJsonStr()); + }); + + } + + /** + * Convert the message to a connect record + * + * @param message message + * @return connect record + */ + @Override + public ConnectRecord convertToConnectRecord(Object message) { + WebhookRequest request = (WebhookRequest) message; + ConnectRecord connectRecord = new ConnectRecord(null, null, System.currentTimeMillis(), request.getPayload()); + connectRecord.addExtension("source", request.getProtocolName()); + connectRecord.addExtension("url", request.getUrl()); + request.getHeaders().forEach((k, v) -> { + if (k.equalsIgnoreCase("extension")) { + JsonObject extension = new JsonObject(v); + extension.forEach(e -> connectRecord.addExtension(e.getKey(), e.getValue())); + } + }); + // check recordUniqueId + if (!connectRecord.getExtensions().containsKey("recordUniqueId")) { + connectRecord.addExtension("recordUniqueId", connectRecord.getRecordId()); + } + + // check data + if (connectRecord.getExtensionObj("isBase64") != null) { + if (Boolean.parseBoolean(connectRecord.getExtensionObj("isBase64").toString())) { + byte[] data = Base64.getDecoder().decode(connectRecord.getData().toString()); + connectRecord.setData(data); + } + } + if (request.getRoutingContext() != null) { + connectRecord.addExtension("routingContext", request.getRoutingContext()); + } + return connectRecord; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/impl/GitHubProtocol.java b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/impl/GitHubProtocol.java new file mode 100644 index 0000000000..e1edbd0faf --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/java/org/apache/eventmesh/connector/http/source/protocol/impl/GitHubProtocol.java @@ -0,0 +1,227 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.http.source.protocol.impl; + +import org.apache.eventmesh.common.Constants; +import org.apache.eventmesh.common.config.connector.http.SourceConnectorConfig; +import org.apache.eventmesh.common.exception.EventMeshException; +import org.apache.eventmesh.connector.http.source.data.CommonResponse; +import org.apache.eventmesh.connector.http.source.data.WebhookRequest; +import org.apache.eventmesh.connector.http.source.protocol.Protocol; +import org.apache.eventmesh.connector.http.source.protocol.WebhookConstants; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; + +import org.apache.commons.lang3.BooleanUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.stream.Collectors; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +import io.netty.handler.codec.http.HttpResponseStatus; +import io.vertx.core.MultiMap; +import io.vertx.core.http.HttpMethod; +import io.vertx.ext.web.Route; +import io.vertx.ext.web.handler.BodyHandler; + +import com.alibaba.fastjson2.JSONObject; + +import lombok.extern.slf4j.Slf4j; + + +/** + * GitHub Protocol. This class represents the GitHub webhook protocol. + */ +@Slf4j +public class GitHubProtocol implements Protocol { + + // Protocol name + public static final String PROTOCOL_NAME = "GitHub"; + + private static final String H_MAC_SHA_265 = "HmacSHA256"; + + private static final String SECRET_KEY = "secret"; + + private String contentType = "application/json"; + + private String secret; + + + /** + * Initialize the protocol. + * + * @param sourceConnectorConfig source connector config + */ + @Override + public void initialize(SourceConnectorConfig sourceConnectorConfig) { + // Initialize the protocol + Map extraConfig = sourceConnectorConfig.getExtraConfig(); + // set the secret, if it is not set, throw an exception + this.secret = extraConfig.get(SECRET_KEY); + if (StringUtils.isBlank(this.secret)) { + throw new EventMeshException("The secret is required for GitHub protocol."); + } + // if the content-type is not set, use the default value + this.contentType = extraConfig.getOrDefault("contentType", contentType); + } + + /** + * Handle the protocol message for GitHub. + * + * @param route route + * @param queue queue info + */ + @Override + public void setHandler(Route route, BlockingQueue queue) { + route.method(HttpMethod.POST) + .handler(BodyHandler.create()) + .handler(ctx -> { + // Get the payload and headers + String payloadStr = ctx.body().asString(Constants.DEFAULT_CHARSET.toString()); + MultiMap headers = ctx.request().headers(); + + // validate the content type + if (!StringUtils.contains(headers.get("Content-Type"), contentType)) { + String errorMsg = String.format("content-type is invalid, please check the content-type. received content-type: %s", + headers.get("Content-Type")); + // Return Bad Request + ctx.fail(HttpResponseStatus.BAD_REQUEST.code(), new EventMeshException(errorMsg)); + return; + } + + // validate the signature + String signature = headers.get(WebhookConstants.GITHUB_SIGNATURE_256); + if (BooleanUtils.isFalse(validateSignature(signature, payloadStr, secret))) { + String errorMsg = String.format("signature is invalid, please check the secret. received signature: %s", signature); + // Return Bad Request + ctx.fail(HttpResponseStatus.BAD_REQUEST.code(), new EventMeshException(errorMsg)); + return; + } + + // if the content type is form data, convert it to json string + if (StringUtils.contains(contentType, "application/x-www-form-urlencoded") + || StringUtils.contains(contentType, "multipart/form-data")) { + /* + Convert form data to json string. There are the following benefits: + 1. Raw form data is not decoded, so it is not easy to process directly. + 2. Converted to reduce storage space by more than 20 percent. Experimental result: 10329 bytes -> 7893 bytes. + */ + JSONObject payloadObj = new JSONObject(); + ctx.request().formAttributes().forEach(entry -> payloadObj.put(entry.getKey(), entry.getValue())); + payloadStr = payloadObj.toJSONString(); + } + + // Create and store the webhook request + Map headerMap = headers.entries().stream() + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + WebhookRequest webhookRequest = new WebhookRequest(PROTOCOL_NAME, ctx.request().absoluteURI(), headerMap, payloadStr, ctx); + + if (!queue.offer(webhookRequest)) { + throw new IllegalStateException("Failed to store the request."); + } + + // Return 200 OK + ctx.response() + .setStatusCode(HttpResponseStatus.OK.code()) + .end(CommonResponse.success().toJsonStr()); + }) + .failureHandler(ctx -> { + log.error("Failed to handle the request from github. ", ctx.failure()); + + // Return Bad Response + ctx.response() + .setStatusCode(ctx.statusCode()) + .end(CommonResponse.base(ctx.failure().getMessage()).toJsonStr()); + }); + } + + + /** + * Validate the signature. + * + * @param signature signature + * @param payload payload + * @param secret secret + * @return boolean + */ + public boolean validateSignature(String signature, String payload, String secret) { + String hash = WebhookConstants.GITHUB_HASH_265_PREFIX; + try { + Mac sha = Mac.getInstance(H_MAC_SHA_265); + SecretKeySpec secretKey = new SecretKeySpec(secret.getBytes(Constants.DEFAULT_CHARSET), H_MAC_SHA_265); + sha.init(secretKey); + byte[] bytes = sha.doFinal(payload.getBytes(Constants.DEFAULT_CHARSET)); + hash += byteArrayToHexString(bytes); + } catch (Exception e) { + throw new EventMeshException("Error occurred while validating the signature.", e); + } + + return hash.equals(signature); + } + + + /** + * Convert the byte array to hex string. + * + * @param bytes bytes + * @return String + */ + private String byteArrayToHexString(byte[] bytes) { + if (bytes == null) { + return ""; + } + + StringBuilder sb = new StringBuilder(); + for (byte b : bytes) { + String hex = Integer.toHexString(0xFF & b); + if (hex.length() == 1) { + // If the length is 1, append 0 + sb.append('0'); + } + sb.append(hex); + } + + return sb.toString(); + } + + + /** + * Convert the message to ConnectRecord. + * + * @param message message + * @return ConnectRecord + */ + @Override + public ConnectRecord convertToConnectRecord(Object message) { + WebhookRequest request = (WebhookRequest) message; + Map headers = request.getHeaders(); + + // Create the ConnectRecord + ConnectRecord connectRecord = new ConnectRecord(null, null, System.currentTimeMillis(), request.getPayload()); + connectRecord.addExtension("id", headers.get(WebhookConstants.GITHUB_DELIVERY)); + connectRecord.addExtension("topic", headers.get(WebhookConstants.GITHUB_EVENT)); + connectRecord.addExtension("source", headers.get(request.getProtocolName())); + connectRecord.addExtension("type", headers.get(WebhookConstants.GITHUB_HOOK_INSTALLATION_TARGET_TYPE)); + return connectRecord; + } + + +} diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.openconnect.api.ConnectorCreateService b/eventmesh-connectors/eventmesh-connector-http/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.openconnect.api.ConnectorCreateService new file mode 100644 index 0000000000..d62ff11992 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.openconnect.api.ConnectorCreateService @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +HTTP-Source=org.apache.eventmesh.connector.http.source.HttpSourceConnector +HTTP-Sink=org.apache.eventmesh.connector.http.sink.HttpSinkConnector diff --git a/eventmesh-connectors/eventmesh-connector-http/src/main/resources/source-config.yml b/eventmesh-connectors/eventmesh-connector-http/src/main/resources/source-config.yml index 9fcc471d3b..0a73e627b0 100644 --- a/eventmesh-connectors/eventmesh-connector-http/src/main/resources/source-config.yml +++ b/eventmesh-connectors/eventmesh-connector-http/src/main/resources/source-config.yml @@ -28,4 +28,9 @@ connectorConfig: connectorName: httpSource path: /test port: 3755 - idleTimeout: 5 \ No newline at end of file + idleTimeout: 5000 # timeunit: ms + maxFormAttributeSize: 1048576 # timeunit: byte, default: 1048576(1MB). This applies only when handling form data submissions. + protocol: CloudEvent # Case insensitive, default: CloudEvent, options: CloudEvent, GitHub, Common + extraConfig: # extra config for different protocol, e.g. GitHub secret + secret: xxxxxxx # GitHub secret + contentType: application/json # GitHub content type \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-http/src/test/java/org/apache/eventmesh/connector/http/source/connector/HttpSinkConnectorTest.java b/eventmesh-connectors/eventmesh-connector-http/src/test/java/org/apache/eventmesh/connector/http/sink/HttpSinkConnectorTest.java similarity index 51% rename from eventmesh-connectors/eventmesh-connector-http/src/test/java/org/apache/eventmesh/connector/http/source/connector/HttpSinkConnectorTest.java rename to eventmesh-connectors/eventmesh-connector-http/src/test/java/org/apache/eventmesh/connector/http/sink/HttpSinkConnectorTest.java index 738df6430b..be2b52e737 100644 --- a/eventmesh-connectors/eventmesh-connector-http/src/test/java/org/apache/eventmesh/connector/http/source/connector/HttpSinkConnectorTest.java +++ b/eventmesh-connectors/eventmesh-connector-http/src/test/java/org/apache/eventmesh/connector/http/sink/HttpSinkConnectorTest.java @@ -15,40 +15,40 @@ * limitations under the License. */ -package org.apache.eventmesh.connector.http.source.connector; +package org.apache.eventmesh.connector.http.sink; + import static org.mockserver.model.HttpRequest.request; -import org.apache.eventmesh.connector.http.sink.HttpSinkConnector; -import org.apache.eventmesh.connector.http.sink.config.HttpSinkConfig; -import org.apache.eventmesh.connector.http.sink.config.HttpWebhookConfig; +import org.apache.eventmesh.common.config.connector.http.HttpSinkConfig; +import org.apache.eventmesh.common.config.connector.http.HttpWebhookConfig; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; import org.apache.eventmesh.openconnect.util.ConfigUtil; +import org.apache.hc.client5.http.fluent.Request; +import org.apache.hc.core5.http.HttpStatus; +import org.apache.hc.core5.net.URIBuilder; + import java.net.URI; +import java.net.URL; import java.util.ArrayList; import java.util.List; import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockserver.integration.ClientAndServer; -import org.mockserver.model.HttpRequest; import org.mockserver.model.HttpResponse; import org.mockserver.model.MediaType; + import com.alibaba.fastjson2.JSON; import com.alibaba.fastjson2.JSONArray; import com.alibaba.fastjson2.JSONObject; -import okhttp3.HttpUrl; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.Response; -import okhttp3.ResponseBody; public class HttpSinkConnectorTest { @@ -56,42 +56,44 @@ public class HttpSinkConnectorTest { private HttpSinkConfig sinkConfig; - private URI severUri; + private URL url; private ClientAndServer mockServer; + private static final AtomicInteger counter = new AtomicInteger(0); @BeforeEach void before() throws Exception { // init sinkConnector - this.sinkConnector = new HttpSinkConnector(); - this.sinkConfig = (HttpSinkConfig) ConfigUtil.parse(sinkConnector.configClass()); - this.sinkConnector.init(this.sinkConfig); - this.sinkConnector.start(); + sinkConnector = new HttpSinkConnector(); + sinkConfig = (HttpSinkConfig) ConfigUtil.parse(sinkConnector.configClass()); + sinkConnector.init(this.sinkConfig); + sinkConnector.start(); - this.severUri = URI.create(sinkConfig.connectorConfig.getUrls()[0]); + url = new URL(sinkConfig.connectorConfig.getUrls()[0]); // start mockServer - mockServer = ClientAndServer.startClientAndServer(severUri.getPort()); + mockServer = ClientAndServer.startClientAndServer(url.getPort()); mockServer.reset() .when( request() .withMethod("POST") - .withPath(severUri.getPath()) + .withPath(url.getPath()) ) .respond( httpRequest -> { - JSONObject requestBody = JSON.parseObject(httpRequest.getBodyAsString()); + // Increase the number of requests received + counter.incrementAndGet(); return HttpResponse.response() .withContentType(MediaType.APPLICATION_JSON) - .withStatusCode(200) + .withStatusCode(HttpStatus.SC_OK) .withBody(new JSONObject() .fluentPut("code", 0) .fluentPut("message", "success") - .fluentPut("data", requestBody.getJSONObject("data").get("data")) .toJSONString() ); // .withDelay(TimeUnit.SECONDS, 10); } ); + } @AfterEach @@ -103,68 +105,61 @@ void after() throws Exception { @Test void testPut() throws Exception { // Create a list of ConnectRecord - final int times = 10; + final int size = 10; List connectRecords = new ArrayList<>(); - for (int i = 0; i < times; i++) { + for (int i = 0; i < size; i++) { ConnectRecord record = createConnectRecord(); connectRecords.add(record); } // Put ConnectRecord sinkConnector.put(connectRecords); - // sleep 5s - Thread.sleep(5000); - - // verify request - HttpRequest[] recordedRequests = mockServer.retrieveRecordedRequests(null); - assert recordedRequests.length == times; + // wait for receiving request + final int times = 5000; // 5 seconds + long start = System.currentTimeMillis(); + while (counter.get() < size) { + if (System.currentTimeMillis() - start > times) { + // timeout + Assertions.fail("The number of requests received=" + counter.get() + " is less than the number of ConnectRecord=" + size); + } else { + Thread.sleep(100); + } + } // verify response HttpWebhookConfig webhookConfig = sinkConfig.connectorConfig.getWebhookConfig(); - String url = new HttpUrl.Builder() - .scheme("http") - .host(severUri.getHost()) - .port(webhookConfig.getPort()) - .addPathSegments(webhookConfig.getExportPath()) - .addQueryParameter("pageNum", "1") - .addQueryParameter("pageSize", "10") - .addQueryParameter("type", "poll") - .build().toString(); - - // build request - Request request = new Request.Builder() - .url(url) - .addHeader("Content-Type", "application/json") + + URI exportUrl = new URIBuilder() + .setScheme("http") + .setHost(url.getHost()) + .setPort(webhookConfig.getPort()) + .setPath(webhookConfig.getExportPath()) + .addParameter("pageNum", "1") + .addParameter("pageSize", "10") + .addParameter("type", "poll") .build(); - OkHttpClient client = new OkHttpClient(); - try (Response response = client.newCall(request).execute()) { - // check response code - if (!response.isSuccessful()) { - throw new RuntimeException("Unexpected response code: " + response); - } - // check response body - ResponseBody responseBody = response.body(); - if (responseBody != null) { - JSONObject jsonObject = JSON.parseObject(responseBody.string()); + Request.get(exportUrl) + .execute() + .handleResponse(response -> { + // check response code + Assertions.assertEquals(HttpStatus.SC_OK, response.getCode()); + // check response body + JSONObject jsonObject = JSON.parseObject(response.getEntity().getContent()); JSONArray pageItems = jsonObject.getJSONArray("pageItems"); - assert pageItems != null && pageItems.size() == times; - - for (int i = 0; i < times; i++) { + Assertions.assertNotNull(pageItems); + Assertions.assertEquals(size, pageItems.size()); + for (int i = 0; i < size; i++) { JSONObject pageItem = pageItems.getJSONObject(i); - assert pageItem != null; - assert pageItem.getJSONObject("data") != null; - assert pageItem.getJSONObject("metadata") != null; + Assertions.assertNotNull(pageItem); } - } - } + return null; + }); } private ConnectRecord createConnectRecord() { - RecordPartition partition = new RecordPartition(); - RecordOffset offset = new RecordOffset(); long timestamp = System.currentTimeMillis(); - return new ConnectRecord(partition, offset, timestamp, UUID.randomUUID().toString()); + return new ConnectRecord(null, null, timestamp, UUID.randomUUID().toString()); } } diff --git a/eventmesh-connectors/eventmesh-connector-http/src/test/java/org/apache/eventmesh/connector/http/source/HttpSourceConnectorTest.java b/eventmesh-connectors/eventmesh-connector-http/src/test/java/org/apache/eventmesh/connector/http/source/HttpSourceConnectorTest.java new file mode 100644 index 0000000000..0dbac47653 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-http/src/test/java/org/apache/eventmesh/connector/http/source/HttpSourceConnectorTest.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.http.source; + + +import org.apache.eventmesh.common.config.connector.http.HttpSourceConfig; +import org.apache.eventmesh.common.config.connector.http.SourceConnectorConfig; +import org.apache.eventmesh.common.utils.JsonUtils; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; +import org.apache.eventmesh.openconnect.util.ConfigUtil; + +import org.apache.hc.client5.http.fluent.Request; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.HttpStatus; +import org.apache.hc.core5.http.io.entity.StringEntity; + +import java.io.IOException; +import java.net.URL; +import java.util.List; +import java.util.Objects; +import java.util.UUID; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + + +class HttpSourceConnectorTest { + + private static HttpSourceConnector connector; + private static String url; + private static final String expectedMessage = "testHttpMessage"; + private static final int batchSize = 10; + + + @BeforeAll + static void setUpAll() throws Exception { + connector = new HttpSourceConnector(); + final HttpSourceConfig sourceConfig = (HttpSourceConfig) ConfigUtil.parse(connector.configClass()); + final SourceConnectorConfig config = sourceConfig.getConnectorConfig(); + // initialize and start the connector + connector.init(sourceConfig); + connector.start(); + + // wait for the connector to start + long timeout = 5000; // 5 seconds + long start = System.currentTimeMillis(); + while (!connector.isStarted()) { + if (System.currentTimeMillis() - start > timeout) { + // timeout + Assertions.fail("Failed to start the connector"); + } else { + Thread.sleep(100); + } + } + + url = new URL("http", "127.0.0.1", config.getPort(), config.getPath()).toString(); + } + + @AfterAll + static void tearDownAll() throws IOException { + connector.stop(); + } + + + @Test + void testPollForBinaryRequest() { + for (int i = 0; i < batchSize; i++) { + try { + // Set the request body + StringEntity entity = new StringEntity(expectedMessage, ContentType.TEXT_PLAIN); + + Request.post(url) + .addHeader("Content-Type", "text/plain") + .addHeader("ce-id", String.valueOf(UUID.randomUUID())) + .addHeader("ce-specversion", "1.0") + .addHeader("ce-type", "com.example.someevent") + .addHeader("ce-source", "/mycontext") + .addHeader("ce-subject", "test") + .body(entity) + .execute() + .handleResponse(res -> { + Assertions.assertEquals(HttpStatus.SC_OK, res.getCode()); + return null; + }); + } catch (IOException e) { + Assertions.fail("Failed to send request", e); + } + } + List res = connector.poll(); + Assertions.assertEquals(batchSize, res.size()); + for (ConnectRecord r : res) { + Assertions.assertEquals(expectedMessage, new String((byte[]) r.getData())); + } + } + + @Test + void testPollForStructuredRequest() { + for (int i = 0; i < batchSize; i++) { + try { + // Create a CloudEvent + TestEvent event = new TestEvent(); + event.id = String.valueOf(UUID.randomUUID()); + event.specversion = "1.0"; + event.type = "com.example.someevent"; + event.source = "/mycontext"; + event.subject = "test"; + event.datacontenttype = "text/plain"; + event.data = expectedMessage; + + // Set the request body + StringEntity entity = new StringEntity(Objects.requireNonNull(JsonUtils.toJSONString(event)), ContentType.APPLICATION_JSON); + + // Send the request and return the response + Request.post(url) + .addHeader("Content-Type", "application/cloudevents+json") + .body(entity) + .execute() + .handleResponse(res -> { + Assertions.assertEquals(HttpStatus.SC_OK, res.getCode()); + return null; + }); + } catch (IOException e) { + Assertions.fail("Failed to send request", e); + } + } + List res = connector.poll(); + Assertions.assertEquals(batchSize, res.size()); + for (ConnectRecord r : res) { + Assertions.assertEquals(expectedMessage, new String((byte[]) r.getData())); + } + } + + + @Test + void testPollForInvalidRequest() { + // Send a bad request. + try { + Request.post(url) + .addHeader("Content-Type", "text/plain") + .execute() + .handleResponse(res -> { + // Check the response code + Assertions.assertEquals(HttpStatus.SC_BAD_REQUEST, res.getCode()); + return null; + }); + } catch (IOException e) { + Assertions.fail("Failed to send request", e); + } + } + + class TestEvent { + + public String specversion; + public String type; + public String source; + public String subject; + public String datacontenttype; + public String id; + + public String data; + } +} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-http/src/test/java/org/apache/eventmesh/connector/http/source/connector/HttpSourceConnectorTest.java b/eventmesh-connectors/eventmesh-connector-http/src/test/java/org/apache/eventmesh/connector/http/source/connector/HttpSourceConnectorTest.java deleted file mode 100644 index 35d58b75c1..0000000000 --- a/eventmesh-connectors/eventmesh-connector-http/src/test/java/org/apache/eventmesh/connector/http/source/connector/HttpSourceConnectorTest.java +++ /dev/null @@ -1,145 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.connector.http.source.connector; - -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.connector.http.source.config.HttpSourceConfig; -import org.apache.eventmesh.connector.http.source.config.SourceConnectorConfig; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.util.ConfigUtil; - -import org.apache.http.HttpHeaders; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.utils.URIBuilder; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; - -import java.util.List; -import java.util.UUID; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -class HttpSourceConnectorTest { - - private HttpSourceConnector connector; - private SourceConnectorConfig config; - private CloseableHttpClient httpClient; - private String uri; - private final String expectedMessage = "testHttpMessage"; - - @BeforeEach - void setUp() throws Exception { - connector = new HttpSourceConnector(); - HttpSourceConfig sourceConfig = (HttpSourceConfig) ConfigUtil.parse(connector.configClass()); - config = sourceConfig.getConnectorConfig(); - connector.init(sourceConfig); - connector.start(); - - uri = new URIBuilder().setScheme("http").setHost("127.0.0.1").setPort(config.getPort()).setPath(config.getPath()).build().toString(); - - httpClient = HttpClients.createDefault(); - } - - @Test - void testPoll() throws Exception { - final int batchSize = 10; - // test binary content mode - for (int i = 0; i < batchSize; i++) { - HttpResponse resp = mockBinaryRequest(); - Assertions.assertEquals(resp.getStatusLine().getStatusCode(), HttpStatus.SC_OK); - - } - List res = connector.poll(); - Assertions.assertEquals(batchSize, res.size()); - for (ConnectRecord r : res) { - Assertions.assertEquals(expectedMessage, new String((byte[]) r.getData())); - } - - // test structured content mode - for (int i = 0; i < batchSize; i++) { - HttpResponse resp = mockStructuredRequest(); - Assertions.assertEquals(resp.getStatusLine().getStatusCode(), HttpStatus.SC_OK); - } - res = connector.poll(); - Assertions.assertEquals(batchSize, res.size()); - for (ConnectRecord r : res) { - Assertions.assertEquals(expectedMessage, new String((byte[]) r.getData())); - } - - // test invalid requests - HttpPost invalidPost = new HttpPost(uri); - invalidPost.setHeader(HttpHeaders.CONTENT_TYPE, "text/plain"); - invalidPost.setHeader("ce-id", String.valueOf(UUID.randomUUID())); - HttpResponse resp = httpClient.execute(invalidPost); - Assertions.assertEquals(HttpStatus.SC_BAD_REQUEST, resp.getStatusLine().getStatusCode()); - } - - HttpResponse mockBinaryRequest() throws Exception { - HttpPost httpPost = new HttpPost(uri); - httpPost.setHeader(HttpHeaders.CONTENT_TYPE, "text/plain"); - httpPost.setHeader("ce-id", String.valueOf(UUID.randomUUID())); - httpPost.setHeader("ce-specversion", "1.0"); - httpPost.setHeader("ce-type", "com.example.someevent"); - httpPost.setHeader("ce-source", "/mycontext"); - httpPost.setHeader("ce-subject", "test"); - httpPost.setEntity(new StringEntity(expectedMessage)); - - return httpClient.execute(httpPost); - } - - HttpResponse mockStructuredRequest() throws Exception { - HttpPost httpPost = new HttpPost(uri); - // according to the CloudEvent specification, a json format event MUST use the media type `application/cloudevents+json` - httpPost.setHeader(HttpHeaders.CONTENT_TYPE, "application/cloudevents+json"); - TestEvent event = new TestEvent(); - event.id = String.valueOf(UUID.randomUUID()); - event.specversion = "1.0"; - event.type = "com.example.someevent"; - event.source = "/mycontext"; - event.subject = "test"; - event.datacontenttype = "text/plain"; - event.data = expectedMessage; - httpPost.setEntity(new StringEntity(JsonUtils.toJSONString(event))); - - return httpClient.execute(httpPost); - } - - @AfterEach - void tearDown() throws Exception { - connector.stop(); - httpClient.close(); - } - - class TestEvent { - - public String specversion; - public String type; - public String source; - public String subject; - public String datacontenttype; - public String id; - - public String data; - } -} \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-http/src/test/resources/source-config.yml b/eventmesh-connectors/eventmesh-connector-http/src/test/resources/source-config.yml index 0a3e68d070..336bb2cb5e 100644 --- a/eventmesh-connectors/eventmesh-connector-http/src/test/resources/source-config.yml +++ b/eventmesh-connectors/eventmesh-connector-http/src/test/resources/source-config.yml @@ -27,4 +27,10 @@ pubSubConfig: connectorConfig: connectorName: httpSource path: /test - port: 3755 \ No newline at end of file + port: 3755 + idleTimeout: 5000 # timeunit: ms + maxFormAttributeSize: 1048576 # timeunit: byte, default: 1048576(1MB). This applies only when handling form data submissions. + protocol: CloudEvent # Case insensitive, default: CloudEvent, options: CloudEvent, GitHub, Common + extraConfig: # extra config for different protocol, e.g. GitHub secret + secret: xxxxxxx # GitHub secret + contentType: application/json # GitHub content type \ No newline at end of file diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/build.gradle b/eventmesh-connectors/eventmesh-connector-jdbc/build.gradle index 7ee333e868..b70bf6d357 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/build.gradle +++ b/eventmesh-connectors/eventmesh-connector-jdbc/build.gradle @@ -34,15 +34,15 @@ packageSources { } dependencies { - antlr("org.antlr:antlr4:4.13.0") + antlr("org.antlr:antlr4:4.13.1") implementation 'org.antlr:antlr4-runtime:4.13.1' - implementation 'com.alibaba:druid:1.2.20' + implementation 'com.alibaba:druid:1.2.23' compileOnly 'org.hibernate:hibernate-core:5.6.15.Final' implementation project(":eventmesh-common") implementation project(":eventmesh-openconnect:eventmesh-openconnect-java") implementation project(":eventmesh-spi") - implementation 'com.zendesk:mysql-binlog-connector-java:0.29.2' - compileOnly 'mysql:mysql-connector-java:8.0.32' + implementation 'com.zendesk:mysql-binlog-connector-java:0.30.1' + compileOnly 'com.mysql:mysql-connector-j' compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/config/JdbcServerConfig.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/config/JdbcServerConfig.java index 451ecf71e9..2b3e614774 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/config/JdbcServerConfig.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/config/JdbcServerConfig.java @@ -17,7 +17,8 @@ package org.apache.eventmesh.connector.jdbc.config; -import org.apache.eventmesh.openconnect.api.config.Config; + +import org.apache.eventmesh.common.config.connector.Config; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/JdbcConnection.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/JdbcConnection.java index d1802d8b96..70d553517f 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/JdbcConnection.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/JdbcConnection.java @@ -17,8 +17,8 @@ package org.apache.eventmesh.connector.jdbc.connection; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcConfig; import org.apache.eventmesh.connector.jdbc.JdbcDriverMetaData; -import org.apache.eventmesh.connector.jdbc.config.JdbcConfig; import org.apache.commons.lang3.StringUtils; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/mysql/MysqlJdbcConnection.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/mysql/MysqlJdbcConnection.java index 2a7514c09a..d62aebba56 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/mysql/MysqlJdbcConnection.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/connection/mysql/MysqlJdbcConnection.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.jdbc.connection.mysql; -import org.apache.eventmesh.connector.jdbc.config.JdbcConfig; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcConfig; import org.apache.eventmesh.connector.jdbc.connection.JdbcConnection; import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlDialectSql; import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/AbstractGeneralDatabaseDialect.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/AbstractGeneralDatabaseDialect.java index 0a7463a187..0ba6ab715f 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/AbstractGeneralDatabaseDialect.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/AbstractGeneralDatabaseDialect.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.jdbc.dialect; -import org.apache.eventmesh.connector.jdbc.config.JdbcConfig; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcConfig; import org.apache.eventmesh.connector.jdbc.connection.JdbcConnection; import org.apache.eventmesh.connector.jdbc.exception.JdbcConnectionException; import org.apache.eventmesh.connector.jdbc.table.catalog.Column; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/DatabaseDialectFactory.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/DatabaseDialectFactory.java index aad7984520..3ad607e455 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/DatabaseDialectFactory.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/DatabaseDialectFactory.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.jdbc.dialect; -import org.apache.eventmesh.connector.jdbc.config.JdbcConfig; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcConfig; import org.apache.eventmesh.spi.EventMeshExtensionType; import org.apache.eventmesh.spi.EventMeshSPI; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/mysql/MysqlDatabaseDialect.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/mysql/MysqlDatabaseDialect.java index acd8730c2e..1a4bb02fb6 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/mysql/MysqlDatabaseDialect.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/mysql/MysqlDatabaseDialect.java @@ -17,9 +17,9 @@ package org.apache.eventmesh.connector.jdbc.dialect.mysql; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcConfig; import org.apache.eventmesh.connector.jdbc.DataTypeConvertor; import org.apache.eventmesh.connector.jdbc.JdbcDriverMetaData; -import org.apache.eventmesh.connector.jdbc.config.JdbcConfig; import org.apache.eventmesh.connector.jdbc.connection.mysql.MysqlJdbcConnection; import org.apache.eventmesh.connector.jdbc.dialect.AbstractGeneralDatabaseDialect; import org.apache.eventmesh.connector.jdbc.dialect.DatabaseType; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/mysql/MysqlDatabaseDialectFactory.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/mysql/MysqlDatabaseDialectFactory.java index eb4fbe3275..b5fb87a016 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/mysql/MysqlDatabaseDialectFactory.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/dialect/mysql/MysqlDatabaseDialectFactory.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.jdbc.dialect.mysql; -import org.apache.eventmesh.connector.jdbc.config.JdbcConfig; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcConfig; import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialectFactory; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/JdbcSinkConnector.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/JdbcSinkConnector.java index 7a5c68f581..cc00f1e142 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/JdbcSinkConnector.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/JdbcSinkConnector.java @@ -17,17 +17,17 @@ package org.apache.eventmesh.connector.jdbc.sink; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcConfig; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSinkConfig; import org.apache.eventmesh.common.utils.JsonUtils; import org.apache.eventmesh.connector.jdbc.JdbcConnectData; -import org.apache.eventmesh.connector.jdbc.config.JdbcConfig; import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialectFactory; -import org.apache.eventmesh.connector.jdbc.sink.config.JdbcSinkConfig; import org.apache.eventmesh.connector.jdbc.sink.handle.DefaultSinkRecordHandler; import org.apache.eventmesh.connector.jdbc.sink.handle.SinkRecordHandler; import org.apache.eventmesh.connector.jdbc.sink.hibernate.HibernateConfiguration; import org.apache.eventmesh.connector.jdbc.source.JdbcAllFactoryLoader; -import org.apache.eventmesh.openconnect.api.config.Config; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; import org.apache.eventmesh.openconnect.api.sink.Sink; @@ -139,6 +139,11 @@ public String name() { return this.sinkConfig.getSinkConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + /** * Stops the Connector. * diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/DefaultSinkRecordHandler.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/DefaultSinkRecordHandler.java index a4ba77ae5d..db684d63c7 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/DefaultSinkRecordHandler.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/sink/handle/DefaultSinkRecordHandler.java @@ -17,6 +17,7 @@ package org.apache.eventmesh.connector.jdbc.sink.handle; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSinkConfig; import org.apache.eventmesh.common.utils.LogUtil; import org.apache.eventmesh.connector.jdbc.CatalogChanges; import org.apache.eventmesh.connector.jdbc.DataChanges; @@ -28,7 +29,6 @@ import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; import org.apache.eventmesh.connector.jdbc.event.DataChangeEventType; import org.apache.eventmesh.connector.jdbc.event.SchemaChangeEventType; -import org.apache.eventmesh.connector.jdbc.sink.config.JdbcSinkConfig; import org.apache.eventmesh.connector.jdbc.source.SourceMateData; import org.apache.eventmesh.connector.jdbc.table.catalog.Column; import org.apache.eventmesh.connector.jdbc.type.Type; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEngine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEngine.java index 08c3823725..2088e75632 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEngine.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractEngine.java @@ -18,9 +18,9 @@ package org.apache.eventmesh.connector.jdbc.source; import org.apache.eventmesh.common.ThreadWrapper; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSourceConfig; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.SourceConnectorConfig; import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig; -import org.apache.eventmesh.connector.jdbc.source.config.SourceConnectorConfig; import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; import org.apache.commons.collections4.CollectionUtils; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractJdbcTaskManager.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractJdbcTaskManager.java index b398ffa17b..ff2fd8ba00 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractJdbcTaskManager.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/AbstractJdbcTaskManager.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.jdbc.source; -import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSourceConfig; import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; import java.util.ArrayList; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcSourceConnector.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcSourceConnector.java index 8d7d9cb66d..ecc5a44154 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcSourceConnector.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/JdbcSourceConnector.java @@ -17,10 +17,12 @@ package org.apache.eventmesh.connector.jdbc.source; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.SourceConfig; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSourceConfig; import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialectFactory; import org.apache.eventmesh.connector.jdbc.event.Event; -import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig; import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.CdcEngine; import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.CdcEngineFactory; import org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotEngine; @@ -28,8 +30,6 @@ import org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotResult; import org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotResult.SnapshotResultStatus; import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; -import org.apache.eventmesh.openconnect.api.config.Config; -import org.apache.eventmesh.openconnect.api.config.SourceConfig; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SourceConnector; import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; @@ -142,7 +142,9 @@ private void doInit() { this.dispatcher = new EventDispatcher(this.sourceJdbcTaskManager); - this.taskManagerCoordinator = new TaskManagerCoordinator(); + this.taskManagerCoordinator = new TaskManagerCoordinator(sourceConfig.getPollConfig().getCapacity(), + sourceConfig.getPollConfig().getMaxBatchSize(), + sourceConfig.getPollConfig().getMaxWaitTime()); this.taskManagerCoordinator.registerTaskManager(SourceJdbcTaskManager.class.getName(), sourceJdbcTaskManager); this.taskManagerCoordinator.init(); } @@ -192,6 +194,11 @@ public String name() { return "JDBC Source Connector"; } + @Override + public void onException(ConnectRecord record) { + + } + /** * Stops the Connector. * @@ -204,9 +211,6 @@ public void stop() throws Exception { @Override public List poll() { - - List connectRecords = this.taskManagerCoordinator.poll(); - - return connectRecords; + return this.taskManagerCoordinator.poll(); } } diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceJdbcTaskManager.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceJdbcTaskManager.java index be38ffb75a..0625dbfad7 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceJdbcTaskManager.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/SourceJdbcTaskManager.java @@ -17,15 +17,17 @@ package org.apache.eventmesh.connector.jdbc.source; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSourceConfig; +import org.apache.eventmesh.common.remote.offset.RecordOffset; +import org.apache.eventmesh.common.remote.offset.RecordPartition; +import org.apache.eventmesh.common.remote.offset.jdbc.JdbcRecordOffset; +import org.apache.eventmesh.common.remote.offset.jdbc.JdbcRecordPartition; import org.apache.eventmesh.connector.jdbc.JdbcConnectData; import org.apache.eventmesh.connector.jdbc.event.Event; -import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig; import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.RandomTaskSelectStrategy; import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.TaskSelectStrategy; import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; import java.util.Collections; import java.util.HashSet; @@ -68,8 +70,8 @@ private void doHandleEvent(Event event) { return; } JdbcConnectData jdbcConnectData = event.getJdbcConnectData(); - RecordPartition partition = new RecordPartition(); - RecordOffset offset = new RecordOffset(); + RecordPartition partition = new JdbcRecordPartition(); + RecordOffset offset = new JdbcRecordOffset(); ConnectRecord record = new ConnectRecord(partition, offset, System.currentTimeMillis(), jdbcConnectData); List records = Collections.singletonList(record); for (TaskManagerListener listener : listeners) { diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/TaskManagerCoordinator.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/TaskManagerCoordinator.java index c299fbc531..8efb8cbc71 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/TaskManagerCoordinator.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/TaskManagerCoordinator.java @@ -40,16 +40,16 @@ @Slf4j public class TaskManagerCoordinator { - private static final int BATCH_MAX = 10; - private static final int DEFAULT_QUEUE_SIZE = 1 << 13; + private final BlockingQueue recordBlockingQueue; + private final Map taskManagerCache = new HashMap<>(8); + private final int maxBatchSize; + private final long maxPollTimeout; - private BlockingQueue recordBlockingQueue = new LinkedBlockingQueue<>(DEFAULT_QUEUE_SIZE); - private Map taskManagerCache = new HashMap<>(8); - /** - * Constructs a new TaskManagerCoordinator. - */ - public TaskManagerCoordinator() { + public TaskManagerCoordinator(int capacity, int maxBatchSize, long maxPollTimeout) { + this.recordBlockingQueue = new LinkedBlockingQueue<>(capacity); + this.maxBatchSize = maxBatchSize; + this.maxPollTimeout = maxPollTimeout; } /** @@ -96,10 +96,13 @@ public void start() { * @return A list of ConnectRecords, up to the maximum batch size defined by BATCH_MAX. */ public List poll() { - List records = new ArrayList<>(BATCH_MAX); - for (int index = 0; index < BATCH_MAX; ++index) { + long startTime = System.currentTimeMillis(); + long remainingTime = maxPollTimeout; + + List records = new ArrayList<>(maxBatchSize); + for (int index = 0; index < maxBatchSize; ++index) { try { - ConnectRecord record = recordBlockingQueue.poll(3, TimeUnit.SECONDS); + ConnectRecord record = recordBlockingQueue.poll(remainingTime, TimeUnit.MILLISECONDS); if (Objects.isNull(record)) { break; } @@ -107,6 +110,10 @@ public List poll() { log.debug("record:{}", JsonUtils.toJSONString(record)); } records.add(record); + + // calculate elapsed time and update remaining time for next poll + long elapsedTime = System.currentTimeMillis() - startTime; + remainingTime = maxPollTimeout > elapsedTime ? maxPollTimeout - elapsedTime : 0; } catch (InterruptedException e) { break; } diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/MysqlAntlr4DdlParser.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/MysqlAntlr4DdlParser.java index 9bb110815f..261da6192b 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/MysqlAntlr4DdlParser.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/MysqlAntlr4DdlParser.java @@ -17,6 +17,7 @@ package org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSourceConfig; import org.apache.eventmesh.connector.jdbc.antlr4.Antlr4DdlParser; import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlLexer; import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser; @@ -25,7 +26,6 @@ import org.apache.eventmesh.connector.jdbc.antlr4.listener.Antlr4DdlParserListener; import org.apache.eventmesh.connector.jdbc.ddl.DdlParserCallback; import org.apache.eventmesh.connector.jdbc.event.Event; -import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig; import org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.listener.MySqlAntlr4DdlParserListener; import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; import org.apache.eventmesh.connector.jdbc.utils.JdbcStringUtils; @@ -113,7 +113,7 @@ public void runIfAllNotNull(Runnable runner, Object... nullableObjects) { */ public TableId parseTableId(String fullIdText) { // Remove special characters from the full ID text - String sanitizedText = StringUtils.replaceEach(fullIdText, new String[]{"'\\''", "\"", "`"}, new String[]{"", "", ""}); + String sanitizedText = StringUtils.replaceEach(fullIdText, new String[] {"'\\''", "\"", "`"}, new String[] {"", "", ""}); // Split the sanitized text by dot (.) to separate catalog and table name String[] split = sanitizedText.split("\\."); diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/CreateDatabaseParserListener.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/CreateDatabaseParserListener.java index a5d15820a9..71b4866e74 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/CreateDatabaseParserListener.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/CreateDatabaseParserListener.java @@ -17,6 +17,7 @@ package org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.listener; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.SourceConnectorConfig; import org.apache.eventmesh.connector.jdbc.CatalogChanges; import org.apache.eventmesh.connector.jdbc.Payload; import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.CreateDatabaseContext; @@ -24,7 +25,6 @@ import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParserBaseListener; import org.apache.eventmesh.connector.jdbc.event.CreateDatabaseEvent; import org.apache.eventmesh.connector.jdbc.event.SchemaChangeEventType; -import org.apache.eventmesh.connector.jdbc.source.config.SourceConnectorConfig; import org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.MysqlAntlr4DdlParser; import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlSourceMateData; import org.apache.eventmesh.connector.jdbc.table.catalog.CatalogSchema; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/CreateTableParserListener.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/CreateTableParserListener.java index 4e30da93f9..044403f778 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/CreateTableParserListener.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/CreateTableParserListener.java @@ -17,6 +17,7 @@ package org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.listener; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.SourceConnectorConfig; import org.apache.eventmesh.connector.jdbc.CatalogChanges; import org.apache.eventmesh.connector.jdbc.Payload; import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.ColumnCreateTableContext; @@ -29,7 +30,6 @@ import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.TableOptionEngineContext; import org.apache.eventmesh.connector.jdbc.event.CreateTableEvent; import org.apache.eventmesh.connector.jdbc.event.SchemaChangeEventType; -import org.apache.eventmesh.connector.jdbc.source.config.SourceConnectorConfig; import org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.MysqlAntlr4DdlParser; import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlSourceMateData; import org.apache.eventmesh.connector.jdbc.table.catalog.Table; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/DropDatabaseParserListener.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/DropDatabaseParserListener.java index 22a25fcd32..c582df4a15 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/DropDatabaseParserListener.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/antlr4/mysql/listener/DropDatabaseParserListener.java @@ -17,13 +17,13 @@ package org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.listener; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.SourceConnectorConfig; import org.apache.eventmesh.connector.jdbc.CatalogChanges; import org.apache.eventmesh.connector.jdbc.Payload; import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParser.DropDatabaseContext; import org.apache.eventmesh.connector.jdbc.antlr4.autogeneration.MySqlParserBaseListener; import org.apache.eventmesh.connector.jdbc.event.DropDatabaseEvent; import org.apache.eventmesh.connector.jdbc.event.SchemaChangeEventType; -import org.apache.eventmesh.connector.jdbc.source.config.SourceConnectorConfig; import org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.MysqlAntlr4DdlParser; import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlSourceMateData; import org.apache.eventmesh.connector.jdbc.table.catalog.CatalogSchema; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/AbstractCdcEngine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/AbstractCdcEngine.java index 1fb1a95579..e7538659bd 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/AbstractCdcEngine.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/AbstractCdcEngine.java @@ -18,13 +18,13 @@ package org.apache.eventmesh.connector.jdbc.source.dialect.cdc; import org.apache.eventmesh.common.ThreadWrapper; +import org.apache.eventmesh.common.config.connector.SourceConfig; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSourceConfig; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.SourceConnectorConfig; import org.apache.eventmesh.connector.jdbc.JdbcContext; import org.apache.eventmesh.connector.jdbc.ddl.DdlParser; import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig; -import org.apache.eventmesh.connector.jdbc.source.config.SourceConnectorConfig; import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; -import org.apache.eventmesh.openconnect.api.config.SourceConfig; import org.apache.commons.collections4.CollectionUtils; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/CdcEngineFactory.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/CdcEngineFactory.java index e08e1c8216..d53499e147 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/CdcEngineFactory.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/CdcEngineFactory.java @@ -17,8 +17,8 @@ package org.apache.eventmesh.connector.jdbc.source.dialect.cdc; +import org.apache.eventmesh.common.config.connector.SourceConfig; import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.openconnect.api.config.SourceConfig; import org.apache.eventmesh.spi.EventMeshExtensionType; import org.apache.eventmesh.spi.EventMeshSPI; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/MysqlCdcEngine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/MysqlCdcEngine.java index 5650c3d0cc..22e9366f1e 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/MysqlCdcEngine.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/MysqlCdcEngine.java @@ -18,13 +18,16 @@ package org.apache.eventmesh.connector.jdbc.source.dialect.cdc.mysql; import org.apache.eventmesh.common.EventMeshThreadFactory; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcConfig; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSourceConfig; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.MysqlConfig; import org.apache.eventmesh.connector.jdbc.CatalogChanges; import org.apache.eventmesh.connector.jdbc.DataChanges; import org.apache.eventmesh.connector.jdbc.DataChanges.Builder; import org.apache.eventmesh.connector.jdbc.Field; import org.apache.eventmesh.connector.jdbc.Payload; import org.apache.eventmesh.connector.jdbc.Schema; -import org.apache.eventmesh.connector.jdbc.config.JdbcConfig; import org.apache.eventmesh.connector.jdbc.connection.mysql.MysqlJdbcConnection; import org.apache.eventmesh.connector.jdbc.dialect.mysql.MysqlDatabaseDialect; import org.apache.eventmesh.connector.jdbc.event.DeleteDataEvent; @@ -33,8 +36,6 @@ import org.apache.eventmesh.connector.jdbc.event.InsertDataEvent; import org.apache.eventmesh.connector.jdbc.event.SchemaChangeEventType; import org.apache.eventmesh.connector.jdbc.event.UpdateDataEvent; -import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig; -import org.apache.eventmesh.connector.jdbc.source.config.MysqlConfig; import org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.MysqlAntlr4DdlParser; import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.AbstractCdcEngine; import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.mysql.RowDeserializers.DeleteRowsEventMeshDeserializer; @@ -51,7 +52,6 @@ import org.apache.eventmesh.connector.jdbc.table.catalog.TableSchema; import org.apache.eventmesh.connector.jdbc.table.catalog.mysql.MysqlDefaultValueConvertorImpl; import org.apache.eventmesh.connector.jdbc.table.type.Pair; -import org.apache.eventmesh.openconnect.api.config.Config; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/MysqlCdcEngineFactory.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/MysqlCdcEngineFactory.java index 35e722fe12..2a87969cf8 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/MysqlCdcEngineFactory.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/cdc/mysql/MysqlCdcEngineFactory.java @@ -17,12 +17,12 @@ package org.apache.eventmesh.connector.jdbc.source.dialect.cdc.mysql; +import org.apache.eventmesh.common.config.connector.SourceConfig; import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; import org.apache.eventmesh.connector.jdbc.dialect.mysql.MysqlDatabaseDialect; import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.AbstractCdcEngineFactory; import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.CdcEngine; import org.apache.eventmesh.connector.jdbc.source.dialect.cdc.CdcEngineFactory; -import org.apache.eventmesh.openconnect.api.config.SourceConfig; import org.apache.commons.lang3.StringUtils; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlJdbcContext.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlJdbcContext.java index 0dcf7ad299..8775d4d488 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlJdbcContext.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/mysql/MysqlJdbcContext.java @@ -17,12 +17,12 @@ package org.apache.eventmesh.connector.jdbc.source.dialect.mysql; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSourceConfig; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.SourceConnectorConfig; import org.apache.eventmesh.connector.jdbc.PartitionOffSetContextPair; import org.apache.eventmesh.connector.jdbc.UniversalJdbcContext; import org.apache.eventmesh.connector.jdbc.context.mysql.MysqlOffsetContext; import org.apache.eventmesh.connector.jdbc.context.mysql.MysqlPartition; -import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig; -import org.apache.eventmesh.connector.jdbc.source.config.SourceConnectorConfig; import org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.MysqlAntlr4DdlParser; import org.apache.commons.lang3.StringUtils; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/AbstractSnapshotEngine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/AbstractSnapshotEngine.java index 96302a5a87..3c46970182 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/AbstractSnapshotEngine.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/AbstractSnapshotEngine.java @@ -18,6 +18,7 @@ package org.apache.eventmesh.connector.jdbc.source.dialect.snapshot; import org.apache.eventmesh.common.ThreadPoolFactory; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSourceConfig; import org.apache.eventmesh.connector.jdbc.DataChanges; import org.apache.eventmesh.connector.jdbc.DataChanges.Builder; import org.apache.eventmesh.connector.jdbc.Field; @@ -33,7 +34,6 @@ import org.apache.eventmesh.connector.jdbc.event.InsertDataEvent; import org.apache.eventmesh.connector.jdbc.source.AbstractEngine; import org.apache.eventmesh.connector.jdbc.source.SourceMateData; -import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig; import org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotResult.SnapshotResultStatus; import org.apache.eventmesh.connector.jdbc.table.catalog.Column; import org.apache.eventmesh.connector.jdbc.table.catalog.TableId; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotEngineFactory.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotEngineFactory.java index d573d7081a..30dfe2e997 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotEngineFactory.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/SnapshotEngineFactory.java @@ -17,9 +17,9 @@ package org.apache.eventmesh.connector.jdbc.source.dialect.snapshot; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSourceConfig; import org.apache.eventmesh.connector.jdbc.JdbcContext; import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; -import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig; import org.apache.eventmesh.spi.EventMeshExtensionType; import org.apache.eventmesh.spi.EventMeshSPI; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/mysql/MysqlSnapshotEngine.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/mysql/MysqlSnapshotEngine.java index e76ab49a3e..b6d6d7f8bf 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/mysql/MysqlSnapshotEngine.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/mysql/MysqlSnapshotEngine.java @@ -17,6 +17,8 @@ package org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.mysql; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSourceConfig; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.MysqlConfig; import org.apache.eventmesh.connector.jdbc.CatalogChanges; import org.apache.eventmesh.connector.jdbc.connection.mysql.MysqlJdbcConnection; import org.apache.eventmesh.connector.jdbc.context.mysql.MysqlOffsetContext; @@ -26,8 +28,6 @@ import org.apache.eventmesh.connector.jdbc.event.EventConsumer; import org.apache.eventmesh.connector.jdbc.event.SchemaChangeEventType; import org.apache.eventmesh.connector.jdbc.source.SourceMateData; -import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig; -import org.apache.eventmesh.connector.jdbc.source.config.MysqlConfig; import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlConstants; import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlDialectSql; import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlJdbcContext; diff --git a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/mysql/MysqlSnapshotEngineFactory.java b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/mysql/MysqlSnapshotEngineFactory.java index 210ded0edf..bac2bdafba 100644 --- a/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/mysql/MysqlSnapshotEngineFactory.java +++ b/eventmesh-connectors/eventmesh-connector-jdbc/src/main/java/org/apache/eventmesh/connector/jdbc/source/dialect/snapshot/mysql/MysqlSnapshotEngineFactory.java @@ -17,9 +17,9 @@ package org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.mysql; +import org.apache.eventmesh.common.config.connector.rdb.jdbc.JdbcSourceConfig; import org.apache.eventmesh.connector.jdbc.dialect.DatabaseDialect; import org.apache.eventmesh.connector.jdbc.dialect.mysql.MysqlDatabaseDialect; -import org.apache.eventmesh.connector.jdbc.source.config.JdbcSourceConfig; import org.apache.eventmesh.connector.jdbc.source.dialect.antlr4.mysql.MysqlAntlr4DdlParser; import org.apache.eventmesh.connector.jdbc.source.dialect.mysql.MysqlJdbcContext; import org.apache.eventmesh.connector.jdbc.source.dialect.snapshot.SnapshotEngine; diff --git a/eventmesh-connectors/eventmesh-connector-kafka/build.gradle b/eventmesh-connectors/eventmesh-connector-kafka/build.gradle index 39dcbf656e..06e4fe97b3 100644 --- a/eventmesh-connectors/eventmesh-connector-kafka/build.gradle +++ b/eventmesh-connectors/eventmesh-connector-kafka/build.gradle @@ -16,9 +16,10 @@ */ dependencies { + implementation project(":eventmesh-common") implementation project(":eventmesh-openconnect:eventmesh-openconnect-java") implementation 'io.cloudevents:cloudevents-kafka:2.5.0' - implementation 'org.apache.kafka:kafka-clients:3.6.2' + implementation 'org.apache.kafka:kafka-clients:3.9.0' compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' } diff --git a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/config/KafkaServerConfig.java b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/config/KafkaServerConfig.java index 2c9ef71fb4..ccbabf2676 100644 --- a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/config/KafkaServerConfig.java +++ b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/config/KafkaServerConfig.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.kafka.config; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/sink/connector/KafkaSinkConnector.java b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/sink/connector/KafkaSinkConnector.java index 49b7e88f19..0adafc1ce6 100644 --- a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/sink/connector/KafkaSinkConnector.java +++ b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/sink/connector/KafkaSinkConnector.java @@ -17,8 +17,8 @@ package org.apache.eventmesh.connector.kafka.sink.connector; -import org.apache.eventmesh.connector.kafka.sink.config.KafkaSinkConfig; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.mq.kafka.KafkaSinkConfig; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; import org.apache.eventmesh.openconnect.api.sink.Sink; @@ -94,6 +94,11 @@ public String name() { return this.sinkConfig.getConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() { producer.close(); diff --git a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/source/connector/KafkaSourceConnector.java b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/source/connector/KafkaSourceConnector.java index 99fa73b580..f771e907cb 100644 --- a/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/source/connector/KafkaSourceConnector.java +++ b/eventmesh-connectors/eventmesh-connector-kafka/src/main/java/org/apache/eventmesh/connector/kafka/source/connector/KafkaSourceConnector.java @@ -17,14 +17,16 @@ package org.apache.eventmesh.connector.kafka.source.connector; -import org.apache.eventmesh.connector.kafka.source.config.KafkaSourceConfig; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.mq.kafka.KafkaSourceConfig; +import org.apache.eventmesh.common.remote.offset.RecordOffset; +import org.apache.eventmesh.common.remote.offset.RecordPartition; +import org.apache.eventmesh.common.remote.offset.kafka.KafkaRecordOffset; +import org.apache.eventmesh.common.remote.offset.kafka.KafkaRecordPartition; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; import org.apache.eventmesh.openconnect.api.source.Source; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; import org.apache.kafka.clients.consumer.ConsumerConfig; import org.apache.kafka.clients.consumer.ConsumerRecord; @@ -34,9 +36,7 @@ import java.time.Duration; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Properties; public class KafkaSourceConnector implements Source { @@ -45,7 +45,7 @@ public class KafkaSourceConnector implements Source { private KafkaConsumer kafkaConsumer; - private int pollTimeOut = 100; + private long maxPollWaitTime; @Override public Class configClass() { @@ -75,7 +75,7 @@ private void doInit() { props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, sourceConfig.getConnectorConfig().getMaxPollRecords()); props.put(ConsumerConfig.AUTO_COMMIT_INTERVAL_MS_CONFIG, sourceConfig.getConnectorConfig().getAutoCommitIntervalMS()); props.put(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, sourceConfig.getConnectorConfig().getSessionTimeoutMS()); - this.pollTimeOut = sourceConfig.getConnectorConfig().getPollTimeOut(); + this.maxPollWaitTime = sourceConfig.getPollConfig().getMaxWaitTime(); this.kafkaConsumer = new KafkaConsumer<>(props); } @@ -94,6 +94,11 @@ public String name() { return this.sourceConfig.getConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() { kafkaConsumer.unsubscribe(); @@ -101,7 +106,7 @@ public void stop() { @Override public List poll() { - ConsumerRecords records = kafkaConsumer.poll(Duration.ofMillis(pollTimeOut)); + ConsumerRecords records = kafkaConsumer.poll(Duration.ofMillis(maxPollWaitTime)); List connectRecords = new ArrayList<>(records.count()); for (ConsumerRecord record : records) { Long timestamp = System.currentTimeMillis(); @@ -118,15 +123,15 @@ public List poll() { } public static RecordOffset convertToRecordOffset(Long offset) { - Map offsetMap = new HashMap<>(); - offsetMap.put("queueOffset", offset + ""); - return new RecordOffset(offsetMap); + KafkaRecordOffset recordOffset = new KafkaRecordOffset(); + recordOffset.setOffset(offset); + return recordOffset; } public static RecordPartition convertToRecordPartition(String topic, int partition) { - Map map = new HashMap<>(); - map.put("topic", topic); - map.put("partition", String.valueOf(partition)); - return new RecordPartition(map); + KafkaRecordPartition recordPartition = new KafkaRecordPartition(); + recordPartition.setTopic(topic); + recordPartition.setPartition(partition); + return recordPartition; } } diff --git a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/config/KnativeServerConfig.java b/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/config/KnativeServerConfig.java index 4125132ad7..9469a5d4b5 100644 --- a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/config/KnativeServerConfig.java +++ b/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/config/KnativeServerConfig.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.knative.config; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/sink/connector/KnativeSinkConnector.java b/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/sink/connector/KnativeSinkConnector.java index fdf5b1056b..b14f77ecd4 100644 --- a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/sink/connector/KnativeSinkConnector.java +++ b/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/sink/connector/KnativeSinkConnector.java @@ -19,10 +19,10 @@ import static org.asynchttpclient.Dsl.asyncHttpClient; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.knative.KnativeSinkConfig; import org.apache.eventmesh.connector.knative.cloudevent.KnativeHeaders; import org.apache.eventmesh.connector.knative.cloudevent.KnativeMessageFactory; -import org.apache.eventmesh.connector.knative.sink.config.KnativeSinkConfig; -import org.apache.eventmesh.openconnect.api.config.Config; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; import org.apache.eventmesh.openconnect.api.sink.Sink; @@ -82,6 +82,11 @@ public String name() { return this.sinkConfig.getConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() { started.compareAndSet(true, false); diff --git a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/source/connector/KnativeSourceConnector.java b/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/source/connector/KnativeSourceConnector.java index 239c39a802..1b0c033e8f 100644 --- a/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/source/connector/KnativeSourceConnector.java +++ b/eventmesh-connectors/eventmesh-connector-knative/src/main/java/org/apache/eventmesh/connector/knative/source/connector/KnativeSourceConnector.java @@ -17,8 +17,8 @@ package org.apache.eventmesh.connector.knative.source.connector; -import org.apache.eventmesh.connector.knative.source.config.KnativeSourceConfig; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.knative.KnativeSourceConfig; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; import org.apache.eventmesh.openconnect.api.source.Source; @@ -65,6 +65,11 @@ public String name() { return this.sourceConfig.getConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() { started.compareAndSet(true, false); diff --git a/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/ConfigUtils.java b/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/ConfigUtils.java new file mode 100644 index 0000000000..f0017397e4 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/ConfigUtils.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.lark; + +import org.apache.eventmesh.common.config.connector.lark.SinkConnectorConfig; + +import org.apache.commons.lang3.StringUtils; + +import com.lark.oapi.service.im.v1.enums.ReceiveIdTypeEnum; + +public class ConfigUtils { + + public static void validateSinkConfiguration(SinkConnectorConfig sinkConnectorConfig) { + // validate blank + if (StringUtils.isAnyBlank(sinkConnectorConfig.getAppId(), sinkConnectorConfig.getAppSecret(), sinkConnectorConfig.getReceiveId())) { + throw new IllegalArgumentException("appId or appSecret or receiveId is blank,please check it."); + } + + // validate receiveIdType + if (!StringUtils.containsAny(sinkConnectorConfig.getReceiveIdType(), ReceiveIdTypeEnum.CHAT_ID.getValue(), + ReceiveIdTypeEnum.EMAIL.getValue(), + ReceiveIdTypeEnum.OPEN_ID.getValue(), + ReceiveIdTypeEnum.USER_ID.getValue(), + ReceiveIdTypeEnum.UNION_ID.getValue())) { + throw new IllegalArgumentException( + String.format("sinkConnectorConfig.receiveIdType=[%s], Invalid.", sinkConnectorConfig.getReceiveIdType())); + } + } +} diff --git a/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/config/LarkConnectServerConfig.java b/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/config/LarkConnectServerConfig.java index 6d0428056e..0106fa4a65 100644 --- a/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/config/LarkConnectServerConfig.java +++ b/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/config/LarkConnectServerConfig.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.lark.config; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/sink/ImServiceHandler.java b/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/sink/ImServiceHandler.java index 75e51e690b..ce5d4a3b85 100644 --- a/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/sink/ImServiceHandler.java +++ b/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/sink/ImServiceHandler.java @@ -19,9 +19,9 @@ import static org.apache.eventmesh.connector.lark.sink.connector.LarkSinkConnector.getTenantAccessToken; +import org.apache.eventmesh.common.config.connector.lark.SinkConnectorConfig; import org.apache.eventmesh.connector.lark.ConnectRecordExtensionKeys; import org.apache.eventmesh.connector.lark.config.LarkMessageTemplateType; -import org.apache.eventmesh.connector.lark.sink.config.SinkConnectorConfig; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; import org.apache.commons.text.StringEscapeUtils; diff --git a/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/sink/connector/LarkSinkConnector.java b/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/sink/connector/LarkSinkConnector.java index d1ee1caa40..9981322e8f 100644 --- a/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/sink/connector/LarkSinkConnector.java +++ b/eventmesh-connectors/eventmesh-connector-lark/src/main/java/org/apache/eventmesh/connector/lark/sink/connector/LarkSinkConnector.java @@ -19,10 +19,11 @@ import static org.apache.eventmesh.connector.lark.sink.ImServiceHandler.create; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.lark.LarkSinkConfig; +import org.apache.eventmesh.common.config.connector.lark.SinkConnectorConfig; +import org.apache.eventmesh.connector.lark.ConfigUtils; import org.apache.eventmesh.connector.lark.sink.ImServiceHandler; -import org.apache.eventmesh.connector.lark.sink.config.LarkSinkConfig; -import org.apache.eventmesh.connector.lark.sink.config.SinkConnectorConfig; -import org.apache.eventmesh.openconnect.api.config.Config; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; import org.apache.eventmesh.openconnect.api.sink.Sink; @@ -87,7 +88,7 @@ public void init(ConnectorContext connectorContext) { this.sinkConfig = (LarkSinkConfig) sinkConnectorContext.getSinkConfig(); SinkConnectorConfig sinkConnectorConfig = sinkConfig.getSinkConnectorConfig(); - sinkConnectorConfig.validateSinkConfiguration(); + ConfigUtils.validateSinkConfiguration(sinkConnectorConfig); imServiceHandler = create(sinkConnectorConfig); } @@ -109,6 +110,11 @@ public String name() { return this.sinkConfig.getSinkConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() { if (!started.compareAndSet(true, false)) { diff --git a/eventmesh-connectors/eventmesh-connector-lark/src/test/java/org/apache/eventmesh/connector/lark/sink/ImServiceHandlerTest.java b/eventmesh-connectors/eventmesh-connector-lark/src/test/java/org/apache/eventmesh/connector/lark/sink/ImServiceHandlerTest.java index f72232a533..9c54717fac 100644 --- a/eventmesh-connectors/eventmesh-connector-lark/src/test/java/org/apache/eventmesh/connector/lark/sink/ImServiceHandlerTest.java +++ b/eventmesh-connectors/eventmesh-connector-lark/src/test/java/org/apache/eventmesh/connector/lark/sink/ImServiceHandlerTest.java @@ -27,11 +27,9 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import org.apache.eventmesh.connector.lark.sink.config.LarkSinkConfig; -import org.apache.eventmesh.connector.lark.sink.config.SinkConnectorConfig; +import org.apache.eventmesh.common.config.connector.lark.LarkSinkConfig; +import org.apache.eventmesh.common.config.connector.lark.SinkConnectorConfig; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; import org.apache.eventmesh.openconnect.util.ConfigUtil; import java.lang.reflect.Field; @@ -103,9 +101,8 @@ public void testRegularSinkAsync() throws Exception { private void regularSink() throws Exception { final int times = 3; for (int i = 0; i < times; i++) { - RecordPartition partition = new RecordPartition(); - RecordOffset offset = new RecordOffset(); - ConnectRecord connectRecord = new ConnectRecord(partition, offset, + + ConnectRecord connectRecord = new ConnectRecord(null, null, System.currentTimeMillis(), "test-lark".getBytes(StandardCharsets.UTF_8)); if (Boolean.parseBoolean(sinkConnectorConfig.getSinkAsync())) { imServiceHandler.sinkAsync(connectRecord); @@ -145,9 +142,8 @@ private void retrySink() throws Exception { long duration = retryDelayInMills * sinkTimes; for (int i = 0; i < times; i++) { - RecordPartition partition = new RecordPartition(); - RecordOffset offset = new RecordOffset(); - ConnectRecord connectRecord = new ConnectRecord(partition, offset, + + ConnectRecord connectRecord = new ConnectRecord(null, null, System.currentTimeMillis(), "test-lark".getBytes(StandardCharsets.UTF_8)); if (Boolean.parseBoolean(sinkConnectorConfig.getSinkAsync())) { imServiceHandler.sinkAsync(connectRecord); diff --git a/eventmesh-connectors/eventmesh-connector-lark/src/test/java/org/apache/eventmesh/connector/lark/sink/LarkSinkConnectorTest.java b/eventmesh-connectors/eventmesh-connector-lark/src/test/java/org/apache/eventmesh/connector/lark/sink/LarkSinkConnectorTest.java index 658fa89223..a02c845dc5 100644 --- a/eventmesh-connectors/eventmesh-connector-lark/src/test/java/org/apache/eventmesh/connector/lark/sink/LarkSinkConnectorTest.java +++ b/eventmesh-connectors/eventmesh-connector-lark/src/test/java/org/apache/eventmesh/connector/lark/sink/LarkSinkConnectorTest.java @@ -24,12 +24,10 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import org.apache.eventmesh.connector.lark.sink.config.LarkSinkConfig; +import org.apache.eventmesh.common.config.connector.lark.LarkSinkConfig; import org.apache.eventmesh.connector.lark.sink.connector.LarkSinkConnector; import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; import org.apache.eventmesh.openconnect.util.ConfigUtil; import java.nio.charset.StandardCharsets; @@ -82,9 +80,7 @@ public void testPut() throws Exception { final int times = 3; List connectRecords = new ArrayList<>(); for (int i = 0; i < times; i++) { - RecordPartition partition = new RecordPartition(); - RecordOffset offset = new RecordOffset(); - ConnectRecord connectRecord = new ConnectRecord(partition, offset, + ConnectRecord connectRecord = new ConnectRecord(null, null, System.currentTimeMillis(), "test-lark".getBytes(StandardCharsets.UTF_8)); connectRecords.add(connectRecord); } diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/config/MongodbServerConfig.java b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/config/MongodbServerConfig.java index 95c651b358..fa5618d4a8 100644 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/config/MongodbServerConfig.java +++ b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/config/MongodbServerConfig.java @@ -17,10 +17,12 @@ package org.apache.eventmesh.connector.mongodb.config; +import org.apache.eventmesh.common.config.connector.Config; + import lombok.Data; @Data -public class MongodbServerConfig { +public class MongodbServerConfig extends Config { private boolean sourceEnable; diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/client/MongodbReplicaSetSinkClient.java b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/client/MongodbReplicaSetSinkClient.java index 638e1a7d3f..0afae2b8de 100644 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/client/MongodbReplicaSetSinkClient.java +++ b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/client/MongodbReplicaSetSinkClient.java @@ -17,8 +17,8 @@ package org.apache.eventmesh.connector.mongodb.sink.client; +import org.apache.eventmesh.common.config.connector.rdb.mongodb.SinkConnectorConfig; import org.apache.eventmesh.connector.mongodb.sink.client.Impl.MongodbSinkClient; -import org.apache.eventmesh.connector.mongodb.sink.config.SinkConnectorConfig; import org.apache.eventmesh.connector.mongodb.utils.MongodbCloudEventUtil; import org.bson.Document; diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/client/MongodbStandaloneSinkClient.java b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/client/MongodbStandaloneSinkClient.java index 45bdf6f699..4a87a4320f 100644 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/client/MongodbStandaloneSinkClient.java +++ b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/client/MongodbStandaloneSinkClient.java @@ -17,9 +17,9 @@ package org.apache.eventmesh.connector.mongodb.sink.client; +import org.apache.eventmesh.common.config.connector.rdb.mongodb.SinkConnectorConfig; import org.apache.eventmesh.connector.mongodb.constant.MongodbConstants; import org.apache.eventmesh.connector.mongodb.sink.client.Impl.MongodbSinkClient; -import org.apache.eventmesh.connector.mongodb.sink.config.SinkConnectorConfig; import org.apache.eventmesh.connector.mongodb.utils.MongodbCloudEventUtil; import org.bson.Document; diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/connector/MongodbSinkConnector.java b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/connector/MongodbSinkConnector.java index 814aaf2882..1001ffa584 100644 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/connector/MongodbSinkConnector.java +++ b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/sink/connector/MongodbSinkConnector.java @@ -17,11 +17,11 @@ package org.apache.eventmesh.connector.mongodb.sink.connector; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.rdb.mongodb.MongodbSinkConfig; import org.apache.eventmesh.connector.mongodb.sink.client.Impl.MongodbSinkClient; import org.apache.eventmesh.connector.mongodb.sink.client.MongodbReplicaSetSinkClient; import org.apache.eventmesh.connector.mongodb.sink.client.MongodbStandaloneSinkClient; -import org.apache.eventmesh.connector.mongodb.sink.config.MongodbSinkConfig; -import org.apache.eventmesh.openconnect.api.config.Config; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; import org.apache.eventmesh.openconnect.api.sink.Sink; @@ -87,6 +87,11 @@ public String name() { return this.sinkConfig.getConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() throws Exception { this.client.stop(); diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/client/MongodbReplicaSetSourceClient.java b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/client/MongodbReplicaSetSourceClient.java index 468cf6c92e..b389c0db9c 100644 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/client/MongodbReplicaSetSourceClient.java +++ b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/client/MongodbReplicaSetSourceClient.java @@ -17,8 +17,8 @@ package org.apache.eventmesh.connector.mongodb.source.client; +import org.apache.eventmesh.common.config.connector.rdb.mongodb.SourceConnectorConfig; import org.apache.eventmesh.connector.mongodb.source.client.Impl.MongodbSourceClient; -import org.apache.eventmesh.connector.mongodb.source.config.SourceConnectorConfig; import org.apache.eventmesh.connector.mongodb.utils.MongodbCloudEventUtil; import java.util.concurrent.BlockingQueue; diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/client/MongodbStandaloneSourceClient.java b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/client/MongodbStandaloneSourceClient.java index e062ab034d..ce7452e0ae 100644 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/client/MongodbStandaloneSourceClient.java +++ b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/client/MongodbStandaloneSourceClient.java @@ -18,9 +18,9 @@ package org.apache.eventmesh.connector.mongodb.source.client; import org.apache.eventmesh.common.ThreadPoolFactory; +import org.apache.eventmesh.common.config.connector.rdb.mongodb.SourceConnectorConfig; import org.apache.eventmesh.connector.mongodb.constant.MongodbConstants; import org.apache.eventmesh.connector.mongodb.source.client.Impl.MongodbSourceClient; -import org.apache.eventmesh.connector.mongodb.source.config.SourceConnectorConfig; import org.apache.eventmesh.connector.mongodb.utils.MongodbCloudEventUtil; import java.util.concurrent.BlockingQueue; diff --git a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/connector/MongodbSourceConnector.java b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/connector/MongodbSourceConnector.java index fec64de56c..1d1dcc1843 100644 --- a/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/connector/MongodbSourceConnector.java +++ b/eventmesh-connectors/eventmesh-connector-mongodb/src/main/java/org/apache/eventmesh/connector/mongodb/source/connector/MongodbSourceConnector.java @@ -17,11 +17,11 @@ package org.apache.eventmesh.connector.mongodb.source.connector; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.rdb.mongodb.MongodbSourceConfig; import org.apache.eventmesh.connector.mongodb.source.client.Impl.MongodbSourceClient; import org.apache.eventmesh.connector.mongodb.source.client.MongodbReplicaSetSourceClient; import org.apache.eventmesh.connector.mongodb.source.client.MongodbStandaloneSourceClient; -import org.apache.eventmesh.connector.mongodb.source.config.MongodbSourceConfig; -import org.apache.eventmesh.openconnect.api.config.Config; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; import org.apache.eventmesh.openconnect.api.source.Source; @@ -42,10 +42,12 @@ public class MongodbSourceConnector implements Source { private MongodbSourceConfig sourceConfig; - private static final int DEFAULT_BATCH_SIZE = 10; - private BlockingQueue queue; + private int maxBatchSize; + + private long maxPollWaitTime; + private MongodbSourceClient client; @Override @@ -67,7 +69,9 @@ public void init(ConnectorContext connectorContext) throws Exception { } private void doInit() { - this.queue = new LinkedBlockingQueue<>(1000); + this.maxBatchSize = sourceConfig.getPollConfig().getMaxBatchSize(); + this.maxPollWaitTime = sourceConfig.getPollConfig().getMaxWaitTime(); + this.queue = new LinkedBlockingQueue<>(sourceConfig.getPollConfig().getCapacity()); String connectorType = sourceConfig.getConnectorConfig().getConnectorType(); if (connectorType.equals(ClusterType.STANDALONE.name())) { this.client = new MongodbStandaloneSourceClient(sourceConfig.getConnectorConfig(), queue); @@ -93,6 +97,11 @@ public String name() { return this.sourceConfig.connectorConfig.getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() throws Exception { this.client.stop(); @@ -100,15 +109,21 @@ public void stop() throws Exception { @Override public List poll() { - List connectRecords = new ArrayList<>(DEFAULT_BATCH_SIZE); - for (int count = 0; count < DEFAULT_BATCH_SIZE; ++count) { + long startTime = System.currentTimeMillis(); + long remainingTime = maxPollWaitTime; + + List connectRecords = new ArrayList<>(maxBatchSize); + for (int count = 0; count < maxBatchSize; ++count) { try { - CloudEvent event = queue.poll(3, TimeUnit.SECONDS); + CloudEvent event = queue.poll(remainingTime, TimeUnit.MILLISECONDS); if (event == null) { break; } - connectRecords.add(CloudEventUtil.convertEventToRecord(event)); + + // calculate elapsed time and update remaining time for next poll + long elapsedTime = System.currentTimeMillis() - startTime; + remainingTime = maxPollWaitTime > elapsedTime ? maxPollWaitTime - elapsedTime : 0; } catch (InterruptedException e) { break; } diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/config/OpenFunctionServerConfig.java b/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/config/OpenFunctionServerConfig.java index 2cf28000f5..b4ae607d5b 100644 --- a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/config/OpenFunctionServerConfig.java +++ b/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/config/OpenFunctionServerConfig.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.openfunction.config; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/sink/connector/OpenFunctionSinkConnector.java b/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/sink/connector/OpenFunctionSinkConnector.java index aed6936004..0f00a7e381 100644 --- a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/sink/connector/OpenFunctionSinkConnector.java +++ b/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/sink/connector/OpenFunctionSinkConnector.java @@ -17,8 +17,8 @@ package org.apache.eventmesh.connector.openfunction.sink.connector; -import org.apache.eventmesh.connector.openfunction.sink.config.OpenFunctionSinkConfig; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.openfunction.OpenFunctionSinkConfig; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; import org.apache.eventmesh.openconnect.api.sink.Sink; @@ -74,6 +74,11 @@ public String name() { return this.sinkConfig.getSinkConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() { } diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/source/connector/OpenFunctionSourceConnector.java b/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/source/connector/OpenFunctionSourceConnector.java index 916d6793db..e40c451ff8 100644 --- a/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/source/connector/OpenFunctionSourceConnector.java +++ b/eventmesh-connectors/eventmesh-connector-openfunction/src/main/java/org/apache/eventmesh/connector/openfunction/source/connector/OpenFunctionSourceConnector.java @@ -17,8 +17,8 @@ package org.apache.eventmesh.connector.openfunction.source.connector; -import org.apache.eventmesh.connector.openfunction.source.config.OpenFunctionSourceConfig; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.openfunction.OpenFunctionSourceConfig; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; import org.apache.eventmesh.openconnect.api.source.Source; @@ -35,12 +35,14 @@ @Slf4j public class OpenFunctionSourceConnector implements Source { - private static final int DEFAULT_BATCH_SIZE = 10; - private OpenFunctionSourceConfig sourceConfig; private BlockingQueue queue; + private int maxBatchSize; + + private long maxPollWaitTime; + @Override public Class configClass() { return OpenFunctionSourceConfig.class; @@ -50,7 +52,7 @@ public Class configClass() { public void init(Config config) throws Exception { // init config for openfunction source connector this.sourceConfig = (OpenFunctionSourceConfig) config; - this.queue = new LinkedBlockingQueue<>(1000); + doInit(); } @Override @@ -58,7 +60,14 @@ public void init(ConnectorContext connectorContext) throws Exception { SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; // init config for openfunction source connector this.sourceConfig = (OpenFunctionSourceConfig) sourceConnectorContext.getSourceConfig(); - this.queue = new LinkedBlockingQueue<>(1000); + doInit(); + } + + private void doInit() { + // init config for openfunction source connector + this.queue = new LinkedBlockingQueue<>(sourceConfig.getPollConfig().getCapacity()); + this.maxBatchSize = sourceConfig.getPollConfig().getMaxBatchSize(); + this.maxPollWaitTime = sourceConfig.getPollConfig().getMaxWaitTime(); } @Override @@ -76,6 +85,11 @@ public String name() { return this.sourceConfig.getSourceConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() { @@ -87,16 +101,21 @@ public BlockingQueue queue() { @Override public List poll() { + long startTime = System.currentTimeMillis(); + long remainingTime = maxPollWaitTime; - List connectRecords = new ArrayList<>(DEFAULT_BATCH_SIZE); - - for (int count = 0; count < DEFAULT_BATCH_SIZE; ++count) { + List connectRecords = new ArrayList<>(maxBatchSize); + for (int count = 0; count < maxBatchSize; ++count) { try { - ConnectRecord connectRecord = queue.poll(3, TimeUnit.SECONDS); + ConnectRecord connectRecord = queue.poll(remainingTime, TimeUnit.MILLISECONDS); if (connectRecord == null) { break; } connectRecords.add(connectRecord); + + // calculate elapsed time and update remaining time for next poll + long elapsedTime = System.currentTimeMillis() - startTime; + remainingTime = maxPollWaitTime > elapsedTime ? maxPollWaitTime - elapsedTime : 0; } catch (InterruptedException e) { Thread currentThread = Thread.currentThread(); log.warn("[OpenFunctionSourceConnector] Interrupting thread {} due to exception {}", diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/test/java/org/apache/eventmesh/connector/openfunction/sink/connector/OpenFunctionSinkConnectorTest.java b/eventmesh-connectors/eventmesh-connector-openfunction/src/test/java/org/apache/eventmesh/connector/openfunction/sink/connector/OpenFunctionSinkConnectorTest.java index efb25b8ea9..6751c0ec17 100644 --- a/eventmesh-connectors/eventmesh-connector-openfunction/src/test/java/org/apache/eventmesh/connector/openfunction/sink/connector/OpenFunctionSinkConnectorTest.java +++ b/eventmesh-connectors/eventmesh-connector-openfunction/src/test/java/org/apache/eventmesh/connector/openfunction/sink/connector/OpenFunctionSinkConnectorTest.java @@ -19,10 +19,8 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; -import org.apache.eventmesh.connector.openfunction.sink.config.OpenFunctionSinkConfig; +import org.apache.eventmesh.common.config.connector.openfunction.OpenFunctionSinkConfig; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; import java.util.ArrayList; import java.util.List; @@ -72,9 +70,7 @@ public void shutdownConnector() { private void writeMockedRecords(int count, String message) throws Exception { List records = new ArrayList<>(); for (int i = 0; i < count; i++) { - RecordPartition partition = new RecordPartition(); - RecordOffset offset = new RecordOffset(); - records.add(new ConnectRecord(partition, offset, System.currentTimeMillis(), message + i)); + records.add(new ConnectRecord(null, null, System.currentTimeMillis(), message + i)); } connector.put(records); } diff --git a/eventmesh-connectors/eventmesh-connector-openfunction/src/test/java/org/apache/eventmesh/connector/openfunction/source/connector/OpenFunctionSourceConnectorTest.java b/eventmesh-connectors/eventmesh-connector-openfunction/src/test/java/org/apache/eventmesh/connector/openfunction/source/connector/OpenFunctionSourceConnectorTest.java index 2663443dbe..880ee701dc 100644 --- a/eventmesh-connectors/eventmesh-connector-openfunction/src/test/java/org/apache/eventmesh/connector/openfunction/source/connector/OpenFunctionSourceConnectorTest.java +++ b/eventmesh-connectors/eventmesh-connector-openfunction/src/test/java/org/apache/eventmesh/connector/openfunction/source/connector/OpenFunctionSourceConnectorTest.java @@ -17,10 +17,8 @@ package org.apache.eventmesh.connector.openfunction.source.connector; -import org.apache.eventmesh.connector.openfunction.source.config.OpenFunctionSourceConfig; +import org.apache.eventmesh.common.config.connector.openfunction.OpenFunctionSourceConfig; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; import java.util.List; import java.util.concurrent.BlockingQueue; @@ -53,9 +51,7 @@ public void testSpringSourceConnector() throws Exception { private void writeMockedRecords(int count, String message) { BlockingQueue queue = connector.queue(); for (int i = 0; i < count; i++) { - RecordPartition partition = new RecordPartition(); - RecordOffset offset = new RecordOffset(); - ConnectRecord record = new ConnectRecord(partition, offset, System.currentTimeMillis(), message + i); + ConnectRecord record = new ConnectRecord(null, null, System.currentTimeMillis(), message + i); queue.offer(record); } } diff --git a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/config/PravegaServerConfig.java b/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/config/PravegaServerConfig.java index bf5d9a4fb7..5945b4a7ae 100644 --- a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/config/PravegaServerConfig.java +++ b/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/config/PravegaServerConfig.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.pravega.config; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/sink/connector/PravegaSinkConnector.java b/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/sink/connector/PravegaSinkConnector.java index d1d29ad95d..e089ef6760 100644 --- a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/sink/connector/PravegaSinkConnector.java +++ b/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/sink/connector/PravegaSinkConnector.java @@ -17,11 +17,11 @@ package org.apache.eventmesh.connector.pravega.sink.connector; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.pravega.PravegaSinkConfig; import org.apache.eventmesh.connector.pravega.client.PravegaCloudEventWriter; import org.apache.eventmesh.connector.pravega.client.PravegaEvent; import org.apache.eventmesh.connector.pravega.exception.PravegaConnectorException; -import org.apache.eventmesh.connector.pravega.sink.config.PravegaSinkConfig; -import org.apache.eventmesh.openconnect.api.config.Config; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; import org.apache.eventmesh.openconnect.api.sink.Sink; @@ -109,6 +109,11 @@ public String name() { return this.sinkConfig.getConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() { writerMap.forEach((topic, writer) -> writer.close()); diff --git a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/source/connector/PravegaSourceConnector.java b/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/source/connector/PravegaSourceConnector.java index c72c38f71d..4b5e4751b3 100644 --- a/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/source/connector/PravegaSourceConnector.java +++ b/eventmesh-connectors/eventmesh-connector-pravega/src/main/java/org/apache/eventmesh/connector/pravega/source/connector/PravegaSourceConnector.java @@ -18,9 +18,9 @@ package org.apache.eventmesh.connector.pravega.source.connector; import org.apache.eventmesh.common.ThreadPoolFactory; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.pravega.PravegaSourceConfig; import org.apache.eventmesh.connector.pravega.client.PravegaEvent; -import org.apache.eventmesh.connector.pravega.source.config.PravegaSourceConfig; -import org.apache.eventmesh.openconnect.api.config.Config; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; import org.apache.eventmesh.openconnect.api.source.Source; @@ -57,8 +57,6 @@ public class PravegaSourceConnector implements Source { private static final AtomicBoolean started = new AtomicBoolean(false); - private static final int DEFAULT_BATCH_SIZE = 10; - private PravegaSourceConfig sourceConfig; private StreamManager streamManager; @@ -71,6 +69,10 @@ public class PravegaSourceConnector implements Source { private BlockingQueue queue; + private int maxBatchSize; + + private long maxPollWaitTime; + private final ThreadPoolExecutor executor = ThreadPoolFactory.createThreadPoolExecutor( Runtime.getRuntime().availableProcessors() * 2, Runtime.getRuntime().availableProcessors() * 2, @@ -89,7 +91,9 @@ public void init(Config config) throws Exception { public void init(ConnectorContext connectorContext) throws Exception { SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; this.sourceConfig = (PravegaSourceConfig) sourceConnectorContext.getSourceConfig(); - this.queue = new LinkedBlockingQueue<>(1000); + this.queue = new LinkedBlockingQueue<>(sourceConfig.getPollConfig().getCapacity()); + this.maxBatchSize = sourceConfig.getPollConfig().getMaxBatchSize(); + this.maxPollWaitTime = sourceConfig.getPollConfig().getMaxWaitTime(); streamManager = StreamManager.create(sourceConfig.getConnectorConfig().getControllerURI()); ClientConfig.ClientConfigBuilder clientConfigBuilder = @@ -148,6 +152,11 @@ public String name() { return this.sourceConfig.getConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() { sourceHandlerMap.forEach((topic, handler) -> { @@ -163,15 +172,21 @@ public void stop() { @Override public List poll() { - List connectRecords = new ArrayList<>(DEFAULT_BATCH_SIZE); - for (int count = 0; count < DEFAULT_BATCH_SIZE; ++count) { + long startTime = System.currentTimeMillis(); + long remainingTime = maxPollWaitTime; + + List connectRecords = new ArrayList<>(maxBatchSize); + for (int count = 0; count < maxBatchSize; ++count) { try { - CloudEvent event = queue.poll(3, TimeUnit.SECONDS); + CloudEvent event = queue.poll(remainingTime, TimeUnit.MILLISECONDS); if (event == null) { break; } - connectRecords.add(CloudEventUtil.convertEventToRecord(event)); + + // calculate elapsed time and update remaining time for next poll + long elapsedTime = System.currentTimeMillis() - startTime; + remainingTime = maxPollWaitTime > elapsedTime ? maxPollWaitTime - elapsedTime : 0; } catch (InterruptedException e) { break; } diff --git a/eventmesh-connectors/eventmesh-connector-prometheus/build.gradle b/eventmesh-connectors/eventmesh-connector-prometheus/build.gradle index 9650575803..97e4ed12f3 100644 --- a/eventmesh-connectors/eventmesh-connector-prometheus/build.gradle +++ b/eventmesh-connectors/eventmesh-connector-prometheus/build.gradle @@ -16,6 +16,7 @@ */ dependencies { + implementation project(":eventmesh-common") implementation project(":eventmesh-openconnect:eventmesh-openconnect-java") implementation 'org.apache.httpcomponents:httpclient' implementation 'com.github.rholder:guava-retrying' diff --git a/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/config/PrometheusServerConfig.java b/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/config/PrometheusServerConfig.java index 3222e28210..a238109c72 100644 --- a/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/config/PrometheusServerConfig.java +++ b/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/config/PrometheusServerConfig.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.prometheus.config; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/source/connector/PrometheusSourceConnector.java b/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/source/connector/PrometheusSourceConnector.java index 2c75f394fc..0cafed73f3 100644 --- a/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/source/connector/PrometheusSourceConnector.java +++ b/eventmesh-connectors/eventmesh-connector-prometheus/src/main/java/org/apache/eventmesh/connector/prometheus/source/connector/PrometheusSourceConnector.java @@ -17,16 +17,18 @@ package org.apache.eventmesh.connector.prometheus.source.connector; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.prometheus.PrometheusSourceConfig; +import org.apache.eventmesh.common.remote.offset.RecordOffset; +import org.apache.eventmesh.common.remote.offset.RecordPartition; +import org.apache.eventmesh.common.remote.offset.prometheus.PrometheusRecordOffset; +import org.apache.eventmesh.common.remote.offset.prometheus.PrometheusRecordPartition; import org.apache.eventmesh.connector.prometheus.model.QueryPrometheusReq; import org.apache.eventmesh.connector.prometheus.model.QueryPrometheusRsp; -import org.apache.eventmesh.connector.prometheus.source.config.PrometheusSourceConfig; -import org.apache.eventmesh.openconnect.api.config.Config; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; import org.apache.eventmesh.openconnect.api.source.Source; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; @@ -143,6 +145,11 @@ public String name() { return this.sourceConfig.getConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() { log.info("prometheus source connector stop."); @@ -185,8 +192,8 @@ public List poll() { private ConnectRecord assembleRecord(String data) { Long timestamp = System.currentTimeMillis(); - RecordPartition recordPartition = new RecordPartition(); - RecordOffset recordOffset = new RecordOffset(); + RecordPartition recordPartition = new PrometheusRecordPartition(); + RecordOffset recordOffset = new PrometheusRecordOffset(); return new ConnectRecord(recordPartition, recordOffset, timestamp, data); } diff --git a/eventmesh-connectors/eventmesh-connector-pulsar/build.gradle b/eventmesh-connectors/eventmesh-connector-pulsar/build.gradle index f087842ea8..62ab590d89 100644 --- a/eventmesh-connectors/eventmesh-connector-pulsar/build.gradle +++ b/eventmesh-connectors/eventmesh-connector-pulsar/build.gradle @@ -16,6 +16,7 @@ */ dependencies { + implementation project(":eventmesh-common") implementation project(":eventmesh-openconnect:eventmesh-openconnect-java") /* diff --git a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/config/PulsarServerConfig.java b/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/config/PulsarServerConfig.java index 56d0b04fed..2c7e939259 100644 --- a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/config/PulsarServerConfig.java +++ b/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/config/PulsarServerConfig.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.pulsar.config; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/sink/connector/PulsarSinkConnector.java b/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/sink/connector/PulsarSinkConnector.java index 5ea0a0147a..3f90c6c1be 100644 --- a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/sink/connector/PulsarSinkConnector.java +++ b/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/sink/connector/PulsarSinkConnector.java @@ -17,8 +17,8 @@ package org.apache.eventmesh.connector.pulsar.sink.connector; -import org.apache.eventmesh.connector.pulsar.sink.config.PulsarSinkConfig; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.mq.pulsar.PulsarSinkConfig; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; import org.apache.eventmesh.openconnect.api.sink.Sink; @@ -85,6 +85,11 @@ public String name() { return this.sinkConfig.getConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() { try { diff --git a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/source/connector/PulsarSourceConnector.java b/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/source/connector/PulsarSourceConnector.java index 718dfc7357..0bc576221e 100644 --- a/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/source/connector/PulsarSourceConnector.java +++ b/eventmesh-connectors/eventmesh-connector-pulsar/src/main/java/org/apache/eventmesh/connector/pulsar/source/connector/PulsarSourceConnector.java @@ -17,13 +17,13 @@ package org.apache.eventmesh.connector.pulsar.source.connector; -import org.apache.eventmesh.connector.pulsar.source.config.PulsarSourceConfig; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.mq.pulsar.PulsarSourceConfig; +import org.apache.eventmesh.common.remote.offset.pulsar.PulsarRecordPartition; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; import org.apache.eventmesh.openconnect.api.source.Source; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; import org.apache.pulsar.client.api.Consumer; import org.apache.pulsar.client.api.Message; @@ -33,9 +33,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; -import java.util.Map; import lombok.extern.slf4j.Slf4j; @@ -89,6 +87,11 @@ public String name() { return this.sourceConfig.getConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() { try { @@ -108,10 +111,9 @@ public List poll() { Message message = (Message) msg; byte[] body = message.getData(); String bodyStr = new String(body, StandardCharsets.UTF_8); - Map map = new HashMap<>(); - map.put("topic", consumer.getTopic()); - map.put("queueId", String.valueOf(message.getSequenceId())); - RecordPartition partition = new RecordPartition(map); + PulsarRecordPartition partition = new PulsarRecordPartition(); + partition.setTopic(consumer.getTopic()); + partition.setQueueId(message.getSequenceId()); ConnectRecord connectRecord = new ConnectRecord(partition, null, timestamp, bodyStr); connectRecord.addExtension("topic", consumer.getTopic()); connectRecords.add(connectRecord); diff --git a/eventmesh-connectors/eventmesh-connector-rabbitmq/build.gradle b/eventmesh-connectors/eventmesh-connector-rabbitmq/build.gradle index 54764c7b63..90d2de781e 100644 --- a/eventmesh-connectors/eventmesh-connector-rabbitmq/build.gradle +++ b/eventmesh-connectors/eventmesh-connector-rabbitmq/build.gradle @@ -20,7 +20,7 @@ dependencies { api project(":eventmesh-openconnect:eventmesh-openconnect-java") implementation project(":eventmesh-common") // rabbitmq - implementation 'com.rabbitmq:amqp-client:5.21.0' + implementation 'com.rabbitmq:amqp-client:5.24.0' implementation 'io.cloudevents:cloudevents-json-jackson' diff --git a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/config/RabbitMQServerConfig.java b/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/config/RabbitMQServerConfig.java index 9b33ce0c83..27b89c62e5 100644 --- a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/config/RabbitMQServerConfig.java +++ b/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/config/RabbitMQServerConfig.java @@ -17,10 +17,14 @@ package org.apache.eventmesh.connector.rabbitmq.config; +import org.apache.eventmesh.common.config.connector.Config; + import lombok.Data; +import lombok.EqualsAndHashCode; @Data -public class RabbitMQServerConfig { +@EqualsAndHashCode(callSuper = true) +public class RabbitMQServerConfig extends Config { private boolean sourceEnable; diff --git a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/sink/connector/RabbitMQSinkConnector.java b/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/sink/connector/RabbitMQSinkConnector.java index 250f31c5bb..08d1cefbac 100644 --- a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/sink/connector/RabbitMQSinkConnector.java +++ b/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/sink/connector/RabbitMQSinkConnector.java @@ -17,12 +17,12 @@ package org.apache.eventmesh.connector.rabbitmq.sink.connector; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.mq.rabbitmq.RabbitMQSinkConfig; import org.apache.eventmesh.connector.rabbitmq.client.RabbitmqClient; import org.apache.eventmesh.connector.rabbitmq.client.RabbitmqConnectionFactory; import org.apache.eventmesh.connector.rabbitmq.cloudevent.RabbitmqCloudEvent; import org.apache.eventmesh.connector.rabbitmq.cloudevent.RabbitmqCloudEventWriter; -import org.apache.eventmesh.connector.rabbitmq.sink.config.RabbitMQSinkConfig; -import org.apache.eventmesh.openconnect.api.config.Config; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; import org.apache.eventmesh.openconnect.api.sink.Sink; @@ -33,6 +33,7 @@ import io.cloudevents.CloudEvent; +import com.rabbitmq.client.BuiltinExchangeType; import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; @@ -77,7 +78,8 @@ public void init(ConnectorContext connectorContext) throws Exception { @Override public void start() throws Exception { if (!started) { - rabbitmqClient.binding(channel, sinkConfig.getConnectorConfig().getExchangeType(), sinkConfig.getConnectorConfig().getExchangeName(), + BuiltinExchangeType builtinExchangeType = BuiltinExchangeType.valueOf(sinkConfig.getConnectorConfig().getExchangeType()); + rabbitmqClient.binding(channel, builtinExchangeType, sinkConfig.getConnectorConfig().getExchangeName(), sinkConfig.getConnectorConfig().getRoutingKey(), sinkConfig.getConnectorConfig().getQueueName()); started = true; } @@ -93,6 +95,11 @@ public String name() { return this.sinkConfig.getConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() { if (started) { diff --git a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/source/connector/RabbitMQSourceConnector.java b/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/source/connector/RabbitMQSourceConnector.java index 95b09034ef..a19b159c1c 100644 --- a/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/source/connector/RabbitMQSourceConnector.java +++ b/eventmesh-connectors/eventmesh-connector-rabbitmq/src/main/java/org/apache/eventmesh/connector/rabbitmq/source/connector/RabbitMQSourceConnector.java @@ -18,12 +18,12 @@ package org.apache.eventmesh.connector.rabbitmq.source.connector; import org.apache.eventmesh.common.ThreadPoolFactory; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.mq.rabbitmq.RabbitMQSourceConfig; +import org.apache.eventmesh.common.config.connector.mq.rabbitmq.SourceConnectorConfig; import org.apache.eventmesh.connector.rabbitmq.client.RabbitmqClient; import org.apache.eventmesh.connector.rabbitmq.client.RabbitmqConnectionFactory; import org.apache.eventmesh.connector.rabbitmq.cloudevent.RabbitmqCloudEvent; -import org.apache.eventmesh.connector.rabbitmq.source.config.RabbitMQSourceConfig; -import org.apache.eventmesh.connector.rabbitmq.source.config.SourceConnectorConfig; -import org.apache.eventmesh.openconnect.api.config.Config; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; import org.apache.eventmesh.openconnect.api.source.Source; @@ -40,6 +40,7 @@ import io.cloudevents.CloudEvent; +import com.rabbitmq.client.BuiltinExchangeType; import com.rabbitmq.client.Channel; import com.rabbitmq.client.Connection; import com.rabbitmq.client.GetResponse; @@ -53,10 +54,12 @@ public class RabbitMQSourceConnector implements Source { private volatile boolean started = false; - private static final int DEFAULT_BATCH_SIZE = 10; - private BlockingQueue queue; + private int maxBatchSize; + + private long maxPollWaitTime; + private final RabbitmqConnectionFactory rabbitmqConnectionFactory = new RabbitmqConnectionFactory(); private RabbitMQSourceHandler rabbitMQSourceHandler; @@ -83,7 +86,9 @@ public void init(Config config) throws Exception { @Override public void init(ConnectorContext connectorContext) throws Exception { - this.queue = new LinkedBlockingQueue<>(1000); + this.queue = new LinkedBlockingQueue<>(sourceConfig.getPollConfig().getCapacity()); + this.maxBatchSize = sourceConfig.getPollConfig().getMaxBatchSize(); + this.maxPollWaitTime = sourceConfig.getPollConfig().getMaxWaitTime(); this.sourceConfig = (RabbitMQSourceConfig) ((SourceConnectorContext) connectorContext).getSourceConfig(); this.rabbitmqClient = new RabbitmqClient(rabbitmqConnectionFactory); this.connection = rabbitmqClient.getConnection(sourceConfig.getConnectorConfig().getHost(), @@ -98,7 +103,8 @@ public void init(ConnectorContext connectorContext) throws Exception { @Override public void start() throws Exception { if (!started) { - rabbitmqClient.binding(channel, sourceConfig.getConnectorConfig().getExchangeType(), sourceConfig.getConnectorConfig().getExchangeName(), + BuiltinExchangeType builtinExchangeType = BuiltinExchangeType.valueOf(sourceConfig.getConnectorConfig().getExchangeType()); + rabbitmqClient.binding(channel, builtinExchangeType, sourceConfig.getConnectorConfig().getExchangeName(), sourceConfig.getConnectorConfig().getRoutingKey(), sourceConfig.getConnectorConfig().getQueueName()); executor.execute(this.rabbitMQSourceHandler); started = true; @@ -115,6 +121,11 @@ public String name() { return this.sourceConfig.getConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() { if (started) { @@ -132,15 +143,21 @@ public void stop() { @Override public List poll() { - List connectRecords = new ArrayList<>(DEFAULT_BATCH_SIZE); - for (int count = 0; count < DEFAULT_BATCH_SIZE; ++count) { + long startTime = System.currentTimeMillis(); + long remainingTime = maxPollWaitTime; + + List connectRecords = new ArrayList<>(maxBatchSize); + for (int count = 0; count < maxBatchSize; ++count) { try { - CloudEvent event = queue.poll(3, TimeUnit.SECONDS); + CloudEvent event = queue.poll(remainingTime, TimeUnit.MILLISECONDS); if (event == null) { break; } - connectRecords.add(CloudEventUtil.convertEventToRecord(event)); + + // calculate elapsed time and update remaining time for next poll + long elapsedTime = System.currentTimeMillis() - startTime; + remainingTime = maxPollWaitTime > elapsedTime ? maxPollWaitTime - elapsedTime : 0; } catch (InterruptedException e) { break; } diff --git a/eventmesh-connectors/eventmesh-connector-redis/build.gradle b/eventmesh-connectors/eventmesh-connector-redis/build.gradle index 425a10570a..fabfe1c983 100644 --- a/eventmesh-connectors/eventmesh-connector-redis/build.gradle +++ b/eventmesh-connectors/eventmesh-connector-redis/build.gradle @@ -16,9 +16,10 @@ */ dependencies { + implementation project(":eventmesh-common") implementation project(":eventmesh-openconnect:eventmesh-openconnect-java") - implementation 'org.redisson:redisson:3.17.3' + implementation 'org.redisson:redisson:3.38.1' api 'io.cloudevents:cloudevents-json-jackson' diff --git a/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/config/RedisServerConfig.java b/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/config/RedisServerConfig.java index 6b55b0e6e9..d4aaee292a 100644 --- a/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/config/RedisServerConfig.java +++ b/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/config/RedisServerConfig.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.redis.config; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; import lombok.Data; diff --git a/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/sink/connector/RedisSinkConnector.java b/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/sink/connector/RedisSinkConnector.java index e07e05e276..5b7d27c3ba 100644 --- a/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/sink/connector/RedisSinkConnector.java +++ b/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/sink/connector/RedisSinkConnector.java @@ -17,9 +17,9 @@ package org.apache.eventmesh.connector.redis.sink.connector; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.redis.RedisSinkConfig; import org.apache.eventmesh.connector.redis.cloudevent.CloudEventCodec; -import org.apache.eventmesh.connector.redis.sink.config.RedisSinkConfig; -import org.apache.eventmesh.openconnect.api.config.Config; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; import org.apache.eventmesh.openconnect.api.sink.Sink; @@ -85,6 +85,11 @@ public String name() { return this.sinkConfig.getConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() throws Exception { this.redissonClient.shutdown(); diff --git a/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/source/connector/RedisSourceConnector.java b/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/source/connector/RedisSourceConnector.java index 2933d545dd..5b858afa30 100644 --- a/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/source/connector/RedisSourceConnector.java +++ b/eventmesh-connectors/eventmesh-connector-redis/src/main/java/org/apache/eventmesh/connector/redis/source/connector/RedisSourceConnector.java @@ -17,9 +17,9 @@ package org.apache.eventmesh.connector.redis.source.connector; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.redis.RedisSourceConfig; import org.apache.eventmesh.connector.redis.cloudevent.CloudEventCodec; -import org.apache.eventmesh.connector.redis.source.config.RedisSourceConfig; -import org.apache.eventmesh.openconnect.api.config.Config; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; import org.apache.eventmesh.openconnect.api.source.Source; @@ -40,8 +40,6 @@ public class RedisSourceConnector implements Source { - private static final int DEFAULT_BATCH_SIZE = 10; - private RTopic topic; private RedisSourceConfig sourceConfig; @@ -50,6 +48,10 @@ public class RedisSourceConnector implements Source { private BlockingQueue queue; + private int maxBatchSize; + + private long maxPollWaitTime; + @Override public Class configClass() { return RedisSourceConfig.class; @@ -73,7 +75,9 @@ private void doInit() { redisConfig.useSingleServer().setAddress(sourceConfig.connectorConfig.getServer()); redisConfig.setCodec(CloudEventCodec.getInstance()); this.redissonClient = Redisson.create(redisConfig); - this.queue = new LinkedBlockingQueue<>(1000); + this.queue = new LinkedBlockingQueue<>(sourceConfig.getPollConfig().getCapacity()); + this.maxBatchSize = sourceConfig.getPollConfig().getMaxBatchSize(); + this.maxPollWaitTime = sourceConfig.getPollConfig().getMaxWaitTime(); } @Override @@ -94,6 +98,11 @@ public String name() { return this.sourceConfig.getConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() throws Exception { this.topic.removeAllListeners(); @@ -102,15 +111,21 @@ public void stop() throws Exception { @Override public List poll() { - List connectRecords = new ArrayList<>(DEFAULT_BATCH_SIZE); - for (int count = 0; count < DEFAULT_BATCH_SIZE; ++count) { + long startTime = System.currentTimeMillis(); + long remainingTime = maxPollWaitTime; + + List connectRecords = new ArrayList<>(maxBatchSize); + for (int count = 0; count < maxBatchSize; ++count) { try { - CloudEvent event = queue.poll(3, TimeUnit.SECONDS); + CloudEvent event = queue.poll(remainingTime, TimeUnit.MILLISECONDS); if (event == null) { break; } - connectRecords.add(CloudEventUtil.convertEventToRecord(event)); + + // calculate elapsed time and update remaining time for next poll + long elapsedTime = System.currentTimeMillis() - startTime; + remainingTime = maxPollWaitTime > elapsedTime ? maxPollWaitTime - elapsedTime : 0; } catch (InterruptedException e) { break; } diff --git a/eventmesh-connectors/eventmesh-connector-redis/src/test/java/org/apache/eventmesh/connector/redis/sink/connector/RedisSinkConnectorTest.java b/eventmesh-connectors/eventmesh-connector-redis/src/test/java/org/apache/eventmesh/connector/redis/sink/connector/RedisSinkConnectorTest.java index 13ec4f7379..c4d153bd25 100644 --- a/eventmesh-connectors/eventmesh-connector-redis/src/test/java/org/apache/eventmesh/connector/redis/sink/connector/RedisSinkConnectorTest.java +++ b/eventmesh-connectors/eventmesh-connector-redis/src/test/java/org/apache/eventmesh/connector/redis/sink/connector/RedisSinkConnectorTest.java @@ -17,12 +17,10 @@ package org.apache.eventmesh.connector.redis.sink.connector; +import org.apache.eventmesh.common.config.connector.redis.RedisSinkConfig; import org.apache.eventmesh.connector.redis.AbstractRedisServer; import org.apache.eventmesh.connector.redis.cloudevent.CloudEventCodec; -import org.apache.eventmesh.connector.redis.sink.config.RedisSinkConfig; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; import org.apache.eventmesh.openconnect.util.ConfigUtil; import java.nio.charset.StandardCharsets; @@ -79,9 +77,7 @@ public void testPutConnectRecords() throws InterruptedException { List records = new ArrayList<>(); for (int i = 0; i < expectedCount; i++) { - RecordPartition partition = new RecordPartition(); - RecordOffset offset = new RecordOffset(); - ConnectRecord connectRecord = new ConnectRecord(partition, offset, System.currentTimeMillis(), + ConnectRecord connectRecord = new ConnectRecord(null, null, System.currentTimeMillis(), expectedMessage.getBytes(StandardCharsets.UTF_8)); connectRecord.addExtension("id", String.valueOf(UUID.randomUUID())); connectRecord.addExtension("source", "testSource"); diff --git a/eventmesh-connectors/eventmesh-connector-redis/src/test/java/org/apache/eventmesh/connector/redis/source/connector/RedisSourceConnectorTest.java b/eventmesh-connectors/eventmesh-connector-redis/src/test/java/org/apache/eventmesh/connector/redis/source/connector/RedisSourceConnectorTest.java index da68a5b170..326798d64a 100644 --- a/eventmesh-connectors/eventmesh-connector-redis/src/test/java/org/apache/eventmesh/connector/redis/source/connector/RedisSourceConnectorTest.java +++ b/eventmesh-connectors/eventmesh-connector-redis/src/test/java/org/apache/eventmesh/connector/redis/source/connector/RedisSourceConnectorTest.java @@ -17,12 +17,10 @@ package org.apache.eventmesh.connector.redis.source.connector; +import org.apache.eventmesh.common.config.connector.redis.RedisSourceConfig; import org.apache.eventmesh.connector.redis.AbstractRedisServer; import org.apache.eventmesh.connector.redis.cloudevent.CloudEventCodec; -import org.apache.eventmesh.connector.redis.source.config.RedisSourceConfig; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; import org.apache.eventmesh.openconnect.util.CloudEventUtil; import org.apache.eventmesh.openconnect.util.ConfigUtil; @@ -76,9 +74,7 @@ public void testPollConnectRecords() throws Exception { private void publishMockEvents() { int mockCount = 5; for (int i = 0; i < mockCount; i++) { - RecordPartition partition = new RecordPartition(); - RecordOffset offset = new RecordOffset(); - ConnectRecord connectRecord = new ConnectRecord(partition, offset, System.currentTimeMillis(), + ConnectRecord connectRecord = new ConnectRecord(null, null, System.currentTimeMillis(), ("\"" + expectedMessage + "\"").getBytes(StandardCharsets.UTF_8)); connectRecord.addExtension("id", String.valueOf(UUID.randomUUID())); connectRecord.addExtension("source", "testSource"); diff --git a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/config/RocketMQServerConfig.java b/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/config/RocketMQServerConfig.java index 4dc574cda2..842094e07f 100644 --- a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/config/RocketMQServerConfig.java +++ b/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/config/RocketMQServerConfig.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.rocketmq.config; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/sink/connector/RocketMQSinkConnector.java b/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/sink/connector/RocketMQSinkConnector.java index a63d92d7f5..31d45a28f4 100644 --- a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/sink/connector/RocketMQSinkConnector.java +++ b/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/sink/connector/RocketMQSinkConnector.java @@ -17,8 +17,9 @@ package org.apache.eventmesh.connector.rocketmq.sink.connector; -import org.apache.eventmesh.connector.rocketmq.sink.config.RocketMQSinkConfig; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.mq.rocketmq.RocketMQSinkConfig; +import org.apache.eventmesh.openconnect.api.ConnectorCreateService; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; import org.apache.eventmesh.openconnect.api.sink.Sink; @@ -34,7 +35,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public class RocketMQSinkConnector implements Sink { +public class RocketMQSinkConnector implements Sink, ConnectorCreateService { private RocketMQSinkConfig sinkConfig; @@ -77,6 +78,11 @@ public String name() { return this.sinkConfig.getConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() { producer.shutdown(); @@ -108,4 +114,9 @@ public Message convertRecordToMessage(ConnectRecord connectRecord) { } return message; } + + @Override + public Sink create() { + return new RocketMQSinkConnector(); + } } diff --git a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/source/connector/RocketMQSourceConnector.java b/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/source/connector/RocketMQSourceConnector.java index 8023be479f..410f927d75 100644 --- a/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/source/connector/RocketMQSourceConnector.java +++ b/eventmesh-connectors/eventmesh-connector-rocketmq/src/main/java/org/apache/eventmesh/connector/rocketmq/source/connector/RocketMQSourceConnector.java @@ -17,14 +17,17 @@ package org.apache.eventmesh.connector.rocketmq.source.connector; -import org.apache.eventmesh.connector.rocketmq.source.config.RocketMQSourceConfig; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.mq.rocketmq.RocketMQSourceConfig; +import org.apache.eventmesh.common.remote.offset.RecordOffset; +import org.apache.eventmesh.common.remote.offset.RecordPartition; +import org.apache.eventmesh.common.remote.offset.rocketmq.RocketMQRecordOffset; +import org.apache.eventmesh.common.remote.offset.rocketmq.RocketMQRecordPartition; +import org.apache.eventmesh.openconnect.api.ConnectorCreateService; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; import org.apache.eventmesh.openconnect.api.source.Source; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetStorageReader; import org.apache.rocketmq.client.consumer.AllocateMessageQueueStrategy; @@ -60,7 +63,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public class RocketMQSourceConnector implements Source { +public class RocketMQSourceConnector implements Source, ConnectorCreateService { private RocketMQSourceConfig sourceConfig; @@ -77,7 +80,7 @@ public class RocketMQSourceConnector implements Source { private final ConcurrentHashMap> prepareCommitOffset = new ConcurrentHashMap<>(); - private ConcurrentHashMap> queue2Offsets = new ConcurrentHashMap<>(); + private final ConcurrentHashMap> queue2Offsets = new ConcurrentHashMap<>(); private final AtomicInteger unAckCounter = new AtomicInteger(); @@ -141,15 +144,14 @@ public void start() throws Exception { for (MessageQueue messageQueue : mqDivided) { try { - Map partitionMap = new HashMap<>(); - partitionMap.put("topic", messageQueue.getTopic()); - partitionMap.put("brokerName", messageQueue.getBrokerName()); - partitionMap.put("queueId", messageQueue.getQueueId() + ""); - RecordPartition recordPartition = new RecordPartition(partitionMap); + RocketMQRecordPartition recordPartition = new RocketMQRecordPartition(); + recordPartition.setBroker(messageQueue.getBrokerName()); + recordPartition.setTopic(messageQueue.getTopic()); + recordPartition.setQueueId(messageQueue.getQueueId() + ""); RecordOffset recordOffset = offsetStorageReader.readOffset(recordPartition); log.info("assigned messageQueue {}, recordOffset {}", messageQueue, recordOffset); if (recordOffset != null) { - long pollOffset = (Long) recordOffset.getOffset().get("queueOffset"); + long pollOffset = ((RocketMQRecordOffset) recordOffset).getQueueOffset(); if (pollOffset != 0) { consumer.seek(messageQueue, pollOffset); } @@ -186,13 +188,13 @@ private List getMessageQueueList(String topic) throws MQClientExce @Override public void commit(ConnectRecord record) { // send success, commit offset - Map map = record.getPosition().getPartition().getPartition(); - String brokerName = (String) map.get("brokerName"); - String topic = (String) map.get("topic"); - int queueId = Integer.parseInt((String) map.get("queueId")); + RocketMQRecordPartition rocketMQRecordPartition = (RocketMQRecordPartition) (record.getPosition().getRecordPartition()); + String brokerName = rocketMQRecordPartition.getBroker(); + String topic = rocketMQRecordPartition.getTopic(); + int queueId = Integer.parseInt(rocketMQRecordPartition.getQueueId()); MessageQueue mq = new MessageQueue(topic, brokerName, queueId); - Map offsetMap = record.getPosition().getOffset().getOffset(); - long offset = Long.parseLong((String) offsetMap.get("queueOffset")); + RocketMQRecordOffset rocketMQRecordOffset = (RocketMQRecordOffset) record.getPosition().getRecordOffset(); + long offset = rocketMQRecordOffset.getQueueOffset(); long canCommitOffset = removeMessage(mq, offset); log.info("commit record {}|mq {}|canCommitOffset {}", record, mq, canCommitOffset); // commit offset to prepareCommitOffset @@ -204,6 +206,11 @@ public String name() { return this.sourceConfig.getConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() { consumer.unsubscribe(sourceConfig.getConnectorConfig().getTopic()); @@ -232,17 +239,18 @@ public List poll() { } public static RecordOffset convertToRecordOffset(Long offset) { - Map offsetMap = new HashMap<>(); - offsetMap.put("queueOffset", offset + ""); - return new RecordOffset(offsetMap); + RocketMQRecordOffset rocketMQRecordOffset = new RocketMQRecordOffset(); + rocketMQRecordOffset.setQueueOffset(offset); + return rocketMQRecordOffset; } public static RecordPartition convertToRecordPartition(String topic, String brokerName, int queueId) { - Map map = new HashMap<>(); - map.put("topic", topic); - map.put("brokerName", brokerName); - map.put("queueId", queueId + ""); - return new RecordPartition(map); + RocketMQRecordPartition rocketMQRecordPartition = new RocketMQRecordPartition(); + rocketMQRecordPartition.setBroker(brokerName); + rocketMQRecordPartition.setTopic(topic); + rocketMQRecordPartition.setQueueId(queueId + ""); + + return rocketMQRecordPartition; } private void putPulledQueueOffset(MessageExt messageExt) { @@ -301,4 +309,9 @@ public void commitOffset(MessageQueue mq, long canCommitOffset) { commitOffset.add(new AtomicLong(nextBeginOffset)); prepareCommitOffset.put(mq, commitOffset); } + + @Override + public Source create() { + return new RocketMQSourceConnector(); + } } diff --git a/eventmesh-connectors/eventmesh-connector-rocketmq/src/test/java/org/apache/eventmesh/connector/rocketmq/sink/connector/RocketMQSinkConnectorTest.java b/eventmesh-connectors/eventmesh-connector-rocketmq/src/test/java/org/apache/eventmesh/connector/rocketmq/sink/connector/RocketMQSinkConnectorTest.java index afd13a3f2f..51d77182a0 100644 --- a/eventmesh-connectors/eventmesh-connector-rocketmq/src/test/java/org/apache/eventmesh/connector/rocketmq/sink/connector/RocketMQSinkConnectorTest.java +++ b/eventmesh-connectors/eventmesh-connector-rocketmq/src/test/java/org/apache/eventmesh/connector/rocketmq/sink/connector/RocketMQSinkConnectorTest.java @@ -21,10 +21,8 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import org.apache.eventmesh.connector.rocketmq.sink.config.RocketMQSinkConfig; +import org.apache.eventmesh.common.config.connector.mq.rocketmq.RocketMQSinkConfig; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; import org.apache.eventmesh.openconnect.util.ConfigUtil; import org.apache.rocketmq.client.producer.DefaultMQProducer; @@ -81,9 +79,7 @@ public void testRocketMQSinkConnector() throws Exception { private List generateMockedRecords(final int messageCount) { List records = new ArrayList<>(); for (int i = 0; i < messageCount; i++) { - RecordPartition partition = new RecordPartition(); - RecordOffset offset = new RecordOffset(); - ConnectRecord connectRecord = new ConnectRecord(partition, offset, System.currentTimeMillis(), + ConnectRecord connectRecord = new ConnectRecord(null, null, System.currentTimeMillis(), EXPECTED_MESSAGE.getBytes(StandardCharsets.UTF_8)); connectRecord.addExtension("id", String.valueOf(UUID.randomUUID())); records.add(connectRecord); diff --git a/eventmesh-connectors/eventmesh-connector-rocketmq/src/test/java/org/apache/eventmesh/connector/rocketmq/source/connector/RocketMQSourceConnectorTest.java b/eventmesh-connectors/eventmesh-connector-rocketmq/src/test/java/org/apache/eventmesh/connector/rocketmq/source/connector/RocketMQSourceConnectorTest.java index 5f5e3410bf..78510e2e4f 100644 --- a/eventmesh-connectors/eventmesh-connector-rocketmq/src/test/java/org/apache/eventmesh/connector/rocketmq/source/connector/RocketMQSourceConnectorTest.java +++ b/eventmesh-connectors/eventmesh-connector-rocketmq/src/test/java/org/apache/eventmesh/connector/rocketmq/source/connector/RocketMQSourceConnectorTest.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.rocketmq.source.connector; -import org.apache.eventmesh.connector.rocketmq.source.config.RocketMQSourceConfig; +import org.apache.eventmesh.common.config.connector.mq.rocketmq.RocketMQSourceConfig; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; import org.apache.eventmesh.openconnect.util.ConfigUtil; diff --git a/eventmesh-connectors/eventmesh-connector-s3/build.gradle b/eventmesh-connectors/eventmesh-connector-s3/build.gradle index 9d004b2f15..af2867917e 100644 --- a/eventmesh-connectors/eventmesh-connector-s3/build.gradle +++ b/eventmesh-connectors/eventmesh-connector-s3/build.gradle @@ -16,6 +16,7 @@ */ dependencies { + implementation project(":eventmesh-common") implementation project(":eventmesh-openconnect:eventmesh-openconnect-java") implementation 'software.amazon.awssdk:s3' compileOnly 'org.projectlombok:lombok' diff --git a/eventmesh-connectors/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/config/S3ServerConfig.java b/eventmesh-connectors/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/config/S3ServerConfig.java index f5c4371d4c..a422c0468c 100644 --- a/eventmesh-connectors/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/config/S3ServerConfig.java +++ b/eventmesh-connectors/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/config/S3ServerConfig.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.s3.config; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/source/connector/S3SourceConnector.java b/eventmesh-connectors/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/source/connector/S3SourceConnector.java index 83ac15398b..078ed7691a 100644 --- a/eventmesh-connectors/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/source/connector/S3SourceConnector.java +++ b/eventmesh-connectors/eventmesh-connector-s3/src/main/java/org/apache/eventmesh/connector/s3/source/connector/S3SourceConnector.java @@ -17,21 +17,21 @@ package org.apache.eventmesh.connector.s3.source.connector; -import org.apache.eventmesh.connector.s3.source.config.S3SourceConfig; -import org.apache.eventmesh.connector.s3.source.config.SourceConnectorConfig; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.s3.S3SourceConfig; +import org.apache.eventmesh.common.config.connector.s3.SourceConnectorConfig; +import org.apache.eventmesh.common.remote.offset.RecordOffset; +import org.apache.eventmesh.common.remote.offset.RecordPartition; +import org.apache.eventmesh.common.remote.offset.S3.S3RecordOffset; +import org.apache.eventmesh.common.remote.offset.S3.S3RecordPartition; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; import org.apache.eventmesh.openconnect.api.source.Source; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; import java.util.Optional; import java.util.concurrent.CompletableFuture; import java.util.concurrent.TimeUnit; @@ -121,6 +121,11 @@ public String name() { return this.sourceConfig.getSourceConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() throws Exception { @@ -157,16 +162,16 @@ public List poll() { } private RecordPartition getRecordPartition() { - Map map = new HashMap<>(); - map.put(REGION, this.sourceConnectorConfig.getRegion()); - map.put(BUCKET, this.sourceConnectorConfig.getBucket()); - map.put(FILE_NAME, this.sourceConnectorConfig.getFileName()); - return new RecordPartition(map); + S3RecordPartition s3RecordPartition = new S3RecordPartition(); + s3RecordPartition.setRegion(this.sourceConnectorConfig.getRegion()); + s3RecordPartition.setBucket(this.sourceConnectorConfig.getBucket()); + s3RecordPartition.setFileName(this.sourceConnectorConfig.getFileName()); + return s3RecordPartition; } private RecordOffset getRecordOffset() { - Map map = new HashMap<>(); - map.put(POSITION, String.valueOf(this.position)); - return new RecordOffset(map); + S3RecordOffset s3RecordOffset = new S3RecordOffset(); + s3RecordOffset.setOffset(this.position); + return s3RecordOffset; } } diff --git a/eventmesh-connectors/eventmesh-connector-s3/src/test/java/org/apache/eventmesh/connector/s3/source/S3SourceConnectorTest.java b/eventmesh-connectors/eventmesh-connector-s3/src/test/java/org/apache/eventmesh/connector/s3/source/S3SourceConnectorTest.java index d6bb08d421..4d5d41093b 100644 --- a/eventmesh-connectors/eventmesh-connector-s3/src/test/java/org/apache/eventmesh/connector/s3/source/S3SourceConnectorTest.java +++ b/eventmesh-connectors/eventmesh-connector-s3/src/test/java/org/apache/eventmesh/connector/s3/source/S3SourceConnectorTest.java @@ -17,8 +17,8 @@ package org.apache.eventmesh.connector.s3.source; -import org.apache.eventmesh.connector.s3.source.config.S3SourceConfig; -import org.apache.eventmesh.connector.s3.source.config.SourceConnectorConfig; +import org.apache.eventmesh.common.config.connector.s3.S3SourceConfig; +import org.apache.eventmesh.common.config.connector.s3.SourceConnectorConfig; import org.apache.eventmesh.connector.s3.source.connector.S3SourceConnector; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; diff --git a/eventmesh-connectors/eventmesh-connector-slack/build.gradle b/eventmesh-connectors/eventmesh-connector-slack/build.gradle index 90b7d9b9b5..665f748b5f 100644 --- a/eventmesh-connectors/eventmesh-connector-slack/build.gradle +++ b/eventmesh-connectors/eventmesh-connector-slack/build.gradle @@ -20,7 +20,7 @@ dependencies { implementation project(":eventmesh-sdks:eventmesh-sdk-java") implementation project(":eventmesh-openconnect:eventmesh-openconnect-java") - implementation "com.slack.api:bolt:1.39.+" + implementation "com.slack.api:bolt:1.42.+" implementation 'com.google.guava:guava' compileOnly 'org.projectlombok:lombok' diff --git a/eventmesh-connectors/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/config/SlackConnectServerConfig.java b/eventmesh-connectors/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/config/SlackConnectServerConfig.java index 515c5af6c7..97479cb87a 100644 --- a/eventmesh-connectors/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/config/SlackConnectServerConfig.java +++ b/eventmesh-connectors/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/config/SlackConnectServerConfig.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.slack.config; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/sink/connector/SlackSinkConnector.java b/eventmesh-connectors/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/sink/connector/SlackSinkConnector.java index a026f2aa22..836409af71 100644 --- a/eventmesh-connectors/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/sink/connector/SlackSinkConnector.java +++ b/eventmesh-connectors/eventmesh-connector-slack/src/main/java/org/apache/eventmesh/connector/slack/sink/connector/SlackSinkConnector.java @@ -17,8 +17,8 @@ package org.apache.eventmesh.connector.slack.sink.connector; -import org.apache.eventmesh.connector.slack.sink.config.SlackSinkConfig; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.slack.SlackSinkConfig; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; import org.apache.eventmesh.openconnect.api.sink.Sink; @@ -84,6 +84,11 @@ public String name() { return this.sinkConfig.getSinkConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() { isRunning = false; diff --git a/eventmesh-connectors/eventmesh-connector-slack/src/test/java/org/apache/eventmesh/connector/slack/sink/connector/SlackSinkConnectorTest.java b/eventmesh-connectors/eventmesh-connector-slack/src/test/java/org/apache/eventmesh/connector/slack/sink/connector/SlackSinkConnectorTest.java index 3f0a32755f..fc5f04c7e1 100644 --- a/eventmesh-connectors/eventmesh-connector-slack/src/test/java/org/apache/eventmesh/connector/slack/sink/connector/SlackSinkConnectorTest.java +++ b/eventmesh-connectors/eventmesh-connector-slack/src/test/java/org/apache/eventmesh/connector/slack/sink/connector/SlackSinkConnectorTest.java @@ -22,10 +22,8 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import org.apache.eventmesh.connector.slack.sink.config.SlackSinkConfig; +import org.apache.eventmesh.common.config.connector.slack.SlackSinkConfig; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; import org.apache.eventmesh.openconnect.util.ConfigUtil; import java.lang.reflect.Field; @@ -74,9 +72,7 @@ public void testSendMessageToSlack() throws Exception { final int times = 3; List records = new ArrayList<>(); for (int i = 0; i < times; i++) { - RecordPartition partition = new RecordPartition(); - RecordOffset offset = new RecordOffset(); - ConnectRecord connectRecord = new ConnectRecord(partition, offset, + ConnectRecord connectRecord = new ConnectRecord(null, null, System.currentTimeMillis(), "Hello, EventMesh!".getBytes(StandardCharsets.UTF_8)); records.add(connectRecord); } diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/config/SpringConnectServerConfig.java b/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/config/SpringConnectServerConfig.java index 7eb2009fbd..88cf8156a8 100644 --- a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/config/SpringConnectServerConfig.java +++ b/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/config/SpringConnectServerConfig.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.spring.config; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/connector/SpringSinkConnector.java b/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/connector/SpringSinkConnector.java index 648e1f8071..9ba99cd547 100644 --- a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/connector/SpringSinkConnector.java +++ b/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/sink/connector/SpringSinkConnector.java @@ -17,8 +17,8 @@ package org.apache.eventmesh.connector.spring.sink.connector; -import org.apache.eventmesh.connector.spring.sink.config.SpringSinkConfig; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.spring.SpringSinkConfig; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; import org.apache.eventmesh.openconnect.api.sink.Sink; @@ -77,6 +77,11 @@ public String name() { return this.sinkConfig.getSinkConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() throws Exception { diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/MessageSendingOperations.java b/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/MessageSendingOperations.java index a337c1cd81..5f38914bb1 100644 --- a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/MessageSendingOperations.java +++ b/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/MessageSendingOperations.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.spring.source; -import org.apache.eventmesh.openconnect.api.callback.SendMessageCallback; +import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendMessageCallback; /** * Operations for sending messages. diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/connector/SpringSourceConnector.java b/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/connector/SpringSourceConnector.java index 5f4d5c89b7..6efed2db3c 100644 --- a/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/connector/SpringSourceConnector.java +++ b/eventmesh-connectors/eventmesh-connector-spring/src/main/java/org/apache/eventmesh/connector/spring/source/connector/SpringSourceConnector.java @@ -17,17 +17,19 @@ package org.apache.eventmesh.connector.spring.source.connector; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.spring.SpringSourceConfig; +import org.apache.eventmesh.common.remote.offset.RecordOffset; +import org.apache.eventmesh.common.remote.offset.RecordPartition; +import org.apache.eventmesh.common.remote.offset.spring.SpringRecordOffset; +import org.apache.eventmesh.common.remote.offset.spring.SpringRecordPartition; import org.apache.eventmesh.connector.spring.source.MessageSendingOperations; -import org.apache.eventmesh.connector.spring.source.config.SpringSourceConfig; import org.apache.eventmesh.openconnect.SourceWorker; -import org.apache.eventmesh.openconnect.api.callback.SendMessageCallback; -import org.apache.eventmesh.openconnect.api.config.Config; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; import org.apache.eventmesh.openconnect.api.source.Source; +import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendMessageCallback; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; import java.util.ArrayList; import java.util.List; @@ -50,14 +52,16 @@ public class SpringSourceConnector implements Source, MessageSendingOperations, private static final String CONNECTOR_PROPERTY_PREFIX = "eventmesh.connector."; - private static final int DEFAULT_BATCH_SIZE = 10; - private ApplicationContext applicationContext; private SpringSourceConfig sourceConfig; private BlockingQueue queue; + private int maxBatchSize; + + private long maxPollWaitTime; + @Override public Class configClass() { return SpringSourceConfig.class; @@ -67,7 +71,7 @@ public Class configClass() { public void init(Config config) throws Exception { // init config for spring source connector this.sourceConfig = (SpringSourceConfig) config; - this.queue = new LinkedBlockingQueue<>(1000); + doInit(); } @Override @@ -75,7 +79,13 @@ public void init(ConnectorContext connectorContext) throws Exception { SourceConnectorContext sourceConnectorContext = (SourceConnectorContext) connectorContext; // init config for spring source connector this.sourceConfig = (SpringSourceConfig) sourceConnectorContext.getSourceConfig(); - this.queue = new LinkedBlockingQueue<>(1000); + doInit(); + } + + private void doInit() { + this.queue = new LinkedBlockingQueue<>(sourceConfig.getPollConfig().getCapacity()); + this.maxBatchSize = sourceConfig.getPollConfig().getMaxBatchSize(); + this.maxPollWaitTime = sourceConfig.getPollConfig().getMaxWaitTime(); } @Override @@ -93,6 +103,11 @@ public String name() { return this.sourceConfig.getSourceConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() throws Exception { @@ -100,15 +115,21 @@ public void stop() throws Exception { @Override public List poll() { - List connectRecords = new ArrayList<>(DEFAULT_BATCH_SIZE); + long startTime = System.currentTimeMillis(); + long remainingTime = maxPollWaitTime; - for (int count = 0; count < DEFAULT_BATCH_SIZE; ++count) { + List connectRecords = new ArrayList<>(maxBatchSize); + for (int count = 0; count < maxBatchSize; ++count) { try { - ConnectRecord connectRecord = queue.poll(3, TimeUnit.SECONDS); + ConnectRecord connectRecord = queue.poll(remainingTime, TimeUnit.MILLISECONDS); if (connectRecord == null) { break; } connectRecords.add(connectRecord); + + // calculate elapsed time and update remaining time for next poll + long elapsedTime = System.currentTimeMillis() - startTime; + remainingTime = maxPollWaitTime > elapsedTime ? maxPollWaitTime - elapsedTime : 0; } catch (InterruptedException e) { Thread currentThread = Thread.currentThread(); log.warn("[SpringSourceConnector] Interrupting thread {} due to exception {}", @@ -121,12 +142,13 @@ public List poll() { /** * Send message. + * * @param message message to send */ @Override public void send(Object message) { - RecordPartition partition = new RecordPartition(); - RecordOffset offset = new RecordOffset(); + RecordPartition partition = new SpringRecordPartition(); + RecordOffset offset = new SpringRecordOffset(); ConnectRecord record = new ConnectRecord(partition, offset, System.currentTimeMillis(), message); addSpringEnvironmentPropertyExtensions(record); queue.offer(record); @@ -134,14 +156,14 @@ public void send(Object message) { /** * Send message with a callback. - * @param message message to send. - * @param workerCallback After the user sends the message to the Connector, - * the SourceWorker will fetch message and invoke. + * + * @param message message to send. + * @param workerCallback After the user sends the message to the Connector, the SourceWorker will fetch message and invoke. */ @Override public void send(Object message, SendMessageCallback workerCallback) { - RecordPartition partition = new RecordPartition(); - RecordOffset offset = new RecordOffset(); + RecordPartition partition = new SpringRecordPartition(); + RecordOffset offset = new SpringRecordOffset(); ConnectRecord record = new ConnectRecord(partition, offset, System.currentTimeMillis(), message); record.addExtension(SourceWorker.CALLBACK_EXTENSION, workerCallback); addSpringEnvironmentPropertyExtensions(record); diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/test/java/org/apache/eventmesh/connector/spring/sink/connector/SpringSinkConnectorTest.java b/eventmesh-connectors/eventmesh-connector-spring/src/test/java/org/apache/eventmesh/connector/spring/sink/connector/SpringSinkConnectorTest.java index ea0ea7c359..767c8803de 100644 --- a/eventmesh-connectors/eventmesh-connector-spring/src/test/java/org/apache/eventmesh/connector/spring/sink/connector/SpringSinkConnectorTest.java +++ b/eventmesh-connectors/eventmesh-connector-spring/src/test/java/org/apache/eventmesh/connector/spring/sink/connector/SpringSinkConnectorTest.java @@ -19,10 +19,8 @@ import static org.junit.jupiter.api.Assertions.assertNotNull; -import org.apache.eventmesh.connector.spring.sink.config.SpringSinkConfig; +import org.apache.eventmesh.common.config.connector.spring.SpringSinkConfig; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; import java.util.ArrayList; import java.util.List; @@ -71,9 +69,7 @@ public void testProcessRecordsInSinkConnectorQueue() throws Exception { private void writeMockedRecords(int count, String message) throws Exception { List records = new ArrayList<>(); for (int i = 0; i < count; i++) { - RecordPartition partition = new RecordPartition(); - RecordOffset offset = new RecordOffset(); - records.add(new ConnectRecord(partition, offset, System.currentTimeMillis(), message + i)); + records.add(new ConnectRecord(null, null, System.currentTimeMillis(), message + i)); } connector.put(records); } diff --git a/eventmesh-connectors/eventmesh-connector-spring/src/test/java/org/apache/eventmesh/connector/spring/source/connector/SpringSourceConnectorTest.java b/eventmesh-connectors/eventmesh-connector-spring/src/test/java/org/apache/eventmesh/connector/spring/source/connector/SpringSourceConnectorTest.java index 65c71aea09..c4c59f0e5f 100644 --- a/eventmesh-connectors/eventmesh-connector-spring/src/test/java/org/apache/eventmesh/connector/spring/source/connector/SpringSourceConnectorTest.java +++ b/eventmesh-connectors/eventmesh-connector-spring/src/test/java/org/apache/eventmesh/connector/spring/source/connector/SpringSourceConnectorTest.java @@ -19,7 +19,7 @@ import static org.mockito.Mockito.doReturn; -import org.apache.eventmesh.connector.spring.source.config.SpringSourceConfig; +import org.apache.eventmesh.common.config.connector.spring.SpringSourceConfig; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; import java.util.List; diff --git a/eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/config/WeChatConnectServerConfig.java b/eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/config/WeChatConnectServerConfig.java index a2634b6b49..2555041c02 100644 --- a/eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/config/WeChatConnectServerConfig.java +++ b/eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/config/WeChatConnectServerConfig.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.wechat.config; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/sink/connector/WeChatSinkConnector.java b/eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/sink/connector/WeChatSinkConnector.java index ac66ec45cc..6908d119b9 100644 --- a/eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/sink/connector/WeChatSinkConnector.java +++ b/eventmesh-connectors/eventmesh-connector-wechat/src/main/java/org/apache/eventmesh/connector/wechat/sink/connector/WeChatSinkConnector.java @@ -17,9 +17,9 @@ package org.apache.eventmesh.connector.wechat.sink.connector; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.wechat.WeChatSinkConfig; import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.connector.wechat.sink.config.WeChatSinkConfig; -import org.apache.eventmesh.openconnect.api.config.Config; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; import org.apache.eventmesh.openconnect.api.sink.Sink; @@ -115,6 +115,11 @@ public String name() { return this.sinkConfig.getSinkConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() throws IOException { isRunning = false; diff --git a/eventmesh-connectors/eventmesh-connector-wechat/src/test/java/org/apache/eventmesh/connector/wechat/sink/connector/WeChatSinkConnectorTest.java b/eventmesh-connectors/eventmesh-connector-wechat/src/test/java/org/apache/eventmesh/connector/wechat/sink/connector/WeChatSinkConnectorTest.java index d993468c18..00432a4e2c 100644 --- a/eventmesh-connectors/eventmesh-connector-wechat/src/test/java/org/apache/eventmesh/connector/wechat/sink/connector/WeChatSinkConnectorTest.java +++ b/eventmesh-connectors/eventmesh-connector-wechat/src/test/java/org/apache/eventmesh/connector/wechat/sink/connector/WeChatSinkConnectorTest.java @@ -21,10 +21,8 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import org.apache.eventmesh.connector.wechat.sink.config.WeChatSinkConfig; +import org.apache.eventmesh.common.config.connector.wechat.WeChatSinkConfig; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; import org.apache.eventmesh.openconnect.util.ConfigUtil; import java.io.IOException; @@ -111,9 +109,7 @@ public void testSendMessageToWeChat() throws Exception { Mockito.doReturn(sendMessageResponse).when(sendMessageRequestCall).execute(); List records = new ArrayList<>(); - RecordPartition partition = new RecordPartition(); - RecordOffset offset = new RecordOffset(); - ConnectRecord connectRecord = new ConnectRecord(partition, offset, + ConnectRecord connectRecord = new ConnectRecord(null, null, System.currentTimeMillis(), "Hello, EventMesh!".getBytes(StandardCharsets.UTF_8)); records.add(connectRecord); @@ -141,9 +137,7 @@ public void testSendMessageToWeChatAbnormally() throws Exception { Mockito.doReturn(sendMessageRequestCall).when(okHttpClient).newCall(Mockito.argThat(sendMessageMatcher)); Mockito.doReturn(sendMessageResponse).when(sendMessageRequestCall).execute(); - RecordPartition partition = new RecordPartition(); - RecordOffset offset = new RecordOffset(); - ConnectRecord connectRecord = new ConnectRecord(partition, offset, + ConnectRecord connectRecord = new ConnectRecord(null, null, System.currentTimeMillis(), "Hello, EventMesh!".getBytes(StandardCharsets.UTF_8)); Method sendMessageMethod = WeChatSinkConnector.class.getDeclaredMethod("sendMessage", ConnectRecord.class); sendMessageMethod.setAccessible(true); diff --git a/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/config/WeComConnectServerConfig.java b/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/config/WeComConnectServerConfig.java index 1f864726bf..38235b3d3b 100644 --- a/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/config/WeComConnectServerConfig.java +++ b/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/config/WeComConnectServerConfig.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.connector.wecom.config; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.config.connector.Config; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/sink/connector/WeComSinkConnector.java b/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/sink/connector/WeComSinkConnector.java index 499104e11a..ca628fa590 100644 --- a/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/sink/connector/WeComSinkConnector.java +++ b/eventmesh-connectors/eventmesh-connector-wecom/src/main/java/org/apache/eventmesh/connector/wecom/sink/connector/WeComSinkConnector.java @@ -18,12 +18,12 @@ package org.apache.eventmesh.connector.wecom.sink.connector; import org.apache.eventmesh.common.Constants; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.wecom.WeComSinkConfig; import org.apache.eventmesh.common.enums.EventMeshDataContentType; import org.apache.eventmesh.common.utils.JsonUtils; import org.apache.eventmesh.connector.wecom.config.WeComMessageTemplateType; import org.apache.eventmesh.connector.wecom.constants.ConnectRecordExtensionKeys; -import org.apache.eventmesh.connector.wecom.sink.config.WeComSinkConfig; -import org.apache.eventmesh.openconnect.api.config.Config; import org.apache.eventmesh.openconnect.api.connector.ConnectorContext; import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; import org.apache.eventmesh.openconnect.api.sink.Sink; @@ -95,6 +95,11 @@ public String name() { return this.sinkConfig.getSinkConnectorConfig().getConnectorName(); } + @Override + public void onException(ConnectRecord record) { + + } + @Override public void stop() throws IOException { isRunning = false; diff --git a/eventmesh-connectors/eventmesh-connector-wecom/src/test/java/org/apache/eventmesh/connector/wecom/connector/MockRecordOffset.java b/eventmesh-connectors/eventmesh-connector-wecom/src/test/java/org/apache/eventmesh/connector/wecom/connector/MockRecordOffset.java new file mode 100644 index 0000000000..066fe3f667 --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-wecom/src/test/java/org/apache/eventmesh/connector/wecom/connector/MockRecordOffset.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.wecom.connector; + +import org.apache.eventmesh.common.remote.offset.RecordOffset; + +public class MockRecordOffset extends RecordOffset { + @Override + public Class getRecordOffsetClass() { + return MockRecordOffset.class; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-wecom/src/test/java/org/apache/eventmesh/connector/wecom/connector/MockRecordPartition.java b/eventmesh-connectors/eventmesh-connector-wecom/src/test/java/org/apache/eventmesh/connector/wecom/connector/MockRecordPartition.java new file mode 100644 index 0000000000..aae552891f --- /dev/null +++ b/eventmesh-connectors/eventmesh-connector-wecom/src/test/java/org/apache/eventmesh/connector/wecom/connector/MockRecordPartition.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.connector.wecom.connector; + +import org.apache.eventmesh.common.remote.offset.RecordPartition; + +public class MockRecordPartition extends RecordPartition { + @Override + public Class getRecordPartitionClass() { + return MockRecordPartition.class; + } +} diff --git a/eventmesh-connectors/eventmesh-connector-wecom/src/test/java/org/apache/eventmesh/connector/wecom/connector/WeComSinkConnectorTest.java b/eventmesh-connectors/eventmesh-connector-wecom/src/test/java/org/apache/eventmesh/connector/wecom/connector/WeComSinkConnectorTest.java index 4cf9f3523b..64b4e19aa3 100644 --- a/eventmesh-connectors/eventmesh-connector-wecom/src/test/java/org/apache/eventmesh/connector/wecom/connector/WeComSinkConnectorTest.java +++ b/eventmesh-connectors/eventmesh-connector-wecom/src/test/java/org/apache/eventmesh/connector/wecom/connector/WeComSinkConnectorTest.java @@ -21,15 +21,15 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import org.apache.eventmesh.common.config.connector.wecom.WeComSinkConfig; +import org.apache.eventmesh.common.remote.offset.RecordOffset; +import org.apache.eventmesh.common.remote.offset.RecordPartition; import org.apache.eventmesh.common.utils.JsonUtils; import org.apache.eventmesh.connector.wecom.config.WeComMessageTemplateType; import org.apache.eventmesh.connector.wecom.constants.ConnectRecordExtensionKeys; -import org.apache.eventmesh.connector.wecom.sink.config.WeComSinkConfig; import org.apache.eventmesh.connector.wecom.sink.connector.SendMessageResponse; import org.apache.eventmesh.connector.wecom.sink.connector.WeComSinkConnector; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; import org.apache.eventmesh.openconnect.util.ConfigUtil; import org.apache.http.HttpEntity; @@ -73,9 +73,8 @@ public void setUp() throws Exception { Mockito.doReturn(httpEntity).when(mockedResponse).getEntity(); WeComSinkConfig sinkConfig = (WeComSinkConfig) ConfigUtil.parse(connector.configClass()); connector.init(sinkConfig); - Field httpClientField = ReflectionSupport.findFields(connector.getClass(), - (f) -> f.getName().equals("httpClient"), - HierarchyTraversalMode.BOTTOM_UP).get(0); + Field httpClientField = + ReflectionSupport.findFields(connector.getClass(), (f) -> f.getName().equals("httpClient"), HierarchyTraversalMode.BOTTOM_UP).get(0); httpClientField.setAccessible(true); httpClientField.set(connector, httpClient); connector.start(); @@ -89,10 +88,10 @@ public void testSendMessageToWeCom() throws IOException { final int times = 3; List records = new ArrayList<>(); for (int i = 0; i < times; i++) { - RecordPartition partition = new RecordPartition(); - RecordOffset offset = new RecordOffset(); - ConnectRecord connectRecord = new ConnectRecord(partition, offset, - System.currentTimeMillis(), "Hello, EventMesh!".getBytes(StandardCharsets.UTF_8)); + RecordPartition partition = new MockRecordPartition(); + RecordOffset offset = new MockRecordOffset(); + ConnectRecord connectRecord = + new ConnectRecord(partition, offset, System.currentTimeMillis(), "Hello, EventMesh!".getBytes(StandardCharsets.UTF_8)); connectRecord.addExtension(ConnectRecordExtensionKeys.WECOM_MESSAGE_TEMPLATE_TYPE, WeComMessageTemplateType.PLAIN_TEXT.getTemplateType()); records.add(connectRecord); diff --git a/eventmesh-examples/build.gradle b/eventmesh-examples/build.gradle index f732a78439..bd90b83495 100644 --- a/eventmesh-examples/build.gradle +++ b/eventmesh-examples/build.gradle @@ -15,7 +15,7 @@ * limitations under the License. */ -def grpcVersion = '1.64.0' +def grpcVersion = '1.68.0' dependencies { implementation project(":eventmesh-sdks:eventmesh-sdk-java") diff --git a/eventmesh-examples/src/main/java/org/apache/eventmesh/spring/pub/SpringPubController.java b/eventmesh-examples/src/main/java/org/apache/eventmesh/spring/pub/SpringPubController.java index b7ea8890ee..a734bb6efa 100644 --- a/eventmesh-examples/src/main/java/org/apache/eventmesh/spring/pub/SpringPubController.java +++ b/eventmesh-examples/src/main/java/org/apache/eventmesh/spring/pub/SpringPubController.java @@ -19,9 +19,9 @@ import org.apache.eventmesh.common.utils.JsonUtils; import org.apache.eventmesh.connector.spring.source.connector.SpringSourceConnector; -import org.apache.eventmesh.openconnect.api.callback.SendExcepionContext; -import org.apache.eventmesh.openconnect.api.callback.SendMessageCallback; -import org.apache.eventmesh.openconnect.api.callback.SendResult; +import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendExceptionContext; +import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendMessageCallback; +import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendResult; import java.util.HashMap; import java.util.Map; @@ -53,8 +53,8 @@ public void onSuccess(SendResult sendResult) { } @Override - public void onException(SendExcepionContext sendExcepionContext) { - log.info("Spring source worker send message to EventMesh failed!", sendExcepionContext.getCause()); + public void onException(SendExceptionContext sendExceptionContext) { + log.info("Spring source worker send message to EventMesh failed!", sendExceptionContext.getCause()); } }); return "success!"; diff --git a/eventmesh-transformer/build.gradle b/eventmesh-function/build.gradle similarity index 92% rename from eventmesh-transformer/build.gradle rename to eventmesh-function/build.gradle index ba88591b41..2944f98194 100644 --- a/eventmesh-transformer/build.gradle +++ b/eventmesh-function/build.gradle @@ -14,8 +14,3 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - -dependencies { - implementation project(":eventmesh-common") -} diff --git a/eventmesh-filter/build.gradle b/eventmesh-function/eventmesh-function-api/build.gradle similarity index 92% rename from eventmesh-filter/build.gradle rename to eventmesh-function/eventmesh-function-api/build.gradle index ba88591b41..2944f98194 100644 --- a/eventmesh-filter/build.gradle +++ b/eventmesh-function/eventmesh-function-api/build.gradle @@ -14,8 +14,3 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - -dependencies { - implementation project(":eventmesh-common") -} diff --git a/eventmesh-function/eventmesh-function-api/src/main/java/org/apache/eventmesh/function/api/AbstractEventMeshFunctionChain.java b/eventmesh-function/eventmesh-function-api/src/main/java/org/apache/eventmesh/function/api/AbstractEventMeshFunctionChain.java new file mode 100644 index 0000000000..8cbb0f9381 --- /dev/null +++ b/eventmesh-function/eventmesh-function-api/src/main/java/org/apache/eventmesh/function/api/AbstractEventMeshFunctionChain.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.function.api; + +import java.util.ArrayList; +import java.util.List; + +/** + * AbstractEventMeshFunctionChain is an abstract class that implements the {@link EventMeshFunction} interface and provides a framework + * for chaining multiple {@link EventMeshFunction} instances that operate on inputs of type {@code T} and produce outputs of type + * {@code R}. This class can be extended to create specific function chains with customized behavior for different + * data types. + * + *

The primary purpose of this class is to allow the sequential execution of functions, where the output of one + * function is passed as the input to the next function in the chain. The chain can be dynamically modified by adding + * functions either at the beginning or the end of the chain.

+ * + * @param the type of the input to the function + * @param the type of the result of the function + */ +public abstract class AbstractEventMeshFunctionChain implements EventMeshFunction { + + protected final List> functions; + + /** + * Default constructor that initializes an empty function chain. + */ + public AbstractEventMeshFunctionChain() { + this.functions = new ArrayList<>(); + } + + /** + * Constructor that initializes the function chain with a given list of functions. The functions will be executed + * in the order they are provided when the {@link #apply(Object)} method is called. + * + * @param functions the initial list of functions to be added to the chain + */ + public AbstractEventMeshFunctionChain(List> functions) { + this.functions = functions; + } + + /** + * Adds a {@link EventMeshFunction} to the beginning of the chain. The function will be executed first when the + * {@link #apply(Object)} method is called. + * + * @param function the function to be added to the beginning of the chain + */ + public void addFirst(EventMeshFunction function) { + this.functions.add(0, function); + } + + /** + * Adds a {@link EventMeshFunction} to the end of the chain. The function will be executed in sequence after all previously + * added functions when the {@link #apply(Object)} method is called. + * + * @param function the function to be added to the end of the chain + */ + public void addLast(EventMeshFunction function) { + this.functions.add(function); + } +} \ No newline at end of file diff --git a/eventmesh-function/eventmesh-function-api/src/main/java/org/apache/eventmesh/function/api/EventMeshFunction.java b/eventmesh-function/eventmesh-function-api/src/main/java/org/apache/eventmesh/function/api/EventMeshFunction.java new file mode 100644 index 0000000000..973f097ae0 --- /dev/null +++ b/eventmesh-function/eventmesh-function-api/src/main/java/org/apache/eventmesh/function/api/EventMeshFunction.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.function.api; + +/** + * EventMesh Interface for a function that accepts one argument and produces a result. This is a functional interface whose functional method is + * {@link #apply(Object)}. + * + *

This interface is similar to {@link java.util.function.Function}, + * but it is specifically designed for use within the EventMesh. It allows defining custom functions to process data or events in the EventMesh. The + * main use case is to encapsulate operations that can be passed around and applied to data or event messages in the EventMesh processing + * pipeline.

+ * + * @param the type of the input to the function + * @param the type of the result of the function + */ +public interface EventMeshFunction { + + /** + * Applies this function to the given argument within the context of the EventMesh module. This method encapsulates the logic for processing the + * input data and producing a result, which can be used in the EventMesh event processing pipeline. + * + * @param t the function argument, representing the input data or event to be processed + * @return the function result, representing the processed output + */ + R apply(T t); + +} \ No newline at end of file diff --git a/eventmesh-function/eventmesh-function-filter/build.gradle b/eventmesh-function/eventmesh-function-filter/build.gradle new file mode 100644 index 0000000000..21e28d7baf --- /dev/null +++ b/eventmesh-function/eventmesh-function-filter/build.gradle @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dependencies { + implementation project(":eventmesh-common") + implementation project(":eventmesh-function:eventmesh-function-api") +} \ No newline at end of file diff --git a/eventmesh-filter/src/main/java/org/apache/eventmesh/filter/PatternEntry.java b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/PatternEntry.java similarity index 94% rename from eventmesh-filter/src/main/java/org/apache/eventmesh/filter/PatternEntry.java rename to eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/PatternEntry.java index 5a2493a371..acc2d5f073 100644 --- a/eventmesh-filter/src/main/java/org/apache/eventmesh/filter/PatternEntry.java +++ b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/PatternEntry.java @@ -15,9 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.filter; +package org.apache.eventmesh.function.filter; -import org.apache.eventmesh.filter.condition.Condition; +import org.apache.eventmesh.function.filter.condition.Condition; import java.util.ArrayList; import java.util.List; diff --git a/eventmesh-filter/src/main/java/org/apache/eventmesh/filter/condition/AnythingButCondition.java b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/AnythingButCondition.java similarity index 97% rename from eventmesh-filter/src/main/java/org/apache/eventmesh/filter/condition/AnythingButCondition.java rename to eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/AnythingButCondition.java index 2d58136a70..d4f209225e 100644 --- a/eventmesh-filter/src/main/java/org/apache/eventmesh/filter/condition/AnythingButCondition.java +++ b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/AnythingButCondition.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.filter.condition; +package org.apache.eventmesh.function.filter.condition; import java.util.ArrayList; import java.util.Iterator; diff --git a/eventmesh-filter/src/main/java/org/apache/eventmesh/filter/condition/Condition.java b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/Condition.java similarity index 94% rename from eventmesh-filter/src/main/java/org/apache/eventmesh/filter/condition/Condition.java rename to eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/Condition.java index fbb4276c7b..9890d5e0d3 100644 --- a/eventmesh-filter/src/main/java/org/apache/eventmesh/filter/condition/Condition.java +++ b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/Condition.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.filter.condition; +package org.apache.eventmesh.function.filter.condition; import com.fasterxml.jackson.databind.JsonNode; diff --git a/eventmesh-filter/src/main/java/org/apache/eventmesh/filter/condition/ConditionsBuilder.java b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/ConditionsBuilder.java similarity index 97% rename from eventmesh-filter/src/main/java/org/apache/eventmesh/filter/condition/ConditionsBuilder.java rename to eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/ConditionsBuilder.java index 4e207663aa..961be85e5b 100644 --- a/eventmesh-filter/src/main/java/org/apache/eventmesh/filter/condition/ConditionsBuilder.java +++ b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/ConditionsBuilder.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.filter.condition; +package org.apache.eventmesh.function.filter.condition; import com.fasterxml.jackson.databind.JsonNode; diff --git a/eventmesh-filter/src/main/java/org/apache/eventmesh/filter/condition/ExistsCondition.java b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/ExistsCondition.java similarity index 95% rename from eventmesh-filter/src/main/java/org/apache/eventmesh/filter/condition/ExistsCondition.java rename to eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/ExistsCondition.java index 53c15bb297..c085ba6585 100644 --- a/eventmesh-filter/src/main/java/org/apache/eventmesh/filter/condition/ExistsCondition.java +++ b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/ExistsCondition.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.filter.condition; +package org.apache.eventmesh.function.filter.condition; import com.fasterxml.jackson.databind.JsonNode; diff --git a/eventmesh-filter/src/main/java/org/apache/eventmesh/filter/condition/NumericCondition.java b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/NumericCondition.java similarity index 97% rename from eventmesh-filter/src/main/java/org/apache/eventmesh/filter/condition/NumericCondition.java rename to eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/NumericCondition.java index 5eb5374c7c..40eb16a75e 100644 --- a/eventmesh-filter/src/main/java/org/apache/eventmesh/filter/condition/NumericCondition.java +++ b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/NumericCondition.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.filter.condition; +package org.apache.eventmesh.function.filter.condition; import java.util.ArrayList; import java.util.List; diff --git a/eventmesh-filter/src/main/java/org/apache/eventmesh/filter/condition/PrefixCondition.java b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/PrefixCondition.java similarity index 95% rename from eventmesh-filter/src/main/java/org/apache/eventmesh/filter/condition/PrefixCondition.java rename to eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/PrefixCondition.java index 633ed1fb02..ff5d0313ce 100644 --- a/eventmesh-filter/src/main/java/org/apache/eventmesh/filter/condition/PrefixCondition.java +++ b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/PrefixCondition.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.filter.condition; +package org.apache.eventmesh.function.filter.condition; import com.fasterxml.jackson.databind.JsonNode; diff --git a/eventmesh-filter/src/main/java/org/apache/eventmesh/filter/condition/SpecifiedCondition.java b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/SpecifiedCondition.java similarity index 95% rename from eventmesh-filter/src/main/java/org/apache/eventmesh/filter/condition/SpecifiedCondition.java rename to eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/SpecifiedCondition.java index f9cc3fb5db..9eefb6b641 100644 --- a/eventmesh-filter/src/main/java/org/apache/eventmesh/filter/condition/SpecifiedCondition.java +++ b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/SpecifiedCondition.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.filter.condition; +package org.apache.eventmesh.function.filter.condition; import com.fasterxml.jackson.databind.JsonNode; diff --git a/eventmesh-filter/src/main/java/org/apache/eventmesh/filter/condition/SuffixCondition.java b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/SuffixCondition.java similarity index 95% rename from eventmesh-filter/src/main/java/org/apache/eventmesh/filter/condition/SuffixCondition.java rename to eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/SuffixCondition.java index 805df0ee17..090df24834 100644 --- a/eventmesh-filter/src/main/java/org/apache/eventmesh/filter/condition/SuffixCondition.java +++ b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/condition/SuffixCondition.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.filter.condition; +package org.apache.eventmesh.function.filter.condition; import com.fasterxml.jackson.databind.JsonNode; diff --git a/eventmesh-filter/src/main/java/org/apache/eventmesh/filter/pattern/Pattern.java b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/pattern/Pattern.java similarity index 75% rename from eventmesh-filter/src/main/java/org/apache/eventmesh/filter/pattern/Pattern.java rename to eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/pattern/Pattern.java index 8abb306b84..955d9f59ef 100644 --- a/eventmesh-filter/src/main/java/org/apache/eventmesh/filter/pattern/Pattern.java +++ b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/pattern/Pattern.java @@ -15,10 +15,11 @@ * limitations under the License. */ -package org.apache.eventmesh.filter.pattern; +package org.apache.eventmesh.function.filter.pattern; import org.apache.eventmesh.common.utils.JsonPathUtils; -import org.apache.eventmesh.filter.PatternEntry; +import org.apache.eventmesh.function.api.EventMeshFunction; +import org.apache.eventmesh.function.filter.PatternEntry; import org.apache.commons.lang3.StringUtils; @@ -29,12 +30,11 @@ import com.fasterxml.jackson.databind.JsonNode; import com.jayway.jsonpath.PathNotFoundException; -public class Pattern { - private List requiredFieldList = new ArrayList<>(); - private List dataList = new ArrayList<>(); +public class Pattern implements EventMeshFunction { - private String content; + private final List requiredFieldList = new ArrayList<>(); + private final List dataList = new ArrayList<>(); public void addRequiredFieldList(PatternEntry patternEntry) { this.requiredFieldList.add(patternEntry); @@ -45,19 +45,22 @@ public void addDataList(PatternEntry patternEntry) { } public boolean filter(String content) { - this.content = content; - // this.jsonNode = JacksonUtils.STRING_TO_JSONNODE(content); + return matchRequiredFieldList(content, requiredFieldList) && matchRequiredFieldList(content, dataList); + } - return matchRequiredFieldList(requiredFieldList) && matchRequiredFieldList(dataList); + @Override + public String apply(String content) { + // filter content + return filter(content) ? content : null; } - private boolean matchRequiredFieldList(List dataList) { + private boolean matchRequiredFieldList(String content, List dataList) { for (final PatternEntry patternEntry : dataList) { JsonNode jsonElement = null; try { // content:filter - String matchRes = JsonPathUtils.matchJsonPathValue(this.content, patternEntry.getPatternPath()); + String matchRes = JsonPathUtils.matchJsonPathValue(content, patternEntry.getPatternPath()); if (StringUtils.isNoneBlank(matchRes)) { jsonElement = JsonPathUtils.parseStrict(matchRes); diff --git a/eventmesh-filter/src/main/java/org/apache/eventmesh/filter/patternbuild/PatternBuilder.java b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/patternbuild/PatternBuilder.java similarity index 85% rename from eventmesh-filter/src/main/java/org/apache/eventmesh/filter/patternbuild/PatternBuilder.java rename to eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/patternbuild/PatternBuilder.java index 5f9a71d262..60193a4efa 100644 --- a/eventmesh-filter/src/main/java/org/apache/eventmesh/filter/patternbuild/PatternBuilder.java +++ b/eventmesh-function/eventmesh-function-filter/src/main/java/org/apache/eventmesh/function/filter/patternbuild/PatternBuilder.java @@ -15,13 +15,13 @@ * limitations under the License. */ -package org.apache.eventmesh.filter.patternbuild; +package org.apache.eventmesh.function.filter.patternbuild; import org.apache.eventmesh.common.exception.JsonException; -import org.apache.eventmesh.filter.PatternEntry; -import org.apache.eventmesh.filter.condition.Condition; -import org.apache.eventmesh.filter.condition.ConditionsBuilder; -import org.apache.eventmesh.filter.pattern.Pattern; +import org.apache.eventmesh.function.filter.PatternEntry; +import org.apache.eventmesh.function.filter.condition.Condition; +import org.apache.eventmesh.function.filter.condition.ConditionsBuilder; +import org.apache.eventmesh.function.filter.pattern.Pattern; import java.util.ArrayDeque; import java.util.Iterator; @@ -38,19 +38,33 @@ public class PatternBuilder { private static final ObjectMapper mapper = new ObjectMapper(); - public static Pattern build(String jsonStr) { - Pattern pattern = new Pattern(); - JsonNode jsonNode = null; + public static Pattern build(String jsonStr) { try { - jsonNode = mapper.readTree(jsonStr); + JsonNode jsonNode = mapper.readTree(jsonStr); + if (jsonNode.isEmpty() || !jsonNode.isObject()) { + return null; + } + return build(jsonNode); } catch (Exception e) { throw new JsonException("INVALID_JSON_STRING", e); } + } - if (jsonNode.isEmpty() || !jsonNode.isObject()) { - return null; + public static Pattern build(Map conditionMap) { + try { + JsonNode jsonNode = mapper.valueToTree(conditionMap); + if (jsonNode.isEmpty() || !jsonNode.isObject()) { + return null; + } + return build(jsonNode); + } catch (Exception e) { + throw new JsonException("INVALID_MAP", e); } + } + + public static Pattern build(JsonNode jsonNode) { + Pattern pattern = new Pattern(); // iter all json data Iterator> iterator = jsonNode.fields(); diff --git a/eventmesh-filter/src/test/java/org/apache/eventmesh/filter/PatternTest.java b/eventmesh-function/eventmesh-function-filter/src/test/java/org/apache/eventmesh/function/filter/PatternTest.java similarity index 82% rename from eventmesh-filter/src/test/java/org/apache/eventmesh/filter/PatternTest.java rename to eventmesh-function/eventmesh-function-filter/src/test/java/org/apache/eventmesh/function/filter/PatternTest.java index 207992b0c1..bc0aeff4ea 100644 --- a/eventmesh-filter/src/test/java/org/apache/eventmesh/filter/PatternTest.java +++ b/eventmesh-function/eventmesh-function-filter/src/test/java/org/apache/eventmesh/function/filter/PatternTest.java @@ -15,10 +15,15 @@ * limitations under the License. */ -package org.apache.eventmesh.filter; +package org.apache.eventmesh.function.filter; -import org.apache.eventmesh.filter.pattern.Pattern; -import org.apache.eventmesh.filter.patternbuild.PatternBuilder; +import org.apache.eventmesh.function.filter.pattern.Pattern; +import org.apache.eventmesh.function.filter.patternbuild.PatternBuilder; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -144,4 +149,20 @@ public void testAnythingButFilter() { Assertions.assertEquals(false, res); } + @Test + public void testPrefixFilterMap() { + // Create the inner Map representing {prefix=eventmesh.} + Map innerMap = new HashMap<>(); + innerMap.put("prefix", "eventmesh."); + // Create a List representing [{prefix=eventmesh.}] + List> sourceList = Collections.singletonList(innerMap); + // Create the condition representing {source=[{prefix=eventmesh.}]} + Map condition = new HashMap<>(); + condition.put("source", sourceList); + + Pattern pattern = PatternBuilder.build(condition); + Boolean res = pattern.filter(event); + Assertions.assertEquals(true, res); + } + } diff --git a/eventmesh-function/eventmesh-function-transformer/build.gradle b/eventmesh-function/eventmesh-function-transformer/build.gradle new file mode 100644 index 0000000000..6939bbd483 --- /dev/null +++ b/eventmesh-function/eventmesh-function-transformer/build.gradle @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +dependencies { + implementation project(":eventmesh-common") + implementation project(":eventmesh-function:eventmesh-function-api") +} diff --git a/eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/ConstantTransformer.java b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/ConstantTransformer.java similarity index 95% rename from eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/ConstantTransformer.java rename to eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/ConstantTransformer.java index dd7c20aace..ae77f149f7 100644 --- a/eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/ConstantTransformer.java +++ b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/ConstantTransformer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.transformer; +package org.apache.eventmesh.function.transformer; public class ConstantTransformer implements Transformer { diff --git a/eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/JsonPathParser.java b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/JsonPathParser.java similarity index 85% rename from eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/JsonPathParser.java rename to eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/JsonPathParser.java index a0ebde12d2..c578310dc4 100644 --- a/eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/JsonPathParser.java +++ b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/JsonPathParser.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.transformer; +package org.apache.eventmesh.function.transformer; import org.apache.eventmesh.common.utils.JsonPathUtils; @@ -35,6 +35,19 @@ public List getVariablesList() { return variablesList; } + /** + * parser input jsonpath map into variable list + * + * @param jsonPathMap jsonpath map + */ + public JsonPathParser(Map jsonPathMap) { + for (Map.Entry entry : jsonPathMap.entrySet()) { + String name = entry.getKey(); + String value = entry.getValue(); + variablesList.add(new Variable(name, value)); + } + } + /** * parser input jsonpath string into variable list * diff --git a/eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/OriginalTransformer.java b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/OriginalTransformer.java similarity index 94% rename from eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/OriginalTransformer.java rename to eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/OriginalTransformer.java index 61aa059d59..59ce0350eb 100644 --- a/eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/OriginalTransformer.java +++ b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/OriginalTransformer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.transformer; +package org.apache.eventmesh.function.transformer; class OriginalTransformer implements Transformer { @@ -23,4 +23,5 @@ class OriginalTransformer implements Transformer { public String transform(String json) { return json; } + } diff --git a/eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/Template.java b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/Template.java similarity index 96% rename from eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/Template.java rename to eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/Template.java index 19c3b5cec3..29d975c371 100644 --- a/eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/Template.java +++ b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/Template.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.transformer; +package org.apache.eventmesh.function.transformer; import org.apache.commons.text.StringSubstitutor; diff --git a/eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/TemplateTransformer.java b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TemplateTransformer.java similarity index 96% rename from eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/TemplateTransformer.java rename to eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TemplateTransformer.java index bc9907ff48..69cee68269 100644 --- a/eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/TemplateTransformer.java +++ b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TemplateTransformer.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.transformer; +package org.apache.eventmesh.function.transformer; import java.util.List; diff --git a/eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/TransformException.java b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TransformException.java similarity index 95% rename from eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/TransformException.java rename to eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TransformException.java index 1b11a29d80..aeb827fc88 100644 --- a/eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/TransformException.java +++ b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TransformException.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.transformer; +package org.apache.eventmesh.function.transformer; /** * Transform exception diff --git a/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/Transformer.java b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/Transformer.java new file mode 100644 index 0000000000..be0e815808 --- /dev/null +++ b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/Transformer.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.function.transformer; + +import org.apache.eventmesh.common.exception.EventMeshException; +import org.apache.eventmesh.function.api.EventMeshFunction; + +import com.fasterxml.jackson.core.JsonProcessingException; + +/** + * EventMesh transformer interface, specified transformer implementation includes: + * 1. Constant + * 2. Original + * 3. Template + */ +public interface Transformer extends EventMeshFunction { + + String transform(String json) throws JsonProcessingException; + + @Override + default String apply(String content) { + try { + return transform(content); + } catch (JsonProcessingException e) { + throw new EventMeshException("Failed to transform content", e); + } + } + +} diff --git a/eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/TransformerBuilder.java b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TransformerBuilder.java similarity index 69% rename from eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/TransformerBuilder.java rename to eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TransformerBuilder.java index e7277af73c..916f1ef7bc 100644 --- a/eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/TransformerBuilder.java +++ b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TransformerBuilder.java @@ -15,7 +15,9 @@ * limitations under the License. */ -package org.apache.eventmesh.transformer; +package org.apache.eventmesh.function.transformer; + +import java.util.Map; public class TransformerBuilder { @@ -32,9 +34,23 @@ public static Transformer buildTransformer(TransformerParam transformerParam) { } } - public static Transformer buildTemplateTransFormer(String jsonContent, String template) { - JsonPathParser jsonPathParser = new JsonPathParser(jsonContent); + /** + * build template transformer + * @param jsonContent json content, support string and map, other type will throw IllegalArgumentException + * @param template template string + * @return transformer + */ + @SuppressWarnings("unchecked") + public static Transformer buildTemplateTransFormer(Object jsonContent, String template) { Template templateEntry = new Template(template); + JsonPathParser jsonPathParser; + if (jsonContent instanceof String) { + jsonPathParser = new JsonPathParser((String) jsonContent); + } else if (jsonContent instanceof Map) { + jsonPathParser = new JsonPathParser((Map) jsonContent); + } else { + throw new TransformException("invalid json content"); + } return new TemplateTransformer(jsonPathParser, templateEntry); } diff --git a/eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/TransformerParam.java b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TransformerParam.java similarity index 97% rename from eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/TransformerParam.java rename to eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TransformerParam.java index d747d7be4c..915111e01d 100644 --- a/eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/TransformerParam.java +++ b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TransformerParam.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.transformer; +package org.apache.eventmesh.function.transformer; public class TransformerParam { diff --git a/eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/TransformerType.java b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TransformerType.java similarity index 97% rename from eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/TransformerType.java rename to eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TransformerType.java index 2dc7809478..969c49ce80 100644 --- a/eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/TransformerType.java +++ b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/TransformerType.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.transformer; +package org.apache.eventmesh.function.transformer; import java.util.Objects; diff --git a/eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/Variable.java b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/Variable.java similarity index 96% rename from eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/Variable.java rename to eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/Variable.java index c9259d335c..aee80e1454 100644 --- a/eventmesh-transformer/src/main/java/org/apache/eventmesh/transformer/Variable.java +++ b/eventmesh-function/eventmesh-function-transformer/src/main/java/org/apache/eventmesh/function/transformer/Variable.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.transformer; +package org.apache.eventmesh.function.transformer; public class Variable { diff --git a/eventmesh-transformer/src/test/java/org/apache/eventmesh/transformer/TransformTest.java b/eventmesh-function/eventmesh-function-transformer/src/test/java/org/apache/eventmesh/function/transformer/TransformTest.java similarity index 88% rename from eventmesh-transformer/src/test/java/org/apache/eventmesh/transformer/TransformTest.java rename to eventmesh-function/eventmesh-function-transformer/src/test/java/org/apache/eventmesh/function/transformer/TransformTest.java index a55cde0baf..f9a444e8f9 100644 --- a/eventmesh-transformer/src/test/java/org/apache/eventmesh/transformer/TransformTest.java +++ b/eventmesh-function/eventmesh-function-transformer/src/test/java/org/apache/eventmesh/function/transformer/TransformTest.java @@ -15,7 +15,10 @@ * limitations under the License. */ -package org.apache.eventmesh.transformer; +package org.apache.eventmesh.function.transformer; + +import java.util.Collections; +import java.util.Map; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -139,4 +142,19 @@ public void testTemplateTransFormerWithConstant() throws JsonProcessingException output); } + @Test + public void testTemplateTransFormerWithStringValueMap() throws JsonProcessingException { + Map content = Collections.singletonMap("data-name", "$.data.name"); + + String template = "Transformers test:data name is ${data-name}"; + Transformer transform = TransformerBuilder.buildTemplateTransFormer(content, template); + String output = transform.transform(EVENT); + Assertions.assertEquals("Transformers test:data name is test-transformer", output); + + Transformer transformer1 = TransformerBuilder.buildTemplateTransFormer(content, template); + String output1 = transformer1.transform(EVENT); + Assertions.assertEquals("Transformers test:data name is test-transformer", output1); + + } + } diff --git a/eventmesh-meta/eventmesh-meta-etcd/src/test/java/org/apache/eventmesh/registry/etcd/service/EtcdCustomServiceTest.java b/eventmesh-meta/eventmesh-meta-etcd/src/test/java/org/apache/eventmesh/registry/etcd/service/EtcdCustomServiceTest.java new file mode 100644 index 0000000000..a20564ae01 --- /dev/null +++ b/eventmesh-meta/eventmesh-meta-etcd/src/test/java/org/apache/eventmesh/registry/etcd/service/EtcdCustomServiceTest.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.registry.etcd.service; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +import org.apache.eventmesh.api.meta.bo.EventMeshAppSubTopicInfo; +import org.apache.eventmesh.api.meta.bo.EventMeshServicePubTopicInfo; +import org.apache.eventmesh.common.utils.JsonUtils; +import org.apache.eventmesh.meta.etcd.service.EtcdCustomService; + +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.mockito.junit.jupiter.MockitoExtension; + +import io.etcd.jetcd.ByteSequence; +import io.etcd.jetcd.Client; +import io.etcd.jetcd.KV; +import io.etcd.jetcd.KeyValue; +import io.etcd.jetcd.kv.GetResponse; +import io.etcd.jetcd.options.GetOption; + +@ExtendWith(MockitoExtension.class) +public class EtcdCustomServiceTest { + + @Mock + private Client etcdClient; + + @Mock + private KV kvClient; + + @Mock + private KeyValue keyValue; + + @Mock + private GetResponse getResponse; + + @Mock + private CompletableFuture futureResponse; + + @InjectMocks + private EtcdCustomService etcdCustomService; + + @BeforeEach + void setUp() { + MockitoAnnotations.openMocks(this); + when(etcdClient.getKVClient()).thenReturn(kvClient); + } + + @Test + public void testFindEventMeshServicePubTopicInfos() throws Exception { + + EventMeshServicePubTopicInfo mockInfo = new EventMeshServicePubTopicInfo(); + mockInfo.setService("testService"); + mockInfo.setTopics(Collections.unmodifiableSet(new HashSet<>(Arrays.asList("topic1", "topic2")))); + + String mockValue = JsonUtils.toJSONString(mockInfo); + ByteSequence mockByteSequence = ByteSequence.from(mockValue, StandardCharsets.UTF_8); + + when(keyValue.getValue()).thenReturn(mockByteSequence); + when(getResponse.getKvs()).thenReturn(Arrays.asList(keyValue)); + when(futureResponse.get()).thenReturn(getResponse); + when(kvClient.get(any(ByteSequence.class), any(GetOption.class))).thenReturn(futureResponse); + + List result = etcdCustomService.findEventMeshServicePubTopicInfos(); + assertNotNull(result); + assertEquals(1, result.size()); + EventMeshServicePubTopicInfo resultInfo = result.get(0); + assertEquals("testService", resultInfo.getService()); + assertEquals(new HashSet<>(Arrays.asList("topic1", "topic2")), resultInfo.getTopics()); + } + + + @Test + public void testFindEventMeshAppSubTopicInfoByGroup() throws Exception { + + String group = "testGroup"; + EventMeshAppSubTopicInfo mockInfo = new EventMeshAppSubTopicInfo(); + + String mockValue = JsonUtils.toJSONString(mockInfo); + ByteSequence mockByteSequence = ByteSequence.from(mockValue, StandardCharsets.UTF_8); + + when(keyValue.getValue()).thenReturn(mockByteSequence); + when(kvClient.get(any(ByteSequence.class), any(GetOption.class))).thenReturn(futureResponse); + when(futureResponse.get()).thenReturn(getResponse); + when(getResponse.getKvs()).thenReturn(Collections.singletonList(keyValue)); + + EventMeshAppSubTopicInfo result = etcdCustomService.findEventMeshAppSubTopicInfoByGroup(group); + + assertNotNull(result); + } + +} diff --git a/eventmesh-meta/eventmesh-meta-raft/build.gradle b/eventmesh-meta/eventmesh-meta-raft/build.gradle index 386b5c6e72..6abc73dd96 100644 --- a/eventmesh-meta/eventmesh-meta-raft/build.gradle +++ b/eventmesh-meta/eventmesh-meta-raft/build.gradle @@ -16,13 +16,15 @@ */ plugins { - id 'com.google.protobuf' version '0.8.17' + id 'com.google.protobuf' version '0.9.4' } -def grpcVersion = '1.50.2' // CURRENT_GRPC_VERSION -def protobufVersion = '3.21.5' +def grpcVersion = '1.68.0' +def protobufVersion = '3.25.4' def protocVersion = protobufVersion +def jraftVersion = '1.3.14' + dependencies { implementation ("io.grpc:grpc-protobuf:${grpcVersion}") { exclude group: "com.google.protobuf", module: "protobuf-java" @@ -36,10 +38,10 @@ dependencies { implementation project(":eventmesh-meta:eventmesh-meta-api") implementation project(":eventmesh-common") - implementation group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.0.1' - implementation "com.alipay.sofa:jraft-core:1.3.14" - implementation "com.alipay.sofa:rpc-grpc-impl:1.3.14" - testImplementation 'org.junit.jupiter:junit-jupiter:5.6.0' + implementation "com.alipay.sofa:jraft-core:${jraftVersion}" + implementation "com.alipay.sofa:rpc-grpc-impl:${jraftVersion}" + implementation group: 'com.caucho', name: 'hessian', version: '4.0.63' + testImplementation 'org.junit.jupiter:junit-jupiter' } protobuf { diff --git a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/JraftMetaServiceImpl.java b/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/JraftMetaServiceImpl.java index 1af6d5c963..1f655eb93e 100644 --- a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/JraftMetaServiceImpl.java +++ b/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/JraftMetaServiceImpl.java @@ -18,13 +18,13 @@ package org.apache.eventmesh.meta.raft; import org.apache.eventmesh.meta.raft.rpc.RequestResponse; +import org.apache.eventmesh.meta.raft.serialize.EventMeshHessianSerializer; import org.apache.commons.lang.StringUtils; import java.nio.ByteBuffer; import com.alipay.remoting.exception.CodecException; -import com.alipay.remoting.serialization.SerializerManager; import com.alipay.sofa.jraft.Status; import com.alipay.sofa.jraft.entity.Task; import com.alipay.sofa.jraft.error.RaftError; @@ -51,7 +51,7 @@ public void applyOperation(EventOperation opreation, EventClosure closure) { try { closure.setEventOperation(opreation); final Task task = new Task(); - task.setData(ByteBuffer.wrap(SerializerManager.getSerializer(SerializerManager.Hessian2).serialize(opreation))); + task.setData(ByteBuffer.wrap(EventMeshHessianSerializer.getInstance().serialize(opreation))); task.setDone(closure); this.server.getNode().apply(task); } catch (CodecException e) { diff --git a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/MetaStateMachine.java b/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/MetaStateMachine.java index a0607f5ab4..0d4690fb1c 100644 --- a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/MetaStateMachine.java +++ b/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/MetaStateMachine.java @@ -21,6 +21,7 @@ import static org.apache.eventmesh.meta.raft.EventOperation.GET; import static org.apache.eventmesh.meta.raft.EventOperation.PUT; +import org.apache.eventmesh.meta.raft.serialize.EventMeshHessianSerializer; import org.apache.eventmesh.meta.raft.snapshot.MetaSnapshotFile; import org.apache.commons.lang.StringUtils; @@ -37,7 +38,6 @@ import java.util.concurrent.atomic.AtomicLong; import com.alipay.remoting.exception.CodecException; -import com.alipay.remoting.serialization.SerializerManager; import com.alipay.sofa.jraft.Closure; import com.alipay.sofa.jraft.Iterator; import com.alipay.sofa.jraft.Status; @@ -121,7 +121,7 @@ public void onApply(Iterator iter) { // Have to parse FetchAddRequest from this user log. final ByteBuffer data = iter.getData(); try { - eventOperation = SerializerManager.getSerializer(SerializerManager.Hessian2) + eventOperation = EventMeshHessianSerializer.getInstance() .deserialize(data.array(), EventOperation.class.getName()); } catch (final CodecException e) { e.printStackTrace(System.err); diff --git a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/serialize/EventMeshHessianSerializer.java b/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/serialize/EventMeshHessianSerializer.java new file mode 100644 index 0000000000..646d69c0d3 --- /dev/null +++ b/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/serialize/EventMeshHessianSerializer.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.meta.raft.serialize; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import com.alipay.remoting.exception.CodecException; +import com.alipay.remoting.serialization.HessianSerializer; +import com.caucho.hessian.io.Hessian2Input; +import com.caucho.hessian.io.Hessian2Output; +import com.caucho.hessian.io.SerializerFactory; + +public class EventMeshHessianSerializer extends HessianSerializer { + + private SerializerFactory customizeSerializerFactory = new EventMeshSerializerFactory(); + + private static EventMeshHessianSerializer instance; + + private EventMeshHessianSerializer() { + } + + public static HessianSerializer getInstance() { + if (instance == null) { + synchronized (EventMeshHessianSerializer.class) { + if (instance == null) { + instance = new EventMeshHessianSerializer(); + } + } + } + return instance; + } + + @Override + public byte[] serialize(Object obj) throws CodecException { + ByteArrayOutputStream byteArray = new ByteArrayOutputStream(); + Hessian2Output output = new Hessian2Output(byteArray); + output.setSerializerFactory(customizeSerializerFactory); + try { + output.writeObject(obj); + output.close(); + } catch (IOException e) { + throw new CodecException("IOException occurred when Hessian serializer encode!", e); + } + + return byteArray.toByteArray(); + } + + @Override + public T deserialize(byte[] data, String classOfT) throws CodecException { + Hessian2Input input = new Hessian2Input(new ByteArrayInputStream(data)); + input.setSerializerFactory(customizeSerializerFactory); + Object resultObject; + try { + resultObject = input.readObject(); + input.close(); + } catch (IOException e) { + throw new CodecException("IOException occurred when Hessian serializer decode!", e); + } + return (T) resultObject; + } +} diff --git a/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/serialize/EventMeshSerializerFactory.java b/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/serialize/EventMeshSerializerFactory.java new file mode 100644 index 0000000000..d16796219d --- /dev/null +++ b/eventmesh-meta/eventmesh-meta-raft/src/main/java/org/apache/eventmesh/meta/raft/serialize/EventMeshSerializerFactory.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.meta.raft.serialize; + +import java.text.SimpleDateFormat; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.WeakHashMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.ConcurrentSkipListMap; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import com.caucho.hessian.io.SerializerFactory; + +public class EventMeshSerializerFactory extends SerializerFactory { + EventMeshSerializerFactory() { + super(); + super.getClassFactory().setWhitelist(true); + allowBasicType(); + allowCollections(); + allowConcurrent(); + allowTime(); + super.getClassFactory().allow("org.apache.eventmesh.*"); + } + + private void allowBasicType() { + super.getClassFactory().allow(boolean.class.getCanonicalName()); + super.getClassFactory().allow(byte.class.getCanonicalName()); + super.getClassFactory().allow(char.class.getCanonicalName()); + super.getClassFactory().allow(double.class.getCanonicalName()); + super.getClassFactory().allow(float.class.getCanonicalName()); + super.getClassFactory().allow(int.class.getCanonicalName()); + super.getClassFactory().allow(long.class.getCanonicalName()); + super.getClassFactory().allow(short.class.getCanonicalName()); + super.getClassFactory().allow(Boolean.class.getCanonicalName()); + super.getClassFactory().allow(Byte.class.getCanonicalName()); + super.getClassFactory().allow(Character.class.getCanonicalName()); + super.getClassFactory().allow(Double.class.getCanonicalName()); + super.getClassFactory().allow(Float.class.getCanonicalName()); + super.getClassFactory().allow(Integer.class.getCanonicalName()); + super.getClassFactory().allow(Long.class.getCanonicalName()); + super.getClassFactory().allow(Short.class.getCanonicalName()); + + super.getClassFactory().allow(Number.class.getCanonicalName()); + super.getClassFactory().allow(Class.class.getCanonicalName()); + super.getClassFactory().allow(String.class.getCanonicalName()); + } + + private void allowCollections() { + super.getClassFactory().allow(List.class.getCanonicalName()); + super.getClassFactory().allow(ArrayList.class.getCanonicalName()); + super.getClassFactory().allow(LinkedList.class.getCanonicalName()); + + super.getClassFactory().allow(Set.class.getCanonicalName()); + super.getClassFactory().allow(HashSet.class.getCanonicalName()); + super.getClassFactory().allow(LinkedHashSet.class.getCanonicalName()); + super.getClassFactory().allow(TreeSet.class.getCanonicalName()); + + super.getClassFactory().allow(Map.class.getCanonicalName()); + super.getClassFactory().allow(HashMap.class.getCanonicalName()); + super.getClassFactory().allow(LinkedHashMap.class.getCanonicalName()); + super.getClassFactory().allow(TreeMap.class.getCanonicalName()); + super.getClassFactory().allow(WeakHashMap.class.getCanonicalName()); + + super.getClassFactory().allow("java.util.Arrays$ArrayList"); + super.getClassFactory().allow("java.util.Collections$EmptyList"); + super.getClassFactory().allow("java.util.Collections$EmptyMap"); + super.getClassFactory().allow("java.util.Collections$SingletonSet"); + super.getClassFactory().allow("java.util.Collections$SingletonList"); + super.getClassFactory().allow("java.util.Collections$UnmodifiableCollection"); + super.getClassFactory().allow("java.util.Collections$UnmodifiableList"); + super.getClassFactory().allow("java.util.Collections$UnmodifiableMap"); + super.getClassFactory().allow("java.util.Collections$UnmodifiableNavigableMap"); + super.getClassFactory().allow("java.util.Collections$UnmodifiableNavigableSet"); + super.getClassFactory().allow("java.util.Collections$UnmodifiableRandomAccessList"); + super.getClassFactory().allow("java.util.Collections$UnmodifiableSet"); + super.getClassFactory().allow("java.util.Collections$UnmodifiableSortedMap"); + super.getClassFactory().allow("java.util.Collections$UnmodifiableSortedSet"); + } + + private void allowConcurrent() { + super.getClassFactory().allow(AtomicBoolean.class.getCanonicalName()); + super.getClassFactory().allow(AtomicInteger.class.getCanonicalName()); + super.getClassFactory().allow(AtomicLong.class.getCanonicalName()); + super.getClassFactory().allow(AtomicReference.class.getCanonicalName()); + + super.getClassFactory().allow(ConcurrentMap.class.getCanonicalName()); + super.getClassFactory().allow(ConcurrentHashMap.class.getCanonicalName()); + super.getClassFactory().allow(ConcurrentSkipListMap.class.getCanonicalName()); + super.getClassFactory().allow(CopyOnWriteArrayList.class.getCanonicalName()); + } + + private void allowTime() { + super.getClassFactory().allow(SimpleDateFormat.class.getCanonicalName()); + super.getClassFactory().allow(DateTimeFormatter.class.getCanonicalName()); + super.getClassFactory().allow(Instant.class.getCanonicalName()); + super.getClassFactory().allow(LocalDate.class.getCanonicalName()); + super.getClassFactory().allow(LocalDateTime.class.getCanonicalName()); + super.getClassFactory().allow(LocalTime.class.getCanonicalName()); + super.getClassFactory().allow(TimeUnit.class.getCanonicalName()); + super.getClassFactory().allow(Date.class.getCanonicalName()); + super.getClassFactory().allow(Calendar.class.getCanonicalName()); + } + + +} diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/Application.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/Application.java index 3252a52e3e..b28f6387a9 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/Application.java +++ b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/Application.java @@ -17,10 +17,10 @@ package org.apache.eventmesh.openconnect; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.SinkConfig; +import org.apache.eventmesh.common.config.connector.SourceConfig; import org.apache.eventmesh.openconnect.api.ConnectorCreateService; -import org.apache.eventmesh.openconnect.api.config.Config; -import org.apache.eventmesh.openconnect.api.config.SinkConfig; -import org.apache.eventmesh.openconnect.api.config.SourceConfig; import org.apache.eventmesh.openconnect.api.connector.Connector; import org.apache.eventmesh.openconnect.api.sink.Sink; import org.apache.eventmesh.openconnect.api.source.Source; diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/SinkWorker.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/SinkWorker.java index a62b1b19ba..57ad4b8ec3 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/SinkWorker.java +++ b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/SinkWorker.java @@ -22,11 +22,11 @@ import org.apache.eventmesh.client.tcp.common.MessageUtils; import org.apache.eventmesh.client.tcp.common.ReceiveMsgHook; import org.apache.eventmesh.client.tcp.conf.EventMeshTCPClientConfig; +import org.apache.eventmesh.common.config.connector.SinkConfig; import org.apache.eventmesh.common.protocol.SubscriptionMode; import org.apache.eventmesh.common.protocol.SubscriptionType; import org.apache.eventmesh.common.protocol.tcp.UserAgent; import org.apache.eventmesh.common.utils.SystemUtils; -import org.apache.eventmesh.openconnect.api.config.SinkConfig; import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; import org.apache.eventmesh.openconnect.api.sink.Sink; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/SourceWorker.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/SourceWorker.java index 6dcbcd2994..2a2162a7af 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/SourceWorker.java +++ b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/SourceWorker.java @@ -24,19 +24,19 @@ import org.apache.eventmesh.client.tcp.common.MessageUtils; import org.apache.eventmesh.client.tcp.conf.EventMeshTCPClientConfig; import org.apache.eventmesh.common.ThreadPoolFactory; +import org.apache.eventmesh.common.config.connector.SourceConfig; +import org.apache.eventmesh.common.config.connector.offset.OffsetStorageConfig; import org.apache.eventmesh.common.exception.EventMeshException; import org.apache.eventmesh.common.protocol.tcp.OPStatus; import org.apache.eventmesh.common.protocol.tcp.Package; import org.apache.eventmesh.common.protocol.tcp.UserAgent; import org.apache.eventmesh.common.utils.JsonUtils; import org.apache.eventmesh.common.utils.SystemUtils; -import org.apache.eventmesh.openconnect.api.callback.SendExcepionContext; -import org.apache.eventmesh.openconnect.api.callback.SendMessageCallback; -import org.apache.eventmesh.openconnect.api.callback.SendResult; -import org.apache.eventmesh.openconnect.api.config.SourceConfig; import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; import org.apache.eventmesh.openconnect.api.source.Source; -import org.apache.eventmesh.openconnect.offsetmgmt.api.config.OffsetStorageConfig; +import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendExceptionContext; +import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendMessageCallback; +import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendResult; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffsetManagement; import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.DefaultOffsetManagementServiceImpl; @@ -152,8 +152,8 @@ public void init() { .map(storageType -> EventMeshExtensionFactory.getExtension(OffsetManagementService.class, storageType)) .orElse(new DefaultOffsetManagementServiceImpl()); this.offsetManagementService.initialize(offsetStorageConfig); - this.offsetStorageWriter = new OffsetStorageWriterImpl(source.name(), offsetManagementService); - this.offsetStorageReader = new OffsetStorageReaderImpl(source.name(), offsetManagementService); + this.offsetStorageWriter = new OffsetStorageWriterImpl(offsetManagementService); + this.offsetStorageReader = new OffsetStorageReaderImpl(offsetManagementService); } @Override @@ -264,8 +264,8 @@ private SendResult convertToSendResult(CloudEvent event) { return result; } - private SendExcepionContext convertToExceptionContext(CloudEvent event, Throwable cause) { - SendExcepionContext exceptionContext = new SendExcepionContext(); + private SendExceptionContext convertToExceptionContext(CloudEvent event, Throwable cause) { + SendExceptionContext exceptionContext = new SendExceptionContext(); exceptionContext.setTopic(event.getId()); exceptionContext.setMessageId(event.getId()); exceptionContext.setCause(cause); diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/Connector.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/Connector.java index 82993b198b..07e44aea94 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/Connector.java +++ b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/Connector.java @@ -17,13 +17,14 @@ package org.apache.eventmesh.openconnect.api.connector; -import org.apache.eventmesh.openconnect.api.config.Config; +import org.apache.eventmesh.common.ComponentLifeCycle; +import org.apache.eventmesh.common.config.connector.Config; import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; /** * Connector */ -public interface Connector { +public interface Connector extends ComponentLifeCycle { /** * Returns the class type of the configuration for this Connector. @@ -33,8 +34,7 @@ public interface Connector { Class configClass(); /** - * This init method is obsolete. For detailed discussion, - * please see here + * This init method is obsolete. For detailed discussion, please see here *

* Initializes the Connector with the provided configuration. * @@ -52,13 +52,6 @@ public interface Connector { */ void init(ConnectorContext connectorContext) throws Exception; - /** - * Starts the Connector. - * - * @throws Exception if the start operation fails - */ - void start() throws Exception; - /** * Commits the specified ConnectRecord object. * @@ -74,10 +67,11 @@ public interface Connector { String name(); /** - * Stops the Connector. + * This method will be called when an exception occurs while processing a ConnectRecord object. This method can be used to handle the exception, + * such as logging error information, or stopping the connector's operation when an exception occurs. * - * @throws Exception if stopping fails + * @param record The ConnectRecord object that was being processed when the exception occurred */ - void stop() throws Exception; + void onException(ConnectRecord record); } diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SinkConnector.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SinkConnector.java index 70b62c1200..9b271746f3 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SinkConnector.java +++ b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SinkConnector.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.openconnect.api.connector; -import org.apache.eventmesh.openconnect.api.config.SinkConfig; +import org.apache.eventmesh.common.config.connector.SinkConfig; import org.apache.eventmesh.openconnect.api.sink.Sink; public abstract class SinkConnector implements Sink { diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SinkConnectorContext.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SinkConnectorContext.java index a7aef4fff2..1ef048b06c 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SinkConnectorContext.java +++ b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SinkConnectorContext.java @@ -17,7 +17,10 @@ package org.apache.eventmesh.openconnect.api.connector; -import org.apache.eventmesh.openconnect.api.config.SinkConfig; +import org.apache.eventmesh.common.config.connector.SinkConfig; +import org.apache.eventmesh.common.remote.job.JobType; + +import java.util.Map; import lombok.Data; @@ -29,4 +32,8 @@ public class SinkConnectorContext implements ConnectorContext { public SinkConfig sinkConfig; + public Map runtimeConfig; + + public JobType jobType; + } diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SourceConnector.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SourceConnector.java index 100612662d..95279c2d41 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SourceConnector.java +++ b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SourceConnector.java @@ -17,7 +17,7 @@ package org.apache.eventmesh.openconnect.api.connector; -import org.apache.eventmesh.openconnect.api.config.SourceConfig; +import org.apache.eventmesh.common.config.connector.SourceConfig; import org.apache.eventmesh.openconnect.api.source.Source; public abstract class SourceConnector implements Source { diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SourceConnectorContext.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SourceConnectorContext.java index 14fd92218a..957452bb10 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SourceConnectorContext.java +++ b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/connector/SourceConnectorContext.java @@ -17,9 +17,14 @@ package org.apache.eventmesh.openconnect.api.connector; -import org.apache.eventmesh.openconnect.api.config.SourceConfig; +import org.apache.eventmesh.common.config.connector.SourceConfig; +import org.apache.eventmesh.common.remote.job.JobType; +import org.apache.eventmesh.common.remote.offset.RecordPosition; import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetStorageReader; +import java.util.List; +import java.util.Map; + import lombok.Data; /** @@ -32,4 +37,11 @@ public class SourceConnectorContext implements ConnectorContext { public SourceConfig sourceConfig; + public Map runtimeConfig; + + public JobType jobType; + + // initial record position + public List recordPositionList; + } diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/factory/ConnectorPluginFactory.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/factory/ConnectorPluginFactory.java new file mode 100644 index 0000000000..7a6f2e0a9c --- /dev/null +++ b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/factory/ConnectorPluginFactory.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.openconnect.api.factory; + +import org.apache.eventmesh.openconnect.api.ConnectorCreateService; +import org.apache.eventmesh.openconnect.api.connector.SinkConnector; +import org.apache.eventmesh.openconnect.api.connector.SourceConnector; +import org.apache.eventmesh.spi.EventMeshExtensionFactory; + + +/** + * The factory to get connector {@link SourceConnector} and {@link SinkConnector} + */ +public class ConnectorPluginFactory { + + /** + * Get ConnectorCreateService instance by plugin name + * + * @param connectorPluginName plugin name + * @return ConnectorCreateService instance + */ + public static ConnectorCreateService createConnector(String connectorPluginName) { + return EventMeshExtensionFactory.getExtension(ConnectorCreateService.class, connectorPluginName); + } + +} diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/monitor/AbstractConnectorMonitor.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/monitor/AbstractConnectorMonitor.java new file mode 100644 index 0000000000..b9205804a4 --- /dev/null +++ b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/monitor/AbstractConnectorMonitor.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.openconnect.api.monitor; + +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.LongAdder; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Getter +public abstract class AbstractConnectorMonitor implements Monitor { + + private final String taskId; + private final String jobId; + private final String ip; + private final LongAdder totalRecordNum; + private final LongAdder totalTimeCost; + protected final AtomicLong startTime; + private final AtomicLong maxTimeCost; + private long averageTime = 0; + private double tps = 0; + + public AbstractConnectorMonitor(String taskId, String jobId, String ip) { + this.taskId = taskId; + this.jobId = jobId; + this.ip = ip; + this.totalRecordNum = new LongAdder(); + this.totalTimeCost = new LongAdder(); + this.startTime = new AtomicLong(System.currentTimeMillis()); + this.maxTimeCost = new AtomicLong(); + } + + @Override + public synchronized void recordProcess(long timeCost) { + totalRecordNum.increment(); + totalTimeCost.add(timeCost); + maxTimeCost.updateAndGet(max -> Math.max(max, timeCost)); + } + + @Override + public synchronized void recordProcess(int recordCount, long timeCost) { + totalRecordNum.add(recordCount); + totalTimeCost.add(timeCost); + maxTimeCost.updateAndGet(max -> Math.max(max, timeCost)); + } + + @Override + public synchronized void printMetrics() { + long totalRecords = totalRecordNum.sum(); + long totalCost = totalTimeCost.sum(); + averageTime = totalRecords > 0 ? totalCost / totalRecords : 0; + long elapsedTime = (System.currentTimeMillis() - startTime.get()) / 1000; // in seconds + tps = elapsedTime > 0 ? (double) totalRecords / elapsedTime : 0; + + log.info("========== Metrics =========="); + log.info("TaskId: {}|JobId: {}|ip: {}", taskId, jobId, ip); + log.info("Total records: {}", totalRecordNum); + log.info("Total time (ms): {}", totalTimeCost); + log.info("Max time per record (ms): {}", maxTimeCost); + log.info("Average time per record (ms): {}", averageTime); + log.info("TPS: {}", tps); + } +} diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/monitor/Monitor.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/monitor/Monitor.java new file mode 100644 index 0000000000..4d4d9efb0c --- /dev/null +++ b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/monitor/Monitor.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.openconnect.api.monitor; + +/** + * Monitor Interface. + * All monitors should implement this interface. + */ +public interface Monitor { + void recordProcess(long timeCost); + + void recordProcess(int recordCount, long timeCost); + + void printMetrics(); +} \ No newline at end of file diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/monitor/MonitorRegistry.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/monitor/MonitorRegistry.java new file mode 100644 index 0000000000..904efc5d3f --- /dev/null +++ b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/monitor/MonitorRegistry.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.openconnect.api.monitor; + +import java.util.ArrayList; +import java.util.List; + +import lombok.Getter; + +public class MonitorRegistry { + + @Getter + private static final List monitors = new ArrayList<>(); + + public static void registerMonitor(Monitor monitor) { + monitors.add(monitor); + } + +} diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/util/ConfigUtil.java b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/util/ConfigUtil.java index 93b2ba9582..066dae3385 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/util/ConfigUtil.java +++ b/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/util/ConfigUtil.java @@ -17,14 +17,15 @@ package org.apache.eventmesh.openconnect.util; -import org.apache.eventmesh.openconnect.api.config.Config; -import org.apache.eventmesh.openconnect.api.config.Constants; -import org.apache.eventmesh.openconnect.api.config.SinkConfig; -import org.apache.eventmesh.openconnect.api.config.SourceConfig; +import org.apache.eventmesh.common.config.connector.Config; +import org.apache.eventmesh.common.config.connector.Constants; +import org.apache.eventmesh.common.config.connector.SinkConfig; +import org.apache.eventmesh.common.config.connector.SourceConfig; import java.io.File; import java.io.FileNotFoundException; import java.net.URL; +import java.util.Map; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -67,6 +68,11 @@ public static T parse(Class c, String filePathName) throws Exception { return objectMapper.readValue(url, c); } + public static T parse(Map map, Class c) throws Exception { + ObjectMapper objectMapper = new ObjectMapper(); + return objectMapper.convertValue(map, c); + } + private static Config parseSourceConfig(Class c) throws Exception { String configFile = System.getProperty(Constants.ENV_SOURCE_CONFIG_FILE, System.getenv(Constants.ENV_SOURCE_CONFIG_FILE)); if (configFile == null || configFile.isEmpty()) { diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-admin/build.gradle b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-admin/build.gradle new file mode 100644 index 0000000000..70defef627 --- /dev/null +++ b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-admin/build.gradle @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dependencies { + implementation project(":eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-api") + implementation project(":eventmesh-common") + testImplementation "org.mockito:mockito-core" + + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' + + implementation "io.grpc:grpc-core" + implementation "io.grpc:grpc-protobuf" + implementation "io.grpc:grpc-stub" + implementation "io.grpc:grpc-netty" + implementation "io.grpc:grpc-netty-shaded" +} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-admin/gradle.properties b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-admin/gradle.properties new file mode 100644 index 0000000000..09957a9d24 --- /dev/null +++ b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-admin/gradle.properties @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +pluginType=offsetMgmt +pluginName=admin \ No newline at end of file diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-admin/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/admin/AdminOffsetService.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-admin/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/admin/AdminOffsetService.java new file mode 100644 index 0000000000..993352a979 --- /dev/null +++ b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-admin/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/admin/AdminOffsetService.java @@ -0,0 +1,291 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.openconnect.offsetmgmt.admin; + +import org.apache.eventmesh.common.config.connector.offset.OffsetStorageConfig; +import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc; +import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc.AdminServiceBlockingStub; +import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc.AdminServiceStub; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Payload; +import org.apache.eventmesh.common.remote.TaskState; +import org.apache.eventmesh.common.remote.datasource.DataSourceType; +import org.apache.eventmesh.common.remote.offset.RecordOffset; +import org.apache.eventmesh.common.remote.offset.RecordPartition; +import org.apache.eventmesh.common.remote.offset.RecordPosition; +import org.apache.eventmesh.common.remote.request.FetchPositionRequest; +import org.apache.eventmesh.common.remote.request.ReportPositionRequest; +import org.apache.eventmesh.common.remote.response.FetchPositionResponse; +import org.apache.eventmesh.common.utils.IPUtils; +import org.apache.eventmesh.common.utils.JsonUtils; +import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.KeyValueStore; +import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.MemoryBasedKeyValueStore; +import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetManagementService; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Random; + +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.stub.StreamObserver; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.protobuf.Any; +import com.google.protobuf.UnsafeByteOperations; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class AdminOffsetService implements OffsetManagementService { + + private String adminServerAddr; + + private ManagedChannel channel; + + private AdminServiceStub adminServiceStub; + + private AdminServiceBlockingStub adminServiceBlockingStub; + + StreamObserver responseObserver; + + StreamObserver requestObserver; + + public KeyValueStore positionStore; + + private String jobId; + + private TaskState jobState; + + private DataSourceType dataSourceType; + + private DataSourceType dataSinkType; + + + @Override + public void start() { + + } + + @Override + public void stop() { + + } + + @Override + public void configure(OffsetStorageConfig config) { + OffsetManagementService.super.configure(config); + } + + @Override + public void persist() { + Map recordMap = positionStore.getKVMap(); + + List recordToSyncList = new ArrayList<>(); + for (Map.Entry entry : recordMap.entrySet()) { + RecordPosition recordPosition = new RecordPosition(entry.getKey(), entry.getValue()); + recordToSyncList.add(recordPosition); + } + + ReportPositionRequest reportPositionRequest = new ReportPositionRequest(); + reportPositionRequest.setJobID(jobId); + reportPositionRequest.setState(jobState); + reportPositionRequest.setDataSourceType(dataSourceType); + reportPositionRequest.setAddress(IPUtils.getLocalAddress()); + + reportPositionRequest.setRecordPositionList(recordToSyncList); + + log.debug("start report position request: {}", JsonUtils.toJSONString(reportPositionRequest)); + + Metadata metadata = Metadata.newBuilder() + .setType(ReportPositionRequest.class.getSimpleName()) + .build(); + Payload payload = Payload.newBuilder() + .setMetadata(metadata) + .setBody(Any.newBuilder().setValue(UnsafeByteOperations.unsafeWrap(Objects.requireNonNull(JsonUtils.toJSONBytes(reportPositionRequest)))) + .build()) + .build(); + requestObserver.onNext(payload); + log.debug("end report position request: {}", JsonUtils.toJSONString(reportPositionRequest)); + + for (Map.Entry entry : recordMap.entrySet()) { + positionStore.remove(entry.getKey()); + } + } + + @Override + public void load() { + + } + + @Override + public void synchronize() { + + } + + @Override + public Map getPositionMap() { + // get from memory storage first + if (positionStore.getKVMap() == null || positionStore.getKVMap().isEmpty()) { + log.info("fetch position from admin server"); + FetchPositionRequest fetchPositionRequest = new FetchPositionRequest(); + fetchPositionRequest.setJobID(jobId); + fetchPositionRequest.setAddress(IPUtils.getLocalAddress()); + fetchPositionRequest.setDataSourceType(dataSourceType); + + Metadata metadata = Metadata.newBuilder() + .setType(FetchPositionRequest.class.getSimpleName()) + .build(); + + Payload request = Payload.newBuilder() + .setMetadata(metadata) + .setBody( + Any.newBuilder().setValue(UnsafeByteOperations.unsafeWrap(Objects.requireNonNull(JsonUtils.toJSONBytes(fetchPositionRequest)))) + .build()) + .build(); + Payload response = adminServiceBlockingStub.invoke(request); + if (response.getMetadata().getType().equals(FetchPositionResponse.class.getSimpleName())) { + FetchPositionResponse fetchPositionResponse = + JsonUtils.parseObject(response.getBody().getValue().toStringUtf8(), FetchPositionResponse.class); + assert fetchPositionResponse != null; + if (fetchPositionResponse.isSuccess()) { + for (RecordPosition recordPosition : fetchPositionResponse.getRecordPosition()) { + positionStore.put(recordPosition.getRecordPartition(), recordPosition.getRecordOffset()); + } + } + } + } + log.info("memory position map {}", positionStore.getKVMap()); + return positionStore.getKVMap(); + } + + @Override + public RecordOffset getPosition(RecordPartition partition) { + // get from memory storage first + if (positionStore.get(partition) == null) { + log.info("fetch position from admin server"); + FetchPositionRequest fetchPositionRequest = new FetchPositionRequest(); + fetchPositionRequest.setJobID(jobId); + fetchPositionRequest.setAddress(IPUtils.getLocalAddress()); + fetchPositionRequest.setDataSourceType(dataSourceType); + RecordPosition fetchRecordPosition = new RecordPosition(); + fetchRecordPosition.setRecordPartition(partition); + fetchPositionRequest.setRecordPosition(fetchRecordPosition); + + Metadata metadata = Metadata.newBuilder() + .setType(FetchPositionRequest.class.getSimpleName()) + .build(); + + Payload request = Payload.newBuilder() + .setMetadata(metadata) + .setBody( + Any.newBuilder().setValue(UnsafeByteOperations.unsafeWrap(Objects.requireNonNull(JsonUtils.toJSONBytes(fetchPositionRequest)))) + .build()) + .build(); + Payload response = adminServiceBlockingStub.invoke(request); + if (response.getMetadata().getType().equals(FetchPositionResponse.class.getSimpleName())) { + FetchPositionResponse fetchPositionResponse = + JsonUtils.parseObject(response.getBody().getValue().toStringUtf8(), FetchPositionResponse.class); + assert fetchPositionResponse != null; + if (fetchPositionResponse.isSuccess()) { + for (RecordPosition recordPosition : fetchPositionResponse.getRecordPosition()) { + positionStore.put(recordPosition.getRecordPartition(), recordPosition.getRecordOffset()); + } + } + } + } + log.info("memory record position {}", positionStore.get(partition)); + return positionStore.get(partition); + } + + @Override + public void putPosition(Map positions) { + positionStore.putAll(positions); + } + + @Override + public void putPosition(RecordPartition partition, RecordOffset position) { + positionStore.put(partition, position); + } + + @Override + public void removePosition(List partitions) { + if (partitions == null) { + return; + } + for (RecordPartition partition : partitions) { + positionStore.remove(partition); + } + } + + @Override + public void initialize(OffsetStorageConfig offsetStorageConfig) { + this.dataSourceType = offsetStorageConfig.getDataSourceType(); + this.dataSinkType = offsetStorageConfig.getDataSinkType(); + + this.adminServerAddr = getRandomAdminServerAddr(offsetStorageConfig.getOffsetStorageAddr()); + this.channel = ManagedChannelBuilder.forTarget(adminServerAddr) + .usePlaintext() + .build(); + this.adminServiceStub = AdminServiceGrpc.newStub(channel).withWaitForReady(); + this.adminServiceBlockingStub = AdminServiceGrpc.newBlockingStub(channel).withWaitForReady(); + + responseObserver = new StreamObserver() { + @Override + public void onNext(Payload response) { + log.info("receive message: {} ", response); + } + + @Override + public void onError(Throwable t) { + log.error("receive error message: {}", t.getMessage()); + } + + @Override + public void onCompleted() { + log.info("finished receive message and completed"); + } + }; + + requestObserver = adminServiceStub.invokeBiStream(responseObserver); + + this.positionStore = new MemoryBasedKeyValueStore<>(); + String offset = offsetStorageConfig.getExtensions().get("offset"); + if (offset != null) { + Map initialRecordOffsetMap = JsonUtils.parseTypeReferenceObject(offset, + new TypeReference>() { + }); + log.info("init record offset {}", initialRecordOffsetMap); + positionStore.putAll(initialRecordOffsetMap); + } + this.jobState = TaskState.RUNNING; + this.jobId = offsetStorageConfig.getExtensions().get("jobId"); + } + + private String getRandomAdminServerAddr(String adminServerAddrList) { + String[] addresses = adminServerAddrList.split(";"); + if (addresses.length == 0) { + throw new IllegalArgumentException("Admin server address list is empty"); + } + Random random = new Random(); + int randomIndex = random.nextInt(addresses.length); + return addresses[randomIndex]; + } +} diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-admin/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetManagementService b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-admin/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetManagementService new file mode 100644 index 0000000000..11b4466d79 --- /dev/null +++ b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-admin/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetManagementService @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +admin=org.apache.eventmesh.openconnect.offsetmgmt.admin.AdminOffsetService \ No newline at end of file diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/build.gradle b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/build.gradle index 97c3b8c33c..1338b0b7d8 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/build.gradle +++ b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/build.gradle @@ -16,6 +16,7 @@ */ dependencies { + implementation project(":eventmesh-common") api project(":eventmesh-spi") api "org.slf4j:slf4j-api" diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/callback/SendExcepionContext.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/callback/SendExceptionContext.java similarity index 90% rename from eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/callback/SendExcepionContext.java rename to eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/callback/SendExceptionContext.java index 0311ceaef5..974b19a547 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/callback/SendExcepionContext.java +++ b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/callback/SendExceptionContext.java @@ -15,15 +15,15 @@ * limitations under the License. */ -package org.apache.eventmesh.openconnect.api.callback; +package org.apache.eventmesh.openconnect.offsetmgmt.api.callback; -public class SendExcepionContext { +public class SendExceptionContext { private String messageId; private String topic; private Throwable cause; - public SendExcepionContext() { + public SendExceptionContext() { } public String getMessageId() { diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/callback/SendMessageCallback.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/callback/SendMessageCallback.java similarity index 87% rename from eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/callback/SendMessageCallback.java rename to eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/callback/SendMessageCallback.java index fd6baba7ec..8346cf36b4 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/callback/SendMessageCallback.java +++ b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/callback/SendMessageCallback.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.openconnect.api.callback; +package org.apache.eventmesh.openconnect.offsetmgmt.api.callback; /** * Message sending callback interface. @@ -24,5 +24,5 @@ public interface SendMessageCallback { void onSuccess(SendResult sendResult); - void onException(SendExcepionContext sendExcepionContext); + void onException(SendExceptionContext sendExceptionContext); } diff --git a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/callback/SendResult.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/callback/SendResult.java similarity index 95% rename from eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/callback/SendResult.java rename to eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/callback/SendResult.java index 8cd861f6de..9afc745f3d 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-java/src/main/java/org/apache/eventmesh/openconnect/api/callback/SendResult.java +++ b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/callback/SendResult.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.eventmesh.openconnect.api.callback; +package org.apache.eventmesh.openconnect.offsetmgmt.api.callback; public class SendResult { diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/ConnectRecord.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/ConnectRecord.java index 119f058b58..0a41e18f7c 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/ConnectRecord.java +++ b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/ConnectRecord.java @@ -17,22 +17,45 @@ package org.apache.eventmesh.openconnect.offsetmgmt.api.data; +import org.apache.eventmesh.common.remote.offset.RecordOffset; +import org.apache.eventmesh.common.remote.offset.RecordPartition; +import org.apache.eventmesh.common.remote.offset.RecordPosition; +import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendMessageCallback; + import java.util.Objects; import java.util.Set; +import java.util.UUID; + +import lombok.Getter; +import lombok.Setter; /** * SourceDataEntries are generated by SourceTasks and passed to specific message queue to store. */ +@Getter public class ConnectRecord { + private final String recordId = UUID.randomUUID().toString(); + + @Setter private Long timestamp; + @Setter private Object data; + @Setter private RecordPosition position; + @Setter private KeyValue extensions; + @Setter + private SendMessageCallback callback; + + public ConnectRecord() { + + } + public ConnectRecord(RecordPartition recordPartition, RecordOffset recordOffset, Long timestamp) { this(recordPartition, recordOffset, timestamp, null); @@ -40,43 +63,15 @@ public ConnectRecord(RecordPartition recordPartition, RecordOffset recordOffset, public ConnectRecord(RecordPartition recordPartition, RecordOffset recordOffset, Long timestamp, Object data) { - this.position = new RecordPosition(recordPartition, recordOffset); - this.timestamp = timestamp; - this.data = data; - } - - public Long getTimestamp() { - return timestamp; - } - - public void setTimestamp(Long timestamp) { + if (recordPartition == null || recordOffset == null) { + this.position = null; + } else { + this.position = new RecordPosition(recordPartition, recordOffset); + } this.timestamp = timestamp; - } - - public Object getData() { - return data; - } - - public void setData(Object data) { this.data = data; } - public KeyValue getExtensions() { - return extensions; - } - - public void setExtensions(KeyValue extensions) { - this.extensions = extensions; - } - - public RecordPosition getPosition() { - return position; - } - - public void setPosition(RecordPosition position) { - this.position = position; - } - public void addExtension(KeyValue extensions) { if (this.extensions == null) { this.extensions = new DefaultKeyValue(); @@ -124,19 +119,20 @@ public boolean equals(Object o) { return false; } ConnectRecord that = (ConnectRecord) o; - return Objects.equals(timestamp, that.timestamp) && Objects.equals(data, that.data) + return Objects.equals(recordId, that.recordId) && Objects.equals(timestamp, that.timestamp) && Objects.equals(data, that.data) && Objects.equals(position, that.position) && Objects.equals(extensions, that.extensions); } @Override public int hashCode() { - return Objects.hash(timestamp, data, position, extensions); + return Objects.hash(recordId, timestamp, data, position, extensions); } @Override public String toString() { return "ConnectRecord{" - + "timestamp=" + timestamp + + "recordId=" + recordId + + ", timestamp=" + timestamp + ", data=" + data + ", position=" + position + ", extensions=" + extensions diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/DefaultKeyValue.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/DefaultKeyValue.java index a0390c1892..891df482be 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/DefaultKeyValue.java +++ b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/DefaultKeyValue.java @@ -23,6 +23,11 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter public class DefaultKeyValue implements KeyValue { private final Map properties; diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/RecordOffsetManagement.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/RecordOffsetManagement.java index 2eaefbef29..7e6b5042f8 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/RecordOffsetManagement.java +++ b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/data/RecordOffsetManagement.java @@ -17,6 +17,10 @@ package org.apache.eventmesh.openconnect.offsetmgmt.api.data; +import org.apache.eventmesh.common.remote.offset.RecordOffset; +import org.apache.eventmesh.common.remote.offset.RecordPartition; +import org.apache.eventmesh.common.remote.offset.RecordPosition; + import java.util.Collections; import java.util.Deque; import java.util.HashMap; @@ -42,12 +46,13 @@ public RecordOffsetManagement() { /** * submit record + * * @param position * @return */ public SubmittedPosition submitRecord(RecordPosition position) { SubmittedPosition submittedPosition = new SubmittedPosition(position); - records.computeIfAbsent(position.getPartition(), e -> new LinkedList<>()).add(submittedPosition); + records.computeIfAbsent(position.getRecordPartition(), e -> new LinkedList<>()).add(submittedPosition); // ensure thread safety in operation synchronized (this) { numUnacked.incrementAndGet(); @@ -63,7 +68,7 @@ private RecordOffset pollOffsetWhile(Deque submittedPositions RecordOffset offset = null; // Stop pulling if there is an uncommitted breakpoint while (canCommitHead(submittedPositions)) { - offset = submittedPositions.poll().getPosition().getOffset(); + offset = submittedPositions.poll().getPosition().getRecordOffset(); } return offset; } @@ -132,8 +137,8 @@ private synchronized void messageAcked() { } /** - * Contains a snapshot of offsets that can be committed for a source task and metadata for that offset commit - * (such as the number of messages for which offsets can and cannot be committed). + * Contains a snapshot of offsets that can be committed for a source task and metadata for that offset commit (such as the number of messages for + * which offsets can and cannot be committed). */ public static class CommittableOffsets { @@ -235,19 +240,19 @@ public void ack() { * @return */ public boolean remove() { - Deque deque = records.get(position.getPartition()); + Deque deque = records.get(position.getRecordPartition()); if (deque == null) { return false; } boolean result = deque.removeLastOccurrence(this); if (deque.isEmpty()) { - records.remove(position.getPartition()); + records.remove(position.getRecordPartition()); } if (result) { messageAcked(); } else { log.warn("Attempted to remove record from submitted queue for partition {}, " - + "but the record has not been submitted or has already been removed", position.getPartition()); + + "but the record has not been submitted or has already been removed", position.getRecordPartition()); } return result; } diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/DefaultOffsetManagementServiceImpl.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/DefaultOffsetManagementServiceImpl.java index e31fc358f0..be72097911 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/DefaultOffsetManagementServiceImpl.java +++ b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/DefaultOffsetManagementServiceImpl.java @@ -17,8 +17,9 @@ package org.apache.eventmesh.openconnect.offsetmgmt.api.storage; -import org.apache.eventmesh.openconnect.offsetmgmt.api.config.OffsetStorageConfig; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; +import org.apache.eventmesh.common.config.connector.offset.OffsetStorageConfig; +import org.apache.eventmesh.common.remote.offset.RecordOffset; +import org.apache.eventmesh.common.remote.offset.RecordPartition; import java.util.List; import java.util.Map; @@ -51,27 +52,27 @@ public void synchronize() { } @Override - public Map getPositionMap() { + public Map getPositionMap() { return null; } @Override - public RecordOffset getPosition(ConnectorRecordPartition partition) { + public RecordOffset getPosition(RecordPartition partition) { return null; } @Override - public void putPosition(Map positions) { + public void putPosition(Map positions) { } @Override - public void putPosition(ConnectorRecordPartition partition, RecordOffset position) { + public void putPosition(RecordPartition partition, RecordOffset position) { } @Override - public void removePosition(List partitions) { + public void removePosition(List partitions) { } diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetManagementService.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetManagementService.java index ef1dc0d30d..62327a1ae9 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetManagementService.java +++ b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetManagementService.java @@ -17,8 +17,9 @@ package org.apache.eventmesh.openconnect.offsetmgmt.api.storage; -import org.apache.eventmesh.openconnect.offsetmgmt.api.config.OffsetStorageConfig; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; +import org.apache.eventmesh.common.config.connector.offset.OffsetStorageConfig; +import org.apache.eventmesh.common.remote.offset.RecordOffset; +import org.apache.eventmesh.common.remote.offset.RecordPartition; import org.apache.eventmesh.spi.EventMeshExtensionType; import org.apache.eventmesh.spi.EventMeshSPI; @@ -49,12 +50,12 @@ default void configure(OffsetStorageConfig config) { } /** - * Persist position info in a persist store. + * Persist position info in a persisted store. */ void persist(); /** - * load position info in a persist store. + * load position info in a persisted store. */ void load(); @@ -68,24 +69,24 @@ default void configure(OffsetStorageConfig config) { * * @return */ - Map getPositionMap(); + Map getPositionMap(); - RecordOffset getPosition(ConnectorRecordPartition partition); + RecordOffset getPosition(RecordPartition partition); /** * Put a position info. */ - void putPosition(Map positions); + void putPosition(Map positions); - void putPosition(ConnectorRecordPartition partition, RecordOffset position); + void putPosition(RecordPartition partition, RecordOffset position); /** * Remove a position info. * * @param partitions */ - void removePosition(List partitions); + void removePosition(List partitions); - void initialize(OffsetStorageConfig connectorConfig); + void initialize(OffsetStorageConfig offsetStorageConfig); } diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageReader.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageReader.java index dc17e29840..30546b96cb 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageReader.java +++ b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageReader.java @@ -17,8 +17,8 @@ package org.apache.eventmesh.openconnect.offsetmgmt.api.storage; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; +import org.apache.eventmesh.common.remote.offset.RecordOffset; +import org.apache.eventmesh.common.remote.offset.RecordPartition; import java.util.Collection; import java.util.Map; diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageReaderImpl.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageReaderImpl.java index efd087404e..ca4ad2c751 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageReaderImpl.java +++ b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageReaderImpl.java @@ -17,8 +17,8 @@ package org.apache.eventmesh.openconnect.offsetmgmt.api.storage; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; +import org.apache.eventmesh.common.remote.offset.RecordOffset; +import org.apache.eventmesh.common.remote.offset.RecordPartition; import java.util.Collection; import java.util.HashMap; @@ -26,29 +26,24 @@ public class OffsetStorageReaderImpl implements OffsetStorageReader { - private final String connectorName; - private OffsetManagementService offsetManagementService; - public OffsetStorageReaderImpl(String connectorName, OffsetManagementService offsetManagementService) { - this.connectorName = connectorName; + public OffsetStorageReaderImpl(OffsetManagementService offsetManagementService) { this.offsetManagementService = offsetManagementService; } @Override public RecordOffset readOffset(RecordPartition partition) { - ConnectorRecordPartition connectorRecordPartition = new ConnectorRecordPartition(connectorName, partition.getPartition()); - return offsetManagementService.getPositionMap().get(connectorRecordPartition); + return offsetManagementService.getPositionMap().get(partition); } @Override public Map readOffsets(Collection partitions) { Map result = new HashMap<>(); - Map allData = offsetManagementService.getPositionMap(); + Map allData = offsetManagementService.getPositionMap(); for (RecordPartition key : partitions) { - ConnectorRecordPartition connectorRecordPartition = new ConnectorRecordPartition(connectorName, key.getPartition()); - if (allData.containsKey(connectorRecordPartition)) { - result.put(key, allData.get(connectorRecordPartition)); + if (allData.containsKey(key)) { + result.put(key, allData.get(key)); } } return result; diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageWriter.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageWriter.java index f8b6bdc45e..fb30acc918 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageWriter.java +++ b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageWriter.java @@ -17,8 +17,8 @@ package org.apache.eventmesh.openconnect.offsetmgmt.api.storage; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; +import org.apache.eventmesh.common.remote.offset.RecordOffset; +import org.apache.eventmesh.common.remote.offset.RecordPartition; import java.util.Map; diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageWriterImpl.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageWriterImpl.java index 3c5ed033a8..ef52602d60 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageWriterImpl.java +++ b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-api/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/api/storage/OffsetStorageWriterImpl.java @@ -17,8 +17,8 @@ package org.apache.eventmesh.openconnect.offsetmgmt.api.storage; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordPartition; +import org.apache.eventmesh.common.remote.offset.RecordOffset; +import org.apache.eventmesh.common.remote.offset.RecordPartition; import java.io.Closeable; import java.io.IOException; @@ -35,29 +35,25 @@ @Slf4j public class OffsetStorageWriterImpl implements OffsetStorageWriter, Closeable { - private final String connectorName; private final ExecutorService executorService = Executors.newSingleThreadExecutor(); - private OffsetManagementService offsetManagementService; + private final OffsetManagementService offsetManagementService; /** * Offset data in Connect format */ - private Map data = new HashMap<>(); - private Map toFlush = null; + private Map data = new HashMap<>(); + private Map toFlush = null; // Unique ID for each flush request to handle callbacks after timeouts private long currentFlushId = 0; - public OffsetStorageWriterImpl(String connectorName, OffsetManagementService offsetManagementService) { - this.connectorName = connectorName; + public OffsetStorageWriterImpl(OffsetManagementService offsetManagementService) { this.offsetManagementService = offsetManagementService; } @Override public void writeOffset(RecordPartition partition, RecordOffset offset) { - ConnectorRecordPartition extendRecordPartition; if (partition != null) { - extendRecordPartition = new ConnectorRecordPartition(connectorName, partition.getPartition()); - data.put(extendRecordPartition, offset); + data.put(partition, offset); } } diff --git a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-nacos/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/nacos/NacosConfigService.java b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-nacos/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/nacos/NacosConfigService.java index 8d4e9dade8..67c53d4d6d 100644 --- a/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-nacos/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/nacos/NacosConfigService.java +++ b/eventmesh-openconnect/eventmesh-openconnect-offsetmgmt-plugin/eventmesh-openconnect-offsetmgmt-nacos/src/main/java/org/apache/eventmesh/openconnect/offsetmgmt/nacos/NacosConfigService.java @@ -17,9 +17,9 @@ package org.apache.eventmesh.openconnect.offsetmgmt.nacos; -import org.apache.eventmesh.openconnect.offsetmgmt.api.config.OffsetStorageConfig; -import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffset; -import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.ConnectorRecordPartition; +import org.apache.eventmesh.common.config.connector.offset.OffsetStorageConfig; +import org.apache.eventmesh.common.remote.offset.RecordOffset; +import org.apache.eventmesh.common.remote.offset.RecordPartition; import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.KeyValueStore; import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.MemoryBasedKeyValueStore; import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetManagementService; @@ -56,7 +56,7 @@ public class NacosConfigService implements OffsetManagementService { private Listener listener; - public KeyValueStore positionStore; + public KeyValueStore positionStore; @Override public void start() { @@ -68,19 +68,19 @@ public void start() { } // merge the updated connectorRecord & recordOffset to memory store - public void mergeOffset(ConnectorRecordPartition connectorRecordPartition, RecordOffset recordOffset) { - if (connectorRecordPartition == null || connectorRecordPartition.getPartition().isEmpty()) { + public void mergeOffset(RecordPartition recordPartition, RecordOffset recordOffset) { + if (recordPartition == null) { return; } - if (positionStore.getKVMap().containsKey(connectorRecordPartition)) { - RecordOffset existedOffset = positionStore.getKVMap().get(connectorRecordPartition); + if (positionStore.getKVMap().containsKey(recordPartition)) { + RecordOffset existedOffset = positionStore.getKVMap().get(recordPartition); // update if (!recordOffset.equals(existedOffset)) { - positionStore.put(connectorRecordPartition, recordOffset); + positionStore.put(recordPartition, recordOffset); } } else { // add new position - positionStore.put(connectorRecordPartition, recordOffset); + positionStore.put(recordPartition, recordOffset); } } @@ -108,12 +108,12 @@ public void load() { @Override public void synchronize() { try { - Map recordMap = positionStore.getKVMap(); + Map recordMap = positionStore.getKVMap(); List> recordToSyncList = new ArrayList<>(); - for (Map.Entry entry : recordMap.entrySet()) { + for (Map.Entry entry : recordMap.entrySet()) { Map synchronizeMap = new HashMap<>(); - synchronizeMap.put("connectorRecordPartition", entry.getKey()); + synchronizeMap.put("recordPartition", entry.getKey()); synchronizeMap.put("recordOffset", entry.getValue()); recordToSyncList.add(synchronizeMap); } @@ -125,13 +125,14 @@ public void synchronize() { } @Override - public Map getPositionMap() { + public Map getPositionMap() { // get from memory storage first if (positionStore.getKVMap() == null || positionStore.getKVMap().isEmpty()) { try { - Map configMap = JacksonUtils.toObj(configService.getConfig(dataId, group, 5000L), - new TypeReference>() { + Map configMap = JacksonUtils.toObj(configService.getConfig(dataId, group, 5000L), + new TypeReference>() { }); + positionStore.putAll(configMap); log.info("nacos position map {}", configMap); return configMap; } catch (NacosException e) { @@ -143,12 +144,12 @@ public Map getPositionMap() { } @Override - public RecordOffset getPosition(ConnectorRecordPartition partition) { + public RecordOffset getPosition(RecordPartition partition) { // get from memory storage first if (positionStore.get(partition) == null) { try { - Map recordMap = JacksonUtils.toObj(configService.getConfig(dataId, group, 5000L), - new TypeReference>() { + Map recordMap = JacksonUtils.toObj(configService.getConfig(dataId, group, 5000L), + new TypeReference>() { }); log.info("nacos record position {}", recordMap.get(partition)); return recordMap.get(partition); @@ -161,21 +162,21 @@ public RecordOffset getPosition(ConnectorRecordPartition partition) { } @Override - public void putPosition(Map positions) { + public void putPosition(Map positions) { positionStore.putAll(positions); } @Override - public void putPosition(ConnectorRecordPartition partition, RecordOffset position) { + public void putPosition(RecordPartition partition, RecordOffset position) { positionStore.put(partition, position); } @Override - public void removePosition(List partitions) { + public void removePosition(List partitions) { if (partitions == null) { return; } - for (ConnectorRecordPartition partition : partitions) { + for (RecordPartition partition : partitions) { positionStore.remove(partition); } } @@ -206,13 +207,13 @@ public void receiveConfigInfo(String configInfo) { }); for (Map recordPartitionOffsetMap : recordOffsetList) { - ConnectorRecordPartition connectorRecordPartition = JacksonUtils.toObj( - JacksonUtils.toJson(recordPartitionOffsetMap.get("connectorRecordPartition")), - ConnectorRecordPartition.class); + RecordPartition recordPartition = JacksonUtils.toObj( + JacksonUtils.toJson(recordPartitionOffsetMap.get("recordPartition")), + RecordPartition.class); RecordOffset recordOffset = JacksonUtils.toObj(JacksonUtils.toJson(recordPartitionOffsetMap.get("recordOffset")), RecordOffset.class); // update the offset in memory store - mergeOffset(connectorRecordPartition, recordOffset); + mergeOffset(recordPartition, recordOffset); } } }; diff --git a/eventmesh-operator/config/samples/eventmesh_v1_runtime.yaml b/eventmesh-operator/config/samples/eventmesh_v1_runtime.yaml index fc640aa639..590928ca97 100644 --- a/eventmesh-operator/config/samples/eventmesh_v1_runtime.yaml +++ b/eventmesh-operator/config/samples/eventmesh_v1_runtime.yaml @@ -41,7 +41,6 @@ data: # HTTP Admin Server eventMesh.server.admin.http.port=10106 ########################## eventMesh tcp configuration ############################ - eventMesh.server.tcp.enabled=true eventMesh.server.tcp.readerIdleSeconds=120 eventMesh.server.tcp.writerIdleSeconds=120 eventMesh.server.tcp.allIdleSeconds=120 diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/build.gradle b/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/build.gradle index 0bc8809e86..e6ffc372b9 100644 --- a/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/build.gradle +++ b/eventmesh-protocol-plugin/eventmesh-protocol-cloudevents/build.gradle @@ -20,10 +20,10 @@ dependencies { implementation "io.cloudevents:cloudevents-core" implementation "com.google.guava:guava" implementation "io.cloudevents:cloudevents-json-jackson" - implementation ("io.grpc:grpc-protobuf:1.64.0") { + implementation ("io.grpc:grpc-protobuf:1.68.0") { exclude group: "com.google.protobuf", module: "protobuf-java" } - implementation("com.google.protobuf:protobuf-java:3.21.5") + implementation("com.google.protobuf:protobuf-java:3.25.4") implementation "io.cloudevents:cloudevents-protobuf" compileOnly 'org.projectlombok:lombok' diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-grpc/build.gradle b/eventmesh-protocol-plugin/eventmesh-protocol-grpc/build.gradle index 37ea099103..5929c72136 100644 --- a/eventmesh-protocol-plugin/eventmesh-protocol-grpc/build.gradle +++ b/eventmesh-protocol-plugin/eventmesh-protocol-grpc/build.gradle @@ -24,8 +24,8 @@ repositories { mavenCentral() } -def grpcVersion = '1.64.0' // CURRENT_GRPC_VERSION -def protobufVersion = '3.21.5' +def grpcVersion = '1.68.0' +def protobufVersion = '3.25.4' def protocVersion = protobufVersion dependencies { @@ -36,7 +36,7 @@ dependencies { implementation "io.grpc:grpc-stub:${grpcVersion}" implementation "com.google.protobuf:protobuf-java-util:${protobufVersion}" implementation "javax.annotation:javax.annotation-api:1.3.2" - testImplementation 'org.junit.jupiter:junit-jupiter:5.6.0' + testImplementation 'org.junit.jupiter:junit-jupiter' } protobuf { diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-http/build.gradle b/eventmesh-protocol-plugin/eventmesh-protocol-http/build.gradle index af9ac9198a..d219c5dc03 100644 --- a/eventmesh-protocol-plugin/eventmesh-protocol-http/build.gradle +++ b/eventmesh-protocol-plugin/eventmesh-protocol-http/build.gradle @@ -20,8 +20,8 @@ dependencies { implementation "io.cloudevents:cloudevents-core" implementation "com.google.guava:guava" implementation "io.cloudevents:cloudevents-json-jackson" - implementation ("io.grpc:grpc-protobuf:1.64.0") { + implementation ("io.grpc:grpc-protobuf:1.68.0") { exclude group: "com.google.protobuf", module: "protobuf-java" } - implementation("com.google.protobuf:protobuf-java:3.21.5") + implementation("com.google.protobuf:protobuf-java:3.25.4") } diff --git a/eventmesh-protocol-plugin/eventmesh-protocol-meshmessage/build.gradle b/eventmesh-protocol-plugin/eventmesh-protocol-meshmessage/build.gradle index 6dbb13157b..3f15d199ff 100644 --- a/eventmesh-protocol-plugin/eventmesh-protocol-meshmessage/build.gradle +++ b/eventmesh-protocol-plugin/eventmesh-protocol-meshmessage/build.gradle @@ -18,10 +18,10 @@ dependencies { implementation project(":eventmesh-protocol-plugin:eventmesh-protocol-api") implementation "io.cloudevents:cloudevents-core" - implementation ("io.grpc:grpc-protobuf:1.64.0") { + implementation ("io.grpc:grpc-protobuf:1.68.0") { exclude group: "com.google.protobuf", module: "protobuf-java" } - implementation("com.google.protobuf:protobuf-java:3.21.5") + implementation("com.google.protobuf:protobuf-java:3.25.4") implementation "io.cloudevents:cloudevents-protobuf" testImplementation project(":eventmesh-protocol-plugin:eventmesh-protocol-api") diff --git a/eventmesh-registry/.gitignore b/eventmesh-registry/.gitignore new file mode 100644 index 0000000000..b63da4551b --- /dev/null +++ b/eventmesh-registry/.gitignore @@ -0,0 +1,42 @@ +.gradle +build/ +!gradle/wrapper/gradle-wrapper.jar +!**/src/main/**/build/ +!**/src/test/**/build/ + +### IntelliJ IDEA ### +.idea/modules.xml +.idea/jarRepositories.xml +.idea/compiler.xml +.idea/libraries/ +*.iws +*.iml +*.ipr +out/ +!**/src/main/**/out/ +!**/src/test/**/out/ + +### Eclipse ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache +bin/ +!**/src/main/**/bin/ +!**/src/test/**/bin/ + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ + +### VS Code ### +.vscode/ + +### Mac OS ### +.DS_Store \ No newline at end of file diff --git a/eventmesh-registry/build.gradle b/eventmesh-registry/build.gradle new file mode 100644 index 0000000000..d973dcedae --- /dev/null +++ b/eventmesh-registry/build.gradle @@ -0,0 +1,16 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ \ No newline at end of file diff --git a/eventmesh-registry/eventmesh-registry-api/build.gradle b/eventmesh-registry/eventmesh-registry-api/build.gradle new file mode 100644 index 0000000000..c0546b6169 --- /dev/null +++ b/eventmesh-registry/eventmesh-registry-api/build.gradle @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dependencies { + implementation project(":eventmesh-spi") + implementation project(":eventmesh-common") + implementation "com.alibaba.nacos:nacos-client" + + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} \ No newline at end of file diff --git a/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/NotifyEvent.java b/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/NotifyEvent.java new file mode 100644 index 0000000000..fdef6a3285 --- /dev/null +++ b/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/NotifyEvent.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.registry; + +import java.util.List; + +import lombok.Getter; + +public class NotifyEvent { + + public NotifyEvent() { + + } + + public NotifyEvent(List instances) { + this(instances, false); + } + + public NotifyEvent(List instances, boolean isIncrement) { + this.isIncrement = isIncrement; + this.instances = instances; + } + + + // means whether it is an increment data + @Getter + private boolean isIncrement; + + @Getter + private List instances; +} diff --git a/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/QueryInstances.java b/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/QueryInstances.java new file mode 100644 index 0000000000..c8c7d61f4d --- /dev/null +++ b/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/QueryInstances.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.registry; + +import java.util.HashMap; +import java.util.Map; + +import lombok.Data; + +@Data +public class QueryInstances { + + private String serviceName; + private boolean health; + private Map extFields = new HashMap<>(); +} diff --git a/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/RegisterServerInfo.java b/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/RegisterServerInfo.java new file mode 100644 index 0000000000..0bf411c037 --- /dev/null +++ b/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/RegisterServerInfo.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.registry; + +import java.util.HashMap; +import java.util.Map; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +@ToString +public class RegisterServerInfo { + + // different implementations will have different formats + @Getter + @Setter + private String serviceName; + + @Getter + @Setter + private String address; + + @Getter + @Setter + private boolean health; + @Getter + private Map metadata = new HashMap<>(); + @Getter + private Map extFields = new HashMap<>(); + + public void setMetadata(Map metadata) { + if (metadata == null) { + this.metadata.clear(); + return; + } + + this.metadata = metadata; + } + + public void addMetadata(String key, String value) { + this.metadata.put(key, value); + } + + public void setExtFields(Map extFields) { + if (extFields == null) { + this.extFields.clear(); + return; + } + + this.extFields = extFields; + } + + public void addExtFields(String key, Object value) { + this.extFields.put(key, value); + } +} diff --git a/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/RegistryFactory.java b/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/RegistryFactory.java new file mode 100644 index 0000000000..d757781c2b --- /dev/null +++ b/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/RegistryFactory.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.registry; + +import org.apache.eventmesh.spi.EventMeshExtensionFactory; + +import java.util.HashMap; +import java.util.Map; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class RegistryFactory { + + private static final Map META_CACHE = new HashMap<>(16); + + public static RegistryService getInstance(String registryPluginType) { + return META_CACHE.computeIfAbsent(registryPluginType, RegistryFactory::registryBuilder); + } + + private static RegistryService registryBuilder(String registryPluginType) { + RegistryService registryServiceExt = EventMeshExtensionFactory.getExtension(RegistryService.class, registryPluginType); + if (registryServiceExt == null) { + String errorMsg = "can't load the registry plugin, please check."; + log.error(errorMsg); + throw new RuntimeException(errorMsg); + } + log.info("build registry plugin [{}] by type [{}] success", registryServiceExt.getClass().getSimpleName(), + registryPluginType); + return registryServiceExt; + } +} diff --git a/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/RegistryListener.java b/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/RegistryListener.java new file mode 100644 index 0000000000..81445fbe20 --- /dev/null +++ b/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/RegistryListener.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.registry; + +/** + * RegistryListener + */ +public interface RegistryListener { + + void onChange(NotifyEvent event) throws Exception; +} diff --git a/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/RegistryService.java b/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/RegistryService.java new file mode 100644 index 0000000000..63243cd339 --- /dev/null +++ b/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/RegistryService.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.registry; + + +import org.apache.eventmesh.registry.exception.RegistryException; +import org.apache.eventmesh.spi.EventMeshExtensionType; +import org.apache.eventmesh.spi.EventMeshSPI; + +import java.util.List; + +/** + * RegistryService + */ +@EventMeshSPI(eventMeshExtensionType = EventMeshExtensionType.REGISTRY) +public interface RegistryService { + void init() throws RegistryException; + + void shutdown() throws RegistryException; + + void subscribe(RegistryListener registryListener, String serviceName); + + void unsubscribe(RegistryListener registryListener, String serviceName); + + List selectInstances(QueryInstances serverInfo); + + boolean register(RegisterServerInfo registerInfo) throws RegistryException; + + boolean unRegister(RegisterServerInfo registerInfo) throws RegistryException; +} diff --git a/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/exception/RegistryException.java b/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/exception/RegistryException.java new file mode 100644 index 0000000000..1aa61bd246 --- /dev/null +++ b/eventmesh-registry/eventmesh-registry-api/src/main/java/org/apache/eventmesh/registry/exception/RegistryException.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.registry.exception; + +public class RegistryException extends RuntimeException { + public RegistryException(String message) { + super(message); + } + + public RegistryException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/eventmesh-registry/eventmesh-registry-nacos/build.gradle b/eventmesh-registry/eventmesh-registry-nacos/build.gradle new file mode 100644 index 0000000000..d371f23812 --- /dev/null +++ b/eventmesh-registry/eventmesh-registry-nacos/build.gradle @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +dependencies { + implementation "com.alibaba.nacos:nacos-client" + implementation project(":eventmesh-registry:eventmesh-registry-api") + implementation project(":eventmesh-common") + + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' +} \ No newline at end of file diff --git a/eventmesh-registry/eventmesh-registry-nacos/gradle.properties b/eventmesh-registry/eventmesh-registry-nacos/gradle.properties new file mode 100644 index 0000000000..cf067e20bf --- /dev/null +++ b/eventmesh-registry/eventmesh-registry-nacos/gradle.properties @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +pluginType=registryCenter +pluginName=nacos \ No newline at end of file diff --git a/eventmesh-registry/eventmesh-registry-nacos/src/main/java/org/apache/eventmesh/registry/nacos/NacosDiscoveryService.java b/eventmesh-registry/eventmesh-registry-nacos/src/main/java/org/apache/eventmesh/registry/nacos/NacosDiscoveryService.java new file mode 100644 index 0000000000..54d9d8b9d3 --- /dev/null +++ b/eventmesh-registry/eventmesh-registry-nacos/src/main/java/org/apache/eventmesh/registry/nacos/NacosDiscoveryService.java @@ -0,0 +1,312 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.registry.nacos; + +import org.apache.eventmesh.common.config.ConfigService; +import org.apache.eventmesh.registry.NotifyEvent; +import org.apache.eventmesh.registry.QueryInstances; +import org.apache.eventmesh.registry.RegisterServerInfo; +import org.apache.eventmesh.registry.RegistryListener; +import org.apache.eventmesh.registry.RegistryService; +import org.apache.eventmesh.registry.exception.RegistryException; + +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Properties; +import java.util.concurrent.Executor; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import java.util.stream.Collectors; + +import com.alibaba.nacos.api.NacosFactory; +import com.alibaba.nacos.api.PropertyKeyConst; +import com.alibaba.nacos.api.exception.NacosException; +import com.alibaba.nacos.api.naming.NamingService; +import com.alibaba.nacos.api.naming.listener.AbstractEventListener; +import com.alibaba.nacos.api.naming.listener.Event; +import com.alibaba.nacos.api.naming.listener.EventListener; +import com.alibaba.nacos.api.naming.listener.NamingEvent; +import com.alibaba.nacos.api.naming.pojo.Instance; +import com.alibaba.nacos.api.naming.pojo.ServiceInfo; +import com.alibaba.nacos.api.naming.utils.NamingUtils; +import com.alibaba.nacos.client.naming.utils.UtilAndComs; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class NacosDiscoveryService implements RegistryService { + + private final AtomicBoolean initFlag = new AtomicBoolean(false); + + private NacosRegistryConfiguration nacosConf; + + private NamingService namingService; + + private final Map> listeners = new HashMap<>(); + + private static final Executor notifyExecutor = new ThreadPoolExecutor(1, 1, 60L, TimeUnit.SECONDS, + new LinkedBlockingQueue<>(20), r -> { + Thread t = new Thread(r); + t.setName("org.apache.eventmesh.registry.nacos.executor"); + t.setDaemon(true); + return t; + }, new ThreadPoolExecutor.DiscardOldestPolicy() + ); + + private final Lock lock = new ReentrantLock(); + + + @Override + public void init() throws RegistryException { + if (!initFlag.compareAndSet(false, true)) { + return; + } + nacosConf = ConfigService.getInstance().buildConfigInstance(NacosRegistryConfiguration.class); + if (nacosConf == null) { + log.info("nacos registry configuration is null"); + } + Properties properties = buildProperties(); + // registry + try { + this.namingService = NacosFactory.createNamingService(properties); + } catch (NacosException e) { + log.error("[NacosRegistryService][start] error", e); + throw new RegistryException(e.getMessage()); + } + } + + private Properties buildProperties() { + Properties properties = new Properties(); + if (nacosConf == null) { + return properties; + } + properties.setProperty(PropertyKeyConst.SERVER_ADDR, nacosConf.getRegistryAddr()); + properties.setProperty(PropertyKeyConst.USERNAME, nacosConf.getEventMeshRegistryPluginUsername()); + properties.setProperty(PropertyKeyConst.PASSWORD, nacosConf.getEventMeshRegistryPluginPassword()); + + String endpoint = nacosConf.getEndpoint(); + if (Objects.nonNull(endpoint) && endpoint.contains(":")) { + int index = endpoint.indexOf(":"); + properties.put(PropertyKeyConst.ENDPOINT, endpoint.substring(0, index)); + properties.put(PropertyKeyConst.ENDPOINT_PORT, endpoint.substring(index + 1)); + } else { + Optional.ofNullable(endpoint).ifPresent(value -> properties.put(PropertyKeyConst.ENDPOINT, endpoint)); + String endpointPort = nacosConf.getEndpointPort(); + Optional.ofNullable(endpointPort).ifPresent(value -> properties.put(PropertyKeyConst.ENDPOINT_PORT, + endpointPort)); + } + String accessKey = nacosConf.getAccessKey(); + Optional.ofNullable(accessKey).ifPresent(value -> properties.put(PropertyKeyConst.ACCESS_KEY, accessKey)); + String secretKey = nacosConf.getSecretKey(); + Optional.ofNullable(secretKey).ifPresent(value -> properties.put(PropertyKeyConst.SECRET_KEY, secretKey)); + String clusterName = nacosConf.getClusterName(); + Optional.ofNullable(clusterName).ifPresent(value -> properties.put(PropertyKeyConst.CLUSTER_NAME, clusterName)); + String logFileName = nacosConf.getLogFileName(); + Optional.ofNullable(logFileName).ifPresent(value -> properties.put(UtilAndComs.NACOS_NAMING_LOG_NAME, + logFileName)); + String logLevel = nacosConf.getLogLevel(); + Optional.ofNullable(logLevel).ifPresent(value -> properties.put(UtilAndComs.NACOS_NAMING_LOG_LEVEL, logLevel)); + Integer pollingThreadCount = nacosConf.getPollingThreadCount(); + Optional.ofNullable(pollingThreadCount).ifPresent(value -> properties.put(PropertyKeyConst.NAMING_POLLING_THREAD_COUNT, pollingThreadCount)); + String namespace = nacosConf.getNamespace(); + Optional.ofNullable(namespace).ifPresent(value -> properties.put(PropertyKeyConst.NAMESPACE, namespace)); + return properties; + } + + @Override + public void shutdown() throws RegistryException { + if (this.namingService != null) { + try { + namingService.shutDown(); + } catch (NacosException e) { + log.warn("shutdown nacos naming service fail", e); + } + } + } + + @Override + public void subscribe(RegistryListener listener, String serviceName) { + lock.lock(); + try { + ServiceInfo serviceInfo = ServiceInfo.fromKey(serviceName); + Map eventListenerMap = listeners.computeIfAbsent(serviceName, + k -> new HashMap<>()); + if (eventListenerMap.containsKey(listener)) { + log.warn("already use same listener subscribe service name {}", serviceName); + return; + } + EventListener eventListener = new AbstractEventListener() { + @Override + public Executor getExecutor() { + return notifyExecutor; + } + + @Override + public void onEvent(Event event) { + if (!(event instanceof NamingEvent)) { + log.warn("received notify event type isn't not as expected"); + return; + } + try { + NamingEvent namingEvent = (NamingEvent) event; + List instances = namingEvent.getInstances(); + List list = new ArrayList<>(); + if (instances != null) { + for (Instance instance : instances) { + RegisterServerInfo info = new RegisterServerInfo(); + info.setAddress(instance.getIp() + ":" + instance.getPort()); + info.setMetadata(instance.getMetadata()); + info.setHealth(instance.isHealthy()); + info.setServiceName( + ServiceInfo.getKey(NamingUtils.getGroupedName(namingEvent.getServiceName(), + namingEvent.getGroupName()), + namingEvent.getClusters())); + list.add(info); + } + } + listener.onChange(new NotifyEvent(list)); + } catch (Exception e) { + log.warn(""); + } + } + }; + List clusters; + if (serviceInfo.getClusters() == null || serviceInfo.getClusters().isEmpty()) { + clusters = new ArrayList<>(); + } else { + clusters = Arrays.stream(serviceInfo.getClusters().split(",")).collect(Collectors.toList()); + } + namingService.subscribe(serviceInfo.getName(), serviceInfo.getGroupName(), clusters, eventListener); + eventListenerMap.put(listener, eventListener); + } catch (Exception e) { + log.error("subscribe service name {} fail", serviceName, e); + } finally { + lock.unlock(); + } + } + + @Override + public void unsubscribe(RegistryListener registryListener, String serviceName) { + lock.lock(); + try { + ServiceInfo serviceInfo = ServiceInfo.fromKey(serviceName); + Map map = listeners.get(serviceName); + if (map == null) { + return; + } + List clusters; + if (serviceInfo.getClusters() == null || serviceInfo.getClusters().isEmpty()) { + clusters = new ArrayList<>(); + } else { + clusters = Arrays.stream(serviceInfo.getClusters().split(",")).collect(Collectors.toList()); + } + EventListener eventListener = map.get(registryListener); + namingService.unsubscribe(serviceInfo.getName(), serviceInfo.getGroupName(), clusters, eventListener); + map.remove(registryListener); + } catch (Exception e) { + log.error("unsubscribe service name {} fail", serviceName, e); + } finally { + lock.unlock(); + } + } + + @Override + public List selectInstances(QueryInstances queryInstances) { + ArrayList list = new ArrayList<>(); + try { + ServiceInfo serviceInfo = ServiceInfo.fromKey(queryInstances.getServiceName()); + ArrayList clusters = new ArrayList<>(); + if (StringUtils.isNotBlank(serviceInfo.getClusters())) { + clusters.addAll(Arrays.asList(serviceInfo.getClusters().split(","))); + } + List instances = namingService.selectInstances(serviceInfo.getName(), + serviceInfo.getGroupName(), clusters, + queryInstances.isHealth()); + if (instances != null) { + instances.forEach(x -> { + RegisterServerInfo instanceInfo = new RegisterServerInfo(); + instanceInfo.setMetadata(x.getMetadata()); + instanceInfo.setHealth(x.isHealthy()); + instanceInfo.setAddress(x.getIp() + ":" + x.getPort()); + instanceInfo.setServiceName( + ServiceInfo.getKey(NamingUtils.getGroupedName(x.getServiceName(), + serviceInfo.getGroupName()), x.getClusterName())); + list.add(instanceInfo); + }); + } + return list; + } catch (Exception e) { + log.error("select instance by query {} from nacos fail", queryInstances, e); + return list; + } + } + + @Override + public boolean register(RegisterServerInfo eventMeshRegisterInfo) throws RegistryException { + try { + String[] ipPort = eventMeshRegisterInfo.getAddress().split(":"); + if (ipPort.length < 2) { + return false; + } + ServiceInfo serviceInfo = ServiceInfo.fromKey(eventMeshRegisterInfo.getServiceName()); + Instance instance = new Instance(); + instance.setClusterName(serviceInfo.getClusters()); + instance.setEnabled(true); + instance.setEphemeral(true); + instance.setHealthy(eventMeshRegisterInfo.isHealth()); + instance.setWeight(1.0); + instance.setIp(ipPort[0]); + instance.setPort(Integer.parseInt(ipPort[1])); + instance.setMetadata(eventMeshRegisterInfo.getMetadata()); + namingService.registerInstance(serviceInfo.getName(), serviceInfo.getGroupName(), instance); + return true; + } catch (Exception e) { + log.error("register instance service {} fail", eventMeshRegisterInfo, e); + return false; + } + } + + @Override + public boolean unRegister(RegisterServerInfo eventMeshRegisterInfo) throws RegistryException { + try { + String[] ipPort = eventMeshRegisterInfo.getAddress().split(":"); + if (ipPort.length < 2) { + return false; + } + ServiceInfo serviceInfo = ServiceInfo.fromKey(eventMeshRegisterInfo.getServiceName()); + namingService.deregisterInstance(serviceInfo.getName(), serviceInfo.getGroupName(), ipPort[0], + Integer.parseInt(ipPort[1]), + serviceInfo.getClusters()); + return true; + } catch (Exception e) { + log.error("unregister instance service {} fail", eventMeshRegisterInfo, e); + return false; + } + } +} diff --git a/eventmesh-registry/eventmesh-registry-nacos/src/main/java/org/apache/eventmesh/registry/nacos/NacosRegistryConfiguration.java b/eventmesh-registry/eventmesh-registry-nacos/src/main/java/org/apache/eventmesh/registry/nacos/NacosRegistryConfiguration.java new file mode 100644 index 0000000000..7c908c9424 --- /dev/null +++ b/eventmesh-registry/eventmesh-registry-nacos/src/main/java/org/apache/eventmesh/registry/nacos/NacosRegistryConfiguration.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.registry.nacos; + +import org.apache.eventmesh.common.config.CommonConfiguration; +import org.apache.eventmesh.common.config.Config; +import org.apache.eventmesh.common.config.ConfigField; + +import com.alibaba.nacos.api.PropertyKeyConst; +import com.alibaba.nacos.client.naming.utils.UtilAndComs; + +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@Config(prefix = "eventMesh.registry.nacos") +public class NacosRegistryConfiguration extends CommonConfiguration { + + @ConfigField(field = PropertyKeyConst.ENDPOINT) + private String endpoint; + + @ConfigField(field = PropertyKeyConst.ENDPOINT_PORT) + private String endpointPort; + + @ConfigField(field = PropertyKeyConst.ACCESS_KEY) + private String accessKey; + + @ConfigField(field = PropertyKeyConst.SECRET_KEY) + private String secretKey; + + @ConfigField(field = PropertyKeyConst.CLUSTER_NAME) + private String clusterName; + + @ConfigField(field = PropertyKeyConst.NAMESPACE) + private String namespace; + + @ConfigField(field = PropertyKeyConst.NAMING_POLLING_THREAD_COUNT) + private Integer pollingThreadCount = Runtime.getRuntime().availableProcessors() / 2 + 1; + + @ConfigField(field = UtilAndComs.NACOS_NAMING_LOG_NAME) + private String logFileName; + + @ConfigField(field = UtilAndComs.NACOS_NAMING_LOG_LEVEL) + private String logLevel; + +} diff --git a/eventmesh-registry/eventmesh-registry-nacos/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.registry.RegistryService b/eventmesh-registry/eventmesh-registry-nacos/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.registry.RegistryService new file mode 100644 index 0000000000..3301d56e5e --- /dev/null +++ b/eventmesh-registry/eventmesh-registry-nacos/src/main/resources/META-INF/eventmesh/org.apache.eventmesh.registry.RegistryService @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +nacos=org.apache.eventmesh.registry.nacos.NacosDiscoveryService \ No newline at end of file diff --git a/eventmesh-runtime-v2/bin/start-v2.sh b/eventmesh-runtime-v2/bin/start-v2.sh new file mode 100644 index 0000000000..fc67c29d3e --- /dev/null +++ b/eventmesh-runtime-v2/bin/start-v2.sh @@ -0,0 +1,200 @@ +#!/bin/bash +# +# Licensed to Apache Software Foundation (ASF) under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Apache Software Foundation (ASF) licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +#=========================================================================================== +# Java Environment Setting +#=========================================================================================== +set -e +# Server configuration may be inconsistent, add these configurations to avoid garbled code problems +export LANG=en_US.UTF-8 +export LC_CTYPE=en_US.UTF-8 +export LC_ALL=en_US.UTF-8 + +TMP_JAVA_HOME="/customize/your/java/home/here" + +# Detect operating system. +OS=$(uname) + +function is_java8_or_11 { + local _java="$1" + [[ -x "$_java" ]] || return 1 + [[ "$("$_java" -version 2>&1)" =~ 'java version "1.8' || "$("$_java" -version 2>&1)" =~ 'openjdk version "1.8' || "$("$_java" -version 2>&1)" =~ 'java version "11' || "$("$_java" -version 2>&1)" =~ 'openjdk version "11' ]] || return 2 + return 0 +} + +function extract_java_version { + local _java="$1" + local version=$("$_java" -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F '.' '{if ($1 == 1 && $2 == 8) print "8"; else if ($1 == 11) print "11"; else print "unknown"}') + echo "$version" +} + +# 0(not running), 1(is running) +#function is_proxyRunning { +# local _pid="$1" +# local pid=`ps ax | grep -i 'org.apache.eventmesh.runtime.boot.EventMeshStartup' |grep java | grep -v grep | awk '{print $1}'|grep $_pid` +# if [ -z "$pid" ] ; then +# return 0 +# else +# return 1 +# fi +#} + +function get_pid { + local ppid="" + if [ -f ${EVENTMESH_HOME}/bin/pid.file ]; then + ppid=$(cat ${EVENTMESH_HOME}/bin/pid.file) + # If the process does not exist, it indicates that the previous process terminated abnormally. + if [ ! -d /proc/$ppid ]; then + # Remove the residual file. + rm ${EVENTMESH_HOME}/bin/pid.file + echo -e "ERROR\t EventMesh process had already terminated unexpectedly before, please check log output." + ppid="" + fi + else + if [[ $OS =~ Msys ]]; then + # There is a Bug on Msys that may not be able to kill the identified process + ppid=`jps -v | grep -i "org.apache.eventmesh.runtime.boot.RuntimeInstanceStarter" | grep java | grep -v grep | awk -F ' ' {'print $1'}` + elif [[ $OS =~ Darwin ]]; then + # Known problem: grep Java may not be able to accurately identify Java processes + ppid=$(/bin/ps -o user,pid,command | grep "java" | grep -i "org.apache.eventmesh.runtime.boot.RuntimeInstanceStarter" | grep -Ev "^root" |awk -F ' ' {'print $2'}) + else + if [ $DOCKER ]; then + # No need to exclude root user in Docker containers. + ppid=$(ps -C java -o user,pid,command --cols 99999 | grep -w $EVENTMESH_HOME | grep -i "org.apache.eventmesh.runtime.boot.RuntimeInstanceStarter" | awk -F ' ' {'print $2'}) + else + # It is required to identify the process as accurately as possible on Linux. + ppid=$(ps -C java -o user,pid,command --cols 99999 | grep -w $EVENTMESH_HOME | grep -i "org.apache.eventmesh.runtime.boot.RuntimeInstanceStarter" | grep -Ev "^root" | awk -F ' ' {'print $2'}) + fi + fi + fi + echo "$ppid"; +} + +#=========================================================================================== +# Locate Java Executable +#=========================================================================================== + +if [[ -d "$TMP_JAVA_HOME" ]] && is_java8_or_11 "$TMP_JAVA_HOME/bin/java"; then + JAVA="$TMP_JAVA_HOME/bin/java" + JAVA_VERSION=$(extract_java_version "$TMP_JAVA_HOME/bin/java") +elif [[ -d "$JAVA_HOME" ]] && is_java8_or_11 "$JAVA_HOME/bin/java"; then + JAVA="$JAVA_HOME/bin/java" + JAVA_VERSION=$(extract_java_version "$JAVA_HOME/bin/java") +elif is_java8_or_11 "$(which java)"; then + JAVA="$(which java)" + JAVA_VERSION=$(extract_java_version "$(which java)") +else + echo -e "ERROR\t Java 8 or 11 not found, operation abort." + exit 9; +fi + +echo "EventMesh using Java version: $JAVA_VERSION, path: $JAVA" + +EVENTMESH_HOME=$(cd "$(dirname "$0")/.." && pwd) +export EVENTMESH_HOME + +EVENTMESH_LOG_HOME="${EVENTMESH_HOME}/logs" +export EVENTMESH_LOG_HOME + +echo -e "EVENTMESH_HOME : ${EVENTMESH_HOME}\nEVENTMESH_LOG_HOME : ${EVENTMESH_LOG_HOME}" + +function make_logs_dir { + if [ ! -e "${EVENTMESH_LOG_HOME}" ]; then mkdir -p "${EVENTMESH_LOG_HOME}"; fi +} + +error_exit () +{ + echo -e "ERROR\t $1 !!" + exit 1 +} + +export JAVA_HOME + +#=========================================================================================== +# JVM Configuration +#=========================================================================================== +#if [ $1 = "prd" -o $1 = "benchmark" ]; then JAVA_OPT="${JAVA_OPT} -server -Xms2048M -Xmx4096M -Xmn2048m -XX:SurvivorRatio=4" +#elif [ $1 = "sit" ]; then JAVA_OPT="${JAVA_OPT} -server -Xms256M -Xmx512M -Xmn256m -XX:SurvivorRatio=4" +#elif [ $1 = "dev" ]; then JAVA_OPT="${JAVA_OPT} -server -Xms128M -Xmx256M -Xmn128m -XX:SurvivorRatio=4" +#fi + +GC_LOG_FILE="${EVENTMESH_LOG_HOME}/eventmesh_gc_%p.log" + +#JAVA_OPT="${JAVA_OPT} -server -Xms2048M -Xmx4096M -Xmn2048m -XX:SurvivorRatio=4" +JAVA_OPT=`cat ${EVENTMESH_HOME}/conf/server.env | grep APP_START_JVM_OPTION::: | awk -F ':::' {'print $2'}` +JAVA_OPT="${JAVA_OPT} -XX:+UseG1GC -XX:G1HeapRegionSize=16m -XX:G1ReservePercent=25 -XX:InitiatingHeapOccupancyPercent=30 -XX:SoftRefLRUPolicyMSPerMB=0 -XX:SurvivorRatio=8 -XX:MaxGCPauseMillis=50" +JAVA_OPT="${JAVA_OPT} -verbose:gc" +if [[ "$JAVA_VERSION" == "8" ]]; then + # Set JAVA_OPT for Java 8 + JAVA_OPT="${JAVA_OPT} -Xloggc:${GC_LOG_FILE} -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=5 -XX:GCLogFileSize=30m" + JAVA_OPT="${JAVA_OPT} -XX:+PrintGCDetails -XX:+PrintGCDateStamps -XX:+PrintGCApplicationStoppedTime -XX:+PrintAdaptiveSizePolicy" +elif [[ "$JAVA_VERSION" == "11" ]]; then + # Set JAVA_OPT for Java 11 + XLOG_PARAM="time,level,tags:filecount=5,filesize=30m" + JAVA_OPT="${JAVA_OPT} -Xlog:gc*:${GC_LOG_FILE}:${XLOG_PARAM}" + JAVA_OPT="${JAVA_OPT} -Xlog:safepoint:${GC_LOG_FILE}:${XLOG_PARAM} -Xlog:ergo*=debug:${GC_LOG_FILE}:${XLOG_PARAM}" +fi +JAVA_OPT="${JAVA_OPT} -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=${EVENTMESH_LOG_HOME} -XX:ErrorFile=${EVENTMESH_LOG_HOME}/hs_err_%p.log" +JAVA_OPT="${JAVA_OPT} -XX:-OmitStackTraceInFastThrow" +JAVA_OPT="${JAVA_OPT} -XX:+AlwaysPreTouch" +JAVA_OPT="${JAVA_OPT} -XX:MaxDirectMemorySize=8G" +JAVA_OPT="${JAVA_OPT} -XX:-UseLargePages -XX:-UseBiasedLocking" +JAVA_OPT="${JAVA_OPT} -Dio.netty.leakDetectionLevel=advanced" +JAVA_OPT="${JAVA_OPT} -Dio.netty.allocator.type=pooled" +JAVA_OPT="${JAVA_OPT} -Djava.security.egd=file:/dev/./urandom" +JAVA_OPT="${JAVA_OPT} -Dlog4j.configurationFile=${EVENTMESH_HOME}/conf/log4j2.xml" +JAVA_OPT="${JAVA_OPT} -Deventmesh.log.home=${EVENTMESH_LOG_HOME}" +JAVA_OPT="${JAVA_OPT} -DconfPath=${EVENTMESH_HOME}/conf" +JAVA_OPT="${JAVA_OPT} -Dlog4j2.AsyncQueueFullPolicy=Discard" +JAVA_OPT="${JAVA_OPT} -Drocketmq.client.logUseSlf4j=true" +JAVA_OPT="${JAVA_OPT} -DeventMeshPluginDir=${EVENTMESH_HOME}/plugin" + +#if [ -f "pid.file" ]; then +# pid=`cat pid.file` +# if ! is_proxyRunning "$pid"; then +# echo "proxy is running already" +# exit 9; +# else +# echo "err pid$pid, rm pid.file" +# rm pid.file +# fi +#fi + +pid=$(get_pid) +if [[ $pid == "ERROR"* ]]; then + echo -e "${pid}" + exit 9 +fi +if [ -n "$pid" ]; then + echo -e "ERROR\t The server is already running (pid=$pid), there is no need to execute start.sh again." + exit 9 +fi + +make_logs_dir + +echo "Using Java version: $JAVA_VERSION, path: $JAVA" >> ${EVENTMESH_LOG_HOME}/eventmesh.out + +EVENTMESH_MAIN=org.apache.eventmesh.runtime.boot.RuntimeInstanceStarter +if [ $DOCKER ]; then + $JAVA $JAVA_OPT -classpath ${EVENTMESH_HOME}/conf:${EVENTMESH_HOME}/apps/*:${EVENTMESH_HOME}/lib/* $EVENTMESH_MAIN >> ${EVENTMESH_LOG_HOME}/eventmesh.out +else + $JAVA $JAVA_OPT -classpath ${EVENTMESH_HOME}/conf:${EVENTMESH_HOME}/apps/*:${EVENTMESH_HOME}/lib/* $EVENTMESH_MAIN >> ${EVENTMESH_LOG_HOME}/eventmesh.out 2>&1 & +echo $!>${EVENTMESH_HOME}/bin/pid.file +fi +exit 0 diff --git a/eventmesh-runtime-v2/bin/stop-v2.sh b/eventmesh-runtime-v2/bin/stop-v2.sh new file mode 100644 index 0000000000..177ae1e129 --- /dev/null +++ b/eventmesh-runtime-v2/bin/stop-v2.sh @@ -0,0 +1,88 @@ +#!/bin/bash +# +# Licensed to Apache Software Foundation (ASF) under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Apache Software Foundation (ASF) licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Detect operating system +OS=$(uname) + +EVENTMESH_HOME=`cd $(dirname $0)/.. && pwd` + +export EVENTMESH_HOME + +function get_pid { + local ppid="" + if [ -f ${EVENTMESH_HOME}/bin/pid.file ]; then + ppid=$(cat ${EVENTMESH_HOME}/bin/pid.file) + # If the process does not exist, it indicates that the previous process terminated abnormally. + if [ ! -d /proc/$ppid ]; then + # Remove the residual file and return an error status. + rm ${EVENTMESH_HOME}/bin/pid.file + echo -e "ERROR\t EventMesh process had already terminated unexpectedly before, please check log output." + ppid="" + fi + else + if [[ $OS =~ Msys ]]; then + # There is a Bug on Msys that may not be able to kill the identified process + ppid=`jps -v | grep -i "org.apache.eventmesh.runtime.boot.RuntimeInstanceStarter" | grep java | grep -v grep | awk -F ' ' {'print $1'}` + elif [[ $OS =~ Darwin ]]; then + # Known problem: grep Java may not be able to accurately identify Java processes + ppid=$(/bin/ps -o user,pid,command | grep "java" | grep -i "org.apache.eventmesh.runtime.boot.RuntimeInstanceStarter" | grep -Ev "^root" |awk -F ' ' {'print $2'}) + else + # It is required to identify the process as accurately as possible on Linux + ppid=$(ps -C java -o user,pid,command --cols 99999 | grep -w $EVENTMESH_HOME | grep -i "org.apache.eventmesh.runtime.boot.RuntimeInstanceStarter" | grep -Ev "^root" |awk -F ' ' {'print $2'}) + fi + fi + echo "$ppid"; +} + +pid=$(get_pid) +if [[ $pid == "ERROR"* ]]; then + echo -e "${pid}" + exit 9 +fi +if [ -z "$pid" ];then + echo -e "ERROR\t No EventMesh server running." + exit 9 +fi + +kill ${pid} +echo "Send shutdown request to EventMesh(${pid}) OK" + +[[ $OS =~ Msys ]] && PS_PARAM=" -W " +stop_timeout=60 +for no in $(seq 1 $stop_timeout); do + if ps $PS_PARAM -p "$pid" 2>&1 > /dev/null; then + if [ $no -lt $stop_timeout ]; then + echo "[$no] server shutting down ..." + sleep 1 + continue + fi + + echo "shutdown server timeout, kill process: $pid" + kill -9 $pid; sleep 1; break; + echo "`date +'%Y-%m-%-d %H:%M:%S'` , pid : [$pid] , error message : abnormal shutdown which can not be closed within 60s" > ../logs/shutdown.error + else + echo "shutdown server ok!"; break; + fi +done + +if [ -f "pid.file" ]; then + rm pid.file +fi + + diff --git a/eventmesh-runtime-v2/build.gradle b/eventmesh-runtime-v2/build.gradle new file mode 100644 index 0000000000..74b9759b10 --- /dev/null +++ b/eventmesh-runtime-v2/build.gradle @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +plugins { + id 'java' +} + +group 'org.apache.eventmesh' +version '1.10.0-release' + +repositories { + mavenCentral() +} + +dependencies { + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' + + api project (":eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-api") + api project (":eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-admin") + implementation project(":eventmesh-openconnect:eventmesh-openconnect-java") + implementation project(":eventmesh-common") + implementation project(":eventmesh-connectors:eventmesh-connector-canal") + implementation project(":eventmesh-connectors:eventmesh-connector-http") + implementation project(":eventmesh-function:eventmesh-function-api") + implementation project(":eventmesh-function:eventmesh-function-filter") + implementation project(":eventmesh-function:eventmesh-function-transformer") + implementation project(":eventmesh-meta:eventmesh-meta-api") + implementation project(":eventmesh-meta:eventmesh-meta-nacos") + implementation project(":eventmesh-registry:eventmesh-registry-api") + implementation project(":eventmesh-registry:eventmesh-registry-nacos") + implementation project(":eventmesh-storage-plugin:eventmesh-storage-api") + implementation project(":eventmesh-storage-plugin:eventmesh-storage-standalone") + + implementation "io.grpc:grpc-core" + implementation "io.grpc:grpc-protobuf" + implementation "io.grpc:grpc-stub" + implementation "io.grpc:grpc-netty" + implementation "io.grpc:grpc-netty-shaded" +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/Runtime.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/Runtime.java new file mode 100644 index 0000000000..608ef96da7 --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/Runtime.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime; + +/** + * Runtime + */ +public interface Runtime { + + void init() throws Exception; + + void start() throws Exception; + + void stop() throws Exception; + +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/RuntimeFactory.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/RuntimeFactory.java new file mode 100644 index 0000000000..ed273030d9 --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/RuntimeFactory.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime; + +/** + * RuntimeFactory + */ +public interface RuntimeFactory extends AutoCloseable { + + void init() throws Exception; + + Runtime createRuntime(RuntimeInstanceConfig runtimeInstanceConfig); + +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/RuntimeInstanceConfig.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/RuntimeInstanceConfig.java new file mode 100644 index 0000000000..caa5330fe3 --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/RuntimeInstanceConfig.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime; + +import org.apache.eventmesh.common.config.Config; +import org.apache.eventmesh.common.enums.ComponentType; + +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@Config(path = "classPath://runtime.yaml") +public class RuntimeInstanceConfig { + + private boolean registryEnabled; + + private String registryServerAddr; + + private String registryPluginType; + + private String storagePluginType; + + private String adminServiceName; + + private String adminServiceAddr; + + private ComponentType componentType; + + private String runtimeInstanceId; + + private String runtimeInstanceName; + + private String runtimeInstanceDesc; + + private String runtimeInstanceVersion; + + private String runtimeInstanceConfig; + + private String runtimeInstanceStatus; + +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/boot/RuntimeInstance.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/boot/RuntimeInstance.java new file mode 100644 index 0000000000..beb1d1eedc --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/boot/RuntimeInstance.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.boot; + +import org.apache.eventmesh.registry.QueryInstances; +import org.apache.eventmesh.registry.RegisterServerInfo; +import org.apache.eventmesh.registry.RegistryFactory; +import org.apache.eventmesh.registry.RegistryService; +import org.apache.eventmesh.runtime.Runtime; +import org.apache.eventmesh.runtime.RuntimeFactory; +import org.apache.eventmesh.runtime.RuntimeInstanceConfig; +import org.apache.eventmesh.runtime.connector.ConnectorRuntimeFactory; +import org.apache.eventmesh.runtime.function.FunctionRuntimeFactory; +import org.apache.eventmesh.runtime.mesh.MeshRuntimeFactory; + +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class RuntimeInstance { + + private String adminServiceAddr; + + private Map adminServerInfoMap = new HashMap<>(); + + private RegistryService registryService; + + private Runtime runtime; + + private RuntimeFactory runtimeFactory; + + private final RuntimeInstanceConfig runtimeInstanceConfig; + + private volatile boolean isStarted = false; + + public RuntimeInstance(RuntimeInstanceConfig runtimeInstanceConfig) { + this.runtimeInstanceConfig = runtimeInstanceConfig; + if (runtimeInstanceConfig.isRegistryEnabled()) { + this.registryService = RegistryFactory.getInstance(runtimeInstanceConfig.getRegistryPluginType()); + } + } + + public void init() throws Exception { + if (registryService != null) { + registryService.init(); + QueryInstances queryInstances = new QueryInstances(); + queryInstances.setServiceName(runtimeInstanceConfig.getAdminServiceName()); + queryInstances.setHealth(true); + List adminServerRegisterInfoList = registryService.selectInstances(queryInstances); + if (!adminServerRegisterInfoList.isEmpty()) { + adminServiceAddr = getRandomAdminServerAddr(adminServerRegisterInfoList); + } else { + throw new RuntimeException("admin server address is empty, please check"); + } + // use registry adminServiceAddr value replace config + runtimeInstanceConfig.setAdminServiceAddr(adminServiceAddr); + } else { + adminServiceAddr = runtimeInstanceConfig.getAdminServiceAddr(); + } + + runtimeFactory = initRuntimeFactory(runtimeInstanceConfig); + runtime = runtimeFactory.createRuntime(runtimeInstanceConfig); + runtime.init(); + } + + public void start() throws Exception { + if (StringUtils.isBlank(adminServiceAddr)) { + throw new RuntimeException("admin server address is empty, please check"); + } else { + if (registryService != null) { + registryService.subscribe((event) -> { + log.info("runtime receive registry event: {}", event); + List registerServerInfoList = event.getInstances(); + Map registerServerInfoMap = new HashMap<>(); + for (RegisterServerInfo registerServerInfo : registerServerInfoList) { + registerServerInfoMap.put(registerServerInfo.getAddress(), registerServerInfo); + } + if (!registerServerInfoMap.isEmpty()) { + adminServerInfoMap = registerServerInfoMap; + updateAdminServerAddr(); + } + }, runtimeInstanceConfig.getAdminServiceName()); + } + runtime.start(); + isStarted = true; + } + } + + public void shutdown() throws Exception { + runtime.stop(); + } + + private void updateAdminServerAddr() throws Exception { + if (isStarted) { + if (!adminServerInfoMap.containsKey(adminServiceAddr)) { + adminServiceAddr = getRandomAdminServerAddr(adminServerInfoMap); + log.info("admin server address changed to: {}", adminServiceAddr); + shutdown(); + start(); + } + } else { + adminServiceAddr = getRandomAdminServerAddr(adminServerInfoMap); + } + } + + private String getRandomAdminServerAddr(Map adminServerInfoMap) { + ArrayList addresses = new ArrayList<>(adminServerInfoMap.keySet()); + Random random = new Random(); + int randomIndex = random.nextInt(addresses.size()); + return addresses.get(randomIndex); + } + + private String getRandomAdminServerAddr(List adminServerRegisterInfoList) { + Random random = new Random(); + int randomIndex = random.nextInt(adminServerRegisterInfoList.size()); + return adminServerRegisterInfoList.get(randomIndex).getAddress(); + } + + private RuntimeFactory initRuntimeFactory(RuntimeInstanceConfig runtimeInstanceConfig) { + switch (runtimeInstanceConfig.getComponentType()) { + case CONNECTOR: + return new ConnectorRuntimeFactory(); + case FUNCTION: + return new FunctionRuntimeFactory(); + case MESH: + return new MeshRuntimeFactory(); + default: + throw new RuntimeException("unsupported runtime type: " + runtimeInstanceConfig.getComponentType()); + } + } + +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/boot/RuntimeInstanceStarter.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/boot/RuntimeInstanceStarter.java new file mode 100644 index 0000000000..0881521879 --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/boot/RuntimeInstanceStarter.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.boot; + +import org.apache.eventmesh.common.config.ConfigService; +import org.apache.eventmesh.runtime.RuntimeInstanceConfig; +import org.apache.eventmesh.runtime.util.BannerUtil; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class RuntimeInstanceStarter { + + public static void main(String[] args) { + try { + RuntimeInstanceConfig runtimeInstanceConfig = ConfigService.getInstance().buildConfigInstance(RuntimeInstanceConfig.class); + RuntimeInstance runtimeInstance = new RuntimeInstance(runtimeInstanceConfig); + BannerUtil.generateBanner(); + runtimeInstance.init(); + runtimeInstance.start(); + + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try { + log.info("runtime shutting down hook begin."); + long start = System.currentTimeMillis(); + runtimeInstance.shutdown(); + long end = System.currentTimeMillis(); + log.info("runtime shutdown cost {}ms", end - start); + } catch (Exception e) { + log.error("exception when shutdown {}", e.getMessage(), e); + } + })); + } catch (Throwable e) { + log.error("runtime start fail {}.", e.getMessage(), e); + System.exit(-1); + } + + } +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/connector/ConnectorRuntime.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/connector/ConnectorRuntime.java new file mode 100644 index 0000000000..92e78256ec --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/connector/ConnectorRuntime.java @@ -0,0 +1,546 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.connector; + +import org.apache.eventmesh.api.consumer.Consumer; +import org.apache.eventmesh.api.factory.StoragePluginFactory; +import org.apache.eventmesh.api.producer.Producer; +import org.apache.eventmesh.common.ThreadPoolFactory; +import org.apache.eventmesh.common.config.ConfigService; +import org.apache.eventmesh.common.config.connector.SinkConfig; +import org.apache.eventmesh.common.config.connector.SourceConfig; +import org.apache.eventmesh.common.config.connector.offset.OffsetStorageConfig; +import org.apache.eventmesh.common.enums.ConnectorStage; +import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc; +import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc.AdminServiceBlockingStub; +import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc.AdminServiceStub; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Payload; +import org.apache.eventmesh.common.remote.JobState; +import org.apache.eventmesh.common.remote.request.FetchJobRequest; +import org.apache.eventmesh.common.remote.response.FetchJobResponse; +import org.apache.eventmesh.common.utils.IPUtils; +import org.apache.eventmesh.common.utils.JsonUtils; +import org.apache.eventmesh.openconnect.api.ConnectorCreateService; +import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; +import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; +import org.apache.eventmesh.openconnect.api.factory.ConnectorPluginFactory; +import org.apache.eventmesh.openconnect.api.sink.Sink; +import org.apache.eventmesh.openconnect.api.source.Source; +import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendExceptionContext; +import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendMessageCallback; +import org.apache.eventmesh.openconnect.offsetmgmt.api.callback.SendResult; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.RecordOffsetManagement; +import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.DefaultOffsetManagementServiceImpl; +import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetManagementService; +import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetStorageReaderImpl; +import org.apache.eventmesh.openconnect.offsetmgmt.api.storage.OffsetStorageWriterImpl; +import org.apache.eventmesh.openconnect.util.ConfigUtil; +import org.apache.eventmesh.runtime.Runtime; +import org.apache.eventmesh.runtime.RuntimeInstanceConfig; +import org.apache.eventmesh.runtime.service.health.HealthService; +import org.apache.eventmesh.runtime.service.monitor.MonitorService; +import org.apache.eventmesh.runtime.service.monitor.SinkMonitor; +import org.apache.eventmesh.runtime.service.monitor.SourceMonitor; +import org.apache.eventmesh.runtime.service.status.StatusService; +import org.apache.eventmesh.runtime.service.verify.VerifyService; +import org.apache.eventmesh.runtime.util.RuntimeUtils; +import org.apache.eventmesh.spi.EventMeshExtensionFactory; + +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; + +import com.google.protobuf.Any; +import com.google.protobuf.UnsafeByteOperations; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ConnectorRuntime implements Runtime { + + private RuntimeInstanceConfig runtimeInstanceConfig; + + private ConnectorRuntimeConfig connectorRuntimeConfig; + + private ManagedChannel channel; + + private AdminServiceStub adminServiceStub; + + private AdminServiceBlockingStub adminServiceBlockingStub; + + private Source sourceConnector; + + private Sink sinkConnector; + + private OffsetStorageWriterImpl offsetStorageWriter; + + private OffsetStorageReaderImpl offsetStorageReader; + + private OffsetManagementService offsetManagementService; + + private RecordOffsetManagement offsetManagement; + + private volatile RecordOffsetManagement.CommittableOffsets committableOffsets; + + private Producer producer; + + private Consumer consumer; + + private final ExecutorService sourceService = ThreadPoolFactory.createSingleExecutor("eventMesh-sourceService"); + + private final ExecutorService sinkService = ThreadPoolFactory.createSingleExecutor("eventMesh-sinkService"); + + + private final BlockingQueue queue; + + private volatile boolean isRunning = false; + + private volatile boolean isFailed = false; + + public static final String CALLBACK_EXTENSION = "callBackExtension"; + + private String adminServerAddr; + + private HealthService healthService; + + private MonitorService monitorService; + + private SourceMonitor sourceMonitor; + + private SinkMonitor sinkMonitor; + + private VerifyService verifyService; + + private StatusService statusService; + + + public ConnectorRuntime(RuntimeInstanceConfig runtimeInstanceConfig) { + this.runtimeInstanceConfig = runtimeInstanceConfig; + this.queue = new LinkedBlockingQueue<>(1000); + } + + @Override + public void init() throws Exception { + + initAdminService(); + + initStorageService(); + + initStatusService(); + + initConnectorService(); + + initMonitorService(); + + initHealthService(); + + initVerfiyService(); + + } + + private void initAdminService() { + adminServerAddr = RuntimeUtils.getRandomAdminServerAddr(runtimeInstanceConfig.getAdminServiceAddr()); + // create gRPC channel + channel = ManagedChannelBuilder.forTarget(adminServerAddr) + .usePlaintext() + .enableRetry() + .maxRetryAttempts(3) + .build(); + + adminServiceStub = AdminServiceGrpc.newStub(channel).withWaitForReady(); + + adminServiceBlockingStub = AdminServiceGrpc.newBlockingStub(channel).withWaitForReady(); + + } + + private void initStorageService() { + // TODO: init producer & consumer + producer = StoragePluginFactory.getMeshMQProducer(runtimeInstanceConfig.getStoragePluginType()); + + consumer = StoragePluginFactory.getMeshMQPushConsumer(runtimeInstanceConfig.getStoragePluginType()); + + } + + private void initStatusService() { + statusService = new StatusService(adminServiceStub, adminServiceBlockingStub); + } + + private void initConnectorService() throws Exception { + + connectorRuntimeConfig = ConfigService.getInstance().buildConfigInstance(ConnectorRuntimeConfig.class); + + FetchJobResponse jobResponse = fetchJobConfig(); + log.info("fetch job config from admin server: {}", JsonUtils.toJSONString(jobResponse)); + + if (jobResponse == null) { + isFailed = true; + stop(); + throw new RuntimeException("fetch job config fail"); + } + + connectorRuntimeConfig.setSourceConnectorType(jobResponse.getTransportType().getSrc().getName()); + connectorRuntimeConfig.setSourceConnectorDesc(jobResponse.getConnectorConfig().getSourceConnectorDesc()); + connectorRuntimeConfig.setSourceConnectorConfig(jobResponse.getConnectorConfig().getSourceConnectorConfig()); + + connectorRuntimeConfig.setSinkConnectorType(jobResponse.getTransportType().getDst().getName()); + connectorRuntimeConfig.setSinkConnectorDesc(jobResponse.getConnectorConfig().getSinkConnectorDesc()); + connectorRuntimeConfig.setSinkConnectorConfig(jobResponse.getConnectorConfig().getSinkConnectorConfig()); + + // spi load offsetMgmtService + this.offsetManagement = new RecordOffsetManagement(); + this.committableOffsets = RecordOffsetManagement.CommittableOffsets.EMPTY; + OffsetStorageConfig offsetStorageConfig = new OffsetStorageConfig(); + offsetStorageConfig.setOffsetStorageAddr(connectorRuntimeConfig.getRuntimeConfig().get("offsetStorageAddr").toString()); + offsetStorageConfig.setOffsetStorageType(connectorRuntimeConfig.getRuntimeConfig().get("offsetStoragePluginType").toString()); + offsetStorageConfig.setDataSourceType(jobResponse.getTransportType().getSrc()); + offsetStorageConfig.setDataSinkType(jobResponse.getTransportType().getDst()); + Map offsetStorageExtensions = new HashMap<>(); + offsetStorageExtensions.put("jobId", connectorRuntimeConfig.getJobID()); + offsetStorageConfig.setExtensions(offsetStorageExtensions); + + this.offsetManagementService = Optional.ofNullable(offsetStorageConfig).map(OffsetStorageConfig::getOffsetStorageType) + .map(storageType -> EventMeshExtensionFactory.getExtension(OffsetManagementService.class, storageType)) + .orElse(new DefaultOffsetManagementServiceImpl()); + this.offsetManagementService.initialize(offsetStorageConfig); + this.offsetStorageWriter = new OffsetStorageWriterImpl(offsetManagementService); + this.offsetStorageReader = new OffsetStorageReaderImpl(offsetManagementService); + + ConnectorCreateService sourceConnectorCreateService = + ConnectorPluginFactory.createConnector(connectorRuntimeConfig.getSourceConnectorType() + "-Source"); + sourceConnector = (Source) sourceConnectorCreateService.create(); + + SourceConfig sourceConfig = (SourceConfig) ConfigUtil.parse(connectorRuntimeConfig.getSourceConnectorConfig(), sourceConnector.configClass()); + SourceConnectorContext sourceConnectorContext = new SourceConnectorContext(); + sourceConnectorContext.setSourceConfig(sourceConfig); + sourceConnectorContext.setRuntimeConfig(connectorRuntimeConfig.getRuntimeConfig()); + sourceConnectorContext.setJobType(jobResponse.getType()); + sourceConnectorContext.setOffsetStorageReader(offsetStorageReader); + if (CollectionUtils.isNotEmpty(jobResponse.getPosition())) { + sourceConnectorContext.setRecordPositionList(jobResponse.getPosition()); + } + sourceConnector.init(sourceConnectorContext); + + ConnectorCreateService sinkConnectorCreateService = + ConnectorPluginFactory.createConnector(connectorRuntimeConfig.getSinkConnectorType() + "-Sink"); + sinkConnector = (Sink) sinkConnectorCreateService.create(); + + SinkConfig sinkConfig = (SinkConfig) ConfigUtil.parse(connectorRuntimeConfig.getSinkConnectorConfig(), sinkConnector.configClass()); + SinkConnectorContext sinkConnectorContext = new SinkConnectorContext(); + sinkConnectorContext.setSinkConfig(sinkConfig); + sinkConnectorContext.setRuntimeConfig(connectorRuntimeConfig.getRuntimeConfig()); + sinkConnectorContext.setJobType(jobResponse.getType()); + sinkConnector.init(sinkConnectorContext); + + statusService.reportJobStatus(connectorRuntimeConfig.getJobID(), JobState.INIT); + + } + + private FetchJobResponse fetchJobConfig() { + String jobId = connectorRuntimeConfig.getJobID(); + FetchJobRequest jobRequest = new FetchJobRequest(); + jobRequest.setJobID(jobId); + + Metadata metadata = Metadata.newBuilder().setType(FetchJobRequest.class.getSimpleName()).build(); + + Payload request = Payload.newBuilder().setMetadata(metadata) + .setBody(Any.newBuilder().setValue(UnsafeByteOperations.unsafeWrap(Objects.requireNonNull(JsonUtils.toJSONBytes(jobRequest)))).build()) + .build(); + Payload response = adminServiceBlockingStub.invoke(request); + if (response.getMetadata().getType().equals(FetchJobResponse.class.getSimpleName())) { + return JsonUtils.parseObject(response.getBody().getValue().toStringUtf8(), FetchJobResponse.class); + } + return null; + } + + private void initMonitorService() { + monitorService = new MonitorService(adminServiceStub, adminServiceBlockingStub); + sourceMonitor = new SourceMonitor(connectorRuntimeConfig.getTaskID(), connectorRuntimeConfig.getJobID(), IPUtils.getLocalAddress()); + monitorService.registerMonitor(sourceMonitor); + sinkMonitor = new SinkMonitor(connectorRuntimeConfig.getTaskID(), connectorRuntimeConfig.getJobID(), IPUtils.getLocalAddress()); + monitorService.registerMonitor(sinkMonitor); + } + + private void initHealthService() { + healthService = new HealthService(adminServiceStub, adminServiceBlockingStub, connectorRuntimeConfig); + } + + private void initVerfiyService() { + verifyService = new VerifyService(adminServiceStub, adminServiceBlockingStub, connectorRuntimeConfig); + } + + @Override + public void start() throws Exception { + // start offsetMgmtService + offsetManagementService.start(); + + monitorService.start(); + + healthService.start(); + + isRunning = true; + // start sinkService + sinkService.execute(() -> { + try { + startSinkConnector(); + } catch (Exception e) { + isFailed = true; + log.error("sink connector start fail", e.getStackTrace()); + try { + this.stop(); + } catch (Exception ex) { + log.error("Failed to stop after exception", ex); + } + } finally { + System.exit(-1); + } + }); + // start sourceService + sourceService.execute(() -> { + try { + startSourceConnector(); + } catch (Exception e) { + isFailed = true; + log.error("source connector start fail", e); + try { + this.stop(); + } catch (Exception ex) { + log.error("Failed to stop after exception", ex); + } + } finally { + System.exit(-1); + } + }); + + statusService.reportJobStatus(connectorRuntimeConfig.getJobID(), JobState.RUNNING); + } + + @Override + public void stop() throws Exception { + log.info("ConnectorRuntime start stop"); + isRunning = false; + if (isFailed) { + statusService.reportJobStatus(connectorRuntimeConfig.getJobID(), JobState.FAIL); + } else { + statusService.reportJobStatus(connectorRuntimeConfig.getJobID(), JobState.COMPLETE); + } + sourceConnector.stop(); + sinkConnector.stop(); + monitorService.stop(); + healthService.stop(); + sourceService.shutdown(); + sinkService.shutdown(); + verifyService.stop(); + statusService.stop(); + if (channel != null && !channel.isShutdown()) { + channel.shutdown().awaitTermination(5, TimeUnit.SECONDS); + } + log.info("ConnectorRuntime stopped"); + } + + private void startSourceConnector() throws Exception { + sourceConnector.start(); + while (isRunning) { + long sourceStartTime = System.currentTimeMillis(); + List connectorRecordList = sourceConnector.poll(); + long sinkStartTime = System.currentTimeMillis(); + // TODO: use producer pub record to storage replace below + if (connectorRecordList != null && !connectorRecordList.isEmpty()) { + for (ConnectRecord record : connectorRecordList) { + // check recordUniqueId + if (record.getExtensions() == null || !record.getExtensions().containsKey("recordUniqueId")) { + record.addExtension("recordUniqueId", record.getRecordId()); + } + + // set a callback for this record + // if used the memory storage callback will be triggered after sink put success + record.setCallback(new SendMessageCallback() { + @Override + public void onSuccess(SendResult result) { + log.debug("send record to sink callback success, record: {}", record); + long sinkEndTime = System.currentTimeMillis(); + sinkMonitor.recordProcess(sinkEndTime - sinkStartTime); + // commit record + sourceConnector.commit(record); + if (record.getPosition() != null) { + Optional submittedRecordPosition = prepareToUpdateRecordOffset(record); + submittedRecordPosition.ifPresent(RecordOffsetManagement.SubmittedPosition::ack); + log.debug("start wait all messages to commit"); + offsetManagement.awaitAllMessages(5000, TimeUnit.MILLISECONDS); + // update & commit offset + updateCommittableOffsets(); + commitOffsets(); + } + Optional callback = + Optional.ofNullable(record.getExtensionObj(CALLBACK_EXTENSION)).map(v -> (SendMessageCallback) v); + callback.ifPresent(cb -> cb.onSuccess(convertToSendResult(record))); + } + + @Override + public void onException(SendExceptionContext sendExceptionContext) { + isFailed = true; + // handle exception + sourceConnector.onException(record); + log.error("send record to sink callback exception, process will shut down, record: {}", record, + sendExceptionContext.getCause()); + try { + stop(); + } catch (Exception e) { + log.error("Failed to stop after exception", e); + } + } + }); + + queue.put(record); + long sourceEndTime = System.currentTimeMillis(); + sourceMonitor.recordProcess(sourceEndTime - sourceStartTime); + + // if enabled incremental data reporting consistency check + if (connectorRuntimeConfig.enableIncrementalDataConsistencyCheck) { + verifyService.reportVerifyRequest(record, ConnectorStage.SOURCE); + } + + } + } + } + } + + private SendResult convertToSendResult(ConnectRecord record) { + SendResult result = new SendResult(); + result.setMessageId(record.getRecordId()); + if (StringUtils.isNotEmpty(record.getExtension("topic"))) { + result.setTopic(record.getExtension("topic")); + } + return result; + } + + + public Optional prepareToUpdateRecordOffset(ConnectRecord record) { + return Optional.of(this.offsetManagement.submitRecord(record.getPosition())); + } + + public void updateCommittableOffsets() { + RecordOffsetManagement.CommittableOffsets newOffsets = offsetManagement.committableOffsets(); + synchronized (this) { + this.committableOffsets = this.committableOffsets.updatedWith(newOffsets); + } + } + + public boolean commitOffsets() { + log.info("Start Committing offsets"); + + long timeout = System.currentTimeMillis() + 5000L; + + RecordOffsetManagement.CommittableOffsets offsetsToCommit; + synchronized (this) { + offsetsToCommit = this.committableOffsets; + this.committableOffsets = RecordOffsetManagement.CommittableOffsets.EMPTY; + } + + if (committableOffsets.isEmpty()) { + log.debug( + "Either no records were produced since the last offset commit, " + + "or every record has been filtered out by a transformation or dropped due to transformation or conversion errors."); + // We continue with the offset commit process here instead of simply returning immediately + // in order to invoke SourceTask::commit and record metrics for a successful offset commit + } else { + log.info("{} Committing offsets for {} acknowledged messages", this, committableOffsets.numCommittableMessages()); + if (committableOffsets.hasPending()) { + log.debug( + "{} There are currently {} pending messages spread across {} source partitions whose offsets will not be committed." + + " The source partition with the most pending messages is {}, with {} pending messages", + this, + committableOffsets.numUncommittableMessages(), committableOffsets.numDeques(), committableOffsets.largestDequePartition(), + committableOffsets.largestDequeSize()); + } else { + log.debug( + "{} There are currently no pending messages for this offset commit; " + + "all messages dispatched to the task's producer since the last commit have been acknowledged", + this); + } + } + + // write offset to memory + offsetsToCommit.offsets().forEach(offsetStorageWriter::writeOffset); + + // begin flush + if (!offsetStorageWriter.beginFlush()) { + return true; + } + + // using offsetManagementService to persist offset + Future flushFuture = offsetStorageWriter.doFlush(); + try { + flushFuture.get(Math.max(timeout - System.currentTimeMillis(), 0), TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + log.warn("{} Flush of offsets interrupted, cancelling", this); + offsetStorageWriter.cancelFlush(); + return false; + } catch (ExecutionException e) { + log.error("{} Flush of offsets threw an unexpected exception: ", this, e); + offsetStorageWriter.cancelFlush(); + return false; + } catch (TimeoutException e) { + log.error("{} Timed out waiting to flush offsets to storage; will try again on next flush interval with latest offsets", this); + offsetStorageWriter.cancelFlush(); + return false; + } + return true; + } + + private void startSinkConnector() throws Exception { + sinkConnector.start(); + while (isRunning) { + // TODO: use consumer sub from storage to replace below + ConnectRecord connectRecord = null; + try { + connectRecord = queue.poll(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.error("poll connect record error", e); + } + if (connectRecord == null) { + continue; + } + List connectRecordList = new ArrayList<>(); + connectRecordList.add(connectRecord); + sinkConnector.put(connectRecordList); + // if enabled incremental data reporting consistency check + if (connectorRuntimeConfig.enableIncrementalDataConsistencyCheck) { + verifyService.reportVerifyRequest(connectRecord, ConnectorStage.SINK); + } + } + } +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/connector/ConnectorRuntimeConfig.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/connector/ConnectorRuntimeConfig.java new file mode 100644 index 0000000000..ab6fc3aaf5 --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/connector/ConnectorRuntimeConfig.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.connector; + +import org.apache.eventmesh.common.config.Config; + +import java.util.Map; + +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +@Config(path = "classPath://connector.yaml") +public class ConnectorRuntimeConfig { + + private String connectorRuntimeInstanceId; + + private String taskID; + + private String jobID; + + private String region; + + private Map runtimeConfig; + + private String sourceConnectorType; + + private String sourceConnectorDesc; + + private Map sourceConnectorConfig; + + private String sinkConnectorType; + + private String sinkConnectorDesc; + + private Map sinkConnectorConfig; + + public boolean enableIncrementalDataConsistencyCheck = true; + +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/connector/ConnectorRuntimeFactory.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/connector/ConnectorRuntimeFactory.java new file mode 100644 index 0000000000..d1ec2ff4e9 --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/connector/ConnectorRuntimeFactory.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.connector; + +import org.apache.eventmesh.runtime.Runtime; +import org.apache.eventmesh.runtime.RuntimeFactory; +import org.apache.eventmesh.runtime.RuntimeInstanceConfig; + +public class ConnectorRuntimeFactory implements RuntimeFactory { + + @Override + public void init() throws Exception { + + } + + @Override + public Runtime createRuntime(RuntimeInstanceConfig runtimeInstanceConfig) { + return new ConnectorRuntime(runtimeInstanceConfig); + } + + @Override + public void close() throws Exception { + + } +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/function/FunctionRuntime.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/function/FunctionRuntime.java new file mode 100644 index 0000000000..4a68001909 --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/function/FunctionRuntime.java @@ -0,0 +1,503 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.function; + +import org.apache.eventmesh.common.ThreadPoolFactory; +import org.apache.eventmesh.common.config.ConfigService; +import org.apache.eventmesh.common.config.connector.SinkConfig; +import org.apache.eventmesh.common.config.connector.SourceConfig; +import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc; +import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc.AdminServiceBlockingStub; +import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc.AdminServiceStub; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Payload; +import org.apache.eventmesh.common.remote.JobState; +import org.apache.eventmesh.common.remote.exception.ErrorCode; +import org.apache.eventmesh.common.remote.job.JobType; +import org.apache.eventmesh.common.remote.request.FetchJobRequest; +import org.apache.eventmesh.common.remote.request.ReportHeartBeatRequest; +import org.apache.eventmesh.common.remote.request.ReportJobRequest; +import org.apache.eventmesh.common.remote.response.FetchJobResponse; +import org.apache.eventmesh.common.utils.IPUtils; +import org.apache.eventmesh.common.utils.JsonUtils; +import org.apache.eventmesh.function.api.AbstractEventMeshFunctionChain; +import org.apache.eventmesh.function.api.EventMeshFunction; +import org.apache.eventmesh.function.filter.pattern.Pattern; +import org.apache.eventmesh.function.filter.patternbuild.PatternBuilder; +import org.apache.eventmesh.function.transformer.Transformer; +import org.apache.eventmesh.function.transformer.TransformerBuilder; +import org.apache.eventmesh.function.transformer.TransformerType; +import org.apache.eventmesh.openconnect.api.ConnectorCreateService; +import org.apache.eventmesh.openconnect.api.connector.SinkConnectorContext; +import org.apache.eventmesh.openconnect.api.connector.SourceConnectorContext; +import org.apache.eventmesh.openconnect.api.factory.ConnectorPluginFactory; +import org.apache.eventmesh.openconnect.api.sink.Sink; +import org.apache.eventmesh.openconnect.api.source.Source; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; +import org.apache.eventmesh.openconnect.util.ConfigUtil; +import org.apache.eventmesh.runtime.Runtime; +import org.apache.eventmesh.runtime.RuntimeInstanceConfig; + +import org.apache.commons.lang3.StringUtils; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Random; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.stub.StreamObserver; + +import com.google.protobuf.Any; +import com.google.protobuf.UnsafeByteOperations; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class FunctionRuntime implements Runtime { + + private final RuntimeInstanceConfig runtimeInstanceConfig; + + private ManagedChannel channel; + + private AdminServiceStub adminServiceStub; + + private AdminServiceBlockingStub adminServiceBlockingStub; + + StreamObserver responseObserver; + + StreamObserver requestObserver; + + private final LinkedBlockingQueue queue; + + private FunctionRuntimeConfig functionRuntimeConfig; + + private AbstractEventMeshFunctionChain functionChain; + + private Sink sinkConnector; + + private Source sourceConnector; + + private final ExecutorService sourceService = ThreadPoolFactory.createSingleExecutor("eventMesh-sourceService"); + + private final ExecutorService sinkService = ThreadPoolFactory.createSingleExecutor("eventMesh-sinkService"); + + private final ScheduledExecutorService heartBeatExecutor = Executors.newSingleThreadScheduledExecutor(); + + private volatile boolean isRunning = false; + + private volatile boolean isFailed = false; + + private String adminServerAddr; + + + public FunctionRuntime(RuntimeInstanceConfig runtimeInstanceConfig) { + this.runtimeInstanceConfig = runtimeInstanceConfig; + this.queue = new LinkedBlockingQueue<>(1000); + } + + + @Override + public void init() throws Exception { + // load function runtime config from local file + this.functionRuntimeConfig = ConfigService.getInstance().buildConfigInstance(FunctionRuntimeConfig.class); + + // init admin service + initAdminService(); + + // get remote config from admin service and update local config + getAndUpdateRemoteConfig(); + + // init connector service + initConnectorService(); + + // report status to admin server + reportJobRequest(functionRuntimeConfig.getJobID(), JobState.INIT); + } + + private void initAdminService() { + adminServerAddr = getRandomAdminServerAddr(runtimeInstanceConfig.getAdminServiceAddr()); + // create gRPC channel + channel = ManagedChannelBuilder.forTarget(adminServerAddr).usePlaintext().build(); + + adminServiceStub = AdminServiceGrpc.newStub(channel).withWaitForReady(); + + adminServiceBlockingStub = AdminServiceGrpc.newBlockingStub(channel).withWaitForReady(); + + responseObserver = new StreamObserver() { + @Override + public void onNext(Payload response) { + log.info("runtime receive message: {} ", response); + } + + @Override + public void onError(Throwable t) { + log.error("runtime receive error message: {}", t.getMessage()); + } + + @Override + public void onCompleted() { + log.info("runtime finished receive message and completed"); + } + }; + + requestObserver = adminServiceStub.invokeBiStream(responseObserver); + } + + private String getRandomAdminServerAddr(String adminServerAddrList) { + String[] addresses = adminServerAddrList.split(";"); + if (addresses.length == 0) { + throw new IllegalArgumentException("Admin server address list is empty"); + } + Random random = new Random(); + int randomIndex = random.nextInt(addresses.length); + return addresses[randomIndex]; + } + + private void getAndUpdateRemoteConfig() { + String jobId = functionRuntimeConfig.getJobID(); + FetchJobRequest jobRequest = new FetchJobRequest(); + jobRequest.setJobID(jobId); + + Metadata metadata = Metadata.newBuilder().setType(FetchJobRequest.class.getSimpleName()).build(); + + Payload request = Payload.newBuilder().setMetadata(metadata) + .setBody(Any.newBuilder().setValue(UnsafeByteOperations.unsafeWrap(Objects.requireNonNull(JsonUtils.toJSONBytes(jobRequest)))).build()) + .build(); + Payload response = adminServiceBlockingStub.invoke(request); + FetchJobResponse jobResponse = null; + if (response.getMetadata().getType().equals(FetchJobResponse.class.getSimpleName())) { + jobResponse = JsonUtils.parseObject(response.getBody().getValue().toStringUtf8(), FetchJobResponse.class); + } + + if (jobResponse == null || jobResponse.getErrorCode() != ErrorCode.SUCCESS) { + if (jobResponse != null) { + log.error("Failed to get remote config from admin server. ErrorCode: {}, Response: {}", + jobResponse.getErrorCode(), jobResponse); + } else { + log.error("Failed to get remote config from admin server. "); + } + isFailed = true; + try { + stop(); + } catch (Exception e) { + log.error("Failed to stop after exception", e); + } + throw new RuntimeException("Failed to get remote config from admin server."); + } + + // update local config + // source + functionRuntimeConfig.setSourceConnectorType(jobResponse.getTransportType().getSrc().getName()); + functionRuntimeConfig.setSourceConnectorDesc(jobResponse.getConnectorConfig().getSourceConnectorDesc()); + functionRuntimeConfig.setSourceConnectorConfig(jobResponse.getConnectorConfig().getSourceConnectorConfig()); + + // sink + functionRuntimeConfig.setSinkConnectorType(jobResponse.getTransportType().getDst().getName()); + functionRuntimeConfig.setSinkConnectorDesc(jobResponse.getConnectorConfig().getSinkConnectorDesc()); + functionRuntimeConfig.setSinkConnectorConfig(jobResponse.getConnectorConfig().getSinkConnectorConfig()); + + // TODO: update functionConfigs + + } + + + private void initConnectorService() throws Exception { + final JobType jobType = (JobType) functionRuntimeConfig.getRuntimeConfig().get("jobType"); + + // create sink connector + ConnectorCreateService sinkConnectorCreateService = + ConnectorPluginFactory.createConnector(functionRuntimeConfig.getSinkConnectorType() + "-Sink"); + this.sinkConnector = (Sink) sinkConnectorCreateService.create(); + + // parse sink config and init sink connector + SinkConfig sinkConfig = (SinkConfig) ConfigUtil.parse(functionRuntimeConfig.getSinkConnectorConfig(), sinkConnector.configClass()); + SinkConnectorContext sinkConnectorContext = new SinkConnectorContext(); + sinkConnectorContext.setSinkConfig(sinkConfig); + sinkConnectorContext.setRuntimeConfig(functionRuntimeConfig.getRuntimeConfig()); + sinkConnectorContext.setJobType(jobType); + sinkConnector.init(sinkConnectorContext); + + // create source connector + ConnectorCreateService sourceConnectorCreateService = + ConnectorPluginFactory.createConnector(functionRuntimeConfig.getSourceConnectorType() + "-Source"); + this.sourceConnector = (Source) sourceConnectorCreateService.create(); + + // parse source config and init source connector + SourceConfig sourceConfig = (SourceConfig) ConfigUtil.parse(functionRuntimeConfig.getSourceConnectorConfig(), sourceConnector.configClass()); + SourceConnectorContext sourceConnectorContext = new SourceConnectorContext(); + sourceConnectorContext.setSourceConfig(sourceConfig); + sourceConnectorContext.setRuntimeConfig(functionRuntimeConfig.getRuntimeConfig()); + sourceConnectorContext.setJobType(jobType); + + sourceConnector.init(sourceConnectorContext); + } + + private void reportJobRequest(String jobId, JobState jobState) { + ReportJobRequest reportJobRequest = new ReportJobRequest(); + reportJobRequest.setJobID(jobId); + reportJobRequest.setState(jobState); + Metadata metadata = Metadata.newBuilder() + .setType(ReportJobRequest.class.getSimpleName()) + .build(); + Payload payload = Payload.newBuilder() + .setMetadata(metadata) + .setBody(Any.newBuilder().setValue(UnsafeByteOperations.unsafeWrap(Objects.requireNonNull(JsonUtils.toJSONBytes(reportJobRequest)))) + .build()) + .build(); + requestObserver.onNext(payload); + } + + + @Override + public void start() throws Exception { + this.isRunning = true; + + // build function chain + this.functionChain = buildFunctionChain(functionRuntimeConfig.getFunctionConfigs()); + + // start heart beat + this.heartBeatExecutor.scheduleAtFixedRate(() -> { + + ReportHeartBeatRequest heartBeat = new ReportHeartBeatRequest(); + heartBeat.setAddress(IPUtils.getLocalAddress()); + heartBeat.setReportedTimeStamp(String.valueOf(System.currentTimeMillis())); + heartBeat.setJobID(functionRuntimeConfig.getJobID()); + + Metadata metadata = Metadata.newBuilder().setType(ReportHeartBeatRequest.class.getSimpleName()).build(); + + Payload request = Payload.newBuilder().setMetadata(metadata) + .setBody(Any.newBuilder().setValue(UnsafeByteOperations.unsafeWrap(Objects.requireNonNull(JsonUtils.toJSONBytes(heartBeat)))).build()) + .build(); + + requestObserver.onNext(request); + }, 5, 5, TimeUnit.SECONDS); + + // start sink service + this.sinkService.execute(() -> { + try { + startSinkConnector(); + } catch (Exception e) { + isFailed = true; + log.error("Sink Connector [{}] failed to start.", sinkConnector.name(), e); + try { + this.stop(); + } catch (Exception ex) { + log.error("Failed to stop after exception", ex); + } + throw new RuntimeException(e); + } + }); + + // start source service + this.sourceService.execute(() -> { + try { + startSourceConnector(); + } catch (Exception e) { + isFailed = true; + log.error("Source Connector [{}] failed to start.", sourceConnector.name(), e); + try { + this.stop(); + } catch (Exception ex) { + log.error("Failed to stop after exception", ex); + } + throw new RuntimeException(e); + } + }); + + reportJobRequest(functionRuntimeConfig.getJobID(), JobState.RUNNING); + } + + private StringEventMeshFunctionChain buildFunctionChain(List> functionConfigs) { + StringEventMeshFunctionChain functionChain = new StringEventMeshFunctionChain(); + + // build function chain + for (Map functionConfig : functionConfigs) { + String functionType = String.valueOf(functionConfig.getOrDefault("functionType", "")); + if (StringUtils.isEmpty(functionType)) { + throw new IllegalArgumentException("'functionType' is required for function"); + } + + // build function based on functionType + EventMeshFunction function; + switch (functionType) { + case "filter": + function = buildFilter(functionConfig); + break; + case "transformer": + function = buildTransformer(functionConfig); + break; + default: + throw new IllegalArgumentException( + "Invalid functionType: '" + functionType + "'. Supported functionType: 'filter', 'transformer'"); + } + + // add function to functionChain + functionChain.addLast(function); + } + + return functionChain; + } + + + @SuppressWarnings("unchecked") + private Pattern buildFilter(Map functionConfig) { + // get condition from attributes + Object condition = functionConfig.get("condition"); + if (condition == null) { + throw new IllegalArgumentException("'condition' is required for filter function"); + } + if (condition instanceof String) { + return PatternBuilder.build(String.valueOf(condition)); + } else if (condition instanceof Map) { + return PatternBuilder.build((Map) condition); + } else { + throw new IllegalArgumentException("Invalid condition"); + } + } + + private Transformer buildTransformer(Map functionConfig) { + // get transformerType from attributes + String transformerTypeStr = String.valueOf(functionConfig.getOrDefault("transformerType", "")).toLowerCase(); + TransformerType transformerType = TransformerType.getItem(transformerTypeStr); + if (transformerType == null) { + throw new IllegalArgumentException( + "Invalid transformerType: '" + transformerTypeStr + + "'. Supported transformerType: 'constant', 'template', 'original' (case insensitive)"); + } + + // build transformer + Transformer transformer = null; + + switch (transformerType) { + case CONSTANT: + // check value + String content = String.valueOf(functionConfig.getOrDefault("content", "")); + if (StringUtils.isEmpty(content)) { + throw new IllegalArgumentException("'content' is required for constant transformer"); + } + transformer = TransformerBuilder.buildConstantTransformer(content); + break; + case TEMPLATE: + // check value and template + Object valueMap = functionConfig.get("valueMap"); + String template = String.valueOf(functionConfig.getOrDefault("template", "")); + if (valueMap == null || StringUtils.isEmpty(template)) { + throw new IllegalArgumentException("'valueMap' and 'template' are required for template transformer"); + } + transformer = TransformerBuilder.buildTemplateTransFormer(valueMap, template); + break; + case ORIGINAL: + // ORIGINAL transformer does not need any parameter + break; + default: + throw new IllegalArgumentException( + "Invalid transformerType: '" + transformerType + "', supported transformerType: 'CONSTANT', 'TEMPLATE', 'ORIGINAL'"); + } + + return transformer; + } + + + private void startSinkConnector() throws Exception { + // start sink connector + this.sinkConnector.start(); + + // try to get data from queue and send it. + while (this.isRunning) { + ConnectRecord connectRecord = null; + try { + connectRecord = queue.poll(5, TimeUnit.SECONDS); + } catch (InterruptedException e) { + log.error("Failed to poll data from queue.", e); + Thread.currentThread().interrupt(); + } + + // send data if not null + if (connectRecord != null) { + sinkConnector.put(Collections.singletonList(connectRecord)); + } + } + } + + private void startSourceConnector() throws Exception { + // start source connector + this.sourceConnector.start(); + + // try to get data from source connector and handle it. + while (this.isRunning) { + List connectorRecordList = sourceConnector.poll(); + + // handle data + if (connectorRecordList != null && !connectorRecordList.isEmpty()) { + for (ConnectRecord connectRecord : connectorRecordList) { + if (connectRecord == null || connectRecord.getData() == null) { + // If data is null, just put it into queue. + this.queue.put(connectRecord); + } else { + // Apply function chain to data + String data = functionChain.apply((String) connectRecord.getData()); + if (data != null) { + if (log.isDebugEnabled()) { + log.debug("Function chain applied. Original data: {}, Transformed data: {}", connectRecord.getData(), data); + } + connectRecord.setData(data); + this.queue.put(connectRecord); + } else if (log.isDebugEnabled()) { + log.debug("Data filtered out by function chain. Original data: {}", connectRecord.getData()); + } + } + } + } + } + } + + + @Override + public void stop() throws Exception { + log.info("FunctionRuntime is stopping..."); + + isRunning = false; + + if (isFailed) { + reportJobRequest(functionRuntimeConfig.getJobID(), JobState.FAIL); + } else { + reportJobRequest(functionRuntimeConfig.getJobID(), JobState.COMPLETE); + } + + sinkConnector.stop(); + sourceConnector.stop(); + sinkService.shutdown(); + sourceService.shutdown(); + heartBeatExecutor.shutdown(); + + requestObserver.onCompleted(); + if (channel != null && !channel.isShutdown()) { + channel.shutdown(); + } + + log.info("FunctionRuntime stopped."); + } +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/function/FunctionRuntimeConfig.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/function/FunctionRuntimeConfig.java new file mode 100644 index 0000000000..4d57c83e82 --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/function/FunctionRuntimeConfig.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.function; + +import org.apache.eventmesh.common.config.Config; + +import java.util.List; +import java.util.Map; + + +import lombok.Data; + +@Data +@Config(path = "classPath://function.yaml") +public class FunctionRuntimeConfig { + + private String functionRuntimeInstanceId; + + private String taskID; + + private String jobID; + + private String region; + + private Map runtimeConfig; + + private String sourceConnectorType; + + private String sourceConnectorDesc; + + private Map sourceConnectorConfig; + + private String sinkConnectorType; + + private String sinkConnectorDesc; + + private Map sinkConnectorConfig; + + private List> functionConfigs; + +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/function/FunctionRuntimeFactory.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/function/FunctionRuntimeFactory.java new file mode 100644 index 0000000000..40346e272f --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/function/FunctionRuntimeFactory.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.function; + +import org.apache.eventmesh.runtime.Runtime; +import org.apache.eventmesh.runtime.RuntimeFactory; +import org.apache.eventmesh.runtime.RuntimeInstanceConfig; + +public class FunctionRuntimeFactory implements RuntimeFactory { + + @Override + public void init() throws Exception { + + } + + @Override + public Runtime createRuntime(RuntimeInstanceConfig runtimeInstanceConfig) { + return new FunctionRuntime(runtimeInstanceConfig); + } + + @Override + public void close() throws Exception { + + } + +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/function/StringEventMeshFunctionChain.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/function/StringEventMeshFunctionChain.java new file mode 100644 index 0000000000..0035999ecb --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/function/StringEventMeshFunctionChain.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.function; + +import org.apache.eventmesh.function.api.AbstractEventMeshFunctionChain; +import org.apache.eventmesh.function.api.EventMeshFunction; + +/** + * ConnectRecord Function Chain. + */ +public class StringEventMeshFunctionChain extends AbstractEventMeshFunctionChain { + + @Override + public String apply(String content) { + for (EventMeshFunction function : functions) { + if (content == null) { + break; + } + content = function.apply(content); + } + return content; + } +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/manager/ConnectorManager.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/manager/ConnectorManager.java new file mode 100644 index 0000000000..2354a350db --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/manager/ConnectorManager.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.manager; + +public class ConnectorManager { +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/manager/FunctionManager.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/manager/FunctionManager.java new file mode 100644 index 0000000000..8c88be9986 --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/manager/FunctionManager.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.manager; + +public class FunctionManager { +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/manager/MeshManager.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/manager/MeshManager.java new file mode 100644 index 0000000000..cc67b9fb40 --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/manager/MeshManager.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.manager; + +public class MeshManager { +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/mesh/MeshRuntime.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/mesh/MeshRuntime.java new file mode 100644 index 0000000000..eb186c7658 --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/mesh/MeshRuntime.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.mesh; + +import org.apache.eventmesh.runtime.Runtime; + +public class MeshRuntime implements Runtime { + + @Override + public void init() throws Exception { + + } + + @Override + public void start() throws Exception { + + } + + @Override + public void stop() throws Exception { + + } +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/mesh/MeshRuntimeConfig.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/mesh/MeshRuntimeConfig.java new file mode 100644 index 0000000000..cd21eb1a11 --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/mesh/MeshRuntimeConfig.java @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.mesh; + +public class MeshRuntimeConfig { +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/mesh/MeshRuntimeFactory.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/mesh/MeshRuntimeFactory.java new file mode 100644 index 0000000000..32a3f2e38e --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/mesh/MeshRuntimeFactory.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.mesh; + +import org.apache.eventmesh.runtime.Runtime; +import org.apache.eventmesh.runtime.RuntimeFactory; +import org.apache.eventmesh.runtime.RuntimeInstanceConfig; + +public class MeshRuntimeFactory implements RuntimeFactory { + + @Override + public void init() throws Exception { + + } + + @Override + public Runtime createRuntime(RuntimeInstanceConfig runtimeInstanceConfig) { + return null; + } + + @Override + public void close() throws Exception { + + } + +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/meta/MetaStorage.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/meta/MetaStorage.java new file mode 100644 index 0000000000..41da6994f7 --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/meta/MetaStorage.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.meta; + +import org.apache.eventmesh.api.exception.MetaException; +import org.apache.eventmesh.api.meta.MetaService; +import org.apache.eventmesh.api.meta.MetaServiceListener; +import org.apache.eventmesh.api.meta.bo.EventMeshAppSubTopicInfo; +import org.apache.eventmesh.api.meta.bo.EventMeshServicePubTopicInfo; +import org.apache.eventmesh.api.meta.dto.EventMeshDataInfo; +import org.apache.eventmesh.api.meta.dto.EventMeshRegisterInfo; +import org.apache.eventmesh.api.meta.dto.EventMeshUnRegisterInfo; +import org.apache.eventmesh.spi.EventMeshExtensionFactory; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class MetaStorage { + + private static final Map META_CACHE = new HashMap<>(16); + + private MetaService metaService; + + private final AtomicBoolean inited = new AtomicBoolean(false); + + private final AtomicBoolean started = new AtomicBoolean(false); + + private final AtomicBoolean shutdown = new AtomicBoolean(false); + + private MetaStorage() { + + } + + public static MetaStorage getInstance(String metaPluginType) { + return META_CACHE.computeIfAbsent(metaPluginType, MetaStorage::metaStorageBuilder); + } + + private static MetaStorage metaStorageBuilder(String metaPluginType) { + MetaService metaServiceExt = EventMeshExtensionFactory.getExtension(MetaService.class, metaPluginType); + if (metaServiceExt == null) { + String errorMsg = "can't load the metaService plugin, please check."; + log.error(errorMsg); + throw new RuntimeException(errorMsg); + } + MetaStorage metaStorage = new MetaStorage(); + metaStorage.metaService = metaServiceExt; + + return metaStorage; + } + + public void init() throws MetaException { + if (!inited.compareAndSet(false, true)) { + return; + } + metaService.init(); + } + + public void start() throws MetaException { + if (!started.compareAndSet(false, true)) { + return; + } + metaService.start(); + } + + public void shutdown() throws MetaException { + inited.compareAndSet(true, false); + started.compareAndSet(true, false); + if (!shutdown.compareAndSet(false, true)) { + return; + } + synchronized (this) { + metaService.shutdown(); + } + } + + public List findEventMeshInfoByCluster(String clusterName) throws MetaException { + return metaService.findEventMeshInfoByCluster(clusterName); + } + + public List findAllEventMeshInfo() throws MetaException { + return metaService.findAllEventMeshInfo(); + } + + public Map> findEventMeshClientDistributionData(String clusterName, String group, String purpose) + throws MetaException { + return metaService.findEventMeshClientDistributionData(clusterName, group, purpose); + } + + public void registerMetadata(Map metadata) { + metaService.registerMetadata(metadata); + } + + public void updateMetaData(Map metadata) { + metaService.updateMetaData(metadata); + } + + public boolean register(EventMeshRegisterInfo eventMeshRegisterInfo) throws MetaException { + return metaService.register(eventMeshRegisterInfo); + } + + public boolean unRegister(EventMeshUnRegisterInfo eventMeshUnRegisterInfo) throws MetaException { + return metaService.unRegister(eventMeshUnRegisterInfo); + } + + public List findEventMeshServicePubTopicInfos() throws Exception { + return metaService.findEventMeshServicePubTopicInfos(); + } + + public EventMeshAppSubTopicInfo findEventMeshAppSubTopicInfo(String group) throws Exception { + return metaService.findEventMeshAppSubTopicInfoByGroup(group); + } + + public Map getMetaData(String key, boolean fuzzyEnabled) { + return metaService.getMetaData(key, fuzzyEnabled); + } + + public void getMetaDataWithListener(MetaServiceListener metaServiceListener, String key) throws Exception { + metaService.getMetaDataWithListener(metaServiceListener, key); + } + + public AtomicBoolean getInited() { + return inited; + } + + public AtomicBoolean getStarted() { + return started; + } +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/health/HealthService.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/health/HealthService.java new file mode 100644 index 0000000000..54f924874b --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/health/HealthService.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.service.health; + +import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Payload; +import org.apache.eventmesh.common.remote.request.ReportHeartBeatRequest; +import org.apache.eventmesh.common.utils.IPUtils; +import org.apache.eventmesh.common.utils.JsonUtils; +import org.apache.eventmesh.runtime.connector.ConnectorRuntimeConfig; + +import java.util.Objects; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import io.grpc.stub.StreamObserver; + +import com.google.protobuf.Any; +import com.google.protobuf.UnsafeByteOperations; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class HealthService { + + private final ScheduledExecutorService scheduler; + + private StreamObserver requestObserver; + + private StreamObserver responseObserver; + + private AdminServiceGrpc.AdminServiceStub adminServiceStub; + + private AdminServiceGrpc.AdminServiceBlockingStub adminServiceBlockingStub; + + private ConnectorRuntimeConfig connectorRuntimeConfig; + + + public HealthService(AdminServiceGrpc.AdminServiceStub adminServiceStub, AdminServiceGrpc.AdminServiceBlockingStub adminServiceBlockingStub, + ConnectorRuntimeConfig connectorRuntimeConfig) { + this.adminServiceStub = adminServiceStub; + this.adminServiceBlockingStub = adminServiceBlockingStub; + this.connectorRuntimeConfig = connectorRuntimeConfig; + + this.scheduler = Executors.newSingleThreadScheduledExecutor(); + + responseObserver = new StreamObserver() { + @Override + public void onNext(Payload response) { + log.debug("health service receive message: {}|{} ", response.getMetadata(), response.getBody()); + } + + @Override + public void onError(Throwable t) { + log.error("health service receive error message: {}", t.getMessage()); + } + + @Override + public void onCompleted() { + log.info("health service finished receive message and completed"); + } + }; + requestObserver = this.adminServiceStub.invokeBiStream(responseObserver); + } + + public void start() { + this.healthReport(); + } + + public void healthReport() { + scheduler.scheduleAtFixedRate(() -> { + ReportHeartBeatRequest heartBeat = new ReportHeartBeatRequest(); + heartBeat.setAddress(IPUtils.getLocalAddress()); + heartBeat.setReportedTimeStamp(String.valueOf(System.currentTimeMillis())); + heartBeat.setJobID(connectorRuntimeConfig.getJobID()); + + Metadata metadata = Metadata.newBuilder().setType(ReportHeartBeatRequest.class.getSimpleName()).build(); + + Payload request = Payload.newBuilder().setMetadata(metadata) + .setBody(Any.newBuilder().setValue(UnsafeByteOperations.unsafeWrap(Objects.requireNonNull(JsonUtils.toJSONBytes(heartBeat)))).build()) + .build(); + + requestObserver.onNext(request); + }, 5, 5, TimeUnit.SECONDS); + } + + + public void stop() { + scheduler.shutdown(); + if (requestObserver != null) { + requestObserver.onCompleted(); + } + } + +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/monitor/MonitorService.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/monitor/MonitorService.java new file mode 100644 index 0000000000..f5af7596c3 --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/monitor/MonitorService.java @@ -0,0 +1,144 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.service.monitor; + +import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Payload; +import org.apache.eventmesh.common.remote.request.ReportMonitorRequest; +import org.apache.eventmesh.common.utils.JsonUtils; +import org.apache.eventmesh.openconnect.api.monitor.Monitor; +import org.apache.eventmesh.openconnect.api.monitor.MonitorRegistry; + +import java.util.List; +import java.util.Objects; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import io.grpc.stub.StreamObserver; + +import com.google.protobuf.Any; +import com.google.protobuf.UnsafeByteOperations; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class MonitorService { + + private final ScheduledExecutorService scheduler; + + private StreamObserver requestObserver; + + private StreamObserver responseObserver; + + private AdminServiceGrpc.AdminServiceStub adminServiceStub; + + private AdminServiceGrpc.AdminServiceBlockingStub adminServiceBlockingStub; + + + public MonitorService(AdminServiceGrpc.AdminServiceStub adminServiceStub, AdminServiceGrpc.AdminServiceBlockingStub adminServiceBlockingStub) { + this.adminServiceStub = adminServiceStub; + this.adminServiceBlockingStub = adminServiceBlockingStub; + + this.scheduler = Executors.newSingleThreadScheduledExecutor(); + + responseObserver = new StreamObserver() { + @Override + public void onNext(Payload response) { + log.debug("monitor service receive message: {}|{} ", response.getMetadata(), response.getBody()); + } + + @Override + public void onError(Throwable t) { + log.error("monitor service receive error message: {}", t.getMessage()); + } + + @Override + public void onCompleted() { + log.info("monitor service finished receive message and completed"); + } + }; + requestObserver = this.adminServiceStub.invokeBiStream(responseObserver); + } + + public void registerMonitor(Monitor monitor) { + MonitorRegistry.registerMonitor(monitor); + } + + public void start() { + this.startReporting(); + } + + public void startReporting() { + scheduler.scheduleAtFixedRate(() -> { + List monitors = MonitorRegistry.getMonitors(); + for (Monitor monitor : monitors) { + monitor.printMetrics(); + reportToAdminService(monitor); + } + }, 5, 30, TimeUnit.SECONDS); + } + + private void reportToAdminService(Monitor monitor) { + ReportMonitorRequest request = new ReportMonitorRequest(); + if (monitor instanceof SourceMonitor) { + SourceMonitor sourceMonitor = (SourceMonitor) monitor; + request.setTaskID(sourceMonitor.getTaskId()); + request.setJobID(sourceMonitor.getJobId()); + request.setAddress(sourceMonitor.getIp()); + request.setConnectorStage(sourceMonitor.getConnectorStage()); + request.setTotalReqNum(sourceMonitor.getTotalRecordNum().longValue()); + request.setTotalTimeCost(sourceMonitor.getTotalTimeCost().longValue()); + request.setMaxTimeCost(sourceMonitor.getMaxTimeCost().longValue()); + request.setAvgTimeCost(sourceMonitor.getAverageTime()); + request.setTps(sourceMonitor.getTps()); + } else if (monitor instanceof SinkMonitor) { + SinkMonitor sinkMonitor = (SinkMonitor) monitor; + request.setTaskID(sinkMonitor.getTaskId()); + request.setJobID(sinkMonitor.getJobId()); + request.setAddress(sinkMonitor.getIp()); + request.setConnectorStage(sinkMonitor.getConnectorStage()); + request.setTotalReqNum(sinkMonitor.getTotalRecordNum().longValue()); + request.setTotalTimeCost(sinkMonitor.getTotalTimeCost().longValue()); + request.setMaxTimeCost(sinkMonitor.getMaxTimeCost().longValue()); + request.setAvgTimeCost(sinkMonitor.getAverageTime()); + request.setTps(sinkMonitor.getTps()); + } else { + throw new IllegalArgumentException("Unsupported monitor: " + monitor); + } + + Metadata metadata = Metadata.newBuilder() + .setType(ReportMonitorRequest.class.getSimpleName()) + .build(); + Payload payload = Payload.newBuilder() + .setMetadata(metadata) + .setBody(Any.newBuilder().setValue(UnsafeByteOperations.unsafeWrap(Objects.requireNonNull(JsonUtils.toJSONBytes(request)))) + .build()) + .build(); + requestObserver.onNext(payload); + } + + public void stop() { + scheduler.shutdown(); + if (requestObserver != null) { + requestObserver.onCompleted(); + } + } + +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/monitor/SinkMonitor.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/monitor/SinkMonitor.java new file mode 100644 index 0000000000..b27b44da7c --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/monitor/SinkMonitor.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.service.monitor; + +import org.apache.eventmesh.common.enums.ConnectorStage; +import org.apache.eventmesh.openconnect.api.monitor.AbstractConnectorMonitor; + +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Getter +@Setter +public class SinkMonitor extends AbstractConnectorMonitor { + + private String connectorStage = ConnectorStage.SINK.name(); + + public SinkMonitor(String taskId, String jobId, String ip) { + super(taskId, jobId, ip); + } + + @Override + public void recordProcess(long timeCost) { + super.recordProcess(timeCost); + } + + @Override + public void recordProcess(int recordCount, long timeCost) { + super.recordProcess(recordCount, timeCost); + } + + @Override + public void printMetrics() { + super.printMetrics(); + } +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/monitor/SourceMonitor.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/monitor/SourceMonitor.java new file mode 100644 index 0000000000..3895c8df14 --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/monitor/SourceMonitor.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.service.monitor; + +import org.apache.eventmesh.common.enums.ConnectorStage; +import org.apache.eventmesh.openconnect.api.monitor.AbstractConnectorMonitor; + +import lombok.Getter; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Getter +@Setter +public class SourceMonitor extends AbstractConnectorMonitor { + + private String connectorStage = ConnectorStage.SOURCE.name(); + + public SourceMonitor(String taskId, String jobId, String ip) { + super(taskId, jobId, ip); + } + + @Override + public void recordProcess(int recordCount, long timeCost) { + super.recordProcess(recordCount, timeCost); + } + + @Override + public void printMetrics() { + super.printMetrics(); + } +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/status/StatusService.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/status/StatusService.java new file mode 100644 index 0000000000..e40686f575 --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/status/StatusService.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.service.status; + +import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Payload; +import org.apache.eventmesh.common.remote.JobState; +import org.apache.eventmesh.common.remote.request.ReportJobRequest; +import org.apache.eventmesh.common.utils.IPUtils; +import org.apache.eventmesh.common.utils.JsonUtils; + +import java.util.Objects; + +import io.grpc.stub.StreamObserver; + +import com.google.protobuf.Any; +import com.google.protobuf.UnsafeByteOperations; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class StatusService { + + private StreamObserver requestObserver; + + private StreamObserver responseObserver; + + private AdminServiceGrpc.AdminServiceStub adminServiceStub; + + private AdminServiceGrpc.AdminServiceBlockingStub adminServiceBlockingStub; + + + public StatusService(AdminServiceGrpc.AdminServiceStub adminServiceStub, AdminServiceGrpc.AdminServiceBlockingStub adminServiceBlockingStub) { + this.adminServiceStub = adminServiceStub; + this.adminServiceBlockingStub = adminServiceBlockingStub; + + responseObserver = new StreamObserver() { + @Override + public void onNext(Payload response) { + log.debug("health service receive message: {}|{} ", response.getMetadata(), response.getBody()); + } + + @Override + public void onError(Throwable t) { + log.error("health service receive error message: {}", t.getMessage()); + } + + @Override + public void onCompleted() { + log.info("health service finished receive message and completed"); + } + }; + requestObserver = this.adminServiceStub.invokeBiStream(responseObserver); + } + + public void reportJobStatus(String jobId, JobState jobState) { + ReportJobRequest reportJobRequest = new ReportJobRequest(); + reportJobRequest.setJobID(jobId); + reportJobRequest.setState(jobState); + reportJobRequest.setAddress(IPUtils.getLocalAddress()); + Metadata metadata = Metadata.newBuilder() + .setType(ReportJobRequest.class.getSimpleName()) + .build(); + Payload payload = Payload.newBuilder() + .setMetadata(metadata) + .setBody(Any.newBuilder().setValue(UnsafeByteOperations.unsafeWrap(Objects.requireNonNull(JsonUtils.toJSONBytes(reportJobRequest)))) + .build()) + .build(); + log.info("report job state request: {}", JsonUtils.toJSONString(reportJobRequest)); + requestObserver.onNext(payload); + } + + public void stop() { + if (requestObserver != null) { + requestObserver.onCompleted(); + } + } +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/verify/VerifyService.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/verify/VerifyService.java new file mode 100644 index 0000000000..8bcb72199c --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/service/verify/VerifyService.java @@ -0,0 +1,138 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.service.verify; + +import org.apache.eventmesh.common.enums.ConnectorStage; +import org.apache.eventmesh.common.protocol.grpc.adminserver.AdminServiceGrpc; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Metadata; +import org.apache.eventmesh.common.protocol.grpc.adminserver.Payload; +import org.apache.eventmesh.common.remote.request.ReportVerifyRequest; +import org.apache.eventmesh.common.utils.IPUtils; +import org.apache.eventmesh.common.utils.JsonUtils; +import org.apache.eventmesh.openconnect.offsetmgmt.api.data.ConnectRecord; +import org.apache.eventmesh.runtime.connector.ConnectorRuntimeConfig; + +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.Arrays; +import java.util.Objects; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import io.grpc.stub.StreamObserver; + +import com.google.protobuf.Any; +import com.google.protobuf.UnsafeByteOperations; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class VerifyService { + + private final ExecutorService reportVerifyExecutor; + + private StreamObserver requestObserver; + + private StreamObserver responseObserver; + + private AdminServiceGrpc.AdminServiceStub adminServiceStub; + + private AdminServiceGrpc.AdminServiceBlockingStub adminServiceBlockingStub; + + private ConnectorRuntimeConfig connectorRuntimeConfig; + + + public VerifyService(AdminServiceGrpc.AdminServiceStub adminServiceStub, AdminServiceGrpc.AdminServiceBlockingStub adminServiceBlockingStub, + ConnectorRuntimeConfig connectorRuntimeConfig) { + this.adminServiceStub = adminServiceStub; + this.adminServiceBlockingStub = adminServiceBlockingStub; + this.connectorRuntimeConfig = connectorRuntimeConfig; + + this.reportVerifyExecutor = Executors.newSingleThreadExecutor(); + + responseObserver = new StreamObserver() { + @Override + public void onNext(Payload response) { + log.debug("verify service receive message: {}|{} ", response.getMetadata(), response.getBody()); + } + + @Override + public void onError(Throwable t) { + log.error("verify service receive error message: {}", t.getMessage()); + } + + @Override + public void onCompleted() { + log.info("verify service finished receive message and completed"); + } + }; + requestObserver = this.adminServiceStub.invokeBiStream(responseObserver); + } + + public void reportVerifyRequest(ConnectRecord record, ConnectorStage connectorStage) { + reportVerifyExecutor.submit(() -> { + try { + byte[] data = (byte[]) record.getData(); + // use record data + recordUniqueId for md5 + String md5Str = md5(Arrays.toString(data) + record.getExtension("recordUniqueId")); + ReportVerifyRequest reportVerifyRequest = new ReportVerifyRequest(); + reportVerifyRequest.setTaskID(connectorRuntimeConfig.getTaskID()); + reportVerifyRequest.setJobID(connectorRuntimeConfig.getJobID()); + reportVerifyRequest.setRecordID(record.getExtension("recordUniqueId")); + reportVerifyRequest.setRecordSig(md5Str); + reportVerifyRequest.setConnectorName( + IPUtils.getLocalAddress() + "_" + connectorRuntimeConfig.getJobID() + "_" + connectorRuntimeConfig.getRegion()); + reportVerifyRequest.setConnectorStage(connectorStage.name()); + reportVerifyRequest.setPosition(JsonUtils.toJSONString(record.getPosition())); + + Metadata metadata = Metadata.newBuilder().setType(ReportVerifyRequest.class.getSimpleName()).build(); + + Payload request = Payload.newBuilder().setMetadata(metadata) + .setBody( + Any.newBuilder().setValue(UnsafeByteOperations.unsafeWrap(Objects.requireNonNull(JsonUtils.toJSONBytes(reportVerifyRequest)))) + .build()) + .build(); + requestObserver.onNext(request); + } catch (Exception e) { + log.error("Failed to report verify request", e); + } + }); + } + + private String md5(String input) { + try { + MessageDigest md = MessageDigest.getInstance("MD5"); + byte[] messageDigest = md.digest(input.getBytes()); + StringBuilder sb = new StringBuilder(); + for (byte b : messageDigest) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException(e); + } + } + + public void stop() { + reportVerifyExecutor.shutdown(); + if (requestObserver != null) { + requestObserver.onCompleted(); + } + } + +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/util/BannerUtil.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/util/BannerUtil.java new file mode 100644 index 0000000000..2569494189 --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/util/BannerUtil.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.util; + +import lombok.extern.slf4j.Slf4j; + +/** + * EventMesh banner util + */ +@Slf4j +public class BannerUtil { + + private static final String LOGO = + " EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEME EMEMEMEME EMEMEMEME " + System.lineSeparator() + + " EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEME EMEMEMEMEMEMEMEME EMEMEMEMEMEMEMEMEM " + System.lineSeparator() + + " EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEM EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEME " + System.lineSeparator() + + "EMEMEMEMEMEM EMEMEMEMEM EMEMEMEMEMEMEMEME EMEMEMEMEME" + System.lineSeparator() + + "EMEMEMEME EMEMEMEMEM EMEMEMEMEMEME EMEMEMEME" + System.lineSeparator() + + "EMEMEME EMEMEMEMEM EMEME EMEMEMEM" + System.lineSeparator() + + "EMEMEME EMEMEMEMEM EMEMEME" + System.lineSeparator() + + "EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEM EMEMEMEMEM EMEMEME" + System.lineSeparator() + + "EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEM EMEMEMEMEM EMEMEME" + System.lineSeparator() + + "EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEM EMEMEMEMEM EMEMEME" + System.lineSeparator() + + "EMEMEME EMEMEMEMEM EMEMEME" + System.lineSeparator() + + "EMEMEME EMEMEMEMEM EMEMEME" + System.lineSeparator() + + "EMEMEMEME EMEMEMEMEM EMEMEMEME" + System.lineSeparator() + + "EMEMEMEMEMEM EMEMEMEMEM EMEMEMEMEMEM" + System.lineSeparator() + + " EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEME EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEM " + System.lineSeparator() + + " EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEM EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEME " + System.lineSeparator() + + " MEMEMEMEMEMEMEMEMEMEMEMEMEMEME EMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEMEME"; + + private static final String LOGONAME = + " ____ _ __ __ _ " + System.lineSeparator() + + " / ____|_ _____ _ __ | |_| \\/ | ___ ___| |__ " + System.lineSeparator() + + " | __|\\ \\ / / _ | '_ \\| __| |\\/| |/ _ |/ __| '_ \\ " + System.lineSeparator() + + " | |___ \\ V / __| | | | |_| | | | __|\\__ \\ | | |" + System.lineSeparator() + + " \\ ____| \\_/ \\___|_| |_|\\__|_| |_|\\___||___/_| |_|"; + + public static void generateBanner() { + String banner = + System.lineSeparator() + + System.lineSeparator() + + LOGO + + System.lineSeparator() + + LOGONAME + + System.lineSeparator(); + if (log.isInfoEnabled()) { + log.info(banner); + } else { + System.out.print(banner); + } + } + +} diff --git a/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/util/RuntimeUtils.java b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/util/RuntimeUtils.java new file mode 100644 index 0000000000..844a9638a3 --- /dev/null +++ b/eventmesh-runtime-v2/src/main/java/org/apache/eventmesh/runtime/util/RuntimeUtils.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.util; + +import java.util.Random; + +public class RuntimeUtils { + + public static String getRandomAdminServerAddr(String adminServerAddrList) { + String[] addresses = adminServerAddrList.split(";"); + if (addresses.length == 0) { + throw new IllegalArgumentException("Admin server address list is empty"); + } + Random random = new Random(); + int randomIndex = random.nextInt(addresses.length); + return addresses[randomIndex]; + } + +} diff --git a/eventmesh-runtime-v2/src/main/resources/connector.yaml b/eventmesh-runtime-v2/src/main/resources/connector.yaml new file mode 100644 index 0000000000..3e407fa3e9 --- /dev/null +++ b/eventmesh-runtime-v2/src/main/resources/connector.yaml @@ -0,0 +1,23 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +taskID: 9c18a0d2-7a61-482c-8275-34f8c2786cea +jobID: a01fd5e1-d295-4b89-99bc-0ae23eb85acf +region: region1 +runtimeConfig: # this used for connector runtime config + offsetStoragePluginType: admin + offsetStorageAddr: "127.0.0.1:8081;127.0.0.1:8081" \ No newline at end of file diff --git a/eventmesh-runtime-v2/src/main/resources/function.yaml b/eventmesh-runtime-v2/src/main/resources/function.yaml new file mode 100644 index 0000000000..eae2b063ec --- /dev/null +++ b/eventmesh-runtime-v2/src/main/resources/function.yaml @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +taskID: c6233632-ab9a-4aba-904f-9d22fba6aa74 +jobID: 8190fe5b-1f9b-4815-8983-2467e76edbf0 +region: region1 + diff --git a/eventmesh-runtime-v2/src/main/resources/runtime.yaml b/eventmesh-runtime-v2/src/main/resources/runtime.yaml new file mode 100644 index 0000000000..9ac36f27b0 --- /dev/null +++ b/eventmesh-runtime-v2/src/main/resources/runtime.yaml @@ -0,0 +1,24 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +componentType: CONNECTOR +registryEnabled: false +registryServerAddr: 127.0.0.1:8085 +registryPluginType: nacos +storagePluginType: memory +adminServiceName: eventmesh-admin +adminServiceAddr: "127.0.0.1:8081;127.0.0.1:8081" diff --git a/eventmesh-runtime/build.gradle b/eventmesh-runtime/build.gradle index 95924faad4..b016e18bfe 100644 --- a/eventmesh-runtime/build.gradle +++ b/eventmesh-runtime/build.gradle @@ -36,8 +36,10 @@ dependencies { implementation "commons-validator:commons-validator" implementation project(":eventmesh-common") - implementation project(":eventmesh-filter") implementation project(":eventmesh-spi") + implementation project(":eventmesh-function:eventmesh-function-api") + implementation project(":eventmesh-function:eventmesh-function-filter") + implementation project(":eventmesh-function:eventmesh-function-transformer") implementation project(":eventmesh-storage-plugin:eventmesh-storage-api") implementation project(":eventmesh-storage-plugin:eventmesh-storage-standalone") implementation project(":eventmesh-storage-plugin:eventmesh-storage-rocketmq") @@ -45,7 +47,6 @@ dependencies { implementation project(":eventmesh-security-plugin:eventmesh-security-acl") implementation project(":eventmesh-security-plugin:eventmesh-security-auth-http-basic") implementation project(":eventmesh-security-plugin:eventmesh-security-auth-token") - implementation project(":eventmesh-transformer") implementation project(":eventmesh-meta:eventmesh-meta-api") implementation project(":eventmesh-meta:eventmesh-meta-nacos") implementation project(":eventmesh-protocol-plugin:eventmesh-protocol-api") diff --git a/eventmesh-runtime/conf/admin-server.jks b/eventmesh-runtime/conf/admin-server.jks new file mode 100644 index 0000000000..92deb897a4 Binary files /dev/null and b/eventmesh-runtime/conf/admin-server.jks differ diff --git a/eventmesh-runtime/conf/eventmesh.properties b/eventmesh-runtime/conf/eventmesh.properties index 4984181ca8..9c2e5ee0d4 100644 --- a/eventmesh-runtime/conf/eventmesh.properties +++ b/eventmesh-runtime/conf/eventmesh.properties @@ -24,11 +24,9 @@ eventMesh.sysid=0000 eventMesh.server.tcp.port=10000 eventMesh.server.http.port=10105 eventMesh.server.grpc.port=10205 -# HTTP Admin Server eventMesh.server.admin.http.port=10106 -########################## EventMesh TCP Configuration ########################## -eventMesh.server.tcp.enabled=true +########################## EventMesh Network Configuration ########################## eventMesh.server.tcp.readerIdleSeconds=120 eventMesh.server.tcp.writerIdleSeconds=120 eventMesh.server.tcp.allIdleSeconds=120 @@ -64,10 +62,30 @@ eventMesh.server.retry.plugin.type=default eventMesh.server.gracefulShutdown.sleepIntervalInMills=1000 eventMesh.server.rebalanceRedirect.sleepIntervalInMills=200 +# TLS +eventMesh.server.useTls.enabled=false +eventMesh.server.ssl.protocol=TLSv1.1 +eventMesh.server.ssl.cer=sChat2.jks +eventMesh.server.ssl.pass=sNetty + # ip address blacklist eventMesh.server.blacklist.ipv4=0.0.0.0/8,127.0.0.0/8,169.254.0.0/16,255.255.255.255/32 eventMesh.server.blacklist.ipv6=::/128,::1/128,ff00::/8 +########################## EventMesh HTTP Admin Configuration ########################## +# thread pool +eventMesh.server.admin.threads.num=2 + +# TLS +eventMesh.server.admin.useTls.enabled=false +eventMesh.server.admin.ssl.protocol=TLSv1.3 +eventMesh.server.admin.ssl.cer=admin-server.jks +eventMesh.server.admin.ssl.pass=eventmesh-admin-server + +# ip address blacklist +eventMesh.server.admin.blacklist.ipv4=0.0.0.0/8,127.0.0.0/8,169.254.0.0/16,255.255.255.255/32 +eventMesh.server.admin.blacklist.ipv6=::/128,::1/128,ff00::/8 + ########################## EventMesh Plugin Configuration ########################## # storage plugin eventMesh.storage.plugin.type=standalone diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/AdminHandlerManager.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/AdminHandlerManager.java index 1da928dc10..35d01a5e3a 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/AdminHandlerManager.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/AdminHandlerManager.java @@ -24,7 +24,6 @@ import org.apache.eventmesh.runtime.admin.handler.v1.HTTPClientHandler; import org.apache.eventmesh.runtime.admin.handler.v1.InsertWebHookConfigHandler; import org.apache.eventmesh.runtime.admin.handler.v1.MetaHandler; -import org.apache.eventmesh.runtime.admin.handler.v1.MetricsHandler; import org.apache.eventmesh.runtime.admin.handler.v1.QueryRecommendEventMeshHandler; import org.apache.eventmesh.runtime.admin.handler.v1.QueryWebHookConfigByIdHandler; import org.apache.eventmesh.runtime.admin.handler.v1.QueryWebHookConfigByManufacturerHandler; @@ -100,7 +99,6 @@ public void registerHttpHandler() { eventMeshTCPServer.getEventMeshTCPConfiguration(), eventMeshHTTPServer.getEventMeshHttpConfiguration(), eventMeshGrpcServer.getEventMeshGrpcConfiguration())); - initHandler(new MetricsHandler(eventMeshHTTPServer, eventMeshTCPServer)); initHandler(new TopicHandler(eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshStoragePluginType())); initHandler(new EventHandler(eventMeshTCPServer.getEventMeshTCPConfiguration().getEventMeshStoragePluginType())); initHandler(new MetaHandler(eventMeshMetaStorage)); diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/MetricsHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/MetricsHandler.java deleted file mode 100644 index a2be02aaf1..0000000000 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v1/MetricsHandler.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.runtime.admin.handler.v1; - -import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.runtime.admin.handler.AbstractHttpHandler; -import org.apache.eventmesh.runtime.admin.response.v1.GetMetricsResponse; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; -import org.apache.eventmesh.runtime.boot.EventMeshTCPServer; -import org.apache.eventmesh.runtime.common.EventMeshHttpHandler; -import org.apache.eventmesh.runtime.metrics.http.HttpMetrics; -import org.apache.eventmesh.runtime.metrics.tcp.TcpMetrics; - -import java.io.IOException; - -import io.netty.channel.ChannelHandlerContext; -import io.netty.handler.codec.http.HttpRequest; - -import lombok.extern.slf4j.Slf4j; - -/** - * This class handles the {@code /metrics} endpoint, corresponding to the {@code eventmesh-dashboard} path {@code /metrics}. - *

- * This handler is responsible for retrieving summary information of metrics, including HTTP and TCP metrics. - * - * @see AbstractHttpHandler - */ - -@Slf4j -@EventMeshHttpHandler(path = "/metrics") -public class MetricsHandler extends AbstractHttpHandler { - - private final HttpMetrics httpMetrics; - private final TcpMetrics tcpMetrics; - - /** - * Constructs a new instance with the provided EventMesh server instance. - * - * @param eventMeshHTTPServer the HTTP server instance of EventMesh - * @param eventMeshTcpServer the TCP server instance of EventMesh - */ - public MetricsHandler(EventMeshHTTPServer eventMeshHTTPServer, - EventMeshTCPServer eventMeshTcpServer) { - super(); - this.httpMetrics = eventMeshHTTPServer.getEventMeshHttpMetricsManager().getHttpMetrics(); - this.tcpMetrics = eventMeshTcpServer.getEventMeshTcpMetricsManager().getTcpMetrics(); - } - - @Override - protected void get(HttpRequest httpRequest, ChannelHandlerContext ctx) throws IOException { - GetMetricsResponse getMetricsResponse = new GetMetricsResponse( - httpMetrics.maxHTTPTPS(), - httpMetrics.avgHTTPTPS(), - httpMetrics.maxHTTPCost(), - httpMetrics.avgHTTPCost(), - httpMetrics.avgHTTPBodyDecodeCost(), - httpMetrics.getHttpDiscard(), - httpMetrics.maxSendBatchMsgTPS(), - httpMetrics.avgSendBatchMsgTPS(), - httpMetrics.getSendBatchMsgNumSum(), - httpMetrics.getSendBatchMsgFailNumSum(), - httpMetrics.getSendBatchMsgFailRate(), - httpMetrics.getSendBatchMsgDiscardNumSum(), - httpMetrics.maxSendMsgTPS(), - httpMetrics.avgSendMsgTPS(), - httpMetrics.getSendMsgNumSum(), - httpMetrics.getSendMsgFailNumSum(), - httpMetrics.getSendMsgFailRate(), - httpMetrics.getReplyMsgNumSum(), - httpMetrics.getReplyMsgFailNumSum(), - httpMetrics.maxPushMsgTPS(), - httpMetrics.avgPushMsgTPS(), - httpMetrics.getHttpPushMsgNumSum(), - httpMetrics.getHttpPushFailNumSum(), - httpMetrics.getHttpPushMsgFailRate(), - httpMetrics.maxHTTPPushLatency(), - httpMetrics.avgHTTPPushLatency(), - httpMetrics.getBatchMsgQueueSize(), - httpMetrics.getSendMsgQueueSize(), - httpMetrics.getPushMsgQueueSize(), - httpMetrics.getHttpRetryQueueSize(), - httpMetrics.avgBatchSendMsgCost(), - httpMetrics.avgSendMsgCost(), - httpMetrics.avgReplyMsgCost(), - - tcpMetrics.getRetrySize(), - tcpMetrics.getClient2eventMeshTPS(), - tcpMetrics.getEventMesh2mqTPS(), - tcpMetrics.getMq2eventMeshTPS(), - tcpMetrics.getEventMesh2clientTPS(), - tcpMetrics.getAllTPS(), - tcpMetrics.getAllConnections(), - tcpMetrics.getSubTopicNum() - ); - String result = JsonUtils.toJSONString(getMetricsResponse); - writeJson(ctx, result); - } -} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v2/ConfigurationHandler.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v2/ConfigurationHandler.java index 1c149ca521..234e0e79cb 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v2/ConfigurationHandler.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/admin/handler/v2/ConfigurationHandler.java @@ -27,6 +27,7 @@ import org.apache.eventmesh.runtime.configuration.EventMeshGrpcConfiguration; import org.apache.eventmesh.runtime.configuration.EventMeshHTTPConfiguration; import org.apache.eventmesh.runtime.configuration.EventMeshTCPConfiguration; +import org.apache.eventmesh.runtime.constants.EventMeshVersion; import org.apache.eventmesh.runtime.util.HttpRequestUtil; import java.lang.reflect.Field; @@ -126,7 +127,7 @@ protected void get(HttpRequest httpRequest, ChannelHandlerContext ctx) { eventMeshTCPConfiguration, eventMeshHTTPConfiguration, eventMeshGrpcConfiguration, - "v1.10.0-release" // TODO get version number after merging https://github.com/apache/eventmesh/pull/4055 + EventMeshVersion.getCurrentVersionDesc() ); String json = JSON.toJSONString(Result.success(getConfigurationResponse), filters.toArray(new Filter[0])); writeJson(ctx, json); diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/AbstractHTTPServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/AbstractHTTPServer.java index 97f05a8764..3fcca51832 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/AbstractHTTPServer.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/AbstractHTTPServer.java @@ -86,6 +86,8 @@ import io.netty.util.ReferenceCountUtil; import io.opentelemetry.api.trace.Span; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; /** @@ -96,6 +98,8 @@ public abstract class AbstractHTTPServer extends AbstractRemotingServer { private final transient EventMeshHTTPConfiguration eventMeshHttpConfiguration; + @Getter + @Setter private EventMeshHttpMetricsManager eventMeshHttpMetricsManager; private static final DefaultHttpDataFactory DEFAULT_HTTP_DATA_FACTORY = new DefaultHttpDataFactory(false); @@ -105,8 +109,14 @@ public abstract class AbstractHTTPServer extends AbstractRemotingServer { } protected final transient AtomicBoolean started = new AtomicBoolean(false); + + @Getter private final transient boolean useTLS; + + @Getter + @Setter private Boolean useTrace = false; // Determine whether trace is enabled + private static final int MAX_CONNECTIONS = 20_000; /** @@ -118,10 +128,13 @@ public abstract class AbstractHTTPServer extends AbstractRemotingServer { private HttpConnectionHandler httpConnectionHandler; private HttpDispatcher httpDispatcher; + @Setter + @Getter private HandlerService handlerService; private final transient ThreadPoolExecutor asyncContextCompleteHandler = ThreadPoolFactory.createThreadPoolExecutor(10, 10, "EventMesh-http-asyncContext"); + @Getter private final HTTPThreadPoolGroup httpThreadPoolGroup; public AbstractHTTPServer(final int port, final boolean useTLS, @@ -523,32 +536,4 @@ protected void initChannel(final SocketChannel channel) { httpDispatcher); } } - - public void setUseTrace(final Boolean useTrace) { - this.useTrace = useTrace; - } - - public Boolean getUseTrace() { - return useTrace; - } - - public EventMeshHttpMetricsManager getEventMeshHttpMetricsManager() { - return eventMeshHttpMetricsManager; - } - - public void setEventMeshHttpMetricsManager(final EventMeshHttpMetricsManager eventMeshHttpMetricsManager) { - this.eventMeshHttpMetricsManager = eventMeshHttpMetricsManager; - } - - public HTTPThreadPoolGroup getHttpThreadPoolGroup() { - return httpThreadPoolGroup; - } - - public HandlerService getHandlerService() { - return handlerService; - } - - public void setHandlerService(HandlerService handlerService) { - this.handlerService = handlerService; - } } diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/AbstractRemotingServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/AbstractRemotingServer.java index fb09116620..e02637ec39 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/AbstractRemotingServer.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/AbstractRemotingServer.java @@ -22,7 +22,6 @@ import org.apache.eventmesh.common.utils.ThreadUtils; import org.apache.eventmesh.runtime.core.protocol.producer.ProducerManager; -import java.util.Objects; import java.util.concurrent.TimeUnit; import io.netty.channel.EventLoopGroup; @@ -31,22 +30,32 @@ import io.netty.channel.nio.NioEventLoopGroup; import io.netty.util.concurrent.EventExecutorGroup; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; /** * The most basic server */ @Slf4j +@Getter public abstract class AbstractRemotingServer implements RemotingServer { private static final int MAX_THREADS = Runtime.getRuntime().availableProcessors(); private static final int DEFAULT_SLEEP_SECONDS = 30; + @Setter private EventLoopGroup bossGroup; + + @Setter private EventLoopGroup ioGroup; + + @Setter private EventExecutorGroup workerGroup; + protected ProducerManager producerManager; + @Setter private int port; protected void buildBossGroup(final String threadPrefix) { @@ -75,10 +84,6 @@ protected void initProducerManager() throws Exception { producerManager.init(); } - public ProducerManager getProducerManager() { - return producerManager; - } - public void init(final String threadPrefix) throws Exception { buildBossGroup(threadPrefix); buildIOGroup(threadPrefix); @@ -94,16 +99,16 @@ public void shutdown() throws Exception { bossGroup.shutdownGracefully(); log.info("shutdown bossGroup"); } - if (Objects.isNull(producerManager)) { + if (producerManager != null) { producerManager.shutdown(); } + ThreadUtils.randomPause(TimeUnit.SECONDS.toMillis(DEFAULT_SLEEP_SECONDS)); if (ioGroup != null) { ioGroup.shutdownGracefully(); log.info("shutdown ioGroup"); } - if (workerGroup != null) { workerGroup.shutdownGracefully(); @@ -114,36 +119,4 @@ public void shutdown() throws Exception { protected boolean useEpoll() { return SystemUtils.isLinuxPlatform() && Epoll.isAvailable(); } - - public EventLoopGroup getBossGroup() { - return bossGroup; - } - - public void setBossGroup(final EventLoopGroup bossGroup) { - this.bossGroup = bossGroup; - } - - public EventLoopGroup getIoGroup() { - return ioGroup; - } - - public void setIoGroup(final EventLoopGroup ioGroup) { - this.ioGroup = ioGroup; - } - - public EventExecutorGroup getWorkerGroup() { - return workerGroup; - } - - public void setWorkerGroup(final EventExecutorGroup workerGroup) { - this.workerGroup = workerGroup; - } - - public int getPort() { - return port; - } - - public void setPort(final int port) { - this.port = port; - } } diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshAdminBootstrap.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshAdminBootstrap.java index e125475ec8..b170afd3ab 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshAdminBootstrap.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshAdminBootstrap.java @@ -17,23 +17,38 @@ package org.apache.eventmesh.runtime.boot; +import static org.apache.eventmesh.common.Constants.ADMIN; + +import org.apache.eventmesh.common.config.ConfigService; +import org.apache.eventmesh.common.utils.ConfigurationContextUtil; +import org.apache.eventmesh.runtime.configuration.EventMeshAdminConfiguration; + +import lombok.Getter; + public class EventMeshAdminBootstrap implements EventMeshBootstrap { + @Getter private EventMeshAdminServer eventMeshAdminServer; - private EventMeshServer eventMeshServer; + private final EventMeshAdminConfiguration eventMeshAdminConfiguration; + + private final EventMeshServer eventMeshServer; public EventMeshAdminBootstrap(EventMeshServer eventMeshServer) { this.eventMeshServer = eventMeshServer; + + ConfigService configService = ConfigService.getInstance(); + this.eventMeshAdminConfiguration = configService.buildConfigInstance(EventMeshAdminConfiguration.class); + + ConfigurationContextUtil.putIfAbsent(ADMIN, eventMeshAdminConfiguration); } @Override public void init() throws Exception { if (eventMeshServer != null) { - eventMeshAdminServer = new EventMeshAdminServer(eventMeshServer); + eventMeshAdminServer = new EventMeshAdminServer(eventMeshServer, eventMeshAdminConfiguration); eventMeshAdminServer.init(); } - } @Override @@ -41,7 +56,6 @@ public void start() throws Exception { if (eventMeshAdminServer != null) { eventMeshAdminServer.start(); } - } @Override diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshAdminServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshAdminServer.java index 4abba74a54..5e98fc690b 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshAdminServer.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshAdminServer.java @@ -17,13 +17,22 @@ package org.apache.eventmesh.runtime.boot; +import org.apache.eventmesh.common.ThreadPoolFactory; +import org.apache.eventmesh.common.protocol.http.common.RequestCode; import org.apache.eventmesh.runtime.admin.handler.AdminHandlerManager; import org.apache.eventmesh.runtime.admin.handler.HttpHandler; +import org.apache.eventmesh.runtime.configuration.EventMeshAdminConfiguration; +import org.apache.eventmesh.runtime.core.protocol.http.processor.AdminMetricsProcessor; import org.apache.eventmesh.runtime.util.HttpResponseUtils; import java.net.URI; import java.util.Objects; import java.util.Optional; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; + +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLEngine; import io.netty.bootstrap.ServerBootstrap; import io.netty.channel.ChannelFutureListener; @@ -41,27 +50,37 @@ import io.netty.handler.codec.http.HttpRequestDecoder; import io.netty.handler.codec.http.HttpResponseEncoder; import io.netty.handler.codec.http.HttpResponseStatus; +import io.netty.handler.ssl.SslHandler; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; @Slf4j public class EventMeshAdminServer extends AbstractHTTPServer { + private final EventMeshAdminConfiguration eventMeshAdminConfiguration; + private HttpConnectionHandler httpConnectionHandler = new HttpConnectionHandler(); private AdminHandlerManager adminHandlerManager; - public EventMeshAdminServer(EventMeshServer eventMeshServer) { - super(eventMeshServer.getEventMeshTCPServer().getEventMeshTCPConfiguration().getEventMeshServerAdminPort(), false, - eventMeshServer.getEventMeshHTTPServer().getEventMeshHttpConfiguration()); + @Getter + private ThreadPoolExecutor adminMetricsExecutor; + + public EventMeshAdminServer(final EventMeshServer eventMeshServer, final EventMeshAdminConfiguration eventMeshAdminConfiguration) { + super(eventMeshAdminConfiguration.getEventMeshServerAdminPort(), + eventMeshAdminConfiguration.isEventMeshServerUseTls(), + eventMeshAdminConfiguration); + this.eventMeshAdminConfiguration = eventMeshAdminConfiguration; adminHandlerManager = new AdminHandlerManager(eventMeshServer); } - @Override public void init() throws Exception { super.init("eventMesh-admin"); + initThreadPool(); adminHandlerManager.registerHttpHandler(); + registerAdminRequestProcessor(); } @Override @@ -71,8 +90,9 @@ public void start() throws Exception { try { bootstrap.group(this.getBossGroup(), this.getIoGroup()) .channel(useEpoll() ? EpollServerSocketChannel.class : NioServerSocketChannel.class) - .childHandler(new AdminServerInitializer()) - .childOption(ChannelOption.SO_KEEPALIVE, Boolean.TRUE); + .childHandler(new AdminServerInitializer( + this.isUseTLS() ? SSLContextFactory.getSslContext(eventMeshAdminConfiguration) : null, this.isUseTLS())) + .childOption(ChannelOption.AUTO_CLOSE, Boolean.TRUE); log.info("AdminHttpServer[port={}] started.", this.getPort()); @@ -95,29 +115,38 @@ public void start() throws Exception { started.compareAndSet(false, true); } - public void parseHttpRequest(ChannelHandlerContext ctx, HttpRequest httpRequest) { - String uriStr = httpRequest.uri(); - URI uri = URI.create(uriStr); - Optional httpHandlerOpt = adminHandlerManager.getHttpHandler(uri.getPath()); - if (httpHandlerOpt.isPresent()) { - try { - httpHandlerOpt.get().handle(httpRequest, ctx); - } catch (Exception e) { - log.error("admin server channelRead error", e); - ctx.writeAndFlush(HttpResponseUtils.buildHttpResponse(Objects.requireNonNull(e.getMessage()), ctx, - HttpHeaderValues.APPLICATION_JSON, HttpResponseStatus.INTERNAL_SERVER_ERROR)).addListener(ChannelFutureListener.CLOSE); - } - } else { - ctx.writeAndFlush(HttpResponseUtils.createNotFound()).addListener(ChannelFutureListener.CLOSE); - } + private void initThreadPool() { + adminMetricsExecutor = ThreadPoolFactory.createThreadPoolExecutor( + eventMeshAdminConfiguration.getEventMeshServerAdminThreadNum(), + eventMeshAdminConfiguration.getEventMeshServerAdminThreadNum(), + new LinkedBlockingQueue<>(50), "eventMesh-admin-metrics", true); + } + + private void registerAdminRequestProcessor() { + final AdminMetricsProcessor adminMetricsProcessor = new AdminMetricsProcessor(this); + registerProcessor(RequestCode.ADMIN_METRICS.getRequestCode(), adminMetricsProcessor); } private class AdminServerInitializer extends ChannelInitializer { + private final transient SSLContext sslContext; + private final transient boolean useTLS; + + public AdminServerInitializer(final SSLContext sslContext, final boolean useTLS) { + this.sslContext = sslContext; + this.useTLS = useTLS; + } + @Override protected void initChannel(final SocketChannel channel) { final ChannelPipeline pipeline = channel.pipeline(); + if (sslContext != null && useTLS) { + final SSLEngine sslEngine = sslContext.createSSLEngine(); + sslEngine.setUseClientMode(false); + pipeline.addFirst(getWorkerGroup(), "ssl", new SslHandler(sslEngine)); + } + pipeline.addLast(getWorkerGroup(), new HttpRequestDecoder(), new HttpResponseEncoder(), @@ -131,5 +160,22 @@ protected void channelRead0(ChannelHandlerContext ctx, HttpRequest msg) { } }); } + + private void parseHttpRequest(ChannelHandlerContext ctx, HttpRequest httpRequest) { + String uriStr = httpRequest.uri(); + URI uri = URI.create(uriStr); + Optional httpHandlerOpt = adminHandlerManager.getHttpHandler(uri.getPath()); + if (httpHandlerOpt.isPresent()) { + try { + httpHandlerOpt.get().handle(httpRequest, ctx); + } catch (Exception e) { + log.error("admin server channelRead error", e); + ctx.writeAndFlush(HttpResponseUtils.buildHttpResponse(Objects.requireNonNull(e.getMessage()), ctx, + HttpHeaderValues.APPLICATION_JSON, HttpResponseStatus.INTERNAL_SERVER_ERROR)).addListener(ChannelFutureListener.CLOSE); + } + } else { + ctx.writeAndFlush(HttpResponseUtils.createNotFound()).addListener(ChannelFutureListener.CLOSE); + } + } } } diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshGrpcBootstrap.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshGrpcBootstrap.java index dc218084b1..b6e493c1bc 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshGrpcBootstrap.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshGrpcBootstrap.java @@ -23,10 +23,13 @@ import org.apache.eventmesh.common.utils.ConfigurationContextUtil; import org.apache.eventmesh.runtime.configuration.EventMeshGrpcConfiguration; +import lombok.Getter; + public class EventMeshGrpcBootstrap implements EventMeshBootstrap { private final EventMeshGrpcConfiguration eventMeshGrpcConfiguration; + @Getter private EventMeshGrpcServer eventMeshGrpcServer; private final EventMeshServer eventMeshServer; @@ -62,12 +65,4 @@ public void shutdown() throws Exception { eventMeshGrpcServer.shutdown(); } } - - public EventMeshGrpcServer getEventMeshGrpcServer() { - return eventMeshGrpcServer; - } - - public void setEventMeshGrpcServer(EventMeshGrpcServer eventMeshGrpcServer) { - this.eventMeshGrpcServer = eventMeshGrpcServer; - } } diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshHTTPServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshHTTPServer.java index cc47cc9727..1089a1cafb 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshHTTPServer.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshHTTPServer.java @@ -31,7 +31,6 @@ import org.apache.eventmesh.runtime.constants.EventMeshConstants; import org.apache.eventmesh.runtime.core.consumer.SubscriptionManager; import org.apache.eventmesh.runtime.core.protocol.http.consumer.ConsumerManager; -import org.apache.eventmesh.runtime.core.protocol.http.processor.AdminMetricsProcessor; import org.apache.eventmesh.runtime.core.protocol.http.processor.BatchSendMessageProcessor; import org.apache.eventmesh.runtime.core.protocol.http.processor.BatchSendMessageV2Processor; import org.apache.eventmesh.runtime.core.protocol.http.processor.CreateTopicProcessor; @@ -68,6 +67,7 @@ import com.google.common.eventbus.EventBus; import com.google.common.util.concurrent.RateLimiter; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -75,6 +75,7 @@ * Add multiple managers to the underlying server */ @Slf4j +@Getter public class EventMeshHTTPServer extends AbstractHTTPServer { private final EventMeshServer eventMeshServer; @@ -95,7 +96,6 @@ public class EventMeshHTTPServer extends AbstractHTTPServer { private transient RateLimiter batchRateLimiter; public EventMeshHTTPServer(final EventMeshServer eventMeshServer, final EventMeshHTTPConfiguration eventMeshHttpConfiguration) { - super(eventMeshHttpConfiguration.getHttpServerPort(), eventMeshHttpConfiguration.isEventMeshServerUseTls(), eventMeshHttpConfiguration); @@ -103,7 +103,6 @@ public EventMeshHTTPServer(final EventMeshServer eventMeshServer, final EventMes this.eventMeshHttpConfiguration = eventMeshHttpConfiguration; this.metaStorage = eventMeshServer.getMetaStorage(); this.acl = eventMeshServer.getAcl(); - } public void init() throws Exception { @@ -251,9 +250,6 @@ private void registerHTTPRequestProcessor() throws Exception { final SendAsyncRemoteEventProcessor sendAsyncRemoteEventProcessor = new SendAsyncRemoteEventProcessor(this); this.getHandlerService().register(sendAsyncRemoteEventProcessor); - final AdminMetricsProcessor adminMetricsProcessor = new AdminMetricsProcessor(this); - registerProcessor(RequestCode.ADMIN_METRICS.getRequestCode(), adminMetricsProcessor); - final HeartBeatProcessor heartProcessor = new HeartBeatProcessor(this); registerProcessor(RequestCode.HEARTBEAT.getRequestCode(), heartProcessor); @@ -299,62 +295,4 @@ private void registerWebhook() throws Exception { this.getHandlerService().register(webHookProcessor, super.getHttpThreadPoolGroup().getWebhookExecutor()); } - - public SubscriptionManager getSubscriptionManager() { - return subscriptionManager; - } - - public ConsumerManager getConsumerManager() { - return consumerManager; - } - - public ProducerManager getProducerManager() { - return producerManager; - } - - public EventMeshHTTPConfiguration getEventMeshHttpConfiguration() { - return eventMeshHttpConfiguration; - } - - public EventBus getEventBus() { - return eventBus; - } - - public HttpRetryer getHttpRetryer() { - return httpRetryer; - } - - public Acl getAcl() { - return acl; - } - - public EventMeshServer getEventMeshServer() { - return eventMeshServer; - } - - public RateLimiter getMsgRateLimiter() { - return msgRateLimiter; - } - - public RateLimiter getBatchRateLimiter() { - return batchRateLimiter; - } - - public FilterEngine getFilterEngine() { - return filterEngine; - } - - public TransformerEngine getTransformerEngine() { - return transformerEngine; - } - - public MetaStorage getMetaStorage() { - return metaStorage; - } - - public HTTPClientPool getHttpClientPool() { - return httpClientPool; - } - - } diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshHttpBootstrap.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshHttpBootstrap.java index 87ce3c23a4..df3b227e42 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshHttpBootstrap.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshHttpBootstrap.java @@ -23,12 +23,15 @@ import org.apache.eventmesh.common.utils.ConfigurationContextUtil; import org.apache.eventmesh.runtime.configuration.EventMeshHTTPConfiguration; -public class EventMeshHttpBootstrap implements EventMeshBootstrap { +import lombok.Getter; - private final EventMeshHTTPConfiguration eventMeshHttpConfiguration; +public class EventMeshHttpBootstrap implements EventMeshBootstrap { + @Getter public EventMeshHTTPServer eventMeshHttpServer; + private final EventMeshHTTPConfiguration eventMeshHttpConfiguration; + private final EventMeshServer eventMeshServer; public EventMeshHttpBootstrap(final EventMeshServer eventMeshServer) { @@ -64,12 +67,4 @@ public void shutdown() throws Exception { eventMeshHttpServer.shutdown(); } } - - public EventMeshHTTPServer getEventMeshHttpServer() { - return eventMeshHttpServer; - } - - public void setEventMeshHttpServer(EventMeshHTTPServer eventMeshHttpServer) { - this.eventMeshHttpServer = eventMeshHttpServer; - } } diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshServer.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshServer.java index b3250795d6..d61580b9c8 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshServer.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshServer.java @@ -43,23 +43,32 @@ import java.util.concurrent.CopyOnWriteArrayList; import java.util.stream.Collectors; +import lombok.Getter; +import lombok.Setter; import lombok.extern.slf4j.Slf4j; @Slf4j public class EventMeshServer { + @Getter private final Acl acl; + @Getter + @Setter private MetaStorage metaStorage; + @Getter private static Trace trace; private final StorageResource storageResource; + @Getter private ServiceState serviceState; + @Getter private ProducerTopicManager producerTopicManager; + @Getter private final CommonConfiguration configuration; // private transient ClientManageController clientManageController; @@ -70,13 +79,17 @@ public class EventMeshServer { private static final ConfigService configService = ConfigService.getInstance(); - private EventMeshAdminBootstrap adminBootstrap; - + @Getter private EventMeshTCPServer eventMeshTCPServer = null; + @Getter + private EventMeshHTTPServer eventMeshHTTPServer = null; + + @Getter private EventMeshGrpcServer eventMeshGrpcServer = null; - private EventMeshHTTPServer eventMeshHTTPServer = null; + @Getter + private EventMeshAdminServer eventMeshAdminServer = null; private EventMeshMetricsManager eventMeshMetricsManager; @@ -105,8 +118,7 @@ public EventMeshServer() { case GRPC: BOOTSTRAP_LIST.add(new EventMeshGrpcBootstrap(this)); break; - default: - // nothing to do + default: // nothing to do } } @@ -115,6 +127,9 @@ public EventMeshServer() { BOOTSTRAP_LIST.add(new EventMeshTcpBootstrap(this)); } + // HTTP Admin Server always enabled + BOOTSTRAP_LIST.add(new EventMeshAdminBootstrap(this)); + List metricsPluginTypes = configuration.getEventMeshMetricsPluginType(); if (CollectionUtils.isNotEmpty(metricsPluginTypes)) { List metricsRegistries = metricsPluginTypes.stream().map(metric -> MetricsPluginFactory.getMetricsRegistry(metric)) @@ -147,6 +162,9 @@ public void init() throws Exception { if (eventMeshBootstrap instanceof EventMeshGrpcBootstrap) { eventMeshGrpcServer = ((EventMeshGrpcBootstrap) eventMeshBootstrap).getEventMeshGrpcServer(); } + if (eventMeshBootstrap instanceof EventMeshAdminBootstrap) { + eventMeshAdminServer = ((EventMeshAdminBootstrap) eventMeshBootstrap).getEventMeshAdminServer(); + } } if (Objects.nonNull(eventMeshTCPServer)) { @@ -168,12 +186,6 @@ public void init() throws Exception { eventMeshMetricsManager.init(); } - - if (Objects.nonNull(eventMeshTCPServer) && Objects.nonNull(eventMeshHTTPServer) && Objects.nonNull(eventMeshGrpcServer)) { - adminBootstrap = new EventMeshAdminBootstrap(this); - adminBootstrap.init(); - } - producerTopicManager = new ProducerTopicManager(this); producerTopicManager.init(); @@ -203,13 +215,10 @@ public void start() throws Exception { eventMeshBootstrap.start(); } - if (Objects.nonNull(adminBootstrap)) { - adminBootstrap.start(); - } producerTopicManager.start(); + serviceState = ServiceState.RUNNING; log.info(SERVER_STATE_MSG, serviceState); - } public void shutdown() throws Exception { @@ -235,48 +244,8 @@ public void shutdown() throws Exception { } producerTopicManager.shutdown(); ConfigurationContextUtil.clear(); - serviceState = ServiceState.STOPPED; + serviceState = ServiceState.STOPPED; log.info(SERVER_STATE_MSG, serviceState); } - - public static Trace getTrace() { - return trace; - } - - public ServiceState getServiceState() { - return serviceState; - } - - public MetaStorage getMetaStorage() { - return metaStorage; - } - - public void setMetaStorage(final MetaStorage metaStorage) { - this.metaStorage = metaStorage; - } - - public Acl getAcl() { - return acl; - } - - public ProducerTopicManager getProducerTopicManager() { - return producerTopicManager; - } - - public CommonConfiguration getConfiguration() { - return configuration; - } - - public EventMeshTCPServer getEventMeshTCPServer() { - return eventMeshTCPServer; - } - - public EventMeshGrpcServer getEventMeshGrpcServer() { - return eventMeshGrpcServer; - } - - public EventMeshHTTPServer getEventMeshHTTPServer() { - return eventMeshHTTPServer; - } } diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshTcpBootstrap.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshTcpBootstrap.java index 9cd665d28a..e098b203c4 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshTcpBootstrap.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/EventMeshTcpBootstrap.java @@ -23,8 +23,11 @@ import org.apache.eventmesh.common.utils.ConfigurationContextUtil; import org.apache.eventmesh.runtime.configuration.EventMeshTCPConfiguration; +import lombok.Getter; + public class EventMeshTcpBootstrap implements EventMeshBootstrap { + @Getter private EventMeshTCPServer eventMeshTcpServer; private final EventMeshTCPConfiguration eventMeshTcpConfiguration; @@ -63,13 +66,4 @@ public void shutdown() throws Exception { eventMeshTcpServer.shutdown(); } } - - public EventMeshTCPServer getEventMeshTcpServer() { - return eventMeshTcpServer; - } - - public void setEventMeshTcpServer(EventMeshTCPServer eventMeshTcpServer) { - this.eventMeshTcpServer = eventMeshTcpServer; - } - } diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/FilterEngine.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/FilterEngine.java index bf6eb9dadc..14677dc690 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/FilterEngine.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/FilterEngine.java @@ -19,8 +19,8 @@ import org.apache.eventmesh.api.meta.MetaServiceListener; import org.apache.eventmesh.common.utils.JsonUtils; -import org.apache.eventmesh.filter.pattern.Pattern; -import org.apache.eventmesh.filter.patternbuild.PatternBuilder; +import org.apache.eventmesh.function.filter.pattern.Pattern; +import org.apache.eventmesh.function.filter.patternbuild.PatternBuilder; import org.apache.eventmesh.runtime.core.protocol.http.consumer.ConsumerGroupManager; import org.apache.eventmesh.runtime.core.protocol.http.consumer.ConsumerManager; import org.apache.eventmesh.runtime.core.protocol.producer.EventMeshProducer; diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/HTTPThreadPoolGroup.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/HTTPThreadPoolGroup.java index bf6c740b56..aa11788501 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/HTTPThreadPoolGroup.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/HTTPThreadPoolGroup.java @@ -23,17 +23,25 @@ import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; +import lombok.Getter; + public class HTTPThreadPoolGroup implements ThreadPoolGroup { private final EventMeshHTTPConfiguration eventMeshHttpConfiguration; + @Getter private ThreadPoolExecutor batchMsgExecutor; + @Getter private ThreadPoolExecutor sendMsgExecutor; + @Getter private ThreadPoolExecutor remoteMsgExecutor; + @Getter private ThreadPoolExecutor replyMsgExecutor; + @Getter private ThreadPoolExecutor pushMsgExecutor; + @Getter private ThreadPoolExecutor clientManageExecutor; - private ThreadPoolExecutor runtimeAdminExecutor; + @Getter private ThreadPoolExecutor webhookExecutor; public HTTPThreadPoolGroup(EventMeshHTTPConfiguration eventMeshHttpConfiguration) { @@ -73,12 +81,6 @@ public void initThreadPool() { new LinkedBlockingQueue<>(eventMeshHttpConfiguration.getEventMeshServerClientManageBlockQSize()), "eventMesh-clientManage", true); - // The runtimeAdminExecutor here is for the runtime.admin package. - runtimeAdminExecutor = ThreadPoolFactory.createThreadPoolExecutor( - eventMeshHttpConfiguration.getEventMeshServerAdminThreadNum(), - eventMeshHttpConfiguration.getEventMeshServerAdminThreadNum(), - new LinkedBlockingQueue<>(50), "eventMesh-runtime-admin", true); - replyMsgExecutor = ThreadPoolFactory.createThreadPoolExecutor( eventMeshHttpConfiguration.getEventMeshServerReplyMsgThreadNum(), eventMeshHttpConfiguration.getEventMeshServerReplyMsgThreadNum(), @@ -95,9 +97,6 @@ public void shutdownThreadPool() { if (batchMsgExecutor != null) { batchMsgExecutor.shutdown(); } - if (runtimeAdminExecutor != null) { - runtimeAdminExecutor.shutdown(); - } if (clientManageExecutor != null) { clientManageExecutor.shutdown(); } @@ -114,36 +113,4 @@ public void shutdownThreadPool() { replyMsgExecutor.shutdown(); } } - - public ThreadPoolExecutor getBatchMsgExecutor() { - return batchMsgExecutor; - } - - public ThreadPoolExecutor getSendMsgExecutor() { - return sendMsgExecutor; - } - - public ThreadPoolExecutor getRemoteMsgExecutor() { - return remoteMsgExecutor; - } - - public ThreadPoolExecutor getReplyMsgExecutor() { - return replyMsgExecutor; - } - - public ThreadPoolExecutor getPushMsgExecutor() { - return pushMsgExecutor; - } - - public ThreadPoolExecutor getClientManageExecutor() { - return clientManageExecutor; - } - - public ThreadPoolExecutor getRuntimeAdminExecutor() { - return runtimeAdminExecutor; - } - - public ThreadPoolExecutor getWebhookExecutor() { - return webhookExecutor; - } } diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/SSLContextFactory.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/SSLContextFactory.java index 0f48220a4d..a0736b430a 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/SSLContextFactory.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/SSLContextFactory.java @@ -17,6 +17,7 @@ package org.apache.eventmesh.runtime.boot; +import org.apache.eventmesh.runtime.configuration.EventMeshAdminConfiguration; import org.apache.eventmesh.runtime.configuration.EventMeshHTTPConfiguration; import org.apache.eventmesh.runtime.constants.EventMeshConstants; @@ -40,23 +41,19 @@ public class SSLContextFactory { - private static String protocol = "TLSv1.1"; - - private static String fileName; - - private static String password; - + /** + * {@link EventMeshAdminConfiguration} will be parsed into {@link EventMeshHTTPConfiguration}. + */ public static SSLContext getSslContext(final EventMeshHTTPConfiguration eventMeshHttpConfiguration) - throws NoSuchAlgorithmException, KeyStoreException, CertificateException, IOException, - UnrecoverableKeyException, KeyManagementException { + throws NoSuchAlgorithmException, KeyStoreException, CertificateException, IOException, UnrecoverableKeyException, KeyManagementException { + + String protocol = eventMeshHttpConfiguration.getEventMeshServerSSLProtocol(); + String fileName = eventMeshHttpConfiguration.getEventMeshServerSSLCer(); + String password = eventMeshHttpConfiguration.getEventMeshServerSSLPass(); SSLContext sslContext; - try (InputStream inputStream = Files.newInputStream(Paths.get(EventMeshConstants.EVENTMESH_CONF_HOME - + File.separator - + fileName), StandardOpenOption.READ)) { - protocol = eventMeshHttpConfiguration.getEventMeshServerSSLProtocol(); - fileName = eventMeshHttpConfiguration.getEventMeshServerSSLCer(); - password = eventMeshHttpConfiguration.getEventMeshServerSSLPass(); + try (InputStream inputStream = Files.newInputStream(Paths.get(EventMeshConstants.EVENTMESH_CONF_HOME + File.separator + fileName), + StandardOpenOption.READ)) { char[] filePass = StringUtils.isNotBlank(password) ? password.toCharArray() : new char[0]; final KeyStore keyStore = KeyStore.getInstance("JKS"); diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/TransformerEngine.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/TransformerEngine.java index 551bcb2799..1d2f8ca30c 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/TransformerEngine.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/boot/TransformerEngine.java @@ -19,14 +19,14 @@ import org.apache.eventmesh.api.meta.MetaServiceListener; import org.apache.eventmesh.common.utils.JsonUtils; +import org.apache.eventmesh.function.transformer.Transformer; +import org.apache.eventmesh.function.transformer.TransformerBuilder; +import org.apache.eventmesh.function.transformer.TransformerParam; import org.apache.eventmesh.runtime.core.protocol.http.consumer.ConsumerGroupManager; import org.apache.eventmesh.runtime.core.protocol.http.consumer.ConsumerManager; import org.apache.eventmesh.runtime.core.protocol.producer.EventMeshProducer; import org.apache.eventmesh.runtime.core.protocol.producer.ProducerManager; import org.apache.eventmesh.runtime.meta.MetaStorage; -import org.apache.eventmesh.transformer.Transformer; -import org.apache.eventmesh.transformer.TransformerBuilder; -import org.apache.eventmesh.transformer.TransformerParam; import org.apache.commons.lang3.StringUtils; diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshAdminConfiguration.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshAdminConfiguration.java new file mode 100644 index 0000000000..dff80eaaa2 --- /dev/null +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshAdminConfiguration.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.configuration; + +import org.apache.eventmesh.common.config.Config; +import org.apache.eventmesh.common.config.ConfigField; + +import java.util.Collections; +import java.util.List; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +import inet.ipaddr.IPAddress; + +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor +@Config(prefix = "eventMesh.server") +public class EventMeshAdminConfiguration extends EventMeshHTTPConfiguration { + + @ConfigField(field = "admin.http.port") + private int eventMeshServerAdminPort = 10106; + + @ConfigField(field = "admin.threads.num") + private int eventMeshServerAdminThreadNum = 2; + + @ConfigField(field = "admin.useTls.enabled") + private boolean eventMeshServerUseTls = false; + + @ConfigField(field = "admin.ssl.protocol") + private String eventMeshServerSSLProtocol = "TLSv1.3"; + + @ConfigField(field = "admin.ssl.cer") + private String eventMeshServerSSLCer = "admin-server.jks"; + + @ConfigField(field = "admin.ssl.pass") + private String eventMeshServerSSLPass = "eventmesh-admin-server"; + + @ConfigField(field = "admin.blacklist.ipv4") + private List eventMeshIpv4BlackList = Collections.emptyList(); // TODO implement after merging #4835 + + @ConfigField(field = "admin.blacklist.ipv6") + private List eventMeshIpv6BlackList = Collections.emptyList(); +} diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshGrpcConfiguration.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshGrpcConfiguration.java index cf08f27553..924a07ab01 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshGrpcConfiguration.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshGrpcConfiguration.java @@ -59,9 +59,6 @@ public class EventMeshGrpcConfiguration extends CommonConfiguration { @ConfigField(field = "metaStorage.threads.num") private int eventMeshServerMetaStorageThreadNum = 10; - @ConfigField(field = "admin.threads.num") - private int eventMeshServerAdminThreadNum = 2; - @ConfigField(field = "retry.threads.num") private int eventMeshServerRetryThreadNum = 2; diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshHTTPConfiguration.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshHTTPConfiguration.java index b0fa18df4c..287c222245 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshHTTPConfiguration.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshHTTPConfiguration.java @@ -63,9 +63,6 @@ public class EventMeshHTTPConfiguration extends CommonConfiguration { @ConfigField(field = "metaStorage.threads.num") private int eventMeshServerMetaStorageThreadNum = 10; - @ConfigField(field = "admin.threads.num") - private int eventMeshServerAdminThreadNum = 2; - @ConfigField(field = "retry.threads.num") private int eventMeshServerRetryThreadNum = 2; @@ -131,5 +128,4 @@ public class EventMeshHTTPConfiguration extends CommonConfiguration { @ConfigField(field = "blacklist.ipv6") private List eventMeshIpv6BlackList = Collections.emptyList(); - } diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshTCPConfiguration.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshTCPConfiguration.java index 891f986167..907d80f686 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshTCPConfiguration.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/configuration/EventMeshTCPConfiguration.java @@ -109,9 +109,6 @@ public class EventMeshTCPConfiguration extends CommonConfiguration { @ConfigField(field = "tcp.RebalanceIntervalInMills") private Integer eventMeshTcpRebalanceIntervalInMills = 30 * 1000; - @ConfigField(field = "admin.http.port") - private int eventMeshServerAdminPort = 10106; - @ConfigField(field = "tcp.sendBack.enabled") private boolean eventMeshTcpSendBackEnabled = Boolean.TRUE; diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/AdminMetricsProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/AdminMetricsProcessor.java index b3fbf0d6a8..9a8b369341 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/AdminMetricsProcessor.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/AdminMetricsProcessor.java @@ -18,7 +18,7 @@ package org.apache.eventmesh.runtime.core.protocol.http.processor; import org.apache.eventmesh.common.protocol.http.HttpCommand; -import org.apache.eventmesh.runtime.boot.EventMeshHTTPServer; +import org.apache.eventmesh.runtime.boot.EventMeshAdminServer; import org.apache.eventmesh.runtime.core.protocol.http.async.AsyncContext; import java.util.concurrent.Executor; @@ -31,7 +31,7 @@ @RequiredArgsConstructor public class AdminMetricsProcessor extends AbstractHttpRequestProcessor { - private final EventMeshHTTPServer eventMeshHTTPServer; + private final EventMeshAdminServer eventMeshAdminServer; @Override public void processRequest(ChannelHandlerContext ctx, AsyncContext asyncContext) throws Exception { @@ -39,6 +39,6 @@ public void processRequest(ChannelHandlerContext ctx, AsyncContext @Override public Executor executor() { - return eventMeshHTTPServer.getHttpThreadPoolGroup().getRuntimeAdminExecutor(); + return eventMeshAdminServer.getAdminMetricsExecutor(); } } diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendAsyncEventProcessor.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendAsyncEventProcessor.java index b30238a28c..0e41d827ab 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendAsyncEventProcessor.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/processor/SendAsyncEventProcessor.java @@ -31,7 +31,8 @@ import org.apache.eventmesh.common.utils.IPUtils; import org.apache.eventmesh.common.utils.JsonUtils; import org.apache.eventmesh.common.utils.RandomStringUtils; -import org.apache.eventmesh.filter.pattern.Pattern; +import org.apache.eventmesh.function.filter.pattern.Pattern; +import org.apache.eventmesh.function.transformer.Transformer; import org.apache.eventmesh.protocol.api.ProtocolAdaptor; import org.apache.eventmesh.protocol.api.ProtocolPluginFactory; import org.apache.eventmesh.runtime.acl.Acl; @@ -44,7 +45,6 @@ import org.apache.eventmesh.runtime.util.EventMeshUtil; import org.apache.eventmesh.runtime.util.RemotingHelper; import org.apache.eventmesh.trace.api.common.EventMeshTraceConstants; -import org.apache.eventmesh.transformer.Transformer; import org.apache.commons.lang3.StringUtils; diff --git a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/push/AsyncHTTPPushRequest.java b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/push/AsyncHTTPPushRequest.java index be95971536..69506ede8a 100644 --- a/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/push/AsyncHTTPPushRequest.java +++ b/eventmesh-runtime/src/main/java/org/apache/eventmesh/runtime/core/protocol/http/push/AsyncHTTPPushRequest.java @@ -30,14 +30,14 @@ import org.apache.eventmesh.common.utils.IPUtils; import org.apache.eventmesh.common.utils.JsonUtils; import org.apache.eventmesh.common.utils.RandomStringUtils; -import org.apache.eventmesh.filter.pattern.Pattern; +import org.apache.eventmesh.function.filter.pattern.Pattern; +import org.apache.eventmesh.function.transformer.Transformer; import org.apache.eventmesh.protocol.api.ProtocolAdaptor; import org.apache.eventmesh.protocol.api.ProtocolPluginFactory; import org.apache.eventmesh.runtime.constants.EventMeshConstants; import org.apache.eventmesh.runtime.core.protocol.http.consumer.HandleMsgContext; import org.apache.eventmesh.runtime.util.EventMeshUtil; import org.apache.eventmesh.runtime.util.WebhookUtil; -import org.apache.eventmesh.transformer.Transformer; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.time.DateFormatUtils; diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/boot/EventMeshServerTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/boot/EventMeshServerTest.java index c546d38e3b..83baf20fb9 100644 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/boot/EventMeshServerTest.java +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/boot/EventMeshServerTest.java @@ -77,7 +77,6 @@ private void assertTCPConfig(EventMeshTCPConfiguration config) { Assertions.assertEquals(15816, config.getEventMeshTcpMsgRetryAsyncDelayInMills()); Assertions.assertEquals(16816, config.getEventMeshTcpMsgRetryQueueSize()); Assertions.assertEquals(Integer.valueOf(17816), config.getEventMeshTcpRebalanceIntervalInMills()); - Assertions.assertEquals(18816, config.getEventMeshServerAdminPort()); Assertions.assertEquals(Boolean.TRUE, config.isEventMeshTcpSendBackEnabled()); Assertions.assertEquals(3, config.getEventMeshTcpSendBackMaxTimes()); Assertions.assertEquals(21816, config.getEventMeshTcpPushFailIsolateTimeInMills()); diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/configuration/EventMeshGrpcConfigurationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/configuration/EventMeshGrpcConfigurationTest.java index 6b206167dc..4dc59a2000 100644 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/configuration/EventMeshGrpcConfigurationTest.java +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/configuration/EventMeshGrpcConfigurationTest.java @@ -50,7 +50,6 @@ private void assertGrpcConfig(EventMeshGrpcConfiguration config) { Assertions.assertEquals(5816, config.getEventMeshServerReplyMsgThreadNum()); Assertions.assertEquals(6816, config.getEventMeshServerSubscribeMsgThreadNum()); Assertions.assertEquals(7816, config.getEventMeshServerMetaStorageThreadNum()); - Assertions.assertEquals(8816, config.getEventMeshServerAdminThreadNum()); Assertions.assertEquals(9816, config.getEventMeshServerRetryThreadNum()); Assertions.assertEquals(11816, config.getEventMeshServerPullMetaStorageInterval()); Assertions.assertEquals(12816, config.getEventMeshServerAsyncAccumulationThreshold()); diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/configuration/EventMeshHTTPConfigurationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/configuration/EventMeshHTTPConfigurationTest.java index d522ff5519..9e99bc511d 100644 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/configuration/EventMeshHTTPConfigurationTest.java +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/configuration/EventMeshHTTPConfigurationTest.java @@ -53,7 +53,6 @@ private void assertHTTPConfig(EventMeshHTTPConfiguration config) throws AddressS Assertions.assertEquals(5816, config.getEventMeshServerReplyMsgThreadNum()); Assertions.assertEquals(6816, config.getEventMeshServerClientManageThreadNum()); Assertions.assertEquals(7816, config.getEventMeshServerMetaStorageThreadNum()); - Assertions.assertEquals(8816, config.getEventMeshServerAdminThreadNum()); Assertions.assertEquals(9816, config.getEventMeshServerRetryThreadNum()); Assertions.assertEquals(11816, config.getEventMeshServerPullMetaStorageInterval()); diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/configuration/EventMeshTCPConfigurationTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/configuration/EventMeshTCPConfigurationTest.java index 1501cf1b5d..2aee84ef72 100644 --- a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/configuration/EventMeshTCPConfigurationTest.java +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/configuration/EventMeshTCPConfigurationTest.java @@ -58,7 +58,6 @@ private void assertTCPConfig(EventMeshTCPConfiguration config) { Assertions.assertEquals(15816, config.getEventMeshTcpMsgRetryAsyncDelayInMills()); Assertions.assertEquals(16816, config.getEventMeshTcpMsgRetryQueueSize()); Assertions.assertEquals(Integer.valueOf(17816), config.getEventMeshTcpRebalanceIntervalInMills()); - Assertions.assertEquals(18816, config.getEventMeshServerAdminPort()); Assertions.assertEquals(Boolean.TRUE, config.isEventMeshTcpSendBackEnabled()); Assertions.assertEquals(3, config.getEventMeshTcpSendBackMaxTimes()); Assertions.assertEquals(21816, config.getEventMeshTcpPushFailIsolateTimeInMills()); diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/EventMeshCloudEventWriterTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/EventMeshCloudEventWriterTest.java new file mode 100644 index 0000000000..e7f5cfcebe --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/EventMeshCloudEventWriterTest.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.util; + +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.OffsetDateTime; +import java.time.ZoneOffset; +import java.util.Map; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class EventMeshCloudEventWriterTest { + + @Test + public void testURIAsValueWithContextAttribute() throws URISyntaxException { + String key = "testKey"; + EventMeshCloudEventWriter eventMeshCloudEventWriter = new EventMeshCloudEventWriter(); + + eventMeshCloudEventWriter.withContextAttribute(key, new URI("file://foo/bars")); + + Map extensionMap = eventMeshCloudEventWriter.getExtensionMap(); + Assertions.assertEquals(extensionMap.get(key), "file://foo/bars"); + } + + @Test + public void testOffsetDateTimeAsValueWithContextAttribute() { + String key = "testKey"; + EventMeshCloudEventWriter eventMeshCloudEventWriter = new EventMeshCloudEventWriter(); + + eventMeshCloudEventWriter.withContextAttribute(key, OffsetDateTime.of(LocalDateTime.of( + LocalDate.of(2023, 6, 17), LocalTime.MIDNIGHT), ZoneOffset.ofTotalSeconds(32400))); + + Map extensionMap = eventMeshCloudEventWriter.getExtensionMap(); + Assertions.assertEquals(extensionMap.get(key), "2023-06-17T00:00:00+09:00"); + } + + @Test + public void testIntegerAsValueWithContextAttribute() { + String key = "testKey"; + EventMeshCloudEventWriter eventMeshCloudEventWriter = new EventMeshCloudEventWriter(); + + eventMeshCloudEventWriter.withContextAttribute(key, 123); + + Map extensionMap = eventMeshCloudEventWriter.getExtensionMap(); + Assertions.assertEquals(extensionMap.get(key), "123"); + } + + @Test + public void testBooleanAsValueWithContextAttribute() { + String key = "testKey"; + EventMeshCloudEventWriter eventMeshCloudEventWriter = new EventMeshCloudEventWriter(); + + eventMeshCloudEventWriter.withContextAttribute(key, Boolean.FALSE); + + Map extensionMap = eventMeshCloudEventWriter.getExtensionMap(); + Assertions.assertEquals(extensionMap.get(key), "false"); + } + + @Test + public void testByteArrayAsValueWithContextAttribute() { + String key = "testKey"; + EventMeshCloudEventWriter eventMeshCloudEventWriter = new EventMeshCloudEventWriter(); + + eventMeshCloudEventWriter.withContextAttribute(key, "bytesArray".getBytes(StandardCharsets.UTF_8)); + + Map extensionMap = eventMeshCloudEventWriter.getExtensionMap(); + String base64EncodedValue = "Ynl0ZXNBcnJheQ=="; + Assertions.assertEquals(extensionMap.get(key), base64EncodedValue); + } + +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/HttpRequestUtilTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/HttpRequestUtilTest.java new file mode 100644 index 0000000000..3d4c868dad --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/HttpRequestUtilTest.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.util; + +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import io.netty.buffer.Unpooled; +import io.netty.handler.codec.http.DefaultFullHttpRequest; +import io.netty.handler.codec.http.HttpMethod; +import io.netty.handler.codec.http.HttpRequest; +import io.netty.handler.codec.http.HttpVersion; + +public class HttpRequestUtilTest { + + @Test + public void testShouldParseHttpGETRequestBody() throws IOException { + HttpRequest httpRequest = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/some-path?q1=xyz"); + Map expected = new HashMap<>(); + expected.put("q1", "xyz"); + Assertions.assertEquals(expected, HttpRequestUtil.parseHttpRequestBody(httpRequest)); + } + + @Test + public void testShouldParseHttpPOSTRequestBody() throws IOException { + HttpRequest httpRequest = new DefaultFullHttpRequest( + HttpVersion.HTTP_1_1, HttpMethod.POST, + "/some-path", + Unpooled.copiedBuffer(("q1=xyz").getBytes()) + ); + Map expected = new HashMap<>(); + expected.put("q1", "xyz"); + Assertions.assertEquals(expected, HttpRequestUtil.parseHttpRequestBody(httpRequest)); + } + + @Test + public void testQueryStringToMap() { + Map expected = new HashMap<>(); + expected.put("q1", "xyz"); + expected.put("q2", "abc"); + Assertions.assertEquals(expected, HttpRequestUtil.queryStringToMap("q1=xyz&q2=abc")); + } + + @Test + public void testGetQueryParam() { + HttpRequest httpRequest = new DefaultFullHttpRequest( + HttpVersion.HTTP_1_1, + HttpMethod.GET, + "/some-path?q1=xyz" + ); + Assertions.assertEquals("xyz", HttpRequestUtil.getQueryParam(httpRequest, "q1", "")); + } + + @Test + public void testGetBodyParam() throws IOException { + HttpRequest httpRequest = new DefaultFullHttpRequest( + HttpVersion.HTTP_1_1, + HttpMethod.POST, + "/some-path", + Unpooled.copiedBuffer(("q1=xyz").getBytes()) + ); + Assertions.assertEquals("xyz", HttpRequestUtil.getBodyParam(httpRequest, "q1")); + } + +} diff --git a/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/TraceUtilsTest.java b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/TraceUtilsTest.java new file mode 100644 index 0000000000..34989b6442 --- /dev/null +++ b/eventmesh-runtime/src/test/java/org/apache/eventmesh/runtime/util/TraceUtilsTest.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.runtime.util; + +import org.apache.eventmesh.runtime.boot.EventMeshServer; +import org.apache.eventmesh.runtime.mock.MockCloudEvent; +import org.apache.eventmesh.runtime.trace.Trace; + +import java.util.Map; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import io.cloudevents.SpecVersion; +import io.opentelemetry.api.trace.Span; + +public class TraceUtilsTest { + @Test + public void testShouldPrepareClientSpan() throws Exception { + Map cloudEventExtensionMap = EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), new MockCloudEvent()); + try (MockedStatic dummyStatic = Mockito.mockStatic(EventMeshServer.class)) { + Trace trace = Trace.getInstance("zipkin", true); + trace.init(); + dummyStatic.when(EventMeshServer::getTrace).thenReturn(trace); + Span testClientSpan = TraceUtils.prepareClientSpan( + cloudEventExtensionMap, + "test client span", + false + ); + Assertions.assertNotNull(testClientSpan); + } + } + + @Test + public void testShouldPrepareServerSpan() throws Exception { + Map cloudEventExtensionMap = EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), new MockCloudEvent()); + try (MockedStatic dummyStatic = Mockito.mockStatic(EventMeshServer.class)) { + Trace trace = Trace.getInstance("zipkin", true); + trace.init(); + dummyStatic.when(EventMeshServer::getTrace).thenReturn(trace); + TraceUtils.prepareClientSpan( + cloudEventExtensionMap, + "test client span", + false + ); + Span testServerSpan = TraceUtils.prepareServerSpan( + cloudEventExtensionMap, + "test server span", + false + ); + Assertions.assertNotNull(testServerSpan); + } + } + + @Test + public void testShouldFinishSpan() throws Exception { + MockCloudEvent cloudEvent = new MockCloudEvent(); + Map cloudEventExtensionMap = EventMeshUtil.getCloudEventExtensionMap(SpecVersion.V1.toString(), cloudEvent); + try (MockedStatic dummyStatic = Mockito.mockStatic(EventMeshServer.class)) { + Trace trace = Trace.getInstance("zipkin", true); + trace.init(); + dummyStatic.when(EventMeshServer::getTrace).thenReturn(trace); + Span testClientSpan = TraceUtils.prepareClientSpan( + cloudEventExtensionMap, + "test client span", + false + ); + + TraceUtils.finishSpan(testClientSpan, cloudEvent); + Assertions.assertFalse(testClientSpan.isRecording()); + } + } +} diff --git a/eventmesh-runtime/src/test/resources/configuration.properties b/eventmesh-runtime/src/test/resources/configuration.properties index 70ff82e05c..836fc9c981 100644 --- a/eventmesh-runtime/src/test/resources/configuration.properties +++ b/eventmesh-runtime/src/test/resources/configuration.properties @@ -78,7 +78,6 @@ eventMesh.server.tcp.writerIdleSeconds=2816 eventMesh.server.tcp.readerIdleSeconds=3816 eventMesh.server.tcp.msgReqnumPerSecond=4816 eventMesh.server.tcp.clientMaxNum=5816 -eventMesh.server.tcp.enabled=true eventMesh.server.global.scheduler=6816 eventMesh.server.tcp.taskHandleExecutorPoolSize=7816 eventMesh.server.tcp.msgDownStreamExecutorPoolSize=8816 @@ -89,7 +88,6 @@ eventMesh.server.retry.sync.pushRetryDelayInMills=14816 eventMesh.server.retry.async.pushRetryDelayInMills=15816 eventMesh.server.retry.pushRetryQueueSize=16816 eventMesh.server.tcp.RebalanceIntervalInMills=17816 -eventMesh.server.admin.http.port=18816 eventMesh.server.tcp.sendBack.enabled=true eventMesh.server.tcp.pushFailIsolateTimeInMills=21816 eventMesh.server.gracefulShutdown.sleepIntervalInMills=22816 diff --git a/eventmesh-sdks/eventmesh-sdk-java/build.gradle b/eventmesh-sdks/eventmesh-sdk-java/build.gradle index d18fba37ea..be55c650a3 100644 --- a/eventmesh-sdks/eventmesh-sdk-java/build.gradle +++ b/eventmesh-sdks/eventmesh-sdk-java/build.gradle @@ -15,7 +15,7 @@ * limitations under the License. */ -def grpcVersion = '1.64.0' +def grpcVersion = '1.68.0' dependencies { api(project(":eventmesh-common")) { @@ -54,7 +54,7 @@ dependencies { implementation "io.grpc:grpc-protobuf:${grpcVersion}" implementation "io.grpc:grpc-stub:${grpcVersion}" - implementation "com.google.protobuf:protobuf-java-util:3.21.5" + implementation "com.google.protobuf:protobuf-java-util:3.25.4" compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' diff --git a/eventmesh-security-plugin/eventmesh-security-auth-token/src/main/java/org/apache/eventmesh/auth/token/impl/auth/AuthTokenUtils.java b/eventmesh-security-plugin/eventmesh-security-auth-token/src/main/java/org/apache/eventmesh/auth/token/impl/auth/AuthTokenUtils.java index 41ceca5490..16005649f4 100644 --- a/eventmesh-security-plugin/eventmesh-security-auth-token/src/main/java/org/apache/eventmesh/auth/token/impl/auth/AuthTokenUtils.java +++ b/eventmesh-security-plugin/eventmesh-security-auth-token/src/main/java/org/apache/eventmesh/auth/token/impl/auth/AuthTokenUtils.java @@ -33,6 +33,7 @@ import java.security.NoSuchAlgorithmException; import java.security.spec.InvalidKeySpecException; import java.security.spec.X509EncodedKeySpec; +import java.util.Objects; import java.util.Set; import io.jsonwebtoken.Claims; @@ -51,41 +52,8 @@ public static void authTokenByPublicKey(AclProperties aclProperties) { throw new AclException("group:" + aclProperties.getExtendedField("group ") + " has no auth to access the topic:" + aclProperties.getTopic()); } - String publicKeyUrl = null; - token = token.replace("Bearer ", ""); - for (String key : ConfigurationContextUtil.KEYS) { - CommonConfiguration commonConfiguration = ConfigurationContextUtil.get(key); - if (commonConfiguration == null) { - continue; - } - if (StringUtils.isBlank(commonConfiguration.getEventMeshSecurityPublickey())) { - throw new AclException("publicKeyUrl cannot be null"); - } - publicKeyUrl = commonConfiguration.getEventMeshSecurityPublickey(); - } - byte[] validationKeyBytes = new byte[0]; - try { - validationKeyBytes = Files.readAllBytes(Paths.get(publicKeyUrl)); - X509EncodedKeySpec spec = new X509EncodedKeySpec(validationKeyBytes); - KeyFactory kf = KeyFactory.getInstance("RSA"); - Key validationKey = kf.generatePublic(spec); - JwtParser signedParser = Jwts.parserBuilder().setSigningKey(validationKey).build(); - Jwt signJwt = signedParser.parseClaimsJws(token); - String sub = signJwt.getBody().get("sub", String.class); - if (!sub.contains(aclProperties.getExtendedField("group").toString()) && !sub.contains("pulsar-admin")) { - throw new AclException("group:" + aclProperties.getExtendedField("group ") + " has no auth to access eventMesh:" - + aclProperties.getTopic()); - } - } catch (IOException e) { - throw new AclException("public key read error!", e); - } catch (NoSuchAlgorithmException e) { - throw new AclException("no such RSA algorithm!", e); - } catch (InvalidKeySpecException e) { - throw new AclException("invalid public key spec!", e); - } catch (JwtException e) { - throw new AclException("invalid token!", e); - } - + String publicKeyUrl = getPublicKeyUrl(); + validateToken(token, publicKeyUrl, aclProperties); } else { throw new AclException("invalid token!"); } @@ -94,40 +62,7 @@ public static void authTokenByPublicKey(AclProperties aclProperties) { public static void helloTaskAuthTokenByPublicKey(AclProperties aclProperties) { String token = aclProperties.getToken(); if (StringUtils.isNotBlank(token)) { - String publicKeyUrl = null; - token = token.replace("Bearer ", ""); - for (String key : ConfigurationContextUtil.KEYS) { - CommonConfiguration commonConfiguration = ConfigurationContextUtil.get(key); - if (commonConfiguration == null) { - continue; - } - if (StringUtils.isBlank(commonConfiguration.getEventMeshSecurityPublickey())) { - throw new AclException("publicKeyUrl cannot be null"); - } - publicKeyUrl = commonConfiguration.getEventMeshSecurityPublickey(); - } - byte[] validationKeyBytes = new byte[0]; - try { - validationKeyBytes = Files.readAllBytes(Paths.get(publicKeyUrl)); - X509EncodedKeySpec spec = new X509EncodedKeySpec(validationKeyBytes); - KeyFactory kf = KeyFactory.getInstance("RSA"); - Key validationKey = kf.generatePublic(spec); - JwtParser signedParser = Jwts.parserBuilder().setSigningKey(validationKey).build(); - Jwt signJwt = signedParser.parseClaimsJws(token); - String sub = signJwt.getBody().get("sub", String.class); - if (!sub.contains(aclProperties.getExtendedField("group").toString()) && !sub.contains("pulsar-admin")) { - throw new AclException("group:" + aclProperties.getExtendedField("group ") + " has no auth to access eventMesh:" - + aclProperties.getTopic()); - } - } catch (IOException e) { - throw new AclException("public key read error!", e); - } catch (NoSuchAlgorithmException e) { - throw new AclException("no such RSA algorithm!", e); - } catch (InvalidKeySpecException e) { - throw new AclException("invalid public key spec!", e); - } catch (JwtException e) { - throw new AclException("invalid token!", e); - } + validateToken(token, getPublicKeyUrl(), aclProperties); } else { throw new AclException("invalid token!"); } @@ -148,4 +83,45 @@ public static boolean authAccess(AclProperties aclProperties) { return groupTopics.contains(topic); } + private static String getPublicKeyUrl() { + String publicKeyUrl = null; + for (String key : ConfigurationContextUtil.KEYS) { + CommonConfiguration commonConfiguration = ConfigurationContextUtil.get(key); + if (null == commonConfiguration) { + continue; + } + if (StringUtils.isBlank(commonConfiguration.getEventMeshSecurityPublickey())) { + throw new AclException("publicKeyUrl cannot be null"); + } + publicKeyUrl = commonConfiguration.getEventMeshSecurityPublickey(); + } + return publicKeyUrl; + } + + private static void validateToken(String token, String publicKeyUrl, AclProperties aclProperties) { + String sub; + token = token.replace("Bearer ", ""); + byte[] validationKeyBytes; + try { + validationKeyBytes = Files.readAllBytes(Paths.get(Objects.requireNonNull(publicKeyUrl))); + X509EncodedKeySpec spec = new X509EncodedKeySpec(validationKeyBytes); + KeyFactory kf = KeyFactory.getInstance("RSA"); + Key validationKey = kf.generatePublic(spec); + JwtParser signedParser = Jwts.parserBuilder().setSigningKey(validationKey).build(); + Jwt signJwt = signedParser.parseClaimsJws(token); + sub = signJwt.getBody().get("sub", String.class); + if (!sub.contains(aclProperties.getExtendedField("group").toString()) && !sub.contains("pulsar-admin")) { + throw new AclException("group:" + aclProperties.getExtendedField("group ") + " has no auth to access eventMesh:" + + aclProperties.getTopic()); + } + } catch (IOException e) { + throw new AclException("public key read error!", e); + } catch (NoSuchAlgorithmException e) { + throw new AclException("no such RSA algorithm!", e); + } catch (InvalidKeySpecException e) { + throw new AclException("invalid public key spec!", e); + } catch (JwtException e) { + throw new AclException("invalid token!", e); + } + } } diff --git a/eventmesh-spi/src/main/java/org/apache/eventmesh/spi/EventMeshExtensionType.java b/eventmesh-spi/src/main/java/org/apache/eventmesh/spi/EventMeshExtensionType.java index f76379f9e8..8de4e1ecfd 100644 --- a/eventmesh-spi/src/main/java/org/apache/eventmesh/spi/EventMeshExtensionType.java +++ b/eventmesh-spi/src/main/java/org/apache/eventmesh/spi/EventMeshExtensionType.java @@ -26,6 +26,7 @@ public enum EventMeshExtensionType { CONNECTOR("connector"), STORAGE("storage"), META("metaStorage"), + REGISTRY("registryCenter"), SECURITY("security"), PROTOCOL("protocol"), METRICS("metrics"), diff --git a/eventmesh-storage-plugin/eventmesh-storage-kafka/build.gradle b/eventmesh-storage-plugin/eventmesh-storage-kafka/build.gradle index 0eafe94636..dbae8d398d 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-kafka/build.gradle +++ b/eventmesh-storage-plugin/eventmesh-storage-kafka/build.gradle @@ -22,9 +22,9 @@ dependencies { implementation group: 'io.cloudevents', name: 'cloudevents-kafka', version: '2.5.0' // https://mvnrepository.com/artifact/org.apache.kafka/kafka-clients - implementation 'org.apache.kafka:kafka-clients:3.6.2' + implementation 'org.apache.kafka:kafka-clients:3.9.0' - testImplementation 'org.junit.jupiter:junit-jupiter:5.6.0' + testImplementation 'org.junit.jupiter:junit-jupiter' testImplementation project(":eventmesh-storage-plugin:eventmesh-storage-api") testImplementation project(":eventmesh-common") diff --git a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/build.gradle b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/build.gradle index 8ca1ec8f8a..3c920da181 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/build.gradle +++ b/eventmesh-storage-plugin/eventmesh-storage-rabbitmq/build.gradle @@ -19,12 +19,12 @@ dependencies { implementation project(":eventmesh-storage-plugin:eventmesh-storage-api") implementation project(":eventmesh-common") // rabbitmq - implementation 'com.rabbitmq:amqp-client:5.21.0' + implementation 'com.rabbitmq:amqp-client:5.24.0' testImplementation project(":eventmesh-storage-plugin:eventmesh-storage-api") testImplementation project(":eventmesh-common") // rabbitmq - testImplementation 'com.rabbitmq:amqp-client:5.21.0' + testImplementation 'com.rabbitmq:amqp-client:5.24.0' implementation 'io.cloudevents:cloudevents-json-jackson' testImplementation 'io.cloudevents:cloudevents-json-jackson' diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/build.gradle b/eventmesh-storage-plugin/eventmesh-storage-redis/build.gradle index bec0767638..4fb1645414 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/build.gradle +++ b/eventmesh-storage-plugin/eventmesh-storage-redis/build.gradle @@ -20,24 +20,16 @@ dependencies { implementation project(":eventmesh-storage-plugin:eventmesh-storage-api") // redisson - implementation('org.redisson:redisson:3.17.3') { - exclude group: 'io.netty', module: 'netty-common' - exclude group: 'io.netty', module: 'netty-buffer' - exclude group: 'io.netty', module: 'netty-codec' - exclude group: 'io.netty', module: 'netty-transport' - exclude group: 'io.netty', module: 'netty-resolver' - exclude group: 'io.netty', module: 'netty-resolver-dns' - exclude group: 'io.netty', module: 'netty-handler' - } + implementation 'org.redisson:redisson:3.38.1' // netty - implementation "io.netty:netty-all" + implementation 'io.netty:netty-all' // auxiliary serialize api 'io.cloudevents:cloudevents-json-jackson' // test dependencies - testImplementation 'ai.grakn:redis-mock:0.1.6' + testImplementation 'com.github.fppt:jedis-mock:1.1.3' testImplementation "org.mockito:mockito-core" compileOnly 'org.projectlombok:lombok' diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/gradle.properties b/eventmesh-storage-plugin/eventmesh-storage-redis/gradle.properties index 899c915a54..6f2a7b3f8b 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/gradle.properties +++ b/eventmesh-storage-plugin/eventmesh-storage-redis/gradle.properties @@ -1,3 +1,4 @@ +# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/client/RedissonClient.java b/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/client/RedissonClient.java index 69fb37b922..9a02064747 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/client/RedissonClient.java +++ b/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/client/RedissonClient.java @@ -73,18 +73,15 @@ private static Redisson create(RedisProperties properties) { throw new StorageRuntimeException(message, ie); } + Config config = new Config(); + config.setCodec(CloudEventCodec.getInstance()); + config.setThreads(properties.getRedissonThreads()); + config.setNettyThreads(properties.getRedissonNettyThreads()); + String serverAddress = properties.getServerAddress(); String serverPassword = properties.getServerPassword(); String masterName = properties.getServerMasterName(); - Config config = OBJECT_MAPPER.convertValue(properties.getRedissonProperties(), Config.class); - - if (config == null) { - config = new Config(); - } - - config.setCodec(CloudEventCodec.getInstance()); - switch (serverType) { case SINGLE: config.useSingleServer() diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/config/RedisProperties.java b/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/config/RedisProperties.java index a478e7a70f..b1e9f1a3af 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/config/RedisProperties.java +++ b/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/java/org/apache/eventmesh/storage/redis/config/RedisProperties.java @@ -20,8 +20,6 @@ import org.apache.eventmesh.common.config.Config; import org.apache.eventmesh.common.config.ConfigField; -import java.util.Properties; - import lombok.Data; @Data @@ -44,7 +42,7 @@ public class RedisProperties { * The address of the redis server following format -- host1:port1,host2:port2,…… */ @ConfigField(field = "serverAddress") - private String serverAddress; + private String serverAddress = "redis://127.0.0.1:6379"; /** * The password for redis authentication. @@ -55,8 +53,11 @@ public class RedisProperties { /** * The redisson options, redisson properties prefix is `eventMesh.server.redis.redisson` */ - @ConfigField(field = "redisson") - private Properties redissonProperties; + @ConfigField(field = "redisson.threads") + private int redissonThreads = 16; + + @ConfigField(field = "redisson.nettyThreads") + private int redissonNettyThreads = 32; public enum ServerType { SINGLE, diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/resources/redis-client.properties b/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/resources/redis-client.properties index e024dca8ac..9baf41f360 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/resources/redis-client.properties +++ b/eventmesh-storage-plugin/eventmesh-storage-redis/src/main/resources/redis-client.properties @@ -15,5 +15,9 @@ # limitations under the License. # -eventMesh.server.redis.serverAddress= +eventMesh.server.redis.serverAddress=redis://127.0.0.1:6379 eventMesh.server.redis.serverPassword= +eventMesh.server.redis.serverType=SINGLE +eventMesh.server.redis.serverMasterName=master +eventMesh.server.redis.redisson.threads= +eventMesh.server.redis.redisson.nettyThreads= diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/AbstractRedisServer.java b/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/AbstractRedisServer.java index 7aab9737d4..cefca9ea4b 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/AbstractRedisServer.java +++ b/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/AbstractRedisServer.java @@ -17,25 +17,19 @@ package org.apache.eventmesh.storage.redis; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; +import java.io.IOException; -import ai.grakn.redismock.RedisServer; +import com.github.fppt.jedismock.RedisServer; public abstract class AbstractRedisServer { - private static RedisServer redisServer; + private static final RedisServer redisServer; - @BeforeAll - public static void setupRedisServer() throws Exception { - redisServer = RedisServer.newRedisServer(6379); - redisServer.start(); - } - - @AfterAll - public static void shutdownRedisServer() { - if (redisServer != null) { - redisServer.stop(); + static { + try { + redisServer = RedisServer.newRedisServer(6379).start(); + } catch (IOException e) { + throw new RuntimeException(e); } } } diff --git a/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/config/RedisPropertiesTest.java b/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/config/RedisPropertiesTest.java index 677dc406e5..c284fb89da 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/config/RedisPropertiesTest.java +++ b/eventmesh-storage-plugin/eventmesh-storage-redis/src/test/java/org/apache/eventmesh/storage/redis/config/RedisPropertiesTest.java @@ -19,8 +19,6 @@ import org.apache.eventmesh.common.config.ConfigService; -import java.util.Properties; - import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -37,11 +35,7 @@ private void assertConfig(RedisProperties config) { Assertions.assertEquals("redis://127.0.0.1:6379", config.getServerAddress()); Assertions.assertEquals(RedisProperties.ServerType.SINGLE, config.getServerType()); Assertions.assertEquals("serverMasterName-success!!!", config.getServerMasterName()); - - Properties properties = new Properties(); - properties.put("threads", "2"); - properties.put("nettyThreads", "2"); - Properties redissonProperties = config.getRedissonProperties(); - Assertions.assertEquals(properties, redissonProperties); + Assertions.assertEquals(2, config.getRedissonThreads()); + Assertions.assertEquals(2, config.getRedissonNettyThreads()); } } diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/build.gradle b/eventmesh-storage-plugin/eventmesh-storage-standalone/build.gradle index a8b1827aa7..22271fb57d 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/build.gradle +++ b/eventmesh-storage-plugin/eventmesh-storage-standalone/build.gradle @@ -18,6 +18,7 @@ dependencies { implementation project(":eventmesh-common") implementation project(":eventmesh-storage-plugin:eventmesh-storage-api") + implementation "com.lmax:disruptor" compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/admin/StandaloneAdmin.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/admin/StandaloneAdmin.java index 7f5ab2da67..72257647ad 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/admin/StandaloneAdmin.java +++ b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/admin/StandaloneAdmin.java @@ -19,7 +19,7 @@ import org.apache.eventmesh.api.admin.AbstractAdmin; import org.apache.eventmesh.api.admin.TopicProperties; -import org.apache.eventmesh.storage.standalone.broker.MessageQueue; +import org.apache.eventmesh.storage.standalone.broker.Channel; import org.apache.eventmesh.storage.standalone.broker.StandaloneBroker; import org.apache.eventmesh.storage.standalone.broker.model.TopicMetadata; @@ -42,11 +42,11 @@ public StandaloneAdmin() { @Override public List getTopic() throws Exception { - ConcurrentHashMap messageContainer = this.standaloneBroker.getMessageContainer(); + ConcurrentHashMap messageContainer = this.standaloneBroker.getMessageContainer(); List topicList = new ArrayList<>(); messageContainer.keySet().forEach(topicMetadata -> { - MessageQueue messageQueue = messageContainer.get(topicMetadata); - final int messageCount = messageQueue.getPutIndex() - messageQueue.getTakeIndex(); + Channel channel = messageContainer.get(topicMetadata); + final int messageCount = channel.getMessageCount(); topicList.add(new TopicProperties( topicMetadata.getTopicName(), messageCount)); @@ -65,25 +65,7 @@ public void deleteTopic(String topicName) { standaloneBroker.deleteTopicIfExist(topicName); } - @Override - public List getEvent(String topicName, int offset, int length) throws Exception { - if (!this.standaloneBroker.checkTopicExist(topicName)) { - throw new Exception("The topic name doesn't exist in the message queue"); - } - ConcurrentHashMap messageContainer = this.standaloneBroker.getMessageContainer(); - long topicOffset = messageContainer.get(new TopicMetadata(topicName)).getTakeIndex(); - List messageList = new ArrayList<>(); - for (int index = 0; index < length; index++) { - long messageOffset = topicOffset + offset + index; - CloudEvent event = this.standaloneBroker.getMessage(topicName, messageOffset); - if (event == null) { - break; - } - messageList.add(event); - } - return messageList; - } @Override public void publish(CloudEvent cloudEvent) throws Exception { diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/Channel.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/Channel.java new file mode 100644 index 0000000000..8de0ca1c54 --- /dev/null +++ b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/Channel.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.storage.standalone.broker; + +import org.apache.eventmesh.api.LifeCycle; +import org.apache.eventmesh.common.EventMeshThreadFactory; +import org.apache.eventmesh.storage.standalone.broker.model.MessageEntity; +import org.apache.eventmesh.storage.standalone.broker.model.TopicMetadata; +import org.apache.eventmesh.storage.standalone.broker.provider.DisruptorProvider; + +import com.lmax.disruptor.BlockingWaitStrategy; +import com.lmax.disruptor.EventHandler; +import com.lmax.disruptor.IgnoreExceptionHandler; +import com.lmax.disruptor.RingBuffer; +import com.lmax.disruptor.dsl.Disruptor; +import com.lmax.disruptor.dsl.ProducerType; + +import lombok.Getter; +import lombok.Setter; + + +public class Channel implements LifeCycle { + + public static final Integer DEFAULT_SIZE = 4096 << 1 << 1; + @Getter + private DisruptorProvider provider; + private final Integer size; + @Setter + private EventHandler eventHandler; + private volatile boolean started = false; + private final TopicMetadata topic; + private static final String THREAD_NAME_PREFIX = "standalone_disruptor_provider_"; + + public Channel(TopicMetadata topic) { + this(DEFAULT_SIZE, topic, null); + } + + public Channel(TopicMetadata topic, EventHandler eventHandler) { + this(DEFAULT_SIZE, topic, eventHandler); + } + + + public Channel(final Integer ringBufferSize, final TopicMetadata topic, final EventHandler eventHandler) { + this.size = ringBufferSize; + this.topic = topic; + this.eventHandler = eventHandler; + } + + + @Override + public boolean isStarted() { + return started; + } + + @Override + public boolean isClosed() { + return !isStarted(); + } + + public synchronized void start() { + if (isClosed()) { + doStart(); + started = true; + } + } + + public void doStart() { + Disruptor disruptor = new Disruptor<>( + MessageEntity::new, + size, + new EventMeshThreadFactory(THREAD_NAME_PREFIX + topic.getTopicName(), true), + ProducerType.MULTI, + new BlockingWaitStrategy() + ); + + disruptor.handleEventsWith(eventHandler); + disruptor.setDefaultExceptionHandler(new IgnoreExceptionHandler()); + RingBuffer ringBuffer = disruptor.getRingBuffer(); + provider = new DisruptorProvider(ringBuffer, disruptor); + provider.start(); + } + + public int getMessageCount() { + return provider.getMessageCount(); + } + + @Override + public synchronized void shutdown() { + if (isStarted()) { + provider.shutdown(); + provider = null; + started = false; + } + } + +} \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/StandaloneBroker.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/StandaloneBroker.java index 5e64b40a70..0cda576332 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/StandaloneBroker.java +++ b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/StandaloneBroker.java @@ -19,42 +19,36 @@ import org.apache.eventmesh.storage.standalone.broker.model.MessageEntity; import org.apache.eventmesh.storage.standalone.broker.model.TopicMetadata; -import org.apache.eventmesh.storage.standalone.broker.task.HistoryMessageClear; -import org.apache.eventmesh.storage.standalone.broker.task.HistoryMessageClearTask; - -import org.apache.commons.lang3.tuple.Pair; +import org.apache.eventmesh.storage.standalone.broker.task.Subscribe; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicLong; import io.cloudevents.CloudEvent; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + /** * This broker used to store event, it just support standalone mode, you shouldn't use this module in production environment */ +@Slf4j public class StandaloneBroker { - private final ConcurrentHashMap messageContainer; + // message source by topic + @Getter + private final ConcurrentHashMap messageContainer; - // todo: move the offset manage to consumer - private final ConcurrentHashMap offsetMap; + @Getter + private final ConcurrentHashMap subscribeContainer; private StandaloneBroker() { this.messageContainer = new ConcurrentHashMap<>(); - this.offsetMap = new ConcurrentHashMap<>(); - startHistoryMessageCleanTask(); - } - - public ConcurrentHashMap getMessageContainer() { - return this.messageContainer; + this.subscribeContainer = new ConcurrentHashMap<>(); } - public ConcurrentHashMap getOffsetMap() { - return this.offsetMap; - } public static StandaloneBroker getInstance() { - return StandaloneBrokerInstanceHolder.instance; + return StandaloneBrokerInstanceHolder.INSTANCE; } /** @@ -62,28 +56,33 @@ public static StandaloneBroker getInstance() { * * @param topicName topic name * @param message message - * @throws InterruptedException */ - public MessageEntity putMessage(String topicName, CloudEvent message) throws InterruptedException { - Pair pair = createTopicIfAbsent(topicName); - AtomicLong topicOffset = pair.getRight(); - MessageQueue messageQueue = pair.getLeft(); - - MessageEntity messageEntity = new MessageEntity( - new TopicMetadata(topicName), message, topicOffset.getAndIncrement(), System.currentTimeMillis()); - messageQueue.put(messageEntity); - + public MessageEntity putMessage(String topicName, CloudEvent message) { + TopicMetadata topicMetadata = new TopicMetadata(topicName); + if (!messageContainer.containsKey(topicMetadata)) { + throw new RuntimeException(String.format("The topic:%s is not created", topicName)); + } + Channel channel = messageContainer.get(topicMetadata); + if (channel.isClosed()) { + throw new RuntimeException(String.format("The topic:%s is not subscribed", topicName)); + } + MessageEntity messageEntity = new MessageEntity(new TopicMetadata(topicName), message); + channel.getProvider().onData(messageEntity); return messageEntity; } + public Channel createTopic(String topicName) { + TopicMetadata topicMetadata = new TopicMetadata(topicName); + return messageContainer.computeIfAbsent(topicMetadata, k -> new Channel(topicMetadata)); + } + /** * Get the message, if the queue is empty then await * * @param topicName */ public CloudEvent takeMessage(String topicName) throws InterruptedException { - TopicMetadata topicMetadata = new TopicMetadata(topicName); - return messageContainer.computeIfAbsent(topicMetadata, k -> new MessageQueue()).take().getMessage(); + return null; } /** @@ -92,12 +91,7 @@ public CloudEvent takeMessage(String topicName) throws InterruptedException { * @param topicName */ public CloudEvent getMessage(String topicName) { - TopicMetadata topicMetadata = new TopicMetadata(topicName); - MessageEntity head = messageContainer.computeIfAbsent(topicMetadata, k -> new MessageQueue()).getHead(); - if (head == null) { - return null; - } - return head.getMessage(); + return null; } /** @@ -108,21 +102,9 @@ public CloudEvent getMessage(String topicName) { * @return CloudEvent */ public CloudEvent getMessage(String topicName, long offset) { - TopicMetadata topicMetadata = new TopicMetadata(topicName); - MessageEntity messageEntity = messageContainer.computeIfAbsent(topicMetadata, k -> new MessageQueue()).getByOffset(offset); - if (messageEntity == null) { - return null; - } - return messageEntity.getMessage(); + return null; } - private void startHistoryMessageCleanTask() { - HistoryMessageClear historyMessageClear = new HistoryMessageClear(messageContainer); - Thread thread = new Thread(new HistoryMessageClearTask(historyMessageClear)); - thread.setDaemon(true); - thread.setName("StandaloneBroker-HistoryMessageCleanTask"); - thread.start(); - } public boolean checkTopicExist(String topicName) { return messageContainer.containsKey(new TopicMetadata(topicName)); @@ -132,13 +114,10 @@ public boolean checkTopicExist(String topicName) { * if the topic does not exist, create the topic * * @param topicName topicName - * @return messageQueue and offset + * @return Channel */ - public Pair createTopicIfAbsent(String topicName) { - TopicMetadata topicMetadata = new TopicMetadata(topicName); - MessageQueue messageQueue = messageContainer.computeIfAbsent(topicMetadata, k -> new MessageQueue()); - AtomicLong offset = offsetMap.computeIfAbsent(topicMetadata, k -> new AtomicLong()); - return Pair.of(messageQueue, offset); + public Channel createTopicIfAbsent(String topicName) { + return createTopic(topicName); } /** @@ -148,18 +127,30 @@ public Pair createTopicIfAbsent(String topicName) { */ public void deleteTopicIfExist(String topicName) { TopicMetadata topicMetadata = new TopicMetadata(topicName); + Channel channel = createTopicIfAbsent(topicName); + channel.shutdown(); messageContainer.remove(topicMetadata); } - public void updateOffset(TopicMetadata topicMetadata, long offset) { - offsetMap.computeIfPresent(topicMetadata, (k, v) -> { - v.set(offset); - return v; - }); + public void subscribed(String topicName, Subscribe subscribe) { + TopicMetadata topicMetadata = new TopicMetadata(topicName); + if (subscribeContainer.containsKey(topicMetadata)) { + log.warn("the topic:{} already subscribed", topicName); + return; + } + Channel channel = getMessageContainer().get(topicMetadata); + if (channel == null) { + log.warn("the topic:{} is not created", topicName); + return; + } + channel.setEventHandler(subscribe); + channel.start(); + subscribeContainer.put(topicMetadata, subscribe); } + private static class StandaloneBrokerInstanceHolder { - private static final StandaloneBroker instance = new StandaloneBroker(); + private static final StandaloneBroker INSTANCE = new StandaloneBroker(); } -} +} \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/model/MessageEntity.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/model/MessageEntity.java index 0f437aee04..3662b30255 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/model/MessageEntity.java +++ b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/model/MessageEntity.java @@ -21,6 +21,9 @@ import io.cloudevents.CloudEvent; +import lombok.NoArgsConstructor; + +@NoArgsConstructor public class MessageEntity implements Serializable { private static final long serialVersionUID = 6646148767540524786L; @@ -40,6 +43,11 @@ public MessageEntity(TopicMetadata topicMetadata, CloudEvent message, long offse this.createTimeMills = currentTimeMills; } + public MessageEntity(TopicMetadata topicMetadata, CloudEvent message) { + this.topicMetadata = topicMetadata; + this.message = message; + } + public TopicMetadata getTopicMetadata() { return topicMetadata; } diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/provider/DisruptorProvider.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/provider/DisruptorProvider.java new file mode 100644 index 0000000000..47b2665a2c --- /dev/null +++ b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/provider/DisruptorProvider.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.eventmesh.storage.standalone.broker.provider; + +import org.apache.eventmesh.api.LifeCycle; +import org.apache.eventmesh.storage.standalone.broker.model.MessageEntity; + +import com.lmax.disruptor.EventTranslatorOneArg; +import com.lmax.disruptor.RingBuffer; +import com.lmax.disruptor.dsl.Disruptor; + +import lombok.extern.slf4j.Slf4j; + +/** + * DisruptorProvider. disruptor provider definition. + */ +@Slf4j +public class DisruptorProvider implements LifeCycle { + + private final RingBuffer ringBuffer; + + private final Disruptor disruptor; + + private volatile boolean start = false; + + private final EventTranslatorOneArg translatorOneArg = (messageEntity, sequence, arg0) -> { + arg0.setOffset(sequence); + arg0.setCreateTimeMills(System.currentTimeMillis()); + messageEntity.setOffset(arg0.getOffset()); + messageEntity.setCreateTimeMills(arg0.getCreateTimeMills()); + messageEntity.setTopicMetadata(arg0.getTopicMetadata()); + messageEntity.setMessage(arg0.getMessage()); + }; + + + /** + * Instantiates a new Disruptor provider. + * + * @param ringBuffer the ring buffer + * @param disruptor the disruptor + */ + public DisruptorProvider(final RingBuffer ringBuffer, final Disruptor disruptor) { + this.ringBuffer = ringBuffer; + this.disruptor = disruptor; + } + + /** + * @param data the data + */ + public MessageEntity onData(final MessageEntity data) { + if (isClosed()) { + throw new IllegalArgumentException("the disruptor is close"); + } + try { + ringBuffer.publishEvent(translatorOneArg, data); + } catch (Exception ex) { + throw new IllegalStateException("send data fail."); + } + return data; + } + + + @Override + public boolean isStarted() { + return start; + } + + @Override + public boolean isClosed() { + return !isStarted(); + } + + @Override + public void start() { + if (null != disruptor) { + disruptor.start(); + start = true; + } + } + + /** + * Shutdown. + */ + public void shutdown() { + if (null != disruptor) { + disruptor.shutdown(); + start = false; + } + } + + public int getMessageCount() { + return ringBuffer.getBufferSize(); + } +} \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/task/Subscribe.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/task/Subscribe.java index 8316270adf..4c84849ac7 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/task/Subscribe.java +++ b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/task/Subscribe.java @@ -21,16 +21,19 @@ import org.apache.eventmesh.api.EventMeshAction; import org.apache.eventmesh.api.EventMeshAsyncConsumeContext; import org.apache.eventmesh.storage.standalone.broker.StandaloneBroker; +import org.apache.eventmesh.storage.standalone.broker.model.MessageEntity; -import java.util.concurrent.atomic.AtomicInteger; import io.cloudevents.CloudEvent; +import com.lmax.disruptor.EventHandler; +import com.lmax.disruptor.WorkHandler; + import lombok.Getter; import lombok.extern.slf4j.Slf4j; @Slf4j -public class Subscribe { +public class Subscribe implements WorkHandler, EventHandler { @Getter private final String topicName; @@ -38,8 +41,6 @@ public class Subscribe { private final EventListener listener; @Getter private volatile boolean isRunning; - @Getter - private AtomicInteger offset; public Subscribe(String topicName, StandaloneBroker standaloneBroker, @@ -51,52 +52,50 @@ public Subscribe(String topicName, } public void subscribe() { + standaloneBroker.subscribed(topicName, this); + } + + public void shutdown() { + isRunning = false; + standaloneBroker.deleteTopicIfExist(topicName); + } + + @Override + public void onEvent(MessageEntity event, long sequence, boolean endOfBatch) { + onEvent(event); + } + + @Override + public void onEvent(MessageEntity event) { try { - log.debug("execute subscribe task, topic: {}, offset: {}", topicName, offset); - if (offset == null) { - CloudEvent message = standaloneBroker.getMessage(topicName); - if (message != null) { - Object tmpOffset = message.getExtension("offset"); - if (tmpOffset instanceof Integer) { - offset = new AtomicInteger(Integer.parseInt(tmpOffset.toString())); - } else { - offset = new AtomicInteger(0); - } - } + if (!isRunning) { + return; } - if (offset != null) { - CloudEvent message = standaloneBroker.getMessage(topicName, offset.get()); - if (message != null) { - EventMeshAsyncConsumeContext consumeContext = new EventMeshAsyncConsumeContext() { + CloudEvent message = event.getMessage(); + if (message != null) { + EventMeshAsyncConsumeContext consumeContext = new EventMeshAsyncConsumeContext() { - @Override - public void commit(EventMeshAction action) { - switch (action) { - case CommitMessage: - // update offset - log.info("message commit, topic: {}, current offset:{}", topicName, offset.get()); - break; - case ManualAck: - // update offset - offset.incrementAndGet(); - log.info("message ack, topic: {}, current offset:{}", topicName, offset.get()); - break; - case ReconsumeLater: - default: - - } + @Override + public void commit(EventMeshAction action) { + switch (action) { + case CommitMessage: + // update offset + log.info("message commit, topic: {}, current offset:{}", topicName, event.getOffset()); + break; + case ManualAck: + // update offset + log.info("message ack, topic: {}, current offset:{}", topicName, event.getOffset()); + break; + case ReconsumeLater: + default: } - }; - listener.consume(message, consumeContext); - } + } + }; + listener.consume(message, consumeContext); } } catch (Exception ex) { - log.error("consumer error, topic: {}, offset: {}", topicName, offset == null ? null : offset.get(), ex); + log.error("consumer error, topic: {}, offset: {}", topicName, event.getOffset(), ex); } } - public void shutdown() { - isRunning = false; - } - -} +} \ No newline at end of file diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/task/SubscribeTask.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/task/SubscribeTask.java deleted file mode 100644 index 0936c79257..0000000000 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/broker/task/SubscribeTask.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.eventmesh.storage.standalone.broker.task; - -import org.apache.eventmesh.common.utils.ThreadUtils; - -import java.util.concurrent.TimeUnit; - -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class SubscribeTask implements Runnable { - - private Subscribe subscribe; - - public SubscribeTask(Subscribe subscribe) { - this.subscribe = subscribe; - } - - @Override - public void run() { - while (subscribe.isRunning()) { - subscribe.subscribe(); - try { - ThreadUtils.sleepWithThrowException(1, TimeUnit.SECONDS); - } catch (InterruptedException e) { - log.error("Thread is interrupted, topic: {}, offset: {} thread name: {}", - subscribe.getTopicName(), - subscribe.getOffset() == null ? null : subscribe.getOffset().get(), - Thread.currentThread().getName(), e); - Thread.currentThread().interrupt(); - } - } - } - -} diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/consumer/StandaloneConsumer.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/consumer/StandaloneConsumer.java index 9eb753e3fa..edb66703f7 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/consumer/StandaloneConsumer.java +++ b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/main/java/org/apache/eventmesh/storage/standalone/consumer/StandaloneConsumer.java @@ -20,17 +20,12 @@ import org.apache.eventmesh.api.AbstractContext; import org.apache.eventmesh.api.EventListener; import org.apache.eventmesh.api.consumer.Consumer; -import org.apache.eventmesh.common.ThreadPoolFactory; import org.apache.eventmesh.storage.standalone.broker.StandaloneBroker; -import org.apache.eventmesh.storage.standalone.broker.model.TopicMetadata; import org.apache.eventmesh.storage.standalone.broker.task.Subscribe; -import org.apache.eventmesh.storage.standalone.broker.task.SubscribeTask; import java.util.List; -import java.util.Objects; import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicBoolean; import io.cloudevents.CloudEvent; @@ -45,16 +40,10 @@ public class StandaloneConsumer implements Consumer { private final ConcurrentHashMap subscribeTable; - private final ExecutorService consumeExecutorService; - public StandaloneConsumer(Properties properties) { this.standaloneBroker = StandaloneBroker.getInstance(); this.subscribeTable = new ConcurrentHashMap<>(16); this.isStarted = new AtomicBoolean(false); - this.consumeExecutorService = ThreadPoolFactory.createThreadPoolExecutor( - Runtime.getRuntime().availableProcessors() * 2, - Runtime.getRuntime().availableProcessors() * 2, - "StandaloneConsumerThread"); } @Override @@ -86,8 +75,6 @@ public void init(Properties keyValue) throws Exception { @Override public void updateOffset(List cloudEvents, AbstractContext context) { - cloudEvents.forEach(cloudEvent -> standaloneBroker.updateOffset( - new TopicMetadata(cloudEvent.getSubject()), Objects.requireNonNull((Long) cloudEvent.getExtension("offset")))); } @@ -99,9 +86,8 @@ public void subscribe(String topic) throws Exception { synchronized (subscribeTable) { standaloneBroker.createTopicIfAbsent(topic); Subscribe subscribe = new Subscribe(topic, standaloneBroker, listener); - SubscribeTask subScribeTask = new SubscribeTask(subscribe); + subscribe.subscribe(); subscribeTable.put(topic, subscribe); - consumeExecutorService.execute(subScribeTask); } } diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/TestUtils.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/TestUtils.java index 5ea0ab6f1a..5571cda950 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/TestUtils.java +++ b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/TestUtils.java @@ -17,9 +17,14 @@ package org.apache.eventmesh.storage.standalone; +import org.apache.eventmesh.storage.standalone.broker.Channel; import org.apache.eventmesh.storage.standalone.broker.MessageQueue; +import org.apache.eventmesh.storage.standalone.broker.StandaloneBroker; import org.apache.eventmesh.storage.standalone.broker.model.MessageEntity; import org.apache.eventmesh.storage.standalone.broker.model.TopicMetadata; +import org.apache.eventmesh.storage.standalone.broker.task.Subscribe; + +import org.apache.commons.lang3.tuple.Pair; import java.net.URI; import java.util.Collections; @@ -29,6 +34,7 @@ import io.cloudevents.CloudEvent; import io.cloudevents.core.builder.CloudEventBuilder; + public class TestUtils { public static final String TEST_TOPIC = "test-topic"; @@ -36,12 +42,18 @@ public class TestUtils { public static final int LENGTH = 5; public static final int EXCEEDED_MESSAGE_STORE_WINDOW = 60 * 60 * 1000 + 1000; - public static ConcurrentHashMap createDefaultMessageContainer() { - ConcurrentHashMap messageContainer = new ConcurrentHashMap<>(1); - messageContainer.put(new TopicMetadata(TEST_TOPIC), new MessageQueue()); - return messageContainer; + public static Pair, ConcurrentHashMap> createDefaultMessageContainer( + StandaloneBroker broker) { + ConcurrentHashMap messageContainer = new ConcurrentHashMap<>(1); + ConcurrentHashMap subscribeContainer = new ConcurrentHashMap<>(1); + + Subscribe subscribe = createSubscribe(broker); + subscribe.subscribe(); + subscribeContainer.put(new TopicMetadata(TEST_TOPIC), subscribe); + return Pair.of(messageContainer, subscribeContainer); } + public static ConcurrentHashMap createMessageContainer(TopicMetadata topicMetadata, MessageEntity messageEntity) throws InterruptedException { ConcurrentHashMap messageContainer = new ConcurrentHashMap<>(1); @@ -79,4 +91,17 @@ public static MessageEntity createMessageEntity(TopicMetadata topicMetadata, Clo offSet, currentTimeMillis); } + + public static Subscribe createSubscribe(StandaloneBroker standaloneBroker) { + standaloneBroker.createTopic(TEST_TOPIC); + return new Subscribe(TEST_TOPIC, standaloneBroker, (cloudEvent, context) -> { + }); + } + + public static Subscribe createSubscribe(StandaloneBroker standaloneBroker, List cloudEvents) { + standaloneBroker.createTopic(TEST_TOPIC); + return new Subscribe(TEST_TOPIC, standaloneBroker, (cloudEvent, context) -> { + cloudEvents.add(cloudEvent); + }); + } } diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/admin/StandaloneAdminTest.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/admin/StandaloneAdminTest.java index 2d84df265c..7200f902ec 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/admin/StandaloneAdminTest.java +++ b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/admin/StandaloneAdminTest.java @@ -17,18 +17,20 @@ package org.apache.eventmesh.storage.standalone.admin; -import static org.apache.eventmesh.storage.standalone.TestUtils.LENGTH; -import static org.apache.eventmesh.storage.standalone.TestUtils.OFF_SET; import static org.apache.eventmesh.storage.standalone.TestUtils.TEST_TOPIC; import static org.apache.eventmesh.storage.standalone.TestUtils.createDefaultCloudEvent; import static org.apache.eventmesh.storage.standalone.TestUtils.createDefaultMessageContainer; import static org.apache.eventmesh.storage.standalone.TestUtils.createDefaultMessageEntity; -import org.apache.eventmesh.api.admin.TopicProperties; +import org.apache.eventmesh.storage.standalone.broker.Channel; import org.apache.eventmesh.storage.standalone.broker.StandaloneBroker; import org.apache.eventmesh.storage.standalone.broker.model.MessageEntity; +import org.apache.eventmesh.storage.standalone.broker.model.TopicMetadata; +import org.apache.eventmesh.storage.standalone.broker.task.Subscribe; -import java.util.List; +import org.apache.commons.lang3.tuple.Pair; + +import java.util.concurrent.ConcurrentHashMap; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -52,6 +54,7 @@ public class StandaloneAdminTest { private StandaloneAdmin standaloneAdmin; + @BeforeEach public void setUp() { initStaticInstance(); @@ -69,13 +72,6 @@ public void testIsClosed() { Assertions.assertTrue(standaloneAdmin.isClosed()); } - @Test - public void testGetTopic() throws Exception { - List topicPropertiesList = standaloneAdmin.getTopic(); - Assertions.assertNotNull(topicPropertiesList); - Assertions.assertFalse(topicPropertiesList.isEmpty()); - } - @Test public void testCreateTopic() { standaloneAdmin.createTopic(TEST_TOPIC); @@ -88,21 +84,6 @@ public void testDeleteTopic() { Mockito.verify(standaloneBroker).deleteTopicIfExist(TEST_TOPIC); } - @Test - public void testGetEvent() throws Exception { - Mockito.when(standaloneBroker.checkTopicExist(TEST_TOPIC)).thenReturn(Boolean.TRUE); - Mockito.when(standaloneBroker.getMessage(TEST_TOPIC, OFF_SET)).thenReturn(createDefaultCloudEvent()); - List events = standaloneAdmin.getEvent(TEST_TOPIC, OFF_SET, LENGTH); - Assertions.assertNotNull(events); - Assertions.assertFalse(events.isEmpty()); - } - - @Test - public void testGetEvent_throwException() { - Mockito.when(standaloneBroker.checkTopicExist(TEST_TOPIC)).thenReturn(Boolean.FALSE); - Exception exception = Assertions.assertThrows(Exception.class, () -> standaloneAdmin.getEvent(TEST_TOPIC, OFF_SET, LENGTH)); - Assertions.assertEquals("The topic name doesn't exist in the message queue", exception.getMessage()); - } @Test public void testPublish() throws Exception { @@ -116,7 +97,11 @@ public void testPublish() throws Exception { private void initStaticInstance() { try (MockedStatic standaloneBrokerMockedStatic = Mockito.mockStatic(StandaloneBroker.class)) { standaloneBrokerMockedStatic.when(StandaloneBroker::getInstance).thenReturn(standaloneBroker); - Mockito.when(standaloneBroker.getMessageContainer()).thenReturn(createDefaultMessageContainer()); + Pair, ConcurrentHashMap> pair = + createDefaultMessageContainer(standaloneBroker); + Mockito.when(standaloneBroker.getSubscribeContainer()).thenReturn(pair.getRight()); + Mockito.when(standaloneBroker.getMessageContainer()).thenReturn(pair.getLeft()); + standaloneAdmin = new StandaloneAdmin(); } } diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/broker/StandaloneBrokerTest.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/broker/StandaloneBrokerTest.java index 3582f95ef5..d57ba6523b 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/broker/StandaloneBrokerTest.java +++ b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/broker/StandaloneBrokerTest.java @@ -17,15 +17,12 @@ package org.apache.eventmesh.storage.standalone.broker; -import static org.apache.eventmesh.storage.standalone.TestUtils.OFF_SET; import static org.apache.eventmesh.storage.standalone.TestUtils.TEST_TOPIC; import static org.apache.eventmesh.storage.standalone.TestUtils.createDefaultCloudEvent; +import static org.apache.eventmesh.storage.standalone.TestUtils.createSubscribe; import org.apache.eventmesh.storage.standalone.broker.model.MessageEntity; - -import org.apache.commons.lang3.tuple.Pair; - -import java.util.concurrent.atomic.AtomicLong; +import org.apache.eventmesh.storage.standalone.broker.task.Subscribe; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -34,6 +31,14 @@ public class StandaloneBrokerTest { + + public StandaloneBroker getStandaloneBroker() { + StandaloneBroker instance = StandaloneBroker.getInstance(); + Subscribe subscribe = createSubscribe(instance); + subscribe.subscribe(); + return instance; + } + @Test public void testGetInstance() { Assertions.assertNotNull(StandaloneBroker.getInstance()); @@ -41,62 +46,27 @@ public void testGetInstance() { @Test public void testCreateTopicIfAbsent() { - StandaloneBroker instance = StandaloneBroker.getInstance(); - Pair pair = instance.createTopicIfAbsent(TEST_TOPIC); + StandaloneBroker instance = getStandaloneBroker(); + Channel pair = instance.createTopicIfAbsent(TEST_TOPIC); Assertions.assertNotNull(pair); } @Test public void testPutMessage() throws InterruptedException { - StandaloneBroker instance = StandaloneBroker.getInstance(); + StandaloneBroker instance = getStandaloneBroker(); CloudEvent cloudEvent = createDefaultCloudEvent(); MessageEntity messageEntity = instance.putMessage(TEST_TOPIC, cloudEvent); Assertions.assertNotNull(messageEntity); } - @Test - public void testTakeMessage() throws InterruptedException { - StandaloneBroker instance = StandaloneBroker.getInstance(); - CloudEvent cloudEvent = createDefaultCloudEvent(); - instance.putMessage(TEST_TOPIC, cloudEvent); - CloudEvent message = instance.takeMessage(TEST_TOPIC); - Assertions.assertNotNull(message); - } - - @Test - public void testGetMessage() throws InterruptedException { - StandaloneBroker instance = StandaloneBroker.getInstance(); - CloudEvent cloudEvent = createDefaultCloudEvent(); - instance.putMessage(TEST_TOPIC, cloudEvent); - CloudEvent cloudEventResult = instance.getMessage(TEST_TOPIC); - Assertions.assertNotNull(cloudEventResult); - } - - @Test - public void testMessageWithOffSet() throws InterruptedException { - StandaloneBroker instance = StandaloneBroker.getInstance(); - CloudEvent cloudEvent = createDefaultCloudEvent(); - instance.putMessage(TEST_TOPIC, cloudEvent); - CloudEvent cloudEventResult = instance.getMessage(TEST_TOPIC, OFF_SET); - Assertions.assertNotNull(cloudEventResult); - } @Test public void testCheckTopicExist() throws InterruptedException { - StandaloneBroker instance = StandaloneBroker.getInstance(); + StandaloneBroker instance = getStandaloneBroker(); CloudEvent cloudEvent = createDefaultCloudEvent(); instance.putMessage(TEST_TOPIC, cloudEvent); boolean exists = instance.checkTopicExist(TEST_TOPIC); Assertions.assertTrue(exists); } - @Test - public void testDeleteTopicIfExist() throws InterruptedException { - StandaloneBroker instance = StandaloneBroker.getInstance(); - CloudEvent cloudEvent = createDefaultCloudEvent(); - instance.putMessage(TEST_TOPIC, cloudEvent); - instance.deleteTopicIfExist(TEST_TOPIC); - boolean exists = instance.checkTopicExist(TEST_TOPIC); - Assertions.assertFalse(exists); - } } diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/broker/task/SubscribeTest.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/broker/task/SubscribeTest.java index bc11c9b0aa..3ef86bdd20 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/broker/task/SubscribeTest.java +++ b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/broker/task/SubscribeTest.java @@ -18,14 +18,11 @@ package org.apache.eventmesh.storage.standalone.broker.task; import static org.apache.eventmesh.storage.standalone.TestUtils.TEST_TOPIC; -import static org.apache.eventmesh.storage.standalone.TestUtils.createDefaultCloudEvent; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyLong; import static org.mockito.ArgumentMatchers.anyString; import org.apache.eventmesh.api.EventListener; -import org.apache.eventmesh.api.EventMeshAsyncConsumeContext; import org.apache.eventmesh.storage.standalone.broker.StandaloneBroker; import org.junit.jupiter.api.Assertions; @@ -35,8 +32,6 @@ import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; -import io.cloudevents.CloudEvent; - @ExtendWith(MockitoExtension.class) public class SubscribeTest { @@ -48,12 +43,9 @@ public class SubscribeTest { @Test public void testSubscribe() { - CloudEvent cloudEvent = createDefaultCloudEvent(); - Mockito.when(standaloneBroker.getMessage(anyString())).thenReturn(cloudEvent); - Mockito.when(standaloneBroker.getMessage(anyString(), anyLong())).thenReturn(cloudEvent); subscribe = new Subscribe(TEST_TOPIC, standaloneBroker, eventListener); subscribe.subscribe(); - Mockito.verify(eventListener).consume(any(CloudEvent.class), any(EventMeshAsyncConsumeContext.class)); + Mockito.verify(standaloneBroker).subscribed(anyString(), any(Subscribe.class)); } @Test diff --git a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/producer/StandaloneProducerTest.java b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/producer/StandaloneProducerTest.java index 37cdc02c6a..20db666831 100644 --- a/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/producer/StandaloneProducerTest.java +++ b/eventmesh-storage-plugin/eventmesh-storage-standalone/src/test/java/org/apache/eventmesh/storage/standalone/producer/StandaloneProducerTest.java @@ -17,8 +17,13 @@ package org.apache.eventmesh.storage.standalone.producer; +import static org.apache.eventmesh.storage.standalone.TestUtils.TEST_TOPIC; +import static org.apache.eventmesh.storage.standalone.TestUtils.createSubscribe; + import org.apache.eventmesh.api.SendResult; import org.apache.eventmesh.storage.standalone.TestUtils; +import org.apache.eventmesh.storage.standalone.broker.StandaloneBroker; +import org.apache.eventmesh.storage.standalone.broker.task.Subscribe; import java.util.Properties; @@ -28,10 +33,13 @@ import io.cloudevents.CloudEvent; + + public class StandaloneProducerTest { private StandaloneProducer standaloneProducer; + @BeforeEach public void setUp() { standaloneProducer = new StandaloneProducer(new Properties()); @@ -61,7 +69,11 @@ public void testShutdown() { @Test public void testPublish() { + StandaloneBroker standaloneBroker = StandaloneBroker.getInstance(); + standaloneBroker.createTopicIfAbsent(TEST_TOPIC); CloudEvent cloudEvent = TestUtils.createDefaultCloudEvent(); + Subscribe subscribe = createSubscribe(standaloneBroker); + subscribe.subscribe(); SendResult sendResult = standaloneProducer.publish(cloudEvent); Assertions.assertNotNull(sendResult); } diff --git a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/build.gradle b/eventmesh-trace-plugin/eventmesh-trace-pinpoint/build.gradle index 0758ba291e..fcb492d692 100644 --- a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/build.gradle +++ b/eventmesh-trace-plugin/eventmesh-trace-pinpoint/build.gradle @@ -15,7 +15,7 @@ * limitations under the License. */ -def pinpointVersion = "2.4.1" +def pinpointVersion = "3.0.0" dependencies { implementation project(":eventmesh-trace-plugin:eventmesh-trace-api") diff --git a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/main/java/org/apache/eventmesh/trace/pinpoint/exporter/PinpointSpanExporter.java b/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/main/java/org/apache/eventmesh/trace/pinpoint/exporter/PinpointSpanExporter.java index 24e41039f3..61186c3062 100644 --- a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/main/java/org/apache/eventmesh/trace/pinpoint/exporter/PinpointSpanExporter.java +++ b/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/main/java/org/apache/eventmesh/trace/pinpoint/exporter/PinpointSpanExporter.java @@ -40,6 +40,8 @@ import java.util.logging.Logger; import java.util.stream.Collectors; +import org.mapstruct.factory.Mappers; + import io.grpc.NameResolverProvider; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.internal.OtelEncodingUtils; @@ -55,6 +57,7 @@ import com.navercorp.pinpoint.bootstrap.context.SpanId; import com.navercorp.pinpoint.bootstrap.context.TraceId; +import com.navercorp.pinpoint.common.profiler.util.TransactionIdUtils; import com.navercorp.pinpoint.common.trace.AnnotationKey; import com.navercorp.pinpoint.common.trace.ServiceType; import com.navercorp.pinpoint.common.util.JvmUtils; @@ -67,6 +70,7 @@ import com.navercorp.pinpoint.profiler.AgentInfoSender; import com.navercorp.pinpoint.profiler.JvmInformation; import com.navercorp.pinpoint.profiler.context.DefaultServerMetaDataRegistryService; +import com.navercorp.pinpoint.profiler.context.DefaultSpanFactory; import com.navercorp.pinpoint.profiler.context.ServerMetaDataRegistryService; import com.navercorp.pinpoint.profiler.context.Span; import com.navercorp.pinpoint.profiler.context.SpanEvent; @@ -75,12 +79,24 @@ import com.navercorp.pinpoint.profiler.context.grpc.GrpcAgentInfoMessageConverter; import com.navercorp.pinpoint.profiler.context.grpc.GrpcSpanMessageConverter; import com.navercorp.pinpoint.profiler.context.grpc.config.GrpcTransportConfig; -import com.navercorp.pinpoint.profiler.context.id.DefaultTraceId; -import com.navercorp.pinpoint.profiler.context.id.DefaultTraceRoot; +import com.navercorp.pinpoint.profiler.context.grpc.config.SpanAutoUriGetter; +import com.navercorp.pinpoint.profiler.context.grpc.config.SpanUriGetter; +import com.navercorp.pinpoint.profiler.context.grpc.mapper.AgentInfoMapper; +import com.navercorp.pinpoint.profiler.context.grpc.mapper.AgentInfoMapperImpl; +import com.navercorp.pinpoint.profiler.context.grpc.mapper.AnnotationValueMapper; +import com.navercorp.pinpoint.profiler.context.grpc.mapper.JvmGcTypeMapper; +import com.navercorp.pinpoint.profiler.context.grpc.mapper.JvmGcTypeMapperImpl; +import com.navercorp.pinpoint.profiler.context.grpc.mapper.SpanMessageMapper; +import com.navercorp.pinpoint.profiler.context.grpc.mapper.SpanMessageMapperImpl; +import com.navercorp.pinpoint.profiler.context.id.DefaultTraceIdFactory; +import com.navercorp.pinpoint.profiler.context.id.DefaultTraceRootFactory; +import com.navercorp.pinpoint.profiler.context.id.TraceIdFactory; import com.navercorp.pinpoint.profiler.context.id.TraceRoot; +import com.navercorp.pinpoint.profiler.context.id.TraceRootFactory; import com.navercorp.pinpoint.profiler.context.provider.AgentInformationProvider; import com.navercorp.pinpoint.profiler.context.provider.grpc.DnsExecutorServiceProvider; import com.navercorp.pinpoint.profiler.context.provider.grpc.GrpcNameResolverProvider; +import com.navercorp.pinpoint.profiler.context.provider.grpc.SSLContextProvider; import com.navercorp.pinpoint.profiler.metadata.MetaDataType; import com.navercorp.pinpoint.profiler.monitor.metric.gc.JvmGcType; import com.navercorp.pinpoint.profiler.receiver.ProfilerCommandLocatorBuilder; @@ -117,6 +133,7 @@ public final class PinpointSpanExporter implements SpanExporter { private final String applicationName; private final GrpcTransportConfig grpcTransportConfig; + private final SSLContextProvider sslContextProvider; private final HeaderFactory headerFactory; @@ -124,22 +141,35 @@ public final class PinpointSpanExporter implements SpanExporter { private final SpanGrpcDataSender spanGrpcDataSender; + private final JvmGcTypeMapper jvmGcTypeMapper = new JvmGcTypeMapperImpl(); + private final AgentInfoMapper agentInfoMapper = new AgentInfoMapperImpl(jvmGcTypeMapper); + + private final SpanUriGetter spanUriGetter = new SpanAutoUriGetter(); + private final AnnotationValueMapper annotationValueMapper = Mappers.getMapper(AnnotationValueMapper.class); + private final SpanMessageMapper spanMessageMapper = new SpanMessageMapperImpl(annotationValueMapper, spanUriGetter); + public PinpointSpanExporter(final String agentId, - final String agentName, - final String applicationName, - final GrpcTransportConfig grpcTransportConfig) { + final String agentName, + final String applicationName, + final GrpcTransportConfig grpcTransportConfig) { this.agentId = Objects.requireNonNull(agentId, "agentId cannot be null"); this.agentName = Objects.requireNonNull(agentName, "agentName cannot be null"); this.applicationName = Objects.requireNonNull(applicationName, "applicationName cannot be null"); this.grpcTransportConfig = Objects.requireNonNull(grpcTransportConfig, "grpcTransportConfig cannot be null"); + if (grpcTransportConfig.getSslOption() != null) { + this.sslContextProvider = new SSLContextProvider(grpcTransportConfig); + } else { + this.sslContextProvider = null; + } + this.headerFactory = new AgentHeaderFactory( - agentId, - agentName, - applicationName, - ServiceType.UNDEFINED.getCode(), - agentStartTime); + agentId, + agentName, + applicationName, + ServiceType.UNDEFINED.getCode(), + agentStartTime); this.agentInfoSender = createAgentInfoSender(); this.agentInfoSender.start(); @@ -155,7 +185,7 @@ private AgentInfoSender createAgentInfoSender() { grpcTransportConfig.getAgentCollectorIp(), grpcTransportConfig.getAgentCollectorPort(), grpcTransportConfig.getAgentSenderExecutorQueueSize(), - new GrpcAgentInfoMessageConverter(), + new GrpcAgentInfoMessageConverter(agentInfoMapper), reconnectExecutor, scheduledExecutorService, agentChannelFactory, @@ -171,17 +201,17 @@ private AgentInfoSender createAgentInfoSender() { ServiceType.STAND_ALONE); final JvmInformation jvmInformation = new JvmInformation( - JvmUtils.getSystemProperty(SystemPropertyKey.JAVA_VERSION), - JvmGcType.UNKNOWN); + JvmUtils.getSystemProperty(SystemPropertyKey.JAVA_VERSION), + JvmGcType.UNKNOWN); final ServerMetaDataRegistryService serverMetaDataRegistryService = new DefaultServerMetaDataRegistryService( - Collections.emptyList()); + Collections.emptyList()); serverMetaDataRegistryService.setServerName(EventMeshTraceConstants.SERVICE_NAME); final AgentInfoFactory agentInfoFactory = new AgentInfoFactory( - agentInformationProvider.createAgentInformation(), - serverMetaDataRegistryService, - jvmInformation); + agentInformationProvider.createAgentInformation(), + serverMetaDataRegistryService, + jvmInformation); return new AgentInfoSender.Builder(agentGrpcDataSender, agentInfoFactory).build(); } @@ -193,7 +223,8 @@ private SpanGrpcDataSender createSpanGrpcDataSender() { new GrpcSpanMessageConverter( agentId, ServiceType.STAND_ALONE.getCode(), - new GrpcSpanProcessorV2()); + new GrpcSpanProcessorV2(), + this.spanMessageMapper); final StreamState streamState = new SimpleStreamState( @@ -201,21 +232,23 @@ private SpanGrpcDataSender createSpanGrpcDataSender() { grpcTransportConfig.getSpanClientOption().getLimitTime()); return new SpanGrpcDataSender( - grpcTransportConfig.getSpanCollectorIp(), - grpcTransportConfig.getSpanCollectorPort(), - grpcTransportConfig.getSpanSenderExecutorQueueSize(), - messageConverter, - reconnectExecutor, - spanChannelFactory, - streamState); + grpcTransportConfig.getSpanCollectorIp(), + grpcTransportConfig.getSpanCollectorPort(), + grpcTransportConfig.getSpanSenderExecutorQueueSize(), + messageConverter, + reconnectExecutor, + spanChannelFactory, + streamState, + grpcTransportConfig.getSpanRpcMaxAgeMillis()); } private ChannelFactory createAgentChannelFactory() { - final ChannelFactoryBuilder channelFactoryBuilder = - new DefaultChannelFactoryBuilder(AGENT_CHANNEL_FACTORY); + final ChannelFactoryBuilder channelFactoryBuilder = new DefaultChannelFactoryBuilder(AGENT_CHANNEL_FACTORY); channelFactoryBuilder.setHeaderFactory(headerFactory); channelFactoryBuilder.setNameResolverProvider(nameResolverProvider); - channelFactoryBuilder.setSslOption(grpcTransportConfig.getSslOption()); + if (this.sslContextProvider != null) { + channelFactoryBuilder.setSslContext(this.sslContextProvider.get()); + } channelFactoryBuilder.setClientOption(grpcTransportConfig.getAgentClientOption()); channelFactoryBuilder.setExecutorQueueSize(grpcTransportConfig.getAgentChannelExecutorQueueSize()); @@ -223,11 +256,12 @@ private ChannelFactory createAgentChannelFactory() { } private ChannelFactory createSpanChannelFactory() { - final ChannelFactoryBuilder channelFactoryBuilder = - new DefaultChannelFactoryBuilder(SPAN_CHANNEL_FACTORY); + final ChannelFactoryBuilder channelFactoryBuilder = new DefaultChannelFactoryBuilder(SPAN_CHANNEL_FACTORY); channelFactoryBuilder.setHeaderFactory(headerFactory); channelFactoryBuilder.setNameResolverProvider(nameResolverProvider); - channelFactoryBuilder.setSslOption(grpcTransportConfig.getSslOption()); + if (this.sslContextProvider != null) { + channelFactoryBuilder.setSslContext(this.sslContextProvider.get()); + } channelFactoryBuilder.setClientOption(grpcTransportConfig.getSpanClientOption()); channelFactoryBuilder.setExecutorQueueSize(grpcTransportConfig.getSpanChannelExecutorQueueSize()); @@ -289,12 +323,21 @@ private Span toSpan(final SpanData spanData) { } }); - final TraceId traceId = new DefaultTraceId(agentId, startTimestamp, transactionId, parentSpanId[0], spanId, - (short) spanData.getKind().ordinal()); - - final TraceRoot traceRoot = new DefaultTraceRoot(traceId, this.agentId, startTimestamp, transactionId); + final TraceIdFactory traceIdFactory = new DefaultTraceIdFactory(this.agentId, startTimestamp); + final TraceRootFactory traceRootFactory = new DefaultTraceRootFactory(this.agentId, traceIdFactory); + + final TraceRoot traceRoot; + if (parentSpanId[0] == SpanId.NULL) { + traceRoot = traceRootFactory.newTraceRoot(transactionId); + } else { + final TraceId traceId = traceIdFactory.continueTraceId( + TransactionIdUtils.formatString(this.agentId, startTimestamp, transactionId), parentSpanId[0], + spanId, (short) spanData.getKind().ordinal()); + traceRoot = traceRootFactory.continueTraceRoot(traceId, transactionId); + } - final Span span = new Span(traceRoot); + final DefaultSpanFactory spanFactory = new DefaultSpanFactory(); + final Span span = spanFactory.newSpan(traceRoot); final StatusData statusData = spanData.getStatus(); if (statusData != null) { @@ -315,15 +358,15 @@ private Span toSpan(final SpanData spanData) { span.setRemoteAddr(UNKNOWN_REQ_IP); Optional.ofNullable(spanData.getAttributes()) - .ifPresent(attributes -> { - span.addAnnotation(Annotations.of(AnnotationKey.HTTP_PARAM_ENTITY.getCode(), - JsonUtils.toJSONString(attributes))); - attributes.forEach((key, value) -> { - if (REQ_IP.equals(key.getKey())) { - span.setRemoteAddr(String.valueOf(value)); - } + .ifPresent(attributes -> { + span.addAnnotation(Annotations.of(AnnotationKey.HTTP_PARAM_ENTITY.getCode(), + JsonUtils.toJSONString(attributes))); + attributes.forEach((key, value) -> { + if (REQ_IP.equals(key.getKey())) { + span.setRemoteAddr(String.valueOf(value)); + } + }); }); - }); if (CollectionUtils.isNotEmpty(spanData.getEvents())) { final AtomicInteger sequence = new AtomicInteger(); @@ -342,7 +385,7 @@ private SpanEvent toSpanEvent(final EventData eventData) { spanEvent.setServiceType(ServiceType.INTERNAL_METHOD.getCode()); spanEvent.setEndPoint(eventData.getName()); spanEvent.addAnnotation(Annotations.of(AnnotationKey.HTTP_PARAM_ENTITY.getCode(), - JsonUtils.toJSONString(eventData.getAttributes()))); + JsonUtils.toJSONString(eventData.getAttributes()))); spanEvent.setElapsedTime((int) toMillis(eventData.getEpochNanos())); return spanEvent; } @@ -354,16 +397,16 @@ private static long toMillis(final long epochNanos) { private static long hex32StringToLong(final String hex32String) { final CharSequence charSequence = new StringBuilder().append(hex32String); return OtelEncodingUtils.isValidBase16String(charSequence) - ? OtelEncodingUtils.longFromBase16String(charSequence, 0) - & OtelEncodingUtils.longFromBase16String(charSequence, 16) - : hex32String.hashCode(); + ? OtelEncodingUtils.longFromBase16String(charSequence, 0) + & OtelEncodingUtils.longFromBase16String(charSequence, 16) + : hex32String.hashCode(); } private static long hex16StringToLong(final String hex16String) { final CharSequence charSequence = new StringBuilder().append(hex16String); return OtelEncodingUtils.isValidBase16String(charSequence) - ? OtelEncodingUtils.longFromBase16String(charSequence, 0) - : hex16String.hashCode(); + ? OtelEncodingUtils.longFromBase16String(charSequence, 0) + : hex16String.hashCode(); } private static String getEndpoint(final Resource resource) { diff --git a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/test/java/org/apache/eventmesh/trace/pinpoint/exporter/PinpointSpanExporterTest.java b/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/test/java/org/apache/eventmesh/trace/pinpoint/exporter/PinpointSpanExporterTest.java index 5475876cef..6ae974825d 100644 --- a/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/test/java/org/apache/eventmesh/trace/pinpoint/exporter/PinpointSpanExporterTest.java +++ b/eventmesh-trace-plugin/eventmesh-trace-pinpoint/src/test/java/org/apache/eventmesh/trace/pinpoint/exporter/PinpointSpanExporterTest.java @@ -21,16 +21,16 @@ import org.apache.eventmesh.trace.api.TracePluginFactory; import org.apache.eventmesh.trace.pinpoint.PinpointTraceService; import org.apache.eventmesh.trace.pinpoint.config.PinpointConfiguration; - + import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.UUID; - + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; - + import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.SpanContext; import io.opentelemetry.api.trace.SpanKind; @@ -43,10 +43,11 @@ import io.opentelemetry.sdk.trace.data.LinkData; import io.opentelemetry.sdk.trace.data.SpanData; import io.opentelemetry.sdk.trace.data.StatusData; - + public class PinpointSpanExporterTest { private PinpointSpanExporter exporter; + public static final String AGENT_ID = "test"; @BeforeEach public void setup() { @@ -73,6 +74,11 @@ public void exportTest() { spans.clear(); spans.add(new SpanDateTest()); Assertions.assertEquals(CompletableResultCode.ofSuccess(), exporter.export(spans)); + + spans.clear(); + spans.add(new SpanDateTest()); + spans.add(new ChildSpanDateTest()); + Assertions.assertEquals(CompletableResultCode.ofSuccess(), exporter.export(spans)); } @Test @@ -87,7 +93,7 @@ public void shutdownTest() { /** * for test - */ + */ private static class SpanDateTest implements SpanData { @Override @@ -171,6 +177,89 @@ public int getTotalAttributeCount() { } } + private static class ChildSpanDateTest implements SpanData { + + @Override + public SpanContext getSpanContext() { + return new SpanContextTest(); + } + + @Override + public SpanContext getParentSpanContext() { + return new SpanContextTest(); + } + + @Override + public Resource getResource() { + return null; + } + + @Override + public InstrumentationLibraryInfo getInstrumentationLibraryInfo() { + return null; + } + + @Override + public String getName() { + return this.getClass().getName(); + } + + @Override + public SpanKind getKind() { + return SpanKind.INTERNAL; + } + + @Override + public long getStartEpochNanos() { + return System.nanoTime(); + } + + @Override + public Attributes getAttributes() { + return null; + } + + @Override + public List getEvents() { + return null; + } + + @Override + public List getLinks() { + return null; + } + + @Override + public StatusData getStatus() { + return StatusData.ok(); + } + + @Override + public long getEndEpochNanos() { + return System.nanoTime(); + } + + @Override + public boolean hasEnded() { + return true; + } + + @Override + public int getTotalRecordedEvents() { + return 0; + } + + @Override + public int getTotalRecordedLinks() { + return 0; + } + + @Override + public int getTotalAttributeCount() { + return 0; + } + } + private static class SpanContextTest implements SpanContext { @Override diff --git a/gradle.properties b/gradle.properties index 789424c0de..f712526c1c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -18,7 +18,7 @@ jdk=1.8 snapshot=false group=org.apache.eventmesh -version=1.10.0-release +version=1.11.0-release #last eight bits of public key signing.keyId= #passphrase for key pairs @@ -33,4 +33,4 @@ signEnabled=false org.gradle.warning.mode=none org.gradle.jvmargs=-Xmx4096m -XX:MaxMetaspaceSize=512m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8 -org.gradle.caching=true \ No newline at end of file +org.gradle.caching=true diff --git a/resources/eventmesh-architecture-5.png b/resources/eventmesh-architecture-5.png index f4111dc176..28a040bdb2 100644 Binary files a/resources/eventmesh-architecture-5.png and b/resources/eventmesh-architecture-5.png differ diff --git a/settings.gradle b/settings.gradle index 9388e2076d..c49a5c4242 100644 --- a/settings.gradle +++ b/settings.gradle @@ -16,8 +16,8 @@ */ plugins { - id 'com.gradle.develocity' version '3.17.4' - id 'com.gradle.common-custom-user-data-gradle-plugin' version '2.0.1' + id 'com.gradle.develocity' version '3.18.1' + id 'com.gradle.common-custom-user-data-gradle-plugin' version '2.0.2' } def isCiServer = System.getenv().containsKey("CI") @@ -47,11 +47,10 @@ include 'eventmesh-common' include 'eventmesh-starter' include 'eventmesh-examples' include 'eventmesh-spi' -include 'eventmesh-filter' -include 'eventmesh-transformer' include 'eventmesh-openconnect:eventmesh-openconnect-java' include 'eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-api' +include 'eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-admin' include 'eventmesh-openconnect:eventmesh-openconnect-offsetmgmt-plugin:eventmesh-openconnect-offsetmgmt-nacos' include 'eventmesh-connectors:eventmesh-connector-openfunction' @@ -75,6 +74,7 @@ include 'eventmesh-connectors:eventmesh-connector-slack' include 'eventmesh-connectors:eventmesh-connector-wechat' include 'eventmesh-connectors:eventmesh-connector-http' include 'eventmesh-connectors:eventmesh-connector-chatgpt' +include 'eventmesh-connectors:eventmesh-connector-canal' include 'eventmesh-storage-plugin:eventmesh-storage-api' include 'eventmesh-storage-plugin:eventmesh-storage-standalone' @@ -125,6 +125,14 @@ include 'eventmesh-webhook:eventmesh-webhook-receive' include 'eventmesh-retry' include 'eventmesh-retry:eventmesh-retry-api' include 'eventmesh-retry:eventmesh-retry-rocketmq' - - +include 'eventmesh-runtime-v2' +include 'eventmesh-admin-server' +include 'eventmesh-registry' +include 'eventmesh-registry:eventmesh-registry-api' +include 'eventmesh-registry:eventmesh-registry-nacos' + +include 'eventmesh-function' +include 'eventmesh-function:eventmesh-function-api' +include 'eventmesh-function:eventmesh-function-filter' +include 'eventmesh-function:eventmesh-function-transformer' diff --git a/tools/dist-license/LICENSE b/tools/dist-license/LICENSE index c5a89ef125..66b61bcaef 100644 --- a/tools/dist-license/LICENSE +++ b/tools/dist-license/LICENSE @@ -203,37 +203,44 @@ ======================================================================= This distribution contains the following third-party artifacts: +FastInfoset 1.2.16 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +FastInfoset 1.2.16 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt +HdrHistogram 2.1.12 licensed under 'CC0-1.0'. For details see: licenses/CC0-1.0.txt +HdrHistogram 2.1.12 licensed under 'BSD-2-Clause'. For details see: licenses/BSD-2-Clause.txt +HikariCP 4.0.3 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +LatencyUtils 2.0.3 licensed under 'CC0-1.0'. For details see: licenses/CC0-1.0.txt ST4 4.3.4 licensed under 'BSD-4-Clause'. For details see: licenses/BSD-4-Clause.txt -accessors-smart 2.4.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +accessors-smart 2.5.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt adapter-rxjava2 2.9.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt alibabacloud-gateway-spi 0.0.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -amqp-client 5.16.0 licensed under 'AL 2.0'. For details see: licenses/AL 2.0.txt -amqp-client 5.16.0 licensed under 'GPL v2'. For details see: licenses/GPL v2.txt -amqp-client 5.16.0 licensed under 'MPL-2.0'. For details see: licenses/MPL-2.0.txt -animal-sniffer-annotations 1.19 licensed under 'MIT'. For details see: licenses/MIT.txt -annotations 2.20.29 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +amqp-client 5.22.0 licensed under 'AL 2.0'. For details see: licenses/AL 2.0.txt +amqp-client 5.22.0 licensed under 'GPL v2'. For details see: licenses/GPL v2.txt +amqp-client 5.22.0 licensed under 'MPL-2.0'. For details see: licenses/MPL-2.0.txt +animal-sniffer-annotations 1.24 licensed under 'MIT'. For details see: licenses/MIT.txt +annotations 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt annotations 4.1.1.4 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt antlr-runtime 3.5.3 licensed under 'BSD licence'. For details see: licenses/BSD licence.txt -antlr4 4.13.0 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt -antlr4-runtime 4.13.0 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt +antlr4 4.13.1 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt +antlr4-runtime 4.13.1 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt aopalliance 1.0 licensed under 'Public Domain'. For details see: licenses/Public Domain.txt -apache-client 2.20.29 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +apache-client 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt api 0.18.2 licensed under 'MIT'. For details see: licenses/MIT.txt -arns 2.20.29 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -asm 9.1 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt -asm 9.2 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt -asm-analysis 9.2 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt -asm-commons 9.2 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt -asm-tree 9.2 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt -asm-util 9.2 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt -assertj-core 2.6.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -async-http-client 2.12.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -async-http-client-netty-utils 2.12.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +arns 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +asm 9.3 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt +asm 9.6 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt +asm-analysis 9.6 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt +asm-commons 9.6 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt +asm-tree 9.6 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt +asm-util 9.6 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt +assertj-core 3.26.3 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +async-http-client 2.12.3 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +async-http-client-netty-utils 2.12.3 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt audience-annotations 0.12.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -auth 2.20.29 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -aws-core 2.20.29 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -aws-query-protocol 2.20.29 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -aws-xml-protocol 2.20.29 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +auth 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +aviator 2.2.1 licensed under 'GNU LESSER GENERAL PUBLIC LICENSE'. For details see: licenses/GNU LESSER GENERAL PUBLIC LICENSE.txt +aws-core 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +aws-query-protocol 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +aws-xml-protocol 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt bcpkix-jdk15on 1.69 licensed under 'Bouncy Castle Licence'. For details see: licenses/Bouncy Castle Licence.txt bcpkix-jdk15on 1.70 licensed under 'Bouncy Castle Licence'. For details see: licenses/Bouncy Castle Licence.txt bcprov-ext-jdk15on 1.69 licensed under 'Bouncy Castle Licence'. For details see: licenses/Bouncy Castle Licence.txt @@ -242,206 +249,305 @@ bcprov-jdk15on 1.69 licensed under 'Bouncy Castle Licence'. For details see: lic bcprov-jdk15on 1.70 licensed under 'Bouncy Castle Licence'. For details see: licenses/Bouncy Castle Licence.txt bcutil-jdk15on 1.69 licensed under 'Bouncy Castle Licence'. For details see: licenses/Bouncy Castle Licence.txt bcutil-jdk15on 1.70 licensed under 'Bouncy Castle Licence'. For details see: licenses/Bouncy Castle Licence.txt -bolt 1.1.6 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -bouncy-castle-bc 2.10.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +bolt 1.42.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +bolt 1.6.4 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt bouncy-castle-bc 2.11.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -bson 3.12.11 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -byte-buddy 1.11.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +bouncy-castle-bc 2.11.4 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +bson 3.12.14 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +byte-buddy 1.14.18 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +byte-buddy 1.15.3 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt cache-api 1.1.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -checker-qual 3.12.0 licensed under 'MIT'. For details see: licenses/MIT.txt +caffeine 2.9.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +canal.common 1.1.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +canal.filter 1.1.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +canal.instance.core 1.1.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +canal.instance.manager 1.1.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +canal.instance.spring 1.1.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +canal.meta 1.1.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +canal.parse 1.1.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +canal.parse.dbsync 1.1.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +canal.parse.driver 1.1.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +canal.protocol 1.1.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +canal.server 1.1.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +canal.sink 1.1.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +canal.store 1.1.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +checker-qual 3.43.0 licensed under 'MIT'. For details see: licenses/MIT.txt +checksums 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +checksums-spi 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt classgraph 4.8.21 licensed under 'MIT'. For details see: licenses/MIT.txt classmate 1.5.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt client 0.18.2 licensed under 'MIT'. For details see: licenses/MIT.txt -cloudevents-api 2.4.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -cloudevents-core 2.4.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -cloudevents-http-vertx 2.3.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -cloudevents-json-jackson 2.4.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -cloudevents-kafka 2.4.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -cloudevents-protobuf 2.4.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +cloudevents-api 3.0.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +cloudevents-core 3.0.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +cloudevents-http-vertx 3.0.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +cloudevents-json-jackson 3.0.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +cloudevents-kafka 2.5.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +cloudevents-protobuf 3.0.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +commons-beanutils 1.8.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt commons-beanutils 1.9.4 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt commons-cli 1.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt commons-codec 1.11 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt commons-codec 1.15 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +commons-codec 1.17.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt commons-collections 3.2.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -commons-collections4 4.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +commons-collections4 4.4 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +commons-compress 1.22 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt commons-digester 2.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -commons-io 2.11.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -commons-lang3 3.6 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +commons-io 2.18.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +commons-lang 2.6 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +commons-lang3 3.17.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt commons-logging 1.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -commons-text 1.9 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -commons-validator 1.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +commons-logging 1.3.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +commons-text 1.12.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +commons-validator 1.9.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +connector.core 1.1.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt consul-api 1.4.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt converter-jackson 2.9.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -credentials-java 0.2.4 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -crt-core 2.20.29 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -curator-client 5.4.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -curator-framework 5.4.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -curator-recipes 5.4.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -dingtalk 2.0.61 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -disruptor 3.4.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +credentials-java 0.3.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +crt-core 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +curator-client 5.7.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +curator-framework 5.7.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +curator-recipes 5.7.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +dingtalk 2.1.27 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +disruptor 3.4.4 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt dom4j 2.0.3 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt -druid 1.2.20 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +druid 1.2.17 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +druid 1.2.23 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +druid-spring-boot-starter 1.2.23 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt endpoint-util 0.0.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -endpoints-spi 2.20.29 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -error_prone_annotations 2.9.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +endpoints-spi 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +error_prone_annotations 2.28.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt eventstream 1.0.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -failureaccess 1.0.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +failsafe 3.3.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +failureaccess 1.0.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt fastjson 1.2.69_noneautotype licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -fastjson2 2.0.48 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +fastjson2 2.0.52 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt gateway-dingtalk 1.0.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt google-auth-library-credentials 0.22.2 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt -grpc-api 1.43.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +grpc-api 1.68.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt grpc-auth 1.39.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -grpc-context 1.43.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -grpc-core 1.43.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +grpc-context 1.68.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +grpc-core 1.68.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt grpc-grpclb 1.17.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -grpc-netty 1.43.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -grpc-netty-shaded 1.43.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -grpc-protobuf 1.42.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -grpc-protobuf 1.43.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -grpc-protobuf-lite 1.42.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -grpc-protobuf-lite 1.43.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -grpc-stub 1.43.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -gson 2.8.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -guava 31.0.1-jre licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +grpc-netty 1.68.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +grpc-netty-shaded 1.68.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +grpc-protobuf 1.68.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +grpc-protobuf-lite 1.68.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +grpc-stub 1.68.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +grpc-util 1.68.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +gson 2.11.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +guava 33.3.0-jre licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt guava-retrying 2.0.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -guice 4.2.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +guice 7.0.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +h2 2.1.210 licensed under 'MPL-2.0'. For details see: licenses/MPL-2.0.txt +h2 2.1.210 licensed under 'EPL 1.0'. For details see: licenses/EPL 1.0.txt +hessian 3.3.6 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +hessian 4.0.63 licensed under 'The Apache Software License, Version 1.1'. For details see: licenses/The Apache Software License, Version 1.1.txt hibernate-validator 6.2.0.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -http-client-spi 2.20.29 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -httpasyncclient 4.1.3 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -httpclient 4.5.13 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -httpcore 4.4.13 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -httpcore-nio 4.4.6 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +http-auth 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +http-auth-aws 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +http-auth-aws-eventstream 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +http-auth-spi 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +http-client-spi 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +httpasyncclient 4.1.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +httpclient 4.5.14 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +httpcore 4.4.16 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +httpcore-nio 4.4.15 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt httpmime 4.5.13 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt icu4j 72.1 licensed under 'Unicode/ICU License'. For details see: licenses/Unicode-ICU License.txt +identity-spi 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt ini4j 0.5.4 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -ipaddress 5.3.3 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +ipaddress 5.5.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +istack-commons-runtime 3.0.8 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt j2objc-annotations 1.3 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -jackson-annotations 2.13.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -jackson-core 2.13.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -jackson-databind 2.13.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -jackson-dataformat-yaml 2.13.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -jackson-datatype-jdk8 2.13.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -jackson-datatype-jsr310 2.13.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -jackson-jr-objects 2.16.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -jackson-module-parameter-names 2.13.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +j2objc-annotations 2.8 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jackson-annotations 2.18.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jackson-core 2.18.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jackson-databind 2.18.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jackson-dataformat-yaml 2.18.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jackson-datatype-jdk8 2.18.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jackson-datatype-jsr310 2.18.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jackson-jr-objects 2.18.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jackson-module-parameter-names 2.18.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jakarta.activation 1.2.2 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt +jakarta.activation-api 1.2.1 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt jakarta.annotation-api 1.3.5 licensed under 'EPL-2.0'. For details see: licenses/EPL-2.0.txt jakarta.annotation-api 1.3.5 licensed under 'GPL-2.0-with-classpath-exception'. For details see: licenses/GPL-2.0-with-classpath-exception.txt +jakarta.annotation-api 2.1.1 licensed under 'EPL-2.0'. For details see: licenses/EPL-2.0.txt +jakarta.annotation-api 2.1.1 licensed under 'GPL-2.0-with-classpath-exception'. For details see: licenses/GPL-2.0-with-classpath-exception.txt +jakarta.inject-api 2.0.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jakarta.servlet-api 4.0.4 licensed under 'EPL-2.0'. For details see: licenses/EPL-2.0.txt +jakarta.servlet-api 4.0.4 licensed under 'GPL-2.0-with-classpath-exception'. For details see: licenses/GPL-2.0-with-classpath-exception.txt jakarta.validation-api 2.0.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -javassist 3.24.0-GA licensed under 'MPL-1.1'. For details see: licenses/MPL-1.1.txt -javassist 3.24.0-GA licensed under 'LGPL-2.1-only'. For details see: licenses/LGPL-2.1-only.txt -javassist 3.24.0-GA licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -javax.inject 1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jakarta.websocket-api 1.1.2 licensed under 'EPL-2.0'. For details see: licenses/EPL-2.0.txt +jakarta.websocket-api 1.1.2 licensed under 'GNU General Public License, version 2 with the GNU Classpath Exception'. For details see: licenses/GNU General Public License, version 2 with the GNU Classpath Exception.txt +jakarta.xml.bind-api 2.3.2 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt +javassist 3.30.2-GA licensed under 'MPL-1.1'. For details see: licenses/MPL-1.1.txt +javassist 3.30.2-GA licensed under 'LGPL-2.1-only'. For details see: licenses/LGPL-2.1-only.txt +javassist 3.30.2-GA licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +javax-websocket-client-impl 9.4.53.v20231009 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +javax-websocket-client-impl 9.4.53.v20231009 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +javax-websocket-server-impl 9.4.53.v20231009 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +javax-websocket-server-impl 9.4.53.v20231009 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt javax.ws.rs-api 2.1 licensed under 'CDDL-1.1'. For details see: licenses/CDDL-1.1.txt javax.ws.rs-api 2.1 licensed under 'GPL-2.0-with-classpath-exception'. For details see: licenses/GPL-2.0-with-classpath-exception.txt jaxb-api 2.3.0 licensed under 'CDDL-1.1'. For details see: licenses/CDDL-1.1.txt jaxb-api 2.3.0 licensed under 'GPL-2.0-with-classpath-exception'. For details see: licenses/GPL-2.0-with-classpath-exception.txt jaxb-core 2.3.0 licensed under 'CDDL-1.1'. For details see: licenses/CDDL-1.1.txt jaxb-impl 2.3.0 licensed under 'CDDL-1.1'. For details see: licenses/CDDL-1.1.txt +jaxb-runtime 2.3.2 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt jboss-logging 3.4.1.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -jboss-marshalling 2.0.11.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -jboss-marshalling-river 2.0.11.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -jcommander 1.78 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jcl-over-slf4j 1.7.12 licensed under 'MIT'. For details see: licenses/MIT.txt jcommander 1.82 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jctools-core 2.1.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt jetcd-common 0.3.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt jetcd-core 0.3.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt jetcd-resolver 0.3.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jetty-annotations 9.4.53.v20231009 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jetty-annotations 9.4.53.v20231009 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +jetty-client 9.4.53.v20231009 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jetty-client 9.4.53.v20231009 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +jetty-continuation 9.4.53.v20231009 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jetty-continuation 9.4.53.v20231009 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +jetty-http 9.4.53.v20231009 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jetty-http 9.4.53.v20231009 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +jetty-io 9.4.53.v20231009 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jetty-io 9.4.53.v20231009 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +jetty-plus 9.4.53.v20231009 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jetty-plus 9.4.53.v20231009 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +jetty-security 9.4.53.v20231009 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jetty-security 9.4.53.v20231009 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +jetty-server 9.4.53.v20231009 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jetty-server 9.4.53.v20231009 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +jetty-servlet 9.4.53.v20231009 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jetty-servlet 9.4.53.v20231009 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +jetty-servlets 9.4.53.v20231009 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jetty-servlets 9.4.53.v20231009 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +jetty-util 9.4.53.v20231009 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jetty-util 9.4.53.v20231009 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +jetty-util-ajax 9.4.53.v20231009 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jetty-util-ajax 9.4.53.v20231009 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +jetty-webapp 9.4.53.v20231009 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jetty-webapp 9.4.53.v20231009 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +jetty-xml 9.4.53.v20231009 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jetty-xml 9.4.53.v20231009 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt jjwt-api 0.11.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt jjwt-impl 0.11.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt jjwt-jackson 0.11.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt jna 4.2.2 licensed under 'LGPL, version 2.1'. For details see: licenses/LGPL, version 2.1.txt jna 4.2.2 licensed under 'ASL, version 2'. For details see: licenses/ASL, version 2.txt -jodd-bean 5.1.6 licensed under 'BSD-2-Clause'. For details see: licenses/BSD-2-Clause.txt -jodd-core 5.1.6 licensed under 'BSD-2-Clause'. For details see: licenses/BSD-2-Clause.txt -json-path 2.7.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -json-smart 2.4.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -json-utils 2.20.29 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jna 5.5.0 licensed under 'LGPL, version 2.1'. For details see: licenses/LGPL, version 2.1.txt +jna 5.5.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +joda-time 2.9.4 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jodd-util 6.3.0 licensed under 'BSD-2-Clause'. For details see: licenses/BSD-2-Clause.txt +jraft-core 1.3.14 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +json-path 2.9.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +json-smart 2.5.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +json-utils 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jsqlparser 4.9 licensed under 'LGPL-2.1-only'. For details see: licenses/LGPL-2.1-only.txt +jsqlparser 4.9 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt jsr305 3.0.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt jtokkit 0.5.1 licensed under 'MIT'. For details see: licenses/MIT.txt -kafka-clients 3.0.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +jts-core 1.20.0 licensed under 'EPL-2.0'. For details see: licenses/EPL-2.0.txt +jts-core 1.20.0 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt +kafka-clients 3.9.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +kryo 5.6.2 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt listenablefuture 9999.0-empty-to-avoid-conflict-with-guava licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -log4j-api 2.22.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -log4j-core 2.22.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -log4j-slf4j2-impl 2.22.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -lz4-java 1.7.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +log4j-api 2.24.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +log4j-core 2.24.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +log4j-slf4j2-impl 2.24.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +logback-adapter 1.1.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +logback-classic 1.2.13 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +logback-classic 1.2.13 licensed under 'GNU Lesser General Public License'. For details see: licenses/GNU Lesser General Public License.txt +logback-core 1.2.13 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +logback-core 1.2.13 licensed under 'GNU Lesser General Public License'. For details see: licenses/GNU Lesser General Public License.txt lz4-java 1.8.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +mapstruct 1.5.5.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt mbknor-jackson-jsonschema_2.12 1.0.34 licensed under 'MIT'. For details see: licenses/MIT.txt -metrics-annotation 4.1.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -metrics-core 4.1.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -metrics-healthchecks 4.1.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -metrics-json 4.1.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -metrics-spi 2.20.29 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -mongodb-driver 3.12.11 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -mongodb-driver-core 3.12.11 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -mysql-binlog-connector-java 0.28.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -nacos-auth-plugin 2.2.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -nacos-client 2.2.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -nacos-encryption-plugin 2.2.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty 3.10.6.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-all 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-buffer 4.1.100.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-buffer 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-buffer 4.1.86.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-codec 4.1.100.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-codec 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-codec 4.1.86.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-codec-dns 4.1.100.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-codec-dns 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-codec-haproxy 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-codec-http 4.1.100.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-codec-http 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-codec-http 4.1.86.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-codec-http2 4.1.100.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-codec-http2 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-codec-http2 4.1.86.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-codec-memcache 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-codec-mqtt 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-codec-redis 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-codec-smtp 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-codec-socks 4.1.100.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-codec-socks 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-codec-stomp 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-codec-xml 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-common 4.1.100.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-common 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-common 4.1.86.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-handler 4.1.100.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-handler 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-handler 4.1.86.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-handler-proxy 4.1.100.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-handler-proxy 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-nio-client 2.20.29 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +metrics-annotation 4.2.26 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +metrics-core 4.2.26 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +metrics-healthchecks 4.2.26 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +metrics-json 4.2.26 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +metrics-spi 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +micrometer-core 1.9.17 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +minlog 1.3.1 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt +mongodb-driver 3.12.14 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +mongodb-driver-core 3.12.14 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +mybatis 3.5.16 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +mybatis 3.5.6 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +mybatis-plus 3.5.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +mybatis-plus-annotation 3.5.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +mybatis-plus-boot-starter 3.5.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +mybatis-plus-core 3.5.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +mybatis-plus-extension 3.5.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +mybatis-plus-spring-boot-autoconfigure 3.5.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +mybatis-spring 2.0.4 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +mybatis-spring 2.1.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +mysql-binlog-connector-java 0.30.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +mysql-connector-java 5.1.48 licensed under 'The GNU General Public License, Version 2'. For details see: licenses/The GNU General Public License, Version 2.txt +nacos-auth-plugin 2.4.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +nacos-client 2.4.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +nacos-encryption-plugin 2.4.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +nacos-log4j2-adapter 2.4.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +nacos-logback-adapter-12 2.4.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty 3.2.10.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-all 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-buffer 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-buffer 4.1.114.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-codec 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-codec 4.1.114.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-codec-dns 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-codec-dns 4.1.114.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-codec-haproxy 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-codec-http 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-codec-http 4.1.114.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-codec-http2 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-codec-http2 4.1.114.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-codec-memcache 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-codec-mqtt 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-codec-redis 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-codec-smtp 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-codec-socks 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-codec-stomp 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-codec-xml 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-common 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-common 4.1.114.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-handler 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-handler 4.1.114.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-handler-proxy 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-handler-ssl-ocsp 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-nio-client 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt netty-reactive-streams 2.0.4 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-resolver 4.1.100.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-resolver 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-resolver 4.1.86.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-resolver-dns 4.1.100.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-resolver-dns 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-resolver-dns-classes-macos 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-resolver-dns-native-macos 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-resolver-dns-native-macos 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-resolver 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-resolver 4.1.114.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-resolver-dns 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-resolver-dns 4.1.114.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-resolver-dns-classes-macos 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-resolver-dns-native-macos 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-resolver-dns-native-macos 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt netty-tcnative-boringssl-static 2.0.48.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt netty-tcnative-boringssl-static 2.0.51.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-tcnative-boringssl-static 2.0.61.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt netty-tcnative-classes 2.0.48.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt netty-tcnative-classes 2.0.51.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-transport 4.1.100.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-transport 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-transport 4.1.86.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-transport-classes-epoll 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-transport-classes-epoll 4.1.86.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-transport-classes-kqueue 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-transport-native-epoll 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-transport-native-epoll 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-transport-native-epoll 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-transport-native-kqueue 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-transport-native-kqueue 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-transport-native-unix-common 4.1.100.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-transport-native-unix-common 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-transport-native-unix-common 4.1.86.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-transport-rxtx 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-transport-sctp 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -netty-transport-udt 4.1.79.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-tcnative-classes 2.0.61.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-transport 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-transport 4.1.114.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-transport-classes-epoll 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-transport-classes-epoll 4.1.114.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-transport-classes-kqueue 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-transport-native-epoll 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-transport-native-epoll 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-transport-native-epoll 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-transport-native-kqueue 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-transport-native-kqueue 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-transport-native-unix-common 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-transport-native-unix-common 4.1.114.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-transport-rxtx 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-transport-sctp 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +netty-transport-udt 4.1.112.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt oapi-sdk 2.0.28 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +objenesis 3.4 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt okhttp 3.14.9 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt okio 1.17.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt openapiutil 0.2.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt @@ -466,46 +572,51 @@ opentelemetry-sdk-trace 1.36.0 licensed under 'Apache-2.0'. For details see: lic opentelemetry-semconv 1.30.1-alpha licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt org.abego.treelayout.core 1.0.3 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt org.jacoco.agent 0.8.4 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt -perfmark-api 0.23.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -pinpoint-annotations 2.4.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -pinpoint-bootstrap 2.4.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -pinpoint-bootstrap-core 2.4.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -pinpoint-commons 2.4.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -pinpoint-commons-buffer 2.4.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -pinpoint-commons-profiler 2.4.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -pinpoint-grpc 2.4.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -pinpoint-profiler 2.4.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -pinpoint-rpc 2.4.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +perfmark-api 0.27.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +pinpoint-annotations 3.0.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +pinpoint-bootstrap 3.0.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +pinpoint-bootstrap-core 3.0.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +pinpoint-commons 3.0.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +pinpoint-commons-buffer 3.0.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +pinpoint-commons-config 3.0.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +pinpoint-commons-profiler 3.0.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +pinpoint-grpc 3.0.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +pinpoint-profiler 3.0.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt pravega-client 0.11.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt pravega-common 0.11.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt pravega-shared-authplugin 0.11.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt pravega-shared-controller-api 0.11.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt pravega-shared-protocol 0.11.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt pravega-shared-security 0.11.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -profiles 2.20.29 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +profiles 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt prometheus-metrics-config 1.1.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt prometheus-metrics-exporter-common 1.1.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt prometheus-metrics-exporter-httpserver 1.1.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt prometheus-metrics-exposition-formats 1.1.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt prometheus-metrics-model 1.1.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt prometheus-metrics-shaded-protobuf 1.1.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -proto-google-common-protos 2.0.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -protobuf-java 3.19.2 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt -protobuf-java 3.21.5 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt -protobuf-java-util 3.15.0 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt -protobuf-java-util 3.21.5 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt +proto-google-common-protos 2.41.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +protobuf-java 3.25.3 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt +protobuf-java 3.25.4 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt +protobuf-java-util 3.21.10 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt +protobuf-java-util 3.25.4 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt protobuf-java-util 3.5.1 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt -protocol-core 2.20.29 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -pulsar-client 2.10.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +protocol-core 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt pulsar-client 2.11.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -pulsar-client-admin-api 2.10.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +pulsar-client 2.11.4 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt pulsar-client-admin-api 2.11.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -pulsar-client-api 2.10.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +pulsar-client-admin-api 2.11.4 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt pulsar-client-api 2.11.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +pulsar-client-api 2.11.4 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt reactive-streams 1.0.3 licensed under 'CC0-1.0'. For details see: licenses/CC0-1.0.txt -reactor-core 3.4.13 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -redisson 3.17.3 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -regions 2.20.29 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +reactive-streams 1.0.4 licensed under 'MIT-0'. For details see: licenses/MIT-0.txt +reactor-core 3.6.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +redisson 3.38.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +reflectasm 1.11.9 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt +regions 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +resilience4j-core 1.7.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +retries 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +retries-spi 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt retrofit 2.9.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt rocketmq-acl 4.9.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt rocketmq-broker 4.9.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt @@ -518,73 +629,116 @@ rocketmq-remoting 4.9.5 licensed under 'Apache-2.0'. For details see: licenses/A rocketmq-srvutil 4.9.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt rocketmq-store 4.9.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt rocketmq-tools 4.9.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +rocksdbjni 8.8.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +rocksdbjni 8.8.1 licensed under 'GNU General Public License, version 2'. For details see: licenses/GNU General Public License, version 2.txt +rpc-grpc-impl 1.3.14 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt rxjava 2.0.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -rxjava 3.0.12 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -s3 2.20.29 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +rxjava 3.1.8 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +s3 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt scala-library 2.12.8 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -sdk-core 2.20.29 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +sdk-core 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt service 0.18.2 licensed under 'MIT'. For details see: licenses/MIT.txt -simpleclient 0.12.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -simpleclient_tracer_common 0.12.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -simpleclient_tracer_otel 0.12.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -simpleclient_tracer_otel_agent 0.12.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -slack-api-client 1.1.6 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -slack-api-model 1.1.6 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -slack-app-backend 1.1.6 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -slf4j-api 2.0.9 licensed under 'MIT'. For details see: licenses/MIT.txt -snakeyaml 1.30 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -snappy-java 1.1.8.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +simpleclient 0.15.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +simpleclient_tracer_common 0.15.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +simpleclient_tracer_otel 0.15.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +simpleclient_tracer_otel_agent 0.15.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +slack-api-client 1.42.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +slack-api-model 1.42.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +slack-app-backend 1.42.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +slf4j-api 2.0.13 licensed under 'MIT'. For details see: licenses/MIT.txt +snakeyaml 2.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +snappy-java 1.1.10.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +sofa-common-tools 1.0.12 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt spring-aop 5.3.15 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -spring-aop 5.3.26 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-aop 5.3.31 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-aop 5.3.9 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt spring-beans 5.3.20 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -spring-beans 5.3.26 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-beans 5.3.31 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-beans 5.3.9 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt spring-boot 2.5.9 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -spring-boot 2.7.10 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-boot 2.7.18 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt spring-boot-autoconfigure 2.5.9 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -spring-boot-autoconfigure 2.7.10 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-boot-autoconfigure 2.7.18 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt spring-boot-starter 2.5.9 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -spring-boot-starter 2.7.10 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -spring-boot-starter-json 2.7.10 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -spring-boot-starter-tomcat 2.7.10 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-boot-starter 2.7.18 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-boot-starter-jdbc 2.7.18 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-boot-starter-jetty 2.7.18 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-boot-starter-json 2.7.18 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-boot-starter-tomcat 2.7.18 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt spring-boot-starter-validation 2.5.9 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -spring-boot-starter-web 2.7.10 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-boot-starter-web 2.7.18 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt spring-context 5.3.15 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -spring-context 5.3.26 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-context 5.3.31 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-context 5.3.9 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt spring-core 5.3.20 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -spring-core 5.3.26 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-core 5.3.31 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-core 5.3.9 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt spring-expression 5.3.15 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -spring-expression 5.3.26 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-expression 5.3.31 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-expression 5.3.9 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt spring-jcl 5.3.20 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -spring-jcl 5.3.26 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-jcl 5.3.31 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-jcl 5.3.9 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-jdbc 5.3.31 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-jdbc 5.3.9 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt spring-messaging 5.3.20 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -spring-web 5.3.26 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -spring-webmvc 5.3.26 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-orm 5.3.9 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-tx 5.3.31 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-tx 5.3.9 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-web 5.3.31 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +spring-webmvc 5.3.31 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt stax-api 1.0-2 licensed under 'GNU General Public Library'. For details see: licenses/GNU General Public Library.txt stax-api 1.0-2 licensed under 'CDDL-1.0'. For details see: licenses/CDDL-1.0.txt +stax-ex 1.8.1 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt tea 1.2.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -tea-openapi 0.2.8 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -tea-util 0.2.21 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +tea-openapi 0.3.3 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +tea-util 0.2.22 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt tea-xml 0.1.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -third-party-jackson-core 2.20.29 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -tomcat-embed-core 9.0.73 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +third-party-jackson-core 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +tomcat-embed-core 9.0.83 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt tomcat-embed-el 9.0.56 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -tomcat-embed-el 9.0.73 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -tomcat-embed-websocket 9.0.73 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -utils 2.20.29 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +tomcat-embed-el 9.0.83 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +tomcat-embed-websocket 9.0.83 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +txw2 2.3.2 licensed under 'BSD-3-Clause'. For details see: licenses/BSD-3-Clause.txt +utils 2.29.5 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt validation-api 1.1.0.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt validation-api 2.0.1.Final licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -vertx-auth-common 4.4.6 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -vertx-auth-common 4.4.6 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt -vertx-bridge-common 4.4.6 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -vertx-bridge-common 4.4.6 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt -vertx-core 4.4.6 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -vertx-core 4.4.6 licensed under 'EPL-2.0'. For details see: licenses/EPL-2.0.txt -vertx-web 4.4.6 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -vertx-web 4.4.6 licensed under 'EPL-2.0'. For details see: licenses/EPL-2.0.txt -vertx-web-client 4.0.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -vertx-web-client 4.0.0 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt -vertx-web-common 4.4.6 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -vertx-web-common 4.4.6 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +vavr 0.10.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +vavr-match 0.10.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +vertx-auth-common 4.5.8 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +vertx-auth-common 4.5.8 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +vertx-auth-common 4.5.9 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +vertx-auth-common 4.5.9 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +vertx-bridge-common 4.5.8 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +vertx-bridge-common 4.5.8 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +vertx-core 4.5.8 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +vertx-core 4.5.8 licensed under 'EPL-2.0'. For details see: licenses/EPL-2.0.txt +vertx-core 4.5.9 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +vertx-core 4.5.9 licensed under 'EPL-2.0'. For details see: licenses/EPL-2.0.txt +vertx-uri-template 4.3.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +vertx-uri-template 4.3.7 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +vertx-uri-template 4.5.9 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +vertx-uri-template 4.5.9 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +vertx-web 4.5.8 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +vertx-web 4.5.8 licensed under 'EPL-2.0'. For details see: licenses/EPL-2.0.txt +vertx-web-client 4.3.7 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +vertx-web-client 4.3.7 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +vertx-web-client 4.5.9 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +vertx-web-client 4.5.9 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +vertx-web-common 4.5.8 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +vertx-web-common 4.5.8 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +vertx-web-common 4.5.9 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +vertx-web-common 4.5.9 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +websocket-api 9.4.53.v20231009 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +websocket-api 9.4.53.v20231009 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +websocket-client 9.4.53.v20231009 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +websocket-client 9.4.53.v20231009 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +websocket-common 9.4.53.v20231009 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +websocket-common 9.4.53.v20231009 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +websocket-server 9.4.53.v20231009 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +websocket-server 9.4.53.v20231009 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt +websocket-servlet 9.4.53.v20231009 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +websocket-servlet 9.4.53.v20231009 licensed under 'EPL-1.0'. For details see: licenses/EPL-1.0.txt xpp3 1.1.4c licensed under 'Indiana University Extreme! Lab Software License, vesion 1.1.1'. For details see: licenses/Indiana University Extreme! Lab Software License, vesion 1.1.1.txt xpp3 1.1.4c licensed under 'Public Domain'. For details see: licenses/Public Domain.txt xpp3 1.1.4c licensed under 'Apache-1.1'. For details see: licenses/Apache-1.1.txt @@ -592,7 +746,10 @@ xsdlib 2013.6.1 licensed under 'BSD-4-Clause'. For details see: licenses/BSD-4-C zipkin 2.27.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt zipkin-reporter 3.3.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt zipkin-sender-okhttp3 3.3.0 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -zookeeper 3.7.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt -zookeeper-jute 3.7.1 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +zkclient 0.10 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +zookeeper 3.9.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt +zookeeper-jute 3.9.2 licensed under 'Apache-2.0'. For details see: licenses/Apache-2.0.txt zstd-jni 1.5.0-2 licensed under 'BSD-2-Clause'. For details see: licenses/BSD-2-Clause.txt zstd-jni 1.5.2-2 licensed under 'BSD-2-Clause'. For details see: licenses/BSD-2-Clause.txt +zstd-jni 1.5.2-5 licensed under 'BSD-2-Clause'. For details see: licenses/BSD-2-Clause.txt +zstd-jni 1.5.6-4 licensed under 'BSD-2-Clause'. For details see: licenses/BSD-2-Clause.txt diff --git a/tools/dist-license/NOTICE b/tools/dist-license/NOTICE index 460b05effd..5d55094918 100644 --- a/tools/dist-license/NOTICE +++ b/tools/dist-license/NOTICE @@ -6,223 +6,150 @@ The Apache Software Foundation (http://www.apache.org/). ======================================================================= -annotations-2.20.29 NOTICE +jetty-webapp-9.4.53.v20231009 NOTICE ======================================================================= -AWS SDK for Java 2.0 -Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - -This product includes software developed by -Amazon Technologies, Inc (http://www.amazon.com/). - -********************** -THIRD PARTY COMPONENTS -********************** -This software includes third party software subject to the following copyrights: -- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. -- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. -- Apache Commons Lang - https://github.com/apache/commons-lang -- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams -- Jackson-core - https://github.com/FasterXML/jackson-core -- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary +============================================================== + Jetty Web Container + Copyright 1995-2018 Mort Bay Consulting Pty Ltd. +============================================================== -The licenses for these third party components are included in LICENSE.txt - -- For Apache Commons Lang see also this required NOTICE: - Apache Commons Lang - Copyright 2001-2020 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (https://www.apache.org/). +The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd +unless otherwise noted. -======================================================================= +Jetty is dual licensed under both -apache-client-2.20.29 NOTICE + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html -======================================================================= + and -AWS SDK for Java 2.0 -Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * The Eclipse Public 1.0 License + http://www.eclipse.org/legal/epl-v10.html -This product includes software developed by -Amazon Technologies, Inc (http://www.amazon.com/). +Jetty may be distributed under either license. -********************** -THIRD PARTY COMPONENTS -********************** -This software includes third party software subject to the following copyrights: -- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. -- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. -- Apache Commons Lang - https://github.com/apache/commons-lang -- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams -- Jackson-core - https://github.com/FasterXML/jackson-core -- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary +------ +Eclipse -The licenses for these third party components are included in LICENSE.txt +The following artifacts are EPL. + * org.eclipse.jetty.orbit:org.eclipse.jdt.core -- For Apache Commons Lang see also this required NOTICE: - Apache Commons Lang - Copyright 2001-2020 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (https://www.apache.org/). +The following artifacts are EPL and ASL2. + * org.eclipse.jetty.orbit:javax.security.auth.message -======================================================================= -arns-2.20.29 NOTICE +The following artifacts are EPL and CDDL 1.0. + * org.eclipse.jetty.orbit:javax.mail.glassfish -======================================================================= -AWS SDK for Java 2.0 -Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +------ +Oracle -This product includes software developed by -Amazon Technologies, Inc (http://www.amazon.com/). +The following artifacts are CDDL + GPLv2 with classpath exception. +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html -********************** -THIRD PARTY COMPONENTS -********************** -This software includes third party software subject to the following copyrights: -- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. -- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. -- Apache Commons Lang - https://github.com/apache/commons-lang -- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams -- Jackson-core - https://github.com/FasterXML/jackson-core -- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + * javax.servlet:javax.servlet-api + * javax.annotation:javax.annotation-api + * javax.transaction:javax.transaction-api + * javax.websocket:javax.websocket-api -The licenses for these third party components are included in LICENSE.txt +------ +Oracle OpenJDK -- For Apache Commons Lang see also this required NOTICE: - Apache Commons Lang - Copyright 2001-2020 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (https://www.apache.org/). +If ALPN is used to negotiate HTTP/2 connections, then the following +artifacts may be included in the distribution or downloaded when ALPN +module is selected. -======================================================================= + * java.sun.security.ssl -assertj-core-2.6.0 NOTICE +These artifacts replace/modify OpenJDK classes. The modififications +are hosted at github and both modified and original are under GPL v2 with +classpath exceptions. +http://openjdk.java.net/legal/gplv2+ce.html -======================================================================= -This product includes software developed by -The Apache Software Foundation (http://www.apache.org/). +------ +OW2 -======================================================================= +The following artifacts are licensed by the OW2 Foundation according to the +terms of http://asm.ow2.org/license.html -audience-annotations-0.12.0 NOTICE +org.ow2.asm:asm-commons +org.ow2.asm:asm -======================================================================= +------ +Apache -Apache Yetus - Audience Annotations -Copyright 2015-2020 The Apache Software Foundation +The following artifacts are ASL2 licensed. -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +org.apache.taglibs:taglibs-standard-spec +org.apache.taglibs:taglibs-standard-impl +------ +MortBay -======================================================================= +The following artifacts are ASL2 licensed. Based on selected classes from +following Apache Tomcat jars, all ASL2 licensed. -auth-2.20.29 NOTICE +org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util -======================================================================= +org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api -AWS SDK for Java 2.0 -Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -This product includes software developed by -Amazon Technologies, Inc (http://www.amazon.com/). +------ +Mortbay -********************** -THIRD PARTY COMPONENTS -********************** -This software includes third party software subject to the following copyrights: -- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. -- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. -- Apache Commons Lang - https://github.com/apache/commons-lang -- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams -- Jackson-core - https://github.com/FasterXML/jackson-core -- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary +The following artifacts are CDDL + GPLv2 with classpath exception. -The licenses for these third party components are included in LICENSE.txt +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html -- For Apache Commons Lang see also this required NOTICE: - Apache Commons Lang - Copyright 2001-2020 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (https://www.apache.org/). +org.eclipse.jetty.toolchain:jetty-schemas -======================================================================= +------ +Assorted -aws-core-2.20.29 NOTICE +The UnixCrypt.java code implements the one way cryptography used by +Unix systems for simple password protection. Copyright 1996 Aki Yoshida, +modified April 2001 by Iris Van den Broeke, Daniel Deville. +Permission to use, copy, modify and distribute UnixCrypt +for non-commercial or commercial purposes and without fee is +granted provided that the copyright notice appears in all copies. ======================================================================= -AWS SDK for Java 2.0 -Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - -This product includes software developed by -Amazon Technologies, Inc (http://www.amazon.com/). - -********************** -THIRD PARTY COMPONENTS -********************** -This software includes third party software subject to the following copyrights: -- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. -- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. -- Apache Commons Lang - https://github.com/apache/commons-lang -- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams -- Jackson-core - https://github.com/FasterXML/jackson-core -- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary - -The licenses for these third party components are included in LICENSE.txt - -- For Apache Commons Lang see also this required NOTICE: - Apache Commons Lang - Copyright 2001-2020 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (https://www.apache.org/). +objenesis-3.4 NOTICE ======================================================================= -aws-query-protocol-2.20.29 NOTICE - -======================================================================= +// ------------------------------------------------------------------ +// NOTICE file corresponding to the section 4d of The Apache License, +// Version 2.0, in this case for Objenesis -AWS SDK for Java 2.0 -Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// ------------------------------------------------------------------ -This product includes software developed by -Amazon Technologies, Inc (http://www.amazon.com/). +Objenesis +Copyright 2006-2024 Joe Walnes, Henri Tremblay, Leonardo Mesquita -********************** -THIRD PARTY COMPONENTS -********************** -This software includes third party software subject to the following copyrights: -- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. -- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. -- Apache Commons Lang - https://github.com/apache/commons-lang -- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams -- Jackson-core - https://github.com/FasterXML/jackson-core -- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary -The licenses for these third party components are included in LICENSE.txt -- For Apache Commons Lang see also this required NOTICE: - Apache Commons Lang - Copyright 2001-2020 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (https://www.apache.org/). ======================================================================= -aws-xml-protocol-2.20.29 NOTICE +protocol-core-2.29.5 NOTICE ======================================================================= @@ -254,13 +181,13 @@ The licenses for these third party components are included in LICENSE.txt ======================================================================= -bouncy-castle-bc-2.10.1-pkg NOTICE +httpcore-4.4.16 NOTICE ======================================================================= -Apache Pulsar :: Bouncy Castle :: BC -Copyright 2017-2020 Apache Software Foundation +Apache HttpCore +Copyright 2005-2022 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). @@ -269,315 +196,365 @@ The Apache Software Foundation (http://www.apache.org/). ======================================================================= -bouncy-castle-bc-2.11.1-pkg NOTICE +jackson-datatype-jsr310-2.18.0 NOTICE ======================================================================= +# Jackson JSON processor -Apache Pulsar :: Bouncy Castle :: BC -Copyright 2017-2020 Apache Software Foundation +Jackson is a high-performance, Free/Open Source JSON processing library. +It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has +been in development since 2007. +It is currently developed by a community of developers. -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +## Licensing + +Jackson components are licensed under Apache (Software) License, version 2.0, +as per accompanying LICENSE file. +## Credits +A list of contributors may be found from CREDITS file, which is included +in some artifacts (usually source distributions); but is always available +from the source code management (SCM) system project uses. ======================================================================= -byte-buddy-1.11.0 NOTICE +jakarta.xml.bind-api-2.3.2 NOTICE ======================================================================= -Copyright 2014 - ${current.year} Rafael Winterhalter +# Notices for Eclipse Project for JAXB -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This content is produced and maintained by the Eclipse Project for JAXB project. - http://www.apache.org/licenses/LICENSE-2.0 +* Project home: https://projects.eclipse.org/projects/ee4j.jaxb -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +## Trademarks -======================================================================= +Eclipse Project for JAXB is a trademark of the Eclipse Foundation. -byte-buddy-1.12.18 NOTICE +## Copyright -======================================================================= +All content is the property of the respective authors or their employers. For +more information regarding authorship of content, please consult the listed +source code repository logs. -Copyright 2014 - ${current.year} Rafael Winterhalter +## Declared Project Licenses -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This program and the accompanying materials are made available under the terms +of the Eclipse Distribution License v. 1.0 which is available +at http://www.eclipse.org/org/documents/edl-v10.php. - http://www.apache.org/licenses/LICENSE-2.0 +SPDX-License-Identifier: BSD-3-Clause -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +## Source Code -======================================================================= +The project maintains the following source code repositories: -classmate-1.5.1 NOTICE +* https://github.com/eclipse-ee4j/jaxb-api -======================================================================= +## Third-party Content -Java ClassMate library was originally written by Tatu Saloranta (tatu.saloranta@iki.fi) +This project leverages the following third party content. -Other developers who have contributed code are: +None -* Brian Langel +## Cryptography +Content may contain encryption software. The country in which you are currently +may have restrictions on the import, possession, and use, and/or re-export to +another country, of encryption software. BEFORE using any encryption software, +please check the country's laws, regulations and policies concerning the import, +possession, or use, and re-export of encryption software, to see if this is +permitted. ======================================================================= -commons-beanutils-1.9.4 NOTICE +commons-logging-1.2 NOTICE ======================================================================= -Apache Commons BeanUtils -Copyright 2000-2019 The Apache Software Foundation +Apache Commons Logging +Copyright 2003-2014 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). + ======================================================================= -commons-cli-1.2 NOTICE +rocketmq-acl-4.9.5 NOTICE ======================================================================= -Apache Commons CLI -Copyright 2001-2009 The Apache Software Foundation -This product includes software developed by +rocketmq-acl 4.9.5 +Copyright 2012-2023 Apache Software Foundation + +This product includes software developed at The Apache Software Foundation (http://www.apache.org/). + + ======================================================================= -commons-codec-1.11 NOTICE +rocketmq-broker-4.9.5 NOTICE ======================================================================= -Apache Commons Codec -Copyright 2002-2017 The Apache Software Foundation + +rocketmq-broker 4.9.5 +Copyright 2012-2023 Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). -src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java -contains test data from http://aspell.net/test/orig/batch0.tab. -Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) - -=============================================================================== -The content of package org.apache.commons.codec.language.bm has been translated -from the original php source code available at http://stevemorse.org/phoneticinfo.htm -with permission from the original authors. -Original source copyright: -Copyright (c) 2008 Alexander Beider & Stephen P. Morse. ======================================================================= -commons-codec-1.15 NOTICE +spring-context-5.3.31 NOTICE ======================================================================= -Apache Commons Codec -Copyright 2002-2020 The Apache Software Foundation +Spring Framework 5.3.31 +Copyright (c) 2002-2023 Pivotal, Inc. -This product includes software developed at -The Apache Software Foundation (https://www.apache.org/). +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. -src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java -contains test data from http://aspell.net/test/orig/batch0.tab. -Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) - -=============================================================================== - -The content of package org.apache.commons.codec.language.bm has been translated -from the original php source code available at http://stevemorse.org/phoneticinfo.htm -with permission from the original authors. -Original source copyright: -Copyright (c) 2008 Alexander Beider & Stephen P. Morse. +This product may include a number of subcomponents with separate +copyright notices and license terms. Your use of the source code for +these subcomponents is subject to the terms and conditions of the +subcomponent's license, as noted in the license.txt file. ======================================================================= -commons-collections-3.2.2 NOTICE +spring-expression-5.3.15 NOTICE ======================================================================= -Apache Commons Collections -Copyright 2001-2015 The Apache Software Foundation +Spring Framework 5.3.15 +Copyright (c) 2002-2022 Pivotal, Inc. -This product includes software developed by -The Apache Software Foundation (http://www.apache.org/). +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. + +This product may include a number of subcomponents with separate +copyright notices and license terms. Your use of the source code for +these subcomponents is subject to the terms and conditions of the +subcomponent's license, as noted in the license.txt file. ======================================================================= -commons-collections4-4.1 NOTICE +jaxb-runtime-2.3.2 NOTICE ======================================================================= -Apache Commons Collections -Copyright 2001-2015 The Apache Software Foundation +# Notices for Eclipse Implementation of JAXB -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +This content is produced and maintained by the Eclipse Implementation of JAXB +project. -======================================================================= +* Project home: https://projects.eclipse.org/projects/ee4j.jaxb-impl -commons-digester-2.1 NOTICE +## Trademarks -======================================================================= +Eclipse Implementation of JAXB is a trademark of the Eclipse Foundation. -Apache Commons Digester -Copyright 2001-2010 The Apache Software Foundation +## Copyright -This product includes software developed by -The Apache Software Foundation (http://www.apache.org/). +All content is the property of the respective authors or their employers. For +more information regarding authorship of content, please consult the listed +source code repository logs. -======================================================================= +## Declared Project Licenses -commons-io-2.11.0 NOTICE +This program and the accompanying materials are made available under the terms +of the Eclipse Distribution License v. 1.0 which is available at +http://www.eclipse.org/org/documents/edl-v10.php. -======================================================================= +SPDX-License-Identifier: BSD-3-Clause -Apache Commons IO -Copyright 2002-2021 The Apache Software Foundation +## Source Code -This product includes software developed at -The Apache Software Foundation (https://www.apache.org/). +The project maintains the following source code repositories: -======================================================================= +* https://github.com/eclipse-ee4j/jaxb-ri +* https://github.com/eclipse-ee4j/jaxb-istack-commons +* https://github.com/eclipse-ee4j/jaxb-dtd-parser +* https://github.com/eclipse-ee4j/jaxb-fi +* https://github.com/eclipse-ee4j/jaxb-stax-ex +* https://github.com/eclipse-ee4j/jax-rpc-ri -commons-lang3-3.6 NOTICE +## Third-party Content -======================================================================= +This project leverages the following third party content. -Apache Commons Lang -Copyright 2001-2017 The Apache Software Foundation +Apache Ant (1.10.2) -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +* License: Apache-2.0 AND W3C AND LicenseRef-Public-Domain -This product includes software from the Spring Framework, -under the Apache License 2.0 (see: StringUtils.containsWhitespace()) +Apache Ant (1.10.2) -======================================================================= +* License: Apache-2.0 AND W3C AND LicenseRef-Public-Domain -commons-logging-1.2 NOTICE +Apache Felix (1.2.0) -======================================================================= +* License: Apache License, 2.0 -Apache Commons Logging -Copyright 2003-2014 The Apache Software Foundation +args4j (2.33) -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +* License: MIT License +dom4j (1.6.1) -======================================================================= +* License: Custom license based on Apache 1.1 -commons-text-1.9 NOTICE +file-management (3.0.0) -======================================================================= +* License: Apache-2.0 +* Project: https://maven.apache.org/shared/file-management/ +* Source: + https://svn.apache.org/viewvc/maven/shared/tags/file-management-3.0.0/ -Apache Commons Text -Copyright 2014-2020 The Apache Software Foundation +JUnit (4.12) -This product includes software developed at -The Apache Software Foundation (https://www.apache.org/). +* License: Eclipse Public License -======================================================================= +JUnit (4.12) -commons-validator-1.7 NOTICE +* License: Eclipse Public License -======================================================================= +maven-compat (3.5.2) -Apache Commons Validator -Copyright 2001-2020 The Apache Software Foundation +* License: Apache-2.0 +* Project: https://maven.apache.org/ref/3.5.2/maven-compat/ +* Source: + https://mvnrepository.com/artifact/org.apache.maven/maven-compat/3.5.2 -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +maven-core (3.5.2) -======================================================================= +* License: Apache-2.0 +* Project: https://maven.apache.org/ref/3.5.2/maven-core/index.html +* Source: https://mvnrepository.com/artifact/org.apache.maven/maven-core/3.5.2 -crt-core-2.20.29 NOTICE +maven-plugin-annotations (3.5) -======================================================================= +* License: Apache-2.0 +* Project: https://maven.apache.org/plugin-tools/maven-plugin-annotations/ +* Source: + https://github.com/apache/maven-plugin-tools/tree/master/maven-plugin-annotations -AWS SDK for Java 2.0 -Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +maven-plugin-api (3.5.2) -This product includes software developed by -Amazon Technologies, Inc (http://www.amazon.com/). +* License: Apache-2.0 -********************** -THIRD PARTY COMPONENTS -********************** -This software includes third party software subject to the following copyrights: -- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. -- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. -- Apache Commons Lang - https://github.com/apache/commons-lang -- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams -- Jackson-core - https://github.com/FasterXML/jackson-core -- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary +maven-resolver-api (1.1.1) -The licenses for these third party components are included in LICENSE.txt +* License: Apache-2.0 -- For Apache Commons Lang see also this required NOTICE: - Apache Commons Lang - Copyright 2001-2020 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (https://www.apache.org/). +maven-resolver-api (1.1.1) -======================================================================= +* License: Apache-2.0 -curator-client-5.4.0 NOTICE +maven-resolver-connector-basic (1.1.1) -======================================================================= +* License: Apache-2.0 +maven-resolver-impl (1.1.1) -Curator Client -Copyright 2011-2022 The Apache Software Foundation +* License: Apache-2.0 -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +maven-resolver-spi (1.1.1) +* License: Apache-2.0 +maven-resolver-transport-file (1.1.1) -======================================================================= +* License: Apache-2.0 +* Project: https://maven.apache.org/resolver/maven-resolver-transport-file/ +* Source: + https://github.com/apache/maven-resolver/tree/master/maven-resolver-transport-file -curator-framework-5.4.0 NOTICE +maven-resolver-util (1.1.1) -======================================================================= +* License: Apache-2.0 +maven-settings (3.5.2) -Curator Framework -Copyright 2011-2022 The Apache Software Foundation +* License: Apache-2.0 +* Source: + https://mvnrepository.com/artifact/org.apache.maven/maven-settings/3.5.2 -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +OSGi Service Platform Core Companion Code (6.0) + +* License: Apache License, 2.0 + +plexus-archiver (3.5) + +* License: Apache-2.0 +* Project: https://codehaus-plexus.github.io/plexus-archiver/ +* Source: https://github.com/codehaus-plexus/plexus-archiver + +plexus-io (3.0.0) +* License: Apache-2.0 + +plexus-utils (3.1.0) + +* License: Apache- 2.0 or Apache- 1.1 or BSD or Public Domain or Indiana + University Extreme! Lab Software License V1.1.1 (Apache 1.1 style) + +relaxng-datatype (1.0) + +* License: New BSD license + +Sax (0.2) + +* License: SAX-PD +* Project: http://www.megginson.com/downloads/SAX/ +* Source: http://sourceforge.net/project/showfiles.php?group_id=29449 + +testng (6.14.2) + +* License: Apache-2.0 AND (MIT OR GPL-1.0+) +* Project: https://testng.org/doc/index.html +* Source: https://github.com/cbeust/testng + +wagon-http-lightweight (3.0.0) + +* License: Pending +* Project: https://maven.apache.org/wagon/ +* Source: + https://mvnrepository.com/artifact/org.apache.maven.wagon/wagon-http-lightweight/3.0.0 + +xz for java (1.8) + +* License: LicenseRef-Public-Domain + +## Cryptography + +Content may contain encryption software. The country in which you are currently +may have restrictions on the import, possession, and use, and/or re-export to +another country, of encryption software. BEFORE using any encryption software, +please check the country's laws, regulations and policies concerning the import, +possession, or use, and re-export of encryption software, to see if this is +permitted. ======================================================================= -curator-recipes-5.4.0 NOTICE +pulsar-client-api-2.11.4 NOTICE ======================================================================= -Curator Recipes -Copyright 2011-2022 The Apache Software Foundation +Pulsar Client :: API +Copyright 2017-2020 Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). @@ -586,855 +563,710 @@ The Apache Software Foundation (http://www.apache.org/). ======================================================================= -endpoints-spi-2.20.29 NOTICE +spring-core-5.3.20 NOTICE ======================================================================= -AWS SDK for Java 2.0 -Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +Spring Framework 5.3.20 +Copyright (c) 2002-2022 Pivotal, Inc. -This product includes software developed by -Amazon Technologies, Inc (http://www.amazon.com/). +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. -********************** -THIRD PARTY COMPONENTS -********************** -This software includes third party software subject to the following copyrights: -- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. -- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. -- Apache Commons Lang - https://github.com/apache/commons-lang -- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams -- Jackson-core - https://github.com/FasterXML/jackson-core -- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary +This product may include a number of subcomponents with separate +copyright notices and license terms. Your use of the source code for +these subcomponents is subject to the terms and conditions of the +subcomponent's license, as noted in the license.txt file. -The licenses for these third party components are included in LICENSE.txt +======================================================================= -- For Apache Commons Lang see also this required NOTICE: - Apache Commons Lang - Copyright 2001-2020 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (https://www.apache.org/). +spring-boot-starter-2.7.18 NOTICE ======================================================================= -fastjson-1.2.69_noneautotype NOTICE +Spring Boot 2.7.18 +Copyright (c) 2012-2023 VMware, Inc. +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. ======================================================================= -/* - * Copyright 1999-2017 Alibaba Group. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +pravega-shared-security-0.11.0 NOTICE + ======================================================================= -grpc-netty-shaded-1.43.2 NOTICE +Copyright (c) 2021 Pravega Authors. +Copyright (c) 2017-2021 Dell Inc., or its subsidiaries. All Rights Reserved. + +This software contains source code from Apache BookKeeper, distributed under +the Apache License Version 2.0, and copyrighted to the Apache Software Foundation. +http://bookkeeper.apache.org ======================================================================= - The Netty Project - ================= +websocket-servlet-9.4.53.v20231009 NOTICE -Please visit the Netty web site for more information: +======================================================================= - * http://netty.io/ +============================================================== + Jetty Web Container + Copyright 1995-2018 Mort Bay Consulting Pty Ltd. +============================================================== -Copyright 2016 The Netty Project +The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd +unless otherwise noted. -The Netty Project licenses this file to you under the Apache License, -version 2.0 (the "License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at: +Jetty is dual licensed under both - http://www.apache.org/licenses/LICENSE-2.0 + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -License for the specific language governing permissions and limitations -under the License. + and -------------------------------------------------------------------------------- -This product contains a forked and modified version of Tomcat Native + * The Eclipse Public 1.0 License + http://www.eclipse.org/legal/epl-v10.html - * LICENSE: - * license/LICENSE.tomcat-native.txt (Apache License 2.0) - * HOMEPAGE: - * http://tomcat.apache.org/native-doc/ - * https://svn.apache.org/repos/asf/tomcat/native/ +Jetty may be distributed under either license. -This product contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. +------ +Eclipse - * LICENSE: - * license/LICENSE.mvn-wrapper.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/takari/maven-wrapper +The following artifacts are EPL. + * org.eclipse.jetty.orbit:org.eclipse.jdt.core -This product contains small piece of code to support AIX, taken from netbsd. +The following artifacts are EPL and ASL2. + * org.eclipse.jetty.orbit:javax.security.auth.message - * LICENSE: - * license/LICENSE.aix-netbsd.txt (OpenSSL License) - * HOMEPAGE: - * https://ftp.netbsd.org/pub/NetBSD/NetBSD-current/src/crypto/external/bsd/openssl/dist +The following artifacts are EPL and CDDL 1.0. + * org.eclipse.jetty.orbit:javax.mail.glassfish -This product contains code from boringssl. - * LICENSE (Combination ISC and OpenSSL license) - * license/LICENSE.boringssl.txt (Combination ISC and OpenSSL license) - * HOMEPAGE: - * https://boringssl.googlesource.com/boringssl/ +------ +Oracle -======================================================================= +The following artifacts are CDDL + GPLv2 with classpath exception. +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html -guice-4.2.2 NOTICE + * javax.servlet:javax.servlet-api + * javax.annotation:javax.annotation-api + * javax.transaction:javax.transaction-api + * javax.websocket:javax.websocket-api -======================================================================= +------ +Oracle OpenJDK +If ALPN is used to negotiate HTTP/2 connections, then the following +artifacts may be included in the distribution or downloaded when ALPN +module is selected. -Google Guice - Core Library -Copyright 2006-2018 Google, Inc. + * java.sun.security.ssl -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +These artifacts replace/modify OpenJDK classes. The modififications +are hosted at github and both modified and original are under GPL v2 with +classpath exceptions. +http://openjdk.java.net/legal/gplv2+ce.html +------ +OW2 -======================================================================= +The following artifacts are licensed by the OW2 Foundation according to the +terms of http://asm.ow2.org/license.html -http-client-spi-2.20.29 NOTICE +org.ow2.asm:asm-commons +org.ow2.asm:asm -======================================================================= -AWS SDK for Java 2.0 -Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +------ +Apache -This product includes software developed by -Amazon Technologies, Inc (http://www.amazon.com/). +The following artifacts are ASL2 licensed. -********************** -THIRD PARTY COMPONENTS -********************** -This software includes third party software subject to the following copyrights: -- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. -- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. -- Apache Commons Lang - https://github.com/apache/commons-lang -- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams -- Jackson-core - https://github.com/FasterXML/jackson-core -- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary +org.apache.taglibs:taglibs-standard-spec +org.apache.taglibs:taglibs-standard-impl -The licenses for these third party components are included in LICENSE.txt -- For Apache Commons Lang see also this required NOTICE: - Apache Commons Lang - Copyright 2001-2020 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (https://www.apache.org/). +------ +MortBay -======================================================================= +The following artifacts are ASL2 licensed. Based on selected classes from +following Apache Tomcat jars, all ASL2 licensed. -httpasyncclient-4.1.3 NOTICE +org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util -======================================================================= +org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api -Apache HttpAsyncClient -Copyright 2010-2017 The Apache Software Foundation +------ +Mortbay -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +The following artifacts are CDDL + GPLv2 with classpath exception. +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html +org.eclipse.jetty.toolchain:jetty-schemas -======================================================================= +------ +Assorted -httpclient-4.5.13 NOTICE +The UnixCrypt.java code implements the one way cryptography used by +Unix systems for simple password protection. Copyright 1996 Aki Yoshida, +modified April 2001 by Iris Van den Broeke, Daniel Deville. +Permission to use, copy, modify and distribute UnixCrypt +for non-commercial or commercial purposes and without fee is +granted provided that the copyright notice appears in all copies. ======================================================================= - -Apache HttpClient -Copyright 1999-2020 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - - +websocket-api-9.4.53.v20231009 NOTICE ======================================================================= -httpcore-4.4.13 NOTICE - -======================================================================= +============================================================== + Jetty Web Container + Copyright 1995-2018 Mort Bay Consulting Pty Ltd. +============================================================== +The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd +unless otherwise noted. -Apache HttpCore -Copyright 2005-2020 The Apache Software Foundation +Jetty is dual licensed under both -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html + and + * The Eclipse Public 1.0 License + http://www.eclipse.org/legal/epl-v10.html -======================================================================= +Jetty may be distributed under either license. -httpcore-nio-4.4.6 NOTICE +------ +Eclipse -======================================================================= +The following artifacts are EPL. + * org.eclipse.jetty.orbit:org.eclipse.jdt.core +The following artifacts are EPL and ASL2. + * org.eclipse.jetty.orbit:javax.security.auth.message -Apache HttpCore NIO -Copyright 2005-2017 The Apache Software Foundation -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +The following artifacts are EPL and CDDL 1.0. + * org.eclipse.jetty.orbit:javax.mail.glassfish +------ +Oracle -======================================================================= +The following artifacts are CDDL + GPLv2 with classpath exception. +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html -httpmime-4.5.13 NOTICE + * javax.servlet:javax.servlet-api + * javax.annotation:javax.annotation-api + * javax.transaction:javax.transaction-api + * javax.websocket:javax.websocket-api -======================================================================= +------ +Oracle OpenJDK +If ALPN is used to negotiate HTTP/2 connections, then the following +artifacts may be included in the distribution or downloaded when ALPN +module is selected. -Apache HttpClient Mime -Copyright 1999-2020 The Apache Software Foundation + * java.sun.security.ssl -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +These artifacts replace/modify OpenJDK classes. The modififications +are hosted at github and both modified and original are under GPL v2 with +classpath exceptions. +http://openjdk.java.net/legal/gplv2+ce.html +------ +OW2 -======================================================================= +The following artifacts are licensed by the OW2 Foundation according to the +terms of http://asm.ow2.org/license.html -jackson-core-2.13.0 NOTICE +org.ow2.asm:asm-commons +org.ow2.asm:asm -======================================================================= -# Jackson JSON processor +------ +Apache -Jackson is a high-performance, Free/Open Source JSON processing library. -It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has -been in development since 2007. -It is currently developed by a community of developers. +The following artifacts are ASL2 licensed. -## Licensing +org.apache.taglibs:taglibs-standard-spec +org.apache.taglibs:taglibs-standard-impl -Jackson 2.x core and extension components are licensed under Apache License 2.0 -To find the details that apply to this artifact see the accompanying LICENSE file. -## Credits +------ +MortBay -A list of contributors may be found from CREDITS(-2.x) file, which is included -in some artifacts (usually source distributions); but is always available -from the source code management (SCM) system project uses. +The following artifacts are ASL2 licensed. Based on selected classes from +following Apache Tomcat jars, all ASL2 licensed. -======================================================================= +org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util -jackson-databind-2.13.0 NOTICE +org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api -======================================================================= -# Jackson JSON processor +------ +Mortbay -Jackson is a high-performance, Free/Open Source JSON processing library. -It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has -been in development since 2007. -It is currently developed by a community of developers. +The following artifacts are CDDL + GPLv2 with classpath exception. -## Licensing +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html -Jackson 2.x core and extension components are licensed under Apache License 2.0 -To find the details that apply to this artifact see the accompanying LICENSE file. +org.eclipse.jetty.toolchain:jetty-schemas -## Credits +------ +Assorted -A list of contributors may be found from CREDITS(-2.x) file, which is included -in some artifacts (usually source distributions); but is always available -from the source code management (SCM) system project uses. +The UnixCrypt.java code implements the one way cryptography used by +Unix systems for simple password protection. Copyright 1996 Aki Yoshida, +modified April 2001 by Iris Van den Broeke, Daniel Deville. +Permission to use, copy, modify and distribute UnixCrypt +for non-commercial or commercial purposes and without fee is +granted provided that the copyright notice appears in all copies. ======================================================================= -jackson-dataformat-yaml-2.13.0 NOTICE +rocketmq-srvutil-4.9.5 NOTICE ======================================================================= -# Jackson JSON processor - -Jackson is a high-performance, Free/Open Source JSON processing library. -It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has -been in development since 2007. -It is currently developed by a community of developers, as well as supported -commercially by FasterXML.com. -## Licensing +rocketmq-srvutil 4.9.5 +Copyright 2012-2023 Apache Software Foundation -Jackson core and extension components may be licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). -## Credits -A list of contributors may be found from CREDITS file, which is included -in some artifacts (usually source distributions); but is always available -from the source code management (SCM) system project uses. ======================================================================= -jakarta.annotation-api-1.3.5 NOTICE +FastInfoset-1.2.16 NOTICE ======================================================================= -# Notices for Jakarta Annotations +# Notices for Eclipse Implementation of JAXB -This content is produced and maintained by the Jakarta Annotations project. +This content is produced and maintained by the Eclipse Implementation of JAXB +project. - * Project home: https://projects.eclipse.org/projects/ee4j.ca +* Project home: https://projects.eclipse.org/projects/ee4j.jaxb-impl ## Trademarks -Jakarta Annotations is a trademark of the Eclipse Foundation. +Eclipse Implementation of JAXB is a trademark of the Eclipse Foundation. + +## Copyright + +All content is the property of the respective authors or their employers. For +more information regarding authorship of content, please consult the listed +source code repository logs. ## Declared Project Licenses This program and the accompanying materials are made available under the terms -of the Eclipse Public License v. 2.0 which is available at -http://www.eclipse.org/legal/epl-2.0. This Source Code may also be made -available under the following Secondary Licenses when the conditions for such -availability set forth in the Eclipse Public License v. 2.0 are satisfied: GNU -General Public License, version 2 with the GNU Classpath Exception which is -available at https://www.gnu.org/software/classpath/license.html. +of the Eclipse Distribution License v. 1.0 which is available at +http://www.eclipse.org/org/documents/edl-v10.php. -SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 +SPDX-License-Identifier: BSD-3-Clause ## Source Code The project maintains the following source code repositories: - * https://github.com/eclipse-ee4j/common-annotations-api +* https://github.com/eclipse-ee4j/jaxb-ri +* https://github.com/eclipse-ee4j/jaxb-istack-commons +* https://github.com/eclipse-ee4j/jaxb-dtd-parser +* https://github.com/eclipse-ee4j/jaxb-fi +* https://github.com/eclipse-ee4j/jaxb-stax-ex +* https://github.com/eclipse-ee4j/jax-rpc-ri ## Third-party Content -## Cryptography +This project leverages the following third party content. -Content may contain encryption software. The country in which you are currently -may have restrictions on the import, possession, and use, and/or re-export to -another country, of encryption software. BEFORE using any encryption software, -please check the country's laws, regulations and policies concerning the import, -possession, or use, and re-export of encryption software, to see if this is -permitted. +Apache Ant (1.10.2) -======================================================================= +* License: Apache-2.0 AND W3C AND LicenseRef-Public-Domain -json-utils-2.20.29 NOTICE +Apache Ant (1.10.2) -======================================================================= +* License: Apache-2.0 AND W3C AND LicenseRef-Public-Domain -AWS SDK for Java 2.0 -Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +Apache Felix (1.2.0) -This product includes software developed by -Amazon Technologies, Inc (http://www.amazon.com/). +* License: Apache License, 2.0 -********************** -THIRD PARTY COMPONENTS -********************** -This software includes third party software subject to the following copyrights: -- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. -- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. -- Apache Commons Lang - https://github.com/apache/commons-lang -- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams -- Jackson-core - https://github.com/FasterXML/jackson-core -- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary +args4j (2.33) -The licenses for these third party components are included in LICENSE.txt +* License: MIT License -- For Apache Commons Lang see also this required NOTICE: - Apache Commons Lang - Copyright 2001-2020 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (https://www.apache.org/). +dom4j (1.6.1) -======================================================================= +* License: Custom license based on Apache 1.1 -kafka-clients-3.0.0 NOTICE +file-management (3.0.0) -======================================================================= +* License: Apache-2.0 +* Project: https://maven.apache.org/shared/file-management/ +* Source: + https://svn.apache.org/viewvc/maven/shared/tags/file-management-3.0.0/ -Apache Kafka -Copyright 2021 The Apache Software Foundation. +JUnit (4.12) -This product includes software developed at -The Apache Software Foundation (https://www.apache.org/). +* License: Eclipse Public License -This distribution has a binary dependency on jersey, which is available under the CDDL -License. The source code of jersey can be found at https://github.com/jersey/jersey/. +JUnit (4.12) -This distribution has a binary test dependency on jqwik, which is available under -the Eclipse Public License 2.0. The source code can be found at -https://github.com/jlink/jqwik. +* License: Eclipse Public License -The streams-scala (streams/streams-scala) module was donated by Lightbend and the original code was copyrighted by them: -Copyright (C) 2018 Lightbend Inc. -Copyright (C) 2017-2018 Alexis Seigneurin. +maven-compat (3.5.2) -This project contains the following code copied from Apache Hadoop: -clients/src/main/java/org/apache/kafka/common/utils/PureJavaCrc32C.java -Some portions of this file Copyright (c) 2004-2006 Intel Corporation and licensed under the BSD license. +* License: Apache-2.0 +* Project: https://maven.apache.org/ref/3.5.2/maven-compat/ +* Source: + https://mvnrepository.com/artifact/org.apache.maven/maven-compat/3.5.2 -This project contains the following code copied from Apache Hive: -streams/src/main/java/org/apache/kafka/streams/state/internals/Murmur3.java -======================================================================= +maven-core (3.5.2) -log4j-api-2.22.1 NOTICE +* License: Apache-2.0 +* Project: https://maven.apache.org/ref/3.5.2/maven-core/index.html +* Source: https://mvnrepository.com/artifact/org.apache.maven/maven-core/3.5.2 -======================================================================= +maven-plugin-annotations (3.5) -Apache Log4j API -Copyright 1999-2023 The Apache Software Foundation +* License: Apache-2.0 +* Project: https://maven.apache.org/plugin-tools/maven-plugin-annotations/ +* Source: + https://github.com/apache/maven-plugin-tools/tree/master/maven-plugin-annotations +maven-plugin-api (3.5.2) -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +* License: Apache-2.0 -======================================================================= +maven-resolver-api (1.1.1) -log4j-core-2.22.1 NOTICE +* License: Apache-2.0 -======================================================================= +maven-resolver-api (1.1.1) -Apache Log4j Core -Copyright 1999-2012 Apache Software Foundation +* License: Apache-2.0 -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +maven-resolver-connector-basic (1.1.1) -ResolverUtil.java -Copyright 2005-2006 Tim Fennell -======================================================================= +* License: Apache-2.0 -log4j-slf4j2-impl-2.22.1 NOTICE +maven-resolver-impl (1.1.1) -======================================================================= +* License: Apache-2.0 -Apache Log4j SLF4J 2.0 Binding -Copyright 1999-2023 The Apache Software Foundation +maven-resolver-spi (1.1.1) +* License: Apache-2.0 -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +maven-resolver-transport-file (1.1.1) -======================================================================= +* License: Apache-2.0 +* Project: https://maven.apache.org/resolver/maven-resolver-transport-file/ +* Source: + https://github.com/apache/maven-resolver/tree/master/maven-resolver-transport-file -metrics-spi-2.20.29 NOTICE +maven-resolver-util (1.1.1) -======================================================================= +* License: Apache-2.0 -AWS SDK for Java 2.0 -Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +maven-settings (3.5.2) -This product includes software developed by -Amazon Technologies, Inc (http://www.amazon.com/). +* License: Apache-2.0 +* Source: + https://mvnrepository.com/artifact/org.apache.maven/maven-settings/3.5.2 -********************** -THIRD PARTY COMPONENTS -********************** -This software includes third party software subject to the following copyrights: -- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. -- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. -- Apache Commons Lang - https://github.com/apache/commons-lang -- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams -- Jackson-core - https://github.com/FasterXML/jackson-core -- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary +OSGi Service Platform Core Companion Code (6.0) -The licenses for these third party components are included in LICENSE.txt +* License: Apache License, 2.0 -- For Apache Commons Lang see also this required NOTICE: - Apache Commons Lang - Copyright 2001-2020 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (https://www.apache.org/). +plexus-archiver (3.5) -======================================================================= +* License: Apache-2.0 +* Project: https://codehaus-plexus.github.io/plexus-archiver/ +* Source: https://github.com/codehaus-plexus/plexus-archiver -nacos-client-2.2.1 NOTICE +plexus-io (3.0.0) -======================================================================= +* License: Apache-2.0 - The Netty Project - ================= +plexus-utils (3.1.0) -Please visit the Netty web site for more information: +* License: Apache- 2.0 or Apache- 1.1 or BSD or Public Domain or Indiana + University Extreme! Lab Software License V1.1.1 (Apache 1.1 style) - * http://netty.io/ +relaxng-datatype (1.0) -Copyright 2016 The Netty Project +* License: New BSD license -The Netty Project licenses this file to you under the Apache License, -version 2.0 (the "License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at: +Sax (0.2) - http://www.apache.org/licenses/LICENSE-2.0 +* License: SAX-PD +* Project: http://www.megginson.com/downloads/SAX/ +* Source: http://sourceforge.net/project/showfiles.php?group_id=29449 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -License for the specific language governing permissions and limitations -under the License. +testng (6.14.2) -------------------------------------------------------------------------------- -This product contains a forked and modified version of Tomcat Native +* License: Apache-2.0 AND (MIT OR GPL-1.0+) +* Project: https://testng.org/doc/index.html +* Source: https://github.com/cbeust/testng - * LICENSE: - * license/LICENSE.tomcat-native.txt (Apache License 2.0) - * HOMEPAGE: - * http://tomcat.apache.org/native-doc/ - * https://svn.apache.org/repos/asf/tomcat/native/ +wagon-http-lightweight (3.0.0) -This product contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. +* License: Pending +* Project: https://maven.apache.org/wagon/ +* Source: + https://mvnrepository.com/artifact/org.apache.maven.wagon/wagon-http-lightweight/3.0.0 - * LICENSE: - * license/LICENSE.mvn-wrapper.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/takari/maven-wrapper +xz for java (1.8) -This product contains small piece of code to support AIX, taken from netbsd. +* License: LicenseRef-Public-Domain - * LICENSE: - * license/LICENSE.aix-netbsd.txt (OpenSSL License) - * HOMEPAGE: - * https://ftp.netbsd.org/pub/NetBSD/NetBSD-current/src/crypto/external/bsd/openssl/dist +## Cryptography +Content may contain encryption software. The country in which you are currently +may have restrictions on the import, possession, and use, and/or re-export to +another country, of encryption software. BEFORE using any encryption software, +please check the country's laws, regulations and policies concerning the import, +possession, or use, and re-export of encryption software, to see if this is +permitted. -This product contains code from boringssl. +======================================================================= - * LICENSE (Combination ISC and OpenSSL license) - * license/LICENSE.boringssl.txt (Combination ISC and OpenSSL license) - * HOMEPAGE: - * https://boringssl.googlesource.com/boringssl/ +log4j-slf4j2-impl-2.24.1 NOTICE ======================================================================= -netty-3.10.6.Final NOTICE +SLF4J 2 Provider for Log4j API +Copyright 1999-2024 The Apache Software Foundation -======================================================================= +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). - The Netty Project - ================= +======================================================================= -Please visit the Netty web site for more information: +spring-core-5.3.31 NOTICE - * http://netty.io/ +======================================================================= -Copyright 2011 The Netty Project +Spring Framework 5.3.31 +Copyright (c) 2002-2023 Pivotal, Inc. -The Netty Project licenses this file to you under the Apache License, -version 2.0 (the "License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at: +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. -http://www.apache.org/licenses/LICENSE-2.0 +This product may include a number of subcomponents with separate +copyright notices and license terms. Your use of the source code for +these subcomponents is subject to the terms and conditions of the +subcomponent's license, as noted in the license.txt file. -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -License for the specific language governing permissions and limitations -under the License. - -Also, please refer to each LICENSE..txt file, which is located in -the 'license' directory of the distribution file, for the license terms of the -components that this product depends on. - -------------------------------------------------------------------------------- -This product contains the extensions to Java Collections Framework which has -been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: - - * LICENSE: - * license/LICENSE.jsr166y.txt (Public Domain) - * HOMEPAGE: - * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ - * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ - -This product contains a modified version of Robert Harder's Public Domain -Base64 Encoder and Decoder, which can be obtained at: - - * LICENSE: - * license/LICENSE.base64.txt (Public Domain) - * HOMEPAGE: - * http://iharder.sourceforge.net/current/java/base64/ - -This product contains a modified version of 'JZlib', a re-implementation of -zlib in pure Java, which can be obtained at: - - * LICENSE: - * license/LICENSE.jzlib.txt (BSD Style License) - * HOMEPAGE: - * http://www.jcraft.com/jzlib/ - -This product contains a modified version of 'Webbit', a Java event based -WebSocket and HTTP server: - - * LICENSE: - * license/LICENSE.webbit.txt (BSD License) - * HOMEPAGE: - * https://github.com/joewalnes/webbit - -This product optionally depends on 'Protocol Buffers', Google's data -interchange format, which can be obtained at: - - * LICENSE: - * license/LICENSE.protobuf.txt (New BSD License) - * HOMEPAGE: - * http://code.google.com/p/protobuf/ - -This product optionally depends on 'Bouncy Castle Crypto APIs' to generate -a temporary self-signed X.509 certificate when the JVM does not provide the -equivalent functionality. It can be obtained at: - - * LICENSE: - * license/LICENSE.bouncycastle.txt (MIT License) - * HOMEPAGE: - * http://www.bouncycastle.org/ - -This product optionally depends on 'SLF4J', a simple logging facade for Java, -which can be obtained at: - - * LICENSE: - * license/LICENSE.slf4j.txt (MIT License) - * HOMEPAGE: - * http://www.slf4j.org/ - -This product optionally depends on 'Apache Commons Logging', a logging -framework, which can be obtained at: - - * LICENSE: - * license/LICENSE.commons-logging.txt (Apache License 2.0) - * HOMEPAGE: - * http://commons.apache.org/logging/ +======================================================================= -This product optionally depends on 'Apache Log4J', a logging framework, -which can be obtained at: +httpcore-nio-4.4.15 NOTICE - * LICENSE: - * license/LICENSE.log4j.txt (Apache License 2.0) - * HOMEPAGE: - * http://logging.apache.org/log4j/ +======================================================================= -This product optionally depends on 'JBoss Logging', a logging framework, -which can be obtained at: - * LICENSE: - * license/LICENSE.jboss-logging.txt (GNU LGPL 2.1) - * HOMEPAGE: - * http://anonsvn.jboss.org/repos/common/common-logging-spi/ +Apache HttpCore NIO +Copyright 2005-2021 The Apache Software Foundation -This product optionally depends on 'Apache Felix', an open source OSGi -framework implementation, which can be obtained at: +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). - * LICENSE: - * license/LICENSE.felix.txt (Apache License 2.0) - * HOMEPAGE: - * http://felix.apache.org/ ======================================================================= -netty-nio-client-2.20.29 NOTICE +pulsar-client-api-2.11.1 NOTICE ======================================================================= -AWS SDK for Java 2.0 -Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -This product includes software developed by -Amazon Technologies, Inc (http://www.amazon.com/). +Pulsar Client :: API +Copyright 2017-2020 Apache Software Foundation -********************** -THIRD PARTY COMPONENTS -********************** -This software includes third party software subject to the following copyrights: -- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. -- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. -- Apache Commons Lang - https://github.com/apache/commons-lang -- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams -- Jackson-core - https://github.com/FasterXML/jackson-core -- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). -The licenses for these third party components are included in LICENSE.txt -- For Apache Commons Lang see also this required NOTICE: - Apache Commons Lang - Copyright 2001-2020 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (https://www.apache.org/). ======================================================================= -netty-tcnative-boringssl-static-2.0.48.Final NOTICE +spring-boot-autoconfigure-2.5.9 NOTICE ======================================================================= - The Netty Project - ================= - -Please visit the Netty web site for more information: - - * http://netty.io/ - -Copyright 2016 The Netty Project - -The Netty Project licenses this file to you under the Apache License, -version 2.0 (the "License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at: - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -License for the specific language governing permissions and limitations -under the License. - -------------------------------------------------------------------------------- -This product contains a forked and modified version of Tomcat Native +Spring Boot 2.5.9 +Copyright (c) 2012-2022 Pivotal, Inc. - * LICENSE: - * license/LICENSE.tomcat-native.txt (Apache License 2.0) - * HOMEPAGE: - * http://tomcat.apache.org/native-doc/ - * https://svn.apache.org/repos/asf/tomcat/native/ +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. +======================================================================= -This product contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. +jetty-util-ajax-9.4.53.v20231009 NOTICE - * LICENSE: - * license/LICENSE.mvn-wrapper.txt (Apache License 2.0) - * HOMEPAGE: - * https://github.com/takari/maven-wrapper +======================================================================= -This product contains small piece of code to support AIX, taken from netbsd. +============================================================== + Jetty Web Container + Copyright 1995-2018 Mort Bay Consulting Pty Ltd. +============================================================== - * LICENSE: - * license/LICENSE.aix-netbsd.txt (OpenSSL License) - * HOMEPAGE: - * https://ftp.netbsd.org/pub/NetBSD/NetBSD-current/src/crypto/external/bsd/openssl/dist +The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd +unless otherwise noted. +Jetty is dual licensed under both -This product contains code from boringssl. + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html - * LICENSE (Combination ISC and OpenSSL license) - * license/LICENSE.boringssl.txt (Combination ISC and OpenSSL license) - * HOMEPAGE: - * https://boringssl.googlesource.com/boringssl/ + and -======================================================================= + * The Eclipse Public 1.0 License + http://www.eclipse.org/legal/epl-v10.html -okhttp-3.14.9 NOTICE +Jetty may be distributed under either license. -======================================================================= +------ +Eclipse -Note that publicsuffixes.gz is compiled from The Public Suffix List: -https://publicsuffix.org/list/public_suffix_list.dat +The following artifacts are EPL. + * org.eclipse.jetty.orbit:org.eclipse.jdt.core -It is subject to the terms of the Mozilla Public License, v. 2.0: -https://mozilla.org/MPL/2.0/ +The following artifacts are EPL and ASL2. + * org.eclipse.jetty.orbit:javax.security.auth.message -======================================================================= -pravega-client-0.11.0 NOTICE +The following artifacts are EPL and CDDL 1.0. + * org.eclipse.jetty.orbit:javax.mail.glassfish -======================================================================= -Copyright (c) 2021 Pravega Authors. -Copyright (c) 2017-2021 Dell Inc., or its subsidiaries. All Rights Reserved. +------ +Oracle -This software contains source code from Apache BookKeeper, distributed under -the Apache License Version 2.0, and copyrighted to the Apache Software Foundation. -http://bookkeeper.apache.org +The following artifacts are CDDL + GPLv2 with classpath exception. +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html -======================================================================= + * javax.servlet:javax.servlet-api + * javax.annotation:javax.annotation-api + * javax.transaction:javax.transaction-api + * javax.websocket:javax.websocket-api -pravega-common-0.11.0 NOTICE +------ +Oracle OpenJDK -======================================================================= +If ALPN is used to negotiate HTTP/2 connections, then the following +artifacts may be included in the distribution or downloaded when ALPN +module is selected. -Copyright (c) 2021 Pravega Authors. -Copyright (c) 2017-2021 Dell Inc., or its subsidiaries. All Rights Reserved. + * java.sun.security.ssl -This software contains source code from Apache BookKeeper, distributed under -the Apache License Version 2.0, and copyrighted to the Apache Software Foundation. -http://bookkeeper.apache.org +These artifacts replace/modify OpenJDK classes. The modififications +are hosted at github and both modified and original are under GPL v2 with +classpath exceptions. +http://openjdk.java.net/legal/gplv2+ce.html -======================================================================= -pravega-shared-authplugin-0.11.0 NOTICE +------ +OW2 -======================================================================= +The following artifacts are licensed by the OW2 Foundation according to the +terms of http://asm.ow2.org/license.html -Copyright (c) 2021 Pravega Authors. -Copyright (c) 2017-2021 Dell Inc., or its subsidiaries. All Rights Reserved. +org.ow2.asm:asm-commons +org.ow2.asm:asm -This software contains source code from Apache BookKeeper, distributed under -the Apache License Version 2.0, and copyrighted to the Apache Software Foundation. -http://bookkeeper.apache.org -======================================================================= +------ +Apache -pravega-shared-controller-api-0.11.0 NOTICE +The following artifacts are ASL2 licensed. -======================================================================= +org.apache.taglibs:taglibs-standard-spec +org.apache.taglibs:taglibs-standard-impl -Copyright (c) 2021 Pravega Authors. -Copyright (c) 2017-2021 Dell Inc., or its subsidiaries. All Rights Reserved. -This software contains source code from Apache BookKeeper, distributed under -the Apache License Version 2.0, and copyrighted to the Apache Software Foundation. -http://bookkeeper.apache.org +------ +MortBay -======================================================================= +The following artifacts are ASL2 licensed. Based on selected classes from +following Apache Tomcat jars, all ASL2 licensed. -pravega-shared-protocol-0.11.0 NOTICE +org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util -======================================================================= +org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api -Copyright (c) 2021 Pravega Authors. -Copyright (c) 2017-2021 Dell Inc., or its subsidiaries. All Rights Reserved. -This software contains source code from Apache BookKeeper, distributed under -the Apache License Version 2.0, and copyrighted to the Apache Software Foundation. -http://bookkeeper.apache.org +------ +Mortbay -======================================================================= +The following artifacts are CDDL + GPLv2 with classpath exception. -pravega-shared-security-0.11.0 NOTICE +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html -======================================================================= +org.eclipse.jetty.toolchain:jetty-schemas -Copyright (c) 2021 Pravega Authors. -Copyright (c) 2017-2021 Dell Inc., or its subsidiaries. All Rights Reserved. +------ +Assorted -This software contains source code from Apache BookKeeper, distributed under -the Apache License Version 2.0, and copyrighted to the Apache Software Foundation. -http://bookkeeper.apache.org +The UnixCrypt.java code implements the one way cryptography used by +Unix systems for simple password protection. Copyright 1996 Aki Yoshida, +modified April 2001 by Iris Van den Broeke, Daniel Deville. +Permission to use, copy, modify and distribute UnixCrypt +for non-commercial or commercial purposes and without fee is +granted provided that the copyright notice appears in all copies. ======================================================================= -profiles-2.20.29 NOTICE +aws-query-protocol-2.29.5 NOTICE ======================================================================= @@ -1466,7 +1298,7 @@ The licenses for these third party components are included in LICENSE.txt ======================================================================= -protocol-core-2.20.29 NOTICE +regions-2.29.5 NOTICE ======================================================================= @@ -1498,164 +1330,215 @@ The licenses for these third party components are included in LICENSE.txt ======================================================================= -pulsar-client-2.10.1 NOTICE +commons-codec-1.17.1 NOTICE ======================================================================= - -Pulsar Client Java -Copyright 2017-2020 Apache Software Foundation +Apache Commons Codec +Copyright 2002-2024 The Apache Software Foundation This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +The Apache Software Foundation (https://www.apache.org/). +======================================================================= +micrometer-core-1.9.17 NOTICE ======================================================================= -pulsar-client-2.10.1 NOTICE +Micrometer -======================================================================= +Copyright (c) 2017-Present VMware, Inc. All Rights Reserved. -Apache Commons Lang -Copyright 2001-2020 The Apache Software Foundation +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at -This product includes software developed at -The Apache Software Foundation (https://www.apache.org/). + https://www.apache.org/licenses/LICENSE-2.0 -======================================================================= +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. -pulsar-client-2.11.1 NOTICE +------------------------------------------------------------------------------- -======================================================================= +This product contains a modified portion of 'io.netty.util.internal.logging', +in the Netty/Common library distributed by The Netty Project: + * Copyright 2013 The Netty Project + * License: Apache License v2.0 + * Homepage: https://netty.io -Pulsar Client Java -Copyright 2017-2020 Apache Software Foundation +This product contains a modified portion of 'StringUtils.isBlank()', +in the Commons Lang library distributed by The Apache Software Foundation: -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). + * Copyright 2001-2019 The Apache Software Foundation + * License: Apache License v2.0 + * Homepage: https://commons.apache.org/proper/commons-lang/ +This product contains a modified portion of 'JsonUtf8Writer', +in the Moshi library distributed by Square, Inc: + * Copyright 2010 Google Inc. + * License: Apache License v2.0 + * Homepage: https://github.com/square/moshi -======================================================================= +This product contains a modified portion of the 'org.springframework.lang' +package in the Spring Framework library, distributed by VMware, Inc: -pulsar-client-2.11.1 NOTICE + * Copyright 2002-2019 the original author or authors. + * License: Apache License v2.0 + * Homepage: https://spring.io/projects/spring-framework ======================================================================= -Apache Commons Lang -Copyright 2001-2020 The Apache Software Foundation +commons-logging-1.3.2 NOTICE -This product includes software developed at +======================================================================= + +Apache Commons Logging +Copyright 2001-2024 The Apache Software Foundation + +This product includes software developed at The Apache Software Foundation (https://www.apache.org/). ======================================================================= -pulsar-client-admin-api-2.10.1 NOTICE +jakarta.websocket-api-1.1.2 NOTICE ======================================================================= +# Notices for Jakarta WebSocket -Pulsar Client Admin :: API -Copyright 2017-2020 Apache Software Foundation +This content is produced and maintained by the Jakarta WebSocket project. -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +* Project home: https://projects.eclipse.org/projects/ee4j.websocket +## Trademarks +Jakarta WebSocket is a trademark of the Eclipse Foundation. -======================================================================= +## Copyright -pulsar-client-admin-api-2.11.1 NOTICE +All content is the property of the respective authors or their employers. For +more information regarding authorship of content, please consult the listed +source code repository logs. -======================================================================= +## Declared Project Licenses + +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v. 2.0 which is available at +http://www.eclipse.org/legal/epl-2.0. This Source Code may also be made +available under the following Secondary Licenses when the conditions for such +availability set forth in the Eclipse Public License v. 2.0 are satisfied: GNU +General Public License, version 2 with the GNU Classpath Exception which is +available at https://www.gnu.org/software/classpath/license.html. +SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 -Pulsar Client Admin :: API -Copyright 2017-2020 Apache Software Foundation +## Source Code -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +The project maintains the following source code repositories: + +* https://github.com/eclipse-ee4j/websocket-api + +## Third-party Content +This project leverages the following third party content. +None + +## Cryptography + +Content may contain encryption software. The country in which you are currently +may have restrictions on the import, possession, and use, and/or re-export to +another country, of encryption software. BEFORE using any encryption software, +please check the country's laws, regulations and policies concerning the import, +possession, or use, and re-export of encryption software, to see if this is +permitted. ======================================================================= -pulsar-client-api-2.10.1 NOTICE +spring-aop-5.3.9 NOTICE ======================================================================= +Spring Framework 5.3.9 +Copyright (c) 2002-2021 Pivotal, Inc. -Pulsar Client :: API -Copyright 2017-2020 Apache Software Foundation +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +This product may include a number of subcomponents with separate +copyright notices and license terms. Your use of the source code for +these subcomponents is subject to the terms and conditions of the +subcomponent's license, as noted in the license.txt file. +======================================================================= +spring-boot-starter-web-2.7.18 NOTICE ======================================================================= -pulsar-client-api-2.11.1 NOTICE +Spring Boot 2.7.18 +Copyright (c) 2012-2023 VMware, Inc. +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. ======================================================================= +jakarta.inject-api-2.0.1 NOTICE -Pulsar Client :: API -Copyright 2017-2020 Apache Software Foundation +======================================================================= -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +# Notices for Eclipse Jakarta Dependency Injection +This content is produced and maintained by the Eclipse Jakarta Dependency Injection project. +* Project home: https://projects.eclipse.org/projects/cdi.batch -======================================================================= +## Trademarks -redisson-3.17.3 NOTICE +Jakarta Dependency Injection is a trademark of the Eclipse Foundation. -======================================================================= +## Copyright -# Jackson JSON processor +All content is the property of the respective authors or their employers. For +more information regarding authorship of content, please consult the listed +source code repository logs. -Jackson is a high-performance, Free/Open Source JSON processing library. -It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has -been in development since 2007. -It is currently developed by a community of developers, as well as supported -commercially by FasterXML.com. +## Declared Project Licenses -## Licensing +This program and the accompanying materials are made available under the terms +of the Apache License, Version 2.0 which is available at +https://www.apache.org/licenses/LICENSE-2.0. -Jackson core and extension components may be licensed under different licenses. -To find the details that apply to this artifact see the accompanying LICENSE file. -For more information, including possible other licensing options, contact -FasterXML.com (http://fasterxml.com). +SPDX-License-Identifier: Apache-2.0 -## Credits +## Source Code -A list of contributors may be found from CREDITS file, which is included -in some artifacts (usually source distributions); but is always available -from the source code management (SCM) system project uses. +The project maintains the following source code repositories: -# Byte Buddy +https://github.com/eclipse-ee4j/injection-api +https://github.com/eclipse-ee4j/injection-spec +https://github.com/eclipse-ee4j/injection-tck -Copyright 2014 - 2019 Rafael Winterhalter +## Third-party Content -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at +This project leverages the following third party content. - http://www.apache.org/licenses/LICENSE-2.0 +None -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. +## Cryptography +None ======================================================================= -regions-2.20.29 NOTICE +http-auth-2.29.5 NOTICE ======================================================================= @@ -1687,204 +1570,293 @@ The licenses for these third party components are included in LICENSE.txt ======================================================================= -rocketmq-acl-4.9.5 NOTICE +spring-boot-2.5.9 NOTICE ======================================================================= +Spring Boot 2.5.9 +Copyright (c) 2012-2022 Pivotal, Inc. -rocketmq-acl 4.9.5 -Copyright 2012-2023 Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. +======================================================================= +checksums-spi-2.29.5 NOTICE ======================================================================= -rocketmq-broker-4.9.5 NOTICE +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -======================================================================= +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary -rocketmq-broker 4.9.5 -Copyright 2012-2023 Apache Software Foundation +The licenses for these third party components are included in LICENSE.txt -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). +======================================================================= +pravega-shared-authplugin-0.11.0 NOTICE ======================================================================= -rocketmq-client-4.9.5 NOTICE +Copyright (c) 2021 Pravega Authors. +Copyright (c) 2017-2021 Dell Inc., or its subsidiaries. All Rights Reserved. + +This software contains source code from Apache BookKeeper, distributed under +the Apache License Version 2.0, and copyrighted to the Apache Software Foundation. +http://bookkeeper.apache.org ======================================================================= +spring-web-5.3.31 NOTICE -rocketmq-client 4.9.5 -Copyright 2012-2023 Apache Software Foundation +======================================================================= -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +Spring Framework 5.3.31 +Copyright (c) 2002-2023 Pivotal, Inc. +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. +This product may include a number of subcomponents with separate +copyright notices and license terms. Your use of the source code for +these subcomponents is subject to the terms and conditions of the +subcomponent's license, as noted in the license.txt file. ======================================================================= -rocketmq-common-4.9.5 NOTICE +jetty-continuation-9.4.53.v20231009 NOTICE ======================================================================= +============================================================== + Jetty Web Container + Copyright 1995-2018 Mort Bay Consulting Pty Ltd. +============================================================== -rocketmq-common 4.9.5 -Copyright 2012-2023 Apache Software Foundation +The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd +unless otherwise noted. -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +Jetty is dual licensed under both + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html + and -======================================================================= + * The Eclipse Public 1.0 License + http://www.eclipse.org/legal/epl-v10.html -rocketmq-filter-4.9.5 NOTICE +Jetty may be distributed under either license. -======================================================================= +------ +Eclipse +The following artifacts are EPL. + * org.eclipse.jetty.orbit:org.eclipse.jdt.core -rocketmq-filter 4.9.5 -Copyright 2012-2023 Apache Software Foundation +The following artifacts are EPL and ASL2. + * org.eclipse.jetty.orbit:javax.security.auth.message -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +The following artifacts are EPL and CDDL 1.0. + * org.eclipse.jetty.orbit:javax.mail.glassfish -======================================================================= +------ +Oracle -rocketmq-logging-4.9.5 NOTICE +The following artifacts are CDDL + GPLv2 with classpath exception. +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html -======================================================================= + * javax.servlet:javax.servlet-api + * javax.annotation:javax.annotation-api + * javax.transaction:javax.transaction-api + * javax.websocket:javax.websocket-api +------ +Oracle OpenJDK -rocketmq-logging 4.9.5 -Copyright 2012-2023 Apache Software Foundation +If ALPN is used to negotiate HTTP/2 connections, then the following +artifacts may be included in the distribution or downloaded when ALPN +module is selected. -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). + * java.sun.security.ssl +These artifacts replace/modify OpenJDK classes. The modififications +are hosted at github and both modified and original are under GPL v2 with +classpath exceptions. +http://openjdk.java.net/legal/gplv2+ce.html -======================================================================= +------ +OW2 -rocketmq-namesrv-4.9.5 NOTICE +The following artifacts are licensed by the OW2 Foundation according to the +terms of http://asm.ow2.org/license.html -======================================================================= +org.ow2.asm:asm-commons +org.ow2.asm:asm -rocketmq-namesrv 4.9.5 -Copyright 2012-2023 Apache Software Foundation +------ +Apache -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +The following artifacts are ASL2 licensed. +org.apache.taglibs:taglibs-standard-spec +org.apache.taglibs:taglibs-standard-impl -======================================================================= +------ +MortBay -rocketmq-remoting-4.9.5 NOTICE +The following artifacts are ASL2 licensed. Based on selected classes from +following Apache Tomcat jars, all ASL2 licensed. -======================================================================= +org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util +org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api -rocketmq-remoting 4.9.5 -Copyright 2012-2023 Apache Software Foundation -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +------ +Mortbay + +The following artifacts are CDDL + GPLv2 with classpath exception. + +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html +org.eclipse.jetty.toolchain:jetty-schemas +------ +Assorted + +The UnixCrypt.java code implements the one way cryptography used by +Unix systems for simple password protection. Copyright 1996 Aki Yoshida, +modified April 2001 by Iris Van den Broeke, Daniel Deville. +Permission to use, copy, modify and distribute UnixCrypt +for non-commercial or commercial purposes and without fee is +granted provided that the copyright notice appears in all copies. ======================================================================= -rocketmq-srvutil-4.9.5 NOTICE +jackson-databind-2.18.0 NOTICE ======================================================================= +# Jackson JSON processor -rocketmq-srvutil 4.9.5 -Copyright 2012-2023 Apache Software Foundation +Jackson is a high-performance, Free/Open Source JSON processing library. +It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has +been in development since 2007. +It is currently developed by a community of developers. -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +## Copyright + +Copyright 2007-, Tatu Saloranta (tatu.saloranta@iki.fi) +## Licensing +Jackson 2.x core and extension components are licensed under Apache License 2.0 +To find the details that apply to this artifact see the accompanying LICENSE file. -======================================================================= +## Credits -rocketmq-store-4.9.5 NOTICE +A list of contributors may be found from CREDITS(-2.x) file, which is included +in some artifacts (usually source distributions); but is always available +from the source code management (SCM) system project uses. ======================================================================= +spring-boot-starter-tomcat-2.7.18 NOTICE -rocketmq-store 4.9.5 -Copyright 2012-2023 Apache Software Foundation +======================================================================= -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +Spring Boot 2.7.18 +Copyright (c) 2012-2023 VMware, Inc. +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. +======================================================================= +jackson-annotations-2.18.0 NOTICE ======================================================================= -rocketmq-tools-4.9.5 NOTICE +# Jackson JSON processor -======================================================================= +Jackson is a high-performance, Free/Open Source JSON processing library. +It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has +been in development since 2007. +It is currently developed by a community of developers. +## Copyright -rocketmq-tools 4.9.5 -Copyright 2012-2023 Apache Software Foundation +Copyright 2007-, Tatu Saloranta (tatu.saloranta@iki.fi) -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +## Licensing + +Jackson 2.x core and extension components are licensed under Apache License 2.0 +To find the details that apply to this artifact see the accompanying LICENSE file. +## Credits +A list of contributors may be found from CREDITS(-2.x) file, which is included +in some artifacts (usually source distributions); but is always available +from the source code management (SCM) system project uses. ======================================================================= -s3-2.20.29 NOTICE +spring-beans-5.3.20 NOTICE ======================================================================= -AWS SDK for Java 2.0 -Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +Spring Framework 5.3.20 +Copyright (c) 2002-2022 Pivotal, Inc. -This product includes software developed by -Amazon Technologies, Inc (http://www.amazon.com/). +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. -********************** -THIRD PARTY COMPONENTS -********************** -This software includes third party software subject to the following copyrights: -- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. -- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. -- Apache Commons Lang - https://github.com/apache/commons-lang -- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams -- Jackson-core - https://github.com/FasterXML/jackson-core -- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary - -The licenses for these third party components are included in LICENSE.txt - -- For Apache Commons Lang see also this required NOTICE: - Apache Commons Lang - Copyright 2001-2020 The Apache Software Foundation - - This product includes software developed at - The Apache Software Foundation (https://www.apache.org/). +This product may include a number of subcomponents with separate +copyright notices and license terms. Your use of the source code for +these subcomponents is subject to the terms and conditions of the +subcomponent's license, as noted in the license.txt file. ======================================================================= -sdk-core-2.20.29 NOTICE +crt-core-2.29.5 NOTICE ======================================================================= @@ -1916,293 +1888,284 @@ The licenses for these third party components are included in LICENSE.txt ======================================================================= -spring-aop-5.3.15 NOTICE +jackson-dataformat-yaml-2.18.0 NOTICE ======================================================================= -Spring Framework 5.3.15 -Copyright (c) 2002-2022 Pivotal, Inc. - -This product is licensed to you under the Apache License, Version 2.0 -(the "License"). You may not use this product except in compliance with -the License. +# Jackson JSON processor -This product may include a number of subcomponents with separate -copyright notices and license terms. Your use of the source code for -these subcomponents is subject to the terms and conditions of the -subcomponent's license, as noted in the license.txt file. +Jackson is a high-performance, Free/Open Source JSON processing library. +It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has +been in development since 2007. +It is currently developed by a community of developers. -======================================================================= +## Copyright -spring-aop-5.3.26 NOTICE +Copyright 2007-, Tatu Saloranta (tatu.saloranta@iki.fi) -======================================================================= +## Licensing -Spring Framework 5.3.26 -Copyright (c) 2002-2023 Pivotal, Inc. +Jackson components are licensed under Apache (Software) License, version 2.0, +as per accompanying LICENSE file. -This product is licensed to you under the Apache License, Version 2.0 -(the "License"). You may not use this product except in compliance with -the License. +## Credits -This product may include a number of subcomponents with separate -copyright notices and license terms. Your use of the source code for -these subcomponents is subject to the terms and conditions of the -subcomponent's license, as noted in the license.txt file. +A list of contributors may be found from CREDITS file, which is included +in some artifacts (usually source distributions); but is always available +from the source code management (SCM) system project uses. ======================================================================= -spring-beans-5.3.20 NOTICE +spring-boot-starter-jetty-2.7.18 NOTICE ======================================================================= -Spring Framework 5.3.20 -Copyright (c) 2002-2022 Pivotal, Inc. +Spring Boot 2.7.18 +Copyright (c) 2012-2023 VMware, Inc. This product is licensed to you under the Apache License, Version 2.0 (the "License"). You may not use this product except in compliance with the License. - -This product may include a number of subcomponents with separate -copyright notices and license terms. Your use of the source code for -these subcomponents is subject to the terms and conditions of the -subcomponent's license, as noted in the license.txt file. - ======================================================================= -spring-beans-5.3.26 NOTICE +jetty-servlets-9.4.53.v20231009 NOTICE ======================================================================= -Spring Framework 5.3.26 -Copyright (c) 2002-2023 Pivotal, Inc. - -This product is licensed to you under the Apache License, Version 2.0 -(the "License"). You may not use this product except in compliance with -the License. +============================================================== + Jetty Web Container + Copyright 1995-2018 Mort Bay Consulting Pty Ltd. +============================================================== -This product may include a number of subcomponents with separate -copyright notices and license terms. Your use of the source code for -these subcomponents is subject to the terms and conditions of the -subcomponent's license, as noted in the license.txt file. +The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd +unless otherwise noted. -======================================================================= +Jetty is dual licensed under both -spring-boot-2.5.9 NOTICE + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html -======================================================================= + and -Spring Boot 2.5.9 -Copyright (c) 2012-2022 Pivotal, Inc. + * The Eclipse Public 1.0 License + http://www.eclipse.org/legal/epl-v10.html -This product is licensed to you under the Apache License, Version 2.0 -(the "License"). You may not use this product except in compliance with -the License. -======================================================================= +Jetty may be distributed under either license. -spring-boot-2.7.10 NOTICE +------ +Eclipse -======================================================================= +The following artifacts are EPL. + * org.eclipse.jetty.orbit:org.eclipse.jdt.core -Spring Boot 2.7.10 -Copyright (c) 2012-2023 VMware, Inc. +The following artifacts are EPL and ASL2. + * org.eclipse.jetty.orbit:javax.security.auth.message -This product is licensed to you under the Apache License, Version 2.0 -(the "License"). You may not use this product except in compliance with -the License. -======================================================================= -spring-boot-autoconfigure-2.5.9 NOTICE +The following artifacts are EPL and CDDL 1.0. + * org.eclipse.jetty.orbit:javax.mail.glassfish -======================================================================= -Spring Boot 2.5.9 -Copyright (c) 2012-2022 Pivotal, Inc. +------ +Oracle -This product is licensed to you under the Apache License, Version 2.0 -(the "License"). You may not use this product except in compliance with -the License. -======================================================================= +The following artifacts are CDDL + GPLv2 with classpath exception. +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html -spring-boot-autoconfigure-2.7.10 NOTICE + * javax.servlet:javax.servlet-api + * javax.annotation:javax.annotation-api + * javax.transaction:javax.transaction-api + * javax.websocket:javax.websocket-api -======================================================================= +------ +Oracle OpenJDK -Spring Boot 2.7.10 -Copyright (c) 2012-2023 VMware, Inc. +If ALPN is used to negotiate HTTP/2 connections, then the following +artifacts may be included in the distribution or downloaded when ALPN +module is selected. -This product is licensed to you under the Apache License, Version 2.0 -(the "License"). You may not use this product except in compliance with -the License. -======================================================================= + * java.sun.security.ssl -spring-boot-starter-2.5.9 NOTICE +These artifacts replace/modify OpenJDK classes. The modififications +are hosted at github and both modified and original are under GPL v2 with +classpath exceptions. +http://openjdk.java.net/legal/gplv2+ce.html -======================================================================= -Spring Boot 2.5.9 -Copyright (c) 2012-2022 Pivotal, Inc. +------ +OW2 -This product is licensed to you under the Apache License, Version 2.0 -(the "License"). You may not use this product except in compliance with -the License. -======================================================================= +The following artifacts are licensed by the OW2 Foundation according to the +terms of http://asm.ow2.org/license.html -spring-boot-starter-2.7.10 NOTICE +org.ow2.asm:asm-commons +org.ow2.asm:asm -======================================================================= -Spring Boot 2.7.10 -Copyright (c) 2012-2023 VMware, Inc. +------ +Apache -This product is licensed to you under the Apache License, Version 2.0 -(the "License"). You may not use this product except in compliance with -the License. -======================================================================= +The following artifacts are ASL2 licensed. -spring-boot-starter-json-2.7.10 NOTICE +org.apache.taglibs:taglibs-standard-spec +org.apache.taglibs:taglibs-standard-impl -======================================================================= -Spring Boot 2.7.10 -Copyright (c) 2012-2023 VMware, Inc. +------ +MortBay -This product is licensed to you under the Apache License, Version 2.0 -(the "License"). You may not use this product except in compliance with -the License. -======================================================================= +The following artifacts are ASL2 licensed. Based on selected classes from +following Apache Tomcat jars, all ASL2 licensed. -spring-boot-starter-tomcat-2.7.10 NOTICE +org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util -======================================================================= +org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api -Spring Boot 2.7.10 -Copyright (c) 2012-2023 VMware, Inc. -This product is licensed to you under the Apache License, Version 2.0 -(the "License"). You may not use this product except in compliance with -the License. -======================================================================= +------ +Mortbay -spring-boot-starter-validation-2.5.9 NOTICE +The following artifacts are CDDL + GPLv2 with classpath exception. -======================================================================= +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html -Spring Boot 2.5.9 -Copyright (c) 2012-2022 Pivotal, Inc. +org.eclipse.jetty.toolchain:jetty-schemas -This product is licensed to you under the Apache License, Version 2.0 -(the "License"). You may not use this product except in compliance with -the License. -======================================================================= +------ +Assorted -spring-boot-starter-web-2.7.10 NOTICE +The UnixCrypt.java code implements the one way cryptography used by +Unix systems for simple password protection. Copyright 1996 Aki Yoshida, +modified April 2001 by Iris Van den Broeke, Daniel Deville. +Permission to use, copy, modify and distribute UnixCrypt +for non-commercial or commercial purposes and without fee is +granted provided that the copyright notice appears in all copies. ======================================================================= -Spring Boot 2.7.10 -Copyright (c) 2012-2023 VMware, Inc. +byte-buddy-1.15.3 NOTICE -This product is licensed to you under the Apache License, Version 2.0 -(the "License"). You may not use this product except in compliance with -the License. ======================================================================= -spring-context-5.3.15 NOTICE - -======================================================================= +Copyright 2014 - Present Rafael Winterhalter -Spring Framework 5.3.15 -Copyright (c) 2002-2022 Pivotal, Inc. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at -This product is licensed to you under the Apache License, Version 2.0 -(the "License"). You may not use this product except in compliance with -the License. + http://www.apache.org/licenses/LICENSE-2.0 -This product may include a number of subcomponents with separate -copyright notices and license terms. Your use of the source code for -these subcomponents is subject to the terms and conditions of the -subcomponent's license, as noted in the license.txt file. +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. ======================================================================= -spring-context-5.3.26 NOTICE +netty-nio-client-2.29.5 NOTICE ======================================================================= -Spring Framework 5.3.26 -Copyright (c) 2002-2023 Pivotal, Inc. +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -This product is licensed to you under the Apache License, Version 2.0 -(the "License"). You may not use this product except in compliance with -the License. +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). -This product may include a number of subcomponents with separate -copyright notices and license terms. Your use of the source code for -these subcomponents is subject to the terms and conditions of the -subcomponent's license, as noted in the license.txt file. +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). ======================================================================= -spring-core-5.3.20 NOTICE +spring-boot-starter-validation-2.5.9 NOTICE ======================================================================= -Spring Framework 5.3.20 -Copyright (c) 2002-2022 Pivotal, Inc. +Spring Boot 2.5.9 +Copyright (c) 2012-2022 Pivotal, Inc. This product is licensed to you under the Apache License, Version 2.0 (the "License"). You may not use this product except in compliance with the License. - -This product may include a number of subcomponents with separate -copyright notices and license terms. Your use of the source code for -these subcomponents is subject to the terms and conditions of the -subcomponent's license, as noted in the license.txt file. - ======================================================================= -spring-core-5.3.26 NOTICE +commons-collections4-4.4 NOTICE ======================================================================= -Spring Framework 5.3.26 -Copyright (c) 2002-2023 Pivotal, Inc. - -This product is licensed to you under the Apache License, Version 2.0 -(the "License"). You may not use this product except in compliance with -the License. +Apache Commons Collections +Copyright 2001-2019 The Apache Software Foundation -This product may include a number of subcomponents with separate -copyright notices and license terms. Your use of the source code for -these subcomponents is subject to the terms and conditions of the -subcomponent's license, as noted in the license.txt file. +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). ======================================================================= -spring-expression-5.3.15 NOTICE +aws-core-2.29.5 NOTICE ======================================================================= -Spring Framework 5.3.15 -Copyright (c) 2002-2022 Pivotal, Inc. +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -This product is licensed to you under the Apache License, Version 2.0 -(the "License"). You may not use this product except in compliance with -the License. +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). -This product may include a number of subcomponents with separate -copyright notices and license terms. Your use of the source code for -these subcomponents is subject to the terms and conditions of the -subcomponent's license, as noted in the license.txt file. +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). ======================================================================= -spring-expression-5.3.26 NOTICE +spring-aop-5.3.31 NOTICE ======================================================================= -Spring Framework 5.3.26 +Spring Framework 5.3.31 Copyright (c) 2002-2023 Pivotal, Inc. This product is licensed to you under the Apache License, Version 2.0 @@ -2234,12 +2197,12 @@ subcomponent's license, as noted in the license.txt file. ======================================================================= -spring-jcl-5.3.26 NOTICE +spring-tx-5.3.9 NOTICE ======================================================================= -Spring Framework 5.3.26 -Copyright (c) 2002-2023 Pivotal, Inc. +Spring Framework 5.3.9 +Copyright (c) 2002-2021 Pivotal, Inc. This product is licensed to you under the Apache License, Version 2.0 (the "License"). You may not use this product except in compliance with @@ -2252,61 +2215,219 @@ subcomponent's license, as noted in the license.txt file. ======================================================================= -spring-messaging-5.3.20 NOTICE +apache-client-2.29.5 NOTICE ======================================================================= -Spring Framework 5.3.20 -Copyright (c) 2002-2022 Pivotal, Inc. +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -This product is licensed to you under the Apache License, Version 2.0 -(the "License"). You may not use this product except in compliance with -the License. +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). -This product may include a number of subcomponents with separate -copyright notices and license terms. Your use of the source code for -these subcomponents is subject to the terms and conditions of the -subcomponent's license, as noted in the license.txt file. +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). ======================================================================= -spring-web-5.3.26 NOTICE +mybatis-3.5.6 NOTICE ======================================================================= -Spring Framework 5.3.26 -Copyright (c) 2002-2023 Pivotal, Inc. +iBATIS + This product includes software developed by + The Apache Software Foundation (http://www.apache.org/). -This product is licensed to you under the Apache License, Version 2.0 -(the "License"). You may not use this product except in compliance with -the License. + Copyright 2010 The Apache Software Foundation -This product may include a number of subcomponents with separate -copyright notices and license terms. Your use of the source code for -these subcomponents is subject to the terms and conditions of the -subcomponent's license, as noted in the license.txt file. + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +OGNL +//-------------------------------------------------------------------------- +// Copyright (c) 2004, Drew Davidson and Luke Blanshard +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// Neither the name of the Drew Davidson nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +// COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +// OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +// AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH +// DAMAGE. +//-------------------------------------------------------------------------- + +Refactored SqlBuilder class (SQL, AbstractSQL) + + This product includes software developed by + Adam Gent (https://gist.github.com/3650165) + + Copyright 2010 Adam Gent + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. ======================================================================= -spring-webmvc-5.3.26 NOTICE +netty-3.2.10.Final NOTICE ======================================================================= -Spring Framework 5.3.26 -Copyright (c) 2002-2023 Pivotal, Inc. -This product is licensed to you under the Apache License, Version 2.0 -(the "License"). You may not use this product except in compliance with -the License. + The Netty Project + ================= -This product may include a number of subcomponents with separate -copyright notices and license terms. Your use of the source code for -these subcomponents is subject to the terms and conditions of the -subcomponent's license, as noted in the license.txt file. +Please visit the Netty web site for more information: + + * http://www.jboss.org/netty/ + +Copyright 2009 Red Hat, Inc. + +Red Hat licenses this product to you under the Apache License, version 2.0 (the +"License"); you may not use this product except in compliance with the License. +You may obtain a copy of the License at: + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed +under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. + +Also, please refer to each LICENSE..txt file, which is located in +the 'license' directory of the distribution file, for the license terms of the +components that this product depends on. + +------------------------------------------------------------------------------- +This product contains the extensions to Java Collections Framework which has +been derived from the works by JSR-166 EG, Doug Lea, and Jason T. Greene: + + * LICENSE: + * license/LICENSE.jsr166y.txt (Public Domain) + * HOMEPAGE: + * http://gee.cs.oswego.edu/cgi-bin/viewcvs.cgi/jsr166/ + * http://viewvc.jboss.org/cgi-bin/viewvc.cgi/jbosscache/experimental/jsr166/ + +This product contains a modified version of Robert Harder's Public Domain +Base64 Encoder and Decoder, which can be obtained at: + + * LICENSE: + * license/LICENSE.base64.txt (Public Domain) + * HOMEPAGE: + * http://iharder.sourceforge.net/current/java/base64/ + +This product contains a modified version of 'JZlib', a re-implementation of +zlib in pure Java, which can be obtained at: + + * LICENSE: + * license/LICENSE.jzlib.txt (BSD Style License) + * HOMEPAGE: + * http://www.jcraft.com/jzlib/ + +This product optionally depends on 'Protocol Buffers', Google's data +interchange format, which can be obtained at: + + * LICENSE: + * license/LICENSE.protobuf.txt (New BSD License) + * HOMEPAGE: + * http://code.google.com/p/protobuf/ + +This product optionally depends on 'SLF4J', a simple logging facade for Java, +which can be obtained at: + + * LICENSE: + * license/LICENSE.slf4j.txt (MIT License) + * HOMEPAGE: + * http://www.slf4j.org/ + +This product optionally depends on 'Apache Commons Logging', a logging +framework, which can be obtained at: + + * LICENSE: + * license/LICENSE.commons-logging.txt (Apache License 2.0) + * HOMEPAGE: + * http://commons.apache.org/logging/ + +This product optionally depends on 'Apache Log4J', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.log4j.txt (Apache License 2.0) + * HOMEPAGE: + * http://logging.apache.org/log4j/ + +This product optionally depends on 'JBoss Logging', a logging framework, +which can be obtained at: + + * LICENSE: + * license/LICENSE.jboss-logging.txt (GNU LGPL 2.1) + * HOMEPAGE: + * http://anonsvn.jboss.org/repos/common/common-logging-spi/ + +This product optionally depends on 'Apache Felix', an open source OSGi +framework implementation, which can be obtained at: + + * LICENSE: + * license/LICENSE.felix.txt (Apache License 2.0) + * HOMEPAGE: + * http://felix.apache.org/ ======================================================================= -third-party-jackson-core-2.20.29 NOTICE +jackson-module-parameter-names-2.18.0 NOTICE ======================================================================= @@ -2319,18 +2440,18 @@ It is currently developed by a community of developers. ## Licensing -Jackson 2.x core and extension components are licensed under Apache License 2.0 -To find the details that apply to this artifact see the accompanying LICENSE file. +Jackson components are licensed under Apache (Software) License, version 2.0, +as per accompanying LICENSE file. ## Credits -A list of contributors may be found from CREDITS(-2.x) file, which is included +A list of contributors may be found from CREDITS file, which is included in some artifacts (usually source distributions); but is always available from the source code management (SCM) system project uses. ======================================================================= -third-party-jackson-core-2.20.29 NOTICE +checksums-2.29.5 NOTICE ======================================================================= @@ -2362,81 +2483,19 @@ The licenses for these third party components are included in LICENSE.txt ======================================================================= -tomcat-embed-core-9.0.73 NOTICE - -======================================================================= - -Apache Tomcat -Copyright 1999-2023 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -The original XML Schemas for Java EE Deployment Descriptors: - - javaee_5.xsd - - javaee_web_services_1_2.xsd - - javaee_web_services_client_1_2.xsd - - javaee_6.xsd - - javaee_web_services_1_3.xsd - - javaee_web_services_client_1_3.xsd - - jsp_2_2.xsd - - web-app_3_0.xsd - - web-common_3_0.xsd - - web-fragment_3_0.xsd - - javaee_7.xsd - - javaee_web_services_1_4.xsd - - javaee_web_services_client_1_4.xsd - - jsp_2_3.xsd - - web-app_3_1.xsd - - web-common_3_1.xsd - - web-fragment_3_1.xsd - - javaee_8.xsd - - web-app_4_0.xsd - - web-common_4_0.xsd - - web-fragment_4_0.xsd - -may be obtained from: -http://www.oracle.com/webfolder/technetwork/jsc/xml/ns/javaee/index.html - -======================================================================= - -tomcat-embed-el-9.0.56 NOTICE - -======================================================================= - -Apache Tomcat -Copyright 1999-2021 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -======================================================================= - -tomcat-embed-el-9.0.73 NOTICE - -======================================================================= - -Apache Tomcat -Copyright 1999-2023 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - -======================================================================= - -tomcat-embed-websocket-9.0.73 NOTICE +spring-boot-2.7.18 NOTICE ======================================================================= -Apache Tomcat -Copyright 1999-2023 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). +Spring Boot 2.7.18 +Copyright (c) 2012-2023 VMware, Inc. +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. ======================================================================= -utils-2.20.29 NOTICE +retries-2.29.5 NOTICE ======================================================================= @@ -2465,3 +2524,4910 @@ The licenses for these third party components are included in LICENSE.txt This product includes software developed at The Apache Software Foundation (https://www.apache.org/). + +======================================================================= + +jetty-server-9.4.53.v20231009 NOTICE + +======================================================================= + +============================================================== + Jetty Web Container + Copyright 1995-2018 Mort Bay Consulting Pty Ltd. +============================================================== + +The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd +unless otherwise noted. + +Jetty is dual licensed under both + + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html + + and + + * The Eclipse Public 1.0 License + http://www.eclipse.org/legal/epl-v10.html + +Jetty may be distributed under either license. + +------ +Eclipse + +The following artifacts are EPL. + * org.eclipse.jetty.orbit:org.eclipse.jdt.core + +The following artifacts are EPL and ASL2. + * org.eclipse.jetty.orbit:javax.security.auth.message + + +The following artifacts are EPL and CDDL 1.0. + * org.eclipse.jetty.orbit:javax.mail.glassfish + + +------ +Oracle + +The following artifacts are CDDL + GPLv2 with classpath exception. +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + + * javax.servlet:javax.servlet-api + * javax.annotation:javax.annotation-api + * javax.transaction:javax.transaction-api + * javax.websocket:javax.websocket-api + +------ +Oracle OpenJDK + +If ALPN is used to negotiate HTTP/2 connections, then the following +artifacts may be included in the distribution or downloaded when ALPN +module is selected. + + * java.sun.security.ssl + +These artifacts replace/modify OpenJDK classes. The modififications +are hosted at github and both modified and original are under GPL v2 with +classpath exceptions. +http://openjdk.java.net/legal/gplv2+ce.html + + +------ +OW2 + +The following artifacts are licensed by the OW2 Foundation according to the +terms of http://asm.ow2.org/license.html + +org.ow2.asm:asm-commons +org.ow2.asm:asm + + +------ +Apache + +The following artifacts are ASL2 licensed. + +org.apache.taglibs:taglibs-standard-spec +org.apache.taglibs:taglibs-standard-impl + + +------ +MortBay + +The following artifacts are ASL2 licensed. Based on selected classes from +following Apache Tomcat jars, all ASL2 licensed. + +org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util + +org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api + + +------ +Mortbay + +The following artifacts are CDDL + GPLv2 with classpath exception. + +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + +org.eclipse.jetty.toolchain:jetty-schemas + +------ +Assorted + +The UnixCrypt.java code implements the one way cryptography used by +Unix systems for simple password protection. Copyright 1996 Aki Yoshida, +modified April 2001 by Iris Van den Broeke, Daniel Deville. +Permission to use, copy, modify and distribute UnixCrypt +for non-commercial or commercial purposes and without fee is +granted provided that the copyright notice appears in all copies. + +======================================================================= + +commons-cli-1.2 NOTICE + +======================================================================= + +Apache Commons CLI +Copyright 2001-2009 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +======================================================================= + +jetty-util-9.4.53.v20231009 NOTICE + +======================================================================= + +============================================================== + Jetty Web Container + Copyright 1995-2018 Mort Bay Consulting Pty Ltd. +============================================================== + +The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd +unless otherwise noted. + +Jetty is dual licensed under both + + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html + + and + + * The Eclipse Public 1.0 License + http://www.eclipse.org/legal/epl-v10.html + +Jetty may be distributed under either license. + +------ +Eclipse + +The following artifacts are EPL. + * org.eclipse.jetty.orbit:org.eclipse.jdt.core + +The following artifacts are EPL and ASL2. + * org.eclipse.jetty.orbit:javax.security.auth.message + + +The following artifacts are EPL and CDDL 1.0. + * org.eclipse.jetty.orbit:javax.mail.glassfish + + +------ +Oracle + +The following artifacts are CDDL + GPLv2 with classpath exception. +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + + * javax.servlet:javax.servlet-api + * javax.annotation:javax.annotation-api + * javax.transaction:javax.transaction-api + * javax.websocket:javax.websocket-api + +------ +Oracle OpenJDK + +If ALPN is used to negotiate HTTP/2 connections, then the following +artifacts may be included in the distribution or downloaded when ALPN +module is selected. + + * java.sun.security.ssl + +These artifacts replace/modify OpenJDK classes. The modififications +are hosted at github and both modified and original are under GPL v2 with +classpath exceptions. +http://openjdk.java.net/legal/gplv2+ce.html + + +------ +OW2 + +The following artifacts are licensed by the OW2 Foundation according to the +terms of http://asm.ow2.org/license.html + +org.ow2.asm:asm-commons +org.ow2.asm:asm + + +------ +Apache + +The following artifacts are ASL2 licensed. + +org.apache.taglibs:taglibs-standard-spec +org.apache.taglibs:taglibs-standard-impl + + +------ +MortBay + +The following artifacts are ASL2 licensed. Based on selected classes from +following Apache Tomcat jars, all ASL2 licensed. + +org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util + +org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api + + +------ +Mortbay + +The following artifacts are CDDL + GPLv2 with classpath exception. + +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + +org.eclipse.jetty.toolchain:jetty-schemas + +------ +Assorted + +The UnixCrypt.java code implements the one way cryptography used by +Unix systems for simple password protection. Copyright 1996 Aki Yoshida, +modified April 2001 by Iris Van den Broeke, Daniel Deville. +Permission to use, copy, modify and distribute UnixCrypt +for non-commercial or commercial purposes and without fee is +granted provided that the copyright notice appears in all copies. + +======================================================================= + +tomcat-embed-core-9.0.83 NOTICE + +======================================================================= + +Apache Tomcat +Copyright 1999-2023 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +The original XML Schemas for Java EE Deployment Descriptors: + - javaee_5.xsd + - javaee_web_services_1_2.xsd + - javaee_web_services_client_1_2.xsd + - javaee_6.xsd + - javaee_web_services_1_3.xsd + - javaee_web_services_client_1_3.xsd + - jsp_2_2.xsd + - web-app_3_0.xsd + - web-common_3_0.xsd + - web-fragment_3_0.xsd + - javaee_7.xsd + - javaee_web_services_1_4.xsd + - javaee_web_services_client_1_4.xsd + - jsp_2_3.xsd + - web-app_3_1.xsd + - web-common_3_1.xsd + - web-fragment_3_1.xsd + - javaee_8.xsd + - web-app_4_0.xsd + - web-common_4_0.xsd + - web-fragment_4_0.xsd + +may be obtained from: +http://www.oracle.com/webfolder/technetwork/jsc/xml/ns/javaee/index.html + +======================================================================= + +curator-framework-5.7.0 NOTICE + +======================================================================= + +Curator Framework +Copyright 2011-2023 The Apache Software Foundation + + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +======================================================================= + +pravega-shared-controller-api-0.11.0 NOTICE + +======================================================================= + +Copyright (c) 2021 Pravega Authors. +Copyright (c) 2017-2021 Dell Inc., or its subsidiaries. All Rights Reserved. + +This software contains source code from Apache BookKeeper, distributed under +the Apache License Version 2.0, and copyrighted to the Apache Software Foundation. +http://bookkeeper.apache.org + +======================================================================= + +spring-boot-autoconfigure-2.7.18 NOTICE + +======================================================================= + +Spring Boot 2.7.18 +Copyright (c) 2012-2023 VMware, Inc. + +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. +======================================================================= + +websocket-common-9.4.53.v20231009 NOTICE + +======================================================================= + +============================================================== + Jetty Web Container + Copyright 1995-2018 Mort Bay Consulting Pty Ltd. +============================================================== + +The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd +unless otherwise noted. + +Jetty is dual licensed under both + + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html + + and + + * The Eclipse Public 1.0 License + http://www.eclipse.org/legal/epl-v10.html + +Jetty may be distributed under either license. + +------ +Eclipse + +The following artifacts are EPL. + * org.eclipse.jetty.orbit:org.eclipse.jdt.core + +The following artifacts are EPL and ASL2. + * org.eclipse.jetty.orbit:javax.security.auth.message + + +The following artifacts are EPL and CDDL 1.0. + * org.eclipse.jetty.orbit:javax.mail.glassfish + + +------ +Oracle + +The following artifacts are CDDL + GPLv2 with classpath exception. +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + + * javax.servlet:javax.servlet-api + * javax.annotation:javax.annotation-api + * javax.transaction:javax.transaction-api + * javax.websocket:javax.websocket-api + +------ +Oracle OpenJDK + +If ALPN is used to negotiate HTTP/2 connections, then the following +artifacts may be included in the distribution or downloaded when ALPN +module is selected. + + * java.sun.security.ssl + +These artifacts replace/modify OpenJDK classes. The modififications +are hosted at github and both modified and original are under GPL v2 with +classpath exceptions. +http://openjdk.java.net/legal/gplv2+ce.html + + +------ +OW2 + +The following artifacts are licensed by the OW2 Foundation according to the +terms of http://asm.ow2.org/license.html + +org.ow2.asm:asm-commons +org.ow2.asm:asm + + +------ +Apache + +The following artifacts are ASL2 licensed. + +org.apache.taglibs:taglibs-standard-spec +org.apache.taglibs:taglibs-standard-impl + + +------ +MortBay + +The following artifacts are ASL2 licensed. Based on selected classes from +following Apache Tomcat jars, all ASL2 licensed. + +org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util + +org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api + + +------ +Mortbay + +The following artifacts are CDDL + GPLv2 with classpath exception. + +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + +org.eclipse.jetty.toolchain:jetty-schemas + +------ +Assorted + +The UnixCrypt.java code implements the one way cryptography used by +Unix systems for simple password protection. Copyright 1996 Aki Yoshida, +modified April 2001 by Iris Van den Broeke, Daniel Deville. +Permission to use, copy, modify and distribute UnixCrypt +for non-commercial or commercial purposes and without fee is +granted provided that the copyright notice appears in all copies. + +======================================================================= + +spring-jdbc-5.3.9 NOTICE + +======================================================================= + +Spring Framework 5.3.9 +Copyright (c) 2002-2021 Pivotal, Inc. + +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. + +This product may include a number of subcomponents with separate +copyright notices and license terms. Your use of the source code for +these subcomponents is subject to the terms and conditions of the +subcomponent's license, as noted in the license.txt file. + +======================================================================= + +http-auth-spi-2.29.5 NOTICE + +======================================================================= + +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). + +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). + +======================================================================= + +guice-7.0.0 NOTICE + +======================================================================= + + +Google Guice - Core Library +Copyright 2006-2023 Google, Inc. + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + + +======================================================================= + +kafka-clients-3.9.0 NOTICE + +======================================================================= + +Apache Kafka +Copyright 2024 The Apache Software Foundation. + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + +This distribution has a binary dependency on jersey, which is available under the CDDL +License. The source code of jersey can be found at https://github.com/jersey/jersey/. + +This distribution has a binary test dependency on jqwik, which is available under +the Eclipse Public License 2.0. The source code can be found at +https://github.com/jlink/jqwik. + +The streams-scala (streams/streams-scala) module was donated by Lightbend and the original code was copyrighted by them: +Copyright (C) 2018 Lightbend Inc. +Copyright (C) 2017-2018 Alexis Seigneurin. + +This project contains the following code copied from Apache Hadoop: +clients/src/main/java/org/apache/kafka/common/utils/PureJavaCrc32C.java +Some portions of this file Copyright (c) 2004-2006 Intel Corporation and licensed under the BSD license. + +This project contains the following code copied from Apache Hive: +streams/src/main/java/org/apache/kafka/streams/state/internals/Murmur3.java + +======================================================================= + +tomcat-embed-websocket-9.0.83 NOTICE + +======================================================================= + +Apache Tomcat +Copyright 1999-2023 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +======================================================================= + +http-client-spi-2.29.5 NOTICE + +======================================================================= + +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). + +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). + +======================================================================= + +http-auth-aws-2.29.5 NOTICE + +======================================================================= + +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). + +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). + +======================================================================= + +redisson-3.38.1 NOTICE + +======================================================================= + +# Jackson JSON processor + +Jackson is a high-performance, Free/Open Source JSON processing library. +It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has +been in development since 2007. +It is currently developed by a community of developers, as well as supported +commercially by FasterXML.com. + +## Licensing + +Jackson core and extension components may be licensed under different licenses. +To find the details that apply to this artifact see the accompanying LICENSE file. +For more information, including possible other licensing options, contact +FasterXML.com (http://fasterxml.com). + +## Credits + +A list of contributors may be found from CREDITS file, which is included +in some artifacts (usually source distributions); but is always available +from the source code management (SCM) system project uses. + +# Byte Buddy + +Copyright 2014 - 2019 Rafael Winterhalter + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +======================================================================= + +pravega-client-0.11.0 NOTICE + +======================================================================= + +Copyright (c) 2021 Pravega Authors. +Copyright (c) 2017-2021 Dell Inc., or its subsidiaries. All Rights Reserved. + +This software contains source code from Apache BookKeeper, distributed under +the Apache License Version 2.0, and copyrighted to the Apache Software Foundation. +http://bookkeeper.apache.org + +======================================================================= + +endpoints-spi-2.29.5 NOTICE + +======================================================================= + +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). + +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). + +======================================================================= + +jetty-io-9.4.53.v20231009 NOTICE + +======================================================================= + +============================================================== + Jetty Web Container + Copyright 1995-2018 Mort Bay Consulting Pty Ltd. +============================================================== + +The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd +unless otherwise noted. + +Jetty is dual licensed under both + + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html + + and + + * The Eclipse Public 1.0 License + http://www.eclipse.org/legal/epl-v10.html + +Jetty may be distributed under either license. + +------ +Eclipse + +The following artifacts are EPL. + * org.eclipse.jetty.orbit:org.eclipse.jdt.core + +The following artifacts are EPL and ASL2. + * org.eclipse.jetty.orbit:javax.security.auth.message + + +The following artifacts are EPL and CDDL 1.0. + * org.eclipse.jetty.orbit:javax.mail.glassfish + + +------ +Oracle + +The following artifacts are CDDL + GPLv2 with classpath exception. +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + + * javax.servlet:javax.servlet-api + * javax.annotation:javax.annotation-api + * javax.transaction:javax.transaction-api + * javax.websocket:javax.websocket-api + +------ +Oracle OpenJDK + +If ALPN is used to negotiate HTTP/2 connections, then the following +artifacts may be included in the distribution or downloaded when ALPN +module is selected. + + * java.sun.security.ssl + +These artifacts replace/modify OpenJDK classes. The modififications +are hosted at github and both modified and original are under GPL v2 with +classpath exceptions. +http://openjdk.java.net/legal/gplv2+ce.html + + +------ +OW2 + +The following artifacts are licensed by the OW2 Foundation according to the +terms of http://asm.ow2.org/license.html + +org.ow2.asm:asm-commons +org.ow2.asm:asm + + +------ +Apache + +The following artifacts are ASL2 licensed. + +org.apache.taglibs:taglibs-standard-spec +org.apache.taglibs:taglibs-standard-impl + + +------ +MortBay + +The following artifacts are ASL2 licensed. Based on selected classes from +following Apache Tomcat jars, all ASL2 licensed. + +org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util + +org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api + + +------ +Mortbay + +The following artifacts are CDDL + GPLv2 with classpath exception. + +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + +org.eclipse.jetty.toolchain:jetty-schemas + +------ +Assorted + +The UnixCrypt.java code implements the one way cryptography used by +Unix systems for simple password protection. Copyright 1996 Aki Yoshida, +modified April 2001 by Iris Van den Broeke, Daniel Deville. +Permission to use, copy, modify and distribute UnixCrypt +for non-commercial or commercial purposes and without fee is +granted provided that the copyright notice appears in all copies. + +======================================================================= + +classmate-1.5.1 NOTICE + +======================================================================= + +Java ClassMate library was originally written by Tatu Saloranta (tatu.saloranta@iki.fi) + +Other developers who have contributed code are: + +* Brian Langel + + +======================================================================= + +stax-ex-1.8.1 NOTICE + +======================================================================= + +# Notices for Eclipse Implementation of JAXB + +This content is produced and maintained by the Eclipse Implementation of JAXB +project. + +* Project home: https://projects.eclipse.org/projects/ee4j.jaxb-impl + +## Trademarks + +Eclipse Implementation of JAXB is a trademark of the Eclipse Foundation. + +## Copyright + +All content is the property of the respective authors or their employers. For +more information regarding authorship of content, please consult the listed +source code repository logs. + +## Declared Project Licenses + +This program and the accompanying materials are made available under the terms +of the Eclipse Distribution License v. 1.0 which is available at +http://www.eclipse.org/org/documents/edl-v10.php. + +SPDX-License-Identifier: BSD-3-Clause + +## Source Code + +The project maintains the following source code repositories: + +* https://github.com/eclipse-ee4j/jaxb-ri +* https://github.com/eclipse-ee4j/jaxb-istack-commons +* https://github.com/eclipse-ee4j/jaxb-dtd-parser +* https://github.com/eclipse-ee4j/jaxb-fi +* https://github.com/eclipse-ee4j/jaxb-stax-ex +* https://github.com/eclipse-ee4j/jax-rpc-ri + +## Third-party Content + +This project leverages the following third party content. + +Apache Ant (1.10.2) + +* License: Apache-2.0 AND W3C AND LicenseRef-Public-Domain + +Apache Ant (1.10.2) + +* License: Apache-2.0 AND W3C AND LicenseRef-Public-Domain + +Apache Felix (1.2.0) + +* License: Apache License, 2.0 + +args4j (2.33) + +* License: MIT License + +dom4j (1.6.1) + +* License: Custom license based on Apache 1.1 + +file-management (3.0.0) + +* License: Apache-2.0 +* Project: https://maven.apache.org/shared/file-management/ +* Source: + https://svn.apache.org/viewvc/maven/shared/tags/file-management-3.0.0/ + +JUnit (4.12) + +* License: Eclipse Public License + +JUnit (4.12) + +* License: Eclipse Public License + +maven-compat (3.5.2) + +* License: Apache-2.0 +* Project: https://maven.apache.org/ref/3.5.2/maven-compat/ +* Source: + https://mvnrepository.com/artifact/org.apache.maven/maven-compat/3.5.2 + +maven-core (3.5.2) + +* License: Apache-2.0 +* Project: https://maven.apache.org/ref/3.5.2/maven-core/index.html +* Source: https://mvnrepository.com/artifact/org.apache.maven/maven-core/3.5.2 + +maven-plugin-annotations (3.5) + +* License: Apache-2.0 +* Project: https://maven.apache.org/plugin-tools/maven-plugin-annotations/ +* Source: + https://github.com/apache/maven-plugin-tools/tree/master/maven-plugin-annotations + +maven-plugin-api (3.5.2) + +* License: Apache-2.0 + +maven-resolver-api (1.1.1) + +* License: Apache-2.0 + +maven-resolver-api (1.1.1) + +* License: Apache-2.0 + +maven-resolver-connector-basic (1.1.1) + +* License: Apache-2.0 + +maven-resolver-impl (1.1.1) + +* License: Apache-2.0 + +maven-resolver-spi (1.1.1) + +* License: Apache-2.0 + +maven-resolver-transport-file (1.1.1) + +* License: Apache-2.0 +* Project: https://maven.apache.org/resolver/maven-resolver-transport-file/ +* Source: + https://github.com/apache/maven-resolver/tree/master/maven-resolver-transport-file + +maven-resolver-util (1.1.1) + +* License: Apache-2.0 + +maven-settings (3.5.2) + +* License: Apache-2.0 +* Source: + https://mvnrepository.com/artifact/org.apache.maven/maven-settings/3.5.2 + +OSGi Service Platform Core Companion Code (6.0) + +* License: Apache License, 2.0 + +plexus-archiver (3.5) + +* License: Apache-2.0 +* Project: https://codehaus-plexus.github.io/plexus-archiver/ +* Source: https://github.com/codehaus-plexus/plexus-archiver + +plexus-io (3.0.0) + +* License: Apache-2.0 + +plexus-utils (3.1.0) + +* License: Apache- 2.0 or Apache- 1.1 or BSD or Public Domain or Indiana + University Extreme! Lab Software License V1.1.1 (Apache 1.1 style) + +relaxng-datatype (1.0) + +* License: New BSD license + +Sax (0.2) + +* License: SAX-PD +* Project: http://www.megginson.com/downloads/SAX/ +* Source: http://sourceforge.net/project/showfiles.php?group_id=29449 + +testng (6.14.2) + +* License: Apache-2.0 AND (MIT OR GPL-1.0+) +* Project: https://testng.org/doc/index.html +* Source: https://github.com/cbeust/testng + +wagon-http-lightweight (3.0.0) + +* License: Pending +* Project: https://maven.apache.org/wagon/ +* Source: + https://mvnrepository.com/artifact/org.apache.maven.wagon/wagon-http-lightweight/3.0.0 + +xz for java (1.8) + +* License: LicenseRef-Public-Domain + +## Cryptography + +Content may contain encryption software. The country in which you are currently +may have restrictions on the import, possession, and use, and/or re-export to +another country, of encryption software. BEFORE using any encryption software, +please check the country's laws, regulations and policies concerning the import, +possession, or use, and re-export of encryption software, to see if this is +permitted. + + +======================================================================= + +curator-client-5.7.0 NOTICE + +======================================================================= + +Curator Client +Copyright 2011-2023 The Apache Software Foundation + + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +======================================================================= + +spring-boot-starter-jdbc-2.7.18 NOTICE + +======================================================================= + +Spring Boot 2.7.18 +Copyright (c) 2012-2023 VMware, Inc. + +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. +======================================================================= + +jackson-datatype-jdk8-2.18.0 NOTICE + +======================================================================= + +# Jackson JSON processor + +Jackson is a high-performance, Free/Open Source JSON processing library. +It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has +been in development since 2007. +It is currently developed by a community of developers. + +## Licensing + +Jackson components are licensed under Apache (Software) License, version 2.0, +as per accompanying LICENSE file. + +## Credits + +A list of contributors may be found from CREDITS file, which is included +in some artifacts (usually source distributions); but is always available +from the source code management (SCM) system project uses. + +======================================================================= + +grpc-netty-shaded-1.68.0 NOTICE + +======================================================================= + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2016 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +------------------------------------------------------------------------------- +This product contains a forked and modified version of Tomcat Native + + * LICENSE: + * license/LICENSE.tomcat-native.txt (Apache License 2.0) + * HOMEPAGE: + * http://tomcat.apache.org/native-doc/ + * https://svn.apache.org/repos/asf/tomcat/native/ + +This product contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. + + * LICENSE: + * license/LICENSE.mvn-wrapper.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/takari/maven-wrapper + +This product contains small piece of code to support AIX, taken from netbsd. + + * LICENSE: + * license/LICENSE.aix-netbsd.txt (OpenSSL License) + * HOMEPAGE: + * https://ftp.netbsd.org/pub/NetBSD/NetBSD-current/src/crypto/external/bsd/openssl/dist + + +This product contains code from boringssl. + + * LICENSE (Combination ISC and OpenSSL license) + * license/LICENSE.boringssl.txt (Combination ISC and OpenSSL license) + * HOMEPAGE: + * https://boringssl.googlesource.com/boringssl/ + +======================================================================= + +rocketmq-namesrv-4.9.5 NOTICE + +======================================================================= + + +rocketmq-namesrv 4.9.5 +Copyright 2012-2023 Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + + +======================================================================= + +spring-boot-starter-json-2.7.18 NOTICE + +======================================================================= + +Spring Boot 2.7.18 +Copyright (c) 2012-2023 VMware, Inc. + +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. +======================================================================= + +spring-beans-5.3.31 NOTICE + +======================================================================= + +Spring Framework 5.3.31 +Copyright (c) 2002-2023 Pivotal, Inc. + +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. + +This product may include a number of subcomponents with separate +copyright notices and license terms. Your use of the source code for +these subcomponents is subject to the terms and conditions of the +subcomponent's license, as noted in the license.txt file. + +======================================================================= + +fastjson-1.2.69_noneautotype NOTICE + +======================================================================= + +/* + * Copyright 1999-2017 Alibaba Group. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +======================================================================= + +identity-spi-2.29.5 NOTICE + +======================================================================= + +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). + +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). + +======================================================================= + +jetty-annotations-9.4.53.v20231009 NOTICE + +======================================================================= + +============================================================== + Jetty Web Container + Copyright 1995-2018 Mort Bay Consulting Pty Ltd. +============================================================== + +The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd +unless otherwise noted. + +Jetty is dual licensed under both + + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html + + and + + * The Eclipse Public 1.0 License + http://www.eclipse.org/legal/epl-v10.html + +Jetty may be distributed under either license. + +------ +Eclipse + +The following artifacts are EPL. + * org.eclipse.jetty.orbit:org.eclipse.jdt.core + +The following artifacts are EPL and ASL2. + * org.eclipse.jetty.orbit:javax.security.auth.message + + +The following artifacts are EPL and CDDL 1.0. + * org.eclipse.jetty.orbit:javax.mail.glassfish + + +------ +Oracle + +The following artifacts are CDDL + GPLv2 with classpath exception. +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + + * javax.servlet:javax.servlet-api + * javax.annotation:javax.annotation-api + * javax.transaction:javax.transaction-api + * javax.websocket:javax.websocket-api + +------ +Oracle OpenJDK + +If ALPN is used to negotiate HTTP/2 connections, then the following +artifacts may be included in the distribution or downloaded when ALPN +module is selected. + + * java.sun.security.ssl + +These artifacts replace/modify OpenJDK classes. The modififications +are hosted at github and both modified and original are under GPL v2 with +classpath exceptions. +http://openjdk.java.net/legal/gplv2+ce.html + + +------ +OW2 + +The following artifacts are licensed by the OW2 Foundation according to the +terms of http://asm.ow2.org/license.html + +org.ow2.asm:asm-commons +org.ow2.asm:asm + + +------ +Apache + +The following artifacts are ASL2 licensed. + +org.apache.taglibs:taglibs-standard-spec +org.apache.taglibs:taglibs-standard-impl + + +------ +MortBay + +The following artifacts are ASL2 licensed. Based on selected classes from +following Apache Tomcat jars, all ASL2 licensed. + +org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util + +org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api + + +------ +Mortbay + +The following artifacts are CDDL + GPLv2 with classpath exception. + +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + +org.eclipse.jetty.toolchain:jetty-schemas + +------ +Assorted + +The UnixCrypt.java code implements the one way cryptography used by +Unix systems for simple password protection. Copyright 1996 Aki Yoshida, +modified April 2001 by Iris Van den Broeke, Daniel Deville. +Permission to use, copy, modify and distribute UnixCrypt +for non-commercial or commercial purposes and without fee is +granted provided that the copyright notice appears in all copies. + +======================================================================= + +utils-2.29.5 NOTICE + +======================================================================= + +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). + +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). + +======================================================================= + +websocket-client-9.4.53.v20231009 NOTICE + +======================================================================= + +============================================================== + Jetty Web Container + Copyright 1995-2018 Mort Bay Consulting Pty Ltd. +============================================================== + +The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd +unless otherwise noted. + +Jetty is dual licensed under both + + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html + + and + + * The Eclipse Public 1.0 License + http://www.eclipse.org/legal/epl-v10.html + +Jetty may be distributed under either license. + +------ +Eclipse + +The following artifacts are EPL. + * org.eclipse.jetty.orbit:org.eclipse.jdt.core + +The following artifacts are EPL and ASL2. + * org.eclipse.jetty.orbit:javax.security.auth.message + + +The following artifacts are EPL and CDDL 1.0. + * org.eclipse.jetty.orbit:javax.mail.glassfish + + +------ +Oracle + +The following artifacts are CDDL + GPLv2 with classpath exception. +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + + * javax.servlet:javax.servlet-api + * javax.annotation:javax.annotation-api + * javax.transaction:javax.transaction-api + * javax.websocket:javax.websocket-api + +------ +Oracle OpenJDK + +If ALPN is used to negotiate HTTP/2 connections, then the following +artifacts may be included in the distribution or downloaded when ALPN +module is selected. + + * java.sun.security.ssl + +These artifacts replace/modify OpenJDK classes. The modififications +are hosted at github and both modified and original are under GPL v2 with +classpath exceptions. +http://openjdk.java.net/legal/gplv2+ce.html + + +------ +OW2 + +The following artifacts are licensed by the OW2 Foundation according to the +terms of http://asm.ow2.org/license.html + +org.ow2.asm:asm-commons +org.ow2.asm:asm + + +------ +Apache + +The following artifacts are ASL2 licensed. + +org.apache.taglibs:taglibs-standard-spec +org.apache.taglibs:taglibs-standard-impl + + +------ +MortBay + +The following artifacts are ASL2 licensed. Based on selected classes from +following Apache Tomcat jars, all ASL2 licensed. + +org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util + +org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api + + +------ +Mortbay + +The following artifacts are CDDL + GPLv2 with classpath exception. + +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + +org.eclipse.jetty.toolchain:jetty-schemas + +------ +Assorted + +The UnixCrypt.java code implements the one way cryptography used by +Unix systems for simple password protection. Copyright 1996 Aki Yoshida, +modified April 2001 by Iris Van den Broeke, Daniel Deville. +Permission to use, copy, modify and distribute UnixCrypt +for non-commercial or commercial purposes and without fee is +granted provided that the copyright notice appears in all copies. + +======================================================================= + +third-party-jackson-core-2.29.5 NOTICE + +======================================================================= + +# Jackson JSON processor + +Jackson is a high-performance, Free/Open Source JSON processing library. +It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has +been in development since 2007. +It is currently developed by a community of developers. + +## Copyright + +Copyright 2007-, Tatu Saloranta (tatu.saloranta@iki.fi) + +## Licensing + +Jackson 2.x core and extension components are licensed under Apache License 2.0 +To find the details that apply to this artifact see the accompanying LICENSE file. + +## Credits + +A list of contributors may be found from CREDITS(-2.x) file, which is included +in some artifacts (usually source distributions); but is always available +from the source code management (SCM) system project uses. + +## FastDoubleParser + +jackson-core bundles a shaded copy of FastDoubleParser . +That code is available under an MIT license +under the following copyright. + +Copyright © 2023 Werner Randelshofer, Switzerland. MIT License. + +See FastDoubleParser-NOTICE for details of other source code included in FastDoubleParser +and the licenses and copyrights that apply to that code. + +======================================================================= + +third-party-jackson-core-2.29.5 NOTICE + +======================================================================= + +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). + +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). + +======================================================================= + +spring-jcl-5.3.31 NOTICE + +======================================================================= + +Spring Framework 5.3.31 +Copyright (c) 2002-2023 Pivotal, Inc. + +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. + +This product may include a number of subcomponents with separate +copyright notices and license terms. Your use of the source code for +these subcomponents is subject to the terms and conditions of the +subcomponent's license, as noted in the license.txt file. + +======================================================================= + +bouncy-castle-bc-2.11.1-pkg NOTICE + +======================================================================= + + +Apache Pulsar :: Bouncy Castle :: BC +Copyright 2017-2020 Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + + +======================================================================= + +jakarta.activation-1.2.2 NOTICE + +======================================================================= + +# Notices for Jakarta Activation + +This content is produced and maintained by Jakarta Activation project. + +* Project home: https://projects.eclipse.org/projects/ee4j.jaf + +## Copyright + +All content is the property of the respective authors or their employers. For +more information regarding authorship of content, please consult the listed +source code repository logs. + +## Declared Project Licenses + +This program and the accompanying materials are made available under the terms +of the Eclipse Distribution License v. 1.0, +which is available at http://www.eclipse.org/org/documents/edl-v10.php. + +SPDX-License-Identifier: BSD-3-Clause + +## Source Code + +The project maintains the following source code repositories: + +* https://github.com/eclipse-ee4j/jaf + +## Third-party Content + +This project leverages the following third party content. + +JUnit (4.12) + +* License: Eclipse Public License + +======================================================================= + +http-auth-aws-eventstream-2.29.5 NOTICE + +======================================================================= + +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). + +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). + +======================================================================= + +pulsar-client-admin-api-2.11.4 NOTICE + +======================================================================= + + +Pulsar Client Admin :: API +Copyright 2017-2020 Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + + +======================================================================= + +tomcat-embed-el-9.0.83 NOTICE + +======================================================================= + +Apache Tomcat +Copyright 1999-2023 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +======================================================================= + +bouncy-castle-bc-2.11.4-pkg NOTICE + +======================================================================= + + +Apache Pulsar :: Bouncy Castle :: BC +Copyright 2017-2020 Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + + +======================================================================= + +log4j-core-2.24.1 NOTICE + +======================================================================= + +Apache Log4j Core +Copyright 1999-2012 Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +ResolverUtil.java +Copyright 2005-2006 Tim Fennell +======================================================================= + +commons-validator-1.9.0 NOTICE + +======================================================================= + +Apache Commons Validator +Copyright 2002-2024 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + +======================================================================= + +rocketmq-remoting-4.9.5 NOTICE + +======================================================================= + + +rocketmq-remoting 4.9.5 +Copyright 2012-2023 Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + + +======================================================================= + +jakarta.activation-api-1.2.1 NOTICE + +======================================================================= + +# Notices for Eclipse Project for JAF + +This content is produced and maintained by the Eclipse Project for JAF project. + +* Project home: https://projects.eclipse.org/projects/ee4j.jaf + +## Copyright + +All content is the property of the respective authors or their employers. For +more information regarding authorship of content, please consult the listed +source code repository logs. + +## Declared Project Licenses + +This program and the accompanying materials are made available under the terms +of the Eclipse Distribution License v. 1.0, +which is available at http://www.eclipse.org/org/documents/edl-v10.php. + +SPDX-License-Identifier: BSD-3-Clause + +## Source Code + +The project maintains the following source code repositories: + +* https://github.com/eclipse-ee4j/jaf + +## Third-party Content + +This project leverages the following third party content. + +JUnit (4.12) + +* License: Eclipse Public License + +======================================================================= + +profiles-2.29.5 NOTICE + +======================================================================= + +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). + +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). + +======================================================================= + +metrics-spi-2.29.5 NOTICE + +======================================================================= + +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). + +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). + +======================================================================= + +sdk-core-2.29.5 NOTICE + +======================================================================= + +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). + +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). + +======================================================================= + +jackson-jr-objects-2.18.0 NOTICE + +======================================================================= + +# Jackson JSON processor + +Jackson is a high-performance, Free/Open Source JSON processing library. +It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has +been in development since 2007. +It is currently developed by a community of developers. + +## Licensing + +Jackson components are licensed under Apache (Software) License, version 2.0, +as per accompanying LICENSE file. + +## Credits + +A list of contributors may be found from CREDITS file, which is included +in some artifacts (usually source distributions); but is always available +from the source code management (SCM) system project uses. + +======================================================================= + +spring-beans-5.3.9 NOTICE + +======================================================================= + +Spring Framework 5.3.9 +Copyright (c) 2002-2021 Pivotal, Inc. + +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. + +This product may include a number of subcomponents with separate +copyright notices and license terms. Your use of the source code for +these subcomponents is subject to the terms and conditions of the +subcomponent's license, as noted in the license.txt file. + +======================================================================= + +commons-codec-1.11 NOTICE + +======================================================================= + +Apache Commons Codec +Copyright 2002-2017 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java +contains test data from http://aspell.net/test/orig/batch0.tab. +Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) + +=============================================================================== + +The content of package org.apache.commons.codec.language.bm has been translated +from the original php source code available at http://stevemorse.org/phoneticinfo.htm +with permission from the original authors. +Original source copyright: +Copyright (c) 2008 Alexander Beider & Stephen P. Morse. + +======================================================================= + +pravega-common-0.11.0 NOTICE + +======================================================================= + +Copyright (c) 2021 Pravega Authors. +Copyright (c) 2017-2021 Dell Inc., or its subsidiaries. All Rights Reserved. + +This software contains source code from Apache BookKeeper, distributed under +the Apache License Version 2.0, and copyrighted to the Apache Software Foundation. +http://bookkeeper.apache.org + +======================================================================= + +rocketmq-client-4.9.5 NOTICE + +======================================================================= + + +rocketmq-client 4.9.5 +Copyright 2012-2023 Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + + +======================================================================= + +commons-compress-1.22 NOTICE + +======================================================================= + +Apache Commons Compress +Copyright 2002-2022 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + +--- + +The files in the package org.apache.commons.compress.archivers.sevenz +were derived from the LZMA SDK, version 9.20 (C/ and CPP/7zip/), +which has been placed in the public domain: + +"LZMA SDK is placed in the public domain." (http://www.7-zip.org/sdk.html) + +--- + +The test file lbzip2_32767.bz2 has been copied from libbzip2's source +repository: + +This program, "bzip2", the associated library "libbzip2", and all +documentation, are copyright (C) 1996-2019 Julian R Seward. All +rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +3. Altered source versions must be plainly marked as such, and must + not be misrepresented as being the original software. + +4. The name of the author may not be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Julian Seward, jseward@acm.org + +======================================================================= + +byte-buddy-1.14.18 NOTICE + +======================================================================= + +Copyright 2014 - Present Rafael Winterhalter + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +======================================================================= + +rocketmq-store-4.9.5 NOTICE + +======================================================================= + + +rocketmq-store 4.9.5 +Copyright 2012-2023 Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + + +======================================================================= + +tomcat-embed-el-9.0.56 NOTICE + +======================================================================= + +Apache Tomcat +Copyright 1999-2021 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +======================================================================= + +jetty-plus-9.4.53.v20231009 NOTICE + +======================================================================= + +============================================================== + Jetty Web Container + Copyright 1995-2018 Mort Bay Consulting Pty Ltd. +============================================================== + +The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd +unless otherwise noted. + +Jetty is dual licensed under both + + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html + + and + + * The Eclipse Public 1.0 License + http://www.eclipse.org/legal/epl-v10.html + +Jetty may be distributed under either license. + +------ +Eclipse + +The following artifacts are EPL. + * org.eclipse.jetty.orbit:org.eclipse.jdt.core + +The following artifacts are EPL and ASL2. + * org.eclipse.jetty.orbit:javax.security.auth.message + + +The following artifacts are EPL and CDDL 1.0. + * org.eclipse.jetty.orbit:javax.mail.glassfish + + +------ +Oracle + +The following artifacts are CDDL + GPLv2 with classpath exception. +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + + * javax.servlet:javax.servlet-api + * javax.annotation:javax.annotation-api + * javax.transaction:javax.transaction-api + * javax.websocket:javax.websocket-api + +------ +Oracle OpenJDK + +If ALPN is used to negotiate HTTP/2 connections, then the following +artifacts may be included in the distribution or downloaded when ALPN +module is selected. + + * java.sun.security.ssl + +These artifacts replace/modify OpenJDK classes. The modififications +are hosted at github and both modified and original are under GPL v2 with +classpath exceptions. +http://openjdk.java.net/legal/gplv2+ce.html + + +------ +OW2 + +The following artifacts are licensed by the OW2 Foundation according to the +terms of http://asm.ow2.org/license.html + +org.ow2.asm:asm-commons +org.ow2.asm:asm + + +------ +Apache + +The following artifacts are ASL2 licensed. + +org.apache.taglibs:taglibs-standard-spec +org.apache.taglibs:taglibs-standard-impl + + +------ +MortBay + +The following artifacts are ASL2 licensed. Based on selected classes from +following Apache Tomcat jars, all ASL2 licensed. + +org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util + +org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api + + +------ +Mortbay + +The following artifacts are CDDL + GPLv2 with classpath exception. + +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + +org.eclipse.jetty.toolchain:jetty-schemas + +------ +Assorted + +The UnixCrypt.java code implements the one way cryptography used by +Unix systems for simple password protection. Copyright 1996 Aki Yoshida, +modified April 2001 by Iris Van den Broeke, Daniel Deville. +Permission to use, copy, modify and distribute UnixCrypt +for non-commercial or commercial purposes and without fee is +granted provided that the copyright notice appears in all copies. + +======================================================================= + +json-utils-2.29.5 NOTICE + +======================================================================= + +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). + +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). + +======================================================================= + +spring-messaging-5.3.20 NOTICE + +======================================================================= + +Spring Framework 5.3.20 +Copyright (c) 2002-2022 Pivotal, Inc. + +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. + +This product may include a number of subcomponents with separate +copyright notices and license terms. Your use of the source code for +these subcomponents is subject to the terms and conditions of the +subcomponent's license, as noted in the license.txt file. + +======================================================================= + +jetty-servlet-9.4.53.v20231009 NOTICE + +======================================================================= + +============================================================== + Jetty Web Container + Copyright 1995-2018 Mort Bay Consulting Pty Ltd. +============================================================== + +The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd +unless otherwise noted. + +Jetty is dual licensed under both + + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html + + and + + * The Eclipse Public 1.0 License + http://www.eclipse.org/legal/epl-v10.html + +Jetty may be distributed under either license. + +------ +Eclipse + +The following artifacts are EPL. + * org.eclipse.jetty.orbit:org.eclipse.jdt.core + +The following artifacts are EPL and ASL2. + * org.eclipse.jetty.orbit:javax.security.auth.message + + +The following artifacts are EPL and CDDL 1.0. + * org.eclipse.jetty.orbit:javax.mail.glassfish + + +------ +Oracle + +The following artifacts are CDDL + GPLv2 with classpath exception. +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + + * javax.servlet:javax.servlet-api + * javax.annotation:javax.annotation-api + * javax.transaction:javax.transaction-api + * javax.websocket:javax.websocket-api + +------ +Oracle OpenJDK + +If ALPN is used to negotiate HTTP/2 connections, then the following +artifacts may be included in the distribution or downloaded when ALPN +module is selected. + + * java.sun.security.ssl + +These artifacts replace/modify OpenJDK classes. The modififications +are hosted at github and both modified and original are under GPL v2 with +classpath exceptions. +http://openjdk.java.net/legal/gplv2+ce.html + + +------ +OW2 + +The following artifacts are licensed by the OW2 Foundation according to the +terms of http://asm.ow2.org/license.html + +org.ow2.asm:asm-commons +org.ow2.asm:asm + + +------ +Apache + +The following artifacts are ASL2 licensed. + +org.apache.taglibs:taglibs-standard-spec +org.apache.taglibs:taglibs-standard-impl + + +------ +MortBay + +The following artifacts are ASL2 licensed. Based on selected classes from +following Apache Tomcat jars, all ASL2 licensed. + +org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util + +org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api + + +------ +Mortbay + +The following artifacts are CDDL + GPLv2 with classpath exception. + +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + +org.eclipse.jetty.toolchain:jetty-schemas + +------ +Assorted + +The UnixCrypt.java code implements the one way cryptography used by +Unix systems for simple password protection. Copyright 1996 Aki Yoshida, +modified April 2001 by Iris Van den Broeke, Daniel Deville. +Permission to use, copy, modify and distribute UnixCrypt +for non-commercial or commercial purposes and without fee is +granted provided that the copyright notice appears in all copies. + +======================================================================= + +pravega-shared-protocol-0.11.0 NOTICE + +======================================================================= + +Copyright (c) 2021 Pravega Authors. +Copyright (c) 2017-2021 Dell Inc., or its subsidiaries. All Rights Reserved. + +This software contains source code from Apache BookKeeper, distributed under +the Apache License Version 2.0, and copyrighted to the Apache Software Foundation. +http://bookkeeper.apache.org + +======================================================================= + +commons-collections-3.2.2 NOTICE + +======================================================================= + +Apache Commons Collections +Copyright 2001-2015 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +======================================================================= + +commons-codec-1.15 NOTICE + +======================================================================= + +Apache Commons Codec +Copyright 2002-2020 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + +src/test/org/apache/commons/codec/language/DoubleMetaphoneTest.java +contains test data from http://aspell.net/test/orig/batch0.tab. +Copyright (C) 2002 Kevin Atkinson (kevina@gnu.org) + +=============================================================================== + +The content of package org.apache.commons.codec.language.bm has been translated +from the original php source code available at http://stevemorse.org/phoneticinfo.htm +with permission from the original authors. +Original source copyright: +Copyright (c) 2008 Alexander Beider & Stephen P. Morse. + +======================================================================= + +annotations-2.29.5 NOTICE + +======================================================================= + +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). + +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). + +======================================================================= + +pulsar-client-admin-api-2.11.1 NOTICE + +======================================================================= + + +Pulsar Client Admin :: API +Copyright 2017-2020 Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + + +======================================================================= + +spring-jdbc-5.3.31 NOTICE + +======================================================================= + +Spring Framework 5.3.31 +Copyright (c) 2002-2023 Pivotal, Inc. + +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. + +This product may include a number of subcomponents with separate +copyright notices and license terms. Your use of the source code for +these subcomponents is subject to the terms and conditions of the +subcomponent's license, as noted in the license.txt file. + +======================================================================= + +httpasyncclient-4.1.5 NOTICE + +======================================================================= + + +Apache HttpAsyncClient +Copyright 2010-2021 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + + +======================================================================= + +commons-lang3-3.17.0 NOTICE + +======================================================================= + +Apache Commons Lang +Copyright 2001-2024 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + +======================================================================= + +commons-beanutils-1.8.2 NOTICE + +======================================================================= + +Apache Commons BeanUtils +Copyright 2000-2009 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +======================================================================= + +rocketmq-tools-4.9.5 NOTICE + +======================================================================= + + +rocketmq-tools 4.9.5 +Copyright 2012-2023 Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + + +======================================================================= + +mybatis-spring-2.0.4 NOTICE + +======================================================================= + +MyBatis Spring +Copyright 2010-2013 + +This product includes software developed by +The MyBatis Team (http://www.mybatis.org/). + +iBATIS + This product includes software developed by + The Apache Software Foundation (http://www.apache.org/). + + Copyright 2010 The Apache Software Foundation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Spring Framework + All Spring projects are licensed under the terms of the Apache License, Version 2.0 + + Copyright 2002-2010 the original author or authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +======================================================================= + +spring-expression-5.3.9 NOTICE + +======================================================================= + +Spring Framework 5.3.9 +Copyright (c) 2002-2021 Pivotal, Inc. + +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. + +This product may include a number of subcomponents with separate +copyright notices and license terms. Your use of the source code for +these subcomponents is subject to the terms and conditions of the +subcomponent's license, as noted in the license.txt file. + +======================================================================= + +javax-websocket-server-impl-9.4.53.v20231009 NOTICE + +======================================================================= + +============================================================== + Jetty Web Container + Copyright 1995-2018 Mort Bay Consulting Pty Ltd. +============================================================== + +The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd +unless otherwise noted. + +Jetty is dual licensed under both + + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html + + and + + * The Eclipse Public 1.0 License + http://www.eclipse.org/legal/epl-v10.html + +Jetty may be distributed under either license. + +------ +Eclipse + +The following artifacts are EPL. + * org.eclipse.jetty.orbit:org.eclipse.jdt.core + +The following artifacts are EPL and ASL2. + * org.eclipse.jetty.orbit:javax.security.auth.message + + +The following artifacts are EPL and CDDL 1.0. + * org.eclipse.jetty.orbit:javax.mail.glassfish + + +------ +Oracle + +The following artifacts are CDDL + GPLv2 with classpath exception. +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + + * javax.servlet:javax.servlet-api + * javax.annotation:javax.annotation-api + * javax.transaction:javax.transaction-api + * javax.websocket:javax.websocket-api + +------ +Oracle OpenJDK + +If ALPN is used to negotiate HTTP/2 connections, then the following +artifacts may be included in the distribution or downloaded when ALPN +module is selected. + + * java.sun.security.ssl + +These artifacts replace/modify OpenJDK classes. The modififications +are hosted at github and both modified and original are under GPL v2 with +classpath exceptions. +http://openjdk.java.net/legal/gplv2+ce.html + + +------ +OW2 + +The following artifacts are licensed by the OW2 Foundation according to the +terms of http://asm.ow2.org/license.html + +org.ow2.asm:asm-commons +org.ow2.asm:asm + + +------ +Apache + +The following artifacts are ASL2 licensed. + +org.apache.taglibs:taglibs-standard-spec +org.apache.taglibs:taglibs-standard-impl + + +------ +MortBay + +The following artifacts are ASL2 licensed. Based on selected classes from +following Apache Tomcat jars, all ASL2 licensed. + +org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util + +org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api + + +------ +Mortbay + +The following artifacts are CDDL + GPLv2 with classpath exception. + +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + +org.eclipse.jetty.toolchain:jetty-schemas + +------ +Assorted + +The UnixCrypt.java code implements the one way cryptography used by +Unix systems for simple password protection. Copyright 1996 Aki Yoshida, +modified April 2001 by Iris Van den Broeke, Daniel Deville. +Permission to use, copy, modify and distribute UnixCrypt +for non-commercial or commercial purposes and without fee is +granted provided that the copyright notice appears in all copies. + +======================================================================= + +retries-spi-2.29.5 NOTICE + +======================================================================= + +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). + +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). + +======================================================================= + +spring-aop-5.3.15 NOTICE + +======================================================================= + +Spring Framework 5.3.15 +Copyright (c) 2002-2022 Pivotal, Inc. + +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. + +This product may include a number of subcomponents with separate +copyright notices and license terms. Your use of the source code for +these subcomponents is subject to the terms and conditions of the +subcomponent's license, as noted in the license.txt file. + +======================================================================= + +jakarta.annotation-api-1.3.5 NOTICE + +======================================================================= + +# Notices for Jakarta Annotations + +This content is produced and maintained by the Jakarta Annotations project. + + * Project home: https://projects.eclipse.org/projects/ee4j.ca + +## Trademarks + +Jakarta Annotations is a trademark of the Eclipse Foundation. + +## Declared Project Licenses + +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v. 2.0 which is available at +http://www.eclipse.org/legal/epl-2.0. This Source Code may also be made +available under the following Secondary Licenses when the conditions for such +availability set forth in the Eclipse Public License v. 2.0 are satisfied: GNU +General Public License, version 2 with the GNU Classpath Exception which is +available at https://www.gnu.org/software/classpath/license.html. + +SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + +## Source Code + +The project maintains the following source code repositories: + + * https://github.com/eclipse-ee4j/common-annotations-api + +## Third-party Content + +## Cryptography + +Content may contain encryption software. The country in which you are currently +may have restrictions on the import, possession, and use, and/or re-export to +another country, of encryption software. BEFORE using any encryption software, +please check the country's laws, regulations and policies concerning the import, +possession, or use, and re-export of encryption software, to see if this is +permitted. + +======================================================================= + +audience-annotations-0.12.0 NOTICE + +======================================================================= + + +Apache Yetus - Audience Annotations +Copyright 2015-2020 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + + +======================================================================= + +commons-io-2.18.0 NOTICE + +======================================================================= + +Apache Commons IO +Copyright 2002-2024 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + +======================================================================= + +mybatis-spring-2.1.2 NOTICE + +======================================================================= + +MyBatis Spring +Copyright 2010-2013 + +This product includes software developed by +The MyBatis Team (http://www.mybatis.org/). + +iBATIS + This product includes software developed by + The Apache Software Foundation (http://www.apache.org/). + + Copyright 2010 The Apache Software Foundation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +Spring Framework + All Spring projects are licensed under the terms of the Apache License, Version 2.0 + + Copyright 2002-2010 the original author or authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +======================================================================= + +okhttp-3.14.9 NOTICE + +======================================================================= + +Note that publicsuffixes.gz is compiled from The Public Suffix List: +https://publicsuffix.org/list/public_suffix_list.dat + +It is subject to the terms of the Mozilla Public License, v. 2.0: +https://mozilla.org/MPL/2.0/ + +======================================================================= + +commons-beanutils-1.9.4 NOTICE + +======================================================================= + +Apache Commons BeanUtils +Copyright 2000-2019 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +======================================================================= + +nacos-client-2.4.1 NOTICE + +======================================================================= + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2016 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +------------------------------------------------------------------------------- +This product contains a forked and modified version of Tomcat Native + + * LICENSE: + * license/LICENSE.tomcat-native.txt (Apache License 2.0) + * HOMEPAGE: + * http://tomcat.apache.org/native-doc/ + * https://svn.apache.org/repos/asf/tomcat/native/ + +This product contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. + + * LICENSE: + * license/LICENSE.mvn-wrapper.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/takari/maven-wrapper + +This product contains small piece of code to support AIX, taken from netbsd. + + * LICENSE: + * license/LICENSE.aix-netbsd.txt (OpenSSL License) + * HOMEPAGE: + * https://ftp.netbsd.org/pub/NetBSD/NetBSD-current/src/crypto/external/bsd/openssl/dist + + +This product contains code from boringssl. + + * LICENSE (Combination ISC and OpenSSL license) + * license/LICENSE.boringssl.txt (Combination ISC and OpenSSL license) + * HOMEPAGE: + * https://boringssl.googlesource.com/boringssl/ + +======================================================================= + +istack-commons-runtime-3.0.8 NOTICE + +======================================================================= + +# Notices for Eclipse Implementation of JAXB + +This content is produced and maintained by the Eclipse Implementation of JAXB +project. + +* Project home: https://projects.eclipse.org/projects/ee4j.jaxb-impl + +## Trademarks + +Eclipse Implementation of JAXB is a trademark of the Eclipse Foundation. + +## Copyright + +All content is the property of the respective authors or their employers. For +more information regarding authorship of content, please consult the listed +source code repository logs. + +## Declared Project Licenses + +This program and the accompanying materials are made available under the terms +of the Eclipse Distribution License v. 1.0 which is available at +http://www.eclipse.org/org/documents/edl-v10.php. + +SPDX-License-Identifier: BSD-3-Clause + +## Source Code + +The project maintains the following source code repositories: + +* https://github.com/eclipse-ee4j/jaxb-ri +* https://github.com/eclipse-ee4j/jaxb-istack-commons +* https://github.com/eclipse-ee4j/jaxb-dtd-parser +* https://github.com/eclipse-ee4j/jaxb-fi +* https://github.com/eclipse-ee4j/jaxb-stax-ex +* https://github.com/eclipse-ee4j/jax-rpc-ri + +## Third-party Content + +This project leverages the following third party content. + +Apache Ant (1.10.2) + +* License: Apache-2.0 AND W3C AND LicenseRef-Public-Domain + +Apache Ant (1.10.2) + +* License: Apache-2.0 AND W3C AND LicenseRef-Public-Domain + +Apache Felix (1.2.0) + +* License: Apache License, 2.0 + +args4j (2.33) + +* License: MIT License + +dom4j (1.6.1) + +* License: Custom license based on Apache 1.1 + +file-management (3.0.0) + +* License: Apache-2.0 +* Project: https://maven.apache.org/shared/file-management/ +* Source: + https://svn.apache.org/viewvc/maven/shared/tags/file-management-3.0.0/ + +JUnit (4.12) + +* License: Eclipse Public License + +JUnit (4.12) + +* License: Eclipse Public License + +maven-compat (3.5.2) + +* License: Apache-2.0 +* Project: https://maven.apache.org/ref/3.5.2/maven-compat/ +* Source: + https://mvnrepository.com/artifact/org.apache.maven/maven-compat/3.5.2 + +maven-core (3.5.2) + +* License: Apache-2.0 +* Project: https://maven.apache.org/ref/3.5.2/maven-core/index.html +* Source: https://mvnrepository.com/artifact/org.apache.maven/maven-core/3.5.2 + +maven-plugin-annotations (3.5) + +* License: Apache-2.0 +* Project: https://maven.apache.org/plugin-tools/maven-plugin-annotations/ +* Source: + https://github.com/apache/maven-plugin-tools/tree/master/maven-plugin-annotations + +maven-plugin-api (3.5.2) + +* License: Apache-2.0 + +maven-resolver-api (1.1.1) + +* License: Apache-2.0 + +maven-resolver-api (1.1.1) + +* License: Apache-2.0 + +maven-resolver-connector-basic (1.1.1) + +* License: Apache-2.0 + +maven-resolver-impl (1.1.1) + +* License: Apache-2.0 + +maven-resolver-spi (1.1.1) + +* License: Apache-2.0 + +maven-resolver-transport-file (1.1.1) + +* License: Apache-2.0 +* Project: https://maven.apache.org/resolver/maven-resolver-transport-file/ +* Source: + https://github.com/apache/maven-resolver/tree/master/maven-resolver-transport-file + +maven-resolver-util (1.1.1) + +* License: Apache-2.0 + +maven-settings (3.5.2) + +* License: Apache-2.0 +* Source: + https://mvnrepository.com/artifact/org.apache.maven/maven-settings/3.5.2 + +OSGi Service Platform Core Companion Code (6.0) + +* License: Apache License, 2.0 + +plexus-archiver (3.5) + +* License: Apache-2.0 +* Project: https://codehaus-plexus.github.io/plexus-archiver/ +* Source: https://github.com/codehaus-plexus/plexus-archiver + +plexus-io (3.0.0) + +* License: Apache-2.0 + +plexus-utils (3.1.0) + +* License: Apache- 2.0 or Apache- 1.1 or BSD or Public Domain or Indiana + University Extreme! Lab Software License V1.1.1 (Apache 1.1 style) + +relaxng-datatype (1.0) + +* License: New BSD license + +Sax (0.2) + +* License: SAX-PD +* Project: http://www.megginson.com/downloads/SAX/ +* Source: http://sourceforge.net/project/showfiles.php?group_id=29449 + +testng (6.14.2) + +* License: Apache-2.0 AND (MIT OR GPL-1.0+) +* Project: https://testng.org/doc/index.html +* Source: https://github.com/cbeust/testng + +wagon-http-lightweight (3.0.0) + +* License: Pending +* Project: https://maven.apache.org/wagon/ +* Source: + https://mvnrepository.com/artifact/org.apache.maven.wagon/wagon-http-lightweight/3.0.0 + +xz for java (1.8) + +* License: LicenseRef-Public-Domain + +## Cryptography + +Content may contain encryption software. The country in which you are currently +may have restrictions on the import, possession, and use, and/or re-export to +another country, of encryption software. BEFORE using any encryption software, +please check the country's laws, regulations and policies concerning the import, +possession, or use, and re-export of encryption software, to see if this is +permitted. + +======================================================================= + +jakarta.annotation-api-2.1.1 NOTICE + +======================================================================= + +# Notices for Jakarta Annotations + +This content is produced and maintained by the Jakarta Annotations project. + + * Project home: https://projects.eclipse.org/projects/ee4j.ca + +## Trademarks + +Jakarta Annotations is a trademark of the Eclipse Foundation. + +## Declared Project Licenses + +This program and the accompanying materials are made available under the terms +of the Eclipse Public License v. 2.0 which is available at +http://www.eclipse.org/legal/epl-2.0. This Source Code may also be made +available under the following Secondary Licenses when the conditions for such +availability set forth in the Eclipse Public License v. 2.0 are satisfied: GNU +General Public License, version 2 with the GNU Classpath Exception which is +available at https://www.gnu.org/software/classpath/license.html. + +SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0 + +## Source Code + +The project maintains the following source code repositories: + + * https://github.com/eclipse-ee4j/common-annotations-api + +## Third-party Content + +## Cryptography + +Content may contain encryption software. The country in which you are currently +may have restrictions on the import, possession, and use, and/or re-export to +another country, of encryption software. BEFORE using any encryption software, +please check the country's laws, regulations and policies concerning the import, +possession, or use, and re-export of encryption software, to see if this is +permitted. + +======================================================================= + +log4j-api-2.24.1 NOTICE + +======================================================================= + +Apache Log4j API +Copyright 1999-2024 The Apache Software Foundation + + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +======================================================================= + +jetty-security-9.4.53.v20231009 NOTICE + +======================================================================= + +============================================================== + Jetty Web Container + Copyright 1995-2018 Mort Bay Consulting Pty Ltd. +============================================================== + +The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd +unless otherwise noted. + +Jetty is dual licensed under both + + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html + + and + + * The Eclipse Public 1.0 License + http://www.eclipse.org/legal/epl-v10.html + +Jetty may be distributed under either license. + +------ +Eclipse + +The following artifacts are EPL. + * org.eclipse.jetty.orbit:org.eclipse.jdt.core + +The following artifacts are EPL and ASL2. + * org.eclipse.jetty.orbit:javax.security.auth.message + + +The following artifacts are EPL and CDDL 1.0. + * org.eclipse.jetty.orbit:javax.mail.glassfish + + +------ +Oracle + +The following artifacts are CDDL + GPLv2 with classpath exception. +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + + * javax.servlet:javax.servlet-api + * javax.annotation:javax.annotation-api + * javax.transaction:javax.transaction-api + * javax.websocket:javax.websocket-api + +------ +Oracle OpenJDK + +If ALPN is used to negotiate HTTP/2 connections, then the following +artifacts may be included in the distribution or downloaded when ALPN +module is selected. + + * java.sun.security.ssl + +These artifacts replace/modify OpenJDK classes. The modififications +are hosted at github and both modified and original are under GPL v2 with +classpath exceptions. +http://openjdk.java.net/legal/gplv2+ce.html + + +------ +OW2 + +The following artifacts are licensed by the OW2 Foundation according to the +terms of http://asm.ow2.org/license.html + +org.ow2.asm:asm-commons +org.ow2.asm:asm + + +------ +Apache + +The following artifacts are ASL2 licensed. + +org.apache.taglibs:taglibs-standard-spec +org.apache.taglibs:taglibs-standard-impl + + +------ +MortBay + +The following artifacts are ASL2 licensed. Based on selected classes from +following Apache Tomcat jars, all ASL2 licensed. + +org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util + +org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api + + +------ +Mortbay + +The following artifacts are CDDL + GPLv2 with classpath exception. + +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + +org.eclipse.jetty.toolchain:jetty-schemas + +------ +Assorted + +The UnixCrypt.java code implements the one way cryptography used by +Unix systems for simple password protection. Copyright 1996 Aki Yoshida, +modified April 2001 by Iris Van den Broeke, Daniel Deville. +Permission to use, copy, modify and distribute UnixCrypt +for non-commercial or commercial purposes and without fee is +granted provided that the copyright notice appears in all copies. + +======================================================================= + +pulsar-client-2.11.1 NOTICE + +======================================================================= + + +Pulsar Client Java +Copyright 2017-2020 Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + + +======================================================================= + +pulsar-client-2.11.1 NOTICE + +======================================================================= + +Apache Commons Lang +Copyright 2001-2020 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + +======================================================================= + +rocketmq-logging-4.9.5 NOTICE + +======================================================================= + + +rocketmq-logging 4.9.5 +Copyright 2012-2023 Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + + +======================================================================= + +spring-tx-5.3.31 NOTICE + +======================================================================= + +Spring Framework 5.3.31 +Copyright (c) 2002-2023 Pivotal, Inc. + +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. + +This product may include a number of subcomponents with separate +copyright notices and license terms. Your use of the source code for +these subcomponents is subject to the terms and conditions of the +subcomponent's license, as noted in the license.txt file. + +======================================================================= + +curator-recipes-5.7.0 NOTICE + +======================================================================= + +Curator Recipes +Copyright 2011-2023 The Apache Software Foundation + + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +======================================================================= + +joda-time-2.9.4 NOTICE + +======================================================================= + +============================================================================= += NOTICE file corresponding to section 4d of the Apache License Version 2.0 = +============================================================================= +This product includes software developed by +Joda.org (http://www.joda.org/). + +======================================================================= + +spring-orm-5.3.9 NOTICE + +======================================================================= + +Spring Framework 5.3.9 +Copyright (c) 2002-2021 Pivotal, Inc. + +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. + +This product may include a number of subcomponents with separate +copyright notices and license terms. Your use of the source code for +these subcomponents is subject to the terms and conditions of the +subcomponent's license, as noted in the license.txt file. + +======================================================================= + +commons-lang-2.6 NOTICE + +======================================================================= + +Apache Commons Lang +Copyright 2001-2011 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +======================================================================= + +aws-xml-protocol-2.29.5 NOTICE + +======================================================================= + +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). + +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). + +======================================================================= + +commons-digester-2.1 NOTICE + +======================================================================= + +Apache Commons Digester +Copyright 2001-2010 The Apache Software Foundation + +This product includes software developed by +The Apache Software Foundation (http://www.apache.org/). + +======================================================================= + +spring-context-5.3.9 NOTICE + +======================================================================= + +Spring Framework 5.3.9 +Copyright (c) 2002-2021 Pivotal, Inc. + +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. + +This product may include a number of subcomponents with separate +copyright notices and license terms. Your use of the source code for +these subcomponents is subject to the terms and conditions of the +subcomponent's license, as noted in the license.txt file. + +======================================================================= + +websocket-server-9.4.53.v20231009 NOTICE + +======================================================================= + +============================================================== + Jetty Web Container + Copyright 1995-2018 Mort Bay Consulting Pty Ltd. +============================================================== + +The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd +unless otherwise noted. + +Jetty is dual licensed under both + + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html + + and + + * The Eclipse Public 1.0 License + http://www.eclipse.org/legal/epl-v10.html + +Jetty may be distributed under either license. + +------ +Eclipse + +The following artifacts are EPL. + * org.eclipse.jetty.orbit:org.eclipse.jdt.core + +The following artifacts are EPL and ASL2. + * org.eclipse.jetty.orbit:javax.security.auth.message + + +The following artifacts are EPL and CDDL 1.0. + * org.eclipse.jetty.orbit:javax.mail.glassfish + + +------ +Oracle + +The following artifacts are CDDL + GPLv2 with classpath exception. +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + + * javax.servlet:javax.servlet-api + * javax.annotation:javax.annotation-api + * javax.transaction:javax.transaction-api + * javax.websocket:javax.websocket-api + +------ +Oracle OpenJDK + +If ALPN is used to negotiate HTTP/2 connections, then the following +artifacts may be included in the distribution or downloaded when ALPN +module is selected. + + * java.sun.security.ssl + +These artifacts replace/modify OpenJDK classes. The modififications +are hosted at github and both modified and original are under GPL v2 with +classpath exceptions. +http://openjdk.java.net/legal/gplv2+ce.html + + +------ +OW2 + +The following artifacts are licensed by the OW2 Foundation according to the +terms of http://asm.ow2.org/license.html + +org.ow2.asm:asm-commons +org.ow2.asm:asm + + +------ +Apache + +The following artifacts are ASL2 licensed. + +org.apache.taglibs:taglibs-standard-spec +org.apache.taglibs:taglibs-standard-impl + + +------ +MortBay + +The following artifacts are ASL2 licensed. Based on selected classes from +following Apache Tomcat jars, all ASL2 licensed. + +org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util + +org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api + + +------ +Mortbay + +The following artifacts are CDDL + GPLv2 with classpath exception. + +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + +org.eclipse.jetty.toolchain:jetty-schemas + +------ +Assorted + +The UnixCrypt.java code implements the one way cryptography used by +Unix systems for simple password protection. Copyright 1996 Aki Yoshida, +modified April 2001 by Iris Van den Broeke, Daniel Deville. +Permission to use, copy, modify and distribute UnixCrypt +for non-commercial or commercial purposes and without fee is +granted provided that the copyright notice appears in all copies. + +======================================================================= + +spring-core-5.3.9 NOTICE + +======================================================================= + +Spring Framework 5.3.9 +Copyright (c) 2002-2021 Pivotal, Inc. + +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. + +This product may include a number of subcomponents with separate +copyright notices and license terms. Your use of the source code for +these subcomponents is subject to the terms and conditions of the +subcomponent's license, as noted in the license.txt file. + +======================================================================= + +arns-2.29.5 NOTICE + +======================================================================= + +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). + +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). + +======================================================================= + +jetty-http-9.4.53.v20231009 NOTICE + +======================================================================= + +============================================================== + Jetty Web Container + Copyright 1995-2018 Mort Bay Consulting Pty Ltd. +============================================================== + +The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd +unless otherwise noted. + +Jetty is dual licensed under both + + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html + + and + + * The Eclipse Public 1.0 License + http://www.eclipse.org/legal/epl-v10.html + +Jetty may be distributed under either license. + +------ +Eclipse + +The following artifacts are EPL. + * org.eclipse.jetty.orbit:org.eclipse.jdt.core + +The following artifacts are EPL and ASL2. + * org.eclipse.jetty.orbit:javax.security.auth.message + + +The following artifacts are EPL and CDDL 1.0. + * org.eclipse.jetty.orbit:javax.mail.glassfish + + +------ +Oracle + +The following artifacts are CDDL + GPLv2 with classpath exception. +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + + * javax.servlet:javax.servlet-api + * javax.annotation:javax.annotation-api + * javax.transaction:javax.transaction-api + * javax.websocket:javax.websocket-api + +------ +Oracle OpenJDK + +If ALPN is used to negotiate HTTP/2 connections, then the following +artifacts may be included in the distribution or downloaded when ALPN +module is selected. + + * java.sun.security.ssl + +These artifacts replace/modify OpenJDK classes. The modififications +are hosted at github and both modified and original are under GPL v2 with +classpath exceptions. +http://openjdk.java.net/legal/gplv2+ce.html + + +------ +OW2 + +The following artifacts are licensed by the OW2 Foundation according to the +terms of http://asm.ow2.org/license.html + +org.ow2.asm:asm-commons +org.ow2.asm:asm + + +------ +Apache + +The following artifacts are ASL2 licensed. + +org.apache.taglibs:taglibs-standard-spec +org.apache.taglibs:taglibs-standard-impl + + +------ +MortBay + +The following artifacts are ASL2 licensed. Based on selected classes from +following Apache Tomcat jars, all ASL2 licensed. + +org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util + +org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api + + +------ +Mortbay + +The following artifacts are CDDL + GPLv2 with classpath exception. + +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + +org.eclipse.jetty.toolchain:jetty-schemas + +------ +Assorted + +The UnixCrypt.java code implements the one way cryptography used by +Unix systems for simple password protection. Copyright 1996 Aki Yoshida, +modified April 2001 by Iris Van den Broeke, Daniel Deville. +Permission to use, copy, modify and distribute UnixCrypt +for non-commercial or commercial purposes and without fee is +granted provided that the copyright notice appears in all copies. + +======================================================================= + +commons-text-1.12.0 NOTICE + +======================================================================= + +Apache Commons Text +Copyright 2014-2024 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + +======================================================================= + +javax-websocket-client-impl-9.4.53.v20231009 NOTICE + +======================================================================= + +============================================================== + Jetty Web Container + Copyright 1995-2018 Mort Bay Consulting Pty Ltd. +============================================================== + +The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd +unless otherwise noted. + +Jetty is dual licensed under both + + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html + + and + + * The Eclipse Public 1.0 License + http://www.eclipse.org/legal/epl-v10.html + +Jetty may be distributed under either license. + +------ +Eclipse + +The following artifacts are EPL. + * org.eclipse.jetty.orbit:org.eclipse.jdt.core + +The following artifacts are EPL and ASL2. + * org.eclipse.jetty.orbit:javax.security.auth.message + + +The following artifacts are EPL and CDDL 1.0. + * org.eclipse.jetty.orbit:javax.mail.glassfish + + +------ +Oracle + +The following artifacts are CDDL + GPLv2 with classpath exception. +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + + * javax.servlet:javax.servlet-api + * javax.annotation:javax.annotation-api + * javax.transaction:javax.transaction-api + * javax.websocket:javax.websocket-api + +------ +Oracle OpenJDK + +If ALPN is used to negotiate HTTP/2 connections, then the following +artifacts may be included in the distribution or downloaded when ALPN +module is selected. + + * java.sun.security.ssl + +These artifacts replace/modify OpenJDK classes. The modififications +are hosted at github and both modified and original are under GPL v2 with +classpath exceptions. +http://openjdk.java.net/legal/gplv2+ce.html + + +------ +OW2 + +The following artifacts are licensed by the OW2 Foundation according to the +terms of http://asm.ow2.org/license.html + +org.ow2.asm:asm-commons +org.ow2.asm:asm + + +------ +Apache + +The following artifacts are ASL2 licensed. + +org.apache.taglibs:taglibs-standard-spec +org.apache.taglibs:taglibs-standard-impl + + +------ +MortBay + +The following artifacts are ASL2 licensed. Based on selected classes from +following Apache Tomcat jars, all ASL2 licensed. + +org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util + +org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api + + +------ +Mortbay + +The following artifacts are CDDL + GPLv2 with classpath exception. + +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + +org.eclipse.jetty.toolchain:jetty-schemas + +------ +Assorted + +The UnixCrypt.java code implements the one way cryptography used by +Unix systems for simple password protection. Copyright 1996 Aki Yoshida, +modified April 2001 by Iris Van den Broeke, Daniel Deville. +Permission to use, copy, modify and distribute UnixCrypt +for non-commercial or commercial purposes and without fee is +granted provided that the copyright notice appears in all copies. + +======================================================================= + +auth-2.29.5 NOTICE + +======================================================================= + +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). + +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). + +======================================================================= + +spring-boot-starter-2.5.9 NOTICE + +======================================================================= + +Spring Boot 2.5.9 +Copyright (c) 2012-2022 Pivotal, Inc. + +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. +======================================================================= + +spring-webmvc-5.3.31 NOTICE + +======================================================================= + +Spring Framework 5.3.31 +Copyright (c) 2002-2023 Pivotal, Inc. + +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. + +This product may include a number of subcomponents with separate +copyright notices and license terms. Your use of the source code for +these subcomponents is subject to the terms and conditions of the +subcomponent's license, as noted in the license.txt file. + +======================================================================= + +txw2-2.3.2 NOTICE + +======================================================================= + +# Notices for Eclipse Implementation of JAXB + +This content is produced and maintained by the Eclipse Implementation of JAXB +project. + +* Project home: https://projects.eclipse.org/projects/ee4j.jaxb-impl + +## Trademarks + +Eclipse Implementation of JAXB is a trademark of the Eclipse Foundation. + +## Copyright + +All content is the property of the respective authors or their employers. For +more information regarding authorship of content, please consult the listed +source code repository logs. + +## Declared Project Licenses + +This program and the accompanying materials are made available under the terms +of the Eclipse Distribution License v. 1.0 which is available at +http://www.eclipse.org/org/documents/edl-v10.php. + +SPDX-License-Identifier: BSD-3-Clause + +## Source Code + +The project maintains the following source code repositories: + +* https://github.com/eclipse-ee4j/jaxb-ri +* https://github.com/eclipse-ee4j/jaxb-istack-commons +* https://github.com/eclipse-ee4j/jaxb-dtd-parser +* https://github.com/eclipse-ee4j/jaxb-fi +* https://github.com/eclipse-ee4j/jaxb-stax-ex +* https://github.com/eclipse-ee4j/jax-rpc-ri + +## Third-party Content + +This project leverages the following third party content. + +Apache Ant (1.10.2) + +* License: Apache-2.0 AND W3C AND LicenseRef-Public-Domain + +Apache Ant (1.10.2) + +* License: Apache-2.0 AND W3C AND LicenseRef-Public-Domain + +Apache Felix (1.2.0) + +* License: Apache License, 2.0 + +args4j (2.33) + +* License: MIT License + +dom4j (1.6.1) + +* License: Custom license based on Apache 1.1 + +file-management (3.0.0) + +* License: Apache-2.0 +* Project: https://maven.apache.org/shared/file-management/ +* Source: + https://svn.apache.org/viewvc/maven/shared/tags/file-management-3.0.0/ + +JUnit (4.12) + +* License: Eclipse Public License + +JUnit (4.12) + +* License: Eclipse Public License + +maven-compat (3.5.2) + +* License: Apache-2.0 +* Project: https://maven.apache.org/ref/3.5.2/maven-compat/ +* Source: + https://mvnrepository.com/artifact/org.apache.maven/maven-compat/3.5.2 + +maven-core (3.5.2) + +* License: Apache-2.0 +* Project: https://maven.apache.org/ref/3.5.2/maven-core/index.html +* Source: https://mvnrepository.com/artifact/org.apache.maven/maven-core/3.5.2 + +maven-plugin-annotations (3.5) + +* License: Apache-2.0 +* Project: https://maven.apache.org/plugin-tools/maven-plugin-annotations/ +* Source: + https://github.com/apache/maven-plugin-tools/tree/master/maven-plugin-annotations + +maven-plugin-api (3.5.2) + +* License: Apache-2.0 + +maven-resolver-api (1.1.1) + +* License: Apache-2.0 + +maven-resolver-api (1.1.1) + +* License: Apache-2.0 + +maven-resolver-connector-basic (1.1.1) + +* License: Apache-2.0 + +maven-resolver-impl (1.1.1) + +* License: Apache-2.0 + +maven-resolver-spi (1.1.1) + +* License: Apache-2.0 + +maven-resolver-transport-file (1.1.1) + +* License: Apache-2.0 +* Project: https://maven.apache.org/resolver/maven-resolver-transport-file/ +* Source: + https://github.com/apache/maven-resolver/tree/master/maven-resolver-transport-file + +maven-resolver-util (1.1.1) + +* License: Apache-2.0 + +maven-settings (3.5.2) + +* License: Apache-2.0 +* Source: + https://mvnrepository.com/artifact/org.apache.maven/maven-settings/3.5.2 + +OSGi Service Platform Core Companion Code (6.0) + +* License: Apache License, 2.0 + +plexus-archiver (3.5) + +* License: Apache-2.0 +* Project: https://codehaus-plexus.github.io/plexus-archiver/ +* Source: https://github.com/codehaus-plexus/plexus-archiver + +plexus-io (3.0.0) + +* License: Apache-2.0 + +plexus-utils (3.1.0) + +* License: Apache- 2.0 or Apache- 1.1 or BSD or Public Domain or Indiana + University Extreme! Lab Software License V1.1.1 (Apache 1.1 style) + +relaxng-datatype (1.0) + +* License: New BSD license + +Sax (0.2) + +* License: SAX-PD +* Project: http://www.megginson.com/downloads/SAX/ +* Source: http://sourceforge.net/project/showfiles.php?group_id=29449 + +testng (6.14.2) + +* License: Apache-2.0 AND (MIT OR GPL-1.0+) +* Project: https://testng.org/doc/index.html +* Source: https://github.com/cbeust/testng + +wagon-http-lightweight (3.0.0) + +* License: Pending +* Project: https://maven.apache.org/wagon/ +* Source: + https://mvnrepository.com/artifact/org.apache.maven.wagon/wagon-http-lightweight/3.0.0 + +xz for java (1.8) + +* License: LicenseRef-Public-Domain + +## Cryptography + +Content may contain encryption software. The country in which you are currently +may have restrictions on the import, possession, and use, and/or re-export to +another country, of encryption software. BEFORE using any encryption software, +please check the country's laws, regulations and policies concerning the import, +possession, or use, and re-export of encryption software, to see if this is +permitted. + + +======================================================================= + +jetty-xml-9.4.53.v20231009 NOTICE + +======================================================================= + +============================================================== + Jetty Web Container + Copyright 1995-2018 Mort Bay Consulting Pty Ltd. +============================================================== + +The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd +unless otherwise noted. + +Jetty is dual licensed under both + + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html + + and + + * The Eclipse Public 1.0 License + http://www.eclipse.org/legal/epl-v10.html + +Jetty may be distributed under either license. + +------ +Eclipse + +The following artifacts are EPL. + * org.eclipse.jetty.orbit:org.eclipse.jdt.core + +The following artifacts are EPL and ASL2. + * org.eclipse.jetty.orbit:javax.security.auth.message + + +The following artifacts are EPL and CDDL 1.0. + * org.eclipse.jetty.orbit:javax.mail.glassfish + + +------ +Oracle + +The following artifacts are CDDL + GPLv2 with classpath exception. +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + + * javax.servlet:javax.servlet-api + * javax.annotation:javax.annotation-api + * javax.transaction:javax.transaction-api + * javax.websocket:javax.websocket-api + +------ +Oracle OpenJDK + +If ALPN is used to negotiate HTTP/2 connections, then the following +artifacts may be included in the distribution or downloaded when ALPN +module is selected. + + * java.sun.security.ssl + +These artifacts replace/modify OpenJDK classes. The modififications +are hosted at github and both modified and original are under GPL v2 with +classpath exceptions. +http://openjdk.java.net/legal/gplv2+ce.html + + +------ +OW2 + +The following artifacts are licensed by the OW2 Foundation according to the +terms of http://asm.ow2.org/license.html + +org.ow2.asm:asm-commons +org.ow2.asm:asm + + +------ +Apache + +The following artifacts are ASL2 licensed. + +org.apache.taglibs:taglibs-standard-spec +org.apache.taglibs:taglibs-standard-impl + + +------ +MortBay + +The following artifacts are ASL2 licensed. Based on selected classes from +following Apache Tomcat jars, all ASL2 licensed. + +org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util + +org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api + + +------ +Mortbay + +The following artifacts are CDDL + GPLv2 with classpath exception. + +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + +org.eclipse.jetty.toolchain:jetty-schemas + +------ +Assorted + +The UnixCrypt.java code implements the one way cryptography used by +Unix systems for simple password protection. Copyright 1996 Aki Yoshida, +modified April 2001 by Iris Van den Broeke, Daniel Deville. +Permission to use, copy, modify and distribute UnixCrypt +for non-commercial or commercial purposes and without fee is +granted provided that the copyright notice appears in all copies. + +======================================================================= + +spring-jcl-5.3.9 NOTICE + +======================================================================= + +Spring Framework 5.3.9 +Copyright (c) 2002-2021 Pivotal, Inc. + +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. + +This product may include a number of subcomponents with separate +copyright notices and license terms. Your use of the source code for +these subcomponents is subject to the terms and conditions of the +subcomponent's license, as noted in the license.txt file. + +======================================================================= + +mybatis-3.5.16 NOTICE + +======================================================================= + +MyBatis +Copyright 2010-2023 + +This product includes software developed by +The MyBatis Team (https://www.mybatis.org/). + +iBATIS + This product includes software developed by + The Apache Software Foundation (https://www.apache.org/). + + Copyright 2010 The Apache Software Foundation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +OGNL +//-------------------------------------------------------------------------- +// Copyright (c) 2004, Drew Davidson and Luke Blanshard +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// Neither the name of the Drew Davidson nor the names of its contributors +// may be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +// COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS +// OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +// AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH +// DAMAGE. +//-------------------------------------------------------------------------- + +Refactored SqlBuilder class (SQL, AbstractSQL) + + This product includes software developed by + Adam Gent (https://gist.github.com/3650165) + + Copyright 2010 Adam Gent + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +======================================================================= + +httpclient-4.5.14 NOTICE + +======================================================================= + + +Apache HttpClient +Copyright 1999-2022 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + + +======================================================================= + +jetty-client-9.4.53.v20231009 NOTICE + +======================================================================= + +============================================================== + Jetty Web Container + Copyright 1995-2018 Mort Bay Consulting Pty Ltd. +============================================================== + +The Jetty Web Container is Copyright Mort Bay Consulting Pty Ltd +unless otherwise noted. + +Jetty is dual licensed under both + + * The Apache 2.0 License + http://www.apache.org/licenses/LICENSE-2.0.html + + and + + * The Eclipse Public 1.0 License + http://www.eclipse.org/legal/epl-v10.html + +Jetty may be distributed under either license. + +------ +Eclipse + +The following artifacts are EPL. + * org.eclipse.jetty.orbit:org.eclipse.jdt.core + +The following artifacts are EPL and ASL2. + * org.eclipse.jetty.orbit:javax.security.auth.message + + +The following artifacts are EPL and CDDL 1.0. + * org.eclipse.jetty.orbit:javax.mail.glassfish + + +------ +Oracle + +The following artifacts are CDDL + GPLv2 with classpath exception. +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + + * javax.servlet:javax.servlet-api + * javax.annotation:javax.annotation-api + * javax.transaction:javax.transaction-api + * javax.websocket:javax.websocket-api + +------ +Oracle OpenJDK + +If ALPN is used to negotiate HTTP/2 connections, then the following +artifacts may be included in the distribution or downloaded when ALPN +module is selected. + + * java.sun.security.ssl + +These artifacts replace/modify OpenJDK classes. The modififications +are hosted at github and both modified and original are under GPL v2 with +classpath exceptions. +http://openjdk.java.net/legal/gplv2+ce.html + + +------ +OW2 + +The following artifacts are licensed by the OW2 Foundation according to the +terms of http://asm.ow2.org/license.html + +org.ow2.asm:asm-commons +org.ow2.asm:asm + + +------ +Apache + +The following artifacts are ASL2 licensed. + +org.apache.taglibs:taglibs-standard-spec +org.apache.taglibs:taglibs-standard-impl + + +------ +MortBay + +The following artifacts are ASL2 licensed. Based on selected classes from +following Apache Tomcat jars, all ASL2 licensed. + +org.mortbay.jasper:apache-jsp + org.apache.tomcat:tomcat-jasper + org.apache.tomcat:tomcat-juli + org.apache.tomcat:tomcat-jsp-api + org.apache.tomcat:tomcat-el-api + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-api + org.apache.tomcat:tomcat-util-scan + org.apache.tomcat:tomcat-util + +org.mortbay.jasper:apache-el + org.apache.tomcat:tomcat-jasper-el + org.apache.tomcat:tomcat-el-api + + +------ +Mortbay + +The following artifacts are CDDL + GPLv2 with classpath exception. + +https://glassfish.dev.java.net/nonav/public/CDDL+GPL.html + +org.eclipse.jetty.toolchain:jetty-schemas + +------ +Assorted + +The UnixCrypt.java code implements the one way cryptography used by +Unix systems for simple password protection. Copyright 1996 Aki Yoshida, +modified April 2001 by Iris Van den Broeke, Daniel Deville. +Permission to use, copy, modify and distribute UnixCrypt +for non-commercial or commercial purposes and without fee is +granted provided that the copyright notice appears in all copies. + +======================================================================= + +rocketmq-common-4.9.5 NOTICE + +======================================================================= + + +rocketmq-common 4.9.5 +Copyright 2012-2023 Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + + +======================================================================= + +netty-tcnative-boringssl-static-2.0.48.Final NOTICE + +======================================================================= + + The Netty Project + ================= + +Please visit the Netty web site for more information: + + * http://netty.io/ + +Copyright 2016 The Netty Project + +The Netty Project licenses this file to you under the Apache License, +version 2.0 (the "License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at: + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +License for the specific language governing permissions and limitations +under the License. + +------------------------------------------------------------------------------- +This product contains a forked and modified version of Tomcat Native + + * LICENSE: + * license/LICENSE.tomcat-native.txt (Apache License 2.0) + * HOMEPAGE: + * http://tomcat.apache.org/native-doc/ + * https://svn.apache.org/repos/asf/tomcat/native/ + +This product contains the Maven wrapper scripts from 'Maven Wrapper', that provides an easy way to ensure a user has everything necessary to run the Maven build. + + * LICENSE: + * license/LICENSE.mvn-wrapper.txt (Apache License 2.0) + * HOMEPAGE: + * https://github.com/takari/maven-wrapper + +This product contains small piece of code to support AIX, taken from netbsd. + + * LICENSE: + * license/LICENSE.aix-netbsd.txt (OpenSSL License) + * HOMEPAGE: + * https://ftp.netbsd.org/pub/NetBSD/NetBSD-current/src/crypto/external/bsd/openssl/dist + + +This product contains code from boringssl. + + * LICENSE (Combination ISC and OpenSSL license) + * license/LICENSE.boringssl.txt (Combination ISC and OpenSSL license) + * HOMEPAGE: + * https://boringssl.googlesource.com/boringssl/ + +======================================================================= + +s3-2.29.5 NOTICE + +======================================================================= + +AWS SDK for Java 2.0 +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + +This product includes software developed by +Amazon Technologies, Inc (http://www.amazon.com/). + +********************** +THIRD PARTY COMPONENTS +********************** +This software includes third party software subject to the following copyrights: +- XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty. +- PKCS#1 PEM encoded private key parsing and utility functions from oauth.googlecode.com - Copyright 1998-2010 AOL Inc. +- Apache Commons Lang - https://github.com/apache/commons-lang +- Netty Reactive Streams - https://github.com/playframework/netty-reactive-streams +- Jackson-core - https://github.com/FasterXML/jackson-core +- Jackson-dataformat-cbor - https://github.com/FasterXML/jackson-dataformats-binary + +The licenses for these third party components are included in LICENSE.txt + +- For Apache Commons Lang see also this required NOTICE: + Apache Commons Lang + Copyright 2001-2020 The Apache Software Foundation + + This product includes software developed at + The Apache Software Foundation (https://www.apache.org/). + +======================================================================= + +pulsar-client-2.11.4 NOTICE + +======================================================================= + + +Pulsar Client Java +Copyright 2017-2020 Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + + +======================================================================= + +pulsar-client-2.11.4 NOTICE + +======================================================================= + +Apache Commons Lang +Copyright 2001-2020 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (https://www.apache.org/). + +======================================================================= + +httpmime-4.5.13 NOTICE + +======================================================================= + + +Apache HttpClient Mime +Copyright 1999-2020 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + + +======================================================================= + +spring-context-5.3.15 NOTICE + +======================================================================= + +Spring Framework 5.3.15 +Copyright (c) 2002-2022 Pivotal, Inc. + +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. + +This product may include a number of subcomponents with separate +copyright notices and license terms. Your use of the source code for +these subcomponents is subject to the terms and conditions of the +subcomponent's license, as noted in the license.txt file. + +======================================================================= + +spring-expression-5.3.31 NOTICE + +======================================================================= + +Spring Framework 5.3.31 +Copyright (c) 2002-2023 Pivotal, Inc. + +This product is licensed to you under the Apache License, Version 2.0 +(the "License"). You may not use this product except in compliance with +the License. + +This product may include a number of subcomponents with separate +copyright notices and license terms. Your use of the source code for +these subcomponents is subject to the terms and conditions of the +subcomponent's license, as noted in the license.txt file. + +======================================================================= + +jackson-core-2.18.0 NOTICE + +======================================================================= + +# Jackson JSON processor + +Jackson is a high-performance, Free/Open Source JSON processing library. +It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has +been in development since 2007. +It is currently developed by a community of developers. + +## Copyright + +Copyright 2007-, Tatu Saloranta (tatu.saloranta@iki.fi) + +## Licensing + +Jackson 2.x core and extension components are licensed under Apache License 2.0 +To find the details that apply to this artifact see the accompanying LICENSE file. + +## Credits + +A list of contributors may be found from CREDITS(-2.x) file, which is included +in some artifacts (usually source distributions); but is always available +from the source code management (SCM) system project uses. + +## FastDoubleParser + +jackson-core bundles a shaded copy of FastDoubleParser . +That code is available under an MIT license +under the following copyright. + +Copyright © 2023 Werner Randelshofer, Switzerland. MIT License. + +See FastDoubleParser-NOTICE for details of other source code included in FastDoubleParser +and the licenses and copyrights that apply to that code. + +======================================================================= + +rocketmq-filter-4.9.5 NOTICE + +======================================================================= + + +rocketmq-filter 4.9.5 +Copyright 2012-2023 Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + + diff --git a/tools/dist-license/licenses/java/AL 2.0-downloaded-LICENSE-2.0.html b/tools/dist-license/licenses/java/AL 2.0-downloaded-LICENSE-2.0.html index 5841d517ad..7ec517ea7f 100644 --- a/tools/dist-license/licenses/java/AL 2.0-downloaded-LICENSE-2.0.html +++ b/tools/dist-license/licenses/java/AL 2.0-downloaded-LICENSE-2.0.html @@ -31,6 +31,7 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +

+ + +
+ +
+
+
+
+
+
+
+
+ +
+ +

Eclipse Public License -v 1.0

+ Version 1.0Submitted: March 8, 2004Submitter: Philip Ma + SPDX short identifier: + EPL-1.0 +

+
+ Steward:
+ + + + Link to license steward's version + +
+
+
+ Open Source Initiative Approved License +
+
+
+
+
+ +
+
+

THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC LICENSE (“AGREEMENT”). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM CONSTITUTES RECIPIENT’S ACCEPTANCE OF THIS AGREEMENT.

+

1. DEFINITIONS

+

“Contribution” means:

+

a) in the case of the initial Contributor, the initial code and documentation distributed under this Agreement, and

+

b) in the case of each subsequent Contributor:

+

i) changes to the Program, and

+

ii) additions to the Program;

+

where such changes and/or additions to the Program originate from and are distributed by that particular Contributor. A Contribution ‘originates’ from a Contributor if it was added to the Program by such Contributor itself or anyone acting on such Contributor’s behalf. Contributions do not include additions to the Program which: (i) are separate modules of software distributed in conjunction with the Program under their own license agreement, and (ii) are not derivative works of the Program.

+

“Contributor” means any person or entity that distributes the Program.

+

“Licensed Patents ” mean patent claims licensable by a Contributor which are necessarily infringed by the use or sale of its Contribution alone or when combined with the Program.

+

“Program” means the Contributions distributed in accordance with this Agreement.

+

“Recipient” means anyone who receives the Program under this Agreement, including all Contributors.

+

2. GRANT OF RIGHTS

+

a) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, distribute and sublicense the Contribution of such Contributor, if any, and such derivative works, in source code and object code form.

+

b) Subject to the terms of this Agreement, each Contributor hereby grants Recipient a non-exclusive, worldwide, royalty-free patent license under Licensed Patents to make, use, sell, offer to sell, import and otherwise transfer the Contribution of such Contributor, if any, in source code and object code form. This patent license shall apply to the combination of the Contribution and the Program if, at the time the Contribution is added by the Contributor, such addition of the Contribution causes such combination to be covered by the Licensed Patents. The patent license shall not apply to any other combinations which include the Contribution. No hardware per se is licensed hereunder.

+

c) Recipient understands that although each Contributor grants the licenses to its Contributions set forth herein, no assurances are provided by any Contributor that the Program does not infringe the patent or other intellectual property rights of any other entity. Each Contributor disclaims any liability to Recipient for claims brought by any other entity based on infringement of intellectual property rights or otherwise. As a condition to exercising the rights and licenses granted hereunder, each Recipient hereby assumes sole responsibility to secure any other intellectual property rights needed, if any. For example, if a third party patent license is required to allow Recipient to distribute the Program, it is Recipient’s responsibility to acquire that license before distributing the Program.

+

d) Each Contributor represents that to its knowledge it has sufficient copyright rights in its Contribution, if any, to grant the copyright license set forth in this Agreement.

+

3. REQUIREMENTS

+

A Contributor may choose to distribute the Program in object code form under its own license agreement, provided that:

+

a) it complies with the terms and conditions of this Agreement; and

+

b) its license agreement:

+

i) effectively disclaims on behalf of all Contributors all warranties and conditions, express and implied, including warranties or conditions of title and non-infringement, and implied warranties or conditions of merchantability and fitness for a particular purpose;

+

ii) effectively excludes on behalf of all Contributors all liability for damages, including direct, indirect, special, incidental and consequential damages, such as lost profits;

+

iii) states that any provisions which differ from this Agreement are offered by that Contributor alone and not by any other party; and

+

iv) states that source code for the Program is available from such Contributor, and informs licensees how to obtain it in a reasonable manner on or through a medium customarily used for software exchange.

+

When the Program is made available in source code form:

+

a) it must be made available under this Agreement; and

+

b) a copy of this Agreement must be included with each copy of the Program.

+

Contributors may not remove or alter any copyright notices contained within the Program.

+

Each Contributor must identify itself as the originator of its Contribution, if any, in a manner that reasonably allows subsequent Recipients to identify the originator of the Contribution.

+

4. COMMERCIAL DISTRIBUTION

+

Commercial distributors of software may accept certain responsibilities with respect to end users, business partners and the like. While this license is intended to facilitate the commercial use of the Program, the Contributor who includes the Program in a commercial product offering should do so in a manner which does not create potential liability for other Contributors. Therefore, if a Contributor includes the Program in a commercial product offering, such Contributor (“Commercial Contributor”) hereby agrees to defend and indemnify every other Contributor (“Indemnified Contributor”) against any losses, damages and costs (collectively “Losses”) arising from claims, lawsuits and other legal actions brought by a third party against the Indemnified Contributor to the extent caused by the acts or omissions of such Commercial Contributor in connection with its distribution of the Program in a commercial product offering. The obligations in this section do not apply to any claims or Losses relating to any actual or alleged intellectual property infringement. In order to qualify, an Indemnified Contributor must: a) promptly notify the Commercial Contributor in writing of such claim, and b) allow the Commercial Contributor to control, and cooperate with the Commercial Contributor in, the defense and any related settlement negotiations. The Indemnified Contributor may participate in any such claim at its own expense.

+

For example, a Contributor might include the Program in a commercial product offering, Product X. That Contributor is then a Commercial Contributor. If that Commercial Contributor then makes performance claims, or offers warranties related to Product X, those performance claims and warranties are such Commercial Contributor’s responsibility alone. Under this section, the Commercial Contributor would have to defend claims against the other Contributors related to those performance claims and warranties, and if a court requires any other Contributor to pay any damages as a result, the Commercial Contributor must pay those damages.

+

5. NO WARRANTY

+

EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON AN “AS IS” BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the appropriateness of using and distributing the Program and assumes all risks associated with its exercise of rights under this Agreement , including but not limited to the risks and costs of program errors, compliance with applicable laws, damage to or loss of data, programs or equipment, and unavailability or interruption of operations.

+

6. DISCLAIMER OF LIABILITY

+

EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.

+

7. GENERAL

+

If any provision of this Agreement is invalid or unenforceable under applicable law, it shall not affect the validity or enforceability of the remainder of the terms of this Agreement, and without further action by the parties hereto, such provision shall be reformed to the minimum extent necessary to make such provision valid and enforceable.

+

If Recipient institutes patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Program itself (excluding combinations of the Program with other software or hardware) infringes such Recipient’s patent(s), then such Recipient’s rights granted under Section 2(b) shall terminate as of the date such litigation is filed.

+

All Recipient’s rights under this Agreement shall terminate if it fails to comply with any of the material terms or conditions of this Agreement and does not cure such failure in a reasonable period of time after becoming aware of such noncompliance. If all Recipient’s rights under this Agreement terminate, Recipient agrees to cease use and distribution of the Program as soon as reasonably practicable. However, Recipient’s obligations under this Agreement and any licenses granted by Recipient relating to the Program shall continue and survive.

+

Everyone is permitted to copy and distribute copies of this Agreement, but in order to avoid inconsistency the Agreement is copyrighted and may only be modified in the following manner. The Agreement Steward reserves the right to publish new versions (including revisions) of this Agreement from time to time. No one other than the Agreement Steward has the right to modify this Agreement. The Eclipse Foundation is the initial Agreement Steward. The Eclipse Foundation may assign the responsibility to serve as the Agreement Steward to a suitable separate entity. Each new version of the Agreement will be given a distinguishing version number. The Program (including Contributions) may always be distributed subject to the version of the Agreement under which it was received. In addition, after a new version of the Agreement is published, Contributor may elect to distribute the Program (including its Contributions) under the new version. Except as expressly stated in Sections 2(a) and 2(b) above, Recipient receives no rights or licenses to the intellectual property of any Contributor under this Agreement, whether expressly, by implication, estoppel or otherwise. All rights in the Program not expressly granted under this Agreement are reserved.

+

This Agreement is governed by the laws of the State of New York and the intellectual property laws of the United States of America. No party to this Agreement will bring a legal action under this Agreement more than one year after the cause of action arose. Each party waives its rights to a jury trial in any resulting litigation.

+ +
+
+
+
+ + +
+ + + +
+
+
+ + +
+ + + + +
+ + + + +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tools/dist-license/licenses/java/EPL 1.0.txt b/tools/dist-license/licenses/java/EPL 1.0.txt new file mode 100644 index 0000000000..dfa398f7fe --- /dev/null +++ b/tools/dist-license/licenses/java/EPL 1.0.txt @@ -0,0 +1 @@ +https://opensource.org/licenses/eclipse-1.0.php \ No newline at end of file diff --git a/tools/dist-license/licenses/java/GNU General Public Library-downloaded-gpl.txt b/tools/dist-license/licenses/java/GNU General Public Library-downloaded-gpl.txt deleted file mode 100644 index f288702d2f..0000000000 --- a/tools/dist-license/licenses/java/GNU General Public Library-downloaded-gpl.txt +++ /dev/null @@ -1,674 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 3, 29 June 2007 - - Copyright (C) 2007 Free Software Foundation, Inc. - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The GNU General Public License is a free, copyleft license for -software and other kinds of works. - - The licenses for most software and other practical works are designed -to take away your freedom to share and change the works. By contrast, -the GNU General Public License is intended to guarantee your freedom to -share and change all versions of a program--to make sure it remains free -software for all its users. We, the Free Software Foundation, use the -GNU General Public License for most of our software; it applies also to -any other work released this way by its authors. You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -them if you wish), that you receive source code or can get it if you -want it, that you can change the software or use pieces of it in new -free programs, and that you know you can do these things. - - To protect your rights, we need to prevent others from denying you -these rights or asking you to surrender the rights. Therefore, you have -certain responsibilities if you distribute copies of the software, or if -you modify it: responsibilities to respect the freedom of others. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must pass on to the recipients the same -freedoms that you received. You must make sure that they, too, receive -or can get the source code. And you must show them these terms so they -know their rights. - - Developers that use the GNU GPL protect your rights with two steps: -(1) assert copyright on the software, and (2) offer you this License -giving you legal permission to copy, distribute and/or modify it. - - For the developers' and authors' protection, the GPL clearly explains -that there is no warranty for this free software. For both users' and -authors' sake, the GPL requires that modified versions be marked as -changed, so that their problems will not be attributed erroneously to -authors of previous versions. - - Some devices are designed to deny users access to install or run -modified versions of the software inside them, although the manufacturer -can do so. This is fundamentally incompatible with the aim of -protecting users' freedom to change the software. The systematic -pattern of such abuse occurs in the area of products for individuals to -use, which is precisely where it is most unacceptable. Therefore, we -have designed this version of the GPL to prohibit the practice for those -products. If such problems arise substantially in other domains, we -stand ready to extend this provision to those domains in future versions -of the GPL, as needed to protect the freedom of users. - - Finally, every program is threatened constantly by software patents. -States should not allow patents to restrict development and use of -software on general-purpose computers, but in those that do, we wish to -avoid the special danger that patents applied to a free program could -make it effectively proprietary. To prevent this, the GPL assures that -patents cannot be used to render the program non-free. - - The precise terms and conditions for copying, distribution and -modification follow. - - TERMS AND CONDITIONS - - 0. Definitions. - - "This License" refers to version 3 of the GNU General Public License. - - "Copyright" also means copyright-like laws that apply to other kinds of -works, such as semiconductor masks. - - "The Program" refers to any copyrightable work licensed under this -License. Each licensee is addressed as "you". "Licensees" and -"recipients" may be individuals or organizations. - - To "modify" a work means to copy from or adapt all or part of the work -in a fashion requiring copyright permission, other than the making of an -exact copy. The resulting work is called a "modified version" of the -earlier work or a work "based on" the earlier work. - - A "covered work" means either the unmodified Program or a work based -on the Program. - - To "propagate" a work means to do anything with it that, without -permission, would make you directly or secondarily liable for -infringement under applicable copyright law, except executing it on a -computer or modifying a private copy. Propagation includes copying, -distribution (with or without modification), making available to the -public, and in some countries other activities as well. - - To "convey" a work means any kind of propagation that enables other -parties to make or receive copies. Mere interaction with a user through -a computer network, with no transfer of a copy, is not conveying. - - An interactive user interface displays "Appropriate Legal Notices" -to the extent that it includes a convenient and prominently visible -feature that (1) displays an appropriate copyright notice, and (2) -tells the user that there is no warranty for the work (except to the -extent that warranties are provided), that licensees may convey the -work under this License, and how to view a copy of this License. If -the interface presents a list of user commands or options, such as a -menu, a prominent item in the list meets this criterion. - - 1. Source Code. - - The "source code" for a work means the preferred form of the work -for making modifications to it. "Object code" means any non-source -form of a work. - - A "Standard Interface" means an interface that either is an official -standard defined by a recognized standards body, or, in the case of -interfaces specified for a particular programming language, one that -is widely used among developers working in that language. - - The "System Libraries" of an executable work include anything, other -than the work as a whole, that (a) is included in the normal form of -packaging a Major Component, but which is not part of that Major -Component, and (b) serves only to enable use of the work with that -Major Component, or to implement a Standard Interface for which an -implementation is available to the public in source code form. A -"Major Component", in this context, means a major essential component -(kernel, window system, and so on) of the specific operating system -(if any) on which the executable work runs, or a compiler used to -produce the work, or an object code interpreter used to run it. - - The "Corresponding Source" for a work in object code form means all -the source code needed to generate, install, and (for an executable -work) run the object code and to modify the work, including scripts to -control those activities. However, it does not include the work's -System Libraries, or general-purpose tools or generally available free -programs which are used unmodified in performing those activities but -which are not part of the work. For example, Corresponding Source -includes interface definition files associated with source files for -the work, and the source code for shared libraries and dynamically -linked subprograms that the work is specifically designed to require, -such as by intimate data communication or control flow between those -subprograms and other parts of the work. - - The Corresponding Source need not include anything that users -can regenerate automatically from other parts of the Corresponding -Source. - - The Corresponding Source for a work in source code form is that -same work. - - 2. Basic Permissions. - - All rights granted under this License are granted for the term of -copyright on the Program, and are irrevocable provided the stated -conditions are met. This License explicitly affirms your unlimited -permission to run the unmodified Program. The output from running a -covered work is covered by this License only if the output, given its -content, constitutes a covered work. This License acknowledges your -rights of fair use or other equivalent, as provided by copyright law. - - You may make, run and propagate covered works that you do not -convey, without conditions so long as your license otherwise remains -in force. You may convey covered works to others for the sole purpose -of having them make modifications exclusively for you, or provide you -with facilities for running those works, provided that you comply with -the terms of this License in conveying all material for which you do -not control copyright. Those thus making or running the covered works -for you must do so exclusively on your behalf, under your direction -and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - - Conveying under any other circumstances is permitted solely under -the conditions stated below. Sublicensing is not allowed; section 10 -makes it unnecessary. - - 3. Protecting Users' Legal Rights From Anti-Circumvention Law. - - No covered work shall be deemed part of an effective technological -measure under any applicable law fulfilling obligations under article -11 of the WIPO copyright treaty adopted on 20 December 1996, or -similar laws prohibiting or restricting circumvention of such -measures. - - When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention -is effected by exercising rights under this License with respect to -the covered work, and you disclaim any intention to limit operation or -modification of the work as a means of enforcing, against the work's -users, your or third parties' legal rights to forbid circumvention of -technological measures. - - 4. Conveying Verbatim Copies. - - You may convey verbatim copies of the Program's source code as you -receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice; -keep intact all notices stating that this License and any -non-permissive terms added in accord with section 7 apply to the code; -keep intact all notices of the absence of any warranty; and give all -recipients a copy of this License along with the Program. - - You may charge any price or no price for each copy that you convey, -and you may offer support or warranty protection for a fee. - - 5. Conveying Modified Source Versions. - - You may convey a work based on the Program, or the modifications to -produce it from the Program, in the form of source code under the -terms of section 4, provided that you also meet all of these conditions: - - a) The work must carry prominent notices stating that you modified - it, and giving a relevant date. - - b) The work must carry prominent notices stating that it is - released under this License and any conditions added under section - 7. This requirement modifies the requirement in section 4 to - "keep intact all notices". - - c) You must license the entire work, as a whole, under this - License to anyone who comes into possession of a copy. This - License will therefore apply, along with any applicable section 7 - additional terms, to the whole of the work, and all its parts, - regardless of how they are packaged. This License gives no - permission to license the work in any other way, but it does not - invalidate such permission if you have separately received it. - - d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your - work need not make them do so. - - A compilation of a covered work with other separate and independent -works, which are not by their nature extensions of the covered work, -and which are not combined with it such as to form a larger program, -in or on a volume of a storage or distribution medium, is called an -"aggregate" if the compilation and its resulting copyright are not -used to limit the access or legal rights of the compilation's users -beyond what the individual works permit. Inclusion of a covered work -in an aggregate does not cause this License to apply to the other -parts of the aggregate. - - 6. Conveying Non-Source Forms. - - You may convey a covered work in object code form under the terms -of sections 4 and 5, provided that you also convey the -machine-readable Corresponding Source under the terms of this License, -in one of these ways: - - a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium - customarily used for software interchange. - - b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a - written offer, valid for at least three years and valid for as - long as you offer spare parts or customer support for that product - model, to give anyone who possesses the object code either (1) a - copy of the Corresponding Source for all the software in the - product that is covered by this License, on a durable physical - medium customarily used for software interchange, for a price no - more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the - Corresponding Source from a network server at no charge. - - c) Convey individual copies of the object code with a copy of the - written offer to provide the Corresponding Source. This - alternative is allowed only occasionally and noncommercially, and - only if you received the object code with such an offer, in accord - with subsection 6b. - - d) Convey the object code by offering access from a designated - place (gratis or for a charge), and offer equivalent access to the - Corresponding Source in the same way through the same place at no - further charge. You need not require recipients to copy the - Corresponding Source along with the object code. If the place to - copy the object code is a network server, the Corresponding Source - may be on a different server (operated by you or a third party) - that supports equivalent copying facilities, provided you maintain - clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the - Corresponding Source, you remain obligated to ensure that it is - available for as long as needed to satisfy these requirements. - - e) Convey the object code using peer-to-peer transmission, provided - you inform other peers where the object code and Corresponding - Source of the work are being offered to the general public at no - charge under subsection 6d. - - A separable portion of the object code, whose source code is excluded -from the Corresponding Source as a System Library, need not be -included in conveying the object code work. - - A "User Product" is either (1) a "consumer product", which means any -tangible personal property which is normally used for personal, family, -or household purposes, or (2) anything designed or sold for incorporation -into a dwelling. In determining whether a product is a consumer product, -doubtful cases shall be resolved in favor of coverage. For a particular -product received by a particular user, "normally used" refers to a -typical or common use of that class of product, regardless of the status -of the particular user or of the way in which the particular user -actually uses, or expects or is expected to use, the product. A product -is a consumer product regardless of whether the product has substantial -commercial, industrial or non-consumer uses, unless such uses represent -the only significant mode of use of the product. - - "Installation Information" for a User Product means any methods, -procedures, authorization keys, or other information required to install -and execute modified versions of a covered work in that User Product from -a modified version of its Corresponding Source. The information must -suffice to ensure that the continued functioning of the modified object -code is in no case prevented or interfered with solely because -modification has been made. - - If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as -part of a transaction in which the right of possession and use of the -User Product is transferred to the recipient in perpetuity or for a -fixed term (regardless of how the transaction is characterized), the -Corresponding Source conveyed under this section must be accompanied -by the Installation Information. But this requirement does not apply -if neither you nor any third party retains the ability to install -modified object code on the User Product (for example, the work has -been installed in ROM). - - The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates -for a work that has been modified or installed by the recipient, or for -the User Product in which it has been modified or installed. Access to a -network may be denied when the modification itself materially and -adversely affects the operation of the network or violates the rules and -protocols for communication across the network. - - Corresponding Source conveyed, and Installation Information provided, -in accord with this section must be in a format that is publicly -documented (and with an implementation available to the public in -source code form), and must require no special password or key for -unpacking, reading or copying. - - 7. Additional Terms. - - "Additional permissions" are terms that supplement the terms of this -License by making exceptions from one or more of its conditions. -Additional permissions that are applicable to the entire Program shall -be treated as though they were included in this License, to the extent -that they are valid under applicable law. If additional permissions -apply only to part of the Program, that part may be used separately -under those permissions, but the entire Program remains governed by -this License without regard to the additional permissions. - - When you convey a copy of a covered work, you may at your option -remove any additional permissions from that copy, or from any part of -it. (Additional permissions may be written to require their own -removal in certain cases when you modify the work.) You may place -additional permissions on material, added by you to a covered work, -for which you have or can give appropriate copyright permission. - - Notwithstanding any other provision of this License, for material you -add to a covered work, you may (if authorized by the copyright holders of -that material) supplement the terms of this License with terms: - - a) Disclaiming warranty or limiting liability differently from the - terms of sections 15 and 16 of this License; or - - b) Requiring preservation of specified reasonable legal notices or - author attributions in that material or in the Appropriate Legal - Notices displayed by works containing it; or - - c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in - reasonable ways as different from the original version; or - - d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - e) Declining to grant rights under trademark law for use of some - trade names, trademarks, or service marks; or - - f) Requiring indemnification of licensors and authors of that - material by anyone who conveys the material (or modified versions of - it) with contractual assumptions of liability to the recipient, for - any liability that these contractual assumptions directly impose on - those licensors and authors. - - All other non-permissive additional terms are considered "further -restrictions" within the meaning of section 10. If the Program as you -received it, or any part of it, contains a notice stating that it is -governed by this License along with a term that is a further -restriction, you may remove that term. If a license document contains -a further restriction but permits relicensing or conveying under this -License, you may add to a covered work material governed by the terms -of that license document, provided that the further restriction does -not survive such relicensing or conveying. - - If you add terms to a covered work in accord with this section, you -must place, in the relevant source files, a statement of the -additional terms that apply to those files, or a notice indicating -where to find the applicable terms. - - Additional terms, permissive or non-permissive, may be stated in the -form of a separately written license, or stated as exceptions; -the above requirements apply either way. - - 8. Termination. - - You may not propagate or modify a covered work except as expressly -provided under this License. Any attempt otherwise to propagate or -modify it is void, and will automatically terminate your rights under -this License (including any patent licenses granted under the third -paragraph of section 11). - - However, if you cease all violation of this License, then your -license from a particular copyright holder is reinstated (a) -provisionally, unless and until the copyright holder explicitly and -finally terminates your license, and (b) permanently, if the copyright -holder fails to notify you of the violation by some reasonable means -prior to 60 days after the cessation. - - Moreover, your license from a particular copyright holder is -reinstated permanently if the copyright holder notifies you of the -violation by some reasonable means, this is the first time you have -received notice of violation of this License (for any work) from that -copyright holder, and you cure the violation prior to 30 days after -your receipt of the notice. - - Termination of your rights under this section does not terminate the -licenses of parties who have received copies or rights from you under -this License. If your rights have been terminated and not permanently -reinstated, you do not qualify to receive new licenses for the same -material under section 10. - - 9. Acceptance Not Required for Having Copies. - - You are not required to accept this License in order to receive or -run a copy of the Program. Ancillary propagation of a covered work -occurring solely as a consequence of using peer-to-peer transmission -to receive a copy likewise does not require acceptance. However, -nothing other than this License grants you permission to propagate or -modify any covered work. These actions infringe copyright if you do -not accept this License. Therefore, by modifying or propagating a -covered work, you indicate your acceptance of this License to do so. - - 10. Automatic Licensing of Downstream Recipients. - - Each time you convey a covered work, the recipient automatically -receives a license from the original licensors, to run, modify and -propagate that work, subject to this License. You are not responsible -for enforcing compliance by third parties with this License. - - An "entity transaction" is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered -work results from an entity transaction, each party to that -transaction who receives a copy of the work also receives whatever -licenses to the work the party's predecessor in interest had or could -give under the previous paragraph, plus a right to possession of the -Corresponding Source of the work from the predecessor in interest, if -the predecessor has it or can get it with reasonable efforts. - - You may not impose any further restrictions on the exercise of the -rights granted or affirmed under this License. For example, you may -not impose a license fee, royalty, or other charge for exercise of -rights granted under this License, and you may not initiate litigation -(including a cross-claim or counterclaim in a lawsuit) alleging that -any patent claim is infringed by making, using, selling, offering for -sale, or importing the Program or any portion of it. - - 11. Patents. - - A "contributor" is a copyright holder who authorizes use under this -License of the Program or a work on which the Program is based. The -work thus licensed is called the contributor's "contributor version". - - A contributor's "essential patent claims" are all patent claims -owned or controlled by the contributor, whether already acquired or -hereafter acquired, that would be infringed by some manner, permitted -by this License, of making, using, or selling its contributor version, -but do not include claims that would be infringed only as a -consequence of further modification of the contributor version. For -purposes of this definition, "control" includes the right to grant -patent sublicenses in a manner consistent with the requirements of -this License. - - Each contributor grants you a non-exclusive, worldwide, royalty-free -patent license under the contributor's essential patent claims, to -make, use, sell, offer for sale, import and otherwise run, modify and -propagate the contents of its contributor version. - - In the following three paragraphs, a "patent license" is any express -agreement or commitment, however denominated, not to enforce a patent -(such as an express permission to practice a patent or covenant not to -sue for patent infringement). To "grant" such a patent license to a -party means to make such an agreement or commitment not to enforce a -patent against the party. - - If you convey a covered work, knowingly relying on a patent license, -and the Corresponding Source of the work is not available for anyone -to copy, free of charge and under the terms of this License, through a -publicly available network server or other readily accessible means, -then you must either (1) cause the Corresponding Source to be so -available, or (2) arrange to deprive yourself of the benefit of the -patent license for this particular work, or (3) arrange, in a manner -consistent with the requirements of this License, to extend the patent -license to downstream recipients. "Knowingly relying" means you have -actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work -in a country, would infringe one or more identifiable patents in that -country that you have reason to believe are valid. - - If, pursuant to or in connection with a single transaction or -arrangement, you convey, or propagate by procuring conveyance of, a -covered work, and grant a patent license to some of the parties -receiving the covered work authorizing them to use, propagate, modify -or convey a specific copy of the covered work, then the patent license -you grant is automatically extended to all recipients of the covered -work and works based on it. - - A patent license is "discriminatory" if it does not include within -the scope of its coverage, prohibits the exercise of, or is -conditioned on the non-exercise of one or more of the rights that are -specifically granted under this License. You may not convey a covered -work if you are a party to an arrangement with a third party that is -in the business of distributing software, under which you make payment -to the third party based on the extent of your activity of conveying -the work, and under which the third party grants, to any of the -parties who would receive the covered work from you, a discriminatory -patent license (a) in connection with copies of the covered work -conveyed by you (or copies made from those copies), or (b) primarily -for and in connection with specific products or compilations that -contain the covered work, unless you entered into that arrangement, -or that patent license was granted, prior to 28 March 2007. - - Nothing in this License shall be construed as excluding or limiting -any implied license or other defenses to infringement that may -otherwise be available to you under applicable patent law. - - 12. No Surrender of Others' Freedom. - - If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot convey a -covered work so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you may -not convey it at all. For example, if you agree to terms that obligate you -to collect a royalty for further conveying from those to whom you convey -the Program, the only way you could satisfy both those terms and this -License would be to refrain entirely from conveying the Program. - - 13. Use with the GNU Affero General Public License. - - Notwithstanding any other provision of this License, you have -permission to link or combine any covered work with a work licensed -under version 3 of the GNU Affero General Public License into a single -combined work, and to convey the resulting work. The terms of this -License will continue to apply to the part which is the covered work, -but the special requirements of the GNU Affero General Public License, -section 13, concerning interaction through a network will apply to the -combination as such. - - 14. Revised Versions of this License. - - The Free Software Foundation may publish revised and/or new versions of -the GNU General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - - Each version is given a distinguishing version number. If the -Program specifies that a certain numbered version of the GNU General -Public License "or any later version" applies to it, you have the -option of following the terms and conditions either of that numbered -version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of the -GNU General Public License, you may choose any version ever published -by the Free Software Foundation. - - If the Program specifies that a proxy can decide which future -versions of the GNU General Public License can be used, that proxy's -public statement of acceptance of a version permanently authorizes you -to choose that version for the Program. - - Later license versions may give you additional or different -permissions. However, no additional obligations are imposed on any -author or copyright holder as a result of your choosing to follow a -later version. - - 15. Disclaimer of Warranty. - - THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY -APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT -HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY -OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, -THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM -IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF -ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - - 16. Limitation of Liability. - - IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS -THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE -USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF -DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD -PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), -EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF -SUCH DAMAGES. - - 17. Interpretation of Sections 15 and 16. - - If the disclaimer of warranty and limitation of liability provided -above cannot be given local legal effect according to their terms, -reviewing courts shall apply local law that most closely approximates -an absolute waiver of all civil liability in connection with the -Program, unless a warranty or assumption of liability accompanies a -copy of the Program in return for a fee. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -state the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - - If the program does terminal interaction, make it output a short -notice like this when it starts in an interactive mode: - - Copyright (C) - This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands -might be different; for a GUI interface, you would use an "about box". - - You should also get your employer (if you work as a programmer) or school, -if any, to sign a "copyright disclaimer" for the program, if necessary. -For more information on this, and how to apply and follow the GNU GPL, see -. - - The GNU General Public License does not permit incorporating your program -into proprietary programs. If your program is a subroutine library, you -may consider it more useful to permit linking proprietary applications with -the library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. But first, please read -. diff --git a/tools/dist-license/licenses/java/GNU General Public License, version 2 with the GNU Classpath Exception-downloaded-secondary-gpl-2.0-cp b/tools/dist-license/licenses/java/GNU General Public License, version 2 with the GNU Classpath Exception-downloaded-secondary-gpl-2.0-cp new file mode 100644 index 0000000000..1f812fa951 --- /dev/null +++ b/tools/dist-license/licenses/java/GNU General Public License, version 2 with the GNU Classpath Exception-downloaded-secondary-gpl-2.0-cp @@ -0,0 +1,907 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 一 (Secondary) GNU General Public License, version 2 with the GNU Classpath Exception | projects.eclipse.org + + + + + + + + +
+ + +
+ +
+
+ + + +
+
+
+
+ +
+
+
+ + +
+
+
+
+
+ + +
+
+ +
+ +
+ + + +
+
+
+ +
+ Eclipse Foundation +
+ + + + +
+ Download + +
+
+
+
+ +
+ + + + + +
+ +
+ + +
+ + +
+ + + +
+
+ +
+ + +
+
+ + +
+ +
+ + + + +
+

+一 (Secondary) GNU General Public License, version 2 with the GNU Classpath Exception +

+ + +
+ + + + + +
+ +

This Source Code may also be made available under the following Secondary Licenses when the conditions for such availability set forth in the Eclipse Public License, v. 2.0 are satisfied: GNU General Public License, version 2 with the GNU Classpath Exception which is available at https://www.gnu.org/software/classpath/license.html.

+
+ +
+
URL
+
https://www.gnu.org/software/classpath/license.html
+
+ + + +
+ + + + + +
+ +
+ +
+ + +
+ +
+ + + +
+
+ + +
+
+ + + +
+
+
+
+ + + + +
+ + +
+ + + + + + + diff --git a/tools/dist-license/licenses/java/GNU General Public License, version 2 with the GNU Classpath Exception.txt b/tools/dist-license/licenses/java/GNU General Public License, version 2 with the GNU Classpath Exception.txt new file mode 100644 index 0000000000..37b86a6d9b --- /dev/null +++ b/tools/dist-license/licenses/java/GNU General Public License, version 2 with the GNU Classpath Exception.txt @@ -0,0 +1 @@ +https://projects.eclipse.org/license/secondary-gpl-2.0-cp \ No newline at end of file diff --git a/tools/dist-license/licenses/java/GNU General Public License, version 2.txt b/tools/dist-license/licenses/java/GNU General Public License, version 2.txt new file mode 100644 index 0000000000..5de3c76da4 --- /dev/null +++ b/tools/dist-license/licenses/java/GNU General Public License, version 2.txt @@ -0,0 +1 @@ +http://www.gnu.org/licenses/gpl-2.0.html \ No newline at end of file diff --git a/tools/dist-license/licenses/java/GNU LESSER GENERAL PUBLIC LICENSE.txt b/tools/dist-license/licenses/java/GNU LESSER GENERAL PUBLIC LICENSE.txt new file mode 100644 index 0000000000..6e4e7fc82e --- /dev/null +++ b/tools/dist-license/licenses/java/GNU LESSER GENERAL PUBLIC LICENSE.txt @@ -0,0 +1 @@ +http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html \ No newline at end of file diff --git a/tools/dist-license/licenses/java/GPL v2-downloaded-gpl-2.0.txt b/tools/dist-license/licenses/java/GPL v2-downloaded-gpl-2.0.txt deleted file mode 100644 index d159169d10..0000000000 --- a/tools/dist-license/licenses/java/GPL v2-downloaded-gpl-2.0.txt +++ /dev/null @@ -1,339 +0,0 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 - - Copyright (C) 1989, 1991 Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - Everyone is permitted to copy and distribute verbatim copies - of this license document, but changing it is not allowed. - - Preamble - - The licenses for most software are designed to take away your -freedom to share and change it. By contrast, the GNU General Public -License is intended to guarantee your freedom to share and change free -software--to make sure the software is free for all its users. This -General Public License applies to most of the Free Software -Foundation's software and to any other program whose authors commit to -using it. (Some other Free Software Foundation software is covered by -the GNU Lesser General Public License instead.) You can apply it to -your programs, too. - - When we speak of free software, we are referring to freedom, not -price. Our General Public Licenses are designed to make sure that you -have the freedom to distribute copies of free software (and charge for -this service if you wish), that you receive source code or can get it -if you want it, that you can change the software or use pieces of it -in new free programs; and that you know you can do these things. - - To protect your rights, we need to make restrictions that forbid -anyone to deny you these rights or to ask you to surrender the rights. -These restrictions translate to certain responsibilities for you if you -distribute copies of the software, or if you modify it. - - For example, if you distribute copies of such a program, whether -gratis or for a fee, you must give the recipients all the rights that -you have. You must make sure that they, too, receive or can get the -source code. And you must show them these terms so they know their -rights. - - We protect your rights with two steps: (1) copyright the software, and -(2) offer you this license which gives you legal permission to copy, -distribute and/or modify the software. - - Also, for each author's protection and ours, we want to make certain -that everyone understands that there is no warranty for this free -software. If the software is modified by someone else and passed on, we -want its recipients to know that what they have is not the original, so -that any problems introduced by others will not reflect on the original -authors' reputations. - - Finally, any free program is threatened constantly by software -patents. We wish to avoid the danger that redistributors of a free -program will individually obtain patent licenses, in effect making the -program proprietary. To prevent this, we have made it clear that any -patent must be licensed for everyone's free use or not licensed at all. - - The precise terms and conditions for copying, distribution and -modification follow. - - GNU GENERAL PUBLIC LICENSE - TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - - 0. This License applies to any program or other work which contains -a notice placed by the copyright holder saying it may be distributed -under the terms of this General Public License. The "Program", below, -refers to any such program or work, and a "work based on the Program" -means either the Program or any derivative work under copyright law: -that is to say, a work containing the Program or a portion of it, -either verbatim or with modifications and/or translated into another -language. (Hereinafter, translation is included without limitation in -the term "modification".) Each licensee is addressed as "you". - -Activities other than copying, distribution and modification are not -covered by this License; they are outside its scope. The act of -running the Program is not restricted, and the output from the Program -is covered only if its contents constitute a work based on the -Program (independent of having been made by running the Program). -Whether that is true depends on what the Program does. - - 1. You may copy and distribute verbatim copies of the Program's -source code as you receive it, in any medium, provided that you -conspicuously and appropriately publish on each copy an appropriate -copyright notice and disclaimer of warranty; keep intact all the -notices that refer to this License and to the absence of any warranty; -and give any other recipients of the Program a copy of this License -along with the Program. - -You may charge a fee for the physical act of transferring a copy, and -you may at your option offer warranty protection in exchange for a fee. - - 2. You may modify your copy or copies of the Program or any portion -of it, thus forming a work based on the Program, and copy and -distribute such modifications or work under the terms of Section 1 -above, provided that you also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices - stating that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in - whole or in part contains or is derived from the Program or any - part thereof, to be licensed as a whole at no charge to all third - parties under the terms of this License. - - c) If the modified program normally reads commands interactively - when run, you must cause it, when started running for such - interactive use in the most ordinary way, to print or display an - announcement including an appropriate copyright notice and a - notice that there is no warranty (or else, saying that you provide - a warranty) and that users may redistribute the program under - these conditions, and telling the user how to view a copy of this - License. (Exception: if the Program itself is interactive but - does not normally print such an announcement, your work based on - the Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If -identifiable sections of that work are not derived from the Program, -and can be reasonably considered independent and separate works in -themselves, then this License, and its terms, do not apply to those -sections when you distribute them as separate works. But when you -distribute the same sections as part of a whole which is a work based -on the Program, the distribution of the whole must be on the terms of -this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest -your rights to work written entirely by you; rather, the intent is to -exercise the right to control the distribution of derivative or -collective works based on the Program. - -In addition, mere aggregation of another work not based on the Program -with the Program (or with a work based on the Program) on a volume of -a storage or distribution medium does not bring the other work under -the scope of this License. - - 3. You may copy and distribute the Program (or a work based on it, -under Section 2) in object code or executable form under the terms of -Sections 1 and 2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable - source code, which must be distributed under the terms of Sections - 1 and 2 above on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three - years, to give any third party, for a charge no more than your - cost of physically performing source distribution, a complete - machine-readable copy of the corresponding source code, to be - distributed under the terms of Sections 1 and 2 above on a medium - customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer - to distribute corresponding source code. (This alternative is - allowed only for noncommercial distribution and only if you - received the program in object code or executable form with such - an offer, in accord with Subsection b above.) - -The source code for a work means the preferred form of the work for -making modifications to it. For an executable work, complete source -code means all the source code for all modules it contains, plus any -associated interface definition files, plus the scripts used to -control compilation and installation of the executable. However, as a -special exception, the source code distributed need not include -anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the -operating system on which the executable runs, unless that component -itself accompanies the executable. - -If distribution of executable or object code is made by offering -access to copy from a designated place, then offering equivalent -access to copy the source code from the same place counts as -distribution of the source code, even though third parties are not -compelled to copy the source along with the object code. - - 4. You may not copy, modify, sublicense, or distribute the Program -except as expressly provided under this License. Any attempt -otherwise to copy, modify, sublicense or distribute the Program is -void, and will automatically terminate your rights under this License. -However, parties who have received copies, or rights, from you under -this License will not have their licenses terminated so long as such -parties remain in full compliance. - - 5. You are not required to accept this License, since you have not -signed it. However, nothing else grants you permission to modify or -distribute the Program or its derivative works. These actions are -prohibited by law if you do not accept this License. Therefore, by -modifying or distributing the Program (or any work based on the -Program), you indicate your acceptance of this License to do so, and -all its terms and conditions for copying, distributing or modifying -the Program or works based on it. - - 6. Each time you redistribute the Program (or any work based on the -Program), the recipient automatically receives a license from the -original licensor to copy, distribute or modify the Program subject to -these terms and conditions. You may not impose any further -restrictions on the recipients' exercise of the rights granted herein. -You are not responsible for enforcing compliance by third parties to -this License. - - 7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not -excuse you from the conditions of this License. If you cannot -distribute so as to satisfy simultaneously your obligations under this -License and any other pertinent obligations, then as a consequence you -may not distribute the Program at all. For example, if a patent -license would not permit royalty-free redistribution of the Program by -all those who receive copies directly or indirectly through you, then -the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under -any particular circumstance, the balance of the section is intended to -apply and the section as a whole is intended to apply in other -circumstances. - -It is not the purpose of this section to induce you to infringe any -patents or other property right claims or to contest validity of any -such claims; this section has the sole purpose of protecting the -integrity of the free software distribution system, which is -implemented by public license practices. Many people have made -generous contributions to the wide range of software distributed -through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing -to distribute software through any other system and a licensee cannot -impose that choice. - -This section is intended to make thoroughly clear what is believed to -be a consequence of the rest of this License. - - 8. If the distribution and/or use of the Program is restricted in -certain countries either by patents or by copyrighted interfaces, the -original copyright holder who places the Program under this License -may add an explicit geographical distribution limitation excluding -those countries, so that distribution is permitted only in or among -countries not thus excluded. In such case, this License incorporates -the limitation as if written in the body of this License. - - 9. The Free Software Foundation may publish revised and/or new versions -of the General Public License from time to time. Such new versions will -be similar in spirit to the present version, but may differ in detail to -address new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any -later version", you have the option of following the terms and conditions -either of that version or of any later version published by the Free -Software Foundation. If the Program does not specify a version number of -this License, you may choose any version ever published by the Free Software -Foundation. - - 10. If you wish to incorporate parts of the Program into other free -programs whose distribution conditions are different, write to the author -to ask for permission. For software which is copyrighted by the Free -Software Foundation, write to the Free Software Foundation; we sometimes -make exceptions for this. Our decision will be guided by the two goals -of preserving the free status of all derivatives of our free software and -of promoting the sharing and reuse of software generally. - - NO WARRANTY - - 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY -FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN -OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES -PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED -OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS -TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE -PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, -REPAIR OR CORRECTION. - - 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING -WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR -REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, -INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING -OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED -TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY -YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER -PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE -POSSIBILITY OF SUCH DAMAGES. - - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs - - If you develop a new program, and you want it to be of the greatest -possible use to the public, the best way to achieve this is to make it -free software which everyone can redistribute and change under these terms. - - To do so, attach the following notices to the program. It is safest -to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least -the "copyright" line and a pointer to where the full notice is found. - - - Copyright (C) - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; either version 2 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this -when it starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author - Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. - This is free software, and you are welcome to redistribute it - under certain conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may -be called something other than `show w' and `show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - `Gnomovision' (which makes passes at compilers) written by James Hacker. - - , 1 April 1989 - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General -Public License instead of this License. diff --git a/tools/dist-license/licenses/java/LGPL, version 2.1-downloaded-licenses.html b/tools/dist-license/licenses/java/LGPL, version 2.1-downloaded-licenses.html deleted file mode 100644 index 923f7b4d4e..0000000000 --- a/tools/dist-license/licenses/java/LGPL, version 2.1-downloaded-licenses.html +++ /dev/null @@ -1,781 +0,0 @@ - - - - - - - - - - - - - - -Licenses -- GNU Project - Free Software Foundation - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
- - - - - - - - - - - -
- - - - -
- - -

Licenses

-
- -
-

Table of Contents

- -
-
- -

-Published software should be free -software. To make it free software, you need to release it under -a free software license. We normally use the GNU -General Public License (GNU GPL), specifying version 3 or any -later version, but occasionally we -use other free -software licenses. We use only licenses that are compatible with -the GNU GPL for GNU software. -

- -

-Documentation for free software should be -free documentation, so that -people can redistribute it and improve it along with the software -it describes. To make it free documentation, you need to release -it under a free documentation license. We normally use the -GNU Free Documentation License (GNU -FDL), but occasionally we use -other free -documentation licenses. -

- -

If you've started a new project and you're not sure what license to -use, “How to -choose a license for your own work” details our -recommendations in an easy-to-follow guide. If you just want a quick -list reference, we have a page that names -our recommended -copyleft licenses.

- -

We also have a page that discusses the BSD License Problem.

- -
-

Evaluating Licenses

- -

If you come across a license not mentioned in our -license list, you can ask us -to evaluate whether it is a free license. Please email a copy of the -license (and the URL where you found it) -to licensing@fsf.org. Our -licensing experts in the staff and the board of directors will review -it. If the license has some unusual conditions, they may pose -difficult philosophical problems, so we can't promise to decide -quickly.

- -

Common Resources for our Software Licenses

- -

We have a number of resources to help people understand and use our -various licenses:

- - - -

The GNU General Public License

- -

-The GNU General Public License is often called the GNU GPL for short; -it is used by most GNU programs, and by more than half of all free -software packages. The latest version is version 3. -

- - - -

The GNU Lesser General Public License

- -

-The GNU Lesser General Public License is used by a few (not by any means -all) GNU libraries. The latest version is version 3. -

- - - -

The GNU Affero General Public License

- -

-The GNU Affero General Public License is based on the GNU GPL, but has an -additional term to allow users who interact with the licensed software over -a network to receive the source for that program. We recommend that people -consider using the GNU AGPL for any software which will commonly be run -over a network. The latest version is version 3. -

- - - -

The GNU Free Documentation License

- -

-The GNU Free Documentation License is a form of copyleft intended -for use on a manual, textbook or other document to assure everyone -the effective freedom to copy and redistribute it, with or without -modifications, either commercially or non-commercially. The latest version -is 1.3. -

- - - -

Exceptions to GNU Licenses

- -

Some GNU programs have additional permissions or special exceptions - to specific terms in one of the main licenses. Since some of those - are commonly used or inspire a lot of questions on their own, we've - started collecting them on - our exceptions page.

- -

License URLs

- -

When linking to our licenses, it's usually best to link to the latest -version; hence the standard URLs such as -https://www.gnu.org/licenses/gpl.html have no version number. -Occasionally, however, you may want to link to a specific version of a -given license. In those situations, you can use the following links -[skip links]:

- -
-
GNU General Public License (GPL)
- -
GPLv3, -GPLv2, -GPLv1
- -
GNU Lesser General Public License (LGPL)
-
LGPLv3, -LGPLv2.1
- -
GNU Affero General Public License (AGPL)
-
GNU - AGPLv3 (The Affero General - Public License version 1 is not a GNU license, but it was - designed to serve a purpose much like the GNU AGPL's.)
- -
GNU Free Documentation License (FDL)
-
FDLv1.3, -FDLv1.2, -FDLv1.1
-
- -

Stable links to each license's alternative - formats are available on its respective page. Not every version of - every license is available in every format. If you need one that is - missing, please email us.

- -

See also the old licenses page.

- - -

Unofficial Translations

- -

-Legally speaking, the original (English) version of the licenses is -what specifies the actual distribution terms for GNU programs and -others that use them. But to help people better understand the -licenses, we give others permission to publish unofficial translations -into other languages, provided that they follow our regulations for -unofficial translations. -

- -

-The FSF does not approve license translations as officially valid. -The reason is that checking them would be difficult and expensive -(needing the help of bilingual lawyers in other countries). Even -worse, if an error did slip through, the results could be disastrous -for the whole free software community. As long as the translations -are unofficial, they can't do any legal harm.

- -

To underscore the fact that these translations are not officially -valid, we do not publish translations. To make that clear, we don't -post them on gnu.org, or on other GNU and FSF web sites; we only link -to them.

- - - -

Verbatim Copying and Distribution

- -

The standard copyright terms for GNU web pages is now the Creative -Commons Attribution-NoDerivs 4.0 International License. It used to -be (and for a few pages still is):

- -

Verbatim -copying and distribution of this entire article are permitted worldwide, -without royalty, in any medium, provided this notice is -preserved.

- -

Please note the following commentary about this -“verbatim license” by Eben Moglen:

- -

-“Our intention in using the phrase ‘verbatim copying in -any medium’ is not to require retention of page headings and -footers or other formatting features. Retention of weblinks in both -hyperlinked and non-hyperlinked media (as notes or some other form of -printed URL in non-HTML media) is required.” -

- -

List of Free Software Licenses

- -
    -
  • - List of Free Software Licenses - -

    If you are contemplating writing a new license, please contact the -FSF by writing to <licensing@fsf.org>. The -proliferation of different free software licenses means increased work -for users in understanding the licenses; we may be able to help you -find an existing Free Software license that meets your needs. -

    - -

    -If that isn't possible, if you really need a new license, with our -help you can ensure that the license really is a Free Software license -and avoid various practical problems. -

    - -
    • -
- - -

What Is Copyleft?

- -

-Copyleft is a general -method for making a program free -software and requiring all modified and extended versions of the -program to be free software as well. -

- -

-The simplest way to make a program free is to put it in the -public -domain, uncopyrighted. This allows people to share the program -and their improvements, if they are so minded. But it also allows -uncooperative people to convert the program into -proprietary -software. They can make changes, many or few, and distribute the -result as a proprietary product. People who receive the program in -that modified form do not have the freedom that the original author -gave them; the middleman has stripped it away. -

- -

-In the GNU project, our aim is -to give all users the freedom to redistribute and change GNU -software. If middlemen could strip off the freedom, we might have -many users, but those users would not have freedom. So instead of -putting GNU software in the public domain, we “copyleft” -it. Copyleft says that anyone who redistributes the software, with or -without changes, must pass along the freedom to further copy and -change it. Copyleft guarantees that every user has freedom. -

- -

-Copyleft also provides an -incentive -for other programmers to add to free software. -Important free programs such as the GNU C++ compiler exist -only because of this. -

- -

-Copyleft also helps programmers who want to contribute -improvements to -free software get permission to -do that. These programmers often work for companies or universities -that would do almost anything to get more money. A programmer may -want to contribute her changes to the community, but her employer may -want to turn the changes into a proprietary software product. -

- -

-When we explain to the employer that it is illegal to distribute the -improved version except as free software, the employer usually decides -to release it as free software rather than throw it away. -

- -

-To copyleft a program, we first state that it is copyrighted; then -we add distribution terms, which are a legal instrument that gives -everyone the rights to use, modify, and redistribute the program's -code or any program derived from it but only if the -distribution terms are unchanged. Thus, the code and the freedoms -become legally inseparable. -

- -

-Proprietary software developers use copyright to take away the users' -freedom; we use copyright to guarantee their freedom. That's why we -reverse the name, changing “copyright” into -“copyleft”. -

- -

-Copyleft is a general concept; there are many ways to fill in the -details. In the GNU Project, the specific distribution terms that we -use are contained in the GNU General Public License, the GNU Lesser -General Public License and the GNU Free Documentation License. -

- -

-The appropriate license is included in many manuals and in each GNU -source code distribution. -

- -

-The GNU GPL is designed so that you can easily apply it to your own -program if you are the copyright holder. You don't have to modify the -GNU GPL to do this, just add notices to your program which refer -properly to the GNU GPL. Please note that you must use the -entire text of the GPL, if you use it. It is an integral whole, and -partial copies are not permitted. (Likewise for the LGPL, AGPL, and FDL.) -

- -

-Using the same distribution terms for many different programs makes it -easy to copy code between various different programs. Since they all -have the same distribution terms, there is no need to think about -whether the terms are compatible. The Lesser GPL includes a -provision that lets you alter the distribution terms to the ordinary -GPL, so that you can copy code into another program covered by the GPL. -

- -

Licenses for Other Types of Works

- -

-We believe that published software and documentation should be -free software and free documentation. -We recommend making all sorts of educational and reference works free -also, using free documentation licenses such as the -GNU Free Documentation License (GNU FDL). -

- -

For essays of opinion and scientific papers, we recommend -either the Creative -Commons Attribution-NoDerivs 3.0 United States License, or the -simple “verbatim copying only” license stated above.

- -

-We don't take the position that artistic or entertainment works must -be free, but if you want to make one free, we recommend -the Free Art -License.

-
- -
- - - - - - - -
- - -
-
-
- ▲ -
-
-
- BACK TO TOP -
- -

Available for this page:

-
-

-[en] English   -[ar] العربية   -[ca] català   -[de] Deutsch   -[el] ελληνικά   -[es] español   -[fr] français   -[it] italiano   -[ja] 日本語   -[nl] Nederlands   -[pl] polski   -[pt-br] português   -[ru] русский   -[sq] Shqip   -[sr] српски   -[tr] Türkçe   -[zh-cn] 简体中文   -[zh-tw] 繁體中文   -

-
-
-
- - - - - - -
- - diff --git a/tools/dist-license/licenses/java/MIT-0.txt b/tools/dist-license/licenses/java/MIT-0.txt new file mode 100644 index 0000000000..a4e9dc9061 --- /dev/null +++ b/tools/dist-license/licenses/java/MIT-0.txt @@ -0,0 +1,16 @@ +MIT No Attribution + +Copyright + +Permission is hereby granted, free of charge, to any person obtaining a copy of this +software and associated documentation files (the "Software"), to deal in the Software +without restriction, including without limitation the rights to use, copy, modify, +merge, publish, distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/tools/dist-license/licenses/java/The Apache Software License, Version 1.1-downloaded-LICENSE-1.1.txt b/tools/dist-license/licenses/java/The Apache Software License, Version 1.1-downloaded-LICENSE-1.1.txt new file mode 100644 index 0000000000..8ee3701375 --- /dev/null +++ b/tools/dist-license/licenses/java/The Apache Software License, Version 1.1-downloaded-LICENSE-1.1.txt @@ -0,0 +1,65 @@ +=== An example Apache Software License 1.1 file === + +The 1.1 version of the Apache License was approved by the ASF in 2000. + +The primary change from the 1.0 license was in the removal of the 'advertising clause' (section 3 of the 1.0 license); derived products are no longer required to include attribution in their advertising materials, only in their documentation. + + +/* ==================================================================== + * The Apache Software License, Version 1.1 + * + * Copyright (c) 2000 The Apache Software Foundation. All rights + * reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. The end-user documentation included with the redistribution, + * if any, must include the following acknowledgment: + * "This product includes software developed by the + * Apache Software Foundation (http://www.apache.org/)." + * Alternately, this acknowledgment may appear in the software itself, + * if and wherever such third-party acknowledgments normally appear. + * + * 4. The names "Apache" and "Apache Software Foundation" must + * not be used to endorse or promote products derived from this + * software without prior written permission. For written + * permission, please contact apache@apache.org. + * + * 5. Products derived from this software may not be called "Apache", + * nor may "Apache" appear in their name, without prior written + * permission of the Apache Software Foundation. + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF + * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND + * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + * Portions of this software are based upon public domain software + * originally written at the National Center for Supercomputing Applications, + * University of Illinois, Urbana-Champaign. + */ + diff --git a/tools/dist-license/licenses/java/The Apache Software License, Version 1.1.txt b/tools/dist-license/licenses/java/The Apache Software License, Version 1.1.txt new file mode 100644 index 0000000000..519548056f --- /dev/null +++ b/tools/dist-license/licenses/java/The Apache Software License, Version 1.1.txt @@ -0,0 +1 @@ +http://www.apache.org/licenses/LICENSE-1.1.txt \ No newline at end of file diff --git a/tools/dist-license/licenses/java/The GNU General Public License, Version 2.txt b/tools/dist-license/licenses/java/The GNU General Public License, Version 2.txt new file mode 100644 index 0000000000..036b20977c --- /dev/null +++ b/tools/dist-license/licenses/java/The GNU General Public License, Version 2.txt @@ -0,0 +1 @@ +http://www.gnu.org/licenses/old-licenses/gpl-2.0.html \ No newline at end of file diff --git a/tools/dist-license/licenses/java/Unicode-ICU License-downloaded-LICENSE b/tools/dist-license/licenses/java/Unicode-ICU License-downloaded-LICENSE new file mode 100644 index 0000000000..180db98fcc --- /dev/null +++ b/tools/dist-license/licenses/java/Unicode-ICU License-downloaded-LICENSE @@ -0,0 +1,542 @@ +UNICODE LICENSE V3 + +COPYRIGHT AND PERMISSION NOTICE + +Copyright © 2016-2024 Unicode, Inc. + +NOTICE TO USER: Carefully read the following legal agreement. BY +DOWNLOADING, INSTALLING, COPYING OR OTHERWISE USING DATA FILES, AND/OR +SOFTWARE, YOU UNEQUIVOCALLY ACCEPT, AND AGREE TO BE BOUND BY, ALL OF THE +TERMS AND CONDITIONS OF THIS AGREEMENT. IF YOU DO NOT AGREE, DO NOT +DOWNLOAD, INSTALL, COPY, DISTRIBUTE OR USE THE DATA FILES OR SOFTWARE. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of data files and any associated documentation (the "Data Files") or +software and any associated documentation (the "Software") to deal in the +Data Files or Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, and/or sell +copies of the Data Files or Software, and to permit persons to whom the +Data Files or Software are furnished to do so, provided that either (a) +this copyright and permission notice appear with all copies of the Data +Files or Software, or (b) this copyright and permission notice appear in +associated Documentation. + +THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF +THIRD PARTY RIGHTS. + +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE +BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, +OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, +ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THE DATA +FILES OR SOFTWARE. + +Except as contained in this notice, the name of a copyright holder shall +not be used in advertising or otherwise to promote the sale, use or other +dealings in these Data Files or Software without prior written +authorization of the copyright holder. + +SPDX-License-Identifier: Unicode-3.0 + +---------------------------------------------------------------------- + +Third-Party Software Licenses + +This section contains third-party software notices and/or additional +terms for licensed third-party software components included within ICU +libraries. + +---------------------------------------------------------------------- + +ICU License - ICU 1.8.1 to ICU 57.1 + +COPYRIGHT AND PERMISSION NOTICE + +Copyright (c) 1995-2016 International Business Machines Corporation and others +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, provided that the above +copyright notice(s) and this permission notice appear in all copies of +the Software and that both the above copyright notice(s) and this +permission notice appear in supporting documentation. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY +SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER +RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF +CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +Except as contained in this notice, the name of a copyright holder +shall not be used in advertising or otherwise to promote the sale, use +or other dealings in this Software without prior written authorization +of the copyright holder. + +All trademarks and registered trademarks mentioned herein are the +property of their respective owners. + +---------------------------------------------------------------------- + +Chinese/Japanese Word Break Dictionary Data (cjdict.txt) + + # The Google Chrome software developed by Google is licensed under + # the BSD license. Other software included in this distribution is + # provided under other licenses, as set forth below. + # + # The BSD License + # http://opensource.org/licenses/bsd-license.php + # Copyright (C) 2006-2008, Google Inc. + # + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are met: + # + # Redistributions of source code must retain the above copyright notice, + # this list of conditions and the following disclaimer. + # Redistributions in binary form must reproduce the above + # copyright notice, this list of conditions and the following + # disclaimer in the documentation and/or other materials provided with + # the distribution. + # Neither the name of Google Inc. nor the names of its + # contributors may be used to endorse or promote products derived from + # this software without specific prior written permission. + # + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + # BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + # + # + # The word list in cjdict.txt are generated by combining three word lists + # listed below with further processing for compound word breaking. The + # frequency is generated with an iterative training against Google web + # corpora. + # + # * Libtabe (Chinese) + # - https://sourceforge.net/project/?group_id=1519 + # - Its license terms and conditions are shown below. + # + # * IPADIC (Japanese) + # - http://chasen.aist-nara.ac.jp/chasen/distribution.html + # - Its license terms and conditions are shown below. + # + # ---------COPYING.libtabe ---- BEGIN-------------------- + # + # /* + # * Copyright (c) 1999 TaBE Project. + # * Copyright (c) 1999 Pai-Hsiang Hsiao. + # * All rights reserved. + # * + # * Redistribution and use in source and binary forms, with or without + # * modification, are permitted provided that the following conditions + # * are met: + # * + # * . Redistributions of source code must retain the above copyright + # * notice, this list of conditions and the following disclaimer. + # * . Redistributions in binary form must reproduce the above copyright + # * notice, this list of conditions and the following disclaimer in + # * the documentation and/or other materials provided with the + # * distribution. + # * . Neither the name of the TaBE Project nor the names of its + # * contributors may be used to endorse or promote products derived + # * from this software without specific prior written permission. + # * + # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # * OF THE POSSIBILITY OF SUCH DAMAGE. + # */ + # + # /* + # * Copyright (c) 1999 Computer Systems and Communication Lab, + # * Institute of Information Science, Academia + # * Sinica. All rights reserved. + # * + # * Redistribution and use in source and binary forms, with or without + # * modification, are permitted provided that the following conditions + # * are met: + # * + # * . Redistributions of source code must retain the above copyright + # * notice, this list of conditions and the following disclaimer. + # * . Redistributions in binary form must reproduce the above copyright + # * notice, this list of conditions and the following disclaimer in + # * the documentation and/or other materials provided with the + # * distribution. + # * . Neither the name of the Computer Systems and Communication Lab + # * nor the names of its contributors may be used to endorse or + # * promote products derived from this software without specific + # * prior written permission. + # * + # * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + # * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # * OF THE POSSIBILITY OF SUCH DAMAGE. + # */ + # + # Copyright 1996 Chih-Hao Tsai @ Beckman Institute, + # University of Illinois + # c-tsai4@uiuc.edu http://casper.beckman.uiuc.edu/~c-tsai4 + # + # ---------------COPYING.libtabe-----END-------------------------------- + # + # + # ---------------COPYING.ipadic-----BEGIN------------------------------- + # + # Copyright 2000, 2001, 2002, 2003 Nara Institute of Science + # and Technology. All Rights Reserved. + # + # Use, reproduction, and distribution of this software is permitted. + # Any copy of this software, whether in its original form or modified, + # must include both the above copyright notice and the following + # paragraphs. + # + # Nara Institute of Science and Technology (NAIST), + # the copyright holders, disclaims all warranties with regard to this + # software, including all implied warranties of merchantability and + # fitness, in no event shall NAIST be liable for + # any special, indirect or consequential damages or any damages + # whatsoever resulting from loss of use, data or profits, whether in an + # action of contract, negligence or other tortuous action, arising out + # of or in connection with the use or performance of this software. + # + # A large portion of the dictionary entries + # originate from ICOT Free Software. The following conditions for ICOT + # Free Software applies to the current dictionary as well. + # + # Each User may also freely distribute the Program, whether in its + # original form or modified, to any third party or parties, PROVIDED + # that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear + # on, or be attached to, the Program, which is distributed substantially + # in the same form as set out herein and that such intended + # distribution, if actually made, will neither violate or otherwise + # contravene any of the laws and regulations of the countries having + # jurisdiction over the User or the intended distribution itself. + # + # NO WARRANTY + # + # The program was produced on an experimental basis in the course of the + # research and development conducted during the project and is provided + # to users as so produced on an experimental basis. Accordingly, the + # program is provided without any warranty whatsoever, whether express, + # implied, statutory or otherwise. The term "warranty" used herein + # includes, but is not limited to, any warranty of the quality, + # performance, merchantability and fitness for a particular purpose of + # the program and the nonexistence of any infringement or violation of + # any right of any third party. + # + # Each user of the program will agree and understand, and be deemed to + # have agreed and understood, that there is no warranty whatsoever for + # the program and, accordingly, the entire risk arising from or + # otherwise connected with the program is assumed by the user. + # + # Therefore, neither ICOT, the copyright holder, or any other + # organization that participated in or was otherwise related to the + # development of the program and their respective officials, directors, + # officers and other employees shall be held liable for any and all + # damages, including, without limitation, general, special, incidental + # and consequential damages, arising out of or otherwise in connection + # with the use or inability to use the program or any product, material + # or result produced or otherwise obtained by using the program, + # regardless of whether they have been advised of, or otherwise had + # knowledge of, the possibility of such damages at any time during the + # project or thereafter. Each user will be deemed to have agreed to the + # foregoing by his or her commencement of use of the program. The term + # "use" as used herein includes, but is not limited to, the use, + # modification, copying and distribution of the program and the + # production of secondary products from the program. + # + # In the case where the program, whether in its original form or + # modified, was distributed or delivered to or received by a user from + # any person, organization or entity other than ICOT, unless it makes or + # grants independently of ICOT any specific warranty to the user in + # writing, such person, organization or entity, will also be exempted + # from and not be held liable to the user for any such damages as noted + # above as far as the program is concerned. + # + # ---------------COPYING.ipadic-----END---------------------------------- + +---------------------------------------------------------------------- + +Lao Word Break Dictionary Data (laodict.txt) + + # Copyright (C) 2016 and later: Unicode, Inc. and others. + # License & terms of use: http://www.unicode.org/copyright.html + # Copyright (c) 2015 International Business Machines Corporation + # and others. All Rights Reserved. + # + # Project: https://github.com/rober42539/lao-dictionary + # Dictionary: https://github.com/rober42539/lao-dictionary/laodict.txt + # License: https://github.com/rober42539/lao-dictionary/LICENSE.txt + # (copied below) + # + # This file is derived from the above dictionary version of Nov 22, 2020 + # ---------------------------------------------------------------------- + # Copyright (C) 2013 Brian Eugene Wilson, Robert Martin Campbell. + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions are met: + # + # Redistributions of source code must retain the above copyright notice, this + # list of conditions and the following disclaimer. Redistributions in binary + # form must reproduce the above copyright notice, this list of conditions and + # the following disclaimer in the documentation and/or other materials + # provided with the distribution. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + # COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + # INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + # OF THE POSSIBILITY OF SUCH DAMAGE. + # -------------------------------------------------------------------------- + +---------------------------------------------------------------------- + +Burmese Word Break Dictionary Data (burmesedict.txt) + + # Copyright (c) 2014 International Business Machines Corporation + # and others. All Rights Reserved. + # + # This list is part of a project hosted at: + # github.com/kanyawtech/myanmar-karen-word-lists + # + # -------------------------------------------------------------------------- + # Copyright (c) 2013, LeRoy Benjamin Sharon + # All rights reserved. + # + # Redistribution and use in source and binary forms, with or without + # modification, are permitted provided that the following conditions + # are met: Redistributions of source code must retain the above + # copyright notice, this list of conditions and the following + # disclaimer. Redistributions in binary form must reproduce the + # above copyright notice, this list of conditions and the following + # disclaimer in the documentation and/or other materials provided + # with the distribution. + # + # Neither the name Myanmar Karen Word Lists, nor the names of its + # contributors may be used to endorse or promote products derived + # from this software without specific prior written permission. + # + # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS + # BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + # EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + # TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + # ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR + # TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + # THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + # SUCH DAMAGE. + # -------------------------------------------------------------------------- + +---------------------------------------------------------------------- + +Time Zone Database + + ICU uses the public domain data and code derived from Time Zone +Database for its time zone support. The ownership of the TZ database +is explained in BCP 175: Procedure for Maintaining the Time Zone +Database section 7. + + # 7. Database Ownership + # + # The TZ database itself is not an IETF Contribution or an IETF + # document. Rather it is a pre-existing and regularly updated work + # that is in the public domain, and is intended to remain in the + # public domain. Therefore, BCPs 78 [RFC5378] and 79 [RFC3979] do + # not apply to the TZ Database or contributions that individuals make + # to it. Should any claims be made and substantiated against the TZ + # Database, the organization that is providing the IANA + # Considerations defined in this RFC, under the memorandum of + # understanding with the IETF, currently ICANN, may act in accordance + # with all competent court orders. No ownership claims will be made + # by ICANN or the IETF Trust on the database or the code. Any person + # making a contribution to the database or code waives all rights to + # future claims in that contribution or in the TZ Database. + +---------------------------------------------------------------------- + +Google double-conversion + +Copyright 2006-2011, the V8 project authors. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +---------------------------------------------------------------------- + +JSON parsing library (nlohmann/json) + +File: vendor/json/upstream/single_include/nlohmann/json.hpp (only for ICU4C) + +MIT License + +Copyright (c) 2013-2022 Niels Lohmann + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +---------------------------------------------------------------------- + +File: aclocal.m4 (only for ICU4C) +Section: pkg.m4 - Macros to locate and utilise pkg-config. + + +Copyright © 2004 Scott James Remnant . +Copyright © 2012-2015 Dan Nicholson + +This program is free software; you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation; either version 2 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, write to the Free Software +Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA +02111-1307, USA. + +As a special exception to the GNU General Public License, if you +distribute this file as part of a program that contains a +configuration script generated by Autoconf, you may include it under +the same distribution terms that you use for the rest of that +program. + + +(The condition for the exception is fulfilled because +ICU4C includes a configuration script generated by Autoconf, +namely the `configure` script.) + +---------------------------------------------------------------------- + +File: config.guess (only for ICU4C) + + +This file is free software; you can redistribute it and/or modify it +under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program; if not, see . + +As a special exception to the GNU General Public License, if you +distribute this file as part of a program that contains a +configuration script generated by Autoconf, you may include it under +the same distribution terms that you use for the rest of that +program. This Exception is an additional permission under section 7 +of the GNU General Public License, version 3 ("GPLv3"). + + +(The condition for the exception is fulfilled because +ICU4C includes a configuration script generated by Autoconf, +namely the `configure` script.) + +---------------------------------------------------------------------- + +File: install-sh (only for ICU4C) + + +Copyright 1991 by the Massachusetts Institute of Technology + +Permission to use, copy, modify, distribute, and sell this software and its +documentation for any purpose is hereby granted without fee, provided that +the above copyright notice appear in all copies and that both that +copyright notice and this permission notice appear in supporting +documentation, and that the name of M.I.T. not be used in advertising or +publicity pertaining to distribution of the software without specific, +written prior permission. M.I.T. makes no representations about the +suitability of this software for any purpose. It is provided "as is" +without express or implied warranty.