From 468f7eac851153dfcc6d0ea904d7cb9e879d1b60 Mon Sep 17 00:00:00 2001 From: wangqi <1942460489@qq.com> Date: Wed, 16 Aug 2023 20:46:57 +0800 Subject: [PATCH 1/3] feature: update seatunnel Dockerfile and github actions --- .github/workflows/docker-seatunnel.yml | 2 +- tools/docker/build/scaleph-seatunnel/Dockerfile | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-seatunnel.yml b/.github/workflows/docker-seatunnel.yml index f02342646..7c8c801b7 100644 --- a/.github/workflows/docker-seatunnel.yml +++ b/.github/workflows/docker-seatunnel.yml @@ -69,7 +69,7 @@ jobs: no-cache: false push: true context: . - build-args: FLINK_VERSION=${{ env.FLINK_VERSION }} + build-args: FLINK_VERSION=${{ env.FLINK_VERSION }}, SEATUNNEL_VERSION=${{ env.SEATUNNEL_VERSION }} platforms: linux/amd64,linux/arm64 file: tools/docker/build/scaleph-seatunnel/Dockerfile tags: ${{ env.HUB }}:${{ env.SEATUNNEL_VERSION }}-flink-${{ env.FLINK_VERSION }} \ No newline at end of file diff --git a/tools/docker/build/scaleph-seatunnel/Dockerfile b/tools/docker/build/scaleph-seatunnel/Dockerfile index 735363ba1..13251537e 100644 --- a/tools/docker/build/scaleph-seatunnel/Dockerfile +++ b/tools/docker/build/scaleph-seatunnel/Dockerfile @@ -18,7 +18,8 @@ ARG FLINK_VERSION=1.17 ARG BASE_RELEASE_IMAGE=flink:${FLINK_VERSION} FROM maven:3.8-eclipse-temurin-11 as build -ENV SEATUNNEL_VERSION=2.3.1 +ARG SEATUNNEL_VERSION=2.3.1 + ENV SEATUNNEL_HOME=/opt/seatunnel RUN mkdir -p $SEATUNNEL_HOME @@ -48,7 +49,6 @@ ARG FLINK_VERSION=1.17 ARG BASE_RELEASE_IMAGE=flink:${FLINK_VERSION} FROM $BASE_RELEASE_IMAGE as release -ENV SEATUNNEL_VERSION=2.3.1 ENV SEATUNNEL_HOME=/opt/seatunnel RUN mkdir -p $SEATUNNEL_HOME From a1833d26f44e98ee0676eb0e95d8e860f16d7a12 Mon Sep 17 00:00:00 2001 From: wangqi <1942460489@qq.com> Date: Wed, 16 Aug 2023 20:51:51 +0800 Subject: [PATCH 2/3] feature: update seatunnel version to 2.3.2 --- .github/workflows/docker-seatunnel.yml | 3 +-- .../sliew/scaleph/common/dict/seatunnel/SeaTunnelVersion.java | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docker-seatunnel.yml b/.github/workflows/docker-seatunnel.yml index 7c8c801b7..99e6ff22f 100644 --- a/.github/workflows/docker-seatunnel.yml +++ b/.github/workflows/docker-seatunnel.yml @@ -22,11 +22,10 @@ on: seatunnelVersion: description: 'seatunnel version' required: true - default: '2.3.1' + default: '2.3.2' type: choice options: - 2.3.2 - - 2.3.1 flinkVersion: description: 'flink version' required: true diff --git a/scaleph-common/src/main/java/cn/sliew/scaleph/common/dict/seatunnel/SeaTunnelVersion.java b/scaleph-common/src/main/java/cn/sliew/scaleph/common/dict/seatunnel/SeaTunnelVersion.java index 80cd83ea8..6d8594739 100644 --- a/scaleph-common/src/main/java/cn/sliew/scaleph/common/dict/seatunnel/SeaTunnelVersion.java +++ b/scaleph-common/src/main/java/cn/sliew/scaleph/common/dict/seatunnel/SeaTunnelVersion.java @@ -28,7 +28,7 @@ @JsonFormat(shape = JsonFormat.Shape.OBJECT) public enum SeaTunnelVersion implements DictInstance { - V_2_3_1("2.3.1", "2.3.1"), + V_2_3_2("2.3.2", "2.3.2"), ; @JsonCreator From b5dd06caf65eab6b29b22d9bf001b8937c5e775a Mon Sep 17 00:00:00 2001 From: wangqi <1942460489@qq.com> Date: Wed, 16 Aug 2023 21:05:52 +0800 Subject: [PATCH 3/3] fix: scaleph seatunnel docker build error --- .github/workflows/docker-seatunnel.yml | 5 ++ .../docker/build/scaleph-seatunnel/Dockerfile | 35 +----------- .../build/scaleph-seatunnel/Dockerfile.build | 56 +++++++++++++++++++ .../build/scaleph-seatunnel/plugin_config | 53 +----------------- 4 files changed, 65 insertions(+), 84 deletions(-) create mode 100644 tools/docker/build/scaleph-seatunnel/Dockerfile.build diff --git a/.github/workflows/docker-seatunnel.yml b/.github/workflows/docker-seatunnel.yml index 99e6ff22f..874728911 100644 --- a/.github/workflows/docker-seatunnel.yml +++ b/.github/workflows/docker-seatunnel.yml @@ -62,6 +62,11 @@ jobs: platforms: amd64,arm64 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 + - name: Install SeaTunnel + run: wget https://archive.apache.org/dist/seatunnel/${{ env.SEATUNNEL_VERSION }}/apache-seatunnel-${{ env.SEATUNNEL_VERSION }}-bin.tar.gz ; \ + tar -zxf apache-seatunnel-${{ env.SEATUNNEL_VERSION }}-bin.tar.gz ; \ + cd apache-seatunnel-${{ env.SEATUNNEL_VERSION }}-bin ; \ + sh -x bin/install-plugin.sh ${{ env.SEATUNNEL_VERSION }} - name: Build image uses: docker/build-push-action@v4 with: diff --git a/tools/docker/build/scaleph-seatunnel/Dockerfile b/tools/docker/build/scaleph-seatunnel/Dockerfile index 13251537e..bcb7a245d 100644 --- a/tools/docker/build/scaleph-seatunnel/Dockerfile +++ b/tools/docker/build/scaleph-seatunnel/Dockerfile @@ -14,43 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -ARG FLINK_VERSION=1.17 -ARG BASE_RELEASE_IMAGE=flink:${FLINK_VERSION} -FROM maven:3.8-eclipse-temurin-11 as build - -ARG SEATUNNEL_VERSION=2.3.1 - -ENV SEATUNNEL_HOME=/opt/seatunnel - -RUN mkdir -p $SEATUNNEL_HOME - -ARG TAR_FILE=apache-seatunnel-incubating-${SEATUNNEL_VERSION}-bin.tar.gz - -#COPY $TAR_FILE $SEATUNNEL_HOME/$TAR_FILE -#RUN cd $SEATUNNEL_HOME ; \ -# tar -zxf $TAR_FILE --strip 1 -C $SEATUNNEL_HOME \ - -# forked from seatunnel k8s quick start -RUN wget https://archive.apache.org/dist/incubator/seatunnel/${SEATUNNEL_VERSION}/$TAR_FILE ; \ - tar -zxf $TAR_FILE --strip 1 -C $SEATUNNEL_HOME - -# strange behavior: replace mvnw by mvn on install-plugin.sh, remove useless comment on plugin_config -RUN rm $SEATUNNEL_HOME/config/plugin_config -COPY tools/docker/build/scaleph-seatunnel/plugin_config $SEATUNNEL_HOME/config - -RUN rm $SEATUNNEL_HOME/bin/install-plugin.sh -COPY tools/docker/build/scaleph-seatunnel/install-plugin.sh $SEATUNNEL_HOME/bin - -RUN cd $SEATUNNEL_HOME ; \ - sh -x bin/install-plugin.sh ${SEATUNNEL_VERSION} - # BASE_RELEASE_IMAGE argument must be defined twice -ARG FLINK_VERSION=1.17 +ARG FLINK_VERSION=1.15 ARG BASE_RELEASE_IMAGE=flink:${FLINK_VERSION} FROM $BASE_RELEASE_IMAGE as release +ARG SEATUNNEL_VERSION=2.3.2 ENV SEATUNNEL_HOME=/opt/seatunnel RUN mkdir -p $SEATUNNEL_HOME -COPY --from=build $SCALEPH_HOME $SCALEPH_HOME/ +COPY apache-seatunnel-${SEATUNNEL_VERSION}-bin/* $SCALEPH_HOME/ diff --git a/tools/docker/build/scaleph-seatunnel/Dockerfile.build b/tools/docker/build/scaleph-seatunnel/Dockerfile.build new file mode 100644 index 000000000..18b92a7d1 --- /dev/null +++ b/tools/docker/build/scaleph-seatunnel/Dockerfile.build @@ -0,0 +1,56 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ARG FLINK_VERSION=1.17 +ARG BASE_RELEASE_IMAGE=flink:${FLINK_VERSION} +FROM maven:3.8-eclipse-temurin-11 as build + +ARG SEATUNNEL_VERSION=2.3.2 + +ENV SEATUNNEL_HOME=/opt/seatunnel + +RUN mkdir -p $SEATUNNEL_HOME + +ARG TAR_FILE=apache-seatunnel-${SEATUNNEL_VERSION}-bin.tar.gz + +#COPY $TAR_FILE $SEATUNNEL_HOME/$TAR_FILE +#RUN cd $SEATUNNEL_HOME ; \ +# tar -zxf $TAR_FILE --strip 1 -C $SEATUNNEL_HOME \ + +# forked from seatunnel k8s quick start +RUN wget https://archive.apache.org/dist/seatunnel/${SEATUNNEL_VERSION}/$TAR_FILE ; \ + tar -zxf $TAR_FILE --strip 1 -C $SEATUNNEL_HOME + +# strange behavior: replace mvnw by mvn on install-plugin.sh, remove useless comment on plugin_config +RUN rm $SEATUNNEL_HOME/config/plugin_config +COPY tools/docker/build/scaleph-seatunnel/plugin_config $SEATUNNEL_HOME/config + +RUN rm $SEATUNNEL_HOME/bin/install-plugin.sh +COPY tools/docker/build/scaleph-seatunnel/install-plugin.sh $SEATUNNEL_HOME/bin + +RUN cd $SEATUNNEL_HOME ; \ + sh -x bin/install-plugin.sh ${SEATUNNEL_VERSION} + +# BASE_RELEASE_IMAGE argument must be defined twice +ARG FLINK_VERSION=1.17 +ARG BASE_RELEASE_IMAGE=flink:${FLINK_VERSION} +FROM $BASE_RELEASE_IMAGE as release + +ENV SEATUNNEL_HOME=/opt/seatunnel + +RUN mkdir -p $SEATUNNEL_HOME + +COPY --from=build $SCALEPH_HOME $SCALEPH_HOME/ diff --git a/tools/docker/build/scaleph-seatunnel/plugin_config b/tools/docker/build/scaleph-seatunnel/plugin_config index 8494cd5f2..5aa1ab787 100644 --- a/tools/docker/build/scaleph-seatunnel/plugin_config +++ b/tools/docker/build/scaleph-seatunnel/plugin_config @@ -1,53 +1,2 @@ -connector-amazondynamodb connector-assert -connector-cassandra -connector-cdc-mysql -connector-cdc-sqlserver -connector-clickhouse -connector-datahub -connector-dingtalk -connector-doris -connector-elasticsearch -connector-email -connector-file-ftp -connector-file-hadoop -connector-file-local -connector-file-oss -connector-file-oss-jindo -connector-file-s3 -connector-file-sftp -connector-google-sheets -connector-hive -connector-http-base -connector-http-feishu -connector-http-gitlab -connector-http-github -connector-http-jira -connector-http-klaviyo -connector-http-lemlist -connector-http-myhours -connector-http-notion -connector-http-onesignal -connector-http-wechat -connector-hudi -connector-iceberg -connector-influxdb -connector-iotdb -connector-jdbc -connector-kafka -connector-kudu -connector-maxcompute -connector-mongodb -connector-neo4j -connector-openmldb -connector-pulsar -connector-rabbitmq -connector-redis -connector-s3-redshift -connector-sentry -connector-slack -connector-socket -connector-starrocks -connector-tablestore -connector-selectdb-cloud -connector-hbase \ No newline at end of file +connector-cassandra \ No newline at end of file