Merge branch 'lakesoul-io:main' into main

Author: darklinght

.github/workflows/consistency-ci.yml

@@ -79,6 +79,7 @@ jobs:
         with:
           spark-version: '3.3.1'
           hadoop-version: '3'
+          spark-url: 'https://mirrors.huaweicloud.com/apache/spark/spark-3.3.1/spark-3.3.1-bin-hadoop3.tgz'
       - run: spark-submit --version
 
       - uses: actions-rs/toolchain@v1

.github/workflows/deployment.yml

@@ -88,9 +88,9 @@ jobs:
     steps:
       - name: Install automake
         run: brew install automake
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Set up JDK 11
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@v5
         with:
           java-version: '11'
           distribution: 'temurin'
@@ -152,23 +152,24 @@ jobs:
           name: lakesoul-nativemetadata-x86_64-pc-windows-msvc
           path: ./rust/target/release/
       - name: Set up JDK 11
-        uses: actions/setup-java@v4
+        uses: actions/setup-java@v5
         with:
           java-version: '11'
           distribution: 'temurin'
           cache: maven
+          server-id: central
+          server-username: MAVEN_USERNAME
+          server-password: MAVEN_PASSWORD
+          gpg-private-key: ${{ secrets.MAVEN_SIGN_GPG_KEY }}
+          gpg-passphrase: GPG_PASSPHRASE
       - name: Install Protoc
         uses: arduino/setup-protoc@v2
         with:
           version: "23.x"
           repo-token: ${{ secrets.GITHUB_TOKEN }}
       - name: Release to Maven Central Repository
-        uses: samuelmeuli/action-maven-publish@v1
-        with:
-          gpg_private_key: ${{ secrets.MAVEN_SIGN_GPG_KEY }}
-          gpg_passphrase: ${{ secrets.MAVEN_SIGN_GPG_PASSPHRASE }}
-          nexus_username: ${{ secrets.OSSRH_USERNAME }}
-          nexus_password: ${{ secrets.OSSRH_TOKEN }}
-          maven_goals_phases: "deploy"
-          maven_args: "-DskipTests -Dmaven.test.skip=true"
-          maven_profiles: "cross-build,release-sign-artifacts"
+        run: mvn --batch-mode deploy -Pcross-build,release-sign-artifacts -DskipTests -Dmaven.test.skip=true
+        env:
+          MAVEN_USERNAME: ${{ secrets.OSSRH_USERNAME }}
+          MAVEN_PASSWORD: ${{ secrets.OSSRH_TOKEN }}
+          GPG_PASSPHRASE: ${{ secrets.MAVEN_SIGN_GPG_PASSPHRASE }}

.github/workflows/flink-cdc-hdfs-test.yml

@@ -58,9 +58,9 @@ jobs:
         run: |
           python -m pip install --upgrade pip setuptools wheel
           pip install pymysql cryptography jproperties --no-cache-dir
-          wget https://repo1.maven.org/maven2/org/apache/flink/flink-s3-fs-hadoop/1.20.1/flink-s3-fs-hadoop-1.20.1.jar -O $HOME/flink-s3-fs-hadoop-1.20.1.jar
-          wget https://repo1.maven.org/maven2/org/apache/parquet/parquet-hadoop-bundle/1.13.1/parquet-hadoop-bundle-1.13.1.jar -O $HOME/parquet-hadoop-bundle-1.13.1.jar
-          wget https://repo1.maven.org/maven2/org/apache/flink/flink-parquet/1.20.1/flink-parquet-1.20.1.jar -O $HOME/flink-parquet-1.20.1.jar
+          wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-s3-fs-hadoop/1.20.1/flink-s3-fs-hadoop-1.20.1.jar -O $HOME/flink-s3-fs-hadoop-1.20.1.jar
+          wget -q https://repo1.maven.org/maven2/org/apache/parquet/parquet-hadoop-bundle/1.13.1/parquet-hadoop-bundle-1.13.1.jar -O $HOME/parquet-hadoop-bundle-1.13.1.jar
+          wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-parquet/1.20.1/flink-parquet-1.20.1.jar -O $HOME/flink-parquet-1.20.1.jar
       - name: Install Protoc
         uses: arduino/setup-protoc@v2
         with:

.github/workflows/flink-cdc-test.yml

@@ -12,17 +12,17 @@ on:
     paths-ignore:
       - "javadoc/**"
       - "website/**"
-      - "cpp/**"
       - "python/**"
+      - "cpp/**"
       - "**.md"
     branches:
       - 'main'
   pull_request:
     paths-ignore:
       - "javadoc/**"
       - "website/**"
-      - "cpp/**"
       - "python/**"
+      - "cpp/**"
       - "**.md"
     branches:
       - 'main'
@@ -202,6 +202,79 @@ jobs:
           retention-days: 5
           if-no-files-found: error
 
+  spark-test-gluten:
+    runs-on: ubuntu-latest
+    needs: [ build-rust-linux-x86_64 ]
+
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres:14.5
+        # Provide the password for postgres
+        env:
+          POSTGRES_PASSWORD: lakesoul_test
+          POSTGRES_USER: lakesoul_test
+          POSTGRES_DB: lakesoul_test
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+          --name lakesoul-test-pg
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          java-version: '11'
+          distribution: 'temurin'
+          cache: maven
+      - name: Install psql
+        run: sudo apt-get install -y postgresql-client-16
+      - name: Init PG
+        run: |
+          ./script/meta_init_for_local_test.sh -j 1
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v2
+        with:
+          version: "23.x"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - uses: actions/download-artifact@v4
+        with:
+          name: lakesoul-nativemetadata-x86_64-unknown-linux-gnu-maven-test
+          path: ./rust/target/release/
+      - uses: actions/download-artifact@v4
+        with:
+          name: lakesoul-nativeio-x86_64-unknown-linux-gnu-maven-test
+          path: ./rust/target/release/
+      - name: Build with Maven
+        run: |
+          sudo apt-get update && sudo apt-get install --only-upgrade tzdata
+          wget -q https://dmetasoul-bucket.obs.cn-southwest-2.myhuaweicloud.com/releases/lakesoul/gluten-1.6.0-84ac41874-snapshot.tar.gz
+          mkdir -p ~/.m2/repository
+          tar xf gluten-1.6.0-84ac41874-snapshot.tar.gz -C ~/.m2/repository
+          rm -f gluten-1.6.0-84ac41874-snapshot.tar.gz
+          MAVEN_OPTS="-Xmx4g -Dio.netty.tryReflectionSetAccessible=true" mvn -B test -pl lakesoul-spark-gluten -am -Pgluten -Pcross-build --file pom.xml -Dtest='LakeSoulGlutenCompatSuite,UpdateGlutenTestSuite,UpsertGlutenTestSuite,DeleteSQLGlutenTestSuite,MergeIntoSQLGlutenTestSuite,' -Dsurefire.failIfNoSpecifiedTests=false
+      - name: Generate Report Site
+        if: always()
+        run: |
+          mvn surefire-report:report-only -pl lakesoul-spark-gluten -am -Pgluten
+      - name: Upload Test Report
+        if: always()
+        continue-on-error: true
+        uses: actions/upload-artifact@v4
+        with:
+          name: maven-test-report-artifact-spark-2
+          path: lakesoul-spark-gluten/target/site
+          retention-days: 5
+          if-no-files-found: error
+
   spark-test-rbac:
     runs-on: ubuntu-latest
     needs: [ build-rust-linux-x86_64 ]

.github/workflows/maven-test.yml

@@ -58,11 +58,11 @@ jobs:
         run: |
           python -m pip install --upgrade pip setuptools wheel
           pip install pymysql cryptography jproperties --no-cache-dir
-          wget https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz -O $HOME/hadoop-3.3.5.tar.gz && tar xf $HOME/hadoop-3.3.5.tar.gz -C $HOME
+          wget -q https://dlcdn.apache.org/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz -O $HOME/hadoop-3.3.5.tar.gz && tar xf $HOME/hadoop-3.3.5.tar.gz -C $HOME
           echo "HADOOP_HOME=$HOME/hadoop-3.3.5" >> $GITHUB_ENV
-          wget https://repo1.maven.org/maven2/org/apache/flink/flink-s3-fs-hadoop/1.20.1/flink-s3-fs-hadoop-1.20.1.jar -O $HOME/flink-s3-fs-hadoop-1.20.1.jar
-          wget https://repo1.maven.org/maven2/org/apache/parquet/parquet-hadoop-bundle/1.12.3/parquet-hadoop-bundle-1.12.3.jar -O $HOME/parquet-hadoop-bundle-1.12.3.jar
-          wget https://repo1.maven.org/maven2/org/apache/flink/flink-parquet/1.20.1/flink-parquet-1.20.1.jar -O $HOME/flink-parquet-1.20.1.jar
+          wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-s3-fs-hadoop/1.20.1/flink-s3-fs-hadoop-1.20.1.jar -O $HOME/flink-s3-fs-hadoop-1.20.1.jar
+          wget -q https://repo1.maven.org/maven2/org/apache/parquet/parquet-hadoop-bundle/1.12.3/parquet-hadoop-bundle-1.12.3.jar -O $HOME/parquet-hadoop-bundle-1.12.3.jar
+          wget -q https://repo1.maven.org/maven2/org/apache/flink/flink-parquet/1.20.1/flink-parquet-1.20.1.jar -O $HOME/flink-parquet-1.20.1.jar
       - name: Install Protoc
         uses: arduino/setup-protoc@v2
         with:

.github/workflows/presto-cdc-test.yml

@@ -0,0 +1,109 @@
+name: CI with Python Test
+on:
+  push:
+    paths-ignore:
+      - "javadoc/**"
+      - "website/**"
+      - "**.md"
+    branches:
+      - 'main'
+      - 'release/**'
+    tags:
+      - 'py-v[0-9]+.[0-9]+.[0-9]+'
+  pull_request:
+    paths-ignore:
+      - "javadoc/**"
+      - "website/**"
+      - "**.md"
+    branches:
+      - 'main'
+      - 'release/**'
+  workflow_dispatch:
+
+jobs:
+  run-pytest:
+    runs-on: ubuntu-latest
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres:14.5
+        # Provide the password for postgres
+        env:
+          POSTGRES_PASSWORD: lakesoul_test
+          POSTGRES_USER: lakesoul_test
+          POSTGRES_DB: lakesoul_test
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+          --name lakesoul-test-pg
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 11
+        uses: actions/setup-java@v4
+        with:
+          java-version: '11'
+          distribution: 'temurin'
+          cache: maven
+      - name: Install psql
+        run: sudo apt-get install -y postgresql-client-16
+      - name: Init PG
+        run: |
+          ./script/meta_init_for_local_test.sh -j 2
+      - name: Install Protoc
+        uses: arduino/setup-protoc@v2
+        with:
+          version: "23.x"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+          cache-suffix: "lakesoul-py-ci"
+          cache-dependency-glob: |
+            pyproject.toml
+            uv.lock
+      - uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: "./rust -> target"
+      - name: pytest
+        env:
+          LAKESOUL_SOURCE_DIR: ${{ github.workspace }}
+        run: |
+          export LD_LIBRARY_PATH=$JAVA_HOME/lib/server:$LD_LIBRARY_PATH
+          echo $LD_LIBRARY_PATH
+          mvn package -pl lakesoul-spark -am -DskipTests
+          uv sync --no-install-project   
+          source .venv/bin/activate
+          bash cpp/compile.sh
+          cp cpp/build/python/lakesoul/metadata/generated/entity_pb2.py python/lakesoul/metadata/generated
+          cp cpp/build/_lakesoul_dataset.so python/lakesoul/arrow
+          cp rust/target/release/liblakesoul_metadata_c.so python/lakesoul/metadata/lib/
+          cargo run --manifest-path rust/Cargo.toml --release -p lakesoul-console -- tpch-gen -p "file:///tmp/lakesoul/tpch_data" --scale-factor 0.1 -n 8 
+          uv run pytest python/tests/ -s
+  publish:
+    runs-on: ubuntu-latest
+    if: startsWith(github.event.ref, 'refs/tags/py-v')
+    needs: ["run-pytest"]
+    env:
+      PIP_INDEX_URL: https://pypi.tuna.tsinghua.edu.cn/simple
+    permissions:
+      id-token: write 
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - name: build  
+        run: |
+          python3 python/docker_build_all_wheels.py
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+      
+        
+

.github/workflows/python-ci.yml

@@ -78,5 +78,7 @@ jobs:
           rm -f Cargo.lock
           cargo clean
           RUST_BACKTRACE=full cargo test --package lakesoul-datafusion
-          RUST_BACKTRACE=full cargo test --package lakesoul-flight --test flight_sql -- --color always
+          RUST_BACKTRACE=full cargo test --package lakesoul-flight --test flight_sql -- --color always --nocapture
+          RUST_BACKTRACE=full cargo test --color=always --package lakesoul-metadata --lib rbac::tests::test_verify_permission -- --nocapture
+          RUST_BACKTRACE=full cargo test --color=always --package lakesoul-s3-proxy --bin lakesoul-s3-proxy tests --profile test --no-fail-fast -- --nocapture
 

.github/workflows/rust-ci.yml

@@ -7,6 +7,7 @@ on:
   push:
     tags:
       - 'v[0-9]+.[0-9]+.[0-9]+'
+      - 'py-v[0-9]+.[0-9]+.[0-9]+'
   workflow_dispatch:
 
 jobs:

.github/workflows/website-publish.yml

@@ -2,13 +2,15 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+.pytest_cache
 /target/
 /build/
 /.idea/
 /spark-warehouse/
 /derby.log
 /.metals/
 /.vscode/
+/**/.venv
 *.iml
 /native-io/lakesoul-io-java/target/
 /aws-test/target/
@@ -32,7 +34,9 @@ __pycache__/
 /python/lakesoul/metadata/generated/entity_pb2_grpc.py
 /python/build/
 /python/lakesoul.egg-info/
+/python/lakesoul/metadata/lib/liblakesoul_metadata_c.so
 *.whl
 /wheelhouse/
 /rust/.idea
+**/.venv/
 .flattened-pom.xml