timeplus-io · yokofly · Mar 4, 2025 · Feb 28, 2025 · Sep 26, 2024 · Aug 16, 2024
diff --git a/.github/workflows/manual_trigger_build.yml b/.github/workflows/manual_trigger_build.yml
@@ -70,6 +70,11 @@ jobs:
         docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE/output:/usr/share/nginx/html:ro -d nginx
         cd $GITHUB_WORKSPACE/docker/server
 
+        # prepare files to be copied to the image
+        mkdir -p resources/protos/google/protobuf
+        cp -r $GITHUB_WORKSPACE/contrib/google-protobuf/src/google/protobuf/*.proto ./resources/protos/google/protobuf/
+        rm -rf resources/protos/google/protobuf/unittest_*
+
         # build docker image
         docker build . --network host --build-arg single_binary_location_url=http://localhost:8080/proton -t ghcr.io/timeplus-io/proton:$TAG
 
@@ -107,6 +112,11 @@ jobs:
         docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE/output:/usr/share/nginx/html:ro -d nginx
         cd $GITHUB_WORKSPACE/docker/server
 
+        # prepare files to be copied to the image
+        mkdir -p resources/protos/google/protobuf
+        cp -r $GITHUB_WORKSPACE/contrib/google-protobuf/src/google/protobuf/*.proto ./resources/protos/google/protobuf/
+        rm -rf resources/protos/google/protobuf/unittest_*
+
         # build docker image
         docker build . --network host --build-arg single_binary_location_url=http://localhost:8080/proton -t ghcr.io/timeplus-io/proton:$TAG
 

diff --git a/.github/workflows/manual_trigger_delete_image.yml b/.github/workflows/manual_trigger_delete_image.yml
@@ -15,9 +15,9 @@ jobs:
       packages: write
       contents: read
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v4.1.7
       - name: Generate Python ENV
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5.4.0
         with:
           python-version: '3.10'
       - name: pip install requests

diff --git a/.github/workflows/manual_trigger_mark_stable.yml b/.github/workflows/manual_trigger_mark_stable.yml
@@ -30,7 +30,7 @@ jobs:
         env:
           TAG: ${{ github.event.inputs.tag }}
       - name: Add tag to image
-        uses: shrink/actions-docker-registry-tag@v3
+        uses: shrink/actions-docker-registry-tag@v4.0.0
         with:
           registry: ghcr.io
           repository: ${{ github.repository }}

diff --git a/.github/workflows/nightly_test.yml b/.github/workflows/nightly_test.yml
@@ -55,6 +55,11 @@ jobs:
             docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE/output:/usr/share/nginx/html:ro -d nginx
             cd $GITHUB_WORKSPACE/docker/server
 
+            # prepare files to be copied to the image
+            mkdir -p resources/protos/google/protobuf
+            cp -r $GITHUB_WORKSPACE/contrib/google-protobuf/src/google/protobuf/*.proto ./resources/protos/google/protobuf/
+            rm -rf resources/protos/google/protobuf/unittest_*
+
             docker build . --network host --build-arg single_binary_location_url=http://localhost:8080/proton -t ghcr.io/timeplus-io/proton:testing-$SANITIZER-$ARCH-$GITHUB_SHA
             docker push ghcr.io/timeplus-io/proton:testing-$SANITIZER-$ARCH-$GITHUB_SHA
     steps:
@@ -193,6 +198,11 @@ jobs:
             docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE/output:/usr/share/nginx/html:ro -d nginx
             cd $GITHUB_WORKSPACE/docker/server
 
+            # prepare files to be copied to the image
+            mkdir -p resources/protos/google/protobuf
+            cp -r $GITHUB_WORKSPACE/contrib/google-protobuf/src/google/protobuf/*.proto ./resources/protos/google/protobuf/
+            rm -rf resources/protos/google/protobuf/unittest_*
+
             docker build . --network host --build-arg single_binary_location_url=http://localhost:8080/proton -t ghcr.io/timeplus-io/proton:testing-$ARCH-$GITHUB_SHA
             docker push ghcr.io/timeplus-io/proton:testing-$ARCH-$GITHUB_SHA
     steps:
@@ -314,6 +324,11 @@ jobs:
             docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE/output:/usr/share/nginx/html:ro -d nginx
             cd $GITHUB_WORKSPACE/docker/server
 
+            # prepare files to be copied to the image
+            mkdir -p resources/protos/google/protobuf
+            cp -r $GITHUB_WORKSPACE/contrib/google-protobuf/src/google/protobuf/*.proto ./resources/protos/google/protobuf/
+            rm -rf resources/protos/google/protobuf/unittest_*
+
             docker build . --network host --build-arg single_binary_location_url=http://localhost:8080/proton -t ghcr.io/timeplus-io/proton:testing-$SANITIZER-$ARCH-$GITHUB_SHA
             docker push ghcr.io/timeplus-io/proton:testing-$SANITIZER-$ARCH-$GITHUB_SHA
     steps:

diff --git a/.github/workflows/proton_ci.yml b/.github/workflows/proton_ci.yml
@@ -17,6 +17,10 @@ on: # yamllint disable-line rule:truthy
       - 'utils/check-style/aspell-ignore/**'
       - 'examples/**'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   verify:
     name: Labels check
@@ -27,7 +31,7 @@ jobs:
       GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
     steps:
       - name: Checkout Repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4.2.2
         with:
           token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
           submodules: false
@@ -81,6 +85,11 @@ jobs:
         docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE/output:/usr/share/nginx/html:ro -d nginx
         cd $GITHUB_WORKSPACE/docker/server
 
+        # prepare files to be copied to the image
+        mkdir -p resources/protos/google/protobuf
+        cp -r $GITHUB_WORKSPACE/contrib/google-protobuf/src/google/protobuf/*.proto ./resources/protos/google/protobuf/
+        rm -rf resources/protos/google/protobuf/unittest_*
+
         # build docker image
         docker build . --network host --build-arg single_binary_location_url=http://localhost:8080/proton -t ghcr.io/timeplus-io/proton:testing-$SANITIZER-$ARCH-$GITHUB_SHA
 

diff --git a/.github/workflows/release_build.yml b/.github/workflows/release_build.yml
@@ -48,6 +48,11 @@ jobs:
         docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE/output:/usr/share/nginx/html:ro -d nginx
         cd $GITHUB_WORKSPACE/docker/server
 
+        # prepare files to be copied to the image
+        mkdir -p resources/protos/google/protobuf
+        cp -r $GITHUB_WORKSPACE/contrib/google-protobuf/src/google/protobuf/*.proto ./resources/protos/google/protobuf/
+        rm -rf resources/protos/google/protobuf/unittest_*
+
         # build docker image
         docker build . --network host --build-arg single_binary_location_url=http://localhost:8080/proton -t ghcr.io/timeplus-io/proton:$GITHUB_SHA-amd64
 
@@ -105,6 +110,11 @@ jobs:
         docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE/output:/usr/share/nginx/html:ro -d nginx
         cd $GITHUB_WORKSPACE/docker/server
 
+        # prepare files to be copied to the image
+        mkdir -p resources/protos/google/protobuf
+        cp -r $GITHUB_WORKSPACE/contrib/google-protobuf/src/google/protobuf/*.proto ./resources/protos/google/protobuf/
+        rm -rf resources/protos/google/protobuf/unittest_*
+
         # build docker image
         docker build . --network host --build-arg single_binary_location_url=http://localhost:8080/proton -t ghcr.io/timeplus-io/proton:$GITHUB_SHA-arm64v8
 

diff --git a/.github/workflows/run_command.yml b/.github/workflows/run_command.yml
@@ -93,7 +93,7 @@ jobs:
       ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
     steps:
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v1
+        uses: aws-actions/configure-aws-credentials@v4.1.0
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -131,24 +131,24 @@ jobs:
       GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
     steps:
       - name: Configure AWS credentials from Test account
-        uses: aws-actions/configure-aws-credentials@v1
+        uses: aws-actions/configure-aws-credentials@v4.1.0
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           aws-region: ${{ secrets.AWS_REGION }}
       - name: Login to ghcr.io
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3.3.0
         with:
           registry: ghcr.io
           username: proton_robot
           password: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
       - name: Checkout Repo
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4.2.2
         with:
           token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
           submodules: ${{ inputs.submodules }}
       - name: Checkout quark
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4.2.2
         with:
           repository: timeplus-io/quark
           path: quark
@@ -162,7 +162,7 @@ jobs:
           ${{ inputs.command }}
       - name: Release binary
         if: ${{ inputs.upload_files != '' }}
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2.2.1
         with:
           files: ${{ inputs.upload_files }}
           token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
@@ -177,7 +177,7 @@ jobs:
     if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
     steps:
       - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v1
+        uses: aws-actions/configure-aws-credentials@v4.1.0
         with:
           aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}

diff --git a/.gitmodules b/.gitmodules
@@ -29,10 +29,6 @@
 [submodule "contrib/jemalloc"]
 	path = contrib/jemalloc
 	url = https://github.com/jemalloc/jemalloc.git
-[submodule "contrib/protobuf"]
-	path = contrib/protobuf
-	url = https://github.com/ClickHouse-Extras/protobuf.git
-	branch = v3.13.0.1
 [submodule "contrib/boost"]
 	path = contrib/boost
 	url = https://github.com/ClickHouse-Extras/boost.git
@@ -79,13 +75,13 @@
 [submodule "contrib/aws"]
 	path = contrib/aws
 	url = https://github.com/ClickHouse-Extras/aws-sdk-cpp.git
-[submodule "aws-c-event-stream"]
+[submodule "contrib/aws-c-event-stream"]
 	path = contrib/aws-c-event-stream
 	url = https://github.com/awslabs/aws-c-event-stream.git
-[submodule "aws-c-common"]
+[submodule "contrib/aws-c-common"]
 	path = contrib/aws-c-common
 	url = https://github.com/ClickHouse/aws-c-common.git
-[submodule "aws-checksums"]
+[submodule "contrib/aws-checksums"]
 	path = contrib/aws-checksums
 	url = https://github.com/awslabs/aws-checksums.git
 [submodule "contrib/curl"]
@@ -142,10 +138,9 @@
 [submodule "contrib/xz"]
 	path = contrib/xz
 	url = https://github.com/xz-mirror/xz
-[submodule "contrib/abseil-cpp"]
+[submodule "contrib/abseil"]
 	path = contrib/abseil-cpp
-	url = https://github.com/abseil/abseil-cpp.git
-	branch = lts_2021_11_02
+	url = https://github.com/ClickHouse/abseil-cpp.git
 [submodule "contrib/dragonbox"]
 	path = contrib/dragonbox
 	url = https://github.com/ClickHouse-Extras/dragonbox.git
@@ -280,9 +275,15 @@
 [submodule "contrib/google-benchmark"]
 	path = contrib/google-benchmark
 	url = https://github.com/google/benchmark.git
+[submodule "contrib/libdivide"]
+	path = contrib/libdivide
+	url = https://github.com/ridiculousfish/libdivide.git
 [submodule "contrib/c-ares"]
 	path = contrib/c-ares
 	url = https://github.com/c-ares/c-ares.git
 [submodule "contrib/pulsar-cmake/pulsar-client-cpp"]
 	path = contrib/pulsar-cmake/pulsar-client-cpp
 	url = https://github.com/apache/pulsar-client-cpp.git
+[submodule "contrib/google-protobuf"]
+	path = contrib/google-protobuf
+	url = https://github.com/ClickHouse/google-protobuf
diff --git a/PreLoad.cmake b/PreLoad.cmake
@@ -48,8 +48,14 @@ if (NOT "$ENV{CFLAGS}" STREQUAL ""
 endif()
 
 # Default toolchain - this is needed to avoid dependency on OS files.
-execute_process(COMMAND uname -s OUTPUT_VARIABLE OS)
-execute_process(COMMAND uname -m OUTPUT_VARIABLE ARCH)
+execute_process(COMMAND uname -s
+    OUTPUT_VARIABLE OS
+    COMMAND_ERROR_IS_FATAL ANY
+)
+execute_process(COMMAND uname -m
+    OUTPUT_VARIABLE ARCH
+    COMMAND_ERROR_IS_FATAL ANY
+)
 
 # By default, prefer clang on Linux
 # But note, that you still may change the compiler with -DCMAKE_C_COMPILER/-DCMAKE_CXX_COMPILER.

diff --git a/base/poco/JSON/src/pdjson.c b/base/poco/JSON/src/pdjson.c
@@ -314,13 +314,13 @@ static int read_unicode(json_stream *json)
 
         if (l < 0xdc00 || l > 0xdfff) {
             json_error(json, "invalid surrogate pair continuation \\u%04lx out "
-                             "of range (dc00-dfff)", l);
+                             "of range (dc00-dfff)", (unsigned long)l);
             return -1;
         }
 
         cp = ((h - 0xd800) * 0x400) + ((l - 0xdc00) + 0x10000);
     } else if (cp >= 0xdc00 && cp <= 0xdfff) {
-            json_error(json, "dangling surrogate \\u%04lx", cp);
+            json_error(json, "dangling surrogate \\u%04lx", (unsigned long)cp);
             return -1;
     }
 

diff --git a/cmake/freebsd/default_libs.cmake b/cmake/freebsd/default_libs.cmake
@@ -1,9 +1,17 @@
 set (DEFAULT_LIBS "-nodefaultlibs")
 
 if (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "amd64")
-    execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-x86_64.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
+    execute_process (COMMAND
+        ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-x86_64.a
+        OUTPUT_VARIABLE BUILTINS_LIBRARY
+        COMMAND_ERROR_IS_FATAL ANY
+        OUTPUT_STRIP_TRAILING_WHITESPACE)
 else ()
-    execute_process (COMMAND ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
+    execute_process (COMMAND
+        ${CMAKE_CXX_COMPILER} --print-file-name=libclang_rt.builtins-${CMAKE_SYSTEM_PROCESSOR}.a
+        OUTPUT_VARIABLE BUILTINS_LIBRARY
+        COMMAND_ERROR_IS_FATAL ANY
+        OUTPUT_STRIP_TRAILING_WHITESPACE)
 endif ()
 
 set (DEFAULT_LIBS "${DEFAULT_LIBS} ${BUILTINS_LIBRARY} ${COVERAGE_OPTION} -lc -lm -lrt -lpthread")

diff --git a/cmake/linux/default_libs.cmake b/cmake/linux/default_libs.cmake
@@ -6,7 +6,11 @@ set (DEFAULT_LIBS "-nodefaultlibs")
 # We need builtins from Clang's RT even without libcxx - for ubsan+int128.
 # See https://bugs.llvm.org/show_bug.cgi?id=16404
 if (COMPILER_CLANG)
-    execute_process (COMMAND ${CMAKE_CXX_COMPILER} --target=${CMAKE_CXX_COMPILER_TARGET} --print-libgcc-file-name --rtlib=compiler-rt OUTPUT_VARIABLE BUILTINS_LIBRARY OUTPUT_STRIP_TRAILING_WHITESPACE)
+    execute_process (COMMAND
+        ${CMAKE_CXX_COMPILER} --target=${CMAKE_CXX_COMPILER_TARGET} --print-libgcc-file-name --rtlib=compiler-rt
+        OUTPUT_VARIABLE BUILTINS_LIBRARY
+        COMMAND_ERROR_IS_FATAL ANY
+        OUTPUT_STRIP_TRAILING_WHITESPACE)
 
     if (NOT EXISTS "${BUILTINS_LIBRARY}")
         set (BUILTINS_LIBRARY "-lgcc")

diff --git a/cmake/tools.cmake b/cmake/tools.cmake
@@ -9,7 +9,11 @@ else ()
 endif ()
 
 # Print details to output
-execute_process(COMMAND ${CMAKE_CXX_COMPILER} --version OUTPUT_VARIABLE COMPILER_SELF_IDENTIFICATION OUTPUT_STRIP_TRAILING_WHITESPACE)
+execute_process(COMMAND ${CMAKE_CXX_COMPILER} --version
+    OUTPUT_VARIABLE COMPILER_SELF_IDENTIFICATION
+    COMMAND_ERROR_IS_FATAL ANY
+    OUTPUT_STRIP_TRAILING_WHITESPACE
+)
 message (STATUS "Using compiler:\n${COMPILER_SELF_IDENTIFICATION}")
 
 # Require minimum compiler versions

diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
@@ -57,7 +57,7 @@ add_contrib (dragonbox-cmake dragonbox)
 add_contrib (vectorscan-cmake vectorscan)
 add_contrib (jemalloc-cmake jemalloc)
 add_contrib (libcpuid-cmake libcpuid)
-add_contrib (libdivide)
+add_contrib (libdivide-cmake)
 add_contrib (libmetrohash)
 add_contrib (lz4-cmake lz4)
 add_contrib (murmurhash)
@@ -79,7 +79,7 @@ add_contrib (thrift-cmake thrift)
 # parquet/arrow/orc
 add_contrib (arrow-cmake arrow) # requires: snappy, thrift, double-conversion
 add_contrib (avro-cmake avro) # requires: snappy
-add_contrib (protobuf-cmake protobuf)
+add_contrib (google-protobuf-cmake google-protobuf)
 add_contrib (openldap-cmake openldap)
 add_contrib (grpc-cmake grpc)
 add_contrib (msgpack-c-cmake msgpack-c)
@@ -136,7 +136,7 @@ add_contrib (fmtlib-cmake fmtlib)
 add_contrib (krb5-cmake krb5)
 add_contrib (cyrus-sasl-cmake cyrus-sasl) # for krb5
 add_contrib (libgsasl-cmake libgsasl) # requires krb5
-add_contrib (libhdfs3-cmake libhdfs3) # requires: protobuf, krb5
+add_contrib (libhdfs3-cmake libhdfs3) # requires: google-protobuf, krb5
 add_contrib (librdkafka-cmake librdkafka) # requires: libgsasl
 add_contrib (libpqxx-cmake libpqxx)
 add_contrib (libpq-cmake libpq)

diff --git a/contrib/abseil-cpp b/contrib/abseil-cpp