From 930e98d24a0158b7a1044a3f7865dabebbaf38c4 Mon Sep 17 00:00:00 2001 From: Ji Bin Date: Mon, 3 Apr 2023 05:05:35 +0800 Subject: [PATCH] Enable build 2.3 for macosx Signed-off-by: Ji Bin --- .github/workflows/main.yml | 166 ++++++++++-------- examples/example.py | 4 +- .../patches/macosx-v2.3.0-beta.patch | 64 +++++++ 3 files changed, 155 insertions(+), 79 deletions(-) create mode 100644 milvus_binary/patches/macosx-v2.3.0-beta.patch diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 809a977..ad12f1a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,7 +17,7 @@ jobs: CCACHE_COMPRESS: 1 CCACHE_COMPRESSLEVEL: 5 CCACHE_MAXSIZE: 2G - timeout-minutes: 120 + timeout-minutes: 180 steps: - name: Checkout uses: actions/checkout@v3 @@ -62,55 +62,55 @@ jobs: name: wheel_linux path: dist/*.whl retention-days: 5 - # build_macos: - # name: Build Wheel - macOS - # runs-on: macos-11 - # env: - # CCACHE_DIR: ${{ github.workspace }}/.ccache - # CCACHE_COMPILERCHECK: content - # CCACHE_COMPRESS: 1 - # CCACHE_COMPRESSLEVEL: 5 - # CCACHE_MAXSIZE: 2G - # timeout-minutes: 120 - # steps: - # - name: Checkout - # uses: actions/checkout@v3 - # - name: Cache go - # uses: actions/cache@v3 - # with: - # path: | - # ~/.cache/go-build - # ~/go/pkg/mod - # key: macos-go-${{ hashFiles('milvus_binary/env.sh') }} - # restore-keys: macos-go- - # - name: Cache conan - # uses: actions/cache@v3 - # with: - # path: | - # ~/.conan/data - # key: macos-conan-${{ hashFiles('milvus_binary/env.sh') }} - # restore-keys: macos-conan- - # - name: Cache ccache - # uses: actions/cache@v3 - # with: - # path: | - # ${{ github.workspace }}/.ccache - # key: macos-ccache-${{ hashFiles('milvus_binary/env.sh') }} - # restore-keys: macos-ccache- - # - name: Setup Go environment - # uses: actions/setup-go@v4 - # with: - # go-version: '~1.18.10' - # cache: false - # - name: Build Wheel - # run: | - # python3 -m pip install --user build wheel 'setuptools>64.0' - # python3 -m build -w -n - # - uses: actions/upload-artifact@v3 - # with: - # name: wheel_macos - # path: dist/*.whl - # retention-days: 5 + build_macos: + name: Build Wheel - macOS + runs-on: macos-11 + env: + CCACHE_DIR: ${{ github.workspace }}/.ccache + CCACHE_COMPILERCHECK: content + CCACHE_COMPRESS: 1 + CCACHE_COMPRESSLEVEL: 5 + CCACHE_MAXSIZE: 2G + timeout-minutes: 180 + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Cache go + uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: macos-go-${{ hashFiles('milvus_binary/env.sh') }} + restore-keys: macos-go- + - name: Cache conan + uses: actions/cache@v3 + with: + path: | + ~/.conan/data + key: macos-conan-${{ hashFiles('milvus_binary/env.sh') }} + restore-keys: macos-conan- + - name: Cache ccache + uses: actions/cache@v3 + with: + path: | + ${{ github.workspace }}/.ccache + key: macos-ccache-${{ hashFiles('milvus_binary/env.sh') }} + restore-keys: macos-ccache- + - name: Setup Go environment + uses: actions/setup-go@v4 + with: + go-version: '~1.18.10' + cache: false + - name: Build Wheel + run: | + python3 -m pip install --user build wheel 'setuptools>64.0' + python3 -m build -w -n + - uses: actions/upload-artifact@v3 + with: + name: wheel_macos + path: dist/*.whl + retention-days: 5 # build_windows: # name: Build Wheel - windows # runs-on: windows-latest @@ -120,7 +120,7 @@ jobs: # CCACHE_COMPRESS: 1 # CCACHE_COMPRESSLEVEL: 5 # CCACHE_MAXSIZE: 2G - # timeout-minutes: 120 + # timeout-minutes: 180 # steps: # - name: Set git to use LF # run: | @@ -210,31 +210,43 @@ jobs: python3 -m pip install --user "$(echo dist/*.whl)[client]" cd examples python3 example.py - # acceptance_test_macos: - # needs: - # - build_macos - # name: Acceptance Test ${{ matrix.os }} - # runs-on: ${{ matrix.os }} - # timeout-minutes: 60 - # strategy: - # fail-fast: false - # matrix: - # os: ["macos-11", "macos-12"] - # steps: - # - uses: actions/checkout@v3 - # - uses: actions/setup-python@v4 - # with: - # python-version: '3.8' - # - uses: actions/download-artifact@v3 - # with: - # name: wheel_macos - # path: dist - # - name: Run hello milvus - # run: | - # python3 -m pip install --user wheel setuptools - # python3 -m pip install --user "$(echo dist/*.whl)[client]" - # cd examples - # python3 example.py + - name: Upload runtime log + uses: actions/upload-artifact@v3 + if: ${{ failure() }} + with: + name: milvus-log + path: ~/.milvus.io/milvus-server/**/*.log + acceptance_test_macos: + needs: + - build_macos + name: Acceptance Test ${{ matrix.os }} + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + os: ["macos-11", "macos-12"] + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.8' + - uses: actions/download-artifact@v3 + with: + name: wheel_macos + path: dist + - name: Run hello milvus + run: | + python3 -m pip install --user wheel setuptools + python3 -m pip install --user "$(echo dist/*.whl)[client]" + cd examples + python3 example.py + - name: Upload runtime log + uses: actions/upload-artifact@v3 + if: ${{ failure() }} + with: + name: milvus-log + path: ~/.milvus.io/milvus-server/**/*.log # acceptance_test_windows: # needs: # - build_windows @@ -259,4 +271,4 @@ jobs: # $x = Get-ChildItem dist\*.whl # pip install "$x[client]" # cd examples - # python example.py \ No newline at end of file + # python example.py diff --git a/examples/example.py b/examples/example.py index 75e9901..03085ed 100644 --- a/examples/example.py +++ b/examples/example.py @@ -21,8 +21,8 @@ # 5. search # Optional, if you want store all related data to specific location -# default it wil using %APPDATA%/milvus-io/milvus-server -default_server.set_base_dir('test_milvus') +# default it wil using ~/.milvus-io/milvus-server/<__version_of_milvus__> +# default_server.set_base_dir('test_milvus') # Optional, if you want cleanup previous data default_server.cleanup() diff --git a/milvus_binary/patches/macosx-v2.3.0-beta.patch b/milvus_binary/patches/macosx-v2.3.0-beta.patch new file mode 100644 index 0000000..312ca41 --- /dev/null +++ b/milvus_binary/patches/macosx-v2.3.0-beta.patch @@ -0,0 +1,64 @@ +diff --git a/scripts/core_build.sh b/scripts/core_build.sh +index 5d079c317..2eccfbd0b 100755 +--- a/scripts/core_build.sh ++++ b/scripts/core_build.sh +@@ -193,12 +193,24 @@ conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/c + unameOut="$(uname -s)" + case "${unameOut}" in + Darwin*) +- llvm_prefix="$(brew --prefix llvm)" ++ # detect llvm version by valid list ++ for llvm_version in 15 14 NOT_FOUND ; do ++ if brew ls --versions llvm@${llvm_version} > /dev/null; then ++ break ++ fi ++ done ++ if [ "${llvm_version}" = "NOT_FOUND" ] ; then ++ echo "llvm@14~15 is not installed" ++ exit 1 ++ fi ++ llvm_prefix="$(brew --prefix llvm@${llvm_version})" + export CLANG_TOOLS_PATH="${llvm_prefix}/bin" ++ export PATH=${CLANG_TOOLS_PATH}:${PATH} + export CC="${llvm_prefix}/bin/clang" + export CXX="${llvm_prefix}/bin/clang++" +- export CFLAGS=-Wno-deprecated-declarations +- export CXXFLAGS=-Wno-deprecated-declarations ++ export CFLAGS="-Wno-deprecated-declarations -I$(brew --prefix libomp)/include" ++ export CXXFLAGS=${CFLAGS} ++ export LDFLAGS="-L$(brew --prefix libomp)/lib" + conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler=clang -s compiler.version=15 -s compiler.libcxx=libc++ || { echo 'conan install failed'; exit 1; } + ;; + Linux*) +diff --git a/scripts/install_deps.sh b/scripts/install_deps.sh +index 93a81669d..5c5440880 100755 +--- a/scripts/install_deps.sh ++++ b/scripts/install_deps.sh +@@ -50,25 +50,10 @@ function install_linux_deps() { + + function install_mac_deps() { + sudo xcode-select --install > /dev/null 2>&1 +- brew install libomp ninja cmake llvm ccache grep +- export PATH="/usr/local/opt/grep/libexec/gnubin:$PATH" +- brew update && brew upgrade && brew cleanup ++ brew install libomp ninja cmake llvm@15 ccache pkg-config openssl librdkafka ++ brew install grep + + pip3 install conan==1.58.0 +- +- if [[ $(arch) == 'arm64' ]]; then +- brew install openssl +- brew install librdkafka +- brew install pkg-config +- sudo mkdir /usr/local/include +- sudo mkdir /usr/local/opt +- sudo ln -s "$(brew --prefix llvm)" "/usr/local/opt/llvm" +- sudo ln -s "$(brew --prefix libomp)/include/omp.h" "/usr/local/include/omp.h" +- sudo ln -s "$(brew --prefix libomp)" "/usr/local/opt/libomp" +- sudo ln -s "$(brew --prefix boost)/include/boost" "/usr/local/include/boost" +- sudo ln -s "$(brew --prefix tbb)/include/tbb" "/usr/local/include/tbb" +- sudo ln -s "$(brew --prefix tbb)/include/oneapi" "/usr/local/include/oneapi" +- fi + } + + if ! command -v go &> /dev/null