diff --git a/.buckrelease b/.buckrelease index 387c490b746..5888535b61b 100644 --- a/.buckrelease +++ b/.buckrelease @@ -1 +1 @@ -v2018.06.25.01 +v2020.09.01.01 diff --git a/.circleci/config.yml b/.circleci/config.yml index 83cd9a0eb10..d35dc09f932 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -3,6 +3,47 @@ version: 2.1 orbs: win: circleci/windows@2.2.0 +install_dlang: &install_dlang + name: Install Dlang + command: | + if [ "${PLATFORM}" == "macos" ]; then + # gpg depends on the latest git, re-install it. + brew remove -f git + brew install git + brew install gpg + fi + curl https://dlang.org/install.sh | bash -s dmd-2.091.0 + source ~/dlang/dmd-2.091.0/activate + dmd --version + +install_lua: &install_lua + name: Install Lua + command: | + cd + lua_version="5.3.5" + curl -R -O http://www.lua.org/ftp/lua-${lua_version}.tar.gz + tar zxf lua-${lua_version}.tar.gz + cd lua-${lua_version} + if [ "${PLATFORM}" == "linux" ]; then + sudo make linux install + elif [ "${PLATFORM}" == "macos" ]; then + sudo make macosx install + fi + lua -v + +install_ocaml: &install_ocaml + name: Install Ocaml + command: | + if [ "${PLATFORM}" == "linux" ]; then + sudo apt install ocaml + elif [ "${PLATFORM}" == "macos" ]; then + brew install opam + opam init -y + opam switch create 4.07.1 + eval `opam env` + fi + ocaml -version + install_openjdk8: &install_openjdk8 name: Install OpenJDK8 command: | @@ -10,28 +51,34 @@ install_openjdk8: &install_openjdk8 sudo apt-get update && sudo apt-get install openjdk-8-jdk sudo update-java-alternatives -s java-1.8.0-openjdk-amd64 elif [ "${PLATFORM}" == "macos" ]; then - brew cask install adoptopenjdk8 + brew install openjdk@8 + export PATH="/usr/local/opt/openjdk@8/bin:$PATH" fi java -version install_android_sdk: &install_android_sdk name: Install Android SDK command: | - sdk_os="linux" - if [ "${PLATFORM}" == "macos" ]; then - sdk_os="darwin" + # skip if ${ANDROID_SDK} is restored from cache + if [ ! -d "${ANDROID_SDK}" ]; then + sdk_os="linux" + if [ "${PLATFORM}" == "macos" ]; then + sdk_os="darwin" + fi + sdk_zip_filename="sdk-tools-${sdk_os}-4333796.zip" + mkdir -p "${ANDROID_SDK}" + cd "${ANDROID_SDK}" + curl -O "https://dl.google.com/android/repository/${sdk_zip_filename}" + unzip "${sdk_zip_filename}" + export PATH="${ANDROID_SDK}/tools/bin:${PATH}" + echo 'y' |sdkmanager --install tools + echo 'y' |sdkmanager --install platform-tools + echo 'y' |sdkmanager --install "build-tools;28.0.0" + echo 'y' |sdkmanager --install "platforms;android-23" + rm "${sdk_zip_filename}" + else + echo "Android SDK restored from cache, skip installation." fi - sdk_zip_filename="sdk-tools-${sdk_os}-4333796.zip" - mkdir -p "${ANDROID_SDK}" - cd "${ANDROID_SDK}" - curl -O "https://dl.google.com/android/repository/${sdk_zip_filename}" - unzip "${sdk_zip_filename}" - export PATH="${ANDROID_SDK}/tools/bin:${PATH}" - echo 'y' |sdkmanager --install tools - echo 'y' |sdkmanager --install platform-tools - echo 'y' |sdkmanager --install "build-tools;28.0.0" - echo 'y' |sdkmanager --install "platforms;android-23" - # Install 32 bit libraries # https://stackoverflow.com/questions/36911709/cannot-run-program-aapt # Needed to run Android build-tools @@ -56,14 +103,29 @@ install_golang: &install_golang go version install_python: &install_python - name: Install Python 3.6.2 + name: Install Python 3.9.4 command: | if [ "${PLATFORM}" == "macos" ]; then brew install pyenv fi - pyenv install -s 3.6.2 - pyenv global 3.6.2 system + if [ "${PLATFORM}" == "linux" ]; then + # Reinstall libffi6 as Ubuntu 20.04 contains only libffi7 + wget http://mirrors.kernel.org/ubuntu/pool/main/libf/libffi/libffi6_3.2.1-8_amd64.deb + sudo apt install ./libffi6_3.2.1-8_amd64.deb + fi + pyenv versions export PATH="$(pyenv root)/shims:${PATH}" + python_version=`python --version` || python_version="Python Not Found" + # Skip installation if Python 3.9.4 restored frm cache. + if [ "${python_version}" != "Python 3.9.4" ]; then + # In the case that pyenv is upgraded to a new version, the cached python + # is broken and should be uninstalled. + pyenv uninstall -f 3.9.4 + pyenv install -s 3.9.4 + pyenv global 3.9.4 system + else + echo "Python3.9.4 restored from cache, skip installation." + fi python --version install_groovy: &install_groovy @@ -87,6 +149,16 @@ install_ghc: &install_ghc fi ghc --version +install_rust: &install_rust + name: Install Rust + command: | + curl https://sh.rustup.rs -sSf -o install_rust.sh + chmod +x install_rust.sh + ./install_rust.sh -q -y + rm install_rust.sh + export PATH="${HOME}/.cargo/bin:${PATH}" + rustc -V + run_ant_build: &run_ant_build name: Run Ant Build command: | @@ -110,8 +182,7 @@ run_buck_build: &run_buck_build set -ex ./bin/buck build buck --out "${BUCK_PEX_LOCATION}" || { cat "buck-out/log/buck-0.log"; exit 1; } -linux_environment: &linux_environment - # Use string constant for values, no environment variables +linux_environment: &linux_environment # Use string constant for values, no environment variables PLATFORM: "linux" BUCKROOT: "/home/circleci/buck" ANDROID_SDK: "/home/circleci/android-sdk" @@ -126,6 +197,7 @@ macos_environment: &macos_environment TERM: "dumb" BUCK_NUM_THREADS: 3 BUCK_PEX_LOCATION: "./new_buck.pex" + JAVA_HOME: "/usr/local/Cellar/openjdk@8/1.8.0+322" windows_environment: &windows_environment PLATFORM: "windows" @@ -135,6 +207,17 @@ windows_environment: &windows_environment BUCK_NUM_THREADS: 3 BUCK_PEX_LOCATION: "./new_buck.pex" +dockerhub: &dockerhub + context: + - DOCKERHUB_TOKEN + +tags_only_filter: &tags_only_filter + filters: + tags: + only: /^v20.*/ + branches: + ignore: /.*/ + jobs: linux_build_openjdk8: environment: @@ -142,22 +225,42 @@ jobs: working_directory: "/home/circleci/buck" machine: # linux VM - image: ubuntu-1604:201903-01 + image: ubuntu-2004:202107-02 steps: # Steps run sequentially in separate shells - checkout - run: <<: *install_openjdk8 + - restore_cache: + # Change this key when upgrading sdk. + key: v-{{ .Environment.CACHE_VERSION }}-buck-andoid-sdk-{{ .Branch }} - run: <<: *install_android_sdk + - save_cache: + paths: + - ~/android-sdk + key: v-{{ .Environment.CACHE_VERSION }}-buck-andoid-sdk-{{ .Branch }} - run: <<: *install_golang + - restore_cache: + # Change this key when upgrading python. + key: v-{{ .Environment.CACHE_VERSION }}-buck-python3.6.2-{{ .Branch }} - run: <<: *install_python + - save_cache: + paths: + - /opt/circleci/.pyenv + key: v-{{ .Environment.CACHE_VERSION }}-buck-python3.6.2-{{ .Branch }} - run: <<: *run_ant_build + - restore_cache: + key: v-{{ .Environment.CACHE_VERSION }}-buck-build-{{ .Branch }} - run: <<: *run_buck_build + - save_cache: + paths: + - ~/buck/buck-out + key: v-{{ .Environment.CACHE_VERSION }}-buck-build-{{ .Branch }} - run: name: Run Build Tests command: | @@ -172,21 +275,33 @@ jobs: <<: *linux_environment working_directory: "/home/circleci/buck" machine: - image: ubuntu-1604:201903-01 + image: ubuntu-2004:202107-02 steps: - checkout - run: <<: *install_openjdk8 + - restore_cache: + # Change this key when upgrading sdk. + key: v-{{ .Environment.CACHE_VERSION }}-buck-andoid-sdk-{{ .Branch }} + - run: + <<: *install_lua + - run: + <<: *install_ocaml - run: <<: *install_android_sdk - run: <<: *install_groovy - run: <<: *install_golang + - restore_cache: + # Change this key when upgrading python. + key: v-{{ .Environment.CACHE_VERSION }}-buck-python3.6.2-{{ .Branch }} - run: <<: *install_python - run: <<: *run_ant_build + - restore_cache: + key: v-{{ .Environment.CACHE_VERSION }}-buck-build-{{ .Branch }} - run: <<: *run_buck_build - run: @@ -205,19 +320,27 @@ jobs: <<: *linux_environment working_directory: "/home/circleci/buck" machine: - image: ubuntu-1604:201903-01 + image: ubuntu-2004:202107-02 steps: - checkout - run: <<: *install_openjdk8 + - restore_cache: + # Change this key when upgrading sdk. + key: v-{{ .Environment.CACHE_VERSION }}-buck-andoid-sdk-{{ .Branch }} - run: <<: *install_android_sdk - run: <<: *install_golang + - restore_cache: + # Change this key when upgrading python. + key: v-{{ .Environment.CACHE_VERSION }}-buck-python3.6.2-{{ .Branch }} - run: <<: *install_python - run: <<: *run_ant_build + - restore_cache: + key: v-{{ .Environment.CACHE_VERSION }}-buck-build-{{ .Branch }} - run: <<: *run_buck_build - run: @@ -235,45 +358,73 @@ jobs: <<: *linux_environment working_directory: "/home/circleci/buck" machine: - image: ubuntu-1604:201903-01 + image: ubuntu-2004:202107-02 steps: - checkout - run: <<: *install_openjdk8 + - restore_cache: + # Change this key when upgrading sdk. + key: v-{{ .Environment.CACHE_VERSION }}-buck-andoid-sdk-{{ .Branch }} - run: <<: *install_android_sdk + - run: + <<: *install_lua + - run: + <<: *install_ocaml - run: <<: *install_ghc - run: <<: *install_golang + - restore_cache: + # Change this key when upgrading python. + key: v-{{ .Environment.CACHE_VERSION }}-buck-python3.6.2-{{ .Branch }} + - run: + <<: *install_rust - run: <<: *install_python - run: <<: *install_groovy + - run: + <<: *install_dlang - run: <<: *run_ant_build + - restore_cache: + key: v-{{ .Environment.CACHE_VERSION }}-buck-build-{{ .Branch }} - run: <<: *run_buck_build - run: name: Run Integration Tests command: | cd "${BUCKROOT}" + export PATH="${HOME}/.cargo/bin:${PATH}" export PATH="${ANDROID_SDK}/tools/bin:${PATH}" export PATH="$(pyenv root)/shims:${PATH}" - export GROOVY_HOME=$HOME/.sdkman/candidates/groovy/current + source ~/dlang/dmd-2.091.0/activate + export GROOVY_HOME="$HOME/.sdkman/candidates/groovy/current" set -eux - ${BUCK_PEX_LOCATION} test --num-threads=$BUCK_NUM_THREADS --all --filter '^(?!(com.facebook.buck.android|com.facebook.buck.jvm.java)).*[Ii]ntegration.*' + # There is a bug in buck, see issue #2435 for details. skip the tests for now. + ${BUCK_PEX_LOCATION} test --num-threads=$BUCK_NUM_THREADS --all --filter '^(?!(com.facebook.buck.android|com.facebook.buck.jvm.java|com.facebook.buck.features.rust.RustBinaryIntegrationTest|com.facebook.buck.features.rust.RustLinkerIntegrationTest|com.facebook.buck.features.lua.LuaBinaryIntegrationTest|com.facebook.buck.apple.AppleBinaryIntegrationTest|com.facebook.buck.features.haskell.HaskellBinaryIntegrationTest|com.facebook.buck.features.go.GoBinaryIntegrationTest)).*[Ii]ntegration.*' + # Run this after the bug has been fixed: + #${BUCK_PEX_LOCATION} test --num-threads=$BUCK_NUM_THREADS --all --filter '^(?!(com.facebook.buck.android|com.facebook.buck.jvm.java)).*[Ii]ntegration.*' linux_test_heavy_integration: environment: <<: *linux_environment working_directory: "/home/circleci/buck" machine: - image: ubuntu-1604:201903-01 + image: ubuntu-2004:202107-02 steps: - checkout - run: <<: *install_openjdk8 + - restore_cache: + # Change this key when upgrading sdk. + key: v-{{ .Environment.CACHE_VERSION }}-buck-andoid-sdk-{{ .Branch }} + - run: + <<: *install_lua + - run: + <<: *install_ocaml - run: <<: *install_android_sdk - run: @@ -284,10 +435,19 @@ jobs: ./scripts/circleci_install_android_ndk.sh - run: <<: *install_golang + - restore_cache: + # Change this key when upgrading python. + key: v-{{ .Environment.CACHE_VERSION }}-buck-python3.6.2-{{ .Branch }} + - run: + <<: *install_rust - run: <<: *install_python + - run: + <<: *install_dlang - run: <<: *run_ant_build + - restore_cache: + key: v-{{ .Environment.CACHE_VERSION }}-buck-build-{{ .Branch }} - run: <<: *run_buck_build - run: @@ -295,6 +455,8 @@ jobs: command: | export NDK_HOME="${HOME}/android-ndk-linux" export ANDROID_HOME="${ANDROID_SDK}" + source ~/dlang/dmd-2.091.0/activate + export PATH="${HOME}/.cargo/bin:${PATH}" set -eux ${BUCK_PEX_LOCATION} build --num-threads=$BUCK_NUM_THREADS //test/com/facebook/buck/android/... //test/com/facebook/buck/jvm/java/... ${BUCK_PEX_LOCATION} test --num-threads=1 //test/com/facebook/buck/android/... //test/com/facebook/buck/jvm/java/... --filter '.*[Ii]ntegration.*' @@ -304,11 +466,14 @@ jobs: <<: *linux_environment working_directory: "/home/circleci/buck" machine: - image: ubuntu-1604:201903-01 + image: ubuntu-2004:202107-02 steps: - checkout - run: <<: *install_openjdk8 + - restore_cache: + # Change this key when upgrading sdk. + key: v-{{ .Environment.CACHE_VERSION }}-buck-andoid-sdk-{{ .Branch }} - run: <<: *install_android_sdk - run: @@ -319,10 +484,19 @@ jobs: ./scripts/circleci_unzip_android_ndk.sh android-ndk-r15c - run: <<: *install_golang + - restore_cache: + # Change this key when upgrading python. + key: v-{{ .Environment.CACHE_VERSION }}-buck-python3.6.2-{{ .Branch }} + - run: + <<: *install_rust - run: <<: *install_python + - run: + <<: *install_dlang - run: <<: *run_ant_build + - restore_cache: + key: v-{{ .Environment.CACHE_VERSION }}-buck-build-{{ .Branch }} - run: <<: *run_buck_build - run: @@ -330,6 +504,8 @@ jobs: command: | export NDK_HOME="${HOME}/android-ndk-linux" export ANDROID_HOME="${ANDROID_SDK}" + source ~/dlang/dmd-2.091.0/activate + export PATH="${HOME}/.cargo/bin:${PATH}" set -eux ${BUCK_PEX_LOCATION} build --num-threads=$BUCK_NUM_THREADS //test/com/facebook/buck/android/... ${BUCK_PEX_LOCATION} test --num-threads=1 //test/com/facebook/buck/android/... --filter '.*[Ii]ntegration.*' @@ -339,11 +515,14 @@ jobs: <<: *linux_environment working_directory: "/home/circleci/buck" machine: - image: ubuntu-1604:201903-01 + image: ubuntu-2004:202107-02 steps: - checkout - run: <<: *install_openjdk8 + - restore_cache: + # Change this key when upgrading sdk. + key: v-{{ .Environment.CACHE_VERSION }}-buck-andoid-sdk-{{ .Branch }} - run: <<: *install_android_sdk - run: @@ -354,10 +533,19 @@ jobs: ./scripts/circleci_unzip_android_ndk.sh android-ndk-r16b - run: <<: *install_golang + - restore_cache: + # Change this key when upgrading python. + key: v-{{ .Environment.CACHE_VERSION }}-buck-python3.6.2-{{ .Branch }} + - run: + <<: *install_rust - run: <<: *install_python + - run: + <<: *install_dlang - run: <<: *run_ant_build + - restore_cache: + key: v-{{ .Environment.CACHE_VERSION }}-buck-build-{{ .Branch }} - run: <<: *run_buck_build - run: @@ -365,6 +553,8 @@ jobs: command: | export NDK_HOME="${HOME}/android-ndk-linux" export ANDROID_HOME="${ANDROID_SDK}" + source ~/dlang/dmd-2.091.0/activate + export PATH="${HOME}/.cargo/bin:${PATH}" set -eux ${BUCK_PEX_LOCATION} build --num-threads=$BUCK_NUM_THREADS //test/com/facebook/buck/android/... ${BUCK_PEX_LOCATION} test --num-threads=1 //test/com/facebook/buck/android/... --filter '.*[Ii]ntegration.*' @@ -374,11 +564,14 @@ jobs: <<: *linux_environment working_directory: "/home/circleci/buck" machine: - image: ubuntu-1604:201903-01 + image: ubuntu-2004:202107-02 steps: - checkout - run: <<: *install_openjdk8 + - restore_cache: + # Change this key when upgrading sdk. + key: v-{{ .Environment.CACHE_VERSION }}-buck-andoid-sdk-{{ .Branch }} - run: <<: *install_android_sdk - run: @@ -389,14 +582,23 @@ jobs: ./scripts/circleci_unzip_android_ndk.sh android-ndk-r17b - run: <<: *install_golang + - restore_cache: + # Change this key when upgrading python. + key: v-{{ .Environment.CACHE_VERSION }}-buck-python3.6.2-{{ .Branch }} + - run: + <<: *install_rust - run: <<: *install_python + - run: + <<: *install_dlang - run: <<: *run_ant_build + - restore_cache: + key: v-{{ .Environment.CACHE_VERSION }}-buck-build-{{ .Branch }} - run: <<: *run_buck_build - run: - name: Need android-27 and android-28 to run the tests, install them. + name: Need android-27 and android-28 to run the tests, install them. command: | export PATH="${ANDROID_SDK}/tools/bin:${PATH}" echo 'y' |sdkmanager --install "platforms;android-27" @@ -404,8 +606,12 @@ jobs: - run: name: Run Android NDK 17 Tests command: | + # libcurses5 is needed for providing libtinfo.so.5 + sudo apt install libncurses5 export NDK_HOME="${HOME}/android-ndk-linux" export ANDROID_HOME="${ANDROID_SDK}" + source ~/dlang/dmd-2.091.0/activate + export PATH="${HOME}/.cargo/bin:${PATH}" set -eux ${BUCK_PEX_LOCATION} build --num-threads=$BUCK_NUM_THREADS //test/com/facebook/buck/android/... ${BUCK_PEX_LOCATION} test --num-threads=1 //test/com/facebook/buck/android/... --filter '.*[Ii]ntegration.*' @@ -416,11 +622,14 @@ jobs: <<: *linux_environment working_directory: "/home/circleci/buck" machine: - image: ubuntu-1604:201903-01 + image: ubuntu-2004:202107-02 steps: - checkout - run: <<: *install_openjdk8 + - restore_cache: + # Change this key when upgrading sdk. + key: v-{{ .Environment.CACHE_VERSION }}-buck-andoid-sdk-{{ .Branch }} - run: <<: *install_android_sdk - run: @@ -431,10 +640,19 @@ jobs: ./scripts/circleci_unzip_android_ndk.sh android-ndk-r18b - run: <<: *install_golang + - restore_cache: + # Change this key when upgrading python. + key: v-{{ .Environment.CACHE_VERSION }}-buck-python3.6.2-{{ .Branch }} + - run: + <<: *install_rust - run: <<: *install_python + - run: + <<: *install_dlang - run: <<: *run_ant_build + - restore_cache: + key: v-{{ .Environment.CACHE_VERSION }}-buck-build-{{ .Branch }} - run: <<: *run_buck_build - run: @@ -442,6 +660,155 @@ jobs: command: | export NDK_HOME="${HOME}/android-ndk-linux" export ANDROID_HOME="${ANDROID_SDK}" + source ~/dlang/dmd-2.091.0/activate + export PATH="${HOME}/.cargo/bin:${PATH}" + set -eux + ${BUCK_PEX_LOCATION} build --num-threads=$BUCK_NUM_THREADS //test/com/facebook/buck/android/... + ${BUCK_PEX_LOCATION} test --num-threads=1 //test/com/facebook/buck/android/... --filter '.*[Ii]ntegration.*' + + linux_test_android_ndk_19: + environment: + <<: *linux_environment + working_directory: "/home/circleci/buck" + machine: + image: ubuntu-2004:202107-02 + steps: + - checkout + - run: + <<: *install_openjdk8 + - restore_cache: + # Change this key when upgrading sdk. + key: v-{{ .Environment.CACHE_VERSION }}-buck-andoid-sdk-{{ .Branch }} + - run: + <<: *install_android_sdk + - run: + name: Install Android NDK 19 + command: | + cd "${BUCKROOT}" + export NDK_HOME="${HOME}/android-ndk-linux" + ./scripts/circleci_unzip_android_ndk.sh android-ndk-r19c + - run: + <<: *install_golang + - restore_cache: + # Change this key when upgrading python. + key: v-{{ .Environment.CACHE_VERSION }}-buck-python3.6.2-{{ .Branch }} + - run: + <<: *install_rust + - run: + <<: *install_python + - run: + <<: *install_dlang + - run: + <<: *run_ant_build + - restore_cache: + key: v-{{ .Environment.CACHE_VERSION }}-buck-build-{{ .Branch }} + - run: + <<: *run_buck_build + - run: + name: Run Android NDK 19 Tests + command: | + export NDK_HOME="${HOME}/android-ndk-linux" + export ANDROID_HOME="${ANDROID_SDK}" + source ~/dlang/dmd-2.091.0/activate + export PATH="${HOME}/.cargo/bin:${PATH}" + set -eux + ${BUCK_PEX_LOCATION} build --num-threads=$BUCK_NUM_THREADS //test/com/facebook/buck/android/... + ${BUCK_PEX_LOCATION} test --num-threads=1 //test/com/facebook/buck/android/... --filter '.*[Ii]ntegration.*' + + linux_test_android_ndk_20: + environment: + <<: *linux_environment + working_directory: "/home/circleci/buck" + machine: + image: ubuntu-2004:202107-02 + steps: + - checkout + - run: + <<: *install_openjdk8 + - restore_cache: + # Change this key when upgrading sdk. + key: v-{{ .Environment.CACHE_VERSION }}-buck-andoid-sdk-{{ .Branch }} + - run: + <<: *install_android_sdk + - run: + name: Install Android NDK 20 + command: | + cd "${BUCKROOT}" + export NDK_HOME="${HOME}/android-ndk-linux" + ./scripts/circleci_unzip_android_ndk.sh android-ndk-r20b + - run: + <<: *install_golang + - restore_cache: + # Change this key when upgrading python. + key: v-{{ .Environment.CACHE_VERSION }}-buck-python3.6.2-{{ .Branch }} + - run: + <<: *install_rust + - run: + <<: *install_python + - run: + <<: *install_dlang + - run: + <<: *run_ant_build + - restore_cache: + key: v-{{ .Environment.CACHE_VERSION }}-buck-build-{{ .Branch }} + - run: + <<: *run_buck_build + - run: + name: Run Android NDK 20 Tests + command: | + export NDK_HOME="${HOME}/android-ndk-linux" + export ANDROID_HOME="${ANDROID_SDK}" + source ~/dlang/dmd-2.091.0/activate + export PATH="${HOME}/.cargo/bin:${PATH}" + set -eux + ${BUCK_PEX_LOCATION} build --num-threads=$BUCK_NUM_THREADS //test/com/facebook/buck/android/... + ${BUCK_PEX_LOCATION} test --num-threads=1 //test/com/facebook/buck/android/... --filter '.*[Ii]ntegration.*' + + linux_test_android_ndk_21: + environment: + <<: *linux_environment + working_directory: "/home/circleci/buck" + machine: + image: ubuntu-2004:202107-02 + steps: + - checkout + - run: + <<: *install_openjdk8 + - restore_cache: + # Change this key when upgrading sdk. + key: v-{{ .Environment.CACHE_VERSION }}-buck-andoid-sdk-{{ .Branch }} + - run: + <<: *install_android_sdk + - run: + name: Install Android NDK 21 + command: | + cd "${BUCKROOT}" + export NDK_HOME="${HOME}/android-ndk-linux" + ./scripts/circleci_unzip_android_ndk.sh android-ndk-r21d + - run: + <<: *install_golang + - restore_cache: + # Change this key when upgrading python. + key: v-{{ .Environment.CACHE_VERSION }}-buck-python3.6.2-{{ .Branch }} + - run: + <<: *install_rust + - run: + <<: *install_python + - run: + <<: *install_dlang + - run: + <<: *run_ant_build + - restore_cache: + key: v-{{ .Environment.CACHE_VERSION }}-buck-build-{{ .Branch }} + - run: + <<: *run_buck_build + - run: + name: Run Android NDK 21 Tests + command: | + export NDK_HOME="${HOME}/android-ndk-linux" + export ANDROID_HOME="${ANDROID_SDK}" + source ~/dlang/dmd-2.091.0/activate + export PATH="${HOME}/.cargo/bin:${PATH}" set -eux ${BUCK_PEX_LOCATION} build --num-threads=$BUCK_NUM_THREADS //test/com/facebook/buck/android/... ${BUCK_PEX_LOCATION} test --num-threads=1 //test/com/facebook/buck/android/... --filter '.*[Ii]ntegration.*' @@ -451,22 +818,42 @@ jobs: <<: *macos_environment working_directory: "/Users/distiller/buck" macos: - xcode: "11.3.1" + xcode: "12.5.1" steps: # Steps run sequentially in separate shells - checkout - run: <<: *install_openjdk8 + - restore_cache: + # Change this key when upgrading sdk. + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-android-sdk-{{ .Branch }} - run: <<: *install_android_sdk + - save_cache: + paths: + - ~/android-sdk + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-android-sdk-{{ .Branch }} - run: <<: *install_golang + - restore_cache: + # Change this key when upgrading python. + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-python3.6.2-{{ .Branch }} - run: <<: *install_python + - save_cache: + paths: + - ~/.pyenv + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-python3.6.2-{{ .Branch }} - run: <<: *run_ant_build + - restore_cache: + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-build-{{ .Branch }} - run: <<: *run_buck_build + - save_cache: + paths: + - ~/buck/buck-out + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-build-{{ .Branch }} - run: name: Run Build Tests command: | @@ -481,21 +868,33 @@ jobs: <<: *macos_environment working_directory: "/Users/distiller/buck" macos: - xcode: "11.3.1" + xcode: "12.5.1" steps: - checkout - run: <<: *install_openjdk8 + - restore_cache: + # Change this key when upgrading sdk. + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-android-sdk-{{ .Branch }} + - run: + <<: *install_lua + - run: + <<: *install_ocaml - run: <<: *install_android_sdk - run: <<: *install_groovy - run: <<: *install_golang + - restore_cache: + # Change this key when upgrading python. + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-python3.6.2-{{ .Branch }} - run: <<: *install_python - run: <<: *run_ant_build + - restore_cache: + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-build-{{ .Branch }} - run: <<: *run_buck_build - run: @@ -513,19 +912,27 @@ jobs: <<: *macos_environment working_directory: "/Users/distiller/buck" macos: - xcode: "11.3.1" + xcode: "12.5.1" steps: - checkout - run: <<: *install_openjdk8 + - restore_cache: + # Change this key when upgrading sdk. + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-android-sdk-{{ .Branch }} - run: <<: *install_android_sdk - run: <<: *install_golang + - restore_cache: + # Change this key when upgrading python. + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-python3.6.2-{{ .Branch }} - run: <<: *install_python - run: <<: *run_ant_build + - restore_cache: + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-build-{{ .Branch }} - run: <<: *run_buck_build - run: @@ -544,31 +951,50 @@ jobs: <<: *macos_environment working_directory: "/Users/distiller/buck" macos: - # The tests do not support xcode 11.x.x yet. - xcode: "10.3.0" + xcode: "12.5.1" steps: - checkout - run: <<: *install_openjdk8 + - restore_cache: + # Change this key when upgrading sdk. + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-android-sdk-{{ .Branch }} + - run: + <<: *install_lua + - run: + <<: *install_ocaml - run: <<: *install_android_sdk - run: <<: *install_ghc - run: <<: *install_golang + - restore_cache: + # Change this key when upgrading python. + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-python3.6.2-{{ .Branch }} - run: - <<: *install_python - - run: - <<: *install_groovy + <<: *install_rust - run: # There is an issue with python in macos 10.14.x. this step is a work around and should be removed later. # https://stackoverflow.com/questions/59269208/errorrootcode-for-hash-md5-was-not-found-not-able-to-use-any-hg-mercurial-co - name: Uninstall Python2 - # Python2 was EOL on 1/1/2020, it does not work with Xcode 10.3.0 + name: Reinstall Python2 command: | - brew uninstall python@2 + brew uninstall python@2 || echo "python2 was not installed." + url=https://www.python.org/ftp/python/2.7.17/python-2.7.17-macosx10.9.pkg + curl -R -L -O $url + sudo installer -pkg python-2.7.17-macosx10.9.pkg -target / + # 2to3 causes python3 installation to fail, delete it. + rm -f /usr/local/bin/2to3 + - run: + <<: *install_python + - run: + <<: *install_groovy + - run: + <<: *install_dlang - run: <<: *run_ant_build + - restore_cache: + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-build-{{ .Branch }} - run: <<: *run_buck_build - run: @@ -577,25 +1003,35 @@ jobs: cd "${BUCKROOT}" export PATH="${ANDROID_SDK}/tools/bin:${PATH}" export PATH="$(pyenv root)/shims:${PATH}" - export "GROOVY_HOME=$HOME/.sdkman/candidates/groovy/current" - export JAVA_HOME=`/usr/libexec/java_home` + export GROOVY_HOME="$HOME/.sdkman/candidates/groovy/current" + export JAVA_HOME="/usr/local/Cellar/openjdk@8/1.8.0+322" export PATH="${JAVA_HOME}/bin:${PATH}" + source ~/dlang/dmd-2.091.0/activate + export PATH="${HOME}/.cargo/bin:${PATH}" set -ex # Should run this: #${BUCK_PEX_LOCATION} test --num-threads=$BUCK_NUM_THREADS --all --filter '^(?!(com.facebook.buck.android|com.facebook.buck.jvm.java)).*[Ii]ntegration.*' # But there are some test failures, exclude them for now and fix them later. - ${BUCK_PEX_LOCATION} test --num-threads=$BUCK_NUM_THREADS --all --filter '^(?!(com.facebook.buck.android|com.facebook.buck.jvm.java|com.facebook.buck.features.haskell.HaskellBinary|com.facebook.buck.swift.SwiftTestIO|com.facebook.buck.apple.AppleTest)).*[Ii]ntegration.*' + ${BUCK_PEX_LOCATION} test --num-threads=$BUCK_NUM_THREADS --all --filter '^(?!(com.facebook.buck.android|com.facebook.buck.jvm.java|com.facebook.buck.features.haskell.HaskellBinary|com.facebook.buck.swift.SwiftTestIO|com.facebook.buck.apple|com.facebook.buck.features.lua.LuaBinaryIntegrationTest|com.facebook.buck.testrunner.RunWithDefaultTimeoutIntegrationTest|com.facebook.buck.features.apple.project.ProjectIntegrationTest|com.facebook.buck.swift.SwiftIOSBundleIntegrationTest)).*[Ii]ntegration.*' + no_output_timeout: 1h macos_test_heavy_integration: environment: <<: *macos_environment working_directory: "/Users/distiller/buck" macos: - xcode: "11.3.1" + xcode: "12.5.1" steps: - checkout - run: <<: *install_openjdk8 + - restore_cache: + # Change this key when upgrading sdk. + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-android-sdk-{{ .Branch }} + - run: + <<: *install_lua + - run: + <<: *install_ocaml - run: <<: *install_android_sdk - run: @@ -606,10 +1042,19 @@ jobs: ./scripts/circleci_install_android_ndk.sh "${PLATFORM}" - run: <<: *install_golang + - restore_cache: + # Change this key when upgrading python. + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-python3.6.2-{{ .Branch }} + - run: + <<: *install_rust - run: <<: *install_python + - run: + <<: *install_dlang - run: <<: *run_ant_build + - restore_cache: + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-build-{{ .Branch }} - run: <<: *run_buck_build - run: @@ -617,8 +1062,10 @@ jobs: command: | export NDK_HOME="${HOME}/android-ndk-${PLATFORM}" export ANDROID_HOME="${ANDROID_SDK}" - export JAVA_HOME=`/usr/libexec/java_home` + export JAVA_HOME="/usr/local/Cellar/openjdk@8/1.8.0+322" export PATH="${JAVA_HOME}/bin:${PATH}" + source ~/dlang/dmd-2.091.0/activate + export PATH="${HOME}/.cargo/bin:${PATH}" set -eux ${BUCK_PEX_LOCATION} build --num-threads=$BUCK_NUM_THREADS //test/com/facebook/buck/android/... //test/com/facebook/buck/jvm/java/... ${BUCK_PEX_LOCATION} test --num-threads=1 //test/com/facebook/buck/android/... //test/com/facebook/buck/jvm/java/... --filter '.*[Ii]ntegration.*' @@ -628,11 +1075,14 @@ jobs: <<: *macos_environment working_directory: "/Users/distiller/buck" macos: - xcode: "11.3.1" + xcode: "12.5.1" steps: - checkout - run: <<: *install_openjdk8 + - restore_cache: + # Change this key when upgrading sdk. + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-android-sdk-{{ .Branch }} - run: <<: *install_android_sdk - run: @@ -643,10 +1093,19 @@ jobs: ./scripts/circleci_unzip_android_ndk.sh android-ndk-r15c "${PLATFORM}" - run: <<: *install_golang + - restore_cache: + # Change this key when upgrading python. + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-python3.6.2-{{ .Branch }} + - run: + <<: *install_rust - run: <<: *install_python + - run: + <<: *install_dlang - run: <<: *run_ant_build + - restore_cache: + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-build-{{ .Branch }} - run: <<: *run_buck_build - run: @@ -654,6 +1113,8 @@ jobs: command: | export NDK_HOME="${HOME}/android-ndk-${PLATFORM}" export ANDROID_HOME="${ANDROID_SDK}" + source ~/dlang/dmd-2.091.0/activate + export PATH="${HOME}/.cargo/bin:${PATH}" set -eux ${BUCK_PEX_LOCATION} build --num-threads=$BUCK_NUM_THREADS //test/com/facebook/buck/android/... ${BUCK_PEX_LOCATION} test --num-threads=1 //test/com/facebook/buck/android/... --filter '.*[Ii]ntegration.*' @@ -663,11 +1124,14 @@ jobs: <<: *macos_environment working_directory: "/Users/distiller/buck" macos: - xcode: "11.3.1" + xcode: "12.5.1" steps: - checkout - run: <<: *install_openjdk8 + - restore_cache: + # Change this key when upgrading sdk. + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-android-sdk-{{ .Branch }} - run: <<: *install_android_sdk - run: @@ -678,10 +1142,19 @@ jobs: ./scripts/circleci_unzip_android_ndk.sh android-ndk-r16b "${PLATFORM}" - run: <<: *install_golang + - restore_cache: + # Change this key when upgrading python. + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-python3.6.2-{{ .Branch }} + - run: + <<: *install_rust - run: <<: *install_python + - run: + <<: *install_dlang - run: <<: *run_ant_build + - restore_cache: + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-build-{{ .Branch }} - run: <<: *run_buck_build - run: @@ -689,6 +1162,8 @@ jobs: command: | export NDK_HOME="${HOME}/android-ndk-${PLATFORM}" export ANDROID_HOME="${ANDROID_SDK}" + source ~/dlang/dmd-2.091.0/activate + export PATH="${HOME}/.cargo/bin:${PATH}" set -eux ${BUCK_PEX_LOCATION} build --num-threads=$BUCK_NUM_THREADS //test/com/facebook/buck/android/... ${BUCK_PEX_LOCATION} test --num-threads=1 //test/com/facebook/buck/android/... --filter '.*[Ii]ntegration.*' @@ -698,11 +1173,14 @@ jobs: <<: *macos_environment working_directory: "/Users/distiller/buck" macos: - xcode: "11.3.1" + xcode: "12.5.1" steps: - checkout - run: <<: *install_openjdk8 + - restore_cache: + # Change this key when upgrading sdk. + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-android-sdk-{{ .Branch }} - run: <<: *install_android_sdk - run: @@ -713,10 +1191,19 @@ jobs: ./scripts/circleci_unzip_android_ndk.sh android-ndk-r17b "${PLATFORM}" - run: <<: *install_golang + - restore_cache: + # Change this key when upgrading python. + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-python3.6.2-{{ .Branch }} + - run: + <<: *install_rust - run: <<: *install_python + - run: + <<: *install_dlang - run: <<: *run_ant_build + - restore_cache: + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-build-{{ .Branch }} - run: <<: *run_buck_build - run: @@ -730,6 +1217,8 @@ jobs: command: | export NDK_HOME="${HOME}/android-ndk-${PLATFORM}" export ANDROID_HOME="${ANDROID_SDK}" + source ~/dlang/dmd-2.091.0/activate + export PATH="${HOME}/.cargo/bin:${PATH}" set -eux ${BUCK_PEX_LOCATION} build --num-threads=$BUCK_NUM_THREADS //test/com/facebook/buck/android/... # Exclude some long running tests, otherwise CircleCI will timeout after 5 hours. @@ -744,11 +1233,14 @@ jobs: <<: *macos_environment working_directory: "/Users/distiller/buck" macos: - xcode: "11.3.1" + xcode: "12.5.1" steps: - checkout - run: <<: *install_openjdk8 + - restore_cache: + # Change this key when upgrading sdk. + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-android-sdk-{{ .Branch }} - run: <<: *install_android_sdk - run: @@ -759,10 +1251,19 @@ jobs: ./scripts/circleci_unzip_android_ndk.sh android-ndk-r18b "${PLATFORM}" - run: <<: *install_golang + - restore_cache: + # Change this key when upgrading python. + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-python3.6.2-{{ .Branch }} + - run: + <<: *install_rust - run: <<: *install_python + - run: + <<: *install_dlang - run: <<: *run_ant_build + - restore_cache: + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-build-{{ .Branch }} - run: <<: *run_buck_build - run: @@ -770,9 +1271,159 @@ jobs: command: | export NDK_HOME="${HOME}/android-ndk-${PLATFORM}" export ANDROID_HOME="${ANDROID_SDK}" + source ~/dlang/dmd-2.091.0/activate + export PATH="${HOME}/.cargo/bin:${PATH}" + set -eux + ${BUCK_PEX_LOCATION} build --num-threads=$BUCK_NUM_THREADS //test/com/facebook/buck/android/... + ${BUCK_PEX_LOCATION} test --num-threads=1 //test/com/facebook/buck/android/... --filter '.*[Ii]ntegration.*' + + macos_test_android_ndk_19: + environment: + <<: *macos_environment + working_directory: "/Users/distiller/buck" + macos: + xcode: "12.5.1" + steps: + - checkout + - run: + <<: *install_openjdk8 + - restore_cache: + # Change this key when upgrading sdk. + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-android-sdk-{{ .Branch }} + - run: + <<: *install_android_sdk + - run: + name: Install Android NDK 19 + command: | + cd "${BUCKROOT}" + export NDK_HOME="${HOME}/android-ndk-${PLATFORM}" + ./scripts/circleci_unzip_android_ndk.sh android-ndk-r19c "${PLATFORM}" + - run: + <<: *install_golang + - restore_cache: + # Change this key when upgrading python. + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-python3.6.2-{{ .Branch }} + - run: + <<: *install_rust + - run: + <<: *install_python + - run: + <<: *install_dlang + - run: + <<: *run_ant_build + - restore_cache: + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-build-{{ .Branch }} + - run: + <<: *run_buck_build + - run: + name: Run Android NDK 19 Tests + command: | + export NDK_HOME="${HOME}/android-ndk-${PLATFORM}" + export ANDROID_HOME="${ANDROID_SDK}" + source ~/dlang/dmd-2.091.0/activate + export PATH="${HOME}/.cargo/bin:${PATH}" set -eux ${BUCK_PEX_LOCATION} build --num-threads=$BUCK_NUM_THREADS //test/com/facebook/buck/android/... ${BUCK_PEX_LOCATION} test --num-threads=1 //test/com/facebook/buck/android/... --filter '.*[Ii]ntegration.*' + + macos_test_android_ndk_20: + environment: + <<: *macos_environment + working_directory: "/Users/distiller/buck" + macos: + xcode: "12.5.1" + steps: + - checkout + - run: + <<: *install_openjdk8 + - restore_cache: + # Change this key when upgrading sdk. + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-android-sdk-{{ .Branch }} + - run: + <<: *install_android_sdk + - run: + name: Install Android NDK 20 + command: | + cd "${BUCKROOT}" + export NDK_HOME="${HOME}/android-ndk-${PLATFORM}" + ./scripts/circleci_unzip_android_ndk.sh android-ndk-r20b "${PLATFORM}" + - run: + <<: *install_golang + - restore_cache: + # Change this key when upgrading python. + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-python3.6.2-{{ .Branch }} + - run: + <<: *install_rust + - run: + <<: *install_python + - run: + <<: *install_dlang + - run: + <<: *run_ant_build + - restore_cache: + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-build-{{ .Branch }} + - run: + <<: *run_buck_build + - run: + name: Run Android NDK 20 Tests + command: | + export NDK_HOME="${HOME}/android-ndk-${PLATFORM}" + export ANDROID_HOME="${ANDROID_SDK}" + source ~/dlang/dmd-2.091.0/activate + export PATH="${HOME}/.cargo/bin:${PATH}" + set -eux + ${BUCK_PEX_LOCATION} build --num-threads=$BUCK_NUM_THREADS //test/com/facebook/buck/android/... + ${BUCK_PEX_LOCATION} test --num-threads=1 //test/com/facebook/buck/android/... --filter '.*[Ii]ntegration.*' + + macos_test_android_ndk_21: + environment: + <<: *macos_environment + working_directory: "/Users/distiller/buck" + macos: + xcode: "12.5.1" + steps: + - checkout + - run: + <<: *install_openjdk8 + - restore_cache: + # Change this key when upgrading sdk. + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-android-sdk-{{ .Branch }} + - run: + <<: *install_android_sdk + - run: + name: Install Android NDK 21 + command: | + cd "${BUCKROOT}" + export NDK_HOME="${HOME}/android-ndk-${PLATFORM}" + ./scripts/circleci_unzip_android_ndk.sh android-ndk-r21d "${PLATFORM}" + - run: + <<: *install_golang + - restore_cache: + # Change this key when upgrading python. + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-python3.6.2-{{ .Branch }} + - run: + <<: *install_rust + - run: + <<: *install_python + - run: + <<: *install_dlang + - run: + <<: *run_ant_build + - restore_cache: + key: v-{{ .Environment.CACHE_VERSION }}-buck-macos-build-{{ .Branch }} + - run: + <<: *run_buck_build + - run: + name: Run Android NDK 21 Tests + command: | + export NDK_HOME="${HOME}/android-ndk-${PLATFORM}" + export ANDROID_HOME="${ANDROID_SDK}" + source ~/dlang/dmd-2.091.0/activate + export PATH="${HOME}/.cargo/bin:${PATH}" + set -eux + ${BUCK_PEX_LOCATION} build --num-threads=$BUCK_NUM_THREADS //test/com/facebook/buck/android/... + ${BUCK_PEX_LOCATION} test --num-threads=1 //test/com/facebook/buck/android/... --filter '.*[Ii]ntegration.*' + windows_build_test: environment: <<: *windows_environment @@ -785,9 +1436,43 @@ jobs: command: choco install ant --version=1.9.7 shell: cmd.exe - run: - name: Install OpenJDK8 - command: choco install adoptopenjdk8 + name: Install OpenJDK11 + command: choco install adoptopenjdk11 shell: cmd.exe + - run: + # on Windows, python3 is installed as python but buck expect the name python3 + name: Create symlink to python + command: $pythonpath=(Split-Path((Get-Command python).Path)); cmd /c mklink ${pythonpath}\python3.exe ${pythonpath}\python.exe + shell: powershell.exe + - run: + name: Ant build + command: cd %BUCKROOT% && refreshenv && ant + shell: cmd.exe + - run: + name: Buck build + command: cd %BUCKROOT% && refreshenv && bin\buck build buck -c python.interpreter=python + shell: cmd.exe + + windows_tests: + environment: + <<: *windows_environment + working_directory: "C:\\Users\\circleci\\buck" + executor: win/default + steps: + - checkout + - run: + name: Install ant + command: choco install ant --version=1.9.7 + shell: cmd.exe + - run: + name: Install OpenJDK11 + command: choco install adoptopenjdk11 + shell: cmd.exe + - run: + # on Windows, python3 is installed as python but buck expect the name python3 + name: Create symlink to python + command: $pythonpath=(Split-Path((Get-Command python).Path)); cmd /c mklink ${pythonpath}\python3.exe ${pythonpath}\python.exe + shell: powershell.exe - run: name: Ant build command: cd %BUCKROOT% && refreshenv && ant @@ -797,12 +1482,132 @@ jobs: command: cd %BUCKROOT% && refreshenv && bin\buck build buck -c python.interpreter=python shell: cmd.exe # We know there are some Windows test failures, comment out for now. They should be fixed. - #- run: - # name: Run Windows tests - # command: | - # cd %BUCKROOT% - # bin\buck test --all --test-selectors=:windows_failures.txt --test-selectors=:windows_cxx_support.txt - # shell: cmd.exe + - run: + name: Run Windows tests + command: | + cd %BUCKROOT% && refreshenv && bin\buck test --all --test-selectors=:windows_failures.txt --test-selectors=:windows_cxx_support.txt + shell: cmd.exe + + macos_publish_release: + environment: + <<: *macos_environment + working_directory: "/Users/distiller/buck" + macos: + xcode: "12.5.1" + steps: + - checkout + - run: + name: Upgrade pip + command: python3 -m pip install --upgrade pip + - run: + name: pip install requests + command: python3 -m pip install requests + - run: + name: pip install python-dateutil + command: python3 -m pip install python-dateutil + - run: + name: pip install python-magic-bin + command: python3 -m pip install python-magic-bin + - run: + name: Run Homebrew Release + command: | + VERSION=${CIRCLE_TAG:1} + python3 ./tools/release/publish_release.py --no-build-chocolatey --no-chocolatey-publish --no-build-deb --github-token ${GITHUB_TOKEN} --use-existing-release --version ${VERSION} --output-dir artifacts + + linux_publish_release: + environment: + <<: *linux_environment + working_directory: "/home/circleci/buck" + machine: + image: ubuntu-2004:202107-02 + steps: + - checkout + - run: + <<: *install_python + - run: + name: Upgrade pip + command: python3 -m pip install --upgrade pip + - run: + name: pip install requests + command: python3 -m pip install requests + - run: + name: pip install python-dateutil + command: python3 -m pip install python-dateutil + - run: + name: pip install python-magic + command: python3 -m pip install python-magic + - run: + name: Run linux release + command: | + VERSION=${CIRCLE_TAG:1} + python3 ./tools/release/publish_release.py --no-build-chocolatey --no-chocolatey-publish --no-build-homebrew --github-token ${GITHUB_TOKEN} --use-existing-release --version ${VERSION} --output-dir artifacts + + windows_publish_release: + environment: + <<: *windows_environment + working_directory: "C:\\Users\\circleci\\buck" + executor: win/default + steps: + - checkout + - run: + name: Make sure Python3 exists + command: | + $pythonpath=(Split-Path((Get-Command python).Path)) + If (Test-Path -Path ${pythonpath}\python3.exe) { + echo "using ${pythonpath}\python3.exe" + } + else { + cmd /c mklink ${pythonpath}\python3.exe ${pythonpath}\python.exe + } + shell: powershell.exe + - run: + name: Upgrade pip + command: python3 -m pip install --upgrade pip + - run: + name: pip install requests + command: python3 -m pip install requests + - run: + name: pip install python-dateutil + command: python3 -m pip install python-dateutil + - run: + name: pip install python-magic-bin + command: python3 -m pip install python-magic-bin + - run: + name: Run windows release + command: | + VERSION=${CIRCLE_TAG:1} + python3 ./tools/release/publish_release.py --no-build-deb --no-build-homebrew --github-token ${GITHUB_TOKEN} --chocolatey-token ${CHOCO_TOKEN} --use-existing-release --version ${VERSION} --output-dir artifacts + shell: bash.exe + + publish_docs: + environment: + <<: *linux_environment + working_directory: "/home/circleci/buck" + machine: + image: ubuntu-2004:202107-02 + steps: + - checkout + - run: + <<: *install_openjdk8 + - run: + # android_sdk needed to build java docs. + <<: *install_android_sdk + - run: + <<: *install_python + - run: + # We do not want to build buck, install the latest release instead. + name: Install Buck + command: | + url=`curl -sH "Authorization: token ${GITHUB_TOKEN}" https://api.github.com/repos/facebook/buck/releases/latest |grep "browser_download_url.*deb" |awk '{gsub("\"", "", $2); print $2}'` + curl -L -O $url + filename=`basename ${url}` + sudo dpkg -i ${filename} || echo "Warning: Buck installed without dependencies." + - run: + name: Publish docs + command: | + export ANDROID_HOME="${ANDROID_SDK}" + cd docs + ./publish.sh --start-soyweb workflows: version: 2.1 @@ -818,6 +1623,9 @@ workflows: - linux_test_android_ndk_16 - linux_test_android_ndk_17 - linux_test_android_ndk_18 + - linux_test_android_ndk_19 + - linux_test_android_ndk_20 + - linux_test_android_ndk_21 macos_jobs: jobs: @@ -830,7 +1638,24 @@ workflows: - macos_test_android_ndk_16 - macos_test_android_ndk_17 - macos_test_android_ndk_18 + - macos_test_android_ndk_19 + - macos_test_android_ndk_20 + - macos_test_android_ndk_21 windows_jobs: jobs: - windows_build_test + - windows_tests + + publish_jobs: + jobs: + - macos_publish_release: + <<: [*tags_only_filter, *dockerhub] + - linux_publish_release: + <<: [*tags_only_filter, *dockerhub] + - windows_publish_release: + <<: [*tags_only_filter, *dockerhub] + - publish_docs: + requires: + - linux_publish_release + <<: *tags_only_filter diff --git a/.idea/libraries/buck_lib.xml b/.idea/libraries/buck_lib.xml index e92530beec5..9eaa20b11fa 100644 --- a/.idea/libraries/buck_lib.xml +++ b/.idea/libraries/buck_lib.xml @@ -5,18 +5,18 @@ - - + + - + - + - - + + @@ -27,18 +27,18 @@ - - + + - + - - + + diff --git a/.idea/libraries/jna.xml b/.idea/libraries/jna.xml index d05e27b8ef4..9e6c23fceff 100644 --- a/.idea/libraries/jna.xml +++ b/.idea/libraries/jna.xml @@ -1,11 +1,13 @@ - + - + + + - + \ No newline at end of file diff --git a/.idea/libraries/jna_4_5_1.xml b/.idea/libraries/jna_4_5_1.xml deleted file mode 100644 index df7d4364463..00000000000 --- a/.idea/libraries/jna_4_5_1.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/libraries/jna_5_6_0.xml b/.idea/libraries/jna_5_6_0.xml new file mode 100644 index 00000000000..bd4782cb4f9 --- /dev/null +++ b/.idea/libraries/jna_5_6_0.xml @@ -0,0 +1,11 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/libraries/jna_platform.xml b/.idea/libraries/jna_platform.xml index 542205ed3ee..8fb34d80ae0 100644 --- a/.idea/libraries/jna_platform.xml +++ b/.idea/libraries/jna_platform.xml @@ -1,11 +1,13 @@ - + - + + + - + \ No newline at end of file diff --git a/.idea/libraries/jna_platform_4_5_1.xml b/.idea/libraries/jna_platform_5_6_0.xml similarity index 73% rename from .idea/libraries/jna_platform_4_5_1.xml rename to .idea/libraries/jna_platform_5_6_0.xml index 24acb1e24ad..ddb25ee37df 100644 --- a/.idea/libraries/jna_platform_4_5_1.xml +++ b/.idea/libraries/jna_platform_5_6_0.xml @@ -1,11 +1,11 @@ - + - + - + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml index f627902d77b..eafa91e5044 100644 --- a/.idea/vcs.xml +++ b/.idea/vcs.xml @@ -1,6 +1,6 @@ - - - - \ No newline at end of file + + + + diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 4912a42e949..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,334 +0,0 @@ -language: android - -matrix: - include: - - os: linux - jdk: oraclejdk8 - python: - - 3.6 - - 2.7 - dist: trusty - addons: - apt: - packages: - - ant - env: CI_ACTION=build - # ANDROID_SDK is required for build. - android: - components: - - tools - - platform-tools - - build-tools-23.0.2 - - android-23 - - os: linux - jdk: openjdk8 - python: - - 3.6 - - 2.7 - dist: trusty - addons: - apt: - packages: - - ant - env: CI_ACTION=build - # ANDROID_SDK is required for build. - android: - components: - - tools - - platform-tools - - build-tools-23.0.2 - - android-23 - - os: linux - jdk: openjdk8 - python: - - 3.6 - - 2.7 - dist: trusty - addons: - apt: - packages: - - ant - - groovy - env: CI_ACTION=unit GROOVY_HOME=/usr/share/groovy/ - # ANDROID_SDK is required for build. - android: - components: - - tools - - platform-tools - - build-tools-23.0.2 - - android-23 - - os: linux - jdk: openjdk8 - python: - - 3.6 - - 2.7 - dist: trusty - addons: - apt: - packages: - - ant - env: CI_ACTION=ant - # ANDROID_HOME required for javadoc verification. - android: - components: - - tools - - platform-tools - - build-tools-23.0.2 - - android-23 - - os: linux - jdk: openjdk8 - python: - - 3.6 - - 2.7 - dist: trusty - addons: - apt: - packages: - - ant - - groovy - # We rely on -gno-record-gcc-switches which was added in 4.7. - - gcc - - g++ - # Haskell tests require GHC (and at least version 7.6). - - ghc - # base ghc package does not include dynamic libraries - # https://stackoverflow.com/a/11711501/1548477 - - ghc-dynamic - - ghc-haddock - env: CI_ACTION=integration GROOVY_HOME=/usr/share/groovy/ - # ANDROID_SDK is required for build. - android: - components: - - tools - - platform-tools - - build-tools-23.0.2 - - android-23 - - os: linux - jdk: openjdk8 - python: - - 3.6 - - 2.7 - android: - components: - - tools - - platform-tools - - build-tools-23.0.2 - - android-23 - - addon-google_apis-google-23 - - android-21 - - addon-google_apis-google-21 - - extra-android-support - dist: trusty - addons: - apt: - packages: - - ant - # Travis is on 64bit and there will be a cryptic aapt error w/o these libs. - # For native code tests, we need some additional libraries if we are in a 64-bit environment. - - libgd2-xpm-dev - - libc6:i386 - - libstdc++6:i386 - - zlib1g:i386 - # We rely on -gno-record-gcc-switches which was added in 4.7. - - gcc - - g++ - # Haskell tests require GHC (and at least version 7.6). - - ghc - # base ghc package does not include dynamic libraries - # https://stackoverflow.com/a/11711501/1548477 - - ghc-dynamic - - ghc-haddock - # https://docs.travis-ci.com/user/caching#Things-not-to-cache - # https://docs.travis-ci.com/user/caching#Explicitly-disabling-caching - cache: - directories: - - $HOME/ndk_cache - env: CI_ACTION=heavy_integration - - os: linux - jdk: openjdk8 - python: - - 3.6 - - 2.7 - android: - components: - - tools - - platform-tools - - build-tools-23.0.2 - - android-23 - - addon-google_apis-google-23 - - android-21 - - addon-google_apis-google-21 - - extra-android-support - dist: trusty - addons: - apt: - packages: - - ant - # Travis is on 64bit and there will be a cryptic aapt error w/o these libs. - # For native code tests, we need some additional libraries if we are in a 64-bit environment. - - libgd2-xpm-dev - - libc6:i386 - - libstdc++6:i386 - - zlib1g:i386 - # We rely on -gno-record-gcc-switches which was added in 4.7. - - gcc - - g++ - # https://docs.travis-ci.com/user/caching#Things-not-to-cache - # https://docs.travis-ci.com/user/caching#Explicitly-disabling-caching - cache: - directories: - - $HOME/ndk_cache - env: CI_ACTION=android_ndk_15 - - os: linux - jdk: openjdk8 - python: - - 3.6 - - 2.7 - android: - components: - - tools - - platform-tools - - build-tools-23.0.2 - - android-23 - - addon-google_apis-google-23 - - android-21 - - addon-google_apis-google-21 - - extra-android-support - dist: trusty - addons: - apt: - packages: - - ant - # Travis is on 64bit and there will be a cryptic aapt error w/o these libs. - # For native code tests, we need some additional libraries if we are in a 64-bit environment. - - libgd2-xpm-dev - - libc6:i386 - - libstdc++6:i386 - - zlib1g:i386 - # We rely on -gno-record-gcc-switches which was added in 4.7. - - gcc - - g++ - # https://docs.travis-ci.com/user/caching#Things-not-to-cache - # https://docs.travis-ci.com/user/caching#Explicitly-disabling-caching - cache: - directories: - - $HOME/ndk_cache - env: CI_ACTION=android_ndk_16 - - os: linux - jdk: openjdk8 - python: - - 3.6 - - 2.7 - android: - components: - - tools - - platform-tools - - build-tools-23.0.2 - - android-23 - - addon-google_apis-google-23 - - android-21 - - addon-google_apis-google-21 - - extra-android-support - dist: trusty - addons: - apt: - packages: - - ant - # Travis is on 64bit and there will be a cryptic aapt error w/o these libs. - # For native code tests, we need some additional libraries if we are in a 64-bit environment. - - libgd2-xpm-dev - - libc6:i386 - - libstdc++6:i386 - - zlib1g:i386 - # We rely on -gno-record-gcc-switches which was added in 4.7. - - gcc - - g++ - # https://docs.travis-ci.com/user/caching#Things-not-to-cache - # https://docs.travis-ci.com/user/caching#Explicitly-disabling-caching - cache: - directories: - - $HOME/ndk_cache - env: CI_ACTION=android_ndk_17 - - os: linux - jdk: openjdk8 - python: - - 3.6 - - 2.7 - android: - components: - - tools - - platform-tools - - build-tools-23.0.2 - - android-23 - - addon-google_apis-google-23 - - android-21 - - addon-google_apis-google-21 - - extra-android-support - dist: trusty - addons: - apt: - packages: - - ant - # Travis is on 64bit and there will be a cryptic aapt error w/o these libs. - # For native code tests, we need some additional libraries if we are in a 64-bit environment. - - libgd2-xpm-dev - - libc6:i386 - - libstdc++6:i386 - - zlib1g:i386 - # We rely on -gno-record-gcc-switches which was added in 4.7. - - gcc - - g++ - # https://docs.travis-ci.com/user/caching#Things-not-to-cache - # https://docs.travis-ci.com/user/caching#Explicitly-disabling-caching - cache: - directories: - - $HOME/ndk_cache - env: CI_ACTION=android_ndk_18 - -# Enable container-based architecture. -sudo: false - -before_install: - - echo "Python version is ${TRAVIS_PYTHON_VERSION}" - # Install ant on MacOS - - if \[ ${TRAVIS_OS_NAME} == "osx" \]; then brew install ant watchman; fi - # Limit Ant's and Buck's memory usage to avoid the OOM killer. - - export ANT_OPTS='-Xmx1000m' - - echo '-Xmx750m' > .buckjavaargs.local - # Set up the Android environment. Only for Linux. - - if \[ ${TRAVIS_OS_NAME} == "linux" \] && \[ "${CI_ACTION}" == "heavy_integration" \]; then - export NDK_HOME="${HOME}/android-ndk-linux" ; - ./scripts/travisci_install_android_ndk.sh ; - fi - - if \[ ${TRAVIS_OS_NAME} == "linux" \] && \[ "${CI_ACTION}" == "android_ndk_15" \]; then - export NDK_HOME="${HOME}/android-ndk-linux" ; - ./scripts/travisci_unzip_android_ndk.sh android-ndk-r15c ; - fi - - if \[ ${TRAVIS_OS_NAME} == "linux" \] && \[ "${CI_ACTION}" == "android_ndk_16" \]; then - export NDK_HOME="${HOME}/android-ndk-linux" ; - ./scripts/travisci_unzip_android_ndk.sh android-ndk-r16b ; - fi - - if \[ ${TRAVIS_OS_NAME} == "linux" \] && \[ "${CI_ACTION}" == "android_ndk_17" \]; then - export NDK_HOME="${HOME}/android-ndk-linux" ; - ./scripts/travisci_unzip_android_ndk.sh android-ndk-r17b ; - fi - - if \[ ${TRAVIS_OS_NAME} == "linux" \] && \[ "${CI_ACTION}" == "android_ndk_18" \]; then - export NDK_HOME="${HOME}/android-ndk-linux" ; - ./scripts/travisci_unzip_android_ndk.sh android-ndk-r18b ; - fi - # Install go 1.10.x, which generates a different .c file name from cgo than previous versions - - eval "$(gimme 1.10.1)" - - echo -e "[go]\n root = ${GOROOT}" >> .buckconfig.local - -# Buck dependencies are checked in, so no need to download dependencies -install: true - -notifications: - slack: - rooms: - secure: SYKQV9DT55kHf5Mpe6g5a3NmGXJb5E7kWiLulRp+EmKDIhf3lVmxGbx4Yr/TKZixbNILsPzhhiB56V0H+0mAgMpygVXaq4M9eSHKLljJEmEdeLKmQaRuOUikMOkpLsHw/epvmqrsvlb3yVpsJZZhhHmi9B0oQc0AnjpL/qLBaZE= - # Send Travis CI notifications to internal sytems like Phabricator. - webhooks: https://code.facebook.com/travis/webhook/ - -script: - - ./scripts/travisci_run.sh diff --git a/README.md b/README.md index 0826d3aef63..7e5abb4f711 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,12 @@ -Buck -==== - -Buck is a build tool. To see what Buck can do for you, -check out the documentation at . +

+ logo +

+

+ Buck +

+

+ Buck is a build tool. To see what Buck can do for you, check out the documentation at http://buck.build/. +

[![Build Status](https://circleci.com/gh/facebook/buck.svg?style=svg)](https://circleci.com/gh/facebook/buck) @@ -17,6 +21,8 @@ Since Buck is used to build Buck, the initial build process involves 2 phases: cd buck ant +You must be using Java 8 or 11 for this to compile successfully. If you see compilation errors from ant, check your `JAVA_HOME` is pointing at one of these versions. + ##### 2. Use bootstrapped version of Buck to build Buck: ./bin/buck build --show-output buck @@ -28,6 +34,8 @@ Since Buck is used to build Buck, the initial build process involves 2 phases: Pre-built binaries of buck for any buck `sha` can be downloaded from `https://jitpack.io/com/github/facebook/buck//buck-.pex`. The very first time a version of buck is requested, it is built via [jitpack](https://jitpack.io/). As a result, it could take a few minutes for this initial binary to become available. Every subsequent request will just serve the built artifact directly. This functionality is available for any fork of buck as well, so you can fetch `https://jitpack.io/com/github//buck//buck-.pex` +For buck binaries built for JDK 11, modify end of the url to `buck--java11.pex`. + Feature Deprecation ------------------- diff --git a/buck.iml b/buck.iml index c0d96fdc408..4935f2b888e 100644 --- a/buck.iml +++ b/buck.iml @@ -123,8 +123,8 @@ - - + + diff --git a/build.xml b/build.xml index 82775c14882..6b98655ea07 100644 --- a/build.xml +++ b/build.xml @@ -117,6 +117,27 @@ + + + + + + + + + + + + + + + @@ -131,9 +152,9 @@ - - - + + + @@ -147,7 +168,7 @@ - + @@ -208,14 +229,14 @@ - + - + @@ -226,8 +247,8 @@ - - + + @@ -1050,12 +1071,16 @@ - + + + + + @@ -1241,14 +1266,23 @@ - + - + + + + + + + + + + - + diff --git a/docs/__common.soy b/docs/__common.soy index 3d22de50097..dad9c30a6d2 100644 --- a/docs/__common.soy +++ b/docs/__common.soy @@ -146,7 +146,9 @@ src="https://www.facebook.com/tr?id=1637165926500152&ev=PageView&noscript=1"
  • GitHub - + {if $navid == 'home'}
    diff --git a/docs/__rust_common.soy b/docs/__rust_common.soy index 01aea09c6c1..19864fa15db 100644 --- a/docs/__rust_common.soy +++ b/docs/__rust_common.soy @@ -196,7 +196,7 @@ to allow for extra warnings, etc. {template .more_examples}

    For more examples, check out our + href="https://github.com/facebook/buck/tree/master/test/com/facebook/buck/features/rust/testdata/"> integration tests.

    {/template} diff --git a/docs/__table_of_contents.soy b/docs/__table_of_contents.soy index 80782e3d8da..596a441114a 100644 --- a/docs/__table_of_contents.soy +++ b/docs/__table_of_contents.soy @@ -253,6 +253,12 @@ {param page: 'visibility' /} {param text: 'Visibility' /} {/call} + {call navigation.link} + {param currentnavid: $navid /} + {param folder: 'concept' /} + {param page: 'flavors' /} + {param text: 'Flavors' /} + {/call} {call navigation.link} {param currentnavid: $navid /} {param folder: 'concept' /} diff --git a/docs/about/showcase.soy b/docs/about/showcase.soy index b6bac62a77c..5fc5c6c4d5e 100644 --- a/docs/about/showcase.soy +++ b/docs/about/showcase.soy @@ -267,6 +267,11 @@ {param imgSrc: 'https://lh5.ggpht.com/CfRup8ZMEbNpd4sf8PDUWzbnFIq0QINZpJ96M6V0-8wLmyqKD0ORSqPOq5EKdL1OskE=w100' /} {/call} + {call .item} + {param name: 'Easynvest' /} + {param imgSrc: 'https://theme.zdassets.com/theme_assets/1130385/0290357291ec4f3e2975f9b58183a608414fbf8d.png' /} + {/call} + {/param} {/call} diff --git a/docs/command/install.soy b/docs/command/install.soy index c3a5fe38e7e..373108cc9b1 100644 --- a/docs/command/install.soy +++ b/docs/command/install.soy @@ -151,7 +151,7 @@ Builds and installs the APK for an android_binary or target. {param name: 'simulator-name' /} {param alias: 'n'/} {param desc} - Use simulator with specific name (defaults to iPhone 6s) + Use simulator with specific name (defaults to iPhone 8) {/param} {/call} diff --git a/docs/concept/buckd.soy b/docs/concept/buckd.soy index 2d505fd8bbc..7a9d7bff3b4 100644 --- a/docs/concept/buckd.soy +++ b/docs/concept/buckd.soy @@ -40,26 +40,26 @@ graph and action graph.

    The Buck daemon writes its port, process id, and log output to files in a .buckd{sp} directory that the daemon creates in the -project root directory. Subsequent Buck commands use these files to +project root directory. Subsequent Buck commands use these files to find the daemon process, and a new Buck daemon process will use them to kill any already-existing daemon process.

    It is safe to run multiple Buck daemons started from different project -directories as they do not interfere with each other, -making buckd suitable for use in shared-server environments +directories as they do not interfere with each other, +making buckd suitable for use in shared-server environments or where several projects are being worked on concurrently.

    While it runs, the Buck daemon process monitors the project's file system and invalidates cached build rules if any build input files -change. The Buck daemon excludes from monitoring any subtrees of the +change. The Buck daemon excludes from monitoring any subtrees of the project file system that are specified in the {call buckconfig.project_ignore /} setting -of .buckconfig. By adding project-specific output -directories and source-control directories, such as.git, to +of .buckconfig. By adding project-specific output +directories and source-control directories, such as.git, to this setting, you can significantly improve performance; this might be necessary to avoid file-change overflows when using Buck daemons to build large projects. @@ -81,7 +81,7 @@ The Buck daemon process is killed if

    -You can also kill the Buck daemon explicitly by +You can also kill the Buck daemon explicitly by running {call buck.cmd_kill /} in the directory tree for your project. Note that if—for some reason—multiple instances of the daemon are running, the buck kill command kills only one of @@ -95,7 +95,7 @@ restarts.

    -To disable the daemon and prevent it from starting, set the environment +To disable the daemon and prevent it from starting, set the environment variable {sp}NO_BUCKD to 1. For example:

    @@ -112,8 +112,8 @@ buck build project_name

    Buck configuration changes invalidate the daemon's state

    If you change Buck's configuration, it invalidates any cached state -stored by the Buck daemon—unless that change is to -a small subset of configuration settings that are whitelisted +stored by the Buck daemon—unless that change is to +a small subset of configuration settings that are supported in AbstractConfigIgnoredByDaemon.java.

    @@ -124,18 +124,18 @@ the following:

     {literal}
    -Invalidating internal cached state: 
    -Buck configuration options changed between invocations. 
    +Invalidating internal cached state:
    +Buck configuration options changed between invocations.
     This may cause slower builds.
     {/literal}
     

    -Note that a Buck configuration change that invalidates the daemon's state can be -caused not only by explicitly changing a setting in one of Buck's -configuration files, such as .buckconfig or .buckconfig.local, but also by using -the --config, --flagfile, or --config-file {call buck.cmd_link}{param name: 'common_parameters' /}{param rendered_text: ' command-line parameters' /}{/call}. +Note that a Buck configuration change that invalidates the daemon's state can be +caused not only by explicitly changing a setting in one of Buck's +configuration files, such as .buckconfig or .buckconfig.local, but also by using +the --config, --flagfile, or --config-file {call buck.cmd_link}{param name: 'common_parameters' /}{param rendered_text: ' command-line parameters' /}{/call}.

    {/param} // content diff --git a/docs/concept/flavors.soy b/docs/concept/flavors.soy new file mode 100644 index 00000000000..ddc005401e0 --- /dev/null +++ b/docs/concept/flavors.soy @@ -0,0 +1,458 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +{namespace buck.flavors} + +/***/ +{template .soyweb} + {call buck.page} + {param title: 'Flavors' /} + {param navid: 'concept_flavors' /} + {param prettify: true /} + {param description} + Flavors in Buck are a way to specify different variations of a build + that otherwise share most configuration specifications. + {/param} + {param content} + +

    +Flavors in Buck are a way to specify different variations of a build that +otherwise share most configuration specifications. +

    + +

    +Flavors can also be used to simultaneously build for two different platforms, +such as iOS and watchOS. Being able to build for both of these at the same +time is important because iOS application can have a dependency on the Watch +application. +

    + +

    +Flavors fall into a number of different categories: +

    + + +

    Syntax for flavors

    + +

    +A flavor is specified as a suffix to a build target, with the hash mark (#) +used as a separator between the target and flavor. +Examples: +

    +{literal} +
    +buck build :cxx_gr_name#default
    +buck build :cxx_gr_name#iphonesimulator-x86_64
    +
    +{/literal} + +

    +You can specify multiple flavors for a single target by separating the flavors +with commas: +

    +{literal} +
    +buck build Libraries/3rdParty/openssl:ssl#android-armv7,static-pic
    +buck build Libraries/3rdParty/openssl:ssl#android-x86,static-pic
    +buck build :bundle#iphoneos-x86_64,strip-all,dwarf-and-dsym
    +
    +{/literal} + +

    +NOTE: Buck supports +a {call buck.concept_link}{param page: 'build_target_pattern' /}{param name: 'build target pattern' /}{/call}, +..., as in, //apps/..., that specifies that Buck +should recurse through build files in subdirectories, building all build +targets in any build files it finds. This build target pattern does not +support specifying flavors. +

    + +

    Default flavors

    +

    +The cxx_library and apple_library rules support +specifying default flavors, which pertain if a build target is +specified 1) explicitly-that is, not using +a {call buck.concept_link}{param page: 'build_target_pattern' /}{param name: 'build target pattern' /}{/call} +-and 2) without a flavor. +

    + +

    Syntax for Flagfile Flavors

    +

    +A flavor may also be specified as a suffix to an @file or --flagfile argument. +These flavors are used within the flagfile only and do not directly change +the target flavor. The separator is '#' as above, and if the separator and +flavor are present, the @file (aka flagfile) must end in ".py" and be a +Python script. The script will be called as follows: +

    + +{literal} +
    +python /path/to/flagfile.py --flavors 
    +
    +{/literal} + +

    Platform flavors #

    +

    +These flavors denote a toolchain to use for compiling. You can also use them to +control conditional fields in the Buck target's rule. +

    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    FlavorMeaning
    android-armv732-bit Android device
    android-x8632-bit Android emulator
    iphoneos
    iphoneos-armv732-bit iPhone device
    iphoneos-arm6464-bit iPhone device
    iphoneos-i386
    iphoneos-x86_64
    iphonesimulator
    iphonesimulator-i386Simulator on 32-bit Mac
    iphonesimulator-x86_64Simulator on 64-bit Mac
    macosx-x86_64Native x86_64 OSX application
    macosx-arm64Native arm64 OSX application
    windows-x86_64Native Windows application
    + +

    +Sighted in the wild: +

    + +{literal} +
    +buck build :main#android-arm64,shared
    +
    +{/literal} + +

    Linker flavors #

    +

    Static

    +{literal} +
    +static
    +
    +{/literal} +

    +Static library (.a) +

    + +

    Position-independent code (PIC)

    +{literal} +
    +static-pic
    +
    +{/literal} +

    +Static library that generates position-independent code (PIC). Note that on the Apple platforms, everything is PIC. +

    + +

    Shared

    +

    +A shared library (so) or dynamically-loaded module (.dylib on Mac, .dll on Windows). +

    + +

    Shared interface

    +{literal} +
    +shared-interface
    +
    +{/literal} +

    +A stub library that only lists the imports and exports of a shared library. +You can link against this library, but it doesn't have any executable code. +

    + +

    Linker Map

    +{literal} +
    +no-linkermap
    +
    +{/literal} + +

    +Specifying this flavor suppresses the generation of a LinkMap.txt file, +which is normally generated by the Apple linker (ld). +

    + +

    Analysis flavors #

    +

    Rust language

    +{literal} +
    +check
    +
    +{/literal} + +

    +The Rust compiler has a build mode which quickly checks syntax and type +correctness, but avoids codegen (which is the slowest phase of Rust +compilation). This flavor invokes check on the suffixed build target. +

    + +{literal} +
    +save-analysis
    +
    +{/literal} +

    +The #save-analysis flavor is an extension of #check. It performs +the same actions as a #check build, but also emits a .json file containing +full type/symbol cross-reference information, for consumption by tools like +RLS (Rust Language Services). +

    + +{literal} +
    +doc
    +
    +{/literal} + +

    +The #doc flavor is the equivalent of the cargo doc command. It +uses the rustdoc tool to generate documentation for a target and its dependencies. +

    + +

    Compilation database

    +

    +If you specify one of these flavors, Buck generates analysis information about +your build. +

    +{literal} +
    +compilation-database
    +
    +{/literal} + +

    +Produces a JSON file that contains (an approximation of) the compiler commands +for compiling each file. The output is in +the +clang compilation database format. +

    + +

    Infer

    +{literal} +
    +infer-analyze
    +infer-capture-all
    +
    +{/literal} + +

    +These flavors run Facebook's Infer tool and +generate intermediate Infer output. +

    + +

    Symbol stripping flavors#

    +

    +These flavor control how symbols are stripped from an output binary. +

    + +

    Strip debug

    +{literal} +
    +strip-debug
    +
    +{/literal} + +

    +Strip debug symbols; equivalent to strip -S. +

    + +

    Strip debug and non-external

    +{literal} +
    +strip-non-global
    +
    +{/literal} + +

    +Strip debug and non-external symbols; equivalent to strip -x. +

    + +

    Strip all

    +{literal} +
    +strip-all
    +
    +{/literal} +

    +Strip all the things; equivalent to strip with no arguments. +

    + +

    Header Flavors#

    +

    +These flavors are used with C++ header files, in the deps or exported_deps +attributes of a Buck target. +

    + +

    Headers used to compile a libraries own files

    +{literal} +
    +private-headers
    +
    +{/literal} +

    +Symlink tree, or header map, of headers used for compiling a library's own files. +

    + +

    Headers used to compile files that depend on this library

    +{literal} +
    +headers
    +
    +{/literal} + +

    +Symlink tree, or header map, of headers used to compile files for other +libraries that depend on this library. +

    + +

    Apple debug flavors#

    +

    +Selects the actual binary used to represent a {call buck.cxx_binary /}. +

    + +

    Stripped binary

    +{literal} +
    +no-debug
    +
    +{/literal} + +

    +Selects the stripped binary +

    + +

    Unstripped binary

    +{literal} +
    +dwarf
    +
    +{/literal} + +

    +Selects the unstripped binary +

    + +

    Unstripped + dsym

    +{literal} +
    +dwarf-and-dsym
    +
    +{/literal} + +

    +Selects the unstripped binary, and the dsym file. +

    + +

    Other

    +{literal} +
    +apple-debuggable-binary
    +
    +{/literal} + +

    +Could be stripped and unstripped, depending on requirements. +

    + +

    Apple bundles flavors#

    +

    +These flavors are legacy methods of creating app and framework bundles. +

    +

    +You should not use these. +

    +

    +Instead of these flavors, we now use the Buck rule, {call buck.apple_bundle /} +

    + +

    App

    +{literal} +
    +app
    +binary#app
    +
    +{/literal} + +

    +Generates an app bundle. +

    + +

    Framework

    +{literal} +
    +framework
    +library#framework
    +
    +{/literal} + +

    +Generates a framework. +

    + + {/param} + {/call} +{/template} diff --git a/docs/concept/visibility.soy b/docs/concept/visibility.soy index 35b1a1e1561..362ef8ba7bf 100644 --- a/docs/concept/visibility.soy +++ b/docs/concept/visibility.soy @@ -44,7 +44,7 @@ targets a target can depend on.

    -Both attributes act as whitelists, with some exceptions. In general, if a target +Both attributes act as allowlists, with some exceptions. In general, if a target is not listed, there may be no dependency relationship. If the within_view list is empty or unset, however, its check is bypassed. Similarly, targets defined in the same build file always act as if diff --git a/docs/files-and-dirs/buckconfig.soy b/docs/files-and-dirs/buckconfig.soy index 29ed31f9581..85a3eaa044f 100644 --- a/docs/files-and-dirs/buckconfig.soy +++ b/docs/files-and-dirs/buckconfig.soy @@ -592,6 +592,7 @@ $ buck targets --resolve-alias app#src_jar

    • ANDROID_SDK
    • +
    • ANDROID_SDK_ROOT
    • ANDROID_HOME
    diff --git a/docs/learning/tutorial.soy b/docs/learning/tutorial.soy index b46ed24a20a..49861d86aa4 100644 --- a/docs/learning/tutorial.soy +++ b/docs/learning/tutorial.soy @@ -19,7 +19,7 @@ /***/ {template .soyweb} {call buck.page} - {param title: 'Tutorial' /} + {param title: 'Tutorials' /} {param navid: 'learning_tutorial' /} {param prettify: true /} {param description} @@ -41,6 +41,35 @@

    +
    + + + + + + + + + + + + + +
    Platform: + Android + MacOS + Linux +
    Development OS: + macOS + Linux +
    Language: + Java + Kotlin + Rust +
    +
    + +

    Path Setup

    @@ -54,11 +83,10 @@ sudo ln -s ${PWD}/bin/buckd /usr/bin/buckd {/literal} -

    Create Project

    - We are going to build a sample Android application. We should start our project in an empty directory, so create a new one and navigate to it: + We are going to build a sample application. We should start our project in an empty directory, so create a new one and navigate to it:

    {literal}
    @@ -74,14 +102,16 @@ cd ~/my-first-buck-project/
     
     

    Compile Your Code

    +

    - Android applications are typically written in Java, so the first thing we will do is configure Buck to compile Java code against the Android API. To do so, Buck needs to know where your Android SDK is. Assuming that your Android SDK is installed in ~/android-sdk, run the following command to set a ANDROID_SDK environment variable that tells Buck where to find your Android SDK: + Android applications are typically written in Java and kotlin, so the first thing we will do is to configure Buck to compile code against the Android API. To do so, Buck needs to know where your Android SDK is. Assuming that your Android SDK is installed in ~/android-sdk, run the following command to set a ANDROID_SDK environment variable that tells Buck where to find your Android SDK:

    {literal}
     export ANDROID_SDK=$HOME/android-sdk
     
    {/literal} +

    Now that Buck can locate your Android SDK, it is time to compile some Java code. First, we create a simple Activity at java/com/example/activity/MyFirstActivity.java:

    @@ -126,6 +156,124 @@ echo "android_library(

    + +

    + Now that Buck can locate your Android SDK, it is time to compile some kotlin code. First, we create a simple Activity at kotlin/com/example/activity/MyFirstActivity.kt: +

    + +{literal}
    +mkdir -p kotlin/com/example/activity/
    +echo "package com.example.activity
    +
    +import android.app.Activity
    +import android.os.Bundle
    +
    +class MyFirstActivity : Activity() {
    +
    +    override fun onCreate(savedInstanceState: Bundle?) {
    +        super.onCreate(savedInstanceState)
    +    }
    +}" > kotlin/com/example/activity/MyFirstActivity.kt
    +
    {/literal} + +

    + Now we need a build file that defines a build rule to compile this kotlin code, so we create an android_library() rule in kotlin/com/example/activity/BUCK: +

    + +{literal}
    +echo "android_library(
    +  name = 'activity',
    +  srcs = glob(['*.kt']),
    +  language = "KOTLIN",
    +  visibility = [ 'PUBLIC' ],
    +)" > kotlin/com/example/activity/BUCK
    +
    {/literal} + +

    + Now we can compile our Java code using Buck: +

    + +
    buck build{sp}//kotlin/com/example/activity:activity
    + +

    +

    + Buck generates its output in the buck-out directory, so this is a good time to specify buck-out as something that should be ignored by your version control system. +
    +

    + + +

    + First, we create rust source files and directories: +

    +{literal}
    +mkdir -p src/front_of_house
    +echo "mod front_of_house;
    +
    +pub use crate::front_of_house::hosting;
    +pub use crate::front_of_house::serving;
    +
    +fn main() {
    +    hosting::add_to_waitlist();
    +    hosting::seat_at_table();
    +    serving::take_order();
    +    serving::serve_order();
    +    serving::take_payment();
    +}" > src/main.rs
    +
    +echo "pub mod hosting;
    +pub mod serving;" > src/front_of_house.rs
    +
    +echo "pub fn add_to_waitlist() {
    +    println!(\"Added to watinglist.\");
    +}
    +
    +pub fn seat_at_table() {
    +    println!(\"Seated at table.\");
    +}" > src/front_of_house/hosting.rs
    +
    +echo "pub fn take_order() {
    +    println!(\"Ordered.\");
    +}
    +
    +pub fn serve_order() {
    +    println!(\"Order served.\");
    +}
    +
    +pub fn take_payment() {
    +    println!(\"Payment done.\");
    +}" > src/front_of_house/serving.rs
    +
    {/literal} + +

    + Now we need a build file that defines a build rule to compile this rust code, so we create an rust_binary() rule in BUCK: +{literal}

    +echo "rust_binary(
    +    name = 'restaurant',
    +    srcs = glob(
    +        ['src/**/*.rs'],
    +    ),
    +)" >BUCK
    +
    {/literal} +

    + +

    +Now we can build our rust code using Buck: +

    + +
    buck build{sp}:restaurant
    + +

    +And run the sample: +

    +
    buck run{sp}:restaurant
    + +

    +

    + Buck generates its output in the buck-out directory, so this is a good time to specify buck-out as something that should be ignored by your version control system. +
    +

    + +

    Package Resources

    @@ -229,8 +377,8 @@ echo "android_binary( manifest_entries = { 'version_code': 1, 'version_name': '1.0', - 'min_sdk_version': 16, - 'target_sdk_version': 23 + 'min_sdk_version': 26, + 'target_sdk_version': 29 }, keystore = ':debug_keystore', deps = [ @@ -279,6 +427,7 @@ echo "[alias]

    buck install app
    +

    Create an IntelliJ Project

    @@ -324,4 +473,100 @@ echo "/.buckd {/param} {/call} + +{literal} + +{/literal} + {/template} diff --git a/docs/publish.sh b/docs/publish.sh index 56d1fd460ad..7e5a01c8641 100755 --- a/docs/publish.sh +++ b/docs/publish.sh @@ -38,6 +38,8 @@ EOF exit 1 } +GIT_USER="buck-bot" +CNAME="buck.build" START_SOYWEB=0 KEEP_FILES=0 SOYWEB_PID=0 @@ -83,14 +85,14 @@ echo "Documentation working directory is ${STATIC_FILES_DIR}" # Create a clean checkout of the gh-pages branch with no data: if [ -z "$1" ] then - git clone git@github.com:facebook/buck.git $STATIC_FILES_DIR + git clone https://${GIT_USER}:${GITHUB_TOKEN}@github.com/facebook/buck.git $STATIC_FILES_DIR else cp -r "$1" $STATIC_FILES_DIR fi cd $STATIC_FILES_DIR # May need to do this if you are creating gh-pages for the first time. -git checkout master +git checkout main git checkout --orphan gh-pages git rm -rf . @@ -101,6 +103,8 @@ cd - # Commit the new version of the docs: cd $STATIC_FILES_DIR +echo "${CNAME}" > CNAME +git config --global user.name "${GIT_USER}" git add . git commit -m "Updated HTML documentation." diff --git a/docs/rule/android_library.soy b/docs/rule/android_library.soy index 012c1df4eb0..d52313ed03a 100644 --- a/docs/rule/android_library.soy +++ b/docs/rule/android_library.soy @@ -163,7 +163,7 @@ compiled class files and resources. {/call} {call buck.arg} - {param name: 'extra_kotlinc_arguments' /} + {param name: 'free_compiler_args' /} {param default : '[]' /} {param desc} List of additional arguments to pass into the Kotlin compiler. diff --git a/docs/rule/kotlin_library.soy b/docs/rule/kotlin_library.soy index 674da84802f..396e2006a5e 100644 --- a/docs/rule/kotlin_library.soy +++ b/docs/rule/kotlin_library.soy @@ -72,8 +72,8 @@ the resources argument. Kotlin compiler plugins to use when compiling this library. This takes a list of source paths, each of which will be prefixed with -Xplugin= and passed as extra arguments to the compiler. - Unlike extra_kotlinc_arguments, these can be source paths, - not just strings (though extra_kotlinc_arguments should be used + Unlike free_compiler_args, these can be source paths, + not just strings (though free_compiler_args should be used to specify plugin compiler options with -P).

    For example, if you want to use{sp} @@ -115,10 +115,109 @@ remote_file( {/call} {call buck.arg} - {param name: 'extra_kotlinc_arguments' /} + {param name: 'free_compiler_args' /} {param default: '[]' /} {param desc} - List of additional arguments to pass into the Kotlin compiler. + A list of additional compiler arguments. + {/param} +{/call} + +{call buck.arg} + {param name: 'all_warnings_as_errors' /} + {param default: 'false' /} + {param desc} + Report an error if there are any warnings. + {/param} +{/call} + +{call buck.arg} + {param name: 'suppress_warnings' /} + {param default: 'false' /} + {param desc} + Generate no warnings. + {/param} +{/call} + +{call buck.arg} + {param name: 'verbose' /} + {param default: 'false' /} + {param desc} + Enable verbose logging output. + {/param} +{/call} + +{call buck.arg} + {param name: 'include_runtime' /} + {param default: 'false' /} + {param desc} + Include Kotlin runtime in to resulting .jar + {/param} +{/call} + +{call buck.arg} + {param name: 'jvm_target' /} + {param default: '1.6' /} + {param desc} + Target version of the generated JVM bytecode (1.6 or 1.8), default is 1.6 + Possible values: "1.6", "1.8" + {/param} +{/call} + +{call buck.arg} + {param name: 'jdk_home' /} + {param default: 'None' /} + {param desc} + Path to JDK home directory to include into classpath, if differs from default JAVA_HOME + {/param} +{/call} + +{call buck.arg} + {param name: 'no_jdk' /} + {param default: 'false' /} + {param desc} + Don't include Java runtime into classpath. + {/param} +{/call} + +{call buck.arg} + {param name: 'no_stdlib' /} + {param default: 'true' /} + {param desc} + Don't include kotlin-stdlib.jar or kotlin-reflect.jar into classpath. + {/param} +{/call} + +{call buck.arg} + {param name: 'no_reflect' /} + {param default: 'true' /} + {param desc} + Don't include kotlin-reflect.jar into classpath. + {/param} +{/call} + +{call buck.arg} + {param name: 'java_parameters' /} + {param default: 'false' /} + {param desc} + Generate metadata for Java 1.8 reflection on method parameters. + {/param} +{/call} + +{call buck.arg} + {param name: 'api_version' /} + {param default: 'None' /} + {param desc} + Allow to use declarations only from the specified version of bundled libraries. + Possible values: "1.0", "1.1", "1.2", "1.3", "1.4 (EXPERIMENTAL)". + {/param} +{/call} + +{call buck.arg} + {param name: 'language_version' /} + {param default: 'None' /} + {param desc} + Provide source compatibility with specified language version. + Possible values: "1.0", "1.1", "1.2", "1.3", "1.4 (EXPERIMENTAL)". {/param} {/call} diff --git a/docs/rule/robolectric_test.soy b/docs/rule/robolectric_test.soy index b31dad439a6..cb8972a2f8e 100644 --- a/docs/rule/robolectric_test.soy +++ b/docs/rule/robolectric_test.soy @@ -69,7 +69,7 @@ with Robolectric test runner. It extends from java_test() rule. {call test_common.contacts_arg /} {call buck.arg} - {param name: 'extra_kotlinc_arguments' /} + {param name: 'free_compiler_args' /} {param default : '[]' /} {param desc} List of additional arguments to pass into the Kotlin compiler. diff --git a/docs/setup/getting_started.soy b/docs/setup/getting_started.soy index edf13122f05..987b3fefed0 100644 --- a/docs/setup/getting_started.soy +++ b/docs/setup/getting_started.soy @@ -302,10 +302,9 @@ xcode-select --install # Download and Install Java SE 8 from: +# https://www.oracle.com/technetwork/java/javase/downloads/index.html. +# This installs the JDK 8, a superset of the JRE. -+# Alternatively, install AdoptOpenJDK 8 with Homebrew Cask: -+brew tap homebrew/cask-cask -+brew tap homebrew/cask-versions -+brew cask install homebrew/cask-versions/adoptopenjdk8 ++# Alternatively, install AdoptOpenJDK 8 with Homebrew: ++brew tap AdoptOpenJDK/openjdk ++brew install --cask adoptopenjdk8

    {/literal} @@ -362,7 +361,7 @@ brew install --HEAD buck Git
  • - Watchman + Watchman
  • @@ -520,6 +519,9 @@ For some distros, the default packages may be older than what you would like. In
  • ANDROID_HOME environment variable
  • +
  • + ANDROID_SDK_ROOT environment variable +
  • The value of the {call buckconfig.android_sdk_path /} property in .buckconfig. @@ -779,6 +781,7 @@ $ env | grep ANDROID_ ANDROID_HOME=<path-to-sdk> ANDROID_NDK_REPOSITORY=<path-to-ndk> ANDROID_SDK=<path-to-sdk> +ANDROID_SDK_ROOT=<path-to-sdk> {/literal} diff --git a/docs/soy2html.py b/docs/soy2html.py index 369eeb7f4c6..89e54f4f85c 100644 --- a/docs/soy2html.py +++ b/docs/soy2html.py @@ -57,7 +57,7 @@ def main(output_dir): ): # Copy the static resource to output_dir. relative_path = os.path.join(root, file_name) - with open(relative_path) as resource_file: + with open(relative_path, "rb") as resource_file: resource = resource_file.read() copy_to_output_dir(relative_path, output_dir, resource) @@ -74,7 +74,7 @@ def ensure_dir(path, output_dir): def copy_to_output_dir(path, output_dir, content): output_file = ensure_dir(path, output_dir) - with open(output_file, "w") as f: + with open(output_file, "wb") as f: f.write(content) @@ -88,7 +88,6 @@ def pollForServerReady(): time.sleep(1) print("Server failed to start after %s seconds." % SERVER_START_POLL) - if __name__ == "__main__": output_dir = sys.argv[1] pollForServerReady() diff --git a/docs/static/buck.css b/docs/static/buck.css index ca6b46247f8..8da24a38a8c 100644 --- a/docs/static/buck.css +++ b/docs/static/buck.css @@ -552,7 +552,17 @@ footer { .display-platform-ios .toggler .button-ios, .display-platform-android .toggler .button-android, .display-platform-other .toggler .button-other, -.display-platform-java .toggler .button-java { +.display-platform-java .toggler .button-java, +.disp-os-mac .toggler .button-mac, +.disp-os-linux .toggler .button-linux, +.disp-os-windows .toggler .button-windows, +.disp-platform-ios .toggler .button-ios, +.disp-platform-android .toggler .button-android, +.disp-platform-macos .toggler .button-macos, +.disp-platform-linuxos .toggler .button-linuxos, +.disp-language-java .toggler .button-java, +.disp-language-kotlin .toggler .button-kotlin, +.disp-language-rust .toggler .button-rust { background-color: #05A5D1; color: white; } @@ -561,6 +571,12 @@ block { display: none; } +.disp-platform-android.disp-os-mac.disp-language-java .android.mac.java, +.disp-platform-android.disp-os-linux.disp-language-java .android.linux.java, +.disp-platform-android.disp-os-mac.disp-language-kotlin .android.mac.kotlin, +.disp-platform-android.disp-os-linux.disp-language-kotlin .android.linux.kotlin, +.disp-platform-macos.disp-os-mac.disp-language-rust .macos.mac.rust, +.disp-platform-linuxos.disp-os-linux.disp-language-rust .linuxos.linux.rust, .display-platform-ios.display-os-mac .ios.mac, .display-platform-ios.display-os-linux .ios.linux, .display-platform-ios.display-os-windows .ios.windows, @@ -576,3 +592,16 @@ block { display: block; } +/* Social Banner */ +.socialBanner { + font-weight: bold; + font-size: 20px; + padding: 20px; + max-width: 768px; + margin: 0 auto; + text-align: center; +} + +.socialBanner a { + text-decoration: underline; +} diff --git a/jitpack.yml b/jitpack.yml index 13525b7c399..880cce4a4cf 100644 --- a/jitpack.yml +++ b/jitpack.yml @@ -1,10 +1,15 @@ install: - set -exo pipefail - mkdir python_tmp - - wget https://github.com/kageiit/jitpack-python/releases/download/2.7/python-2.7.tar.gz -O python_tmp/python.tar.gz - - tar -C python_tmp -xf python_tmp/python.tar.gz --strip-components=1 + - wget https://github.com/kageiit/jitpack-python/releases/download/3.8/python-3.8-ubuntu-16.tar.gz -O python_tmp/python.tar.gz + - tar -C python_tmp -xf python_tmp/python.tar.gz - export PATH="$PATH:python_tmp/bin" - ant - PEX=$(bin/buck build buck --show-output | awk '{print $2}') - SHA=$(git rev-parse HEAD) - mvn install:install-file -Dfile="$PEX" -DgroupId="$GROUP" -DartifactId="$ARTIFACT" -Dversion="$SHA" -Dpackaging="pex" -DgeneratePom=true + - export JAVA_HOME="/usr/lib/jvm/jdk-11" + - export PATH="/usr/lib/jvm/jdk-11:$PATH" + - ant clean && ant + - JAVA11_PEX=$(bin/buck build --config java.target_level=11 --config java.source_level=11 buck --show-output | awk '{print $2}') + - mvn install:install-file -Dfile="$JAVA11_PEX" -DgroupId="$GROUP" -DartifactId="$ARTIFACT" -Dversion="$SHA" -Dclassifier="java11" -Dpackaging="pex" -DgeneratePom=true diff --git a/programs/classpaths b/programs/classpaths index 8ab03688bdd..029e0bf8055 100644 --- a/programs/classpaths +++ b/programs/classpaths @@ -59,20 +59,20 @@ third-party/java/jackson/jackson-datatype-jdk8-2.9.7.jar third-party/java/jackson/jackson-module-kotlin-2.9.9.jar third-party/java/jdom/jdom-2.0.6.jar third-party/java/jetty/jetty-all-9.4.9.v20180320-uber.jar -third-party/java/jna/jna-4.5.1.jar -third-party/java/jna/jna-platform-4.5.1.jar +third-party/java/jna/jna-5.6.0.jar +third-party/java/jna/jna-platform-5.6.0.jar third-party/java/jopt-simple/jopt-simple-4.6.jar third-party/java/json-simple/json-simple-1.1.1.jar third-party/java/jsr/javax.inject-1.jar third-party/java/jsr/jsr305.jar third-party/java/kxml2/kxml2-2.3.0.jar -third-party/java/nailgun/nailgun-server-1.0.0.jar -third-party/java/nuprocess/nuprocess-1.2.4.jar -third-party/java/ObjCBridge/ObjCBridge.jar +third-party/java/nailgun/nailgun-server-1.0.1.jar +third-party/java/nuprocess/nuprocess-2.0.1.jar +third-party/java/ObjCBridge/java-objc-bridge-1.1-SNAPSHOT.jar third-party/java/objenesis/objenesis-1.2.jar third-party/java/okhttp/okhttp-3.9.0.jar third-party/java/okio/okio-1.13.0.jar -third-party/java/oshi/oshi-core-3.3-SNAPSHOT.jar +third-party/java/oshi/oshi-core-5.2.5.jar third-party/java/opencensus/opencensus-api-0.24.0.jar third-party/java/opencensus/opencensus-contrib-grpc-metrics-0.24.0.jar third-party/java/pf4j/pf4j-2.0.0-SNAPSHOT.jar diff --git a/programs/gen_buck_info.py b/programs/gen_buck_info.py index 747a07b957b..1eecb7bcd46 100644 --- a/programs/gen_buck_info.py +++ b/programs/gen_buck_info.py @@ -29,9 +29,16 @@ def main(argv): parser = argparse.ArgumentParser() - parser.add_argument("--release-version", help="The buck release version") parser.add_argument( - "--release-timestamp", help="The unix timestamp when the release happened" + "--release-version", + nargs = "?", + default = None, + help="The buck release version") + parser.add_argument( + "--release-timestamp", + nargs = "?", + default = None, + help="The unix timestamp when the release happened" ) parser.add_argument( "--java-version", @@ -52,7 +59,7 @@ def main(argv): candidate_paths = [] vcs_module = None while vcs_module is None and os.path.dirname(path) != path: - while not os.path.exists(os.path.join(path, ".buckconfig")): + while not os.path.exists(os.path.join(path, ".buckconfig")) and os.path.dirname(path) != path: path = os.path.dirname(path) for vcs_dir, module in SUPPORTED_VCS.items(): if os.path.exists(os.path.join(path, vcs_dir)): diff --git a/python-dsl/buck_parser/buck.py b/python-dsl/buck_parser/buck.py index c9c56767534..a7ce468f5f1 100644 --- a/python-dsl/buck_parser/buck.py +++ b/python-dsl/buck_parser/buck.py @@ -303,7 +303,7 @@ def invoke(self, *args, **kwargs): # Optimistically hope that name is the first arg. It generally is... name = args[0] raise IncorrectArgumentsException( - self.func.func_name, name, missing_args, extra_args + self.func.__name__, name, missing_args, extra_args ) raise @@ -2262,14 +2262,14 @@ def main(): configs = {} if options.config is not None: - with open(options.config, "rb") as f: + with open(options.config, "r") as f: for section, contents in iteritems(json.load(f)): for field, value in iteritems(contents): configs[(section, field)] = value ignore_paths = [] if options.ignore_paths is not None: - with open(options.ignore_paths, "rb") as f: + with open(options.ignore_paths, "r") as f: ignore_paths = [make_glob(i) for i in json.load(f)] build_file_processor = BuildFileProcessor( diff --git a/python-dsl/buck_parser/module_whitelist.py b/python-dsl/buck_parser/module_whitelist.py index 1c9c1f7d7af..ac032202b11 100644 --- a/python-dsl/buck_parser/module_whitelist.py +++ b/python-dsl/buck_parser/module_whitelist.py @@ -75,7 +75,12 @@ def _custom_import(self, name, globals=None, locals=None, fromlist=(), level=-1) name = name.split(".")[0] frame = util.get_caller_frame(skip=[__name__]) + # __import__() can be called during the call to inspect.getframeinfo() and that should + # be handled by the ORIGINAL_IMPORT + current_import = builtins.__import__ + builtins.__import__ = ORIGINAL_IMPORT filename = inspect.getframeinfo(frame).filename + builtins.__import__ = current_import # The import will be always allowed if it was not called from a project file. if name in self._import_whitelist or not self._path_predicate(filename): diff --git a/scripts/slice_trace.py b/scripts/slice_trace.py index fcd2343b346..e1ffc925ed8 100644 --- a/scripts/slice_trace.py +++ b/scripts/slice_trace.py @@ -16,8 +16,6 @@ import argparse import codecs -import collections -import json import math import os diff --git a/src/com/facebook/buck/android/AndroidApkBuildable.java b/src/com/facebook/buck/android/AndroidApkBuildable.java new file mode 100644 index 00000000000..c1432eb4c24 --- /dev/null +++ b/src/com/facebook/buck/android/AndroidApkBuildable.java @@ -0,0 +1,113 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.buck.android; + +import com.facebook.buck.android.apkmodule.APKModule; +import com.facebook.buck.android.exopackage.ExopackageMode; +import com.facebook.buck.android.toolchain.AndroidSdkLocation; +import com.facebook.buck.core.model.BuildTarget; +import com.facebook.buck.core.sourcepath.SourcePath; +import com.facebook.buck.core.sourcepath.resolver.SourcePathResolverAdapter; +import com.facebook.buck.core.toolchain.tool.Tool; +import com.facebook.buck.io.filesystem.ProjectFilesystem; +import com.facebook.buck.step.Step; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedSet; +import java.nio.file.Path; +import java.util.EnumSet; +import java.util.Optional; +import java.util.function.Supplier; + +/** + * The class is responsible to create unoptimized apk through {@link ApkBuilderStep}. + */ +public class AndroidApkBuildable extends AndroidBinaryBuildable { + + AndroidApkBuildable( + BuildTarget buildTarget, + ProjectFilesystem filesystem, + AndroidSdkLocation androidSdkLocation, + SourcePath keystorePath, + SourcePath keystorePropertiesPath, + EnumSet exopackageModes, + int xzCompressionLevel, + boolean packageAssetLibraries, + boolean compressAssetLibraries, + Optional assetCompressionAlgorithm, + Tool javaRuntimeLauncher, + SourcePath androidManifestPath, + DexFilesInfo dexFilesInfo, + NativeFilesInfo nativeFilesInfo, + ResourceFilesInfo resourceFilesInfo, + ImmutableSortedSet apkModules, + ImmutableMap moduleResourceApkPaths, + Optional bundleConfigFilePath, + BinaryType binaryType) { + super( + buildTarget, + filesystem, + androidSdkLocation, + keystorePath, + keystorePropertiesPath, + exopackageModes, + xzCompressionLevel, + packageAssetLibraries, + compressAssetLibraries, + assetCompressionAlgorithm, + javaRuntimeLauncher, + androidManifestPath, + dexFilesInfo, + nativeFilesInfo, + resourceFilesInfo, + apkModules, + moduleResourceApkPaths, + bundleConfigFilePath, + binaryType, + false); + } + + + @Override + void getBinaryTypeSpecificBuildSteps( + ImmutableList.Builder steps, + ImmutableModuleInfo.Builder baseModuleInfo, + Supplier keystoreProperties, + ImmutableSet.Builder nativeLibraryDirectoriesBuilder, + ImmutableSet allAssetDirectories, + SourcePathResolverAdapter pathResolver, + ImmutableSet thirdPartyJars, + ImmutableSet.Builder zipFiles, + ImmutableSet.Builder modulesInfo) { + steps.add( + new ApkBuilderStep( + getProjectFilesystem(), + pathResolver.getAbsolutePath(resourceFilesInfo.resourcesApkPath), + AndroidBinaryPathUtility.getSignedApkPath(filesystem, buildTarget, binaryType), + pathResolver.getRelativePath(dexFilesInfo.primaryDexPath), + allAssetDirectories, + nativeLibraryDirectoriesBuilder.build(), + zipFiles.build(), + thirdPartyJars, + pathResolver.getAbsolutePath(keystorePath), + keystoreProperties, + false, + javaRuntimeLauncher.getCommandPrefix(pathResolver), + androidSdkLocation)); + } +} diff --git a/src/com/facebook/buck/android/AndroidApkOptimizer.java b/src/com/facebook/buck/android/AndroidApkOptimizer.java new file mode 100644 index 00000000000..9f1d73dbc5c --- /dev/null +++ b/src/com/facebook/buck/android/AndroidApkOptimizer.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.buck.android; + +import static com.facebook.buck.android.BinaryType.APK; + +import com.facebook.buck.android.redex.RedexOptions; +import com.facebook.buck.android.toolchain.AndroidPlatformTarget; +import com.facebook.buck.android.toolchain.AndroidSdkLocation; +import com.facebook.buck.core.model.BuildTarget; +import com.facebook.buck.core.sourcepath.SourcePath; +import com.facebook.buck.io.filesystem.ProjectFilesystem; +import com.facebook.buck.step.Step; +import com.google.common.collect.ImmutableList; +import java.nio.file.Path; +import java.util.Optional; +import java.util.function.Supplier; + +/** The class executes all binary steps responsible for optimizing apk specific components. */ +public class AndroidApkOptimizer extends AndroidBinaryOptimizer { + + AndroidApkOptimizer( + BuildTarget buildTarget, + ProjectFilesystem filesystem, + AndroidSdkLocation androidSdkLocation, + AndroidPlatformTarget androidPlatformTarget, + SourcePath keystorePath, + SourcePath keystorePropertiesPath, + Optional redexOptions, + boolean packageAssetLibraries, + boolean compressAssetLibraries, + Optional assetCompressionAlgorithm, + boolean isCompressResources) { + super( + buildTarget, + filesystem, + androidSdkLocation, + androidPlatformTarget, + keystorePath, + keystorePropertiesPath, + redexOptions, + packageAssetLibraries, + compressAssetLibraries, + assetCompressionAlgorithm, + isCompressResources, + APK); + } + + @Override + void getBinaryTypeSpecificBuildSteps( + ImmutableList.Builder steps, + Path apkToAlign, + Path finalApkPath, + Supplier keystoreProperties, + boolean applyRedex) { + Path zipalignedApkPath = + AndroidBinaryPathUtility.getZipalignedApkPath(filesystem, buildTarget, binaryType); + steps.add( + new ZipalignStep( + filesystem.getRootPath(), androidPlatformTarget, apkToAlign, zipalignedApkPath)); + steps.add( + new ApkSignerStep( + filesystem, zipalignedApkPath, finalApkPath, keystoreProperties, applyRedex)); + } +} diff --git a/src/com/facebook/buck/android/AndroidAppModularityDescription.java b/src/com/facebook/buck/android/AndroidAppModularityDescription.java index 1f0333b90f7..ede606029c4 100644 --- a/src/com/facebook/buck/android/AndroidAppModularityDescription.java +++ b/src/com/facebook/buck/android/AndroidAppModularityDescription.java @@ -55,6 +55,7 @@ public BuildRule createBuildRule( args.getApplicationModuleDependencies(), APKModuleGraph.extractTargetsFromQueries(args.getApplicationModuleBlacklist()), ImmutableSet.of(), + ImmutableSet.of(), context.getTargetGraph(), buildTarget); diff --git a/src/com/facebook/buck/android/AndroidBinary.java b/src/com/facebook/buck/android/AndroidBinary.java index 2d779477d76..7a187e25d1e 100644 --- a/src/com/facebook/buck/android/AndroidBinary.java +++ b/src/com/facebook/buck/android/AndroidBinary.java @@ -16,6 +16,8 @@ package com.facebook.buck.android; +import static com.facebook.buck.android.BinaryType.APK; + import com.facebook.buck.android.FilterResourcesSteps.ResourceFilter; import com.facebook.buck.android.ResourcesFilter.ResourceCompressionMode; import com.facebook.buck.android.apkmodule.APKModule; @@ -118,6 +120,7 @@ public class AndroidBinary extends AbstractBuildRule private final BuildRuleParams buildRuleParams; @AddToRuleKey private final AndroidBinaryBuildable buildable; + @AddToRuleKey private final AndroidBinaryOptimizer optimizer; private final Supplier> transitiveClasspathDepsSupplier; @@ -203,17 +206,12 @@ public class AndroidBinary extends AbstractBuildRule } this.buildable = - new AndroidBinaryBuildable( + new AndroidApkBuildable( getBuildTarget(), getProjectFilesystem(), androidSdkLocation, - androidPlatformTarget, keystore.getPathToStore(), keystore.getPathToPropertiesFile(), - redexOptions, - redexOptions - .map(options -> enhancementResult.getAdditionalRedexInputs()) - .orElse(ImmutableList.of()), exopackageModes, xzCompressionLevel, packageAssetLibraries, @@ -221,14 +219,27 @@ public class AndroidBinary extends AbstractBuildRule assetCompressionAlgorithm, javaRuntimeLauncher, enhancementResult.getAndroidManifestPath(), - resourceCompressionMode.isCompressResources(), dexFilesInfo, nativeFilesInfo, resourceFilesInfo, apkModules, enhancementResult.getModuleResourceApkPaths(), Optional.empty(), - true); + APK); + + this.optimizer = + new AndroidApkOptimizer( + getBuildTarget(), + getProjectFilesystem(), + androidSdkLocation, + androidPlatformTarget, + keystore.getPathToStore(), + keystore.getPathToPropertiesFile(), + redexOptions, + packageAssetLibraries, + compressAssetLibraries, + assetCompressionAlgorithm, + resourceCompressionMode.isCompressResources()); this.exopackageInfo = exopackageInfo; params = @@ -352,12 +363,17 @@ public boolean inputBasedRuleKeyIsEnabled() { @Override public ImmutableList getBuildSteps( BuildContext context, BuildableContext buildableContext) { - return buildable.getBuildSteps(context, buildableContext); + return ImmutableList.builder() + .addAll(buildable.getBuildSteps(context, buildableContext)) + .addAll(optimizer.getBuildSteps(context, buildableContext)) + .build(); } @Override public SourcePath getSourcePathToOutput() { - return ExplicitBuildTargetSourcePath.of(getBuildTarget(), buildable.getFinalApkPath()); + return ExplicitBuildTargetSourcePath.of( + getBuildTarget(), + AndroidBinaryPathUtility.getFinalApkPath(getProjectFilesystem(), getBuildTarget(), APK)); } public AndroidPackageableCollection getAndroidPackageableCollection() { diff --git a/src/com/facebook/buck/android/AndroidBinaryBuildable.java b/src/com/facebook/buck/android/AndroidBinaryBuildable.java index da561923880..866123776a7 100644 --- a/src/com/facebook/buck/android/AndroidBinaryBuildable.java +++ b/src/com/facebook/buck/android/AndroidBinaryBuildable.java @@ -16,11 +16,10 @@ package com.facebook.buck.android; +import static com.facebook.buck.android.BinaryType.APK; + import com.facebook.buck.android.apkmodule.APKModule; import com.facebook.buck.android.exopackage.ExopackageMode; -import com.facebook.buck.android.redex.ReDexStep; -import com.facebook.buck.android.redex.RedexOptions; -import com.facebook.buck.android.toolchain.AndroidPlatformTarget; import com.facebook.buck.android.toolchain.AndroidSdkLocation; import com.facebook.buck.core.build.buildable.context.BuildableContext; import com.facebook.buck.core.build.context.BuildContext; @@ -44,7 +43,6 @@ import com.facebook.buck.unarchive.UnzipStep; import com.facebook.buck.util.MoreSuppliers; import com.facebook.buck.util.stream.RichStream; -import com.facebook.buck.zip.RepackZipEntriesStep; import com.facebook.buck.zip.ZipScrubberStep; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -64,7 +62,11 @@ import java.util.Optional; import java.util.function.Supplier; -class AndroidBinaryBuildable implements AddsToRuleKey { +/** + * The class is responsible to build android binary types {@link BinaryType} with the help of + * respective builder steps. + */ +abstract class AndroidBinaryBuildable implements AddsToRuleKey { /** * This is the path from the root of the APK that should contain the metadata.txt and @@ -77,55 +79,42 @@ class AndroidBinaryBuildable implements AddsToRuleKey { @AddToRuleKey private final SourcePath androidManifestPath; - @AddToRuleKey private final DexFilesInfo dexFilesInfo; + @AddToRuleKey protected final DexFilesInfo dexFilesInfo; @AddToRuleKey private final NativeFilesInfo nativeFilesInfo; - @AddToRuleKey private final ResourceFilesInfo resourceFilesInfo; + @AddToRuleKey protected final ResourceFilesInfo resourceFilesInfo; @AddToRuleKey final boolean packageAssetLibraries; @AddToRuleKey final boolean compressAssetLibraries; @AddToRuleKey final Optional assetCompressionAlgorithm; - @AddToRuleKey private final Optional redexOptions; - - @AddToRuleKey - // Redex accesses some files that are indirectly referenced through the proguard command-line.txt. - // TODO(cjhopman): Redex shouldn't do that, or this list should be constructed more carefully. - private final ImmutableList additionalRedexInputs; - @AddToRuleKey private final int xzCompressionLevel; - @AddToRuleKey private final SourcePath keystorePath; + @AddToRuleKey protected final SourcePath keystorePath; @AddToRuleKey private final SourcePath keystorePropertiesPath; - @AddToRuleKey private final ImmutableSortedSet apkModules; + @AddToRuleKey protected final ImmutableSortedSet apkModules; // The java launcher is used for ApkBuilder. - @AddToRuleKey private final Tool javaRuntimeLauncher; - - // Post-process resource compression - @AddToRuleKey private final boolean isCompressResources; + @AddToRuleKey protected final Tool javaRuntimeLauncher; @AddToRuleKey private final ImmutableMap moduleResourceApkPaths; - private final boolean isApk; + protected final BinaryType binaryType; // Path to Bundles config file - @AddToRuleKey private final Optional bundleConfigFilePath; + @AddToRuleKey protected final Optional bundleConfigFilePath; // These should be the only things not added to the rulekey. - private final ProjectFilesystem filesystem; - private final BuildTarget buildTarget; - private final AndroidSdkLocation androidSdkLocation; - private final AndroidPlatformTarget androidPlatformTarget; + protected final ProjectFilesystem filesystem; + protected final BuildTarget buildTarget; + protected final AndroidSdkLocation androidSdkLocation; + protected final boolean useDynamicFeature; AndroidBinaryBuildable( BuildTarget buildTarget, ProjectFilesystem filesystem, AndroidSdkLocation androidSdkLocation, - AndroidPlatformTarget androidPlatformTarget, SourcePath keystorePath, SourcePath keystorePropertiesPath, - Optional redexOptions, - ImmutableList additionalRedexInputs, EnumSet exopackageModes, int xzCompressionLevel, boolean packageAssetLibraries, @@ -133,27 +122,23 @@ class AndroidBinaryBuildable implements AddsToRuleKey { Optional assetCompressionAlgorithm, Tool javaRuntimeLauncher, SourcePath androidManifestPath, - boolean isCompressResources, DexFilesInfo dexFilesInfo, NativeFilesInfo nativeFilesInfo, ResourceFilesInfo resourceFilesInfo, ImmutableSortedSet apkModules, ImmutableMap moduleResourceApkPaths, Optional bundleConfigFilePath, - boolean isApk) { + BinaryType binaryType, + boolean useDynamicFeature) { this.filesystem = filesystem; this.buildTarget = buildTarget; this.androidSdkLocation = androidSdkLocation; - this.androidPlatformTarget = androidPlatformTarget; this.keystorePath = keystorePath; this.keystorePropertiesPath = keystorePropertiesPath; - this.redexOptions = redexOptions; - this.additionalRedexInputs = additionalRedexInputs; this.exopackageModes = exopackageModes; this.xzCompressionLevel = xzCompressionLevel; this.javaRuntimeLauncher = javaRuntimeLauncher; this.androidManifestPath = androidManifestPath; - this.isCompressResources = isCompressResources; this.apkModules = apkModules; this.moduleResourceApkPaths = moduleResourceApkPaths; this.dexFilesInfo = dexFilesInfo; @@ -163,7 +148,8 @@ class AndroidBinaryBuildable implements AddsToRuleKey { this.assetCompressionAlgorithm = assetCompressionAlgorithm; this.resourceFilesInfo = resourceFilesInfo; this.bundleConfigFilePath = bundleConfigFilePath; - this.isApk = isApk; + this.binaryType = binaryType; + this.useDynamicFeature = useDynamicFeature; } @SuppressWarnings("PMD.PrematureDeclaration") @@ -174,7 +160,7 @@ public ImmutableList getBuildSteps( // The `HasInstallableApk` interface needs access to the manifest, so make sure we create our // own copy of this so that we don't have a runtime dep on the `AaptPackageResources` step. - Path manifestPath = getManifestPath(); + Path manifestPath = AndroidBinaryPathUtility.getManifestPath(filesystem, buildTarget); steps.add( MkdirStep.of( BuildCellRelativePath.fromCellRelativePath( @@ -247,7 +233,8 @@ public ImmutableList getBuildSteps( .build(); SourcePathResolverAdapter resolver = context.getSourcePathResolver(); - Path signedApkPath = getSignedApkPath(); + Path signedApkPath = + AndroidBinaryPathUtility.getSignedApkPath(filesystem, buildTarget, binaryType); Path pathToKeystore = resolver.getAbsolutePath(keystorePath); Supplier keystoreProperties = getKeystorePropertiesSupplier(resolver, pathToKeystore); @@ -256,127 +243,21 @@ public ImmutableList getBuildSteps( resourceFilesInfo.pathsToThirdPartyJars.stream() .map(resolver::getAbsolutePath) .collect(ImmutableSet.toImmutableSet()); - if (isApk) { - steps.add( - new ApkBuilderStep( - getProjectFilesystem(), - pathResolver.getAbsolutePath(resourceFilesInfo.resourcesApkPath), - getSignedApkPath(), - pathResolver.getRelativePath(dexFilesInfo.primaryDexPath), - allAssetDirectories, - nativeLibraryDirectoriesBuilder.build(), - zipFiles.build(), - thirdPartyJars, - pathToKeystore, - keystoreProperties, - false, - javaRuntimeLauncher.getCommandPrefix(pathResolver))); - } else { - ImmutableSet moduleNames = - apkModules.stream().map(APKModule::getName).collect(ImmutableSet.toImmutableSet()); - - for (Path path : dexFilesInfo.getSecondaryDexDirs(getProjectFilesystem(), pathResolver)) { - if (path.getFileName().toString().equals("additional_dexes")) { - File[] assetFiles = path.toFile().listFiles(); - if (assetFiles == null) { - continue; - } - for (File assetFile : assetFiles) { - if (!assetFile.getName().equals("assets")) { - continue; - } - File[] modules = assetFile.listFiles(); - if (modules == null) { - continue; - } - for (File module : modules) { - if (moduleNames.contains(module.getName())) { - continue; - } - baseModuleInfo.putAssetDirectories(module.toPath(), "assets"); - } - } - } else { - baseModuleInfo.putAssetDirectories(path, ""); - } - } - baseModuleInfo - .setResourceApk(pathResolver.getAbsolutePath(resourceFilesInfo.resourcesApkPath)) - .addDexFile(pathResolver.getRelativePath(dexFilesInfo.primaryDexPath)) - .setJarFilesThatMayContainResources(thirdPartyJars) - .setZipFiles(zipFiles.build()); - - modulesInfo.add(baseModuleInfo.build()); - - Optional bundleConfigPath = bundleConfigFilePath.map(pathResolver::getAbsolutePath); - - steps.add( - new AabBuilderStep( - getProjectFilesystem(), - getSignedApkPath(), - bundleConfigPath, - buildTarget, - false, - modulesInfo.build())); - } + getBinaryTypeSpecificBuildSteps( + steps, + baseModuleInfo, + keystoreProperties, + nativeLibraryDirectoriesBuilder, + allAssetDirectories, + pathResolver, + thirdPartyJars, + zipFiles, + modulesInfo); // The `ApkBuilderStep` delegates to android tools to build a ZIP with timestamps in it, making // the output non-deterministic. So use an additional scrubbing step to zero these out. steps.add(ZipScrubberStep.of(getProjectFilesystem().resolve(signedApkPath))); - - Path apkToRedexAndAlign; - // Optionally, compress the resources file in the .apk. - if (isCompressResources) { - Path compressedApkPath = getCompressedResourcesApkPath(); - apkToRedexAndAlign = compressedApkPath; - steps.add(createRepackZipEntriesStep(signedApkPath, compressedApkPath)); - } else { - apkToRedexAndAlign = signedApkPath; - } - - boolean applyRedex = redexOptions.isPresent(); - Path apkToAlign = apkToRedexAndAlign; - Path v2SignedApkPath = getFinalApkPath(); - - if (applyRedex) { - Path redexedApk = getRedexedApkPath(); - apkToAlign = redexedApk; - steps.addAll( - createRedexSteps( - context, - buildableContext, - resolver, - keystoreProperties, - apkToRedexAndAlign, - redexedApk)); - } - - if (isApk) { - Path zipalignedApkPath = getZipalignedApkPath(); - steps.add( - new ZipalignStep( - getProjectFilesystem().getRootPath(), - androidPlatformTarget, - apkToAlign, - zipalignedApkPath)); - steps.add( - new ApkSignerStep( - getProjectFilesystem(), - zipalignedApkPath, - v2SignedApkPath, - keystoreProperties, - applyRedex)); - - } else { - steps.add( - new ZipalignStep( - getProjectFilesystem().getRootPath(), - androidPlatformTarget, - apkToAlign, - v2SignedApkPath)); - } - buildableContext.recordArtifact(v2SignedApkPath); return steps.build(); } @@ -408,7 +289,8 @@ private void processModule( assetDirectoriesBuilderForThisModule); } - boolean shouldPackageAssetLibraries = packageAssetLibraries || !module.isRootModule(); + // Do not package native libraries as assets for dynamic feature modules. To keep the earlier implementation intact added check for useDynamicFeature. + boolean shouldPackageAssetLibraries = !useDynamicFeature && (packageAssetLibraries || !module.isRootModule()); if (!ExopackageMode.enabledForNativeLibraries(exopackageModes) && nativeFilesInfo.nativeLibsDirs.isPresent() && nativeFilesInfo.nativeLibsDirs.get().containsKey(module)) { @@ -447,7 +329,7 @@ private void processModule( addModuleResourceDirectory(module, context, moduleResourcesDirectories, steps); } - if (!addThisModule || isApk) { + if (!addThisModule || binaryType == APK) { return; } @@ -528,7 +410,8 @@ private void addNativeLibraryAsAssetDirectory( Preconditions.checkState( ExopackageMode.enabledForModules(exopackageModes) || !ExopackageMode.enabledForResources(exopackageModes)); - Path pathForNativeLibsAsAssets = getPathForNativeLibsAsAssets(); + Path pathForNativeLibsAsAssets = + AndroidBinaryPathUtility.getPathForNativeLibsAsAssets(filesystem, buildTarget); Path libSubdirectory = pathForNativeLibsAsAssets @@ -724,49 +607,12 @@ private Supplier getKeystorePropertiesSupplier( }); } - private RepackZipEntriesStep createRepackZipEntriesStep( - Path signedApkPath, Path compressedApkPath) { - return new RepackZipEntriesStep( - getProjectFilesystem(), - signedApkPath, - compressedApkPath, - ImmutableSet.of("resources.arsc")); - } - - private Iterable createRedexSteps( - BuildContext context, - BuildableContext buildableContext, - SourcePathResolverAdapter resolver, - Supplier keystoreProperties, - Path apkToRedexAndAlign, - Path redexedApk) { - ImmutableList.Builder steps = ImmutableList.builder(); - Path proguardConfigDir = getProguardTextFilesPath(); - steps.add( - MkdirStep.of( - BuildCellRelativePath.fromCellRelativePath( - context.getBuildCellRootPath(), getProjectFilesystem(), redexedApk.getParent()))); - ImmutableList redexSteps = - ReDexStep.createSteps( - getProjectFilesystem(), - androidSdkLocation, - resolver, - redexOptions.get(), - apkToRedexAndAlign, - redexedApk, - keystoreProperties, - proguardConfigDir, - buildableContext); - steps.addAll(redexSteps); - return steps.build(); - } - private CopyStep createCopyProguardFilesStep( SourcePathResolverAdapter pathResolver, SourcePath proguardTextFilesPath) { return CopyStep.forDirectory( getProjectFilesystem(), pathResolver.getRelativePath(proguardTextFilesPath), - getProguardTextFilesPath(), + AndroidBinaryPathUtility.getProguardTextFilesPath(filesystem, buildTarget), CopyStep.DirectoryMode.CONTENTS_ONLY); } @@ -778,73 +624,6 @@ public BuildTarget getBuildTarget() { return buildTarget; } - public Path getManifestPath() { - return BuildTargetPaths.getGenPath( - getProjectFilesystem(), getBuildTarget(), "%s/AndroidManifest.xml"); - } - - /** All native-libs-as-assets are copied to this directory before running apkbuilder. */ - private Path getPathForNativeLibsAsAssets() { - return BuildTargetPaths.getScratchPath( - getProjectFilesystem(), getBuildTarget(), "__native_libs_as_assets_%s__"); - } - - /** The APK at this path will be jar signed, but not zipaligned. */ - private Path getSignedApkPath() { - return Paths.get( - getUnsignedApkPath() - .replaceAll("\\.unsigned\\.apk$", ".signed.apk") - .replaceAll("\\.unsigned\\.aab$", ".signed.aab")); - } - - /** The APK at this path will be zipaligned and jar signed. */ - private Path getZipalignedApkPath() { - return Paths.get( - getUnsignedApkPath() - .replaceAll("\\.unsigned\\.apk$", ".zipaligned.apk") - .replaceAll("\\.unsigned\\.aab$", ".signed.aab")); - } - - /** The APK at this path will be zipaligned and v2 signed. */ - Path getFinalApkPath() { - return Paths.get( - getUnsignedApkPath() - .replaceAll("\\.unsigned\\.apk$", ".apk") - .replaceAll("\\.unsigned\\.aab$", ".aab")); - } - - /** The APK at this path will have compressed resources, but will not be zipaligned. */ - private Path getCompressedResourcesApkPath() { - return Paths.get( - getUnsignedApkPath() - .replaceAll("\\.unsigned\\.apk$", ".compressed.apk") - .replaceAll("\\.unsigned\\.aab$", ".compressed.aab")); - } - - private String getUnsignedApkPath() { - return getPath("%s.unsigned." + getExtension()).toString(); - } - - private String getExtension() { - return isApk ? "apk" : "aab"; - } - - private Path getPath(String format) { - return BuildTargetPaths.getGenPath(getProjectFilesystem(), getBuildTarget(), format); - } - - private Path getRedexedApkPath() { - Path path = BuildTargetPaths.getGenPath(getProjectFilesystem(), getBuildTarget(), "%s__redex"); - return path.resolve(getBuildTarget().getShortName() + ".redex." + getExtension()); - } - - /** - * Directory of text files used by proguard. Unforunately, this contains both inputs and outputs. - */ - private Path getProguardTextFilesPath() { - return BuildTargetPaths.getGenPath(getProjectFilesystem(), getBuildTarget(), "%s/proguard"); - } - @VisibleForTesting static Path getProguardOutputFromInputClasspath(Path proguardConfigDir, Path classpathEntry) { // Hehe, this is so ridiculously fragile. @@ -857,4 +636,15 @@ static Path getProguardOutputFromInputClasspath(Path proguardConfigDir, Path cla Path dirName = classpathEntry.getParent(); return proguardConfigDir.resolve(dirName).resolve(obfuscatedName); } + + abstract void getBinaryTypeSpecificBuildSteps( + ImmutableList.Builder steps, + ImmutableModuleInfo.Builder baseModuleInfo, + Supplier keystoreProperties, + ImmutableSet.Builder nativeLibraryDirectoriesBuilder, + ImmutableSet allAssetDirectories, + SourcePathResolverAdapter pathResolver, + ImmutableSet thirdPartyJars, + ImmutableSet.Builder zipFiles, + ImmutableSet.Builder modulesInfo); } diff --git a/src/com/facebook/buck/android/AndroidBinaryGraphEnhancerFactory.java b/src/com/facebook/buck/android/AndroidBinaryGraphEnhancerFactory.java index 68ee34d731b..cec81633313 100644 --- a/src/com/facebook/buck/android/AndroidBinaryGraphEnhancerFactory.java +++ b/src/com/facebook/buck/android/AndroidBinaryGraphEnhancerFactory.java @@ -117,6 +117,7 @@ public AndroidBinaryGraphEnhancer create( args.getApplicationModuleDependencies(), APKModuleGraph.extractTargetsFromQueries(args.getApplicationModuleBlacklist()), args.getApplicationModulesWithResources(), + args.getUseDynamicFeature()? args.getApplicationModulesWithManifest(): args.getApplicationModulesWithResources(), targetGraph, buildTarget); } diff --git a/src/com/facebook/buck/android/AndroidBinaryOptimizer.java b/src/com/facebook/buck/android/AndroidBinaryOptimizer.java new file mode 100644 index 00000000000..f869d71139f --- /dev/null +++ b/src/com/facebook/buck/android/AndroidBinaryOptimizer.java @@ -0,0 +1,200 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.buck.android; + +import com.facebook.buck.android.redex.ReDexStep; +import com.facebook.buck.android.redex.RedexOptions; +import com.facebook.buck.android.toolchain.AndroidPlatformTarget; +import com.facebook.buck.android.toolchain.AndroidSdkLocation; +import com.facebook.buck.core.build.buildable.context.BuildableContext; +import com.facebook.buck.core.build.context.BuildContext; +import com.facebook.buck.core.model.BuildTarget; +import com.facebook.buck.core.rulekey.AddToRuleKey; +import com.facebook.buck.core.rulekey.AddsToRuleKey; +import com.facebook.buck.core.sourcepath.SourcePath; +import com.facebook.buck.core.sourcepath.resolver.SourcePathResolverAdapter; +import com.facebook.buck.io.BuildCellRelativePath; +import com.facebook.buck.io.filesystem.ProjectFilesystem; +import com.facebook.buck.step.Step; +import com.facebook.buck.step.fs.MkdirStep; +import com.facebook.buck.util.MoreSuppliers; +import com.facebook.buck.zip.RepackZipEntriesStep; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableList.Builder; +import com.google.common.collect.ImmutableSet; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Optional; +import java.util.function.Supplier; + +/** The class executes all common binary steps responsible for optimizing aab/apk. */ +public abstract class AndroidBinaryOptimizer implements AddsToRuleKey { + + @AddToRuleKey final boolean packageAssetLibraries; + @AddToRuleKey final boolean compressAssetLibraries; + @AddToRuleKey final Optional assetCompressionAlgorithm; + + @AddToRuleKey private final Optional redexOptions; + + @AddToRuleKey private final SourcePath keystorePath; + @AddToRuleKey private final SourcePath keystorePropertiesPath; + + // Post-process resource compression + @AddToRuleKey private final boolean isCompressResources; + + protected final BinaryType binaryType; + + // These should be the only things not added to the rulekey. + protected final ProjectFilesystem filesystem; + protected final BuildTarget buildTarget; + private final AndroidSdkLocation androidSdkLocation; + protected final AndroidPlatformTarget androidPlatformTarget; + + AndroidBinaryOptimizer( + BuildTarget buildTarget, + ProjectFilesystem filesystem, + AndroidSdkLocation androidSdkLocation, + AndroidPlatformTarget androidPlatformTarget, + SourcePath keystorePath, + SourcePath keystorePropertiesPath, + Optional redexOptions, + boolean packageAssetLibraries, + boolean compressAssetLibraries, + Optional assetCompressionAlgorithm, + boolean isCompressResources, + BinaryType binaryType) { + this.filesystem = filesystem; + this.buildTarget = buildTarget; + this.androidSdkLocation = androidSdkLocation; + this.androidPlatformTarget = androidPlatformTarget; + this.keystorePath = keystorePath; + this.keystorePropertiesPath = keystorePropertiesPath; + this.redexOptions = redexOptions; + this.isCompressResources = isCompressResources; + this.packageAssetLibraries = packageAssetLibraries; + this.compressAssetLibraries = compressAssetLibraries; + this.assetCompressionAlgorithm = assetCompressionAlgorithm; + this.binaryType = binaryType; + } + + @SuppressWarnings("PMD.PrematureDeclaration") + public ImmutableList getBuildSteps( + BuildContext context, BuildableContext buildableContext) { + ImmutableList.Builder steps = ImmutableList.builder(); + SourcePathResolverAdapter resolver = context.getSourcePathResolver(); + + Path signedApkPath = + AndroidBinaryPathUtility.getSignedApkPath(filesystem, buildTarget, binaryType); + + Path apkToRedexAndAlign; + // Optionally, compress the resources file in the .apk. + if (isCompressResources) { + Path compressedApkPath = + AndroidBinaryPathUtility.getCompressedResourcesApkPath( + filesystem, buildTarget, binaryType); + apkToRedexAndAlign = compressedApkPath; + steps.add(createRepackZipEntriesStep(signedApkPath, compressedApkPath)); + } else { + apkToRedexAndAlign = signedApkPath; + } + + boolean applyRedex = redexOptions.isPresent(); + Path apkToAlign = apkToRedexAndAlign; + Path v2SignedApkPath = + AndroidBinaryPathUtility.getFinalApkPath(filesystem, buildTarget, binaryType); + + Path pathToKeystore = resolver.getAbsolutePath(keystorePath); + Supplier keystoreProperties = + getKeystorePropertiesSupplier(resolver, pathToKeystore); + + if (applyRedex) { + Path redexedApk = + AndroidBinaryPathUtility.getRedexedApkPath(filesystem, buildTarget, binaryType); + apkToAlign = redexedApk; + steps.addAll( + createRedexSteps( + context, + buildableContext, + resolver, + keystoreProperties, + apkToRedexAndAlign, + redexedApk)); + } + + getBinaryTypeSpecificBuildSteps( + steps, apkToAlign, v2SignedApkPath, keystoreProperties, applyRedex); + buildableContext.recordArtifact(v2SignedApkPath); + return steps.build(); + } + + private RepackZipEntriesStep createRepackZipEntriesStep( + Path signedApkPath, Path compressedApkPath) { + return new RepackZipEntriesStep( + filesystem, + signedApkPath, + compressedApkPath, + ImmutableSet.of("resources.arsc")); + } + + private Iterable createRedexSteps( + BuildContext context, + BuildableContext buildableContext, + SourcePathResolverAdapter resolver, + Supplier keystoreProperties, + Path apkToRedexAndAlign, + Path redexedApk) { + ImmutableList.Builder steps = ImmutableList.builder(); + Path proguardConfigDir = AndroidBinaryPathUtility.getProguardTextFilesPath(filesystem, buildTarget); + steps.add( + MkdirStep.of( + BuildCellRelativePath.fromCellRelativePath( + context.getBuildCellRootPath(), filesystem, redexedApk.getParent()))); + ImmutableList redexSteps = + ReDexStep.createSteps( + filesystem, + androidSdkLocation, + resolver, + redexOptions.get(), + apkToRedexAndAlign, + redexedApk, + keystoreProperties, + proguardConfigDir, + buildableContext); + steps.addAll(redexSteps); + return steps.build(); + } + + private Supplier getKeystorePropertiesSupplier( + SourcePathResolverAdapter resolver, Path pathToKeystore) { + return MoreSuppliers.memoize( + () -> { + try { + return KeystoreProperties.createFromPropertiesFile( + pathToKeystore, resolver.getAbsolutePath(keystorePropertiesPath), filesystem); + } catch (IOException e) { + throw new RuntimeException(); + } + }); + } + + abstract void getBinaryTypeSpecificBuildSteps( + Builder steps, + Path apkToAlign, + Path finalApkPath, + Supplier keystoreProperties, + boolean applyRedex); +} diff --git a/src/com/facebook/buck/android/AndroidBinaryPathUtility.java b/src/com/facebook/buck/android/AndroidBinaryPathUtility.java new file mode 100644 index 00000000000..994f4148533 --- /dev/null +++ b/src/com/facebook/buck/android/AndroidBinaryPathUtility.java @@ -0,0 +1,110 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.buck.android; + +import static com.facebook.buck.android.BinaryType.APK; + +import com.facebook.buck.core.model.BuildTarget; +import com.facebook.buck.core.model.impl.BuildTargetPaths; +import com.facebook.buck.io.filesystem.ProjectFilesystem; +import java.nio.file.Path; +import java.nio.file.Paths; + +/** Utility class to resolve path conflicts in aab/apk. */ +public final class AndroidBinaryPathUtility { + + private AndroidBinaryPathUtility() {} + + /** The APK at this path will have compressed resources, but will not be zipaligned. */ + static Path getCompressedResourcesApkPath( + ProjectFilesystem projectFilesystem, BuildTarget buildTarget, BinaryType binaryType) { + return Paths.get( + getUnsignedApkPath(projectFilesystem, buildTarget, binaryType) + .replaceAll("\\.unsigned\\.apk$", ".compressed.apk") + .replaceAll("\\.unsigned\\.aab$", ".compressed.aab")); + } + + /** The APK at this path will be zipaligned and jar signed. */ + static Path getZipalignedApkPath( + ProjectFilesystem projectFilesystem, BuildTarget buildTarget, BinaryType binaryType) { + return Paths.get( + getUnsignedApkPath(projectFilesystem, buildTarget, binaryType) + .replaceAll("\\.unsigned\\.apk$", ".zipaligned.apk") + .replaceAll("\\.unsigned\\.aab$", ".zipaligned.aab")); + } + + /** The APK at this path will be jar signed, but not zipaligned. */ + static Path getSignedApkPath( + ProjectFilesystem projectFilesystem, BuildTarget buildTarget, BinaryType binaryType) { + return Paths.get( + getUnsignedApkPath(projectFilesystem, buildTarget, binaryType) + .replaceAll("\\.unsigned\\.apk$", ".signed.apk") + .replaceAll("\\.unsigned\\.aab$", ".signed.aab")); + } + + /** The APK at this path will be zipaligned and v2 signed. */ + static Path getFinalApkPath( + ProjectFilesystem projectFilesystem, BuildTarget buildTarget, BinaryType binaryType) { + return Paths.get( + getUnsignedApkPath(projectFilesystem, buildTarget, binaryType) + .replaceAll("\\.unsigned\\.apk$", ".apk") + .replaceAll("\\.unsigned\\.aab$", ".aab")); + } + + /** + * Directory of text files used by proguard. Unforunately, this contains both inputs and outputs. + */ + static Path getProguardTextFilesPath( + ProjectFilesystem projectFilesystem, BuildTarget buildTarget) { + return BuildTargetPaths.getGenPath(projectFilesystem, buildTarget, "%s/proguard"); + } + + /** The APK at this path will be optimized with redex */ + static Path getRedexedApkPath( + ProjectFilesystem projectFilesystem, BuildTarget buildTarget, BinaryType binaryType) { + Path path = BuildTargetPaths.getGenPath(projectFilesystem, buildTarget, "%s__redex"); + return path.resolve(buildTarget.getShortName() + ".redex." + getExtension(binaryType)); + } + + /** The APK at this path will be unsigned */ + private static String getUnsignedApkPath( + ProjectFilesystem projectFilesystem, BuildTarget buildTarget, BinaryType binaryType) { + return getPath("%s.unsigned." + getExtension(binaryType), projectFilesystem, buildTarget) + .toString(); + } + + /** The path to AndroidManifest file */ + static Path getManifestPath(ProjectFilesystem projectFilesystem, BuildTarget buildTarget) { + return BuildTargetPaths.getGenPath(projectFilesystem, buildTarget, "%s/AndroidManifest.xml"); + } + + /** All native-libs-as-assets are copied to this directory before running apkbuilder. */ + static Path getPathForNativeLibsAsAssets( + ProjectFilesystem projectFilesystem, BuildTarget buildTarget) { + return BuildTargetPaths.getScratchPath( + projectFilesystem, buildTarget, "__native_libs_as_assets_%s__"); + } + + private static String getExtension(BinaryType binaryType) { + return binaryType == APK ? "apk" : "aab"; + } + + private static Path getPath( + String format, ProjectFilesystem projectFilesystem, BuildTarget buildTarget) { + return BuildTargetPaths.getGenPath(projectFilesystem, buildTarget, format); + } +} diff --git a/src/com/facebook/buck/android/AndroidBuckConfig.java b/src/com/facebook/buck/android/AndroidBuckConfig.java index f52deb21f0b..f6871b31755 100644 --- a/src/com/facebook/buck/android/AndroidBuckConfig.java +++ b/src/com/facebook/buck/android/AndroidBuckConfig.java @@ -64,7 +64,7 @@ public enum NdkSearchOrderEntry { } private static final ImmutableList DEFAULT_SDK_PATH_SEARCH_ORDER = - ImmutableList.of("ANDROID_SDK", "ANDROID_HOME", CONFIG_ENTRY_IN_SDK_PATH_SEARCH_ORDER); + ImmutableList.of("ANDROID_SDK", "ANDROID_HOME", "ANDROID_SDK_ROOT", CONFIG_ENTRY_IN_SDK_PATH_SEARCH_ORDER); private static final ImmutableList DEFAULT_ADB_SEARCH_ORDER = ImmutableList.of(SDK_ENTRY_IN_ADB_PATH_SEARCH_ORDER, PATH_ENTRY_IN_ADB_PATH_SEARCH_ORDER); private static final ImmutableList DEFAULT_NDK_SEARCH_ORDER = @@ -154,7 +154,7 @@ public Optional getSdkPath() { * SDK (for example, {@code ANDROID_SDK}). * *

    If nothing is specified in {@code .buckconfig} the default order is: {@code ANDROID_SDK}, - * {@code ANDROID_HOME}, {@code } + * {@code ANDROID_HOME}, {@code ANDROID_SDK_ROOT}, {@code } */ public ImmutableList getSdkPathSearchOrder() { return delegate diff --git a/src/com/facebook/buck/android/AndroidBundle.java b/src/com/facebook/buck/android/AndroidBundle.java index e4af16f79ba..c14298bf9ed 100644 --- a/src/com/facebook/buck/android/AndroidBundle.java +++ b/src/com/facebook/buck/android/AndroidBundle.java @@ -16,6 +16,8 @@ package com.facebook.buck.android; +import static com.facebook.buck.android.BinaryType.AAB; + import com.facebook.buck.android.FilterResourcesSteps.ResourceFilter; import com.facebook.buck.android.ResourcesFilter.ResourceCompressionMode; import com.facebook.buck.android.apkmodule.APKModule; @@ -67,7 +69,7 @@ * * *

    - * android_binary(
    + * android_bundle(
      *   name = 'messenger',
      *   manifest = 'AndroidManifest.xml',
      *   deps = [
    @@ -75,6 +77,26 @@
      *   ],
      * )
      * 
    + * + * Configuration for dynamic feature (enable use_split_dex, use_dynamic_feature and application_module_configs flags) : + *
    A new configuration application_modules_with_manifest is defined to decouple the manifest behaviour from the resources. This approach is aligned with dynamic features heuristics where base manifest has complete information of the feature manifest
    + *
    + * android_bundle(
    + *   name = 'messenger',
    + *   manifest = 'AndroidManifest.xml',
    + *   deps = [
    + *     '//src/com/facebook/messenger:messenger_library',
    + *   ],
    + *   use_split_dex = True,
    + *   use_dynamic_feature = True,
    + *   application_module_configs = {
    + *     "feature1":['//feature1:module_root'],
    + *   },
    + *   application_modules_with_manifest = [
    + *     "feature1",
    + *   ]
    + * )
    + * 
    */ public class AndroidBundle extends AbstractBuildRule implements SupportsInputBasedRuleKey, @@ -107,6 +129,7 @@ public class AndroidBundle extends AbstractBuildRule private final BuildRuleParams buildRuleParams; @AddToRuleKey private final AndroidBinaryBuildable buildable; + @AddToRuleKey private final AndroidBinaryOptimizer optimizer; // TODO(cjhopman): What's the difference between shouldProguard and skipProguard? AndroidBundle( @@ -141,7 +164,8 @@ public class AndroidBundle extends AbstractBuildRule ResourceFilesInfo resourceFilesInfo, ImmutableSortedSet apkModules, Optional exopackageInfo, - Optional bundleConfigFilePath) { + Optional bundleConfigFilePath, + boolean useDynamicFeature) { super(buildTarget, projectFilesystem); Preconditions.checkArgument(params.getExtraDeps().get().isEmpty()); this.ruleFinder = ruleFinder; @@ -185,17 +209,12 @@ public class AndroidBundle extends AbstractBuildRule } this.buildable = - new AndroidBinaryBuildable( + new AndroidBundleBuildable( getBuildTarget(), getProjectFilesystem(), androidSdkLocation, - androidPlatformTarget, keystore.getPathToStore(), keystore.getPathToPropertiesFile(), - redexOptions, - redexOptions - .map(options -> enhancementResult.getAdditionalRedexInputs()) - .orElse(ImmutableList.of()), exopackageModes, xzCompressionLevel, packageAssetLibraries, @@ -203,14 +222,27 @@ public class AndroidBundle extends AbstractBuildRule assetCompressionAlgorithm, javaRuntimeLauncher, enhancementResult.getAndroidManifestPath(), - resourceCompressionMode.isCompressResources(), dexFilesInfo, nativeFilesInfo, resourceFilesInfo, apkModules, enhancementResult.getModuleResourceApkPaths(), bundleConfigFilePath, - false); + AAB, + useDynamicFeature); + this.optimizer = + new AndroidBundleOptimizer( + getBuildTarget(), + getProjectFilesystem(), + androidSdkLocation, + androidPlatformTarget, + keystore.getPathToStore(), + keystore.getPathToPropertiesFile(), + redexOptions, + packageAssetLibraries, + compressAssetLibraries, + assetCompressionAlgorithm, + resourceCompressionMode.isCompressResources()); this.exopackageInfo = exopackageInfo; params = @@ -298,12 +330,17 @@ public boolean inputBasedRuleKeyIsEnabled() { @Override public ImmutableList getBuildSteps( BuildContext context, BuildableContext buildableContext) { - return buildable.getBuildSteps(context, buildableContext); + return ImmutableList.builder() + .addAll(buildable.getBuildSteps(context, buildableContext)) + .addAll(optimizer.getBuildSteps(context, buildableContext)) + .build(); } @Override public SourcePath getSourcePathToOutput() { - return ExplicitBuildTargetSourcePath.of(getBuildTarget(), buildable.getFinalApkPath()); + return ExplicitBuildTargetSourcePath.of( + getBuildTarget(), + AndroidBinaryPathUtility.getFinalApkPath(getProjectFilesystem(), getBuildTarget(), AAB)); } public Keystore getKeystore() { diff --git a/src/com/facebook/buck/android/AndroidBundleBuildable.java b/src/com/facebook/buck/android/AndroidBundleBuildable.java new file mode 100644 index 00000000000..563e0a06a99 --- /dev/null +++ b/src/com/facebook/buck/android/AndroidBundleBuildable.java @@ -0,0 +1,153 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.buck.android; + +import com.facebook.buck.android.apkmodule.APKModule; +import com.facebook.buck.android.exopackage.ExopackageMode; +import com.facebook.buck.android.toolchain.AndroidSdkLocation; +import com.facebook.buck.core.model.BuildTarget; +import com.facebook.buck.core.sourcepath.SourcePath; +import com.facebook.buck.core.sourcepath.resolver.SourcePathResolverAdapter; +import com.facebook.buck.core.toolchain.tool.Tool; +import com.facebook.buck.io.filesystem.ProjectFilesystem; +import com.facebook.buck.step.Step; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedSet; +import java.io.File; +import java.nio.file.Path; +import java.util.EnumSet; +import java.util.Optional; +import java.util.function.Supplier; + +/** + * The class is responsible to create final bundle by taking individual module information and + * passing to {@link com.android.bundle.Config.Bundletool} through {@link AabBuilderStep}. + */ +public class AndroidBundleBuildable extends AndroidBinaryBuildable { + + AndroidBundleBuildable( + BuildTarget buildTarget, + ProjectFilesystem filesystem, + AndroidSdkLocation androidSdkLocation, + SourcePath keystorePath, + SourcePath keystorePropertiesPath, + EnumSet exopackageModes, + int xzCompressionLevel, + boolean packageAssetLibraries, + boolean compressAssetLibraries, + Optional assetCompressionAlgorithm, + Tool javaRuntimeLauncher, + SourcePath androidManifestPath, + DexFilesInfo dexFilesInfo, + NativeFilesInfo nativeFilesInfo, + ResourceFilesInfo resourceFilesInfo, + ImmutableSortedSet apkModules, + ImmutableMap moduleResourceApkPaths, + Optional bundleConfigFilePath, + BinaryType binaryType, + boolean useDynamicFeature) { + super( + buildTarget, + filesystem, + androidSdkLocation, + keystorePath, + keystorePropertiesPath, + exopackageModes, + xzCompressionLevel, + packageAssetLibraries, + compressAssetLibraries, + assetCompressionAlgorithm, + javaRuntimeLauncher, + androidManifestPath, + dexFilesInfo, + nativeFilesInfo, + resourceFilesInfo, + apkModules, + moduleResourceApkPaths, + bundleConfigFilePath, + binaryType, + useDynamicFeature); + } + + @Override + void getBinaryTypeSpecificBuildSteps( + ImmutableList.Builder steps, + ImmutableModuleInfo.Builder baseModuleInfo, + Supplier keystoreProperties, + ImmutableSet.Builder nativeLibraryDirectoriesBuilder, + ImmutableSet allAssetDirectories, + SourcePathResolverAdapter pathResolver, + ImmutableSet thirdPartyJars, + ImmutableSet.Builder zipFiles, + ImmutableSet.Builder modulesInfo) { + + addAdditionalDexes(baseModuleInfo, pathResolver); + + baseModuleInfo + .setResourceApk(pathResolver.getAbsolutePath(resourceFilesInfo.resourcesApkPath)) + .addDexFile(pathResolver.getRelativePath(dexFilesInfo.primaryDexPath)) + .setJarFilesThatMayContainResources(thirdPartyJars) + .setZipFiles(zipFiles.build()); + + modulesInfo.add(baseModuleInfo.build()); + + Optional bundleConfigPath = bundleConfigFilePath.map(pathResolver::getAbsolutePath); + + steps.add( + new AabBuilderStep( + getProjectFilesystem(), + AndroidBinaryPathUtility.getSignedApkPath(filesystem, buildTarget, binaryType), + bundleConfigPath, + buildTarget, + false, + modulesInfo.build())); + } + + private void addAdditionalDexes( + ImmutableModuleInfo.Builder baseModuleInfo, SourcePathResolverAdapter pathResolver) { + ImmutableSet moduleNames = + apkModules.stream().map(APKModule::getName).collect(ImmutableSet.toImmutableSet()); + + for (Path path : dexFilesInfo.getSecondaryDexDirs(getProjectFilesystem(), pathResolver)) { + if (path.getFileName().toString().equals("additional_dexes")) { + File[] assetFiles = path.toFile().listFiles(); + if (assetFiles == null) { + continue; + } + for (File assetFile : assetFiles) { + if (!assetFile.getName().equals("assets")) { + continue; + } + File[] modules = assetFile.listFiles(); + if (modules == null) { + continue; + } + for (File module : modules) { + if (moduleNames.contains(module.getName())) { + continue; + } + baseModuleInfo.putAssetDirectories(module.toPath(), "assets"); + } + } + } else { + baseModuleInfo.putAssetDirectories(path, ""); + } + } + } +} diff --git a/src/com/facebook/buck/android/AndroidBundleFactory.java b/src/com/facebook/buck/android/AndroidBundleFactory.java index 02351871726..2c41c20fcda 100644 --- a/src/com/facebook/buck/android/AndroidBundleFactory.java +++ b/src/com/facebook/buck/android/AndroidBundleFactory.java @@ -135,6 +135,7 @@ public AndroidBundle create( filesInfo.getResourceFilesInfo(), ImmutableSortedSet.copyOf(result.getAPKModuleGraph().getAPKModules()), filesInfo.getExopackageInfo(), - args.getBundleConfigFile()); + args.getBundleConfigFile(), + args.getUseDynamicFeature()); } } diff --git a/src/com/facebook/buck/android/AndroidBundleOptimizer.java b/src/com/facebook/buck/android/AndroidBundleOptimizer.java new file mode 100644 index 00000000000..35358950cb3 --- /dev/null +++ b/src/com/facebook/buck/android/AndroidBundleOptimizer.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.buck.android; + +import static com.facebook.buck.android.BinaryType.AAB; + +import com.facebook.buck.android.redex.RedexOptions; +import com.facebook.buck.android.toolchain.AndroidPlatformTarget; +import com.facebook.buck.android.toolchain.AndroidSdkLocation; +import com.facebook.buck.core.model.BuildTarget; +import com.facebook.buck.core.sourcepath.SourcePath; +import com.facebook.buck.io.filesystem.ProjectFilesystem; +import com.facebook.buck.step.Step; +import com.google.common.collect.ImmutableList; +import java.nio.file.Path; +import java.util.Optional; +import java.util.function.Supplier; + +/** The class executes all binary steps responsible for optimizing aab specific components. */ +public class AndroidBundleOptimizer extends AndroidBinaryOptimizer { + + AndroidBundleOptimizer( + BuildTarget buildTarget, + ProjectFilesystem filesystem, + AndroidSdkLocation androidSdkLocation, + AndroidPlatformTarget androidPlatformTarget, + SourcePath keystorePath, + SourcePath keystorePropertiesPath, + Optional redexOptions, + boolean packageAssetLibraries, + boolean compressAssetLibraries, + Optional assetCompressionAlgorithm, + boolean isCompressResources) { + super( + buildTarget, + filesystem, + androidSdkLocation, + androidPlatformTarget, + keystorePath, + keystorePropertiesPath, + redexOptions, + packageAssetLibraries, + compressAssetLibraries, + assetCompressionAlgorithm, + isCompressResources, + AAB); + } + + @Override + void getBinaryTypeSpecificBuildSteps( + ImmutableList.Builder steps, + Path apkToAlign, + Path finalApkPath, + Supplier keystoreProperties, + boolean applyRedex) { + steps.add( + new ZipalignStep( + filesystem.getRootPath(), androidPlatformTarget, apkToAlign, finalApkPath)); + } +} diff --git a/src/com/facebook/buck/android/AndroidGraphEnhancerArgs.java b/src/com/facebook/buck/android/AndroidGraphEnhancerArgs.java index ef20676dd33..2bfc1855485 100644 --- a/src/com/facebook/buck/android/AndroidGraphEnhancerArgs.java +++ b/src/com/facebook/buck/android/AndroidGraphEnhancerArgs.java @@ -101,6 +101,11 @@ default Set getApplicationModulesWithResources() { return ImmutableSet.of(); } + @Value.Default + default Set getApplicationModulesWithManifest() { + return ImmutableSet.of(); + } + Optional>> getApplicationModuleDependencies(); @Value.Default @@ -203,6 +208,11 @@ default boolean isSkipProguard() { return false; } + @Value.Default + default boolean getUseDynamicFeature() { + return false; + } + @Value.Default default ImmutableSet getExtraFilteredResources() { return ImmutableSet.of(); diff --git a/src/com/facebook/buck/android/AndroidInstrumentationApkDescription.java b/src/com/facebook/buck/android/AndroidInstrumentationApkDescription.java index 1a69996484a..0907880a9b9 100644 --- a/src/com/facebook/buck/android/AndroidInstrumentationApkDescription.java +++ b/src/com/facebook/buck/android/AndroidInstrumentationApkDescription.java @@ -123,7 +123,7 @@ public BuildRule createBuildRule( .map(BuildRule::getBuildTarget) .forEach(buildTargetsToExclude::add); - APKModule rootAPKModule = APKModule.of(APKModuleGraph.ROOT_APKMODULE_NAME, true); + APKModule rootAPKModule = APKModule.of(APKModuleGraph.ROOT_APKMODULE_NAME, true, true); AndroidPackageableCollection.ResourceDetails resourceDetails = apkUnderTest.getAndroidPackageableCollection().getResourceDetails().get(rootAPKModule); ImmutableSet resourcesToExclude = diff --git a/src/com/facebook/buck/android/AndroidManifestFactory.java b/src/com/facebook/buck/android/AndroidManifestFactory.java index e89781e76e7..76cd43a966e 100644 --- a/src/com/facebook/buck/android/AndroidManifestFactory.java +++ b/src/com/facebook/buck/android/AndroidManifestFactory.java @@ -42,7 +42,7 @@ public AndroidManifest createBuildRule( projectFilesystem, resolver, skeleton, - APKModule.of(APKModuleGraph.ROOT_APKMODULE_NAME, true), + APKModule.of(APKModuleGraph.ROOT_APKMODULE_NAME, true, true), manifestFiles); } } diff --git a/src/com/facebook/buck/android/ApkBuilderStep.java b/src/com/facebook/buck/android/ApkBuilderStep.java index 5e439f60f4c..8223a77e65e 100644 --- a/src/com/facebook/buck/android/ApkBuilderStep.java +++ b/src/com/facebook/buck/android/ApkBuilderStep.java @@ -26,6 +26,7 @@ import com.facebook.buck.step.Step; import com.facebook.buck.step.StepExecutionResult; import com.facebook.buck.step.StepExecutionResults; +import com.facebook.buck.android.toolchain.AndroidSdkLocation; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMultimap; @@ -58,6 +59,7 @@ public class ApkBuilderStep implements Step { private final boolean debugMode; private final ImmutableList javaRuntimeLauncher; private final AppBuilderBase appBuilderBase; + private final AndroidSdkLocation androidSdkLocation; /** * @param resourceApk Path to the Apk which only contains resources, no dex files. @@ -80,7 +82,8 @@ public ApkBuilderStep( Path pathToKeystore, Supplier keystorePropertiesSupplier, boolean debugMode, - ImmutableList javaRuntimeLauncher) { + ImmutableList javaRuntimeLauncher, + AndroidSdkLocation androidSdkLocation) { this.filesystem = filesystem; this.resourceApk = resourceApk; this.pathToOutputApkFile = pathToOutputApkFile; @@ -93,6 +96,7 @@ public ApkBuilderStep( this.javaRuntimeLauncher = javaRuntimeLauncher; this.appBuilderBase = new AppBuilderBase(filesystem, keystorePropertiesSupplier, pathToKeystore); + this.androidSdkLocation = androidSdkLocation; } @Override @@ -161,9 +165,7 @@ public String getDescription(ExecutionContext context) { args.addAll(javaRuntimeLauncher); args.add( "-classpath", - // TODO(mbolin): Make the directory that corresponds to $ANDROID_HOME a field that is - // accessible via an AndroidPlatformTarget and insert that here in place of "$ANDROID_HOME". - "$ANDROID_HOME/tools/lib/sdklib.jar", + androidSdkLocation.getSdkRootPath().toString() + "/tools/lib/sdklib.jar", "com.android.sdklib.build.ApkBuilderMain"); args.add(String.valueOf(pathToOutputApkFile)); args.add("-v" /* verbose */); diff --git a/src/com/facebook/buck/android/BUCK b/src/com/facebook/buck/android/BUCK index 6539b9c5dae..b766e7965a4 100644 --- a/src/com/facebook/buck/android/BUCK +++ b/src/com/facebook/buck/android/BUCK @@ -48,7 +48,12 @@ RULES_SRCS = [ "AndroidAppModularityDescription.java", "AndroidBinary.java", "AndroidBundle.java", + "AndroidApkOptimizer.java", + "AndroidBinaryOptimizer.java", + "AndroidBundleOptimizer.java", + "AndroidApkBuildable.java", "AndroidBinaryBuildable.java", + "AndroidBundleBuildable.java", "AndroidBinaryDescription.java", "AndroidBinaryFactory.java", "AndroidBundleDescription.java", diff --git a/src/com/facebook/buck/android/BinaryType.java b/src/com/facebook/buck/android/BinaryType.java new file mode 100644 index 00000000000..ef93fa54f66 --- /dev/null +++ b/src/com/facebook/buck/android/BinaryType.java @@ -0,0 +1,23 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.buck.android; + +/** Supported android binary types */ +public enum BinaryType { + APK, + AAB +} diff --git a/src/com/facebook/buck/android/apkmodule/APKModule.java b/src/com/facebook/buck/android/apkmodule/APKModule.java index 5c56a8b5efc..4820b12bb59 100644 --- a/src/com/facebook/buck/android/apkmodule/APKModule.java +++ b/src/com/facebook/buck/android/apkmodule/APKModule.java @@ -24,8 +24,8 @@ @BuckStyleValue public abstract class APKModule implements Comparable, AddsToRuleKey { - public static APKModule of(String name, boolean hasResources) { - return ImmutableAPKModule.of(name, hasResources); + public static APKModule of(String name, boolean hasResources, boolean hasManifest) { + return ImmutableAPKModule.of(name, hasResources, hasManifest); } @AddToRuleKey @@ -34,6 +34,9 @@ public static APKModule of(String name, boolean hasResources) { @AddToRuleKey public abstract boolean hasResources(); + @AddToRuleKey + public abstract boolean hasManifest(); + @Value.Derived public boolean isRootModule() { return getName().equals(APKModuleGraph.ROOT_APKMODULE_NAME); diff --git a/src/com/facebook/buck/android/apkmodule/APKModuleGraph.java b/src/com/facebook/buck/android/apkmodule/APKModuleGraph.java index 5b20a3a8a1e..c2335f111ac 100644 --- a/src/com/facebook/buck/android/apkmodule/APKModuleGraph.java +++ b/src/com/facebook/buck/android/apkmodule/APKModuleGraph.java @@ -81,6 +81,7 @@ public class APKModuleGraph implements AddsToRuleKey { @AddToRuleKey private final Optional> blacklistedModules; @AddToRuleKey private final Set modulesWithResources; + @AddToRuleKey private final Set modulesWithManifest; private final Optional> seedTargets; private final Map> buildTargetsMap = new HashMap<>(); @@ -140,6 +141,7 @@ public APKModuleGraph(TargetGraph targetGraph, BuildTarget target) { Optional.empty(), Optional.empty(), ImmutableSet.of(), + ImmutableSet.of(), targetGraph, target); } @@ -156,6 +158,9 @@ public APKModuleGraph(TargetGraph targetGraph, BuildTarget target) { * target is required by both these modules, we can safely place it in the minimal cover which * is the APKModule m2. * @param blacklistedModules A list of targets that will NOT be included in any module. + * @param modulesWithResources A list of modules with resources + * @param modulesWithManifest A list of modules with manifests. This parameter will be later used + * to fetch Manifest information for each module. * @param targetGraph The full target graph of the build * @param target The root target to use to traverse the graph */ @@ -164,12 +169,14 @@ public APKModuleGraph( Optional>> appModuleDependencies, Optional> blacklistedModules, Set modulesWithResources, + Set modulesWithManifest, TargetGraph targetGraph, BuildTarget target) { this.targetGraph = targetGraph; this.appModuleDependencies = appModuleDependencies; this.blacklistedModules = blacklistedModules; this.modulesWithResources = modulesWithResources; + this.modulesWithManifest = modulesWithManifest; this.target = target; this.seedTargets = Optional.empty(); this.suppliedSeedConfigMap = seedConfigMap; @@ -191,6 +198,7 @@ public APKModuleGraph( this.appModuleDependencies = Optional.empty(); this.blacklistedModules = Optional.empty(); this.modulesWithResources = ImmutableSet.of(); + this.modulesWithManifest = ImmutableSet.of(); } public ImmutableSortedMap> toOutgoingEdgesMap() { @@ -301,6 +309,18 @@ public APKModule findResourceModuleForTarget(BuildTarget target) { return (module == null || !module.hasResources()) ? rootAPKModuleSupplier.get() : module; } + /** + * Get the Module that should contain the manifest for the given target or else fallback to the same logic of findResourceModuleForTarget. + * + * @param target target to search for in modules + * @return the module that contains the target + */ + public APKModule findManifestModuleForTarget(BuildTarget target) { + APKModule module = targetToModuleMapSupplier.get().get(target); + return (module == null || !module.hasManifest()) ? findResourceModuleForTarget(target) : module; + } + + /** * Group the classes in the input jars into a multimap based on the APKModule they belong to * @@ -438,7 +458,7 @@ public Iterable> visit(TargetNode node) { } }.start(); } - APKModule rootModule = APKModule.of(ROOT_APKMODULE_NAME, true); + APKModule rootModule = APKModule.of(ROOT_APKMODULE_NAME, true, true); buildTargetsMap.put(rootModule, ImmutableSet.copyOf(rootTargets)); return rootModule; } @@ -548,7 +568,7 @@ public int compare(TreeSet left, TreeSet right) { for (TreeSet moduleCover : sortedContainingModuleSets) { String moduleName = moduleCover.size() == 1 ? moduleCover.iterator().next() : "shared" + currentId++; - APKModule module = APKModule.of(moduleName, modulesWithResources.contains(moduleName)); + APKModule module = APKModule.of(moduleName, modulesWithResources.contains(moduleName), modulesWithManifest.contains(moduleName)); combinedModuleHashToModuleMap.put(ImmutableSet.copyOf(moduleCover), module); } diff --git a/src/com/facebook/buck/android/dalvik/DalvikStatsCache.java b/src/com/facebook/buck/android/dalvik/DalvikStatsCache.java index 1b11d44d8db..a44dbb82a45 100644 --- a/src/com/facebook/buck/android/dalvik/DalvikStatsCache.java +++ b/src/com/facebook/buck/android/dalvik/DalvikStatsCache.java @@ -32,10 +32,13 @@ class DalvikStatsCache { } DalvikStatsTool.Stats getStats(FileLike entry) { - String name = entry.getRelativePath(); - if (!name.endsWith(".class")) { + String[] pathParts = entry.getRelativePath().split("/"); + String name = pathParts[pathParts.length-1]; + if (!name.endsWith(".class") || name.equals("module-info.class")) { // Probably something like a pom.properties file in a JAR: this does not contribute // to the linear alloc size, so return zero. + // skipping special class files like module descriptor - here no classes will be + // declared and class visitor throws error. return DalvikStatsTool.Stats.ZERO; } diff --git a/src/com/facebook/buck/android/packageable/AndroidPackageableCollector.java b/src/com/facebook/buck/android/packageable/AndroidPackageableCollector.java index f834be683b6..a5e1863d185 100644 --- a/src/com/facebook/buck/android/packageable/AndroidPackageableCollector.java +++ b/src/com/facebook/buck/android/packageable/AndroidPackageableCollector.java @@ -315,7 +315,7 @@ public AndroidPackageableCollector addManifestPiece(BuildTarget owner, SourcePat return this; } collectionBuilder.putAndroidManifestPieces( - apkModuleGraph.findResourceModuleForTarget(owner), manifest); + apkModuleGraph.findManifestModuleForTarget(owner), manifest); return this; } diff --git a/src/com/facebook/buck/android/toolchain/impl/AndroidSdkDirectoryResolver.java b/src/com/facebook/buck/android/toolchain/impl/AndroidSdkDirectoryResolver.java index ebad242c703..8c2b430437b 100644 --- a/src/com/facebook/buck/android/toolchain/impl/AndroidSdkDirectoryResolver.java +++ b/src/com/facebook/buck/android/toolchain/impl/AndroidSdkDirectoryResolver.java @@ -32,7 +32,7 @@ public class AndroidSdkDirectoryResolver extends BaseAndroidToolchainResolver { @VisibleForTesting static final String SDK_NOT_FOUND_MESSAGE = "Android SDK could not be found. Make sure to set " - + "one of these environment variables: ANDROID_SDK, ANDROID_HOME, " + + "one of these environment variables: ANDROID_SDK, ANDROID_HOME, ANDROID_SDK_ROOT, " + "or android.sdk_path in your .buckconfig"; private Optional sdkErrorMessage; diff --git a/src/com/facebook/buck/android/toolchain/ndk/impl/AndroidNdkResolver.java b/src/com/facebook/buck/android/toolchain/ndk/impl/AndroidNdkResolver.java index 9a1edd077bb..d08416f3331 100644 --- a/src/com/facebook/buck/android/toolchain/ndk/impl/AndroidNdkResolver.java +++ b/src/com/facebook/buck/android/toolchain/ndk/impl/AndroidNdkResolver.java @@ -47,7 +47,7 @@ public class AndroidNdkResolver extends BaseAndroidToolchainResolver { private static final Logger LOG = Logger.get(AndroidNdkResolver.class); /** Android NDK versions starting with this number are not supported. */ - private static final String NDK_MIN_UNSUPPORTED_VERSION = "21"; + private static final String NDK_MIN_UNSUPPORTED_VERSION = "22"; // Pre r11 NDKs store the version at RELEASE.txt. @VisibleForTesting static final String NDK_PRE_R11_VERSION_FILENAME = "RELEASE.TXT"; diff --git a/src/com/facebook/buck/android/toolchain/ndk/impl/NdkCxxPlatforms.java b/src/com/facebook/buck/android/toolchain/ndk/impl/NdkCxxPlatforms.java index 3cf595fd08c..f45d41427d0 100644 --- a/src/com/facebook/buck/android/toolchain/ndk/impl/NdkCxxPlatforms.java +++ b/src/com/facebook/buck/android/toolchain/ndk/impl/NdkCxxPlatforms.java @@ -184,8 +184,10 @@ public static String getDefaultClangVersionForNdk(String ndkVersion) { return "7.0.2"; } else if (ndkMajorVersion < 20) { return "8.0.2"; - } else { + } else if (ndkMajorVersion < 21) { return "8.0.7"; + } else { + return "9.0.8"; } } diff --git a/src/com/facebook/buck/apple/ActoolStep.java b/src/com/facebook/buck/apple/ActoolStep.java index 809412344be..d4564aa3b01 100644 --- a/src/com/facebook/buck/apple/ActoolStep.java +++ b/src/com/facebook/buck/apple/ActoolStep.java @@ -39,6 +39,7 @@ class ActoolStep extends ShellStep { private final Path outputPlist; private final Optional appIcon; private final Optional launchImage; + private final Optional productType; private final AppleAssetCatalogsCompilationOptions compilationOptions; public ActoolStep( @@ -52,6 +53,7 @@ public ActoolStep( Path outputPlist, Optional appIcon, Optional launchImage, + Optional productType, AppleAssetCatalogsCompilationOptions compilationOptions) { super(workingDirectory); this.applePlatformName = applePlatformName; @@ -63,6 +65,7 @@ public ActoolStep( this.outputPlist = outputPlist; this.appIcon = appIcon; this.launchImage = launchImage; + this.productType = productType; this.compilationOptions = compilationOptions; } @@ -104,6 +107,10 @@ protected ImmutableList getShellCommandInternal(ExecutionContext context commandBuilder.add("--launch-image", launchImage.get()); } + if (productType.isPresent()) { + commandBuilder.add("--product-type", productType.get()); + } + if (compilationOptions.getNotices()) { commandBuilder.add("--notices"); } diff --git a/src/com/facebook/buck/apple/AppleAssetCatalog.java b/src/com/facebook/buck/apple/AppleAssetCatalog.java index 92e308d42df..2cc436b5873 100644 --- a/src/com/facebook/buck/apple/AppleAssetCatalog.java +++ b/src/com/facebook/buck/apple/AppleAssetCatalog.java @@ -73,6 +73,8 @@ public class AppleAssetCatalog extends AbstractBuildRule { @AddToRuleKey private final Optional launchImage; + @AddToRuleKey private final Optional productType; + @AddToRuleKey private final AppleAssetCatalogsCompilationOptions compilationOptions; private final Supplier> buildDepsSupplier; @@ -103,6 +105,7 @@ public class AppleAssetCatalog extends AbstractBuildRule { ImmutableSortedSet assetCatalogDirs, Optional appIcon, Optional launchImage, + Optional productType, AppleAssetCatalogsCompilationOptions compilationOptions, String bundleName) { super(buildTarget, projectFilesystem); @@ -117,6 +120,7 @@ public class AppleAssetCatalog extends AbstractBuildRule { BuildTargetPaths.getScratchPath(getProjectFilesystem(), buildTarget, "%s-output.plist"); this.appIcon = appIcon; this.launchImage = launchImage; + this.productType = productType; this.compilationOptions = compilationOptions; this.buildDepsSupplier = BuildableSupport.buildDepsSupplier(this, ruleFinder); } @@ -148,6 +152,7 @@ public ImmutableList getBuildSteps( getProjectFilesystem().resolve(outputPlist), appIcon, launchImage, + productType, compilationOptions)); buildableContext.recordArtifact(getOutputDir()); diff --git a/src/com/facebook/buck/apple/AppleBinaryDescription.java b/src/com/facebook/buck/apple/AppleBinaryDescription.java index f897c59198d..d3045ef315b 100644 --- a/src/com/facebook/buck/apple/AppleBinaryDescription.java +++ b/src/com/facebook/buck/apple/AppleBinaryDescription.java @@ -177,25 +177,30 @@ public Optional>> flavorDomains( @Override public boolean hasFlavors( - ImmutableSet flavors, TargetConfiguration toolchainTargetConfiguration) { - if (FluentIterable.from(flavors).allMatch(SUPPORTED_FLAVORS::contains)) { + ImmutableSet flavors, TargetConfiguration toolchainTargetConfiguration) { + Set unmatchedFlavors = Sets.difference(flavors, SUPPORTED_FLAVORS); + if (unmatchedFlavors.isEmpty()) { return true; } ImmutableSet delegateFlavors = ImmutableSet.copyOf(Sets.difference(flavors, NON_DELEGATE_FLAVORS)); - if (swiftDelegate - .map(swift -> swift.hasFlavors(delegateFlavors, toolchainTargetConfiguration)) - .orElse(false)) { + ImmutableSet supportedDelegateFlavors = swiftDelegate + .map(swift -> swift.getSupportedFlavors(delegateFlavors, toolchainTargetConfiguration)) + .orElse(ImmutableSet.of()); + unmatchedFlavors = Sets.difference(unmatchedFlavors, supportedDelegateFlavors); + if (unmatchedFlavors.isEmpty()) { return true; } + ImmutableSet immutableUnmatchedFlavors = ImmutableSet.copyOf(unmatchedFlavors); ImmutableList> thinFlavorSets = - generateThinDelegateFlavors(delegateFlavors); + generateThinDelegateFlavors(immutableUnmatchedFlavors); if (thinFlavorSets.size() > 0) { return Iterables.all( thinFlavorSets, inputFlavors -> cxxBinaryFlavored.hasFlavors(inputFlavors, toolchainTargetConfiguration)); } else { - return cxxBinaryFlavored.hasFlavors(delegateFlavors, toolchainTargetConfiguration); + return cxxBinaryFlavored.hasFlavors(immutableUnmatchedFlavors, + toolchainTargetConfiguration); } } @@ -419,12 +424,14 @@ private BuildRule createBundleBuildRule( Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), appleConfig.getCodesignTimeout(), swiftBuckConfig.getCopyStdlibToFrameworks(), swiftBuckConfig.getUseLipoThin(), cxxBuckConfig.shouldCacheStrip(), appleConfig.useEntitlementsWhenAdhocCodeSigning(), - Predicates.alwaysTrue()); + Predicates.alwaysTrue(), + Optional.empty()); } private BuildRule createBinary( diff --git a/src/com/facebook/buck/apple/AppleBundle.java b/src/com/facebook/buck/apple/AppleBundle.java index b48c385569a..5b95eec9391 100644 --- a/src/com/facebook/buck/apple/AppleBundle.java +++ b/src/com/facebook/buck/apple/AppleBundle.java @@ -190,6 +190,7 @@ public class AppleBundle extends AbstractBuildRule private final Duration codesignTimeout; private final BuildRuleParams buildRuleParams; private BuildableSupport.DepsSupplier depsSupplier; + private final Optional isAppClip; AppleBundle( BuildTarget buildTarget, @@ -224,7 +225,8 @@ public class AppleBundle extends AbstractBuildRule Duration codesignTimeout, boolean copySwiftStdlibToFrameworks, boolean useLipoThin, - boolean useEntitlementsWhenAdhocCodeSigning) { + boolean useEntitlementsWhenAdhocCodeSigning, + Optional isAppClip) { super(buildTarget, projectFilesystem); this.buildRuleParams = params; this.extension = @@ -301,6 +303,7 @@ public class AppleBundle extends AbstractBuildRule this.useLipoThin = useLipoThin; this.useEntitlementsWhenAdhocCodeSigning = useEntitlementsWhenAdhocCodeSigning; this.depsSupplier = BuildableSupport.buildDepsSupplier(this, graphBuilder); + this.isAppClip = isAppClip; } public static String getBinaryName(BuildTarget buildTarget, Optional productName) { @@ -609,7 +612,8 @@ public ImmutableList getBuildSteps( sdkPath, lipo, bundleBinaryPath, - destinations); + destinations, + isAppClip.orElse(false)); for (BuildRule extraBinary : extraBinaries) { Path outputPath = getBundleBinaryPathForBuildRule(extraBinary); @@ -664,7 +668,8 @@ public ImmutableList getBuildSteps( sdkPath, lipo, bundleBinaryPath, - destinations); + destinations, + isAppClip.orElse(false)); } // Ensure the bundle directory is archived so we can fetch it later. @@ -809,7 +814,8 @@ public void addSwiftStdlibStepIfNeeded( sdkPath, lipo, bundleBinaryPath, - destinations); + destinations, + isAppClip.orElse(false)); } private void copyAnotherCopyOfWatchKitStub( @@ -991,6 +997,10 @@ private ImmutableMap getInfoPlistAdditionalKeys() { return keys.build(); } + public Boolean getIsAppClip() { + return isAppClip.orElse(false); + } + @Override public boolean isTestedBy(BuildTarget testRule) { if (tests.contains(testRule)) { diff --git a/src/com/facebook/buck/apple/AppleBundleDescription.java b/src/com/facebook/buck/apple/AppleBundleDescription.java index 28abfe871d1..cee0c9b9b2c 100644 --- a/src/com/facebook/buck/apple/AppleBundleDescription.java +++ b/src/com/facebook/buck/apple/AppleBundleDescription.java @@ -216,12 +216,14 @@ public AppleBundle createBuildRule( args.getCodesignIdentity(), args.getIbtoolModuleFlag(), args.getIbtoolFlags(), + args.getXcodeProductType(), appleConfig.getCodesignTimeout(), swiftBuckConfig.getCopyStdlibToFrameworks(), swiftBuckConfig.getUseLipoThin(), cxxBuckConfig.shouldCacheStrip(), appleConfig.useEntitlementsWhenAdhocCodeSigning(), - resourceFilter); + resourceFilter, + args.getIsAppClip()); } /** diff --git a/src/com/facebook/buck/apple/AppleBundleDestination.java b/src/com/facebook/buck/apple/AppleBundleDestination.java index 7313506ca7b..94a293cad7b 100644 --- a/src/com/facebook/buck/apple/AppleBundleDestination.java +++ b/src/com/facebook/buck/apple/AppleBundleDestination.java @@ -25,6 +25,7 @@ public enum AppleBundleDestination { RESOURCES, FRAMEWORKS, + APPCLIPS, EXECUTABLES, PLUGINS, XPCSERVICES; @@ -43,6 +44,8 @@ public Path getPath(AppleBundleDestinations destinations) { return destinations.getResourcesPath(); case EXECUTABLES: return destinations.getExecutablesPath(); + case APPCLIPS: + return destinations.getAppClipsPath(); case FRAMEWORKS: return destinations.getFrameworksPath(); case PLUGINS: diff --git a/src/com/facebook/buck/apple/AppleBundleDestinations.java b/src/com/facebook/buck/apple/AppleBundleDestinations.java index cbbfd0435c0..f49240ef058 100644 --- a/src/com/facebook/buck/apple/AppleBundleDestinations.java +++ b/src/com/facebook/buck/apple/AppleBundleDestinations.java @@ -35,6 +35,9 @@ abstract class AppleBundleDestinations implements AddsToRuleKey { @AddToRuleKey(stringify = true) public abstract Path getExecutablesPath(); + @AddToRuleKey(stringify = true) + public abstract Path getAppClipsPath(); + @AddToRuleKey(stringify = true) public abstract Path getFrameworksPath(); @@ -62,6 +65,7 @@ abstract class AppleBundleDestinations implements AddsToRuleKey { OSX_CONTENTS_PATH, OSX_CONTENTS_PATH.resolve("Resources"), OSX_CONTENTS_PATH.resolve("MacOS"), + OSX_CONTENTS_PATH, OSX_CONTENTS_PATH.resolve("Frameworks"), OSX_CONTENTS_PATH.resolve("PlugIns"), OSX_CONTENTS_PATH, @@ -76,6 +80,7 @@ abstract class AppleBundleDestinations implements AddsToRuleKey { OSX_FRAMEWORK_CONTENTS_PATH.resolve("Resources"), OSX_FRAMEWORK_CONTENTS_PATH.resolve("Resources"), OSX_FRAMEWORK_CONTENTS_PATH, + OSX_FRAMEWORK_CONTENTS_PATH, OSX_FRAMEWORK_CONTENTS_PATH.resolve("Frameworks"), OSX_FRAMEWORK_CONTENTS_PATH, OSX_FRAMEWORK_CONTENTS_PATH, @@ -90,6 +95,7 @@ abstract class AppleBundleDestinations implements AddsToRuleKey { IOS_CONTENTS_PATH, IOS_CONTENTS_PATH, IOS_CONTENTS_PATH, + IOS_CONTENTS_PATH.resolve("AppClips"), IOS_CONTENTS_PATH.resolve("Frameworks"), IOS_CONTENTS_PATH.resolve("PlugIns"), IOS_CONTENTS_PATH.resolve("Watch"), @@ -104,6 +110,7 @@ abstract class AppleBundleDestinations implements AddsToRuleKey { IOS_FRAMEWORK_CONTENTS_PATH, IOS_FRAMEWORK_CONTENTS_PATH, IOS_FRAMEWORK_CONTENTS_PATH, + IOS_FRAMEWORK_CONTENTS_PATH, IOS_FRAMEWORK_CONTENTS_PATH.resolve("Frameworks"), IOS_FRAMEWORK_CONTENTS_PATH, IOS_FRAMEWORK_CONTENTS_PATH, diff --git a/src/com/facebook/buck/apple/AppleDescriptions.java b/src/com/facebook/buck/apple/AppleDescriptions.java index bdff5dc069e..b0fca9cabf9 100644 --- a/src/com/facebook/buck/apple/AppleDescriptions.java +++ b/src/com/facebook/buck/apple/AppleDescriptions.java @@ -402,6 +402,7 @@ public static Optional createBuildRuleForTransitiveAssetCatal BuildTarget buildTarget, ProjectFilesystem projectFilesystem, SourcePathRuleFinder ruleFinder, + Optional productType, ApplePlatform applePlatform, String targetSDKVersion, Tool actool, @@ -479,6 +480,7 @@ public static Optional createBuildRuleForTransitiveAssetCatal assetCatalogDirs, appIcon, launchImage, + productType, appleAssetCatalogsCompilationOptions, MERGED_ASSET_CATALOG_NAME)); } @@ -667,12 +669,14 @@ static AppleBundle createAppleBundle( Optional codesignAdhocIdentity, Optional ibtoolModuleFlag, Optional> ibtoolFlags, + Optional productType, Duration codesignTimeout, boolean copySwiftStdlibToFrameworks, boolean useLipoThin, boolean cacheStrips, boolean useEntitlementsWhenAdhocCodeSigning, - Predicate filter) { + Predicate filter, + Optional isAppClip) { AppleCxxPlatform appleCxxPlatform = ApplePlatforms.getAppleCxxPlatformForBuildTarget( graphBuilder, @@ -751,6 +755,7 @@ static AppleBundle createAppleBundle( buildTargetWithoutBundleSpecificFlavors, projectFilesystem, graphBuilder, + productType, appleCxxPlatform.getAppleSdk().getApplePlatform(), appleCxxPlatform.getMinVersion(), appleCxxPlatform.getActool(), @@ -914,7 +919,8 @@ static AppleBundle createAppleBundle( codesignTimeout, copySwiftStdlibToFrameworks, useLipoThin, - useEntitlementsWhenAdhocCodeSigning); + useEntitlementsWhenAdhocCodeSigning, + isAppClip); } /** @@ -1096,6 +1102,9 @@ private static ImmutableMap collectFirstLevelAppleDependency destinationPath = destinations.getWatchAppPath(); } else if (appleBundle.isLegacyWatchApp()) { destinationPath = destinations.getResourcesPath(); + } + else if (appleBundle.getIsAppClip()) { + destinationPath = destinations.getAppClipsPath(); } else { destinationPath = destinations.getPlugInsPath(); } diff --git a/src/com/facebook/buck/apple/AppleLibraryDescription.java b/src/com/facebook/buck/apple/AppleLibraryDescription.java index 4ccea916d87..14093369cf0 100644 --- a/src/com/facebook/buck/apple/AppleLibraryDescription.java +++ b/src/com/facebook/buck/apple/AppleLibraryDescription.java @@ -461,12 +461,14 @@ private BuildRule createFramework Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), appleConfig.getCodesignTimeout(), swiftBuckConfig.getCopyStdlibToFrameworks(), swiftBuckConfig.getUseLipoThin(), cxxBuckConfig.shouldCacheStrip(), appleConfig.useEntitlementsWhenAdhocCodeSigning(), - Predicates.alwaysTrue()); + Predicates.alwaysTrue(), + Optional.empty()); } /** @@ -887,7 +889,7 @@ private static CxxHeaders createSwiftModuleHeaders( BuildTarget swiftCompileTarget = baseTarget.withAppendedFlavors(Type.SWIFT_COMPILE.getFlavor()); SwiftCompile compile = (SwiftCompile) graphBuilder.requireRule(swiftCompileTarget); - return CxxHeadersDir.of(CxxPreprocessables.IncludeType.LOCAL, compile.getOutputPath()); + return CxxHeadersDir.of(CxxPreprocessables.IncludeType.LOCAL, compile.getSwiftModuleOutputPath()); } private static CxxHeaders createSwiftObjcHeaders( diff --git a/src/com/facebook/buck/apple/AppleResourceProcessing.java b/src/com/facebook/buck/apple/AppleResourceProcessing.java index 75a56144fbe..fc2b67ad2c7 100644 --- a/src/com/facebook/buck/apple/AppleResourceProcessing.java +++ b/src/com/facebook/buck/apple/AppleResourceProcessing.java @@ -222,12 +222,14 @@ public static void addSwiftStdlibStepIfNeeded( Path sdkPath, Tool lipo, Path bundleBinaryPath, - AppleBundleDestinations destinations) { + AppleBundleDestinations destinations, + boolean isAppClip) { // It's apparently safe to run this even on a non-swift bundle (in that case, no libs // are copied over). boolean shouldCopySwiftStdlib = !bundleExtension.equals(AppleBundleExtension.APPEX.toFileExtension()) && (!bundleExtension.equals(AppleBundleExtension.FRAMEWORK.toFileExtension()) + && !isAppClip || copySwiftStdlibToFrameworks); if (swiftStdlibTool.isPresent() && shouldCopySwiftStdlib) { diff --git a/src/com/facebook/buck/apple/AppleTest.java b/src/com/facebook/buck/apple/AppleTest.java index fba2d03f8d9..1729d0d26ba 100644 --- a/src/com/facebook/buck/apple/AppleTest.java +++ b/src/com/facebook/buck/apple/AppleTest.java @@ -104,6 +104,7 @@ public class AppleTest extends AbstractBuildRuleWithDeclaredAndExtraDeps private final Path testLogsPath; private final Optional> snapshotReferenceImagesPath; + private final Optional> snapshotImagesDiffPath; private Optional testRuleTimeoutMs; @@ -216,6 +217,7 @@ public ImmutableList getTestCaseSummaries() { Optional testRuleTimeoutMs, boolean isUiTest, Optional> snapshotReferenceImagesPath, + Optional> snapshotImagesDiffPath, Optional> testSpecificEnvironmentVariables, boolean useIdb, Path idbPath) { @@ -244,6 +246,7 @@ public ImmutableList getTestCaseSummaries() { this.testLogLevel = testLogLevel; this.isUiTest = isUiTest; this.snapshotReferenceImagesPath = snapshotReferenceImagesPath; + this.snapshotImagesDiffPath = snapshotImagesDiffPath; this.testSpecificEnvironmentVariables = testSpecificEnvironmentVariables; this.useIdb = useIdb; this.idbPath = idbPath; @@ -345,23 +348,8 @@ public Pair, ExternalTestRunnerTestSpec> getTestCommand( destinationSpecifierArg = defaultDestinationSpecifier; } - Optional snapshotReferenceImagesPath = Optional.empty(); - if (this.snapshotReferenceImagesPath.isPresent()) { - if (this.snapshotReferenceImagesPath.get().isLeft()) { - snapshotReferenceImagesPath = - Optional.of( - buildContext - .getSourcePathResolver() - .getAbsolutePath(this.snapshotReferenceImagesPath.get().getLeft()) - .toString()); - } else if (this.snapshotReferenceImagesPath.get().isRight()) { - snapshotReferenceImagesPath = - Optional.of( - getProjectFilesystem() - .getPathForRelativePath(this.snapshotReferenceImagesPath.get().getRight()) - .toString()); - } - } + Optional snapshotReferenceImagesPath = getAbsoluteSnapshotTestingArgumentPath(this.snapshotReferenceImagesPath, buildContext); + Optional snapshotImagesDiffPath = getAbsoluteSnapshotTestingArgumentPath(this.snapshotImagesDiffPath, buildContext); XctoolRunTestsStep xctoolStep = new XctoolRunTestsStep( @@ -384,7 +372,8 @@ public Pair, ExternalTestRunnerTestSpec> getTestCommand( Optional.of(testLogLevelEnvironmentVariable), Optional.of(testLogLevel), testRuleTimeoutMs, - snapshotReferenceImagesPath); + snapshotReferenceImagesPath, + snapshotImagesDiffPath); if (useIdb) { idbStdoutReader = Optional.of(new AppleTestIdbStdoutReader(testReportingCallback)); @@ -456,6 +445,25 @@ public Pair, ExternalTestRunnerTestSpec> getTestCommand( return new Pair<>(steps.build(), externalSpec.build()); } + private Optional getAbsoluteSnapshotTestingArgumentPath( + Optional> snapshotTestArgument, BuildContext buildContext) { + if (snapshotTestArgument.isPresent()) { + if (snapshotTestArgument.get().isLeft()) { + return Optional.of( + buildContext + .getSourcePathResolver() + .getAbsolutePath(snapshotTestArgument.get().getLeft()) + .toString()); + } else if (snapshotTestArgument.get().isRight()) { + return Optional.of( + getProjectFilesystem() + .getPathForRelativePath(snapshotTestArgument.get().getRight()) + .toString()); + } + } + return Optional.empty(); + } + static Optional extractBundlePathForBundle( Optional bundle, SourcePathResolverAdapter sourcePathResolverAdapter) { if (!bundle.isPresent()) { diff --git a/src/com/facebook/buck/apple/AppleTestDescription.java b/src/com/facebook/buck/apple/AppleTestDescription.java index 6d4dde1473c..c908157837a 100644 --- a/src/com/facebook/buck/apple/AppleTestDescription.java +++ b/src/com/facebook/buck/apple/AppleTestDescription.java @@ -368,12 +368,14 @@ public BuildRule createBuildRule( args.getCodesignIdentity(), Optional.empty(), Optional.empty(), + Optional.empty(), appleConfig.getCodesignTimeout(), swiftBuckConfig.getCopyStdlibToFrameworks(), swiftBuckConfig.getUseLipoThin(), cxxBuckConfig.shouldCacheStrip(), appleConfig.useEntitlementsWhenAdhocCodeSigning(), - Predicates.alwaysTrue()))); + Predicates.alwaysTrue(), + Optional.empty()))); Optional xctool = getXctool(projectFilesystem, params, targetConfiguration, graphBuilder); @@ -420,6 +422,7 @@ public BuildRule createBuildRule( AppleDeveloperDirectoryForTestsProvider.class), args.getIsUiTest(), args.getSnapshotReferenceImagesPath(), + args.getSnapshotImagesDiffPath(), appleConfig.useIdb(), appleConfig.getIdbPath()); } @@ -457,6 +460,7 @@ public BuildRule createBuildRule( .getDefaultTestRuleTimeoutMs()), args.getIsUiTest(), args.getSnapshotReferenceImagesPath(), + args.getSnapshotImagesDiffPath(), args.getEnv(), appleConfig.useIdb(), appleConfig.getIdbPath()); @@ -951,6 +955,9 @@ default boolean getIsUiTest() { // for use with FBSnapshotTestcase, injects the path as FB_REFERENCE_IMAGE_DIR Optional> getSnapshotReferenceImagesPath(); + // for use with FBSnapshotTestcase, injects the path as IMAGE_DIFF_DIR + Optional> getSnapshotImagesDiffPath(); + // Bundle related fields. ImmutableMap getDestinationSpecifier(); diff --git a/src/com/facebook/buck/apple/AppleTestX.java b/src/com/facebook/buck/apple/AppleTestX.java index 88379f905ab..8bcd27d2b30 100644 --- a/src/com/facebook/buck/apple/AppleTestX.java +++ b/src/com/facebook/buck/apple/AppleTestX.java @@ -104,6 +104,7 @@ public class AppleTestX extends AbstractBuildRuleWithDeclaredAndExtraDeps AppleDeveloperDirectoryForTestsProvider appleDeveloperDirectoryForTestsProvider, boolean isUiTest, Optional> snapshotReferenceImagesPath, + Optional> snapshotImagesDiffPath, boolean useIdb, Path idbPath) { super(buildTarget, projectFilesystem, params); @@ -121,6 +122,7 @@ public class AppleTestX extends AbstractBuildRuleWithDeclaredAndExtraDeps appleDeveloperDirectoryForTestsProvider, isUiTest, snapshotReferenceImagesPath, + snapshotImagesDiffPath, useIdb, idbPath); @@ -248,6 +250,7 @@ private static class AppleConfigs implements AddsToRuleKey { public static final String DEFAULT_DESTINATION = "default_destination"; public static final String DEVELOPER_DIRECTORY_FOR_TESTS = "developer_directory_for_tests"; public static final String SNAPSHOT_REFERENCE_IMG_PATH = "snapshot_reference_img_path"; + public static final String SNAPSHOT_IMAGES_DIFF_PATH = "snapshot_images_diff_path"; private static final String UI_TEST_TARGET_APP = "ui_test_target_app"; private static final String TEST_HOST_APP = "test_host_app"; @@ -270,6 +273,7 @@ private static class AppleConfigs implements AddsToRuleKey { @AddToRuleKey private final boolean isUiTest; private final Optional> snapshotReferenceImagesPath; + private final Optional> snapshotImagesDiffPath; private final boolean useIdb; private final Path idbPath; @@ -285,6 +289,7 @@ private AppleConfigs( AppleDeveloperDirectoryForTestsProvider appleDeveloperDirectoryForTestsProvider, boolean isUiTest, Optional> snapshotReferenceImagesPath, + Optional> snapshotImagesDiffPath, boolean useIdb, Path idbPath) { this.testHostApp = testHostApp; @@ -298,6 +303,7 @@ private AppleConfigs( this.appleDeveloperDirectoryForTestsProvider = appleDeveloperDirectoryForTestsProvider; this.isUiTest = isUiTest; this.snapshotReferenceImagesPath = snapshotReferenceImagesPath; + this.snapshotImagesDiffPath = snapshotImagesDiffPath; this.useIdb = useIdb; this.idbPath = idbPath; } @@ -341,6 +347,19 @@ private String asJson( }) .orElse("")); + generator.writeStringField( + SNAPSHOT_IMAGES_DIFF_PATH, + snapshotImagesDiffPath + .map( + pathOrStr -> { + if (pathOrStr.isLeft()) { + return sourcePathResolver.getAbsolutePath(pathOrStr.getLeft()).toString(); + } + + return filesystem.getPathForRelativePath(pathOrStr.getRight()).toString(); + }) + .orElse("")); + generator.writeObjectField( UI_TEST_TARGET_APP, AppleTest.extractBundlePathForBundle(uiTestTargetApp, sourcePathResolver) diff --git a/src/com/facebook/buck/apple/AppleToolchainDescription.java b/src/com/facebook/buck/apple/AppleToolchainDescription.java index 7072e2300f4..e9847c2e6ef 100644 --- a/src/com/facebook/buck/apple/AppleToolchainDescription.java +++ b/src/com/facebook/buck/apple/AppleToolchainDescription.java @@ -130,7 +130,8 @@ public BuildRule createBuildRule( args.getArchitecture(), "apple", applePlatform.getSwiftName().orElse(applePlatform.getName()), - args.getMinVersion()); + args.getMinVersion(), + applePlatform.getIsSimulator()); Optional swiftPlatform = swiftToolchainRule .map(SwiftToolchainBuildRule.class::cast) diff --git a/src/com/facebook/buck/apple/HasAppleBundleFields.java b/src/com/facebook/buck/apple/HasAppleBundleFields.java index 14d66b29ae9..f86c61bfda9 100644 --- a/src/com/facebook/buck/apple/HasAppleBundleFields.java +++ b/src/com/facebook/buck/apple/HasAppleBundleFields.java @@ -31,6 +31,8 @@ public interface HasAppleBundleFields { Optional getXcodeProductType(); + Optional getIsAppClip(); + ImmutableMap getInfoPlistSubstitutions(); @Value.Default diff --git a/src/com/facebook/buck/apple/XctoolRunTestsStep.java b/src/com/facebook/buck/apple/XctoolRunTestsStep.java index cbe4e8b405c..aa1396f6ef8 100644 --- a/src/com/facebook/buck/apple/XctoolRunTestsStep.java +++ b/src/com/facebook/buck/apple/XctoolRunTestsStep.java @@ -74,6 +74,7 @@ class XctoolRunTestsStep implements Step { Executors.newSingleThreadScheduledExecutor(); private static final String XCTOOL_ENV_VARIABLE_PREFIX = "XCTOOL_TEST_ENV_"; private static final String FB_REFERENCE_IMAGE_DIR = "FB_REFERENCE_IMAGE_DIR"; + private static final String IMAGE_DIFF_DIR = "IMAGE_DIFF_DIR"; private final ProjectFilesystem filesystem; @@ -96,6 +97,7 @@ public interface StdoutReadingCallback { private final Optional logLevel; private final Optional timeoutInMs; private final Optional snapshotReferenceImagesPath; + private final Optional snapshotImagesDiffPath; private final Map> appTestPathsToTestHostAppPathsToTestTargetAppPaths; private final boolean isUsingXCodeBuildTool; @@ -172,7 +174,8 @@ public XctoolRunTestsStep( Optional logLevelEnvironmentVariable, Optional logLevel, Optional timeoutInMs, - Optional snapshotReferenceImagesPath) { + Optional snapshotReferenceImagesPath, + Optional snapshotImagesDiffPath) { Preconditions.checkArgument( !(logicTestBundlePaths.isEmpty() && appTestBundleToHostAppPaths.isEmpty() @@ -207,6 +210,7 @@ public XctoolRunTestsStep( this.logLevel = logLevel; this.timeoutInMs = timeoutInMs; this.snapshotReferenceImagesPath = snapshotReferenceImagesPath; + this.snapshotImagesDiffPath = snapshotImagesDiffPath; // Super hacky, but xcodebuildtool is an alternative wrapper // around xcodebuild and forwarding the -f arguments only makes // sense in that context. @@ -240,6 +244,11 @@ public ImmutableMap getEnv(ExecutionContext context) { XCTOOL_ENV_VARIABLE_PREFIX + FB_REFERENCE_IMAGE_DIR, snapshotReferenceImagesPath.get()); } + if (snapshotImagesDiffPath.isPresent()) { + environment.put( + XCTOOL_ENV_VARIABLE_PREFIX + IMAGE_DIFF_DIR, snapshotImagesDiffPath.get()); + } + environment.putAll(this.environmentOverrides); return ImmutableMap.copyOf(environment); } diff --git a/src/com/facebook/buck/apple/toolchain/ApplePlatform.java b/src/com/facebook/buck/apple/toolchain/ApplePlatform.java index a6342d9f28b..b897350b033 100644 --- a/src/com/facebook/buck/apple/toolchain/ApplePlatform.java +++ b/src/com/facebook/buck/apple/toolchain/ApplePlatform.java @@ -43,7 +43,7 @@ public abstract class ApplePlatform implements Comparable, AddsTo ImmutableApplePlatform.builder() .setName("iphonesimulator") .setSwiftName("ios") - .setArchitectures(ImmutableList.of("i386", "x86_64")) + .setArchitectures(ImmutableList.of("i386", "x86_64", "arm64")) .setMinVersionFlagPrefix("-mios-simulator-version-min=") // only used for legacy watch apps .setStubBinaryPath(Optional.of("Library/Application Support/WatchKit/WK")) @@ -59,7 +59,7 @@ public abstract class ApplePlatform implements Comparable, AddsTo public static final ApplePlatform WATCHSIMULATOR = ImmutableApplePlatform.builder() .setName("watchsimulator") - .setArchitectures(ImmutableList.of("i386", "x86_64")) + .setArchitectures(ImmutableList.of("i386", "x86_64", "arm64")) .setMinVersionFlagPrefix("-mwatchos-simulator-version-min=") .setStubBinaryPath(Optional.of("Library/Application Support/WatchKit/WK")) .build(); @@ -73,14 +73,14 @@ public abstract class ApplePlatform implements Comparable, AddsTo public static final ApplePlatform APPLETVSIMULATOR = ImmutableApplePlatform.builder() .setName("appletvsimulator") - .setArchitectures(ImmutableList.of("x86_64")) + .setArchitectures(ImmutableList.of("arm64", "x86_64")) .setMinVersionFlagPrefix("-mtvos-simulator-version-min=") .setSwiftName("tvos") .build(); public static final ApplePlatform MACOSX = ImmutableApplePlatform.builder() .setName("macosx") - .setArchitectures(ImmutableList.of("i386", "x86_64")) + .setArchitectures(ImmutableList.of("i386", "x86_64", "arm64")) .setAppIncludesFrameworks(true) .build(); public static final ApplePlatform DRIVERKIT = @@ -144,6 +144,10 @@ public ApplePlatformType getType() { return ApplePlatformType.of(getName()); } + public boolean getIsSimulator() { + return isSimulator(getName()); + } + public static boolean needsCodeSign(String name) { return name.startsWith(IPHONEOS.getName()) || name.startsWith(IPHONESIMULATOR.getName()) diff --git a/src/com/facebook/buck/apple/toolchain/impl/AppleCxxPlatforms.java b/src/com/facebook/buck/apple/toolchain/impl/AppleCxxPlatforms.java index d4ab4c01d85..b65f3a7c9fb 100644 --- a/src/com/facebook/buck/apple/toolchain/impl/AppleCxxPlatforms.java +++ b/src/com/facebook/buck/apple/toolchain/impl/AppleCxxPlatforms.java @@ -61,6 +61,11 @@ import com.facebook.buck.swift.toolchain.SwiftPlatform; import com.facebook.buck.swift.toolchain.SwiftTargetTriple; import com.facebook.buck.swift.toolchain.impl.SwiftPlatformFactory; +import com.facebook.buck.util.Console; +import com.facebook.buck.util.DefaultProcessExecutor; +import com.facebook.buck.util.MoreSuppliers; +import com.facebook.buck.util.ProcessExecutor; +import com.facebook.buck.util.ProcessExecutorParams; import com.facebook.buck.util.environment.Platform; import com.facebook.buck.util.stream.RichStream; import com.google.common.annotations.VisibleForTesting; @@ -76,9 +81,12 @@ import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.text.ParseException; +import java.util.EnumSet; import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.function.Supplier; import java.util.regex.Pattern; import javax.xml.parsers.ParserConfigurationException; import org.xml.sax.SAXException; @@ -133,7 +141,7 @@ public static ImmutableList buildAppleCxxPlatforms( return appleCxxPlatformsBuilder.build(); } - private static Tool getXcodeTool( + private static VersionedTool getXcodeTool( ProjectFilesystem filesystem, ImmutableList toolSearchPaths, XcodeToolFinder xcodeToolFinder, @@ -184,15 +192,6 @@ public static AppleCxxPlatform buildWithXcodeToolFinder( AppleConfig appleConfig = buckConfig.getView(AppleConfig.class); - ImmutableList.Builder ldflagsBuilder = ImmutableList.builder(); - ldflagsBuilder.addAll(Linkers.iXlinker("-sdk_version", targetSdk.getVersion())); - if (appleConfig.linkAllObjC()) { - ldflagsBuilder.addAll(Linkers.iXlinker("-ObjC")); - } - if (targetSdk.getApplePlatform().equals(ApplePlatform.WATCHOS)) { - ldflagsBuilder.addAll(Linkers.iXlinker("-bitcode_verify")); - } - // Populate Xcode version keys from Xcode's own Info.plist if available. Optional xcodeBuildVersion = Optional.empty(); Optional developerPath = sdkPaths.getDeveloperPath(); @@ -423,7 +422,9 @@ public static AppleCxxPlatform buildWithXcodeToolFinder( LOG.debug( "Headers verification platform whitelist: %s", headerVerification.getPlatformWhitelist()); ImmutableList ldFlags = - ImmutableList.builder().addAll(cflags).addAll(ldflagsBuilder.build()).build(); + getLdFlags(targetSdk, filesystem, xcodeToolFinder, toolSearchPaths, appleConfig, version); + ImmutableList combinedLdFlags = + ImmutableList.builder().addAll(cflags).addAll(ldFlags).build(); ImmutableList cflagsArgs = ImmutableList.copyOf(StringArg.from(cflags)); CxxPlatform cxxPlatform = @@ -442,7 +443,7 @@ public static AppleCxxPlatform buildWithXcodeToolFinder( new ConstantToolProvider(clangXxPath), config.shouldCacheLinks(), appleConfig.shouldLinkScrubConcurrently()), - StringArg.from(ldFlags), + StringArg.from(combinedLdFlags), ImmutableMultimap.of(), strip, ArchiverProvider.from(new BsdArchiver(ar)), @@ -477,7 +478,8 @@ public static AppleCxxPlatform buildWithXcodeToolFinder( targetArchitecture, "apple", applePlatform.getSwiftName().orElse(applePlatform.getName()), - minVersion), + minVersion, + applePlatform.getIsSimulator()), version, targetSdk, swiftSdkPathsBuilder.build(), @@ -513,6 +515,30 @@ public static AppleCxxPlatform buildWithXcodeToolFinder( return platformBuilder.build(); } + private static ImmutableList getLdFlags( + AppleSdk targetSdk, + ProjectFilesystem filesystem, + XcodeToolFinder xcodeToolFinder, + ImmutableList toolSearchPaths, + AppleConfig appleConfig, + String version) { + ImmutableList.Builder builder = ImmutableList.builder(); + + if (shouldSetSDKVersion(filesystem, xcodeToolFinder, toolSearchPaths, appleConfig, version) + .get()) { + builder.addAll(Linkers.iXlinker("-sdk_version", targetSdk.getVersion())); + } + + if (appleConfig.linkAllObjC()) { + builder.addAll(Linkers.iXlinker("-ObjC")); + } + if (targetSdk.getApplePlatform().equals(ApplePlatform.WATCHOS)) { + builder.addAll(Linkers.iXlinker("-bitcode_verify")); + } + + return builder.build(); + } + private static Optional getSwiftPlatform( SwiftTargetTriple swiftTarget, String version, @@ -605,6 +631,77 @@ private static Path getToolPath( return result.get(); } + private static Supplier shouldSetSDKVersion( + ProjectFilesystem filesystem, + XcodeToolFinder xcodeToolFinder, + ImmutableList toolSearchPaths, + AppleConfig appleConfig, + String version) { + return MoreSuppliers.memoize( + () -> { + // If the Clang driver detects ld version at 520 or above, it will pass -platform_version, + // otherwise it will pass -_version_min. As -platform_version is incompatible + // with -sdk_version (which Buck passes), we should only be passing -sdk_version if we + // believe the driver will not pass it. + // https://reviews.llvm.org/rG25ce33a6e4f3b13732c0f851e68390dc2acb9123 + Optional ldVersion = + getLdVersion(filesystem, xcodeToolFinder, toolSearchPaths, appleConfig, version); + if (ldVersion.isPresent()) { + return ldVersion.get() < 520; + } + return true; + }); + } + + private static Optional getLdVersion( + ProjectFilesystem filesystem, + XcodeToolFinder xcodeToolFinder, + ImmutableList toolSearchPaths, + AppleConfig appleConfig, + String version) { + + VersionedTool ld = + getXcodeTool(filesystem, toolSearchPaths, xcodeToolFinder, appleConfig, "ld", version); + + ProcessExecutor executor = new DefaultProcessExecutor(Console.createNullConsole()); + ProcessExecutorParams processExecutorParams = + ProcessExecutorParams.builder() + .setCommand(ImmutableList.of(ld.getPath().toString(), "-v")) + .build(); + Set options = EnumSet.of(ProcessExecutor.Option.EXPECTING_STD_OUT); + ProcessExecutor.Result result; + try { + result = + executor.launchAndExecute( + processExecutorParams, + options, + /* stdin */ Optional.empty(), + /* timeOutMs */ Optional.empty(), + /* timeOutHandler */ Optional.empty()); + } catch (InterruptedException | IOException e) { + LOG.debug("Could not execute ld -v, continuing with setting the sdk_version."); + return Optional.empty(); + } + + if (result.getExitCode() != 0) { + throw new RuntimeException( + result.getMessageForUnexpectedResult(ld.getPath().toString() + " -v")); + } + + Double parsedVersion = 0.0; + try { + // We expect the version string to be of the form "@(#)PROGRAM:ld PROJECT:ld64-556.6" + String versionStr = result.getStderr().get().split("\n")[0]; + parsedVersion = Double.parseDouble(versionStr.split("ld64-")[1]); + } catch (Exception e) { + LOG.debug( + "Unable to parse the ld version from %s, continuing with setting the sdk_version.", + result.getStderr().get()); + } + + return Optional.of(parsedVersion); + } + @VisibleForTesting public static class XcodeBuildVersionCache { private final Map> cache = new HashMap<>(); diff --git a/src/com/facebook/buck/apple/xcode/XCScheme.java b/src/com/facebook/buck/apple/xcode/XCScheme.java index a205a6ca69e..2f955ca6aff 100644 --- a/src/com/facebook/buck/apple/xcode/XCScheme.java +++ b/src/com/facebook/buck/apple/xcode/XCScheme.java @@ -16,6 +16,7 @@ package com.facebook.buck.apple.xcode; +import com.facebook.buck.apple.xcode.xcodeproj.PBXTarget; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.util.ArrayList; @@ -252,7 +253,11 @@ public enum WatchInterface { private final Optional watchInterface; private final LaunchStyle launchStyle; private final Optional> environmentVariables; + private final Optional expandVariablesBasedOn; + private final Optional> commandLineArguments; private final Optional notificationPayloadFile; + private final Optional applicationLanguage; + private final Optional applicationRegion; public LaunchAction( BuildableReference buildableReference, @@ -262,9 +267,13 @@ public LaunchAction( Optional watchInterface, LaunchStyle launchStyle, Optional> environmentVariables, + Optional expandVariablesBasedOn, + Optional> commandLineArguments, Optional> preActions, Optional> postActions, - Optional notificationPayloadFile) { + Optional notificationPayloadFile, + Optional applicationLanguage, + Optional applicationRegion) { super(preActions, postActions); this.buildableReference = buildableReference; this.buildConfiguration = buildConfiguration; @@ -273,7 +282,11 @@ public LaunchAction( this.watchInterface = watchInterface; this.launchStyle = launchStyle; this.environmentVariables = environmentVariables; + this.expandVariablesBasedOn = expandVariablesBasedOn; + this.commandLineArguments = commandLineArguments; this.notificationPayloadFile = notificationPayloadFile; + this.applicationLanguage = applicationLanguage; + this.applicationRegion = applicationRegion; } public BuildableReference getBuildableReference() { @@ -307,23 +320,45 @@ public LaunchStyle getLaunchStyle() { public Optional> getEnvironmentVariables() { return environmentVariables; } + + public Optional getExpandVariablesBasedOn() { + return expandVariablesBasedOn; + } + + public Optional> getCommandLineArguments() { + return commandLineArguments; + } + + public Optional getApplicationLanguage() { + return applicationLanguage; + } + + public Optional getApplicationRegion() { + return applicationRegion; + } } public static class ProfileAction extends SchemeAction { BuildableReference buildableReference; private final String buildConfiguration; private final Optional> environmentVariables; + private final Optional expandVariablesBasedOn; + private final Optional> commandLineArguments; public ProfileAction( BuildableReference buildableReference, String buildConfiguration, Optional> environmentVariables, + Optional expandVariablesBasedOn, + Optional> commandLineArguments, Optional> preActions, Optional> postActions) { super(preActions, postActions); this.buildableReference = buildableReference; this.buildConfiguration = buildConfiguration; this.environmentVariables = environmentVariables; + this.expandVariablesBasedOn = expandVariablesBasedOn; + this.commandLineArguments = commandLineArguments; } public BuildableReference getBuildableReference() { @@ -337,22 +372,42 @@ public String getBuildConfiguration() { public Optional> getEnvironmentVariables() { return environmentVariables; } + + public Optional getExpandVariablesBasedOn() { + return expandVariablesBasedOn; + } + + public Optional> getCommandLineArguments() { + return commandLineArguments; + } } public static class TestAction extends SchemeAction { List testables; private final String buildConfiguration; private final Optional> environmentVariables; + private final Optional expandVariablesBasedOn; + private final Optional> commandLineArguments; + private final Optional applicationLanguage; + private final Optional applicationRegion; public TestAction( String buildConfiguration, Optional> environmentVariables, + Optional expandVariablesBasedOn, + Optional> commandLineArguments, Optional> preActions, - Optional> postActions) { + Optional> postActions, + Optional applicationLanguage, + Optional applicationRegion) { super(preActions, postActions); this.testables = new ArrayList<>(); this.buildConfiguration = buildConfiguration; this.environmentVariables = environmentVariables; + this.expandVariablesBasedOn = expandVariablesBasedOn; + this.commandLineArguments = commandLineArguments; + this.applicationLanguage = applicationLanguage; + this.applicationRegion = applicationRegion; } public void addTestableReference(TestableReference testable) { @@ -370,6 +425,22 @@ public String getBuildConfiguration() { public Optional> getEnvironmentVariables() { return environmentVariables; } + + public Optional getExpandVariablesBasedOn() { + return expandVariablesBasedOn; + } + + public Optional> getCommandLineArguments() { + return commandLineArguments; + } + + public Optional getApplicationLanguage() { + return applicationLanguage; + } + + public Optional getApplicationRegion() { + return applicationRegion; + } } public static class TestableReference { diff --git a/src/com/facebook/buck/apple/xcode/xcodeproj/PBXCopyFilesBuildPhase.java b/src/com/facebook/buck/apple/xcode/xcodeproj/PBXCopyFilesBuildPhase.java index a40bcb0f963..e0c9c62e456 100644 --- a/src/com/facebook/buck/apple/xcode/xcodeproj/PBXCopyFilesBuildPhase.java +++ b/src/com/facebook/buck/apple/xcode/xcodeproj/PBXCopyFilesBuildPhase.java @@ -28,6 +28,7 @@ public enum Destination { WRAPPER(1), EXECUTABLES(6), RESOURCES(7), + APPCLIPS(16), FRAMEWORKS(10), SHARED_FRAMEWORKS(11), SHARED_SUPPORT(12), diff --git a/src/com/facebook/buck/apple/xcode/xcodeproj/ProductTypes.java b/src/com/facebook/buck/apple/xcode/xcodeproj/ProductTypes.java index be094938e50..97905594b24 100644 --- a/src/com/facebook/buck/apple/xcode/xcodeproj/ProductTypes.java +++ b/src/com/facebook/buck/apple/xcode/xcodeproj/ProductTypes.java @@ -32,7 +32,10 @@ public final class ProductTypes { ProductType.of("com.apple.product-type.framework.static"); public static final ProductType APPLICATION = ProductType.of("com.apple.product-type.application"); + public static final ProductType APP_CLIP = + ProductType.of("com.apple.product-type.application.on-demand-install-capable"); public static final ProductType WATCH_APPLICATION = + ProductType.of("com.apple.product-type.application.watchapp2"); public static final ProductType UNIT_TEST = ProductType.of("com.apple.product-type.bundle.unit-test"); diff --git a/src/com/facebook/buck/artifact_cache/AbstractAsynchronousCache.java b/src/com/facebook/buck/artifact_cache/AbstractAsynchronousCache.java index c628616a1a1..6ea0b992fcd 100644 --- a/src/com/facebook/buck/artifact_cache/AbstractAsynchronousCache.java +++ b/src/com/facebook/buck/artifact_cache/AbstractAsynchronousCache.java @@ -496,8 +496,9 @@ public final ListenableFuture store(ArtifactInfo info, BorrowablePath outp long artifactSizeBytes = getFileSize(output.getPath()); if (artifactExceedsMaximumSize(artifactSizeBytes)) { LOG.info( - "Artifact too big so not storing it in the %s cache. " + "file=[%s] buildTarget=[%s]", - name, output.getPath(), info.getBuildTarget()); + "Artifact too big so not storing it in the %s cache. " + + "file=[%s] buildTarget=[%s] size=[%d]", + name, output.getPath(), info.getBuildTarget(), artifactSizeBytes); return Futures.immediateFuture(Unit.UNIT); } diff --git a/src/com/facebook/buck/artifact_cache/ArtifactCaches.java b/src/com/facebook/buck/artifact_cache/ArtifactCaches.java index dda992bf87d..2dde2089796 100644 --- a/src/com/facebook/buck/artifact_cache/ArtifactCaches.java +++ b/src/com/facebook/buck/artifact_cache/ArtifactCaches.java @@ -690,6 +690,7 @@ private static ArtifactCache createHttpArtifactCache( .setHttpFetchExecutorService(httpFetchExecutorService) .setErrorTextTemplate(cacheDescription.getErrorMessageFormat()) .setErrorTextLimit(cacheDescription.getErrorMessageLimit()) + .setMaxStoreSizeBytes(cacheDescription.getMaxStoreSize()) .build()); } diff --git a/src/com/facebook/buck/cli/BuckDaemon.java b/src/com/facebook/buck/cli/BuckDaemon.java index 4af0a1ed418..5dfc3895797 100644 --- a/src/com/facebook/buck/cli/BuckDaemon.java +++ b/src/com/facebook/buck/cli/BuckDaemon.java @@ -48,7 +48,7 @@ public final class BuckDaemon { * *

    See: https://github.com/java-native-access/jna/issues/652 */ - public static final int JNA_POINTER_SIZE = Pointer.SIZE; + public static final int JNA_POINTER_SIZE = Native.POINTER_SIZE; private static final int AFTER_COMMAND_AUTO_GC_DELAY_MS = 5000; private static final int SUBSEQUENT_GC_DELAY_MS = 10000; diff --git a/src/com/facebook/buck/cli/InstallCommand.java b/src/com/facebook/buck/cli/InstallCommand.java index b6189999a06..24da7e3d3f4 100644 --- a/src/com/facebook/buck/cli/InstallCommand.java +++ b/src/com/facebook/buck/cli/InstallCommand.java @@ -105,7 +105,7 @@ public class InstallCommand extends BuildCommand { private static final long APPLE_SIMULATOR_WAIT_MILLIS = 20000; private static final ImmutableList APPLE_SIMULATOR_APPS = ImmutableList.of("Simulator.app", "iOS Simulator.app"); - private static final String DEFAULT_APPLE_SIMULATOR_NAME = "iPhone 6s"; + private static final String DEFAULT_APPLE_SIMULATOR_NAME = "iPhone 8"; private static final String DEFAULT_APPLE_TV_SIMULATOR_NAME = "Apple TV"; private static final InstallResult FAILURE = ImmutableInstallResult.of(ExitCode.RUN_ERROR, Optional.empty()); diff --git a/src/com/facebook/buck/cli/MainRunner.java b/src/com/facebook/buck/cli/MainRunner.java index b04648ac307..4f3b8abd57b 100644 --- a/src/com/facebook/buck/cli/MainRunner.java +++ b/src/com/facebook/buck/cli/MainRunner.java @@ -235,7 +235,7 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.ThreadFactoryBuilder; -import com.sun.jna.Pointer; +import com.sun.jna.Native; import java.io.Closeable; import java.io.FileNotFoundException; import java.io.IOException; @@ -284,7 +284,7 @@ public final class MainRunner { * *

    See: https://github.com/java-native-access/jna/issues/652 */ - public static final int JNA_POINTER_SIZE = Pointer.SIZE; + public static final int JNA_POINTER_SIZE = Native.POINTER_SIZE; private static final Optional BUCKD_LAUNCH_TIME_NANOS = Optional.ofNullable(System.getProperty("buck.buckd_launch_time_nanos")); diff --git a/src/com/facebook/buck/core/path/BUCK b/src/com/facebook/buck/core/path/BUCK index 7e456a01f8e..7f937496578 100644 --- a/src/com/facebook/buck/core/path/BUCK +++ b/src/com/facebook/buck/core/path/BUCK @@ -2,7 +2,7 @@ java_library( name = "path", srcs = glob(["*.java"]), tests = [ - "//test/com/facebook/buck/core/path:test", + "//test/com/facebook/buck/core/path:path", ], visibility = [ "PUBLIC", diff --git a/src/com/facebook/buck/cxx/CxxBinaryFlavored.java b/src/com/facebook/buck/cxx/CxxBinaryFlavored.java index 4afa7e945a1..6dae691822e 100644 --- a/src/com/facebook/buck/cxx/CxxBinaryFlavored.java +++ b/src/com/facebook/buck/cxx/CxxBinaryFlavored.java @@ -65,6 +65,8 @@ public boolean hasFlavors( CxxDescriptionEnhancer.CXX_LINK_MAP_FLAVOR, CxxDescriptionEnhancer.HEADER_SYMLINK_TREE_FLAVOR, CxxDescriptionEnhancer.INCREMENTAL_THINLTO, + CxxDescriptionEnhancer.STATIC_FLAVOR, + CxxDescriptionEnhancer.SHARED_FLAVOR, CxxCompilationDatabase.COMPILATION_DATABASE, CxxCompilationDatabase.UBER_COMPILATION_DATABASE, CxxInferEnhancer.InferFlavors.INFER.getFlavor(), diff --git a/src/com/facebook/buck/features/apple/common/XcodeWorkspaceConfigDescription.java b/src/com/facebook/buck/features/apple/common/XcodeWorkspaceConfigDescription.java index 3f1721c6e89..be3088704f9 100644 --- a/src/com/facebook/buck/features/apple/common/XcodeWorkspaceConfigDescription.java +++ b/src/com/facebook/buck/features/apple/common/XcodeWorkspaceConfigDescription.java @@ -100,6 +100,15 @@ interface AbstractXcodeWorkspaceConfigDescriptionArg extends BuildRuleArg { Optional>> getEnvironmentVariables(); + Optional>> + getCommandLineArguments(); + + Optional getApplicationLanguage(); + + Optional getApplicationRegion(); + + Optional> getExpandVariablesBasedOn(); + /** * Add value to scheme to indicate it will be used to work on an app extension. This should * cause Xcode to automatically begin debugging the extension when it's launched. diff --git a/src/com/facebook/buck/features/apple/project/NewNativeTargetProjectMutator.java b/src/com/facebook/buck/features/apple/project/NewNativeTargetProjectMutator.java index 59c81fabfe4..7e73c6de581 100644 --- a/src/com/facebook/buck/features/apple/project/NewNativeTargetProjectMutator.java +++ b/src/com/facebook/buck/features/apple/project/NewNativeTargetProjectMutator.java @@ -633,6 +633,8 @@ private PBXCopyFilesBuildPhase.Destination pbxCopyPhaseDestination( switch (destination) { case FRAMEWORKS: return PBXCopyFilesBuildPhase.Destination.FRAMEWORKS; + case APPCLIPS: + return PBXCopyFilesBuildPhase.Destination.APPCLIPS; case EXECUTABLES: return PBXCopyFilesBuildPhase.Destination.EXECUTABLES; case RESOURCES: diff --git a/src/com/facebook/buck/features/apple/project/ProjectGenerator.java b/src/com/facebook/buck/features/apple/project/ProjectGenerator.java index 9e3378f0e80..900d4f57e71 100644 --- a/src/com/facebook/buck/features/apple/project/ProjectGenerator.java +++ b/src/com/facebook/buck/features/apple/project/ProjectGenerator.java @@ -2549,7 +2549,7 @@ private ImmutableMap getFrameworkAndLibrarySearchPathConfigs( librarySearchPaths.add("$DT_TOOLCHAIN_DIR/usr/lib/swift/$PLATFORM_NAME"); if (options.shouldLinkSystemSwift()) { - librarySearchPaths.add("$DT_TOOLCHAIN_DIR/usr/lib/swift-5.0/$PLATFORM_NAME"); + librarySearchPaths.add("$DT_TOOLCHAIN_DIR/usr/lib/swift-5.2/$PLATFORM_NAME"); } } @@ -3394,6 +3394,11 @@ private Optional getDestinationSpec(TargetNode CopyFilePhaseDestinationSpec.of( PBXCopyFilesBuildPhase.Destination.PRODUCTS, Optional.of("$(CONTENTS_FOLDER_PATH)/Watch"))); + } else if (isAppClipApplicationNode(targetNode)) { + return Optional.of( + CopyFilePhaseDestinationSpec.of( + PBXCopyFilesBuildPhase.Destination.APPCLIPS, + Optional.of("$(CONTENTS_FOLDER_PATH)/AppClips"))); } else { return Optional.of( CopyFilePhaseDestinationSpec.of(PBXCopyFilesBuildPhase.Destination.EXECUTABLES)); @@ -3434,6 +3439,11 @@ private Optional getDestinationSpec(TargetNode return Optional.of( CopyFilePhaseDestinationSpec.of(PBXCopyFilesBuildPhase.Destination.RESOURCES)); } else if (targetNode.getDescription() instanceof PrebuiltAppleFrameworkDescription) { + PrebuiltAppleFrameworkDescriptionArg frameworkDescriptionArg = + (PrebuiltAppleFrameworkDescriptionArg) targetNode.getConstructorArg(); + if (frameworkDescriptionArg.getPreferredLinkage() == NativeLinkableGroup.Linkage.STATIC) { + return Optional.empty(); + } return Optional.of( CopyFilePhaseDestinationSpec.of(PBXCopyFilesBuildPhase.Destination.FRAMEWORKS)); } else if (targetNode.getDescription() instanceof PrebuiltCxxLibraryDescription) { @@ -4709,6 +4719,9 @@ private ProductType bundleToTargetProductType( } } else if (binaryNode.getDescription() instanceof AppleBinaryDescription) { if (extension == AppleBundleExtension.APP) { + if (targetNode.getConstructorArg().getIsAppClip().orElse(false)) { + return ProductTypes.APP_CLIP; + } return ProductTypes.APPLICATION; } } else if (binaryNode.getDescription() instanceof AppleTestDescription) { @@ -4866,6 +4879,21 @@ private static boolean isWatchApplicationNode(TargetNode targetNode) { return false; } + /** + * Determines if a target node is for AppClip application + * + * @param targetNode A target node + * @return If the given target node is for an AppClip application + */ + private static boolean isAppClipApplicationNode(TargetNode targetNode) { + if (targetNode.getDescription() instanceof AppleBundleDescription) { + AppleBundleDescriptionArg arg = (AppleBundleDescriptionArg) targetNode.getConstructorArg(); + return arg.getXcodeProductType() + .equals(Optional.of(ProductTypes.APP_CLIP.getIdentifier())); + } + return false; + } + private Optional getPrefixHeaderSourcePath(CxxLibraryDescription.CommonArg arg) { // The prefix header could be stored in either the `prefix_header` or the `precompiled_header` // field. Use either, but prefer the prefix_header. diff --git a/src/com/facebook/buck/features/apple/project/SchemeGenerator.java b/src/com/facebook/buck/features/apple/project/SchemeGenerator.java index 46e5be45912..01af304e4dc 100644 --- a/src/com/facebook/buck/features/apple/project/SchemeGenerator.java +++ b/src/com/facebook/buck/features/apple/project/SchemeGenerator.java @@ -39,6 +39,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; @@ -87,10 +88,15 @@ class SchemeGenerator { private final XCScheme.LaunchAction.LaunchStyle launchStyle; private final Optional>> environmentVariables; + private final Optional> expandVariablesBasedOn; + private final Optional>> + commandLineArguments; private Optional< ImmutableMap< SchemeActionType, ImmutableMap>>> additionalSchemeActions; + private final Optional applicationLanguage; + private final Optional applicationRegion; public SchemeGenerator( ProjectFilesystem projectFilesystem, @@ -107,6 +113,8 @@ public SchemeGenerator( ImmutableMap actionConfigNames, ImmutableMap targetToProjectPathMap, Optional>> environmentVariables, + Optional> expandVariablesBasedOn, + Optional>> commandLineArguments, Optional< ImmutableMap< SchemeActionType, @@ -114,7 +122,9 @@ public SchemeGenerator( additionalSchemeActions, XCScheme.LaunchAction.LaunchStyle launchStyle, Optional watchInterface, - Optional notificationPayloadFile) { + Optional notificationPayloadFile, + Optional applicationLanguage, + Optional applicationRegion) { this.projectFilesystem = projectFilesystem; this.primaryTarget = primaryTarget; this.watchInterface = watchInterface; @@ -131,8 +141,12 @@ public SchemeGenerator( this.actionConfigNames = actionConfigNames; this.targetToProjectPathMap = targetToProjectPathMap; this.environmentVariables = environmentVariables; + this.expandVariablesBasedOn = expandVariablesBasedOn; + this.commandLineArguments = commandLineArguments; this.additionalSchemeActions = additionalSchemeActions; this.notificationPayloadFile = notificationPayloadFile; + this.applicationLanguage = applicationLanguage; + this.applicationRegion = applicationRegion; LOG.verbose( "Generating scheme with build targets %s, test build targets %s, test bundle targets %s", @@ -236,20 +250,34 @@ public Path writeScheme() throws IOException { } ImmutableMap> envVariables = ImmutableMap.of(); + Map envVariablesBasedOn = ImmutableMap.of(); if (environmentVariables.isPresent()) { envVariables = environmentVariables.get(); + if (expandVariablesBasedOn.isPresent()) { + envVariablesBasedOn = expandVariablesBasedOn.get().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> buildTargetToBuildableReferenceMap.get(e.getValue()))); + } + } + + ImmutableMap> commandLineArgs = ImmutableMap.of(); + if (commandLineArguments.isPresent()) { + commandLineArgs = commandLineArguments.get(); } XCScheme.TestAction testAction = new XCScheme.TestAction( Objects.requireNonNull(actionConfigNames.get(SchemeActionType.TEST)), Optional.ofNullable(envVariables.get(SchemeActionType.TEST)), + Optional.ofNullable(envVariablesBasedOn.get(SchemeActionType.TEST)), + Optional.ofNullable(commandLineArgs.get(SchemeActionType.TEST)), additionalCommandsForSchemeAction( SchemeActionType.TEST, AdditionalActions.PRE_SCHEME_ACTIONS, primaryBuildReference), additionalCommandsForSchemeAction( SchemeActionType.TEST, AdditionalActions.POST_SCHEME_ACTIONS, - primaryBuildReference)); + primaryBuildReference), + applicationLanguage, + applicationRegion); for (PBXTarget target : orderedRunTestTargets) { XCScheme.BuildableReference buildableReference = @@ -276,6 +304,8 @@ public Path writeScheme() throws IOException { watchInterface, launchStyle, Optional.ofNullable(envVariables.get(SchemeActionType.LAUNCH)), + Optional.ofNullable(envVariablesBasedOn.get(SchemeActionType.LAUNCH)), + Optional.ofNullable(commandLineArgs.get(SchemeActionType.LAUNCH)), additionalCommandsForSchemeAction( SchemeActionType.LAUNCH, AdditionalActions.PRE_SCHEME_ACTIONS, @@ -284,7 +314,9 @@ public Path writeScheme() throws IOException { SchemeActionType.LAUNCH, AdditionalActions.POST_SCHEME_ACTIONS, primaryBuildReference), - notificationPayloadFile)); + notificationPayloadFile, + applicationLanguage, + applicationRegion)); profileAction = Optional.of( @@ -292,6 +324,8 @@ public Path writeScheme() throws IOException { primaryBuildableReference, Objects.requireNonNull(actionConfigNames.get(SchemeActionType.PROFILE)), Optional.ofNullable(envVariables.get(SchemeActionType.PROFILE)), + Optional.ofNullable(envVariablesBasedOn.get(SchemeActionType.PROFILE)), + Optional.ofNullable(commandLineArgs.get(SchemeActionType.PROFILE)), additionalCommandsForSchemeAction( SchemeActionType.PROFILE, AdditionalActions.PRE_SCHEME_ACTIONS, @@ -387,6 +421,26 @@ public static Element serializeEnvironmentVariables( return rootElement; } + public static Element serializeCommandLineArguments( + Document doc, ImmutableMap commandLineArguments) { + Element rootElement = doc.createElement("CommandLineArguments"); + for (String argumentKey : commandLineArguments.keySet()) { + Element argumentElement = doc.createElement("CommandLineArgument"); + argumentElement.setAttribute("argument", argumentKey); + argumentElement.setAttribute("isEnabled", commandLineArguments.get(argumentKey)); + rootElement.appendChild(argumentElement); + } + return rootElement; + } + + public static Element serializeExpandVariablesBasedOn( + Document doc, XCScheme.BuildableReference reference) { + Element referenceElement = serializeBuildableReference(doc, reference); + Element rootElement = doc.createElement("MacroExpansion"); + rootElement.appendChild(referenceElement); + return rootElement; + } + public static Element serializeBuildAction(Document doc, XCScheme.BuildAction buildAction) { Element buildActionElem = doc.createElement("BuildAction"); serializePrePostActions(doc, buildAction, buildActionElem); @@ -442,6 +496,12 @@ public static Element serializeTestAction(Document doc, XCScheme.TestAction test } if (testAction.getEnvironmentVariables().isPresent()) { + if (testAction.getExpandVariablesBasedOn().isPresent()) { + Element expandBasedOnElement = + serializeExpandVariablesBasedOn(doc, testAction.getExpandVariablesBasedOn().get()); + testActionElem.appendChild(expandBasedOnElement); + } + // disable the default override that makes Test use Launch's environment variables testActionElem.setAttribute("shouldUseLaunchSchemeArgsEnv", "NO"); Element environmentVariablesElement = @@ -449,6 +509,20 @@ public static Element serializeTestAction(Document doc, XCScheme.TestAction test testActionElem.appendChild(environmentVariablesElement); } + if (testAction.getCommandLineArguments().isPresent()) { + Element commandLineArgumentElement = + serializeCommandLineArguments(doc, testAction.getCommandLineArguments().get()); + testActionElem.appendChild(commandLineArgumentElement); + } + + if (testAction.getApplicationLanguage().isPresent()) { + testActionElem.setAttribute("language", testAction.getApplicationLanguage().get()); + } + + if (testAction.getApplicationRegion().isPresent()) { + testActionElem.setAttribute("region", testAction.getApplicationRegion().get()); + } + return testActionElem; } @@ -514,11 +588,31 @@ public static Element serializeLaunchAction(Document doc, XCScheme.LaunchAction "launchStyle", launchStyle == XCScheme.LaunchAction.LaunchStyle.AUTO ? "0" : "1"); if (launchAction.getEnvironmentVariables().isPresent()) { + if (launchAction.getExpandVariablesBasedOn().isPresent()) { + Element expandBasedOnElement = + serializeExpandVariablesBasedOn(doc, launchAction.getExpandVariablesBasedOn().get()); + launchActionElem.appendChild(expandBasedOnElement); + } + Element environmentVariablesElement = serializeEnvironmentVariables(doc, launchAction.getEnvironmentVariables().get()); launchActionElem.appendChild(environmentVariablesElement); } + if (launchAction.getCommandLineArguments().isPresent()) { + Element commandLineArgumentElement = + serializeCommandLineArguments(doc, launchAction.getCommandLineArguments().get()); + launchActionElem.appendChild(commandLineArgumentElement); + } + + if (launchAction.getApplicationLanguage().isPresent()) { + launchActionElem.setAttribute("language", launchAction.getApplicationLanguage().get()); + } + + if (launchAction.getApplicationRegion().isPresent()) { + launchActionElem.setAttribute("region", launchAction.getApplicationRegion().get()); + } + return launchActionElem; } @@ -536,6 +630,12 @@ public static Element serializeProfileAction(Document doc, XCScheme.ProfileActio productRunnableElem.appendChild(refElem); if (profileAction.getEnvironmentVariables().isPresent()) { + if (profileAction.getExpandVariablesBasedOn().isPresent()) { + Element expandBasedOnElement = + serializeExpandVariablesBasedOn(doc, profileAction.getExpandVariablesBasedOn().get()); + profileActionElem.appendChild(expandBasedOnElement); + } + // disable the default override that makes Profile use Launch's environment variables profileActionElem.setAttribute("shouldUseLaunchSchemeArgsEnv", "NO"); Element environmentVariablesElement = @@ -543,6 +643,12 @@ public static Element serializeProfileAction(Document doc, XCScheme.ProfileActio profileActionElem.appendChild(environmentVariablesElement); } + if (profileAction.getCommandLineArguments().isPresent()) { + Element commandLineArgumentElement = + serializeCommandLineArguments(doc, profileAction.getCommandLineArguments().get()); + profileActionElem.appendChild(commandLineArgumentElement); + } + return profileActionElem; } diff --git a/src/com/facebook/buck/features/apple/project/WorkspaceAndProjectGenerator.java b/src/com/facebook/buck/features/apple/project/WorkspaceAndProjectGenerator.java index 2ebde178a08..20f0233e691 100644 --- a/src/com/facebook/buck/features/apple/project/WorkspaceAndProjectGenerator.java +++ b/src/com/facebook/buck/features/apple/project/WorkspaceAndProjectGenerator.java @@ -23,6 +23,7 @@ import com.facebook.buck.apple.AppleConfig; import com.facebook.buck.apple.AppleDependenciesCache; import com.facebook.buck.apple.AppleTestDescriptionArg; +import com.facebook.buck.apple.PrebuiltAppleFrameworkDescription; import com.facebook.buck.apple.XCodeDescriptions; import com.facebook.buck.apple.xcode.XCScheme; import com.facebook.buck.apple.xcode.xcodeproj.PBXProject; @@ -909,16 +910,16 @@ private static void addExtraWorkspaceSchemes( private static ImmutableSet filterRulesForProjectDirectory( TargetGraph projectGraph, ImmutableSet projectBuildTargets) { - // ProjectGenerator implicitly generates targets for all apple_binary rules which - // are referred to by apple_bundle rules' 'binary' field. - // - // We used to support an explicit xcode_project_config() which - // listed all dependencies explicitly, but now that we synthesize - // one, we need to ensure we continue to only pass apple_binary - // targets which do not belong to apple_bundle rules. - ImmutableSet.Builder binaryTargetsInsideBundlesBuilder = ImmutableSet.builder(); + ImmutableSet.Builder excludedTargetsBuilder = ImmutableSet.builder(); for (TargetNode projectTargetNode : projectGraph.getAll(projectBuildTargets)) { if (projectTargetNode.getDescription() instanceof AppleBundleDescription) { + // ProjectGenerator implicitly generates targets for all apple_binary rules which + // are referred to by apple_bundle rules' 'binary' field. + // + // We used to support an explicit xcode_project_config() which + // listed all dependencies explicitly, but now that we synthesize + // one, we need to ensure we continue to only pass apple_binary + // targets which do not belong to apple_bundle rules. AppleBundleDescriptionArg appleBundleDescriptionArg = (AppleBundleDescriptionArg) projectTargetNode.getConstructorArg(); // We don't support apple_bundle rules referring to apple_binary rules @@ -938,15 +939,15 @@ private static ImmutableSet filterRulesForProjectDirectory( projectTargetNode.getBuildTarget(), appleBundleDescriptionArg.getBinary(), projectTargetNode.getBuildTarget().getCellRelativeBasePath()); - binaryTargetsInsideBundlesBuilder.add(binaryTarget); + excludedTargetsBuilder.add(binaryTarget); + } else if (projectTargetNode.getDescription() instanceof PrebuiltAppleFrameworkDescription) { + // prebuilt frameworks are unbuildable, they just propagate linker flags and dependencies + excludedTargetsBuilder.add(projectTargetNode.getBuildTarget()); } } - ImmutableSet binaryTargetsInsideBundles = - binaryTargetsInsideBundlesBuilder.build(); - // Remove all apple_binary targets which are inside bundles from - // the rest of the build targets in the project. - return ImmutableSet.copyOf(Sets.difference(projectBuildTargets, binaryTargetsInsideBundles)); + ImmutableSet excludedBuildTargets = excludedTargetsBuilder.build(); + return ImmutableSet.copyOf(Sets.difference(projectBuildTargets, excludedBuildTargets)); } /** @@ -1171,6 +1172,7 @@ private void writeWorkspaceSchemesForProjects( orderedBuildTestTargets, orderedRunTestTargets, Optional.empty(), + Optional.empty(), Optional.empty()); schemeGenerator.writeScheme(); @@ -1222,6 +1224,14 @@ private void writeWorkspaceSchemes( ? outputDirectory : outputDirectory.resolve(workspaceName + ".xcworkspace"); + Optional> expandVariablesBasedOn = Optional.empty(); + if (schemeConfigArg.getExpandVariablesBasedOn().isPresent()) { + Map mapTargets = + schemeConfigArg.getExpandVariablesBasedOn().get().entrySet() + .stream().collect(Collectors.toMap(Map.Entry::getKey, e -> buildTargetToPBXTarget.get(e.getValue()) )); + expandVariablesBasedOn = Optional.of(ImmutableMap.copyOf(mapTargets)); + } + SchemeGenerator schemeGenerator = buildSchemeGenerator( targetToProjectPathMap, @@ -1233,7 +1243,8 @@ private void writeWorkspaceSchemes( orderedBuildTestTargets, orderedRunTestTargets, runnablePath, - remoteRunnablePath); + remoteRunnablePath, + expandVariablesBasedOn); schemeGenerator.writeScheme(); schemeGenerators.put(schemeName, schemeGenerator); } @@ -1249,9 +1260,12 @@ private SchemeGenerator buildSchemeGenerator( ImmutableSet orderedBuildTestTargets, ImmutableSet orderedRunTestTargets, Optional runnablePath, - Optional remoteRunnablePath) { + Optional remoteRunnablePath, + Optional> expandVariablesBasedOn) { Optional>> environmentVariables = Optional.empty(); + Optional>> commandLineArguments = + Optional.empty(); Optional< ImmutableMap< SchemeActionType, ImmutableMap>>> @@ -1260,14 +1274,19 @@ private SchemeGenerator buildSchemeGenerator( Optional watchInterface = Optional.empty(); Optional notificationPayloadFile = Optional.empty(); Optional wasCreatedForAppExtension = Optional.empty(); + Optional applicationLanguage = Optional.empty(); + Optional applicationRegion = Optional.empty(); if (schemeConfigArg.isPresent()) { environmentVariables = schemeConfigArg.get().getEnvironmentVariables(); + commandLineArguments = schemeConfigArg.get().getCommandLineArguments(); additionalSchemeActions = schemeConfigArg.get().getAdditionalSchemeActions(); launchStyle = schemeConfigArg.get().getLaunchStyle().orElse(launchStyle); watchInterface = schemeConfigArg.get().getWatchInterface(); notificationPayloadFile = schemeConfigArg.get().getNotificationPayloadFile(); wasCreatedForAppExtension = schemeConfigArg.get().getWasCreatedForAppExtension(); + applicationLanguage = schemeConfigArg.get().getApplicationLanguage(); + applicationRegion = schemeConfigArg.get().getApplicationRegion(); } return new SchemeGenerator( @@ -1285,9 +1304,13 @@ private SchemeGenerator buildSchemeGenerator( XcodeWorkspaceConfigDescription.getActionConfigNamesFromArg(schemeConfigArg), targetToProjectPathMap, environmentVariables, + expandVariablesBasedOn, + commandLineArguments, additionalSchemeActions, launchStyle, watchInterface, - notificationPayloadFile); + notificationPayloadFile, + applicationLanguage, + applicationRegion); } } diff --git a/src/com/facebook/buck/features/apple/projectV2/SchemeGenerator.java b/src/com/facebook/buck/features/apple/projectV2/SchemeGenerator.java index 24e0181be32..851f4f46a1a 100644 --- a/src/com/facebook/buck/features/apple/projectV2/SchemeGenerator.java +++ b/src/com/facebook/buck/features/apple/projectV2/SchemeGenerator.java @@ -40,6 +40,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; @@ -88,9 +89,14 @@ class SchemeGenerator { private final XCScheme.LaunchAction.LaunchStyle launchStyle; private final Optional>> environmentVariables; + private final Optional> expandVariablesBasedOn; + private final Optional>> + commandLineArguments; private Optional< ImmutableMap>>> additionalSchemeActions; + private final Optional applicationLanguage; + private final Optional applicationRegion; public SchemeGenerator( ProjectFilesystem projectFilesystem, @@ -107,13 +113,17 @@ public SchemeGenerator( ImmutableMap actionConfigNames, ImmutableMap targetToProjectPathMap, Optional>> environmentVariables, + Optional> expandVariablesBasedOn, + Optional>> commandLineArguments, Optional< ImmutableMap< SchemeActionType, ImmutableMap>>> additionalSchemeActions, XCScheme.LaunchAction.LaunchStyle launchStyle, Optional watchInterface, - Optional notificationPayloadFile) { + Optional notificationPayloadFile, + Optional applicationLanguage, + Optional applicationRegion) { this.projectFilesystem = projectFilesystem; this.primaryTarget = primaryTarget; this.watchInterface = watchInterface; @@ -130,8 +140,12 @@ public SchemeGenerator( this.actionConfigNames = actionConfigNames; this.targetToProjectPathMap = targetToProjectPathMap; this.environmentVariables = environmentVariables; + this.expandVariablesBasedOn = expandVariablesBasedOn; + this.commandLineArguments = commandLineArguments; this.additionalSchemeActions = additionalSchemeActions; this.notificationPayloadFile = notificationPayloadFile; + this.applicationLanguage = applicationLanguage; + this.applicationRegion = applicationRegion; LOG.verbose( "Generating scheme with build targets %s, test build targets %s, test bundle targets %s", @@ -231,20 +245,34 @@ public Path writeScheme() throws IOException { } ImmutableMap> envVariables = ImmutableMap.of(); + Map envVariablesBasedOn = ImmutableMap.of(); if (environmentVariables.isPresent()) { envVariables = environmentVariables.get(); + if (expandVariablesBasedOn.isPresent()) { + envVariablesBasedOn = expandVariablesBasedOn.get().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, e -> buildTargetToBuildableReferenceMap.get(e.getValue()))); + } + } + + ImmutableMap> commandLineArgs = ImmutableMap.of(); + if (commandLineArguments.isPresent()) { + commandLineArgs = commandLineArguments.get(); } XCScheme.TestAction testAction = new XCScheme.TestAction( Objects.requireNonNull(actionConfigNames.get(SchemeActionType.TEST)), Optional.ofNullable(envVariables.get(SchemeActionType.TEST)), + Optional.ofNullable(envVariablesBasedOn.get(SchemeActionType.TEST)), + Optional.ofNullable(commandLineArgs.get(SchemeActionType.TEST)), additionalCommandsForSchemeAction( SchemeActionType.TEST, AdditionalActions.PRE_SCHEME_ACTIONS, primaryBuildReference), additionalCommandsForSchemeAction( SchemeActionType.TEST, AdditionalActions.POST_SCHEME_ACTIONS, - primaryBuildReference)); + primaryBuildReference), + applicationLanguage, + applicationRegion); for (PBXTarget target : orderedRunTestTargets) { XCScheme.BuildableReference buildableReference = @@ -271,6 +299,8 @@ public Path writeScheme() throws IOException { watchInterface, launchStyle, Optional.ofNullable(envVariables.get(SchemeActionType.LAUNCH)), + Optional.ofNullable(envVariablesBasedOn.get(SchemeActionType.LAUNCH)), + Optional.ofNullable(commandLineArgs.get(SchemeActionType.LAUNCH)), additionalCommandsForSchemeAction( SchemeActionType.LAUNCH, AdditionalActions.PRE_SCHEME_ACTIONS, @@ -279,7 +309,9 @@ public Path writeScheme() throws IOException { SchemeActionType.LAUNCH, AdditionalActions.POST_SCHEME_ACTIONS, primaryBuildReference), - notificationPayloadFile)); + notificationPayloadFile, + applicationLanguage, + applicationRegion)); profileAction = Optional.of( @@ -287,6 +319,8 @@ public Path writeScheme() throws IOException { primaryBuildableReference, Objects.requireNonNull(actionConfigNames.get(SchemeActionType.PROFILE)), Optional.ofNullable(envVariables.get(SchemeActionType.PROFILE)), + Optional.ofNullable(envVariablesBasedOn.get(SchemeActionType.PROFILE)), + Optional.ofNullable(commandLineArgs.get(SchemeActionType.PROFILE)), additionalCommandsForSchemeAction( SchemeActionType.PROFILE, AdditionalActions.PRE_SCHEME_ACTIONS, @@ -424,6 +458,26 @@ public static Element serializeEnvironmentVariables( return rootElement; } + public static Element serializeCommandLineArguments( + Document doc, ImmutableMap commandLineArguments) { + Element rootElement = doc.createElement("CommandLineArguments"); + for (String argumentKey : commandLineArguments.keySet()) { + Element argumentElement = doc.createElement("CommandLineArgument"); + argumentElement.setAttribute("argument", argumentKey); + argumentElement.setAttribute("isEnabled", commandLineArguments.get(argumentKey)); + rootElement.appendChild(argumentElement); + } + return rootElement; + } + + public static Element serializeExpandVariablesBasedOn( + Document doc, XCScheme.BuildableReference reference) { + Element referenceElement = serializeBuildableReference(doc, reference); + Element rootElement = doc.createElement("MacroExpansion"); + rootElement.appendChild(referenceElement); + return rootElement; + } + public static Element serializeBuildAction(Document doc, XCScheme.BuildAction buildAction) { Element buildActionElem = doc.createElement("BuildAction"); serializePrePostActions(doc, buildAction, buildActionElem); @@ -479,6 +533,12 @@ public static Element serializeTestAction(Document doc, XCScheme.TestAction test } if (testAction.getEnvironmentVariables().isPresent()) { + if (testAction.getExpandVariablesBasedOn().isPresent()) { + Element expandBasedOnElement = + serializeExpandVariablesBasedOn(doc, testAction.getExpandVariablesBasedOn().get()); + testActionElem.appendChild(expandBasedOnElement); + } + // disable the default override that makes Test use Launch's environment variables testActionElem.setAttribute("shouldUseLaunchSchemeArgsEnv", "NO"); Element environmentVariablesElement = @@ -486,6 +546,19 @@ public static Element serializeTestAction(Document doc, XCScheme.TestAction test testActionElem.appendChild(environmentVariablesElement); } + if (testAction.getCommandLineArguments().isPresent()) { + Element commandLineArgumentElement = + serializeCommandLineArguments(doc, testAction.getCommandLineArguments().get()); + testActionElem.appendChild(commandLineArgumentElement); + } + + if (testAction.getApplicationLanguage().isPresent()) { + testActionElem.setAttribute("language", testAction.getApplicationLanguage().get()); + } + + if (testAction.getApplicationRegion().isPresent()) { + testActionElem.setAttribute("region", testAction.getApplicationRegion().get()); + } return testActionElem; } @@ -551,11 +624,31 @@ public static Element serializeLaunchAction(Document doc, XCScheme.LaunchAction "launchStyle", launchStyle == XCScheme.LaunchAction.LaunchStyle.AUTO ? "0" : "1"); if (launchAction.getEnvironmentVariables().isPresent()) { + if (launchAction.getExpandVariablesBasedOn().isPresent()) { + Element expandBasedOnElement = + serializeExpandVariablesBasedOn(doc, launchAction.getExpandVariablesBasedOn().get()); + launchActionElem.appendChild(expandBasedOnElement); + } + Element environmentVariablesElement = serializeEnvironmentVariables(doc, launchAction.getEnvironmentVariables().get()); launchActionElem.appendChild(environmentVariablesElement); } + if (launchAction.getCommandLineArguments().isPresent()) { + Element commandLineArgumentElement = + serializeCommandLineArguments(doc, launchAction.getCommandLineArguments().get()); + launchActionElem.appendChild(commandLineArgumentElement); + } + + if (launchAction.getApplicationLanguage().isPresent()) { + launchActionElem.setAttribute("language", launchAction.getApplicationLanguage().get()); + } + + if (launchAction.getApplicationRegion().isPresent()) { + launchActionElem.setAttribute("region", launchAction.getApplicationRegion().get()); + } + return launchActionElem; } @@ -573,6 +666,12 @@ public static Element serializeProfileAction(Document doc, XCScheme.ProfileActio productRunnableElem.appendChild(refElem); if (profileAction.getEnvironmentVariables().isPresent()) { + if (profileAction.getExpandVariablesBasedOn().isPresent()) { + Element expandBasedOnElement = + serializeExpandVariablesBasedOn(doc, profileAction.getExpandVariablesBasedOn().get()); + profileActionElem.appendChild(expandBasedOnElement); + } + // disable the default override that makes Profile use Launch's environment variables profileActionElem.setAttribute("shouldUseLaunchSchemeArgsEnv", "NO"); Element environmentVariablesElement = @@ -580,6 +679,12 @@ public static Element serializeProfileAction(Document doc, XCScheme.ProfileActio profileActionElem.appendChild(environmentVariablesElement); } + if (profileAction.getCommandLineArguments().isPresent()) { + Element commandLineArgumentElement = + serializeCommandLineArguments(doc, profileAction.getCommandLineArguments().get()); + profileActionElem.appendChild(commandLineArgumentElement); + } + return profileActionElem; } diff --git a/src/com/facebook/buck/features/apple/projectV2/WorkspaceAndProjectGenerator.java b/src/com/facebook/buck/features/apple/projectV2/WorkspaceAndProjectGenerator.java index 167cac3a575..5e472e623f7 100644 --- a/src/com/facebook/buck/features/apple/projectV2/WorkspaceAndProjectGenerator.java +++ b/src/com/facebook/buck/features/apple/projectV2/WorkspaceAndProjectGenerator.java @@ -77,6 +77,7 @@ import java.util.Optional; import java.util.Set; import java.util.function.Function; +import java.util.stream.Collector; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -899,6 +900,14 @@ private void writeWorkspaceSchemes( Path schemeOutputDirectory = outputDirectory.resolve(workspaceName + ".xcworkspace"); + Optional> expandVariablesBasedOn = Optional.empty(); + if (schemeConfigArg.getExpandVariablesBasedOn().isPresent()) { + Map mapTargets = + schemeConfigArg.getExpandVariablesBasedOn().get().entrySet() + .stream().collect(Collectors.toMap(Map.Entry::getKey, e -> buildTargetToPBXTarget.get(e.getValue()) )); + expandVariablesBasedOn = Optional.of(ImmutableMap.copyOf(mapTargets)); + } + SchemeGenerator schemeGenerator = buildSchemeGenerator( targetToProjectPathMap, @@ -910,7 +919,8 @@ private void writeWorkspaceSchemes( orderedBuildTestTargets, orderedRunTestTargets, runnablePath, - remoteRunnablePath); + remoteRunnablePath, + expandVariablesBasedOn); schemeGenerator.writeScheme(); schemeGenerators.put(schemeName, schemeGenerator); } @@ -926,9 +936,12 @@ private SchemeGenerator buildSchemeGenerator( ImmutableSet orderedBuildTestTargets, ImmutableSet orderedRunTestTargets, Optional runnablePath, - Optional remoteRunnablePath) { + Optional remoteRunnablePath, + Optional> expandVariablesBasedOn) { Optional>> environmentVariables = Optional.empty(); + Optional>> commandLineArguments = + Optional.empty(); Optional< ImmutableMap< SchemeActionType, ImmutableMap>>> @@ -937,14 +950,19 @@ private SchemeGenerator buildSchemeGenerator( Optional watchInterface = Optional.empty(); Optional notificationPayloadFile = Optional.empty(); Optional wasCreatedForAppExtension = Optional.empty(); + Optional applicationLanguage = Optional.empty(); + Optional applicationRegion = Optional.empty(); if (schemeConfigArg.isPresent()) { environmentVariables = schemeConfigArg.get().getEnvironmentVariables(); + commandLineArguments = schemeConfigArg.get().getCommandLineArguments(); additionalSchemeActions = schemeConfigArg.get().getAdditionalSchemeActions(); launchStyle = schemeConfigArg.get().getLaunchStyle().orElse(launchStyle); watchInterface = schemeConfigArg.get().getWatchInterface(); notificationPayloadFile = schemeConfigArg.get().getNotificationPayloadFile(); wasCreatedForAppExtension = schemeConfigArg.get().getWasCreatedForAppExtension(); + applicationLanguage = schemeConfigArg.get().getApplicationLanguage(); + applicationRegion = schemeConfigArg.get().getApplicationRegion(); } return new SchemeGenerator( @@ -962,9 +980,14 @@ private SchemeGenerator buildSchemeGenerator( XcodeWorkspaceConfigDescription.getActionConfigNamesFromArg(schemeConfigArg), targetToProjectPathMap, environmentVariables, + expandVariablesBasedOn, + commandLineArguments, additionalSchemeActions, launchStyle, watchInterface, - notificationPayloadFile); + notificationPayloadFile, + applicationLanguage, + applicationRegion + ); } } diff --git a/src/com/facebook/buck/features/apple/projectV2/XcodeNativeTargetGenerator.java b/src/com/facebook/buck/features/apple/projectV2/XcodeNativeTargetGenerator.java index 31b82f09734..760d96f91c5 100644 --- a/src/com/facebook/buck/features/apple/projectV2/XcodeNativeTargetGenerator.java +++ b/src/com/facebook/buck/features/apple/projectV2/XcodeNativeTargetGenerator.java @@ -1863,7 +1863,7 @@ private ImmutableMap getFrameworkAndLibrarySearchPathConfigs( // folder changes in a future release this can be revisited. librarySearchPaths.add("$DT_TOOLCHAIN_DIR/usr/lib/swift/$PLATFORM_NAME"); if (options.shouldLinkSystemSwift()) { - librarySearchPaths.add("$DT_TOOLCHAIN_DIR/usr/lib/swift-5.0/$PLATFORM_NAME"); + librarySearchPaths.add("$DT_TOOLCHAIN_DIR/usr/lib/swift-5.2/$PLATFORM_NAME"); } } diff --git a/src/com/facebook/buck/features/js/JsUtil.java b/src/com/facebook/buck/features/js/JsUtil.java index 450341cc73a..f4fb21fe331 100644 --- a/src/com/facebook/buck/features/js/JsUtil.java +++ b/src/com/facebook/buck/features/js/JsUtil.java @@ -112,6 +112,7 @@ static WorkerShellStep jsonWorkerShellStepAddingFlavors( tool.getCommandPrefix(pathResolver), tool.getEnvironment(pathResolver), worker.getMaxWorkers(), + worker.isAsync(), worker.isPersistent() ? Optional.of( WorkerProcessIdentity.of(buildTarget.toString(), worker.getInstanceKey())) diff --git a/src/com/facebook/buck/features/python/toolchain/impl/PythonInterpreterFromConfig.java b/src/com/facebook/buck/features/python/toolchain/impl/PythonInterpreterFromConfig.java index 9bc44ff2efe..73745f0d152 100644 --- a/src/com/facebook/buck/features/python/toolchain/impl/PythonInterpreterFromConfig.java +++ b/src/com/facebook/buck/features/python/toolchain/impl/PythonInterpreterFromConfig.java @@ -29,9 +29,8 @@ public class PythonInterpreterFromConfig implements PythonInterpreter { - // Prefer "python2" where available (Linux), but fall back to "python" (Mac). private static final ImmutableList PYTHON_INTERPRETER_NAMES = - ImmutableList.of("python2", "python", "python3"); + ImmutableList.of("python3", "python", "python2"); private final PythonBuckConfig pythonBuckConfig; private final ExecutableFinder executableFinder; diff --git a/src/com/facebook/buck/json/BuildFilePythonResultDeserializer.java b/src/com/facebook/buck/json/BuildFilePythonResultDeserializer.java index c2e1a2a06ca..33c69898690 100644 --- a/src/com/facebook/buck/json/BuildFilePythonResultDeserializer.java +++ b/src/com/facebook/buck/json/BuildFilePythonResultDeserializer.java @@ -110,7 +110,21 @@ private static List deserializeList(JsonParser jp) throws IOException { ImmutableList.Builder builder = ImmutableList.builder(); JsonToken token; while ((token = jp.nextToken()) != JsonToken.END_ARRAY) { - builder.add(deserializeRecursive(jp, token)); + Object obj = deserializeRecursive(jp, token); + if (obj != null) { + builder.add(obj); + } + else { + // null elements can't be added to ImmutableList, an NPE will be thrown. + // Throw a meaningful exception here instead. + StringBuilder message = new StringBuilder(); + message.append("null value can't be added to ["); + for (Object element: builder.build()) { + message.append("'").append(element).append("'").append(", "); + } + message.append("]"); + throw new IllegalArgumentException(message.toString()); + } } if (token != JsonToken.END_ARRAY) { throw new JsonParseException(jp, "Missing expected END_ARRAY"); diff --git a/src/com/facebook/buck/jvm/groovy/GroovyConfiguredCompilerFactory.java b/src/com/facebook/buck/jvm/groovy/GroovyConfiguredCompilerFactory.java index b90b960e79e..3054d549a65 100644 --- a/src/com/facebook/buck/jvm/groovy/GroovyConfiguredCompilerFactory.java +++ b/src/com/facebook/buck/jvm/groovy/GroovyConfiguredCompilerFactory.java @@ -18,6 +18,7 @@ import com.facebook.buck.core.model.BuildTarget; import com.facebook.buck.core.model.TargetConfiguration; +import com.facebook.buck.core.rules.ActionGraphBuilder; import com.facebook.buck.core.rules.BuildRuleResolver; import com.facebook.buck.core.toolchain.ToolchainProvider; import com.facebook.buck.core.util.Optionals; @@ -43,6 +44,7 @@ public GroovyConfiguredCompilerFactory(GroovyBuckConfig groovyBuckConfig) { public CompileToJarStepFactory configure( @Nullable JvmLibraryArg args, JavacOptions javacOptions, + ActionGraphBuilder actionGraphBuilder, BuildRuleResolver buildRuleResolver, TargetConfiguration targetConfiguration, ToolchainProvider toolchainProvider) { diff --git a/src/com/facebook/buck/jvm/java/ConfiguredCompilerFactory.java b/src/com/facebook/buck/jvm/java/ConfiguredCompilerFactory.java index 19604db0e75..3530cf3cee0 100644 --- a/src/com/facebook/buck/jvm/java/ConfiguredCompilerFactory.java +++ b/src/com/facebook/buck/jvm/java/ConfiguredCompilerFactory.java @@ -18,6 +18,7 @@ import com.facebook.buck.core.model.BuildTarget; import com.facebook.buck.core.model.TargetConfiguration; +import com.facebook.buck.core.rules.ActionGraphBuilder; import com.facebook.buck.core.rules.BuildRuleResolver; import com.facebook.buck.core.toolchain.ToolchainProvider; import com.facebook.buck.jvm.java.abi.AbiGenerationMode; @@ -31,11 +32,12 @@ public abstract class ConfiguredCompilerFactory { // TODO(jkeljo): args is not actually Nullable in all subclasses, but it is also not // straightforward to create a safe "empty" default value. Find a fix. public abstract CompileToJarStepFactory configure( - @Nullable JvmLibraryArg args, - JavacOptions javacOptions, - BuildRuleResolver buildRuleResolver, - TargetConfiguration targetConfiguration, - ToolchainProvider toolchainProvider); + @Nullable JvmLibraryArg args, + JavacOptions javacOptions, + ActionGraphBuilder actionGraphBuilder, + BuildRuleResolver buildRuleResolver, + TargetConfiguration targetConfiguration, + ToolchainProvider toolchainProvider); public abstract Optional getExtraClasspathProvider( ToolchainProvider toolchainProvider, TargetConfiguration toolchainTargetConfiguration); diff --git a/src/com/facebook/buck/jvm/java/DefaultJavaLibraryRules.java b/src/com/facebook/buck/jvm/java/DefaultJavaLibraryRules.java index 9538c125438..1b838a61d55 100644 --- a/src/com/facebook/buck/jvm/java/DefaultJavaLibraryRules.java +++ b/src/com/facebook/buck/jvm/java/DefaultJavaLibraryRules.java @@ -302,10 +302,15 @@ AbiGenerationMode getAbiGenerationMode() { if (args != null) { result = args.getAbiGenerationMode().orElse(null); } - if (result == null) { - result = Objects.requireNonNull(getConfiguredCompilerFactory()).getAbiGenerationMode(); + + // Respect user input if provided + if (result != null) { + return result; } + // Infer ABI generation mode based on properties of build target + result = Objects.requireNonNull(getConfiguredCompilerFactory()).getAbiGenerationMode(); + if (result == AbiGenerationMode.CLASS) { return result; } @@ -558,6 +563,7 @@ CompileToJarStepFactory getConfiguredCompiler() { getArgs(), getJavacOptions(), getActionGraphBuilder(), + getActionGraphBuilder(), getInitialBuildTarget().getTargetConfiguration(), getToolchainProvider()); } @@ -569,6 +575,7 @@ CompileToJarStepFactory getConfiguredCompilerForSourceOnlyAbi() { getArgs(), getJavacOptionsForSourceOnlyAbi(), getActionGraphBuilder(), + getActionGraphBuilder(), getInitialBuildTarget().getTargetConfiguration(), getToolchainProvider()); } diff --git a/src/com/facebook/buck/jvm/java/JavaConfiguredCompilerFactory.java b/src/com/facebook/buck/jvm/java/JavaConfiguredCompilerFactory.java index fca2ec326e5..4531da08fab 100644 --- a/src/com/facebook/buck/jvm/java/JavaConfiguredCompilerFactory.java +++ b/src/com/facebook/buck/jvm/java/JavaConfiguredCompilerFactory.java @@ -17,6 +17,7 @@ package com.facebook.buck.jvm.java; import com.facebook.buck.core.model.TargetConfiguration; +import com.facebook.buck.core.rules.ActionGraphBuilder; import com.facebook.buck.core.rules.BuildRuleResolver; import com.facebook.buck.core.toolchain.ToolchainProvider; import com.facebook.buck.jvm.java.abi.AbiGenerationMode; @@ -86,6 +87,7 @@ public boolean shouldGenerateSourceOnlyAbi() { public CompileToJarStepFactory configure( @Nullable JvmLibraryArg arg, JavacOptions javacOptions, + ActionGraphBuilder actionGraphBuilder, BuildRuleResolver buildRuleResolver, TargetConfiguration targetConfiguration, ToolchainProvider toolchainProvider) { diff --git a/src/com/facebook/buck/jvm/java/JavaTest.java b/src/com/facebook/buck/jvm/java/JavaTest.java index ad0e42336a6..b81aebd7f1b 100644 --- a/src/com/facebook/buck/jvm/java/JavaTest.java +++ b/src/com/facebook/buck/jvm/java/JavaTest.java @@ -83,6 +83,7 @@ import java.util.SortedSet; import java.util.concurrent.Callable; import java.util.logging.Level; +import java.util.stream.Collectors; import java.util.stream.Stream; import javax.annotation.Nullable; @@ -393,10 +394,11 @@ public Path getPathToTestOutputDirectory() { /** @return a test case result, named "main", signifying a failure of the entire test class. */ private TestCaseSummary getTestClassFailedSummary(String testClass, String message, long time) { return new TestCaseSummary( - testClass, - ImmutableList.of( - new TestResultSummary( - testClass, "main", ResultType.FAILURE, time, message, "", "", ""))); + testClass, + false, + ImmutableList.of( + new TestResultSummary( + testClass, "main", ResultType.FAILURE, time, message, "", "", ""))); } @Override @@ -444,7 +446,12 @@ public Callable interpretTestResults( // definitively say whether or not a class should be run. It's not possible, for example, // to filter testClassNames here at the buck end. } else if (Files.isRegularFile(testResultFile)) { - summaries.add(XmlTestResultParser.parse(testResultFile)); + TestCaseSummary summary = XmlTestResultParser.parse(testResultFile); + if (summary.isTestSuite()) { + summaries.addAll(unrollTestSuiteSummary(summary)); + } else { + summaries.add(summary); + } } } @@ -615,6 +622,15 @@ protected ImmutableSet getRuntimeClasspath(BuildContext buildContext) { .build(); } + private List unrollTestSuiteSummary(TestCaseSummary summary) { + Map> summariesByActualTestCase = summary.getTestResults() + .stream().collect(Collectors.groupingBy(TestResultSummary::getTestCaseName)); + + return summariesByActualTestCase.entrySet().stream() + .map(entry -> new TestCaseSummary(entry.getKey(), false, entry.getValue())) + .collect(Collectors.toList()); + } + public interface AdditionalClasspathEntriesProvider { ImmutableList getAdditionalClasspathEntries(SourcePathResolverAdapter resolver); } diff --git a/src/com/facebook/buck/jvm/java/abi/AbiFilteringClassVisitor.java b/src/com/facebook/buck/jvm/java/abi/AbiFilteringClassVisitor.java index c32893f7f0b..3973645bac6 100644 --- a/src/com/facebook/buck/jvm/java/abi/AbiFilteringClassVisitor.java +++ b/src/com/facebook/buck/jvm/java/abi/AbiFilteringClassVisitor.java @@ -46,18 +46,17 @@ class AbiFilteringClassVisitor extends ClassVisitor { private boolean hasVisibleConstructor = false; private Set includedInnerClasses = new HashSet<>(); private List nestMembers = new ArrayList<>(); - - public AbiFilteringClassVisitor(ClassVisitor cv, List methodsWithRetainedBody) { - this(cv, methodsWithRetainedBody, null); - } + private final boolean isKotlinClass; public AbiFilteringClassVisitor( ClassVisitor cv, List methodsWithRetainedBody, - @Nullable Set referencedClassNames) { + @Nullable Set referencedClassNames, + boolean isKotlinClass) { super(Opcodes.ASM7, cv); this.methodsWithRetainedBody = methodsWithRetainedBody; this.referencedClassNames = referencedClassNames; + this.isKotlinClass = isKotlinClass; } @Override @@ -213,7 +212,7 @@ private boolean shouldInclude(int access) { return false; } - return (access & (Opcodes.ACC_SYNTHETIC | Opcodes.ACC_BRIDGE)) != Opcodes.ACC_SYNTHETIC; + return (isKotlinClass || (access & (Opcodes.ACC_SYNTHETIC | Opcodes.ACC_BRIDGE)) != Opcodes.ACC_SYNTHETIC); } private boolean isInterface(int access) { diff --git a/src/com/facebook/buck/jvm/java/abi/StubJarClassEntry.java b/src/com/facebook/buck/jvm/java/abi/StubJarClassEntry.java index ac52d60e4fc..6df86aa117a 100644 --- a/src/com/facebook/buck/jvm/java/abi/StubJarClassEntry.java +++ b/src/com/facebook/buck/jvm/java/abi/StubJarClassEntry.java @@ -44,6 +44,7 @@ class StubJarClassEntry extends StubJarEntry { private final Path path; private final ClassNode stub; private final boolean retainEverything; + private final boolean isKotlinClass; @Nullable public static StubJarClassEntry of( @@ -70,7 +71,7 @@ public static StubJarClassEntry of( // used within an inline function, and in these cases we need to retain the whole class. input.visitClass(path, stub, false); return new StubJarClassEntry( - path, stub, Collections.emptySet(), Collections.emptyList(), true); + path, stub, Collections.emptySet(), Collections.emptyList(), true, isKotlinClass); } ClassNode dummyStub = new ClassNode(Opcodes.ASM7); input.visitClass(path, dummyStub, true); @@ -94,7 +95,7 @@ public static StubJarClassEntry of( // ABI methods and fields, and will use that information later to filter the InnerClasses table. ClassReferenceTracker referenceTracker = new ClassReferenceTracker(stub); ClassVisitor firstLevelFiltering = - new AbiFilteringClassVisitor(referenceTracker, methodBodiesToRetain); + new AbiFilteringClassVisitor(referenceTracker, methodBodiesToRetain, null, isKotlinClass); // If we want ABIs that are compatible with those generated from source, we add a visitor // at the very start of the chain which transforms the event stream coming out of `ClassNode` @@ -106,11 +107,13 @@ public static StubJarClassEntry of( // The synthetic package-info class is how package annotations are recorded; that one is // actually used by the compiler - if (!isAnonymousOrLocalOrSyntheticClass(stub) + // Kotlin top functions reside in synthetic classes, we should output ABIs for them. + if ((isSyntheticClass(stub) && isKotlinModule) + || !(isSyntheticClass(stub) || isAnonymousOrLocalClass(stub)) || retainAllMethodBodies || stub.name.endsWith("/package-info")) { return new StubJarClassEntry( - path, stub, referenceTracker.getReferencedClassNames(), methodBodiesToRetain, false); + path, stub, referenceTracker.getReferencedClassNames(), methodBodiesToRetain, false, isKotlinClass); } return null; @@ -121,12 +124,14 @@ private StubJarClassEntry( ClassNode stub, Set referencedClassNames, List methodBodiesToRetain, - boolean retainEverything) { + boolean retainEverything, + boolean isKotlinClass) { this.path = path; this.stub = stub; this.referencedClassNames = referencedClassNames; this.methodBodiesToRetain = methodBodiesToRetain; this.retainEverything = retainEverything; + this.isKotlinClass = isKotlinClass; } @Override @@ -144,7 +149,7 @@ private InputStream openInputStream() { ClassVisitor visitor = writer; if (!retainEverything) { visitor = new InnerClassSortingClassVisitor(stub.name, visitor); - visitor = new AbiFilteringClassVisitor(visitor, methodBodiesToRetain, referencedClassNames); + visitor = new AbiFilteringClassVisitor(visitor, methodBodiesToRetain, referencedClassNames, isKotlinClass); } stub.accept(visitor); @@ -152,11 +157,11 @@ private InputStream openInputStream() { return new ByteArrayInputStream(writer.toByteArray()); } - private static boolean isAnonymousOrLocalOrSyntheticClass(ClassNode node) { - if ((node.access & Opcodes.ACC_SYNTHETIC) == Opcodes.ACC_SYNTHETIC) { - return true; - } + private static boolean isSyntheticClass(ClassNode node) { + return ((node.access & Opcodes.ACC_SYNTHETIC) == Opcodes.ACC_SYNTHETIC); + } + private static boolean isAnonymousOrLocalClass(ClassNode node) { InnerClassNode innerClass = getInnerClassMetadata(node); while (innerClass != null) { if (innerClass.outerName == null) { diff --git a/src/com/facebook/buck/jvm/kotlin/KotlinConfiguredCompilerFactory.java b/src/com/facebook/buck/jvm/kotlin/KotlinConfiguredCompilerFactory.java index e66cb56bd4d..b33461c8365 100644 --- a/src/com/facebook/buck/jvm/kotlin/KotlinConfiguredCompilerFactory.java +++ b/src/com/facebook/buck/jvm/kotlin/KotlinConfiguredCompilerFactory.java @@ -16,8 +16,12 @@ package com.facebook.buck.jvm.kotlin; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Comparator.comparing; + import com.facebook.buck.core.model.BuildTarget; import com.facebook.buck.core.model.TargetConfiguration; +import com.facebook.buck.core.rules.ActionGraphBuilder; import com.facebook.buck.core.rules.BuildRule; import com.facebook.buck.core.rules.BuildRuleResolver; import com.facebook.buck.core.sourcepath.SourcePath; @@ -34,11 +38,19 @@ import com.facebook.buck.jvm.kotlin.KotlinLibraryDescription.AnnotationProcessingTool; import com.facebook.buck.jvm.kotlin.KotlinLibraryDescription.CoreArg; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSortedSet; +import com.google.common.collect.Maps; import java.nio.file.Path; import java.util.Objects; import java.util.Optional; import java.util.function.BiFunction; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; +import java.util.function.Predicate; import javax.annotation.Nullable; public class KotlinConfiguredCompilerFactory extends ConfiguredCompilerFactory { @@ -71,19 +83,21 @@ public KotlinConfiguredCompilerFactory( public CompileToJarStepFactory configure( @Nullable JvmLibraryArg args, JavacOptions javacOptions, + ActionGraphBuilder actionGraphBuilder, BuildRuleResolver buildRuleResolver, TargetConfiguration targetConfiguration, ToolchainProvider toolchainProvider) { CoreArg kotlinArgs = Objects.requireNonNull((CoreArg) args); Path pathToAbiGenerationPluginJar = - shouldGenerateSourceAbi() ? kotlinBuckConfig.getPathToAbiGenerationPluginJar() : null; + shouldGenerateSourceAbi(kotlinArgs, kotlinBuckConfig) ? kotlinBuckConfig.getPathToAbiGenerationPluginJar() : null; return new KotlincToJarStepFactory( kotlinBuckConfig.getKotlinc(), kotlinBuckConfig.getKotlinHomeLibraries(), pathToAbiGenerationPluginJar, - kotlinArgs.getExtraKotlincArguments(), + condenseCompilerArguments(kotlinArgs), kotlinArgs.getKotlincPlugins(), - getFriendSourcePaths(buildRuleResolver, kotlinArgs.getFriendPaths(), kotlinBuckConfig), + getFriendSourcePaths( + actionGraphBuilder, buildRuleResolver, kotlinArgs.getFriendPaths(), kotlinBuckConfig), kotlinArgs.getAnnotationProcessingTool().orElse(AnnotationProcessingTool.KAPT), kotlinArgs.getKaptApOptions(), extraClasspathProviderSupplier.apply(toolchainProvider, targetConfiguration), @@ -131,7 +145,12 @@ public boolean sourceAbiCopiesFromLibraryTargetOutput() { return true; } + private static boolean shouldGenerateSourceAbi(CoreArg kotlinArgs, KotlinBuckConfig kotlinBuckConfig) { + return kotlinArgs.getAbiGenerationMode().orElse(kotlinBuckConfig.getAbiGenerationMode()).isSourceAbi(); + } + private static ImmutableList getFriendSourcePaths( + ActionGraphBuilder actionGraphBuilder, BuildRuleResolver buildRuleResolver, ImmutableSortedSet friendPaths, KotlinBuckConfig kotlinBuckConfig) { @@ -142,13 +161,11 @@ private static ImmutableList getFriendSourcePaths( if (shouldCompileAgainstAbis && rule instanceof HasJavaAbi) { Optional abiJarTarget = ((HasJavaAbi) rule).getAbiJar(); if (abiJarTarget.isPresent()) { - Optional abiJarRule = buildRuleResolver.getRuleOptional(abiJarTarget.get()); - if (abiJarRule.isPresent()) { - SourcePath abiJarPath = abiJarRule.get().getSourcePathToOutput(); - if (abiJarPath != null) { - sourcePaths.add(abiJarPath); - continue; - } + BuildRule abiJarRule = actionGraphBuilder.requireRule(abiJarTarget.get()); + SourcePath abiJarPath = abiJarRule.getSourcePathToOutput(); + if (abiJarPath != null) { + sourcePaths.add(abiJarPath); + continue; } } } @@ -161,4 +178,64 @@ private static ImmutableList getFriendSourcePaths( return sourcePaths.build(); } + + ImmutableList condenseCompilerArguments(CoreArg kotlinArgs) { + ImmutableMap.Builder> optionBuilder = ImmutableMap.builder(); + LinkedHashMap> freeArgs = Maps.newLinkedHashMap(); + kotlinArgs.getFreeCompilerArgs() + .forEach(arg -> freeArgs.put(arg, Optional.empty())); + optionBuilder.putAll(freeArgs); + + // Args from CommonToolArguments.kt and KotlinCommonToolOptions.kt + if (kotlinArgs.getAllWarningsAsErrors()) { + optionBuilder.put("-Werror", Optional.empty()); + } + if (kotlinArgs.getSuppressWarnings()) { + optionBuilder.put("-nowarn", Optional.empty()); + } + if (kotlinArgs.getVerbose()) { + optionBuilder.put("-verbose", Optional.empty()); + } + + // Args from K2JVMCompilerArguments.kt and KotlinJvmOptions.kt + optionBuilder.put("-jvm-target", Optional.of(kotlinArgs.getJvmTarget())); + if (kotlinArgs.getIncludeRuntime()) { + optionBuilder.put("-include-runtime", Optional.empty()); + } + kotlinArgs.getJdkHome().ifPresent(jdkHome -> optionBuilder.put("-jdk-home", Optional.of(jdkHome))); + if (kotlinArgs.getNoJdk()) { + optionBuilder.put("-no-jdk", Optional.empty()); + } + if (kotlinArgs.getNoStdlib()) { + optionBuilder.put("-no-stdlib", Optional.empty()); + } + if (kotlinArgs.getNoReflect()) { + optionBuilder.put("-no-reflect", Optional.empty()); + } + if (kotlinArgs.getJavaParameters()) { + optionBuilder.put("-java-parameters", Optional.empty()); + } + kotlinArgs.getApiVersion().ifPresent(apiVersion -> optionBuilder.put("-api-version", Optional.of(apiVersion))); + kotlinArgs.getLanguageVersion().ifPresent(languageVersion -> optionBuilder.put("-language-version", Optional.of(languageVersion))); + + // Return de-duping keys and sorting by them. + return optionBuilder.build() + .entrySet() + .stream() + .filter(distinctByKey(Map.Entry::getKey)) + .sorted(comparing(Map.Entry::getKey, String.CASE_INSENSITIVE_ORDER)) + .flatMap(entry -> { + if (entry.getValue().isPresent()) { + return ImmutableList.of(entry.getKey(), entry.getValue().get()).stream(); + } else { + return ImmutableList.of(entry.getKey()).stream(); + } + }) + .collect(toImmutableList()); + } + + static Predicate distinctByKey(Function keyExtractor) { + Set seen = ConcurrentHashMap.newKeySet(); + return t -> seen.add(keyExtractor.apply(t)); + } } diff --git a/src/com/facebook/buck/jvm/kotlin/KotlinLibraryDescription.java b/src/com/facebook/buck/jvm/kotlin/KotlinLibraryDescription.java index 15ad99c9f37..1b37b2c029a 100644 --- a/src/com/facebook/buck/jvm/kotlin/KotlinLibraryDescription.java +++ b/src/com/facebook/buck/jvm/kotlin/KotlinLibraryDescription.java @@ -201,7 +201,101 @@ public enum AnnotationProcessingTool { } public interface CoreArg extends JavaLibraryDescription.CoreArg { - ImmutableList getExtraKotlincArguments(); + + /** + * A list of additional compiler arguments. + */ + ImmutableList getFreeCompilerArgs(); + + /** + * Report an error if there are any warnings. + */ + @Value.Default + default boolean getAllWarningsAsErrors() { + return false; + } + + /** + * Generate no warnings. + */ + @Value.Default + default boolean getSuppressWarnings() { + return false; + } + + /** + * Enable verbose logging output. + */ + @Value.Default + default boolean getVerbose() { + return false; + } + + /** + * Include Kotlin runtime in to resulting .jar + */ + @Value.Default + default boolean getIncludeRuntime() { + return false; + } + + /** + * Target version of the generated JVM bytecode (1.6 or 1.8), default is 1.6 + * Possible values: "1.6", "1.8" + */ + @Value.Default + default String getJvmTarget() { + return "1.6"; + } + + /** + * Path to JDK home directory to include into classpath, if differs from default JAVA_HOME + */ + Optional getJdkHome(); + + /** + * Don't include Java runtime into classpath. + */ + @Value.Default + default boolean getNoJdk() { + return false; + } + + /** + * Don't include kotlin-stdlib.jar or kotlin-reflect.jar into classpath. + */ + @Value.Default + default boolean getNoStdlib() { + return true; + } + + /** + * Don't include kotlin-reflect.jar into classpath. + */ + @Value.Default + default boolean getNoReflect() { + return true; + } + + /** + * Generate metadata for Java 1.8 reflection on method parameters. + */ + @Value.Default + default boolean getJavaParameters() { + return false; + } + + /** + * Allow to use declarations only from the specified version of bundled libraries. + * Possible values: "1.0", "1.1", "1.2", "1.3", "1.4". + */ + Optional getApiVersion(); + + /** + * Provide source compatibility with specified language version. + * Possible values: "1.0", "1.1", "1.2", "1.3", "1.4". + */ + Optional getLanguageVersion(); Optional getAnnotationProcessingTool(); diff --git a/src/com/facebook/buck/jvm/kotlin/KotlincStep.java b/src/com/facebook/buck/jvm/kotlin/KotlincStep.java index 06f1c548e27..b32cf38edbc 100644 --- a/src/com/facebook/buck/jvm/kotlin/KotlincStep.java +++ b/src/com/facebook/buck/jvm/kotlin/KotlincStep.java @@ -40,15 +40,12 @@ public class KotlincStep implements Step { private static final String CLASSPATH_FLAG = "-classpath"; private static final String DESTINATION_FLAG = "-d"; - private static final String INCLUDE_RUNTIME_FLAG = "-include-runtime"; - private static final String EXCLUDE_REFLECT = "-no-reflect"; - private static final String VERBOSE = "-verbose"; + private static final String VERBOSE_FLAG = "-verbose"; private final Kotlinc kotlinc; private final ImmutableSortedSet combinedClassPathEntries; private final Path outputDirectory; private final ImmutableList extraArguments; - private final ImmutableList verboseModeOnlyExtraArguments; private final ImmutableSortedSet sourceFilePaths; private final ProjectFilesystem filesystem; private final Path pathToSrcsList; @@ -63,7 +60,6 @@ public class KotlincStep implements Step { ImmutableSortedSet combinedClassPathEntries, Kotlinc kotlinc, ImmutableList extraArguments, - ImmutableList verboseModeOnlyExtraArguments, ProjectFilesystem filesystem, Optional workingDirectory) { this.invokingRule = invokingRule; @@ -73,7 +69,6 @@ public class KotlincStep implements Step { this.kotlinc = kotlinc; this.combinedClassPathEntries = combinedClassPathEntries; this.extraArguments = extraArguments; - this.verboseModeOnlyExtraArguments = verboseModeOnlyExtraArguments; this.filesystem = filesystem; this.workingDirectory = workingDirectory; } @@ -156,10 +151,6 @@ ImmutableList getOptions( path -> filesystem.resolve(path).toAbsolutePath().toString()))); } - builder.add(INCLUDE_RUNTIME_FLAG); - builder.add(EXCLUDE_REFLECT); - builder.add(VERBOSE); - if (!extraArguments.isEmpty()) { for (String extraArgument : extraArguments) { if (!extraArgument.isEmpty()) { @@ -168,13 +159,8 @@ ImmutableList getOptions( } } - if (context.getVerbosity().shouldUseVerbosityFlagIfAvailable() - && !verboseModeOnlyExtraArguments.isEmpty()) { - for (String extraArgument : verboseModeOnlyExtraArguments) { - if (!extraArgument.isEmpty()) { - builder.add(extraArgument); - } - } + if (context.getVerbosity().shouldUseVerbosityFlagIfAvailable()) { + builder.add(VERBOSE_FLAG); } return builder.build(); diff --git a/src/com/facebook/buck/jvm/kotlin/KotlincToJarStepFactory.java b/src/com/facebook/buck/jvm/kotlin/KotlincToJarStepFactory.java index 8adc339534f..2f614804a2b 100644 --- a/src/com/facebook/buck/jvm/kotlin/KotlincToJarStepFactory.java +++ b/src/com/facebook/buck/jvm/kotlin/KotlincToJarStepFactory.java @@ -105,9 +105,6 @@ public class KotlincToJarStepFactory extends CompileToJarStepFactory implements private static final String AP_OPTIONS = KAPT3_PLUGIN + "apoptions="; private static final String KAPT_GENERATED = "kapt.kotlin.generated"; private static final String MODULE_NAME = "-module-name"; - private static final String NO_STDLIB = "-no-stdlib"; - private static final String NO_REFLECT = "-no-reflect"; - private static final String VERBOSE = "-verbose"; private static final PathMatcher KOTLIN_PATH_MATCHER = FileExtensionMatcher.of("kt"); private static final PathMatcher SRC_ZIP_MATCHER = GlobPatternMatcher.of("**.src.zip"); @@ -295,13 +292,11 @@ public void createCompileStep( .addAll(getKotlincPluginsArgs(resolver)) .addAll(annotationProcessingOptionsBuilder.build()) .add(MODULE_NAME) - .add(moduleName) - .add(NO_STDLIB) - .add(NO_REFLECT); + .add(moduleName); - Path tmpSourceAbiFolder; if (abiGenerationPlugin != null) { - tmpSourceAbiFolder = JavaAbis.getTmpGenPathForSourceAbi(projectFilesystem, invokingRule); + Path tmpSourceAbiFolder = JavaAbis.getTmpGenPathForSourceAbi(projectFilesystem, invokingRule); + addCreateFolderStep(steps, projectFilesystem, buildContext, tmpSourceAbiFolder); extraArguments.add("-Xplugin=" + abiGenerationPlugin); extraArguments.add( "-P", "plugin:org.jetbrains.kotlin.jvm.abi:outputDir=" + tmpSourceAbiFolder); @@ -316,7 +311,6 @@ public void createCompileStep( allClasspaths, kotlinc, extraArguments.build(), - ImmutableList.of(VERBOSE), projectFilesystem, Optional.of(parameters.getOutputPaths().getWorkingDirectory()))); diff --git a/src/com/facebook/buck/jvm/scala/ScalaConfiguredCompilerFactory.java b/src/com/facebook/buck/jvm/scala/ScalaConfiguredCompilerFactory.java index c209fbd2345..841ad1c5cf1 100644 --- a/src/com/facebook/buck/jvm/scala/ScalaConfiguredCompilerFactory.java +++ b/src/com/facebook/buck/jvm/scala/ScalaConfiguredCompilerFactory.java @@ -18,6 +18,7 @@ import com.facebook.buck.core.model.BuildTarget; import com.facebook.buck.core.model.TargetConfiguration; +import com.facebook.buck.core.rules.ActionGraphBuilder; import com.facebook.buck.core.rules.BuildRuleResolver; import com.facebook.buck.core.toolchain.ToolchainProvider; import com.facebook.buck.core.toolchain.tool.Tool; @@ -71,6 +72,7 @@ private Tool getScalac(BuildRuleResolver resolver, TargetConfiguration targetCon public CompileToJarStepFactory configure( @Nullable JvmLibraryArg arg, JavacOptions javacOptions, + ActionGraphBuilder actionGraphBuilder, BuildRuleResolver buildRuleResolver, TargetConfiguration targetConfiguration, ToolchainProvider toolchainProvider) { diff --git a/src/com/facebook/buck/parser/ParserPythonInterpreterProvider.java b/src/com/facebook/buck/parser/ParserPythonInterpreterProvider.java index c2a8ee7e33c..b46f7b7c19f 100644 --- a/src/com/facebook/buck/parser/ParserPythonInterpreterProvider.java +++ b/src/com/facebook/buck/parser/ParserPythonInterpreterProvider.java @@ -31,7 +31,7 @@ public class ParserPythonInterpreterProvider { private static final ImmutableList PYTHON_INTERPRETER_NAMES = - ImmutableList.of("python2", "python"); + ImmutableList.of("python3", "python", "python2"); private final ParserConfig parserConfig; private final ExecutableFinder executableFinder; diff --git a/src/com/facebook/buck/parser/PythonDslProjectBuildFileParser.java b/src/com/facebook/buck/parser/PythonDslProjectBuildFileParser.java index 16eb76ca916..e21baa34319 100644 --- a/src/com/facebook/buck/parser/PythonDslProjectBuildFileParser.java +++ b/src/com/facebook/buck/parser/PythonDslProjectBuildFileParser.java @@ -455,18 +455,24 @@ protected BuildFileManifest getAllRulesInternal(Path buildFile) } } currentBuildFile.set(buildFile); - BuildFilePythonResult resultObject = - performJsonRequest( - ImmutableMap.of( - "buildFile", - buildFile.toString(), - "watchRoot", - watchRoot, - "projectPrefix", - projectPrefix, - "packageImplicitLoad", - packageImplicitIncludeFinder.findIncludeForBuildFile(getBasePath(buildFile)))); Path buckPyPath = getPathToBuckPy(options.getDescriptions()); + BuildFilePythonResult resultObject; + try { + resultObject = + performJsonRequest( + ImmutableMap.of( + "buildFile", + buildFile.toString(), + "watchRoot", + watchRoot, + "projectPrefix", + projectPrefix, + "packageImplicitLoad", + packageImplicitIncludeFinder.findIncludeForBuildFile(getBasePath(buildFile)))); + } catch (IllegalArgumentException iae) { + throw BuildFileParseException.createForBuildFileParseError( + buildFile, createParseException(buildFile, buckPyPath.getParent(), iae.getMessage(), null)); + } handleDiagnostics( buildFile, buckPyPath.getParent(), resultObject.getDiagnostics(), buckEventBus); values = resultObject.getValues(); diff --git a/src/com/facebook/buck/rules/macros/WorkerMacroArg.java b/src/com/facebook/buck/rules/macros/WorkerMacroArg.java index aa9c1dbccee..512928a422c 100644 --- a/src/com/facebook/buck/rules/macros/WorkerMacroArg.java +++ b/src/com/facebook/buck/rules/macros/WorkerMacroArg.java @@ -133,6 +133,10 @@ public int getMaxWorkers() { return workerTool.getMaxWorkers(); } + public boolean isAsync() { + return workerTool.isAsync(); + } + public String getJobArgs(SourcePathResolverAdapter pathResolver) { return Arg.stringify(arg, pathResolver).trim(); } diff --git a/src/com/facebook/buck/shell/DefaultWorkerToolRule.java b/src/com/facebook/buck/shell/DefaultWorkerToolRule.java index ca8383a9826..ba1b1c46134 100644 --- a/src/com/facebook/buck/shell/DefaultWorkerToolRule.java +++ b/src/com/facebook/buck/shell/DefaultWorkerToolRule.java @@ -59,6 +59,7 @@ protected DefaultWorkerToolRule( SourcePathRuleFinder ruleFinder, Tool tool, int maxWorkers, + boolean isAsync, boolean isPersistent) { super( buildTarget, @@ -71,6 +72,7 @@ protected DefaultWorkerToolRule( new DefaultWorkerTool( new DefaultWorkerToolDelegatingTool(tool, getSourcePathToOutput()), maxWorkers, + isAsync, isPersistent, buildTarget, generateNewUUID()); @@ -141,6 +143,13 @@ static class DefaultWorkerTool implements WorkerTool { @CustomFieldBehavior(DefaultFieldSerialization.class) private final Integer maxWorkers; + /** + * Important : Do not add this field into RuleKey. Rule key should not change in case of async + * variable modification. + */ + @CustomFieldBehavior(DefaultFieldSerialization.class) + private final boolean isAsync; + /** * Important : Do not add this field into RuleKey. Rule key should not change in case of * instance key modification (that is calculated during creation as random UUID). @@ -149,11 +158,17 @@ static class DefaultWorkerTool implements WorkerTool { private HashCode instanceKey; DefaultWorkerTool( - Tool tool, int maxWorkers, boolean isPersistent, BuildTarget buildTarget, UUID uuid) { + Tool tool, + int maxWorkers, + boolean isAsync, + boolean isPersistent, + BuildTarget buildTarget, + UUID uuid) { this.tool = tool; this.maxWorkers = maxWorkers; this.isPersistent = isPersistent; this.buildTarget = buildTarget; + this.isAsync = isAsync; this.instanceKey = calculateInstanceKey(uuid); } @@ -180,6 +195,11 @@ public int getMaxWorkers() { return maxWorkers; } + @Override + public boolean isAsync() { + return isAsync; + } + @Override public boolean isPersistent() { return isPersistent; diff --git a/src/com/facebook/buck/shell/GenruleBuildable.java b/src/com/facebook/buck/shell/GenruleBuildable.java index bd9310eeafd..9c5eb2ab341 100644 --- a/src/com/facebook/buck/shell/GenruleBuildable.java +++ b/src/com/facebook/buck/shell/GenruleBuildable.java @@ -652,7 +652,8 @@ public final boolean shouldExecuteRemotely() { *
  • GEN_DIR, Buck's gendir *
  • SRCDIR, the symlink-populated source directory readable to the command *
  • TMP, the temp directory usable by the command - *
  • ANDROID_HOME, the path to the Android SDK (if present) + *
  • ANDROID_HOME, deprecated, the path to the Android SDK (if present) + *
  • ANDROID_SDK_ROOT, the path to the Android SDK (if present) *
  • DX, the path to the Android DX executable (if present) *
  • ZIPALIGN, the path to the Android Zipalign executable (if present) *
  • AAPT, the path to the Android AAPT executable (if present) @@ -704,6 +705,8 @@ public void addEnvironmentVariables( androidTools.ifPresent( tools -> { environmentVariablesBuilder.put("ANDROID_HOME", tools.getAndroidSdkLocation().toString()); + environmentVariablesBuilder.put( + "ANDROID_SDK_ROOT", tools.getAndroidSdkLocation().toString()); environmentVariablesBuilder.put("DX", tools.getAndroidPathToDx().toString()); environmentVariablesBuilder.put("ZIPALIGN", tools.getAndroidPathToZipalign().toString()); environmentVariablesBuilder.put( @@ -758,6 +761,7 @@ private static Optional convertToWorkerJobParams( workerMacroArg.getStartupCommand(), workerMacroArg.getEnvironment(), workerMacroArg.getMaxWorkers(), + workerMacroArg.isAsync(), workerMacroArg.getPersistentWorkerKey().isPresent() ? Optional.of( WorkerProcessIdentity.of( diff --git a/src/com/facebook/buck/shell/WorkerShellStep.java b/src/com/facebook/buck/shell/WorkerShellStep.java index 520bc1eea67..513b60acfec 100644 --- a/src/com/facebook/buck/shell/WorkerShellStep.java +++ b/src/com/facebook/buck/shell/WorkerShellStep.java @@ -28,13 +28,14 @@ import com.facebook.buck.worker.WorkerJobParams; import com.facebook.buck.worker.WorkerJobResult; import com.facebook.buck.worker.WorkerProcessPool; -import com.facebook.buck.worker.WorkerProcessPool.BorrowedWorkerProcess; import com.facebook.buck.worker.WorkerProcessPoolFactory; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Throwables; import com.google.common.collect.ImmutableMap; import java.io.IOException; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ExecutionException; import java.util.stream.Collectors; public class WorkerShellStep implements Step { @@ -78,9 +79,14 @@ public StepExecutionResult execute(ExecutionContext context) WorkerJobParams paramsToUse = getWorkerJobParamsToUse(context.getPlatform()); WorkerProcessPool pool = factory.getWorkerProcessPool(context, paramsToUse.getWorkerProcessParams()); - WorkerJobResult result; - try (BorrowedWorkerProcess process = pool.borrowWorkerProcess()) { - result = process.submitAndWaitForJob(getExpandedJobArgs(context)); + WorkerJobResult result = null; + try { + result = pool.submitJob(getExpandedJobArgs(context)).get(); + } catch (ExecutionException e) { + if (e.getCause() != null) { + Throwables.throwIfUnchecked(e.getCause()); + } + throw new RuntimeException(e); } Verbosity verbosity = context.getVerbosity(); diff --git a/src/com/facebook/buck/shell/WorkerTool.java b/src/com/facebook/buck/shell/WorkerTool.java index ab65baf444b..8ca978fb1dc 100644 --- a/src/com/facebook/buck/shell/WorkerTool.java +++ b/src/com/facebook/buck/shell/WorkerTool.java @@ -34,4 +34,6 @@ public interface WorkerTool extends AddsToRuleKey { boolean isPersistent(); HashCode getInstanceKey(); + + boolean isAsync(); } diff --git a/src/com/facebook/buck/shell/WorkerToolDescription.java b/src/com/facebook/buck/shell/WorkerToolDescription.java index 006aeafc822..0d1c0833641 100644 --- a/src/com/facebook/buck/shell/WorkerToolDescription.java +++ b/src/com/facebook/buck/shell/WorkerToolDescription.java @@ -124,6 +124,8 @@ public BuildRule createBuildRule( builder.addEnv(e.getKey(), macrosConverter.convert(e.getValue())); } + boolean async = args.getSoloAsync().orElse(false); + Preconditions.checkArgument( !(args.getMaxWorkers().isPresent() && args.getMaxWorkersPerThreadPercent().isPresent()), "max_workers and max_workers_per_thread_percent must not be used together."); @@ -154,6 +156,7 @@ public BuildRule createBuildRule( graphBuilder, tool, maxWorkers, + async, args.getPersistent() .orElse(buckConfig.getBooleanValue(CONFIG_SECTION, CONFIG_PERSISTENT_KEY, false))); } @@ -197,5 +200,7 @@ default Either> getArgs() { Optional getMaxWorkersPerThreadPercent(); Optional getPersistent(); + + Optional getSoloAsync(); } } diff --git a/src/com/facebook/buck/swift/SwiftCompile.java b/src/com/facebook/buck/swift/SwiftCompile.java index a7796d3a8f3..30c95d157fc 100644 --- a/src/com/facebook/buck/swift/SwiftCompile.java +++ b/src/com/facebook/buck/swift/SwiftCompile.java @@ -21,11 +21,13 @@ import com.facebook.buck.core.build.execution.context.ExecutionContext; import com.facebook.buck.core.filesystems.AbsPath; import com.facebook.buck.core.model.BuildTarget; +import com.facebook.buck.core.model.Flavor; import com.facebook.buck.core.model.impl.BuildTargetPaths; import com.facebook.buck.core.rulekey.AddToRuleKey; import com.facebook.buck.core.rules.ActionGraphBuilder; import com.facebook.buck.core.rules.BuildRule; import com.facebook.buck.core.rules.BuildRuleResolver; +import com.facebook.buck.core.rules.attr.SupportsInputBasedRuleKey; import com.facebook.buck.core.rules.common.BuildableSupport; import com.facebook.buck.core.rules.impl.AbstractBuildRule; import com.facebook.buck.core.sourcepath.ExplicitBuildTargetSourcePath; @@ -34,19 +36,19 @@ import com.facebook.buck.core.toolchain.tool.Tool; import com.facebook.buck.cxx.CxxDescriptionEnhancer; import com.facebook.buck.cxx.PreprocessorFlags; -import com.facebook.buck.cxx.toolchain.CxxPlatform; import com.facebook.buck.cxx.toolchain.HeaderVisibility; +import com.facebook.buck.rules.coercer.FrameworkPath; import com.facebook.buck.cxx.toolchain.LinkerMapMode; import com.facebook.buck.cxx.toolchain.PathShortener; import com.facebook.buck.cxx.toolchain.Preprocessor; import com.facebook.buck.io.BuildCellRelativePath; import com.facebook.buck.io.file.MostFiles; import com.facebook.buck.io.filesystem.ProjectFilesystem; +import com.facebook.buck.rules.args.AddsToRuleKeyFunction; import com.facebook.buck.rules.args.Arg; import com.facebook.buck.rules.args.FileListableLinkerInputArg; import com.facebook.buck.rules.args.SourcePathArg; import com.facebook.buck.rules.args.StringArg; -import com.facebook.buck.rules.coercer.FrameworkPath; import com.facebook.buck.step.Step; import com.facebook.buck.step.StepExecutionResult; import com.facebook.buck.step.StepExecutionResults; @@ -68,10 +70,9 @@ import java.util.Arrays; import java.util.Optional; import java.util.SortedSet; -import java.util.function.Function; /** A build rule which compiles one or more Swift sources into a Swift module. */ -public class SwiftCompile extends AbstractBuildRule { +public class SwiftCompile extends AbstractBuildRule implements SupportsInputBasedRuleKey { private static final String INCLUDE_FLAG = "-I"; @@ -82,13 +83,20 @@ public class SwiftCompile extends AbstractBuildRule { @AddToRuleKey(stringify = true) private final Path outputPath; + @AddToRuleKey(stringify = true) private final Path objectFilePath; + @AddToRuleKey(stringify = true) private final Path modulePath; + @AddToRuleKey(stringify = true) private final Path moduleObjectPath; + @AddToRuleKey(stringify = true) private final ImmutableList objectPaths; private final Optional swiftFileListPath; @AddToRuleKey private final boolean shouldEmitSwiftdocs; + @AddToRuleKey private final boolean useModulewrap; + @AddToRuleKey private final boolean compileForceCache; + @AddToRuleKey(stringify = true) private final Path swiftdocPath; @AddToRuleKey private final ImmutableSortedSet srcs; @@ -96,15 +104,16 @@ public class SwiftCompile extends AbstractBuildRule { @AddToRuleKey private final Optional version; @AddToRuleKey private final ImmutableList compilerFlags; + @AddToRuleKey(stringify = true) private final Path headerPath; - private final CxxPlatform cxxPlatform; - private final ImmutableSet frameworks; + @AddToRuleKey private final ImmutableSet frameworks; + @AddToRuleKey private final AddsToRuleKeyFunction frameworkPathToSearchPath; + @AddToRuleKey(stringify = true) + private final Flavor flavor; - private final boolean enableObjcInterop; + @AddToRuleKey private final boolean enableObjcInterop; @AddToRuleKey private final Optional bridgingHeader; - private final SwiftBuckConfig swiftBuckConfig; - @AddToRuleKey private final Preprocessor cPreprocessor; @AddToRuleKey private final PreprocessorFlags cxxDeps; @@ -114,7 +123,6 @@ public class SwiftCompile extends AbstractBuildRule { private BuildableSupport.DepsSupplier depsSupplier; SwiftCompile( - CxxPlatform cxxPlatform, SwiftBuckConfig swiftBuckConfig, BuildTarget buildTarget, SwiftTargetTriple swiftTarget, @@ -122,6 +130,8 @@ public class SwiftCompile extends AbstractBuildRule { ActionGraphBuilder graphBuilder, Tool swiftCompiler, ImmutableSet frameworks, + AddsToRuleKeyFunction frameworkPathToSearchPath, + Flavor flavor, String moduleName, Path outputPath, Iterable srcs, @@ -133,9 +143,9 @@ public class SwiftCompile extends AbstractBuildRule { PreprocessorFlags cxxDeps, boolean importUnderlyingModule) { super(buildTarget, projectFilesystem); - this.cxxPlatform = cxxPlatform; this.frameworks = frameworks; - this.swiftBuckConfig = swiftBuckConfig; + this.frameworkPathToSearchPath = frameworkPathToSearchPath; + this.flavor = flavor; this.swiftCompiler = swiftCompiler; this.outputPath = outputPath; this.importUnderlyingModule = importUnderlyingModule; @@ -161,6 +171,8 @@ public class SwiftCompile extends AbstractBuildRule { : Optional.empty(); this.shouldEmitSwiftdocs = swiftBuckConfig.getEmitSwiftdocs(); + this.useModulewrap = swiftBuckConfig.getUseModulewrap(); + this.compileForceCache = swiftBuckConfig.getCompileForceCache(); this.swiftdocPath = outputPath.resolve(escapedModuleName + ".swiftdoc"); this.srcs = ImmutableSortedSet.copyOf(srcs); @@ -203,9 +215,6 @@ private SwiftCompileStep makeCompileStep(SourcePathResolverAdapter resolver) { compilerCommand.add("-import-underlying-module"); } - Function frameworkPathToSearchPath = - CxxDescriptionEnhancer.frameworkPathToSearchPath(cxxPlatform, resolver); - compilerCommand.addAll( Streams.concat(frameworks.stream(), cxxDeps.getFrameworkPaths().stream()) .filter(x -> !x.isSDKROOTFrameworkPath()) @@ -318,7 +327,7 @@ public boolean isCacheable() { // means that Obj-C headers can be included multiple times if the machines which // populated the cache and the machine which is building have placed the source // repository at different paths (usually the case with CI and developer machines). - return !bridgingHeader.isPresent() || swiftBuckConfig.getCompileForceCache(); + return !bridgingHeader.isPresent() || compileForceCache; } @Override @@ -345,7 +354,7 @@ public ImmutableList getBuildSteps( path -> steps.add(makeFileListStep(context.getSourcePathResolver(), path))); steps.add(makeCompileStep(context.getSourcePathResolver())); - if (swiftBuckConfig.getUseModulewrap()) { + if (useModulewrap) { steps.add(makeModulewrapStep(context.getSourcePathResolver())); } @@ -402,7 +411,7 @@ ImmutableList getSwiftIncludeArgs(SourcePathResolverAdapter resolver) { .toToolFlags( resolver, PathShortener.byRelativizingToWorkingDir(getProjectFilesystem().getRootPath()), - CxxDescriptionEnhancer.frameworkPathToSearchPath(cxxPlatform, resolver), + frameworkPathToSearchPath, cPreprocessor, Optional.empty()) .getAllFlags(); @@ -416,7 +425,7 @@ ImmutableList getSwiftIncludeArgs(SourcePathResolverAdapter resolver) { getProjectFilesystem(), getBuildTarget().withFlavors(), headerVisibility, - cxxPlatform.getFlavor()); + flavor); args.add(INCLUDE_FLAG.concat(headerPath.toString())); } } @@ -425,7 +434,7 @@ ImmutableList getSwiftIncludeArgs(SourcePathResolverAdapter resolver) { } public ImmutableList getAstLinkArgs() { - if (!swiftBuckConfig.getUseModulewrap()) { + if (!useModulewrap) { return ImmutableList.builder() .addAll(StringArg.from("-Xlinker", "-add_ast_path")) .add(SourcePathArg.of(ExplicitBuildTargetSourcePath.of(getBuildTarget(), modulePath))) @@ -475,4 +484,17 @@ public SourcePath getObjCGeneratedHeaderPath() { public SourcePath getOutputPath() { return ExplicitBuildTargetSourcePath.of(getBuildTarget(), outputPath); } + + /** + * @return {@link SourcePath} to the .swiftmodule output from the compilation process. A + * swiftmodule file contains the public interface for a module, and is basically a binary file + * format equivalent to header files for a C framework or library. + * + * A swiftmodule file contains serialized ASTs (and possibly SIL), it conforms to + * Swift Binary Serialization Format, more details about this binary format can be found here: + * https://github.com/apple/swift/blob/7e6d62dae4bae4eb3737a6f76c0e51534c1bcca3/docs/Serialization.rst. + */ + public SourcePath getSwiftModuleOutputPath() { + return ExplicitBuildTargetSourcePath.of(getBuildTarget(), modulePath); + } } diff --git a/src/com/facebook/buck/swift/SwiftLibraryDescription.java b/src/com/facebook/buck/swift/SwiftLibraryDescription.java index 73659a904e3..ddb6d239440 100644 --- a/src/com/facebook/buck/swift/SwiftLibraryDescription.java +++ b/src/com/facebook/buck/swift/SwiftLibraryDescription.java @@ -288,7 +288,6 @@ public Iterable visit(BuildRule rule) { BuildTarget buildTargetCopy = buildTarget; return new SwiftCompile( - cxxPlatform, swiftBuckConfig, buildTarget, args.getTargetSdkVersion() @@ -298,6 +297,8 @@ public Iterable visit(BuildRule rule) { graphBuilder, swiftPlatform.get().getSwiftc(), args.getFrameworks(), + CxxDescriptionEnhancer.frameworkPathToSearchPath(cxxPlatform, graphBuilder.getSourcePathResolver()), + cxxPlatform.getFlavor(), args.getModuleName().orElse(buildTarget.getShortName()), BuildTargetPaths.getGenPath(projectFilesystem, buildTarget, "%s"), args.getSrcs(), @@ -460,7 +461,6 @@ public static SwiftCompile createSwiftCompileRule( args.getSrcs().forEach(src -> srcsDepsBuilder.add(src)); return new SwiftCompile( - cxxPlatform, swiftBuckConfig, buildTarget, swiftTarget.orElse(swiftPlatform.getSwiftTarget()), @@ -468,6 +468,8 @@ public static SwiftCompile createSwiftCompileRule( graphBuilder, swiftPlatform.getSwiftc(), args.getFrameworks(), + CxxDescriptionEnhancer.frameworkPathToSearchPath(cxxPlatform, graphBuilder.getSourcePathResolver()), + cxxPlatform.getFlavor(), args.getModuleName().orElse(buildTarget.getShortName()), BuildTargetPaths.getGenPath(projectFilesystem, buildTarget, "%s"), args.getSrcs(), @@ -500,6 +502,16 @@ private FlavorDomain getCxxPlatforms( .getUnresolvedCxxPlatforms(); } + public ImmutableSet getSupportedFlavors( + ImmutableSet flavors, TargetConfiguration toolchainTargetConfiguration) { + ImmutableSet currentSupportedFlavors = + ImmutableSet.copyOf(Sets.filter(flavors, SUPPORTED_FLAVORS::contains)); + ImmutableSet supportedCxxPlatformsFlavors = + ImmutableSet.copyOf(Sets.filter(flavors, getCxxPlatforms(toolchainTargetConfiguration)::contains)); + + return ImmutableSet.copyOf(Sets.union(currentSupportedFlavors, supportedCxxPlatformsFlavors)); + } + @RuleArg interface AbstractSwiftLibraryDescriptionArg extends BuildRuleArg, HasDeclaredDeps, HasSrcs { Optional getModuleName(); diff --git a/src/com/facebook/buck/swift/toolchain/SwiftTargetTriple.java b/src/com/facebook/buck/swift/toolchain/SwiftTargetTriple.java index d52cef447cd..a3b997d8df7 100644 --- a/src/com/facebook/buck/swift/toolchain/SwiftTargetTriple.java +++ b/src/com/facebook/buck/swift/toolchain/SwiftTargetTriple.java @@ -40,19 +40,26 @@ public abstract class SwiftTargetTriple implements AddsToRuleKey { @AddToRuleKey public abstract String getTargetSdkVersion(); + @AddToRuleKey + public abstract Boolean getIsSimulator(); + public String getTriple() { - return getArchitecture() + "-" + getVendor() + "-" + getPlatformName() + getTargetSdkVersion(); + String triple = getArchitecture() + "-" + getVendor() + "-" + getPlatformName() + getTargetSdkVersion(); + if(getIsSimulator()) { + triple = triple + "-simulator"; + } + return triple; } public static SwiftTargetTriple of( - String architecture, String vendor, String platformName, String targetSdkVersion) { - return ImmutableSwiftTargetTriple.of(architecture, vendor, platformName, targetSdkVersion); + String architecture, String vendor, String platformName, String targetSdkVersion, Boolean isSimulator) { + return ImmutableSwiftTargetTriple.of(architecture, vendor, platformName, targetSdkVersion, isSimulator); } public SwiftTargetTriple withTargetSdkVersion(String targetSdkVersion) { if (targetSdkVersion.equals(getTargetSdkVersion())) { return this; } - return of(getArchitecture(), getVendor(), getPlatformName(), targetSdkVersion); + return of(getArchitecture(), getVendor(), getPlatformName(), targetSdkVersion, getIsSimulator()); } } diff --git a/src/com/facebook/buck/swift/toolchain/impl/SwiftPlatformFactory.java b/src/com/facebook/buck/swift/toolchain/impl/SwiftPlatformFactory.java index 156c82128d3..da3ff2f7c0c 100644 --- a/src/com/facebook/buck/swift/toolchain/impl/SwiftPlatformFactory.java +++ b/src/com/facebook/buck/swift/toolchain/impl/SwiftPlatformFactory.java @@ -139,10 +139,10 @@ public static Optional findSwiftCompatibilityRuntimePath( } // Currently Xcode includes swift and swift-5.0 directories, and each contains different - // contents. Specifically, swift/platform contains the libswiftCompatibility50 and + // contents. Specifically, swift/platform contains the libswiftCompatibility51 and // libswiftCompatibilityDynamicReplacements libraries which are *also* necessary. Path swiftRuntimePath = toolchainPath.resolve("usr/lib/swift").resolve(platformName); - String libSwiftCompatibilityLibraryName = "libswiftCompatibility50.a"; + String libSwiftCompatibilityLibraryName = "libswiftCompatibility51.a"; LOG.debug("Searching for swift compatibility toolchain in %s", toolchainPath.toString()); if (Files.exists(swiftRuntimePath.resolve(libSwiftCompatibilityLibraryName))) { LOG.debug("Found swift compatibility toolchain at %s", toolchainPath.toString()); diff --git a/src/com/facebook/buck/test/TestCaseSummary.java b/src/com/facebook/buck/test/TestCaseSummary.java index 9d3494c99ac..cd0f2f1abdb 100644 --- a/src/com/facebook/buck/test/TestCaseSummary.java +++ b/src/com/facebook/buck/test/TestCaseSummary.java @@ -29,6 +29,7 @@ public class TestCaseSummary implements TestCaseSummaryExternalInterface testResults; private final boolean isDryRun; private final boolean hasAssumptionViolations; @@ -38,7 +39,15 @@ public class TestCaseSummary implements TestCaseSummaryExternalInterface testResults) { + this(testCaseName, false, testResults); + } + + public TestCaseSummary( + String testCaseName, + boolean testSuite, + List testResults) { this.testCaseName = testCaseName; + this.testSuite = testSuite; this.testResults = ImmutableList.copyOf(testResults); boolean isDryRun = false; @@ -110,6 +119,11 @@ public long getTotalTime() { return totalTime; } + /** @return whether this summary represents a suite of tests or a single class */ + public boolean isTestSuite() { + return testSuite; + } + /** @return a one-line, printable summary */ public String getOneLineSummary(Locale locale, boolean hasPassingDependencies, Ansi ansi) { String statusText; diff --git a/src/com/facebook/buck/test/XmlTestResultParser.java b/src/com/facebook/buck/test/XmlTestResultParser.java index 027305fca41..1fb756e1704 100644 --- a/src/com/facebook/buck/test/XmlTestResultParser.java +++ b/src/com/facebook/buck/test/XmlTestResultParser.java @@ -139,6 +139,7 @@ private static TestCaseSummary doParse(String xml) throws IOException, SAXExcept Element root = doc.getDocumentElement(); Preconditions.checkState("testcase".equals(root.getTagName())); String testCaseName = root.getAttribute("name"); + boolean testSuite = false; NodeList testElements = doc.getElementsByTagName("test"); List testResults = Lists.newArrayListWithCapacity(testElements.getLength()); @@ -176,13 +177,20 @@ private static TestCaseSummary doParse(String xml) throws IOException, SAXExcept stdErr = null; } + String actualTestCaseName = node.getAttribute("suite"); + if (actualTestCaseName != null && !actualTestCaseName.isEmpty()) { + testSuite |= !actualTestCaseName.equals(testCaseName); + } else { + actualTestCaseName = testCaseName; + } + TestResultSummary testResult = new TestResultSummary( - testCaseName, testName, type, time, message, stacktrace, stdOut, stdErr); + actualTestCaseName, testName, type, time, message, stacktrace, stdOut, stdErr); testResults.add(testResult); } - return new TestCaseSummary(testCaseName, testResults); + return new TestCaseSummary(testCaseName, testSuite, testResults); } private static String createDetailedExceptionMessage(Path xmlFile, String xmlFileContents) { diff --git a/src/com/facebook/buck/testrunner/BaseRunner.java b/src/com/facebook/buck/testrunner/BaseRunner.java index 17f0e0fbefe..4550424859c 100755 --- a/src/com/facebook/buck/testrunner/BaseRunner.java +++ b/src/com/facebook/buck/testrunner/BaseRunner.java @@ -28,6 +28,7 @@ import java.io.StringWriter; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; @@ -82,7 +83,7 @@ private static String removeCRIfNeeded(String text) { * The test result file is written as XML to avoid introducing a dependency on JSON (see class * overview). */ - protected void writeResult(String testClassName, List results) + protected void writeResult(String testClassName, Collection results) throws IOException, ParserConfigurationException, TransformerException { // XML writer logic taken from: // http://www.genedavis.com/library/xml/java_dom_xml_creation.jsp diff --git a/src/com/facebook/buck/testrunner/TestNGRunner.java b/src/com/facebook/buck/testrunner/TestNGRunner.java index 185f9026323..884354ba258 100644 --- a/src/com/facebook/buck/testrunner/TestNGRunner.java +++ b/src/com/facebook/buck/testrunner/TestNGRunner.java @@ -25,12 +25,12 @@ import java.lang.reflect.Constructor; import java.lang.reflect.Method; import java.lang.reflect.Modifier; -import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; -import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.stream.Collectors; import javax.annotation.Nullable; import org.testng.IAnnotationTransformer; @@ -51,7 +51,7 @@ import org.testng.reporters.SuiteHTMLReporter; import org.testng.reporters.XMLReporter; -/** Class that runs a set of TestNG tests and writes the results to a directory. */ +/** Class that runs a set of TestNG tests and outputs the results to a directory. */ public final class TestNGRunner extends BaseRunner { @Override @@ -60,11 +60,13 @@ public void run() throws Throwable { Class testClass = Class.forName(className); - List results; + Collection results; if (!mightBeATestClass(testClass)) { results = Collections.emptyList(); } else { - results = new ArrayList<>(); + // TestNG can run a test class's tests in parallel via DataProvider. + // Use concurrency-safe collection to avoid comodification errors. + results = new ConcurrentLinkedQueue<>(); TestNG testng = new TestNG(); testng.setUseDefaultListeners(false); testng.addListener(new FilteringAnnotationTransformer(results)); @@ -155,9 +157,9 @@ private static String getTestMethodNameWithParameters(ITestResult iTestResult) { } public class FilteringAnnotationTransformer implements IAnnotationTransformer { - final List results; + final Collection results; - FilteringAnnotationTransformer(List results) { + FilteringAnnotationTransformer(Collection results) { this.results = results; } @@ -199,13 +201,13 @@ public void transform( } private static class TestListener implements ITestListener, IConfigurationListener { - private final List results; + private final Collection results; private boolean mustRestoreStdoutAndStderr; private PrintStream originalOut, originalErr, stdOutStream, stdErrStream; private ByteArrayOutputStream rawStdOutBytes, rawStdErrBytes; private Map failedConfigurationTestClasses = new HashMap<>(); - public TestListener(List results) { + public TestListener(Collection results) { this.results = results; } diff --git a/src/com/facebook/buck/util/ProcessHelper.java b/src/com/facebook/buck/util/ProcessHelper.java index 730ab31ba56..69fdc598d1e 100644 --- a/src/com/facebook/buck/util/ProcessHelper.java +++ b/src/com/facebook/buck/util/ProcessHelper.java @@ -68,7 +68,7 @@ public static ProcessHelper getInstance() { try { LOG.verbose("Getting process tree..."); OperatingSystem os = OSHI.getOperatingSystem(); - OSProcess[] processes = os.getProcesses(100, OperatingSystem.ProcessSort.NEWEST); + List processes = os.getProcesses(100, OperatingSystem.ProcessSort.NEWEST); for (OSProcess process : processes) { tree.add(process); } diff --git a/src/com/facebook/buck/worker/BUCK b/src/com/facebook/buck/worker/BUCK index 933d4b1de7f..5f76301949d 100644 --- a/src/com/facebook/buck/worker/BUCK +++ b/src/com/facebook/buck/worker/BUCK @@ -39,6 +39,8 @@ java_immutables_library( "WorkerProcess.java", "WorkerProcessCommand.java", "WorkerProcessPool.java", + "WorkerProcessPoolAsync.java", + "WorkerProcessPoolSync.java", "WorkerProcessProtocol.java", "WorkerProcessProtocolZero.java", ], diff --git a/src/com/facebook/buck/worker/WorkerProcess.java b/src/com/facebook/buck/worker/WorkerProcess.java index b912ebd6213..393c8e40edc 100644 --- a/src/com/facebook/buck/worker/WorkerProcess.java +++ b/src/com/facebook/buck/worker/WorkerProcess.java @@ -25,15 +25,22 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.SettableFuture; import java.io.Closeable; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.HashSet; import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicInteger; import javax.annotation.Nullable; +import javax.annotation.concurrent.ThreadSafe; +@ThreadSafe public class WorkerProcess implements Closeable { private static final Logger LOG = Logger.get(WorkerProcess.class); @@ -45,8 +52,12 @@ public class WorkerProcess implements Closeable { private final Path stdErr; private final AtomicInteger currentMessageID = new AtomicInteger(); private boolean handshakePerformed = false; + private final ConcurrentHashMap> commandExitCodes = + new ConcurrentHashMap<>(); @Nullable private WorkerProcessProtocol.CommandSender protocol; @Nullable private ProcessExecutor.LaunchedProcess launchedProcess; + private final Thread readerThread; + private volatile boolean shutdownReaderThread = false; /** * Worker process is a process that stays alive and receives commands which describe jobs. Worker @@ -73,10 +84,14 @@ public WorkerProcess( processParams.withRedirectError(ProcessBuilder.Redirect.to(stdErr.toFile())); this.filesystem = filesystem; this.tmpPath = tmpPath; + this.readerThread = new Thread(this::readerLoop); + this.readerThread.setDaemon(true); + this.readerThread.setName( + "Worker Process IO Thread: " + Joiner.on(' ').join(processParams.getCommand())); } public boolean isAlive() { - return launchedProcess != null && launchedProcess.isAlive(); + return launchedProcess != null && launchedProcess.isAlive() && !shutdownReaderThread; } public synchronized void ensureLaunchAndHandshake() throws IOException { @@ -84,7 +99,7 @@ public synchronized void ensureLaunchAndHandshake() throws IOException { return; } LOG.debug( - "Starting up process %d using command: \'%s\'", + "Starting up process %d using command: '%s'", this.hashCode(), Joiner.on(' ').join(processParams.getCommand())); launchedProcess = executor.launchProcess(processParams); protocol = @@ -102,14 +117,11 @@ public synchronized void ensureLaunchAndHandshake() throws IOException { LOG.debug("Handshaking with process %d", this.hashCode()); protocol.handshake(currentMessageID.getAndIncrement()); handshakePerformed = true; + readerThread.start(); } - public synchronized WorkerJobResult submitAndWaitForJob(String jobArgs) throws IOException { - Preconditions.checkState( - protocol != null, - "Tried to submit a job to the worker process before the handshake was performed."); - - int messageID = currentMessageID.getAndAdd(1); + public ListenableFuture submitJob(String jobArgs) throws IOException { + int messageID = currentMessageID.getAndIncrement(); Path argsPath = Paths.get(tmpPath.toString(), String.format("%d.args", messageID)); Path stdoutPath = Paths.get(tmpPath.toString(), String.format("%d.out", messageID)); Path stderrPath = Paths.get(tmpPath.toString(), String.format("%d.err", messageID)); @@ -117,31 +129,66 @@ public synchronized WorkerJobResult submitAndWaitForJob(String jobArgs) throws I filesystem.deleteFileAtPathIfExists(stderrPath); filesystem.writeContentsToPath(jobArgs, argsPath); - LOG.debug( - "Sending job %d to process %d \n" + " job arguments: \'%s\'", - messageID, this.hashCode(), jobArgs); - protocol.send(messageID, ImmutableWorkerProcessCommand.of(argsPath, stdoutPath, stderrPath)); - LOG.debug("Receiving response for job %d from process %d", messageID, this.hashCode()); - int exitCode = protocol.receiveCommandResponse(messageID); - Optional stdout = filesystem.readFileIfItExists(stdoutPath); - Optional stderr = filesystem.readFileIfItExists(stderrPath); - LOG.debug( - "Job %d for process %d finished \n" - + " exit code: %d \n" - + " stdout: %s \n" - + " stderr: %s", - messageID, this.hashCode(), exitCode, stdout.orElse(""), stderr.orElse("")); + SettableFuture exitCodeFuture = SettableFuture.create(); + commandExitCodes.put(messageID, exitCodeFuture); + + try { + synchronized (this) { + Preconditions.checkState( + protocol != null, + "Tried to submit a job to the worker process before the handshake was performed."); + + Preconditions.checkState(!shutdownReaderThread, "Submitting job to a closed worker"); + + LOG.debug( + "Sending job %d to process %d \n" + " job arguments: '%s'", + messageID, this.hashCode(), jobArgs); + protocol.send( + messageID, ImmutableWorkerProcessCommand.of(argsPath, stdoutPath, stderrPath)); + } - return WorkerJobResult.of(exitCode, stdout, stderr); + } catch (Throwable t) { + commandExitCodes.remove(messageID); + throw t; + } + + // Notify the reader thread to start reading if it isn't already. + synchronized (readerThread) { + readerThread.notify(); + } + + return exitCodeFuture.transform( + (exitCode) -> { + LOG.debug( + "Receiving response for job %d from process %d - %d", + messageID, this.hashCode(), exitCode); + Optional stdout = filesystem.readFileIfItExists(stdoutPath); + Optional stderr = filesystem.readFileIfItExists(stderrPath); + LOG.debug( + "Job %d for process %d finished \n" + + " exit code: %d \n" + + " stdout: %s \n" + + " stderr: %s", + messageID, this.hashCode(), exitCode, stdout.orElse(""), stderr.orElse("")); + + return WorkerJobResult.of(exitCode, stdout, stderr); + }, + MoreExecutors.directExecutor()); } @Override public synchronized void close() { LOG.debug("Closing process %d", this.hashCode()); try { + // Notify the reader thread to exit. + synchronized (readerThread) { + shutdownReaderThread = true; + readerThread.notify(); + } if (protocol != null) { protocol.close(); } + readerThread.join(5000); Files.deleteIfExists(stdErr); } catch (Exception e) { LOG.debug(e, "Error closing worker process %s.", processParams.getCommand()); @@ -170,7 +217,85 @@ public synchronized void close() { } @VisibleForTesting - void setProtocol(WorkerProcessProtocol.CommandSender protocolMock) { + void launchForTesting(WorkerProcessProtocol.CommandSender protocolMock) { this.protocol = protocolMock; + handshakePerformed = true; + readerThread.start(); + } + + private void processNextCommandResponse() throws Throwable { + Preconditions.checkState( + protocol != null, + "Tried to submit a job to the worker process before the handshake was performed."); + + Preconditions.checkState( + !Thread.holdsLock(this), + "About to block on input, should not be holding the lock that prevents new jobs"); + WorkerProcessProtocol.CommandResponse commandResponse = protocol.receiveNextCommandResponse(); + SettableFuture result = commandExitCodes.remove(commandResponse.getCommandId()); + Preconditions.checkState( + result != null, + "Received message id %s with no corresponding waiter! (result was %s)", + commandResponse.getCommandId(), + commandResponse.getExitCode()); + + result.set(commandResponse.getExitCode()); + } + + private void readerLoop() { + boolean readerAlive = true; + while (true) { + // A dance here to avoid calling `processNextCommandResponse` if we're not waiting for any + // commands. In the best case scenario, where we don't call `close()` until all commands are + // done, this will avoid painful exceptions in `close()`. + // + // This uses `readerThread` instead of `this` as the lock so `close()` can block out + // `submitJob`, but still wait for this thread to exit. + synchronized (readerThread) { + while (!shutdownReaderThread && commandExitCodes.isEmpty()) { + try { + readerThread.wait(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } + if (shutdownReaderThread) { + break; + } + } + + if (!readerAlive) { + failAllFutures(); + continue; + } + + try { + processNextCommandResponse(); + } catch (Throwable t) { + // The WorkerProcessProtocol isn't really conducive to concurrency, so we just assume + // that shutdowns will cause some kind of exception. In case of a orderly shutdown, we + // wouldn't have any jobs running, so we wouldn't even enter processNextCommandResponse. + // The code here can only happen if e.g. the worker process crashed. + + // TODO(mikekap): Maybe fail the jobs themselves with this exception, but kill the noise + // if 5 jobs fail with the same message. + LOG.error(t, "Worker pool process failed"); + readerAlive = false; + } + } + + failAllFutures(); + } + + private void failAllFutures() { + while (!commandExitCodes.isEmpty()) { + HashSet keys = new HashSet<>(commandExitCodes.keySet()); + for (Integer key : keys) { + SettableFuture result = commandExitCodes.remove(key); + if (result != null) { + result.setException(new RuntimeException("Worker process error")); + } + } + } } } diff --git a/src/com/facebook/buck/worker/WorkerProcessParams.java b/src/com/facebook/buck/worker/WorkerProcessParams.java index b122e3579c1..9b901da341b 100644 --- a/src/com/facebook/buck/worker/WorkerProcessParams.java +++ b/src/com/facebook/buck/worker/WorkerProcessParams.java @@ -44,6 +44,12 @@ public interface WorkerProcessParams { /** Maximum number of tools that pool can have. */ int getMaxWorkers(); + /** + * Whether we use synchronous 1-command-at-a-time processes, or one process with a max number of + * tasks + */ + boolean isAsync(); + /** * Identifies the instance of the persisted worker process pool. Defines when worker process pool * should be invalidated. @@ -58,8 +64,9 @@ static WorkerProcessParams of( ImmutableList startupCommand, ImmutableMap startupEnvironment, int maxWorkers, + boolean isAsync, Optional workerProcessIdentity) { return ImmutableWorkerProcessParams.of( - tempDir, startupCommand, startupEnvironment, maxWorkers, workerProcessIdentity); + tempDir, startupCommand, startupEnvironment, maxWorkers, isAsync, workerProcessIdentity); } } diff --git a/src/com/facebook/buck/worker/WorkerProcessPool.java b/src/com/facebook/buck/worker/WorkerProcessPool.java index 2c87c2f0115..46d1ad00c6f 100644 --- a/src/com/facebook/buck/worker/WorkerProcessPool.java +++ b/src/com/facebook/buck/worker/WorkerProcessPool.java @@ -16,205 +16,19 @@ package com.facebook.buck.worker; -import com.facebook.buck.core.util.log.Logger; -import com.facebook.buck.util.concurrent.LinkedBlockingStack; -import com.facebook.buck.util.function.ThrowingSupplier; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; import com.google.common.hash.HashCode; +import com.google.common.util.concurrent.ListenableFuture; import java.io.Closeable; import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.function.Consumer; -import javax.annotation.Nullable; -import javax.annotation.concurrent.ThreadSafe; -/** - * A pool of {@link WorkerProcess} instances. - * - *

    This pool supports acquisition and release of worker processes on different threads. Workers - * are acquired with {@link #borrowWorkerProcess()}, which returns - */ -@ThreadSafe -public class WorkerProcessPool implements Closeable { - private static final Logger LOG = Logger.get(WorkerProcessPool.class); - - private final int capacity; - private final BlockingQueue availableWorkers; - private final WorkerLifecycle[] workerLifecycles; - private final HashCode poolHash; +public interface WorkerProcessPool extends Closeable { + HashCode getPoolHash(); - public WorkerProcessPool( - int maxWorkers, - HashCode poolHash, - ThrowingSupplier startWorkerProcess) { - capacity = maxWorkers; - availableWorkers = new LinkedBlockingStack<>(); - workerLifecycles = new WorkerLifecycle[maxWorkers]; - this.poolHash = poolHash; + int getCapacity(); - Arrays.setAll( - workerLifecycles, - ignored -> new WorkerLifecycle(startWorkerProcess, availableWorkers::add)); - Collections.addAll(availableWorkers, workerLifecycles); - } - - /** - * If there are available workers, returns one. Otherwise blocks until one becomes available and - * returns it. Borrowed worker processes must be relased by calling {@link - * BorrowedWorkerProcess#close()} after using them. - */ - public BorrowedWorkerProcess borrowWorkerProcess() throws InterruptedException { - return new BorrowedWorkerProcess(availableWorkers.take()); - } - - @VisibleForTesting - Optional borrowWorkerProcess(int timeout, TimeUnit unit) - throws InterruptedException { - return Optional.ofNullable(availableWorkers.poll(timeout, unit)) - .map(BorrowedWorkerProcess::new); - } + ListenableFuture submitJob(String expandedJobArgs) + throws IOException, InterruptedException; @Override - public synchronized void close() { - Throwable caughtWhileClosing = null; - - // remove all available workers - int numAvailableWorkers = availableWorkers.drainTo(new ArrayList<>(capacity)); - for (WorkerLifecycle lifecycle : this.workerLifecycles) { - try { - lifecycle.close(); - } catch (Throwable t) { - caughtWhileClosing = t; - } - } - - Preconditions.checkState( - numAvailableWorkers == capacity, - "WorkerProcessPool was still running when shutdown was called."); - if (caughtWhileClosing != null) { - throw new RuntimeException(caughtWhileClosing); - } - } - - public int getCapacity() { - return capacity; - } - - HashCode getPoolHash() { - return poolHash; - } - - /** - * Represents the lifecycle of one specific worker in a {@link WorkerProcessPool}. - * - *

    Concurrency is controlled by the pool, which supports acquiring and releasing workers with - * {@link WorkerProcessPool#availableWorkers}. - * - *

    {@link #get()} and {@link #close()} are synchronized to allow closing as part of closing the - * pool with a consumer trying to acquire a worker in parallel. - */ - @ThreadSafe - private static class WorkerLifecycle - implements Closeable, ThrowingSupplier { - - private final ThrowingSupplier startWorkerProcess; - private final Consumer onWorkerProcessReturn; - private boolean isClosed = false; - @Nullable private WorkerProcess workerProcess; - - private WorkerLifecycle( - ThrowingSupplier startWorkerProcess, - Consumer onWorkerProcessReturn) { - this.startWorkerProcess = startWorkerProcess; - this.onWorkerProcessReturn = onWorkerProcessReturn; - } - - /** Allows to retrieve the wrapped worker process, starting it up if necessary. */ - @Override - public synchronized WorkerProcess get() throws IOException { - Preconditions.checkState(!isClosed, "Worker was already terminated"); - // If the worker is broken, destroy it - if (workerProcess != null && !workerProcess.isAlive()) { - try { - workerProcess.close(); - } catch (Exception ex) { - LOG.error(ex, "Failed to close dead worker process; ignoring."); - } finally { - workerProcess = null; - } - } - - // start a worker if necessary, this might throw IOException - if (workerProcess == null) { - workerProcess = startWorkerProcess.get(); - } - - return workerProcess; - } - - public void makeAvailable() { - onWorkerProcessReturn.accept(this); - } - - @Override - public synchronized void close() { - isClosed = true; - if (workerProcess != null) { - workerProcess.close(); - workerProcess = null; - } - } - } - - /** - * Represents a {@link WorkerProcess} borrowed from a {@link WorkerProcessPool}. - * - *

    Ownership must be returned to the pool by calling {@link #close()} after finishing to use - * the worker. - * - *

    Since BorrowedWorkerProcess implements Closable, it can be used with a try-with-resources - * statement. - * - *

    BorrowedWorkerProcess is not threadsafe, and is expected to be used by one thread at a time - * only. Concurrency control is handled by {@link WorkerProcessPool} and {@link WorkerLifecycle}. - */ - public static class BorrowedWorkerProcess implements Closeable { - @Nullable private WorkerLifecycle lifecycle; - - private BorrowedWorkerProcess(WorkerLifecycle lifecycle) { - this.lifecycle = Objects.requireNonNull(lifecycle); - } - - /** Returns ownership of the borrowed worker process back to the pool it was retrieved from. */ - @Override - public void close() { - if (lifecycle != null) { - WorkerLifecycle lifecycle = this.lifecycle; - this.lifecycle = null; - lifecycle.makeAvailable(); - } - } - - /** - * Submits a job to the worker, and returns the result. - * - * @throws IOException - */ - public WorkerJobResult submitAndWaitForJob(String expandedJobArgs) throws IOException { - return get().submitAndWaitForJob(expandedJobArgs); - } - - @VisibleForTesting - WorkerProcess get() throws IOException { - Preconditions.checkState(lifecycle != null, "BorrowedWorker has already been closed."); - return lifecycle.get(); - } - } + void close(); } diff --git a/src/com/facebook/buck/worker/WorkerProcessPoolAsync.java b/src/com/facebook/buck/worker/WorkerProcessPoolAsync.java new file mode 100644 index 00000000000..e884ddb0dfb --- /dev/null +++ b/src/com/facebook/buck/worker/WorkerProcessPoolAsync.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.buck.worker; + +import com.facebook.buck.util.function.ThrowingSupplier; +import com.google.common.hash.HashCode; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import java.io.IOException; +import java.util.concurrent.Semaphore; +import javax.annotation.Nullable; + +public class WorkerProcessPoolAsync implements WorkerProcessPool { + + private final HashCode poolHash; + private final int maxRequests; + private final ThrowingSupplier startWorkerProcess; + private final Semaphore concurrencyLimiter; + @Nullable private WorkerProcess workerProcess; + + public WorkerProcessPoolAsync( + int maxRequests, + HashCode poolHash, + ThrowingSupplier startWorkerProcess) { + this.poolHash = poolHash; + this.maxRequests = maxRequests; + this.startWorkerProcess = startWorkerProcess; + this.concurrencyLimiter = new Semaphore(maxRequests <= 0 ? Integer.MAX_VALUE : maxRequests); + } + + @Override + public HashCode getPoolHash() { + return poolHash; + } + + @Override + public int getCapacity() { + return maxRequests; + } + + @Override + public ListenableFuture submitJob(String expandedJobArgs) + throws IOException, InterruptedException { + synchronized (this) { + if (workerProcess == null || !workerProcess.isAlive()) { + workerProcess = startWorkerProcess.get(); + } + } + + concurrencyLimiter.acquire(); + try { + ListenableFuture result = workerProcess.submitJob(expandedJobArgs); + result.addListener(concurrencyLimiter::release, MoreExecutors.directExecutor()); + return result; + } catch (Throwable t) { + concurrencyLimiter.release(); + throw t; + } + } + + @Override + public void close() { + synchronized (this) { + if (workerProcess != null) { + workerProcess.close(); + } + } + } +} diff --git a/src/com/facebook/buck/worker/WorkerProcessPoolFactory.java b/src/com/facebook/buck/worker/WorkerProcessPoolFactory.java index 5c7a987549d..f9944039fe8 100644 --- a/src/com/facebook/buck/worker/WorkerProcessPoolFactory.java +++ b/src/com/facebook/buck/worker/WorkerProcessPoolFactory.java @@ -22,6 +22,7 @@ import com.facebook.buck.util.Escaper; import com.facebook.buck.util.ProcessExecutorParams; import com.facebook.buck.util.environment.Platform; +import com.facebook.buck.util.function.ThrowingSupplier; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; @@ -69,7 +70,7 @@ public WorkerProcessPool getWorkerProcessPool( } else { processPoolMap = context.getWorkerProcessPools(); key = Joiner.on(' ').join(getCommand(context.getPlatform(), paramsToUse)); - workerHash = Hashing.sha1().hashString(key, StandardCharsets.UTF_8); + workerHash = Hashing.sha256().hashString(key, StandardCharsets.UTF_8); } // If the worker pool has a different hash, recreate the pool. @@ -90,10 +91,17 @@ public WorkerProcessPool getWorkerProcessPool( context.postEvent( ConsoleEvent.warning( "There are two 'worker_tool' targets declared with the same command (%s), but " - + "different 'max_worker' settings (%d and %d). Only the first capacity is applied. " + + "different 'max_worker' settings (%d and %d). Only the former capacity is applied. " + "Consolidate these workers to avoid this warning.", key, poolCapacity, paramsToUse.getMaxWorkers())); } + if ((pool instanceof WorkerProcessPoolAsync) != paramsToUse.isAsync()) { + context.postEvent( + ConsoleEvent.warning( + "There are two 'worker_tool' targets declared with the same command (%s), but " + + "different 'solo_async' settings. Consolidate these workers to avoid this warning.", + key)); + } return pool; } @@ -113,18 +121,24 @@ private WorkerProcessPool createWorkerProcessPool( Path workerTmpDir = paramsToUse.getTempDir(); AtomicInteger workerNumber = new AtomicInteger(0); - - WorkerProcessPool newPool = - new WorkerProcessPool( - paramsToUse.getMaxWorkers(), - workerHash, - () -> { - Path tmpDir = workerTmpDir.resolve(Integer.toString(workerNumber.getAndIncrement())); - filesystem.mkdirs(tmpDir); - WorkerProcess process = createWorkerProcess(processParams, context, tmpDir); - process.ensureLaunchAndHandshake(); - return process; - }); + ThrowingSupplier startWorkerProcess = + () -> { + Path tmpDir = workerTmpDir.resolve(Integer.toString(workerNumber.getAndIncrement())); + filesystem.mkdirs(tmpDir); + WorkerProcess process = + WorkerProcessPoolFactory.this.createWorkerProcess(processParams, context, tmpDir); + process.ensureLaunchAndHandshake(); + return process; + }; + + WorkerProcessPool newPool; + if (paramsToUse.isAsync()) { + newPool = + new WorkerProcessPoolAsync(paramsToUse.getMaxWorkers(), workerHash, startWorkerProcess); + } else { + newPool = + new WorkerProcessPoolSync(paramsToUse.getMaxWorkers(), workerHash, startWorkerProcess); + } WorkerProcessPool previousPool = processPoolMap.putIfAbsent(key, newPool); // If putIfAbsent does not return null, then that means another thread beat this thread // into putting an WorkerProcessPool in the map for this key. If that's the case, then we diff --git a/src/com/facebook/buck/worker/WorkerProcessPoolSync.java b/src/com/facebook/buck/worker/WorkerProcessPoolSync.java new file mode 100644 index 00000000000..f745e8fcc3e --- /dev/null +++ b/src/com/facebook/buck/worker/WorkerProcessPoolSync.java @@ -0,0 +1,244 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.buck.worker; + +import com.facebook.buck.core.util.log.Logger; +import com.facebook.buck.util.concurrent.LinkedBlockingStack; +import com.facebook.buck.util.function.ThrowingSupplier; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Throwables; +import com.google.common.hash.HashCode; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.SettableFuture; +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; +import javax.annotation.Nullable; +import javax.annotation.concurrent.ThreadSafe; + +/** A synchronous pool of {@link WorkerProcess} instances. */ +@ThreadSafe +public class WorkerProcessPoolSync implements Closeable, WorkerProcessPool { + private static final Logger LOG = Logger.get(WorkerProcessPoolSync.class); + + private final int capacity; + private final BlockingQueue availableWorkers; + private final WorkerLifecycle[] workerLifecycles; + private final HashCode poolHash; + + public WorkerProcessPoolSync( + int maxWorkers, + HashCode poolHash, + ThrowingSupplier startWorkerProcess) { + capacity = maxWorkers; + availableWorkers = new LinkedBlockingStack<>(); + workerLifecycles = new WorkerLifecycle[maxWorkers]; + this.poolHash = poolHash; + + Arrays.setAll( + workerLifecycles, + ignored -> new WorkerLifecycle(startWorkerProcess, availableWorkers::add)); + Collections.addAll(availableWorkers, workerLifecycles); + } + + @Override + public ListenableFuture submitJob(String expandedJobArgs) + throws InterruptedException { + BorrowedWorkerProcess process = borrowWorkerProcess(); + try { + ListenableFuture result = process.submitJob(expandedJobArgs); + result.addListener(process::close, MoreExecutors.directExecutor()); + process = null; + return result; + } catch (Throwable t) { + Throwables.throwIfUnchecked(t); + + SettableFuture result = SettableFuture.create(); + result.setException(t); + return result; + } finally { + if (process != null) { + process.close(); + } + } + } + + /** + * If there are available workers, returns one. Otherwise blocks until one becomes available and + * returns it. Borrowed worker processes must be relased by calling {@link + * BorrowedWorkerProcess#close()} after using them. + */ + public BorrowedWorkerProcess borrowWorkerProcess() throws InterruptedException { + return new BorrowedWorkerProcess(availableWorkers.take()); + } + + @VisibleForTesting + Optional borrowWorkerProcess(int timeout, TimeUnit unit) + throws InterruptedException { + return Optional.ofNullable(availableWorkers.poll(timeout, unit)) + .map(BorrowedWorkerProcess::new); + } + + @Override + public synchronized void close() { + Throwable caughtWhileClosing = null; + + // remove all available workers + int numAvailableWorkers = availableWorkers.drainTo(new ArrayList<>(capacity)); + for (WorkerLifecycle lifecycle : this.workerLifecycles) { + try { + lifecycle.close(); + } catch (Throwable t) { + caughtWhileClosing = t; + } + } + + Preconditions.checkState( + numAvailableWorkers == capacity, + "WorkerProcessPool was still running when shutdown was called."); + if (caughtWhileClosing != null) { + throw new RuntimeException(caughtWhileClosing); + } + } + + @Override + public int getCapacity() { + return capacity; + } + + @Override + public HashCode getPoolHash() { + return poolHash; + } + + /** + * Represents the lifecycle of one specific worker in a {@link WorkerProcessPoolSync}. + * + *

    Concurrency is controlled by the pool, which supports acquiring and releasing workers with + * {@link WorkerProcessPoolSync#availableWorkers}. + * + *

    {@link #get()} and {@link #close()} are synchronized to allow closing as part of closing the + * pool with a consumer trying to acquire a worker in parallel. + */ + @ThreadSafe + private static class WorkerLifecycle + implements Closeable, ThrowingSupplier { + + private final ThrowingSupplier startWorkerProcess; + private final Consumer onWorkerProcessReturn; + private boolean isClosed = false; + @Nullable private WorkerProcess workerProcess; + + private WorkerLifecycle( + ThrowingSupplier startWorkerProcess, + Consumer onWorkerProcessReturn) { + this.startWorkerProcess = startWorkerProcess; + this.onWorkerProcessReturn = onWorkerProcessReturn; + } + + /** Allows to retrieve the wrapped worker process, starting it up if necessary. */ + @Override + public synchronized WorkerProcess get() throws IOException { + Preconditions.checkState(!isClosed, "Worker was already terminated"); + // If the worker is broken, destroy it + if (workerProcess != null && !workerProcess.isAlive()) { + try { + workerProcess.close(); + } catch (Exception ex) { + LOG.error(ex, "Failed to close dead worker process; ignoring."); + } finally { + workerProcess = null; + } + } + + // start a worker if necessary, this might throw IOException + if (workerProcess == null) { + workerProcess = startWorkerProcess.get(); + } + + return workerProcess; + } + + public void makeAvailable() { + onWorkerProcessReturn.accept(this); + } + + @Override + public synchronized void close() { + isClosed = true; + if (workerProcess != null) { + workerProcess.close(); + workerProcess = null; + } + } + } + + /** + * Represents a {@link WorkerProcess} borrowed from a {@link WorkerProcessPoolSync}. + * + *

    Ownership must be returned to the pool by calling {@link #close()} after finishing to use + * the worker. + * + *

    Since BorrowedWorkerProcess implements Closable, it can be used with a try-with-resources + * statement. + * + *

    BorrowedWorkerProcess is not threadsafe, and is expected to be used by one thread at a time + * only. Concurrency control is handled by {@link WorkerProcessPoolSync} and {@link + * WorkerLifecycle}. + */ + public static class BorrowedWorkerProcess implements Closeable { + @Nullable private WorkerLifecycle lifecycle; + + private BorrowedWorkerProcess(WorkerLifecycle lifecycle) { + this.lifecycle = Objects.requireNonNull(lifecycle); + } + + /** Returns ownership of the borrowed worker process back to the pool it was retrieved from. */ + @Override + public void close() { + if (lifecycle != null) { + WorkerLifecycle lifecycle = this.lifecycle; + this.lifecycle = null; + lifecycle.makeAvailable(); + } + } + + /** + * Submits a job to the worker, and returns the result. + * + * @throws IOException + */ + public ListenableFuture submitJob(String expandedJobArgs) throws IOException { + return get().submitJob(expandedJobArgs); + } + + @VisibleForTesting + WorkerProcess get() throws IOException { + Preconditions.checkState(lifecycle != null, "BorrowedWorker has already been closed."); + return lifecycle.get(); + } + } +} diff --git a/src/com/facebook/buck/worker/WorkerProcessProtocol.java b/src/com/facebook/buck/worker/WorkerProcessProtocol.java index 4b9bf2c3608..7cef68bd7b7 100644 --- a/src/com/facebook/buck/worker/WorkerProcessProtocol.java +++ b/src/com/facebook/buck/worker/WorkerProcessProtocol.java @@ -20,13 +20,30 @@ import java.io.IOException; public interface WorkerProcessProtocol { + class CommandResponse { + private final int commandId; + private final int exitCode; + + public CommandResponse(int commandId, int exitCode) { + this.commandId = commandId; + this.exitCode = exitCode; + } + + public int getExitCode() { + return exitCode; + } + + public int getCommandId() { + return commandId; + } + } interface CommandSender extends Closeable { void handshake(int messageId) throws IOException; void send(int messageId, WorkerProcessCommand command) throws IOException; - int receiveCommandResponse(int messageID) throws IOException; + CommandResponse receiveNextCommandResponse() throws IOException; /** Instructs the CommandReceiver to shut itself down. */ @Override diff --git a/src/com/facebook/buck/worker/WorkerProcessProtocolZero.java b/src/com/facebook/buck/worker/WorkerProcessProtocolZero.java index 12ce99eff97..02b2efb90b3 100644 --- a/src/com/facebook/buck/worker/WorkerProcessProtocolZero.java +++ b/src/com/facebook/buck/worker/WorkerProcessProtocolZero.java @@ -129,7 +129,7 @@ public void send(int messageId, WorkerProcessCommand command) throws IOException } */ @Override - public int receiveCommandResponse(int messageID) throws IOException { + public WorkerProcessProtocol.CommandResponse receiveNextCommandResponse() throws IOException { int id = -1; int exitCode = -1; String type = ""; @@ -157,12 +157,6 @@ public int receiveCommandResponse(int messageID) throws IOException { getStdErrorOutput(stdErr)); } - if (id != messageID) { - throw new HumanReadableException( - String.format( - "Expected response's \"id\" value to be " + "\"%d\", got \"%d\" instead.", - messageID, id)); - } if (!type.equals(TYPE_RESULT) && !type.equals(TYPE_ERROR)) { throw new HumanReadableException( String.format( @@ -170,7 +164,7 @@ public int receiveCommandResponse(int messageID) throws IOException { + "to be one of [\"%s\",\"%s\"], got \"%s\" instead.", TYPE_RESULT, TYPE_ERROR, type)); } - return exitCode; + return new WorkerProcessProtocol.CommandResponse(id, exitCode); } @Override diff --git a/test/com/facebook/buck/android/AndroidAppBundleIntegrationTest.java b/test/com/facebook/buck/android/AndroidAppBundleIntegrationTest.java index 6046d1e0a1e..537f6d48e62 100644 --- a/test/com/facebook/buck/android/AndroidAppBundleIntegrationTest.java +++ b/test/com/facebook/buck/android/AndroidAppBundleIntegrationTest.java @@ -38,11 +38,14 @@ import com.facebook.buck.testutil.integration.ProjectWorkspace; import com.facebook.buck.testutil.integration.TestDataHelper; import com.facebook.buck.testutil.integration.ZipInspector; +import com.facebook.buck.util.MoreStringsForTests; import com.facebook.buck.util.zip.ZipConstants; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Date; +import java.util.regex.Pattern; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; import org.apache.commons.compress.archivers.zip.ZipUtil; @@ -62,6 +65,8 @@ public void setUp() throws IOException { workspace = TestDataHelper.createProjectWorkspaceForScenario(this, "android_project", tmpFolder); workspace.setUp(); + workspace.addTemplateToWorkspace(Paths.get("test/com/facebook/buck/toolchains/kotlin")); + setWorkspaceCompilationMode(workspace); AssumeAndroidPlatform.get(workspace).assumeSdkIsAvailable(); AssumeAndroidPlatform.get(workspace).assumeNdkIsAvailable(); @@ -181,4 +186,72 @@ public void testAppBundleWithMultipleModules() throws IOException { zipInspector.assertFileExists("small_with_no_resource_deps/manifest/AndroidManifest.xml"); zipInspector.assertFileExists("small_with_no_resource_deps/resources.pb"); } + + @Test + public void testAppBundleWithDynamicFeatures() throws IOException { + String target = "//apps/dynamic_features:app_dynamic_features"; + ProcessResult result = workspace.runBuckCommand("build", target); + result.assertSuccess(); + + Path aab = + workspace.getPath( + BuildTargetPaths.getGenPath( + filesystem, BuildTargetFactory.newInstance(target), "%s.signed.aab")); + + ZipInspector zipInspector = new ZipInspector(aab); + + // Verify dex are built for specific module + zipInspector.assertFileExists("base/dex/classes.dex"); + zipInspector.assertFileExists("native/dex/classes.dex"); + zipInspector.assertFileExists("java/dex/classes.dex"); + zipInspector.assertFileExists("kotlin/dex/classes.dex"); + zipInspector.assertFileExists("initialInstall/dex/classes.dex"); + + // Verify native libs are present only in native module + zipInspector.assertFileExists("native/lib/x86/libprebuilt.so"); + zipInspector.assertFileDoesNotExist("base/lib/x86/libprebuilt.so"); + + // Verify only x86 libs are present in the generated bundle file + zipInspector.assertFileExists("native/lib/x86/libprebuilt.so"); + zipInspector.assertFileDoesNotExist("base/lib/arm64-v8a/libprebuilt.so"); + + // Verify manifest properties and delivery modes of respective modules + zipInspector.assertFileExists("base/manifest/AndroidManifest.xml"); + zipInspector.assertFileExists("native/manifest/AndroidManifest.xml"); + zipInspector.assertFileExists("java/manifest/AndroidManifest.xml"); + zipInspector.assertFileExists("kotlin/manifest/AndroidManifest.xml"); + zipInspector.assertFileExists("initialInstall/manifest/AndroidManifest.xml"); + + String nativeManifestContent = new String( + zipInspector.getFileContents("native/manifest/AndroidManifest.xml")); + assertTrue(Pattern.compile(".*(split[\\u0000-\\u007F]+native).*").matcher( + MoreStringsForTests.containsIgnoringPlatformNewlines(nativeManifestContent).toString()).matches()); + assertTrue(nativeManifestContent.contains("on-demand")); + + String javaManifestContent = new String( + zipInspector.getFileContents("java/manifest/AndroidManifest.xml")); + assertTrue(Pattern.compile(".*(split[\\u0000-\\u007F]+java).*").matcher( + MoreStringsForTests.containsIgnoringPlatformNewlines(javaManifestContent).toString()).matches()); + assertTrue(javaManifestContent.contains("install-time")); + assertTrue(javaManifestContent.contains("conditions")); + + String kotlinManifestContent = new String( + zipInspector.getFileContents("kotlin/manifest/AndroidManifest.xml")); + assertTrue(Pattern.compile(".*(split[\\u0000-\\u007F]+kotlin).*").matcher( + MoreStringsForTests.containsIgnoringPlatformNewlines(kotlinManifestContent).toString()).matches()); + assertTrue(kotlinManifestContent.contains("on-demand")); + + String initialInstallManifestContent = new String( + zipInspector.getFileContents("initialInstall/manifest/AndroidManifest.xml")); + assertTrue(Pattern.compile(".*(split[\\u0000-\\u007F]+initialInstall).*").matcher( + MoreStringsForTests.containsIgnoringPlatformNewlines(initialInstallManifestContent).toString()).matches()); + assertTrue(initialInstallManifestContent.contains("install-time")); + + // Verify base manifest should include details of all feature Manifest files + String baseManifestContent = new String(zipInspector.getFileContents("base/manifest/AndroidManifest.xml")); + assertTrue(baseManifestContent.contains("OnDemandNativeFeatureActivity")); + assertTrue(baseManifestContent.contains("ConditionalJavaFeatureActivity")); + assertTrue(baseManifestContent.contains("OnDemandKotlinFeatureActivity")); + assertTrue(baseManifestContent.contains("AtInstallFeatureActivity")); + } } diff --git a/test/com/facebook/buck/android/AndroidLibraryIntegrationTest.java b/test/com/facebook/buck/android/AndroidLibraryIntegrationTest.java index 7410b52699f..dd04dfb3f1d 100644 --- a/test/com/facebook/buck/android/AndroidLibraryIntegrationTest.java +++ b/test/com/facebook/buck/android/AndroidLibraryIntegrationTest.java @@ -78,7 +78,7 @@ public void testAndroidKotlinLibraryExtraArgumentsCompilation() throws Exception AssumeAndroidPlatform.get(workspace).assumeSdkIsAvailable(); KotlinTestAssumptions.assumeCompilerAvailable(workspace.asCell().getBuckConfig()); ProcessResult result = - workspace.runBuckBuild("//kotlin/com/sample/lib:lib_extra_kotlinc_arguments"); + workspace.runBuckBuild("//kotlin/com/sample/lib:lib_free_compiler_args"); result.assertSuccess(); } diff --git a/test/com/facebook/buck/android/AndroidPrebuiltAarIntegrationTest.java b/test/com/facebook/buck/android/AndroidPrebuiltAarIntegrationTest.java index dffab334bc1..242ba970ec5 100644 --- a/test/com/facebook/buck/android/AndroidPrebuiltAarIntegrationTest.java +++ b/test/com/facebook/buck/android/AndroidPrebuiltAarIntegrationTest.java @@ -64,7 +64,7 @@ public void thatAndroidToolchainIsNotRequired() throws IOException { .runBuckCommandWithEnvironmentOverridesAndContext( tmp.getRoot(), Optional.empty(), - ImmutableMap.of("ANDROID_SDK", badSdkPath, "ANDROID_HOME", badSdkPath), + ImmutableMap.of("ANDROID_SDK", badSdkPath, "ANDROID_HOME", badSdkPath, "ANDROID_SDK_ROOT", badSdkPath), "targets", "--show-rulekey", "//:aar") diff --git a/test/com/facebook/buck/android/BUCK b/test/com/facebook/buck/android/BUCK index ee3134fc9f3..9be0f43d34a 100644 --- a/test/com/facebook/buck/android/BUCK +++ b/test/com/facebook/buck/android/BUCK @@ -743,6 +743,7 @@ java_test( "//test/com/facebook/buck/step:testutil", "//test/com/facebook/buck/testutil:testutil", "//test/com/facebook/buck/testutil/integration:util", + "//test/com/facebook/buck/util:testutil", "//third-party/java/bundletool:bundletool", "//third-party/java/commons-compress:commons-compress", "//third-party/java/guava:guava", diff --git a/test/com/facebook/buck/android/GenerateManifestStepTest.java b/test/com/facebook/buck/android/GenerateManifestStepTest.java index 6661ae691f6..f9caade17dc 100644 --- a/test/com/facebook/buck/android/GenerateManifestStepTest.java +++ b/test/com/facebook/buck/android/GenerateManifestStepTest.java @@ -75,7 +75,7 @@ public void testManifestGeneration() throws Exception { new GenerateManifestStep( filesystem, skeletonPath, - APKModule.of(APKModuleGraph.ROOT_APKMODULE_NAME, true), + APKModule.of(APKModuleGraph.ROOT_APKMODULE_NAME, true, true), libraryManifestFiles, outputPath, mergeReportPath); @@ -119,7 +119,7 @@ public void testManifestGenerationWithModule() throws Exception { new GenerateManifestStep( filesystem, skeletonPath, - APKModule.of("MODULE_NAME", false), + APKModule.of("MODULE_NAME", false, false), libraryManifestFiles, outputPath, mergeReportPath); diff --git a/test/com/facebook/buck/android/PreDexedFilesSorterTest.java b/test/com/facebook/buck/android/PreDexedFilesSorterTest.java index ab998ab6f6a..124d667dfa7 100644 --- a/test/com/facebook/buck/android/PreDexedFilesSorterTest.java +++ b/test/com/facebook/buck/android/PreDexedFilesSorterTest.java @@ -62,7 +62,7 @@ public void setUp() { TargetGraph.EMPTY, BuildTargetFactory.newInstance("//fakeTarget:yes"), Optional.empty()); - extraModule = APKModule.of("extra", false); + extraModule = APKModule.of("extra", false, false); } @Test diff --git a/test/com/facebook/buck/android/SplitZipStepTest.java b/test/com/facebook/buck/android/SplitZipStepTest.java index b569cc7a111..a2b26354836 100644 --- a/test/com/facebook/buck/android/SplitZipStepTest.java +++ b/test/com/facebook/buck/android/SplitZipStepTest.java @@ -75,7 +75,7 @@ public void testMetaList() throws IOException { ImmutableSet requires = ImmutableSet.of(); SplitZipStep.writeMetaList( writer, - APKModule.of(SplitZipStep.SECONDARY_DEX_ID, false), + APKModule.of(SplitZipStep.SECONDARY_DEX_ID, false, false), requires, ImmutableList.of(outJar), DexStore.JAR); @@ -112,9 +112,9 @@ public void testMetaListApkModuule() throws IOException { StringWriter stringWriter = new StringWriter(); try (BufferedWriter writer = new BufferedWriter(stringWriter)) { - ImmutableSet requires = ImmutableSet.of(APKModule.of("dependency", false)); + ImmutableSet requires = ImmutableSet.of(APKModule.of("dependency", false, false)); SplitZipStep.writeMetaList( - writer, APKModule.of("module", false), requires, ImmutableList.of(outJar), DexStore.JAR); + writer, APKModule.of("module", false, false), requires, ImmutableList.of(outJar), DexStore.JAR); } List lines = CharStreams.readLines(new StringReader(stringWriter.toString())); assertEquals(3, lines.size()); diff --git a/test/com/facebook/buck/android/apkmodule/APKModuleTest.java b/test/com/facebook/buck/android/apkmodule/APKModuleTest.java index 1641a6f5f0c..26f4f5a1a49 100644 --- a/test/com/facebook/buck/android/apkmodule/APKModuleTest.java +++ b/test/com/facebook/buck/android/apkmodule/APKModuleTest.java @@ -363,6 +363,7 @@ public void testAPKModuleGraphWithDeclaredDependency() { Optional.of(appModuleDependencies.build()), Optional.empty(), ImmutableSet.of(), + ImmutableSet.of(), graph, androidBinaryTarget); @@ -503,6 +504,7 @@ public void testAPKModuleGraphSharedWithDeclaredDependency() { Optional.of(appModuleDependencies.build()), Optional.empty(), ImmutableSet.of(), + ImmutableSet.of(), graph, androidBinaryTarget); @@ -654,6 +656,7 @@ public void testAPKModuleGraphWithMultiLevelDependencies() { Optional.of(appModuleDependencies.build()), Optional.empty(), ImmutableSet.of(), + ImmutableSet.of(), graph, androidBinaryTarget); @@ -810,6 +813,7 @@ public void testAPKModuleGraphThatLowestDeclaredDepTakesCareOfMultipleLevelsOfIn Optional.of(appModuleDependencies.build()), Optional.empty(), ImmutableSet.of(), + ImmutableSet.of(), graph, androidBinaryTarget); diff --git a/test/com/facebook/buck/android/packageable/AndroidPackageableCollectorTest.java b/test/com/facebook/buck/android/packageable/AndroidPackageableCollectorTest.java index aff5773a976..231c45698b9 100644 --- a/test/com/facebook/buck/android/packageable/AndroidPackageableCollectorTest.java +++ b/test/com/facebook/buck/android/packageable/AndroidPackageableCollectorTest.java @@ -316,7 +316,7 @@ public void testGetAndroidResourceDeps() { .map(BuildRule::getBuildTarget) .collect(ImmutableList.toImmutableList()); - APKModule rootModule = APKModule.of(APKModuleGraph.ROOT_APKMODULE_NAME, true); + APKModule rootModule = APKModule.of(APKModuleGraph.ROOT_APKMODULE_NAME, true, true); assertEquals( "Android resources should be topologically sorted.", @@ -401,7 +401,7 @@ public void testGetAndroidResourceDepsWithDuplicateResourcePaths() { AndroidPackageableCollection.ResourceDetails resourceDetails = androidPackageableCollection .getResourceDetails() - .get(APKModule.of(APKModuleGraph.ROOT_APKMODULE_NAME, true)); + .get(APKModule.of(APKModuleGraph.ROOT_APKMODULE_NAME, true, true)); assertThat( resourceDetails.getResourceDirectories(), Matchers.contains(resAPath, resPath, resBPath)); } diff --git a/test/com/facebook/buck/android/resources/ExoResourcesRewriterTest.java b/test/com/facebook/buck/android/resources/ExoResourcesRewriterTest.java index f4d0d8b6498..3f4f1c17f32 100644 --- a/test/com/facebook/buck/android/resources/ExoResourcesRewriterTest.java +++ b/test/com/facebook/buck/android/resources/ExoResourcesRewriterTest.java @@ -85,7 +85,7 @@ public void testRewriteResources() throws IOException { primaryResourceTable.dump(new PrintStream(baos)); String content = new String(baos.toByteArray(), Charsets.UTF_8); Path expectedPath = filesystem.resolve(filesystem.getPath(APK_NAME + ".resources.primary")); - String expected = filesystem.readFileIfItExists(expectedPath).get(); + String expected = filesystem.readFileIfItExists(expectedPath).get().replaceAll("\r\n", "\n"); assertEquals(expected, content); @@ -96,7 +96,7 @@ public void testRewriteResources() throws IOException { exoResourceTable.dump(new PrintStream(baos)); content = new String(baos.toByteArray(), Charsets.UTF_8); expectedPath = filesystem.resolve(filesystem.getPath(APK_NAME + ".resources.exo")); - expected = filesystem.readFileIfItExists(expectedPath).get(); + expected = filesystem.readFileIfItExists(expectedPath).get().replaceAll("\r\n", "\n"); assertEquals(expected, content); } diff --git a/test/com/facebook/buck/android/resources/ResourceTableTest.java b/test/com/facebook/buck/android/resources/ResourceTableTest.java index 233fb1f881b..d35a589456f 100644 --- a/test/com/facebook/buck/android/resources/ResourceTableTest.java +++ b/test/com/facebook/buck/android/resources/ResourceTableTest.java @@ -208,7 +208,7 @@ public void testSliceResourceTable() throws Exception { } resourceTable = ResourceTable.slice(resourceTable, counts); Path resourcesOutput = filesystem.resolve(filesystem.getPath(APK_NAME + ".resources.sliced")); - String expected = filesystem.readFileIfItExists(resourcesOutput).get(); + String expected = filesystem.readFileIfItExists(resourcesOutput).get().replaceAll("\r\n", "\n"); ByteArrayOutputStream baos = new ByteArrayOutputStream(); resourceTable.dump(new PrintStream(baos)); diff --git a/test/com/facebook/buck/android/resources/ResourcesXmlTest.java b/test/com/facebook/buck/android/resources/ResourcesXmlTest.java index ee767d3e880..8fcd99b324d 100644 --- a/test/com/facebook/buck/android/resources/ResourcesXmlTest.java +++ b/test/com/facebook/buck/android/resources/ResourcesXmlTest.java @@ -90,7 +90,7 @@ public void testAaptDumpXmlTree() throws Exception { String content = new String(baos.toByteArray(), Charsets.UTF_8); Path xmltreeOutput = filesystem.resolve(filesystem.getPath(APK_NAME + ".manifest")); - String expected = new String(Files.readAllBytes(xmltreeOutput)); + String expected = new String(Files.readAllBytes(xmltreeOutput)).replaceAll("\r\n", "\n"); MoreAsserts.assertLargeStringsEqual(expected, content); } } @@ -148,7 +148,7 @@ public void testAaptDumpReversedXmlTree() throws Exception { String content = new String(baos.toByteArray(), Charsets.UTF_8); Path xmltreeOutput = filesystem.resolve(filesystem.getPath(APK_NAME + ".manifest.reversed")); - String expected = new String(Files.readAllBytes(xmltreeOutput)); + String expected = new String(Files.readAllBytes(xmltreeOutput)).replaceAll("\r\n", "\n"); MoreAsserts.assertLargeStringsEqual(expected, content); } } diff --git a/test/com/facebook/buck/android/resources/StringPoolTest.java b/test/com/facebook/buck/android/resources/StringPoolTest.java index 75ca7ee0a6c..582c95deace 100644 --- a/test/com/facebook/buck/android/resources/StringPoolTest.java +++ b/test/com/facebook/buck/android/resources/StringPoolTest.java @@ -143,7 +143,7 @@ public void testAaptDumpStrings() throws Exception { String content = new String(baos.toByteArray(), Charsets.UTF_8); Path stringsOutput = filesystem.resolve(filesystem.getPath(APK_NAME + ".strings")); - String expected = new String(Files.readAllBytes(stringsOutput)); + String expected = new String(Files.readAllBytes(stringsOutput)).replaceAll("\r\n", "\n"); MoreAsserts.assertLargeStringsEqual(expected, content); } diff --git a/test/com/facebook/buck/android/testdata/android_project/apps/dynamic_features/AndroidManifest.xml b/test/com/facebook/buck/android/testdata/android_project/apps/dynamic_features/AndroidManifest.xml new file mode 100644 index 00000000000..8a5fa33694b --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/apps/dynamic_features/AndroidManifest.xml @@ -0,0 +1,14 @@ + + + + + + + + + diff --git a/test/com/facebook/buck/android/testdata/android_project/apps/dynamic_features/BUCK.fixture b/test/com/facebook/buck/android/testdata/android_project/apps/dynamic_features/BUCK.fixture new file mode 100644 index 00000000000..39c05f42352 --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/apps/dynamic_features/BUCK.fixture @@ -0,0 +1,38 @@ +android_bundle( + name = "app_dynamic_features", + # Configurations needed for dynamic features: Start + application_module_configs = { + "kotlin": ["//kotlin/com/sample/dynamic_features/on_demand:src_release"], + "java": ["//java/com/sample/dynamic_features/conditional:src_release"], + "native": ["//native/dynamic_features/on_demand:src_release"], + "initialInstall": ["//java/com/sample/dynamic_features/at_install:src_release"], + }, + application_modules_with_manifest = { + "kotlin", + "java", + "native", + "initialInstall" + }, + module_manifest_skeleton = "ModuleManifest.xml", + use_dynamic_feature = True, + use_split_dex = True, + # Configurations needed for dynamic features: End + bundle_config_file = "bundle-config.json", + cpu_filters = [ + "x86", + ], + keystore = "//keystores:debug", + manifest_skeleton = "AndroidManifest.xml", + package_type = "debug", + primary_dex_patterns = [ + "/MyApplication^", + ], + deps = [ + "//java/com/sample/app:app", + "//res/com/sample/dynamic_features/base:base", + "//kotlin/com/sample/dynamic_features/on_demand:src_release", + "//java/com/sample/dynamic_features/conditional:src_release", + "//native/dynamic_features/on_demand:src_release", + "//java/com/sample/dynamic_features/at_install:src_release", + ], +) diff --git a/test/com/facebook/buck/android/testdata/android_project/apps/dynamic_features/ModuleManifest.xml b/test/com/facebook/buck/android/testdata/android_project/apps/dynamic_features/ModuleManifest.xml new file mode 100644 index 00000000000..d9f0d614ccc --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/apps/dynamic_features/ModuleManifest.xml @@ -0,0 +1,5 @@ + + diff --git a/test/com/facebook/buck/android/testdata/android_project/apps/dynamic_features/bundle-config.json b/test/com/facebook/buck/android/testdata/android_project/apps/dynamic_features/bundle-config.json new file mode 100644 index 00000000000..93375b3391a --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/apps/dynamic_features/bundle-config.json @@ -0,0 +1,21 @@ +{ + "compression": {}, + "optimizations": { + "splits_config": { + "split_dimension": [ + { + "value": "ABI", + "negate": false + }, + { + "value": "SCREEN_DENSITY", + "negate": true + }, + { + "value": "LANGUAGE", + "negate": true + } + ] + } + } +} diff --git a/test/com/facebook/buck/android/testdata/android_project/java/com/resourceref/generator.py b/test/com/facebook/buck/android/testdata/android_project/java/com/resourceref/generator.py index 954bfa33609..8ee3f794f3e 100644 --- a/test/com/facebook/buck/android/testdata/android_project/java/com/resourceref/generator.py +++ b/test/com/facebook/buck/android/testdata/android_project/java/com/resourceref/generator.py @@ -6,17 +6,17 @@ def main(argv): RES_COUNT = 3000 if argv[1] == "res": - print """\ + print ("""\ -""" +""") for c in range(RES_COUNT): - print ' #000' % c - print ' hi' % c - print ' ' % c - print """\ + print (' #000' % c) + print (' hi' % c) + print (' ' % c) + print ("""\ -""" +""") if __name__ == "__main__": diff --git a/test/com/facebook/buck/android/testdata/android_project/java/com/sample/dynamic_features/at_install/AndroidManifest.xml b/test/com/facebook/buck/android/testdata/android_project/java/com/sample/dynamic_features/at_install/AndroidManifest.xml new file mode 100644 index 00000000000..534503578f8 --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/java/com/sample/dynamic_features/at_install/AndroidManifest.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + diff --git a/test/com/facebook/buck/android/testdata/android_project/java/com/sample/dynamic_features/at_install/AtInstallFeatureActivity.java b/test/com/facebook/buck/android/testdata/android_project/java/com/sample/dynamic_features/at_install/AtInstallFeatureActivity.java new file mode 100644 index 00000000000..f85d9382c1b --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/java/com/sample/dynamic_features/at_install/AtInstallFeatureActivity.java @@ -0,0 +1,34 @@ +/* + * Copyright 2014-present Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.sample.dynamic_features.at_install; + +import android.app.Activity; +import android.os.Bundle; +import android.widget.TextView; +import com.sample.initialInstall.R; + +public class AtInstallFeatureActivity extends Activity { + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.layout_initial_install); + + TextView tv = (TextView) findViewById(R.id.initial_install_text_view); + tv.setText(R.string.initial_install_text); + } +} diff --git a/test/com/facebook/buck/android/testdata/android_project/java/com/sample/dynamic_features/at_install/BUCK.fixture b/test/com/facebook/buck/android/testdata/android_project/java/com/sample/dynamic_features/at_install/BUCK.fixture new file mode 100644 index 00000000000..5fecc31598c --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/java/com/sample/dynamic_features/at_install/BUCK.fixture @@ -0,0 +1,11 @@ +android_library( + name = "src_release", + srcs = glob(["*.java"]), + manifest = "AndroidManifest.xml", + visibility = [ + "PUBLIC", + ], + deps = [ + "//res/com/sample/dynamic_features/at_install:initialInstall", + ], +) diff --git a/test/com/facebook/buck/android/testdata/android_project/java/com/sample/dynamic_features/conditional/AndroidManifest.xml b/test/com/facebook/buck/android/testdata/android_project/java/com/sample/dynamic_features/conditional/AndroidManifest.xml new file mode 100644 index 00000000000..10f8cd55a47 --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/java/com/sample/dynamic_features/conditional/AndroidManifest.xml @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/com/facebook/buck/android/testdata/android_project/java/com/sample/dynamic_features/conditional/BUCK.fixture b/test/com/facebook/buck/android/testdata/android_project/java/com/sample/dynamic_features/conditional/BUCK.fixture new file mode 100644 index 00000000000..907001fc981 --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/java/com/sample/dynamic_features/conditional/BUCK.fixture @@ -0,0 +1,11 @@ +android_library( + name = "src_release", + srcs = glob(["*.java"]), + manifest = "AndroidManifest.xml", + visibility = [ + "PUBLIC", + ], + deps = [ + "//res/com/sample/dynamic_features/conditional:java", + ], +) diff --git a/test/com/facebook/buck/android/testdata/android_project/java/com/sample/dynamic_features/conditional/ConditionalJavaFeatureActivity.java b/test/com/facebook/buck/android/testdata/android_project/java/com/sample/dynamic_features/conditional/ConditionalJavaFeatureActivity.java new file mode 100644 index 00000000000..ba9e43157c1 --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/java/com/sample/dynamic_features/conditional/ConditionalJavaFeatureActivity.java @@ -0,0 +1,34 @@ +/* + * Copyright 2014-present Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.sample.dynamic_features.conditional; + +import android.app.Activity; +import android.os.Bundle; +import android.widget.TextView; +import com.sample.java.R; + +public class ConditionalJavaFeatureActivity extends Activity { + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.layout_conditional_install); + + TextView tv = (TextView) findViewById(R.id.conditional_java_text_view); + tv.setText(R.string.conditional_java_text); + } +} diff --git a/test/com/facebook/buck/android/testdata/android_project/kotlin/com/sample/dynamic_features/on_demand/AndroidManifest.xml b/test/com/facebook/buck/android/testdata/android_project/kotlin/com/sample/dynamic_features/on_demand/AndroidManifest.xml new file mode 100644 index 00000000000..b95677e0e9b --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/kotlin/com/sample/dynamic_features/on_demand/AndroidManifest.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + diff --git a/test/com/facebook/buck/android/testdata/android_project/kotlin/com/sample/dynamic_features/on_demand/BUCK.fixture b/test/com/facebook/buck/android/testdata/android_project/kotlin/com/sample/dynamic_features/on_demand/BUCK.fixture new file mode 100644 index 00000000000..bafa6dccfe4 --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/kotlin/com/sample/dynamic_features/on_demand/BUCK.fixture @@ -0,0 +1,14 @@ +android_library( + name = "src_release", + srcs = glob([ + "OnDemandKotlinFeatureActivity.kt", + ]), + manifest = "AndroidManifest.xml", + language = "KOTLIN", + visibility = [ + "PUBLIC", + ], + deps = [ + "//res/com/sample/dynamic_features/on_demand/kotlin:kotlin", + ], +) diff --git a/test/com/facebook/buck/android/testdata/android_project/kotlin/com/sample/dynamic_features/on_demand/OnDemandKotlinFeatureActivity.kt b/test/com/facebook/buck/android/testdata/android_project/kotlin/com/sample/dynamic_features/on_demand/OnDemandKotlinFeatureActivity.kt new file mode 100644 index 00000000000..8b6017e4021 --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/kotlin/com/sample/dynamic_features/on_demand/OnDemandKotlinFeatureActivity.kt @@ -0,0 +1,31 @@ +/* + * Copyright 2014-present Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package com.sample.dynamic_features.on_demand; + +import android.app.Activity +import android.os.Bundle +import android.widget.TextView +import com.sample.kotlin.R + +public class OnDemandKotlinFeatureActivity : Activity() { + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + setContentView(R.layout.layout_on_demand_kotlin) + val tv: TextView = findViewById(R.id.on_demand_kotlin_text_view) as TextView + tv.text = getString(R.string.on_demand_kotlin_text) + } +} diff --git a/test/com/facebook/buck/android/testdata/android_project/kotlin/com/sample/lib/BUCK.fixture b/test/com/facebook/buck/android/testdata/android_project/kotlin/com/sample/lib/BUCK.fixture index be11af1830d..415f14ebb73 100644 --- a/test/com/facebook/buck/android/testdata/android_project/kotlin/com/sample/lib/BUCK.fixture +++ b/test/com/facebook/buck/android/testdata/android_project/kotlin/com/sample/lib/BUCK.fixture @@ -47,13 +47,13 @@ android_library( ) android_library( - name = "lib_extra_kotlinc_arguments", + name = "lib_free_compiler_args", srcs = glob([ "Activity.kt", "Sample.kt", "Sample2.kt", ]), - extra_kotlinc_arguments = [ + free_compiler_args = [ "-Xplugin=", ], language = "KOTLIN", diff --git a/test/com/facebook/buck/android/testdata/android_project/native/dynamic_features/on_demand/AndroidManifest.xml b/test/com/facebook/buck/android/testdata/android_project/native/dynamic_features/on_demand/AndroidManifest.xml new file mode 100644 index 00000000000..d81d1e96e3a --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/native/dynamic_features/on_demand/AndroidManifest.xml @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + diff --git a/test/com/facebook/buck/android/testdata/android_project/native/dynamic_features/on_demand/BUCK.fixture b/test/com/facebook/buck/android/testdata/android_project/native/dynamic_features/on_demand/BUCK.fixture new file mode 100644 index 00000000000..9e0a82c0d8c --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/native/dynamic_features/on_demand/BUCK.fixture @@ -0,0 +1,18 @@ +android_library( + name = "src_release", + srcs = glob(["*.java"]), + manifest = "AndroidManifest.xml", + visibility = [ + "PUBLIC", + ], + deps = [ + "//res/com/sample/dynamic_features/on_demand/native:native", + ":native_libs", + ], +) + +prebuilt_native_library( + name = "native_libs", + native_libs = "libs", + visibility = ["PUBLIC"], +) diff --git a/test/com/facebook/buck/android/testdata/android_project/native/dynamic_features/on_demand/OnDemandNativeFeatureActivity.java b/test/com/facebook/buck/android/testdata/android_project/native/dynamic_features/on_demand/OnDemandNativeFeatureActivity.java new file mode 100644 index 00000000000..e9a2c92bcf9 --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/native/dynamic_features/on_demand/OnDemandNativeFeatureActivity.java @@ -0,0 +1,34 @@ +/* + * Copyright 2014-present Facebook, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +package dynamic_features.on_demand; + +import android.app.Activity; +import android.os.Bundle; +import android.widget.TextView; +import com.sample.ccode.R; + +public class OnDemandNativeFeatureActivity extends Activity { + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + setContentView(R.layout.layout_on_demand_native); + + TextView tv = (TextView) findViewById(R.id.on_demand_native_text_view); + tv.setText(R.string.on_demand_native_text); + } +} diff --git a/test/com/facebook/buck/android/testdata/android_project/native/dynamic_features/on_demand/libs/arm64-v8a/libprebuilt.so b/test/com/facebook/buck/android/testdata/android_project/native/dynamic_features/on_demand/libs/arm64-v8a/libprebuilt.so new file mode 100755 index 00000000000..de2559e238a Binary files /dev/null and b/test/com/facebook/buck/android/testdata/android_project/native/dynamic_features/on_demand/libs/arm64-v8a/libprebuilt.so differ diff --git a/test/com/facebook/buck/android/testdata/android_project/native/dynamic_features/on_demand/libs/armeabi-v7a/libprebuilt.so b/test/com/facebook/buck/android/testdata/android_project/native/dynamic_features/on_demand/libs/armeabi-v7a/libprebuilt.so new file mode 100755 index 00000000000..3ccc26fa4e2 Binary files /dev/null and b/test/com/facebook/buck/android/testdata/android_project/native/dynamic_features/on_demand/libs/armeabi-v7a/libprebuilt.so differ diff --git a/test/com/facebook/buck/android/testdata/android_project/native/dynamic_features/on_demand/libs/x86/libprebuilt.so b/test/com/facebook/buck/android/testdata/android_project/native/dynamic_features/on_demand/libs/x86/libprebuilt.so new file mode 100755 index 00000000000..6a5651534a6 Binary files /dev/null and b/test/com/facebook/buck/android/testdata/android_project/native/dynamic_features/on_demand/libs/x86/libprebuilt.so differ diff --git a/test/com/facebook/buck/android/testdata/android_project/native/dynamic_features/on_demand/libs/x86_64/libprebuilt.so b/test/com/facebook/buck/android/testdata/android_project/native/dynamic_features/on_demand/libs/x86_64/libprebuilt.so new file mode 100755 index 00000000000..6b69f58dfbd Binary files /dev/null and b/test/com/facebook/buck/android/testdata/android_project/native/dynamic_features/on_demand/libs/x86_64/libprebuilt.so differ diff --git a/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/at_install/BUCK.fixture b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/at_install/BUCK.fixture new file mode 100644 index 00000000000..1f3784fc6c7 --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/at_install/BUCK.fixture @@ -0,0 +1,10 @@ +android_resource( + name = "initialInstall", + package = "com.sample.initialInstall", + res = "res", + visibility = [ + "PUBLIC", + ], + deps = [ + ], +) diff --git a/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/at_install/res/layout/layout_initial_install.xml b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/at_install/res/layout/layout_initial_install.xml new file mode 100644 index 00000000000..87f9aa58718 --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/at_install/res/layout/layout_initial_install.xml @@ -0,0 +1,12 @@ + + + + diff --git a/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/at_install/res/values/strings.xml b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/at_install/res/values/strings.xml new file mode 100644 index 00000000000..ade053aab36 --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/at_install/res/values/strings.xml @@ -0,0 +1,4 @@ + + + String from at-install module + diff --git a/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/base/BUCK.fixture b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/base/BUCK.fixture new file mode 100644 index 00000000000..a5a6eed742c --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/base/BUCK.fixture @@ -0,0 +1,10 @@ +android_resource( + name = "base", + package = "com.sample", + res = "res", + visibility = [ + "PUBLIC", + ], + deps = [ + ], +) diff --git a/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/base/res/drawable-hdpi/app_icon.png b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/base/res/drawable-hdpi/app_icon.png new file mode 100755 index 00000000000..92fc86a7918 Binary files /dev/null and b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/base/res/drawable-hdpi/app_icon.png differ diff --git a/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/base/res/drawable-mdpi/app_icon.png b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/base/res/drawable-mdpi/app_icon.png new file mode 100755 index 00000000000..9c2dadd0574 Binary files /dev/null and b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/base/res/drawable-mdpi/app_icon.png differ diff --git a/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/base/res/drawable-xhdpi/app_icon.png b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/base/res/drawable-xhdpi/app_icon.png new file mode 100755 index 00000000000..a307cb64fe3 Binary files /dev/null and b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/base/res/drawable-xhdpi/app_icon.png differ diff --git a/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/base/res/values/strings.xml b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/base/res/values/strings.xml new file mode 100644 index 00000000000..3313590c264 --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/base/res/values/strings.xml @@ -0,0 +1,8 @@ + + + Dynamic Features App + Initial Installed module title + On-demand java module title + On-demand kotlin module title + On-demand native module title + diff --git a/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/conditional/BUCK.fixture b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/conditional/BUCK.fixture new file mode 100644 index 00000000000..2ddb755f49d --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/conditional/BUCK.fixture @@ -0,0 +1,10 @@ +android_resource( + name = "java", + package = "com.sample.java", + res = "res", + visibility = [ + "PUBLIC", + ], + deps = [ + ], +) diff --git a/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/conditional/res/layout/layout_conditional_install.xml b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/conditional/res/layout/layout_conditional_install.xml new file mode 100644 index 00000000000..26d205ccd64 --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/conditional/res/layout/layout_conditional_install.xml @@ -0,0 +1,12 @@ + + + + diff --git a/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/conditional/res/values/strings.xml b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/conditional/res/values/strings.xml new file mode 100644 index 00000000000..c72a8ddf012 --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/conditional/res/values/strings.xml @@ -0,0 +1,4 @@ + + + String from conditional module + diff --git a/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/on_demand/kotlin/BUCK.fixture b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/on_demand/kotlin/BUCK.fixture new file mode 100644 index 00000000000..e58f50d56ab --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/on_demand/kotlin/BUCK.fixture @@ -0,0 +1,10 @@ +android_resource( + name = "kotlin", + package = "com.sample.kotlin", + res = "res", + visibility = [ + "PUBLIC", + ], + deps = [ + ], +) diff --git a/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/on_demand/kotlin/res/layout/layout_on_demand_kotlin.xml b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/on_demand/kotlin/res/layout/layout_on_demand_kotlin.xml new file mode 100644 index 00000000000..37add8d65c7 --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/on_demand/kotlin/res/layout/layout_on_demand_kotlin.xml @@ -0,0 +1,12 @@ + + + + diff --git a/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/on_demand/kotlin/res/values/strings.xml b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/on_demand/kotlin/res/values/strings.xml new file mode 100644 index 00000000000..e283020171f --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/on_demand/kotlin/res/values/strings.xml @@ -0,0 +1,4 @@ + + + String from kotlin on-demand module + diff --git a/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/on_demand/native/BUCK.fixture b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/on_demand/native/BUCK.fixture new file mode 100644 index 00000000000..57fa074f8b2 --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/on_demand/native/BUCK.fixture @@ -0,0 +1,10 @@ +android_resource( + name = "native", + package = "com.sample.ccode", + res = "res", + visibility = [ + "PUBLIC", + ], + deps = [ + ], +) diff --git a/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/on_demand/native/res/layout/layout_on_demand_native.xml b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/on_demand/native/res/layout/layout_on_demand_native.xml new file mode 100644 index 00000000000..e64621a5fbb --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/on_demand/native/res/layout/layout_on_demand_native.xml @@ -0,0 +1,12 @@ + + + + diff --git a/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/on_demand/native/res/values/strings.xml b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/on_demand/native/res/values/strings.xml new file mode 100644 index 00000000000..e224015f321 --- /dev/null +++ b/test/com/facebook/buck/android/testdata/android_project/res/com/sample/dynamic_features/on_demand/native/res/values/strings.xml @@ -0,0 +1,4 @@ + + + String from native on-demand module + diff --git a/test/com/facebook/buck/android/toolchain/impl/AndroidSdkDirectoryResolverTest.java b/test/com/facebook/buck/android/toolchain/impl/AndroidSdkDirectoryResolverTest.java index 847026ba94a..25fb5b359d3 100644 --- a/test/com/facebook/buck/android/toolchain/impl/AndroidSdkDirectoryResolverTest.java +++ b/test/com/facebook/buck/android/toolchain/impl/AndroidSdkDirectoryResolverTest.java @@ -33,6 +33,7 @@ public class AndroidSdkDirectoryResolverTest { @Rule public TemporaryPaths tmpDir = new TemporaryPaths(); @Rule public TemporaryPaths tmpDir2 = new TemporaryPaths(); + @Rule public TemporaryPaths tmpDir3 = new TemporaryPaths(); @Rule public ExpectedException expectedException = ExpectedException.none(); @@ -122,12 +123,13 @@ public void testEnvVariableInSdkPathSearchOrderIsUsed() throws IOException { public void testFirstEnvVariableInSdkPathSearchOrderIsUsed() throws IOException { Path env1Dir = tmpDir.newFolder("sdk"); Path env2Dir = tmpDir2.newFolder("sdk"); + Path env3Dir = tmpDir3.newFolder("sdk"); AndroidSdkDirectoryResolver resolver = new AndroidSdkDirectoryResolver( tmpDir.getRoot().getFileSystem(), - ImmutableMap.of("ANDROID_SDK", env1Dir.toString(), "ANDROID_HOME", env2Dir.toString()), + ImmutableMap.of("ANDROID_SDK", env1Dir.toString(), "ANDROID_HOME", env2Dir.toString(), "ANDROID_SDK_ROOT", env3Dir.toString()), FakeAndroidBuckConfig.builder() - .setSdkPathSearchOrder("ANDROID_SDK, ANDROID_HOME") + .setSdkPathSearchOrder("ANDROID_SDK, ANDROID_HOME, ANDROID_SDK_ROOT") .build()); assertEquals(env1Dir, resolver.getSdkOrThrow()); } diff --git a/test/com/facebook/buck/android/toolchain/ndk/impl/AndroidNdkResolverTest.java b/test/com/facebook/buck/android/toolchain/ndk/impl/AndroidNdkResolverTest.java index 61c1d976c01..0be49bc832c 100644 --- a/test/com/facebook/buck/android/toolchain/ndk/impl/AndroidNdkResolverTest.java +++ b/test/com/facebook/buck/android/toolchain/ndk/impl/AndroidNdkResolverTest.java @@ -440,12 +440,12 @@ public void testUnsupportedVersionNotUsed() throws IOException { Path expectedPath = createTmpNdkVersions( NDK_POST_R11_VERSION_FILENAME, - "ndk-dir-r19b", - "Pkg.Desc = Android NDK\nPkg.Revision = 19.1.5304403", "ndk-dir-r20", "Pkg.Desc = Android NDK\nPkg.Revision = 20.0.5594570", - "ndk-dir-r21-xxx", - "Pkg.Desc = Android NDK\nPkg.Revision = 21.0.5000000")[1]; + "ndk-dir-r21", + "Pkg.Desc = Android NDK\nPkg.Revision = 21.0.6113669", + "ndk-dir-r22-xxx", + "Pkg.Desc = Android NDK\nPkg.Revision = 22.0.5000000")[1]; AndroidNdkResolver resolver = new AndroidNdkResolver( tmpDir.getRoot().getFileSystem(), diff --git a/test/com/facebook/buck/apple/AppleBinaryIntegrationTest.java b/test/com/facebook/buck/apple/AppleBinaryIntegrationTest.java index 7a7b768e8a3..2c56855250a 100644 --- a/test/com/facebook/buck/apple/AppleBinaryIntegrationTest.java +++ b/test/com/facebook/buck/apple/AppleBinaryIntegrationTest.java @@ -1530,7 +1530,7 @@ public void testAppleBinaryBuildsFatBinariesWithSwift() throws Exception { workspace.setUp(); BuildTarget target = BuildTargetFactory.newInstance( - "//:DemoAppBinary#iphonesimulator-i386,iphonesimulator-x86_64,no-linkermap"); + "//:DemoAppBinary#iphonesimulator-i386,iphonesimulator-x86_64,iphonesimulator-arm64,no-linkermap"); workspace.runBuckCommand("build", target.getFullyQualifiedName()).assertSuccess(); Path output = @@ -1541,7 +1541,7 @@ public void testAppleBinaryBuildsFatBinariesWithSwift() throws Exception { workspace.runCommand("file", output.toString()).getStdout().get(), containsString("executable")); ProcessExecutor.Result lipoVerifyResult = - workspace.runCommand("lipo", output.toString(), "-verify_arch", "i386", "x86_64"); + workspace.runCommand("lipo", output.toString(), "-verify_arch", "i386", "x86_64", "arm64"); assertEquals(lipoVerifyResult.getStderr().orElse(""), 0, lipoVerifyResult.getExitCode()); } diff --git a/test/com/facebook/buck/apple/AppleBundleIntegrationTest.java b/test/com/facebook/buck/apple/AppleBundleIntegrationTest.java index e9cb6a84de3..33565ff20a7 100644 --- a/test/com/facebook/buck/apple/AppleBundleIntegrationTest.java +++ b/test/com/facebook/buck/apple/AppleBundleIntegrationTest.java @@ -1163,6 +1163,34 @@ public void watchApplicationBundle() throws IOException { assertTrue(Files.exists(watchAppPath.resolve("Interface.plist"))); } + @Test + public void appClipApplicationBundle() throws IOException { + ProjectWorkspace workspace = + TestDataHelper.createProjectWorkspaceForScenario(this, "app_bundle_with_appclip", tmp); + workspace.setUp(); + + BuildTarget target = BuildTargetFactory.newInstance("//:ExampleApp#iphonesimulator-x86_64"); + workspace.runBuckCommand("build", target.getFullyQualifiedName()).assertSuccess(); + + Path appPath = + workspace.getPath( + BuildTargetPaths.getGenPath( + filesystem, + target.withAppendedFlavors( + AppleDebugFormat.DWARF_AND_DSYM.getFlavor(), + AppleDescriptions.NO_INCLUDE_FRAMEWORKS_FLAVOR), + "%s") + .resolve(target.getShortName() + ".app")); + assertTrue(Files.exists(appPath.resolve("ExampleApp"))); + assertTrue(Files.exists(appPath.resolve("Info.plist"))); + + Path appClipPath = appPath.resolve("AppClips/Clip.app/"); + + assertTrue(Files.exists(appClipPath.resolve("Clip"))); + assertTrue(Files.exists(appClipPath.resolve("Info.plist"))); + assertFalse(Files.exists(appClipPath.resolve("Frameworks"))); + } + @Test public void copiesFrameworkBundleIntoFrameworkDirectory() throws Exception { assumeTrue( diff --git a/test/com/facebook/buck/apple/FakeAppleRuleDescriptions.java b/test/com/facebook/buck/apple/FakeAppleRuleDescriptions.java index 23008f45907..01993ef0807 100644 --- a/test/com/facebook/buck/apple/FakeAppleRuleDescriptions.java +++ b/test/com/facebook/buck/apple/FakeAppleRuleDescriptions.java @@ -112,24 +112,25 @@ private FakeAppleRuleDescriptions() {} () -> { ProjectFilesystem filesystem = FakeProjectFilesystem.createJavaOnlyFilesystem(); Stream.of( + "Platforms/iPhoneOS.platform/Developer/usr/bin/libtool", + "Toolchains/XcodeDefault.xctoolchain/usr/bin/ar", "Toolchains/XcodeDefault.xctoolchain/usr/bin/clang", "Toolchains/XcodeDefault.xctoolchain/usr/bin/clang++", "Toolchains/XcodeDefault.xctoolchain/usr/bin/dsymutil", + "Toolchains/XcodeDefault.xctoolchain/usr/bin/ld", + "Toolchains/XcodeDefault.xctoolchain/usr/bin/libtool", "Toolchains/XcodeDefault.xctoolchain/usr/bin/lipo", + "Toolchains/XcodeDefault.xctoolchain/usr/bin/nm", "Toolchains/XcodeDefault.xctoolchain/usr/bin/ranlib", "Toolchains/XcodeDefault.xctoolchain/usr/bin/strip", "Toolchains/XcodeDefault.xctoolchain/usr/bin/swiftc", "Toolchains/XcodeDefault.xctoolchain/usr/bin/swift-stdlib-tool", - "Toolchains/XcodeDefault.xctoolchain/usr/bin/nm", - "Toolchains/XcodeDefault.xctoolchain/usr/bin/ar", - "Toolchains/XcodeDefault.xctoolchain/usr/bin/libtool", - "Platforms/iPhoneOS.platform/Developer/usr/bin/libtool", + "Tools/otest", "usr/bin/actool", "usr/bin/ibtool", "usr/bin/momc", "usr/bin/copySceneKitAssets", "usr/bin/lldb", - "Tools/otest", "usr/bin/xctest") .forEach( path -> { diff --git a/test/com/facebook/buck/apple/PrebuiltAppleFrameworkIntegrationTest.java b/test/com/facebook/buck/apple/PrebuiltAppleFrameworkIntegrationTest.java index a08cc29d273..de5d3b34cd1 100644 --- a/test/com/facebook/buck/apple/PrebuiltAppleFrameworkIntegrationTest.java +++ b/test/com/facebook/buck/apple/PrebuiltAppleFrameworkIntegrationTest.java @@ -128,11 +128,19 @@ public void testStaticWithDependencies() throws IOException, InterruptedExceptio ProjectFilesystem filesystem = TestProjectFilesystems.createProjectFilesystem(workspace.getDestPath()); - BuildTarget target = BuildTargetFactory.newInstance("//app:TestApp#static,macosx-x86_64"); + BuildTarget target = + BuildTargetFactory.newInstance( + "//app:TestAppBundle#dwarf-and-dsym,macosx-x86_64,no-include-frameworks"); ProcessResult result = workspace.runBuckCommand("build", target.getFullyQualifiedName()); result.assertSuccess(); - Path testBinaryPath = workspace.getPath(BuildTargetPaths.getGenPath(filesystem, target, "%s")); + Path testAppBundlePath = + workspace + .getPath(BuildTargetPaths.getGenPath(filesystem, target, "%s")) + .resolve("TestApp.app"); + Path testBinaryPath = testAppBundlePath.resolve("Contents/MacOS/TestApp"); + Path transitiveResourcePath = testAppBundlePath.resolve("Contents/Resources/resource_file"); + assertTrue(Files.exists(transitiveResourcePath)); ProcessExecutor.Result otoolResult = workspace.runCommand("otool", "-L", testBinaryPath.toString()); diff --git a/test/com/facebook/buck/apple/XctoolRunTestsStepTest.java b/test/com/facebook/buck/apple/XctoolRunTestsStepTest.java index 66a64e628a1..ac94182dcba 100644 --- a/test/com/facebook/buck/apple/XctoolRunTestsStepTest.java +++ b/test/com/facebook/buck/apple/XctoolRunTestsStepTest.java @@ -76,6 +76,7 @@ public void xctoolCommandWithOnlyLogicTests() throws Exception { Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), Optional.empty()); ProcessExecutorParams xctoolParams = ProcessExecutorParams.builder() @@ -128,6 +129,7 @@ public void xctoolCommandWhichFailsPrintsStderrToConsole() throws Exception { Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), Optional.empty()); ProcessExecutorParams xctoolParams = ProcessExecutorParams.builder() @@ -185,6 +187,7 @@ public void xctoolCommandWithOnlyAppTests() throws Exception { Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), Optional.empty()); ProcessExecutorParams xctoolParams = @@ -242,6 +245,7 @@ public void xctoolCommandWithOnlyUITests() throws Exception { Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), Optional.empty()); ProcessExecutorParams xctoolParams = @@ -297,6 +301,7 @@ public void xctoolCommandWithAppAndLogicTests() throws Exception { Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), Optional.empty()); ProcessExecutorParams xctoolParams = @@ -354,6 +359,7 @@ public void xctoolCommandWhichReturnsExitCode1DoesNotFailStep() throws Exception Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), Optional.empty()); ProcessExecutorParams xctoolParams = @@ -409,6 +415,7 @@ public void xctoolCommandWhichReturnsExitCode400FailsStep() throws Exception { Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), Optional.empty()); ProcessExecutorParams xctoolParams = @@ -463,6 +470,7 @@ public void xctoolCommandWithTestSelectorFiltersTests() throws Exception { Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), Optional.empty()); ProcessExecutorParams xctoolListOnlyParams = @@ -554,6 +562,7 @@ public void xctoolCommandWithTestSelectorFailsIfListTestsOnlyFails() throws Exce Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), Optional.empty()); ProcessExecutorParams xctoolListOnlyParams = @@ -620,6 +629,7 @@ public void xctoolCommandWithTestSelectorMatchingNothingDoesNotFail() throws Exc Optional.empty(), Optional.empty(), Optional.empty(), + Optional.empty(), Optional.empty()); ProcessExecutorParams xctoolListOnlyParams = ProcessExecutorParams.builder() @@ -682,7 +692,8 @@ public void testDirectoryAndLevelPassedInEnvironment() throws Exception { Optional.of("TEST_LOG_LEVEL"), Optional.of("verbose"), Optional.empty(), - Optional.of("/path/to/snapshotimages")); + Optional.of("/path/to/snapshotimages"), + Optional.of("/path/to/snapshotdiffimages")); ProcessExecutorParams xctoolParams = ProcessExecutorParams.builder() .setCommand( @@ -698,12 +709,14 @@ public void testDirectoryAndLevelPassedInEnvironment() throws Exception { // This is the important part of this test: only if the process is executed // with a matching environment will the test pass. .setEnvironment( - ImmutableMap.of( - "DEVELOPER_DIR", "/path/to/developer/dir", - "XCTOOL_TEST_ENV_TEST_LOG_PATH", "/path/to/test-logs", - "XCTOOL_TEST_ENV_TEST_LOG_LEVEL", "verbose", - "XCTOOL_TEST_ENV_FB_REFERENCE_IMAGE_DIR", "/path/to/snapshotimages", - "LLVM_PROFILE_FILE", "/tmp/some.profraw")) + ImmutableMap.builder() + .put("DEVELOPER_DIR", "/path/to/developer/dir") + .put("XCTOOL_TEST_ENV_TEST_LOG_PATH", "/path/to/test-logs") + .put("XCTOOL_TEST_ENV_TEST_LOG_LEVEL", "verbose") + .put("XCTOOL_TEST_ENV_FB_REFERENCE_IMAGE_DIR", "/path/to/snapshotimages") + .put("XCTOOL_TEST_ENV_IMAGE_DIFF_DIR", "/path/to/snapshotdiffimages") + .put("LLVM_PROFILE_FILE", "/tmp/some.profraw") + .build()) .setDirectory(projectFilesystem.getRootPath().getPath()) .setRedirectOutput(ProcessBuilder.Redirect.PIPE) .build(); diff --git a/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/AppClipApp/AppDelegate.swift b/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/AppClipApp/AppDelegate.swift new file mode 100644 index 00000000000..53dfa23c169 --- /dev/null +++ b/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/AppClipApp/AppDelegate.swift @@ -0,0 +1,19 @@ +// +// AppDelegate.swift +// AppClipApp +// +// Created by Lucas Marcal on 08/10/20. +// + +import UIKit + +class AppDelegate: UIResponder, UIApplicationDelegate { + var window: UIWindow? + func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool { + self.window = UIWindow(frame: UIScreen.main.bounds) + self.window?.rootViewController = ViewController() + self.window?.makeKeyAndVisible() + return true + } +} + diff --git a/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/AppClipApp/Info.plist b/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/AppClipApp/Info.plist new file mode 100644 index 00000000000..a47c2e96f14 --- /dev/null +++ b/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/AppClipApp/Info.plist @@ -0,0 +1,43 @@ + + + + + CFBundleDevelopmentRegion + EN + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + com.buck.test.appclipapp + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + LSRequiresIPhoneOS + + UIApplicationSupportsIndirectInputEvents + + UIRequiredDeviceCapabilities + + armv7 + + UISupportedInterfaceOrientations + + UIInterfaceOrientationPortrait + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + UISupportedInterfaceOrientations~ipad + + UIInterfaceOrientationPortrait + UIInterfaceOrientationPortraitUpsideDown + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + + diff --git a/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/AppClipApp/ViewController.swift b/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/AppClipApp/ViewController.swift new file mode 100644 index 00000000000..231edffe815 --- /dev/null +++ b/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/AppClipApp/ViewController.swift @@ -0,0 +1,16 @@ +// +// ViewController.swift +// AppClipApp +// +// Created by Lucas Marcal on 08/10/20. +// + +import UIKit + +class ViewController: UIViewController { + + override func viewDidLoad() { + super.viewDidLoad() + view.backgroundColor = .yellow + } +} \ No newline at end of file diff --git a/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/AppClipApp/main.swift b/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/AppClipApp/main.swift new file mode 100644 index 00000000000..98d33b62d2c --- /dev/null +++ b/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/AppClipApp/main.swift @@ -0,0 +1,7 @@ +import Foundation +import UIKit + +let argc = CommandLine.argc +let argv = UnsafeMutableRawPointer(CommandLine.unsafeArgv).bindMemory(to: UnsafeMutablePointer.self, capacity: Int(CommandLine.argc)) + +UIApplicationMain(argc, argv, nil, NSStringFromClass(AppDelegate.self)) \ No newline at end of file diff --git a/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/BUCK.fixture b/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/BUCK.fixture new file mode 100644 index 00000000000..cfcb788ef53 --- /dev/null +++ b/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/BUCK.fixture @@ -0,0 +1,17 @@ +apple_bundle( + name = "ExampleApp", + visibility = ["PUBLIC"], + extension = "app", + binary = ":ExampleAppBinary#iphonesimulator-x86_64", + product_name = "ExampleApp", + info_plist = "AppClipApp/Info.plist", + deps = ["//Clip:Clip"] +) + +apple_binary( + name = "ExampleAppBinary", + visibility = ["PUBLIC"], + swift_version = "5", + target_sdk_version = "14.0", + srcs = glob(["AppClipApp/*.swift",]) +) diff --git a/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/Clip/AppDelegate.swift b/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/Clip/AppDelegate.swift new file mode 100644 index 00000000000..7d0600759d5 --- /dev/null +++ b/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/Clip/AppDelegate.swift @@ -0,0 +1,20 @@ +// +// AppDelegate.swift +// Clip +// +// Created by Lucas Marcal on 07/10/20. +// + +import UIKit + +class AppDelegate: UIResponder, UIApplicationDelegate { + var window: UIWindow? + func application(_ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]?) -> Bool { + self.window = UIWindow(frame: UIScreen.main.bounds) + self.window?.rootViewController = UIViewController() + self.window?.rootViewController?.view.backgroundColor = .red + self.window?.makeKeyAndVisible() + return true + } +} + diff --git a/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/Clip/BUCK.fixture b/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/Clip/BUCK.fixture new file mode 100644 index 00000000000..cef33ed829a --- /dev/null +++ b/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/Clip/BUCK.fixture @@ -0,0 +1,19 @@ +apple_bundle( + name = "Clip", + visibility = ["PUBLIC"], + extension = "app", + binary = ":ClipBinary#iphonesimulator-x86_64", + product_name = "Clip", + info_plist = "Info.plist", + is_app_clip = True, + xcode_product_type = "com.apple.product-type.application.on-demand-install-capable" +) + +apple_binary( + name = "ClipBinary", + visibility = ["PUBLIC"], + target_sdk_version = "14.0", + swift_version = "5", + srcs = glob(["*.swift",]), + entitlements_file = "Clip.entitlements" +) diff --git a/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/Clip/Clip.entitlements b/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/Clip/Clip.entitlements new file mode 100644 index 00000000000..7fd43c0e50b --- /dev/null +++ b/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/Clip/Clip.entitlements @@ -0,0 +1,10 @@ + + + + + com.apple.developer.parent-application-identifiers + + $(AppIdentifierPrefix)com.buck.test.appclipapp + + + diff --git a/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/Clip/Info.plist b/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/Clip/Info.plist new file mode 100644 index 00000000000..d83ecb26e8f --- /dev/null +++ b/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/Clip/Info.plist @@ -0,0 +1,52 @@ + + + + + CFBundleDevelopmentRegion + EN + CFBundleDisplayName + AppWithClip + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + com.buck.test.appclip + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + LSRequiresIPhoneOS + + NSAppClip + + NSAppClipRequestEphemeralUserNotification + + NSAppClipRequestLocationConfirmation + + + UIApplicationSupportsIndirectInputEvents + + UIRequiredDeviceCapabilities + + armv7 + + UISupportedInterfaceOrientations + + UIInterfaceOrientationPortrait + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + UISupportedInterfaceOrientations~ipad + + UIInterfaceOrientationPortrait + UIInterfaceOrientationPortraitUpsideDown + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + + diff --git a/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/Clip/main.swift b/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/Clip/main.swift new file mode 100644 index 00000000000..98d33b62d2c --- /dev/null +++ b/test/com/facebook/buck/apple/testdata/app_bundle_with_appclip/Clip/main.swift @@ -0,0 +1,7 @@ +import Foundation +import UIKit + +let argc = CommandLine.argc +let argv = UnsafeMutableRawPointer(CommandLine.unsafeArgv).bindMemory(to: UnsafeMutablePointer.self, capacity: Int(CommandLine.argc)) + +UIApplicationMain(argc, argv, nil, NSStringFromClass(AppDelegate.self)) \ No newline at end of file diff --git a/test/com/facebook/buck/apple/testdata/ios-project/xcode-developer-dir/Toolchains/XcodeDefault.xctoolchain/usr/bin/ld b/test/com/facebook/buck/apple/testdata/ios-project/xcode-developer-dir/Toolchains/XcodeDefault.xctoolchain/usr/bin/ld new file mode 100755 index 00000000000..e69de29bb2d diff --git a/test/com/facebook/buck/apple/testdata/prebuilt_apple_framework_static/app/BUCK.fixture b/test/com/facebook/buck/apple/testdata/prebuilt_apple_framework_static/app/BUCK.fixture index 16cb8e0b474..d4573819935 100644 --- a/test/com/facebook/buck/apple/testdata/prebuilt_apple_framework_static/app/BUCK.fixture +++ b/test/com/facebook/buck/apple/testdata/prebuilt_apple_framework_static/app/BUCK.fixture @@ -1,3 +1,11 @@ +apple_bundle( + name = "TestAppBundle", + binary = ":TestApp", + extension = "app", + info_plist = "Info.plist", + product_name = "TestApp", +) + apple_binary( name = "TestApp", srcs = ["main.m"], diff --git a/test/com/facebook/buck/apple/testdata/prebuilt_apple_framework_static/app/Info.plist b/test/com/facebook/buck/apple/testdata/prebuilt_apple_framework_static/app/Info.plist new file mode 100644 index 00000000000..838f5f73ee7 --- /dev/null +++ b/test/com/facebook/buck/apple/testdata/prebuilt_apple_framework_static/app/Info.plist @@ -0,0 +1,26 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIconFile + + CFBundleIdentifier + bundle.identifier + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + LSMinimumSystemVersion + 10.0 + + diff --git a/test/com/facebook/buck/apple/testdata/prebuilt_apple_framework_static/dep/BUCK.fixture b/test/com/facebook/buck/apple/testdata/prebuilt_apple_framework_static/dep/BUCK.fixture index f3a9aef4569..fc26861efb4 100644 --- a/test/com/facebook/buck/apple/testdata/prebuilt_apple_framework_static/dep/BUCK.fixture +++ b/test/com/facebook/buck/apple/testdata/prebuilt_apple_framework_static/dep/BUCK.fixture @@ -2,6 +2,14 @@ apple_library( name = "Dep", srcs = ["strings.m"], exported_headers = ["strings.h"], + deps = [":DepResources"], frameworks = ["$SDKROOT/System/Library/Frameworks/Foundation.framework"], visibility = ["PUBLIC"], ) + +apple_resource( + name = "DepResources", + files = ["resource_file"], + dirs = [], +) + diff --git a/test/com/facebook/buck/apple/testdata/prebuilt_apple_framework_static/dep/resource_file b/test/com/facebook/buck/apple/testdata/prebuilt_apple_framework_static/dep/resource_file new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/com/facebook/buck/apple/toolchain/impl/AppleCxxPlatformsTest.java b/test/com/facebook/buck/apple/toolchain/impl/AppleCxxPlatformsTest.java index e9610bcae2b..aa65945dca7 100644 --- a/test/com/facebook/buck/apple/toolchain/impl/AppleCxxPlatformsTest.java +++ b/test/com/facebook/buck/apple/toolchain/impl/AppleCxxPlatformsTest.java @@ -132,11 +132,12 @@ private static ImmutableSet getCommonKnownPaths(Path root) { root.resolve("Toolchains/XcodeDefault.xctoolchain/usr/bin/clang"), root.resolve("Toolchains/XcodeDefault.xctoolchain/usr/bin/clang++"), root.resolve("Toolchains/XcodeDefault.xctoolchain/usr/bin/dsymutil"), + root.resolve("Toolchains/XcodeDefault.xctoolchain/usr/bin/ld"), root.resolve("Toolchains/XcodeDefault.xctoolchain/usr/bin/libtool"), root.resolve("Toolchains/XcodeDefault.xctoolchain/usr/bin/lipo"), + root.resolve("Toolchains/XcodeDefault.xctoolchain/usr/bin/nm"), root.resolve("Toolchains/XcodeDefault.xctoolchain/usr/bin/ranlib"), root.resolve("Toolchains/XcodeDefault.xctoolchain/usr/bin/strip"), - root.resolve("Toolchains/XcodeDefault.xctoolchain/usr/bin/nm"), root.resolve("usr/bin/actool"), root.resolve("usr/bin/ibtool"), root.resolve("usr/bin/momc"), @@ -1198,7 +1199,7 @@ public void checkSwiftPlatformUsesCorrectMinTargetSdk() { assertThat(swiftc, instanceOf(VersionedTool.class)); assertThat( swiftPlatform.getSwiftTarget(), - equalTo(SwiftTargetTriple.of("i386", "apple", "ios", "7.0"))); + equalTo(SwiftTargetTriple.of("i386", "apple", "ios", "7.0", false))); } @Test diff --git a/test/com/facebook/buck/apple/toolchain/impl/AppleSdkDiscoveryTest.java b/test/com/facebook/buck/apple/toolchain/impl/AppleSdkDiscoveryTest.java index eb29f93f3b5..f001d3f1d22 100644 --- a/test/com/facebook/buck/apple/toolchain/impl/AppleSdkDiscoveryTest.java +++ b/test/com/facebook/buck/apple/toolchain/impl/AppleSdkDiscoveryTest.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; import com.facebook.buck.apple.AppleConfig; import com.facebook.buck.apple.toolchain.ApplePlatform; @@ -33,6 +34,7 @@ import com.facebook.buck.testutil.integration.ProjectWorkspace; import com.facebook.buck.testutil.integration.TestDataHelper; import com.facebook.buck.util.CreateSymlinksForTests; +import com.facebook.buck.util.environment.Platform; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -102,7 +104,7 @@ public void shouldResolveSdkVersionConflicts() throws IOException { .setName("macosx") .setVersion("10.9") .setApplePlatform(ApplePlatform.MACOSX) - .addArchitectures("i386", "x86_64") + .addArchitectures("i386", "x86_64", "arm64") .addAllToolchains(toolchains.values()) .build(); AppleSdk macosxDebugSdk = @@ -110,7 +112,7 @@ public void shouldResolveSdkVersionConflicts() throws IOException { .setName("macosx-Debug") .setVersion("10.9") .setApplePlatform(ApplePlatform.MACOSX) - .addArchitectures("i386", "x86_64") + .addArchitectures("i386", "x86_64", "arm64") .addAllToolchains(toolchains.values()) .build(); AppleSdkPaths macosxReleasePaths = @@ -165,7 +167,7 @@ public void shouldFindPlatformsInExtraPlatformDirectories() throws IOException { .setName("macosx10.9") .setVersion("10.9") .setApplePlatform(ApplePlatform.MACOSX) - .addArchitectures("i386", "x86_64") + .addArchitectures("i386", "x86_64", "arm64") .addAllToolchains(toolchains.values()) .build(); AppleSdkPaths macosx109Paths = @@ -208,7 +210,7 @@ public void ignoresInvalidExtraPlatformDirectories() throws IOException { .setName("macosx10.9") .setVersion("10.9") .setApplePlatform(ApplePlatform.MACOSX) - .addArchitectures("i386", "x86_64") + .addArchitectures("i386", "x86_64", "arm64") .addAllToolchains(toolchains.values()) .build(); AppleSdkPaths macosx109Paths = @@ -282,7 +284,7 @@ public void shouldIgnoreSdkWithBadSymlink() throws Exception { .setName("macosx10.9") .setVersion("10.9") .setApplePlatform(ApplePlatform.MACOSX) - .addArchitectures("i386", "x86_64") + .addArchitectures("i386", "x86_64", "arm64") .addAllToolchains(toolchains.values()) .build(); AppleSdkPaths macosx109Paths = @@ -324,7 +326,7 @@ public void appleSdkPathsBuiltFromDirectory() throws Exception { .setName("macosx10.9") .setVersion("10.9") .setApplePlatform(ApplePlatform.MACOSX) - .addArchitectures("i386", "x86_64") + .addArchitectures("i386", "x86_64", "arm64") .addToolchains(getDefaultToolchain(root)) .build(); AppleSdkPaths macosx109Paths = @@ -356,7 +358,7 @@ public void appleSdkPathsBuiltFromDirectory() throws Exception { .setName("iphonesimulator8.0") .setVersion("8.0") .setApplePlatform(ApplePlatform.IPHONESIMULATOR) - .addArchitectures("i386", "x86_64") + .addArchitectures("i386", "x86_64", "arm64") .addToolchains(getDefaultToolchain(root)) .build(); AppleSdkPaths iphonesimulator80Paths = @@ -390,7 +392,7 @@ public void appleSdkPathsBuiltFromDirectory() throws Exception { .setName("watchsimulator2.0") .setVersion("2.0") .setApplePlatform(ApplePlatform.WATCHSIMULATOR) - .addArchitectures("i386", "x86_64") + .addArchitectures("i386", "x86_64", "arm64") .addToolchains(getDefaultToolchain(root)) .build(); AppleSdkPaths watchsimulator20Paths = @@ -423,7 +425,7 @@ public void appleSdkPathsBuiltFromDirectory() throws Exception { .setName("appletvsimulator9.1") .setVersion("9.1") .setApplePlatform(ApplePlatform.APPLETVSIMULATOR) - .addArchitectures("x86_64") + .addArchitectures("x86_64", "arm64") .addToolchains(getDefaultToolchain(root)) .build(); AppleSdkPaths appletvsimulator91Paths = @@ -499,7 +501,7 @@ public void multipleAppleSdkPathsPerPlatformBuiltFromDirectory() throws Exceptio .setName("macosx10.9") .setVersion("10.9") .setApplePlatform(ApplePlatform.MACOSX) - .addArchitectures("i386", "x86_64") + .addArchitectures("i386", "x86_64", "arm64") .addToolchains(getDefaultToolchain(root)) .build(); AppleSdkPaths macosx109Paths = @@ -531,7 +533,7 @@ public void multipleAppleSdkPathsPerPlatformBuiltFromDirectory() throws Exceptio .setName("iphonesimulator8.0") .setVersion("8.0") .setApplePlatform(ApplePlatform.IPHONESIMULATOR) - .addArchitectures("i386", "x86_64") + .addArchitectures("i386", "x86_64", "arm64") .addToolchains(getDefaultToolchain(root)) .build(); AppleSdkPaths iphonesimulator80Paths = @@ -565,7 +567,7 @@ public void multipleAppleSdkPathsPerPlatformBuiltFromDirectory() throws Exceptio .setName("iphonesimulator8.1") .setVersion("8.1") .setApplePlatform(ApplePlatform.IPHONESIMULATOR) - .addArchitectures("i386", "x86_64") + .addArchitectures("i386", "x86_64", "arm64") .addToolchains(getDefaultToolchain(root)) .build(); AppleSdkPaths iphonesimulator81Paths = @@ -623,7 +625,7 @@ public void shouldDiscoverRealSdkThroughAbsoluteSymlink() throws IOException { .setName("macosx10.9") .setVersion("10.9") .setApplePlatform(ApplePlatform.MACOSX) - .addArchitectures("i386", "x86_64") + .addArchitectures("i386", "x86_64", "arm64") .addAllToolchains(toolchains.values()) .build(); AppleSdkPaths macosx109Paths = @@ -651,6 +653,7 @@ public void shouldDiscoverRealSdkThroughAbsoluteSymlink() throws IOException { @Test public void shouldScanRealDirectoryOnlyOnce() throws IOException { + assumeTrue(Platform.detect() != Platform.WINDOWS); ProjectWorkspace workspace = TestDataHelper.createProjectWorkspaceForScenario(this, "sdk-discovery-symlink", temp); workspace.setUp(); @@ -748,7 +751,7 @@ public void overrideToolchains() throws IOException { .setName("macosx10.9") .setVersion("10.9") .setApplePlatform(ApplePlatform.MACOSX) - .addArchitectures("i386", "x86_64") + .addArchitectures("i386", "x86_64", "arm64") .addAllToolchains(ImmutableList.of(overrideToolchain1, overrideToolchain2)) .build(); AppleSdkPaths macosx109Paths = diff --git a/test/com/facebook/buck/cli/BuildCommandErrorsIntegrationTest.java b/test/com/facebook/buck/cli/BuildCommandErrorsIntegrationTest.java index e239958ec8d..5a3e683c2ac 100644 --- a/test/com/facebook/buck/cli/BuildCommandErrorsIntegrationTest.java +++ b/test/com/facebook/buck/cli/BuildCommandErrorsIntegrationTest.java @@ -338,7 +338,7 @@ public void suggestionsWhenBuildTargetDoesntExist() { Matchers.stringContainsInOrder( "BUILD FAILED: The rule //missing_target:bar could not be found.", "Please check the spelling and whether it is one of the 23 targets in ", - Paths.get("missing_target", "BUCK").toString() + ". (915 bytes)", + Paths.get("missing_target", "BUCK").toString(), "3 similar targets in ", " //missing_target:barWithSomeLongSuffix", " //missing_target:baz", diff --git a/test/com/facebook/buck/cli/TestRunningTest.java b/test/com/facebook/buck/cli/TestRunningTest.java index 40737d98769..f3394c94f4e 100644 --- a/test/com/facebook/buck/cli/TestRunningTest.java +++ b/test/com/facebook/buck/cli/TestRunningTest.java @@ -312,44 +312,45 @@ public void testXmlGeneration() throws Exception { // Check for exactly one tag. NodeList testsList = doc.getElementsByTagName("tests"); - assertEquals(testsList.getLength(), 1); + assertEquals(1, testsList.getLength()); // Check for exactly one tag. Element testsEl = (Element) testsList.item(0); NodeList testList = testsEl.getElementsByTagName("test"); - assertEquals(testList.getLength(), 1); + assertEquals(1, testList.getLength()); // Check the target has been set Element testEl = (Element) testList.item(0); - assertEquals(testEl.getAttribute("target"), "//foo/bar:baz"); + assertEquals("//foo/bar:baz", testEl.getAttribute("target")); + + assertEquals("TestCase", testEl.getAttribute("name")); // Check for exactly three tags. // There should be two failures and one success. NodeList resultsList = testEl.getElementsByTagName("testresult"); - assertEquals(resultsList.getLength(), 3); + assertEquals(3, resultsList.getLength()); // Verify the text elements of the first tag. Element passResultEl = (Element) resultsList.item(0); - assertEquals(passResultEl.getAttribute("name"), "passTest"); - assertEquals(passResultEl.getAttribute("time"), "5000"); - assertEquals(passResultEl.getAttribute("status"), "PASS"); + assertEquals("passTest", passResultEl.getAttribute("name")); + assertEquals("5000", passResultEl.getAttribute("time")); + assertEquals("PASS", passResultEl.getAttribute("status")); checkXmlTextContents(passResultEl, "message", ""); checkXmlTextContents(passResultEl, "stacktrace", ""); // Verify the text elements of the second tag. - assertEquals(testEl.getAttribute("name"), "TestCase"); Element failResultEl1 = (Element) resultsList.item(1); - assertEquals(failResultEl1.getAttribute("name"), "failWithMsg"); - assertEquals(failResultEl1.getAttribute("time"), "7000"); - assertEquals(failResultEl1.getAttribute("status"), "FAIL"); + assertEquals("failWithMsg", failResultEl1.getAttribute("name")); + assertEquals("7000", failResultEl1.getAttribute("time")); + assertEquals("FAIL", failResultEl1.getAttribute("status")); checkXmlTextContents(failResultEl1, "message", "Index out of bounds!"); checkXmlTextContents(failResultEl1, "stacktrace", "Stacktrace"); // Verify the text elements of the third tag. Element failResultEl2 = (Element) resultsList.item(2); - assertEquals(failResultEl2.getAttribute("name"), "failNoMsg"); - assertEquals(failResultEl2.getAttribute("time"), "4000"); - assertEquals(failResultEl2.getAttribute("status"), "PASS"); + assertEquals("failNoMsg", failResultEl2.getAttribute("name")); + assertEquals("4000", failResultEl2.getAttribute("time")); + assertEquals("PASS", failResultEl2.getAttribute("status")); checkXmlTextContents(failResultEl2, "message", ""); checkXmlTextContents(failResultEl2, "stacktrace", ""); } diff --git a/test/com/facebook/buck/cli/buck_run_test.py b/test/com/facebook/buck/cli/buck_run_test.py index 0217e000181..d2a23623fa3 100644 --- a/test/com/facebook/buck/cli/buck_run_test.py +++ b/test/com/facebook/buck/cli/buck_run_test.py @@ -31,7 +31,7 @@ def test_buck_run(self): self.assertEqual(0, workspace.run_buck("run", "//:hello-python")) subdir = workspace.resolve_path("subdir") os.mkdir(subdir) - proc = run_buck_process(["run", "//:pwd"], subdir) + proc = run_buck_process(["run", "//:pwd-python"], subdir) stdout, stderr = proc.communicate() sys.stdout.write(stdout.decode("utf8")) sys.stdout.flush() diff --git a/test/com/facebook/buck/cli/testdata/buck_run/BUCK.fixture b/test/com/facebook/buck/cli/testdata/buck_run/BUCK.fixture index e4b110cc55d..7b1672bf1e0 100644 --- a/test/com/facebook/buck/cli/testdata/buck_run/BUCK.fixture +++ b/test/com/facebook/buck/cli/testdata/buck_run/BUCK.fixture @@ -20,6 +20,6 @@ python_binary( ) python_binary( - name = "pwd", - main = "pwd.py", + name = "pwd-python", + main = "pwdbase.py", ) diff --git a/test/com/facebook/buck/cli/testdata/buck_run/hello.py b/test/com/facebook/buck/cli/testdata/buck_run/hello.py index 57c7635cebb..bfdd43b261d 100644 --- a/test/com/facebook/buck/cli/testdata/buck_run/hello.py +++ b/test/com/facebook/buck/cli/testdata/buck_run/hello.py @@ -1,2 +1,2 @@ if __name__ == '__main__': - print "hello" + print ("hello") diff --git a/test/com/facebook/buck/cli/testdata/buck_run/pwd.py b/test/com/facebook/buck/cli/testdata/buck_run/pwdbase.py similarity index 86% rename from test/com/facebook/buck/cli/testdata/buck_run/pwd.py rename to test/com/facebook/buck/cli/testdata/buck_run/pwdbase.py index 0085e1c65f2..727d872163a 100644 --- a/test/com/facebook/buck/cli/testdata/buck_run/pwd.py +++ b/test/com/facebook/buck/cli/testdata/buck_run/pwdbase.py @@ -2,6 +2,6 @@ import sys cwdbase = os.path.basename(os.getcwd()) -print cwdbase +print (cwdbase) sys.exit(0 if cwdbase == 'subdir' else 1) diff --git a/test/com/facebook/buck/cli/testdata/test_coverage/BUCK.fixture b/test/com/facebook/buck/cli/testdata/test_coverage/BUCK.fixture index 8a2da970d22..0221542c9d8 100644 --- a/test/com/facebook/buck/cli/testdata/test_coverage/BUCK.fixture +++ b/test/com/facebook/buck/cli/testdata/test_coverage/BUCK.fixture @@ -18,6 +18,7 @@ genrule( srcs = [":foo"], out = "out.jar", cmd = "cp $SRCS $OUT", + cmd_exe = "copy %SRCS% %OUT%", ) prebuilt_jar( diff --git a/test/com/facebook/buck/core/build/engine/impl/BUCK b/test/com/facebook/buck/core/build/engine/impl/BUCK index 27feb446aea..b0d4251e255 100644 --- a/test/com/facebook/buck/core/build/engine/impl/BUCK +++ b/test/com/facebook/buck/core/build/engine/impl/BUCK @@ -21,6 +21,7 @@ java_test( "//src/com/facebook/buck/util/json:json", "//src/com/facebook/buck/util/stream:stream", "//src/com/facebook/buck/util/zip:zip", + "//src/com/facebook/buck/io/pathformat:pathformat", "//test/com/facebook/buck/artifact_cache:testutil", "//test/com/facebook/buck/core/build/context:testutil", "//test/com/facebook/buck/core/build/engine/cache/manager:testutil", diff --git a/test/com/facebook/buck/core/build/engine/impl/CachingBuildEngineTest.java b/test/com/facebook/buck/core/build/engine/impl/CachingBuildEngineTest.java index a1c9f062860..0ace3875a15 100644 --- a/test/com/facebook/buck/core/build/engine/impl/CachingBuildEngineTest.java +++ b/test/com/facebook/buck/core/build/engine/impl/CachingBuildEngineTest.java @@ -117,6 +117,7 @@ import com.facebook.buck.io.file.LazyPath; import com.facebook.buck.io.filesystem.ProjectFilesystem; import com.facebook.buck.io.filesystem.TestProjectFilesystems; +import com.facebook.buck.io.pathformat.PathFormatter; import com.facebook.buck.rules.keys.DefaultDependencyFileRuleKeyFactory; import com.facebook.buck.rules.keys.DefaultRuleKeyFactory; import com.facebook.buck.rules.keys.DependencyFileEntry; @@ -2045,7 +2046,7 @@ public void inputBasedRuleKeyCacheHitAvoidsBuildingLocally() throws Exception { TarInspector.readTarZst(fetchedArtifact); assertEquals( - Sets.union(ImmutableSet.of(metadataDirectory + "/"), artifactEntries.keySet()), + Sets.union(ImmutableSet.of(PathFormatter.pathWithUnixSeparators(metadataDirectory) + "/"), artifactEntries.keySet()), fetchedArtifactEntries.keySet()); assertThat( fetchedArtifactEntries, @@ -2476,9 +2477,10 @@ public SourcePath getSourcePathToOutput() { assertThat( fetchedArtifactEntries, Matchers.hasEntry( - BuildInfo.getPathToArtifactMetadataDirectory(target, filesystem) - .resolve(BuildInfo.MetadataKey.DEP_FILE) - .toString(), + PathFormatter.pathWithUnixSeparators( + BuildInfo.getPathToArtifactMetadataDirectory(target, filesystem) + .resolve(BuildInfo.MetadataKey.DEP_FILE) + .toString()), ObjectMappers.WRITER .writeValueAsString(ImmutableList.of(fileToDepFileEntryString(input))) .getBytes(UTF_8))); diff --git a/test/com/facebook/buck/core/module/impl/jarwithouthash/BUCK b/test/com/facebook/buck/core/module/impl/jarwithouthash/BUCK index b5a65f57b29..a8fb4456fcc 100644 --- a/test/com/facebook/buck/core/module/impl/jarwithouthash/BUCK +++ b/test/com/facebook/buck/core/module/impl/jarwithouthash/BUCK @@ -30,7 +30,7 @@ genrule( # Running tests as a shell script to reconstruct the layout of modules and how they are loaded python_test( - name = "test", + name = "jarwithouthash-test", srcs = [ "//test/com/facebook/buck/core/module/impl:test_app.py", ], diff --git a/test/com/facebook/buck/core/module/impl/moduleclass/BUCK b/test/com/facebook/buck/core/module/impl/moduleclass/BUCK index 74b927f8bcd..d47425cff12 100644 --- a/test/com/facebook/buck/core/module/impl/moduleclass/BUCK +++ b/test/com/facebook/buck/core/module/impl/moduleclass/BUCK @@ -35,6 +35,10 @@ genrule( "mkdir $OUT && ", "cp $(location :{}) $OUT/module-binary-hash.txt".format("known-hash.txt"), ]), + cmd_exe = " ".join([ + "mkdir $OUT && ", + "copy $(location :{}) $OUT\module-binary-hash.txt".format("known-hash.txt"), + ]), ) zip_file( diff --git a/test/com/facebook/buck/core/rules/configsetting/testdata/project_with_constraints/BUCK.fixture b/test/com/facebook/buck/core/rules/configsetting/testdata/project_with_constraints/BUCK.fixture index 410ee49ea88..83183fe7adf 100644 --- a/test/com/facebook/buck/core/rules/configsetting/testdata/project_with_constraints/BUCK.fixture +++ b/test/com/facebook/buck/core/rules/configsetting/testdata/project_with_constraints/BUCK.fixture @@ -127,6 +127,7 @@ genrule( }), out = "cat_out.txt", cmd = "cat $SRCS > $OUT", + cmd_exe = "type %SRCS% > %OUT%", ) genrule( @@ -137,6 +138,7 @@ genrule( }), out = "cat_out.txt", cmd = "cat $SRCS > $OUT", + cmd_exe = "type %SRCS% > %OUT%", ) genrule( @@ -147,6 +149,7 @@ genrule( }), out = "cat_out.txt", cmd = "cat $SRCS > $OUT", + cmd_exe = "type %SRCS% > %OUT%", ) genrule( @@ -157,4 +160,5 @@ genrule( }), out = "cat_out.txt", cmd = "cat $SRCS > $OUT", + cmd_exe = "type %SRCS% > %OUT%", ) diff --git a/test/com/facebook/buck/core/rules/configsetting/testdata/simple_project/BUCK.fixture b/test/com/facebook/buck/core/rules/configsetting/testdata/simple_project/BUCK.fixture index 618c70ab13e..40eae8d3384 100644 --- a/test/com/facebook/buck/core/rules/configsetting/testdata/simple_project/BUCK.fixture +++ b/test/com/facebook/buck/core/rules/configsetting/testdata/simple_project/BUCK.fixture @@ -6,6 +6,7 @@ genrule( }), out = "cat_out.txt", cmd = "cat $SRCS > $OUT", + cmd_exe = "type %SRCS% > %OUT%", ) genrule( @@ -17,6 +18,7 @@ genrule( }), out = "cat_out.txt", cmd = "cat $SRCS > $OUT", + cmd_exe = "type %SRCS% > %OUT%", ) genrule( @@ -27,6 +29,7 @@ genrule( }), out = "cat_out.txt", cmd = "cat $SRCS > $OUT", + cmd_exe = "type %SRCS% > %OUT%", ) config_setting( @@ -180,6 +183,7 @@ genrule( }), out = "cat_out.txt", cmd = "cat $SRCS > $OUT", + cmd_exe = "type %SRCS% > %OUT%", ) genrule( @@ -190,6 +194,7 @@ genrule( }) + ["c.txt"], out = "select_concat_out.txt", cmd = "cat $SRCS > $OUT", + cmd_exe = "type %SRCS% > %OUT%", ) config_setting( diff --git a/test/com/facebook/buck/cxx/CxxCompilationDatabaseTest.java b/test/com/facebook/buck/cxx/CxxCompilationDatabaseTest.java index ae974248b75..ff6af45d864 100644 --- a/test/com/facebook/buck/cxx/CxxCompilationDatabaseTest.java +++ b/test/com/facebook/buck/cxx/CxxCompilationDatabaseTest.java @@ -47,6 +47,7 @@ import com.facebook.buck.step.Step; import com.facebook.buck.step.fs.MkdirStep; import com.facebook.buck.testutil.MoreAsserts; +import com.facebook.buck.util.environment.Platform; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -67,7 +68,13 @@ public void testCompilationDatabase() { BuildTargetFactory.newInstance("//foo:baz") .withAppendedFlavors(ImmutableSet.of(CxxCompilationDatabase.COMPILATION_DATABASE)); - String root = "/Users/user/src"; + String root; + if (Platform.detect() == Platform.WINDOWS) { + root = "C:\\Users\\user\\src"; + } + else { + root = "/Users/user/src"; + } Path fakeRoot = Paths.get(root); ProjectFilesystem filesystem = new FakeProjectFilesystem(CanonicalCellName.rootCell(), AbsPath.of(fakeRoot)); diff --git a/test/com/facebook/buck/cxx/CxxErrorTransformerTest.java b/test/com/facebook/buck/cxx/CxxErrorTransformerTest.java index e30186da6cc..8f7bac29981 100644 --- a/test/com/facebook/buck/cxx/CxxErrorTransformerTest.java +++ b/test/com/facebook/buck/cxx/CxxErrorTransformerTest.java @@ -19,6 +19,7 @@ import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.oneOf; +import static org.junit.Assume.assumeTrue; import com.facebook.buck.core.rules.BuildRuleResolver; import com.facebook.buck.core.rules.resolver.impl.TestActionGraphBuilder; @@ -27,6 +28,7 @@ import com.facebook.buck.io.filesystem.ProjectFilesystem; import com.facebook.buck.io.filesystem.impl.FakeProjectFilesystem; import com.facebook.buck.util.Ansi; +import com.facebook.buck.util.environment.Platform; import com.google.common.collect.ImmutableList; import java.nio.file.Path; import java.nio.file.Paths; @@ -94,6 +96,7 @@ public void setUp() { @Test public void shouldProperlyTransformErrorLinesInPrimaryIncludeTrace() { + assumeTrue(Platform.detect() != Platform.WINDOWS); assertThat( transformer.transformLine( pathResolver, String.format("In file included from %s:", originalPath)), @@ -114,6 +117,7 @@ public void shouldProperlyTransformErrorLinesInPrimaryIncludeTrace() { @Test public void shouldProperlyTransformLinesInSubsequentIncludeTrace() { + assumeTrue(Platform.detect() != Platform.WINDOWS); assertThat( transformer.transformLine(pathResolver, String.format(" from %s:", originalPath)), equalTo(String.format(" from %s:", expectedPath))); @@ -130,6 +134,7 @@ public void shouldProperlyTransformLinesInSubsequentIncludeTrace() { @Test public void shouldProperlyTransformLinesInErrorMessages() { + assumeTrue(Platform.detect() != Platform.WINDOWS); assertThat( transformer.transformLine(pathResolver, String.format("%s: something bad", originalPath)), equalTo(String.format("%s: something bad", expectedPath))); @@ -144,6 +149,7 @@ public void shouldProperlyTransformLinesInErrorMessages() { @Test public void shouldProperlyTransformColoredLinesInErrorMessages() { + assumeTrue(Platform.detect() != Platform.WINDOWS); Ansi ansi = new Ansi(/* isAnsiTerminal */ true); assertThat( transformer.transformLine( diff --git a/test/com/facebook/buck/cxx/CxxGtestTestTest.java b/test/com/facebook/buck/cxx/CxxGtestTestTest.java index 1de47ffa6f2..cfdd905fd89 100644 --- a/test/com/facebook/buck/cxx/CxxGtestTestTest.java +++ b/test/com/facebook/buck/cxx/CxxGtestTestTest.java @@ -133,7 +133,25 @@ public void testParseResults() throws Exception { ObjectMappers.readValue(summaries, SUMMARIES_REFERENCE); ImmutableList actualSummaries = test.parseResults(exitCode, output, results); - assertEquals(sample, expectedSummaries, actualSummaries); + // The lines in expectedSummaries are read from a data file and are separated by "\n". On Windows + // platform, the generated actualSummaries are separated by "\r\n", which is actually the expected + // result. In order to make the test pass, change the actualSummaries. + ImmutableList.Builder linuxActualSummaryBuilder = ImmutableList.builder(); + + for (TestResultSummary testSummary: actualSummaries) { + linuxActualSummaryBuilder.add(new TestResultSummary( + testSummary.getTestCaseName(), + testSummary.getTestName(), + testSummary.getType(), + testSummary.getTime(), + testSummary.getMessage(), + testSummary.getStacktrace(), + testSummary.getStdOut().replaceAll("\r", ""), + testSummary.getStdErr() + )); + } + + assertEquals(sample, expectedSummaries, linuxActualSummaryBuilder.build()); } } } diff --git a/test/com/facebook/buck/cxx/testdata/platform_linker_flags/BUCK.fixture b/test/com/facebook/buck/cxx/testdata/platform_linker_flags/BUCK.fixture index 395a71b79ac..ffb1583779d 100644 --- a/test/com/facebook/buck/cxx/testdata/platform_linker_flags/BUCK.fixture +++ b/test/com/facebook/buck/cxx/testdata/platform_linker_flags/BUCK.fixture @@ -3,7 +3,7 @@ TARGETS = { "macos": ["-Wl,-flat_namespace,-undefined,suppress"], } -for target, flags in TARGETS.iteritems(): +for target, flags in TARGETS.items(): cxx_binary( name = "binary_matches_default_exactly_" + target, srcs = [ diff --git a/test/com/facebook/buck/event/EventSerializationTest.java b/test/com/facebook/buck/event/EventSerializationTest.java index 8b80652d3a6..da7ed47f57d 100644 --- a/test/com/facebook/buck/event/EventSerializationTest.java +++ b/test/com/facebook/buck/event/EventSerializationTest.java @@ -237,8 +237,8 @@ public void testTestRunEventFinished() throws IOException { String message = ObjectMappers.WRITER.writeValueAsString(event); assertJsonEquals( "{%s," - + "\"results\":[{\"testCases\":[{\"testCaseName\":\"Test1\",\"testResults\":[{" - + "\"testCaseName\":\"Test1\",\"type\":\"FAILURE\",\"time\":0}]," + + "\"results\":[{\"testCases\":[{\"testCaseName\":\"Test1\",\"testSuite\":false," + + "\"testResults\":[{\"testCaseName\":\"Test1\",\"type\":\"FAILURE\",\"time\":0}]," + "\"failureCount\":1,\"skippedCount\":0,\"totalTime\":0," + "\"success\":false}]," + "\"failureCount\":1,\"contacts\":[],\"labels\":[]," @@ -324,8 +324,8 @@ public void testIndividualTestEventFinished() throws IOException { String message = ObjectMappers.WRITER.writeValueAsString(event); assertJsonEquals( "{%s,\"eventKey\":{\"value\":-594614477}," - + "\"results\":{\"testCases\":[{\"testCaseName\":\"Test1\",\"testResults\":[{" - + "\"testCaseName\":\"Test1\",\"type\":\"FAILURE\",\"time\":0}]," + + "\"results\":{\"testCases\":[{\"testCaseName\":\"Test1\",\"testSuite\":false," + + "\"testResults\":[{\"testCaseName\":\"Test1\",\"type\":\"FAILURE\",\"time\":0}]," + "\"failureCount\":1,\"skippedCount\":0,\"totalTime\":0," + "\"success\":false}]," + "\"failureCount\":1,\"contacts\":[],\"labels\":[]," diff --git a/test/com/facebook/buck/features/apple/project/ProjectIntegrationTest.java b/test/com/facebook/buck/features/apple/project/ProjectIntegrationTest.java index f5f8aaf8e4a..bba8fc19495 100644 --- a/test/com/facebook/buck/features/apple/project/ProjectIntegrationTest.java +++ b/test/com/facebook/buck/features/apple/project/ProjectIntegrationTest.java @@ -627,6 +627,19 @@ public void testHalide() throws IOException { workspace.verify(); } + @Test + public void testBuckProjectGeneratedSchemeWithEnvVariablesAndExpandSetting() throws IOException { + ProjectWorkspace workspace = + TestDataHelper.createProjectWorkspaceForScenario( + this, "project_generated_scheme_with_env_variables_and_expand_setting", temporaryFolder); + workspace.setUp(); + + ProcessResult result = workspace.runBuckCommand("project", "//Apps:workspace"); + result.assertSuccess(); + + workspace.verify(); + } + private void runXcodebuild(ProjectWorkspace workspace, String workspacePath, String schemeName) throws IOException, InterruptedException { ProcessExecutor.Result processResult = diff --git a/test/com/facebook/buck/features/apple/project/SchemeGeneratorTest.java b/test/com/facebook/buck/features/apple/project/SchemeGeneratorTest.java index 4ba92c22ce6..f42e117b94e 100644 --- a/test/com/facebook/buck/features/apple/project/SchemeGeneratorTest.java +++ b/test/com/facebook/buck/features/apple/project/SchemeGeneratorTest.java @@ -128,9 +128,13 @@ public void schemeWithMultipleTargetsBuildsInCorrectOrder() throws Exception { targetToProjectPathMapBuilder.build(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); String schemeXml = projectFilesystem.readFileIfItExists(schemePath).get(); @@ -214,9 +218,13 @@ public void schemeBuildsAndTestsAppleTestTargets() throws Exception { targetToProjectPathMapBuilder.build(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); String schemeXml = projectFilesystem.readFileIfItExists(schemePath).get(); @@ -309,9 +317,13 @@ public void schemeIncludesAllExpectedActions() throws Exception { targetToProjectPathMapBuilder.build(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); String schemeXml = projectFilesystem.readFileIfItExists(schemePath).get(); @@ -372,27 +384,31 @@ public void buildableReferenceShouldHaveExpectedProperties() throws Exception { Path pbxprojectPath = Paths.get("foo/Foo.xcodeproj/project.pbxproj"); targetToProjectPathMapBuilder.put(rootTarget, pbxprojectPath); - SchemeGenerator schemeGenerator = - new SchemeGenerator( - projectFilesystem, - Optional.of(rootTarget), - ImmutableSet.of(rootTarget), - ImmutableSet.of(), - ImmutableSet.of(), - "TestScheme", - Paths.get("_gen/Foo.xcworkspace/scshareddata/xcshemes"), - false /* parallelizeBuild */, - Optional.empty() /* wasCreatedForAppExtension */, - Optional.empty() /* runnablePath */, - Optional.empty() /* remoteRunnablePath */, - SchemeActionType.DEFAULT_CONFIG_NAMES, - targetToProjectPathMapBuilder.build(), - Optional.empty(), - Optional.empty(), - XCScheme.LaunchAction.LaunchStyle.AUTO, - Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + new SchemeGenerator( + projectFilesystem, + Optional.of(rootTarget), + ImmutableSet.of(rootTarget), + ImmutableSet.of(), + ImmutableSet.of(), + "TestScheme", + Paths.get("_gen/Foo.xcworkspace/scshareddata/xcshemes"), + false /* parallelizeBuild */, + Optional.empty() /* wasCreatedForAppExtension */, + Optional.empty() /* runnablePath */, + Optional.empty() /* remoteRunnablePath */, + SchemeActionType.DEFAULT_CONFIG_NAMES, + targetToProjectPathMapBuilder.build(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + XCScheme.LaunchAction.LaunchStyle.AUTO, + Optional.empty(), /* watchAdapter */ + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); + Path schemePath = schemeGenerator.writeScheme(); @@ -453,9 +469,13 @@ public void allActionsShouldBePresentInSchemeWithDefaultBuildConfigurations() th targetToProjectPathMapBuilder.build(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); @@ -540,9 +560,13 @@ public void schemeIsRewrittenIfContentsHaveChanged() throws IOException { targetToProjectPathMapBuilder.build(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); assertThat( @@ -578,9 +602,13 @@ public void schemeIsRewrittenIfContentsHaveChanged() throws IOException { ImmutableMap.of(rootTarget, pbxprojectPath), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); assertThat( @@ -619,9 +647,13 @@ public void schemeIsNotRewrittenIfContentsHaveNotChanged() throws IOException { ImmutableMap.of(rootTarget, pbxprojectPath), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); assertThat( @@ -657,9 +689,13 @@ public void schemeIsNotRewrittenIfContentsHaveNotChanged() throws IOException { ImmutableMap.of(rootTarget, pbxprojectPath), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); assertThat( projectFilesystem.getLastModifiedTime(schemePath), equalTo(FileTime.fromMillis(49152L))); @@ -719,9 +755,13 @@ public void schemeWithNoPrimaryRuleCanIncludeTests() throws Exception { targetToProjectPathMapBuilder.build(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); String schemeXml = projectFilesystem.readFileIfItExists(schemePath).get(); @@ -828,9 +868,13 @@ public void launchActionShouldNotContainRemoteRunnableWhenNotProvided() throws E targetToProjectPathMapBuilder.build(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); @@ -881,9 +925,13 @@ public void launchActionShouldContainRemoteRunnableWhenProvided() throws Excepti targetToProjectPathMapBuilder.build(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* payloadNotificationFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); @@ -962,10 +1010,14 @@ public void prePostActionsSerializedWithRootBuildable() throws Exception { SchemeActionType.DEFAULT_CONFIG_NAMES, targetToProjectPathMapBuilder.build(), Optional.empty(), + Optional.empty(), + Optional.empty(), Optional.of(schemeActions), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); @@ -1039,9 +1091,13 @@ public void enablingParallelizeBuild() throws Exception { targetToProjectPathMapBuilder.build(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notifcationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); @@ -1102,9 +1158,13 @@ public void serializesEnvironmentVariables() throws Exception { targetToProjectPathMapBuilder.build(), Optional.of(environmentVariables), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); @@ -1124,6 +1184,132 @@ public void serializesEnvironmentVariables() throws Exception { assertThat(envVar.getAttributes().getNamedItem("value").getNodeValue(), equalTo("IS_SET")); } + @Test + public void serializesCommandLineArguments() throws Exception { + ImmutableMap.Builder targetToProjectPathMapBuilder = ImmutableMap.builder(); + + PBXTarget rootTarget = + new PBXNativeTarget("rootRule", AbstractPBXObjectFactory.DefaultFactory()); + rootTarget.setGlobalID("rootGID"); + rootTarget.setProductReference( + new PBXFileReference( + "root.a", "root.a", PBXReference.SourceTree.BUILT_PRODUCTS_DIR, Optional.empty())); + rootTarget.setProductType(ProductTypes.STATIC_LIBRARY); + + Path pbxprojectPath = Paths.get("foo/Foo.xcodeproj/project.pbxproj"); + targetToProjectPathMapBuilder.put(rootTarget, pbxprojectPath); + + ImmutableMap> commandLineArguments = + ImmutableMap.of(SchemeActionType.LAUNCH, ImmutableMap.of("COMMAND_ARG", "YES")); + + SchemeGenerator schemeGenerator = + new SchemeGenerator( + projectFilesystem, + Optional.of(rootTarget), + ImmutableSet.of(rootTarget), + ImmutableSet.of(), + ImmutableSet.of(), + "TestScheme", + Paths.get("_gen/Foo.xcworkspace/xcshareddata/xcshemes"), + true /* parallelizeBuild */, + Optional.of(true) /* wasCreatedForAppExtension */, + Optional.empty() /* runnablePath */, + Optional.empty() /* remoteRunnablePath */, + SchemeActionType.DEFAULT_CONFIG_NAMES, + targetToProjectPathMapBuilder.build(), + Optional.empty(), + Optional.empty(), + Optional.of(commandLineArguments), /* commandLineArguments */ + Optional.empty(), + XCScheme.LaunchAction.LaunchStyle.AUTO, + Optional.empty(), /* watchAdapter */ + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); + + Path schemePath = schemeGenerator.writeScheme(); + + DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); + Document scheme = dBuilder.parse(projectFilesystem.newFileInputStream(schemePath)); + + XPathFactory xpathFactory = XPathFactory.newInstance(); + XPath buildActionXpath = xpathFactory.newXPath(); + XPathExpression buildActionExpr = + buildActionXpath.compile("//LaunchAction/CommandLineArguments/CommandLineArgument"); + NodeList commandLineArgsList = (NodeList) buildActionExpr.evaluate(scheme, XPathConstants.NODESET); + + assertThat(commandLineArgsList.getLength(), is(1)); + Node commandLineArgument = commandLineArgsList.item(0); + assertThat(commandLineArgument.getAttributes().getNamedItem("argument").getNodeValue(), equalTo("COMMAND_ARG")); + assertThat(commandLineArgument.getAttributes().getNamedItem("isEnabled").getNodeValue(), equalTo("YES")); + } + + @Test + public void serializesRegionAndLanguage() throws Exception { + ImmutableMap.Builder targetToProjectPathMapBuilder = ImmutableMap.builder(); + + PBXTarget rootTarget = + new PBXNativeTarget("rootRule", AbstractPBXObjectFactory.DefaultFactory()); + rootTarget.setGlobalID("rootGID"); + rootTarget.setProductReference( + new PBXFileReference( + "root.a", "root.a", PBXReference.SourceTree.BUILT_PRODUCTS_DIR, Optional.empty())); + rootTarget.setProductType(ProductTypes.STATIC_LIBRARY); + + Path pbxprojectPath = Paths.get("foo/Foo.xcodeproj/project.pbxproj"); + targetToProjectPathMapBuilder.put(rootTarget, pbxprojectPath); + + String region = "pt-BR"; + String language = "pt"; + + SchemeGenerator schemeGenerator = + new SchemeGenerator( + projectFilesystem, + Optional.of(rootTarget), + ImmutableSet.of(rootTarget), + ImmutableSet.of(), + ImmutableSet.of(), + "TestScheme", + Paths.get("_gen/Foo.xcworkspace/scshareddata/xcshemes"), + true /* parallelizeBuild */, + Optional.empty() /* wasCreatedForAppExtension */, + Optional.empty() /* runnablePath */, + Optional.empty() /* remoteRunnablePath */, + SchemeActionType.DEFAULT_CONFIG_NAMES, + targetToProjectPathMapBuilder.build(), + Optional.empty() /* environmentVariables */, + Optional.empty(), + Optional.empty(), + Optional.empty(), + XCScheme.LaunchAction.LaunchStyle.AUTO, + Optional.empty(), /* watchAdapter */ + Optional.empty(), /* notificationPayloadFile */ + Optional.of(region), + Optional.of(language)); + + Path schemePath = schemeGenerator.writeScheme(); + + DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); + Document scheme = dBuilder.parse(projectFilesystem.newFileInputStream(schemePath)); + + XPathFactory xpathFactory = XPathFactory.newInstance(); + XPath buildActionXpath = xpathFactory.newXPath(); + + XPathExpression launchActionExpr = buildActionXpath.compile("//LaunchAction"); + XPathExpression testActionExpr = buildActionXpath.compile("//TestAction"); + + Node launchActionNode = (Node) launchActionExpr.evaluate(scheme, XPathConstants.NODE); + Node testActionNode = (Node) testActionExpr.evaluate(scheme, XPathConstants.NODE); + + assertThat(launchActionNode.getAttributes().getNamedItem("language").getNodeValue(), equalTo("pt-BR")); + assertThat(launchActionNode.getAttributes().getNamedItem("region").getNodeValue(), equalTo("pt")); + + assertThat(testActionNode.getAttributes().getNamedItem("language").getNodeValue(), equalTo("pt-BR")); + assertThat(testActionNode.getAttributes().getNamedItem("region").getNodeValue(), equalTo("pt")); + } + /** * Include `wasCreatedForAppExtension` when true. * @@ -1161,9 +1347,13 @@ public void serializesWasCreatedForAppExtension() throws Exception { targetToProjectPathMapBuilder.build(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); @@ -1224,9 +1414,13 @@ public void excludesWasCreatedForAppExtension() throws Exception { targetToProjectPathMapBuilder.build(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); diff --git a/test/com/facebook/buck/features/apple/project/testdata/project_generated_scheme_with_env_variables_and_expand_setting/.buckconfig b/test/com/facebook/buck/features/apple/project/testdata/project_generated_scheme_with_env_variables_and_expand_setting/.buckconfig new file mode 100644 index 00000000000..8e8064188b4 --- /dev/null +++ b/test/com/facebook/buck/features/apple/project/testdata/project_generated_scheme_with_env_variables_and_expand_setting/.buckconfig @@ -0,0 +1,2 @@ +[project] + ide = xcode diff --git a/test/com/facebook/buck/features/apple/project/testdata/project_generated_scheme_with_env_variables_and_expand_setting/Apps/BUCK.fixture b/test/com/facebook/buck/features/apple/project/testdata/project_generated_scheme_with_env_variables_and_expand_setting/Apps/BUCK.fixture new file mode 100644 index 00000000000..add098c977c --- /dev/null +++ b/test/com/facebook/buck/features/apple/project/testdata/project_generated_scheme_with_env_variables_and_expand_setting/Apps/BUCK.fixture @@ -0,0 +1,46 @@ +EMPTY_CONFIGS = { + "Debug": {}, + "Release": {}, +} + +apple_binary( + name = "TestAppBinary", + srcs = [], + configs = EMPTY_CONFIGS, + frameworks = [], + deps = [ + ], +) + +apple_bundle( + name = "TestApp", + binary = ":TestAppBinary", + extension = "app", + info_plist = "Info.plist", + tests = [":TestAppTests"], + deps = [":TestAppBinary"], +) + +apple_test( + name = "TestAppTests", + srcs = [], + configs = EMPTY_CONFIGS, + frameworks = [], + info_plist = "Test.plist", + deps = [ + ":TestApp", + ], +) + +xcode_workspace_config( + name = "workspace", + src_target = "//Apps:TestApp", + environment_variables = { + 'Test': { + 'TEST_ENV': 'SET' + } + }, + expand_variables_based_on = { + 'Test': ':TestApp' + } +) diff --git a/test/com/facebook/buck/features/apple/project/testdata/project_generated_scheme_with_env_variables_and_expand_setting/Apps/Info.plist b/test/com/facebook/buck/features/apple/project/testdata/project_generated_scheme_with_env_variables_and_expand_setting/Apps/Info.plist new file mode 100644 index 00000000000..1c8b7382b3e --- /dev/null +++ b/test/com/facebook/buck/features/apple/project/testdata/project_generated_scheme_with_env_variables_and_expand_setting/Apps/Info.plist @@ -0,0 +1,43 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + DemoApp + CFBundleIdentifier + com.example.DemoApp + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + DemoApp + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleSignature + ???? + CFBundleVersion + 1 + LSRequiresIPhoneOS + + UIRequiredDeviceCapabilities + + armv7 + + UISupportedInterfaceOrientations + + UIInterfaceOrientationPortrait + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + UISupportedInterfaceOrientations~ipad + + UIInterfaceOrientationPortrait + UIInterfaceOrientationPortraitUpsideDown + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + + diff --git a/test/com/facebook/buck/features/apple/project/testdata/project_generated_scheme_with_env_variables_and_expand_setting/Apps/Test.plist b/test/com/facebook/buck/features/apple/project/testdata/project_generated_scheme_with_env_variables_and_expand_setting/Apps/Test.plist new file mode 100644 index 00000000000..7c13f891a0a --- /dev/null +++ b/test/com/facebook/buck/features/apple/project/testdata/project_generated_scheme_with_env_variables_and_expand_setting/Apps/Test.plist @@ -0,0 +1,22 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + ${EXECUTABLE_NAME} + CFBundleIdentifier + com.facebook.${PRODUCT_NAME:rfc1034identifier} + CFBundleInfoDictionaryVersion + 6.0 + CFBundlePackageType + BNDL + CFBundleShortVersionString + 1.0 + CFBundleSignature + ???? + CFBundleVersion + 1 + + diff --git a/test/com/facebook/buck/features/apple/project/testdata/project_generated_scheme_with_env_variables_and_expand_setting/Apps/TestApp.xcworkspace/xcshareddata/xcschemes/TestApp.xcscheme.expected b/test/com/facebook/buck/features/apple/project/testdata/project_generated_scheme_with_env_variables_and_expand_setting/Apps/TestApp.xcworkspace/xcshareddata/xcschemes/TestApp.xcscheme.expected new file mode 100644 index 00000000000..5620bbe23f9 --- /dev/null +++ b/test/com/facebook/buck/features/apple/project/testdata/project_generated_scheme_with_env_variables_and_expand_setting/Apps/TestApp.xcworkspace/xcshareddata/xcschemes/TestApp.xcscheme.expected @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/com/facebook/buck/features/apple/projectV2/ProjectIntegrationTest.java b/test/com/facebook/buck/features/apple/projectV2/ProjectIntegrationTest.java index bb7ed85cd0f..e591a8589b8 100644 --- a/test/com/facebook/buck/features/apple/projectV2/ProjectIntegrationTest.java +++ b/test/com/facebook/buck/features/apple/projectV2/ProjectIntegrationTest.java @@ -515,6 +515,21 @@ public void testBuckProjectWithSwiftDependencyOnModularObjectiveCLibrary() // workspace.verify(); // } + @Test + public void testBuckProjectGeneratedSchemeWithEnvVariablesAndExpandSetting() + throws IOException, InterruptedException { + assumeTrue(Platform.detect() == Platform.MACOS); + assumeTrue(AppleNativeIntegrationTestUtils.isApplePlatformAvailable(ApplePlatform.MACOSX)); + + ProjectWorkspace workspace = + createWorkspace(this, "project_generated_scheme_with_env_variables_and_expand_setting"); + + ProcessResult result = workspace.runBuckCommand("project", "//Apps:workspace"); + result.assertSuccess(); + + runXcodebuild(workspace, "Apps/TestApp.xcworkspace", "TestApp"); + } + private void runXcodebuild(ProjectWorkspace workspace, String workspacePath, String schemeName) throws IOException, InterruptedException { ProcessExecutor.Result processResult = diff --git a/test/com/facebook/buck/features/apple/projectV2/SchemeGeneratorTest.java b/test/com/facebook/buck/features/apple/projectV2/SchemeGeneratorTest.java index 2a88294ccaf..c9f0a0498ec 100644 --- a/test/com/facebook/buck/features/apple/projectV2/SchemeGeneratorTest.java +++ b/test/com/facebook/buck/features/apple/projectV2/SchemeGeneratorTest.java @@ -38,6 +38,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import java.io.File; import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; @@ -128,9 +129,13 @@ public void schemeWithMultipleTargetsBuildsInCorrectOrder() throws Exception { targetToProjectPathMapBuilder.build(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationLanguage */ + Optional.empty() /* applicationRegion */); Path schemePath = schemeGenerator.writeScheme(); String schemeXml = projectFilesystem.readFileIfItExists(schemePath).get(); @@ -214,9 +219,13 @@ public void schemeBuildsAndTestsAppleTestTargets() throws Exception { targetToProjectPathMapBuilder.build(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); String schemeXml = projectFilesystem.readFileIfItExists(schemePath).get(); @@ -309,9 +318,13 @@ public void schemeIncludesAllExpectedActions() throws Exception { targetToProjectPathMapBuilder.build(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); String schemeXml = projectFilesystem.readFileIfItExists(schemePath).get(); @@ -390,9 +403,13 @@ public void buildableReferenceShouldHaveExpectedProperties() throws Exception { targetToProjectPathMapBuilder.build(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); @@ -453,9 +470,13 @@ public void allActionsShouldBePresentInSchemeWithDefaultBuildConfigurations() th targetToProjectPathMapBuilder.build(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); @@ -540,9 +561,13 @@ public void schemeIsRewrittenIfContentsHaveChanged() throws IOException { targetToProjectPathMapBuilder.build(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); assertThat( @@ -578,9 +603,13 @@ public void schemeIsRewrittenIfContentsHaveChanged() throws IOException { ImmutableMap.of(rootTarget, pbxprojectPath), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); assertThat( @@ -619,9 +648,13 @@ public void schemeIsNotRewrittenIfContentsHaveNotChanged() throws IOException { ImmutableMap.of(rootTarget, pbxprojectPath), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); assertThat( @@ -657,9 +690,13 @@ public void schemeIsNotRewrittenIfContentsHaveNotChanged() throws IOException { ImmutableMap.of(rootTarget, pbxprojectPath), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); assertThat( projectFilesystem.getLastModifiedTime(schemePath), equalTo(FileTime.fromMillis(49152L))); @@ -719,9 +756,13 @@ public void schemeWithNoPrimaryRuleCanIncludeTests() throws Exception { targetToProjectPathMapBuilder.build(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); String schemeXml = projectFilesystem.readFileIfItExists(schemePath).get(); @@ -828,9 +869,13 @@ public void launchActionShouldNotContainRemoteRunnableWhenNotProvided() throws E targetToProjectPathMapBuilder.build(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); @@ -881,9 +926,13 @@ public void launchActionShouldContainRemoteRunnableWhenProvided() throws Excepti targetToProjectPathMapBuilder.build(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* payloadNotificationFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); @@ -962,10 +1011,14 @@ public void prePostActionsSerializedWithRootBuildable() throws Exception { SchemeActionType.DEFAULT_CONFIG_NAMES, targetToProjectPathMapBuilder.build(), Optional.empty(), + Optional.empty(), + Optional.empty(), Optional.of(schemeActions), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); @@ -1039,9 +1092,13 @@ public void enablingParallelizeBuild() throws Exception { targetToProjectPathMapBuilder.build(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notifcationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); @@ -1102,9 +1159,13 @@ public void serializesEnvironmentVariables() throws Exception { targetToProjectPathMapBuilder.build(), Optional.of(environmentVariables), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); @@ -1124,6 +1185,199 @@ public void serializesEnvironmentVariables() throws Exception { assertThat(envVar.getAttributes().getNamedItem("value").getNodeValue(), equalTo("IS_SET")); } + @Test + public void serializesCommandLineArguments() throws Exception { + ImmutableMap.Builder targetToProjectPathMapBuilder = ImmutableMap.builder(); + + PBXTarget rootTarget = + new PBXNativeTarget("rootRule", AbstractPBXObjectFactory.DefaultFactory()); + rootTarget.setGlobalID("rootGID"); + rootTarget.setProductReference( + new PBXFileReference( + "root.a", "root.a", PBXReference.SourceTree.BUILT_PRODUCTS_DIR, Optional.empty())); + rootTarget.setProductType(ProductTypes.STATIC_LIBRARY); + + Path pbxprojectPath = Paths.get("foo/Foo.xcodeproj/project.pbxproj"); + targetToProjectPathMapBuilder.put(rootTarget, pbxprojectPath); + + ImmutableMap> commandLineArguments = + ImmutableMap.of(SchemeActionType.LAUNCH, ImmutableMap.of("COMMAND_ARG", "YES")); + + SchemeGenerator schemeGenerator = + new SchemeGenerator( + projectFilesystem, + Optional.of(rootTarget), + ImmutableSet.of(rootTarget), + ImmutableSet.of(), + ImmutableSet.of(), + "TestScheme", + Paths.get("_gen/Foo.xcworkspace/xcshareddata/xcshemes"), + true /* parallelizeBuild */, + Optional.of(true) /* wasCreatedForAppExtension */, + Optional.empty() /* runnablePath */, + Optional.empty() /* remoteRunnablePath */, + SchemeActionType.DEFAULT_CONFIG_NAMES, + targetToProjectPathMapBuilder.build(), + Optional.empty(), + Optional.empty(), + Optional.of(commandLineArguments), /* commandLineArguments */ + Optional.empty(), + XCScheme.LaunchAction.LaunchStyle.AUTO, + Optional.empty(), /* watchAdapter */ + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); + + Path schemePath = schemeGenerator.writeScheme(); + + DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); + Document scheme = dBuilder.parse(projectFilesystem.newFileInputStream(schemePath)); + + XPathFactory xpathFactory = XPathFactory.newInstance(); + XPath buildActionXpath = xpathFactory.newXPath(); + XPathExpression buildActionExpr = + buildActionXpath.compile("//LaunchAction/CommandLineArguments/CommandLineArgument"); + NodeList commandLineArgsList = (NodeList) buildActionExpr.evaluate(scheme, XPathConstants.NODESET); + + assertThat(commandLineArgsList.getLength(), is(1)); + Node commandLineArgument = commandLineArgsList.item(0); + assertThat(commandLineArgument.getAttributes().getNamedItem("argument").getNodeValue(), equalTo("COMMAND_ARG")); + assertThat(commandLineArgument.getAttributes().getNamedItem("isEnabled").getNodeValue(), equalTo("YES")); + } + + @Test + public void serializesRegionAndLanguage() throws Exception { + ImmutableMap.Builder targetToProjectPathMapBuilder = ImmutableMap.builder(); + + PBXTarget rootTarget = + new PBXNativeTarget("rootRule", AbstractPBXObjectFactory.DefaultFactory()); + rootTarget.setGlobalID("rootGID"); + rootTarget.setProductReference( + new PBXFileReference( + "root.a", "root.a", PBXReference.SourceTree.BUILT_PRODUCTS_DIR, Optional.empty())); + rootTarget.setProductType(ProductTypes.STATIC_LIBRARY); + + Path pbxprojectPath = Paths.get("foo/Foo.xcodeproj/project.pbxproj"); + targetToProjectPathMapBuilder.put(rootTarget, pbxprojectPath); + + String region = "pt-BR"; + String language = "pt"; + + SchemeGenerator schemeGenerator = + new SchemeGenerator( + projectFilesystem, + Optional.of(rootTarget), + ImmutableSet.of(rootTarget), + ImmutableSet.of(), + ImmutableSet.of(), + "TestScheme", + Paths.get("_gen/Foo.xcworkspace/scshareddata/xcshemes"), + true /* parallelizeBuild */, + Optional.empty() /* wasCreatedForAppExtension */, + Optional.empty() /* runnablePath */, + Optional.empty() /* remoteRunnablePath */, + SchemeActionType.DEFAULT_CONFIG_NAMES, + targetToProjectPathMapBuilder.build(), + Optional.empty() /* environmentVariables */, + Optional.empty(), + Optional.empty(), + Optional.empty(), + XCScheme.LaunchAction.LaunchStyle.AUTO, + Optional.empty(), /* watchAdapter */ + Optional.empty(), /* notificationPayloadFile */ + Optional.of(region), + Optional.of(language)); + + Path schemePath = schemeGenerator.writeScheme(); + + DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); + Document scheme = dBuilder.parse(projectFilesystem.newFileInputStream(schemePath)); + + XPathFactory xpathFactory = XPathFactory.newInstance(); + XPath buildActionXpath = xpathFactory.newXPath(); + + XPathExpression launchActionExpr = buildActionXpath.compile("//LaunchAction"); + XPathExpression testActionExpr = buildActionXpath.compile("//TestAction"); + + Node launchActionNode = (Node) launchActionExpr.evaluate(scheme, XPathConstants.NODE); + Node testActionNode = (Node) testActionExpr.evaluate(scheme, XPathConstants.NODE); + + assertThat(launchActionNode.getAttributes().getNamedItem("language").getNodeValue(), equalTo("pt-BR")); + assertThat(launchActionNode.getAttributes().getNamedItem("region").getNodeValue(), equalTo("pt")); + + assertThat(testActionNode.getAttributes().getNamedItem("language").getNodeValue(), equalTo("pt-BR")); + assertThat(testActionNode.getAttributes().getNamedItem("region").getNodeValue(), equalTo("pt")); + } + + @Test + public void expandVariablesBasedOnSetIfPresent() throws Exception { + ImmutableMap.Builder targetToProjectPathMapBuilder = ImmutableMap.builder(); + + PBXTarget rootTarget = + new PBXNativeTarget("rootRule", AbstractPBXObjectFactory.DefaultFactory()); + rootTarget.setGlobalID("rootGID"); + rootTarget.setProductReference( + new PBXFileReference( + "root.a", "root.a", PBXReference.SourceTree.BUILT_PRODUCTS_DIR, Optional.empty())); + rootTarget.setProductType(ProductTypes.STATIC_LIBRARY); + + Path pbxprojectPath = Paths.get("foo/Foo.xcodeproj/project.pbxproj"); + targetToProjectPathMapBuilder.put(rootTarget, pbxprojectPath); + + ImmutableMap> environmentVariables = + ImmutableMap.of(SchemeActionType.TEST, ImmutableMap.of("ENV_VARIABLE", "IS_SET")); + ImmutableMap expandVariablesBasedOn = + ImmutableMap.of(SchemeActionType.TEST, rootTarget); + + SchemeGenerator schemeGenerator = + new SchemeGenerator( + projectFilesystem, + Optional.of(rootTarget), + ImmutableSet.of(rootTarget), + ImmutableSet.of(), + ImmutableSet.of(), + "TestScheme", + Paths.get("_gen/Foo.xcworkspace/scshareddata/xcshemes"), + true /* parallelizeBuild */, + Optional.empty() /* wasCreatedForAppExtension */, + Optional.empty() /* runnablePath */, + Optional.empty() /* remoteRunnablePath */, + SchemeActionType.DEFAULT_CONFIG_NAMES, + targetToProjectPathMapBuilder.build(), + Optional.of(environmentVariables), + Optional.of(expandVariablesBasedOn), + Optional.empty(), + Optional.empty(), + XCScheme.LaunchAction.LaunchStyle.AUTO, + Optional.empty(), /* watchAdapter */ + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); + + Path schemePath = schemeGenerator.writeScheme(); + + DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder dBuilder = dbFactory.newDocumentBuilder(); + Document scheme = dBuilder.parse(projectFilesystem.newFileInputStream(schemePath)); + + XPathFactory xpathFactory = XPathFactory.newInstance(); + XPath buildActionXpath = xpathFactory.newXPath(); + XPathExpression buildActionExpr = + buildActionXpath.compile("//TestAction/MacroExpansion/BuildableReference"); + NodeList referncesVariableList = (NodeList) buildActionExpr.evaluate(scheme, XPathConstants.NODESET); + + assertThat(referncesVariableList.getLength(), is(1)); + Node reference = referncesVariableList.item(0); + assertThat(reference.getAttributes().getNamedItem("BuildableIdentifier").getNodeValue(), equalTo("primary")); + assertThat(reference.getAttributes().getNamedItem("BlueprintIdentifier").getNodeValue(), equalTo("rootGID")); + assertThat(reference.getAttributes().getNamedItem("BuildableName").getNodeValue(), equalTo("root.a")); + assertThat(reference.getAttributes().getNamedItem("BlueprintName").getNodeValue(), equalTo("rootRule")); + assertThat(reference.getAttributes().getNamedItem("ReferencedContainer").getNodeValue().replace('\\', '/'), + equalTo("container:../../../foo/Foo.xcodeproj/project.pbxproj")); + } + /** * Include `wasCreatedForAppExtension` when true. * @@ -1161,9 +1415,13 @@ public void serializesWasCreatedForAppExtension() throws Exception { targetToProjectPathMapBuilder.build(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); @@ -1224,9 +1482,13 @@ public void excludesWasCreatedForAppExtension() throws Exception { targetToProjectPathMapBuilder.build(), Optional.empty(), Optional.empty(), + Optional.empty(), + Optional.empty(), XCScheme.LaunchAction.LaunchStyle.AUTO, Optional.empty(), /* watchAdapter */ - Optional.empty() /* notificationPayloadFile */); + Optional.empty(), /* notificationPayloadFile */ + Optional.empty(), /* applicationRegion */ + Optional.empty() /* applicationLanguage */); Path schemePath = schemeGenerator.writeScheme(); diff --git a/test/com/facebook/buck/features/apple/projectV2/testdata/project_generated_scheme_with_env_variables_and_expand_setting/.buckconfig b/test/com/facebook/buck/features/apple/projectV2/testdata/project_generated_scheme_with_env_variables_and_expand_setting/.buckconfig new file mode 100644 index 00000000000..8e8064188b4 --- /dev/null +++ b/test/com/facebook/buck/features/apple/projectV2/testdata/project_generated_scheme_with_env_variables_and_expand_setting/.buckconfig @@ -0,0 +1,2 @@ +[project] + ide = xcode diff --git a/test/com/facebook/buck/features/apple/projectV2/testdata/project_generated_scheme_with_env_variables_and_expand_setting/Apps/BUCK.fixture b/test/com/facebook/buck/features/apple/projectV2/testdata/project_generated_scheme_with_env_variables_and_expand_setting/Apps/BUCK.fixture new file mode 100644 index 00000000000..add098c977c --- /dev/null +++ b/test/com/facebook/buck/features/apple/projectV2/testdata/project_generated_scheme_with_env_variables_and_expand_setting/Apps/BUCK.fixture @@ -0,0 +1,46 @@ +EMPTY_CONFIGS = { + "Debug": {}, + "Release": {}, +} + +apple_binary( + name = "TestAppBinary", + srcs = [], + configs = EMPTY_CONFIGS, + frameworks = [], + deps = [ + ], +) + +apple_bundle( + name = "TestApp", + binary = ":TestAppBinary", + extension = "app", + info_plist = "Info.plist", + tests = [":TestAppTests"], + deps = [":TestAppBinary"], +) + +apple_test( + name = "TestAppTests", + srcs = [], + configs = EMPTY_CONFIGS, + frameworks = [], + info_plist = "Test.plist", + deps = [ + ":TestApp", + ], +) + +xcode_workspace_config( + name = "workspace", + src_target = "//Apps:TestApp", + environment_variables = { + 'Test': { + 'TEST_ENV': 'SET' + } + }, + expand_variables_based_on = { + 'Test': ':TestApp' + } +) diff --git a/test/com/facebook/buck/features/apple/projectV2/testdata/project_generated_scheme_with_env_variables_and_expand_setting/Apps/Info.plist b/test/com/facebook/buck/features/apple/projectV2/testdata/project_generated_scheme_with_env_variables_and_expand_setting/Apps/Info.plist new file mode 100644 index 00000000000..1c8b7382b3e --- /dev/null +++ b/test/com/facebook/buck/features/apple/projectV2/testdata/project_generated_scheme_with_env_variables_and_expand_setting/Apps/Info.plist @@ -0,0 +1,43 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + DemoApp + CFBundleIdentifier + com.example.DemoApp + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + DemoApp + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleSignature + ???? + CFBundleVersion + 1 + LSRequiresIPhoneOS + + UIRequiredDeviceCapabilities + + armv7 + + UISupportedInterfaceOrientations + + UIInterfaceOrientationPortrait + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + UISupportedInterfaceOrientations~ipad + + UIInterfaceOrientationPortrait + UIInterfaceOrientationPortraitUpsideDown + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + + diff --git a/test/com/facebook/buck/features/apple/projectV2/testdata/project_generated_scheme_with_env_variables_and_expand_setting/Apps/Test.plist b/test/com/facebook/buck/features/apple/projectV2/testdata/project_generated_scheme_with_env_variables_and_expand_setting/Apps/Test.plist new file mode 100644 index 00000000000..7c13f891a0a --- /dev/null +++ b/test/com/facebook/buck/features/apple/projectV2/testdata/project_generated_scheme_with_env_variables_and_expand_setting/Apps/Test.plist @@ -0,0 +1,22 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + ${EXECUTABLE_NAME} + CFBundleIdentifier + com.facebook.${PRODUCT_NAME:rfc1034identifier} + CFBundleInfoDictionaryVersion + 6.0 + CFBundlePackageType + BNDL + CFBundleShortVersionString + 1.0 + CFBundleSignature + ???? + CFBundleVersion + 1 + + diff --git a/test/com/facebook/buck/features/apple/projectV2/testdata/project_generated_scheme_with_env_variables_and_expand_setting/Apps/TestApp.xcworkspace/xcshareddata/xcschemes/TestApp.xcscheme.expected b/test/com/facebook/buck/features/apple/projectV2/testdata/project_generated_scheme_with_env_variables_and_expand_setting/Apps/TestApp.xcworkspace/xcshareddata/xcschemes/TestApp.xcscheme.expected new file mode 100644 index 00000000000..5620bbe23f9 --- /dev/null +++ b/test/com/facebook/buck/features/apple/projectV2/testdata/project_generated_scheme_with_env_variables_and_expand_setting/Apps/TestApp.xcworkspace/xcshareddata/xcschemes/TestApp.xcscheme.expected @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/test/com/facebook/buck/features/d/testdata/test/test_spinning.d b/test/com/facebook/buck/features/d/testdata/test/test_spinning.d index 665fc3f10d9..fc4496e07cc 100644 --- a/test/com/facebook/buck/features/d/testdata/test/test_spinning.d +++ b/test/com/facebook/buck/features/d/testdata/test/test_spinning.d @@ -1,10 +1,10 @@ public import core.thread; unittest { -} - -void main() { for( ; ; ) { Thread.sleep( dur!("seconds")( 5 ) ); } -} \ No newline at end of file +} + +void main() { +} diff --git a/test/com/facebook/buck/features/ocaml/OCamlIntegrationTest.java b/test/com/facebook/buck/features/ocaml/OCamlIntegrationTest.java index 7bba2eebc20..ef3448badae 100644 --- a/test/com/facebook/buck/features/ocaml/OCamlIntegrationTest.java +++ b/test/com/facebook/buck/features/ocaml/OCamlIntegrationTest.java @@ -31,6 +31,7 @@ import com.facebook.buck.core.model.BuildTarget; import com.facebook.buck.core.model.BuildTargetFactory; import com.facebook.buck.core.model.UnconfiguredTargetConfiguration; +import com.facebook.buck.core.model.impl.BuildTargetPaths; import com.facebook.buck.core.rules.BuildRuleResolver; import com.facebook.buck.core.rules.resolver.impl.TestActionGraphBuilder; import com.facebook.buck.core.toolchain.ToolchainCreationContext; @@ -229,7 +230,12 @@ public void testNativePlugin() throws Exception { workspace.runBuckCommand("build", binTarget.toString()).assertSuccess(); Path ocamlNativePluginDir = - workspace.getDestPath().resolve("buck-out").resolve("gen").resolve("ocaml_native_plugin"); + workspace.getPath( + BuildTargetPaths.getGenPath( + workspace.getProjectFileSystem(), + pluginTarget, + "%s" + )).getParent(); Path pluginCmxsFile = ocamlNativePluginDir.resolve("plugin#default").resolve("libplugin.cmxs"); @@ -517,8 +523,8 @@ public void testConfigWarningsFlags() throws IOException { workspace.runBuckCommand("build", target.toString()).assertFailure(); BuckBuildLog buildLog = workspace.getBuildLog(); assertTrue(buildLog.getAllTargets().containsAll(targets)); - buildLog.assertTargetCanceled(target); - buildLog.assertTargetCanceled(binary); + buildLog.assertTargetFailed(target.toString()); + buildLog.assertTargetFailed(binary.toString()); workspace.resetBuildLogFile(); workspace.replaceFileContents(".buckconfig", "warnings_flags=+a", ""); @@ -552,8 +558,8 @@ public void testConfigInteropIncludes() throws IOException, InterruptedException workspace.runBuckCommand("build", target.toString()).assertFailure(); BuckBuildLog buildLog = workspace.getBuildLog(); assertThat(buildLog.getAllTargets(), Matchers.hasItems(targets.toArray(new BuildTarget[0]))); - buildLog.assertTargetCanceled(target); - buildLog.assertTargetCanceled(binary); + buildLog.assertTargetFailed(target.toString()); + buildLog.assertTargetFailed(binary.toString()); workspace.resetBuildLogFile(); diff --git a/test/com/facebook/buck/features/project/intellij/testdata/project_with_zipfile/dep/BUCK.fixture b/test/com/facebook/buck/features/project/intellij/testdata/project_with_zipfile/dep/BUCK.fixture index 68a18cc731e..2ad1291e4dc 100644 --- a/test/com/facebook/buck/features/project/intellij/testdata/project_with_zipfile/dep/BUCK.fixture +++ b/test/com/facebook/buck/features/project/intellij/testdata/project_with_zipfile/dep/BUCK.fixture @@ -3,6 +3,7 @@ genrule( srcs = ["A.java"], out = ".", cmd = "cp $SRCS $OUT", + cmd_exe = "copy %SRCS% %OUT%", ) zip_file( diff --git a/test/com/facebook/buck/features/project/intellij/testdata/project_with_zipfile/dep1/BUCK.fixture b/test/com/facebook/buck/features/project/intellij/testdata/project_with_zipfile/dep1/BUCK.fixture index 1caa52b2c9c..3c48e80cda8 100644 --- a/test/com/facebook/buck/features/project/intellij/testdata/project_with_zipfile/dep1/BUCK.fixture +++ b/test/com/facebook/buck/features/project/intellij/testdata/project_with_zipfile/dep1/BUCK.fixture @@ -3,6 +3,7 @@ genrule( srcs = ["A.java"], out = ".", cmd = "cp $SRCS $OUT", + cmd_exe = "copy %SRCS% %OUT%", ) zip_file( diff --git a/test/com/facebook/buck/features/project/intellij/testdata/project_with_zipfile/dep2/BUCK.fixture b/test/com/facebook/buck/features/project/intellij/testdata/project_with_zipfile/dep2/BUCK.fixture index 744f9320f68..6132615aa47 100644 --- a/test/com/facebook/buck/features/project/intellij/testdata/project_with_zipfile/dep2/BUCK.fixture +++ b/test/com/facebook/buck/features/project/intellij/testdata/project_with_zipfile/dep2/BUCK.fixture @@ -3,6 +3,7 @@ genrule( srcs = ["A.java"], out = ".", cmd = "cp $SRCS $OUT", + cmd_exe = "copy %SRCS% %OUT%", ) zip_file( diff --git a/test/com/facebook/buck/features/project/intellij/testdata/project_with_zipfile/prod/BUCK.fixture b/test/com/facebook/buck/features/project/intellij/testdata/project_with_zipfile/prod/BUCK.fixture index ff17ff6a79b..55f93ca5143 100644 --- a/test/com/facebook/buck/features/project/intellij/testdata/project_with_zipfile/prod/BUCK.fixture +++ b/test/com/facebook/buck/features/project/intellij/testdata/project_with_zipfile/prod/BUCK.fixture @@ -14,6 +14,7 @@ genrule( srcs = ["A.java"], out = ".", cmd = "cp $SRCS $OUT", + cmd_exe = "copy %SRCS% %OUT%", ) zip_file( diff --git a/test/com/facebook/buck/features/python/PythonBinaryIntegrationTest.java b/test/com/facebook/buck/features/python/PythonBinaryIntegrationTest.java index edd44f33814..c8363b2815d 100644 --- a/test/com/facebook/buck/features/python/PythonBinaryIntegrationTest.java +++ b/test/com/facebook/buck/features/python/PythonBinaryIntegrationTest.java @@ -608,7 +608,7 @@ public void inplaceBinaryUsesInterpreterFlags() throws IOException { (path, attr) -> path.getFileName().toString().endsWith(".pyc")) .map(path -> tmp.getRoot().relativize(path)) .collect(ImmutableList.toImmutableList()); - Assert.assertEquals(3, pycFiles.size()); + Assert.assertEquals(4, pycFiles.size()); } @Test diff --git a/test/com/facebook/buck/features/python/testdata/python_platform/file.py b/test/com/facebook/buck/features/python/testdata/python_platform/file.py index ecae0477078..fc8541d0d25 100644 --- a/test/com/facebook/buck/features/python/testdata/python_platform/file.py +++ b/test/com/facebook/buck/features/python/testdata/python_platform/file.py @@ -1 +1 @@ -print "I'm a file. A lonely, lonely file." +print ("I'm a file. A lonely, lonely file.") diff --git a/test/com/facebook/buck/features/python/toolchain/impl/PythonInterpreterFromConfigTest.java b/test/com/facebook/buck/features/python/toolchain/impl/PythonInterpreterFromConfigTest.java index a652506d594..eef92c987fe 100644 --- a/test/com/facebook/buck/features/python/toolchain/impl/PythonInterpreterFromConfigTest.java +++ b/test/com/facebook/buck/features/python/toolchain/impl/PythonInterpreterFromConfigTest.java @@ -98,9 +98,9 @@ public void whenPythonPlusExtensionOnPathIsExecutableFileThenItIsUsed() throws I } @Test - public void whenPython2OnPathThenItIsUsed() throws IOException { + public void whenPython3OnPathThenItIsUsed() throws IOException { temporaryFolder.newExecutableFile("python"); - Path python2 = temporaryFolder.newExecutableFile("python2"); + Path python3 = temporaryFolder.newExecutableFile("python3"); PythonBuckConfig config = new PythonBuckConfig( FakeBuckConfig.builder() @@ -114,8 +114,8 @@ public void whenPython2OnPathThenItIsUsed() throws IOException { new PythonInterpreterFromConfig(config, new ExecutableFinder()); assertEquals( - "Should return path to python2.", - python2.toAbsolutePath(), + "Should return path to python3.", + python3.toAbsolutePath(), pythonInterpreter.getPythonInterpreterPath(config.getDefaultSection())); } @@ -141,8 +141,8 @@ public void whenPythonOnPathNotFoundThenThrow() { @Test public void whenMultiplePythonExecutablesOnPathFirstIsUsed() throws IOException { - Path pythonA = temporaryFolder.newExecutableFile("python2"); - temporaryFolder2.newExecutableFile("python2"); + Path pythonA = temporaryFolder.newExecutableFile("python3"); + temporaryFolder2.newExecutableFile("python3"); String path = temporaryFolder.getRoot().toAbsolutePath() + File.pathSeparator diff --git a/test/com/facebook/buck/features/rust/RustBinaryIntegrationTest.java b/test/com/facebook/buck/features/rust/RustBinaryIntegrationTest.java index bd51ebe8b35..73ec847acb7 100644 --- a/test/com/facebook/buck/features/rust/RustBinaryIntegrationTest.java +++ b/test/com/facebook/buck/features/rust/RustBinaryIntegrationTest.java @@ -17,6 +17,7 @@ package com.facebook.buck.features.rust; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.either; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; @@ -199,7 +200,8 @@ public void simpleBinaryWarnings() throws IOException { assertThat( workspace.runBuckBuild("//:xyzzy").assertSuccess().getStderr(), Matchers.allOf( - containsString("warning: constant item is never used: `foo`"), + either(containsString("warning: constant item is never used: `foo`")) + .or(containsString("warning: constant is never used: `foo`")), containsString("warning: constant `foo` should have an upper case name"))); BuckBuildLog buildLog = workspace.getBuildLog(); diff --git a/test/com/facebook/buck/features/rust/RustLibraryIntegrationTest.java b/test/com/facebook/buck/features/rust/RustLibraryIntegrationTest.java index 683cca3540e..168a3cede84 100644 --- a/test/com/facebook/buck/features/rust/RustLibraryIntegrationTest.java +++ b/test/com/facebook/buck/features/rust/RustLibraryIntegrationTest.java @@ -17,6 +17,7 @@ package com.facebook.buck.features.rust; import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.either; import static org.junit.Assert.assertThat; import com.facebook.buck.testutil.ProcessResult; @@ -96,7 +97,8 @@ public void rustLibraryCheckWarning() throws IOException { "rust.rustc_check_flags=-Dwarnings --cfg \"feature=\\\"warning\\\"\"", "//messenger:messenger#check") .getStderr(), - containsString("error: method is never used: `unused`")); + either(containsString("error: method is never used: `unused`")) + .or(containsString("error: associated function is never used: `unused`"))); } @Test diff --git a/test/com/facebook/buck/features/zip/rules/testdata/zip-rule/example/BUCK.fixture b/test/com/facebook/buck/features/zip/rules/testdata/zip-rule/example/BUCK.fixture index 80b47e97b81..c18e025cf62 100644 --- a/test/com/facebook/buck/features/zip/rules/testdata/zip-rule/example/BUCK.fixture +++ b/test/com/facebook/buck/features/zip/rules/testdata/zip-rule/example/BUCK.fixture @@ -107,6 +107,7 @@ genrule( srcs = ["cake.txt"], out = "copy_out", cmd = "mkdir -p $OUT && cp $SRCS $OUT", + cmd_exe = "mkdir $OUT && copy %SRCS% %OUT%", ) zip_file( diff --git a/test/com/facebook/buck/jvm/java/JarGenruleTest.java b/test/com/facebook/buck/jvm/java/JarGenruleTest.java index 43ba0a69784..c5b5c0c9708 100644 --- a/test/com/facebook/buck/jvm/java/JarGenruleTest.java +++ b/test/com/facebook/buck/jvm/java/JarGenruleTest.java @@ -39,6 +39,6 @@ public void jarGenruleIsExecutable() throws Exception { Path output = workspace.buildAndReturnOutput("//:execute-jar-genrule"); String outputContents = workspace.getFileContents(output); - assertThat(outputContents, is(equalTo("I am a test resource file.\n"))); + assertThat(outputContents, is(equalTo("I am a test resource file." + System.lineSeparator()))); } } diff --git a/test/com/facebook/buck/jvm/java/JavacStepTest.java b/test/com/facebook/buck/jvm/java/JavacStepTest.java index 59de7c600f1..a634f07f699 100644 --- a/test/com/facebook/buck/jvm/java/JavacStepTest.java +++ b/test/com/facebook/buck/jvm/java/JavacStepTest.java @@ -38,6 +38,7 @@ import com.facebook.buck.testutil.TestConsole; import com.facebook.buck.util.FakeProcess; import com.facebook.buck.util.FakeProcessExecutor; +import com.facebook.buck.util.environment.Platform; import com.google.common.base.Functions; import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; @@ -145,14 +146,15 @@ public void failedCompileSendsStdoutAndStderrToConsole() throws Exception { // JavacStep itself writes stdout to the console on error; we expect the Build class to write // the stderr stream returned in the StepExecutionResult + String separator = System.getProperty("line.separator"); assertThat( result, equalTo( StepExecutionResult.builder() .setExitCode(StepExecutionResults.ERROR_EXIT_CODE) - .setStderr(Optional.of("javac stderr\n")) + .setStderr(Optional.of("javac stderr" + separator)) .build())); - assertThat(listener.getLogMessages(), equalTo(ImmutableList.of("javac stdout\n"))); + assertThat(listener.getLogMessages(), equalTo(ImmutableList.of("javac stdout" + separator))); } @Test @@ -160,6 +162,13 @@ public void existingBootclasspathDirSucceeds() throws Exception { FakeJavac fakeJavac = new FakeJavac(); BuildRuleResolver buildRuleResolver = new TestActionGraphBuilder(); ProjectFilesystem fakeFilesystem = FakeProjectFilesystem.createJavaOnlyFilesystem(); + String bootClassPath; + if (Platform.detect() == Platform.WINDOWS) { + bootClassPath = "C:\\this-totally-exists"; + } + else { + bootClassPath = "/this-totally-exists"; + } JavacOptions javacOptions = JavacOptions.builder() .setLanguageLevelOptions( @@ -167,7 +176,7 @@ public void existingBootclasspathDirSucceeds() throws Exception { .setSourceLevel("8.0") .setTargetLevel("8.0") .build()) - .setBootclasspath("/this-totally-exists") + .setBootclasspath(bootClassPath) .build(); ClasspathChecker classpathChecker = new ClasspathChecker( @@ -207,6 +216,13 @@ public void bootclasspathResolvedToAbsolutePath() { FakeJavac fakeJavac = new FakeJavac(); BuildRuleResolver buildRuleResolver = new TestActionGraphBuilder(); ProjectFilesystem fakeFilesystem = FakeProjectFilesystem.createJavaOnlyFilesystem(); + String bootClassPath; + if (Platform.detect() == Platform.WINDOWS) { + bootClassPath = "C:\\this-totally-exists;relative-path"; + } + else { + bootClassPath = "/this-totally-exists:relative-path"; + } JavacOptions javacOptions = JavacOptions.builder() .setLanguageLevelOptions( @@ -214,7 +230,7 @@ public void bootclasspathResolvedToAbsolutePath() { .setSourceLevel("8.0") .setTargetLevel("8.0") .build()) - .setBootclasspath("/this-totally-exists:relative-path") + .setBootclasspath(bootClassPath) .build(); ClasspathChecker classpathChecker = new ClasspathChecker( diff --git a/test/com/facebook/buck/jvm/java/abi/AbiFilteringClassVisitorTest.java b/test/com/facebook/buck/jvm/java/abi/AbiFilteringClassVisitorTest.java index 3191cd4d6b0..3da3ea5addf 100644 --- a/test/com/facebook/buck/jvm/java/abi/AbiFilteringClassVisitorTest.java +++ b/test/com/facebook/buck/jvm/java/abi/AbiFilteringClassVisitorTest.java @@ -36,7 +36,7 @@ public class AbiFilteringClassVisitorTest { public void setUp() { mockVisitor = createMock(ClassVisitor.class); filteringVisitor = - new AbiFilteringClassVisitor(mockVisitor, ImmutableList.of(), ImmutableSet.of()); + new AbiFilteringClassVisitor(mockVisitor, ImmutableList.of(), ImmutableSet.of(), false); } @Test @@ -92,7 +92,7 @@ public void testExcludesPrivateMethods() { @Test public void testIncludesPrivateMethodsWhenRetained() { filteringVisitor = - new AbiFilteringClassVisitor(mockVisitor, ImmutableList.of("foo"), ImmutableSet.of()); + new AbiFilteringClassVisitor(mockVisitor, ImmutableList.of("foo"), ImmutableSet.of(), false); testIncludesMethodWithAccess(Opcodes.ACC_PRIVATE); } @@ -166,7 +166,7 @@ public void testIncludesInnerClassEntryForInnerClass() { @Test public void testIncludesInnerClassEntryForReferencedOtherClassInnerClass() { filteringVisitor = - new AbiFilteringClassVisitor(mockVisitor, ImmutableList.of(), ImmutableSet.of("Bar$Inner")); + new AbiFilteringClassVisitor(mockVisitor, ImmutableList.of(), ImmutableSet.of("Bar$Inner"), false); visitClass(mockVisitor, "Foo"); mockVisitor.visitInnerClass("Bar$Inner", "Bar", "Inner", Opcodes.ACC_PUBLIC); diff --git a/test/com/facebook/buck/jvm/java/abi/StubJarTest.java b/test/com/facebook/buck/jvm/java/abi/StubJarTest.java index 66ab7ba6a2a..2cbaaa03c82 100644 --- a/test/com/facebook/buck/jvm/java/abi/StubJarTest.java +++ b/test/com/facebook/buck/jvm/java/abi/StubJarTest.java @@ -1612,7 +1612,10 @@ public void kotlinClassWithInlineExtensionMethodThatUsesRunnable() throws IOExce "", " // access flags 0x0", " (Lkotlin/jvm/functions/Function0;)V", - "}") + "", + " // access flags 0x1011", + " public final synthetic run()V", + "}") .createAndCheckStubJar(); } diff --git a/test/com/facebook/buck/jvm/java/testdata/jar_genrule/BUCK.fixture b/test/com/facebook/buck/jvm/java/testdata/jar_genrule/BUCK.fixture index cfccbcb3c7b..0c559b7458c 100644 --- a/test/com/facebook/buck/jvm/java/testdata/jar_genrule/BUCK.fixture +++ b/test/com/facebook/buck/jvm/java/testdata/jar_genrule/BUCK.fixture @@ -13,6 +13,7 @@ jar_genrule( name = "jar-genrule", srcs = ["resource.txt"], cmd = "cp $(location :bin) $OUT && jar uf $OUT resource.txt", + cmd_exe = "copy $(location :bin) $OUT && jar uf $OUT resource.txt", ) genrule( diff --git a/test/com/facebook/buck/jvm/kotlin/BUCK b/test/com/facebook/buck/jvm/kotlin/BUCK index 02823c90125..1049c281bf4 100644 --- a/test/com/facebook/buck/jvm/kotlin/BUCK +++ b/test/com/facebook/buck/jvm/kotlin/BUCK @@ -195,7 +195,10 @@ standard_java_test( "//src/com/facebook/buck/worker:worker_pool_factory", "//src/com/facebook/buck/worker:worker_process", "//test/com/facebook/buck/core/config:FakeBuckConfig", + "//test/com/facebook/buck/core/model:testutil", "//test/com/facebook/buck/io/filesystem:testutil", + "//test/com/facebook/buck/jvm/java:javac-env", + "//test/com/facebook/buck/jvm/java:testutil", "//test/com/facebook/buck/jvm/kotlin:testutil", "//test/com/facebook/buck/testutil:testutil", "//test/com/facebook/buck/testutil/integration:util", diff --git a/test/com/facebook/buck/jvm/kotlin/KotlinConfiguredCompilerFactoryTest.java b/test/com/facebook/buck/jvm/kotlin/KotlinConfiguredCompilerFactoryTest.java new file mode 100644 index 00000000000..0c9f04cb4cd --- /dev/null +++ b/test/com/facebook/buck/jvm/kotlin/KotlinConfiguredCompilerFactoryTest.java @@ -0,0 +1,94 @@ +/* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.facebook.buck.jvm.kotlin; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.facebook.buck.core.config.FakeBuckConfig; +import com.facebook.buck.core.model.BuildTargetFactory; +import com.facebook.buck.jvm.java.JavaCompilationConstants; +import com.facebook.buck.jvm.java.JavacFactory; +import com.facebook.buck.jvm.java.JavacFactoryHelper; +import com.facebook.buck.testutil.TemporaryPaths; +import com.google.common.collect.ImmutableList; +import org.junit.Before; +import org.junit.Test; + +public class KotlinConfiguredCompilerFactoryTest { + private KotlinBuckConfig config; + private JavacFactory jFactory; + private KotlinConfiguredCompilerFactory kFactory; + private KotlinLibraryDescriptionArg.Builder kotlinArgsBuilder; + + @Before + public void setUp() { + config = new KotlinBuckConfig(FakeBuckConfig.builder().build()); + jFactory = JavacFactoryHelper.createJavacFactory(JavaCompilationConstants.DEFAULT_JAVA_CONFIG); + kFactory = new KotlinConfiguredCompilerFactory(config, jFactory); + kotlinArgsBuilder = FauxKotlinLibraryBuilder.createBuilder( + BuildTargetFactory.newInstance("//:rule")).getArgForPopulating(); + } + + @Test + public void testCondenseCompilerArguments_validateAllArguments() { + kotlinArgsBuilder.addFreeCompilerArgs("-test", "-test"); + kotlinArgsBuilder.setAllWarningsAsErrors(true); + kotlinArgsBuilder.setSuppressWarnings(true); + kotlinArgsBuilder.setVerbose(true); + kotlinArgsBuilder.setJvmTarget("1.8"); + kotlinArgsBuilder.setIncludeRuntime(true); + kotlinArgsBuilder.setJdkHome("jdk_home"); + kotlinArgsBuilder.setNoJdk(true); + kotlinArgsBuilder.setNoStdlib(true); + kotlinArgsBuilder.setNoReflect(true); + kotlinArgsBuilder.setJavaParameters(true); + kotlinArgsBuilder.setApiVersion("1.3"); + kotlinArgsBuilder.setLanguageVersion("1.3"); + + ImmutableList condensedArgs = + kFactory.condenseCompilerArguments(kotlinArgsBuilder.build()); + + assertEquals(condensedArgs.indexOf("-test"), condensedArgs.lastIndexOf("-test")); + + assertTrue(condensedArgs.contains("-Werror")); + assertTrue(condensedArgs.contains("-nowarn")); + assertTrue(condensedArgs.contains("-verbose")); + assertTrue(condensedArgs.contains("-include-runtime")); + assertTrue(condensedArgs.contains("-no-jdk")); + assertTrue(condensedArgs.contains("-no-stdlib")); + assertTrue(condensedArgs.contains("-no-reflect")); + assertTrue(condensedArgs.contains("-java-parameters")); + + assertTrue(condensedArgs.contains("-jvm-target")); + int jvmTargetIndex = condensedArgs.indexOf("-jvm-target") + 1; + assertEquals(condensedArgs.get(jvmTargetIndex), "1.8"); + + assertTrue(condensedArgs.contains("-jdk-home")); + int jdkHomeIndex = condensedArgs.indexOf("-jdk-home") + 1; + assertEquals(condensedArgs.get(jdkHomeIndex), "jdk_home"); + + assertTrue(condensedArgs.contains("-language-version")); + int languageVersionIndex = condensedArgs.indexOf("-language-version") + 1; + assertEquals(condensedArgs.get(languageVersionIndex), "1.3"); + + assertTrue(condensedArgs.contains("-api-version")); + int apiVersionIndex = condensedArgs.indexOf("-api-version") + 1; + assertEquals(condensedArgs.get(apiVersionIndex), "1.3"); + + } +} diff --git a/test/com/facebook/buck/log/test_log_rotation.py b/test/com/facebook/buck/log/test_log_rotation.py index e8abbc2924a..e442c52456f 100644 --- a/test/com/facebook/buck/log/test_log_rotation.py +++ b/test/com/facebook/buck/log/test_log_rotation.py @@ -15,11 +15,13 @@ import glob import os import unittest +import platform from project_workspace import ProjectWorkspace class LogRotationTest(unittest.TestCase): + @unittest.skipIf(platform.system() == "Windows", "Skip on Windows platform.") def test_log_retention(self): """ Tests that the default java.util.logging setup can maintain at least 'a couple' of log files. """ diff --git a/test/com/facebook/buck/parser/ParserIntegrationTest.java b/test/com/facebook/buck/parser/ParserIntegrationTest.java index a7010ccbeba..495c64400b0 100644 --- a/test/com/facebook/buck/parser/ParserIntegrationTest.java +++ b/test/com/facebook/buck/parser/ParserIntegrationTest.java @@ -436,7 +436,7 @@ public void testDisablingImplicitNativeRules() throws Exception { "//python/implicit_in_extension_bzl:main", "-c", "parser.disable_implicit_native_rules=true"), - "NameError: global name 'java_library' is not defined", + "NameError: name 'java_library' is not defined", "extension.bzl\", line 5", "BUCK\", line 5"); assertParseFailedWithSubstrings( diff --git a/test/com/facebook/buck/shell/FakeWorkerBuilder.java b/test/com/facebook/buck/shell/FakeWorkerBuilder.java index fd6f3577d4f..0fd3bc6a247 100644 --- a/test/com/facebook/buck/shell/FakeWorkerBuilder.java +++ b/test/com/facebook/buck/shell/FakeWorkerBuilder.java @@ -97,6 +97,11 @@ public boolean isPersistent() { return false; } + @Override + public boolean isAsync() { + return false; + } + @Override public HashCode getInstanceKey() { return hashCode; diff --git a/test/com/facebook/buck/shell/GenruleBuildableTest.java b/test/com/facebook/buck/shell/GenruleBuildableTest.java index b80a1bed87d..347d3b05d78 100644 --- a/test/com/facebook/buck/shell/GenruleBuildableTest.java +++ b/test/com/facebook/buck/shell/GenruleBuildableTest.java @@ -255,6 +255,7 @@ public void testShouldIncludeAndroidSpecificEnvInEnvironmentIfPresent() { assertEquals("aapt", env.get("AAPT")); assertEquals("aapt2", env.get("AAPT2")); assertEquals(sdkDir.toString(), env.get("ANDROID_HOME")); + assertEquals(sdkDir.toString(), env.get("ANDROID_SDK_ROOT")); assertEquals(ndkDir.toString(), env.get("NDK_HOME")); } diff --git a/test/com/facebook/buck/shell/WorkerShellStepTest.java b/test/com/facebook/buck/shell/WorkerShellStepTest.java index 86f72028c63..0e05367218f 100644 --- a/test/com/facebook/buck/shell/WorkerShellStepTest.java +++ b/test/com/facebook/buck/shell/WorkerShellStepTest.java @@ -41,6 +41,7 @@ import com.facebook.buck.worker.WorkerProcessParams; import com.facebook.buck.worker.WorkerProcessPool; import com.facebook.buck.worker.WorkerProcessPoolFactory; +import com.facebook.buck.worker.WorkerProcessPoolSync; import com.google.common.base.Charsets; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -103,7 +104,7 @@ private WorkerJobParams createJobParams( ImmutableMap startupEnv, String jobArgs, int maxWorkers) { - return createJobParams(startupCommand, startupEnv, jobArgs, maxWorkers, null, null); + return createJobParams(startupCommand, startupEnv, jobArgs, maxWorkers, false, null, null); } private WorkerJobParams createJobParams( @@ -111,6 +112,7 @@ private WorkerJobParams createJobParams( ImmutableMap startupEnv, String jobArgs, int maxWorkers, + boolean isAsync, @Nullable String persistentWorkerKey, @Nullable HashCode workerHash) { return WorkerJobParams.of( @@ -120,6 +122,7 @@ private WorkerJobParams createJobParams( startupCommand, startupEnv, maxWorkers, + isAsync, persistentWorkerKey == null || workerHash == null ? Optional.empty() : Optional.of(WorkerProcessIdentity.of(persistentWorkerKey, workerHash)))); @@ -139,18 +142,18 @@ private ExecutionContext createExecutionContextWith( private ExecutionContext createExecutionContextWith( ImmutableMap jobArgs, int poolCapacity) { WorkerProcessPool workerProcessPool = - new WorkerProcessPool( + new WorkerProcessPoolSync( poolCapacity, - Hashing.sha1().hashString(fakeWorkerStartupCommand, Charsets.UTF_8), + Hashing.sha256().hashString(fakeWorkerStartupCommand, Charsets.UTF_8), () -> new FakeWorkerProcess(jobArgs)); ConcurrentHashMap workerProcessMap = new ConcurrentHashMap<>(); workerProcessMap.put(fakeWorkerStartupCommand, workerProcessPool); WorkerProcessPool persistentWorkerProcessPool = - new WorkerProcessPool( + new WorkerProcessPoolSync( poolCapacity, - Hashing.sha1().hashString(fakePersistentWorkerStartupCommand, Charsets.UTF_8), + Hashing.sha256().hashString(fakePersistentWorkerStartupCommand, Charsets.UTF_8), () -> new FakeWorkerProcess(jobArgs)); ConcurrentHashMap persistentWorkerProcessMap = new ConcurrentHashMap<>(); @@ -282,8 +285,9 @@ public void testPersistentJobIsExecutedAndResultIsReceived() ImmutableMap.of(), "myJobArgs", 1, + false, persistentWorkerKey, - Hashing.sha1().hashString(fakePersistentWorkerStartupCommand, Charsets.UTF_8)), + Hashing.sha256().hashString(fakePersistentWorkerStartupCommand, Charsets.UTF_8)), null, null); @@ -499,12 +503,47 @@ public void testWarningIsPrintedForIdenticalWorkerToolsWithDifferentCapacity() t assertThat(consoleEvent.getLevel(), Matchers.is(Level.WARNING)); assertThat( consoleEvent.getMessage(), - Matchers.is( - String.format( - "There are two 'worker_tool' targets declared with the same command (%s), but different " - + "'max_worker' settings (%d and %d). Only the first capacity is applied. Consolidate " - + "these workers to avoid this warning.", - fakeWorkerStartupCommand, existingPoolSize, stepPoolSize))); + Matchers.allOf( + Matchers.containsString("max_worker"), + Matchers.containsString(fakeWorkerStartupCommand), + Matchers.containsString(String.format("%d and %d", existingPoolSize, stepPoolSize)))); + } + + @Test + public void testWarningIsPrintedForAsyncAndNonAsyncPools() throws Exception { + int poolSize = 2; + + ExecutionContext context = + createExecutionContextWith( + ImmutableMap.of("jobArgs", WorkerJobResult.of(0, Optional.of(""), Optional.of(""))), + poolSize); + + FakeBuckEventListener listener = new FakeBuckEventListener(); + context.getBuckEventBus().register(listener); + + WorkerJobParams params = + createJobParams( + ImmutableList.of(startupCommand, startupArg), + ImmutableMap.of(), + "jobArgs", + poolSize, + true, + null, + null); + + WorkerShellStep step = createWorkerShellStep(params, null, null); + step.execute(context); + + BuckEvent firstEvent = listener.getEvents().get(0); + assertThat(firstEvent, Matchers.instanceOf(ConsoleEvent.class)); + + ConsoleEvent consoleEvent = (ConsoleEvent) firstEvent; + assertThat(consoleEvent.getLevel(), Matchers.is(Level.WARNING)); + assertThat( + consoleEvent.getMessage(), + Matchers.allOf( + Matchers.containsString("solo_async"), + Matchers.containsString(fakeWorkerStartupCommand))); } private static class ConcurrentExecution extends Thread { diff --git a/test/com/facebook/buck/shell/WorkerToolRuleIntegrationTest.java b/test/com/facebook/buck/shell/WorkerToolRuleIntegrationTest.java index 28b250dc866..14b51571690 100644 --- a/test/com/facebook/buck/shell/WorkerToolRuleIntegrationTest.java +++ b/test/com/facebook/buck/shell/WorkerToolRuleIntegrationTest.java @@ -111,6 +111,16 @@ public void testPersistentWorkerToolRules() throws Exception { .assertSuccess(); } + @Test + public void testAsyncWorkerToolRules() throws Exception { + BuildTarget target1 = workspace.newBuildTarget("//:async-test0"); + BuildTarget target2 = workspace.newBuildTarget("//:async-test1"); + + workspace + .runBuckBuild(target1.getFullyQualifiedName(), target2.getFullyQualifiedName()) + .assertSuccess(); + } + @Test public void testPersistentWorkerToolReusesProcess() throws Exception { ProjectFilesystem filesystem = workspace.getProjectFileSystem(); diff --git a/test/com/facebook/buck/shell/testdata/genrule_zip_scrubber/BUCK.fixture b/test/com/facebook/buck/shell/testdata/genrule_zip_scrubber/BUCK.fixture index 4df1e4783c8..705148b2c94 100644 --- a/test/com/facebook/buck/shell/testdata/genrule_zip_scrubber/BUCK.fixture +++ b/test/com/facebook/buck/shell/testdata/genrule_zip_scrubber/BUCK.fixture @@ -75,7 +75,7 @@ genrule( ], out = "output.zip", bash = "cp $SRCS $OUT", - cmd_exe = "copy $SRCS $OUT", + cmd_exe = "copy %SRCS% %OUT%", ) genrule( diff --git a/test/com/facebook/buck/shell/testdata/worker_tool_test/BUCK.fixture b/test/com/facebook/buck/shell/testdata/worker_tool_test/BUCK.fixture index dd0ab515073..7faa87a85a2 100644 --- a/test/com/facebook/buck/shell/testdata/worker_tool_test/BUCK.fixture +++ b/test/com/facebook/buck/shell/testdata/worker_tool_test/BUCK.fixture @@ -139,3 +139,30 @@ genrule( out = "output.txt", cmd = "$(worker :worker_crash_on_start) $OUT", ) + +worker_tool( + name = "worker_async", + args = [ + "--num-jobs", + "2", + "--async", + "1" + ], + exe = ":external_tool", + solo_async = True, + max_workers = -1, +) + +genrule( + name = "async-test0", + srcs = [], + out = "output.txt", + cmd = "$(worker :worker_async) $OUT", +) + +genrule( + name = "async-test1", + srcs = [], + out = "output.txt", + cmd = "$(worker :worker_async) $OUT", +) diff --git a/test/com/facebook/buck/shell/testdata/worker_tool_test/external_tool.sh b/test/com/facebook/buck/shell/testdata/worker_tool_test/external_tool.sh index dbabdba9a34..e7e2048e939 100755 --- a/test/com/facebook/buck/shell/testdata/worker_tool_test/external_tool.sh +++ b/test/com/facebook/buck/shell/testdata/worker_tool_test/external_tool.sh @@ -14,6 +14,10 @@ for i in "$@"; do loc="$2" shift ;; + --async) + async="$2" + shift + ;; *) shift ;; @@ -49,9 +53,16 @@ do # Write to the output file. echo "the startup arguments were: $ARGS" > $output_path # Send the job result reply. - printf ",{\"id\":%s, \"type\":\"result\", \"exit_code\":0}" "$message_id" + if [ "$async" != "" ]; then + printf ",{\"id\":%s, \"type\":\"result\", \"exit_code\":0}" "$message_id" >> $TMP/output + else + printf ",{\"id\":%s, \"type\":\"result\", \"exit_code\":0}" "$message_id" + fi done +if [ "$async" != "" ]; then + cat $TMP/output +fi # Read in the end of the JSON array and reply with a corresponding closing bracket. read -d "]" echo ] diff --git a/test/com/facebook/buck/support/state/BuckGlobalStateLifecycleManagerTest.java b/test/com/facebook/buck/support/state/BuckGlobalStateLifecycleManagerTest.java index 8ace06d39c9..3d40ca95a0d 100644 --- a/test/com/facebook/buck/support/state/BuckGlobalStateLifecycleManagerTest.java +++ b/test/com/facebook/buck/support/state/BuckGlobalStateLifecycleManagerTest.java @@ -468,6 +468,7 @@ private Cells createCellWithAndroidSdk(Path androidSdkPath) { .setFilesystem(filesystem) .addEnvironmentVariable("ANDROID_HOME", androidSdkPath.toString()) .addEnvironmentVariable("ANDROID_SDK", androidSdkPath.toString()) + .addEnvironmentVariable("ANDROID_SDK_ROOT", androidSdkPath.toString()) .build(); } diff --git a/test/com/facebook/buck/swift/SwiftNativeLinkableGroupTest.java b/test/com/facebook/buck/swift/SwiftNativeLinkableGroupTest.java index cd53fb999e2..5b632dace34 100644 --- a/test/com/facebook/buck/swift/SwiftNativeLinkableGroupTest.java +++ b/test/com/facebook/buck/swift/SwiftNativeLinkableGroupTest.java @@ -111,7 +111,7 @@ public void testStaticLinkerFlagsOnMobile() { swiftcTool, Optional.of(swiftStdTool), true, - SwiftTargetTriple.of("x86_64", "apple", "ios", "9.3")); + SwiftTargetTriple.of("x86_64", "apple", "ios", "9.3", false)); ImmutableList.Builder staticArgsBuilder = ImmutableList.builder(); SwiftRuntimeNativeLinkableGroup.populateLinkerArguments( @@ -152,7 +152,7 @@ public void testStaticLinkerFlagsOnMac() { swiftcTool, Optional.of(swiftStdTool), true, - SwiftTargetTriple.of("x86_64", "apple", "ios", "9.3")); + SwiftTargetTriple.of("x86_64", "apple", "ios", "9.3", false)); ImmutableList.Builder sharedArgsBuilder = ImmutableList.builder(); SwiftRuntimeNativeLinkableGroup.populateLinkerArguments( diff --git a/test/com/facebook/buck/swift/toolchain/impl/SwiftPlatformFactoryIntegrationTest.java b/test/com/facebook/buck/swift/toolchain/impl/SwiftPlatformFactoryIntegrationTest.java index 92682d12995..2235f1382a8 100644 --- a/test/com/facebook/buck/swift/toolchain/impl/SwiftPlatformFactoryIntegrationTest.java +++ b/test/com/facebook/buck/swift/toolchain/impl/SwiftPlatformFactoryIntegrationTest.java @@ -86,7 +86,7 @@ public void setUp() { @Test public void testBuildSwiftPlatformWithEmptyToolchainPaths() throws IOException { Path developerDir = tmp.newFolder("Developer"); - SwiftTargetTriple triple = SwiftTargetTriple.of("x86_64", "apple", "ios", "9.3"); + SwiftTargetTriple triple = SwiftTargetTriple.of("x86_64", "apple", "ios", "9.3", false); SwiftPlatform swiftPlatform = SwiftPlatformFactory.build( createAppleSdk(), @@ -113,7 +113,7 @@ public void testBuildSwiftPlatformWithNonEmptyLookupPathWithoutTools() throws IO swiftcTool, Optional.of(swiftStdTool), true, - SwiftTargetTriple.of("x86_64", "apple", "ios", "9.3")); + SwiftTargetTriple.of("x86_64", "apple", "ios", "9.3", false)); assertThat(swiftPlatform.getSwiftRuntimePathsForBundling(), empty()); assertThat(swiftPlatform.getSwiftStaticRuntimePaths(), empty()); } @@ -136,7 +136,7 @@ public void testBuildSwiftPlatformWithNonEmptyLookupPathWithTools() throws IOExc swiftcTool, Optional.of(swiftStdTool), true, - SwiftTargetTriple.of("x86_64", "apple", "ios", "9.3")); + SwiftTargetTriple.of("x86_64", "apple", "ios", "9.3", false)); assertThat(swiftPlatform.getSwiftRuntimePathsForBundling(), hasSize(1)); assertThat(swiftPlatform.getSwiftStaticRuntimePaths(), hasSize(2)); } diff --git a/test/com/facebook/buck/testrunner/RunWithAnnotationIntegrationTest.java b/test/com/facebook/buck/testrunner/RunWithAnnotationIntegrationTest.java index dedc0031a77..189123d5955 100644 --- a/test/com/facebook/buck/testrunner/RunWithAnnotationIntegrationTest.java +++ b/test/com/facebook/buck/testrunner/RunWithAnnotationIntegrationTest.java @@ -51,7 +51,12 @@ public void testSimpleSuiteRun2TestCases() throws IOException { ProcessResult suiteTestResult = workspace.runBuckCommand("test", "//:SimpleSuiteTest"); suiteTestResult.assertSuccess("Test should pass"); - assertThat(suiteTestResult.getStderr(), containsString("2 Passed")); + assertThat( + suiteTestResult.getStderr(), + containsString("1 Passed 0 Skipped 0 Failed com.example.Subtest1")); + assertThat( + suiteTestResult.getStderr(), + containsString("1 Passed 0 Skipped 0 Failed com.example.Subtest2")); } @Test @@ -62,8 +67,15 @@ public void testFailingSuiteRun3TestCasesWith1Failure() throws IOException { ProcessResult suiteTestResult = workspace.runBuckCommand("test", "//:FailingSuiteTest"); suiteTestResult.assertTestFailure("Test should fail because of one of subtests failure"); - assertThat(suiteTestResult.getStderr(), containsString("2 Passed")); - assertThat(suiteTestResult.getStderr(), containsString("1 Failed")); + assertThat( + suiteTestResult.getStderr(), + containsString("1 Passed 0 Skipped 0 Failed com.example.Subtest1")); + assertThat( + suiteTestResult.getStderr(), + containsString("1 Passed 0 Skipped 0 Failed com.example.Subtest2")); + assertThat( + suiteTestResult.getStderr(), + containsString("0 Passed 0 Skipped 1 Failed com.example.FailingSubtest")); } @Test diff --git a/test/com/facebook/buck/util/ListeningProcessExecutorTest.java b/test/com/facebook/buck/util/ListeningProcessExecutorTest.java index d55f7a153f7..74108e8a123 100644 --- a/test/com/facebook/buck/util/ListeningProcessExecutorTest.java +++ b/test/com/facebook/buck/util/ListeningProcessExecutorTest.java @@ -212,7 +212,10 @@ public void clearsEnvWhenExplicitlySet() throws Exception { ListeningProcessExecutor.LaunchedProcess process = executor.launchProcess(params, listener); int returnCode = executor.waitForProcess(process); assertThat(returnCode, equalTo(0)); - assertThat(listener.capturedStdout.toString("UTF-8"), is(emptyString())); + if (Platform.detect() != Platform.WINDOWS) { + // Variable %COMSPEC% is always in Windows environment so don't assert empty on it. + assertThat(listener.capturedStdout.toString("UTF-8"), is(emptyString())); + } assertThat(listener.capturedStderr.toString("UTF-8"), is(emptyString())); } } diff --git a/test/com/facebook/buck/util/concurrent/JobLimiterTest.java b/test/com/facebook/buck/util/concurrent/JobLimiterTest.java index 86adfd54d44..5418f22867e 100644 --- a/test/com/facebook/buck/util/concurrent/JobLimiterTest.java +++ b/test/com/facebook/buck/util/concurrent/JobLimiterTest.java @@ -68,8 +68,8 @@ public void testLimitsJobs() throws Exception { } // Two jobs should start. - assertTrue(jobStarted.tryAcquire(50, TimeUnit.MILLISECONDS)); - assertTrue(jobStarted.tryAcquire(50, TimeUnit.MILLISECONDS)); + assertTrue(jobStarted.tryAcquire(200, TimeUnit.MILLISECONDS)); + assertTrue(jobStarted.tryAcquire(200, TimeUnit.MILLISECONDS)); assertEquals(2, jobsRunning.get()); diff --git a/test/com/facebook/buck/util/versioncontrol/HgCmdLineInterfaceIntegrationTest.java b/test/com/facebook/buck/util/versioncontrol/HgCmdLineInterfaceIntegrationTest.java index 798192d097e..655c764f1be 100644 --- a/test/com/facebook/buck/util/versioncontrol/HgCmdLineInterfaceIntegrationTest.java +++ b/test/com/facebook/buck/util/versioncontrol/HgCmdLineInterfaceIntegrationTest.java @@ -175,7 +175,9 @@ public void testDiffBetweenDiffs() repoThreeCmdLine.diffBetweenRevisions("b1fd7e", "2911b3").get().get()) { InputStreamReader diffFileReader = new InputStreamReader(diffFileStream, Charsets.UTF_8); String actualDiff = CharStreams.toString(diffFileReader); - assertEquals(String.join("\n", expectedValue), actualDiff); + // The output message from FB hg is a bit more than that from open source hg, use contains + // here. + assertThat(String.join("\n", expectedValue), Matchers.containsString(actualDiff)); } } diff --git a/test/com/facebook/buck/worker/FakeWorkerProcess.java b/test/com/facebook/buck/worker/FakeWorkerProcess.java index 9706a1ba0f2..608345aa660 100644 --- a/test/com/facebook/buck/worker/FakeWorkerProcess.java +++ b/test/com/facebook/buck/worker/FakeWorkerProcess.java @@ -22,6 +22,8 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.SettableFuture; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Paths; @@ -41,7 +43,6 @@ public FakeWorkerProcess(ImmutableMap jobArgsToJobResul Paths.get("tmp").toAbsolutePath().normalize()); this.jobArgsToJobResultMap = jobArgsToJobResultMap; this.isAlive = false; - this.setProtocol(new FakeWorkerProcessProtocol.FakeCommandSender()); } @Override @@ -55,13 +56,15 @@ public synchronized void ensureLaunchAndHandshake() { } @Override - public synchronized WorkerJobResult submitAndWaitForJob(String jobArgs) { + public synchronized ListenableFuture submitJob(String jobArgs) { WorkerJobResult result = this.jobArgsToJobResultMap.get(jobArgs); if (result == null) { throw new IllegalArgumentException( String.format("No fake WorkerJobResult found for job arguments '%s'", jobArgs)); } - return result; + SettableFuture out = SettableFuture.create(); + out.set(result); + return out; } @Override diff --git a/test/com/facebook/buck/worker/FakeWorkerProcessProtocol.java b/test/com/facebook/buck/worker/FakeWorkerProcessProtocol.java index 523eb4e937a..3c53887ab7e 100644 --- a/test/com/facebook/buck/worker/FakeWorkerProcessProtocol.java +++ b/test/com/facebook/buck/worker/FakeWorkerProcessProtocol.java @@ -17,27 +17,47 @@ package com.facebook.buck.worker; import java.io.IOException; +import java.util.concurrent.ArrayBlockingQueue; public class FakeWorkerProcessProtocol { public static class FakeCommandSender implements WorkerProcessProtocol.CommandSender { - private boolean isClosed = false; + private volatile boolean isClosed = false; + private final ArrayBlockingQueue messageIds = new ArrayBlockingQueue<>(10); @Override public void handshake(int messageId) {} @Override - public void send(int messageId, WorkerProcessCommand command) {} + public void send(int messageId, WorkerProcessCommand command) throws IOException { + try { + messageIds.put(messageId); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } @Override - public int receiveCommandResponse(int messageID) throws IOException { - return 0; + public WorkerProcessProtocol.CommandResponse receiveNextCommandResponse() throws IOException { + if (isClosed) { + throw new RuntimeException("Closed"); + } + try { + return new WorkerProcessProtocol.CommandResponse(messageIds.take(), 0); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } } @Override - public void close() { + public synchronized void close() { isClosed = true; + try { + messageIds.put(-1); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } } public boolean isClosed() { diff --git a/test/com/facebook/buck/worker/WorkerProcessPoolTest.java b/test/com/facebook/buck/worker/WorkerProcessPoolSyncTest.java similarity index 90% rename from test/com/facebook/buck/worker/WorkerProcessPoolTest.java rename to test/com/facebook/buck/worker/WorkerProcessPoolSyncTest.java index fb615e66331..6d7bee87966 100644 --- a/test/com/facebook/buck/worker/WorkerProcessPoolTest.java +++ b/test/com/facebook/buck/worker/WorkerProcessPoolSyncTest.java @@ -24,7 +24,7 @@ import com.facebook.buck.util.Threads; import com.facebook.buck.util.function.ThrowingSupplier; -import com.facebook.buck.worker.WorkerProcessPool.BorrowedWorkerProcess; +import com.facebook.buck.worker.WorkerProcessPoolSync.BorrowedWorkerProcess; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -50,7 +50,7 @@ import org.junit.Before; import org.junit.Test; -public class WorkerProcessPoolTest { +public class WorkerProcessPoolSyncTest { private static final int WAIT_FOR_TEST_THREADS_TIMEOUT = 1000; private TestThreads testThreads; @@ -69,7 +69,7 @@ public void tearDown() { public void testProvidesWorkersAccordingToCapacityThenBlocks() throws Exception { int maxWorkers = 3; Set createdWorkers = new HashSet<>(); - WorkerProcessPool pool = createPool(maxWorkers, createdWorkers::add); + WorkerProcessPoolSync pool = createPool(maxWorkers, createdWorkers::add); AtomicReference extraWorkerProcess = new AtomicReference<>(); // acquire enough workers to exhaust the pool @@ -91,7 +91,7 @@ public void testReusesWorkerProcesses() throws Exception { int secondBatch = 3; Set usedWorkers = new HashSet<>(); - WorkerProcessPool pool = createPool(maxWorkers, usedWorkers::add); + WorkerProcessPoolSync pool = createPool(maxWorkers, usedWorkers::add); // create two worker processes, and release them. acquireWorkersThenRelease(pool, firstBatch); @@ -106,7 +106,7 @@ public void testReusesWorkerProcesses() throws Exception { public void testLargePool() throws Exception { int numConcurrentConsumers = 128; Set createdWorkers = new HashSet<>(); - WorkerProcessPool pool = createPool(numConcurrentConsumers * 2, createdWorkers::add); + WorkerProcessPoolSync pool = createPool(numConcurrentConsumers * 2, createdWorkers::add); acquireWorkersThenRelease(pool, numConcurrentConsumers); @@ -117,7 +117,7 @@ public void testLargePool() throws Exception { public void testReusesWorkerProcessesInLargePools() throws Exception { int numConcurrentConsumers = 128; Set createdWorkers = new HashSet<>(); - WorkerProcessPool pool = createPool(numConcurrentConsumers * 2, createdWorkers::add); + WorkerProcessPoolSync pool = createPool(numConcurrentConsumers * 2, createdWorkers::add); acquireWorkersThenRelease(pool, numConcurrentConsumers / 2); acquireWorkersThenRelease(pool, numConcurrentConsumers); @@ -128,7 +128,7 @@ public void testReusesWorkerProcessesInLargePools() throws Exception { @Test(timeout = WAIT_FOR_TEST_THREADS_TIMEOUT) public void destroysProcessOnFailure() throws Exception { Set createdWorkers = new HashSet<>(); - WorkerProcessPool pool = createPool(1, createdWorkers::add); + WorkerProcessPoolSync pool = createPool(1, createdWorkers::add); acquireWorkersThenRelease(pool, 1); assertThat(createdWorkers.size(), is(1)); @@ -151,7 +151,7 @@ public void destroysProcessOnFailure() throws Exception { @Test(timeout = WAIT_FOR_TEST_THREADS_TIMEOUT) public void returnAndDestroyDoNotInterrupt() throws InterruptedException, IOException { - WorkerProcessPool pool = createPool(1); + WorkerProcessPoolSync pool = createPool(1); WorkerProcess process; try (BorrowedWorkerProcess worker = pool.borrowWorkerProcess()) { @@ -174,7 +174,7 @@ public void returnAndDestroyDoNotInterrupt() throws InterruptedException, IOExce @Test public void cleansUpDeadProcesses() throws InterruptedException, IOException { - WorkerProcessPool pool = createPool(1); + WorkerProcessPoolSync pool = createPool(1); WorkerProcess process; try (BorrowedWorkerProcess worker = pool.borrowWorkerProcess()) { @@ -199,7 +199,7 @@ public void cleansUpDeadProcesses() throws InterruptedException, IOException { public void notifiesWaitingThreadsWhenCleaningDeadProcesses() throws Exception { int maxWorkers = 2; Set createdProcesses = concurrentSet(); - WorkerProcessPool pool = createPool(maxWorkers, createdProcesses::add); + WorkerProcessPoolSync pool = createPool(maxWorkers, createdProcesses::add); acquireWorkersThenRunActionThenRelease( pool, @@ -216,7 +216,7 @@ public void notifiesWaitingThreadsWhenCleaningDeadProcesses() throws Exception { @Test(timeout = WAIT_FOR_TEST_THREADS_TIMEOUT) public void canStartupMultipleWorkersInParallel() throws InterruptedException, IOException { ArrayBlockingQueue> workers = new ArrayBlockingQueue<>(1); - WorkerProcessPool pool = createPool(2, workers); + WorkerProcessPoolSync pool = createPool(2, workers); // thread 1, attempting to borrow a worker testThreads.startThread(borrowWorkerProcessWithoutReturning(pool, concurrentSet())); @@ -246,7 +246,7 @@ public void canStartupMultipleWorkersInParallel() throws InterruptedException, I @Test public void canReturnAndBorrowWorkersWhileStartingUpOtherWorkers() throws Exception { SynchronousQueue> workers = new SynchronousQueue<>(); - WorkerProcessPool pool = createPool(2, workers); + WorkerProcessPoolSync pool = createPool(2, workers); CountDownLatch firstThreadWaitingToBorrowProcess = new CountDownLatch(1); CountDownLatch secondThreadWaitingForWorker = new CountDownLatch(1); @@ -296,7 +296,7 @@ public void testEmptyPoolDoesNotBlockForever() throws InterruptedException { ConcurrentHashMap usedWorkers = new ConcurrentHashMap<>(); Phaser phaser = new Phaser(CAPACITY + 1); // + 1 for test main thread - WorkerProcessPool pool = + WorkerProcessPoolSync pool = createPool( CAPACITY, () -> { @@ -325,7 +325,7 @@ public void testPoolClosesCleanyIfNoWorkersUsed() { @Test public void testPoolClosesCleanlyAfterSomeWorkersWereUsedAndReturned() throws Exception { int maxWorkers = 6; - WorkerProcessPool pool = createPool(maxWorkers); + WorkerProcessPoolSync pool = createPool(maxWorkers); acquireWorkersThenRelease(pool, maxWorkers / 2); pool.close(); } @@ -333,7 +333,7 @@ public void testPoolClosesCleanlyAfterSomeWorkersWereUsedAndReturned() throws Ex @Test public void testPoolClosesCleanlyAfterAllWorkersWereUsedAndReturned() throws Exception { int maxWorkers = 6; - WorkerProcessPool pool = createPool(maxWorkers); + WorkerProcessPoolSync pool = createPool(maxWorkers); acquireWorkersThenRelease(pool, maxWorkers); pool.close(); } @@ -341,7 +341,7 @@ public void testPoolClosesCleanlyAfterAllWorkersWereUsedAndReturned() throws Exc @Test public void testPoolClosesCleanlyAfterSomeWorkersWereReused() throws Exception { int maxWorkers = 6; - WorkerProcessPool pool = createPool(maxWorkers); + WorkerProcessPoolSync pool = createPool(maxWorkers); for (int i = 0; i < 2; ++i) { // first iteration starts up workers, second iteration reuses acquireWorkersThenRelease(pool, 2); @@ -353,7 +353,7 @@ public void testPoolClosesCleanlyAfterSomeWorkersWereReused() throws Exception { public void testThrowsWhenClosingWithoutAllWorkersReturned() throws InterruptedException, IOException { int arbitraryNumber = 3; - WorkerProcessPool pool = createPool(arbitraryNumber); + WorkerProcessPoolSync pool = createPool(arbitraryNumber); BorrowedWorkerProcess worker = pool.borrowWorkerProcess(); worker.get(); // use worker pool.close(); @@ -363,17 +363,17 @@ public void testThrowsWhenClosingWithoutAllWorkersReturned() @Test(expected = IllegalStateException.class) public void testThrowsWhenClosingWithoutAllUnusedWorkersReturned() throws InterruptedException { int arbitraryNumber = 5; - WorkerProcessPool pool = createPool(arbitraryNumber); + WorkerProcessPoolSync pool = createPool(arbitraryNumber); BorrowedWorkerProcess worker = pool.borrowWorkerProcess(); pool.close(); worker.close(); } - private static WorkerProcessPool createPool( + private static WorkerProcessPoolSync createPool( int maxWorkers, ThrowingSupplier startWorkerProcess) { - return new WorkerProcessPool( + return new WorkerProcessPoolSync( maxWorkers, - Hashing.sha1().hashLong(0), + Hashing.sha256().hashLong(0), () -> { WorkerProcess workerProcess = startWorkerProcess.get(); workerProcess.ensureLaunchAndHandshake(); @@ -381,11 +381,11 @@ private static WorkerProcessPool createPool( }); } - private static WorkerProcessPool createPool(int maxWorkers) { + private static WorkerProcessPoolSync createPool(int maxWorkers) { return createPool(maxWorkers, x -> {}); } - private static WorkerProcessPool createPool( + private static WorkerProcessPoolSync createPool( int maxWorkers, Consumer onWorkerCreated) { return createPool( maxWorkers, @@ -396,7 +396,7 @@ private static WorkerProcessPool createPool( }); } - private static WorkerProcessPool createPool( + private static WorkerProcessPoolSync createPool( int maxWorkers, BlockingQueue> workers) { return createPool( maxWorkers, @@ -414,7 +414,7 @@ private static Set concurrentSet() { } private static UnsafeRunnable borrowWorkerProcessWithoutReturning( - WorkerProcessPool pool, Set createdWorkers) { + WorkerProcessPoolSync pool, Set createdWorkers) { return () -> { BorrowedWorkerProcess worker = pool.borrowWorkerProcess(); WorkerProcess process = worker.get(); @@ -424,7 +424,7 @@ private static UnsafeRunnable borrowWorkerProcessWithoutReturning( } private static UnsafeRunnable borrowAndReturnWorkerProcess( - WorkerProcessPool pool, ConcurrentHashMap usedWorkers) { + WorkerProcessPoolSync pool, ConcurrentHashMap usedWorkers) { return () -> { try (BorrowedWorkerProcess worker = pool.borrowWorkerProcess()) { WorkerProcess workerProcess = worker.get(); @@ -434,13 +434,13 @@ private static UnsafeRunnable borrowAndReturnWorkerProcess( }; } - private static void acquireWorkersThenRelease(WorkerProcessPool pool, int numWorkers) + private static void acquireWorkersThenRelease(WorkerProcessPoolSync pool, int numWorkers) throws Exception { acquireWorkersThenRunActionThenRelease(pool, numWorkers, () -> {}); } private static void acquireWorkersThenRunActionThenRelease( - WorkerProcessPool pool, int numWorkers, UnsafeRunnable action) throws Exception { + WorkerProcessPoolSync pool, int numWorkers, UnsafeRunnable action) throws Exception { if (numWorkers < 1) { action.run(); return; diff --git a/test/com/facebook/buck/worker/WorkerProcessProtocolZeroTest.java b/test/com/facebook/buck/worker/WorkerProcessProtocolZeroTest.java index 07bce010336..44a89200dcc 100644 --- a/test/com/facebook/buck/worker/WorkerProcessProtocolZeroTest.java +++ b/test/com/facebook/buck/worker/WorkerProcessProtocolZeroTest.java @@ -16,6 +16,7 @@ package com.facebook.buck.worker; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -126,7 +127,7 @@ public void testReceiveCommandResponse() throws IOException { new WorkerProcessProtocolZero.CommandSender( dummyOutputStream, jsonReader, newTempFile(), () -> {}, () -> true); - protocol.receiveCommandResponse(messageID); + assertEquals(protocol.receiveNextCommandResponse().getCommandId(), messageID); } @Test @@ -140,25 +141,7 @@ public void testReceiveCommandResponseWithMalformedJSON() throws IOException { new WorkerProcessProtocolZero.CommandSender( dummyOutputStream, inputStream(malformedJson), newTempFile(), () -> {}, () -> true); - protocol.receiveCommandResponse(123); - } - - @Test - public void testReceiveCommandResponseWithIncorrectMessageID() throws IOException { - int messageID = 123; - expectedException.expect(HumanReadableException.class); - expectedException.expectMessage( - String.format("Expected response's \"id\" value to be \"%d\"", messageID)); - - int differentMessageID = 456; - InputStream jsonReader = - createMockJsonReaderForReceiveCommandResponse(differentMessageID, "result", 0); - - WorkerProcessProtocol.CommandSender protocol = - new WorkerProcessProtocolZero.CommandSender( - dummyOutputStream, jsonReader, newTempFile(), () -> {}, () -> true); - - protocol.receiveCommandResponse(messageID); + assertEquals(protocol.receiveNextCommandResponse().getCommandId(), 123); } @Test @@ -174,7 +157,7 @@ public void testReceiveCommandResponseWithInvalidType() throws IOException { new WorkerProcessProtocolZero.CommandSender( dummyOutputStream, jsonReader, newTempFile(), () -> {}, () -> true); - protocol.receiveCommandResponse(messageID); + assertEquals(protocol.receiveNextCommandResponse().getCommandId(), messageID); } @Test diff --git a/test/com/facebook/buck/worker/WorkerProcessTest.java b/test/com/facebook/buck/worker/WorkerProcessTest.java index c882a660eec..90dfbbba193 100644 --- a/test/com/facebook/buck/worker/WorkerProcessTest.java +++ b/test/com/facebook/buck/worker/WorkerProcessTest.java @@ -38,11 +38,16 @@ import com.facebook.buck.util.Verbosity; import com.facebook.buck.util.environment.Platform; import com.google.common.collect.ImmutableList; +import com.google.common.util.concurrent.ListenableFuture; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.hamcrest.Matchers; import org.junit.Rule; import org.junit.Test; @@ -56,7 +61,7 @@ private ProcessExecutorParams createDummyParams() { } @Test - public void testSubmitAndWaitForJob() throws IOException { + public void testJob() throws IOException, ExecutionException, InterruptedException { ProjectFilesystem filesystem = new FakeProjectFilesystem(); Path tmpPath = Files.createTempDirectory("tmp").toAbsolutePath().normalize(); Path argsPath = Paths.get(tmpPath.toString(), "0.args"); @@ -72,23 +77,105 @@ public void testSubmitAndWaitForJob() throws IOException { try (WorkerProcess process = new WorkerProcess( new FakeProcessExecutor(), createDummyParams(), filesystem, workerStdErr, tmpPath)) { - process.setProtocol( + process.launchForTesting( new FakeWorkerProcessProtocol.FakeCommandSender() { @Override - public int receiveCommandResponse(int messageID) throws IOException { - // simulate the external tool and write the stdout and stderr files + public void send(int messageId, WorkerProcessCommand command) throws IOException { filesystem.writeContentsToPath(stdout.get(), stdoutPath); filesystem.writeContentsToPath(stderr.get(), stderrPath); - return super.receiveCommandResponse(messageID); + super.send(messageId, command); } }); WorkerJobResult expectedResult = WorkerJobResult.of(exitCode, stdout, stderr); - assertThat(process.submitAndWaitForJob(jobArgs), Matchers.equalTo(expectedResult)); + assertThat(process.submitJob(jobArgs).get(), Matchers.equalTo(expectedResult)); assertThat(filesystem.readFileIfItExists(argsPath).get(), Matchers.equalTo(jobArgs)); } } + @Test(timeout = 20 * 1000) + public void testUncleanShutdown() throws IOException, ExecutionException, InterruptedException { + FakeWorkerProcessProtocol.FakeCommandSender protocol = + new FakeWorkerProcessProtocol.FakeCommandSender() { + @Override + public void send(int messageId, WorkerProcessCommand command) throws IOException { + // Black hole all messages. + } + }; + + try (WorkerProcess process = + new WorkerProcess( + new FakeProcessExecutor(), + createDummyParams(), + new FakeProjectFilesystem(), + Paths.get("stderr"), + Paths.get("tmp").toAbsolutePath().normalize())) { + process.launchForTesting(protocol); + + ListenableFuture job = process.submitJob("do stuff"); + try { + job.get(1000, TimeUnit.MILLISECONDS); + fail("Should have thrown timeout exception?"); + } catch (TimeoutException ignored) { + } + + assertFalse(protocol.isClosed()); + process.close(); + assertTrue(protocol.isClosed()); + + try { + job.get(); + fail("Should have thrown exception from job.get"); + } catch (ExecutionException ignored) { + } + } + } + + @Test(timeout = 20 * 1000) + public void testDeadProcess() throws IOException, ExecutionException, InterruptedException { + FakeWorkerProcessProtocol.FakeCommandSender protocol = + new FakeWorkerProcessProtocol.FakeCommandSender() { + @Override + public void send(int messageId, WorkerProcessCommand command) throws IOException { + // Noop + } + + @Override + public WorkerProcessProtocol.CommandResponse receiveNextCommandResponse() + throws IOException { + throw new IOException("IO Exception!"); + } + }; + + try (WorkerProcess process = + new WorkerProcess( + new FakeProcessExecutor(), + createDummyParams(), + new FakeProjectFilesystem(), + Paths.get("stderr"), + Paths.get("tmp").toAbsolutePath().normalize())) { + process.launchForTesting(protocol); + + ListenableFuture job = process.submitJob("do stuff"); + try { + job.get(); + fail("Should have thrown execution exception?"); + } catch (ExecutionException ignored) { + } + + ListenableFuture job2 = process.submitJob("do more stuff"); + try { + job2.get(); + fail("Should have thrown execution exception?"); + } catch (ExecutionException ignored) { + } + + assertFalse(protocol.isClosed()); + process.close(); + assertTrue(protocol.isClosed()); + } + } + @Test public void testClose() { FakeWorkerProcessProtocol.FakeCommandSender protocol = @@ -101,7 +188,7 @@ public void testClose() { new FakeProjectFilesystem(), Paths.get("stderr"), Paths.get("tmp").toAbsolutePath().normalize())) { - process.setProtocol(protocol); + process.launchForTesting(protocol); assertFalse(protocol.isClosed()); process.close(); @@ -109,6 +196,58 @@ public void testClose() { } } + @Test(timeout = 20 * 1000) + public void testConcurrentExecution() + throws IOException, InterruptedException, ExecutionException { + CountDownLatch latch = new CountDownLatch(2); + + ProjectFilesystem filesystem = new FakeProjectFilesystem(); + Path tmpPath = Files.createTempDirectory("tmp").toAbsolutePath().normalize(); + Path workerStdErr = Paths.get(tmpPath.toString(), "stderr"); + + Optional stdout = Optional.of("my stdout"); + Optional stderr = Optional.of("my stderr"); + + FakeWorkerProcessProtocol.FakeCommandSender protocol = + new FakeWorkerProcessProtocol.FakeCommandSender() { + @Override + public void send(int messageId, WorkerProcessCommand command) throws IOException { + filesystem.writeContentsToPath(stdout.get(), command.getStdOutPath()); + filesystem.writeContentsToPath(stderr.get(), command.getStdErrPath()); + latch.countDown(); + super.send(messageId, command); + } + + @Override + public WorkerProcessProtocol.CommandResponse receiveNextCommandResponse() + throws IOException { + try { + latch.await(); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return super.receiveNextCommandResponse(); + } + }; + + try (WorkerProcess process = + new WorkerProcess( + new FakeProcessExecutor(), createDummyParams(), filesystem, workerStdErr, tmpPath)) { + process.launchForTesting(protocol); + + ListenableFuture result1 = process.submitJob("job1"); + try { + result1.get(1000, TimeUnit.MILLISECONDS); + fail("Should have thrown timeout exception?"); + } catch (TimeoutException ignored) { + } + + ListenableFuture result2 = process.submitJob("job2"); + assertThat(result1.get(), Matchers.equalTo(WorkerJobResult.of(0, stdout, stderr))); + assertThat(result2.get(), Matchers.equalTo(WorkerJobResult.of(0, stdout, stderr))); + } + } + @Test(timeout = 20 * 1000) public void testDoesNotBlockOnLargeStderr() throws IOException { ProjectWorkspace workspace = diff --git a/test/com/facebook/buck/worker/testdata/worker_process/script.bat b/test/com/facebook/buck/worker/testdata/worker_process/script.bat index 29e88441a67..4dcb7864795 100644 --- a/test/com/facebook/buck/worker/testdata/worker_process/script.bat +++ b/test/com/facebook/buck/worker/testdata/worker_process/script.bat @@ -9,7 +9,7 @@ FOR /L %%i IN (1,1,1024) DO ( REM Write invalid JSON to stdout to trigger an exception when Buck tries to REM consume our output. -ECHO "}" +ECHO } REM Consume all of stdin to avoid a race with Buck trying to talk to us. TYPE CON diff --git a/third-party/java/ObjCBridge/BUCK b/third-party/java/ObjCBridge/BUCK index 46dab407b28..122f890ca2e 100644 --- a/third-party/java/ObjCBridge/BUCK +++ b/third-party/java/ObjCBridge/BUCK @@ -1,9 +1,10 @@ prebuilt_jar( name = "ObjCBridge", - binary_jar = "ObjCBridge.jar", + binary_jar = "java-objc-bridge-1.1-SNAPSHOT.jar", licenses = [ "LICENSE", ], + source_jar = "java-objc-bridge-1.1-SNAPSHOT-sources.jar", visibility = ["PUBLIC"], deps = [ "//third-party/java/jna:jna", diff --git a/third-party/java/ObjCBridge/ObjCBridge.jar b/third-party/java/ObjCBridge/ObjCBridge.jar deleted file mode 100644 index f558a6f5bb7..00000000000 Binary files a/third-party/java/ObjCBridge/ObjCBridge.jar and /dev/null differ diff --git a/third-party/java/ObjCBridge/README.facebook b/third-party/java/ObjCBridge/README.facebook index 193e7075d84..caa5ca9702c 100644 --- a/third-party/java/ObjCBridge/README.facebook +++ b/third-party/java/ObjCBridge/README.facebook @@ -1,6 +1,16 @@ Project: Java Objective-C Bridge -Version: Git revision c22107743c2751f1fbaef3f509d59683f16adb60 -Release Date: 2014-08-06 +Version: Git revision c38d607820e18bdc607067457ba6680a6277f7cd +Release Date: Sun Nov 3 19:35:06 2019 +0100 Website: https://github.com/shannah/Java-Objective-C-Bridge/ License: Apache 2.0 Description: JNA library to allow Java programs to invoke Objective-C APIs +Instructions: + brew install maven + brew install gpg + gpg --gen-key + git clone https://github.com/shannah/Java-Objective-C-Bridge.git + cd Java-Objective-C-Bridge + Edit pom.xml (or apply patch.diff) + Change to version 8 + Change to version 8 + mvn clean install -Drelease=true diff --git a/third-party/java/ObjCBridge/java-objc-bridge-1.1-SNAPSHOT-javadoc.jar b/third-party/java/ObjCBridge/java-objc-bridge-1.1-SNAPSHOT-javadoc.jar new file mode 100644 index 00000000000..c6ea5287adf Binary files /dev/null and b/third-party/java/ObjCBridge/java-objc-bridge-1.1-SNAPSHOT-javadoc.jar differ diff --git a/third-party/java/ObjCBridge/java-objc-bridge-1.1-SNAPSHOT-sources.jar b/third-party/java/ObjCBridge/java-objc-bridge-1.1-SNAPSHOT-sources.jar new file mode 100644 index 00000000000..996c6b79e3f Binary files /dev/null and b/third-party/java/ObjCBridge/java-objc-bridge-1.1-SNAPSHOT-sources.jar differ diff --git a/third-party/java/ObjCBridge/java-objc-bridge-1.1-SNAPSHOT.jar b/third-party/java/ObjCBridge/java-objc-bridge-1.1-SNAPSHOT.jar new file mode 100644 index 00000000000..d4bb935503d Binary files /dev/null and b/third-party/java/ObjCBridge/java-objc-bridge-1.1-SNAPSHOT.jar differ diff --git a/third-party/java/ObjCBridge/libjcocoa.dylib b/third-party/java/ObjCBridge/libjcocoa.dylib index f01c68b9c18..9cb1873bf13 100755 Binary files a/third-party/java/ObjCBridge/libjcocoa.dylib and b/third-party/java/ObjCBridge/libjcocoa.dylib differ diff --git a/third-party/java/ObjCBridge/libjcocoa.dylib.dSYM/Contents/Info.plist b/third-party/java/ObjCBridge/libjcocoa.dylib.dSYM/Contents/Info.plist new file mode 100644 index 00000000000..58347f605d3 --- /dev/null +++ b/third-party/java/ObjCBridge/libjcocoa.dylib.dSYM/Contents/Info.plist @@ -0,0 +1,20 @@ + + + + + CFBundleDevelopmentRegion + English + CFBundleIdentifier + com.apple.xcode.dsym.libjcocoa.dylib + CFBundleInfoDictionaryVersion + 6.0 + CFBundlePackageType + dSYM + CFBundleSignature + ???? + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + + diff --git a/third-party/java/ObjCBridge/libjcocoa.dylib.dSYM/Contents/Resources/DWARF/libjcocoa.dylib b/third-party/java/ObjCBridge/libjcocoa.dylib.dSYM/Contents/Resources/DWARF/libjcocoa.dylib new file mode 100644 index 00000000000..270550ec424 Binary files /dev/null and b/third-party/java/ObjCBridge/libjcocoa.dylib.dSYM/Contents/Resources/DWARF/libjcocoa.dylib differ diff --git a/third-party/java/ObjCBridge/patch.diff b/third-party/java/ObjCBridge/patch.diff new file mode 100644 index 00000000000..4de66405a49 --- /dev/null +++ b/third-party/java/ObjCBridge/patch.diff @@ -0,0 +1,27 @@ +diff --git a/pom.xml b/pom.xml +index 76d7547..50fef9a 100644 +--- a/pom.xml ++++ b/pom.xml +@@ -40,8 +40,8 @@ + + + UTF-8 +- 11 +- 11 ++ 8 ++ 8 + Debug + + +diff --git a/src/main/java/ca/weblite/objc/Proxy.java b/src/main/java/ca/weblite/objc/Proxy.java +index 0654519..7aba1f5 100644 +--- a/src/main/java/ca/weblite/objc/Proxy.java ++++ b/src/main/java/ca/weblite/objc/Proxy.java +@@ -501,7 +501,6 @@ public class Proxy implements Peerable { + */ + @Override + public String toString(){ +- System.out.println("The peer is "+getPeer()); + if ( getPeer() == null ){ + return "null"; + } diff --git a/third-party/java/jna/BUCK b/third-party/java/jna/BUCK index 7c3e5e17bd1..0ce430edc44 100644 --- a/third-party/java/jna/BUCK +++ b/third-party/java/jna/BUCK @@ -1,19 +1,19 @@ prebuilt_jar( name = "jna", - binary_jar = "jna-4.5.1.jar", + binary_jar = "jna-5.6.0.jar", licenses = [ "LICENSE", ], - source_jar = "jna-4.5.1-sources.jar", + source_jar = "jna-5.6.0-sources.jar", visibility = ["PUBLIC"], ) prebuilt_jar( name = "jna-platform", - binary_jar = "jna-platform-4.5.1.jar", + binary_jar = "jna-platform-5.6.0.jar", licenses = [ "LICENSE", ], - source_jar = "jna-platform-4.5.1-sources.jar", + source_jar = "jna-platform-5.6.0-sources.jar", visibility = ["PUBLIC"], ) diff --git a/third-party/java/jna/README.facebook b/third-party/java/jna/README.facebook index 7f4bacea943..7e9df94b41c 100644 --- a/third-party/java/jna/README.facebook +++ b/third-party/java/jna/README.facebook @@ -2,6 +2,6 @@ README for JNA URL: https://github.com/java-native-access/jna Downloaded From: - http://central.maven.org/maven2/net/java/dev/jna/jna/4.5.1/ - http://central.maven.org/maven2/net/java/dev/jna/jna-platform/4.5.1/ + https://search.maven.org/artifact/net.java.dev.jna/jna/5.6.0/jar + https://search.maven.org/artifact/net.java.dev.jna/jna-platform/5.6.0/jar License: Apache 2.0 and LPGL 2.1 dual license diff --git a/third-party/java/jna/jna-4.5.1-sources.jar b/third-party/java/jna/jna-4.5.1-sources.jar deleted file mode 100644 index c4b516c1773..00000000000 Binary files a/third-party/java/jna/jna-4.5.1-sources.jar and /dev/null differ diff --git a/third-party/java/jna/jna-4.5.1.jar b/third-party/java/jna/jna-4.5.1.jar deleted file mode 100644 index 68216afbea0..00000000000 Binary files a/third-party/java/jna/jna-4.5.1.jar and /dev/null differ diff --git a/third-party/java/jna/jna-5.6.0-javadoc.jar b/third-party/java/jna/jna-5.6.0-javadoc.jar new file mode 100644 index 00000000000..8be1f339e34 Binary files /dev/null and b/third-party/java/jna/jna-5.6.0-javadoc.jar differ diff --git a/third-party/java/jna/jna-5.6.0-sources.jar b/third-party/java/jna/jna-5.6.0-sources.jar new file mode 100644 index 00000000000..36f599d36d1 Binary files /dev/null and b/third-party/java/jna/jna-5.6.0-sources.jar differ diff --git a/third-party/java/jna/jna-5.6.0.jar b/third-party/java/jna/jna-5.6.0.jar new file mode 100644 index 00000000000..fda5aa4132a Binary files /dev/null and b/third-party/java/jna/jna-5.6.0.jar differ diff --git a/third-party/java/jna/jna-platform-4.5.1-sources.jar b/third-party/java/jna/jna-platform-4.5.1-sources.jar deleted file mode 100644 index 98308622da3..00000000000 Binary files a/third-party/java/jna/jna-platform-4.5.1-sources.jar and /dev/null differ diff --git a/third-party/java/jna/jna-platform-5.6.0-javadoc.jar b/third-party/java/jna/jna-platform-5.6.0-javadoc.jar new file mode 100644 index 00000000000..1fae1207962 Binary files /dev/null and b/third-party/java/jna/jna-platform-5.6.0-javadoc.jar differ diff --git a/third-party/java/jna/jna-platform-5.6.0-sources.jar b/third-party/java/jna/jna-platform-5.6.0-sources.jar new file mode 100644 index 00000000000..2c3ee27a365 Binary files /dev/null and b/third-party/java/jna/jna-platform-5.6.0-sources.jar differ diff --git a/third-party/java/jna/jna-platform-4.5.1.jar b/third-party/java/jna/jna-platform-5.6.0.jar similarity index 55% rename from third-party/java/jna/jna-platform-4.5.1.jar rename to third-party/java/jna/jna-platform-5.6.0.jar index 5aeb3e4e35f..91b263520b6 100644 Binary files a/third-party/java/jna/jna-platform-4.5.1.jar and b/third-party/java/jna/jna-platform-5.6.0.jar differ diff --git a/third-party/java/log4j2/BUCK b/third-party/java/log4j2/BUCK index 50e756bfec8..d5211bc4602 100644 --- a/third-party/java/log4j2/BUCK +++ b/third-party/java/log4j2/BUCK @@ -1,6 +1,6 @@ prebuilt_jar( name = "log4j2-api", - binary_jar = "log4j-api-2.13.0.jar", + binary_jar = "log4j-api-2.17.1.jar", licenses = [ "LICENSE", ], @@ -12,7 +12,7 @@ prebuilt_jar( prebuilt_jar( name = "log4j2-core", - binary_jar = "log4j-core-2.13.0.jar", + binary_jar = "log4j-core-2.17.1.jar", licenses = [ "LICENSE", ], diff --git a/third-party/java/log4j2/log4j-api-2.17.1-sources.jar b/third-party/java/log4j2/log4j-api-2.17.1-sources.jar new file mode 100644 index 00000000000..2ebb2c51880 Binary files /dev/null and b/third-party/java/log4j2/log4j-api-2.17.1-sources.jar differ diff --git a/third-party/java/log4j2/log4j-api-2.17.1.jar b/third-party/java/log4j2/log4j-api-2.17.1.jar new file mode 100644 index 00000000000..605c45d0433 Binary files /dev/null and b/third-party/java/log4j2/log4j-api-2.17.1.jar differ diff --git a/third-party/java/log4j2/log4j-core-2.17.1-sources.jar b/third-party/java/log4j2/log4j-core-2.17.1-sources.jar new file mode 100644 index 00000000000..cc1894f3e65 Binary files /dev/null and b/third-party/java/log4j2/log4j-core-2.17.1-sources.jar differ diff --git a/third-party/java/log4j2/log4j-core-2.17.1.jar b/third-party/java/log4j2/log4j-core-2.17.1.jar new file mode 100644 index 00000000000..bbead1267d9 Binary files /dev/null and b/third-party/java/log4j2/log4j-core-2.17.1.jar differ diff --git a/third-party/java/nailgun/BUCK b/third-party/java/nailgun/BUCK index f59eb231472..ef6f117ce2e 100644 --- a/third-party/java/nailgun/BUCK +++ b/third-party/java/nailgun/BUCK @@ -1,10 +1,10 @@ prebuilt_jar( name = "nailgun", - binary_jar = "nailgun-server-1.0.0.jar", + binary_jar = "nailgun-server-1.0.1.jar", licenses = [ "LICENSE", ], - source_jar = "nailgun-server-1.0.0-sources.jar", + source_jar = "nailgun-server-1.0.1-sources.jar", visibility = [ "//src/com/facebook/buck/cli:cli", "//src/com/facebook/buck/cli/exceptions/handlers:handlers", diff --git a/third-party/java/nailgun/README.txt b/third-party/java/nailgun/README.txt index c0c629ea0db..54eeefe177f 100644 --- a/third-party/java/nailgun/README.txt +++ b/third-party/java/nailgun/README.txt @@ -1,7 +1,8 @@ -nailgun-server-1.0.0.jar and nailgun-server-1.0.0-sources.jar were -built from https://github.com/facebook/nailgun +Repository: https://github.com/facebook/nailgun +URL: https://search.maven.org/artifact/com.facebook/nailgun-server/1.0.1/jar +Version: 1.0.1 -To regenerate these jars: +Build Instructions: 0) install maven (brew install maven) 1) git clone https://github.com/facebook/nailgun diff --git a/third-party/java/nailgun/nailgun-server-1.0.1-javadoc.jar b/third-party/java/nailgun/nailgun-server-1.0.1-javadoc.jar new file mode 100644 index 00000000000..9ab091a2003 Binary files /dev/null and b/third-party/java/nailgun/nailgun-server-1.0.1-javadoc.jar differ diff --git a/third-party/java/nailgun/nailgun-server-1.0.0-sources.jar b/third-party/java/nailgun/nailgun-server-1.0.1-sources.jar similarity index 65% rename from third-party/java/nailgun/nailgun-server-1.0.0-sources.jar rename to third-party/java/nailgun/nailgun-server-1.0.1-sources.jar index c6d5d58c6e4..7f24928e862 100644 Binary files a/third-party/java/nailgun/nailgun-server-1.0.0-sources.jar and b/third-party/java/nailgun/nailgun-server-1.0.1-sources.jar differ diff --git a/third-party/java/nailgun/nailgun-server-1.0.0.jar b/third-party/java/nailgun/nailgun-server-1.0.1.jar similarity index 82% rename from third-party/java/nailgun/nailgun-server-1.0.0.jar rename to third-party/java/nailgun/nailgun-server-1.0.1.jar index 1c9333c8c85..112e4f8c879 100644 Binary files a/third-party/java/nailgun/nailgun-server-1.0.0.jar and b/third-party/java/nailgun/nailgun-server-1.0.1.jar differ diff --git a/third-party/java/nuprocess/BUCK b/third-party/java/nuprocess/BUCK index e05f557e50e..9fb5c1436ac 100644 --- a/third-party/java/nuprocess/BUCK +++ b/third-party/java/nuprocess/BUCK @@ -1,10 +1,10 @@ prebuilt_jar( name = "nuprocess", - binary_jar = "nuprocess-1.2.4.jar", + binary_jar = "nuprocess-2.0.1.jar", licenses = [ "LICENSE", ], - source_jar = "nuprocess-1.2.4-sources.jar", + source_jar = "nuprocess-2.0.1-sources.jar", visibility = ["PUBLIC"], deps = [ "//third-party/java/jna:jna", diff --git a/third-party/java/nuprocess/README.facebook b/third-party/java/nuprocess/README.facebook index 34c7618bad4..35a933801e1 100644 --- a/third-party/java/nuprocess/README.facebook +++ b/third-party/java/nuprocess/README.facebook @@ -1,6 +1,8 @@ README for NuProcess -URL: https://github.com/brettwooldridge/NuProcess -Revision: 562e0b39cd893ca389ba2d0b3689dde305b7632d -Built With: mvn package +Repository: https://github.com/brettwooldridge/NuProcess +URL: https://search.maven.org/artifact/com.zaxxer/nuprocess/2.0.1/bundle +Version: 2.01 +Build Instructions: + mvn package License: Apache 2.0 license diff --git a/third-party/java/nuprocess/nuprocess-1.2.4-sources.jar b/third-party/java/nuprocess/nuprocess-1.2.4-sources.jar deleted file mode 100644 index 2c7c2c2fc5c..00000000000 Binary files a/third-party/java/nuprocess/nuprocess-1.2.4-sources.jar and /dev/null differ diff --git a/third-party/java/nuprocess/nuprocess-1.2.4.jar b/third-party/java/nuprocess/nuprocess-1.2.4.jar deleted file mode 100644 index 7475418b426..00000000000 Binary files a/third-party/java/nuprocess/nuprocess-1.2.4.jar and /dev/null differ diff --git a/third-party/java/nuprocess/nuprocess-2.0.1-javadoc.jar b/third-party/java/nuprocess/nuprocess-2.0.1-javadoc.jar new file mode 100644 index 00000000000..735f3b7b057 Binary files /dev/null and b/third-party/java/nuprocess/nuprocess-2.0.1-javadoc.jar differ diff --git a/third-party/java/nuprocess/nuprocess-2.0.1-sources.jar b/third-party/java/nuprocess/nuprocess-2.0.1-sources.jar new file mode 100644 index 00000000000..d1d90c21d2d Binary files /dev/null and b/third-party/java/nuprocess/nuprocess-2.0.1-sources.jar differ diff --git a/third-party/java/nuprocess/nuprocess-2.0.1.jar b/third-party/java/nuprocess/nuprocess-2.0.1.jar new file mode 100644 index 00000000000..7c856ed78df Binary files /dev/null and b/third-party/java/nuprocess/nuprocess-2.0.1.jar differ diff --git a/third-party/java/oshi/BUCK b/third-party/java/oshi/BUCK index 237c82d46e3..b0bf74e1ceb 100644 --- a/third-party/java/oshi/BUCK +++ b/third-party/java/oshi/BUCK @@ -1,10 +1,10 @@ prebuilt_jar( name = "oshi-core", - binary_jar = "oshi-core-3.3-SNAPSHOT.jar", + binary_jar = "oshi-core-5.2.5.jar", licenses = [ "LICENSE.html", ], - source_jar = "oshi-core-3.3-SNAPSHOT-sources.jar", + source_jar = "oshi-core-5.2.5-sources.jar", visibility = ["PUBLIC"], deps = [ "//third-party/java/jna:jna", diff --git a/third-party/java/oshi/README.facebook b/third-party/java/oshi/README.facebook index b8dbfef4dcf..3dcaf4e924a 100644 --- a/third-party/java/oshi/README.facebook +++ b/third-party/java/oshi/README.facebook @@ -1,8 +1,9 @@ README for OSHI -URL: https://github.com/oshi/oshi/commit/e51f8e84f20bc4b132aa02601a4760b20bd4a198 -Version: 3.3-SNAPSHOT e51f8e84f20bc4b132aa02601a4760b20bd4a198 -Built with: +Repository: https://github.com/oshi/oshi +URL: https://search.maven.org/artifact/com.github.oshi/oshi-core/5.2.5/jar +Version: 5.2.5 +Build Instructions: mvn package mvn source:jar License: Eclipse Public License v1.0 diff --git a/third-party/java/oshi/oshi-core-3.3-SNAPSHOT-sources.jar b/third-party/java/oshi/oshi-core-3.3-SNAPSHOT-sources.jar deleted file mode 100644 index 9fde3592b57..00000000000 Binary files a/third-party/java/oshi/oshi-core-3.3-SNAPSHOT-sources.jar and /dev/null differ diff --git a/third-party/java/oshi/oshi-core-3.3-SNAPSHOT.jar b/third-party/java/oshi/oshi-core-3.3-SNAPSHOT.jar deleted file mode 100644 index 5772648ad55..00000000000 Binary files a/third-party/java/oshi/oshi-core-3.3-SNAPSHOT.jar and /dev/null differ diff --git a/third-party/java/oshi/oshi-core-5.2.5-javadoc.jar b/third-party/java/oshi/oshi-core-5.2.5-javadoc.jar new file mode 100644 index 00000000000..4499e583381 Binary files /dev/null and b/third-party/java/oshi/oshi-core-5.2.5-javadoc.jar differ diff --git a/third-party/java/oshi/oshi-core-5.2.5-sources.jar b/third-party/java/oshi/oshi-core-5.2.5-sources.jar new file mode 100644 index 00000000000..e21677e6e16 Binary files /dev/null and b/third-party/java/oshi/oshi-core-5.2.5-sources.jar differ diff --git a/third-party/java/oshi/oshi-core-5.2.5.jar b/third-party/java/oshi/oshi-core-5.2.5.jar new file mode 100644 index 00000000000..c53d2451eec Binary files /dev/null and b/third-party/java/oshi/oshi-core-5.2.5.jar differ diff --git a/third-party/py/pathlib/test_pathlib.py b/third-party/py/pathlib/test_pathlib.py index 7147e4ee5a3..5522fdcc34d 100755 --- a/third-party/py/pathlib/test_pathlib.py +++ b/third-party/py/pathlib/test_pathlib.py @@ -1,4 +1,3 @@ -import collections import io import os import errno @@ -19,8 +18,10 @@ raise ImportError("unittest2 is required for tests on pre-2.7") try: + import collections.abc as collections_abc from test import support except ImportError: + import collections as collections_abc # Fallback for PY3.2. from test import test_support as support TESTFN = support.TESTFN @@ -1395,7 +1396,7 @@ def _check(glob, expected): P = self.cls p = P(BASE) it = p.glob("fileA") - self.assertIsInstance(it, collections.Iterator) + self.assertIsInstance(it, collections_abc.Iterator) _check(it, ["fileA"]) _check(p.glob("fileB"), []) _check(p.glob("dir*/file*"), ["dirB/fileB", "dirC/fileC"]) @@ -1420,7 +1421,7 @@ def _check(glob, expected): P = self.cls p = P(BASE) it = p.rglob("fileA") - self.assertIsInstance(it, collections.Iterator) + self.assertIsInstance(it, collections_abc.Iterator) # XXX cannot test because of symlink loops in the test setup #_check(it, ["fileA"]) #_check(p.rglob("fileB"), ["dirB/fileB"]) diff --git a/third-party/py/pex/README.facebook b/third-party/py/pex/README.facebook index fd7aee556fc..f08e6b58844 100644 --- a/third-party/py/pex/README.facebook +++ b/third-party/py/pex/README.facebook @@ -13,3 +13,5 @@ Local modifications: - Back-ported Python 3.6 compatibility commit c5ab73fd4d8e816e21a89d48c8d0c8095ef5a49c - Back-ported namespaced packages fix, commit 7d2dc7f500aa7ae227c3ddca4b278b807d353a5e - Fixed Python 3 issue with writing bytes to a text file (`with open(path, 'wb') as fp:` on line 68 in `compiler.py`) + - Imported from collections.abc instead of collections to support Python 3.10 + - Back-ported removal of MarkerEvaluation from pieces of commit ba5633b3c7b9317b87130a2ea671d8c008a673d6 and a718819d2849196e902808301c9a95724510c5c1 diff --git a/third-party/py/pex/pex/base.py b/third-party/py/pex/pex/base.py index 7d90443dd3c..2a3d775f9e0 100644 --- a/third-party/py/pex/pex/base.py +++ b/third-party/py/pex/pex/base.py @@ -3,7 +3,10 @@ from __future__ import absolute_import -from collections import Iterable +try: + from collections.abc import Iterable +except ImportError: # For PY3.2 + from collections import Iterable from pkg_resources import Requirement diff --git a/third-party/py/pex/pex/link.py b/third-party/py/pex/pex/link.py index 542f344e09a..5815adc94d2 100644 --- a/third-party/py/pex/pex/link.py +++ b/third-party/py/pex/pex/link.py @@ -5,12 +5,17 @@ import os import posixpath -from collections import Iterable from .compatibility import string as compatible_string from .compatibility import PY3 from .util import Memoizer + +try: + from collections.abc import Iterable +except ImportError: # For PY3.2 + from collections import Iterable + if PY3: import urllib.parse as urlparse else: diff --git a/third-party/py/pex/pex/orderedset.py b/third-party/py/pex/pex/orderedset.py index 3eb5cb45414..1a8db948504 100644 --- a/third-party/py/pex/pex/orderedset.py +++ b/third-party/py/pex/pex/orderedset.py @@ -8,10 +8,14 @@ # modifications # -import collections +try: + import collections.abc as collections_abc +except ImportError: # For PY3.2 + import collections as collections_abc -class OrderedSet(collections.MutableSet): + +class OrderedSet(collections_abc.MutableSet): KEY, PREV, NEXT = range(3) def __init__(self, iterable=None): diff --git a/third-party/py/pywatchman/pywatchman/pybser.py b/third-party/py/pywatchman/pywatchman/pybser.py index 91dff190c03..01f0c42bc0a 100644 --- a/third-party/py/pywatchman/pywatchman/pybser.py +++ b/third-party/py/pywatchman/pywatchman/pybser.py @@ -32,7 +32,6 @@ # no unicode literals import binascii -import collections import ctypes import struct import sys @@ -41,6 +40,11 @@ compat, ) +try: + import collections.abc as collections_abc +except ImportError: + import collections as collections_abc # Fallback for PY3.2. + BSER_ARRAY = b'\x00' BSER_OBJECT = b'\x01' BSER_BYTESTRING = b'\x02' @@ -177,8 +181,8 @@ def append_recursive(self, val): self.ensure_size(needed) struct.pack_into(b'=cd', self.buf, self.wpos, BSER_REAL, val) self.wpos += needed - elif isinstance(val, collections.Mapping) and \ - isinstance(val, collections.Sized): + elif isinstance(val, collections_abc.Mapping) and \ + isinstance(val, collections_abc.Sized): val_len = len(val) size = _int_size(val_len) needed = 2 + size @@ -205,8 +209,8 @@ def append_recursive(self, val): for k, v in iteritems: self.append_string(k) self.append_recursive(v) - elif isinstance(val, collections.Iterable) and \ - isinstance(val, collections.Sized): + elif isinstance(val, collections_abc.Iterable) and \ + isinstance(val, collections_abc.Sized): val_len = len(val) size = _int_size(val_len) needed = 2 + size diff --git a/third-party/py/pywatchman/tests/tests.py b/third-party/py/pywatchman/tests/tests.py index 02d9a0b6396..7e77da43ede 100755 --- a/third-party/py/pywatchman/tests/tests.py +++ b/third-party/py/pywatchman/tests/tests.py @@ -6,7 +6,6 @@ # no unicode literals import binascii -import collections import inspect import unittest import os diff --git a/third-party/py/setuptools/pkg_resources/__init__.py b/third-party/py/setuptools/pkg_resources/__init__.py index d09e0b6f9af..262399db8e3 100644 --- a/third-party/py/setuptools/pkg_resources/__init__.py +++ b/third-party/py/setuptools/pkg_resources/__init__.py @@ -29,7 +29,6 @@ import functools import pkgutil import token -import symbol import operator import platform import collections @@ -1402,202 +1401,30 @@ def to_filename(name): return name.replace('-','_') -class MarkerEvaluation(object): - values = { - 'os_name': lambda: os.name, - 'sys_platform': lambda: sys.platform, - 'python_full_version': platform.python_version, - 'python_version': lambda: platform.python_version()[:3], - 'platform_version': platform.version, - 'platform_machine': platform.machine, - 'platform_python_implementation': platform.python_implementation, - 'python_implementation': platform.python_implementation, - } - - @classmethod - def is_invalid_marker(cls, text): - """ - Validate text as a PEP 426 environment marker; return an exception - if invalid or False otherwise. - """ - try: - cls.evaluate_marker(text) - except SyntaxError as e: - return cls.normalize_exception(e) - return False - - @staticmethod - def normalize_exception(exc): - """ - Given a SyntaxError from a marker evaluation, normalize the error - message: - - Remove indications of filename and line number. - - Replace platform-specific error messages with standard error - messages. - """ - subs = { - 'unexpected EOF while parsing': 'invalid syntax', - 'parenthesis is never closed': 'invalid syntax', - } - exc.filename = None - exc.lineno = None - exc.msg = subs.get(exc.msg, exc.msg) - return exc - - @classmethod - def and_test(cls, nodelist): - # MUST NOT short-circuit evaluation, or invalid syntax can be skipped! - items = [ - cls.interpret(nodelist[i]) - for i in range(1, len(nodelist), 2) - ] - return functools.reduce(operator.and_, items) - - @classmethod - def test(cls, nodelist): - # MUST NOT short-circuit evaluation, or invalid syntax can be skipped! - items = [ - cls.interpret(nodelist[i]) - for i in range(1, len(nodelist), 2) - ] - return functools.reduce(operator.or_, items) - - @classmethod - def atom(cls, nodelist): - t = nodelist[1][0] - if t == token.LPAR: - if nodelist[2][0] == token.RPAR: - raise SyntaxError("Empty parentheses") - return cls.interpret(nodelist[2]) - msg = "Language feature not supported in environment markers" - raise SyntaxError(msg) - - @classmethod - def comparison(cls, nodelist): - if len(nodelist) > 4: - msg = "Chained comparison not allowed in environment markers" - raise SyntaxError(msg) - comp = nodelist[2][1] - cop = comp[1] - if comp[0] == token.NAME: - if len(nodelist[2]) == 3: - if cop == 'not': - cop = 'not in' - else: - cop = 'is not' - try: - cop = cls.get_op(cop) - except KeyError: - msg = repr(cop) + " operator not allowed in environment markers" - raise SyntaxError(msg) - return cop(cls.evaluate(nodelist[1]), cls.evaluate(nodelist[3])) - - @classmethod - def get_op(cls, op): - ops = { - symbol.test: cls.test, - symbol.and_test: cls.and_test, - symbol.atom: cls.atom, - symbol.comparison: cls.comparison, - 'not in': lambda x, y: x not in y, - 'in': lambda x, y: x in y, - '==': operator.eq, - '!=': operator.ne, - '<': operator.lt, - '>': operator.gt, - '<=': operator.le, - '>=': operator.ge, - } - if hasattr(symbol, 'or_test'): - ops[symbol.or_test] = cls.test - return ops[op] - - @classmethod - def evaluate_marker(cls, text, extra=None): - """ - Evaluate a PEP 426 environment marker on CPython 2.4+. - Return a boolean indicating the marker result in this environment. - Raise SyntaxError if marker is invalid. - - This implementation uses the 'parser' module, which is not implemented - on - Jython and has been superseded by the 'ast' module in Python 2.6 and - later. - """ - return cls.interpret(parser.expr(text).totuple(1)[1]) - - @staticmethod - def _translate_metadata2(env): - """ - Markerlib implements Metadata 1.2 (PEP 345) environment markers. - Translate the variables to Metadata 2.0 (PEP 426). - """ - return dict( - (key.replace('.', '_'), value) - for key, value in env - ) - - @classmethod - def _markerlib_evaluate(cls, text): - """ - Evaluate a PEP 426 environment marker using markerlib. - Return a boolean indicating the marker result in this environment. - Raise SyntaxError if marker is invalid. - """ - import _markerlib - - env = cls._translate_metadata2(_markerlib.default_environment()) - try: - result = _markerlib.interpret(text, env) - except NameError as e: - raise SyntaxError(e.args[0]) - return result - - if 'parser' not in globals(): - # Fall back to less-complete _markerlib implementation if 'parser' module - # is not available. - evaluate_marker = _markerlib_evaluate - - @classmethod - def interpret(cls, nodelist): - while len(nodelist)==2: nodelist = nodelist[1] - try: - op = cls.get_op(nodelist[0]) - except KeyError: - raise SyntaxError("Comparison or logical expression expected") - return op(nodelist) +def invalid_marker(text): + """ + Validate text as a PEP 508 environment marker; return an exception + if invalid or False otherwise. + """ + try: + evaluate_marker(text) + except packaging.markers.InvalidMarker as e: + e.filename = None + e.lineno = None + return e + return False - @classmethod - def evaluate(cls, nodelist): - while len(nodelist)==2: nodelist = nodelist[1] - kind = nodelist[0] - name = nodelist[1] - if kind==token.NAME: - try: - op = cls.values[name] - except KeyError: - raise SyntaxError("Unknown name %r" % name) - return op() - if kind==token.STRING: - s = nodelist[1] - if not cls._safe_string(s): - raise SyntaxError( - "Only plain strings allowed in environment markers") - return s[1:-1] - msg = "Language feature not supported in environment markers" - raise SyntaxError(msg) - @staticmethod - def _safe_string(cand): - return ( - cand[:1] in "'\"" and - not cand.startswith('"""') and - not cand.startswith("'''") and - '\\' not in cand - ) +def evaluate_marker(text, extra=None): + """ + Evaluate a PEP 508 environment marker. + Return a boolean indicating the marker result in this environment. + Raise InvalidMarker if marker is invalid. + This implementation uses the 'pyparsing' module. + """ + marker = packaging.markers.Marker(text) + return marker.evaluate() -invalid_marker = MarkerEvaluation.is_invalid_marker -evaluate_marker = MarkerEvaluation.evaluate_marker class NullProvider: """Try to implement resources and metadata for arbitrary PEP 302 loaders""" diff --git a/third-party/py/setuptools/pkg_resources/_vendor/packaging/markers.py b/third-party/py/setuptools/pkg_resources/_vendor/packaging/markers.py new file mode 100644 index 00000000000..80e63676661 --- /dev/null +++ b/third-party/py/setuptools/pkg_resources/_vendor/packaging/markers.py @@ -0,0 +1,221 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. +from __future__ import absolute_import, division, print_function + +import operator +import os +import platform +import sys + +from pkg_resources._vendor.pyparsing import ParseException, ParseResults, stringStart, stringEnd +from pkg_resources._vendor.pyparsing import ZeroOrMore, Group, Forward, QuotedString +from pkg_resources._vendor.pyparsing import Literal as L # noqa + +from ._compat import string_types +from .specifiers import Specifier, InvalidSpecifier + + +__all__ = [ + "InvalidMarker", "UndefinedComparison", "Marker", "default_environment", +] + + +class InvalidMarker(ValueError): + """ + An invalid marker was found, users should refer to PEP 508. + """ + + +class UndefinedComparison(ValueError): + """ + An invalid operation was attempted on a value that doesn't support it. + """ + + +class Node(object): + def __init__(self, value): + self.value = value + def __str__(self): + return str(self.value) + def __repr__(self): + return "<{0}({1!r})>".format(self.__class__.__name__, str(self)) + + +class Variable(Node): + pass + + +class Value(Node): + pass + +VARIABLE = ( + L("implementation_version") | + L("platform_python_implementation") | + L("implementation_name") | + L("python_full_version") | + L("platform_release") | + L("platform_version") | + L("platform_machine") | + L("platform_system") | + L("python_version") | + L("sys_platform") | + L("os_name") | + L("extra") +) +VARIABLE.setParseAction(lambda s, l, t: Variable(t[0])) + +VERSION_CMP = ( + L("===") | + L("==") | + L(">=") | + L("<=") | + L("!=") | + L("~=") | + L(">") | + L("<") +) +MARKER_OP = VERSION_CMP | L("not in") | L("in") +MARKER_VALUE = QuotedString("'") | QuotedString('"') +MARKER_VALUE.setParseAction(lambda s, l, t: Value(t[0])) +BOOLOP = L("and") | L("or") +MARKER_VAR = VARIABLE | MARKER_VALUE +MARKER_ITEM = Group(MARKER_VAR + MARKER_OP + MARKER_VAR) +MARKER_ITEM.setParseAction(lambda s, l, t: tuple(t[0])) +LPAREN = L("(").suppress() +RPAREN = L(")").suppress() +MARKER_EXPR = Forward() +MARKER_ATOM = MARKER_ITEM | Group(LPAREN + MARKER_EXPR + RPAREN) +MARKER_EXPR << MARKER_ATOM + ZeroOrMore(BOOLOP + MARKER_EXPR) +MARKER = stringStart + MARKER_EXPR + stringEnd + + +def _coerce_parse_result(results): + if isinstance(results, ParseResults): + return [_coerce_parse_result(i) for i in results] + else: + return results + + +def _format_marker(marker, first=True): + assert isinstance(marker, (list, tuple, string_types)) + # Sometimes we have a structure like [[...]] which is a single item list + # where the single item is itself it's own list. In that case we want skip + # the rest of this function so that we don't get extraneous () on the + # outside. + if (isinstance(marker, list) and len(marker) == 1 + and isinstance(marker[0], (list, tuple))): + return _format_marker(marker[0]) + if isinstance(marker, list): + inner = (_format_marker(m, first=False) for m in marker) + if first: + return " ".join(inner) + else: + return "(" + " ".join(inner) + ")" + elif isinstance(marker, tuple): + return '{0} {1} "{2}"'.format(*marker) + else: + return marker + + +_operators = { + "in": lambda lhs, rhs: lhs in rhs, + "not in": lambda lhs, rhs: lhs not in rhs, + "<": operator.lt, + "<=": operator.le, + "==": operator.eq, + "!=": operator.ne, + ">=": operator.ge, + ">": operator.gt, +} + + +def _eval_op(lhs, op, rhs): + try: + spec = Specifier("".join([op, rhs])) + except InvalidSpecifier: + pass + else: + return spec.contains(lhs) + oper = _operators.get(op) + if oper is None: + raise UndefinedComparison( + "Undefined {0!r} on {1!r} and {2!r}.".format(op, lhs, rhs) + ) + return oper(lhs, rhs) + + +def _evaluate_markers(markers, environment): + groups = [[]] + for marker in markers: + assert isinstance(marker, (list, tuple, string_types)) + if isinstance(marker, list): + groups[-1].append(_evaluate_markers(marker, environment)) + elif isinstance(marker, tuple): + lhs, op, rhs = marker + if isinstance(lhs, Variable): + value = _eval_op(environment[lhs.value], op, rhs.value) + else: + value = _eval_op(lhs.value, op, environment[rhs.value]) + groups[-1].append(value) + else: + assert marker in ["and", "or"] + if marker == "or": + groups.append([]) + return any(all(item) for item in groups) + + +def format_full_version(info): + version = '{0.major}.{0.minor}.{0.micro}'.format(info) + kind = info.releaselevel + if kind != 'final': + version += kind[0] + str(info.serial) + return version + + +def default_environment(): + if hasattr(sys, 'implementation'): + iver = format_full_version(sys.implementation.version) + implementation_name = sys.implementation.name + else: + iver = '0' + implementation_name = '' + return { + "implementation_name": implementation_name, + "implementation_version": iver, + "os_name": os.name, + "platform_machine": platform.machine(), + "platform_release": platform.release(), + "platform_system": platform.system(), + "platform_version": platform.version(), + "python_full_version": platform.python_version(), + "platform_python_implementation": platform.python_implementation(), + "python_version": platform.python_version()[:3], + "sys_platform": sys.platform, + } + + +class Marker(object): + def __init__(self, marker): + try: + self._markers = _coerce_parse_result(MARKER.parseString(marker)) + except ParseException: + self._markers = None + # We do this because we can't do raise ... from None in Python 2.x + if self._markers is None: + raise InvalidMarker("Invalid marker: {0!r}".format(marker)) + def __str__(self): + return _format_marker(self._markers) + def __repr__(self): + return "".format(str(self)) + def evaluate(self, environment=None): + """Evaluate a marker. + Return the boolean from evaluating the given marker against the + environment. environment is an optional argument to override all or + part of the determined environment. + The environment is determined from the current Python process. + """ + current_environment = default_environment() + if environment is not None: + current_environment.update(environment) + return _evaluate_markers(self._markers, current_environment) diff --git a/third-party/py/setuptools/pkg_resources/tests/test_markers.py b/third-party/py/setuptools/pkg_resources/tests/test_markers.py index d8844e74e3e..dec9856cffd 100644 --- a/third-party/py/setuptools/pkg_resources/tests/test_markers.py +++ b/third-party/py/setuptools/pkg_resources/tests/test_markers.py @@ -6,11 +6,6 @@ from pkg_resources import evaluate_marker -@mock.patch.dict('pkg_resources.MarkerEvaluation.values', - python_full_version=mock.Mock(return_value='2.7.10')) -def test_lexicographic_ordering(): - """ - Although one might like 2.7.10 to be greater than 2.7.3, - the marker spec only supports lexicographic ordering. - """ - assert evaluate_marker("python_full_version > '2.7.3'") is False +@mock.patch('platform.python_version', return_value='2.7.10') +def test_ordering(python_version_mock): + assert evaluate_marker("python_full_version > '2.7.3'") is True diff --git a/tools/import_deps/.gitignore b/tools/import_deps/.gitignore new file mode 100644 index 00000000000..b983e390fde --- /dev/null +++ b/tools/import_deps/.gitignore @@ -0,0 +1,6 @@ +libs +buck-out +.buckd +BUCK +venv +third-party \ No newline at end of file diff --git a/tools/import_deps/README.md b/tools/import_deps/README.md new file mode 100644 index 00000000000..d60b16f8d8c --- /dev/null +++ b/tools/import_deps/README.md @@ -0,0 +1,32 @@ +## Import Gradle Dependencies to BUCK + +The tool is designed to help with migrating from gradle to buck. +It imports all dependencies defined in a build.gradle and generates BUCK files. + +Steps: +1. Create a dependencies file: + +``` +./gradlew -q :app:dependencies --configuration debugRuntimeClasspath >> report_file.txt +./gradlew -q :app:dependencies --configuration debugRuntimeClasspath >> report_file.txt +./gradlew -q :app:dependencies --configuration debugAnnotationProcessorClasspath >> report_file.txt +``` + +The gradle build resolves versions. The tool will import the packages of the versions +preferred by gradle. + +2. Run import dependencies script: + +``` +./importdeps.py --gdeps report_file.txt --libs third-party +``` + +The tool will import dependencies and generate BUCK files. + +3. Test generated BUCK file: + +``` +buck targets //third-party: +``` + +4. Use imported targets in your project. diff --git a/tools/import_deps/dependency/__init__.py b/tools/import_deps/dependency/__init__.py new file mode 100644 index 00000000000..2f66fdbab07 --- /dev/null +++ b/tools/import_deps/dependency/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) Meta Platforms, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tools/import_deps/dependency/buck.py b/tools/import_deps/dependency/buck.py new file mode 100644 index 00000000000..e0d94a1a153 --- /dev/null +++ b/tools/import_deps/dependency/buck.py @@ -0,0 +1,145 @@ +# Copyright (c) Meta Platforms, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import io +import logging +import os +import re +from pathlib import Path + +from dependency.pom import MavenDependency +from dependency.repo import Repo, ArtifactFlavor + + +class Buck: + BUCK_FILE = "BUCK" + + def __init__(self, repo: Repo) -> None: + super().__init__() + self.repo = repo + + def create(self, buck_path: Path, dependency: MavenDependency, visited: set): + """ + Creates or updates a BUCK file for a single dependency. + :param buck_path: a path to BUCK file. + :param dependency: a dependency. + :param visited: a set of visited dependencies + :return: + """ + assert dependency is not None + if visited.__contains__(dependency): + return + + mode = "a+" + + with buck_path.open(mode) as f: + f.write(self.generate_prebuilt_jar_or_aar_rule(dependency)) + f.write("\n") + f.write(self.android_library(dependency)) + + visited.add(dependency) + + def get_all_dependencies(self, pom: MavenDependency, visited=set()) -> []: + if visited.__contains__(pom): + return visited + visited.add(pom) + for d in pom.dependsOn: + self.get_all_dependencies(d, visited) + return visited + + def get_path(self) -> Path: + p = Path(os.path.join(self.repo.get_root(), Buck.BUCK_FILE)) + return p + + def get_buck_path_for_dependency(self, dependency: MavenDependency) -> Path: + artifact_base_dir = Path(self.repo.root) + for d in dependency.group_id.split("."): + artifact_base_dir = os.path.join(artifact_base_dir, d) + p = Path(os.path.join(artifact_base_dir, Buck.BUCK_FILE)) + return p + + def fix_path(self, file: Path): + f = str(file) + f = f.replace("..", "/") + return f + + def generate_prebuilt_jar_or_aar_rule(self, pom: MavenDependency) -> str: + """ + Generate android_prebuilt_aar or prebuilt_jar rules for an + artifact. + :param pom: a dependency. + :return: + """ + artifact = pom.get_artifact() + if artifact is None: + logging.debug(f"pom has no artifacts: {pom}") + return "" + # assert artifact is not None, f"pom has no artifacts: {pom}" + template = None + if re.match(r".*.aar$", artifact): + template = self.get_android_prebuilt_aar(pom) + elif re.match(r".*.jar$", artifact): + template = self.get_prebuilt_jar(pom) + return template + + def android_library(self, pom: MavenDependency): + artifact = pom.get_artifact() + artifact_path = self.fix_path(artifact) + header = f""" +android_library( + name = "{pom.artifact_id}", +""" + footer = f""" visibility = [ "PUBLIC" ], +) +""" + with io.StringIO() as out: + out.write(header) + out.write(f""" exported_deps = [ \n ":_{pom.artifact_id}", \n""") + dependencies = pom.get_dependencies() + for d in dependencies: + out.write(f' "{self.repo.get_buck_target(d)}",\n') + out.write(f""" ],\n""") + out.write(footer) + contents = out.getvalue() + return contents + + def get_android_prebuilt_aar(self, dep: MavenDependency): + artifact_name = self.repo.get_artifact_name(dep, ArtifactFlavor.AAR) + artifact_path = f"{dep.artifact_id}/{dep.version}/{artifact_name}" + with io.StringIO() as out: + out.write( + f""" +android_prebuilt_aar( + name = '_{dep.artifact_id}', + aar = '{artifact_path}', +""" + ) + out.write(")\n") + contents = out.getvalue() + return contents + + def get_prebuilt_jar(self, dep: MavenDependency): + artifact = dep.get_artifact() + artifact_name = self.repo.get_artifact_name(dep, ArtifactFlavor.JAR) + artifact_path = f"{dep.artifact_id}/{dep.version}/{artifact_name}" + with io.StringIO() as out: + out.write( + f""" +prebuilt_jar( + name = '_{dep.artifact_id}', + binary_jar = '{artifact_path}', +""" + ) + out.write(")\n") + contents = out.getvalue() + return contents diff --git a/tools/import_deps/dependency/gradle.py b/tools/import_deps/dependency/gradle.py new file mode 100644 index 00000000000..a1faadeda2a --- /dev/null +++ b/tools/import_deps/dependency/gradle.py @@ -0,0 +1,117 @@ +# Copyright (c) Meta Platforms, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import re +from pathlib import Path + +from dependency.pom import MavenDependency + + +class GradleDependencies: + """ + Import gradle dependencies. + """ + + def __init__(self) -> None: + super().__init__() + self.re_deps_version = re.compile(r"([\d.]+) -> ([\d.]+)(( \([c*]\))|())$") + self.re_deps_version3 = re.compile( + r"(\{strictly [\d.]+\}) -> ([\d.]+)(( \([c*]\))|())$" + ) + self.re_deps_version2 = re.compile(r"([\d.]+)( \([c*]\))|()$") + self.re_deps_line = re.compile(r"^[+|\\].*$") + self.re_deps_groups = re.compile(r"^([+|\\][+-\\| ]+)(.*):(.*):(.*)$") + + def import_gradle_dependencies(self, report_file: Path): + """ + Import gradle dependencies created as + ./gradlew -q :app:dependencies --configuration debugCompileClasspath > report_file.txt + + Create a set of dependencies. + The method will pick a resolved version of the dependency. + I.e. the version that is used by the gradle build. + :param report_file: a report file path + :return: a set of dependencies. + """ + all_dependencies = set() + dependencies_stack = [] + with open(report_file, "r") as f: + line = f.readline() + while line: + line = f.readline() + if self.re_deps_line.match(line): + m = self.re_deps_groups.match(line) + line_prefix = m.group(1) + extracted_dependency = self.extract_dependency(line) + pd = self.find_dependency(all_dependencies, extracted_dependency) + all_dependencies.add(pd) + if len(dependencies_stack) == 0: + dependencies_stack.append((line_prefix, pd)) + continue + if len(dependencies_stack) > 0: + parent = dependencies_stack[len(dependencies_stack) - 1] + parent_prefix = parent[0] + parent_dep = parent[1] + if len(line_prefix) > len(parent_prefix): + parent_dep.add_dependency(pd) + dependencies_stack.append((line_prefix, pd)) + elif len(line_prefix) < len(parent_prefix): + parent = dependencies_stack.pop() + while len(parent[0]) > len(line_prefix): + parent = dependencies_stack.pop() + if len(dependencies_stack) > 0: + parent = dependencies_stack[len(dependencies_stack) - 1] + parent_dep = parent[1] + parent_dep.add_dependency(pd) + dependencies_stack.append((line_prefix, pd)) + else: + dependencies_stack.pop() + dependencies_stack.append((line_prefix, pd)) + if len(dependencies_stack) >= 2: + grandparent = dependencies_stack[ + len(dependencies_stack) - 2 + ] + grandparent_dep = grandparent[1] + grandparent_dep.add_dependency(pd) + return all_dependencies + + def find_dependency( + self, all_dependencies: set, pd: MavenDependency + ) -> MavenDependency: + for d in all_dependencies: + if pd.is_keys_equal(d): + return d + return pd + + def extract_dependency(self, line: str) -> MavenDependency: + m = self.re_deps_groups.match(line) + group_id = m.group(2) + artifact_id = m.group(3) + version = self.match_version(m.group(4)) + pd = MavenDependency(group_id, artifact_id, version, "") + return pd + + def match_version(self, version: str): + v = version + vm = self.re_deps_version.match(version) + if vm: + v = vm.group(2) + else: + vm = self.re_deps_version2.match(version) + if vm: + v = vm.group(1) + else: + vm = self.re_deps_version3.match(version) + if vm: + v = vm.group(2) + return v diff --git a/tools/import_deps/dependency/manager.py b/tools/import_deps/dependency/manager.py new file mode 100644 index 00000000000..0757e2ec3f5 --- /dev/null +++ b/tools/import_deps/dependency/manager.py @@ -0,0 +1,233 @@ +# Copyright (c) Meta Platforms, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import os +from collections import deque + +from .pom import MavenDependency, PomXmlAdapter, Artifact +from .repo import Repo, ArtifactFlavor + + +class Manager: + """ + Provides methods to import Maven dependencies to a local repository (../libs). + """ + + # A stack of pom files to be updated and parsed as we add more dependencies. + pom_files = [] + + def __init__(self, repo: Repo) -> None: + super().__init__() + self.repo = repo + + def import_maven_dep(self, gradle_dependency: str): + """ + Import a maven dependency as well as its dependencies. + Create BUCK files. + :param gradle_dependency: + :return: a PomDependency + """ + pom_dependency = self.to_maven_dependency(gradle_dependency) + self.import_dep(pom_dependency) + return pom_dependency + + def import_dep(self, dependency: MavenDependency) -> set: + all_dependencies = set() + visited = set() + return self._import_dep(dependency, all_dependencies, visited) + + def _import_dep( + self, dependency: MavenDependency, all_dependencies, visited=set() + ) -> set: + """ + Import a maven dependency with its transitive dependencies. + :param all_dependencies: + :param dependency: + :param all_dependencies: set of all dependencies + :param visited: a set of visited dependencies + :return: + """ + if visited.__contains__(dependency.__str__()): + return all_dependencies + visited.add(dependency.__str__()) + all_dependencies.add(dependency) + + for flavor in Repo.artifact_flavors: + file = self.repo.download_maven_dep(dependency, flavor) + if not os.path.exists(file): + logging.debug(f"Unable to download: {file}") + else: + dependency.add_artifact(Artifact(file)) + if flavor == ArtifactFlavor.POM: + self.pom_files.append(file) + + # parse pom + while len(self.pom_files) > 0: + pom_file = self.pom_files.pop() + pom_xml_adapter = PomXmlAdapter(pom_file) + deps = pom_xml_adapter.get_deps() + for d in deps: + dependency.add_dependency(d) + self._import_dep(d, all_dependencies, visited) + + return all_dependencies + + def import_dep_shallow(self, dependency: MavenDependency) -> []: + """ + Import a maven dependency artifacts without its transitive dependencies. + :param dependency: + :return: a list of downloaded files + """ + downloaded_files = [] + for flavor in Repo.artifact_flavors: + file = self.repo.download_maven_dep(dependency, flavor) + if os.path.exists(file): + downloaded_files.append(file) + dependency.add_artifact(Artifact(file)) + return downloaded_files + + def resolve_deps_versions(self, dependency: MavenDependency, deps: []) -> None: + """ + In some cases POM does not have versions for dependencies. + This method obtains a release version for such dependencies. + :param dependency: a dependncy to be updated + :param deps: dependencies + :return: None + """ + for d in deps: + if d.version is None: + d.version = self.repo.get_release_version(d) + dependency.add_dependency(d) + + def import_deps_shallow(self, deps: set): + """ + Import dependencies from the set. + Do not import dependsOn dependencies. + + :param deps: a set of dependencies + :return: + """ + for d in deps: + self.import_dep_shallow(d) + + def to_maven_dependency(self, gradle_dependency: str) -> MavenDependency: + d = gradle_dependency.split(":") + pd = MavenDependency(d[0], d[1], d[2], "") + return pd + + def import_missing_dependencies(self, deps: set, versions: dict) -> set: + visited = set() + self._import_missing_dependencies(deps, versions, visited) + return visited + + def _import_missing_dependencies(self, deps: set, versions: dict, visited: set): + queue = deque(deps) + while len(queue) > 0: + dep: MavenDependency = queue.popleft() + if dep in visited: + continue + visited.add(dep) + dep = self.get_larger_version(dep, versions) + missing_dependencies = self.check_missing_dependencies(dep) + for md in missing_dependencies: + if md.version is None: + md.version = self.repo.get_release_version(md) + md = self.get_larger_version(md, versions) + assert md.version is not None + dep.add_dependency(md) + self.import_dep_shallow(md) + queue.append(md) + + def get_larger_version( + self, dep: MavenDependency, versions: dict + ) -> MavenDependency: + """ + Find a matching version in the versions dict. Return it if it is a larger version. + Or return the dep. + :param dep: + :param versions: a repository of objects with the larger version + :return: + """ + key = dep.get_group_and_artifactid() + if key not in versions.keys(): + versions[key] = dep + return dep + larger_version = dep.get_larger_version(versions[key].version, dep.version) + if dep != larger_version: + return versions[key] + return dep + + def check_missing_dependencies(self, dep: MavenDependency) -> set: + pom_file = self.repo.download_maven_dep(dep, ArtifactFlavor.POM) + pom_xml_adapter = PomXmlAdapter(pom_file) + pom_deps = pom_xml_adapter.get_deps() + missing = set(pom_deps) - dep.get_dependencies() + return missing + + def update_versions( + self, dep: MavenDependency, versions: dict, visited=set() + ) -> None: + """ + Build a set of all dependencies with the largest version numbers./ + :param dep: + :param versions: + :param visited: + :param all_dependencies: + :return: + """ + if visited.__contains__(dep): + return + visited.add(dep) + + # if the dep has a lover version than in versions, then return also + group_and_artifactid = dep.get_group_and_artifactid() + if group_and_artifactid in versions.keys(): + v = versions[group_and_artifactid] + largest_version = dep.get_larger_version(dep.version, v) + if largest_version is v: + return + + # at this point dep either unique or has a large version + # set or bump the version up + versions[group_and_artifactid] = dep.version + + pom_file = self.repo.download_maven_dep(dep, ArtifactFlavor.POM) + pom_xml_adapter = PomXmlAdapter(pom_file) + pom_deps = pom_xml_adapter.get_deps() + + for d in pom_deps: + self.get_all_dependencies(d, versions, visited) + + def update_dependencies( + self, dep: MavenDependency, versions: dict, visited=set() + ) -> None: + if visited.__contains__(dep): + return + visited.add(dep) + + pom_file = self.repo.download_maven_dep(dep, ArtifactFlavor.POM) + pom_xml_adapter = PomXmlAdapter(pom_file) + pom_deps = pom_xml_adapter.get_deps() + + deps = dep.get_dependencies() + + for d in pom_deps: + if d not in deps: + v = versions[d.get_group_and_artifactid()] + x = d + if v is not d.version: + x = MavenDependency(d.group_id, d.artifact_id, v) + self.import_dep_shallow(x) + dep.add_dependency(x) + self.update_dependencies(d, versions, visited) diff --git a/tools/import_deps/dependency/pom.py b/tools/import_deps/dependency/pom.py new file mode 100644 index 00000000000..5147ba4e7a4 --- /dev/null +++ b/tools/import_deps/dependency/pom.py @@ -0,0 +1,387 @@ +# Copyright (c) Meta Platforms, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import os +import re +import xml.etree.ElementTree as ET +from pathlib import Path +from typing import List, Optional + + +class Artifact: + """ + A specific file of a dependency: jar, aar, pom, etc. + """ + + def __init__(self, file: Path): + self.file = file + + def _key(self): + return self.file + + def __hash__(self) -> int: + return hash(self._key()) + + def __eq__(self, o: object) -> bool: + if not isinstance(o, Artifact): + return False + return o.file == self.file + + +class MavenDependency: + """ + A Maven dependency that contains artifacts (jar, aar, ...) and has its + own dependencies. + """ + + def __init__( + self, group_id: str, artifact_id: str, version: str, scope: str = "runtime" + ) -> None: + self.group_id = group_id + self.artifact_id = artifact_id + self.version = version + self.scope = scope + self.dependsOn = set() + self.artifacts = set() + + def __str__(self) -> str: + return f"{self.group_id}:{self.artifact_id}:{self.version}:{self.scope}" + + def __hash__(self) -> int: + return hash(self._key()) + + def add_dependency(self, dependency: "MavenDependency") -> None: + """ + Adds a dependency. The dependency will be added if + it has a different group and artifactId than self, + or it is new to the collection of dependencies, + or if its version is larger than the version of an existing + dependency. In this case, the dependency is substituted. + :param dependency: a dependency to be added. + """ + if dependency == self: + return + if self.get_group_and_artifactid() == dependency.get_group_and_artifactid(): + return + if self.dependsOn.__contains__(dependency): + return + partial_match = self.find_dep(dependency.group_id, dependency.artifact_id) + if partial_match is not None: + larger_version = self.get_larger_version( + partial_match.version, dependency.version + ) + if partial_match.version is not larger_version: + self.dependsOn.remove(partial_match) + self.dependsOn.add(dependency) + return + return + self.dependsOn.add(dependency) + + def get_larger_version(self, v1: str, v2: str) -> str: + x = [v1, v2] + x.sort() + return x[1] + + def add_artifact(self, artifact: Artifact): + self.artifacts.add(artifact) + + def get_artifact(self): + for artifact in self.artifacts: + file = artifact.file + if ( + re.match(r".*.aar$", file) + or re.match(r".*.jar$", file) + and not re.match(r".*-sources.jar$", file) + ): + return file + + def get_artifacts(self): + return self.artifacts + + def is_keys_equal(self, pd: "MavenDependency"): + return ( + pd.artifact_id == self.artifact_id + and pd.group_id == self.group_id + and pd.version == self.version + ) + + def is_group_and_artifactid_equal(self, pd: "MavenDependency"): + return pd.artifact_id == self.artifact_id and pd.group_id == self.group_id + + def get_group_and_artifactid(self): + return f"{self.group_id}_{self.artifact_id}" + + def get_version(self): + return self.version + + def update_version(self, version: str): + self.version = version + + def get_dependencies(self) -> set: + return self.dependsOn + + def get_all_dependencies(self) -> set: + """ + Find all dependencies recursively. + :return: a set of dependencies. + """ + visited = set() + queue = set() + queue.add(self) + while len(queue) > 0: + d = queue.pop() + if d in visited: + continue + visited.add(d) + for n in d.get_dependencies(): + queue.add(n) + return visited + + def print_dependency_tree(self): + visited = set() + prefix = "" + self._print_dependency_tree(prefix, visited) + + def _print_dependency_tree(self, prefix: str, visited: set): + if self in visited: + if len(self.get_dependencies()) > 0: + print(f"{prefix} {self.__str__()} ...") + else: + print(f"{prefix} {self.__str__()}") + return + visited.add(self) + print(f"{prefix} {self.__str__()}") + prefix = f"{prefix} >" + for d in self.get_dependencies(): + d._print_dependency_tree(prefix, visited) + + def get_compile_dependencies(self) -> set: + compile_dependencies = set() + for d in self.dependsOn: + if d.scope == "compile": + compile_dependencies.add(d) + return compile_dependencies + + def get_runtime_dependencies(self) -> set: + runtime_dependencies = set() + for d in self.dependsOn: + if d.scope != "compile": + runtime_dependencies.add(d) + return runtime_dependencies + + def _key(self): + return self.group_id, self.artifact_id, self.version, self.scope + + def get_libs_path(self): + l = self.group_id.split(".") + l.append(self.artifact_id) + l.append(self.version) + return l + + def get_artifact_versions_dir(self) -> []: + """ + Provides a master folder for the artifact. This is a base for sub-folders with specific artifact versions. + :return: a folder path. + """ + l = self.group_id.split(".") + l.append(self.artifact_id) + return l + + def __eq__(self, o: object) -> bool: + if not isinstance(o, MavenDependency): + return False + return ( + self.group_id == o.group_id + and self.artifact_id == o.artifact_id + and self.version == o.version + and self.scope == o.scope + ) + + def get_full_name(self): + return f"{self.group_id}:{self.artifact_id}" + + def get_group_id(self): + return self.group_id + + def get_artifact_id(self): + return self.artifact_id + + def get_scope(self): + return self.scope + + def set_scope(self, scope: str): + self.scope = scope + + def find_dep(self, group_id: str, artifact_id: str): + for d in self.dependsOn: + if d.get_full_name() == f"{group_id}:{artifact_id}": + return d + + +class PomXmlAdapter: + _NAMESPACES = {"xmlns": "http://maven.apache.org/POM/4.0.0"} + + def __init__(self, pom_file: str): + self._pom_file = pom_file + + def get_deps(self) -> List[MavenDependency]: + deps = [] + if not os.path.exists(self._pom_file): + logging.warning(f"The pom file is not found: {self._pom_file}") + return deps + tree = ET.parse(self._pom_file) + root = tree.getroot() + dependencies = root.findall( + "./xmlns:dependencies/xmlns:dependency", namespaces=self._NAMESPACES + ) + for dep_node in dependencies: + dependency = self._create_pom_dependency(dep_node, root) + deps.append(dependency) + # TODO: revisit the "test" scope + deps = [d for d in deps if d.version != "" and d.scope != "test"] + return deps + + def get_group_id(self, root): + group_id = root.find("./xmlns:groupId", namespaces=self._NAMESPACES) + if group_id is None: + group_id = root.find( + "./xmlns:parent/xmlns:groupId", namespaces=self._NAMESPACES + ) + return group_id.text + + def get_package_version(self, root): + """ + Provides a version of the main artifact of the POM. + :param root: an XML tree root + :return: an artifact version + """ + version = root.find("./xmlns:version", namespaces=self._NAMESPACES) + if version is None: + version = root.find( + "./xmlns:parent/xmlns:version", namespaces=self._NAMESPACES + ) + if version is None: + version = root.find("./version") + if version is None: + logging.warning(f"Unable to find the version for {self._pom_file}") + version = root.find("version") + logging.debug(f"Project version: {version}") + return version + + def get_dependency_version(self, dependency_node, root): + """ + Provides a version of a dependency. + :param dependency_node: a dependency XML node + :param artifact_version: a version of the artifact + :param root: an XML tree root + :return: a dependency version. None if no version is provided. + """ + version_element = dependency_node.find( + "xmlns:version", namespaces=self._NAMESPACES + ) + if version_element is None: + # Some POMs have dependencies without versions. For example: + # third-party/com/google/guava/guava-testlib/31.1-jre/guava-testlib-31.1-jre.pom + return None + # Sometimes version is specified with a property or a reference to a + # project (main artifact) version. + # Check if the version is specified with a variable that starts with ${... + x = re.compile(r"\${(.*)}") + r = x.match(version_element.text) + if r is not None: + property_name = r.group(1) + if property_name.startswith("project"): + version_element = self.get_package_version(root) + else: + # ${hamcrestVersion} + version_element = root.find( + f"./xmlns:properties/xmlns:{property_name}", + namespaces=self._NAMESPACES, + ) + if version_element is not None: + logging.warning(f"Unable to find definition of {property_name}") + version = None + if version_element is not None: + version_text = PomXmlAdapter.get_or_empty(version_element) + version = self.resolve_complicated_version(version_text) + return version + + def _create_pom_dependency(self, dep_node, root) -> MavenDependency: + group_id = dep_node.find("xmlns:groupId", namespaces=self._NAMESPACES).text + if group_id == "${project.groupId}": + group_id = self.get_group_id(root) + assert group_id is not None, f"groupId is not found for {self._pom_file}" + artifact_id = dep_node.find( + "xmlns:artifactId", namespaces=self._NAMESPACES + ).text + version_text = self.get_dependency_version(dep_node, root) + scope = PomXmlAdapter.get_or_empty( + dep_node.find("xmlns:scope", namespaces=self._NAMESPACES) + ) + logging.debug( + f"groupid: {group_id}, artifactId: {artifact_id}, version: {version_text}" + ) + pd = MavenDependency(group_id, artifact_id, version_text, scope) + return pd + + def get_or_empty(e: Optional[ET.Element]) -> str: + if e is None: + return "" + else: + # pyre-ignore + return e.text + + def resolve_complicated_version(self, v: str, default_version: str = None) -> str: + """ + Eventually we want to resolve all uncommon pom.xml version patterns: + - 1.0.1 - simple version (Supported) + - [1.0.1] - exact version (Supported) + - [1.0.0,2.0.0) - range with upper bound (NOT supported) + - [1.0.0,) - range without upper bound (NOT supported) + Will not support: + - LATEST - removed in Maven 3.0 + - RELEASE - removed in Maven 3.0 + - {VERSION_VARIABLE_FROM_POM} - too complicated + """ + # (v), [v), or [v] + if v.isdecimal(): + return v + if re.match(r"\d+\.\d+\.\d+", v): + return v + if re.match(r"\d+\.\d+", v): + return v + bounded = re.search(r"^[\[\(].*[\]\)]$", v) + if bounded: + return v[1:-1] + else: + # default + return default_version + + +class MavenMetadataParser: + """ + This is a maven-metadata.xml parser. + """ + + def get_release_version(self, file: str) -> str: + """ + Parses maven-metadata.xml and returns a release version. + :param file: a maven-metadata.xml + :return: a release version + """ + assert os.path.exists(file) + tree = ET.parse(file) + root = tree.getroot() + version = root.find("./versioning/release") + return version.text diff --git a/tools/import_deps/dependency/repo.py b/tools/import_deps/dependency/repo.py new file mode 100644 index 00000000000..4f62221f587 --- /dev/null +++ b/tools/import_deps/dependency/repo.py @@ -0,0 +1,211 @@ +# Copyright (c) Meta Platforms, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import os +from enum import Enum +from pathlib import Path +from typing import List + +import urllib3 + +from .pom import Artifact, PomXmlAdapter, MavenMetadataParser +from .pom import MavenDependency + + +class ArtifactFlavor(Enum): + JAR = ".jar" + AAR = ".aar" + POM = ".pom" + SOURCE_JAR = "-sources.jar" + + +class Repo: + repo_url_maven = "https://repo1.maven.org/maven2/" + repo_url_google = "https://maven.google.com/" + artifact_flavors: List[ArtifactFlavor] = [ + ArtifactFlavor.JAR, + ArtifactFlavor.AAR, + ArtifactFlavor.POM, + ArtifactFlavor.SOURCE_JAR, + ] + + def __init__(self, libs="libs") -> None: + super().__init__() + self.libs = libs + self.root = f"./{libs}" + + def get_libs_name(self): + """ + :return: a name of the repo libs folder + """ + return self.libs + + def get_base_name(self) -> str: + return f"//{os.path.basename(self.libs)}" + + def get_buck_target(self, dep: MavenDependency) -> str: + """ + Provides a fully qualified name for the dependency library in the repo. + :param dep: a dependency + :return: a fully qualified name for the dependency library in the repo. + """ + artifact_base_dir = f"{self.get_base_name()}" + for p in dep.group_id.split("."): + artifact_base_dir = artifact_base_dir + "/" + p + target = artifact_base_dir + ":" + dep.artifact_id + return target + + def get_root(self): + return self.root + + def mkdirs(self, path: list) -> bool: + """ + Create directories in libs folder. + """ + if not os.path.exists(self.root) or not os.path.isdir(self.root): + return False + + d = self.get_path(path) + if not os.path.isdir(d): + os.makedirs(name=d, exist_ok=True) + return os.path.isdir(d) + + def get_path(self, path: list) -> Path: + d = self.root + for folder in path: + d = os.path.join(d, folder) + return d + + def get_dependency_dir(self, dep: MavenDependency) -> Path: + d = Path(self.root) + for folder in dep.get_libs_path(): + d = os.path.join(d, folder) + return d + + def get_dependency_path(self, dep: MavenDependency, flavor: ArtifactFlavor) -> Path: + """ + Provides a path to an artifact file in the local repo. + """ + d = self.get_dependency_dir(dep) + file_name = self.get_artifact_name(dep, flavor) + path = os.path.join(d, file_name) + return path + + def get_artifact_name(self, dep: MavenDependency, flavor: ArtifactFlavor): + file_name = f"{dep.artifact_id}-{dep.version}{flavor.value}" + return file_name + + def get_base_url(self, dep: MavenDependency): + group = dep.group_id.replace(".", "/") + maven_repo = self.get_maven_repo(dep) + base_url: str = f"{maven_repo}{group}/{dep.artifact_id}/{dep.version}/{dep.artifact_id}-{dep.version}" + return base_url + + def get_maven_repo(self, dep: MavenDependency): + if dep.group_id.startswith("androidx") or dep.group_id.startswith( + "com.google.android" + ): + return self.repo_url_google + return self.repo_url_maven + + def get_release_version(self, dep: MavenDependency) -> str: + """ + Check Maven repos and obtain a release version of a dependency. + + :param dep: + :return: a version from maven-metadata.xml. + """ + maven_repo = self.get_maven_repo(dep) + group = dep.group_id.replace(".", "/") + maven_metadata_url: str = ( + f"{maven_repo}{group}/{dep.artifact_id}/maven-metadata.xml" + ) + artifact_base_dir = Path(self.root) + for d in dep.get_artifact_versions_dir(): + artifact_base_dir = os.path.join(artifact_base_dir, d) + if not os.path.exists(artifact_base_dir): + os.makedirs(artifact_base_dir, exist_ok=True) + maven_metadata_file = os.path.join(artifact_base_dir, "maven-metadata.xml") + downloaded = self.download(maven_metadata_url, maven_metadata_file) + if not downloaded: + return None + parser = MavenMetadataParser() + version = parser.get_release_version(maven_metadata_file) + return version + + def get_url(self, dep: MavenDependency, flavor: ArtifactFlavor): + return f"{self.get_base_url(dep)}{flavor.value}" + + def download(self, url, path) -> bool: + with urllib3.PoolManager() as http: + r = http.request("GET", url, preload_content=False) + if r.status != 200: + return False + with open(path, "wb") as out: + while True: + data = r.read() + if not data: + break + out.write(data) + r.release_conn() + return True + + def download_maven_dep( + self, dep: MavenDependency, flavor: ArtifactFlavor, force: bool = False + ) -> Path: + """ + Download a Maven dependency + :param dep: the base dependency + :param flavor: a flavor: jar, aar, pom, etc + :param force: download even if the file exists + :return: a dependency file path + """ + destination = self.get_dependency_dir(dep) + if not os.path.exists(destination): + os.makedirs(destination, exist_ok=True) + url = self.get_url(dep, flavor) + repo_file = self.get_dependency_path(dep, flavor) + if not os.path.exists(repo_file) or force: + self.download(url, repo_file) + logging.debug(f"url: {url}, file: {os.path.abspath(repo_file)}") + else: + logging.debug(f"url: {url}, file: {repo_file} exists") + return repo_file + + def load_artifacts(self, pom: MavenDependency): + """ + Check the disk for available artifacts and add them to the + collection. + :return: + """ + destination = self.get_dependency_dir(pom) + if not os.path.exists(destination): + return + + for flavor in Repo.artifact_flavors: + repo_file = self.get_dependency_path(pom, flavor) + if os.path.exists(repo_file): + pom.add_artifact(Artifact(repo_file)) + + def update_scope_from_pom(self, dep: MavenDependency): + pom_file = self.download_maven_dep(dep, ArtifactFlavor.POM) + pom_xml_adapter = PomXmlAdapter(pom_file) + pom_deps = pom_xml_adapter.get_deps() + for pd in pom_deps: + d = dep.find_dep(pd.get_group_id(), pd.get_artifact_id()) + if d is None: + logging.debug(f"dependency not found: {pd} in {dep}") + return + if d.get_scope() != pd.get_scope(): + d.set_scope(pd.get_scope()) diff --git a/tools/import_deps/importdeps.py b/tools/import_deps/importdeps.py new file mode 100755 index 00000000000..57eb3fc4811 --- /dev/null +++ b/tools/import_deps/importdeps.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# Copyright (c) Meta Platforms, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import logging +import os + +from dependency.buck import Buck +from dependency.gradle import GradleDependencies +from dependency.manager import Manager +from dependency.repo import Repo + + +def main(): + parser = argparse.ArgumentParser( + description="""Import gradle dependencies and generate BUCK file. + Created gradle dependencies file by running + ./gradlew -q :app:dependencies --configuration debugCompileClasspath > report_file.txt + """ + ) + + parser.add_argument("--gdeps", required=True, help="gradle dependencies file") + parser.add_argument( + "--libs", default="third-party", help="libs folder for imported dependencies" + ) + try: + args = parser.parse_args() + print(f"args: {args}") + gdeps = args.gdeps + libs = args.libs + print(f"gdeps: {gdeps}, libs: {libs}") + import_deps(gdeps, libs) + except argparse.ArgumentError: + print("Invalid arguments") + + +def import_deps(gradle_deps: str, libs: str) -> None: + """ + Import dependencies and create BUCK file. + :param gradle_deps: file with gradle dependencies + :param libs: path to a folder where to store the dependencies + :return: None + """ + logging.basicConfig(level=logging.DEBUG) + assert os.path.exists(gradle_deps), f"Unable to open {gradle_deps}" + if not os.path.exists(libs): + os.mkdir(libs) + assert os.path.exists(libs) + gd = GradleDependencies() + # Importing dependencies from a file, will not determine the scope. + # I.e. whether it is a runtime or compile time dependency. + # We will need to parse pom for the dependency and update it. + deps = gd.import_gradle_dependencies(gradle_deps) + repo = Repo(libs=libs) + buck = Buck(repo) + # Remove ./libs/BUCK file if it exists + buck_file = buck.get_path() + if os.path.exists(buck_file): + os.remove(buck_file) + manager = Manager(repo) + + versions = dict() + for dep in deps: + manager.import_dep_shallow(dep) + versions[dep.get_group_and_artifactid()] = dep + + # Check for missing dependencies + manager.import_missing_dependencies(deps, versions) + + visited = set() + for dep in versions.values(): + buck_file = buck.get_buck_path_for_dependency(dep) + # This will also update a BUCK file + print(f"adding {dep}") + buck.create(buck_file, dep, visited) + visited.add(dep) + + +main() diff --git a/tools/import_deps/tests/__init__.py b/tools/import_deps/tests/__init__.py new file mode 100644 index 00000000000..2f66fdbab07 --- /dev/null +++ b/tools/import_deps/tests/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) Meta Platforms, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tools/import_deps/tests/data/deps.txt b/tools/import_deps/tests/data/deps.txt new file mode 100644 index 00000000000..7e56c0de9ab --- /dev/null +++ b/tools/import_deps/tests/data/deps.txt @@ -0,0 +1,198 @@ + +------------------------------------------------------------ +Project ':app' +------------------------------------------------------------ + +debugCompileClasspath - Resolved configuration for compilation for variant: debug ++--- androidx.appcompat:appcompat:1.4.1 +| +--- androidx.annotation:annotation:1.3.0 +| +--- androidx.core:core:1.7.0 +| | +--- androidx.annotation:annotation:1.2.0 -> 1.3.0 +| | +--- androidx.annotation:annotation-experimental:1.1.0 +| | +--- androidx.lifecycle:lifecycle-runtime:2.3.1 -> 2.4.0 +| | | +--- androidx.lifecycle:lifecycle-common:2.4.0 +| | | | \--- androidx.annotation:annotation:1.1.0 -> 1.3.0 +| | | +--- androidx.arch.core:core-common:2.1.0 +| | | | \--- androidx.annotation:annotation:1.1.0 -> 1.3.0 +| | | \--- androidx.annotation:annotation:1.1.0 -> 1.3.0 +| | \--- androidx.versionedparcelable:versionedparcelable:1.1.1 +| | +--- androidx.annotation:annotation:1.1.0 -> 1.3.0 +| | \--- androidx.collection:collection:1.0.0 -> 1.1.0 +| | \--- androidx.annotation:annotation:1.1.0 -> 1.3.0 +| +--- androidx.cursoradapter:cursoradapter:1.0.0 +| | \--- androidx.annotation:annotation:1.0.0 -> 1.3.0 +| +--- androidx.activity:activity:1.2.4 -> 1.3.1 +| | +--- androidx.annotation:annotation:1.1.0 -> 1.3.0 +| | +--- androidx.core:core:1.1.0 -> 1.7.0 (*) +| | +--- androidx.lifecycle:lifecycle-runtime:2.3.1 -> 2.4.0 (*) +| | +--- androidx.lifecycle:lifecycle-viewmodel:2.3.1 +| | | \--- androidx.annotation:annotation:1.1.0 -> 1.3.0 +| | +--- androidx.savedstate:savedstate:1.1.0 +| | | \--- androidx.annotation:annotation:1.1.0 -> 1.3.0 +| | \--- androidx.lifecycle:lifecycle-viewmodel-savedstate:2.3.1 +| | +--- androidx.annotation:annotation:1.0.0 -> 1.3.0 +| | +--- androidx.savedstate:savedstate:1.1.0 (*) +| | +--- androidx.lifecycle:lifecycle-livedata-core:2.3.1 +| | | \--- androidx.lifecycle:lifecycle-common:2.3.1 -> 2.4.0 (*) +| | \--- androidx.lifecycle:lifecycle-viewmodel:2.3.1 (*) +| +--- androidx.fragment:fragment:1.3.6 +| | +--- androidx.annotation:annotation:1.1.0 -> 1.3.0 +| | +--- androidx.core:core:1.2.0 -> 1.7.0 (*) +| | +--- androidx.collection:collection:1.1.0 (*) +| | +--- androidx.viewpager:viewpager:1.0.0 +| | | +--- androidx.annotation:annotation:1.0.0 -> 1.3.0 +| | | +--- androidx.core:core:1.0.0 -> 1.7.0 (*) +| | | \--- androidx.customview:customview:1.0.0 -> 1.1.0 +| | | +--- androidx.annotation:annotation:1.1.0 -> 1.3.0 +| | | \--- androidx.core:core:1.3.0 -> 1.7.0 (*) +| | +--- androidx.loader:loader:1.0.0 +| | | +--- androidx.annotation:annotation:1.0.0 -> 1.3.0 +| | | +--- androidx.core:core:1.0.0 -> 1.7.0 (*) +| | | +--- androidx.lifecycle:lifecycle-livedata:2.0.0 +| | | | +--- androidx.arch.core:core-runtime:2.0.0 -> 2.1.0 +| | | | | +--- androidx.annotation:annotation:1.1.0 -> 1.3.0 +| | | | | \--- androidx.arch.core:core-common:2.1.0 (*) +| | | | +--- androidx.lifecycle:lifecycle-livedata-core:2.0.0 -> 2.3.1 (*) +| | | | \--- androidx.arch.core:core-common:2.0.0 -> 2.1.0 (*) +| | | \--- androidx.lifecycle:lifecycle-viewmodel:2.0.0 -> 2.3.1 (*) +| | +--- androidx.activity:activity:1.2.4 -> 1.3.1 (*) +| | +--- androidx.lifecycle:lifecycle-livedata-core:2.3.1 (*) +| | +--- androidx.lifecycle:lifecycle-viewmodel:2.3.1 (*) +| | +--- androidx.lifecycle:lifecycle-viewmodel-savedstate:2.3.1 (*) +| | +--- androidx.savedstate:savedstate:1.1.0 (*) +| | \--- androidx.annotation:annotation-experimental:1.0.0 -> 1.1.0 +| +--- androidx.appcompat:appcompat-resources:1.4.1 +| | +--- androidx.annotation:annotation:1.2.0 -> 1.3.0 +| | +--- androidx.core:core:1.0.1 -> 1.7.0 (*) +| | +--- androidx.vectordrawable:vectordrawable:1.1.0 +| | | +--- androidx.annotation:annotation:1.1.0 -> 1.3.0 +| | | +--- androidx.core:core:1.1.0 -> 1.7.0 (*) +| | | \--- androidx.collection:collection:1.1.0 (*) +| | \--- androidx.vectordrawable:vectordrawable-animated:1.1.0 +| | +--- androidx.vectordrawable:vectordrawable:1.1.0 (*) +| | +--- androidx.interpolator:interpolator:1.0.0 +| | | \--- androidx.annotation:annotation:1.0.0 -> 1.3.0 +| | \--- androidx.collection:collection:1.1.0 (*) +| +--- androidx.drawerlayout:drawerlayout:1.0.0 -> 1.1.1 +| | +--- androidx.annotation:annotation:1.1.0 -> 1.3.0 +| | +--- androidx.core:core:1.2.0 -> 1.7.0 (*) +| | \--- androidx.customview:customview:1.1.0 (*) +| \--- androidx.savedstate:savedstate:1.1.0 (*) ++--- com.google.android.material:material:1.5.0 +| +--- androidx.annotation:annotation:1.2.0 -> 1.3.0 +| +--- androidx.appcompat:appcompat:1.1.0 -> 1.4.1 (*) +| +--- androidx.cardview:cardview:1.0.0 +| | \--- androidx.annotation:annotation:1.0.0 -> 1.3.0 +| +--- androidx.coordinatorlayout:coordinatorlayout:1.1.0 +| | +--- androidx.annotation:annotation:1.1.0 -> 1.3.0 +| | +--- androidx.core:core:1.1.0 -> 1.7.0 (*) +| | +--- androidx.customview:customview:1.0.0 -> 1.1.0 (*) +| | \--- androidx.collection:collection:1.0.0 -> 1.1.0 (*) +| +--- androidx.constraintlayout:constraintlayout:2.0.1 -> 2.1.3 +| +--- androidx.core:core:1.5.0 -> 1.7.0 (*) +| +--- androidx.drawerlayout:drawerlayout:1.1.1 (*) +| +--- androidx.dynamicanimation:dynamicanimation:1.0.0 +| | +--- androidx.core:core:1.0.0 -> 1.7.0 (*) +| | +--- androidx.collection:collection:1.0.0 -> 1.1.0 (*) +| | \--- androidx.legacy:legacy-support-core-utils:1.0.0 +| | +--- androidx.annotation:annotation:1.0.0 -> 1.3.0 +| | +--- androidx.core:core:1.0.0 -> 1.7.0 (*) +| | +--- androidx.documentfile:documentfile:1.0.0 +| | | \--- androidx.annotation:annotation:1.0.0 -> 1.3.0 +| | +--- androidx.loader:loader:1.0.0 (*) +| | +--- androidx.localbroadcastmanager:localbroadcastmanager:1.0.0 +| | | \--- androidx.annotation:annotation:1.0.0 -> 1.3.0 +| | \--- androidx.print:print:1.0.0 +| | \--- androidx.annotation:annotation:1.0.0 -> 1.3.0 +| +--- androidx.annotation:annotation-experimental:1.0.0 -> 1.1.0 +| +--- androidx.fragment:fragment:1.0.0 -> 1.3.6 (*) +| +--- androidx.lifecycle:lifecycle-runtime:2.0.0 -> 2.4.0 (*) +| +--- androidx.recyclerview:recyclerview:1.0.0 -> 1.1.0 +| | +--- androidx.annotation:annotation:1.1.0 -> 1.3.0 +| | +--- androidx.core:core:1.1.0 -> 1.7.0 (*) +| | +--- androidx.customview:customview:1.0.0 -> 1.1.0 (*) +| | \--- androidx.collection:collection:1.0.0 -> 1.1.0 (*) +| +--- androidx.transition:transition:1.2.0 +| | +--- androidx.annotation:annotation:1.1.0 -> 1.3.0 +| | +--- androidx.core:core:1.0.1 -> 1.7.0 (*) +| | \--- androidx.collection:collection:1.0.0 -> 1.1.0 (*) +| +--- androidx.vectordrawable:vectordrawable:1.1.0 (*) +| \--- androidx.viewpager2:viewpager2:1.0.0 +| +--- androidx.annotation:annotation:1.1.0 -> 1.3.0 +| +--- androidx.fragment:fragment:1.1.0 -> 1.3.6 (*) +| +--- androidx.recyclerview:recyclerview:1.1.0 (*) +| +--- androidx.core:core:1.1.0 -> 1.7.0 (*) +| \--- androidx.collection:collection:1.1.0 (*) ++--- androidx.constraintlayout:constraintlayout:2.1.3 ++--- com.google.dagger:hilt-android:2.41 +| +--- com.google.dagger:dagger:2.41 +| | \--- javax.inject:javax.inject:1 +| +--- com.google.dagger:dagger-lint-aar:2.41 +| +--- com.google.dagger:hilt-core:2.41 +| | +--- com.google.dagger:dagger:2.41 (*) +| | +--- com.google.code.findbugs:jsr305:3.0.2 +| | \--- javax.inject:javax.inject:1 +| +--- com.google.code.findbugs:jsr305:3.0.2 +| +--- androidx.activity:activity:1.3.1 (*) +| +--- androidx.annotation:annotation:1.2.0 -> 1.3.0 +| +--- androidx.fragment:fragment:1.3.6 (*) +| +--- androidx.lifecycle:lifecycle-common:2.3.1 -> 2.4.0 (*) +| +--- androidx.lifecycle:lifecycle-viewmodel:2.3.1 (*) +| +--- androidx.lifecycle:lifecycle-viewmodel-savedstate:2.3.1 (*) +| +--- androidx.savedstate:savedstate:1.1.0 (*) +| +--- javax.inject:javax.inject:1 +| \--- org.jetbrains.kotlin:kotlin-stdlib:1.5.32 +| +--- org.jetbrains:annotations:13.0 +| \--- org.jetbrains.kotlin:kotlin-stdlib-common:1.5.32 ++--- androidx.appcompat:appcompat:{strictly 1.4.1} -> 1.4.1 (c) ++--- com.google.android.material:material:{strictly 1.5.0} -> 1.5.0 (c) ++--- androidx.constraintlayout:constraintlayout:{strictly 2.1.3} -> 2.1.3 (c) ++--- com.google.dagger:hilt-android:{strictly 2.41} -> 2.41 (c) ++--- androidx.annotation:annotation:{strictly 1.3.0} -> 1.3.0 (c) ++--- androidx.core:core:{strictly 1.7.0} -> 1.7.0 (c) ++--- androidx.cursoradapter:cursoradapter:{strictly 1.0.0} -> 1.0.0 (c) ++--- androidx.activity:activity:{strictly 1.3.1} -> 1.3.1 (c) ++--- androidx.fragment:fragment:{strictly 1.3.6} -> 1.3.6 (c) ++--- androidx.appcompat:appcompat-resources:{strictly 1.4.1} -> 1.4.1 (c) ++--- androidx.drawerlayout:drawerlayout:{strictly 1.1.1} -> 1.1.1 (c) ++--- androidx.savedstate:savedstate:{strictly 1.1.0} -> 1.1.0 (c) ++--- androidx.cardview:cardview:{strictly 1.0.0} -> 1.0.0 (c) ++--- androidx.coordinatorlayout:coordinatorlayout:{strictly 1.1.0} -> 1.1.0 (c) ++--- androidx.dynamicanimation:dynamicanimation:{strictly 1.0.0} -> 1.0.0 (c) ++--- androidx.annotation:annotation-experimental:{strictly 1.1.0} -> 1.1.0 (c) ++--- androidx.lifecycle:lifecycle-runtime:{strictly 2.4.0} -> 2.4.0 (c) ++--- androidx.recyclerview:recyclerview:{strictly 1.1.0} -> 1.1.0 (c) ++--- androidx.transition:transition:{strictly 1.2.0} -> 1.2.0 (c) ++--- androidx.vectordrawable:vectordrawable:{strictly 1.1.0} -> 1.1.0 (c) ++--- androidx.viewpager2:viewpager2:{strictly 1.0.0} -> 1.0.0 (c) ++--- com.google.dagger:dagger:{strictly 2.41} -> 2.41 (c) ++--- com.google.dagger:dagger-lint-aar:{strictly 2.41} -> 2.41 (c) ++--- com.google.dagger:hilt-core:{strictly 2.41} -> 2.41 (c) ++--- com.google.code.findbugs:jsr305:{strictly 3.0.2} -> 3.0.2 (c) ++--- androidx.lifecycle:lifecycle-common:{strictly 2.4.0} -> 2.4.0 (c) ++--- androidx.lifecycle:lifecycle-viewmodel:{strictly 2.3.1} -> 2.3.1 (c) ++--- androidx.lifecycle:lifecycle-viewmodel-savedstate:{strictly 2.3.1} -> 2.3.1 (c) ++--- javax.inject:javax.inject:{strictly 1} -> 1 (c) ++--- org.jetbrains.kotlin:kotlin-stdlib:{strictly 1.5.32} -> 1.5.32 (c) ++--- androidx.versionedparcelable:versionedparcelable:{strictly 1.1.1} -> 1.1.1 (c) ++--- androidx.collection:collection:{strictly 1.1.0} -> 1.1.0 (c) ++--- androidx.viewpager:viewpager:{strictly 1.0.0} -> 1.0.0 (c) ++--- androidx.loader:loader:{strictly 1.0.0} -> 1.0.0 (c) ++--- androidx.lifecycle:lifecycle-livedata-core:{strictly 2.3.1} -> 2.3.1 (c) ++--- androidx.vectordrawable:vectordrawable-animated:{strictly 1.1.0} -> 1.1.0 (c) ++--- androidx.customview:customview:{strictly 1.1.0} -> 1.1.0 (c) ++--- androidx.legacy:legacy-support-core-utils:{strictly 1.0.0} -> 1.0.0 (c) ++--- androidx.arch.core:core-common:{strictly 2.1.0} -> 2.1.0 (c) ++--- org.jetbrains:annotations:{strictly 13.0} -> 13.0 (c) ++--- org.jetbrains.kotlin:kotlin-stdlib-common:{strictly 1.5.32} -> 1.5.32 (c) ++--- androidx.lifecycle:lifecycle-livedata:{strictly 2.0.0} -> 2.0.0 (c) ++--- androidx.interpolator:interpolator:{strictly 1.0.0} -> 1.0.0 (c) ++--- androidx.documentfile:documentfile:{strictly 1.0.0} -> 1.0.0 (c) ++--- androidx.localbroadcastmanager:localbroadcastmanager:{strictly 1.0.0} -> 1.0.0 (c) ++--- androidx.print:print:{strictly 1.0.0} -> 1.0.0 (c) +\--- androidx.arch.core:core-runtime:{strictly 2.1.0} -> 2.1.0 (c) + +(c) - dependency constraint +(*) - dependencies omitted (listed previously) + +A web-based, searchable dependency report is available by adding the --scan option. diff --git a/tools/import_deps/tests/data/deps_with_missing.txt b/tools/import_deps/tests/data/deps_with_missing.txt new file mode 100644 index 00000000000..32290dab424 --- /dev/null +++ b/tools/import_deps/tests/data/deps_with_missing.txt @@ -0,0 +1,16 @@ + +------------------------------------------------------------ +Project ':app' +------------------------------------------------------------ + +debugCompileClasspath - Resolved configuration for compilation for variant: debug ++--- androidx.appcompat:appcompat:1.4.1 +| +--- androidx.annotation:annotation:1.3.0 +| +--- androidx.core:core:1.7.0 +| | +--- androidx.annotation:annotation:1.2.0 -> 1.3.0 +\--- androidx.arch.core:core-runtime:{strictly 2.1.0} -> 2.1.0 (c) + +(c) - dependency constraint +(*) - dependencies omitted (listed previously) + +A web-based, searchable dependency report is available by adding the --scan option. diff --git a/tools/import_deps/tests/test_base.py b/tools/import_deps/tests/test_base.py new file mode 100644 index 00000000000..79908962b9f --- /dev/null +++ b/tools/import_deps/tests/test_base.py @@ -0,0 +1,36 @@ +# Copyright (c) Meta Platforms, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import glob +import os +import unittest + +from dependency.repo import Repo + + +class BaseTestCase(unittest.TestCase): + @classmethod + def setUpClass(self): + self.repo = Repo("../third-party") + self.deps_file = "data/deps.txt" + + @classmethod + def tearDownClass(self): + print(f"cleaning up the {self.repo.get_libs_name()}") + files = glob.glob(f"{self.repo.get_libs_name()}/**/*.*", recursive=True) + for f in files: + try: + if os.path.isfile(f): + os.remove(f) + except OSError as e: + self.fail(e) diff --git a/tools/import_deps/tests/test_buck.py b/tools/import_deps/tests/test_buck.py new file mode 100644 index 00000000000..92e1a104cb4 --- /dev/null +++ b/tools/import_deps/tests/test_buck.py @@ -0,0 +1,121 @@ +# Copyright (c) Meta Platforms, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import filecmp +import os +import unittest +from pathlib import Path + +from dependency.buck import Buck +from dependency.gradle import GradleDependencies +from dependency.manager import Manager +from dependency.pom import MavenDependency +from tests.test_base import BaseTestCase + + +class BuckTestCase(BaseTestCase): + def test_create(self): + buck = Buck(self.repo) + dependency = MavenDependency("com.google.dagger", "dagger", "2.41", "") + buck_file = buck.get_buck_path_for_dependency(dependency) + manager = Manager(self.repo) + downloaded_files = manager.import_dep_shallow(dependency) + buck.create(buck_file, dependency, set()) + self.assertTrue(os.path.exists(buck_file)) + self.assertTrue( + filecmp.cmp("data/buck/buck1.txt", buck_file), + f"files are different: {buck_file}", + ) + + def test_create_with_some_dependencies(self): + buck = Buck(self.repo) + dependency = MavenDependency("com.google.dagger", "dagger", "2.41", "") + buck_file = buck.get_buck_path_for_dependency(dependency) + manager = Manager(self.repo) + downloaded_files = manager.import_dep(dependency) + self.assertEqual(3, len(dependency.get_artifacts())) + self.assertEqual(1, len(dependency.get_dependencies())) + buck.create(buck_file, dependency, set()) + self.assertTrue(os.path.exists(buck_file)) + self.assertTrue( + filecmp.cmp("data/buck/buck2.txt", buck_file), + "dagger BUCK files are different", + ) + + def test_create_with_aar_dependencies(self): + buck = Buck(self.repo) + dependency = MavenDependency("com.google.dagger", "hilt-android", "2.41", "") + buck_file = buck.get_buck_path_for_dependency(dependency) + expected_file = os.path.join(Path(self.repo.root), "com/google/dagger/BUCK") + self.assertEqual(expected_file, buck_file.__str__()) + manager = Manager(self.repo) + downloaded_files = manager.import_dep(dependency) + self.assertEqual(3, len(dependency.get_artifacts())) + self.assertEqual(13, len(dependency.get_dependencies())) + buck.create(buck_file, dependency, set()) + self.assertTrue(os.path.exists(expected_file)) + + def test_create_with_all_dependencies(self): + """ + Read all dependencies from the gradle. + Create BUCK for all these dependencies. + """ + file = self.deps_file + self.assertTrue(os.path.exists(file)) + gd = GradleDependencies() + # Importing dependencies from a file, will not determine the scope. + # I.e. whether it is runtime or a compile time dependency. + # We will need to parse pom for the dependency and update it. + deps = gd.import_gradle_dependencies(file) + self.assertEqual(47, len(deps)) + buck = Buck(self.repo) + # Remove ./libs/BUCK file if it exists + buck_file = buck.get_path() + if os.path.exists(buck_file): + os.remove(buck_file) + manager = Manager(self.repo) + versions = dict() + for dep in deps: + manager.import_dep_shallow(dep) + versions[dep.get_group_and_artifactid()] = dep + + # Check for missing dependencies + manager.import_missing_dependencies(deps, versions) + + visited = set() + for dep in deps: + buck_file = buck.get_buck_path_for_dependency(dep) + # This will also update a BUCK file + buck.create(buck_file, dep, visited) + visited.add(dep) + + def test_get_path_for_dependency(self): + buck = Buck(self.repo) + dependency = MavenDependency( + "com.google.guava", "guava-testlib", "31.1-jre", "" + ) + path = buck.get_buck_path_for_dependency(dependency) + self.assertEqual(Path("../third-party/com/google/guava/BUCK"), path) + + def tearDown(self) -> None: + super().tearDown() + buck1 = os.path.join(self.repo.root, "com/google/dagger/BUCK") + if os.path.exists(buck1): + os.rename(buck1, os.path.join(self.repo.root, "com/google/dagger/BUCK.bak")) + buck2 = os.path.join(self.repo.root, "javax/inject/BUCK") + if os.path.exists(buck2): + os.rename(buck2, os.path.join(self.repo.root, "javax/inject/BUCK.bak")) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/import_deps/tests/test_gradle.py b/tools/import_deps/tests/test_gradle.py new file mode 100644 index 00000000000..04b8ab8e819 --- /dev/null +++ b/tools/import_deps/tests/test_gradle.py @@ -0,0 +1,116 @@ +# Copyright (c) Meta Platforms, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import unittest + +from dependency.gradle import GradleDependencies +from dependency.manager import Manager +from dependency.pom import MavenDependency +from tests.test_base import BaseTestCase + + +class MyTestCase(BaseTestCase): + def test_import_gradle_dependencies(self): + file = "data/deps.txt" + self.assertTrue(os.path.exists(file)) + gd = GradleDependencies() + deps = gd.import_gradle_dependencies(file) + self.assertEqual(47, len(deps)) + self.print_dependencies_tree(deps, ">") + core_runtime: MavenDependency = gd.find_dependency( + deps, MavenDependency("androidx.arch.core", "core-runtime", "2.1.0") + ) + self.repo.load_artifacts(core_runtime) + if len(core_runtime.get_artifacts()) < 1: + manager = Manager(self.repo) + manager.import_dep_shallow(core_runtime) + self.assertEqual(3, len(core_runtime.get_artifacts())) + core_runtime_deps = core_runtime.get_dependencies() + self.assertEqual(2, len(core_runtime_deps)) + core_common = gd.find_dependency( + core_runtime_deps, + MavenDependency("androidx.arch.core", "core-common", "2.1.0"), + ) + self.assertEqual(1, len(core_common.get_dependencies())) + expected_annotation = MavenDependency( + "androidx.annotation", "annotation", "1.3.0" + ) + actual_annotation = core_common.get_dependencies().pop() + self.assertTrue(expected_annotation.is_keys_equal(actual_annotation)) + + def print_dependencies_tree(self, dependencies: [], prefix: str = ""): + for d in dependencies: + print(f"{prefix} {d}") + if len(d.get_dependencies()) > 0: + self.print_dependencies_tree(d.get_dependencies(), f">{prefix}") + + def test_match_version(self): + gd = GradleDependencies() + self.assertEqual("1.3.0", gd.match_version("1.3.0")) + self.assertEqual("1.3.0", gd.match_version("1.3.0 (*)")) + self.assertEqual("1.3.0", gd.match_version("1.1.0 -> 1.3.0")) + self.assertEqual("1.3.0", gd.match_version("1.1.0 -> 1.3.0 (*)")) + self.assertEqual("1", gd.match_version("1")) + self.assertEqual("1", gd.match_version("1 (*)")) + self.assertEqual("2", gd.match_version("1 -> 2")) + self.assertEqual("2", gd.match_version("1 -> 2 (*)")) + self.assertEqual("2", gd.match_version("1 -> 2 (c)")) + self.assertEqual("1.5.0", gd.match_version("{strictly 1.5.0} -> 1.5.0 (c)")) + + def test_extract_dependency(self): + gd = GradleDependencies() + self.assertEqual( + MavenDependency("androidx.appcompat", "appcompat", "1.4.1", ""), + gd.extract_dependency("+--- androidx.appcompat:appcompat:1.4.1"), + ) + self.assertEqual( + MavenDependency("androidx.annotation", "annotation", "1.3.0", ""), + gd.extract_dependency("| +--- androidx.annotation:annotation:1.3.0"), + ) + self.assertEqual( + MavenDependency("androidx.lifecycle", "lifecycle-runtime", "2.4.0", ""), + gd.extract_dependency( + "| | +--- androidx.lifecycle:lifecycle-runtime:2.3.1 -> 2.4.0 (*)" + ), + ) + self.assertEqual( + MavenDependency("androidx.lifecycle", "lifecycle-viewmodel", "2.3.1", ""), + gd.extract_dependency( + "| | \\--- androidx.lifecycle:lifecycle-viewmodel:2.3.1 (*)" + ), + ) + self.assertEqual( + MavenDependency("androidx.lifecycle", "lifecycle-livedata", "2.0.0", ""), + gd.extract_dependency( + "| | | +--- androidx.lifecycle:lifecycle-livedata:2.0.0" + ), + ) + self.assertEqual( + MavenDependency("javax.inject", "javax.inject", "1", ""), + gd.extract_dependency("| | \\--- javax.inject:javax.inject:1"), + ) + self.assertEqual( + MavenDependency("com.google.dagger", "dagger", "2.41", ""), + gd.extract_dependency("| | +--- com.google.dagger:dagger:2.41 (*)"), + ) + self.assertEqual( + MavenDependency("com.google.android.material", "material", "1.5.0", ""), + gd.extract_dependency( + "+--- com.google.android.material:material:{strictly 1.5.0} -> 1.5.0 (c)" + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/import_deps/tests/test_manager.py b/tools/import_deps/tests/test_manager.py new file mode 100644 index 00000000000..5fc1eb19df1 --- /dev/null +++ b/tools/import_deps/tests/test_manager.py @@ -0,0 +1,289 @@ +# Copyright (c) Meta Platforms, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import os +import unittest + +from dependency.buck import Buck +from dependency.gradle import GradleDependencies +from dependency.manager import Manager +from dependency.pom import MavenDependency, PomXmlAdapter +from dependency.repo import ArtifactFlavor +from tests.test_base import BaseTestCase + + +class TestManager(BaseTestCase): + def test_import_maven_dep(self): + gradle_dep = "com.google.dagger:hilt-android:2.41" + manager = Manager(self.repo) + pd = manager.import_maven_dep(gradle_dep) + self.assertEqual( + MavenDependency("com.google.dagger", "hilt-android", "2.41", ""), pd + ) + self.assertFalse( + os.path.exists( + f"{self.repo.root}/com/google/dagger/hilt-android/2.41/hilt-android-2.41.jar" + ) + ) + self.assertTrue( + os.path.exists( + f"{self.repo.root}/com/google/dagger/hilt-android/2.41/hilt-android-2.41.pom" + ) + ) + self.assertTrue( + os.path.exists( + f"{self.repo.root}/com/google/dagger/hilt-android/2.41/hilt-android-2.41.aar" + ) + ) + + def test_toPomDependency(self): + gradle_dep = "com.google.dagger:hilt-android:2.41" + manager = Manager(self.repo) + pd = manager.to_maven_dependency(gradle_dep) + self.assertEqual( + MavenDependency("com.google.dagger", "hilt-android", "2.41", ""), pd + ) + + def test_import_dep_shallow(self): + self.assertTrue(os.path.exists(self.deps_file)) + gd = GradleDependencies() + deps = gd.import_gradle_dependencies(self.deps_file) + self.assertEqual(47, len(deps)) + manager = Manager(self.repo) + for d in deps: + downloaded_files = manager.import_dep_shallow(d) + self.assertTrue(len(downloaded_files) > 0, f"Unable to download: {d}") + + def test_import_dep(self): + gradle_dep = "com.google.dagger:hilt-android:2.41" + manager = Manager(self.repo) + pd = manager.to_maven_dependency(gradle_dep) + all_dependencies = manager.import_dep(pd) + for n in all_dependencies: + print(n) + self.assertEqual(52, len(all_dependencies)) + + def test_check_missing_dependencies_partial_list(self): + manager = Manager(self.repo) + pom = MavenDependency("androidx.viewpager", "viewpager", "1.0.0", "") + manager.import_dep_shallow(pom) + core = MavenDependency("androidx.core", "core", "1.0.0", "compile") + pom.add_dependency(core) + missing_deps = manager.check_missing_dependencies(pom) + self.assertEqual(2, len(missing_deps)) + + def test_check_missing_dependencies_empty_list(self): + manager = Manager(self.repo) + dependency = MavenDependency( + "androidx.concurrent", "concurrent-futures", "1.0.0", "runtime" + ) + manager.import_dep_shallow(dependency) + missing_deps = manager.check_missing_dependencies(dependency) + logging.debug("missing dependencies:") + dependency.print_dependency_tree() + for d in missing_deps: + logging.debug(d) + self.assertEqual(2, len(missing_deps)) + annotation = MavenDependency( + "androidx.annotation", "annotation", "1.1.0", "compile" + ) + listenablefuture = MavenDependency( + "com.google.guava", "listenablefuture", "1.0", "compile" + ) + self.assertEqual({annotation, listenablefuture}, missing_deps) + + def test_import_missing_dependencies_single_dep(self): + manager = Manager(self.repo) + dependency = MavenDependency("androidx.viewpager", "viewpager", "1.0.0", "") + manager.import_dep_shallow(dependency) + versions = dict() + annotations = MavenDependency( + "androidx.annotation", "annotation", "1.3.0", "compile" + ) + versions[annotations.get_group_and_artifactid()] = annotations + manager.import_missing_dependencies({dependency}, versions) + self.assertEqual(3, len(dependency.get_dependencies())) + self.assertTrue(annotations in dependency.get_dependencies()) + all_dependencies = dependency.get_all_dependencies() + self.assertEqual(9, len(all_dependencies)) + print("\n\n*** all dependencies") + for d in all_dependencies: + print(d) + print("\n\n*** dependencies tree") + dependency.print_dependency_tree() + print("\n\n*** versions") + for k, v in versions.items(): + print(f"{k}:{v}") + self.assertEqual(9, len(versions)) + + def test_import_missing_dependencies_multiple_deps(self): + manager = Manager(self.repo) + dependency = MavenDependency( + "androidx.concurrent", "concurrent-futures", "1.0.0", "runtime" + ) + manager.import_dep_shallow(dependency) + versions = dict() + annotations = MavenDependency( + "androidx.annotation", "annotation", "1.3.0", "compile" + ) + versions[annotations.get_group_and_artifactid()] = annotations + manager.import_missing_dependencies({dependency, annotations}, versions) + self.assertEqual(2, len(dependency.get_dependencies())) + self.assertTrue(annotations in dependency.get_dependencies()) + all_dependencies = dependency.get_all_dependencies() + self.assertEqual(3, len(all_dependencies)) + print("\n\n*** all dependencies") + for d in all_dependencies: + print(d) + print("\n\n*** dependencies tree") + dependency.print_dependency_tree() + print("\n\n*** versions") + for k, v in versions.items(): + print(f"{k}:{v}") + self.assertEqual(3, len(versions)) + + def test_import_missing_dependencies_updates_versions(self): + manager = Manager(self.repo) + dependency = MavenDependency("androidx.appcompat", "appcompat", "1.4.1", "") + manager.import_dep_shallow(dependency) + versions = dict() + annotations = MavenDependency( + "androidx.annotation", "annotation", "1.3.0", "compile" + ) + versions[annotations.get_group_and_artifactid()] = annotations + manager.import_missing_dependencies({dependency, annotations}, versions) + self.assertEqual(14, len(dependency.get_dependencies())) + self.assertTrue(annotations in dependency.get_dependencies()) + all_dependencies = dependency.get_all_dependencies() + print("\n\n*** all dependencies") + keys = set() + for d in all_dependencies: + print(d) + keys.add(d.get_group_and_artifactid()) + print("\n\n*** dependencies tree") + dependency.print_dependency_tree() + self.assertEqual(34, len(all_dependencies)) + print("\n\n*** versions") + for k, v in versions.items(): + print(f"{k}:{v}") + self.assertEqual(set(versions.keys()), keys) + + def test_import_missing_dependencies_all_gradle_deps(self): + file = "data/deps_with_missing.txt" + self.assertTrue(os.path.exists(file)) + gd = GradleDependencies() + deps = gd.import_gradle_dependencies(file) + buck = Buck(self.repo) + # Remove ./libs/BUCK file if it exists + buck_file = buck.get_path() + if os.path.exists(buck_file): + os.remove(buck_file) + manager = Manager(self.repo) + versions = dict() + print("\n\n*** all dependencies") + for dep in deps: + manager.import_dep_shallow(dep) + versions[dep.get_group_and_artifactid()] = dep + print(dep) + self.assertEqual(4, len(versions)) + # Check for missing dependencies + manager.import_missing_dependencies(deps, versions) + self.assertTrue("androidx.annotation_annotation-experimental" in versions) + self.assertEqual( + "androidx.annotation:annotation-experimental:1.1.0:compile", + versions["androidx.annotation_annotation-experimental"].__str__(), + ) + # self.assertEqual("androidx.emoji2:emoji2:1.0.0:runtime", + # versions["androidx.emoji2_emoji2"].__str__()) + print("\n\n*** versions") + for k, v in versions.items(): + print(f"{k}:{v}") + packages = set() + print("\n\n*** dependencies tree") + appcompat = versions["androidx.appcompat_appcompat"] + appcompat.print_dependency_tree() + for dep in deps: + packages.add(dep) + dep_tree = dep.get_all_dependencies() + packages = packages.union(dep_tree) + # dep.print_dependency_tree() + # print("\n") + unique_package_keys = set() + for p in packages: + unique_package_keys.add(p.get_group_and_artifactid()) + self.assertEqual(set(versions.keys()), unique_package_keys) + # find dup packages + for k in unique_package_keys: + dups = [] + for p in packages: + if p.get_group_and_artifactid() == k: + dups.append(p) + if len(dups) > 1: + print(f"dups: {dups}") + self.assertEqual(len(versions), len(packages)) + self.assertEqual(34, len(versions)) + + def test_import_missing_dependencies_resove_missing_versions(self): + dependency = MavenDependency("com.google.guava", "guava", "31.0.1-jre", "") + versions = dict() + manager = Manager(self.repo) + manager.import_dep_shallow(dependency) + jar_file = ( + f"{self.repo.root}/com/google/guava/guava/31.0.1-jre/guava-31.0.1-jre.jar" + ) + self.assertTrue(os.path.exists(jar_file)) + visited = manager.import_missing_dependencies({dependency}, versions) + keys = set() + for d in visited: + keys.add(d.__str__()) + self.assertSetEqual( + { + "com.google.guava:guava:31.0.1-jre:", + "com.google.guava:failureaccess:1.0.1:", + "com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava:", + "com.google.code.findbugs:jsr305:3.0.2:", + "org.checkerframework:checker-qual:3.22.0:", + "com.google.errorprone:error_prone_annotations:2.13.1:", + "com.google.j2objc:j2objc-annotations:1.3:", + }, + keys, + ) + + def test_resolve_deps_versions(self): + dependency = MavenDependency("com.google.guava", "guava", "31.0.1-jre", "") + manager = Manager(self.repo) + manager.import_dep_shallow(dependency) + pom_file = self.repo.download_maven_dep(dependency, ArtifactFlavor.POM) + self.assertTrue(os.path.exists(pom_file)) + pom_xml_adapter = PomXmlAdapter(pom_file) + pom_deps = pom_xml_adapter.get_deps() + manager.resolve_deps_versions(dependency, pom_deps) + keys = set() + for d in pom_deps: + keys.add(d.__str__()) + self.assertSetEqual( + { + "com.google.guava:failureaccess:1.0.1:", + "com.google.guava:listenablefuture:9999.0-empty-to-avoid-conflict-with-guava:", + "com.google.code.findbugs:jsr305:3.0.2:", + "org.checkerframework:checker-qual:3.22.0:", + "com.google.errorprone:error_prone_annotations:2.13.1:", + "com.google.j2objc:j2objc-annotations:1.3:", + }, + keys, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/import_deps/tests/test_pom.py b/tools/import_deps/tests/test_pom.py new file mode 100644 index 00000000000..c29ae866a07 --- /dev/null +++ b/tools/import_deps/tests/test_pom.py @@ -0,0 +1,171 @@ +# Copyright (c) Meta Platforms, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import unittest + +from dependency.pom import MavenDependency, PomXmlAdapter +from dependency.repo import Repo, ArtifactFlavor +from tests.test_base import BaseTestCase + + +class TestPomDependency(BaseTestCase): + def test_str(self): + dependency = MavenDependency("com.google.dagger", "dagger", "2.41", "") + self.assertEqual("com.google.dagger:dagger:2.41:", dependency.__str__()) + + def test_get_libs_path(self): + dependency = MavenDependency("com.google.dagger", "dagger", "2.41", "") + path = dependency.get_libs_path() + self.assertEqual(["com", "google", "dagger", "dagger", "2.41"], path) + + def test_parse_pom(self): + dependency = MavenDependency("com.google.dagger", "dagger", "2.41", "") + self.repo.download_maven_dep(dependency, ArtifactFlavor.POM) + file = self.repo.download_maven_dep(dependency, ArtifactFlavor.POM) + self.assertTrue(os.path.exists(file)) + self.assertEqual( + f"{self.repo.get_libs_name()}/com/google/dagger/dagger/2.41/dagger-2.41.pom", + file, + ) + pom_xml_adapter = PomXmlAdapter(file) + deps = pom_xml_adapter.get_deps() + self.assertEqual(1, len(deps)) + self.assertEqual( + MavenDependency("javax.inject", "javax.inject", "1", "").__str__(), + deps[0].__str__(), + ) + + def test_resolve_pom_variables(self): + """ + Verifies that the method replaces variables in + ${project.groupId} + guava + ${project.version} + with the project groupId and version: "com.google.guava:guava:31.1-jre:". + """ + dependency = MavenDependency( + "com.google.guava", "guava-testlib", "31.1-jre", "" + ) + self.repo.download_maven_dep(dependency, ArtifactFlavor.POM) + file = self.repo.download_maven_dep(dependency, ArtifactFlavor.POM) + self.assertTrue(os.path.exists(file)) + self.assertEqual( + f"{self.repo.get_libs_name()}/com/google/guava/guava-testlib/31.1-jre/guava-testlib-31.1-jre.pom", + file, + ) + pom_xml_adapter = PomXmlAdapter(file) + deps = pom_xml_adapter.get_deps() + keys = set() + for d in deps: + s = d.__str__() + print(f"dep: {s}") + keys.add(s) + self.assertSetEqual( + { + "com.google.code.findbugs:jsr305:None:", + "org.checkerframework:checker-qual:None:", + "com.google.errorprone:error_prone_annotations:None:", + "com.google.j2objc:j2objc-annotations:None:", + "com.google.guava:guava:31.1-jre:", + "junit:junit:None:compile", + # "com.google.truth:javax.truth:None:test", - the test is excluded for now + }, + keys, + ) + + def test_resolve_pom_properties(self): + dependency = MavenDependency("junit", "junit", "4.13.2", "") + self.repo.download_maven_dep(dependency, ArtifactFlavor.POM) + file = self.repo.download_maven_dep(dependency, ArtifactFlavor.POM) + self.assertTrue(os.path.exists(file)) + self.assertEqual( + f"{self.repo.get_libs_name()}/junit/junit/4.13.2/junit-4.13.2.pom", + file, + ) + pom_xml_adapter = PomXmlAdapter(file) + deps = pom_xml_adapter.get_deps() + keys = set() + for d in deps: + s = d.__str__() + print(f"dep: {s}") + keys.add(s) + self.assertSetEqual( + { + "org.hamcrest:hamcrest-core:1.3:", + # TODO: check about the test scope + }, + keys, + ) + + def test_resolve_pom_complex_properties(self): + dependency = MavenDependency("com.squareup", "javapoet", "1.13.0", "") + self.repo.download_maven_dep(dependency, ArtifactFlavor.POM) + file = self.repo.download_maven_dep(dependency, ArtifactFlavor.POM) + self.assertTrue(os.path.exists(file)) + self.assertEqual( + f"{self.repo.get_libs_name()}/com/squareup/javapoet/1.13.0/javapoet-1.13.0.pom", + file, + ) + pom_xml_adapter = PomXmlAdapter(file) + deps = pom_xml_adapter.get_deps() + self.assertEqual(0, len(deps)) + # TODO: update for "test" scope + + def test_get_deps(self): + dependency = MavenDependency("com.google.dagger", "hilt-core", "2.41", "") + self.repo.download_maven_dep(dependency, ArtifactFlavor.POM) + pom_file = f"{self.repo.get_libs_name()}/com/google/dagger/hilt-core/2.41/hilt-core-2.41.pom" + self.assertTrue(os.path.exists(pom_file)) + pa = PomXmlAdapter(pom_file) + deps = pa.get_deps() + self.assertEqual(3, len(deps)) + keys = set() + for d in deps: + keys.add(d.__str__()) + self.assertSetEqual( + { + "com.google.dagger:dagger:2.41:", + "com.google.code.findbugs:jsr305:3.0.2:", + "javax.inject:javax.inject:1:", + }, + keys, + ) + + def test_add_dependency_version_upgrade(self): + dependency = MavenDependency("com.google.dagger", "hilt-core", "2.41", "") + d1 = MavenDependency("androidx.core", "core", "1.7.0", "") + d2 = MavenDependency("androidx.core", "core", "1.1.0", "") + d3 = MavenDependency("androidx.core", "core", "1.8.0", "") + dependency.add_dependency(d1) + dependency.add_dependency(d2) + dependency.add_dependency(d3) + deps = dependency.get_dependencies() + self.assertEqual(1, len(deps)) + self.assertEqual(deps.pop(), d3) + + def test_add_dependency_excludes_cycles(self): + dependency = MavenDependency("com.google.dagger", "hilt-core", "2.41", "") + d1 = MavenDependency("androidx.core", "core", "1.7.0", "") + d2 = MavenDependency("androidx.core", "core", "1.1.0", "") + d3 = MavenDependency("com.google.dagger", "hilt-core", "2.41", "") + dependency.add_dependency(d1) + dependency.add_dependency(d2) + dependency.add_dependency(d3) + deps = dependency.get_dependencies() + self.assertEqual(1, len(deps)) + self.assertEqual(deps.pop(), d1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/import_deps/tests/test_repo.py b/tools/import_deps/tests/test_repo.py new file mode 100644 index 00000000000..385f299341d --- /dev/null +++ b/tools/import_deps/tests/test_repo.py @@ -0,0 +1,108 @@ +# Copyright (c) Meta Platforms, Inc. and its affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import time +import unittest + +from dependency.pom import MavenDependency +from dependency.repo import Repo, ArtifactFlavor +from tests.test_base import BaseTestCase + + +class TestRepo(BaseTestCase): + def test_mkdirs(self): + dirs = ["one", "two", "three"] + self.assertTrue(self.repo.mkdirs(dirs)) + timeout = time.time() + 5 + while time.time() < timeout: + if os.path.exists("../libs/one/two/three"): + break + time.sleep(1) + self.assertTrue(os.path.exists(f"{self.repo.root}/one")) + self.assertTrue(os.path.exists(f"{self.repo.root}/one/two")) + self.assertTrue(os.path.exists(f"{self.repo.root}/one/two/three")) + os.rmdir(f"{self.repo.root}/one/two/three") + os.rmdir(f"{self.repo.root}/one/two") + os.rmdir(f"{self.repo.root}/one") + + def test_get_dependency_dir(self): + pom = MavenDependency("com.google.dagger", "dagger", "2.41", "") + path = self.repo.get_dependency_dir(pom) + self.assertEqual( + f"{self.repo.get_libs_name()}/com/google/dagger/dagger/2.41", path.__str__() + ) + + def test_get_dependency_path(self): + pom = MavenDependency("com.google.dagger", "dagger", "2.41", "") + path = self.repo.get_dependency_path(pom, ArtifactFlavor.JAR) + self.assertEqual( + f"{self.repo.get_libs_name()}/com/google/dagger/dagger/2.41/dagger-2.41.jar", + path.__str__(), + ) + + def test_get_base_url(self): + pom = MavenDependency("com.google.dagger", "dagger", "2.41", "") + url = self.repo.get_base_url(pom) + self.assertEqual( + "https://repo1.maven.org/maven2/com/google/dagger/dagger/2.41/dagger-2.41", + url, + ) + + def test_get_url(self): + pom = MavenDependency("com.google.dagger", "dagger", "2.41", "") + url = self.repo.get_url(pom, ArtifactFlavor.JAR) + self.assertEqual( + "https://repo1.maven.org/maven2/com/google/dagger/dagger/2.41/dagger-2.41.jar", + url, + ) + url = self.repo.get_url(pom, ArtifactFlavor.POM) + self.assertEqual( + "https://repo1.maven.org/maven2/com/google/dagger/dagger/2.41/dagger-2.41.pom", + url, + ) + + def test_download_maven_dep(self): + pom = MavenDependency("com.google.dagger", "dagger", "2.41", "") + self.repo.download_maven_dep(pom, ArtifactFlavor.JAR) + self.assertTrue( + os.path.exists( + f"{self.repo.get_libs_name()}/com/google/dagger/dagger/2.41/dagger-2.41.jar" + ) + ) + file = self.repo.download_maven_dep(pom, ArtifactFlavor.POM) + self.assertTrue(os.path.exists(file)) + self.assertEqual( + f"{self.repo.get_libs_name()}/com/google/dagger/dagger/2.41/dagger-2.41.pom", + file, + ) + self.repo.download_maven_dep(pom, ArtifactFlavor.AAR) + self.assertFalse( + os.path.exists( + f"{self.repo.get_libs_name()}/com/google/dagger/dagger/2.41/dagger-2.41.aar" + ) + ) + + def test_get_release_version(self): + dep = MavenDependency("com.google.dagger", "dagger", "2.41", "") + result = self.repo.get_release_version(dep) + self.assertTrue( + os.path.exists( + f"{self.repo.get_libs_name()}/com/google/dagger/dagger/maven-metadata.xml" + ) + ) + self.assertEqual("2.42", result) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/release/platforms/chocolatey.py b/tools/release/platforms/chocolatey.py index 36109693259..8c3a27314ce 100644 --- a/tools/release/platforms/chocolatey.py +++ b/tools/release/platforms/chocolatey.py @@ -100,7 +100,6 @@ def build_chocolatey( windows_host, [ "build", - "--isolation=" + docker_isolation, "-m", docker_memory, # Default memory is 1G "-t", diff --git a/tools/release/platforms/chocolatey/Dockerfile b/tools/release/platforms/chocolatey/Dockerfile index 25df1861d61..6f10ec9e3a9 100644 --- a/tools/release/platforms/chocolatey/Dockerfile +++ b/tools/release/platforms/chocolatey/Dockerfile @@ -5,12 +5,17 @@ SHELL ["powershell", "-command"] ARG version= ARG timestamp= ARG repository=facebook/buck +ARG chocolateyUseWindowsCompression=true # Install chocolatey RUN Set-ExecutionPolicy Bypass -Scope Process -Force; iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) # Download enough to bootstrap a build -RUN choco install -y python2 git jdk8 +RUN choco install -y python --version=3.6.2 +RUN choco install -y git jdk8 + +# Choco installs python3 and named it to python, buck expects python3 so create a symlink to python +RUN $pythonpath=(Split-Path((Get-Command python).Path)); cmd /c mklink ${pythonpath}\python3.exe ${pythonpath}\python.exe # The JRE package that ant depends on seems to be somewhat spotty, # install the jdk instead and use its java binary. diff --git a/tools/release/platforms/chocolatey/buck.nuspec b/tools/release/platforms/chocolatey/buck.nuspec index 87cc11adc32..15a64be4122 100644 --- a/tools/release/platforms/chocolatey/buck.nuspec +++ b/tools/release/platforms/chocolatey/buck.nuspec @@ -18,8 +18,8 @@ A fast build system that encourages the creation of small, reusable modules over a variety of platforms and languages including iOS, Android, C++, Java, and more. - - + + diff --git a/tools/release/platforms/chocolatey/build.py b/tools/release/platforms/chocolatey/build.py index eacb799f67a..651ad5528c5 100644 --- a/tools/release/platforms/chocolatey/build.py +++ b/tools/release/platforms/chocolatey/build.py @@ -88,8 +88,8 @@ def write_license_file(original_license): def write_verification_txt(original_verification_txt, version, timestamp): dest = "VERIFICATION.txt" - with open(original_verification_txt, "r") as fin, open(dest, "w") as fout: - verification_text = fin.read().decode("utf-8") + with open(original_verification_txt, "r", encoding="utf-8") as fin, open(dest, "w") as fout: + verification_text = fin.read() verification_text = verification_text.format( release_version=version, release_timestamp=timestamp ) diff --git a/tools/release/platforms/debian/Dockerfile b/tools/release/platforms/debian/Dockerfile index 3bb8f8dd26b..a350de8b5d5 100644 --- a/tools/release/platforms/debian/Dockerfile +++ b/tools/release/platforms/debian/Dockerfile @@ -9,12 +9,16 @@ RUN apt-get install -y --no-install-recommends curl ca-certificates \ zlib1g-dev libarchive-dev \ ca-certificates-java \ ant \ - python \ + python3 \ + python3-distutils \ groovy \ ghc \ equivs && \ apt-get clean +# docker needs the old name python, create a symlink to python3 +RUN python_path=`which python3`;ln -f -s ${python_path} ${python_path%3} + RUN git clone --branch v${version} --depth 1 https://github.com/${repository}.git src WORKDIR /src diff --git a/tools/release/platforms/debian/buck.equivs b/tools/release/platforms/debian/buck.equivs index ed628020d8b..a972daac6e2 100644 --- a/tools/release/platforms/debian/buck.equivs +++ b/tools/release/platforms/debian/buck.equivs @@ -7,7 +7,7 @@ Package: buck Version: v Maintainer: team@buckaroo.pm Recommends: watchman, nailgun -Depends: openjdk-8-jre-headless, python2.7 +Depends: openjdk-8-jre-headless, python3, python3-distutils Architecture: all Copyright: LICENSE Readme: README.md diff --git a/tools/release/platforms/homebrew.py b/tools/release/platforms/homebrew.py index 4b6601d761a..095f4c76839 100644 --- a/tools/release/platforms/homebrew.py +++ b/tools/release/platforms/homebrew.py @@ -24,6 +24,7 @@ from platforms.common import ReleaseException, run from releases import get_version_and_timestamp_from_release +from releases import get_current_user def brew(homebrew_dir, command, *run_args, **run_kwargs): @@ -104,13 +105,13 @@ def update_formula_before_bottle( all_data = fin.read() all_data = re.sub( r"BUCK_VERSION = .*$", - 'BUCK_VERSION = "{}".freeze'.format(release_version), + 'BUCK_VERSION = "{}"'.format(release_version), all_data, flags=re.MULTILINE, ) all_data = re.sub( r"BUCK_RELEASE_TIMESTAMP = .*$", - 'BUCK_RELEASE_TIMESTAMP = "{}".freeze'.format(release_timestamp), + 'BUCK_RELEASE_TIMESTAMP = "{}"'.format(release_timestamp), all_data, flags=re.MULTILINE, ) @@ -230,7 +231,7 @@ def update_formula_after_bottle(formula_path, sha, target_macos_version_spec): logging.info("Updated formula with new bottle sha") -def push_tap(git_repository, tap_path, version): +def push_tap(git_repository, tap_path, version, github_token): """ Grab any working directory changes for the tap, clone a new tap repository, and push those changes upstream. The original tap path is in a clean state @@ -243,7 +244,8 @@ def push_tap(git_repository, tap_path, version): """ logging.info("Gathering git diff from {}".format(tap_path)) git_diff = run(["git", "diff"], tap_path, True).stdout - git_url = "git@github.com:{}.git".format(git_repository) + user = get_current_user(github_token) + git_url = "https://{}:{}@github.com/{}.git".format(user["login"], github_token, git_repository) with tempfile.TemporaryDirectory() as temp_dir: logging.info("Cloning {} into {}".format(git_url, temp_dir)) @@ -297,13 +299,13 @@ def audit_tap(homebrew_dir, tap_repository): brew(homebrew_dir, ["audit", brew_target]) -def publish_tap_changes(homebrew_dir, tap_repository, version): +def publish_tap_changes(homebrew_dir, tap_repository, version, github_token): git_user, git_repo = tap_repository.split("/") full_git_repo = "{}/homebrew-{}".format(git_user, git_repo) formula_path = get_formula_path(homebrew_dir, tap_repository) tap_path = os.path.dirname(formula_path) - push_tap(full_git_repo, tap_path, version) + push_tap(full_git_repo, tap_path, version, github_token) def log_about_manual_tap_push(homebrew_dir, tap_repository): diff --git a/tools/release/publish_release.py b/tools/release/publish_release.py index d9cfaa2e24f..86682e329a9 100755 --- a/tools/release/publish_release.py +++ b/tools/release/publish_release.py @@ -221,6 +221,11 @@ def parse_args(args): "https://github.com/chocolatey/chocolatey.org/issues/584" ), ) + parser.add_argument( + "--docker-login", + action="store_true", + help="If set, run 'docker login' using DOCKERHUB_USERNAME and DOCKERHUB_TOKEN", + ) parsed_kwargs = dict(parser.parse_args(args)._get_kwargs()) if parsed_kwargs["deb_file"]: parsed_kwargs["build_deb"] = False @@ -283,6 +288,15 @@ def validate_repo_upstream(args): ) +def docker_login(): + username = os.environ.get("DOCKERHUB_USERNAME") + token = os.environ.get("DOCKERHUB_TOKEN") + if username and token: + run(["docker", "login", "--username", username, "--password-stdin"], input=token) + else: + logging.error("Both DOCKERHUB_USERNAME and DOCKERHUB_TOKEN must be set to login to dockerhub") + + def validate_environment(args): """ Make sure we can build """ @@ -398,7 +412,7 @@ def publish( add_assets(release, github_token, homebrew_file) validate_tap(homebrew_dir, args.tap_repository, args.version) if args.homebrew_push_tap: - publish_tap_changes(homebrew_dir, args.tap_repository, args.version) + publish_tap_changes(homebrew_dir, args.tap_repository, args.version, github_token) else: log_about_manual_tap_push(args.tap_repository) @@ -410,6 +424,10 @@ def main(): github_token = ( args.github_token if args.github_token else get_token(args.github_token_file) ) + + if args.docker_login: + docker_login() + if args.chocolatey_publish: chocolatey_token = ( args.chocolatey_token @@ -467,7 +485,6 @@ def main(): homebrew_dir, chocolatey_file, ) - except ReleaseException as e: logging.error(str(e)) finally: diff --git a/tools/release/releases.py b/tools/release/releases.py index 53c97da7ba1..d0476d6176f 100644 --- a/tools/release/releases.py +++ b/tools/release/releases.py @@ -19,6 +19,7 @@ import dateutil.parser import magic import requests +import time from platforms.common import ReleaseException, run @@ -39,7 +40,8 @@ def get_version_and_timestamp_from_release(release): """ Returns the version (without a 'v' prefix) and the timestamp of the release """ release_version = release["tag_name"].lstrip("v") - release_timestamp = dateutil.parser.parse(release["created_at"]).strftime("%s") + created_at = dateutil.parser.parse(release["created_at"]) + release_timestamp = str(int(time.mktime(created_at.timetuple()))) return release_version, release_timestamp @@ -62,7 +64,8 @@ def get_current_user(github_token, prefer_fb_email=True): response = requests.get(url, headers=headers) response.raise_for_status() ret = response.json() - if not ret["email"].endswith("@fb.com") and prefer_fb_email: + default_email = ret["email"] + if default_email is None or (not default_email.endswith("@fb.com") and prefer_fb_email): while emails_url: response = requests.get(emails_url, headers=headers) response.raise_for_status() @@ -70,7 +73,7 @@ def get_current_user(github_token, prefer_fb_email=True): ( email["email"] for email in response.json() - if email["verified"] and email["email"].endswith("@fb.com") + if email["verified"] and email["email"].endswith("@fb.com") and (email["visibility"] is None or email["visibility"].lower() == "public") ), None, ) @@ -79,6 +82,17 @@ def get_current_user(github_token, prefer_fb_email=True): break else: emails_url = response.links.get("next", {}).get("url") + if default_email is None: + default_email = next( + ( + email["email"] + for email in response.json() + if email["verified"] and (email["visibility"] is None or email["visibility"].lower() == "public") + ), + None, + ) + if ret["email"] is None and not default_email is None: + ret["email"] = default_email return ret diff --git a/windows_cxx_support.txt b/windows_cxx_support.txt index 4ab3bf24810..4b5e3630b88 100644 --- a/windows_cxx_support.txt +++ b/windows_cxx_support.txt @@ -4,3 +4,7 @@ !com.facebook.buck.cxx.PrecompiledHeaderFeatureTest !com.facebook.buck.cxx.PrecompiledHeaderFeatureTest.* !com.facebook.buck.cxx.CxxLinkableEnhancerTest#createCxxLinkableBuildRuleExecutableVsShared +!com.facebook.buck.cxx.CxxRawHeadersIntegrationTest +!com.facebook.buck.features.go.GoBinaryIntegrationTest +!com.facebook.buck.features.go.GoTestIntegrationTest +!com.facebook.buck.features.go.GoProjectIntegrationTest diff --git a/windows_failures.txt b/windows_failures.txt index 8cb4ba4e6ba..515a46a9582 100644 --- a/windows_failures.txt +++ b/windows_failures.txt @@ -31,3 +31,8 @@ !com.facebook.buck.core.build.engine.impl.CachingBuildEngineTest !com.facebook.buck.core.build.engine.impl.CachingBuildEngineTest.* !com.facebook.buck.core.model.graph.action.computation.ActionGraphCacheIntegrationTest +!com.facebook.buck.cli.bootstrapper.class_loader_test.ClassLoaderTest +!com.facebook.buck.features.python.PexStepTest +!com.facebook.buck.maven.ResolverIntegrationTest +!com.facebook.buck.features.python.PythonBinaryIntegrationTest +!com.facebook.buck.rules.modern.builders.HybridLocalStrategyTest#testJobsGetStolenConsistently