merge master into branch

Author: mattip

.bazelrc

@@ -1,15 +1,11 @@
 # Must be first. Enables build:windows, build:linux, build:macos, build:freebsd, build:openbsd
 build --enable_platform_specific_config
 
-build:linux --workspace_status_command="bash ./bazel/workspace_status.sh"
-
-# Provides users an option to turn on strict action env.
-# TODO(aslonnie): make this default, fix the python tests.
-# NOTE(edoakes): enable this by default locally by adding a .user.bazelrc file with:
-# build --config=strict
-# test --config=strict
+build --incompatible_strict_action_env
 build:strict --incompatible_strict_action_env
 
+build:linux --workspace_status_command="bash ./bazel/workspace_status.sh"
+
 # To distinguish different incompatible environments.
 build --action_env=RAY_BUILD_ENV
 
@@ -84,8 +80,6 @@ build:iwyu --output_groups=report
 build:windows --attempt_to_print_relative_paths
 # Save disk space by hardlinking cache hits instead of copying
 build:windows --experimental_repository_cache_hardlinks
-# Clean the environment before building, to make builds more deterministic
-build:windows --incompatible_strict_action_env
 # For colored output (seems necessary on Windows)
 build:windows --color=yes
 # For compiler colored output (seems necessary on Windows)
@@ -168,6 +162,18 @@ test:ci-base --test_output=errors
 test:ci-base --test_verbose_timeout_warnings
 test:ci-base --flaky_test_attempts=3
 
+# Sending in PATH is required for tests to run on CI, after we enable
+# --incompatible_strict_action_env, until we either convert all Python tests to
+# hermetic tests -- which not only requires pinning all Python dependencies with bazel,
+# but also requires building ray(test) wheel with bazel. Alternatively, we can
+# also stop using bazel test to run ray's Python tests.
+#
+# This PATH test_env is intentionally not enabled on non-CI so that C/C++
+# tests, which are all hermetic, can build, test and cache as intended, ray
+# Python developers do not really use bazel test to run tests locally, but more
+# often just run tests with "pytest" directly.
+test:ci-base --test_env=PATH
+
 build:ci --color=yes
 build:ci --curses=no
 build:ci --keep_going

.buildkite/_forge.rayci.yml

@@ -1,186 +1,8 @@
 group: forge
+sort_key: "_forge"
 steps:
   - name: forge
     wanda: ci/docker/forge.wanda.yaml
 
   - name: manylinux
     wanda: ci/docker/manylinux.wanda.yaml
-
-  - name: raycpubase
-    label: "wanda: ray.py{{matrix}}.cpu.base"
-    tags:
-      - python_dependencies
-      - docker
-    wanda: ci/docker/ray.cpu.base.wanda.yaml
-    matrix:
-      - "3.9"
-      - "3.10"
-      - "3.11"
-      - "3.12"
-    env:
-      PYTHON_VERSION: "{{matrix}}"
-      ARCH_SUFFIX: ""
-
-  - name: raycpubaseextra
-    label: "wanda: ray.py{{matrix}}.cpu.base-extra"
-    wanda: ci/docker/ray.cpu.base-extra.wanda.yaml
-    matrix:
-      - "3.9"
-      - "3.10"
-      - "3.11"
-      - "3.12"
-    env:
-      PYTHON_VERSION: "{{matrix}}"
-      IMAGE_TYPE: "ray"
-      ARCH_SUFFIX: ""
-    depends_on: raycpubase
-
-  - name: raycudabase
-    label: "wanda: ray.py{{matrix.python}}.cu{{matrix.cuda}}.base"
-    tags:
-      - python_dependencies
-      - docker
-    wanda: ci/docker/ray.cuda.base.wanda.yaml
-    matrix:
-      setup:
-        python:
-          - "3.9"
-          - "3.10"
-          - "3.11"
-          - "3.12"
-        cuda:
-          - "11.7.1-cudnn8"
-          - "11.8.0-cudnn8"
-          - "12.1.1-cudnn8"
-          - "12.3.2-cudnn9"
-          - "12.4.1-cudnn"
-          - "12.5.1-cudnn"
-          - "12.6.3-cudnn"
-          - "12.8.1-cudnn"
-    env:
-      PYTHON_VERSION: "{{matrix.python}}"
-      CUDA_VERSION: "{{matrix.cuda}}"
-      ARCH_SUFFIX: ""
-
-  - name: raycudabaseextra
-    label: "wanda: ray.py{{matrix.python}}.cu{{matrix.cuda}}.base-extra"
-    wanda: ci/docker/ray.cuda.base-extra.wanda.yaml
-    matrix:
-      setup:
-        python:
-          - "3.9"
-          - "3.10"
-          - "3.11"
-          - "3.12"
-        cuda:
-          - "11.7.1-cudnn8"
-          - "11.8.0-cudnn8"
-          - "12.1.1-cudnn8"
-          - "12.3.2-cudnn9"
-          - "12.4.1-cudnn"
-          - "12.5.1-cudnn"
-          - "12.6.3-cudnn"
-          - "12.8.1-cudnn"
-    env:
-      PYTHON_VERSION: "{{matrix.python}}"
-      CUDA_VERSION: "{{matrix.cuda}}"
-      IMAGE_TYPE: "ray"
-      ARCH_SUFFIX: ""
-    depends_on: raycudabase
-
-  - name: ray-llmbase
-    label: "wanda: ray-llm.py{{matrix.python}}.cu{{matrix.cuda}}.base"
-    tags:
-      - python_dependencies
-      - docker
-    wanda: ci/docker/ray-llm.base.wanda.yaml
-    depends_on: raycudabase
-    matrix:
-      setup:
-        python:
-          - "3.11"
-        cuda:
-          - "12.8.1-cudnn"
-    env:
-      PYTHON_VERSION: "{{matrix.python}}"
-      CUDA_VERSION: "{{matrix.cuda}}"
-
-  - name: ray-llmbaseextra
-    label: "wanda: ray-llm.py{{matrix.python}}.cu{{matrix.cuda}}.base-extra"
-    wanda: ci/docker/ray.cuda.base-extra.wanda.yaml
-    matrix:
-      setup:
-        python:
-          - "3.11"
-        cuda:
-          - "12.8.1-cudnn"
-    env:
-      PYTHON_VERSION: "{{matrix.python}}"
-      CUDA_VERSION: "{{matrix.cuda}}"
-      IMAGE_TYPE: "ray-llm"
-      ARCH_SUFFIX: ""
-    depends_on: ray-llmbase
-
-  - name: ray-mlcudabase
-    label: "wanda: ray-ml.py{{matrix.python}}.cu{{matrix.cuda}}.base"
-    tags:
-      - python_dependencies
-      - docker
-    wanda: ci/docker/ray-ml.cuda.base.wanda.yaml
-    depends_on: raycudabase
-    matrix:
-      setup:
-        python:
-          - "3.9"
-          - "3.10"
-          - "3.11"
-        cuda:
-          - "12.1.1-cudnn8"
-    env:
-      PYTHON_VERSION: "{{matrix.python}}"
-      CUDA_VERSION: "{{matrix.cuda}}"
-
-  - name: ray-mlcudabaseextra
-    label: "wanda: ray-ml.py{{matrix.python}}.cu{{matrix.cuda}}.base-extra"
-    wanda: ci/docker/ray.cuda.base-extra.wanda.yaml
-    matrix:
-      setup:
-        python:
-          - "3.9"
-          - "3.10"
-          - "3.11"
-        cuda:
-          - "12.1.1-cudnn8"
-    env:
-      PYTHON_VERSION: "{{matrix.python}}"
-      CUDA_VERSION: "{{matrix.cuda}}"
-      IMAGE_TYPE: "ray-ml"
-      ARCH_SUFFIX: ""
-    depends_on: ray-mlcudabase
-
-  - name: ray-mlcpubase
-    label: "wanda: ray-ml.py{{matrix}}.cpu.base"
-    tags:
-      - python_dependencies
-      - docker
-    wanda: ci/docker/ray-ml.cpu.base.wanda.yaml
-    depends_on: raycpubase
-    matrix:
-      - "3.9"
-      - "3.10"
-      - "3.11"
-    env:
-      PYTHON_VERSION: "{{matrix}}"
-
-  - name: ray-mlcpubaseextra
-    label: "wanda: ray-ml.py{{matrix}}.cpu.base-extra"
-    wanda: ci/docker/ray.cpu.base-extra.wanda.yaml
-    matrix:
-      - "3.9"
-      - "3.10"
-      - "3.11"
-    env:
-      PYTHON_VERSION: "{{matrix}}"
-      IMAGE_TYPE: "ray-ml"
-      ARCH_SUFFIX: ""
-    depends_on: ray-mlcpubase