test-bot: shard dependent runs across runners

GunniBusch · GunniBusch · commit fe25bbf6a5de · 2026-03-31T03:26:16.000+02:00
diff --git a/Library/Homebrew/github_runner_matrix.rb b/Library/Homebrew/github_runner_matrix.rb
@@ -64,28 +64,40 @@ def initialize(testing_formulae, deleted_formulae, all_supported:, dependent_mat
     @dependent_matrix = dependent_matrix
     @compatible_testing_formulae = T.let({}, T::Hash[GitHubRunner, T::Array[TestRunnerFormula]])
     @formulae_with_untested_dependents = T.let({}, T::Hash[GitHubRunner, T::Array[TestRunnerFormula]])
-
+    @compatible_untested_dependent_names = T.let({}, T::Hash[GitHubRunner, T::Array[String]])
     @runners = T.let([], T::Array[GitHubRunner])
     generate_runners!
 
     freeze
   end
 
-  sig { returns(T::Array[RunnerSpecHash]) }
+  sig { returns(T::Array[T::Hash[Symbol, T.untyped]]) }
   def active_runner_specs_hash
-    runners.select(&:active)
-           .map(&:spec)
-           .map(&:to_h)
+    runners.filter(&:active).flat_map do |r|
+      Array.new(shard_count = selected_runner_count_for(r)) do |i|
+        (spec = r.spec.to_h).merge(
+          name:        (shard_count > 1) ? "#{spec[:name]} (shard #{i + 1}/#{shard_count})" : spec[:name],
+          shard_index: i,
+          shard_total: shard_count,
+        )
+      end
+    end
   end
 
   private
 
   SELF_HOSTED_LINUX_RUNNER = "linux-self-hosted-1"
+  DEPS_SHARDING_ENV = "HOMEBREW_DEPS_SHARDING"
+  DEPS_SHARD_MAX_RUNNERS_ENV = "HOMEBREW_DEPS_SHARD_MAX_RUNNERS"
+  DEPS_SHARD_BASE_THRESHOLD_ENV = "HOMEBREW_DEPS_SHARD_BASE_THRESHOLD"
+  DEPS_SHARD_RUNNER_PENALTY_ENV = "HOMEBREW_DEPS_SHARD_RUNNER_PENALTY"
   # ARM macOS timeout, keep this under 1/2 of GitHub's job execution time limit for self-hosted runners.
   # https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners/about-self-hosted-runners#usage-limits
   GITHUB_ACTIONS_LONG_TIMEOUT = 2160 # 36 hours
   GITHUB_ACTIONS_SHORT_TIMEOUT = 60
-  private_constant :SELF_HOSTED_LINUX_RUNNER, :GITHUB_ACTIONS_LONG_TIMEOUT, :GITHUB_ACTIONS_SHORT_TIMEOUT
+  private_constant :SELF_HOSTED_LINUX_RUNNER, :DEPS_SHARDING_ENV, :DEPS_SHARD_MAX_RUNNERS_ENV,
+                   :DEPS_SHARD_BASE_THRESHOLD_ENV, :DEPS_SHARD_RUNNER_PENALTY_ENV,
+                   :GITHUB_ACTIONS_LONG_TIMEOUT, :GITHUB_ACTIONS_SHORT_TIMEOUT
 
   sig { params(arch: Symbol).returns(LinuxRunnerSpec) }
   def linux_runner_spec(arch)
@@ -261,6 +273,62 @@ def generate_runners!
     @runners.freeze
   end
 
+  sig { params(runner: GitHubRunner).returns(Integer) }
+  def selected_runner_count_for(runner)
+    return 1 if !@dependent_matrix || %w[1 true].exclude?(ENV.fetch(DEPS_SHARDING_ENV, "false").downcase)
+
+    max_available_runners = T.must(sharding_integer_env(DEPS_SHARD_MAX_RUNNERS_ENV, runner, 1))
+    base_spillover_threshold = sharding_integer_env(DEPS_SHARD_BASE_THRESHOLD_ENV, runner, nil)
+    additional_runner_reluctance = sharding_integer_env(DEPS_SHARD_RUNNER_PENALTY_ENV, runner, nil)
+    raise ArgumentError, "#{DEPS_SHARD_MAX_RUNNERS_ENV} must be positive" unless max_available_runners.positive?
+    raise ArgumentError, "#{DEPS_SHARD_BASE_THRESHOLD_ENV} must be positive" if base_spillover_threshold&.<= 0
+
+    if additional_runner_reluctance&.negative?
+      raise ArgumentError,
+            "#{DEPS_SHARD_RUNNER_PENALTY_ENV} must be non-negative"
+    end
+    return 1 if max_available_runners <= 1 || base_spillover_threshold.nil? || additional_runner_reluctance.nil?
+
+    dependent_names = @compatible_untested_dependent_names[runner] ||= compatible_testing_formulae(runner)
+                                                                       .flat_map do |formula|
+      compatible_untested_dependent_names_for_formula(
+        formula, runner
+      )
+    end.uniq.sort
+    dependent_count = dependent_names.count
+    return 1 if dependent_count.zero?
+
+    runner_count = if additional_runner_reluctance.zero?
+      dependent_count.to_f / base_spillover_threshold
+    else
+      threshold_difference = base_spillover_threshold - additional_runner_reluctance
+      discriminant = (threshold_difference**2) + (4 * additional_runner_reluctance * dependent_count)
+      (Math.sqrt(discriminant.to_f) - threshold_difference) / (2 * additional_runner_reluctance)
+    end
+
+    runner_count.ceil.clamp(1, max_available_runners)
+  end
+
+  sig { params(base_env_name: String, runner: GitHubRunner, default: T.nilable(Integer)).returns(T.nilable(Integer)) }
+  def sharding_integer_env(base_env_name, runner, default)
+    platform = runner.platform.to_s.upcase
+    arch = runner.arch.to_s.upcase
+    version = runner.macos_version&.to_sym
+    version = version.to_s.upcase if version
+
+    [
+      ("#{base_env_name}_#{platform}_#{arch}_#{version}" if version),
+      "#{base_env_name}_#{platform}_#{arch}",
+      "#{base_env_name}_#{platform}",
+      base_env_name,
+    ].compact.each do |env_name|
+      env_value = ENV.fetch(env_name, nil)
+      return Integer(env_value, 10) if env_value.present?
+    end
+
+    default
+  end
+
   sig { params(runner: GitHubRunner).returns(T::Array[String]) }
   def testable_formulae(runner)
     formulae = if @dependent_matrix
@@ -301,27 +369,30 @@ def compatible_testing_formulae(runner)
 
   sig { params(runner: GitHubRunner).returns(T::Array[TestRunnerFormula]) }
   def formulae_with_untested_dependents(runner)
-    @formulae_with_untested_dependents[runner] ||= begin
-      platform = runner.platform
-      arch = runner.arch
-      macos_version = runner.macos_version
+    @formulae_with_untested_dependents[runner] ||= compatible_testing_formulae(runner).select do |formula|
+      compatible_untested_dependent_names_for_formula(formula, runner).present?
+    end
+  end
 
-      compatible_testing_formulae(runner).select do |formula|
-        compatible_dependents = formula.dependents(platform:, arch:, macos_version: macos_version&.to_sym)
-                                       .select do |dependent_f|
-          Homebrew::SimulateSystem.with(os: platform, arch: Homebrew::SimulateSystem.arch_symbols.fetch(arch)) do
-            simulated_dependent_f = TestRunnerFormula.new(Formulary.factory(dependent_f.name))
-            next false if macos_version && !simulated_dependent_f.compatible_with?(macos_version)
+  sig { params(formula: TestRunnerFormula, runner: GitHubRunner).returns(T::Array[String]) }
+  def compatible_untested_dependent_names_for_formula(formula, runner)
+    platform = runner.platform
+    arch = runner.arch
+    macos_version = runner.macos_version
 
-            simulated_dependent_f.public_send(:"#{platform}_compatible?") &&
-              simulated_dependent_f.public_send(:"#{arch}_compatible?")
-          end
-        end
+    compatible_dependents = formula.dependents(platform:, arch:, macos_version: macos_version&.to_sym)
+                                   .select do |dependent_f|
+      Homebrew::SimulateSystem.with(os: platform, arch: Homebrew::SimulateSystem.arch_symbols.fetch(arch)) do
+        simulated_dependent_f = TestRunnerFormula.new(Formulary.factory(dependent_f.name))
+        next false if macos_version && !simulated_dependent_f.compatible_with?(macos_version)
 
-        # These arrays will generally have been generated by different Formulary caches,
-        # so we can only compare them by name and not directly.
-        (compatible_dependents.map(&:name) - @testing_formulae.map(&:name)).present?
+        simulated_dependent_f.public_send(:"#{platform}_compatible?") &&
+          simulated_dependent_f.public_send(:"#{arch}_compatible?")
       end
     end
+
+    # These arrays will generally have been generated by different Formulary caches,
+    # so we can only compare them by name and not directly.
+    compatible_dependents.map(&:name) - @testing_formulae.map(&:name)
   end
 end
diff --git a/Library/Homebrew/test/github_runner_matrix_spec.rb b/Library/Homebrew/test/github_runner_matrix_spec.rb
@@ -292,6 +292,34 @@
               matrix = described_class.new([testball], ["deleted"], all_supported: false, dependent_matrix: true)
               expect(get_runner_names(matrix)).to eq(["Linux x86_64"])
             end
+
+            it "shards dependent runners using the most specific sharding settings" do
+              allow(Homebrew::EnvConfig).to receive(:eval_all?).and_return(true)
+              allow(Formula).to receive(:all).and_return([
+                testball,
+                testball_depender_linux,
+                setup_test_runner_formula("testball-depender-linux-two", ["testball", :linux]),
+              ].map(&:formula))
+              allow(ENV).to receive(:[]).with("HOMEBREW_LINUX_RUNNER").and_return("linux-self-hosted-1")
+              env_fetch_overrides = {
+                ["HOMEBREW_DEPS_SHARDING", "false"]                      => "1",
+                ["HOMEBREW_DEPS_SHARD_BASE_THRESHOLD_LINUX_X86_64", nil] => "1",
+                ["HOMEBREW_DEPS_SHARD_BASE_THRESHOLD_LINUX", nil]        => "10",
+                ["HOMEBREW_DEPS_SHARD_RUNNER_PENALTY", nil]              => "0",
+                ["HOMEBREW_DEPS_SHARD_MAX_RUNNERS_LINUX", nil]           => "2",
+              }
+              allow(ENV).to receive(:fetch).and_wrap_original do |original, key, default = nil|
+                if env_fetch_overrides.key?([key, default])
+                  env_fetch_overrides[[key, default]]
+                else
+                  original.call(key, default)
+                end
+              end
+              matrix = described_class.new([testball], [], all_supported: false, dependent_matrix: true)
+
+              expect(matrix.active_runner_specs_hash.map { |spec| spec.fetch(:name) })
+                .to eq(["Linux x86_64 (shard 1/2)", "Linux x86_64 (shard 2/2)"])
+            end
           end
 
           context "when dependent formulae require macOS" do
diff --git a/Library/Homebrew/test/test_bot/formulae_spec.rb b/Library/Homebrew/test/test_bot/formulae_spec.rb
@@ -46,4 +46,12 @@
       end
     end
   end
+
+  it "skips handled formulae from the assigned shard" do
+    deps = Homebrew::TestBot::FormulaeDependents.new(tap: nil, git: "git", dry_run: true, fail_fast: false,
+                                                     verbose: false)
+    deps.instance_variable_set(:@assigned_dependent_formula_names, Set["alpha"])
+    deps.instance_variable_set(:@handled_dependent_formula_names, Set["alpha"])
+    expect(deps.send(:sharded_dependents, [instance_double(Formula, full_name: "alpha")])).to eq([])
+  end
 end
diff --git a/Library/Homebrew/test_bot/formulae_dependents.rb b/Library/Homebrew/test_bot/formulae_dependents.rb
@@ -4,6 +4,8 @@
 module Homebrew
   module TestBot
     class FormulaeDependents < TestFormulae
+      DEPS_SHARD_INDEX_ENV = "HOMEBREW_DEPS_SHARD_INDEX"
+      DEPS_SHARD_TOTAL_ENV = "HOMEBREW_DEPS_SHARD_TOTAL"
       sig { params(testing_formulae: T::Array[String]).returns(T::Array[String]) }
       attr_writer :testing_formulae
 
@@ -24,6 +26,8 @@ def initialize(tap:, git:, dry_run:, fail_fast:, verbose:)
         @testing_formulae_with_tested_dependents = T.let([], T::Array[String])
         @tested_dependents_list = T.let(nil, T.nilable(Pathname))
         @dependent_testing_formulae = T.let([], T::Array[String])
+        @assigned_dependent_formula_names = T.let(nil, T.nilable(T::Set[String]))
+        @handled_dependent_formula_names = T.let(Set.new, T::Set[String])
       end
 
       sig { params(args: Homebrew::Cmd::TestBotCmd::Args).void }
@@ -41,6 +45,7 @@ def run!(args:)
         @tested_dependents_list = Pathname("tested-dependents-#{Utils::Bottles.tag}.txt")
 
         @dependent_testing_formulae = sorted_formulae - skipped_or_failed_formulae
+        configure_dependent_sharding!(args:)
 
         install_formulae_if_needed_from_bottles!(installable_bottles, args:)
 
@@ -145,33 +150,41 @@ def dependent_formulae!(formula_name, args:)
         bottled_dependents.each do |dependent|
           install_dependent(dependent, testable_dependents, args:)
         end
+        return if @assigned_dependent_formula_names.nil?
+
+        (source_dependents + bottled_dependents).each do |dependent|
+          @handled_dependent_formula_names.add(dependent.full_name)
+        end
       end
 
       sig {
-        params(formula: Formula, formula_name: String, args: Homebrew::Cmd::TestBotCmd::Args)
-          .returns([T::Array[Formula], T::Array[Formula], T::Array[Formula]])
+        params(formula: Formula, args: Homebrew::Cmd::TestBotCmd::Args)
+          .returns(T::Boolean)
       }
-      def dependents_for_formula(formula, formula_name, args:)
-        info_header "Determining dependents..."
-
+      def skip_recursive_dependents_for(formula, args:)
         # Always skip recursive dependents on Intel. It's really slow.
         # Also skip recursive dependents on Linux unless it's a Linux-only formula.
         #
         # TODO: move to extend/os
         # rubocop:todo Homebrew/MoveToExtendOS
-        skip_recursive_dependents = args.skip_recursive_dependents? ||
-                                    (OS.mac? && Hardware::CPU.intel?) ||
-                                    (OS.linux? && formula.requirements.exclude?(LinuxRequirement.new))
+        args.skip_recursive_dependents? ||
+          (OS.mac? && Hardware::CPU.intel?) ||
+          (OS.linux? && formula.requirements.exclude?(LinuxRequirement.new))
         # rubocop:enable Homebrew/MoveToExtendOS
+      end
 
+      sig {
+        params(formula_name: String, skip_recursive_dependents: T::Boolean)
+          .returns(T::Array[String])
+      }
+      def dependent_formula_names(formula_name, skip_recursive_dependents:)
         uses_args = %w[--formula --eval-all]
         uses_include_test_args = [*uses_args, "--include-test"]
         uses_include_test_args << "--recursive" unless skip_recursive_dependents
         dependents = with_env(HOMEBREW_STDERR: "1") do
           Utils.safe_popen_read("brew", "uses", *uses_include_test_args, formula_name)
                .split("\n")
         end
-
         # TODO: Consider handling the following case better.
         #       `foo` has a build dependency on `bar`, and `bar` has a runtime dependency on
         #       `baz`. When testing `baz` with `--build-dependents-from-source`, `foo` is
@@ -182,9 +195,21 @@ def dependents_for_formula(formula, formula_name, args:)
         end
         dependents.uniq!
         dependents.sort!
+        dependents
+      end
 
+      sig {
+        params(formula: Formula, formula_name: String, args: Homebrew::Cmd::TestBotCmd::Args)
+          .returns([T::Array[Formula], T::Array[Formula], T::Array[Formula]])
+      }
+      def dependents_for_formula(formula, formula_name, args:)
+        info_header "Determining dependents..."
+
+        skip_recursive_dependents = skip_recursive_dependents_for(formula, args:)
+        dependents = dependent_formula_names(formula_name, skip_recursive_dependents:)
         dependents -= @tested_formulae
         dependents = dependents.map { |d| Formulary.factory(d) }
+        dependents = sharded_dependents(dependents)
 
         dependents = dependents.zip(dependents.map do |f|
           if skip_recursive_dependents
@@ -256,6 +281,41 @@ def dependents_for_formula(formula, formula_name, args:)
         [source_dependents, bottled_dependents, testable_dependents]
       end
 
+      sig { params(args: Homebrew::Cmd::TestBotCmd::Args).void }
+      def configure_dependent_sharding!(args:)
+        shard_index = (env_value = ENV.fetch(DEPS_SHARD_INDEX_ENV, nil)).blank? ? 0 : Integer(env_value, 10)
+        shard_total = (env_value = ENV.fetch(DEPS_SHARD_TOTAL_ENV, nil)).blank? ? 1 : Integer(env_value, 10)
+        raise ArgumentError, "#{DEPS_SHARD_TOTAL_ENV} must be positive" unless shard_total.positive?
+        raise ArgumentError, "#{DEPS_SHARD_INDEX_ENV} must be between 0 and #{shard_total - 1}" unless
+          (0...shard_total).cover?(shard_index)
+
+        @assigned_dependent_formula_names = nil
+        @handled_dependent_formula_names.clear
+        return if shard_total == 1
+
+        all_dependent_formula_names = @dependent_testing_formulae.flat_map do |formula_name|
+          formula = Formulary.factory(formula_name)
+          skip_recursive_dependents = skip_recursive_dependents_for(formula, args:)
+          dependent_formula_names(formula_name, skip_recursive_dependents:)
+        end.uniq.sort
+        @assigned_dependent_formula_names = Set.new(
+          all_dependent_formula_names.each_with_index.filter_map do |name, position|
+            name if position % shard_total == shard_index
+          end,
+        )
+      end
+
+      sig { params(dependents: T::Array[Formula]).returns(T::Array[Formula]) }
+      def sharded_dependents(dependents)
+        assigned_dependent_formula_names = @assigned_dependent_formula_names
+        return dependents if assigned_dependent_formula_names.nil?
+
+        dependents.select do |dependent|
+          assigned_dependent_formula_names.include?(dependent.full_name) &&
+            @handled_dependent_formula_names.exclude?(dependent.full_name)
+        end
+      end
+
       sig {
         params(
           dependent:           Formula,
