flaky_test_bisector.rb

# Usage:
#   FLAKY_TEST="ClassNameTest#test_method_name" bin/rails test
#
# Make sure to have this file required before any tests are run, e.g., by placing it
# at the top of the test/test_helper.rb file.

if ENV["FLAKY_TEST"]

  ENV["SINGLE"] = "true"  # Disable parallel test execution

  module Minitest
    def self.__run(reporter, options)
      flaky = ENV["FLAKY_TEST"]
      suites = Runnable.runnables
      tests = suites.flat_map { |suite| suite.runnable_methods.map { |method| [ suite, method ] } }

      flaky, suspects = tests.partition { |suite, method| "#{suite.inspect}##{method}" == flaky }

      guilty = bisect(suspects, flaky)

      if guilty.empty?
        puts
        puts "No guilty tests found"
      else
        puts
        puts "Found some guilty tests"
        guilty.each { |suite, method| puts " - #{suite.inspect}##{method}" }

        puts
        puts "Try running this command to confirm:"
        puts
        print "    bin/rails test "
        guilty.each { |suite, method| print "\"#{to_arg(suite, method)}\" "  }
        flaky.each { |suite, method| print "\"#{to_arg(suite, method)}\" "  }
        print " --verbose"
        puts
        puts
      end
    end

    private

      def self.bisect(suspects, flaky)
        return [] if suspects.empty?

        puts "Bisecting #{suspects.size} suspect(s)"

        if suspects.size == 1
          return causes_flaky_failure(suspects, flaky) ? suspects : []
        end

        mid = suspects.size / 2
        left = suspects.slice(...mid)
        right = suspects.slice(mid..)

        if causes_flaky_failure(left, flaky)
          result = bisect(left, flaky)
          result.size > 0 ? result : left
        elsif causes_flaky_failure(right, flaky)
          result = bisect(right, flaky)
          result.size > 0 ? result : right
        else
          []
        end
      end

      def self.causes_flaky_failure(suspects, flaky)
        run_tests(suspects)
        results = run_tests(flaky)
        puts "\n"
        !results.all?(&:passed?)
      end

      def self.run_tests(tests)
        tests.map do |suite, method|
          result = Minitest.run_one_method(suite, method)
          putc result.result_code
          result
        end
      end

      def self.to_arg(suite, method)
        current_dir = Dir.pwd
        file, line = suite.new(method).method(method.to_sym).source_location
        "#{file}:#{line}".sub("#{current_dir}/", "")
      end
  end

end

jinx

      def self.linear_search(suspects, flaky)
        return nil if suspects.empty?

        puts "Linear search through #{suspects.size} suspects"

        zipped = suspects.flat_map { |sus| [ sus, flaky ] }

        until suspects.empty? || guilty
          prime_suspect = suspects.pop
          guilty = prime_suspect if causes_flaky_failure([ prime_suspect ], flaky, separator: " ", print: false)
          searched += 1
          puts "#{searched}/#{search_size} Cleared #{prime_suspect}"
        end

        guilty || []
      end