Created
October 16, 2025 23:49
-
-
Save elbartostrikesagain/ec06fadecec47bdaf6492d721d60ddf3 to your computer and use it in GitHub Desktop.
Matcher to like perform_under but gets rid of results outside a standard deviation range
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| it "maintains acceptable performance for the index action", skip_db_cleaner: true do # for some reason skip_db_cleaner: true when running the whole test suite wasn't being inherited, so I had to dup it :( . When running the describe, it was being inherited by the 'it' blocks, and once you run it and try to run the whole suite again, it starts passing. | |
| expect { get :index, params: params, format: :json }.to meet_performance_threshold(4.0) | |
| .sample(5) | |
| .under_standard_deviations(2) | |
| # Also see rspec-benchmark. This matcher is like perform_under but it removes results outside a defined standard deviation range reduce high variance. | |
| RSpec::Matchers.define :meet_performance_threshold do |threshold_seconds| | |
| chain :sample do |sample_size| | |
| @with_sample_size = sample_size | |
| end | |
| chain :under_standard_deviations do |deviations| | |
| @std_devs_for_outlier = deviations | |
| end | |
| chain :warmup do |warmup_count| | |
| @warmup_count = warmup_count | |
| end | |
| def supports_block_expectations? | |
| true | |
| end | |
| match do |action_to_benchmark| | |
| raise "expect must take a block or be passed a Proc or Lambda" unless action_to_benchmark.is_a?(Proc) | |
| @with_sample_size ||= 5 | |
| @warmup_count ||= 1 | |
| raise "under_standard_deviations must be provided and larger than 0 otherwise you might as well use perform_under from rspec-benchmark" if @std_devs_for_outlier.blank? || @std_devs_for_outlier <= 0 | |
| raise "with_sample_size must at least be 1" if @with_sample_size < 1 | |
| @times = (@warmup_count + @with_sample_size).times.map do | |
| Benchmark.measure { action_to_benchmark.call }.real | |
| end | |
| @times = @times.drop(@warmup_count) | |
| initial_avg = @times.sum / @times.size.to_f | |
| sum_of_squares = @times.map { |x| (x - initial_avg)**2 }.sum | |
| std_dev = Math.sqrt(sum_of_squares / @times.size) | |
| outlier_threshold = initial_avg + (@std_devs_for_outlier * std_dev) | |
| @filtered_times = @times.reject { |t| t > outlier_threshold } | |
| puts "filtered out #{(@times - @filtered_times).size} outlier(s) from #{@times}" | |
| # If all runs were outliers, fall back to the initial set of times. | |
| raise "All code execution times were outside of #{@std_devs_for_outlier} standard deviations. " if @filtered_times.empty? | |
| final_times = @filtered_times.empty? ? @times : @filtered_times | |
| @avg_time = final_times.sum / final_times.size.to_f | |
| @avg_time < threshold_seconds | |
| end | |
| failure_message do |_| | |
| "expected average time to be < #{threshold_seconds}s, but was #{@avg_time.round(4)}s.\n" \ | |
| "Runs: #{@times.map { |t| t.round(4) }.join("s, ")}s" | |
| end | |
| failure_message_when_negated do |_| | |
| "expected average time not to be < #{threshold_seconds}s, but was #{@avg_time.round(4)}s" | |
| end | |
| description do | |
| "perform under #{threshold_seconds}s on average" | |
| end | |
| end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment