Experiment analysis using bayesian techniques

bayesian_experiment_analysis_v2.py

from numpy import mean, quantile
from scipy.stats import beta

################# Parameters #################
control_successes = 10
control_failures = 20
treatment_successes = 40
treatment_failures = 23
##############################################

control_distribution_values = beta.rvs(control_successes, control_failures, size=100000)
treatment_distribution_values = beta.rvs(
    treatment_successes, treatment_failures, size=100000
)

differences = (treatment_distribution_values - control_distribution_values) * 100

mean_difference = mean(differences)
absolute_mean_difference = round(abs(mean_difference), 1)

upper_bound_error = mean_difference - quantile(differences, 0.975)
lower_bound_error = mean_difference - quantile(differences, 0.025)
average_abs_error = round((abs(upper_bound_error) + abs(lower_bound_error)) / 2, 1)

proportion_positive = round(
    (len(differences[differences > 0]) / len(differences)) * 100, 1
)

# Override 100%, never truly 100% sure
if proportion_positive == 100.0:
    proportion_positive = 99.9

# Summarizing the outcome
print("Recommendation:")

if proportion_positive > 98:
    print("The new version won! We recommend you install the new version.")
elif proportion_positive > 90:
    print(
        """
- High risk experiments: We recommend you wait longer to see if the chances improve.  Or, if you've already waited awhile, end the experiment and keep the old version.
- Medium risk experiments: The new version won! We recommend you install the new version.
- Low risk/progress blocking experiments: The new version won! We recommend you install the new version.
        """
    )
elif proportion_positive > 70:
    print(
        """
- High risk experiments: We recommend you wait longer to see if the chances improve.  Or, if you've already waited awhile, end the experiment and keep the old version.
- Medium risk experiments: We recommend you wait longer to see if the chances improve.  Or, if you've already waited awhile, end the experiment and keep the old version.
- Low risk/progress blocking experiments: We (probably) recommend you install the new version. But if you're not in a rush, waiting longer would improve your chances of making the right decision.
        """
    )
else:
    print(
        """
We recommend you wait longer to see if the chances improve. \
Or, if you've already waited awhile, end the experiment and keep the old version.
        """
    )

# Analysis details
print("\n\nAnalysis details")
print(
    f"""
There is approximately a {proportion_positive}% chance that the new version will \
have a higher conversion rate than the control.
    """
)

if mean_difference > 0:
    print(
        f"""
The new version's conversion rate is {absolute_mean_difference}% higher \
than the control's conversion rate (+/- {average_abs_error}%).
        """
    )
else:
    print(
        f"""
The new version's conversion rate is {absolute_mean_difference}% lower \
than the control's conversion rate (+/- {average_abs_error}%).
        """
    )