position_biais.py

from transformers import pipeline
from string import Template

template = Template(
    """Please act as an impartial judge and evaluate the quality of the responses provided by
two AI assistants to the user question displayed below. Your evaluation should consider
factors such as the helpfulness and relevance. Ensure that the order in which the responses
were presented does not influence your decision. Answer just by [[A]] if assistant A is better,
[[B]] if assistant B is better, and [[C]] for a tie.

[User Question]
What are the benefits of regular exercise?

[Assistant A's Answer]
$answer_a

[Assistant B's Answer]
$answer_b

[Judge verdict]
"""
)

answer_a = "Regular exercise helps improve cardiovascular health, maintain a healthy weight, and boost mental well-being."
answer_b = "Engaging in regular physical activity can increase energy levels, enhance mood, and reduce the risk of chronic diseases."
pipe = pipeline(model="mistralai/Mistral-7B-Instruct-v0.3", task="text-generation")

pipe(template.substitute(answer_a=answer_a, answer_b=answer_b), max_new_tokens=3)  # [[A]]
pipe(template.substitute(answer_a=answer_b, answer_b=answer_a), max_new_tokens=3)  # [[A]]