csharp-metadata-example

Program.cs

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Threading.Tasks;
using Braintrust.Sdk;
using Braintrust.Sdk.Eval;
using Braintrust.Sdk.Instrumentation.OpenAI;
using OpenAI;
using OpenAI.Chat;

namespace Braintrust.TestApps;

/// <summary>
/// A scorer that returns a base score of 0.5, adjusted by any "weight" value in metadata.
/// </summary>
public class WeightedScorer : IScorer<string, string>
{
    public string Name => "weighted_score";

    public IReadOnlyList<Score> Score(TaskResult<string, string> taskResult)
    {
        double score = 0.5;

        // Check if there's a "weight" key in metadata and adjust the score
        if (taskResult.DatasetCase.Metadata.TryGetValue("weight", out var weightValue))
        {
            if (weightValue is double weight)
            {
                score += weight;
            }
            else if (weightValue is int weightInt)
            {
                score += weightInt;
            }
            else if (double.TryParse(weightValue?.ToString(), out var parsed))
            {
                score += parsed;
            }
        }

        // Clamp score to [0, 1] range
        score = Math.Max(0.0, Math.Min(1.0, score));

        return [new Score(Name, score)];
    }
}

public class Program
{
    public static async Task Main(string[] args)
    {
        try
        {
            var braintrust = Sdk.Braintrust.Get();
            var activitySource = braintrust.GetActivitySource();

            // Wrap OpenAI client for automatic tracing
            var openAIApiKey = Environment.GetEnvironmentVariable("OPENAI_API_KEY");
            if (string.IsNullOrEmpty(openAIApiKey))
            {
                Console.WriteLine("ERROR: OPENAI_API_KEY environment variable not set. Bailing.");
                return;
            }

            OpenAIClient openAIClient = BraintrustOpenAI.WrapOpenAI(activitySource, openAIApiKey);

            // Eval
            string GreetingTask(string input)
            {
                return $"Hello {input}";
            }

            var eval = await braintrust
                .EvalBuilder<string, string>()
                .Name($"dotnet-test-eval-{DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()}")
                .Cases(
                    // Case without weight metadata - will score 0.5
                    DatasetCase.Of("World", "Hello World"),
                    // Case with weight metadata - will score 0.5 + 0.2 = 0.7
                    DatasetCase.Of(
                        "Alice",
                        "Hello Alice",
                        new List<string>(),
                        new Dictionary<string, object> { { "weight", 0.2 } }
                    ),
                    // Case with higher weight - will score 0.5 + 0.4 = 0.9
                    DatasetCase.Of(
                        "Bob",
                        "Hello Bob",
                        new List<string> { "vip" },
                        new Dictionary<string, object> { { "weight", 0.4 }, { "priority", "high" } }
                    )
                )
                .TaskFunction(GreetingTask)
                .Scorers(
                    new FunctionScorer<string, string>(
                        "exact_match",
                        (expected, actual) => expected == actual ? 1.0 : 0.0
                    ),
                    new WeightedScorer()
                )
                .BuildAsync();

            var result = await eval.RunAsync();
            Console.WriteLine($"\n\n{result.CreateReportString()}");
        }
        catch (Exception e)
        {
            Console.Error.WriteLine($"Error: {e.Message}");
            Console.Error.WriteLine(e.StackTrace);
        }
    }

    private static async Task CallOpenAI(OpenAIClient openAIClient)
    {
        var chatClient = openAIClient.GetChatClient("gpt-3.5-turbo");

        var messages = new ChatMessage[]
        {
            new UserChatMessage("Say hello from OpenAI!")
        };

        var options = new ChatCompletionOptions
        {
            MaxOutputTokenCount = 50
        };

        await chatClient.CompleteChatAsync(messages, options);
    }
}