Custom code evaluators

import { Eval, type EvalScorer } from "braintrust";
import OpenAI from "openai";

const client = new OpenAI();

const DATASET = [
  {
    input: "What is 2+2?",
    expected: "4",
  },
  {
    input: "What is the capital of France?",
    expected: "Paris",
  },
];

async function task(input: string): Promise<string> {
  const response = await client.responses.create({
    model: "gpt-5-mini",
    input: [
      { role: "user", content: input },
    ],
  });
  return response.output_text ?? "";
}

const equalityScorer: EvalScorer<string, string, string> = ({ output, expected }) => {
  if (!expected) return null;
  const matches = output === expected;
  return {
    name: "Equality",
    score: matches ? 1 : 0,
    metadata: { exact_match: matches },
  };
};

const containsScorer: EvalScorer<string, string, string> = ({ output, expected }) => {
  if (!expected) return null;
  const contains = output.toLowerCase().includes(expected.toLowerCase());
  return {
    name: "Contains expected",
    score: contains ? 1 : 0,
  };
};

Eval("Custom Code Scorer Example", {
  data: DATASET,
  task,
  scores: [equalityScorer, containsScorer],
});

from braintrust import Eval
from openai import OpenAI

client = OpenAI()

DATASET = [
    {
        "input": "What is 2+2?",
        "expected": "4",
    },
    {
        "input": "What is the capital of France?",
        "expected": "Paris",
    },
]


def task(input):
    response = client.responses.create(
        model="gpt-5-mini",
        input=[
            {"role": "user", "content": input},
        ],
    )
    return response.output_text


def equality_scorer(input, output, expected, metadata):
    if not expected:
        return None
    matches = output == expected
    return {
        "name": "Equality",
        "score": 1 if matches else 0,
        "metadata": {"exact_match": matches},
    }


def contains_scorer(input, output, expected, metadata):
    if not expected:
        return None
    contains = expected.lower() in output.lower()
    return {
        "name": "Contains expected",
        "score": 1 if contains else 0,
    }


Eval(
    "Custom Code Scorer Example",
    data=DATASET,
    task=task,
    scores=[equality_scorer, contains_scorer],
)

require "braintrust"
require "openai"

Braintrust.init

client = OpenAI::Client.new(api_key: ENV.fetch("OPENAI_API_KEY", nil))

DATASET = [
  {input: "What is 2+2?", expected: "4"},
  {input: "What is the capital of France?", expected: "Paris"},
]

equality_scorer = Braintrust::Scorer.new("equality") do |output:, expected:|
  next nil unless expected
  matches = output == expected
  {name: "Equality", score: matches ? 1.0 : 0.0, metadata: {exact_match: matches}}
end

contains_scorer = Braintrust::Scorer.new("contains_expected") do |output:, expected:|
  next nil unless expected
  contains = output.downcase.include?(expected.downcase)
  {name: "Contains expected", score: contains ? 1.0 : 0.0}
end

Braintrust::Eval.run(
  project: "Custom Code Scorer Example",
  cases: DATASET,
  task: lambda do |input:|
    response = client.chat.completions.create(
      model: "gpt-5-mini",
      messages: [{role: "user", content: input}]
    )
    response.choices.first.message.content || ""
  end,
  scorers: [equality_scorer, contains_scorer]
)

OpenTelemetry.tracer_provider.shutdown

using Braintrust.Sdk;
using Braintrust.Sdk.Eval;
using Braintrust.Sdk.OpenAI;
using OpenAI;
using OpenAI.Chat;

sealed class ContainsScorer : IScorer<string, string>
{
    public string Name => "Contains expected";

    public Task<IReadOnlyList<Score>> Score(TaskResult<string, string> taskResult)
    {
        if (taskResult.DatasetCase.Expected is null)
            return Task.FromResult<IReadOnlyList<Score>>([]);

        var contains = taskResult.Result.Contains(
            taskResult.DatasetCase.Expected, StringComparison.OrdinalIgnoreCase);
        return Task.FromResult<IReadOnlyList<Score>>(
            [new Score(Name, contains ? 1.0 : 0.0)]);
    }
}

class Program
{
    static readonly DatasetCase<string, string>[] Dataset =
    [
        DatasetCase.Of("What is 2+2?", "4"),
        DatasetCase.Of("What is the capital of France?", "Paris"),
    ];

    static async Task Main(string[] args)
    {
        var equalityScorer = new FunctionScorer<string, string>(
            "Equality",
            (expected, actual) => actual == expected ? 1.0 : 0.0);

        var braintrust = Braintrust.Sdk.Braintrust.Get();
        var activitySource = braintrust.GetActivitySource();
        var openAIClient = BraintrustOpenAI.WrapOpenAI(
            activitySource, Environment.GetEnvironmentVariable("OPENAI_API_KEY")!);

        async Task<string> Task(string input)
        {
            var response = await openAIClient.GetChatClient("gpt-5-mini")
                .CompleteChatAsync([new UserChatMessage(input)]);
            return response.Value.Content[0].Text;
        }

        var eval = await braintrust
            .EvalBuilder<string, string>()
            .Name("Custom Code Scorer Example")
            .Cases(Dataset)
            .TaskFunction(Task)
            .Scorers(equalityScorer, new ContainsScorer())
            .BuildAsync();

        var result = await eval.RunAsync();
        Console.WriteLine(result.CreateReportString());
    }
}

import { Eval, wrapOpenAI, wrapTraced, type EvalScorer } from "braintrust";
import OpenAI from "openai";

const client = wrapOpenAI(new OpenAI());

const SUPPORT_DATASET = [
  { input: "My order hasn't arrived yet. Order #12345." },
  { input: "I need help resetting my password." },
];

const callLLM = wrapTraced(async function callLLM(messages: Array<{ role: string; content: string }>) {
  const response = await client.chat.completions.create({
    model: "gpt-5-mini",
    messages,
  });
  return response.choices[0].message.content || "";
});

async function supportTask(input: string): Promise<string> {
  const messages: Array<{ role: string; content: string }> = [
    { role: "system", content: "You are a helpful customer support agent." }
  ];

  messages.push({ role: "user", content: input });
  const response1 = await callLLM(messages);
  messages.push({ role: "assistant", content: response1 });

  messages.push({ role: "user", content: "Can you provide more details?" });
  const response2 = await callLLM(messages);
  messages.push({ role: "assistant", content: response2 });

  messages.push({ role: "user", content: "Thank you for your help!" });
  const response3 = await callLLM(messages);

  return response3;
}

const politenessScorer: EvalScorer<string, string, unknown> = async ({ trace }) => {
  if (!trace) return 0;

  const thread = await trace.getThread();
  const lastAssistantMsg = thread.reverse().find(msg => msg.role === "assistant");
  const content = lastAssistantMsg?.content?.toLowerCase() || "";

  const politeWords = ["welcome", "glad", "happy", "pleasure", "thank"];
  const isPolite = politeWords.some(word => content.includes(word));

  return {
    name: "Politeness",
    score: isPolite ? 1 : 0,
    metadata: { checked_message_preview: content.slice(0, 80) },
  };
};

const efficiencyScorer: EvalScorer<string, string, unknown> = async ({ trace }) => {
  if (!trace) return 0;

  const llmSpans = await trace.getSpans({ spanType: ["llm"] });
  const isEfficient = llmSpans.length >= 3 && llmSpans.length <= 5;

  return {
    name: "Efficiency",
    score: isEfficient ? 1 : 0,
    metadata: { llm_calls: llmSpans.length },
  };
};

Eval("Support Quality", {
  data: SUPPORT_DATASET,
  task: supportTask,
  scores: [politenessScorer, efficiencyScorer],
});

from braintrust import Eval, wrap_openai, traced
from openai import AsyncOpenAI

client = wrap_openai(AsyncOpenAI())

SUPPORT_DATASET = [
    {"input": "My order hasn't arrived yet. Order #12345."},
    {"input": "I need help resetting my password."},
]


@traced
async def call_llm(messages):
    response = await client.chat.completions.create(
        model="gpt-5-mini",
        messages=messages,
    )
    return response.choices[0].message.content or ""


async def support_task(input):
    messages = [
        {"role": "system", "content": "You are a helpful customer support agent."}
    ]

    messages.append({"role": "user", "content": input})
    response1 = await call_llm(messages)
    messages.append({"role": "assistant", "content": response1})

    messages.append({"role": "user", "content": "Can you provide more details?"})
    response2 = await call_llm(messages)
    messages.append({"role": "assistant", "content": response2})

    messages.append({"role": "user", "content": "Thank you for your help!"})
    response3 = await call_llm(messages)

    return response3


async def politeness_scorer(input, output, expected, trace=None):
    if not trace:
        return 0

    thread = await trace.get_thread()
    last_assistant_msg = next(
        (msg for msg in reversed(thread) if msg.get("role") == "assistant"), None
    )
    content = (last_assistant_msg.get("content") or "").lower() if last_assistant_msg else ""

    polite_words = ["welcome", "glad", "happy", "pleasure", "thank"]
    is_polite = any(word in content for word in polite_words)

    return {
        "name": "Politeness",
        "score": 1 if is_polite else 0,
        "metadata": {"checked_message_preview": content[:80]},
    }


async def efficiency_scorer(input, output, expected, trace=None):
    if not trace:
        return 0

    llm_spans = await trace.get_spans(span_type=["llm"])
    is_efficient = 3 <= len(llm_spans) <= 5

    return {
        "name": "Efficiency",
        "score": 1 if is_efficient else 0,
        "metadata": {"llm_calls": len(llm_spans)},
    }


Eval(
    "Support Quality",
    data=SUPPORT_DATASET,
    task=support_task,
    scores=[politeness_scorer, efficiency_scorer],
)

require "braintrust"
require "openai"

Braintrust.init

client = OpenAI::Client.new(api_key: ENV.fetch("OPENAI_API_KEY", nil))

SUPPORT_DATASET = [
  {input: "My order hasn't arrived yet. Order #12345."},
  {input: "I need help resetting my password."},
]

def chat(client, messages)
  client.chat.completions.create(model: "gpt-5-mini", messages: messages)
    .choices.first.message.content || ""
end

support_task = Braintrust::Task.new("support") do |input:|
  messages = [{role: "system", content: "You are a helpful customer support agent."}]

  messages << {role: "user", content: input}
  messages << {role: "assistant", content: chat(client, messages)}

  messages << {role: "user", content: "Can you provide more details?"}
  messages << {role: "assistant", content: chat(client, messages)}

  messages << {role: "user", content: "Thank you for your help!"}
  chat(client, messages)
end

politeness_scorer = Braintrust::Scorer.new("politeness") do |trace:|
  next 0 unless trace

  thread = trace.thread
  last_assistant = thread.reverse.find { |msg| msg["role"] == "assistant" }
  content = (last_assistant&.dig("content") || "").downcase

  polite_words = ["welcome", "glad", "happy", "pleasure", "thank"]
  is_polite = polite_words.any? { |word| content.include?(word) }

  {score: is_polite ? 1.0 : 0.0, metadata: {checked_message_preview: content[0, 80]}}
end

efficiency_scorer = Braintrust::Scorer.new("efficiency") do |trace:|
  next 0 unless trace

  llm_spans = trace.spans(span_type: "llm")
  is_efficient = llm_spans.length.between?(3, 5)

  {score: is_efficient ? 1.0 : 0.0, metadata: {llm_calls: llm_spans.length}}
end

Braintrust::Eval.run(
  project: "Support Quality",
  cases: SUPPORT_DATASET,
  task: support_task,
  scorers: [politeness_scorer, efficiency_scorer]
)

OpenTelemetry.tracer_provider.shutdown

using Braintrust.Sdk.Eval;
using Braintrust.Sdk.OpenAI;
using OpenAI.Chat;

var braintrust = Braintrust.Sdk.Braintrust.Get();
var activitySource = braintrust.GetActivitySource();
var openAIClient = BraintrustOpenAI.WrapOpenAI(
    activitySource, Environment.GetEnvironmentVariable("OPENAI_API_KEY"));
var chatClient = openAIClient.GetChatClient("gpt-5-mini");

string SupportTask(string input)
{
    var messages = new List<ChatMessage>
    {
        new SystemChatMessage("You are a helpful customer support agent."),
        new UserChatMessage(input),
    };

    messages.Add(new AssistantChatMessage(chatClient.CompleteChat(messages).Value.Content[0].Text));
    messages.Add(new UserChatMessage("Can you provide more details?"));
    messages.Add(new AssistantChatMessage(chatClient.CompleteChat(messages).Value.Content[0].Text));
    messages.Add(new UserChatMessage("Thank you for your help!"));

    return chatClient.CompleteChat(messages).Value.Content[0].Text;
}

var eval = await braintrust
    .EvalBuilder<string, string>()
    .Name("Support Quality")
    .Cases(
        DatasetCase.Of("My order hasn't arrived yet. Order #12345.", ""),
        DatasetCase.Of("I need help resetting my password.", ""))
    .TaskFunction(SupportTask)
    .Scorers(new PolitenessScorer(), new EfficiencyScorer())
    .BuildAsync();

await eval.RunAsync();

// Scores the last assistant message in the conversation thread reconstructed from the trace
class PolitenessScorer : ITracedScorer<string, string>
{
    public string Name => "Politeness";

    public Task<IReadOnlyList<Score>> Score(TaskResult<string, string> taskResult) =>
        Task.FromResult<IReadOnlyList<Score>>([new Score(Name, 0.0)]);

    public async Task<IReadOnlyList<Score>> Score(
        TaskResult<string, string> taskResult, EvalTrace trace)
    {
        var thread = await trace.GetThreadAsync();
        var lastAssistant = thread.LastOrDefault(m =>
            m.TryGetValue("role", out var role) && role as string == "assistant");
        var content = (lastAssistant?.GetValueOrDefault("content") as string ?? "").ToLowerInvariant();

        string[] politeWords = ["welcome", "glad", "happy", "pleasure", "thank"];
        var isPolite = politeWords.Any(content.Contains);

        return [new Score(Name, isPolite ? 1.0 : 0.0,
            new Dictionary<string, object> { ["checked_message_preview"] = content[..Math.Min(80, content.Length)] })];
    }
}

// Scores efficiency based on the number of LLM spans in the trace
class EfficiencyScorer : ITracedScorer<string, string>
{
    public string Name => "Efficiency";

    public Task<IReadOnlyList<Score>> Score(TaskResult<string, string> taskResult) =>
        Task.FromResult<IReadOnlyList<Score>>([new Score(Name, 0.0)]);

    public async Task<IReadOnlyList<Score>> Score(
        TaskResult<string, string> taskResult, EvalTrace trace)
    {
        var llmSpans = await trace.GetSpansAsync("llm");
        var isEfficient = llmSpans.Count is >= 3 and <= 5;

        return [new Score(Name, isEfficient ? 1.0 : 0.0,
            new Dictionary<string, object> { ["llm_calls"] = llmSpans.Count })];
    }
}

// Pass thresholds are not supported in the Java SDK.
// Use the UI or push a TypeScript/Python scorer via the CLI to set a pass threshold.

// Pass thresholds are not supported in the C# SDK.
// Use the UI or push a TypeScript/Python scorer via the CLI to set a pass threshold.