Write scorers

import { Eval } from "braintrust";
import { LLMClassifierFromTemplate } from "autoevals";
import OpenAI from "openai";

const client = new OpenAI();

// Inline dataset: movie descriptions and expected titles (same as Python llm_eval_simple.py)
const MOVIE_DATASET = [
  {
    input:
      "A detective investigates a series of murders based on the seven deadly sins.",
    expected: "Se7en",
  },
  {
    input:
      "A thief who steals corporate secrets through the use of dream-sharing technology is given the inverse task of planting an idea into the mind of a C.E.O.",
    expected: "Inception",
  },
];

async function task(input: string): Promise<string> {
  const response = await client.responses.create({
    model: "gpt-4o-mini",
    input: [
      {
        role: "system",
        content:
          "Based on the following description, identify the movie. Reply with only the movie title.",
      },
      { role: "user", content: input },
    ],
  });
  return response.output_text ?? "";
}

// LLM-as-judge: scores whether the model's output matches the expected title (exact or equivalent)
const llmJudge = LLMClassifierFromTemplate({
  name: "Correctness",
  promptTemplate: `
You are evaluating a movie-identification task.

Output (model's answer): {{output}}
Expected (correct movie): {{expected}}

Does the output correctly identify the same movie as the expected answer?
Consider alternate titles (e.g. "Harry Potter 1" vs "Harry Potter and the Sorcerer's Stone") as correct.

Return "correct" if the output is the right movie (exact or equivalent title).
Return "incorrect" otherwise.
`,
  choiceScores: { correct: 1, incorrect: 0 },
  model: "gpt-4o-mini",
});

Eval("Movie Matcher", {
  data: MOVIE_DATASET,
  task,
  scores: [llmJudge],
});

import braintrust from "braintrust";
import { z } from "zod";

const project = braintrust.projects.create({ name: "my-project" });

project.scorers.create({
  name: "Equality scorer",
  slug: "equality-scorer",
  description: "Check if output equals expected",
  parameters: z.object({
    output: z.string(),
    expected: z.string(),
  }),
  handler: async ({ output, expected }) => {
    const matches = output === expected;
    return {
      score: matches ? 1 : 0,
      metadata: { exact_match: matches },
    };
  },
  metadata: {
    __pass_threshold: 0.5,
  },
});

import { Eval, type Scorer } from "braintrust";

// Inline dataset
const CONVERSATION_DATASET = [
  {
    input: "What is the capital of France?",
    expected: "multi-turn",
  },
  {
    input: "Tell me about quantum physics",
    expected: "multi-turn",
  },
];

// Simulated multi-turn conversation task
async function conversationTask(input: string): Promise<string> {
  // In a real scenario, this would be a multi-turn conversation
  return `Here's information about ${input}`;
}

// Trace-level scorer using conversation thread
const threadLengthScorer: Scorer = async ({ trace }) => {
  if (!trace) return 0;

  // Get preprocessed conversation thread
  const thread = await trace.getThread();

  const conversationLength = thread.length;

  return {
    name: "Thread length",
    score: conversationLength >= 3 ? 1 : 0,
    metadata: {
      conversation_length: conversationLength,
      has_minimum_turns: conversationLength >= 3,
    },
  };
};

// Trace-level scorer using span analysis
const llmCallCounter: Scorer = async ({ trace }) => {
  if (!trace) return 0;

  // Get only LLM spans for workflow analysis
  const llmSpans = await trace.getSpans({ spanType: ["llm"] });

  // Or get all spans: await trace.getSpans()
  // Or multiple types: await trace.getSpans({ spanType: ["llm", "tool"] })

  return {
    name: "LLM call count",
    score: llmSpans.length > 0 ? 1 : 0,
    metadata: { llm_count: llmSpans.length },
  };
};

Eval("Conversation Quality", {
  data: CONVERSATION_DATASET,
  task: conversationTask,
  scores: [threadLengthScorer, llmCallCounter],
  traceScorers: true, // Enable trace-level scoring
});