jev-lab

import { experimental_evaluate as evaluate } from 'ai';

const { answers } = await evaluate({
  model: 'typesafe-ai/jev',
  state,
  questions: {
    topic: { type: 'choice', instructions: 'Which team should handle `ticket.message`? Classify the customer\'s primary request.',
      criteria: { billing: 'Charges, invoices, refunds, or subscriptions. Not for order tracking or account access.', orders: 'Order status, delivery, cancellation, or returns. Not for charges or account access.', account: 'Login, profile, permissions, or security. Not for charges or order tracking.' } },
    requests_credentials: { type: 'boolean', instructions: 'Does `ticket.message` ask the recipient to disclose one of the credentials listed in `policy.sensitive_credentials`?' },
    sender_identity_mismatch: { type: 'boolean', instructions: 'Does `ticket.sender.display_name` claim an organization that conflicts with the domain of `ticket.sender.email`?' },
    refund_requested: { type: 'boolean', instructions: 'Does the customer in `ticket.message` explicitly request a refund or credit? Require a requested remedy, not a billing complaint alone.' },
    mentions_open_order: { type: 'boolean', instructions: 'Does `ticket.message` refer, by id or identifying details, to an order listed in `customer.open_orders`?' },
    mentions_login_issue: { type: 'boolean', instructions: 'Does `ticket.message` describe a login or session problem?' },
    frustration: { type: 'score', instructions: 'How frustrated does the customer in `ticket.message` appear? Judge expressed frustration, not issue severity.',
      criteria: ['Calm and matter-of-fact: neutral wording', 'Frustrated but civil: expresses annoyance, remains constructive', 'Very angry or threatening to leave: hostile language, threatens cancellation'] },
  },
});