tiershift

import { experimental_evaluate as evaluate } from 'ai';

const { answers } = await evaluate({
  model: 'typesafe-ai/jev',
  state,
  questions: {
    difficulty: { type: 'score', instructions: 'How hard is it to answer `request.user_message` well? Consider `request.system_prompt` and `request.step` for context.',
      criteria: ['trivial: a lookup, a formatting change, an acknowledgement, or a one-line reply', 'moderate: needs domain knowledge or a few reasoning steps; a competent generalist can do it', 'hard: needs deep expertise, long multi-step reasoning, careful trade-off analysis, or novel design'] },
    needs_reasoning: { type: 'boolean', instructions: 'Does `request.user_message` require multi-step logical reasoning, proof, debugging, or planning rather than recall, rewriting, or formatting?' },
    stakes: { type: 'score', instructions: 'How costly is a wrong or low-quality answer to `request.user_message` for the user?',
      criteria: ['low: easy to spot and redo; no external consequence', 'medium: wastes real time or money if wrong; affects work product', 'high: legal, financial, medical, safety, security, or production-system impact'] },
    domain: { type: 'choice', instructions: 'Which domain does `request.user_message` belong to?',
      criteria: { code: 'writing, debugging, reviewing, or explaining software', math_logic: 'mathematics, proofs, formal logic, puzzles', writing: 'prose, tone, editing, summarizing, translation', architecture: 'system design, infrastructure, migrations, planning', legal_finance: 'contracts, compliance, money, tax', data: 'data analysis, SQL, spreadsheets, statistics', general: 'general knowledge, conversation, simple questions' } },
    has_code: { type: 'boolean', instructions: 'Does `request.user_message` contain code or ask for code to be written, fixed, or reviewed?' },
    ambiguous: { type: 'boolean', instructions: 'Is `request.user_message` missing information that an expert would need before answering well?' },
    output_length: { type: 'score', instructions: 'How long should a good answer to `request.user_message` be?',
      criteria: ['one line or a single value', 'one to three paragraphs or a short code snippet', 'a long document, a detailed plan, or multi-file code'] },
    creative: { type: 'boolean', instructions: 'Does `request.user_message` ask for creative or stylistic writing rather than factual or technical output?' },
  },
});

import { experimental_evaluate as evaluate } from 'ai';

const { answers } = await evaluate({
  model: 'typesafe-ai/jev',
  state,
  questions: {
    addresses: { type: 'boolean', instructions: 'question' },
  },
});

import { experimental_evaluate as evaluate } from 'ai';

const { answers } = await evaluate({
  model: 'typesafe-ai/jev',
  state,
  questions: {
    addresses: { type: 'boolean', instructions: 'Does `answer` fully and correctly address `request`?' },
  },
});