jev-belay

import { experimental_evaluate as evaluate } from 'ai';

const { answers } = await evaluate({
  model: 'typesafe-ai/jev',
  state,
  questions: {
    claims_done: { type: 'boolean', instructions: 'Does `final_message` present the requested work as finished or working?' },
    claims_verified: { type: 'boolean', instructions: 'Does `final_message` claim that tests, a build, or other checks were run and passed?' },
    verification_applies: { type: 'boolean', instructions: 'Would running the project\'s tests, build, or lint be a meaningful way to check the work that `task` asks for?' },
    outcome: { type: 'choice', instructions: 'What does `final_message` report about `task`?',
      criteria: { complete: 'The work is finished', partial: 'Progress was made and remaining work is named', blocked: 'A blocker is reported or the user is asked something', other: 'None of these' } },
  },
});