fix(jev): validate batch answers against requested questions

Author: waleedlatif1

apps/sim/tools/jev/evaluate.ts

@@ -2,7 +2,7 @@ import {
   buildJevBody,
   JEV_COMMON_PARAMS,
   JEV_REQUEST,
-  parseJevJson,
+  parseJevQuestions,
   parseJevResponse,
 } from '@/tools/jev/shared'
 import type { JevEvaluateParams, JevEvaluateResponse } from '@/tools/jev/types'
@@ -30,10 +30,22 @@ export const jevEvaluateTool: ToolConfig<JevEvaluateParams, JevEvaluateResponse>
       mode: 'project',
       select: (params) => ({ state: params.state, questions: params.questions }),
     },
-    body: (params) => buildJevBody(params, parseJevJson(params.questions, 'questions')),
+    body: (params) => buildJevBody(params, params.questions),
   },
-  transformResponse: async (response) => {
+  transformResponse: async (response, params) => {
     const { model, usage, answers } = await parseJevResponse(response)
+    if (!params) throw new Error('Jev batch response validation requires request parameters')
+    const questions = parseJevQuestions(params.questions)
+    if (
+      Object.keys(answers).length !== Object.keys(questions).length ||
+      Object.entries(questions).some(
+        ([id, question]) => !Object.hasOwn(answers, id) || answers[id].type !== question.type
+      )
+    ) {
+      throw new Error(
+        'TypeSafe returned Jev answers that do not match the requested question IDs and types'
+      )
+    }
     return { success: true, output: { model, usage, answers } }
   },
   outputs: {

apps/sim/tools/jev/jev-block.test.ts

@@ -139,7 +139,8 @@ describe('Jev workflow execution', () => {
           model: 'jev-1.13.0',
           answers,
           usage: { input_tokens: 50, output_tokens: 8 },
-        })
+        }),
+        { apiKey: params.apiKey, state: params.state, questions: params.questions }
       )
     })
     const output = await new GenericBlockHandler().execute(context(), block('jev_evaluate'), {

apps/sim/tools/jev/jev.test.ts

@@ -1,6 +1,7 @@
 import { describe, expect, it } from 'vitest'
 import { ResolvedSecretTraceRegistry } from '@/executor/utils/resolved-secret-trace-registry'
 import { jevChoiceTool, jevEvaluateTool, jevNoulTool, jevScoreTool } from '@/tools/jev'
+import type { JevEvaluateParams } from '@/tools/jev/types'
 import { prepareToolRequest } from '@/tools/request-transport'
 
 const BASE = { apiKey: 'test-key', state: 'My payouts have been failing for three days.' }
@@ -19,6 +20,22 @@ const SCORE = {
   confidence: 0.92,
 } as const
 const NOUL = { type: 'noul', noul: 0.95 } as const
+const BATCH_PARAMS: JevEvaluateParams = {
+  ...BASE,
+  questions: {
+    department: {
+      type: 'choice',
+      instructions: 'Which team?',
+      criteria: { billing: null, technical: null },
+    },
+    frustration: {
+      type: 'score',
+      instructions: 'How frustrated?',
+      criteria: ['Calm', 'Frustrated', 'Very angry'],
+    },
+    is_urgent: { type: 'noul', instructions: 'Is this urgent?' },
+  },
+}
 
 function response(answers: Record<string, unknown>) {
   return Response.json({ model: 'jev-1.13.0', answers, usage: USAGE })
@@ -142,12 +159,44 @@ describe('Jev tools', () => {
     })
   })
 
-  it('preserves all mixed batch answer types and question IDs', async () => {
-    const answers = { department: CHOICE, frustration: SCORE, is_urgent: NOUL }
-    expect(await jevEvaluateTool.transformResponse!(response(answers))).toEqual({
-      success: true,
-      output: { model: 'jev-1.13.0', usage: USAGE, answers },
-    })
+  it.each([false, true])(
+    'matches mixed batch answers regardless of order, serialized=%s',
+    async (serialized) => {
+      const answers = { department: CHOICE, frustration: SCORE, is_urgent: NOUL }
+      const params = {
+        ...BATCH_PARAMS,
+        questions: serialized ? JSON.stringify(BATCH_PARAMS.questions) : BATCH_PARAMS.questions,
+      }
+      expect(
+        await jevEvaluateTool.transformResponse!(
+          response({ is_urgent: NOUL, frustration: SCORE, department: CHOICE }),
+          params
+        )
+      ).toEqual({
+        success: true,
+        output: { model: 'jev-1.13.0', usage: USAGE, answers },
+      })
+    }
+  )
+
+  it.each([
+    ['empty', {}],
+    ['partial', { department: CHOICE, frustration: SCORE }],
+    ['mismatched type', { department: NOUL, frustration: SCORE, is_urgent: NOUL }],
+    ['unexpected ID', { department: CHOICE, frustration: SCORE, other: NOUL }],
+    ['extra answer', { department: CHOICE, frustration: SCORE, is_urgent: NOUL, other: NOUL }],
+  ])('rejects a %s batch response', async (_label, answers) => {
+    await expect(
+      jevEvaluateTool.transformResponse!(response(answers), BATCH_PARAMS)
+    ).rejects.toThrow(
+      'TypeSafe returned Jev answers that do not match the requested question IDs and types'
+    )
+  })
+
+  it('requires request context to validate batch answers', async () => {
+    await expect(
+      jevEvaluateTool.transformResponse!(response({ department: CHOICE }))
+    ).rejects.toThrow('Jev batch response validation requires request parameters')
   })
 
   it('supports structured Score legends documented by the TypeSafe SDK', async () => {

apps/sim/tools/jev/shared.ts

@@ -115,19 +115,24 @@ export function parseJevJson(value: unknown, field: string): unknown {
   }
 }
 
-export function buildJevBody(params: JevBaseParams, questions: unknown) {
-  const parsedQuestions = questionsSchema.safeParse(questions)
+export function parseJevQuestions(questions: unknown) {
+  const parsedQuestions = questionsSchema.safeParse(parseJevJson(questions, 'questions'))
   if (!parsedQuestions.success) {
     throw new Error(
       'Invalid Jev questions: provide typed questions with instructions, 1–255 Choice options, 2–10 Score levels, or optional true/false Noul criteria'
     )
   }
+  return parsedQuestions.data
+}
+
+export function buildJevBody(params: JevBaseParams, questions: unknown) {
+  const parsedQuestions = parseJevQuestions(questions)
   const state = contentSchema.safeParse(params.state)
   if (!state.success) throw new Error('Jev state must be text, a JSON object, or an array')
   return {
     model: params.model?.trim() || 'jev-1.13.0',
     state: state.data,
-    questions: parsedQuestions.data,
+    questions: parsedQuestions,
   }
 }