fix(jev): expose typed answer fields in reference pickers

Author: waleedlatif1

apps/sim/blocks/agent-evaluation.test.ts

@@ -5,6 +5,7 @@ import {
   getEffectiveBlockOutputs,
   getEffectiveBlockOutputType,
 } from '@/lib/workflows/blocks/block-outputs'
+import { getBlockReferenceTags } from '@/lib/workflows/blocks/block-reference-tags'
 import { evaluateSubBlockCondition } from '@/lib/workflows/subblocks/visibility'
 import { AgentBlock } from '@/blocks/blocks/agent'
 import { getAgentModelOptions, getModelOptions } from '@/blocks/utils'
@@ -75,6 +76,110 @@ describe('Agent evaluation configuration', () => {
     expect(getModelOptions().map((option) => option.id)).not.toContain('jev-1.13.0')
   })
 
+  describe('evaluation answer references', () => {
+    const questions = {
+      category: { type: 'choice', instructions: 'Choose a category', criteria: { a: 'A', b: 'B' } },
+      rating: { type: 'score', instructions: 'Rate the result', criteria: ['Low', 'High'] },
+      passed: { type: 'noul', instructions: 'Did it pass?' },
+    }
+
+    it.each(['jev-1.13.0', 'jev-latest', 'jev-preview', '<start.model>', '{{MODEL_ID}}'])(
+      'exposes typed question fields for %s without an execution result',
+      (model) => {
+        const values = {
+          model: { value: model },
+          evaluationQuestions: { value: JSON.stringify(questions) },
+          responseFormat: {
+            value: { schema: { type: 'object', properties: { title: { type: 'string' } } } },
+          },
+        }
+        const tags = getBlockReferenceTags({
+          block: { id: 'agent-test', type: 'agent', name: 'Evaluate', subBlocks: values },
+        })
+        const fields = {
+          'answers.category.choice': 'string',
+          'answers.category.confidence': 'number',
+          'answers.category.probabilities': 'json',
+          'answers.category.type': 'string',
+          'answers.rating.score': 'number',
+          'answers.rating.confidence': 'number',
+          'answers.rating.legend': 'json',
+          'answers.passed.noul': 'number',
+        }
+        for (const [path, type] of Object.entries(fields)) {
+          expect(tags).toContain(`evaluate.${path}`)
+          expect(getEffectiveBlockOutputType('agent', path, values)).toBe(type)
+        }
+        expect(getEffectiveBlockOutputType('agent', 'answers', values)).toBe('json')
+        expect(getEffectiveBlockOutputType('agent', 'answers.category', values)).toBe('json')
+        expect(tags).not.toContain('evaluate.answers.passed.confidence')
+        expect(tags.includes('evaluate.title')).toBe(!model.startsWith('jev-'))
+      }
+    )
+
+    it.each([undefined, '', '{', '<start.questions>', '{{QUESTIONS}}', [], null, { unknown: {} }])(
+      'keeps the answers object selectable when questions cannot be inferred: %j',
+      (value) => {
+        const values = { model: { value: 'jev-latest' }, evaluationQuestions: { value } }
+        expect(getEffectiveBlockOutputPaths('agent', values)).toContain('answers')
+        expect(getEffectiveBlockOutputType('agent', 'answers', values)).toBe('json')
+      }
+    )
+
+    it('uses structured questions and follows edits without leaking fields into chat models', () => {
+      const values = {
+        model: { value: 'jev-latest' },
+        evaluationQuestions: { value: { result: questions.category } },
+      }
+      expect(getEffectiveBlockOutputPaths('agent', values)).toContain('answers.result.choice')
+      expect(
+        getEffectiveBlockOutputPaths('agent', {
+          ...values,
+          evaluationQuestions: { value: { result: questions.passed } },
+        })
+      ).not.toContain('answers.result.choice')
+      expect(
+        getEffectiveBlockOutputPaths('agent', {
+          ...values,
+          model: { value: 'gpt-4o' },
+        }).some((path) => path.startsWith('answers'))
+      ).toBe(false)
+    })
+
+    it('does not offer ambiguous reference paths for special question IDs', () => {
+      const values = {
+        model: { value: 'jev-latest' },
+        evaluationQuestions: {
+          value: {
+            'with.dot': questions.passed,
+            'with space': questions.passed,
+            'with[0]': questions.passed,
+            '<start.question>': questions.passed,
+            'valid-id_1': questions.passed,
+          },
+        },
+      }
+      const paths = getEffectiveBlockOutputPaths('agent', values)
+      expect(paths.filter((path) => path.startsWith('answers.'))).toEqual([
+        'answers.valid-id_1.noul',
+        'answers.valid-id_1.type',
+      ])
+      expect(getEffectiveBlockOutputType('agent', 'answers', values)).toBe('json')
+    })
+
+    it.each(['type', 'properties', 'description', '__proto__'])(
+      'resolves the question named %s through schema properties',
+      (id) => {
+        const values = {
+          model: { value: 'jev-latest' },
+          evaluationQuestions: { value: { [id]: questions.passed } },
+        }
+        expect(getEffectiveBlockOutputPaths('agent', values)).toContain(`answers.${id}.noul`)
+        expect(getEffectiveBlockOutputType('agent', `answers.${id}.noul`, values)).toBe('number')
+      }
+    )
+  })
+
   it.each([false, true])(
     'serializes native fields without requiring messages, advanced=%s',
     (advancedMode) => {

apps/sim/lib/workflows/blocks/block-outputs.ts

@@ -3,6 +3,7 @@ import {
   extractFieldsFromSchema,
   parseResponseFormatSafely,
 } from '@/lib/core/utils/response-format'
+import { getJevAnswerOutput } from '@/lib/workflows/blocks/jev-outputs'
 import { normalizeInputFormatValue } from '@/lib/workflows/input-format'
 import { containsReference } from '@/lib/workflows/sanitization/references'
 import {
@@ -435,18 +436,19 @@ export function getEffectiveBlockOutputs(
 
   if (blockType === 'agent') {
     const model = subBlocks?.model?.value
-    if (typeof model !== 'string' || !isEvaluationModel(model)) {
-      const responseFormatOutputs = getResponseFormatOutputs(subBlocks, 'agent')
-      if (responseFormatOutputs) {
-        /** A referenced model may select either evaluation or chat at execution time. */
-        return typeof model === 'string' && containsReference(model)
-          ? {
-              ...getBlockOutputs('agent', subBlocks, false, { includeHidden }),
-              ...responseFormatOutputs,
-            }
-          : responseFormatOutputs
+    const mayEvaluate =
+      typeof model === 'string' && (isEvaluationModel(model) || containsReference(model))
+    if (mayEvaluate) {
+      const outputs = getBlockOutputs('agent', subBlocks, false, { includeHidden })
+      const answers = getJevAnswerOutput(subBlocks?.evaluationQuestions?.value)
+      return {
+        ...outputs,
+        ...(containsReference(model) ? getResponseFormatOutputs(subBlocks, 'agent') : undefined),
+        ...(answers ? { answers } : undefined),
       }
     }
+    const responseFormatOutputs = getResponseFormatOutputs(subBlocks, 'agent')
+    if (responseFormatOutputs) return responseFormatOutputs
   }
 
   let baseOutputs: OutputDefinition
@@ -559,9 +561,7 @@ function traverseOutputPath(outputs: OutputDefinition, pathParts: string[]): unk
 
     const currentObj = current as Record<string, unknown>
 
-    if (part in currentObj) {
-      current = currentObj[part]
-    } else if (
+    if (
       'type' in currentObj &&
       (currentObj.type === 'object' || currentObj.type === 'json') &&
       'properties' in currentObj &&
@@ -592,6 +592,8 @@ function traverseOutputPath(outputs: OutputDefinition, pathParts: string[]): unk
       } else {
         return null
       }
+    } else if (part in currentObj) {
+      current = currentObj[part]
     } else {
       return null
     }

apps/sim/lib/workflows/blocks/jev-outputs.ts

@@ -0,0 +1,52 @@
+import { isRecordLike } from '@sim/utils/object'
+import type { JevAnswer } from '@/providers/typesafe/types'
+import type { OutputProperty } from '@/tools/types'
+
+const ANSWER_FIELDS = {
+  choice: {
+    choice: { type: 'string' },
+    confidence: { type: 'number' },
+    probabilities: { type: 'json' },
+    type: { type: 'string' },
+  },
+  score: {
+    score: { type: 'number' },
+    confidence: { type: 'number' },
+    probabilities: { type: 'json' },
+    legend: { type: 'json' },
+    type: { type: 'string' },
+  },
+  noul: {
+    noul: { type: 'number' },
+    type: { type: 'string' },
+  },
+} satisfies {
+  [Type in JevAnswer['type']]: Record<keyof Extract<JevAnswer, { type: Type }>, OutputProperty>
+}
+
+/**
+ * Describes answers known from the editor's questions without requiring resolved inputs.
+ * Keys containing reference syntax remain accessible through the whole answers object.
+ */
+export function getJevAnswerOutput(
+  questions: unknown
+): (OutputProperty & { type: 'json' }) | undefined {
+  if (typeof questions === 'string') {
+    try {
+      questions = JSON.parse(questions)
+    } catch {
+      return undefined
+    }
+  }
+  if (!isRecordLike(questions)) return undefined
+
+  const entries: Array<[string, OutputProperty]> = []
+  for (const [id, question] of Object.entries(questions)) {
+    if (!/^[\w-]+$/.test(id) || !isRecordLike(question)) continue
+    const { type } = question
+    if (type !== 'choice' && type !== 'score' && type !== 'noul') continue
+    entries.push([id, { type: 'json', properties: ANSWER_FIELDS[type] }])
+  }
+
+  return entries.length > 0 ? { type: 'json', properties: Object.fromEntries(entries) } : undefined
+}