|
| 1 | +/** @vitest-environment node */ |
| 2 | +import { resetEnvFlagsMock, setEnvFlags } from '@sim/testing' |
| 3 | +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' |
| 4 | +import { |
| 5 | + getEffectiveBlockOutputPaths, |
| 6 | + getEffectiveBlockOutputs, |
| 7 | + getEffectiveBlockOutputType, |
| 8 | +} from '@/lib/workflows/blocks/block-outputs' |
| 9 | +import { getBlockReferenceTags } from '@/lib/workflows/blocks/block-reference-tags' |
| 10 | +import { evaluateSubBlockCondition } from '@/lib/workflows/subblocks/visibility' |
| 11 | +import { AgentBlock } from '@/blocks/blocks/agent' |
| 12 | +import { getAgentModelOptions, getModelOptions } from '@/blocks/utils' |
| 13 | +import { getBaseModelProviders } from '@/providers/models' |
| 14 | +import { Serializer } from '@/serializer' |
| 15 | +import { useProvidersStore } from '@/stores/providers/store' |
| 16 | +import type { BlockState } from '@/stores/workflows/workflow/types' |
| 17 | + |
| 18 | +const { mockGetBlock } = vi.hoisted(() => ({ mockGetBlock: vi.fn() })) |
| 19 | + |
| 20 | +vi.mock('@/blocks', () => ({ getBlock: mockGetBlock })) |
| 21 | + |
| 22 | +describe('Agent evaluation configuration', () => { |
| 23 | + afterEach(resetEnvFlagsMock) |
| 24 | + beforeEach(() => { |
| 25 | + mockGetBlock.mockReturnValue(AgentBlock) |
| 26 | + }) |
| 27 | + |
| 28 | + it.each(['jev-1.13.0', 'jev-latest', 'jev-preview'])( |
| 29 | + 'shows native fields and credentials for %s', |
| 30 | + (model) => { |
| 31 | + const visible = AgentBlock.subBlocks |
| 32 | + .filter((field) => evaluateSubBlockCondition(field.condition, { model })) |
| 33 | + .map((field) => field.id) |
| 34 | + expect(visible).toEqual(['model', 'apiKey', 'evaluationState', 'evaluationQuestions']) |
| 35 | + } |
| 36 | + ) |
| 37 | + |
| 38 | + it('keeps evaluation inputs configurable for a model reference', () => { |
| 39 | + for (const field of AgentBlock.subBlocks.filter((field) => field.id.startsWith('evaluation'))) { |
| 40 | + expect(evaluateSubBlockCondition(field.condition, { model: '<start.model>' })).toBe(true) |
| 41 | + } |
| 42 | + }) |
| 43 | + |
| 44 | + it.each([false, true])('shows TypeSafe credentials only when needed, hosted=%s', (hosted) => { |
| 45 | + setEnvFlags({ isHosted: hosted }) |
| 46 | + const apiKey = AgentBlock.subBlocks.find((field) => field.id === 'apiKey')! |
| 47 | + expect(evaluateSubBlockCondition(apiKey.condition, { model: 'jev-latest' })).toBe(!hosted) |
| 48 | + }) |
| 49 | + |
| 50 | + it.each(['jev-1.13.0', '<start.model>', '{{MODEL_ID}}'])( |
| 51 | + 'exposes answers for %s in downstream selectors', |
| 52 | + (model) => { |
| 53 | + const values = { model: { value: model } } |
| 54 | + expect(getEffectiveBlockOutputs('agent', values)).toHaveProperty('answers') |
| 55 | + expect(getEffectiveBlockOutputPaths('agent', values)).toContain('answers') |
| 56 | + expect(getEffectiveBlockOutputType('agent', 'answers', values)).toBe('json') |
| 57 | + } |
| 58 | + ) |
| 59 | + |
| 60 | + it('does not expose evaluation answers for a known chat model', () => { |
| 61 | + expect(getEffectiveBlockOutputs('agent', { model: { value: 'gpt-4o' } })).not.toHaveProperty( |
| 62 | + 'answers' |
| 63 | + ) |
| 64 | + }) |
| 65 | + |
| 66 | + it.each(['jev-1.13.0', '<start.model>'])( |
| 67 | + 'keeps answers accessible with a saved chat schema for %s', |
| 68 | + (model) => { |
| 69 | + const outputs = getEffectiveBlockOutputs('agent', { |
| 70 | + model: { value: model }, |
| 71 | + responseFormat: { |
| 72 | + value: { schema: { type: 'object', properties: { title: { type: 'string' } } } }, |
| 73 | + }, |
| 74 | + }) |
| 75 | + expect(outputs).toHaveProperty('answers') |
| 76 | + if (model === 'jev-1.13.0') expect(outputs).not.toHaveProperty('title') |
| 77 | + else expect(outputs).toHaveProperty('title') |
| 78 | + } |
| 79 | + ) |
| 80 | + |
| 81 | + it('shows Jev only in the model picker that supports evaluation inputs', () => { |
| 82 | + useProvidersStore.getState().setProviderModels('base', Object.keys(getBaseModelProviders())) |
| 83 | + expect(getAgentModelOptions().map((option) => option.id)).toContain('jev-1.13.0') |
| 84 | + expect(getModelOptions().map((option) => option.id)).not.toContain('jev-1.13.0') |
| 85 | + }) |
| 86 | + |
| 87 | + describe('evaluation answer references', () => { |
| 88 | + const questions = { |
| 89 | + category: { type: 'choice', instructions: 'Choose a category', criteria: { a: 'A', b: 'B' } }, |
| 90 | + rating: { type: 'score', instructions: 'Rate the result', criteria: ['Low', 'High'] }, |
| 91 | + passed: { type: 'noul', instructions: 'Did it pass?' }, |
| 92 | + } |
| 93 | + |
| 94 | + it.each(['jev-1.13.0', 'jev-latest', 'jev-preview', '<start.model>', '{{MODEL_ID}}'])( |
| 95 | + 'exposes typed question fields for %s without an execution result', |
| 96 | + (model) => { |
| 97 | + const values = { |
| 98 | + model: { value: model }, |
| 99 | + evaluationQuestions: { value: JSON.stringify(questions) }, |
| 100 | + responseFormat: { |
| 101 | + value: { schema: { type: 'object', properties: { title: { type: 'string' } } } }, |
| 102 | + }, |
| 103 | + } |
| 104 | + const tags = getBlockReferenceTags({ |
| 105 | + block: { id: 'agent-test', type: 'agent', name: 'Evaluate', subBlocks: values }, |
| 106 | + }) |
| 107 | + const fields = { |
| 108 | + 'answers.category.choice': 'string', |
| 109 | + 'answers.category.confidence': 'number', |
| 110 | + 'answers.category.probabilities': 'json', |
| 111 | + 'answers.category.type': 'string', |
| 112 | + 'answers.rating.score': 'number', |
| 113 | + 'answers.rating.confidence': 'number', |
| 114 | + 'answers.rating.legend': 'json', |
| 115 | + 'answers.passed.noul': 'number', |
| 116 | + } |
| 117 | + for (const [path, type] of Object.entries(fields)) { |
| 118 | + expect(tags).toContain(`evaluate.${path}`) |
| 119 | + expect(getEffectiveBlockOutputType('agent', path, values)).toBe(type) |
| 120 | + } |
| 121 | + expect(getEffectiveBlockOutputType('agent', 'answers', values)).toBe('json') |
| 122 | + expect(getEffectiveBlockOutputType('agent', 'answers.category', values)).toBe('json') |
| 123 | + expect(tags).not.toContain('evaluate.answers.passed.confidence') |
| 124 | + expect(tags.includes('evaluate.title')).toBe(!model.startsWith('jev-')) |
| 125 | + } |
| 126 | + ) |
| 127 | + |
| 128 | + it.each([undefined, '', '{', '<start.questions>', '{{QUESTIONS}}', [], null, { unknown: {} }])( |
| 129 | + 'keeps the answers object selectable when questions cannot be inferred: %j', |
| 130 | + (value) => { |
| 131 | + const values = { model: { value: 'jev-latest' }, evaluationQuestions: { value } } |
| 132 | + expect(getEffectiveBlockOutputPaths('agent', values)).toContain('answers') |
| 133 | + expect(getEffectiveBlockOutputType('agent', 'answers', values)).toBe('json') |
| 134 | + } |
| 135 | + ) |
| 136 | + |
| 137 | + it('uses structured questions and follows edits without leaking fields into chat models', () => { |
| 138 | + const values = { |
| 139 | + model: { value: 'jev-latest' }, |
| 140 | + evaluationQuestions: { value: { result: questions.category } }, |
| 141 | + } |
| 142 | + expect(getEffectiveBlockOutputPaths('agent', values)).toContain('answers.result.choice') |
| 143 | + expect( |
| 144 | + getEffectiveBlockOutputPaths('agent', { |
| 145 | + ...values, |
| 146 | + evaluationQuestions: { value: { result: questions.passed } }, |
| 147 | + }) |
| 148 | + ).not.toContain('answers.result.choice') |
| 149 | + expect( |
| 150 | + getEffectiveBlockOutputPaths('agent', { |
| 151 | + ...values, |
| 152 | + model: { value: 'gpt-4o' }, |
| 153 | + }).some((path) => path.startsWith('answers')) |
| 154 | + ).toBe(false) |
| 155 | + }) |
| 156 | + |
| 157 | + it('does not offer ambiguous reference paths for special question IDs', () => { |
| 158 | + const values = { |
| 159 | + model: { value: 'jev-latest' }, |
| 160 | + evaluationQuestions: { |
| 161 | + value: { |
| 162 | + 'with.dot': questions.passed, |
| 163 | + 'with space': questions.passed, |
| 164 | + 'with[0]': questions.passed, |
| 165 | + '<start.question>': questions.passed, |
| 166 | + 'valid-id_1': questions.passed, |
| 167 | + }, |
| 168 | + }, |
| 169 | + } |
| 170 | + const paths = getEffectiveBlockOutputPaths('agent', values) |
| 171 | + expect(paths.filter((path) => path.startsWith('answers.'))).toEqual([ |
| 172 | + 'answers.valid-id_1.noul', |
| 173 | + 'answers.valid-id_1.type', |
| 174 | + ]) |
| 175 | + expect(getEffectiveBlockOutputType('agent', 'answers', values)).toBe('json') |
| 176 | + }) |
| 177 | + |
| 178 | + it.each(['type', 'properties', 'description', '__proto__'])( |
| 179 | + 'resolves the question named %s through schema properties', |
| 180 | + (id) => { |
| 181 | + const values = { |
| 182 | + model: { value: 'jev-latest' }, |
| 183 | + evaluationQuestions: { value: { [id]: questions.passed } }, |
| 184 | + } |
| 185 | + expect(getEffectiveBlockOutputPaths('agent', values)).toContain(`answers.${id}.noul`) |
| 186 | + expect(getEffectiveBlockOutputType('agent', `answers.${id}.noul`, values)).toBe('number') |
| 187 | + } |
| 188 | + ) |
| 189 | + }) |
| 190 | + |
| 191 | + it.each([false, true])( |
| 192 | + 'serializes native fields without requiring messages, advanced=%s', |
| 193 | + (advancedMode) => { |
| 194 | + const values = { |
| 195 | + model: 'jev-1.13.0', |
| 196 | + apiKey: '{{TYPESAFE_API_KEY}}', |
| 197 | + evaluationState: '42', |
| 198 | + evaluationQuestions: '{"passed":{"type":"noul","instructions":"Did it pass?"}}', |
| 199 | + messages: JSON.stringify([{ role: 'user', content: 'Old chat prompt' }]), |
| 200 | + } |
| 201 | + const block: BlockState = { |
| 202 | + id: 'agent-test', |
| 203 | + type: 'agent', |
| 204 | + name: 'Evaluator', |
| 205 | + position: { x: 0, y: 0 }, |
| 206 | + enabled: true, |
| 207 | + advancedMode, |
| 208 | + outputs: {}, |
| 209 | + subBlocks: Object.fromEntries( |
| 210 | + Object.entries(values).map(([id, value]) => [ |
| 211 | + id, |
| 212 | + { id, value, type: AgentBlock.subBlocks.find((field) => field.id === id)!.type }, |
| 213 | + ]) |
| 214 | + ), |
| 215 | + } |
| 216 | + const result = new Serializer().serializeWorkflow({ [block.id]: block }, [], {}, {}, true) |
| 217 | + expect(result.blocks[0].config.tool).toBe('typesafe') |
| 218 | + expect(result.blocks[0].config.params).toMatchObject({ |
| 219 | + evaluationState: '42', |
| 220 | + evaluationQuestions: values.evaluationQuestions, |
| 221 | + }) |
| 222 | + expect(result.blocks[0].config.params).not.toHaveProperty('messages') |
| 223 | + } |
| 224 | + ) |
| 225 | +}) |
0 commit comments