From 26d34976aa1c0c30f2ee8574f4ccf3b2550d52e3 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Thu, 24 Sep 2026 16:11:13 +0200 Subject: [PATCH 1/6] feat(node): Add TypeSafe integration Co-Authored-By: Claude Opus 5.5 --- .../tracing/typesafe/instrument-manual.mjs | 12 ++ .../suites/tracing/typesafe/instrument.mjs | 9 + .../suites/tracing/typesafe/scenario.mjs | 71 +++++++ .../suites/tracing/typesafe/test.ts | 107 +++++++++++ packages/astro/src/index.server.ts | 2 + packages/aws-serverless/src/index.ts | 2 + packages/bun/src/index.ts | 2 + packages/cloudflare/src/index.ts | 1 + packages/deno/src/index.ts | 1 + packages/elysia/src/index.ts | 2 + packages/google-cloud-serverless/src/index.ts | 2 + packages/node/src/index.ts | 2 + packages/remix/src/server/index.ts | 1 + packages/server-utils/src/ai/index.ts | 1 + .../server-utils/src/ai/typesafe/constants.ts | 5 + .../server-utils/src/ai/typesafe/index.ts | 175 ++++++++++++++++++ packages/server-utils/src/index.ts | 1 + .../server-utils/src/integrations/index.ts | 2 + .../server-utils/src/integrations/typesafe.ts | 62 +++++++ .../server-utils/src/orchestrion/channels.ts | 2 + .../config/channel-integration-definitions.ts | 1 + .../src/orchestrion/config/index.ts | 2 + .../src/orchestrion/config/typesafe.ts | 18 ++ packages/sveltekit/src/server/index.ts | 1 + packages/vercel-edge/src/index.ts | 1 + 25 files changed, 485 insertions(+) create mode 100644 dev-packages/node-integration-tests/suites/tracing/typesafe/instrument-manual.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/typesafe/instrument.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/typesafe/scenario.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts create mode 100644 packages/server-utils/src/ai/typesafe/constants.ts create mode 100644 packages/server-utils/src/ai/typesafe/index.ts create mode 100644 packages/server-utils/src/integrations/typesafe.ts create mode 100644 packages/server-utils/src/orchestrion/config/typesafe.ts diff --git a/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument-manual.mjs b/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument-manual.mjs new file mode 100644 index 000000000000..b405a672e86c --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument-manual.mjs @@ -0,0 +1,12 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, + // `instrumentTypeSafeClient` is the manual path for runtimes without the orchestrion hook. + // Drop the automatic integration so the scenario exercises it alone. + integrations: integrations => integrations.filter(integration => integration.name !== 'TypeSafe'), +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument.mjs new file mode 100644 index 000000000000..46a27dd03b74 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument.mjs @@ -0,0 +1,9 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/typesafe/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/typesafe/scenario.mjs new file mode 100644 index 000000000000..609265208b9b --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/typesafe/scenario.mjs @@ -0,0 +1,71 @@ +import * as Sentry from '@sentry/node'; +import { noul, score, TypeSafeClient } from '@typesafe-ai/sdk'; +import express from 'express'; + +function startMockServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1/systemone', (req, res) => { + if (req.body.model === 'error-model') { + res.status(400).json({ error: 'Unknown model' }); + return; + } + + res.json({ + model: 'jev-1.13.0', + answers: { + authIssue: { type: 'noul', noul: 0.98 }, + urgency: { + type: 'score', + score: 1.58, + confidence: 0.9, + legend: { 0: 'low', 1: 'medium', 2: 'high' }, + probabilities: { 0: 0, 1: 0.42, 2: 0.58 }, + }, + }, + usage: { input_tokens: 275, output_tokens: 20 }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockServer(); + const typesafe = new TypeSafeClient({ + apiKey: 'mock-api-key', + baseURL: `http://localhost:${server.address().port}`, + retry: { maxRetries: 0 }, + }); + const client = Sentry.getClient().getIntegrationByName('TypeSafe') + ? typesafe + : Sentry.instrumentTypeSafeClient(typesafe); + const state = 'I cannot log in, and I also want a refund for last month.'; + const questions = { + authIssue: noul('Is there a login problem?'), + urgency: score('How urgent is this ticket?', ['low', 'medium', 'high']), + }; + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + await client.systemOne({ state, questions }); + + // The instrumentation must not read the body the caller gets from `asResponse()`. + const response = await client.systemOne({ model: 'jev-1.13', state, questions }).asResponse(); + await response.json(); + + try { + await client.systemOne({ model: 'error-model', state, questions }); + } catch { + // expected + } + }); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts b/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts new file mode 100644 index 000000000000..b2268845f30b --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts @@ -0,0 +1,107 @@ +import { + GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MODEL, + GEN_AI_RESPONSE_MODEL, + GEN_AI_USAGE_INPUT_TOKENS, + GEN_AI_USAGE_OUTPUT_TOKENS, + GEN_AI_USAGE_TOTAL_TOKENS, + SENTRY_OP, + SENTRY_ORIGIN, +} from '@sentry/conventions/attributes'; +import { afterAll, describe, expect } from 'vitest'; +import { cleanupChildProcesses, createEsmAndCjsTests } from '../../../utils/runner'; + +describe('TypeSafe integration', () => { + afterAll(() => { + cleanupChildProcesses(); + }); + + describe.each([ + ['automatic', 'instrument.mjs'], + ['manual', 'instrument-manual.mjs'], + ])('%s instrumentation', (_, instrumentFile) => { + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + instrumentFile, + (createRunner, test) => { + test('creates evaluate spans for systemOne', async () => { + await createRunner() + .unordered() + .expect({ + span: container => { + const evaluateSpans = container.items.filter( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.evaluate', + ); + expect(evaluateSpans).toHaveLength(3); + for (const span of evaluateSpans) { + expect(span.attributes[SENTRY_ORIGIN]?.value).toBe('auto.ai.typesafe'); + expect(span.attributes[GEN_AI_OPERATION_NAME]?.value).toBe('evaluate'); + expect(span.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe('typesafe'); + } + + const defaultModelSpan = evaluateSpans.find(span => span.name === 'evaluate jev-latest')!; + expect(defaultModelSpan).toBeDefined(); + expect(defaultModelSpan.status).toBe('ok'); + expect(defaultModelSpan.attributes[GEN_AI_REQUEST_MODEL]?.value).toBe('jev-latest'); + expect(defaultModelSpan.attributes[GEN_AI_RESPONSE_MODEL]?.value).toBe('jev-1.13.0'); + expect(defaultModelSpan.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(275); + expect(defaultModelSpan.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]?.value).toBe(20); + expect(defaultModelSpan.attributes[GEN_AI_USAGE_TOTAL_TOKENS]?.value).toBe(295); + expect(JSON.parse(defaultModelSpan.attributes[GEN_AI_INPUT_MESSAGES]?.value as string)).toEqual([ + { + type: 'evaluation', + state: 'I cannot log in, and I also want a refund for last month.', + questions: { + authIssue: { type: 'noul', instructions: 'Is there a login problem?' }, + urgency: { + type: 'score', + instructions: 'How urgent is this ticket?', + criteria: ['low', 'medium', 'high'], + }, + }, + }, + ]); + expect(JSON.parse(defaultModelSpan.attributes[GEN_AI_OUTPUT_MESSAGES]?.value as string)).toEqual([ + { + type: 'evaluation', + answers: { + authIssue: { type: 'noul', noul: 0.98 }, + urgency: { + type: 'score', + score: 1.58, + confidence: 0.9, + legend: { 0: 'low', 1: 'medium', 2: 'high' }, + probabilities: { 0: 0, 1: 0.42, 2: 0.58 }, + }, + }, + }, + ]); + + const asResponseSpan = evaluateSpans.find(span => span.name === 'evaluate jev-1.13')!; + expect(asResponseSpan).toBeDefined(); + expect(asResponseSpan.status).toBe('ok'); + expect(asResponseSpan.attributes[GEN_AI_RESPONSE_MODEL]?.value).toBe('jev-1.13.0'); + + const errorSpan = evaluateSpans.find(span => span.name === 'evaluate error-model')!; + expect(errorSpan).toBeDefined(); + expect(errorSpan.status).toBe('error'); + expect(errorSpan.attributes[GEN_AI_RESPONSE_MODEL]).toBeUndefined(); + expect(errorSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); + }, + }) + .start() + .completed(); + }); + }, + { + additionalDependencies: { + '@typesafe-ai/sdk': '^0.6.0', + }, + }, + ); + }); +}); diff --git a/packages/astro/src/index.server.ts b/packages/astro/src/index.server.ts index 8f765fc367f8..3121d53e68e3 100644 --- a/packages/astro/src/index.server.ts +++ b/packages/astro/src/index.server.ts @@ -95,6 +95,7 @@ export { openAIIntegration, groqIntegration, togetherAIIntegration, + typesafeIntegration, langChainIntegration, langGraphIntegration, createFlueInstrumentation, @@ -161,6 +162,7 @@ export { supabaseIntegration, instrumentSupabaseClient, instrumentMistralAiClient, + instrumentTypeSafeClient, instrumentOpenAiClient, instrumentAnthropicAiClient, instrumentGoogleGenAIClient, diff --git a/packages/aws-serverless/src/index.ts b/packages/aws-serverless/src/index.ts index c467d1b2266f..09dc84610073 100644 --- a/packages/aws-serverless/src/index.ts +++ b/packages/aws-serverless/src/index.ts @@ -63,6 +63,7 @@ export { openAIIntegration, groqIntegration, togetherAIIntegration, + typesafeIntegration, langChainIntegration, langGraphIntegration, mastraIntegration, @@ -144,6 +145,7 @@ export { supabaseIntegration, instrumentSupabaseClient, instrumentMistralAiClient, + instrumentTypeSafeClient, instrumentOpenAiClient, instrumentAnthropicAiClient, instrumentGoogleGenAIClient, diff --git a/packages/bun/src/index.ts b/packages/bun/src/index.ts index 10aa2d422844..51660e47c37c 100644 --- a/packages/bun/src/index.ts +++ b/packages/bun/src/index.ts @@ -85,6 +85,7 @@ export { openAIIntegration, groqIntegration, togetherAIIntegration, + typesafeIntegration, langChainIntegration, langGraphIntegration, mastraIntegration, @@ -161,6 +162,7 @@ export { supabaseIntegration, instrumentSupabaseClient, instrumentMistralAiClient, + instrumentTypeSafeClient, instrumentOpenAiClient, instrumentAnthropicAiClient, instrumentGoogleGenAIClient, diff --git a/packages/cloudflare/src/index.ts b/packages/cloudflare/src/index.ts index acd492c06c15..2a6413cb2fdc 100644 --- a/packages/cloudflare/src/index.ts +++ b/packages/cloudflare/src/index.ts @@ -127,6 +127,7 @@ export { getOtlpTracesEndpoint, prismaIntegration, instrumentMistralAiClient, + instrumentTypeSafeClient, instrumentOpenAiClient, instrumentAnthropicAiClient, instrumentGoogleGenAIClient, diff --git a/packages/deno/src/index.ts b/packages/deno/src/index.ts index 3b44d8b6d875..88c795f5031b 100644 --- a/packages/deno/src/index.ts +++ b/packages/deno/src/index.ts @@ -147,6 +147,7 @@ export { openAIIntegration, groqIntegration, togetherAIIntegration, + typesafeIntegration, postgresIntegration, postgresJsIntegration, tediousIntegration, diff --git a/packages/elysia/src/index.ts b/packages/elysia/src/index.ts index 80e3c9e303ba..56e64cffb3ba 100644 --- a/packages/elysia/src/index.ts +++ b/packages/elysia/src/index.ts @@ -64,6 +64,7 @@ export { openAIIntegration, groqIntegration, togetherAIIntegration, + typesafeIntegration, langChainIntegration, langGraphIntegration, createFlueInstrumentation, @@ -138,6 +139,7 @@ export { supabaseIntegration, instrumentSupabaseClient, instrumentMistralAiClient, + instrumentTypeSafeClient, instrumentOpenAiClient, instrumentAnthropicAiClient, instrumentGoogleGenAIClient, diff --git a/packages/google-cloud-serverless/src/index.ts b/packages/google-cloud-serverless/src/index.ts index e86252691726..9b7bc0ff8abc 100644 --- a/packages/google-cloud-serverless/src/index.ts +++ b/packages/google-cloud-serverless/src/index.ts @@ -63,6 +63,7 @@ export { openAIIntegration, groqIntegration, togetherAIIntegration, + typesafeIntegration, langChainIntegration, langGraphIntegration, mastraIntegration, @@ -141,6 +142,7 @@ export { systemErrorIntegration, instrumentSupabaseClient, instrumentMistralAiClient, + instrumentTypeSafeClient, instrumentOpenAiClient, instrumentAnthropicAiClient, instrumentGoogleGenAIClient, diff --git a/packages/node/src/index.ts b/packages/node/src/index.ts index c4f2edc27610..ee9638ed98aa 100644 --- a/packages/node/src/index.ts +++ b/packages/node/src/index.ts @@ -31,6 +31,7 @@ export { mysql2Integration, openAIIntegration, togetherAIIntegration, + typesafeIntegration, postgresIntegration, postgresJsIntegration, redisIntegration, @@ -45,6 +46,7 @@ export { instrumentAnthropicAiClient, instrumentGoogleGenAIClient, instrumentMistralAiClient, + instrumentTypeSafeClient, createLangChainCallbackHandler, instrumentLangChainEmbeddings, instrumentStateGraph, diff --git a/packages/remix/src/server/index.ts b/packages/remix/src/server/index.ts index e47c09b72b01..3d713e88b0f3 100644 --- a/packages/remix/src/server/index.ts +++ b/packages/remix/src/server/index.ts @@ -127,6 +127,7 @@ export { instrumentAnthropicAiClient, instrumentGoogleGenAIClient, instrumentMistralAiClient, + instrumentTypeSafeClient, instrumentStateGraph, instrumentStateGraphCompile, zodErrorsIntegration, diff --git a/packages/server-utils/src/ai/index.ts b/packages/server-utils/src/ai/index.ts index d45ece7e300f..3fc0f4bd5fc9 100644 --- a/packages/server-utils/src/ai/index.ts +++ b/packages/server-utils/src/ai/index.ts @@ -8,6 +8,7 @@ export { instrumentOpenAiClient } from './openai'; export { instrumentAnthropicAiClient } from './anthropic-ai'; export { instrumentGoogleGenAIClient } from './google-genai'; export { instrumentMistralAiClient } from './mistral'; +export { instrumentTypeSafeClient } from './typesafe'; export { instrumentWorkersAiClient } from './workers-ai'; export { createLangChainCallbackHandler, instrumentLangChainEmbeddings } from './langchain'; export { instrumentStateGraph, instrumentStateGraphCompile, instrumentCreateReactAgent } from './langgraph'; diff --git a/packages/server-utils/src/ai/typesafe/constants.ts b/packages/server-utils/src/ai/typesafe/constants.ts new file mode 100644 index 000000000000..47c71bda6fb9 --- /dev/null +++ b/packages/server-utils/src/ai/typesafe/constants.ts @@ -0,0 +1,5 @@ +export const TYPESAFE_INTEGRATION_NAME = 'TypeSafe' as const; + +export const TYPESAFE_PROVIDER_NAME = 'typesafe' as const; + +export const TYPESAFE_ORIGIN = 'auto.ai.typesafe' as const; diff --git a/packages/server-utils/src/ai/typesafe/index.ts b/packages/server-utils/src/ai/typesafe/index.ts new file mode 100644 index 000000000000..55c25b2892ed --- /dev/null +++ b/packages/server-utils/src/ai/typesafe/index.ts @@ -0,0 +1,175 @@ +import { + GEN_AI_INPUT_MESSAGES, + GEN_AI_OPERATION_NAME, + GEN_AI_OUTPUT_MESSAGES, + GEN_AI_PROVIDER_NAME, + GEN_AI_REQUEST_MODEL, + GEN_AI_RESPONSE_MODEL, + GEN_AI_USAGE_INPUT_TOKENS, + GEN_AI_USAGE_OUTPUT_TOKENS, + GEN_AI_USAGE_TOTAL_TOKENS, +} from '@sentry/conventions/attributes'; +import type { Span, SpanAttributes } from '@sentry/core'; +import { + isObjectLike, + SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN, + SPAN_STATUS_ERROR, + startInactiveSpan, + stringify, + withActiveSpan, +} from '@sentry/core'; +import type { GenAiOptions } from '../core/utils'; +import { getGenAiSpanOp, resolveAIRecordingOptions } from '../core/utils'; +import { TYPESAFE_ORIGIN, TYPESAFE_PROVIDER_NAME } from './constants'; + +/** + * Start the span for a `systemOne(request)` call. The request model falls back to the client's + * `defaultModel`, the same way the SDK resolves it. + */ +export function startEvaluateSpan(request: unknown, client: unknown, recordInputs: boolean): Span { + const params = isObjectLike(request) ? request : {}; + const defaultModel = isObjectLike(client) ? client.defaultModel : undefined; + const model = + typeof params.model === 'string' ? params.model : typeof defaultModel === 'string' ? defaultModel : undefined; + + return startInactiveSpan({ + name: model ? `evaluate ${model}` : 'evaluate', + op: getGenAiSpanOp('evaluate'), + attributes: getRequestAttributes(params, model, recordInputs), + }); +} + +function getRequestAttributes( + request: Record, + model: string | undefined, + recordInputs: boolean, +): SpanAttributes { + return { + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: TYPESAFE_ORIGIN, + [GEN_AI_OPERATION_NAME]: 'evaluate', + [GEN_AI_PROVIDER_NAME]: TYPESAFE_PROVIDER_NAME, + ...(model ? { [GEN_AI_REQUEST_MODEL]: model } : {}), + ...(recordInputs + ? { + [GEN_AI_INPUT_MESSAGES]: stringify([ + { type: 'evaluation', state: request.state, questions: request.questions }, + ]), + } + : {}), + }; +} + +/** Add the response model, token usage and (optionally) the answers of a `systemOne` result. */ +export function addResponseAttributes(span: Span, result: unknown, recordOutputs: boolean): void { + if (!isObjectLike(result)) { + return; + } + + if (typeof result.model === 'string') { + span.setAttribute(GEN_AI_RESPONSE_MODEL, result.model); + } + + const usage = isObjectLike(result.usage) ? result.usage : undefined; + const inputTokens = typeof usage?.input_tokens === 'number' ? usage.input_tokens : undefined; + const outputTokens = typeof usage?.output_tokens === 'number' ? usage.output_tokens : undefined; + if (inputTokens !== undefined) { + span.setAttribute(GEN_AI_USAGE_INPUT_TOKENS, inputTokens); + } + if (outputTokens !== undefined) { + span.setAttribute(GEN_AI_USAGE_OUTPUT_TOKENS, outputTokens); + } + if (inputTokens !== undefined && outputTokens !== undefined) { + span.setAttribute(GEN_AI_USAGE_TOTAL_TOKENS, inputTokens + outputTokens); + } + + if (recordOutputs && result.answers !== undefined) { + span.setAttribute(GEN_AI_OUTPUT_MESSAGES, stringify([{ type: 'evaluation', answers: result.answers }])); + } +} + +/** + * `systemOne` returns a lazy `APIPromise` before the request settles, and parses the body on the first + * `.then()`. Wait on `asResponse()` (the raw fetch, which does not parse the body) and read a clone of + * the body, so the caller's own parse is unaffected. Call this before the caller can await the result, + * so the clone is taken first. Returns `false` when `result` is not an `APIPromise`. + */ +export function onSystemOneResponse( + result: unknown, + onBody: (body: unknown) => void, + onError: (error: unknown) => void, +): boolean { + if (!isObjectLike(result) || typeof result.asResponse !== 'function') { + return false; + } + + (result.asResponse() as Promise).then( + response => + response + .clone() + .json() + .then( + body => onBody(body), + () => onBody(undefined), + ), + error => onError(error), + ); + + return true; +} + +/** + * Instrument a TypeSafe client (`@typesafe-ai/sdk`) with Sentry tracing. + * Can be used across Node.js, Cloudflare Workers, and Vercel Edge. + */ +export function instrumentTypeSafeClient(client: T, options?: GenAiOptions): T { + return new Proxy(client, { + get(target, prop) { + // Read with the real client as receiver: the SDK keeps its API key in a private class field. + const value: unknown = Reflect.get(target, prop, target); + if (typeof value !== 'function') { + return value; + } + if (prop === 'systemOne') { + return (...args: unknown[]) => + instrumentSystemOne(value as (...args: unknown[]) => unknown, target, args, options); + } + return value.bind(target); + }, + }); +} + +function instrumentSystemOne( + systemOne: (...args: unknown[]) => unknown, + client: object, + args: unknown[], + options: GenAiOptions | undefined, +): unknown { + const { recordInputs, recordOutputs } = resolveAIRecordingOptions(options); + const span = startEvaluateSpan(args[0], client, recordInputs); + const endWithError = (): void => { + span.setStatus({ code: SPAN_STATUS_ERROR, message: 'internal_error' }); + span.end(); + }; + + let result: unknown; + try { + result = withActiveSpan(span, () => systemOne.apply(client, args)); + } catch (error) { + endWithError(); + throw error; + } + + const waiting = onSystemOneResponse( + result, + body => { + addResponseAttributes(span, body, recordOutputs); + span.end(); + }, + endWithError, + ); + if (!waiting) { + span.end(); + } + + return result; +} diff --git a/packages/server-utils/src/index.ts b/packages/server-utils/src/index.ts index 54e82b0dae6f..10c81b070d8a 100644 --- a/packages/server-utils/src/index.ts +++ b/packages/server-utils/src/index.ts @@ -53,6 +53,7 @@ export { mongooseIntegration } from './integrations/mongoose'; export { mistralAIIntegration } from './integrations/mistral'; export { groqIntegration } from './integrations/groq'; export { togetherAIIntegration } from './integrations/together-ai'; +export { typesafeIntegration } from './integrations/typesafe'; export { mysqlIntegration } from './integrations/mysql'; export { mysql2Integration } from './integrations/mysql2'; export { openAIIntegration } from './integrations/openai'; diff --git a/packages/server-utils/src/integrations/index.ts b/packages/server-utils/src/integrations/index.ts index fa6cf1759906..1c22770d2d1e 100644 --- a/packages/server-utils/src/integrations/index.ts +++ b/packages/server-utils/src/integrations/index.ts @@ -23,6 +23,7 @@ import { googleGenAIIntegration } from './google-genai'; import { mistralAIIntegration } from './mistral'; import { groqIntegration } from './groq'; import { togetherAIIntegration } from './together-ai'; +import { typesafeIntegration } from './typesafe'; import { postgresJsIntegration } from './postgres-js'; import { firebaseIntegration } from './firebase'; import { expressIntegration } from './express'; @@ -63,6 +64,7 @@ export function getTracingIntegrations(): Integration[] { mistralAIIntegration(), groqIntegration(), togetherAIIntegration(), + typesafeIntegration(), postgresJsIntegration(), firebaseIntegration(), ]; diff --git a/packages/server-utils/src/integrations/typesafe.ts b/packages/server-utils/src/integrations/typesafe.ts new file mode 100644 index 000000000000..56c10bb2db39 --- /dev/null +++ b/packages/server-utils/src/integrations/typesafe.ts @@ -0,0 +1,62 @@ +import * as diagnosticsChannel from 'node:diagnostics_channel'; +import type { IntegrationFn } from '@sentry/core'; +import { defineIntegration } from '@sentry/core'; +import type { GenAiOptions } from '../ai/core/utils'; +import { resolveAIRecordingOptions } from '../ai/core/utils'; +import { addResponseAttributes, onSystemOneResponse, startEvaluateSpan } from '../ai/typesafe'; +import { TYPESAFE_INTEGRATION_NAME } from '../ai/typesafe/constants'; +import { CHANNELS } from '../orchestrion/channels'; +import { typesafeModuleNames } from '../orchestrion/config/typesafe'; +import { invokeOrchestrionInstrumentation } from '../orchestrion/instrumentation'; +import { bindTracingChannelToSpan } from '../tracing-channel'; + +/** + * The context orchestrion shares across the tracing-channel lifecycle hooks: `arguments` is the live + * args array passed to `systemOne(request, options)`, `self` the `TypeSafeClient`, and `result` the + * returned `APIPromise` (replaced by the parsed response body once it arrives). `Sync` tracing keeps + * orchestrion from calling `.then()` on it, see `orchestrion/config/typesafe.ts`. + */ +interface TypeSafeChannelContext { + arguments: unknown[]; + self?: unknown; + result?: unknown; +} + +const _typesafeIntegration = ((options: GenAiOptions = {}) => { + return { + name: TYPESAFE_INTEGRATION_NAME, + setup(client) { + invokeOrchestrionInstrumentation(client, typesafeModuleNames, instrumentTypeSafe, [options]); + }, + }; +}) satisfies IntegrationFn; + +function instrumentTypeSafe(options: GenAiOptions): void { + bindTracingChannelToSpan( + diagnosticsChannel.tracingChannel(CHANNELS.TYPESAFE_SYSTEM_ONE), + data => startEvaluateSpan(data.arguments?.[0], data.self, resolveAIRecordingOptions(options).recordInputs), + { + beforeSpanEnd: (span, data) => { + if (!('error' in data)) { + addResponseAttributes(span, data.result, resolveAIRecordingOptions(options).recordOutputs); + } + }, + deferSpanEnd: ({ data, end }) => + onSystemOneResponse( + data.result, + body => { + data.result = body; + end(); + }, + error => end(error), + ), + }, + ); +} + +/** + * Diagnostics-channel-based integration for the TypeSafe SDK (`@typesafe-ai/sdk` >= 0.5.0 < 1). + * Subscribes to the `orchestrion:@typesafe-ai/sdk:system-one` channel injected into + * `TypeSafeClient.systemOne`, so it requires the Sentry runtime hook or bundler plugin. + */ +export const typesafeIntegration = defineIntegration(_typesafeIntegration); diff --git a/packages/server-utils/src/orchestrion/channels.ts b/packages/server-utils/src/orchestrion/channels.ts index a88495d6ed8c..0700f6e0e885 100644 --- a/packages/server-utils/src/orchestrion/channels.ts +++ b/packages/server-utils/src/orchestrion/channels.ts @@ -30,6 +30,7 @@ import { redisChannels } from './config/redis'; import { remixChannels } from './config/remix'; import { tediousChannels } from './config/tedious'; import { togetherAiChannels } from './config/together-ai'; +import { typesafeChannels } from './config/typesafe'; import { vercelAiChannels } from './config/vercel-ai'; /** @@ -81,6 +82,7 @@ export const CHANNELS = { ...remixChannels, ...tediousChannels, ...togetherAiChannels, + ...typesafeChannels, ...vercelAiChannels, } as const; diff --git a/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts b/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts index 881a38da55fd..92b3b4967efc 100644 --- a/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts +++ b/packages/server-utils/src/orchestrion/config/channel-integration-definitions.ts @@ -32,6 +32,7 @@ export const CHANNEL_INTEGRATION_DEFINITIONS = [ { exportName: 'mistralAIIntegration', modules: ['@mistralai/mistralai'] }, { exportName: 'groqIntegration', modules: ['groq-sdk'] }, { exportName: 'togetherAIIntegration', modules: ['together-ai'] }, + { exportName: 'typesafeIntegration', modules: ['@typesafe-ai/sdk'] }, { exportName: 'vercelAIIntegration', modules: ['ai'] }, { exportName: 'langChainIntegration', diff --git a/packages/server-utils/src/orchestrion/config/index.ts b/packages/server-utils/src/orchestrion/config/index.ts index 0ed9055935d4..8faa9464fcc5 100644 --- a/packages/server-utils/src/orchestrion/config/index.ts +++ b/packages/server-utils/src/orchestrion/config/index.ts @@ -35,6 +35,7 @@ import { redisConfig } from './redis'; import { remixConfig } from './remix'; import { tediousConfig } from './tedious'; import { togetherAiConfig } from './together-ai'; +import { typesafeConfig } from './typesafe'; import { vercelAiConfig } from './vercel-ai'; // Kept sorted alphabetically by module so concurrent additions insert at different // points rather than all appending to the end (fewer merge conflicts). @@ -86,6 +87,7 @@ export const SENTRY_INSTRUMENTATIONS: InstrumentationConfig[] = [ ...remixConfig, ...tediousConfig, ...togetherAiConfig, + ...typesafeConfig, ...vercelAiConfig, ]; diff --git a/packages/server-utils/src/orchestrion/config/typesafe.ts b/packages/server-utils/src/orchestrion/config/typesafe.ts new file mode 100644 index 000000000000..b62e97d2fd23 --- /dev/null +++ b/packages/server-utils/src/orchestrion/config/typesafe.ts @@ -0,0 +1,18 @@ +import type { InstrumentationConfig } from '../apmTypes'; + +import { getModuleNames } from './module-names'; + +// `systemOne` returns a lazy `APIPromise` whose body is parsed on the first `.then()`. `kind: 'Sync'` +// keeps orchestrion from calling `.then()` on it (which `Auto` would), so a caller's `asResponse()` +// still gets an unread body. The SDK ships one bundled file per module format. +export const typesafeConfig = ['dist/index.mjs', 'dist/index.cjs'].map(filePath => ({ + channelName: 'system-one', + module: { name: '@typesafe-ai/sdk', versionRange: '>=0.5.0 <1', filePath }, + functionQuery: { className: 'TypeSafeClient', methodName: 'systemOne', kind: 'Sync' as const }, +})) satisfies InstrumentationConfig[]; + +export const typesafeModuleNames = getModuleNames(typesafeConfig); + +export const typesafeChannels = { + TYPESAFE_SYSTEM_ONE: 'orchestrion:@typesafe-ai/sdk:system-one', +} as const; diff --git a/packages/sveltekit/src/server/index.ts b/packages/sveltekit/src/server/index.ts index 45932345abfb..0dc4dcc20c26 100644 --- a/packages/sveltekit/src/server/index.ts +++ b/packages/sveltekit/src/server/index.ts @@ -131,6 +131,7 @@ export { instrumentAnthropicAiClient, instrumentGoogleGenAIClient, instrumentMistralAiClient, + instrumentTypeSafeClient, instrumentStateGraph, instrumentStateGraphCompile, zodErrorsIntegration, diff --git a/packages/vercel-edge/src/index.ts b/packages/vercel-edge/src/index.ts index 9d32602e0c76..30d4a428eefb 100644 --- a/packages/vercel-edge/src/index.ts +++ b/packages/vercel-edge/src/index.ts @@ -105,6 +105,7 @@ export { openTelemetryIntegration, getOtlpTracesEndpoint, instrumentMistralAiClient, + instrumentTypeSafeClient, instrumentOpenAiClient, instrumentAnthropicAiClient, instrumentGoogleGenAIClient, From 59126fd3b89a7080594d4a7aed2de0c8f895fca1 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Fri, 25 Sep 2026 09:15:51 +0200 Subject: [PATCH 2/6] Cover lossless answers and disabled recording in evaluate tests --- .../tracing/typesafe/instrument-manual.mjs | 2 + .../suites/tracing/typesafe/instrument.mjs | 2 + .../suites/tracing/typesafe/scenario.mjs | 19 ++++++++- .../suites/tracing/typesafe/test.ts | 41 ++++++++++++++++++- 4 files changed, 62 insertions(+), 2 deletions(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument-manual.mjs b/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument-manual.mjs index b405a672e86c..49f6d0a0d5e7 100644 --- a/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument-manual.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument-manual.mjs @@ -6,6 +6,8 @@ Sentry.init({ release: '1.0', tracesSampleRate: 1.0, transport: loggingTransport, + // `NO_RECORDING` turns off recording of inputs and outputs for the privacy test. + ...(process.env.NO_RECORDING ? { dataCollection: { genAI: { inputs: false, outputs: false } } } : {}), // `instrumentTypeSafeClient` is the manual path for runtimes without the orchestrion hook. // Drop the automatic integration so the scenario exercises it alone. integrations: integrations => integrations.filter(integration => integration.name !== 'TypeSafe'), diff --git a/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument.mjs index 46a27dd03b74..595be514d00c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument.mjs @@ -6,4 +6,6 @@ Sentry.init({ release: '1.0', tracesSampleRate: 1.0, transport: loggingTransport, + // `NO_RECORDING` turns off recording of inputs and outputs for the privacy test. + ...(process.env.NO_RECORDING ? { dataCollection: { genAI: { inputs: false, outputs: false } } } : {}), }); diff --git a/dev-packages/node-integration-tests/suites/tracing/typesafe/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/typesafe/scenario.mjs index 609265208b9b..919e76893950 100644 --- a/dev-packages/node-integration-tests/suites/tracing/typesafe/scenario.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/typesafe/scenario.mjs @@ -1,5 +1,5 @@ import * as Sentry from '@sentry/node'; -import { noul, score, TypeSafeClient } from '@typesafe-ai/sdk'; +import { choice, noul, score, TypeSafeClient } from '@typesafe-ai/sdk'; import express from 'express'; function startMockServer() { @@ -16,6 +16,12 @@ function startMockServer() { model: 'jev-1.13.0', answers: { authIssue: { type: 'noul', noul: 0.98 }, + department: { + type: 'choice', + choice: 'billing', + confidence: 0.28, + probabilities: { billing: 0.64, technical: 0.36 }, + }, urgency: { type: 'score', score: 1.58, @@ -48,6 +54,10 @@ async function run() { const state = 'I cannot log in, and I also want a refund for last month.'; const questions = { authIssue: noul('Is there a login problem?'), + department: choice('Which team should handle this?', { + billing: 'Charges and refunds', + technical: 'Bugs and outages', + }), urgency: score('How urgent is this ticket?', ['low', 'medium', 'high']), }; @@ -63,6 +73,13 @@ async function run() { } catch { // expected } + + // The SDK rejects empty questions with a `TypeSafeError` before sending the request. + try { + await client.systemOne({ model: 'validation-error', state, questions: {} }); + } catch { + // expected + } }); server.close(); diff --git a/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts b/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts index b2268845f30b..5f8d95e9b743 100644 --- a/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts @@ -36,7 +36,7 @@ describe('TypeSafe integration', () => { const evaluateSpans = container.items.filter( span => span.attributes[SENTRY_OP]?.value === 'gen_ai.evaluate', ); - expect(evaluateSpans).toHaveLength(3); + expect(evaluateSpans).toHaveLength(4); for (const span of evaluateSpans) { expect(span.attributes[SENTRY_ORIGIN]?.value).toBe('auto.ai.typesafe'); expect(span.attributes[GEN_AI_OPERATION_NAME]?.value).toBe('evaluate'); @@ -57,6 +57,11 @@ describe('TypeSafe integration', () => { state: 'I cannot log in, and I also want a refund for last month.', questions: { authIssue: { type: 'noul', instructions: 'Is there a login problem?' }, + department: { + type: 'choice', + instructions: 'Which team should handle this?', + criteria: { billing: 'Charges and refunds', technical: 'Bugs and outages' }, + }, urgency: { type: 'score', instructions: 'How urgent is this ticket?', @@ -70,6 +75,12 @@ describe('TypeSafe integration', () => { type: 'evaluation', answers: { authIssue: { type: 'noul', noul: 0.98 }, + department: { + type: 'choice', + choice: 'billing', + confidence: 0.28, + probabilities: { billing: 0.64, technical: 0.36 }, + }, urgency: { type: 'score', score: 1.58, @@ -91,6 +102,34 @@ describe('TypeSafe integration', () => { expect(errorSpan.status).toBe('error'); expect(errorSpan.attributes[GEN_AI_RESPONSE_MODEL]).toBeUndefined(); expect(errorSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); + + const validationErrorSpan = evaluateSpans.find(span => span.name === 'evaluate validation-error')!; + expect(validationErrorSpan).toBeDefined(); + expect(validationErrorSpan.status).toBe('error'); + expect(validationErrorSpan.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); + }, + }) + .start() + .completed(); + }); + + test('does not record inputs or outputs when recording is off', async () => { + await createRunner() + .withEnv({ NO_RECORDING: 'true' }) + .unordered() + .expect({ + span: container => { + const evaluateSpans = container.items.filter( + span => span.attributes[SENTRY_OP]?.value === 'gen_ai.evaluate', + ); + expect(evaluateSpans).toHaveLength(4); + expect(evaluateSpans.filter(span => span.status === 'error')).toHaveLength(2); + for (const span of evaluateSpans) { + expect(span.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); + expect(span.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); + // State, questions and answers must not come back through another attribute (e.g. an error message). + expect(JSON.stringify(span)).not.toMatch(/cannot log in|Charges and refunds|0\.98/); + } }, }) .start() From 1b61cafab7dd397fb23cac19b56eae246c8493ae Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Fri, 25 Sep 2026 10:26:33 +0200 Subject: [PATCH 3/6] Check only span attributes in evaluate privacy tests --- .../node-integration-tests/suites/tracing/typesafe/test.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts b/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts index 5f8d95e9b743..ad492d196acc 100644 --- a/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts @@ -128,7 +128,11 @@ describe('TypeSafe integration', () => { expect(span.attributes[GEN_AI_INPUT_MESSAGES]).toBeUndefined(); expect(span.attributes[GEN_AI_OUTPUT_MESSAGES]).toBeUndefined(); // State, questions and answers must not come back through another attribute (e.g. an error message). - expect(JSON.stringify(span)).not.toMatch(/cannot log in|Charges and refunds|0\.98/); + // Only the attributes are checked (timestamps could match a number), and `probabilities` only + // occurs in answers. + expect(JSON.stringify(span.attributes)).not.toMatch( + /cannot log in|Charges and refunds|probabilities/, + ); } }, }) From 58a108efb991574fd6b9d7f6747e11dab1c3bc5c Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Fri, 25 Sep 2026 11:32:49 +0200 Subject: [PATCH 4/6] Drop runtime list from instrumentTypeSafeClient docs --- packages/server-utils/src/ai/typesafe/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server-utils/src/ai/typesafe/index.ts b/packages/server-utils/src/ai/typesafe/index.ts index 55c25b2892ed..b8fe48878024 100644 --- a/packages/server-utils/src/ai/typesafe/index.ts +++ b/packages/server-utils/src/ai/typesafe/index.ts @@ -119,7 +119,6 @@ export function onSystemOneResponse( /** * Instrument a TypeSafe client (`@typesafe-ai/sdk`) with Sentry tracing. - * Can be used across Node.js, Cloudflare Workers, and Vercel Edge. */ export function instrumentTypeSafeClient(client: T, options?: GenAiOptions): T { return new Proxy(client, { From c149d104979d4d70153bb3a94cd73a4eb0248b39 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Fri, 25 Sep 2026 14:17:49 +0200 Subject: [PATCH 5/6] Use separate instrument files for disabled recording --- .../instrument-manual-no-recording.mjs | 13 +++++++++++++ .../tracing/typesafe/instrument-manual.mjs | 2 -- .../typesafe/instrument-no-recording.mjs | 10 ++++++++++ .../suites/tracing/typesafe/instrument.mjs | 2 -- .../suites/tracing/typesafe/test.ts | 19 +++++++++++++++---- 5 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/typesafe/instrument-manual-no-recording.mjs create mode 100644 dev-packages/node-integration-tests/suites/tracing/typesafe/instrument-no-recording.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument-manual-no-recording.mjs b/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument-manual-no-recording.mjs new file mode 100644 index 000000000000..605a0ca368f2 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument-manual-no-recording.mjs @@ -0,0 +1,13 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, + dataCollection: { genAI: { inputs: false, outputs: false } }, + // `instrumentTypeSafeClient` is the manual path for runtimes without the orchestrion hook. + // Drop the automatic integration so the scenario exercises it alone. + integrations: integrations => integrations.filter(integration => integration.name !== 'TypeSafe'), +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument-manual.mjs b/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument-manual.mjs index 49f6d0a0d5e7..b405a672e86c 100644 --- a/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument-manual.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument-manual.mjs @@ -6,8 +6,6 @@ Sentry.init({ release: '1.0', tracesSampleRate: 1.0, transport: loggingTransport, - // `NO_RECORDING` turns off recording of inputs and outputs for the privacy test. - ...(process.env.NO_RECORDING ? { dataCollection: { genAI: { inputs: false, outputs: false } } } : {}), // `instrumentTypeSafeClient` is the manual path for runtimes without the orchestrion hook. // Drop the automatic integration so the scenario exercises it alone. integrations: integrations => integrations.filter(integration => integration.name !== 'TypeSafe'), diff --git a/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument-no-recording.mjs b/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument-no-recording.mjs new file mode 100644 index 000000000000..ed7a6cfe9f0a --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument-no-recording.mjs @@ -0,0 +1,10 @@ +import * as Sentry from '@sentry/node'; +import { loggingTransport } from '@sentry-internal/node-integration-tests'; + +Sentry.init({ + dsn: 'https://public@dsn.ingest.sentry.io/1337', + release: '1.0', + tracesSampleRate: 1.0, + transport: loggingTransport, + dataCollection: { genAI: { inputs: false, outputs: false } }, +}); diff --git a/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument.mjs b/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument.mjs index 595be514d00c..46a27dd03b74 100644 --- a/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/typesafe/instrument.mjs @@ -6,6 +6,4 @@ Sentry.init({ release: '1.0', tracesSampleRate: 1.0, transport: loggingTransport, - // `NO_RECORDING` turns off recording of inputs and outputs for the privacy test. - ...(process.env.NO_RECORDING ? { dataCollection: { genAI: { inputs: false, outputs: false } } } : {}), }); diff --git a/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts b/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts index ad492d196acc..eb49b623ed93 100644 --- a/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts @@ -20,9 +20,9 @@ describe('TypeSafe integration', () => { }); describe.each([ - ['automatic', 'instrument.mjs'], - ['manual', 'instrument-manual.mjs'], - ])('%s instrumentation', (_, instrumentFile) => { + ['automatic', 'instrument.mjs', 'instrument-no-recording.mjs'], + ['manual', 'instrument-manual.mjs', 'instrument-manual-no-recording.mjs'], + ])('%s instrumentation', (_, instrumentFile, noRecordingInstrumentFile) => { createEsmAndCjsTests( __dirname, 'scenario.mjs', @@ -112,10 +112,21 @@ describe('TypeSafe integration', () => { .start() .completed(); }); + }, + { + additionalDependencies: { + '@typesafe-ai/sdk': '^0.6.0', + }, + }, + ); + createEsmAndCjsTests( + __dirname, + 'scenario.mjs', + noRecordingInstrumentFile, + (createRunner, test) => { test('does not record inputs or outputs when recording is off', async () => { await createRunner() - .withEnv({ NO_RECORDING: 'true' }) .unordered() .expect({ span: container => { From f21c21cf232a9f612d7e89fe86fd79c223fa6930 Mon Sep 17 00:00:00 2001 From: Andrei Borza Date: Fri, 25 Sep 2026 17:04:15 +0200 Subject: [PATCH 6/6] Split TypeSafe scenario into automatic and manual variants --- .../tracing/typesafe/scenario-manual.mjs | 87 +++++++++++++++++++ .../suites/tracing/typesafe/scenario.mjs | 5 +- .../suites/tracing/typesafe/test.ts | 10 +-- 3 files changed, 93 insertions(+), 9 deletions(-) create mode 100644 dev-packages/node-integration-tests/suites/tracing/typesafe/scenario-manual.mjs diff --git a/dev-packages/node-integration-tests/suites/tracing/typesafe/scenario-manual.mjs b/dev-packages/node-integration-tests/suites/tracing/typesafe/scenario-manual.mjs new file mode 100644 index 000000000000..30dcd790a49d --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/typesafe/scenario-manual.mjs @@ -0,0 +1,87 @@ +import * as Sentry from '@sentry/node'; +import { choice, noul, score, TypeSafeClient } from '@typesafe-ai/sdk'; +import express from 'express'; + +function startMockServer() { + const app = express(); + app.use(express.json()); + + app.post('/v1/systemone', (req, res) => { + if (req.body.model === 'error-model') { + res.status(400).json({ error: 'Unknown model' }); + return; + } + + res.json({ + model: 'jev-1.13.0', + answers: { + authIssue: { type: 'noul', noul: 0.98 }, + department: { + type: 'choice', + choice: 'billing', + confidence: 0.28, + probabilities: { billing: 0.64, technical: 0.36 }, + }, + urgency: { + type: 'score', + score: 1.58, + confidence: 0.9, + legend: { 0: 'low', 1: 'medium', 2: 'high' }, + probabilities: { 0: 0, 1: 0.42, 2: 0.58 }, + }, + }, + usage: { input_tokens: 275, output_tokens: 20 }, + }); + }); + + return new Promise(resolve => { + const server = app.listen(0, () => { + resolve(server); + }); + }); +} + +async function run() { + const server = await startMockServer(); + const client = Sentry.instrumentTypeSafeClient( + new TypeSafeClient({ + apiKey: 'mock-api-key', + baseURL: `http://localhost:${server.address().port}`, + retry: { maxRetries: 0 }, + }), + ); + const state = 'I cannot log in, and I also want a refund for last month.'; + const questions = { + authIssue: noul('Is there a login problem?'), + department: choice('Which team should handle this?', { + billing: 'Charges and refunds', + technical: 'Bugs and outages', + }), + urgency: score('How urgent is this ticket?', ['low', 'medium', 'high']), + }; + + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + await client.systemOne({ state, questions }); + + // The instrumentation must not read the body the caller gets from `asResponse()`. + const response = await client.systemOne({ model: 'jev-1.13', state, questions }).asResponse(); + await response.json(); + + try { + await client.systemOne({ model: 'error-model', state, questions }); + } catch { + // expected + } + + // The SDK rejects empty questions with a `TypeSafeError` before sending the request. + try { + await client.systemOne({ model: 'validation-error', state, questions: {} }); + } catch { + // expected + } + }); + + server.close(); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/typesafe/scenario.mjs b/dev-packages/node-integration-tests/suites/tracing/typesafe/scenario.mjs index 919e76893950..694a9fd97d59 100644 --- a/dev-packages/node-integration-tests/suites/tracing/typesafe/scenario.mjs +++ b/dev-packages/node-integration-tests/suites/tracing/typesafe/scenario.mjs @@ -43,14 +43,11 @@ function startMockServer() { async function run() { const server = await startMockServer(); - const typesafe = new TypeSafeClient({ + const client = new TypeSafeClient({ apiKey: 'mock-api-key', baseURL: `http://localhost:${server.address().port}`, retry: { maxRetries: 0 }, }); - const client = Sentry.getClient().getIntegrationByName('TypeSafe') - ? typesafe - : Sentry.instrumentTypeSafeClient(typesafe); const state = 'I cannot log in, and I also want a refund for last month.'; const questions = { authIssue: noul('Is there a login problem?'), diff --git a/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts b/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts index eb49b623ed93..e5feb8919745 100644 --- a/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/typesafe/test.ts @@ -20,12 +20,12 @@ describe('TypeSafe integration', () => { }); describe.each([ - ['automatic', 'instrument.mjs', 'instrument-no-recording.mjs'], - ['manual', 'instrument-manual.mjs', 'instrument-manual-no-recording.mjs'], - ])('%s instrumentation', (_, instrumentFile, noRecordingInstrumentFile) => { + ['automatic', 'scenario.mjs', 'instrument.mjs', 'instrument-no-recording.mjs'], + ['manual', 'scenario-manual.mjs', 'instrument-manual.mjs', 'instrument-manual-no-recording.mjs'], + ])('%s instrumentation', (_, scenarioFile, instrumentFile, noRecordingInstrumentFile) => { createEsmAndCjsTests( __dirname, - 'scenario.mjs', + scenarioFile, instrumentFile, (createRunner, test) => { test('creates evaluate spans for systemOne', async () => { @@ -122,7 +122,7 @@ describe('TypeSafe integration', () => { createEsmAndCjsTests( __dirname, - 'scenario.mjs', + scenarioFile, noRecordingInstrumentFile, (createRunner, test) => { test('does not record inputs or outputs when recording is off', async () => {