diff --git a/.agents/skills/source-command-add-connector/SKILL.md b/.agents/skills/source-command-add-connector/SKILL.md new file mode 100644 index 00000000000..1b70b7a426f --- /dev/null +++ b/.agents/skills/source-command-add-connector/SKILL.md @@ -0,0 +1,534 @@ +--- +name: "source-command-add-connector" +description: "Add a knowledge base connector for syncing documents from an external source" +--- + +# source-command-add-connector + +Use this skill when the user asks to run the migrated source command `add-connector`. + +## Command Template + +# Add Connector Skill + +You are an expert at adding knowledge base connectors to Sim. A connector syncs documents from an external source (Confluence, Google Drive, Notion, etc.) into a knowledge base. + +## Your Task + +When the user asks you to create a connector: +1. Use Context7 or WebFetch to read the service's API documentation +2. Determine the auth mode: **OAuth** (if Sim already has an OAuth provider for the service) or **API key** (if the service uses API key / Bearer token auth) +3. Create the connector directory and config +4. Register it in the connector registry + +## Directory Structure + +Create files in `apps/sim/connectors/{service}/`: +``` +connectors/{service}/ +├── index.ts # Barrel export +└── {service}.ts # ConnectorConfig definition +``` + +## Authentication + +Connectors use a discriminated union for auth config (`ConnectorAuthConfig` in `connectors/types.ts`): + +```typescript +type ConnectorAuthConfig = + | { mode: 'oauth'; provider: OAuthService; requiredScopes?: string[] } + | { mode: 'apiKey'; label?: string; placeholder?: string } +``` + +### OAuth mode +For services with existing OAuth providers in `apps/sim/lib/oauth/types.ts`. The `provider` must match an `OAuthService`. The modal shows a credential picker and handles token refresh automatically. + +### API key mode +For services that use API key / Bearer token auth. The modal shows a password input with the configured `label` and `placeholder`. The API key is encrypted at rest using AES-256-GCM and stored in a dedicated `encryptedApiKey` column on the connector record. The sync engine decrypts it automatically — connectors receive the raw access token in `listDocuments`, `getDocument`, and `validateConfig`. + +## ConnectorConfig Structure + +### OAuth connector example + +```typescript +import { createLogger } from '@sim/logger' +import { {Service}Icon } from '@/components/icons' +import { fetchWithRetry } from '@/lib/knowledge/documents/utils' +import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' + +const logger = createLogger('{Service}Connector') + +export const {service}Connector: ConnectorConfig = { + id: '{service}', + name: '{Service}', + description: 'Sync documents from {Service} into your knowledge base', + version: '1.0.0', + icon: {Service}Icon, + + auth: { + mode: 'oauth', + provider: '{service}', // Must match OAuthService in lib/oauth/types.ts + requiredScopes: ['read:...'], + }, + + configFields: [ + // Rendered dynamically by the add-connector modal UI + // Supports 'short-input' and 'dropdown' types + ], + + listDocuments: async (accessToken, sourceConfig, cursor) => { + // Return metadata stubs with contentDeferred: true (if per-doc content fetch needed) + // Or full documents with content (if list API returns content inline) + // Return { documents: ExternalDocument[], nextCursor?, hasMore } + }, + + getDocument: async (accessToken, sourceConfig, externalId) => { + // Fetch full content for a single document + // Return ExternalDocument with contentDeferred: false, or null + }, + + validateConfig: async (accessToken, sourceConfig) => { + // Return { valid: true } or { valid: false, error: 'message' } + }, + + // Optional: map source metadata to semantic tag keys (translated to slots by sync engine) + mapTags: (metadata) => { + // Return Record with keys matching tagDefinitions[].id + }, +} +``` + +### API key connector example + +```typescript +export const {service}Connector: ConnectorConfig = { + id: '{service}', + name: '{Service}', + description: 'Sync documents from {Service} into your knowledge base', + version: '1.0.0', + icon: {Service}Icon, + + auth: { + mode: 'apiKey', + label: 'API Key', // Shown above the input field + placeholder: 'Enter your {Service} API key', // Input placeholder + }, + + configFields: [ /* ... */ ], + listDocuments: async (accessToken, sourceConfig, cursor) => { /* ... */ }, + getDocument: async (accessToken, sourceConfig, externalId) => { /* ... */ }, + validateConfig: async (accessToken, sourceConfig) => { /* ... */ }, +} +``` + +## ConfigField Types + +The add-connector modal renders these automatically — no custom UI needed. + +Three field types are supported: `short-input`, `dropdown`, and `selector`. + +```typescript +// Text input +{ + id: 'domain', + title: 'Domain', + type: 'short-input', + placeholder: 'yoursite.example.com', + required: true, +} + +// Dropdown (static options) +{ + id: 'contentType', + title: 'Content Type', + type: 'dropdown', + required: false, + options: [ + { label: 'Pages only', id: 'page' }, + { label: 'Blog posts only', id: 'blogpost' }, + { label: 'All content', id: 'all' }, + ], +} +``` + +## Dynamic Selectors (Canonical Pairs) + +Use `type: 'selector'` to fetch options dynamically from the existing selector registry (`hooks/selectors/registry.ts`). Selectors are always paired with a manual fallback input using the **canonical pair** pattern — a `selector` field (basic mode) and a `short-input` field (advanced mode) linked by `canonicalParamId`. + +The user sees a toggle button (ArrowLeftRight) to switch between the selector dropdown and manual text input. On submit, the modal resolves each canonical pair to the active mode's value, keyed by `canonicalParamId`. + +### Rules + +1. **Every selector field MUST have a canonical pair** — a corresponding `short-input` (or `dropdown`) field with the same `canonicalParamId` and `mode: 'advanced'`. +2. **`required` must be set identically on both fields** in a pair. If the selector is required, the manual input must also be required. +3. **`canonicalParamId` must match the key the connector expects in `sourceConfig`** (e.g. `baseId`, `channel`, `teamId`). The advanced field's `id` should typically match `canonicalParamId`. +4. **`dependsOn` references the selector field's `id`**, not the `canonicalParamId`. The modal propagates dependency clearing across canonical siblings automatically — changing either field in a parent pair clears dependent children. + +### Selector canonical pair example (Airtable base → table cascade) + +```typescript +configFields: [ + // Base: selector (basic) + manual (advanced) + { + id: 'baseSelector', + title: 'Base', + type: 'selector', + selectorKey: 'airtable.bases', // Must exist in hooks/selectors/registry.ts + canonicalParamId: 'baseId', + mode: 'basic', + placeholder: 'Select a base', + required: true, + }, + { + id: 'baseId', + title: 'Base ID', + type: 'short-input', + canonicalParamId: 'baseId', + mode: 'advanced', + placeholder: 'e.g. appXXXXXXXXXXXXXX', + required: true, + }, + // Table: selector depends on base (basic) + manual (advanced) + { + id: 'tableSelector', + title: 'Table', + type: 'selector', + selectorKey: 'airtable.tables', + canonicalParamId: 'tableIdOrName', + mode: 'basic', + dependsOn: ['baseSelector'], // References the selector field ID + placeholder: 'Select a table', + required: true, + }, + { + id: 'tableIdOrName', + title: 'Table Name or ID', + type: 'short-input', + canonicalParamId: 'tableIdOrName', + mode: 'advanced', + placeholder: 'e.g. Tasks', + required: true, + }, + // Non-selector fields stay as-is + { id: 'maxRecords', title: 'Max Records', type: 'short-input', ... }, +] +``` + +### Selector with domain dependency (Jira/Confluence pattern) + +When a selector depends on a plain `short-input` field (no canonical pair), `dependsOn` references that field's `id` directly. The `domain` field's value maps to `SelectorContext.domain` automatically via `SELECTOR_CONTEXT_FIELDS`. + +```typescript +configFields: [ + { + id: 'domain', + title: 'Jira Domain', + type: 'short-input', + placeholder: 'yoursite.atlassian.net', + required: true, + }, + { + id: 'projectSelector', + title: 'Project', + type: 'selector', + selectorKey: 'jira.projects', + canonicalParamId: 'projectKey', + mode: 'basic', + dependsOn: ['domain'], + placeholder: 'Select a project', + required: true, + }, + { + id: 'projectKey', + title: 'Project Key', + type: 'short-input', + canonicalParamId: 'projectKey', + mode: 'advanced', + placeholder: 'e.g. ENG, PROJ', + required: true, + }, +] +``` + +### How `dependsOn` maps to `SelectorContext` + +The connector selector field builds a `SelectorContext` from dependency values. For the mapping to work, each dependency's `canonicalParamId` (or field `id` for non-canonical fields) must exist in `SELECTOR_CONTEXT_FIELDS` (`lib/workflows/subblocks/context.ts`): + +``` +oauthCredential, domain, teamId, projectId, knowledgeBaseId, planId, +siteId, collectionId, spreadsheetId, fileId, baseId, datasetId, serviceDeskId +``` + +### Available selector keys + +Check `hooks/selectors/types.ts` for the full `SelectorKey` union. Common ones for connectors: + +| SelectorKey | Context Deps | Returns | +|-------------|-------------|---------| +| `airtable.bases` | credential | Base ID + name | +| `airtable.tables` | credential, `baseId` | Table ID + name | +| `slack.channels` | credential | Channel ID + name | +| `gmail.labels` | credential | Label ID + name | +| `google.calendar` | credential | Calendar ID + name | +| `linear.teams` | credential | Team ID + name | +| `linear.projects` | credential, `teamId` | Project ID + name | +| `jira.projects` | credential, `domain` | Project key + name | +| `confluence.spaces` | credential, `domain` | Space key + name | +| `notion.databases` | credential | Database ID + name | +| `asana.workspaces` | credential | Workspace GID + name | +| `microsoft.teams` | credential | Team ID + name | +| `microsoft.channels` | credential, `teamId` | Channel ID + name | +| `webflow.sites` | credential | Site ID + name | +| `outlook.folders` | credential | Folder ID + name | + +## ExternalDocument Shape + +Every document returned from `listDocuments`/`getDocument` must include: + +```typescript +{ + externalId: string // Source-specific unique ID + title: string // Document title + content: string // Extracted plain text (or '' if contentDeferred) + contentDeferred?: boolean // true = content will be fetched via getDocument + mimeType: 'text/plain' // Always text/plain (content is extracted) + contentHash: string // Metadata-based hash for change detection + sourceUrl?: string // Link back to original (stored on document record) + metadata?: Record // Source-specific data (fed to mapTags) +} +``` + +## Content Deferral (Required for file/content-download connectors) + +**All connectors that require per-document API calls to fetch content MUST use `contentDeferred: true`.** This is the standard pattern — `listDocuments` returns lightweight metadata stubs, and content is fetched lazily by the sync engine via `getDocument` only for new/changed documents. + +This pattern is critical for reliability: the sync engine processes documents in batches and enqueues each batch for processing immediately. If a sync times out, all previously-batched documents are already queued. Without deferral, content downloads during listing can exhaust the sync task's time budget before any documents are saved. + +### When to use `contentDeferred: true` + +- The service's list API does NOT return document content (only metadata) +- Content requires a separate download/export API call per document +- Examples: Google Drive, OneDrive, SharePoint, Dropbox, Notion, Confluence, Gmail, Obsidian, Evernote, GitHub + +### When NOT to use `contentDeferred` + +- The list API already returns the full content inline (e.g., Slack messages, Reddit posts, HubSpot notes) +- No per-document API call is needed to get content + +### Content Hash Strategy + +Use a **metadata-based** `contentHash` — never a content-based hash. The hash must be derivable from the list response metadata alone, so the sync engine can detect changes without downloading content. + +Good metadata hash sources: +- `modifiedTime` / `lastModifiedDateTime` — changes when file is edited +- Git blob SHA — unique per content version +- API-provided content hash (e.g., Dropbox `content_hash`) +- Version number (e.g., Confluence page version) + +Format: `{service}:{id}:{changeIndicator}` + +```typescript +// Google Drive: modifiedTime changes on edit +contentHash: `gdrive:${file.id}:${file.modifiedTime ?? ''}` + +// GitHub: blob SHA is a content-addressable hash +contentHash: `gitsha:${item.sha}` + +// Dropbox: API provides content_hash +contentHash: `dropbox:${entry.id}:${entry.content_hash ?? entry.server_modified}` + +// Confluence: version number increments on edit +contentHash: `confluence:${page.id}:${page.version.number}` +``` + +**Critical invariant:** The `contentHash` MUST be identical whether produced by `listDocuments` (stub) or `getDocument` (full doc). Both should use the same stub function to guarantee this. + +### Implementation Pattern + +```typescript +// 1. Create a stub function (sync, no API calls) +function fileToStub(file: ServiceFile): ExternalDocument { + return { + externalId: file.id, + title: file.name || 'Untitled', + content: '', + contentDeferred: true, + mimeType: 'text/plain', + sourceUrl: `https://service.com/file/${file.id}`, + contentHash: `service:${file.id}:${file.modifiedTime ?? ''}`, + metadata: { /* fields needed by mapTags */ }, + } +} + +// 2. listDocuments returns stubs (fast, metadata only) +listDocuments: async (accessToken, sourceConfig, cursor) => { + const response = await fetchWithRetry(listUrl, { ... }) + const files = (await response.json()).files + const documents = files.map(fileToStub) + return { documents, nextCursor, hasMore } +} + +// 3. getDocument fetches content and returns full doc with SAME contentHash +getDocument: async (accessToken, sourceConfig, externalId) => { + const metadata = await fetchWithRetry(metadataUrl, { ... }) + const file = await metadata.json() + if (file.trashed) return null + + try { + const content = await fetchContent(accessToken, file) + if (!content.trim()) return null + const stub = fileToStub(file) + return { ...stub, content, contentDeferred: false } + } catch (error) { + logger.warn(`Failed to fetch content for: ${file.name}`, { error }) + return null + } +} +``` + +### Reference Implementations + +- **Google Drive**: `connectors/google-drive/google-drive.ts` — file download/export with `modifiedTime` hash +- **GitHub**: `connectors/github/github.ts` — git blob SHA hash +- **Notion**: `connectors/notion/notion.ts` — blocks API with `last_edited_time` hash +- **Confluence**: `connectors/confluence/confluence.ts` — version number hash + +## tagDefinitions — Declared Tag Definitions + +Declare which tags the connector populates using semantic IDs. Shown in the add-connector modal as opt-out checkboxes. +On connector creation, slots are **dynamically assigned** via `getNextAvailableSlot` — connectors never hardcode slot names. + +```typescript +tagDefinitions: [ + { id: 'labels', displayName: 'Labels', fieldType: 'text' }, + { id: 'version', displayName: 'Version', fieldType: 'number' }, + { id: 'lastModified', displayName: 'Last Modified', fieldType: 'date' }, +], +``` + +Each entry has: +- `id`: Semantic key matching a key returned by `mapTags` (e.g. `'labels'`, `'version'`) +- `displayName`: Human-readable name shown in the UI (e.g. "Labels", "Last Modified") +- `fieldType`: `'text'` | `'number'` | `'date'` | `'boolean'` — determines which slot pool to draw from + +Users can opt out of specific tags in the modal. Disabled IDs are stored in `sourceConfig.disabledTagIds`. +The assigned mapping (`semantic id → slot`) is stored in `sourceConfig.tagSlotMapping`. + +## mapTags — Metadata to Semantic Keys + +Maps source metadata to semantic tag keys. Required if `tagDefinitions` is set. +The sync engine calls this automatically and translates semantic keys to actual DB slots +using the `tagSlotMapping` stored on the connector. + +Return keys must match the `id` values declared in `tagDefinitions`. + +```typescript +mapTags: (metadata: Record): Record => { + const result: Record = {} + + // Validate arrays before casting — metadata may be malformed + const labels = Array.isArray(metadata.labels) ? (metadata.labels as string[]) : [] + if (labels.length > 0) result.labels = labels.join(', ') + + // Validate numbers — guard against NaN + if (metadata.version != null) { + const num = Number(metadata.version) + if (!Number.isNaN(num)) result.version = num + } + + // Validate dates — guard against Invalid Date + if (typeof metadata.lastModified === 'string') { + const date = new Date(metadata.lastModified) + if (!Number.isNaN(date.getTime())) result.lastModified = date + } + + return result +} +``` + +## External API Calls — Use `fetchWithRetry` + +All external API calls must use `fetchWithRetry` from `@/lib/knowledge/documents/utils` instead of raw `fetch()`. This provides exponential backoff with retries on 429/502/503/504 errors. It returns a standard `Response` — all `.ok`, `.json()`, `.text()` checks work unchanged. + +For `validateConfig` (user-facing, called on save), pass `VALIDATE_RETRY_OPTIONS` to cap wait time at ~7s. Background operations (`listDocuments`, `getDocument`) use the built-in defaults (5 retries, ~31s max). + +```typescript +import { VALIDATE_RETRY_OPTIONS, fetchWithRetry } from '@/lib/knowledge/documents/utils' + +// Background sync — use defaults +const response = await fetchWithRetry(url, { + method: 'GET', + headers: { Authorization: `Bearer ${accessToken}` }, +}) + +// validateConfig — tighter retry budget +const response = await fetchWithRetry(url, { ... }, VALIDATE_RETRY_OPTIONS) +``` + +## sourceUrl + +If `ExternalDocument.sourceUrl` is set, the sync engine stores it on the document record. Always construct the full URL (not a relative path). + +## Sync Engine Behavior (Do Not Modify) + +The sync engine (`lib/knowledge/connectors/sync-engine.ts`) is connector-agnostic. It: +1. Calls `listDocuments` with pagination until `hasMore` is false +2. Compares `contentHash` to detect new/changed/unchanged documents +3. Stores `sourceUrl` and calls `mapTags` on insert/update automatically +4. Handles soft-delete of removed documents +5. Resolves access tokens automatically — OAuth tokens are refreshed, API keys are decrypted from the `encryptedApiKey` column + +You never need to modify the sync engine when adding a connector. + +## Icon + +The `icon` field on `ConnectorConfig` is used throughout the UI — in the connector list, the add-connector modal, and as the document icon in the knowledge base table (replacing the generic file type icon for connector-sourced documents). The icon is read from `CONNECTOR_REGISTRY[connectorType].icon` at runtime — no separate icon map to maintain. + +If the service already has an icon in `apps/sim/components/icons.tsx` (from a tool integration), reuse it. Otherwise, ask the user to provide the SVG. + +## Registering + +Add one line to `apps/sim/connectors/registry.ts`: + +```typescript +import { {service}Connector } from '@/connectors/{service}' + +export const CONNECTOR_REGISTRY: ConnectorRegistry = { + // ... existing connectors ... + {service}: {service}Connector, +} +``` + +## Reference Implementations + +- **OAuth + contentDeferred**: `apps/sim/connectors/google-drive/google-drive.ts` — file download with metadata-based hash, `orderBy` for deterministic pagination +- **OAuth + contentDeferred (blocks API)**: `apps/sim/connectors/notion/notion.ts` — complex block content extraction deferred to `getDocument` +- **OAuth + contentDeferred (git)**: `apps/sim/connectors/github/github.ts` — blob SHA hash, tree listing +- **OAuth + inline content**: `apps/sim/connectors/confluence/confluence.ts` — multiple config field types, `mapTags`, label fetching +- **API key**: `apps/sim/connectors/fireflies/fireflies.ts` — GraphQL API with Bearer token auth + +## Checklist + +- [ ] Created `connectors/{service}/{service}.ts` with full ConnectorConfig +- [ ] Created `connectors/{service}/index.ts` barrel export +- [ ] **Auth configured correctly:** + - OAuth: `auth.provider` matches an existing `OAuthService` in `lib/oauth/types.ts` + - API key: `auth.label` and `auth.placeholder` set appropriately +- [ ] **Selector fields configured correctly (if applicable):** + - Every `type: 'selector'` field has a canonical pair (`short-input` or `dropdown` with same `canonicalParamId` and `mode: 'advanced'`) + - `required` is identical on both fields in each canonical pair + - `selectorKey` exists in `hooks/selectors/registry.ts` + - `dependsOn` references selector field IDs (not `canonicalParamId`) + - Dependency `canonicalParamId` values exist in `SELECTOR_CONTEXT_FIELDS` +- [ ] `listDocuments` handles pagination with metadata-based content hashes +- [ ] `contentDeferred: true` used if content requires per-doc API calls (file download, export, blocks fetch) +- [ ] `contentHash` is metadata-based (not content-based) and identical between stub and `getDocument` +- [ ] `sourceUrl` set on each ExternalDocument (full URL, not relative) +- [ ] `metadata` includes source-specific data for tag mapping +- [ ] `tagDefinitions` declared for each semantic key returned by `mapTags` +- [ ] `mapTags` implemented if source has useful metadata (labels, dates, versions) +- [ ] `validateConfig` verifies the source is accessible +- [ ] All external API calls use `fetchWithRetry` (not raw `fetch`) +- [ ] All optional config fields validated in `validateConfig` +- [ ] Icon exists in `components/icons.tsx` (or asked user to provide SVG) +- [ ] Registered in `connectors/registry.ts` diff --git a/.agents/skills/source-command-add-tools/SKILL.md b/.agents/skills/source-command-add-tools/SKILL.md new file mode 100644 index 00000000000..45ef9b85959 --- /dev/null +++ b/.agents/skills/source-command-add-tools/SKILL.md @@ -0,0 +1,327 @@ +--- +name: "source-command-add-tools" +description: "Create tool configurations for a Sim integration by reading API docs" +--- + +# source-command-add-tools + +Use this skill when the user asks to run the migrated source command `add-tools`. + +## Command Template + +# Add Tools Skill + +You are an expert at creating tool configurations for Sim integrations. Your job is to read API documentation and create properly structured tool files. + +## Your Task + +When the user asks you to create tools for a service: +1. Use Context7 or WebFetch to read the service's API documentation +2. Create the tools directory structure +3. Generate properly typed tool configurations + +## Directory Structure + +Create files in `apps/sim/tools/{service}/`: +``` +tools/{service}/ +├── index.ts # Barrel export +├── types.ts # Parameter & response types +└── {action}.ts # Individual tool files (one per operation) +``` + +## Tool Configuration Structure + +Every tool MUST follow this exact structure: + +```typescript +import type { {ServiceName}{Action}Params } from '@/tools/{service}/types' +import type { ToolConfig } from '@/tools/types' + +interface {ServiceName}{Action}Response { + success: boolean + output: { + // Define output structure here + } +} + +export const {serviceName}{Action}Tool: ToolConfig< + {ServiceName}{Action}Params, + {ServiceName}{Action}Response +> = { + id: '{service}_{action}', // snake_case, matches tool name + name: '{Service} {Action}', // Human readable + description: 'Brief description', // One sentence + version: '1.0.0', + + // OAuth config (if service uses OAuth) + oauth: { + required: true, + provider: '{service}', // Must match OAuth provider ID + }, + + params: { + // Hidden params (system-injected, only use hidden for oauth accessToken) + accessToken: { + type: 'string', + required: true, + visibility: 'hidden', + description: 'OAuth access token', + }, + // User-only params (credentials, api key, IDs user must provide) + someId: { + type: 'string', + required: true, + visibility: 'user-only', + description: 'The ID of the resource', + }, + // User-or-LLM params (everything else, can be provided by user OR computed by LLM) + query: { + type: 'string', + required: false, // Use false for optional + visibility: 'user-or-llm', + description: 'Search query', + }, + }, + + request: { + url: (params) => `https://api.service.com/v1/resource/${params.id}`, + method: 'POST', + headers: (params) => ({ + Authorization: `Bearer ${params.accessToken}`, + 'Content-Type': 'application/json', + }), + body: (params) => ({ + // Request body - only for POST/PUT/PATCH + // Trim ID fields to prevent copy-paste whitespace errors: + // userId: params.userId?.trim(), + }), + }, + + transformResponse: async (response: Response) => { + const data = await response.json() + return { + success: true, + output: { + // Map API response to output + // Use ?? null for nullable fields + // Use ?? [] for optional arrays + }, + } + }, + + outputs: { + // Define each output field + }, +} +``` + +## Critical Rules for Parameters + +### Visibility Options +- `'hidden'` - System-injected (OAuth tokens, internal params). User never sees. +- `'user-only'` - User must provide (credentials, api keys, account-specific IDs) +- `'user-or-llm'` - User provides OR LLM can compute (search queries, content, filters, most fall into this category) + +### Parameter Types +- `'string'` - Text values +- `'number'` - Numeric values +- `'boolean'` - True/false +- `'json'` - Complex objects (NOT 'object', use 'json') +- `'file'` - Single file +- `'file[]'` - Multiple files + +### Required vs Optional +- Always explicitly set `required: true` or `required: false` +- Optional params should have `required: false` + +## Critical Rules for Outputs + +### Output Types +- `'string'`, `'number'`, `'boolean'` - Primitives +- `'json'` - Complex objects (use this, NOT 'object') +- `'array'` - Arrays with `items` property +- `'object'` - Objects with `properties` property + +### Optional Outputs +Add `optional: true` for fields that may not exist in the response: +```typescript +closedAt: { + type: 'string', + description: 'When the issue was closed', + optional: true, +}, +``` + +### Typed JSON Outputs + +When using `type: 'json'` and you know the object shape in advance, **always define the inner structure** using `properties` so downstream consumers know what fields are available: + +```typescript +// BAD: Opaque json with no info about what's inside +metadata: { + type: 'json', + description: 'Response metadata', +}, + +// GOOD: Define the known properties +metadata: { + type: 'json', + description: 'Response metadata', + properties: { + id: { type: 'string', description: 'Unique ID' }, + status: { type: 'string', description: 'Current status' }, + count: { type: 'number', description: 'Total count' }, + }, +}, +``` + +For arrays of objects, define the item structure: +```typescript +items: { + type: 'array', + description: 'List of items', + items: { + type: 'object', + properties: { + id: { type: 'string', description: 'Item ID' }, + name: { type: 'string', description: 'Item name' }, + }, + }, +}, +``` + +Only use bare `type: 'json'` without `properties` when the shape is truly dynamic or unknown. + +## Critical Rules for transformResponse + +### Handle Nullable Fields +ALWAYS use `?? null` for fields that may be undefined: +```typescript +transformResponse: async (response: Response) => { + const data = await response.json() + return { + success: true, + output: { + id: data.id, + title: data.title, + body: data.body ?? null, // May be undefined + assignee: data.assignee ?? null, // May be undefined + labels: data.labels ?? [], // Default to empty array + closedAt: data.closed_at ?? null, // May be undefined + }, + } +} +``` + +### Never Output Raw JSON Dumps +DON'T do this: +```typescript +output: { + data: data, // BAD - raw JSON dump +} +``` + +DO this instead - extract meaningful fields: +```typescript +output: { + id: data.id, + name: data.name, + status: data.status, + metadata: { + createdAt: data.created_at, + updatedAt: data.updated_at, + }, +} +``` + +## Types File Pattern + +Create `types.ts` with interfaces for all params and responses: + +```typescript +import type { ToolResponse } from '@/tools/types' + +// Parameter interfaces +export interface {Service}{Action}Params { + accessToken: string + requiredField: string + optionalField?: string +} + +// Response interfaces (extend ToolResponse) +export interface {Service}{Action}Response extends ToolResponse { + output: { + field1: string + field2: number + optionalField?: string | null + } +} +``` + +## Index.ts Barrel Export Pattern + +```typescript +// Export all tools +export { serviceTool1 } from './{action1}' +export { serviceTool2 } from './{action2}' + +// Export types +export * from './types' +``` + +## Registering Tools + +After creating tools, remind the user to: +1. Import tools in `apps/sim/tools/registry.ts` +2. Add to the `tools` object with snake_case keys: +```typescript +import { serviceActionTool } from '@/tools/{service}' + +export const tools = { + // ... existing tools ... + {service}_{action}: serviceActionTool, +} +``` + +## V2 Tool Pattern + +If creating V2 tools (API-aligned outputs), use `_v2` suffix: +- Tool ID: `{service}_{action}_v2` +- Variable name: `{action}V2Tool` +- Version: `'2.0.0'` +- Outputs: Flat, API-aligned (no content/metadata wrapper) + +## Naming Convention + +All tool IDs MUST use `snake_case`: `{service}_{action}` (e.g., `x_create_tweet`, `slack_send_message`). Never use camelCase or PascalCase for tool IDs. + +## Checklist Before Finishing + +- [ ] All tool IDs use snake_case +- [ ] All params have explicit `required: true` or `required: false` +- [ ] All params have appropriate `visibility` +- [ ] All nullable response fields use `?? null` +- [ ] All optional outputs have `optional: true` +- [ ] No raw JSON dumps in outputs +- [ ] Types file has all interfaces +- [ ] Index.ts exports all tools + +## Final Validation (Required) + +After creating all tools, you MUST validate every tool before finishing: + +1. **Read every tool file** you created — do not skip any +2. **Cross-reference with the API docs** to verify: + - All required params are marked `required: true` + - All optional params are marked `required: false` + - Param types match the API (string, number, boolean, json) + - Request URL, method, headers, and body match the API spec + - `transformResponse` extracts the correct fields from the API response + - All output fields match what the API actually returns + - No fields are missing from outputs that the API provides + - No extra fields are defined in outputs that the API doesn't return +3. **Verify consistency** across tools: + - Shared types in `types.ts` match all tools that use them + - Tool IDs in the barrel export match the tool file definitions + - Error handling is consistent (error checks, meaningful messages) diff --git a/.agents/skills/source-command-add-trigger/SKILL.md b/.agents/skills/source-command-add-trigger/SKILL.md new file mode 100644 index 00000000000..4b1db657114 --- /dev/null +++ b/.agents/skills/source-command-add-trigger/SKILL.md @@ -0,0 +1,503 @@ +--- +name: "source-command-add-trigger" +description: "Create webhook or polling triggers for a Sim integration" +--- + +# source-command-add-trigger + +Use this skill when the user asks to run the migrated source command `add-trigger`. + +## Command Template + +# Add Trigger + +You are an expert at creating webhook and polling triggers for Sim. You understand the trigger system, the generic `buildTriggerSubBlocks` helper, polling infrastructure, and how triggers connect to blocks. + +## Your Task + +1. Research what webhook events the service supports — if the service lacks reliable webhooks, use polling +2. Create the trigger files using the generic builder (webhook) or manual config (polling) +3. Create a provider handler (webhook) or polling handler (polling) +4. Register triggers and connect them to the block + +## Directory Structure + +``` +apps/sim/triggers/{service}/ +├── index.ts # Barrel exports +├── utils.ts # Service-specific helpers (options, instructions, extra fields, outputs) +├── {event_a}.ts # Primary trigger (includes dropdown) +├── {event_b}.ts # Secondary trigger (no dropdown) +└── webhook.ts # Generic webhook trigger (optional, for "all events") + +apps/sim/lib/webhooks/ +├── provider-subscription-utils.ts # Shared subscription helpers (getProviderConfig, getNotificationUrl) +├── providers/ +│ ├── {service}.ts # Provider handler (auth, formatInput, matchEvent, subscriptions) +│ ├── types.ts # WebhookProviderHandler interface +│ ├── utils.ts # Shared helpers (createHmacVerifier, verifyTokenAuth, skipByEventTypes) +│ └── registry.ts # Handler map + default handler +``` + +## Step 1: Create `utils.ts` + +This file contains all service-specific helpers used by triggers. + +```typescript +import type { SubBlockConfig } from '@/blocks/types' +import type { TriggerOutput } from '@/triggers/types' + +export const {service}TriggerOptions = [ + { label: 'Event A', id: '{service}_event_a' }, + { label: 'Event B', id: '{service}_event_b' }, +] + +export function {service}SetupInstructions(eventType: string): string { + const instructions = [ + 'Copy the Webhook URL above', + 'Go to {Service} Settings > Webhooks', + `Select the ${eventType} event type`, + 'Paste the webhook URL and save', + 'Click "Save" above to activate your trigger', + ] + return instructions + .map((instruction, index) => + `
${index + 1}. ${instruction}
` + ) + .join('') +} + +export function build{Service}ExtraFields(triggerId: string): SubBlockConfig[] { + return [ + { + id: 'projectId', + title: 'Project ID (Optional)', + type: 'short-input', + placeholder: 'Leave empty for all projects', + mode: 'trigger', + condition: { field: 'selectedTriggerId', value: triggerId }, + }, + ] +} + +export function build{Service}Outputs(): Record { + return { + eventType: { type: 'string', description: 'The type of event' }, + resourceId: { type: 'string', description: 'ID of the affected resource' }, + resource: { + id: { type: 'string', description: 'Resource ID' }, + name: { type: 'string', description: 'Resource name' }, + }, + } +} +``` + +## Step 2: Create Trigger Files + +**Primary trigger** — MUST include `includeDropdown: true`: + +```typescript +import { {Service}Icon } from '@/components/icons' +import { buildTriggerSubBlocks } from '@/triggers' +import { build{Service}ExtraFields, build{Service}Outputs, {service}SetupInstructions, {service}TriggerOptions } from '@/triggers/{service}/utils' +import type { TriggerConfig } from '@/triggers/types' + +export const {service}EventATrigger: TriggerConfig = { + id: '{service}_event_a', + name: '{Service} Event A', + provider: '{service}', + description: 'Trigger workflow when Event A occurs', + version: '1.0.0', + icon: {Service}Icon, + subBlocks: buildTriggerSubBlocks({ + triggerId: '{service}_event_a', + triggerOptions: {service}TriggerOptions, + includeDropdown: true, + setupInstructions: {service}SetupInstructions('Event A'), + extraFields: build{Service}ExtraFields('{service}_event_a'), + }), + outputs: build{Service}Outputs(), + webhook: { method: 'POST', headers: { 'Content-Type': 'application/json' } }, +} +``` + +**Secondary triggers** — NO `includeDropdown` (it's already in the primary): + +```typescript +export const {service}EventBTrigger: TriggerConfig = { + // Same as above but: id: '{service}_event_b', no includeDropdown +} +``` + +## Step 3: Register and Wire + +### `apps/sim/triggers/{service}/index.ts` + +```typescript +export { {service}EventATrigger } from './event_a' +export { {service}EventBTrigger } from './event_b' +``` + +### `apps/sim/triggers/registry.ts` + +```typescript +import { {service}EventATrigger, {service}EventBTrigger } from '@/triggers/{service}' + +export const TRIGGER_REGISTRY: TriggerRegistry = { + // ... existing ... + {service}_event_a: {service}EventATrigger, + {service}_event_b: {service}EventBTrigger, +} +``` + +### Block file (`apps/sim/blocks/blocks/{service}.ts`) + +Wire triggers into the block so the trigger UI appears and `generate-docs.ts` discovers them. Two changes are needed: + +1. **Spread trigger subBlocks** at the end of the block's `subBlocks` array +2. **Add `triggers` property** after `outputs` with `enabled: true` and `available: [...]` + +```typescript +import { getTrigger } from '@/triggers' + +export const {Service}Block: BlockConfig = { + // ... + subBlocks: [ + // Regular tool subBlocks first... + ...getTrigger('{service}_event_a').subBlocks, + ...getTrigger('{service}_event_b').subBlocks, + ], + // ... tools, inputs, outputs ... + triggers: { + enabled: true, + available: ['{service}_event_a', '{service}_event_b'], + }, +} +``` + +**Versioned blocks (V1 + V2):** Many integrations have a hidden V1 block and a visible V2 block. Where you add the trigger wiring depends on how V2 inherits from V1: + +- **V2 uses `...V1Block` spread** (e.g., Google Calendar): Add trigger to V1 — V2 inherits both `subBlocks` and `triggers` automatically. +- **V2 defines its own `subBlocks`** (e.g., Google Sheets): Add trigger to V2 (the visible block). V1 is hidden and doesn't need it. +- **Single block, no V2** (e.g., Google Drive): Add trigger directly. + +`generate-docs.ts` deduplicates by base type (first match wins). If V1 is processed first without triggers, the V2 triggers won't appear in `integrations.json`. Always verify by checking the output after running the script. + +## Provider Handler + +All provider-specific webhook logic lives in a single handler file: `apps/sim/lib/webhooks/providers/{service}.ts`. + +### When to Create a Handler + +| Behavior | Method | Examples | +|---|---|---| +| HMAC signature auth | `verifyAuth` via `createHmacVerifier` | Ashby, Jira, Linear, Typeform | +| Custom token auth | `verifyAuth` via `verifyTokenAuth` | Generic, Google Forms | +| Event filtering | `matchEvent` | GitHub, Jira, Attio, HubSpot | +| Idempotency dedup | `extractIdempotencyId` | Slack, Stripe, Linear, Jira | +| Custom input formatting | `formatInput` | Slack, Teams, Attio, Ashby | +| Auto webhook creation | `createSubscription` | Ashby, Grain, Calendly, Airtable | +| Auto webhook deletion | `deleteSubscription` | Ashby, Grain, Calendly, Airtable | +| Challenge/verification | `handleChallenge` | Slack, WhatsApp, Teams | +| Custom success response | `formatSuccessResponse` | Slack, Twilio Voice, Teams | + +If none apply, you don't need a handler. The default handler provides bearer token auth. + +### Example Handler + +```typescript +import crypto from 'crypto' +import { createLogger } from '@sim/logger' +import { safeCompare } from '@/lib/core/security/encryption' +import type { EventMatchContext, FormatInputContext, FormatInputResult, WebhookProviderHandler } from '@/lib/webhooks/providers/types' +import { createHmacVerifier } from '@/lib/webhooks/providers/utils' + +const logger = createLogger('WebhookProvider:{Service}') + +function validate{Service}Signature(secret: string, signature: string, body: string): boolean { + if (!secret || !signature || !body) return false + const computed = crypto.createHmac('sha256', secret).update(body, 'utf8').digest('hex') + return safeCompare(computed, signature) +} + +export const {service}Handler: WebhookProviderHandler = { + verifyAuth: createHmacVerifier({ + configKey: 'webhookSecret', + headerName: 'X-{Service}-Signature', + validateFn: validate{Service}Signature, + providerLabel: '{Service}', + }), + + async matchEvent({ body, requestId, providerConfig }: EventMatchContext) { + const triggerId = providerConfig.triggerId as string | undefined + if (triggerId && triggerId !== '{service}_webhook') { + const { is{Service}EventMatch } = await import('@/triggers/{service}/utils') + if (!is{Service}EventMatch(triggerId, body as Record)) return false + } + return true + }, + + async formatInput({ body }: FormatInputContext): Promise { + const b = body as Record + return { + input: { + eventType: b.type, + resourceId: (b.data as Record)?.id || '', + resource: b.data, + }, + } + }, + + extractIdempotencyId(body: unknown) { + const obj = body as Record + return obj.id && obj.type ? `${obj.type}:${obj.id}` : null + }, +} +``` + +### Register the Handler + +In `apps/sim/lib/webhooks/providers/registry.ts`: + +```typescript +import { {service}Handler } from '@/lib/webhooks/providers/{service}' + +const PROVIDER_HANDLERS: Record = { + // ... existing (alphabetical) ... + {service}: {service}Handler, +} +``` + +## Output Alignment (Critical) + +There are two sources of truth that **MUST be aligned**: + +1. **Trigger `outputs`** — schema defining what fields SHOULD be available (UI tag dropdown) +2. **`formatInput` on the handler** — implementation that transforms raw payload into actual data + +If they differ: the tag dropdown shows fields that don't exist, or actual data has fields users can't discover. + +**Rules for `formatInput`:** +- Return `{ input: { ... } }` where inner keys match trigger `outputs` exactly +- Return `{ input: ..., skip: { message: '...' } }` to skip execution +- No wrapper objects or duplication +- Use `null` for missing optional data + +## Automatic Webhook Registration + +If the service API supports programmatic webhook creation, implement `createSubscription` and `deleteSubscription` on the handler. The orchestration layer calls these automatically — **no code touches `route.ts`, `provider-subscriptions.ts`, or `deploy.ts`**. + +```typescript +import { getNotificationUrl, getProviderConfig } from '@/lib/webhooks/provider-subscription-utils' +import type { DeleteSubscriptionContext, SubscriptionContext, SubscriptionResult } from '@/lib/webhooks/providers/types' + +export const {service}Handler: WebhookProviderHandler = { + async createSubscription(ctx: SubscriptionContext): Promise { + const config = getProviderConfig(ctx.webhook) + const apiKey = config.apiKey as string + if (!apiKey) throw new Error('{Service} API Key is required.') + + const res = await fetch('https://api.{service}.com/webhooks', { + method: 'POST', + headers: { Authorization: `Bearer ${apiKey}`, 'Content-Type': 'application/json' }, + body: JSON.stringify({ url: getNotificationUrl(ctx.webhook) }), + }) + + if (!res.ok) throw new Error(`{Service} error: ${res.status}`) + const { id } = (await res.json()) as { id: string } + return { providerConfigUpdates: { externalId: id } } + }, + + async deleteSubscription(ctx: DeleteSubscriptionContext): Promise { + const config = getProviderConfig(ctx.webhook) + const { apiKey, externalId } = config as { apiKey?: string; externalId?: string } + if (!apiKey || !externalId) return + await fetch(`https://api.{service}.com/webhooks/${externalId}`, { + method: 'DELETE', + headers: { Authorization: `Bearer ${apiKey}` }, + }).catch(() => {}) + }, +} +``` + +**Key points:** +- Throw from `createSubscription` — orchestration rolls back the DB webhook +- Never throw from `deleteSubscription` — log non-fatally +- Return `{ providerConfigUpdates: { externalId } }` — orchestration merges into `providerConfig` +- Add `apiKey` field to `build{Service}ExtraFields` with `password: true` + +## Trigger Outputs Schema + +Trigger outputs use the same schema as block outputs (NOT tool outputs). + +**Supported:** `type` + `description` for leaf fields, nested objects for complex data. +**NOT supported:** `optional: true`, `items` (those are tool-output-only features). + +```typescript +export function buildOutputs(): Record { + return { + eventType: { type: 'string', description: 'Event type' }, + timestamp: { type: 'string', description: 'When it occurred' }, + payload: { type: 'json', description: 'Full event payload' }, + resource: { + id: { type: 'string', description: 'Resource ID' }, + name: { type: 'string', description: 'Resource name' }, + }, + } +} +``` + +## Polling Triggers + +Use polling when the service lacks reliable webhooks (e.g., Google Sheets, Google Drive, Google Calendar, Gmail, RSS, IMAP). Polling triggers do NOT use `buildTriggerSubBlocks` — they define subBlocks manually. + +### Directory Structure + +``` +apps/sim/triggers/{service}/ +├── index.ts # Barrel export +└── poller.ts # TriggerConfig with polling: true + +apps/sim/lib/webhooks/polling/ +└── {service}.ts # PollingProviderHandler implementation +``` + +### Polling Handler (`apps/sim/lib/webhooks/polling/{service}.ts`) + +```typescript +import { pollingIdempotency } from '@/lib/core/idempotency/service' +import type { PollingProviderHandler, PollWebhookContext } from '@/lib/webhooks/polling/types' +import { markWebhookFailed, markWebhookSuccess, resolveOAuthCredential, updateWebhookProviderConfig } from '@/lib/webhooks/polling/utils' +import { processPolledWebhookEvent } from '@/lib/webhooks/processor' + +export const {service}PollingHandler: PollingProviderHandler = { + provider: '{service}', + label: '{Service}', + + async pollWebhook(ctx: PollWebhookContext): Promise<'success' | 'failure'> { + const { webhookData, workflowData, requestId, logger } = ctx + const webhookId = webhookData.id + + try { + // For OAuth services: + const accessToken = await resolveOAuthCredential(webhookData, '{service}', requestId, logger) + const config = webhookData.providerConfig as unknown as {Service}WebhookConfig + + // First poll: seed state, emit nothing + if (!config.lastCheckedTimestamp) { + await updateWebhookProviderConfig(webhookId, { lastCheckedTimestamp: new Date().toISOString() }, logger) + await markWebhookSuccess(webhookId, logger) + return 'success' + } + + // Fetch changes since last poll, process with idempotency + // ... + + await markWebhookSuccess(webhookId, logger) + return 'success' + } catch (error) { + logger.error(`[${requestId}] Error processing {service} webhook ${webhookId}:`, error) + await markWebhookFailed(webhookId, logger) + return 'failure' + } + }, +} +``` + +**Key patterns:** +- First poll seeds state and emits nothing (avoids flooding with existing data) +- Use `pollingIdempotency.executeWithIdempotency(provider, key, callback)` for dedup +- Use `processPolledWebhookEvent(webhookData, workflowData, payload, requestId)` to fire the workflow +- Use `updateWebhookProviderConfig(webhookId, partialConfig, logger)` for read-merge-write on state +- Use the latest server-side timestamp from API responses (not wall clock) to avoid clock skew + +### Trigger Config (`apps/sim/triggers/{service}/poller.ts`) + +```typescript +import { {Service}Icon } from '@/components/icons' +import type { TriggerConfig } from '@/triggers/types' + +export const {service}PollingTrigger: TriggerConfig = { + id: '{service}_poller', + name: '{Service} Trigger', + provider: '{service}', + description: 'Triggers when ...', + version: '1.0.0', + icon: {Service}Icon, + polling: true, // REQUIRED — routes to polling infrastructure + + subBlocks: [ + { id: 'triggerCredentials', type: 'oauth-input', title: 'Credentials', serviceId: '{service}', requiredScopes: [], required: true, mode: 'trigger', supportsCredentialSets: true }, + // ... service-specific config fields (dropdowns, inputs, switches) ... + { id: 'triggerInstructions', type: 'text', title: 'Setup Instructions', hideFromPreview: true, mode: 'trigger', defaultValue: '...' }, + ], + + outputs: { + // Must match the payload shape from processPolledWebhookEvent + }, +} +``` + +### Registration (3 places) + +1. **`apps/sim/triggers/constants.ts`** — add provider to `POLLING_PROVIDERS` Set +2. **`apps/sim/lib/webhooks/polling/registry.ts`** — import handler, add to `POLLING_HANDLERS` +3. **`apps/sim/triggers/registry.ts`** — import trigger config, add to `TRIGGER_REGISTRY` + +### Helm Cron Job + +Add to `helm/sim/values.yaml` under the existing polling cron jobs: + +```yaml +{service}WebhookPoll: + schedule: "*/1 * * * *" + concurrencyPolicy: Forbid + url: "http://sim:3000/api/webhooks/poll/{service}" +``` + +### Reference Implementations + +- Simple: `apps/sim/lib/webhooks/polling/rss.ts` + `apps/sim/triggers/rss/poller.ts` +- Complex (OAuth, attachments): `apps/sim/lib/webhooks/polling/gmail.ts` + `apps/sim/triggers/gmail/poller.ts` +- Cursor-based (changes API): `apps/sim/lib/webhooks/polling/google-drive.ts` +- Timestamp-based: `apps/sim/lib/webhooks/polling/google-calendar.ts` + +## Checklist + +### Trigger Definition +- [ ] Created `utils.ts` with options, instructions, extra fields, and output builders +- [ ] Primary trigger has `includeDropdown: true`; secondary triggers do NOT +- [ ] All triggers use `buildTriggerSubBlocks` helper +- [ ] Created `index.ts` barrel export + +### Registration +- [ ] All triggers in `triggers/registry.ts` → `TRIGGER_REGISTRY` +- [ ] Block has `triggers.enabled: true` and lists all trigger IDs in `triggers.available` +- [ ] Block spreads all trigger subBlocks: `...getTrigger('id').subBlocks` + +### Provider Handler (if needed) +- [ ] Handler file at `apps/sim/lib/webhooks/providers/{service}.ts` +- [ ] Registered in `providers/registry.ts` (alphabetical) +- [ ] Signature validator is a private function inside the handler file +- [ ] `formatInput` output keys match trigger `outputs` exactly +- [ ] Event matching uses dynamic `await import()` for trigger utils + +### Auto Registration (if supported) +- [ ] `createSubscription` and `deleteSubscription` on the handler +- [ ] NO changes to `route.ts`, `provider-subscriptions.ts`, or `deploy.ts` +- [ ] API key field uses `password: true` + +### Polling Trigger (if applicable) +- [ ] Handler implements `PollingProviderHandler` at `lib/webhooks/polling/{service}.ts` +- [ ] Trigger config has `polling: true` and defines subBlocks manually (no `buildTriggerSubBlocks`) +- [ ] Provider string matches across: trigger config, handler, `POLLING_PROVIDERS`, polling registry +- [ ] First poll seeds state and emits nothing +- [ ] Added provider to `POLLING_PROVIDERS` in `triggers/constants.ts` +- [ ] Added handler to `POLLING_HANDLERS` in `lib/webhooks/polling/registry.ts` +- [ ] Added cron job to `helm/sim/values.yaml` +- [ ] Payload shape matches trigger `outputs` schema + +### Testing +- [ ] `bun run type-check` passes +- [ ] Manually verify output keys match trigger `outputs` keys +- [ ] Trigger UI shows correctly in the block diff --git a/.agents/skills/source-command-council/SKILL.md b/.agents/skills/source-command-council/SKILL.md new file mode 100644 index 00000000000..ecfeb0c25ce --- /dev/null +++ b/.agents/skills/source-command-council/SKILL.md @@ -0,0 +1,18 @@ +--- +name: "source-command-council" +description: "Spawn task agents to explore a given area of interest in the codebase" +--- + +# source-command-council + +Use this skill when the user asks to run the migrated source command `council`. + +## Command Template + +Based on the given area of interest, please: + +1. Dig around the codebase in terms of that given area of interest, gather general information such as keywords and architecture overview. +2. Spawn off n=10 (unless specified otherwise) task agents to dig deeper into the codebase in terms of that given area of interest, some of them should be out of the box for variance. +3. Once the task agents are done, use the information to do what the user wants. + +If user is in plan mode, use the information to create the plan. diff --git a/.agents/skills/source-command-ship/SKILL.md b/.agents/skills/source-command-ship/SKILL.md new file mode 100644 index 00000000000..1beef2a8a84 --- /dev/null +++ b/.agents/skills/source-command-ship/SKILL.md @@ -0,0 +1,90 @@ +--- +name: "source-command-ship" +description: "Commit, push, and open a PR to staging in one shot" +--- + +# source-command-ship + +Use this skill when the user asks to run the migrated source command `ship`. + +## Command Template + +# Ship Command + +You help ship code by creating commits, pushing to the remote branch, and creating PRs in the user's voice. + +## Your Task + +When the user runs `/ship`: + +1. **Check git status** - See what files have changed +2. **Generate a commit message** following this format: `type(scope): description` + - Types: `fix`, `feat`, `improvement`, `chore` + - Scope: short identifier (e.g., `undo-redo`, `api`, `ui`) + - Keep it concise + +3. **Run pre-ship checks** from the repo root before staging: + - `bun run lint` to fix formatting issues + - `bun run check:api-validation:strict` to catch boundary contract failures before CI + +4. **Stage and commit** the changes with the generated message + +5. **Push to origin** using the current branch name + +6. **Create a PR** to staging with a description in the user's voice + +## Commit Message Format + +Based on the repo's commit history: +``` +fix(scope): description for bug fixes +feat(scope): description for new features +improvement(scope): description for enhancements +chore(scope): description for maintenance +``` + +## PR Description Format + +Use this exact template in the user's voice (concise, bullet points): + +```markdown +## Summary +- bullet point describing what changed +- another bullet point if needed + +## Type of Change +- [x] Bug fix (or appropriate type) + +## Testing +Tested manually (or describe testing) + +## Checklist +- [x] Code follows project style guidelines +- [x] Self-reviewed my changes +- [ ] Tests added/updated and passing +- [x] No new warnings introduced +- [x] I confirm that I have read and agree to the terms outlined in the [Contributor License Agreement (CLA)](./CONTRIBUTING.md#contributor-license-agreement-cla) +``` + +## PR Creation Command + +Use this command structure: +```bash +gh pr create --base staging --title "COMMIT_MESSAGE" --body "PR_BODY" +``` + +## Important Notes + +- Always confirm the commit message and PR description with the user before executing +- The PR should be created against `staging` branch +- Keep descriptions concise and in active voice +- Match the user's previous PR style: direct, no fluff, bullet points +- **DO NOT add "Co-Authored-By" lines to commits** - keep commit messages clean + +## User's Voice Characteristics (based on previous PRs) + +- Short, direct bullet points +- No unnecessary explanation +- "Tested manually" is acceptable for testing section; include lint and boundary validation results when run +- Checkboxes filled in appropriately +- No screenshots section unless UI changes diff --git a/.agents/skills/source-command-validate-connector/SKILL.md b/.agents/skills/source-command-validate-connector/SKILL.md new file mode 100644 index 00000000000..d1808355a3c --- /dev/null +++ b/.agents/skills/source-command-validate-connector/SKILL.md @@ -0,0 +1,322 @@ +--- +name: "source-command-validate-connector" +description: "Validate an existing knowledge base connector against its service's API docs" +--- + +# source-command-validate-connector + +Use this skill when the user asks to run the migrated source command `validate-connector`. + +## Command Template + +# Validate Connector Skill + +You are an expert auditor for Sim knowledge base connectors. Your job is to thoroughly validate that an existing connector is correct, complete, and follows all conventions. + +## Your Task + +When the user asks you to validate a connector: +1. Read the service's API documentation (via Context7 or WebFetch) +2. Read the connector implementation, OAuth config, and registry entries +3. Cross-reference everything against the API docs and Sim conventions +4. Report all issues found, grouped by severity (critical, warning, suggestion) +5. Fix all issues after reporting them + +## Step 1: Gather All Files + +Read **every** file for the connector — do not skip any: + +``` +apps/sim/connectors/{service}/{service}.ts # Connector implementation +apps/sim/connectors/{service}/index.ts # Barrel export +apps/sim/connectors/registry.ts # Connector registry entry +apps/sim/connectors/types.ts # ConnectorConfig interface, ExternalDocument, etc. +apps/sim/connectors/utils.ts # Shared utilities (computeContentHash, htmlToPlainText, etc.) +apps/sim/lib/oauth/oauth.ts # OAUTH_PROVIDERS — single source of truth for scopes +apps/sim/lib/oauth/utils.ts # getCanonicalScopesForProvider, getScopesForService, SCOPE_DESCRIPTIONS +apps/sim/lib/oauth/types.ts # OAuthService union type +apps/sim/components/icons.tsx # Icon definition for the service +``` + +If the connector uses selectors, also read: +``` +apps/sim/hooks/selectors/registry.ts # Selector key definitions +apps/sim/hooks/selectors/types.ts # SelectorKey union type +apps/sim/lib/workflows/subblocks/context.ts # SELECTOR_CONTEXT_FIELDS +``` + +## Step 2: Pull API Documentation + +Fetch the official API docs for the service. This is the **source of truth** for: +- Endpoint URLs, HTTP methods, and auth headers +- Required vs optional parameters +- Parameter types and allowed values +- Response shapes and field names +- Pagination patterns (cursor, offset, next token) +- Rate limits and error formats +- OAuth scopes and their meanings + +Use Context7 (resolve-library-id → query-docs) or WebFetch to retrieve documentation. If both fail, note which claims are based on training knowledge vs verified docs. + +## Step 3: Validate API Endpoints + +For **every** API call in the connector (`listDocuments`, `getDocument`, `validateConfig`, and any helper functions), verify against the API docs: + +### URLs and Methods +- [ ] Base URL is correct for the service's API version +- [ ] Endpoint paths match the API docs exactly +- [ ] HTTP method is correct (GET, POST, PUT, PATCH, DELETE) +- [ ] Path parameters are correctly interpolated and URI-encoded where needed +- [ ] Query parameters use correct names and formats per the API docs + +### Headers +- [ ] Authorization header uses the correct format: + - OAuth: `Authorization: Bearer ${accessToken}` + - API Key: correct header name per the service's docs +- [ ] `Content-Type` is set for POST/PUT/PATCH requests +- [ ] Any service-specific headers are present (e.g., `Notion-Version`, `Dropbox-API-Arg`) +- [ ] No headers are sent that the API doesn't support or silently ignores + +### Request Bodies +- [ ] POST/PUT body fields match API parameter names exactly +- [ ] Required fields are always sent +- [ ] Optional fields are conditionally included (not sent as `null` or empty unless the API expects that) +- [ ] Field value types match API expectations (string vs number vs boolean) + +### Input Sanitization +- [ ] User-controlled values interpolated into query strings are properly escaped: + - OData `$filter`: single quotes escaped with `''` (e.g., `externalId.replace(/'/g, "''")`) + - SOQL: single quotes escaped with `\'` + - GraphQL variables: passed as variables, not interpolated into query strings + - URL path segments: `encodeURIComponent()` applied +- [ ] URL-type config fields (e.g., `siteUrl`, `instanceUrl`) are normalized: + - Strip `https://` / `http://` prefix if the API expects bare domains + - Strip trailing `/` + - Apply `.trim()` before validation + +### Response Parsing +- [ ] Response structure is correctly traversed (e.g., `data.results` vs `data.items` vs `data`) +- [ ] Field names extracted match what the API actually returns +- [ ] Nullable fields are handled with `?? null` or `|| undefined` +- [ ] Error responses are checked before accessing data fields + +## Step 4: Validate OAuth Scopes (if OAuth connector) + +Scopes must be correctly declared and sufficient for all API calls the connector makes. + +### Connector requiredScopes +- [ ] `requiredScopes` in the connector's `auth` config lists all scopes needed by the connector +- [ ] Each scope in `requiredScopes` is a real, valid scope recognized by the service's API +- [ ] No invalid, deprecated, or made-up scopes are listed +- [ ] No unnecessary excess scopes beyond what the connector actually needs + +### Scope Subset Validation (CRITICAL) +- [ ] Every scope in `requiredScopes` exists in the OAuth provider's `scopes` array in `lib/oauth/oauth.ts` +- [ ] Find the provider in `OAUTH_PROVIDERS[providerGroup].services[serviceId].scopes` +- [ ] Verify: `requiredScopes` ⊆ `OAUTH_PROVIDERS scopes` (every required scope is present in the provider config) +- [ ] If a required scope is NOT in the provider config, flag as **critical** — the connector will fail at runtime + +### Scope Sufficiency +For each API endpoint the connector calls: +- [ ] Identify which scopes are required per the API docs +- [ ] Verify those scopes are included in the connector's `requiredScopes` +- [ ] If the connector calls endpoints requiring scopes not in `requiredScopes`, flag as **warning** + +### Token Refresh Config +- [ ] Check the `getOAuthTokenRefreshConfig` function in `lib/oauth/oauth.ts` for this provider +- [ ] `useBasicAuth` matches the service's token exchange requirements +- [ ] `supportsRefreshTokenRotation` matches whether the service issues rotating refresh tokens +- [ ] Token endpoint URL is correct + +## Step 5: Validate Pagination + +### listDocuments Pagination +- [ ] Cursor/pagination parameter name matches the API docs +- [ ] Response pagination field is correctly extracted (e.g., `next_cursor`, `nextPageToken`, `@odata.nextLink`, `offset`) +- [ ] `hasMore` is correctly determined from the response +- [ ] `nextCursor` is correctly passed back for the next page +- [ ] `maxItems` / `maxRecords` cap is correctly applied across pages using `syncContext.totalDocsFetched` +- [ ] Page size is within the API's allowed range (not exceeding max page size) +- [ ] Last page precision: when a `maxItems` cap exists, the final page request uses `Math.min(PAGE_SIZE, remaining)` to avoid fetching more records than needed +- [ ] No off-by-one errors in pagination tracking +- [ ] The connector does NOT hit known API pagination limits silently (e.g., HubSpot search 10k cap) + +### Pagination State Across Pages +- [ ] `syncContext` is used to cache state across pages (user names, field maps, instance URLs, portal IDs, etc.) +- [ ] Cached state in `syncContext` is correctly initialized on first page and reused on subsequent pages + +## Step 6: Validate Data Transformation + +### ExternalDocument Construction +- [ ] `externalId` is a stable, unique identifier from the source API +- [ ] `title` is extracted from the correct field and has a sensible fallback (e.g., `'Untitled'`) +- [ ] `content` is plain text — HTML content is stripped using `htmlToPlainText` from `@/connectors/utils` +- [ ] `mimeType` is `'text/plain'` +- [ ] `contentHash` is computed using `computeContentHash` from `@/connectors/utils` +- [ ] `sourceUrl` is a valid, complete URL back to the original resource (not relative) +- [ ] `metadata` contains all fields referenced by `mapTags` and `tagDefinitions` + +### Content Extraction +- [ ] Rich text / HTML fields are converted to plain text before indexing +- [ ] Important content is not silently dropped (e.g., nested blocks, table cells, code blocks) +- [ ] Content is not silently truncated without logging a warning +- [ ] Empty/blank documents are properly filtered out +- [ ] Size checks use `Buffer.byteLength(text, 'utf8')` not `text.length` when comparing against byte-based limits (e.g., `MAX_FILE_SIZE` in bytes) + +## Step 7: Validate Tag Definitions and mapTags + +### tagDefinitions +- [ ] Each `tagDefinition` has an `id`, `displayName`, and `fieldType` +- [ ] `fieldType` matches the actual data type: `'text'` for strings, `'number'` for numbers, `'date'` for dates, `'boolean'` for booleans +- [ ] Every `id` in `tagDefinitions` is returned by `mapTags` +- [ ] No `tagDefinition` references a field that `mapTags` never produces + +### mapTags +- [ ] Return keys match `tagDefinition` `id` values exactly +- [ ] Date values are properly parsed using `parseTagDate` from `@/connectors/utils` +- [ ] Array values are properly joined using `joinTagArray` from `@/connectors/utils` +- [ ] Number values are validated (not `NaN`) +- [ ] Metadata field names accessed in `mapTags` match what `listDocuments`/`getDocument` store in `metadata` + +## Step 8: Validate Config Fields and Validation + +### configFields +- [ ] Every field has `id`, `title`, `type` +- [ ] `required` is set explicitly (not omitted) +- [ ] Dropdown fields have `options` with `label` and `id` for each option +- [ ] Selector fields follow the canonical pair pattern: + - A `type: 'selector'` field with `selectorKey`, `canonicalParamId`, `mode: 'basic'` + - A `type: 'short-input'` field with the same `canonicalParamId`, `mode: 'advanced'` + - `required` is identical on both fields in the pair +- [ ] `selectorKey` values exist in the selector registry +- [ ] `dependsOn` references selector field `id` values, not `canonicalParamId` + +### validateConfig +- [ ] Validates all required fields are present before making API calls +- [ ] Validates optional numeric fields (checks `Number.isNaN`, positive values) +- [ ] Makes a lightweight API call to verify access (e.g., fetch 1 record, get profile) +- [ ] Uses `VALIDATE_RETRY_OPTIONS` for retry budget +- [ ] Returns `{ valid: true }` on success +- [ ] Returns `{ valid: false, error: 'descriptive message' }` on failure +- [ ] Catches exceptions and returns user-friendly error messages +- [ ] Does NOT make expensive calls (full data listing, large queries) + +## Step 9: Validate getDocument + +- [ ] Fetches a single document by `externalId` +- [ ] Returns `null` for 404 / not found (does not throw) +- [ ] Returns the same `ExternalDocument` shape as `listDocuments` +- [ ] Handles all content types that `listDocuments` can produce (e.g., if `listDocuments` returns both pages and blogposts, `getDocument` must handle both — not hardcode one endpoint) +- [ ] Forwards `syncContext` if it needs cached state (user names, field maps, etc.) +- [ ] Error handling is graceful (catches, logs, returns null or throws with context) +- [ ] Does not redundantly re-fetch data already included in the initial API response (e.g., if comments come back with the post, don't fetch them again separately) + +## Step 10: Validate General Quality + +### fetchWithRetry Usage +- [ ] All external API calls use `fetchWithRetry` from `@/lib/knowledge/documents/utils` +- [ ] No raw `fetch()` calls to external APIs +- [ ] `VALIDATE_RETRY_OPTIONS` used in `validateConfig` +- [ ] If `validateConfig` calls a shared helper (e.g., `linearGraphQL`, `resolveId`), that helper must accept and forward `retryOptions` to `fetchWithRetry` +- [ ] Default retry options used in `listDocuments`/`getDocument` + +### API Efficiency +- [ ] APIs that support field selection (e.g., `$select`, `sysparm_fields`, `fields`) should request only the fields the connector needs — in both `listDocuments` AND `getDocument` +- [ ] No redundant API calls: if a helper already fetches data (e.g., site metadata), callers should reuse the result instead of making a second call for the same information +- [ ] Sequential per-item API calls (fetching details for each document in a loop) should be batched with `Promise.all` and a concurrency limit of 3-5 + +### Error Handling +- [ ] Individual document failures are caught and logged without aborting the sync +- [ ] API error responses include status codes in error messages +- [ ] No unhandled promise rejections in concurrent operations + +### Concurrency +- [ ] Concurrent API calls use reasonable batch sizes (3-5 is typical) +- [ ] No unbounded `Promise.all` over large arrays + +### Logging +- [ ] Uses `createLogger` from `@sim/logger` (not `console.log`) +- [ ] Logs sync progress at `info` level +- [ ] Logs errors at `warn` or `error` level with context + +### Registry +- [ ] Connector is exported from `connectors/{service}/index.ts` +- [ ] Connector is registered in `connectors/registry.ts` +- [ ] Registry key matches the connector's `id` field + +## Step 11: Report and Fix + +### Report Format + +Group findings by severity: + +**Critical** (will cause runtime errors, data loss, or auth failures): +- Wrong API endpoint URL or HTTP method +- Invalid or missing OAuth scopes (not in provider config) +- Incorrect response field mapping (accessing wrong path) +- SOQL/query fields that don't exist on the target object +- Pagination that silently hits undocumented API limits +- Missing error handling that would crash the sync +- `requiredScopes` not a subset of OAuth provider scopes +- Query/filter injection: user-controlled values interpolated into OData `$filter`, SOQL, or query strings without escaping + +**Warning** (incorrect behavior, data quality issues, or convention violations): +- HTML content not stripped via `htmlToPlainText` +- `getDocument` not forwarding `syncContext` +- `getDocument` hardcoded to one content type when `listDocuments` returns multiple (e.g., only pages but not blogposts) +- Missing `tagDefinition` for metadata fields returned by `mapTags` +- Incorrect `useBasicAuth` or `supportsRefreshTokenRotation` in token refresh config +- Invalid scope names that the API doesn't recognize (even if silently ignored) +- Private resources excluded from name-based lookup despite scopes being available +- Silent data truncation without logging +- Size checks using `text.length` (character count) instead of `Buffer.byteLength` (byte count) for byte-based limits +- URL-type config fields not normalized (protocol prefix, trailing slashes cause API failures) +- `VALIDATE_RETRY_OPTIONS` not threaded through helper functions called by `validateConfig` + +**Suggestion** (minor improvements): +- Missing incremental sync support despite API supporting it +- Overly broad scopes that could be narrowed (not wrong, but could be tighter) +- Source URL format could be more specific +- Missing `orderBy` for deterministic pagination +- Redundant API calls that could be cached in `syncContext` +- Sequential per-item API calls that could be batched with `Promise.all` (concurrency 3-5) +- API supports field selection but connector fetches all fields (e.g., missing `$select`, `sysparm_fields`, `fields`) +- `getDocument` re-fetches data already included in the initial API response (e.g., comments returned with post) +- Last page of pagination requests full `PAGE_SIZE` when fewer records remain (`Math.min(PAGE_SIZE, remaining)`) + +### Fix All Issues + +After reporting, fix every **critical** and **warning** issue. Apply **suggestions** where they don't add unnecessary complexity. + +### Validation Output + +After fixing, confirm: +1. `bun run lint` passes +2. TypeScript compiles clean +3. Re-read all modified files to verify fixes are correct + +## Checklist Summary + +- [ ] Read connector implementation, types, utils, registry, and OAuth config +- [ ] Pulled and read official API documentation for the service +- [ ] Validated every API endpoint URL, method, headers, and body against API docs +- [ ] Validated input sanitization: no query/filter injection, URL fields normalized +- [ ] Validated OAuth scopes: `requiredScopes` ⊆ OAuth provider `scopes` in `oauth.ts` +- [ ] Validated each scope is real and recognized by the service's API +- [ ] Validated scopes are sufficient for all API endpoints the connector calls +- [ ] Validated token refresh config (`useBasicAuth`, `supportsRefreshTokenRotation`) +- [ ] Validated pagination: cursor names, page sizes, hasMore logic, no silent caps +- [ ] Validated data transformation: plain text extraction, HTML stripping, content hashing +- [ ] Validated tag definitions match mapTags output, correct fieldTypes +- [ ] Validated config fields: canonical pairs, selector keys, required flags +- [ ] Validated validateConfig: lightweight check, error messages, retry options +- [ ] Validated getDocument: null on 404, all content types handled, no redundant re-fetches, syncContext forwarding +- [ ] Validated fetchWithRetry used for all external calls (no raw fetch), VALIDATE_RETRY_OPTIONS threaded through helpers +- [ ] Validated API efficiency: field selection used, no redundant calls, sequential fetches batched +- [ ] Validated error handling: graceful failures, no unhandled rejections +- [ ] Validated logging: createLogger, no console.log +- [ ] Validated registry: correct export, correct key +- [ ] Reported all issues grouped by severity +- [ ] Fixed all critical and warning issues +- [ ] Ran `bun run lint` after fixes +- [ ] Verified TypeScript compiles clean diff --git a/.agents/skills/source-command-validate-integration/SKILL.md b/.agents/skills/source-command-validate-integration/SKILL.md new file mode 100644 index 00000000000..a6b8e1bc96f --- /dev/null +++ b/.agents/skills/source-command-validate-integration/SKILL.md @@ -0,0 +1,295 @@ +--- +name: "source-command-validate-integration" +description: "Validate an existing Sim integration (tools, block, registry) against the service's API docs" +--- + +# source-command-validate-integration + +Use this skill when the user asks to run the migrated source command `validate-integration`. + +## Command Template + +# Validate Integration Skill + +You are an expert auditor for Sim integrations. Your job is to thoroughly validate that an existing integration is correct, complete, and follows all conventions. + +## Your Task + +When the user asks you to validate an integration: +1. Read the service's API documentation (via WebFetch or Context7) +2. Read every tool, the block, and registry entries +3. Cross-reference everything against the API docs and Sim conventions +4. Report all issues found, grouped by severity (critical, warning, suggestion) +5. Fix all issues after reporting them + +## Step 1: Gather All Files + +Read **every** file for the integration — do not skip any: + +``` +apps/sim/tools/{service}/ # All tool files, types.ts, index.ts +apps/sim/blocks/blocks/{service}.ts # Block definition +apps/sim/tools/registry.ts # Tool registry entries for this service +apps/sim/blocks/registry.ts # Block registry entry for this service +apps/sim/components/icons.tsx # Icon definition +apps/sim/lib/auth/auth.ts # OAuth config — should use getCanonicalScopesForProvider() +apps/sim/lib/oauth/oauth.ts # OAuth provider config — single source of truth for scopes +apps/sim/lib/oauth/utils.ts # Scope utilities, SCOPE_DESCRIPTIONS for modal UI +``` + +## Step 2: Pull API Documentation + +Fetch the official API docs for the service. This is the **source of truth** for: +- Endpoint URLs, HTTP methods, and auth headers +- Required vs optional parameters +- Parameter types and allowed values +- Response shapes and field names +- Pagination patterns (which param name, which response field) +- Rate limits and error formats + +## Step 3: Validate Tools + +For **every** tool file, check: + +### Tool ID and Naming +- [ ] Tool ID uses `snake_case`: `{service}_{action}` (e.g., `x_create_tweet`, `slack_send_message`) +- [ ] Tool `name` is human-readable (e.g., `'X Create Tweet'`) +- [ ] Tool `description` is a concise one-liner describing what it does +- [ ] Tool `version` is set (`'1.0.0'` or `'2.0.0'` for V2) + +### Params +- [ ] All required API params are marked `required: true` +- [ ] All optional API params are marked `required: false` +- [ ] Every param has explicit `required: true` or `required: false` — never omitted +- [ ] Param types match the API (`'string'`, `'number'`, `'boolean'`, `'json'`) +- [ ] Visibility is correct: + - `'hidden'` — ONLY for OAuth access tokens and system-injected params + - `'user-only'` — for API keys, credentials, and account-specific IDs the user must provide + - `'user-or-llm'` — for everything else (search queries, content, filters, IDs that could come from other blocks) +- [ ] Every param has a `description` that explains what it does + +### Request +- [ ] URL matches the API endpoint exactly (correct base URL, path segments, path params) +- [ ] HTTP method matches the API spec (GET, POST, PUT, PATCH, DELETE) +- [ ] Headers include correct auth pattern: + - OAuth: `Authorization: Bearer ${params.accessToken}` + - API Key: correct header name and format per the service's docs +- [ ] `Content-Type` header is set for POST/PUT/PATCH requests +- [ ] Body sends all required fields and only includes optional fields when provided +- [ ] For GET requests with query params: URL is constructed correctly with query string +- [ ] ID fields in URL paths are `.trim()`-ed to prevent copy-paste whitespace errors +- [ ] Path params use template literals correctly: `` `https://api.service.com/v1/${params.id.trim()}` `` + +### Response / transformResponse +- [ ] Correctly parses the API response (`await response.json()`) +- [ ] Extracts the right fields from the response structure (e.g., `data.data` vs `data` vs `data.results`) +- [ ] All nullable fields use `?? null` +- [ ] All optional arrays use `?? []` +- [ ] Error cases are handled: checks for missing/empty data and returns meaningful error +- [ ] Does NOT do raw JSON dumps — extracts meaningful, individual fields + +### Outputs +- [ ] All output fields match what the API actually returns +- [ ] No fields are missing that the API provides and users would commonly need +- [ ] No phantom fields defined that the API doesn't return +- [ ] `optional: true` is set on fields that may not exist in all responses +- [ ] When using `type: 'json'` and the shape is known, `properties` defines the inner fields +- [ ] When using `type: 'array'`, `items` defines the item structure with `properties` +- [ ] Field descriptions are accurate and helpful + +### Types (types.ts) +- [ ] Has param interfaces for every tool (e.g., `XCreateTweetParams`) +- [ ] Has response interfaces for every tool (extending `ToolResponse`) +- [ ] Optional params use `?` in the interface (e.g., `replyTo?: string`) +- [ ] Field names in types match actual API field names +- [ ] Shared response types are properly reused (e.g., `XTweetResponse` shared across tweet tools) + +### Barrel Export (index.ts) +- [ ] Every tool is exported +- [ ] All types are re-exported (`export * from './types'`) +- [ ] No orphaned exports (tools that don't exist) + +### Tool Registry (tools/registry.ts) +- [ ] Every tool is imported and registered +- [ ] Registry keys use snake_case and match tool IDs exactly +- [ ] Entries are in alphabetical order within the file + +## Step 4: Validate Block + +### Block ↔ Tool Alignment (CRITICAL) + +This is the most important validation — the block must be perfectly aligned with every tool it references. + +For **each tool** in `tools.access`: +- [ ] The operation dropdown has an option whose ID matches the tool ID (or the `tools.config.tool` function correctly maps to it) +- [ ] Every **required** tool param (except `accessToken`) has a corresponding subBlock input that is: + - Shown when that operation is selected (correct `condition`) + - Marked as `required: true` (or conditionally required) +- [ ] Every **optional** tool param has a corresponding subBlock input (or is intentionally omitted if truly never needed) +- [ ] SubBlock `id` values are unique across the entire block — no duplicates even across different conditions +- [ ] The `tools.config.tool` function returns the correct tool ID for every possible operation value +- [ ] The `tools.config.params` function correctly maps subBlock IDs to tool param names when they differ + +### SubBlocks +- [ ] Operation dropdown lists ALL tool operations available in `tools.access` +- [ ] Dropdown option labels are human-readable and descriptive +- [ ] Conditions use correct syntax: + - Single value: `{ field: 'operation', value: 'x_create_tweet' }` + - Multiple values (OR): `{ field: 'operation', value: ['x_create_tweet', 'x_delete_tweet'] }` + - Negation: `{ field: 'operation', value: 'delete', not: true }` + - Compound: `{ field: 'op', value: 'send', and: { field: 'type', value: 'dm' } }` +- [ ] Condition arrays include ALL operations that use that field — none missing +- [ ] `dependsOn` is set for fields that need other values (selectors depending on credential, cascading dropdowns) +- [ ] SubBlock types match tool param types: + - Enum/fixed options → `dropdown` + - Free text → `short-input` + - Long text/content → `long-input` + - True/false → `dropdown` with Yes/No options (not `switch` unless purely UI toggle) + - Credentials → `oauth-input` with correct `serviceId` +- [ ] Dropdown `value: () => 'default'` is set for dropdowns with a sensible default + +### Advanced Mode +- [ ] Optional, rarely-used fields are set to `mode: 'advanced'`: + - Pagination tokens / next tokens + - Time range filters (start/end time) + - Sort order / direction options + - Max results / per page limits + - Reply settings / threading options + - Rarely used IDs (reply-to, quote-tweet, etc.) + - Exclude filters +- [ ] **Required** fields are NEVER set to `mode: 'advanced'` +- [ ] Fields that users fill in most of the time are NOT set to `mode: 'advanced'` + +### WandConfig +- [ ] Timestamp fields have `wandConfig` with `generationType: 'timestamp'` +- [ ] Comma-separated list fields have `wandConfig` with a descriptive prompt +- [ ] Complex filter/query fields have `wandConfig` with format examples in the prompt +- [ ] All `wandConfig` prompts end with "Return ONLY the [format] - no explanations, no extra text." +- [ ] `wandConfig.placeholder` describes what to type in natural language + +### Tools Config +- [ ] `tools.access` lists **every** tool ID the block can use — none missing +- [ ] `tools.config.tool` returns the correct tool ID for each operation +- [ ] Type coercions are in `tools.config.params` (runs at execution time), NOT in `tools.config.tool` (runs at serialization time before variable resolution) +- [ ] `tools.config.params` handles: + - `Number()` conversion for numeric params that come as strings from inputs + - `Boolean` / string-to-boolean conversion for toggle params + - Empty string → `undefined` conversion for optional dropdown values + - Any subBlock ID → tool param name remapping +- [ ] No `Number()`, `JSON.parse()`, or other coercions in `tools.config.tool` — these would destroy dynamic references like `` + +### Block Outputs +- [ ] Outputs cover the key fields returned by ALL tools (not just one operation) +- [ ] Output types are correct (`'string'`, `'number'`, `'boolean'`, `'json'`) +- [ ] `type: 'json'` outputs either: + - Describe inner fields in the description string (GOOD): `'User profile (id, name, username, bio)'` + - Use nested output definitions (BEST): `{ id: { type: 'string' }, name: { type: 'string' } }` +- [ ] No opaque `type: 'json'` with vague descriptions like `'Response data'` +- [ ] Outputs that only appear for certain operations use `condition` if supported, or document which operations return them + +### Block Metadata +- [ ] `type` is snake_case (e.g., `'x'`, `'cloudflare'`) +- [ ] `name` is human-readable (e.g., `'X'`, `'Cloudflare'`) +- [ ] `description` is a concise one-liner +- [ ] `longDescription` provides detail for docs +- [ ] `docsLink` points to `'https://docs.sim.ai/tools/{service}'` +- [ ] `category` is `'tools'` +- [ ] `bgColor` uses the service's brand color hex +- [ ] `icon` references the correct icon component from `@/components/icons` +- [ ] `authMode` is set correctly (`AuthMode.OAuth` or `AuthMode.ApiKey`) +- [ ] Block is registered in `blocks/registry.ts` alphabetically + +### Block Inputs +- [ ] `inputs` section lists all subBlock params that the block accepts +- [ ] Input types match the subBlock types +- [ ] When using `canonicalParamId`, inputs list the canonical ID (not the raw subBlock IDs) + +## Step 5: Validate OAuth Scopes (if OAuth service) + +Scopes are centralized — the single source of truth is `OAUTH_PROVIDERS` in `lib/oauth/oauth.ts`. + +- [ ] Scopes defined in `lib/oauth/oauth.ts` under `OAUTH_PROVIDERS[provider].services[service].scopes` +- [ ] `auth.ts` uses `getCanonicalScopesForProvider(providerId)` — NOT a hardcoded array +- [ ] Block `requiredScopes` uses `getScopesForService(serviceId)` — NOT a hardcoded array +- [ ] No hardcoded scope arrays in `auth.ts` or block files (should all use utility functions) +- [ ] Each scope has a human-readable description in `SCOPE_DESCRIPTIONS` within `lib/oauth/utils.ts` +- [ ] No excess scopes that aren't needed by any tool + +## Step 6: Validate Pagination Consistency + +If any tools support pagination: +- [ ] Pagination param names match the API docs (e.g., `pagination_token` vs `next_token` vs `cursor`) +- [ ] Different API endpoints that use different pagination param names have separate subBlocks in the block +- [ ] Pagination response fields (`nextToken`, `cursor`, etc.) are included in tool outputs +- [ ] Pagination subBlocks are set to `mode: 'advanced'` + +## Step 7: Validate Error Handling + +- [ ] `transformResponse` checks for error conditions before accessing data +- [ ] Error responses include meaningful messages (not just generic "failed") +- [ ] HTTP error status codes are handled (check `response.ok` or status codes) + +## Step 8: Report and Fix + +### Report Format + +Group findings by severity: + +**Critical** (will cause runtime errors or incorrect behavior): +- Wrong endpoint URL or HTTP method +- Missing required params or wrong `required` flag +- Incorrect response field mapping (accessing wrong path in response) +- Missing error handling that would cause crashes +- Tool ID mismatch between tool file, registry, and block `tools.access` +- OAuth scopes missing in `auth.ts` that tools need +- `tools.config.tool` returning wrong tool ID for an operation +- Type coercions in `tools.config.tool` instead of `tools.config.params` + +**Warning** (follows conventions incorrectly or has usability issues): +- Optional field not set to `mode: 'advanced'` +- Missing `wandConfig` on timestamp/complex fields +- Wrong `visibility` on params (e.g., `'hidden'` instead of `'user-or-llm'`) +- Missing `optional: true` on nullable outputs +- Opaque `type: 'json'` without property descriptions +- Missing `.trim()` on ID fields in request URLs +- Missing `?? null` on nullable response fields +- Block condition array missing an operation that uses that field +- Hardcoded scope arrays instead of using `getScopesForService()` / `getCanonicalScopesForProvider()` +- Missing scope description in `SCOPE_DESCRIPTIONS` within `lib/oauth/utils.ts` + +**Suggestion** (minor improvements): +- Better description text +- Inconsistent naming across tools +- Missing `longDescription` or `docsLink` +- Pagination fields that could benefit from `wandConfig` + +### Fix All Issues + +After reporting, fix every **critical** and **warning** issue. Apply **suggestions** where they don't add unnecessary complexity. + +### Validation Output + +After fixing, confirm: +1. `bun run lint` passes with no fixes needed +2. TypeScript compiles clean (no type errors) +3. Re-read all modified files to verify fixes are correct + +## Checklist Summary + +- [ ] Read ALL tool files, block, types, index, and registries +- [ ] Pulled and read official API documentation +- [ ] Validated every tool's ID, params, request, response, outputs, and types against API docs +- [ ] Validated block ↔ tool alignment (every tool param has a subBlock, every condition is correct) +- [ ] Validated advanced mode on optional/rarely-used fields +- [ ] Validated wandConfig on timestamps and complex inputs +- [ ] Validated tools.config mapping, tool selector, and type coercions +- [ ] Validated block outputs match what tools return, with typed JSON where possible +- [ ] Validated OAuth scopes use centralized utilities (getScopesForService, getCanonicalScopesForProvider) — no hardcoded arrays +- [ ] Validated scope descriptions exist in `SCOPE_DESCRIPTIONS` within `lib/oauth/utils.ts` for all scopes +- [ ] Validated pagination consistency across tools and block +- [ ] Validated error handling (error checks, meaningful messages) +- [ ] Validated registry entries (tools and block, alphabetical, correct imports) +- [ ] Reported all issues grouped by severity +- [ ] Fixed all critical and warning issues +- [ ] Ran `bun run lint` after fixes +- [ ] Verified TypeScript compiles clean diff --git a/.agents/skills/source-command-validate-trigger/SKILL.md b/.agents/skills/source-command-validate-trigger/SKILL.md new file mode 100644 index 00000000000..30d04dadcfa --- /dev/null +++ b/.agents/skills/source-command-validate-trigger/SKILL.md @@ -0,0 +1,218 @@ +--- +name: "source-command-validate-trigger" +description: "Validate an existing Sim webhook trigger against provider API docs and repository conventions" +--- + +# source-command-validate-trigger + +Use this skill when the user asks to run the migrated source command `validate-trigger`. + +## Command Template + +# Validate Trigger + +You are an expert auditor for Sim webhook triggers. Your job is to validate that an existing trigger implementation is correct, complete, secure, and aligned across all layers. + +## Your Task + +1. Read the service's webhook/API documentation (via WebFetch) +2. Read every trigger file, provider handler, and registry entry +3. Cross-reference against the API docs and Sim conventions +4. Report all issues grouped by severity (critical, warning, suggestion) +5. Fix all issues after reporting them + +## Step 1: Gather All Files + +Read **every** file for the trigger — do not skip any: + +``` +apps/sim/triggers/{service}/ # All trigger files, utils.ts, index.ts +apps/sim/lib/webhooks/providers/{service}.ts # Provider handler (if exists) +apps/sim/lib/webhooks/providers/registry.ts # Handler registry +apps/sim/triggers/registry.ts # Trigger registry +apps/sim/blocks/blocks/{service}.ts # Block definition (trigger wiring) +``` + +Also read for reference: +``` +apps/sim/lib/webhooks/providers/types.ts # WebhookProviderHandler interface +apps/sim/lib/webhooks/providers/utils.ts # Shared helpers (createHmacVerifier, etc.) +apps/sim/lib/webhooks/provider-subscription-utils.ts # Subscription helpers +apps/sim/lib/webhooks/processor.ts # Central webhook processor +``` + +## Step 2: Pull API Documentation + +Fetch the service's official webhook documentation. This is the **source of truth** for: +- Webhook event types and payload shapes +- Signature/auth verification method (HMAC algorithm, header names, secret format) +- Challenge/verification handshake requirements +- Webhook subscription API (create/delete endpoints, if applicable) +- Retry behavior and delivery guarantees + +## Step 3: Validate Trigger Definitions + +### utils.ts +- [ ] `{service}TriggerOptions` lists all trigger IDs accurately +- [ ] `{service}SetupInstructions` provides clear, correct steps for the service +- [ ] `build{Service}ExtraFields` includes relevant filter/config fields with correct `condition` +- [ ] Output builders expose all meaningful fields from the webhook payload +- [ ] Output builders do NOT use `optional: true` or `items` (tool-output-only features) +- [ ] Nested output objects correctly model the payload structure + +### Trigger Files +- [ ] Exactly one primary trigger has `includeDropdown: true` +- [ ] All secondary triggers do NOT have `includeDropdown` +- [ ] All triggers use `buildTriggerSubBlocks` helper (not hand-rolled subBlocks) +- [ ] Every trigger's `id` matches the convention `{service}_{event_name}` +- [ ] Every trigger's `provider` matches the service name used in the handler registry +- [ ] `index.ts` barrel exports all triggers + +### Trigger ↔ Provider Alignment (CRITICAL) +- [ ] Every trigger ID referenced in `matchEvent` logic exists in `{service}TriggerOptions` +- [ ] Event matching logic in the provider correctly maps trigger IDs to service event types +- [ ] Event matching logic in `is{Service}EventMatch` (if exists) correctly identifies events per the API docs + +## Step 4: Validate Provider Handler + +### Auth Verification +- [ ] `verifyAuth` correctly validates webhook signatures per the service's documentation +- [ ] HMAC algorithm matches (SHA-1, SHA-256, SHA-512) +- [ ] Signature header name matches the API docs exactly +- [ ] Signature format is handled (raw hex, `sha256=` prefix, base64, etc.) +- [ ] Uses `safeCompare` for timing-safe comparison (no `===`) +- [ ] If `webhookSecret` is required, handler rejects when it's missing (fail-closed) +- [ ] Signature is computed over raw body (not parsed JSON) + +### Event Matching +- [ ] `matchEvent` returns `boolean` (not `NextResponse` or other values) +- [ ] Challenge/verification events are excluded from matching (e.g., `endpoint.url_validation`) +- [ ] When `triggerId` is a generic webhook ID, all events pass through +- [ ] When `triggerId` is specific, only matching events pass +- [ ] Event matching logic uses dynamic `await import()` for trigger utils (avoids circular deps) + +### formatInput (CRITICAL) +- [ ] Every key in the `formatInput` return matches a key in the trigger `outputs` schema +- [ ] Every key in the trigger `outputs` schema is populated by `formatInput` +- [ ] No extra undeclared keys that users can't discover in the UI +- [ ] No wrapper objects (`webhook: { ... }`, `{service}: { ... }`) +- [ ] Nested output paths exist at the correct depth (e.g., `resource.id` actually has `resource: { id: ... }`) +- [ ] `null` is used for missing optional fields (not empty strings or empty objects) +- [ ] Returns `{ input: { ... } }` — not a bare object + +### Idempotency +- [ ] `extractIdempotencyId` returns a stable, unique key per delivery +- [ ] Uses provider-specific delivery IDs when available (e.g., `X-Request-Id`, `Linear-Delivery`, `svix-id`) +- [ ] Falls back to content-based ID (e.g., `${type}:${id}`) when no delivery header exists +- [ ] Does NOT include timestamps in the idempotency key (would break dedup on retries) + +### Challenge Handling (if applicable) +- [ ] `handleChallenge` correctly implements the service's URL verification handshake +- [ ] Returns the expected response format per the API docs +- [ ] Env-backed secrets are resolved via `resolveEnvVarsInObject` if needed + +## Step 5: Validate Automatic Subscription Lifecycle + +If the service supports programmatic webhook creation: + +### createSubscription +- [ ] Calls the correct API endpoint to create a webhook +- [ ] Sends the correct event types/filters +- [ ] Passes the notification URL from `getNotificationUrl(ctx.webhook)` +- [ ] Returns `{ providerConfigUpdates: { externalId } }` with the external webhook ID +- [ ] Throws on failure (orchestration handles rollback) +- [ ] Provides user-friendly error messages (401 → "Invalid API Key", etc.) + +### deleteSubscription +- [ ] Calls the correct API endpoint to delete the webhook +- [ ] Handles 404 gracefully (webhook already deleted) +- [ ] Never throws — catches errors and logs non-fatally +- [ ] Skips gracefully when `apiKey` or `externalId` is missing + +### Orchestration Isolation +- [ ] NO provider-specific logic in `route.ts`, `provider-subscriptions.ts`, or `deploy.ts` +- [ ] All subscription logic lives on the handler (`createSubscription`/`deleteSubscription`) + +## Step 6: Validate Registration and Block Wiring + +### Trigger Registry (`triggers/registry.ts`) +- [ ] All triggers are imported and registered +- [ ] Registry keys match trigger IDs exactly +- [ ] No orphaned entries (triggers that don't exist) + +### Provider Handler Registry (`providers/registry.ts`) +- [ ] Handler is imported and registered (if handler exists) +- [ ] Registry key matches the `provider` field on the trigger configs +- [ ] Entries are in alphabetical order + +### Block Wiring (`blocks/blocks/{service}.ts`) +- [ ] Block has `triggers.enabled: true` +- [ ] `triggers.available` lists all trigger IDs +- [ ] All trigger subBlocks are spread into `subBlocks`: `...getTrigger('id').subBlocks` +- [ ] No trigger IDs in `triggers.available` that aren't in the registry +- [ ] No trigger subBlocks spread that aren't in `triggers.available` + +## Step 7: Validate Security + +- [ ] Webhook secrets are never logged (not even at debug level) +- [ ] Auth verification runs before any event processing +- [ ] No secret comparison uses `===` (must use `safeCompare` or `crypto.timingSafeEqual`) +- [ ] Timestamp/replay protection is reasonable (not too tight for retries, not too loose for security) +- [ ] Raw body is used for signature verification (not re-serialized JSON) + +## Step 8: Report and Fix + +### Report Format + +Group findings by severity: + +**Critical** (runtime errors, security issues, or data loss): +- Wrong HMAC algorithm or header name +- `formatInput` keys don't match trigger `outputs` +- Missing `verifyAuth` when the service sends signed webhooks +- `matchEvent` returns non-boolean values +- Provider-specific logic leaking into shared orchestration files +- Trigger IDs mismatch between trigger files, registry, and block +- `createSubscription` calling wrong API endpoint +- Auth comparison using `===` instead of `safeCompare` + +**Warning** (convention violations or usability issues): +- Missing `extractIdempotencyId` when the service provides delivery IDs +- Timestamps in idempotency keys (breaks dedup on retries) +- Missing challenge handling when the service requires URL verification +- Output schema missing fields that `formatInput` returns (undiscoverable data) +- Overly tight timestamp skew window that rejects legitimate retries +- `matchEvent` not filtering challenge/verification events +- Setup instructions missing important steps + +**Suggestion** (minor improvements): +- More specific output field descriptions +- Additional output fields that could be exposed +- Better error messages in `createSubscription` +- Logging improvements + +### Fix All Issues + +After reporting, fix every **critical** and **warning** issue. Apply **suggestions** where they don't add unnecessary complexity. + +### Validation Output + +After fixing, confirm: +1. `bun run type-check` passes +2. Re-read all modified files to verify fixes are correct +3. Provider handler tests pass (if they exist): `bun test {service}` + +## Checklist Summary + +- [ ] Read all trigger files, provider handler, types, registries, and block +- [ ] Pulled and read official webhook/API documentation +- [ ] Validated trigger definitions: options, instructions, extra fields, outputs +- [ ] Validated primary/secondary trigger distinction (`includeDropdown`) +- [ ] Validated provider handler: auth, matchEvent, formatInput, idempotency +- [ ] Validated output alignment: every `outputs` key ↔ every `formatInput` key +- [ ] Validated subscription lifecycle: createSubscription, deleteSubscription, no shared-file edits +- [ ] Validated registration: trigger registry, handler registry, block wiring +- [ ] Validated security: safe comparison, no secret logging, replay protection +- [ ] Reported all issues grouped by severity +- [ ] Fixed all critical and warning issues +- [ ] `bun run type-check` passes after fixes diff --git a/apps/sim/app/api/mcp/oauth/callback/route.ts b/apps/sim/app/api/mcp/oauth/callback/route.ts new file mode 100644 index 00000000000..721675dac9d --- /dev/null +++ b/apps/sim/app/api/mcp/oauth/callback/route.ts @@ -0,0 +1,181 @@ +import { auth as mcpAuth } from '@modelcontextprotocol/sdk/client/auth.js' +import { db } from '@sim/db' +import { mcpServers } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { and, eq, isNull } from 'drizzle-orm' +import type { NextRequest } from 'next/server' +import { NextResponse } from 'next/server' +import { mcpOauthCallbackContract } from '@/lib/api/contracts/mcp' +import { parseRequest } from '@/lib/api/server' +import { getSession } from '@/lib/auth' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { + assertSafeOauthServerUrl, + clearState, + clearVerifier, + loadOauthRowByState, + loadPreregisteredClient, + type McpOauthCallbackReason, + SimMcpOauthProvider, +} from '@/lib/mcp/oauth' +import { mcpService } from '@/lib/mcp/service' + +const logger = createLogger('McpOauthCallbackAPI') + +export const dynamic = 'force-dynamic' + +function escapeHtml(value: string): string { + return value + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, ''') +} + +function jsonLiteral(value: string | undefined): string { + if (value === undefined) return 'undefined' + return JSON.stringify(value).replace(//g, '\\u003e') +} + +function htmlClose( + message: string, + ok: boolean, + reason: McpOauthCallbackReason, + serverId?: string +): NextResponse { + const safeMessage = escapeHtml(message) + const title = ok ? 'Connected' : 'Connection failed' + const body = `${title}

${safeMessage}

` + return new NextResponse(body, { + headers: { 'Content-Type': 'text/html; charset=utf-8' }, + }) +} + +export const GET = withRouteHandler(async (request: NextRequest) => { + const parsed = await parseRequest(mcpOauthCallbackContract, request, {}) + if (!parsed.success) { + return htmlClose('Malformed authorization callback.', false, 'missing_params') + } + const { state, code, error: errorParam } = parsed.data.query + + const initialRow = state ? await loadOauthRowByState(state).catch(() => null) : null + const stateRowServerId = initialRow?.mcpServerId + + if (errorParam) { + logger.warn(`MCP OAuth callback received error: ${errorParam}`) + if (initialRow) await clearState(initialRow.id).catch(() => {}) + return htmlClose( + `Authorization failed: ${errorParam}`, + false, + 'provider_error', + stateRowServerId + ) + } + if (!state || !code) { + return htmlClose( + 'Missing state or code in callback URL.', + false, + 'missing_params', + stateRowServerId + ) + } + + let serverId: string | undefined + try { + const session = await getSession() + if (!session?.user?.id) { + return htmlClose( + 'You must be signed in to complete authorization.', + false, + 'unauthenticated', + stateRowServerId + ) + } + + const row = initialRow + if (!row) { + return htmlClose('Invalid or expired authorization state.', false, 'invalid_state') + } + serverId = row.mcpServerId + + if (session.user.id !== row.userId) { + return htmlClose( + 'You must be signed in as the same user that initiated the flow.', + false, + 'user_mismatch', + serverId + ) + } + + const [server] = await db + .select({ id: mcpServers.id, url: mcpServers.url, workspaceId: mcpServers.workspaceId }) + .from(mcpServers) + .where(and(eq(mcpServers.id, row.mcpServerId), isNull(mcpServers.deletedAt))) + .limit(1) + if (!server || !server.url) { + return htmlClose('Server no longer exists.', false, 'server_gone', serverId) + } + if (server.workspaceId !== row.workspaceId) { + return htmlClose( + 'Workspace mismatch on authorization callback.', + false, + 'invalid_state', + serverId + ) + } + try { + assertSafeOauthServerUrl(server.url) + } catch { + return htmlClose( + 'MCP OAuth requires https (or http://localhost for development).', + false, + 'insecure_url', + serverId + ) + } + + // Burn state before token exchange so a replayed callback cannot reuse it. + await clearState(row.id) + + const preregistered = await loadPreregisteredClient(server.id) + const provider = new SimMcpOauthProvider({ row, preregistered }) + let result: Awaited> + try { + result = await mcpAuth(provider, { + serverUrl: server.url, + authorizationCode: code, + }) + } catch (e) { + logger.error('Token exchange failed during MCP OAuth callback', e) + return htmlClose( + 'Token exchange failed. Please try again.', + false, + 'token_exchange_failed', + server.id + ) + } finally { + await clearVerifier(row.id) + } + + if (result !== 'AUTHORIZED') { + return htmlClose('Authorization did not complete.', false, 'token_exchange_failed', server.id) + } + + try { + await mcpService.clearCache(server.workspaceId) + await mcpService.discoverServerTools(session.user.id, server.id, server.workspaceId) + } catch (e) { + logger.warn('Post-auth tools refresh failed', toError(e).message) + } + + return htmlClose('Connected. You can close this window.', true, 'authorized', server.id) + } catch (error) { + logger.error('MCP OAuth callback failed', error) + return htmlClose('Authorization failed. Please try again.', false, 'unknown', serverId) + } +}) diff --git a/apps/sim/app/api/mcp/oauth/start/route.test.ts b/apps/sim/app/api/mcp/oauth/start/route.test.ts new file mode 100644 index 00000000000..7c81138ca42 --- /dev/null +++ b/apps/sim/app/api/mcp/oauth/start/route.test.ts @@ -0,0 +1,137 @@ +/** + * @vitest-environment node + */ +import { + dbChainMock, + dbChainMockFns, + hybridAuthMock, + hybridAuthMockFns, + McpOauthRedirectRequiredMock, + mcpOauthMock, + mcpOauthMockFns, + permissionsMock, + permissionsMockFns, + resetDbChainMock, + schemaMock, +} from '@sim/testing' +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const { mockMcpAuth } = vi.hoisted(() => ({ + mockMcpAuth: vi.fn(), +})) + +vi.mock('@sim/db', () => dbChainMock) +vi.mock('@sim/db/schema', () => schemaMock) +vi.mock('drizzle-orm', () => ({ + and: vi.fn(), + eq: vi.fn(), + isNull: vi.fn(), +})) +vi.mock('@modelcontextprotocol/sdk/client/auth.js', () => ({ + auth: mockMcpAuth, +})) +vi.mock('@/lib/auth/hybrid', () => hybridAuthMock) +vi.mock('@/lib/workspaces/permissions/utils', () => permissionsMock) +vi.mock('@/lib/mcp/oauth', () => mcpOauthMock) + +import { GET } from './route' + +describe('MCP OAuth start route', () => { + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + hybridAuthMockFns.mockCheckSessionOrInternalAuth.mockResolvedValue({ + success: true, + userId: 'user-2', + userName: 'User Two', + userEmail: 'user2@example.com', + authType: 'session', + }) + permissionsMockFns.mockGetUserEntityPermissions.mockResolvedValue('write') + dbChainMockFns.limit.mockResolvedValue([ + { + id: 'server-1', + name: 'Exa', + url: 'https://mcp.exa.ai/mcp', + workspaceId: 'workspace-1', + authType: 'oauth', + deletedAt: null, + }, + ]) + mcpOauthMockFns.mockGetOrCreateOauthRow.mockResolvedValue({ + id: 'oauth-row-1', + mcpServerId: 'server-1', + userId: 'user-1', + workspaceId: 'workspace-1', + clientInformation: null, + tokens: null, + codeVerifier: null, + state: null, + stateCreatedAt: null, + updatedAt: new Date(), + }) + mcpOauthMockFns.mockLoadPreregisteredClient.mockResolvedValue(undefined) + mockMcpAuth.mockRejectedValue(new McpOauthRedirectRequiredMock('https://mcp.exa.ai/authorize')) + }) + + it('requires workspace write permission via MCP auth middleware', async () => { + const request = new NextRequest( + 'http://localhost:3000/api/mcp/oauth/start?workspaceId=workspace-1&serverId=server-1' + ) + + await GET(request) + + expect(permissionsMockFns.mockGetUserEntityPermissions).toHaveBeenCalledWith( + 'user-2', + 'workspace', + 'workspace-1' + ) + }) + + it('uses a workspace-scoped OAuth row and stamps the latest authorizing user', async () => { + const request = new NextRequest( + 'http://localhost:3000/api/mcp/oauth/start?workspaceId=workspace-1&serverId=server-1' + ) + + const response = await GET(request) + const body = await response.json() + + expect(response.status).toBe(200) + expect(body).toEqual({ + status: 'redirect', + authorizationUrl: 'https://mcp.exa.ai/authorize', + }) + expect(mcpOauthMockFns.mockGetOrCreateOauthRow).toHaveBeenCalledWith({ + mcpServerId: 'server-1', + userId: 'user-2', + workspaceId: 'workspace-1', + }) + expect(mcpOauthMockFns.mockSetOauthRowUser).toHaveBeenCalledWith('oauth-row-1', 'user-2') + }) + + it('rejects a second user starting OAuth while another authorization is active', async () => { + mcpOauthMockFns.mockGetOrCreateOauthRow.mockResolvedValueOnce({ + id: 'oauth-row-1', + mcpServerId: 'server-1', + userId: 'user-1', + workspaceId: 'workspace-1', + clientInformation: null, + tokens: null, + codeVerifier: null, + state: 'hashed-active-state', + stateCreatedAt: new Date(), + updatedAt: new Date(), + }) + const request = new NextRequest( + 'http://localhost:3000/api/mcp/oauth/start?workspaceId=workspace-1&serverId=server-1' + ) + + const response = await GET(request) + const body = await response.json() + + expect(response.status).toBe(409) + expect(body.error).toBe('OAuth authorization already in progress for this server') + expect(mockMcpAuth).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/api/mcp/oauth/start/route.ts b/apps/sim/app/api/mcp/oauth/start/route.ts new file mode 100644 index 00000000000..55a7d41956f --- /dev/null +++ b/apps/sim/app/api/mcp/oauth/start/route.ts @@ -0,0 +1,122 @@ +import { auth as mcpAuth } from '@modelcontextprotocol/sdk/client/auth.js' +import { db } from '@sim/db' +import { mcpServers } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { toError } from '@sim/utils/errors' +import { and, eq, isNull } from 'drizzle-orm' +import type { NextRequest } from 'next/server' +import { NextResponse } from 'next/server' +import { startMcpOauthContract } from '@/lib/api/contracts/mcp' +import { parseRequest } from '@/lib/api/server' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { withMcpAuth } from '@/lib/mcp/middleware' +import { + assertSafeOauthServerUrl, + getOrCreateOauthRow, + loadPreregisteredClient, + McpOauthInsecureUrlError, + McpOauthRedirectRequired, + SimMcpOauthProvider, + setOauthRowUser, +} from '@/lib/mcp/oauth' +import { createMcpErrorResponse } from '@/lib/mcp/utils' + +const logger = createLogger('McpOauthStartAPI') +const OAUTH_START_TTL_MS = 10 * 60 * 1000 + +export const dynamic = 'force-dynamic' + +export const GET = withRouteHandler( + withMcpAuth('write')(async (request: NextRequest, { userId, workspaceId }) => { + try { + const parsed = await parseRequest(startMcpOauthContract, request, {}) + if (!parsed.success) return parsed.response + const { serverId } = parsed.data.query + + const [server] = await db + .select() + .from(mcpServers) + .where( + and( + eq(mcpServers.id, serverId), + eq(mcpServers.workspaceId, workspaceId), + isNull(mcpServers.deletedAt) + ) + ) + .limit(1) + + if (!server) { + return createMcpErrorResponse(new Error('Server not found'), 'Server not found', 404) + } + if (server.authType !== 'oauth') { + return createMcpErrorResponse( + new Error(`Server authType is "${server.authType}", not oauth`), + 'Server is not configured for OAuth', + 400 + ) + } + if (!server.url) { + return createMcpErrorResponse(new Error('Server has no URL'), 'Missing server URL', 400) + } + try { + assertSafeOauthServerUrl(server.url) + } catch (e) { + if (e instanceof McpOauthInsecureUrlError) { + return createMcpErrorResponse( + e, + 'MCP OAuth requires https (or http://localhost for development)', + 400 + ) + } + throw e + } + + const row = await getOrCreateOauthRow({ + mcpServerId: server.id, + userId, + workspaceId, + }) + const hasActiveFlow = + !!row.state && + !!row.stateCreatedAt && + row.stateCreatedAt.getTime() > Date.now() - OAUTH_START_TTL_MS + if (hasActiveFlow && row.userId && row.userId !== userId) { + return createMcpErrorResponse( + new Error('OAuth authorization already in progress'), + 'OAuth authorization already in progress for this server', + 409 + ) + } + if (row.userId !== userId) { + await setOauthRowUser(row.id, userId) + row.userId = userId + } + const preregistered = await loadPreregisteredClient(server.id) + const provider = new SimMcpOauthProvider({ row, preregistered }) + + try { + const result = await mcpAuth(provider, { serverUrl: server.url }) + if (result === 'AUTHORIZED') { + return NextResponse.json({ status: 'already_authorized' }) + } + return createMcpErrorResponse( + new Error('Provider did not capture redirect URL'), + 'Failed to start OAuth flow', + 500 + ) + } catch (e) { + if (e instanceof McpOauthRedirectRequired) { + logger.info(`OAuth redirect for server ${serverId}`) + return NextResponse.json({ + status: 'redirect', + authorizationUrl: e.authorizationUrl, + }) + } + throw e + } + } catch (error) { + logger.error('Error starting MCP OAuth flow:', error) + return createMcpErrorResponse(toError(error), 'Failed to start OAuth flow', 500) + } + }) +) diff --git a/apps/sim/app/api/mcp/servers/[id]/route.ts b/apps/sim/app/api/mcp/servers/[id]/route.ts index 6795f6383e1..4242fdef119 100644 --- a/apps/sim/app/api/mcp/servers/[id]/route.ts +++ b/apps/sim/app/api/mcp/servers/[id]/route.ts @@ -45,23 +45,25 @@ export const PATCH = withRouteHandler( } ) - // Remove workspaceId from body to prevent it from being updated - const { workspaceId: _, ...updateData } = body - const result = await performUpdateMcpServer({ workspaceId, userId, actorName: userName, actorEmail: userEmail, serverId, - name: updateData.name, - description: updateData.description, - transport: updateData.transport, - url: updateData.url, - headers: updateData.headers, - timeout: updateData.timeout, - retries: updateData.retries, - enabled: updateData.enabled, + name: body.name, + description: body.description, + transport: body.transport, + url: body.url, + headers: body.headers, + timeout: body.timeout, + retries: body.retries, + enabled: body.enabled, + authType: body.authType, + oauthClientId: body.oauthClientId || null, + oauthClientIdProvided: body.oauthClientId !== undefined, + oauthClientSecret: body.oauthClientSecret, + oauthClientSecretProvided: body.oauthClientSecret !== undefined, request, }) if (!result.success || !result.server) { @@ -75,7 +77,10 @@ export const PATCH = withRouteHandler( logger.info(`[${requestId}] Successfully updated MCP server: ${serverId}`) - return createMcpSuccessResponse({ server: updatedServer }) + const { oauthClientSecret: _secret, ...rest } = updatedServer + return createMcpSuccessResponse({ + server: { ...rest, hasOauthClientSecret: !!_secret }, + }) } catch (error) { logger.error(`[${requestId}] Error updating MCP server:`, error) return createMcpErrorResponse(toError(error), 'Failed to update MCP server', 500) diff --git a/apps/sim/app/api/mcp/servers/route.ts b/apps/sim/app/api/mcp/servers/route.ts index f0f2744b053..1d02caeef74 100644 --- a/apps/sim/app/api/mcp/servers/route.ts +++ b/apps/sim/app/api/mcp/servers/route.ts @@ -27,11 +27,16 @@ export const GET = withRouteHandler( try { logger.info(`[${requestId}] Listing MCP servers for workspace ${workspaceId}`) - const servers = await db + const rows = await db .select() .from(mcpServers) .where(and(eq(mcpServers.workspaceId, workspaceId), isNull(mcpServers.deletedAt))) + const servers = rows.map(({ oauthClientSecret: _secret, ...rest }) => ({ + ...rest, + hasOauthClientSecret: !!_secret, + })) + logger.info( `[${requestId}] Listed ${servers.length} MCP servers for workspace ${workspaceId}` ) @@ -45,13 +50,6 @@ export const GET = withRouteHandler( /** * POST - Register a new MCP server for the workspace (requires write permission) - * - * Uses deterministic server IDs based on URL hash to ensure that re-adding - * the same server produces the same ID. This prevents "server not found" errors - * when workflows reference the old server ID after delete/re-add cycles. - * - * If a server with the same ID already exists (same URL in same workspace), - * it will be updated instead of creating a duplicate. */ export const POST = withRouteHandler( withMcpAuth('write')( @@ -96,6 +94,11 @@ export const POST = withRouteHandler( retries: body.retries, enabled: body.enabled, source, + authType: body.authType, + oauthClientId: body.oauthClientId || null, + oauthClientIdProvided: body.oauthClientId !== undefined, + oauthClientSecret: body.oauthClientSecret, + oauthClientSecretProvided: body.oauthClientSecret !== undefined, request, }) if (!result.success || !result.serverId) { @@ -112,8 +115,8 @@ export const POST = withRouteHandler( return createMcpSuccessResponse( result.updated - ? { serverId: result.serverId, updated: true } - : { serverId: result.serverId }, + ? { serverId: result.serverId, updated: true, authType: result.authType } + : { serverId: result.serverId, authType: result.authType }, result.updated ? 200 : 201 ) } catch (error) { diff --git a/apps/sim/app/api/mcp/tools/discover/route.ts b/apps/sim/app/api/mcp/tools/discover/route.ts index e94f2f56328..b125fa7ff2b 100644 --- a/apps/sim/app/api/mcp/tools/discover/route.ts +++ b/apps/sim/app/api/mcp/tools/discover/route.ts @@ -1,3 +1,4 @@ +import { UnauthorizedError } from '@modelcontextprotocol/sdk/client/auth.js' import { createLogger } from '@sim/logger' import type { NextRequest } from 'next/server' import { mcpToolDiscoveryQuerySchema, refreshMcpToolsBodySchema } from '@/lib/api/contracts/mcp' @@ -5,7 +6,7 @@ import { validationErrorResponse } from '@/lib/api/server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { getParsedBody, withMcpAuth } from '@/lib/mcp/middleware' import { mcpService } from '@/lib/mcp/service' -import type { McpToolDiscoveryResponse } from '@/lib/mcp/types' +import { McpOauthAuthorizationRequiredError, type McpToolDiscoveryResponse } from '@/lib/mcp/types' import { categorizeError, createMcpErrorResponse, createMcpSuccessResponse } from '@/lib/mcp/utils' const logger = createLogger('McpToolDiscoveryAPI') @@ -46,6 +47,12 @@ export const GET = withRouteHandler( ) return createMcpSuccessResponse(responseData) } catch (error) { + if ( + error instanceof McpOauthAuthorizationRequiredError || + error instanceof UnauthorizedError + ) { + return createMcpErrorResponse(error, 'OAuth re-authorization required', 401) + } logger.error(`[${requestId}] Error discovering MCP tools:`, error) const { message, status } = categorizeError(error) return createMcpErrorResponse(new Error(message), 'Failed to discover MCP tools', status) @@ -100,6 +107,12 @@ export const POST = withRouteHandler( }, }) } catch (error) { + if ( + error instanceof McpOauthAuthorizationRequiredError || + error instanceof UnauthorizedError + ) { + return createMcpErrorResponse(error, 'OAuth re-authorization required', 401) + } logger.error(`[${requestId}] Error refreshing tool discovery:`, error) const { message, status } = categorizeError(error) return createMcpErrorResponse(new Error(message), 'Failed to refresh tool discovery', status) diff --git a/apps/sim/app/api/mcp/tools/execute/route.ts b/apps/sim/app/api/mcp/tools/execute/route.ts index d9458deceab..8599a5fcadf 100644 --- a/apps/sim/app/api/mcp/tools/execute/route.ts +++ b/apps/sim/app/api/mcp/tools/execute/route.ts @@ -1,5 +1,7 @@ +import { UnauthorizedError } from '@modelcontextprotocol/sdk/client/auth.js' import { createLogger } from '@sim/logger' import type { NextRequest } from 'next/server' +import { NextResponse } from 'next/server' import { mcpToolExecutionBodySchema } from '@/lib/api/contracts/mcp' import { getHighestPrioritySubscription } from '@/lib/billing/core/plan' import { getExecutionTimeout } from '@/lib/core/execution-limits' @@ -7,8 +9,14 @@ import type { SubscriptionPlan } from '@/lib/core/rate-limiter/types' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { SIM_VIA_HEADER } from '@/lib/execution/call-chain' import { getParsedBody, withMcpAuth } from '@/lib/mcp/middleware' +import { McpOauthRedirectRequired } from '@/lib/mcp/oauth' import { mcpService } from '@/lib/mcp/service' -import type { McpTool, McpToolCall, McpToolResult } from '@/lib/mcp/types' +import { + McpOauthAuthorizationRequiredError, + type McpTool, + type McpToolCall, + type McpToolResult, +} from '@/lib/mcp/types' import { categorizeError, createMcpErrorResponse, createMcpSuccessResponse } from '@/lib/mcp/utils' import { assertPermissionsAllowed, @@ -43,6 +51,7 @@ function hasType(prop: unknown): prop is SchemaProperty { */ export const POST = withRouteHandler( withMcpAuth('read')(async (request: NextRequest, { userId, workspaceId, requestId }) => { + let serverId: string | undefined try { const rawBody = getParsedBody(request) ?? (await request.json()) const parsedBody = mcpToolExecutionBodySchema.safeParse(rawBody) @@ -63,7 +72,8 @@ export const POST = withRouteHandler( userId: userId, }) - const { serverId, toolName, arguments: rawArgs } = body + const { toolName, arguments: rawArgs } = body + serverId = body.serverId const args = rawArgs || {} try { @@ -101,7 +111,8 @@ export const POST = withRouteHandler( if (tool.inputSchema?.properties) { for (const [paramName, paramSchema] of Object.entries(tool.inputSchema.properties)) { - const schema = paramSchema as any + const schema = hasType(paramSchema) ? paramSchema : null + if (!schema) continue const value = args[paramName] if (value === undefined || value === null) { @@ -185,12 +196,18 @@ export const POST = withRouteHandler( extraHeaders[SIM_VIA_HEADER] = simViaHeader } + let timeoutHandle: ReturnType | undefined const result = await Promise.race([ mcpService.executeTool(userId, serverId, toolCall, workspaceId, extraHeaders), - new Promise((_, reject) => - setTimeout(() => reject(new Error('Tool execution timeout')), executionTimeout) - ), - ]) + new Promise((_, reject) => { + timeoutHandle = setTimeout( + () => reject(new Error('Tool execution timeout')), + executionTimeout + ) + }), + ]).finally(() => { + if (timeoutHandle !== undefined) clearTimeout(timeoutHandle) + }) const transformedResult = transformToolResult(result) @@ -218,6 +235,27 @@ export const POST = withRouteHandler( return createMcpSuccessResponse(transformedResult) } catch (error) { + if ( + error instanceof McpOauthAuthorizationRequiredError || + error instanceof McpOauthRedirectRequired || + error instanceof UnauthorizedError + ) { + const errorServerId = + error instanceof McpOauthAuthorizationRequiredError ? error.serverId : serverId + logger.warn(`[${requestId}] OAuth re-authorization required for MCP tool execution`, { + serverId: errorServerId, + }) + return NextResponse.json( + { + success: false, + error: 'OAuth re-authorization required', + code: 'reauth_required', + serverId: errorServerId, + }, + { status: 401 } + ) + } + logger.error(`[${requestId}] Error executing MCP tool:`, error) const { message, status } = categorizeError(error) diff --git a/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/utils.ts b/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/utils.ts index afdf52bdaab..4f142b0c67d 100644 --- a/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/utils.ts +++ b/apps/sim/app/workspace/[workspaceId]/logs/components/log-details/utils.ts @@ -8,6 +8,20 @@ import { getBlock, getBlockByToolName } from '@/blocks' import { PROVIDER_DEFINITIONS } from '@/providers/models' import { normalizeToolId } from '@/tools/normalize' +/** + * Extracts the bare tool name from an MCP tool id of the form + * `mcp-{serverId}-{toolName}`. Returns null when the id is not MCP-shaped. + * Kept local to avoid importing from `@/lib/mcp/utils`, which pulls in + * `next/server` and breaks client bundles. + */ +function tryParseMcpToolName(toolId: string): string | null { + if (!toolId.startsWith('mcp-')) return null + const parts = toolId.split('-') + if (parts.length < 3) return null + const toolName = parts.slice(2).join('-') + return toolName.length > 0 ? toolName : null +} + export const DEFAULT_BLOCK_COLOR = '#6b7280' export interface BlockIconAndColor { @@ -41,6 +55,10 @@ export function getBlockIconAndColor( ): BlockIconAndColor { const lowerType = type.toLowerCase() if (lowerType === 'tool' && toolName) { + if (tryParseMcpToolName(toolName)) { + const mcpBlock = getBlock('mcp') + if (mcpBlock) return { icon: mcpBlock.icon, bgColor: mcpBlock.bgColor } + } const normalized = normalizeToolId(toolName) if (normalized === 'load_skill') return { icon: AgentSkillsIcon, bgColor: '#8B5CF6' } const toolBlock = getBlockByToolName(normalized) @@ -90,7 +108,11 @@ export function formatTps( } export function getDisplayName(span: TraceSpan): string { - if (span.type?.toLowerCase() === 'tool') return normalizeToolId(span.name) + if (span.type?.toLowerCase() === 'tool') { + const mcpToolName = tryParseMcpToolName(span.name) + if (mcpToolName) return mcpToolName + return normalizeToolId(span.name) + } return span.name } diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/mcp/components/form-field/form-field.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/mcp/components/form-field/form-field.tsx index 04beeb1484a..cad5381d1d2 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/components/mcp/components/form-field/form-field.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/mcp/components/form-field/form-field.tsx @@ -9,7 +9,7 @@ interface FormFieldProps { export function FormField({ label, children, optional }: FormFieldProps) { return (
-
) : ( -
+
+ + -
- Headers +
{(formData.headers || []).map((header, index) => ( ))}
-
+ + + + {showAdvanced && ( +
+ + { + if (testResult) clearTestResult() + if (submitError) setSubmitError(null) + setFormData((prev) => ({ ...prev, oauthClientId: e.target.value })) + }} + className='h-9' + /> + + + { + if (testResult) clearTestResult() + if (submitError) setSubmitError(null) + setOauthClientSecretTouched(value.length > 0) + setFormData((prev) => ({ ...prev, oauthClientSecret: value })) + }} + className='h-9' + /> + +

+ Only needed for servers that don't support automatic client registration. +

+
+ )}
)} - + {submitError && ( -

{submitError}

+

{submitError}

)}
@@ -716,7 +812,7 @@ export function McpServerFormModal({ )}
- {formMode === 'json' ? ( diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/mcp/mcp.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/mcp/mcp.tsx index a21da0f563e..e8bc927a284 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/components/mcp/mcp.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/mcp/mcp.tsx @@ -1,6 +1,6 @@ 'use client' -import { useCallback, useEffect, useMemo, useState } from 'react' +import { useEffect, useState } from 'react' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { ChevronDown, Plus, Search } from 'lucide-react' @@ -27,6 +27,7 @@ import { type McpToolIssue, } from '@/lib/mcp/tool-validation' import type { McpTransport } from '@/lib/mcp/types' +import { useMcpOauthPopup } from '@/hooks/mcp/use-mcp-oauth-popup' import { type McpServer, type McpTool, @@ -102,7 +103,10 @@ function ServerListItem({ ({transportLabel})

{isRefreshing ? 'Refreshing...' @@ -123,14 +127,29 @@ function ServerListItem({ ) } +function buildEditInitialData(server: McpServer) { + const entries: { key: string; value: string }[] = server.headers + ? Object.entries(server.headers).map(([key, value]) => ({ key, value })) + : [] + if (entries.length === 0) entries.push({ key: '', value: '' }) + const last = entries[entries.length - 1] + if (last.key !== '' || last.value !== '') entries.push({ key: '', value: '' }) + + return { + name: server.name || '', + transport: (server.transport as McpTransport) || 'streamable-http', + url: server.url || '', + timeout: 30000, + headers: entries, + oauthClientId: server.oauthClientId || undefined, + hasOauthClientSecret: server.hasOauthClientSecret === true, + } +} + interface MCPProps { initialServerId?: string | null } -/** - * MCP Settings component for managing Model Context Protocol servers. - * Handles server CRUD operations, connection testing, and environment variable integration. - */ export function MCP({ initialServerId }: MCPProps) { const params = useParams() const workspaceId = params.workspaceId as string @@ -147,7 +166,8 @@ export function MCP({ initialServerId }: MCPProps) { isFetching: toolsFetching, } = useMcpToolsQuery(workspaceId) const { data: storedTools = [], refetch: refetchStoredTools } = useStoredMcpTools(workspaceId) - const forceRefreshTools = useForceRefreshMcpTools() + const forceRefreshToolsMutation = useForceRefreshMcpTools() + const forceRefreshTools = forceRefreshToolsMutation.mutate const createServerMutation = useCreateMcpServer() const deleteServerMutation = useDeleteMcpServer() const refreshServerMutation = useRefreshMcpServer() @@ -156,23 +176,16 @@ export function MCP({ initialServerId }: MCPProps) { const { data: allowedMcpDomains = null } = useAllowedMcpDomains() const [showAddModal, setShowAddModal] = useState(false) - const [showEditModal, setShowEditModal] = useState(false) - const [editInitialData, setEditInitialData] = useState< - | { - name: string - transport: McpTransport - url?: string - timeout?: number - headers?: { key: string; value: string }[] - } - | undefined - >(undefined) + const [editingServerId, setEditingServerId] = useState(null) const [searchTerm, setSearchTerm] = useState('') const [deletingServers, setDeletingServers] = useState>(() => new Set()) + const { connectingServers: connectingOauthServers, startOauthForServer } = useMcpOauthPopup({ + workspaceId, + }) - const [showDeleteDialog, setShowDeleteDialog] = useState(false) - const [serverToDelete, setServerToDelete] = useState<{ id: string; name: string } | null>(null) + const [serverToDeleteId, setServerToDeleteId] = useState(null) + const showDeleteDialog = serverToDeleteId !== null const [selectedServerId, setSelectedServerId] = useState(initialServerId ?? null) @@ -185,28 +198,23 @@ export function MCP({ initialServerId }: MCPProps) { } }, []) - const [refreshingServers, setRefreshingServers] = useState< - Record - >({}) const [expandedTools, setExpandedTools] = useState>(() => new Set()) - const handleRemoveServer = useCallback((serverId: string, serverName: string) => { - setServerToDelete({ id: serverId, name: serverName }) - setShowDeleteDialog(true) - }, []) + const handleRemoveServer = (serverId: string) => { + setServerToDeleteId(serverId) + } - const confirmDeleteServer = useCallback(async () => { - if (!serverToDelete) return + const confirmDeleteServer = async () => { + if (!serverToDeleteId) return - setShowDeleteDialog(false) - const { id: serverId, name: serverName } = serverToDelete - setServerToDelete(null) + const serverId = serverToDeleteId + setServerToDeleteId(null) setDeletingServers((prev) => new Set(prev).add(serverId)) try { await deleteServerMutation.mutateAsync({ workspaceId, serverId }) - logger.info(`Removed MCP server: ${serverName}`) + logger.info(`Removed MCP server: ${serverId}`) } catch (error) { logger.error('Failed to remove MCP server:', error) } finally { @@ -216,43 +224,36 @@ export function MCP({ initialServerId }: MCPProps) { return newSet }) } - }, [serverToDelete, deleteServerMutation, workspaceId]) - - const toolsByServer = useMemo(() => { - return (mcpToolsData || []).reduce( - (acc, tool) => { - if (!tool?.serverId) return acc - if (!acc[tool.serverId]) { - acc[tool.serverId] = [] - } - acc[tool.serverId].push(tool) - return acc - }, - {} as Record - ) - }, [mcpToolsData]) - - const filteredServers = useMemo(() => { - return (servers || []).filter((server) => - server.name?.toLowerCase().includes(searchTerm.toLowerCase()) - ) - }, [servers, searchTerm]) + } - const handleViewDetails = useCallback( - (serverId: string) => { - setSelectedServerId(serverId) - forceRefreshTools(workspaceId) - refetchStoredTools() + const toolsByServer = (mcpToolsData || []).reduce( + (acc, tool) => { + if (!tool?.serverId) return acc + if (!acc[tool.serverId]) { + acc[tool.serverId] = [] + } + acc[tool.serverId].push(tool) + return acc }, - [workspaceId, forceRefreshTools, refetchStoredTools] + {} as Record + ) + + const filteredServers = (servers || []).filter((server) => + server.name?.toLowerCase().includes(searchTerm.toLowerCase()) ) - const handleBackToList = useCallback(() => { + const handleViewDetails = (serverId: string) => { + setSelectedServerId(serverId) + forceRefreshTools(workspaceId) + refetchStoredTools() + } + + const handleBackToList = () => { setSelectedServerId(null) setExpandedTools(new Set()) - }, []) + } - const toggleToolExpanded = useCallback((toolName: string) => { + const toggleToolExpanded = (toolName: string) => { setExpandedTools((prev) => { const newSet = new Set(prev) if (newSet.has(toolName)) { @@ -262,131 +263,109 @@ export function MCP({ initialServerId }: MCPProps) { } return newSet }) - }, []) + } - const handleRefreshServer = useCallback( - async (serverId: string) => { - try { - setRefreshingServers((prev) => ({ ...prev, [serverId]: { status: 'refreshing' } })) - const result = await refreshServerMutation.mutateAsync({ workspaceId, serverId }) - logger.info( - `Refreshed MCP server: ${serverId}, workflows updated: ${result.workflowsUpdated}` - ) - - const activeWorkflowId = useWorkflowRegistry.getState().activeWorkflowId - if (activeWorkflowId && result.updatedWorkflowIds?.includes(activeWorkflowId)) { - logger.info(`Active workflow ${activeWorkflowId} was updated, reloading subblock values`) - try { - const { data: workflowData } = await requestJson(getWorkflowStateContract, { - params: { id: activeWorkflowId }, - }) - if (workflowData?.state?.blocks) { - useSubBlockStore - .getState() - .initializeFromWorkflow( - activeWorkflowId, - workflowData.state.blocks as Record - ) - } - } catch (reloadError) { - logger.warn('Failed to reload workflow subblock values:', reloadError) - } - } + const handleRefreshServer = async (serverId: string) => { + try { + const result = await refreshServerMutation.mutateAsync({ workspaceId, serverId }) + logger.info( + `Refreshed MCP server: ${serverId}, workflows updated: ${result.workflowsUpdated}` + ) - setRefreshingServers((prev) => ({ - ...prev, - [serverId]: { status: 'refreshed', workflowsUpdated: result.workflowsUpdated }, - })) - setTimeout(() => { - setRefreshingServers((prev) => { - const newState = { ...prev } - delete newState[serverId] - return newState + const activeWorkflowId = useWorkflowRegistry.getState().activeWorkflowId + if (activeWorkflowId && result.updatedWorkflowIds?.includes(activeWorkflowId)) { + logger.info(`Active workflow ${activeWorkflowId} was updated, reloading subblock values`) + try { + const { data: workflowData } = await requestJson(getWorkflowStateContract, { + params: { id: activeWorkflowId }, }) - }, 3000) - } catch (error) { - logger.error('Failed to refresh MCP server:', error) - setRefreshingServers((prev) => { - const newState = { ...prev } - delete newState[serverId] - return newState - }) + if (workflowData?.state?.blocks) { + useSubBlockStore + .getState() + .initializeFromWorkflow( + activeWorkflowId, + workflowData.state.blocks as Record + ) + } + } catch (reloadError) { + logger.warn('Failed to reload workflow subblock values:', reloadError) + } } - }, - [refreshServerMutation, workspaceId] - ) - - const handleOpenEditModal = useCallback((server: McpServer) => { - const headers: { key: string; value: string }[] = server.headers - ? Object.entries(server.headers).map(([key, value]) => ({ key, value })) - : [{ key: '', value: '' }] - if (headers.length === 0) headers.push({ key: '', value: '' }) - - const lastHeader = headers[headers.length - 1] - if (lastHeader.key !== '' || lastHeader.value !== '') { - headers.push({ key: '', value: '' }) + } catch (error) { + logger.error('Failed to refresh MCP server:', error) } + } - setEditInitialData({ - name: server.name || '', - transport: (server.transport as McpTransport) || 'streamable-http', - url: server.url || '', - timeout: 30000, - headers, - }) - setShowEditModal(true) - }, []) - - const selectedServer = useMemo(() => { + useEffect(() => { + if (!refreshServerMutation.isSuccess) return + const timeout = window.setTimeout(() => refreshServerMutation.reset(), 3000) + return () => window.clearTimeout(timeout) + // eslint-disable-next-line react-hooks/exhaustive-deps -- mutation object is unstable; isSuccess flag is the trigger + }, [refreshServerMutation.isSuccess]) + + const refreshingServerId = refreshServerMutation.isPending + ? refreshServerMutation.variables?.serverId + : null + const refreshedServerId = refreshServerMutation.isSuccess + ? refreshServerMutation.variables?.serverId + : null + const refreshedWorkflowsUpdated = refreshServerMutation.data?.workflowsUpdated + + const editingServer = editingServerId + ? (servers.find((s) => s.id === editingServerId) as McpServer | undefined) + : undefined + const editInitialData = editingServer ? buildEditInitialData(editingServer) : undefined + + const selectedServer = (() => { if (!selectedServerId) return null const server = servers.find((s) => s.id === selectedServerId) as McpServer | undefined if (!server) return null const serverTools = (toolsByServer[selectedServerId] || []) as McpTool[] return { server, tools: serverTools } - }, [selectedServerId, servers, toolsByServer]) + })() + + const getStoredToolIssues = ( + serverId: string, + toolName: string + ): { issue: McpToolIssue; workflowName: string }[] => { + const relevantStoredTools = storedTools.filter( + (st) => st.serverId === serverId && st.toolName === toolName + ) - const getStoredToolIssues = useCallback( - (serverId: string, toolName: string): { issue: McpToolIssue; workflowName: string }[] => { - const relevantStoredTools = storedTools.filter( - (st) => st.serverId === serverId && st.toolName === toolName + const serverStates = servers.map((s) => ({ + id: s.id, + url: s.url, + connectionStatus: s.connectionStatus, + lastError: s.lastError || undefined, + })) + + const discoveredTools = mcpToolsData.map((t) => ({ + serverId: t.serverId, + name: t.name, + inputSchema: t.inputSchema, + })) + + const issues: { issue: McpToolIssue; workflowName: string }[] = [] + + for (const storedTool of relevantStoredTools) { + const issue = getMcpToolIssue( + { + serverId: storedTool.serverId, + serverUrl: storedTool.serverUrl, + toolName: storedTool.toolName, + schema: storedTool.schema, + }, + serverStates, + discoveredTools ) - const serverStates = servers.map((s) => ({ - id: s.id, - url: s.url, - connectionStatus: s.connectionStatus, - lastError: s.lastError || undefined, - })) - - const discoveredTools = mcpToolsData.map((t) => ({ - serverId: t.serverId, - name: t.name, - inputSchema: t.inputSchema, - })) - - const issues: { issue: McpToolIssue; workflowName: string }[] = [] - - for (const storedTool of relevantStoredTools) { - const issue = getMcpToolIssue( - { - serverId: storedTool.serverId, - serverUrl: storedTool.serverUrl, - toolName: storedTool.toolName, - schema: storedTool.schema, - }, - serverStates, - discoveredTools - ) - - if (issue) { - issues.push({ issue, workflowName: storedTool.workflowName }) - } + if (issue) { + issues.push({ issue, workflowName: storedTool.workflowName }) } + } - return issues - }, - [storedTools, servers, mcpToolsData] - ) + return issues + } const error = toolsError || serversError const hasServers = servers && servers.length > 0 @@ -422,12 +401,32 @@ export function MCP({ initialServerId }: MCPProps) { {server.connectionStatus === 'error' && (

Status -

+

{server.lastError || 'Unable to connect'}

)} + {server.authType === 'oauth' && server.connectionStatus !== 'connected' && ( +
+ + Authentication + +
+ +
+
+ )} +
Tools ({tools.length}) @@ -450,11 +449,12 @@ export function MCP({ initialServerId }: MCPProps) { key={tool.name} className='overflow-hidden rounded-md border bg-[var(--surface-3)]' > - + {isExpanded && hasParams && (
@@ -563,25 +563,27 @@ export function MCP({ initialServerId }: MCPProps) { -
{ + if (!open) setEditingServerId(null) + }} mode='edit' initialData={editInitialData} onSubmit={async (config) => { @@ -620,7 +622,7 @@ export function MCP({ initialServerId }: MCPProps) { />
@@ -628,7 +630,7 @@ export function MCP({ initialServerId }: MCPProps) {
{error ? (
-

+

{getErrorMessage(error, 'Failed to load MCP servers')}

@@ -656,8 +658,8 @@ export function MCP({ initialServerId }: MCPProps) { tools={tools} isDeleting={deletingServers.has(server.id)} isLoadingTools={isLoadingTools} - isRefreshing={refreshingServers[server.id]?.status === 'refreshing'} - onRemove={() => handleRemoveServer(server.id, server.name || 'this server')} + isRefreshing={refreshingServerId === server.id} + onRemove={() => handleRemoveServer(server.id)} onViewDetails={() => handleViewDetails(server.id)} /> ) @@ -677,28 +679,38 @@ export function MCP({ initialServerId }: MCPProps) { onOpenChange={setShowAddModal} mode='add' onSubmit={async (config) => { - await createServerMutation.mutateAsync({ + const result = await createServerMutation.mutateAsync({ workspaceId, config: { ...config, enabled: true }, }) + if (result.authType === 'oauth') { + await startOauthForServer(result.serverId) + } }} workspaceId={workspaceId} availableEnvVars={availableEnvVars} allowedMcpDomains={allowedMcpDomains} /> - + { + if (!open) setServerToDeleteId(null) + }} + > Delete MCP Server Are you sure you want to delete{' '} - {serverToDelete?.name} + + {servers.find((s) => s.id === serverToDeleteId)?.name || 'this server'} + ? This action cannot be undone. -