From 7d330097cee049454353bdb1b9cd7da97ad4ec88 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sun, 13 Sep 2026 21:56:25 -0700 Subject: [PATCH 1/5] feat(network): add organization-specific HTTPS egress --- .../docs/platform/enterprise/index.mdx | 1 + .../docs/platform/enterprise/meta.json | 1 + .../docs/platform/enterprise/security.mdx | 32 + .../platform/enterprise/session-policies.mdx | 2 +- apps/sim/app/api/auth/sso/register/route.ts | 5 +- apps/sim/app/api/link-preview/route.ts | 4 +- apps/sim/app/api/mcp/oauth/callback/route.ts | 14 +- .../[id]/data-drains/[drainId]/test/route.ts | 8 +- .../api/organizations/[id]/network/route.ts | 24 + .../settings/[section]/settings.tsx | 10 +- .../settings/navigation.test.ts | 4 +- .../o/[organizationId]/settings/navigation.ts | 8 +- .../settings/[section]/page.test.tsx | 2 +- .../settings/[section]/settings.tsx | 14 +- .../settings-empty-state.tsx | 2 +- .../[workspaceId]/settings/navigation.test.ts | 5 +- .../[workspaceId]/settings/navigation.ts | 1 + apps/sim/background/webhook-execution.ts | 797 ++++++------- .../components/settings/navigation.test.ts | 8 +- apps/sim/components/settings/navigation.ts | 23 +- .../settings/organization-security.tsx | 82 ++ apps/sim/connectors/airtable/airtable.ts | 3 +- apps/sim/connectors/asana/asana.ts | 3 +- apps/sim/connectors/ashby/ashby.ts | 3 +- .../connectors/azure-devops/azure-devops.ts | 3 +- apps/sim/connectors/bitbucket/bitbucket.ts | 3 +- apps/sim/connectors/box/box.test.ts | 4 +- apps/sim/connectors/box/box.ts | 3 +- .../connectors/circleback/circleback.test.ts | 4 +- apps/sim/connectors/circleback/circleback.ts | 3 +- apps/sim/connectors/clickup/clickup.ts | 3 +- apps/sim/connectors/confluence/confluence.ts | 2 +- apps/sim/connectors/confluence/permissions.ts | 3 +- apps/sim/connectors/databricks/databricks.ts | 3 +- apps/sim/connectors/discord/discord.ts | 3 +- apps/sim/connectors/docusign/docusign.ts | 3 +- apps/sim/connectors/dropbox/dropbox.ts | 3 +- apps/sim/connectors/fathom/fathom.ts | 3 +- apps/sim/connectors/github/request.ts | 6 +- .../connectors/gmail/company-crawl.test.ts | 6 +- apps/sim/connectors/gmail/gmail.test.ts | 4 +- apps/sim/connectors/gmail/gmail.ts | 3 +- apps/sim/connectors/gong/gong.ts | 3 +- .../google-calendar/google-calendar.ts | 3 +- .../sim/connectors/google-chat/google-chat.ts | 3 +- .../sim/connectors/google-docs/google-docs.ts | 7 +- .../connectors/google-forms/google-forms.ts | 3 +- .../sim/connectors/google-meet/google-meet.ts | 3 +- .../connectors/google-sheets/google-sheets.ts | 3 +- .../connectors/google-slides/google-slides.ts | 3 +- .../connectors/google-vault/google-vault.ts | 7 +- apps/sim/connectors/grain/grain.ts | 3 +- apps/sim/connectors/granola/granola.test.ts | 4 +- apps/sim/connectors/granola/granola.ts | 3 +- apps/sim/connectors/greenhouse/greenhouse.ts | 3 +- apps/sim/connectors/hubspot/hubspot.ts | 3 +- apps/sim/connectors/incidentio/incidentio.ts | 3 +- apps/sim/connectors/intercom/intercom.ts | 3 +- apps/sim/connectors/jira/jira.ts | 7 +- apps/sim/connectors/jsm/jsm.test.ts | 4 +- apps/sim/connectors/jsm/jsm.ts | 3 +- apps/sim/connectors/linear/linear.test.ts | 4 +- apps/sim/connectors/linear/linear.ts | 3 +- .../microsoft-excel/microsoft-excel.ts | 3 +- .../microsoft-teams/microsoft-teams.test.ts | 4 +- .../microsoft-teams/microsoft-teams.ts | 3 +- apps/sim/connectors/monday/monday.test.ts | 4 +- apps/sim/connectors/monday/monday.ts | 3 +- apps/sim/connectors/notion/notion.test.ts | 4 +- apps/sim/connectors/notion/notion.ts | 2 +- apps/sim/connectors/onedrive/onedrive.test.ts | 4 +- apps/sim/connectors/onedrive/onedrive.ts | 7 +- apps/sim/connectors/outlook/outlook.ts | 3 +- apps/sim/connectors/pagerduty/pagerduty.ts | 3 +- apps/sim/connectors/reddit/reddit.ts | 3 +- apps/sim/connectors/rootly/rootly.ts | 3 +- .../connectors/salesforce/salesforce.test.ts | 4 +- apps/sim/connectors/salesforce/salesforce.ts | 3 +- apps/sim/connectors/servicenow/servicenow.ts | 3 +- .../connectors/sharepoint/sharepoint.test.ts | 4 +- apps/sim/connectors/sharepoint/sharepoint.ts | 7 +- apps/sim/connectors/slack/slack.ts | 7 +- apps/sim/connectors/trello/trello.ts | 3 +- apps/sim/connectors/typeform/typeform.ts | 3 +- apps/sim/connectors/utils.test.ts | 6 +- apps/sim/connectors/webflow/webflow.ts | 3 +- apps/sim/connectors/wordpress/wordpress.ts | 3 +- apps/sim/connectors/workday/workday.ts | 3 +- apps/sim/connectors/x/x.ts | 3 +- apps/sim/connectors/youtube/youtube.ts | 3 +- apps/sim/connectors/zoho-desk/zoho-desk.ts | 3 +- apps/sim/connectors/zoom/zoom.test.ts | 4 +- apps/sim/connectors/zoom/zoom.ts | 3 +- .../handlers/workflow/workflow-handler.ts | 6 +- .../sim/hooks/queries/organization-network.ts | 22 + .../lib/api/contracts/organization-network.ts | 23 + .../lib/copilot/generated/docs-manifest.ts | 1 + .../sim/lib/copilot/tool-executor/executor.ts | 124 +- apps/sim/lib/copilot/tools/server/router.ts | 5 +- .../authorized-workspace-use-case.test.ts | 35 + .../authorized-workspace-use-case.ts | 35 +- apps/sim/lib/core/config/appconfig.test.ts | 154 ++- apps/sim/lib/core/config/appconfig.ts | 128 ++- apps/sim/lib/core/config/env.ts | 4 + .../read-organization-network.test.ts | 105 ++ .../application/read-organization-network.ts | 55 + .../lib/core/network/config.server.test.ts | 179 +++ apps/sim/lib/core/network/config.server.ts | 251 +++++ .../lib/core/network/context.server.test.ts | 46 + apps/sim/lib/core/network/context.server.ts | 32 + .../fixtures/gateway-runtime.fixture.ts | 54 + apps/sim/lib/core/network/gateway-metadata.ts | 6 + .../lib/core/network/gateway.server.test.ts | 339 ++++++ apps/sim/lib/core/network/gateway.server.ts | 136 +++ .../network/resource-scope.server.test.ts | 100 ++ .../lib/core/network/resource-scope.server.ts | 19 + apps/sim/lib/core/network/routing.test.ts | 48 + apps/sim/lib/core/network/routing.ts | 86 ++ .../lib/core/network/transport.server.test.ts | 109 ++ apps/sim/lib/core/network/transport.server.ts | 95 ++ .../guarded-request-fetch.server.test.ts | 148 ++- .../core/security/input-validation.server.ts | 374 +++--- .../core/security/pinned-fetch.server.test.ts | 4 +- ...ecure-fetch-request-framing.server.test.ts | 12 + .../secure-fetch-response-cap.server.test.ts | 276 ++++- apps/sim/lib/core/utils/fetch-deadline.ts | 2 +- .../application/public-enrollment.ts | 3 +- .../application/organization-credentials.ts | 10 +- apps/sim/lib/data-drains/service.ts | 32 +- apps/sim/lib/execution/files.ts | 5 +- apps/sim/lib/execution/isolated-vm-worker.cjs | 17 +- apps/sim/lib/execution/isolated-vm.test.ts | 1 + apps/sim/lib/execution/isolated-vm.ts | 23 +- .../authorized-knowledge-use-case.ts | 13 +- .../connectors/external-group-sync.ts | 129 ++- .../connectors/member-sync-engine.ts | 1004 +++++++++-------- .../lib/knowledge/connectors/sync-engine.ts | 977 ++++++++-------- .../documents/secure-fetch.server.test.ts | 79 ++ .../documents/secure-fetch.server.ts | 61 + apps/sim/lib/knowledge/documents/service.ts | 882 ++++++++------- .../sim/lib/knowledge/documents/utils.test.ts | 3 +- apps/sim/lib/knowledge/documents/utils.ts | 57 +- apps/sim/lib/mcp/connection-manager.test.ts | 29 + apps/sim/lib/mcp/connection-manager.ts | 163 +-- apps/sim/lib/mcp/middleware.ts | 6 +- apps/sim/lib/mcp/pinned-fetch.ts | 6 +- .../selectors/application/execute-selector.ts | 4 +- apps/sim/lib/webhooks/polling/orchestrator.ts | 15 +- apps/sim/lib/webhooks/polling/outlook.ts | 2 +- .../lib/webhooks/provider-subscriptions.ts | 36 +- .../lib/workflows/executor/execution-core.ts | 11 +- .../workspaces/admin-move-source-impact.ts | 2 +- .../tools/jira/attachment-downloads.test.ts | 2 +- apps/sim/tools/jira/utils.ts | 2 +- ...check-tool-registry-boundary.baseline.json | 14 +- 155 files changed, 5390 insertions(+), 2518 deletions(-) create mode 100644 apps/docs/content/docs/platform/enterprise/security.mdx create mode 100644 apps/sim/app/api/organizations/[id]/network/route.ts create mode 100644 apps/sim/components/settings/organization-security.tsx create mode 100644 apps/sim/hooks/queries/organization-network.ts create mode 100644 apps/sim/lib/api/contracts/organization-network.ts create mode 100644 apps/sim/lib/core/network/application/read-organization-network.test.ts create mode 100644 apps/sim/lib/core/network/application/read-organization-network.ts create mode 100644 apps/sim/lib/core/network/config.server.test.ts create mode 100644 apps/sim/lib/core/network/config.server.ts create mode 100644 apps/sim/lib/core/network/context.server.test.ts create mode 100644 apps/sim/lib/core/network/context.server.ts create mode 100644 apps/sim/lib/core/network/fixtures/gateway-runtime.fixture.ts create mode 100644 apps/sim/lib/core/network/gateway-metadata.ts create mode 100644 apps/sim/lib/core/network/gateway.server.test.ts create mode 100644 apps/sim/lib/core/network/gateway.server.ts create mode 100644 apps/sim/lib/core/network/resource-scope.server.test.ts create mode 100644 apps/sim/lib/core/network/resource-scope.server.ts create mode 100644 apps/sim/lib/core/network/routing.test.ts create mode 100644 apps/sim/lib/core/network/routing.ts create mode 100644 apps/sim/lib/core/network/transport.server.test.ts create mode 100644 apps/sim/lib/core/network/transport.server.ts create mode 100644 apps/sim/lib/knowledge/documents/secure-fetch.server.test.ts diff --git a/apps/docs/content/docs/platform/enterprise/index.mdx b/apps/docs/content/docs/platform/enterprise/index.mdx index a781bcc40d9..be354a1a18e 100644 --- a/apps/docs/content/docs/platform/enterprise/index.mdx +++ b/apps/docs/content/docs/platform/enterprise/index.mdx @@ -11,6 +11,7 @@ Sim Enterprise adds organization controls for access, provisioning, operations, | [Single sign-on](/platform/enterprise/sso) | Connect a SAML or OIDC identity provider | | [Verified domains](/platform/enterprise/verified-domains) | Prove domain ownership for SSO and provisioning | | [Directory provisioning](/platform/enterprise/scim) | Sync users and group-based access from your directory | +| [Security](/platform/enterprise/security) | Manage session policies and view configured outbound IP addresses | | [Session policies](/platform/enterprise/session-policies) | Set session lifetimes and revoke member sessions | | [Audit logs](/platform/enterprise/audit-logs) | Investigate configuration and security events | | [Usage tracking](/platform/enterprise/usage-tracking) | Review usage by member, workspace, model, and source | diff --git a/apps/docs/content/docs/platform/enterprise/meta.json b/apps/docs/content/docs/platform/enterprise/meta.json index 123eec90b29..c87d6bad42d 100644 --- a/apps/docs/content/docs/platform/enterprise/meta.json +++ b/apps/docs/content/docs/platform/enterprise/meta.json @@ -5,6 +5,7 @@ "sso", "scim", "verified-domains", + "security", "session-policies", "access-control", "custom-blocks", diff --git a/apps/docs/content/docs/platform/enterprise/security.mdx b/apps/docs/content/docs/platform/enterprise/security.mdx new file mode 100644 index 00000000000..237aec5bb5b --- /dev/null +++ b/apps/docs/content/docs/platform/enterprise/security.mdx @@ -0,0 +1,32 @@ +--- +title: Security +description: Manage organization session policies and view configured outbound IP addresses +--- + +Organization owners and admins open **Settings → Security** to manage session policies and view outbound IP addresses. + +## Session policies + +Set **Max session lifetime** and **Idle timeout** in hours, then select **Save**. Leave either field empty to use its default behavior. **Discard** restores your saved values. + +**Sign out all members** opens a confirmation before revoking the organization's browser sessions, except your current session. It does not revoke API keys. + +See [Session policies](/platform/enterprise/session-policies) for limits, defaults, and how changes affect existing sessions. + +## Outbound IP addresses + +Copy your organization's configured addresses in `/32` format and allowlist **every listed address** on the destination firewall. These addresses apply to supported HTTPS connections from Sim and its background workers. + +Contact Sim support or your deployment administrator to configure dedicated IPs and confirm connection coverage. + +### Connection coverage + +Dedicated routing supports public IPv4 destinations over HTTPS on port 443. It applies to supported Search connector API requests, HTTP request blocks, MCP and A2A connections, and HTTP requests made through Sim's local Function runtime. Background jobs use the same organization assignment. + +Native database connections, AWS SDK integrations, remote sandbox traffic, and browser requests keep their existing network paths. Other provider SDKs and OAuth authorization or token refresh calls require separate coverage confirmation. Dedicated IPs do not change access permissions in connected services. + +If dedicated routing is unavailable, affected requests fail instead of using shared IPs. Organizations without dedicated routing keep their existing behavior. + +## Availability + +On Sim Cloud, Security settings require an Enterprise organization and an owner or admin role. On self-hosted deployments, the outbound IP section is available to organization administrators; session controls appear only when session policies are enabled. See [self-hosted enterprise configuration](/platform/enterprise/self-hosted). diff --git a/apps/docs/content/docs/platform/enterprise/session-policies.mdx b/apps/docs/content/docs/platform/enterprise/session-policies.mdx index 73ddd8fd2a4..b4b0d4267dd 100644 --- a/apps/docs/content/docs/platform/enterprise/session-policies.mdx +++ b/apps/docs/content/docs/platform/enterprise/session-policies.mdx @@ -11,7 +11,7 @@ Session Policies let organization owners and admins on Enterprise plans control ## Setup -Go to **Settings → Organization → Session policies** in your organization settings. +Go to **Settings → Security → Session policies** in your organization settings. Both limits are optional. Leave a field empty to keep the default behavior: sessions last 30 days and extend automatically while a member stays active. diff --git a/apps/sim/app/api/auth/sso/register/route.ts b/apps/sim/app/api/auth/sso/register/route.ts index ff657118256..913eccc4386 100644 --- a/apps/sim/app/api/auth/sso/register/route.ts +++ b/apps/sim/app/api/auth/sso/register/route.ts @@ -9,6 +9,7 @@ import { getValidationErrorMessage, parseRequest } from '@/lib/api/server' import { auth, getSession } from '@/lib/auth' import { hasSSOAccess } from '@/lib/billing' import { isSsoEnabled } from '@/lib/core/config/env-flags' +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' import { secureFetchWithPinnedIP, validateUrlWithDNS, @@ -369,7 +370,9 @@ export const POST = withRouteHandler(async (request: NextRequest) => { !oidcConfig.authorizationEndpoint || !oidcConfig.tokenEndpoint || !oidcConfig.jwksEndpoint const discoveryUrl = `${issuer.replace(/\/$/, '')}/.well-known/openid-configuration` - const discoveryResult = await fetchOIDCDiscoveryDocument(discoveryUrl) + const discoveryResult = await runWithOutboundOrganization(membership.organizationId, () => + fetchOIDCDiscoveryDocument(discoveryUrl) + ) if (needsDiscovery) { logger.info('Fetching OIDC discovery document for missing endpoints', { diff --git a/apps/sim/app/api/link-preview/route.ts b/apps/sim/app/api/link-preview/route.ts index 8aaba14c0db..27e810a593a 100644 --- a/apps/sim/app/api/link-preview/route.ts +++ b/apps/sim/app/api/link-preview/route.ts @@ -10,6 +10,7 @@ import { getLinkPreviewContract } from '@/lib/api/contracts/link-preview' import { parseRequest } from '@/lib/api/server' import { getSession } from '@/lib/auth' import { getRedisClient } from '@/lib/core/config/redis' +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' import { enforceUserRateLimit } from '@/lib/core/rate-limiter/route-helpers' import { secureFetchWithValidation } from '@/lib/core/security/input-validation.server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' @@ -104,7 +105,8 @@ export const GET = withRouteHandler(async (request: NextRequest) => { let preview: LinkPreview = null try { - preview = await fetchPreview(url) + /** Link previews have no organization owner and use a shared URL cache. */ + preview = await runWithOutboundOrganization(null, () => fetchPreview(url)) } catch (error) { logger.info('Link preview fetch failed; returning null preview', { host: new URL(url).hostname, diff --git a/apps/sim/app/api/mcp/oauth/callback/route.ts b/apps/sim/app/api/mcp/oauth/callback/route.ts index dd31f4a24b2..6b7655ec2b0 100644 --- a/apps/sim/app/api/mcp/oauth/callback/route.ts +++ b/apps/sim/app/api/mcp/oauth/callback/route.ts @@ -8,6 +8,7 @@ import { NextResponse } from 'next/server' import { mcpOauthCallbackContract } from '@/lib/api/contracts/mcp' import { parseRequest } from '@/lib/api/server' import { getSession } from '@/lib/auth' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { credentialGroupOAuthAttemptPrincipal } from '@/lib/credential-groups/application/enrollment-auth' import { completePublicCredentialGroupMcpOAuth } from '@/lib/credential-groups/application/public-enrollment' @@ -238,11 +239,10 @@ export const GET = withRouteHandler(async (request: NextRequest) => { const provider = new SimMcpOauthProvider({ row, preregistered }) let result: Awaited> try { - result = await timedStep('mcpAuthGuarded', 120_000, () => - mcpAuthGuarded(provider, { - serverUrl, - authorizationCode: code, - }) + result = await withResourceOutboundScope({ workspaceId: serverWorkspaceId }, () => + timedStep('mcpAuthGuarded', 120_000, () => + mcpAuthGuarded(provider, { serverUrl, authorizationCode: code }) + ) ) } catch (e) { logger.error('Token exchange failed during MCP OAuth callback', e) @@ -267,7 +267,9 @@ export const GET = withRouteHandler(async (request: NextRequest) => { try { // forceRefresh: skip any stale cache from before re-auth. await timedStep('discoverServerTools', 60_000, () => - mcpService.discoverServerTools(session.user.id, server.id, serverWorkspaceId, 'force') + withResourceOutboundScope({ workspaceId: serverWorkspaceId }, () => + mcpService.discoverServerTools(session.user.id, server.id, serverWorkspaceId, 'force') + ) ) } catch (e) { logger.warn('Post-auth tools refresh failed', toError(e).message) diff --git a/apps/sim/app/api/organizations/[id]/data-drains/[drainId]/test/route.ts b/apps/sim/app/api/organizations/[id]/data-drains/[drainId]/test/route.ts index 5550ff9eb4c..c14d01b3c02 100644 --- a/apps/sim/app/api/organizations/[id]/data-drains/[drainId]/test/route.ts +++ b/apps/sim/app/api/organizations/[id]/data-drains/[drainId]/test/route.ts @@ -4,6 +4,7 @@ import { toError } from '@sim/utils/errors' import { type NextRequest, NextResponse } from 'next/server' import { testDataDrainContract } from '@/lib/api/contracts/data-drains' import { parseRequest } from '@/lib/api/server' +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { authorizeDrainAccess, loadDrain } from '@/lib/data-drains/access' import { getDestination } from '@/lib/data-drains/destinations/registry' @@ -29,7 +30,8 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Route } const destination = getDestination(drain.destinationType) - if (!destination.test) { + const testConnection = destination.test + if (!testConnection) { return NextResponse.json( { error: `Destination '${drain.destinationType}' does not support connection testing` }, { status: 400 } @@ -44,7 +46,9 @@ export const POST = withRouteHandler(async (request: NextRequest, context: Route const controller = new AbortController() const timeout = setTimeout(() => controller.abort(), TEST_TIMEOUT_MS) try { - await destination.test({ config, credentials, signal: controller.signal }) + await runWithOutboundOrganization(drain.organizationId, () => + testConnection({ config, credentials, signal: controller.signal }) + ) recordAudit({ workspaceId: null, actorId: access.session.user.id, diff --git a/apps/sim/app/api/organizations/[id]/network/route.ts b/apps/sim/app/api/organizations/[id]/network/route.ts new file mode 100644 index 00000000000..a6e87d2cb75 --- /dev/null +++ b/apps/sim/app/api/organizations/[id]/network/route.ts @@ -0,0 +1,24 @@ +import { getOrganizationNetworkContract } from '@/lib/api/contracts/organization-network' +import { + defineInternalJsonRoute, + internalOrchestrationErrorPolicy, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { + readOrganizationNetwork, + readOrganizationNetworkOperation, +} from '@/lib/core/network/application/read-organization-network' + +export const dynamic = 'force-dynamic' + +export const GET = defineInternalJsonRoute({ + contract: getOrganizationNetworkContract, + auth: internalSessionAuth, + operation: readOrganizationNetworkOperation, + rateLimit: internalRateLimits.user({ bucketName: 'organization-network-read' }), + errorPolicy: internalOrchestrationErrorPolicy, + mapInput: ({ params }) => ({ organizationId: params.id }), + useCase: readOrganizationNetwork, + staticResponseHeaders: { 'Cache-Control': 'private, no-store' }, +}) diff --git a/apps/sim/app/o/[organizationId]/settings/[section]/settings.tsx b/apps/sim/app/o/[organizationId]/settings/[section]/settings.tsx index f5adfa63f19..94221fc31fc 100644 --- a/apps/sim/app/o/[organizationId]/settings/[section]/settings.tsx +++ b/apps/sim/app/o/[organizationId]/settings/[section]/settings.tsx @@ -51,11 +51,6 @@ const AuditLogs = dynamic(() => import('@/ee/audit-logs/components/audit-logs').then((m) => m.AuditLogs) ) const SSO = dynamic(() => import('@/ee/sso/components/sso-settings').then((m) => m.SSO)) -const SessionPolicySettings = dynamic(() => - import('@/ee/session-policy/components/session-policy-settings').then( - (m) => m.SessionPolicySettings - ) -) const DataRetentionSettings = dynamic(() => import('@/ee/data-retention/components/data-retention-settings').then( (m) => m.DataRetentionSettings @@ -64,6 +59,9 @@ const DataRetentionSettings = dynamic(() => const DataDrainsSettings = dynamic(() => import('@/ee/data-drains/components/data-drains-settings').then((m) => m.DataDrainsSettings) ) +const OrganizationSecuritySettings = dynamic(() => + import('@/components/settings/organization-security').then((m) => m.OrganizationSecuritySettings) +) const UsageMonitoring = dynamic(() => import('@/ee/organization-usage/components/usage-monitoring').then((m) => m.UsageMonitoring) ) @@ -113,7 +111,7 @@ export function OrganizationSettings({ section }: OrganizationSettingsProps) { /> )} {section === 'sso' && } - {section === 'sessions' && } + {section === 'security' && } {section === 'data-retention' && } {section === 'data-drains' && } {section === 'whitelabeling' && } diff --git a/apps/sim/app/o/[organizationId]/settings/navigation.test.ts b/apps/sim/app/o/[organizationId]/settings/navigation.test.ts index 36c23895f8c..94ec37b962a 100644 --- a/apps/sim/app/o/[organizationId]/settings/navigation.test.ts +++ b/apps/sim/app/o/[organizationId]/settings/navigation.test.ts @@ -73,6 +73,8 @@ describe('organization settings navigation', () => { ) expect(resolveOrganizationSettingsSection('subscription')).toBe('billing') expect(resolveOrganizationSettingsSection('domains')).toBe('sso') + expect(resolveOrganizationSettingsSection('sessions')).toBe('security') + expect(resolveOrganizationSettingsSection('/o/one/settings/network')).toBeNull() expect(resolveOrganizationSettingsSection('skills')).toBeNull() expect(buildOrganizationNavItems('org', true).map(({ id }) => id)).toEqual([ 'home', @@ -92,7 +94,7 @@ describe('organization settings navigation', () => { 'governance:audit-logs', 'governance:access-control', 'governance:sso', - 'governance:sessions', + 'governance:security', 'governance:data-retention', 'governance:data-drains', 'sim-search:integrations', diff --git a/apps/sim/app/o/[organizationId]/settings/navigation.ts b/apps/sim/app/o/[organizationId]/settings/navigation.ts index 124ff827666..0a96e8a0754 100644 --- a/apps/sim/app/o/[organizationId]/settings/navigation.ts +++ b/apps/sim/app/o/[organizationId]/settings/navigation.ts @@ -37,7 +37,13 @@ export function resolveOrganizationSettingsSection( path, items: ORGANIZATION_SETTINGS_ITEMS, defaultSection: null, - aliases: { organization: 'members', team: 'members', subscription: 'billing', domains: 'sso' }, + aliases: { + organization: 'members', + team: 'members', + subscription: 'billing', + domains: 'sso', + sessions: 'security', + }, }) } diff --git a/apps/sim/app/workspace/[workspaceId]/settings/[section]/page.test.tsx b/apps/sim/app/workspace/[workspaceId]/settings/[section]/page.test.tsx index 034c639a179..bfd7f13b9f7 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/[section]/page.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/[section]/page.test.tsx @@ -51,7 +51,7 @@ vi.mock('@/app/workspace/[workspaceId]/settings/navigation', () => ({ 'access-control', 'audit-logs', 'sso', - 'sessions', + 'security', 'data-retention', 'data-drains', 'whitelabeling', diff --git a/apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx b/apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx index 1247284618e..ae626e7a9d9 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx @@ -93,11 +93,6 @@ const AuditLogs = dynamic(() => import('@/ee/audit-logs/components/audit-logs').then((m) => m.AuditLogs) ) const SSO = dynamic(() => import('@/ee/sso/components/sso-settings').then((m) => m.SSO)) -const SessionPolicySettings = dynamic(() => - import('@/ee/session-policy/components/session-policy-settings').then( - (m) => m.SessionPolicySettings - ) -) const DataRetentionSettings = dynamic(() => import('@/ee/data-retention/components/data-retention-settings').then( (m) => m.DataRetentionSettings @@ -106,6 +101,9 @@ const DataRetentionSettings = dynamic(() => const DataDrainsSettings = dynamic(() => import('@/ee/data-drains/components/data-drains-settings').then((m) => m.DataDrainsSettings) ) +const OrganizationSecuritySettings = dynamic(() => + import('@/components/settings/organization-security').then((m) => m.OrganizationSecuritySettings) +) const UsageMonitoring = dynamic(() => import('@/ee/organization-usage/components/usage-monitoring').then((m) => m.UsageMonitoring) ) @@ -201,15 +199,15 @@ export function SettingsPage({ section }: SettingsPageProps) { /> )} {effectiveSection === 'sso' && organizationId && } - {effectiveSection === 'sessions' && organizationId && ( - - )} {effectiveSection === 'data-retention' && organizationId && ( )} {effectiveSection === 'data-drains' && organizationId && ( )} + {effectiveSection === 'security' && organizationId && ( + + )} {effectiveSection === 'whitelabeling' && organizationId && ( )} diff --git a/apps/sim/app/workspace/[workspaceId]/settings/components/settings-empty-state/settings-empty-state.tsx b/apps/sim/app/workspace/[workspaceId]/settings/components/settings-empty-state/settings-empty-state.tsx index ec541844f65..cb1ab6b89e7 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/components/settings-empty-state/settings-empty-state.tsx +++ b/apps/sim/app/workspace/[workspaceId]/settings/components/settings-empty-state/settings-empty-state.tsx @@ -55,7 +55,7 @@ export function SettingsQueryErrorState({ return (
- {getErrorMessage(error, fallback)} + {getErrorMessage(error, fallback)} {isRetrying ? 'Retrying…' : 'Try again'} diff --git a/apps/sim/app/workspace/[workspaceId]/settings/navigation.test.ts b/apps/sim/app/workspace/[workspaceId]/settings/navigation.test.ts index 42172436605..fdad1a693a0 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/navigation.test.ts +++ b/apps/sim/app/workspace/[workspaceId]/settings/navigation.test.ts @@ -44,7 +44,7 @@ describe('unified settings navigation', () => { { id: 'recently-deleted', label: 'Recently deleted', section: 'workspace' }, { id: 'self-host', label: 'Self hosting', section: 'platform' }, { id: 'sso', label: 'Single sign-on', section: 'organization' }, - { id: 'sessions', label: 'Session policies', section: 'organization' }, + { id: 'security', label: 'Security', section: 'organization' }, { id: 'data-retention', label: 'Data retention', section: 'organization' }, { id: 'data-drains', label: 'Data drains', section: 'organization' }, { id: 'whitelabeling', label: 'White-labeling', section: 'organization' }, @@ -90,7 +90,7 @@ describe('unified settings navigation', () => { 'audit-logs', 'whitelabeling', 'sso', - 'sessions', + 'security', 'data-retention', 'data-drains', ]) @@ -124,6 +124,7 @@ describe('resolveSettingsSection', () => { team: 'organization', 'api-keys': 'apikeys', domains: 'sso', + sessions: 'security', } as const it('keeps legacy section links working', () => { diff --git a/apps/sim/app/workspace/[workspaceId]/settings/navigation.ts b/apps/sim/app/workspace/[workspaceId]/settings/navigation.ts index 8427439f40a..973e5b78fb5 100644 --- a/apps/sim/app/workspace/[workspaceId]/settings/navigation.ts +++ b/apps/sim/app/workspace/[workspaceId]/settings/navigation.ts @@ -44,6 +44,7 @@ const SECTION_ALIASES: Readonly> = { 'api-keys': 'apikeys', /** Verified domains moved into the SSO page. */ domains: 'sso', + sessions: 'security', } export interface ResolvedSettingsSection { diff --git a/apps/sim/background/webhook-execution.ts b/apps/sim/background/webhook-execution.ts index 6ebcac7d17a..ff54b85dcd9 100644 --- a/apps/sim/background/webhook-execution.ts +++ b/apps/sim/background/webhook-execution.ts @@ -45,6 +45,7 @@ import { WEBHOOK_IN_PROGRESS_LEASE_SECONDS, webhookIdempotency, } from '@/lib/core/idempotency' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { type EnvironmentResolutionSnapshot, getEffectiveEnvironmentSnapshot, @@ -817,460 +818,462 @@ async function executeWebhookJobInternal( throw new Error(`Workflow ${payload.workflowId} has no associated workspace`) } - const workflowVariables = (workflowRecord.variables as Record) || {} + return withResourceOutboundScope({ workspaceId }, async () => { + const workflowVariables = (workflowRecord.variables as Record) || {} - let deploymentVersionId: string | undefined - /** - * Flipped immediately before `executeWorkflowCore` is invoked. While false, - * no block has run and no execution effect exists, so a retryable - * infrastructure error may be surfaced as a `RetryableSetupError` and the - * whole delivery safely re-attempted. Once true, errors are never - * reclassified as retryable — retrying after the executor started could - * double-run the workflow. - */ - let workflowCoreStarted = false + let deploymentVersionId: string | undefined + /** + * Flipped immediately before `executeWorkflowCore` is invoked. While false, + * no block has run and no execution effect exists, so a retryable + * infrastructure error may be surfaced as a `RetryableSetupError` and the + * whole delivery safely re-attempted. Once true, errors are never + * reclassified as retryable — retrying after the executor started could + * double-run the workflow. + */ + let workflowCoreStarted = false - try { - const workflowStatePromise = payload.deploymentVersionId - ? loadWorkflowDeploymentVersionState( - payload.workflowId, - payload.deploymentVersionId, - workspaceId + try { + const workflowStatePromise = payload.deploymentVersionId + ? loadWorkflowDeploymentVersionState( + payload.workflowId, + payload.deploymentVersionId, + workspaceId + ) + : loadDeployedWorkflowState(payload.workflowId, workspaceId) + const [workflowData, webhookRows, resolvedCredentialUserId] = await Promise.all([ + workflowStatePromise, + db.select().from(webhook).where(eq(webhook.id, payload.webhookId)).limit(1), + payload.credentialId + ? resolveCredentialAccountUserId(payload.credentialId) + : Promise.resolve(undefined), + ]) + const credentialAccountUserId = resolvedCredentialUserId + if (payload.credentialId && !credentialAccountUserId) { + logger.warn( + `[${requestId}] Failed to resolve credential account for credential ${payload.credentialId}` ) - : loadDeployedWorkflowState(payload.workflowId, workspaceId) - const [workflowData, webhookRows, resolvedCredentialUserId] = await Promise.all([ - workflowStatePromise, - db.select().from(webhook).where(eq(webhook.id, payload.webhookId)).limit(1), - payload.credentialId - ? resolveCredentialAccountUserId(payload.credentialId) - : Promise.resolve(undefined), - ]) - const credentialAccountUserId = resolvedCredentialUserId - if (payload.credentialId && !credentialAccountUserId) { - logger.warn( - `[${requestId}] Failed to resolve credential account for credential ${payload.credentialId}` - ) - } - - if (!workflowData) { - throw new Error( - 'Workflow state not found. The workflow may not be deployed or the deployment data may be corrupted.' - ) - } + } - const { blocks, edges, loops, parallels } = workflowData - deploymentVersionId = - 'deploymentVersionId' in workflowData - ? (workflowData.deploymentVersionId as string) - : undefined + if (!workflowData) { + throw new Error( + 'Workflow state not found. The workflow may not be deployed or the deployment data may be corrupted.' + ) + } - const handler = getProviderHandler(payload.provider) + const { blocks, edges, loops, parallels } = workflowData + deploymentVersionId = + 'deploymentVersionId' in workflowData + ? (workflowData.deploymentVersionId as string) + : undefined - let input: Record | null = null - let skipMessage: string | undefined + const handler = getProviderHandler(payload.provider) - const webhookRecord = webhookRows[0] - if (!webhookRecord) { - throw new Error(`Webhook record not found: ${payload.webhookId}`) - } + let input: Record | null = null + let skipMessage: string | undefined - const secretScope = { userId: workflowRecord.userId, workspaceId } - let resolvedSecretTraceRegistry = createIncompleteResolvedSecretTraceRegistry(secretScope) - const resolvedWebhookRecord = await resolveWebhookExecutionProviderConfig( - webhookRecord, - payload.provider, - workflowRecord.userId, - workspaceId, - { - /** - * The identity preprocessing already elected for this run, so the - * provider config resolves against exactly the workspace variables the - * run's own blocks will see rather than against a second, narrower - * selection derived from the workflow owner. - */ - actorUserId, - onEnvironmentSnapshot: async (secretEnvironment) => { - try { - resolvedSecretTraceRegistry = await createResolvedSecretTraceRegistry({ - personalEncrypted: secretEnvironment.personalEncrypted, - workspaceEncrypted: secretEnvironment.workspaceEncrypted, - personalDecrypted: secretEnvironment.personalDecrypted, - workspaceDecrypted: secretEnvironment.workspaceDecrypted, - decryptionFailures: secretEnvironment.decryptionFailures, - personalOwners: secretEnvironment.personalOwners, - workspaceUnredactedKeys: secretEnvironment.workspaceUnredactedKeys, - scope: secretScope, - }) - } catch (error) { - logger.warn( - `[${requestId}] Failed to build webhook trace secret catalog`, - loggingSession.projectDiagnosticError(error) - ) - resolvedSecretTraceRegistry = createIncompleteResolvedSecretTraceRegistry(secretScope) - } - loggingSession.setResolvedSecretTraceRegistry(resolvedSecretTraceRegistry) - }, - onResolved: (name, value) => { - resolvedSecretTraceRegistry.recordResolved(name, value) - }, + const webhookRecord = webhookRows[0] + if (!webhookRecord) { + throw new Error(`Webhook record not found: ${payload.webhookId}`) } - ) - if (handler.formatInput) { - const result = await handler.formatInput({ - webhook: resolvedWebhookRecord, - workflow: { id: payload.workflowId, userId: payload.userId }, - body: payload.body, - headers: payload.headers, - query: payload.query ?? {}, - method: payload.method ?? '', - requestId, - }) - input = result.input as Record | null - skipMessage = result.skip?.message - } else { - input = payload.body as Record | null - } + const secretScope = { userId: workflowRecord.userId, workspaceId } + let resolvedSecretTraceRegistry = createIncompleteResolvedSecretTraceRegistry(secretScope) + const resolvedWebhookRecord = await resolveWebhookExecutionProviderConfig( + webhookRecord, + payload.provider, + workflowRecord.userId, + workspaceId, + { + /** + * The identity preprocessing already elected for this run, so the + * provider config resolves against exactly the workspace variables the + * run's own blocks will see rather than against a second, narrower + * selection derived from the workflow owner. + */ + actorUserId, + onEnvironmentSnapshot: async (secretEnvironment) => { + try { + resolvedSecretTraceRegistry = await createResolvedSecretTraceRegistry({ + personalEncrypted: secretEnvironment.personalEncrypted, + workspaceEncrypted: secretEnvironment.workspaceEncrypted, + personalDecrypted: secretEnvironment.personalDecrypted, + workspaceDecrypted: secretEnvironment.workspaceDecrypted, + decryptionFailures: secretEnvironment.decryptionFailures, + personalOwners: secretEnvironment.personalOwners, + workspaceUnredactedKeys: secretEnvironment.workspaceUnredactedKeys, + scope: secretScope, + }) + } catch (error) { + logger.warn( + `[${requestId}] Failed to build webhook trace secret catalog`, + loggingSession.projectDiagnosticError(error) + ) + resolvedSecretTraceRegistry = createIncompleteResolvedSecretTraceRegistry(secretScope) + } + loggingSession.setResolvedSecretTraceRegistry(resolvedSecretTraceRegistry) + }, + onResolved: (name, value) => { + resolvedSecretTraceRegistry.recordResolved(name, value) + }, + } + ) - if (!input && handler.handleEmptyInput) { - const skipResult = handler.handleEmptyInput(requestId) - if (skipResult) { - skipMessage = skipResult.message + if (handler.formatInput) { + const result = await handler.formatInput({ + webhook: resolvedWebhookRecord, + workflow: { id: payload.workflowId, userId: payload.userId }, + body: payload.body, + headers: payload.headers, + query: payload.query ?? {}, + method: payload.method ?? '', + requestId, + }) + input = result.input as Record | null + skipMessage = result.skip?.message + } else { + input = payload.body as Record | null } - } - if (skipMessage) { - await loggingSession.safeStart({ - userId: actorUserId, - actorUserId, - billingAttribution, - workspaceId, - variables: {}, - triggerData: { - isTest: false, - correlation, - }, - deploymentVersionId, - }) + if (!input && handler.handleEmptyInput) { + const skipResult = handler.handleEmptyInput(requestId) + if (skipResult) { + skipMessage = skipResult.message + } + } - await loggingSession.safeComplete({ - endedAt: new Date().toISOString(), - totalDurationMs: 0, - finalOutput: { message: skipMessage }, - traceSpans: [], - }) + if (skipMessage) { + await loggingSession.safeStart({ + userId: actorUserId, + actorUserId, + billingAttribution, + workspaceId, + variables: {}, + triggerData: { + isTest: false, + correlation, + }, + deploymentVersionId, + }) - return { - success: true, - workflowId: payload.workflowId, - executionId, - output: { message: skipMessage }, - executedAt: new Date().toISOString(), - } - } + await loggingSession.safeComplete({ + endedAt: new Date().toISOString(), + totalDurationMs: 0, + finalOutput: { message: skipMessage }, + traceSpans: [], + }) - if (input && payload.blockId && blocks[payload.blockId]) { - try { - const triggerBlock = blocks[payload.blockId] - const rawSelectedTriggerId = triggerBlock?.subBlocks?.selectedTriggerId?.value - const rawTriggerId = triggerBlock?.subBlocks?.triggerId?.value + return { + success: true, + workflowId: payload.workflowId, + executionId, + output: { message: skipMessage }, + executedAt: new Date().toISOString(), + } + } - let resolvedTriggerId = [rawSelectedTriggerId, rawTriggerId].find( - (candidate): candidate is string => - typeof candidate === 'string' && isTriggerValid(candidate) - ) + if (input && payload.blockId && blocks[payload.blockId]) { + try { + const triggerBlock = blocks[payload.blockId] + const rawSelectedTriggerId = triggerBlock?.subBlocks?.selectedTriggerId?.value + const rawTriggerId = triggerBlock?.subBlocks?.triggerId?.value + + let resolvedTriggerId = [rawSelectedTriggerId, rawTriggerId].find( + (candidate): candidate is string => + typeof candidate === 'string' && isTriggerValid(candidate) + ) + + if (!resolvedTriggerId) { + const blockConfig = getBlock(triggerBlock.type) + if (blockConfig?.category === 'triggers' && isTriggerValid(triggerBlock.type)) { + resolvedTriggerId = triggerBlock.type + } else if (triggerBlock.triggerMode && blockConfig?.triggers?.enabled) { + const available = blockConfig.triggers?.available?.[0] + if (available && isTriggerValid(available)) { + resolvedTriggerId = available + } + } + } - if (!resolvedTriggerId) { - const blockConfig = getBlock(triggerBlock.type) - if (blockConfig?.category === 'triggers' && isTriggerValid(triggerBlock.type)) { - resolvedTriggerId = triggerBlock.type - } else if (triggerBlock.triggerMode && blockConfig?.triggers?.enabled) { - const available = blockConfig.triggers?.available?.[0] - if (available && isTriggerValid(available)) { - resolvedTriggerId = available + if (resolvedTriggerId) { + const triggerConfig = getTrigger(resolvedTriggerId) + + if (triggerConfig.outputs) { + const processedInput = await processTriggerFileOutputs(input, triggerConfig.outputs, { + workspaceId, + workflowId: payload.workflowId, + executionId, + requestId, + userId: payload.userId, + projectDiagnosticError: (error, details) => + loggingSession.projectDiagnosticError(error, details), + }) + safeAssign(input, processedInput as Record) } } + } catch (error) { + logger.error( + `[${requestId}] Error processing trigger file outputs`, + loggingSession.projectDiagnosticError(error) + ) } + } - if (resolvedTriggerId) { - const triggerConfig = getTrigger(resolvedTriggerId) - - if (triggerConfig.outputs) { - const processedInput = await processTriggerFileOutputs(input, triggerConfig.outputs, { - workspaceId, - workflowId: payload.workflowId, - executionId, - requestId, - userId: payload.userId, - projectDiagnosticError: (error, details) => - loggingSession.projectDiagnosticError(error, details), - }) - safeAssign(input, processedInput as Record) - } + if (input && handler.processInputFiles && payload.blockId && blocks[payload.blockId]) { + try { + await handler.processInputFiles({ + input, + blocks, + blockId: payload.blockId, + workspaceId, + workflowId: payload.workflowId, + executionId, + requestId, + userId: payload.userId, + }) + } catch (error) { + logger.error( + `[${requestId}] Error processing provider-specific files`, + loggingSession.projectDiagnosticError(error) + ) } - } catch (error) { - logger.error( - `[${requestId}] Error processing trigger file outputs`, - loggingSession.projectDiagnosticError(error) - ) } - } - if (input && handler.processInputFiles && payload.blockId && blocks[payload.blockId]) { - try { - await handler.processInputFiles({ - input, + logger.info(`[${requestId}] Executing workflow for ${payload.provider} webhook`) + + const metadata: ExecutionMetadata = { + requestId, + executionId, + workflowId: payload.workflowId, + workspaceId, + userId: actorUserId!, + principal, + billingAttribution, + sessionUserId: undefined, + workflowUserId: workflowRecord.userId, + triggerType: payload.provider || 'webhook', + triggerBlockId: payload.blockId, + useDraftState: false, + startTime: new Date().toISOString(), + isClientSession: false, + credentialAccountUserId, + correlation, + workflowStateOverride: { blocks, - blockId: payload.blockId, - workspaceId, - workflowId: payload.workflowId, - executionId, - requestId, - userId: payload.userId, - }) - } catch (error) { - logger.error( - `[${requestId}] Error processing provider-specific files`, - loggingSession.projectDiagnosticError(error) - ) + edges, + loops: loops || {}, + parallels: parallels || {}, + deploymentVersionId, + }, } - } - logger.info(`[${requestId}] Executing workflow for ${payload.provider} webhook`) + const triggerInput = input || {} - const metadata: ExecutionMetadata = { - requestId, - executionId, - workflowId: payload.workflowId, - workspaceId, - userId: actorUserId!, - principal, - billingAttribution, - sessionUserId: undefined, - workflowUserId: workflowRecord.userId, - triggerType: payload.provider || 'webhook', - triggerBlockId: payload.blockId, - useDraftState: false, - startTime: new Date().toISOString(), - isClientSession: false, - credentialAccountUserId, - correlation, - workflowStateOverride: { - blocks, - edges, - loops: loops || {}, - parallels: parallels || {}, - deploymentVersionId, - }, - } + /** + * Surface the pre-execution latency that per-block timings cannot see: the + * gap between webhook receipt and the first block running, and — for + * trigger_id-bound providers like Slack — the true age of the interaction + * against its 3s expiry window. Logged structured so it is queryable/alarmable. + */ + if (payload.webhookReceivedAt !== undefined || payload.triggerTimestampMs !== undefined) { + const now = Date.now() + logger.info(`[${requestId}] Webhook dispatch latency`, { + workflowId: payload.workflowId, + provider: payload.provider, + dispatchLatencyMs: + payload.webhookReceivedAt !== undefined ? now - payload.webhookReceivedAt : undefined, + triggerAgeMs: + payload.triggerTimestampMs !== undefined ? now - payload.triggerTimestampMs : undefined, + }) + } - const triggerInput = input || {} + const persistedProviderConfig = isRecordLike(resolvedWebhookRecord.providerConfig) + ? resolvedWebhookRecord.providerConfig + : {} + const slackStreamConfig = + payload.provider === 'slack' || payload.provider === 'slack_app' + ? readSlackStreamResponseConfig(persistedProviderConfig) + : null + if (slackStreamConfig && payload.provider !== 'slack') { + throw new Error('Slack trigger response streaming is only supported for custom bots') + } + const slackStreamCredentialId = + typeof persistedProviderConfig.credentialId === 'string' + ? persistedProviderConfig.credentialId + : null + if (slackStreamConfig && !slackStreamCredentialId) { + throw new Error('Slack stream configuration is missing its custom bot credential') + } + const slackStreamController = slackStreamConfig + ? await SlackExecutionStreamController.create({ + credentialId: slackStreamCredentialId!, + workspaceId, + workflowId: payload.workflowId, + executionId, + userId: actorUserId, + triggerInput, + config: slackStreamConfig, + loggingSession, + abortSignal: timeoutController.signal, + }) + : null - /** - * Surface the pre-execution latency that per-block timings cannot see: the - * gap between webhook receipt and the first block running, and — for - * trigger_id-bound providers like Slack — the true age of the interaction - * against its 3s expiry window. Logged structured so it is queryable/alarmable. - */ - if (payload.webhookReceivedAt !== undefined || payload.triggerTimestampMs !== undefined) { - const now = Date.now() - logger.info(`[${requestId}] Webhook dispatch latency`, { - workflowId: payload.workflowId, - provider: payload.provider, - dispatchLatencyMs: - payload.webhookReceivedAt !== undefined ? now - payload.webhookReceivedAt : undefined, - triggerAgeMs: - payload.triggerTimestampMs !== undefined ? now - payload.triggerTimestampMs : undefined, - }) - } + const snapshot = new ExecutionSnapshot( + metadata, + workflowRecord, + triggerInput, + workflowVariables, + slackStreamController?.selectedOutputs ?? [] + ) - const persistedProviderConfig = isRecordLike(resolvedWebhookRecord.providerConfig) - ? resolvedWebhookRecord.providerConfig - : {} - const slackStreamConfig = - payload.provider === 'slack' || payload.provider === 'slack_app' - ? readSlackStreamResponseConfig(persistedProviderConfig) - : null - if (slackStreamConfig && payload.provider !== 'slack') { - throw new Error('Slack trigger response streaming is only supported for custom bots') - } - const slackStreamCredentialId = - typeof persistedProviderConfig.credentialId === 'string' - ? persistedProviderConfig.credentialId - : null - if (slackStreamConfig && !slackStreamCredentialId) { - throw new Error('Slack stream configuration is missing its custom bot credential') - } - const slackStreamController = slackStreamConfig - ? await SlackExecutionStreamController.create({ - credentialId: slackStreamCredentialId!, - workspaceId, - workflowId: payload.workflowId, - executionId, - userId: actorUserId, - triggerInput, - config: slackStreamConfig, + workflowCoreStarted = true + let executionResult: ExecutionResult + try { + executionResult = await executeWorkflowCore({ + snapshot, + callbacks: slackStreamController?.callbacks ?? {}, loggingSession, + trustedInitialResolvedSecretTraceProvenance: + resolvedSecretTraceRegistry.exportProvenanceForValue(triggerInput), + includeFileBase64: false, + base64MaxBytes: undefined, abortSignal: timeoutController.signal, }) - : null - - const snapshot = new ExecutionSnapshot( - metadata, - workflowRecord, - triggerInput, - workflowVariables, - slackStreamController?.selectedOutputs ?? [] - ) - - workflowCoreStarted = true - let executionResult: ExecutionResult - try { - executionResult = await executeWorkflowCore({ - snapshot, - callbacks: slackStreamController?.callbacks ?? {}, - loggingSession, - trustedInitialResolvedSecretTraceProvenance: - resolvedSecretTraceRegistry.exportProvenanceForValue(triggerInput), - includeFileBase64: false, - base64MaxBytes: undefined, - abortSignal: timeoutController.signal, - }) - } catch (error) { + } catch (error) { + if (slackStreamController) { + await slackStreamController.finalize({ + success: false, + output: {}, + error: toError(error).message, + }) + } + throw error + } if (slackStreamController) { - await slackStreamController.finalize({ - success: false, - output: {}, - error: toError(error).message, - }) + await slackStreamController.finalize(executionResult) + slackStreamController.assertSucceeded() } - throw error - } - if (slackStreamController) { - await slackStreamController.finalize(executionResult) - slackStreamController.assertSucceeded() - } - - await handleExecutionResult(executionResult, { - loggingSession, - timeoutController, - requestId, - executionId, - workflowId: payload.workflowId, - }) - - logger.info(`[${requestId}] Webhook execution completed`, { - success: executionResult.success, - workflowId: payload.workflowId, - provider: payload.provider, - }) - return { - success: executionResult.success, - workflowId: payload.workflowId, - executionId, - output: executionResult.output, - executedAt: new Date().toISOString(), - provider: payload.provider, - } - } catch (error: unknown) { - const errorMessage = toError(error).message - const errorStack = error instanceof Error ? error.stack : undefined - - /** - * Mirrors the schedule executor's setup boundary: an infrastructure error - * raised before the workflow core started left no execution effect, so it - * is surfaced as a `RetryableSetupError` — releasing the idempotency claim - * and, while attempts remain, requeueing without recording a terminal - * failed row for an attempt that will be retried. Exhausted retries fall - * through to normal failure handling but still throw typed so a provider - * redelivery is not rejected for a run that never happened. - */ - const retryableSetupCause = - !workflowCoreStarted && isRetryableInfrastructureError(error) - ? describeRetryableInfrastructureError(error) - : undefined - if (retryableSetupCause && hasRemainingWebhookInfraRetry(payload)) { - logger.warn(`[${requestId}] Retryable setup failure before webhook workflow started`, { + await handleExecutionResult(executionResult, { + loggingSession, + timeoutController, + requestId, + executionId, workflowId: payload.workflowId, - provider: payload.provider, - cause: retryableSetupCause, }) - throw new RetryableSetupError(errorMessage, { cause: retryableSetupCause }) - } - logger.error( - `[${requestId}] Webhook execution failed`, - loggingSession.projectDiagnosticError(error, { + logger.info(`[${requestId}] Webhook execution completed`, { + success: executionResult.success, workflowId: payload.workflowId, provider: payload.provider, }) - ) - - // The finalized flag is set inside a fire-and-forget post-execution promise; await it so the - // signal is reliable and the failure is fully persisted before we decide fault vs error. - await loggingSession.waitForPostExecution() - // A failure inside workflow execution (block error, provider 4xx, missing required field, etc.) - // is finalized by core and already recorded in the execution logs. That is a user/workflow error, - // not a trigger.dev job fault — complete the run normally so we don't fire a false alert. Errors - // that were not finalized came from the webhook pipeline itself, so we re-throw to fault below. - if (wasExecutionFinalizedByCore(error, executionId)) { return { - success: false, + success: executionResult.success, workflowId: payload.workflowId, executionId, - output: hasExecutionResult(error) ? error.executionResult.output : {}, + output: executionResult.output, executedAt: new Date().toISOString(), provider: payload.provider, } - } + } catch (error: unknown) { + const errorMessage = toError(error).message + const errorStack = error instanceof Error ? error.stack : undefined - try { - await loggingSession.safeStart({ - userId: actorUserId, - actorUserId, - billingAttribution, - workspaceId, - variables: {}, - triggerData: { - isTest: false, - correlation, - }, - deploymentVersionId, - }) + /** + * Mirrors the schedule executor's setup boundary: an infrastructure error + * raised before the workflow core started left no execution effect, so it + * is surfaced as a `RetryableSetupError` — releasing the idempotency claim + * and, while attempts remain, requeueing without recording a terminal + * failed row for an attempt that will be retried. Exhausted retries fall + * through to normal failure handling but still throw typed so a provider + * redelivery is not rejected for a run that never happened. + */ + const retryableSetupCause = + !workflowCoreStarted && isRetryableInfrastructureError(error) + ? describeRetryableInfrastructureError(error) + : undefined + if (retryableSetupCause && hasRemainingWebhookInfraRetry(payload)) { + logger.warn(`[${requestId}] Retryable setup failure before webhook workflow started`, { + workflowId: payload.workflowId, + provider: payload.provider, + cause: retryableSetupCause, + }) + throw new RetryableSetupError(errorMessage, { cause: retryableSetupCause }) + } - const executionResult = hasExecutionResult(error) - ? error.executionResult - : { - success: false, - output: {}, - logs: [], - } - const { traceSpans } = buildTraceSpans(executionResult) - - await loggingSession.safeCompleteWithError({ - endedAt: new Date().toISOString(), - totalDurationMs: 0, - error: { - message: errorMessage || 'Webhook execution failed', - stackTrace: errorStack, - }, - traceSpans, - executionState: executionResult.executionState, - }) - } catch (loggingError) { logger.error( - `[${requestId}] Failed to complete logging session`, - loggingSession.projectDiagnosticError(loggingError) + `[${requestId}] Webhook execution failed`, + loggingSession.projectDiagnosticError(error, { + workflowId: payload.workflowId, + provider: payload.provider, + }) ) - } - if (retryableSetupCause) { - throw new RetryableSetupError(errorMessage, { cause: retryableSetupCause }) + // The finalized flag is set inside a fire-and-forget post-execution promise; await it so the + // signal is reliable and the failure is fully persisted before we decide fault vs error. + await loggingSession.waitForPostExecution() + + // A failure inside workflow execution (block error, provider 4xx, missing required field, etc.) + // is finalized by core and already recorded in the execution logs. That is a user/workflow error, + // not a trigger.dev job fault — complete the run normally so we don't fire a false alert. Errors + // that were not finalized came from the webhook pipeline itself, so we re-throw to fault below. + if (wasExecutionFinalizedByCore(error, executionId)) { + return { + success: false, + workflowId: payload.workflowId, + executionId, + output: hasExecutionResult(error) ? error.executionResult.output : {}, + executedAt: new Date().toISOString(), + provider: payload.provider, + } + } + + try { + await loggingSession.safeStart({ + userId: actorUserId, + actorUserId, + billingAttribution, + workspaceId, + variables: {}, + triggerData: { + isTest: false, + correlation, + }, + deploymentVersionId, + }) + + const executionResult = hasExecutionResult(error) + ? error.executionResult + : { + success: false, + output: {}, + logs: [], + } + const { traceSpans } = buildTraceSpans(executionResult) + + await loggingSession.safeCompleteWithError({ + endedAt: new Date().toISOString(), + totalDurationMs: 0, + error: { + message: errorMessage || 'Webhook execution failed', + stackTrace: errorStack, + }, + traceSpans, + executionState: executionResult.executionState, + }) + } catch (loggingError) { + logger.error( + `[${requestId}] Failed to complete logging session`, + loggingSession.projectDiagnosticError(loggingError) + ) + } + + if (retryableSetupCause) { + throw new RetryableSetupError(errorMessage, { cause: retryableSetupCause }) + } + throw error } - throw error - } + }) } export const webhookExecution = task({ diff --git a/apps/sim/components/settings/navigation.test.ts b/apps/sim/components/settings/navigation.test.ts index 7e29f8ca48c..e469f904048 100644 --- a/apps/sim/components/settings/navigation.test.ts +++ b/apps/sim/components/settings/navigation.test.ts @@ -116,7 +116,7 @@ describe('settings navigation boundaries', () => { 'recently-deleted', 'self-host', 'sso', - 'sessions', + 'security', 'data-retention', 'data-drains', 'whitelabeling', @@ -237,7 +237,7 @@ describe('settings navigation boundaries', () => { 'access-control': false, 'audit-logs': false, sso: true, - sessions: true, + security: true, 'data-retention': false, 'data-drains': false, usage: true, @@ -301,7 +301,7 @@ describe('settings navigation boundaries', () => { 'data-drains', 'data-retention', 'organization', - 'sessions', + 'security', 'sso', 'usage', 'whitelabeling', @@ -319,7 +319,7 @@ describe('settings navigation boundaries', () => { 'access-control': 'access-control', 'audit-logs': 'audit-logs', sso: 'sso', - sessions: 'sessions', + security: 'security', 'data-retention': 'data-retention', 'data-drains': 'data-drains', whitelabeling: 'whitelabeling', diff --git a/apps/sim/components/settings/navigation.ts b/apps/sim/components/settings/navigation.ts index 2665b09e256..7721f1c7841 100644 --- a/apps/sim/components/settings/navigation.ts +++ b/apps/sim/components/settings/navigation.ts @@ -2,7 +2,6 @@ import type { ComponentType } from 'react' import { ChartColumn, ClipboardList, - Clock, Credit, Database, Globe, @@ -55,7 +54,7 @@ export type OrganizationSettingsSection = | 'access-control' | 'audit-logs' | 'sso' - | 'sessions' + | 'security' | 'data-retention' | 'data-drains' | 'whitelabeling' @@ -115,7 +114,7 @@ export type UnifiedSettingsSection = | 'inbox' | 'sandboxes' | 'admin' - | 'sessions' + | 'security' | 'data-retention' | 'data-drains' | 'mothership' @@ -701,18 +700,18 @@ export const SETTINGS_SECTION_REGISTRY: readonly SettingsSectionRegistryEntry[] }, }, { - label: 'Session policies', - icon: Clock, - docsLink: 'https://docs.sim.ai/platform/enterprise/session-policies', + label: 'Security', + icon: Lock, + docsLink: 'https://docs.sim.ai/platform/enterprise/security', unified: { - id: 'sessions', - description: 'Limit session lifetimes and sign out members org-wide.', + id: 'security', + description: 'Manage session policies and view outbound IP addresses.', group: 'organization', order: 8, requiresHosted: true, requiresEnterprise: true, - selfHostedOverride: 'sessionPolicies', - organizationSection: 'sessions', + selfHostedOverride: 'always', + organizationSection: 'security', }, }, { @@ -905,7 +904,7 @@ const ORGANIZATION_SECTION_GROUPS: Record + {hosted || features.sessionPolicies ? ( + + + + ) : ( + + )} + +
+ ) +} + +function OrganizationNetworkSection({ organizationId }: OrganizationSecuritySettingsProps) { + const { data, error, isPending, isFetching, refetch } = useOrganizationNetwork(organizationId) + + return ( + + {isPending ? ( + + Loading network settings… + + ) : error || data?.mode === 'unavailable' ? ( + void refetch()} + variant='inline' + /> + ) : data?.mode === 'gateway' && data.publicIps.length > 0 ? ( +
+

+ Allowlist every address below for supported HTTPS connections. +

+
+ {data.publicIps.map((ip) => ( + + ))} +
+
+ ) : data ? ( +

+ {data.mode === 'blocked' + ? 'Supported HTTPS connections are paused.' + : data.mode === 'gateway' + ? 'Your outbound IP addresses aren’t available yet.' + : 'Outbound IP addresses are not configured for this organization.'} +

+ ) : null} + + ) +} diff --git a/apps/sim/connectors/airtable/airtable.ts b/apps/sim/connectors/airtable/airtable.ts index fdddb724f41..541dc55b8ca 100644 --- a/apps/sim/connectors/airtable/airtable.ts +++ b/apps/sim/connectors/airtable/airtable.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { airtableConnectorMeta } from '@/connectors/airtable/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { diff --git a/apps/sim/connectors/asana/asana.ts b/apps/sim/connectors/asana/asana.ts index f6c3758a61a..83fdf9e2a9c 100644 --- a/apps/sim/connectors/asana/asana.ts +++ b/apps/sim/connectors/asana/asana.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { asanaConnectorMeta } from '@/connectors/asana/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { joinTagArray, parseTagDate } from '@/connectors/utils' diff --git a/apps/sim/connectors/ashby/ashby.ts b/apps/sim/connectors/ashby/ashby.ts index 3caf9a9f2dd..ba781a63867 100644 --- a/apps/sim/connectors/ashby/ashby.ts +++ b/apps/sim/connectors/ashby/ashby.ts @@ -5,7 +5,8 @@ import { readResponseJsonWithLimit, readResponseTextWithLimit, } from '@/lib/core/utils/stream-limits' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { ashbyConnectorMeta } from '@/connectors/ashby/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { parseTagDate } from '@/connectors/utils' diff --git a/apps/sim/connectors/azure-devops/azure-devops.ts b/apps/sim/connectors/azure-devops/azure-devops.ts index 1277e9fff9c..8c5a19fed41 100644 --- a/apps/sim/connectors/azure-devops/azure-devops.ts +++ b/apps/sim/connectors/azure-devops/azure-devops.ts @@ -1,7 +1,8 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { decodeTextBuffer } from '@/lib/file-parsers/utils' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { azureDevopsConnectorMeta } from '@/connectors/azure-devops/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { diff --git a/apps/sim/connectors/bitbucket/bitbucket.ts b/apps/sim/connectors/bitbucket/bitbucket.ts index 03b3d38d3f6..94f1dd28e7c 100644 --- a/apps/sim/connectors/bitbucket/bitbucket.ts +++ b/apps/sim/connectors/bitbucket/bitbucket.ts @@ -1,7 +1,8 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { decodeTextBuffer } from '@/lib/file-parsers/utils' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { bitbucketConnectorMeta } from '@/connectors/bitbucket/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { diff --git a/apps/sim/connectors/box/box.test.ts b/apps/sim/connectors/box/box.test.ts index 1ab7d1b4d6f..4f4bbab61d2 100644 --- a/apps/sim/connectors/box/box.test.ts +++ b/apps/sim/connectors/box/box.test.ts @@ -5,9 +5,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) -vi.mock('@/lib/knowledge/documents/utils', () => ({ +vi.mock('@/lib/knowledge/documents/utils', () => ({ VALIDATE_RETRY_OPTIONS: {} })) +vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => ({ fetchWithRetry: mockFetchWithRetry, - VALIDATE_RETRY_OPTIONS: {}, })) vi.mock('@/components/icons', () => ({ BoxCompanyIcon: () => null })) diff --git a/apps/sim/connectors/box/box.ts b/apps/sim/connectors/box/box.ts index d5f453ea759..aa33acd3d33 100644 --- a/apps/sim/connectors/box/box.ts +++ b/apps/sim/connectors/box/box.ts @@ -2,7 +2,8 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' import { decodeTextBuffer } from '@/lib/file-parsers/utils' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { boxConnectorMeta } from '@/connectors/box/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { diff --git a/apps/sim/connectors/circleback/circleback.test.ts b/apps/sim/connectors/circleback/circleback.test.ts index f3b653371d2..39795a208f4 100644 --- a/apps/sim/connectors/circleback/circleback.test.ts +++ b/apps/sim/connectors/circleback/circleback.test.ts @@ -10,9 +10,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) -vi.mock('@/lib/knowledge/documents/utils', () => ({ +vi.mock('@/lib/knowledge/documents/utils', () => ({ VALIDATE_RETRY_OPTIONS: {} })) +vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => ({ fetchWithRetry: mockFetchWithRetry, - VALIDATE_RETRY_OPTIONS: {}, })) vi.mock('@/components/icons', () => ({ CirclebackIcon: () => null })) diff --git a/apps/sim/connectors/circleback/circleback.ts b/apps/sim/connectors/circleback/circleback.ts index 3f0f82ce5f5..fe4a791d150 100644 --- a/apps/sim/connectors/circleback/circleback.ts +++ b/apps/sim/connectors/circleback/circleback.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { circlebackConnectorMeta } from '@/connectors/circleback/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { joinTagArray, parseTagDate } from '@/connectors/utils' diff --git a/apps/sim/connectors/clickup/clickup.ts b/apps/sim/connectors/clickup/clickup.ts index 87a9ac3bce6..313a572d1f3 100644 --- a/apps/sim/connectors/clickup/clickup.ts +++ b/apps/sim/connectors/clickup/clickup.ts @@ -1,7 +1,8 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { isRecordLike } from '@sim/utils/object' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { clickupConnectorMeta } from '@/connectors/clickup/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { diff --git a/apps/sim/connectors/confluence/confluence.ts b/apps/sim/connectors/confluence/confluence.ts index 8a80a0d0ddf..263d50c7db0 100644 --- a/apps/sim/connectors/confluence/confluence.ts +++ b/apps/sim/connectors/confluence/confluence.ts @@ -11,9 +11,9 @@ import { confluencePageAcl, } from '@/lib/knowledge/access/confluence-permissions' import type { MirroredDocumentAcl } from '@/lib/knowledge/access/types' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' import { createRetryableHttpError, - fetchWithRetry, type RetryOptions, VALIDATE_RETRY_OPTIONS, } from '@/lib/knowledge/documents/utils' diff --git a/apps/sim/connectors/confluence/permissions.ts b/apps/sim/connectors/confluence/permissions.ts index 99405116121..e354fc80ba4 100644 --- a/apps/sim/connectors/confluence/permissions.ts +++ b/apps/sim/connectors/confluence/permissions.ts @@ -5,7 +5,8 @@ import { type ConfluenceRestriction, confluenceSubjectToken, } from '@/lib/knowledge/access/confluence-permissions' -import { fetchWithRetry, type RetryOptions } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import type { RetryOptions } from '@/lib/knowledge/documents/utils' import { extractCursor } from '@/connectors/confluence/cursor' import type { ConnectorDirectory, diff --git a/apps/sim/connectors/databricks/databricks.ts b/apps/sim/connectors/databricks/databricks.ts index 88f1adfdc80..d29f65a9dd6 100644 --- a/apps/sim/connectors/databricks/databricks.ts +++ b/apps/sim/connectors/databricks/databricks.ts @@ -3,7 +3,8 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { truncate } from '@sim/utils/string' import { validateDatabricksWorkspaceHost } from '@/lib/core/security/input-validation' import { decodeTextBuffer } from '@/lib/file-parsers/utils' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DATABRICKS_CONTENT_TYPES, type DatabricksContentType, diff --git a/apps/sim/connectors/discord/discord.ts b/apps/sim/connectors/discord/discord.ts index 27ed685fafd..1fd694d0cb8 100644 --- a/apps/sim/connectors/discord/discord.ts +++ b/apps/sim/connectors/discord/discord.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DEFAULT_MAX_MESSAGES, discordConnectorMeta } from '@/connectors/discord/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { computeContentHash, parseTagDate } from '@/connectors/utils' diff --git a/apps/sim/connectors/docusign/docusign.ts b/apps/sim/connectors/docusign/docusign.ts index 412ba5ace7d..365d362aa0c 100644 --- a/apps/sim/connectors/docusign/docusign.ts +++ b/apps/sim/connectors/docusign/docusign.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { getDocusignOAuthUrl, getDocusignWebBase } from '@/lib/oauth/docusign' import { docusignConnectorMeta } from '@/connectors/docusign/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' diff --git a/apps/sim/connectors/dropbox/dropbox.ts b/apps/sim/connectors/dropbox/dropbox.ts index 132e6deb4ea..07226e8d51b 100644 --- a/apps/sim/connectors/dropbox/dropbox.ts +++ b/apps/sim/connectors/dropbox/dropbox.ts @@ -1,7 +1,8 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { decodeTextBuffer } from '@/lib/file-parsers/utils' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { dropboxConnectorMeta } from '@/connectors/dropbox/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { diff --git a/apps/sim/connectors/fathom/fathom.ts b/apps/sim/connectors/fathom/fathom.ts index bce6b9f4ff3..befb45c92bf 100644 --- a/apps/sim/connectors/fathom/fathom.ts +++ b/apps/sim/connectors/fathom/fathom.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { fathomConnectorMeta } from '@/connectors/fathom/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { parseTagDate } from '@/connectors/utils' diff --git a/apps/sim/connectors/github/request.ts b/apps/sim/connectors/github/request.ts index 13198b79ae5..aa1557cd004 100644 --- a/apps/sim/connectors/github/request.ts +++ b/apps/sim/connectors/github/request.ts @@ -7,8 +7,8 @@ import { } from '@/lib/core/rate-limiter/provider-capacity-error' import type { ProviderCapacityQuota } from '@/lib/core/rate-limiter/provider-capacity-state' import { readResponseTextWithLimit } from '@/lib/core/utils/stream-limits' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' import { - fetchWithRetry, hasRateLimitEvidence, type RetryOptions, resolveRetryDelayMs, @@ -69,7 +69,7 @@ export async function fetchGitHubWithRetry( .digest('hex') return fetchWithRetry(url, options, { ...retryOptions, - fetcher: async (input, init) => { + fetcher: async (input, init, transport) => { const signal = init?.signal ?? undefined let lease try { @@ -119,7 +119,7 @@ export async function fetchGitHubWithRetry( } try { - const response = await fetch(input, init) + const response = await transport(input, init) quota = readRequestQuota(response.headers) let secondaryLimit = false let forbiddenBody: string | undefined diff --git a/apps/sim/connectors/gmail/company-crawl.test.ts b/apps/sim/connectors/gmail/company-crawl.test.ts index 6918529e1ce..5030a0f4948 100644 --- a/apps/sim/connectors/gmail/company-crawl.test.ts +++ b/apps/sim/connectors/gmail/company-crawl.test.ts @@ -9,10 +9,8 @@ const { fetchProvider, listUsers, getUser } = vi.hoisted(() => ({ getUser: vi.fn(), })) -vi.mock('@/lib/knowledge/documents/utils', () => ({ - fetchWithRetry: fetchProvider, - VALIDATE_RETRY_OPTIONS: {}, -})) +vi.mock('@/lib/knowledge/documents/utils', () => ({ VALIDATE_RETRY_OPTIONS: {} })) +vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => ({ fetchWithRetry: fetchProvider })) vi.mock('@/components/icons', () => ({ GmailIcon: () => null })) vi.mock('@/connectors/google-workspace/users', () => ({ GOOGLE_WORKSPACE_USERS_PAGE_SIZE: 100, diff --git a/apps/sim/connectors/gmail/gmail.test.ts b/apps/sim/connectors/gmail/gmail.test.ts index 346021bb07b..f4d80fd1fd4 100644 --- a/apps/sim/connectors/gmail/gmail.test.ts +++ b/apps/sim/connectors/gmail/gmail.test.ts @@ -5,9 +5,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) -vi.mock('@/lib/knowledge/documents/utils', () => ({ +vi.mock('@/lib/knowledge/documents/utils', () => ({ VALIDATE_RETRY_OPTIONS: {} })) +vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => ({ fetchWithRetry: mockFetchWithRetry, - VALIDATE_RETRY_OPTIONS: {}, })) vi.mock('@/components/icons', () => ({ GmailIcon: () => null })) vi.mock('@/lib/knowledge/documents/service', () => ({ diff --git a/apps/sim/connectors/gmail/gmail.ts b/apps/sim/connectors/gmail/gmail.ts index c85f7df81c0..f63ea781624 100644 --- a/apps/sim/connectors/gmail/gmail.ts +++ b/apps/sim/connectors/gmail/gmail.ts @@ -3,7 +3,8 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { isPlainRecord } from '@sim/utils/object' import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import { isPayloadSizeLimitError, readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DEFAULT_MAX_THREADS, gmailConnectorMeta } from '@/connectors/gmail/meta' import { getGoogleWorkspaceDocument, diff --git a/apps/sim/connectors/gong/gong.ts b/apps/sim/connectors/gong/gong.ts index 77ed6655aa3..638c572eb0d 100644 --- a/apps/sim/connectors/gong/gong.ts +++ b/apps/sim/connectors/gong/gong.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { gongConnectorMeta } from '@/connectors/gong/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { joinTagArray, parseTagDate } from '@/connectors/utils' diff --git a/apps/sim/connectors/google-calendar/google-calendar.ts b/apps/sim/connectors/google-calendar/google-calendar.ts index b56cd329196..52405ccb028 100644 --- a/apps/sim/connectors/google-calendar/google-calendar.ts +++ b/apps/sim/connectors/google-calendar/google-calendar.ts @@ -1,7 +1,8 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { z } from 'zod' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DEFAULT_MAX_EVENTS, googleCalendarConnectorMeta } from '@/connectors/google-calendar/meta' import { getGoogleWorkspaceDocument, diff --git a/apps/sim/connectors/google-chat/google-chat.ts b/apps/sim/connectors/google-chat/google-chat.ts index 107e2acea72..b726cba04c5 100644 --- a/apps/sim/connectors/google-chat/google-chat.ts +++ b/apps/sim/connectors/google-chat/google-chat.ts @@ -1,7 +1,8 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DEFAULT_MAX_MESSAGES, googleChatConnectorMeta, diff --git a/apps/sim/connectors/google-docs/google-docs.ts b/apps/sim/connectors/google-docs/google-docs.ts index f5a0d2faac7..411627179bc 100644 --- a/apps/sim/connectors/google-docs/google-docs.ts +++ b/apps/sim/connectors/google-docs/google-docs.ts @@ -1,11 +1,8 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { isPlainRecord } from '@sim/utils/object' -import { - fetchWithRetry, - readBoundedHttpErrorBody, - VALIDATE_RETRY_OPTIONS, -} from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { readBoundedHttpErrorBody, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { googleDocsConnectorMeta } from '@/connectors/google-docs/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { diff --git a/apps/sim/connectors/google-forms/google-forms.ts b/apps/sim/connectors/google-forms/google-forms.ts index 957e039cb93..c678fb906ee 100644 --- a/apps/sim/connectors/google-forms/google-forms.ts +++ b/apps/sim/connectors/google-forms/google-forms.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { googleFormsConnectorMeta, MAX_RESPONSES_PER_FORM } from '@/connectors/google-forms/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { diff --git a/apps/sim/connectors/google-meet/google-meet.ts b/apps/sim/connectors/google-meet/google-meet.ts index 5f7f2a3813d..6eb43c42fa2 100644 --- a/apps/sim/connectors/google-meet/google-meet.ts +++ b/apps/sim/connectors/google-meet/google-meet.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { googleMeetConnectorMeta } from '@/connectors/google-meet/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { joinTagArray, parseTagDate } from '@/connectors/utils' diff --git a/apps/sim/connectors/google-sheets/google-sheets.ts b/apps/sim/connectors/google-sheets/google-sheets.ts index aba40c3d6f8..5604dd899e1 100644 --- a/apps/sim/connectors/google-sheets/google-sheets.ts +++ b/apps/sim/connectors/google-sheets/google-sheets.ts @@ -1,7 +1,8 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' import type { RetryOptions } from '@/lib/knowledge/documents/utils' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { googleSheetsConnectorMeta } from '@/connectors/google-sheets/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { diff --git a/apps/sim/connectors/google-slides/google-slides.ts b/apps/sim/connectors/google-slides/google-slides.ts index 1257cf60aa5..c0739a728f6 100644 --- a/apps/sim/connectors/google-slides/google-slides.ts +++ b/apps/sim/connectors/google-slides/google-slides.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { googleSlidesConnectorMeta } from '@/connectors/google-slides/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { diff --git a/apps/sim/connectors/google-vault/google-vault.ts b/apps/sim/connectors/google-vault/google-vault.ts index 85e912ebb86..5d7f3eb48a1 100644 --- a/apps/sim/connectors/google-vault/google-vault.ts +++ b/apps/sim/connectors/google-vault/google-vault.ts @@ -1,10 +1,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' -import { - fetchWithRetry, - type RetryOptions, - VALIDATE_RETRY_OPTIONS, -} from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { type RetryOptions, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { googleVaultConnectorMeta } from '@/connectors/google-vault/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { computeContentHash, parseTagDate, takeIndexableWithinCap } from '@/connectors/utils' diff --git a/apps/sim/connectors/grain/grain.ts b/apps/sim/connectors/grain/grain.ts index 564a334a3b5..54cc21ccf27 100644 --- a/apps/sim/connectors/grain/grain.ts +++ b/apps/sim/connectors/grain/grain.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { grainConnectorMeta } from '@/connectors/grain/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { joinTagArray, parseTagDate } from '@/connectors/utils' diff --git a/apps/sim/connectors/granola/granola.test.ts b/apps/sim/connectors/granola/granola.test.ts index 3b5a8cec4a9..49d469c4ad5 100644 --- a/apps/sim/connectors/granola/granola.test.ts +++ b/apps/sim/connectors/granola/granola.test.ts @@ -10,9 +10,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) -vi.mock('@/lib/knowledge/documents/utils', () => ({ +vi.mock('@/lib/knowledge/documents/utils', () => ({ VALIDATE_RETRY_OPTIONS: {} })) +vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => ({ fetchWithRetry: mockFetchWithRetry, - VALIDATE_RETRY_OPTIONS: {}, })) vi.mock('@/components/icons', () => ({ GranolaIcon: () => null })) diff --git a/apps/sim/connectors/granola/granola.ts b/apps/sim/connectors/granola/granola.ts index d92d65974c7..3926000802e 100644 --- a/apps/sim/connectors/granola/granola.ts +++ b/apps/sim/connectors/granola/granola.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { granolaConnectorMeta } from '@/connectors/granola/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { htmlToPlainText, joinTagArray, looksLikeHtml, parseTagDate } from '@/connectors/utils' diff --git a/apps/sim/connectors/greenhouse/greenhouse.ts b/apps/sim/connectors/greenhouse/greenhouse.ts index e4b78e007fa..570a815cfe9 100644 --- a/apps/sim/connectors/greenhouse/greenhouse.ts +++ b/apps/sim/connectors/greenhouse/greenhouse.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { greenhouseConnectorMeta } from '@/connectors/greenhouse/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { htmlToPlainText, parseTagDate } from '@/connectors/utils' diff --git a/apps/sim/connectors/hubspot/hubspot.ts b/apps/sim/connectors/hubspot/hubspot.ts index 936ca11fd73..1f72847125d 100644 --- a/apps/sim/connectors/hubspot/hubspot.ts +++ b/apps/sim/connectors/hubspot/hubspot.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { hubspotConnectorMeta } from '@/connectors/hubspot/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { htmlToPlainText, looksLikeHtml, parseTagDate } from '@/connectors/utils' diff --git a/apps/sim/connectors/incidentio/incidentio.ts b/apps/sim/connectors/incidentio/incidentio.ts index bed02ee269a..5e040dcc06e 100644 --- a/apps/sim/connectors/incidentio/incidentio.ts +++ b/apps/sim/connectors/incidentio/incidentio.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { incidentioConnectorMeta } from '@/connectors/incidentio/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { htmlToPlainText, parseTagDate } from '@/connectors/utils' diff --git a/apps/sim/connectors/intercom/intercom.ts b/apps/sim/connectors/intercom/intercom.ts index 2fba2a0dc52..e44685e89bd 100644 --- a/apps/sim/connectors/intercom/intercom.ts +++ b/apps/sim/connectors/intercom/intercom.ts @@ -1,7 +1,8 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { z } from 'zod' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DEFAULT_INTERCOM_REGION, DEFAULT_MAX_ITEMS, diff --git a/apps/sim/connectors/jira/jira.ts b/apps/sim/connectors/jira/jira.ts index 917bac7873e..52c0a766a89 100644 --- a/apps/sim/connectors/jira/jira.ts +++ b/apps/sim/connectors/jira/jira.ts @@ -7,11 +7,8 @@ import { normalizeAtlassianSiteUrl, resolveAtlassianCloudId, } from '@/lib/atlassian/discovery' -import { - fetchWithRetry, - type RetryOptions, - VALIDATE_RETRY_OPTIONS, -} from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { type RetryOptions, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { jiraConnectorMeta } from '@/connectors/jira/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { diff --git a/apps/sim/connectors/jsm/jsm.test.ts b/apps/sim/connectors/jsm/jsm.test.ts index 6aab2f5e135..797a3150b78 100644 --- a/apps/sim/connectors/jsm/jsm.test.ts +++ b/apps/sim/connectors/jsm/jsm.test.ts @@ -5,9 +5,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) -vi.mock('@/lib/knowledge/documents/utils', () => ({ +vi.mock('@/lib/knowledge/documents/utils', () => ({ VALIDATE_RETRY_OPTIONS: {} })) +vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => ({ fetchWithRetry: mockFetchWithRetry, - VALIDATE_RETRY_OPTIONS: {}, })) vi.mock('@/components/icons', () => ({ JiraServiceManagementIcon: () => null })) vi.mock('@/tools/jira/utils', () => ({ diff --git a/apps/sim/connectors/jsm/jsm.ts b/apps/sim/connectors/jsm/jsm.ts index a1a05066dca..ebc527b5e1d 100644 --- a/apps/sim/connectors/jsm/jsm.ts +++ b/apps/sim/connectors/jsm/jsm.ts @@ -4,7 +4,8 @@ import { AtlassianSiteNotAccessibleError, AtlassianSiteNotMatchedError, } from '@/lib/atlassian/discovery' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { jsmConnectorMeta } from '@/connectors/jsm/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { diff --git a/apps/sim/connectors/linear/linear.test.ts b/apps/sim/connectors/linear/linear.test.ts index 873612db586..4adba9216b8 100644 --- a/apps/sim/connectors/linear/linear.test.ts +++ b/apps/sim/connectors/linear/linear.test.ts @@ -7,9 +7,9 @@ const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn(), })) -vi.mock('@/lib/knowledge/documents/utils', () => ({ +vi.mock('@/lib/knowledge/documents/utils', () => ({ VALIDATE_RETRY_OPTIONS: { maxRetries: 0 } })) +vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => ({ fetchWithRetry: mockFetchWithRetry, - VALIDATE_RETRY_OPTIONS: { maxRetries: 0 }, })) import { linearConnector } from '@/connectors/linear/linear' diff --git a/apps/sim/connectors/linear/linear.ts b/apps/sim/connectors/linear/linear.ts index 47536312153..c4e5fe4531a 100644 --- a/apps/sim/connectors/linear/linear.ts +++ b/apps/sim/connectors/linear/linear.ts @@ -2,8 +2,9 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' import { backoffWithJitter } from '@sim/utils/retry' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' import type { RetryOptions } from '@/lib/knowledge/documents/utils' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { linearConnectorMeta } from '@/connectors/linear/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { diff --git a/apps/sim/connectors/microsoft-excel/microsoft-excel.ts b/apps/sim/connectors/microsoft-excel/microsoft-excel.ts index e9085e137a2..6e39d672d16 100644 --- a/apps/sim/connectors/microsoft-excel/microsoft-excel.ts +++ b/apps/sim/connectors/microsoft-excel/microsoft-excel.ts @@ -1,7 +1,8 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' import type { RetryOptions } from '@/lib/knowledge/documents/utils' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { microsoftExcelConnectorMeta } from '@/connectors/microsoft-excel/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { diff --git a/apps/sim/connectors/microsoft-teams/microsoft-teams.test.ts b/apps/sim/connectors/microsoft-teams/microsoft-teams.test.ts index 306289f09c5..68663929a18 100644 --- a/apps/sim/connectors/microsoft-teams/microsoft-teams.test.ts +++ b/apps/sim/connectors/microsoft-teams/microsoft-teams.test.ts @@ -5,9 +5,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) -vi.mock('@/lib/knowledge/documents/utils', () => ({ +vi.mock('@/lib/knowledge/documents/utils', () => ({ VALIDATE_RETRY_OPTIONS: {} })) +vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => ({ fetchWithRetry: mockFetchWithRetry, - VALIDATE_RETRY_OPTIONS: {}, })) vi.mock('@/components/icons', () => ({ MicrosoftTeamsIcon: () => null })) diff --git a/apps/sim/connectors/microsoft-teams/microsoft-teams.ts b/apps/sim/connectors/microsoft-teams/microsoft-teams.ts index 1ae19ae97ef..958f44c9e05 100644 --- a/apps/sim/connectors/microsoft-teams/microsoft-teams.ts +++ b/apps/sim/connectors/microsoft-teams/microsoft-teams.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DEFAULT_MAX_MESSAGES, microsoftTeamsConnectorMeta, diff --git a/apps/sim/connectors/monday/monday.test.ts b/apps/sim/connectors/monday/monday.test.ts index 7ab350c5cff..46e723dea6c 100644 --- a/apps/sim/connectors/monday/monday.test.ts +++ b/apps/sim/connectors/monday/monday.test.ts @@ -5,9 +5,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) -vi.mock('@/lib/knowledge/documents/utils', () => ({ +vi.mock('@/lib/knowledge/documents/utils', () => ({ VALIDATE_RETRY_OPTIONS: {} })) +vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => ({ fetchWithRetry: mockFetchWithRetry, - VALIDATE_RETRY_OPTIONS: {}, })) vi.mock('@/components/icons', () => ({ MondayIcon: () => null })) diff --git a/apps/sim/connectors/monday/monday.ts b/apps/sim/connectors/monday/monday.ts index 0a50a7819e5..25e251da8d7 100644 --- a/apps/sim/connectors/monday/monday.ts +++ b/apps/sim/connectors/monday/monday.ts @@ -2,7 +2,8 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { sleep } from '@sim/utils/helpers' import { backoffWithJitter } from '@sim/utils/retry' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { mondayConnectorMeta } from '@/connectors/monday/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { diff --git a/apps/sim/connectors/notion/notion.test.ts b/apps/sim/connectors/notion/notion.test.ts index a128aa1a383..17ec2ed635a 100644 --- a/apps/sim/connectors/notion/notion.test.ts +++ b/apps/sim/connectors/notion/notion.test.ts @@ -9,10 +9,12 @@ const { mockFetchWithRetry, mockReadBoundedHttpErrorPayload } = vi.hoisted(() => })) vi.mock('@/lib/knowledge/documents/utils', () => ({ - fetchWithRetry: mockFetchWithRetry, readBoundedHttpErrorPayload: mockReadBoundedHttpErrorPayload, VALIDATE_RETRY_OPTIONS: {}, })) +vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => ({ + fetchWithRetry: mockFetchWithRetry, +})) vi.mock('@/components/icons', () => ({ NotionIcon: () => null })) import { notionConnector } from '@/connectors/notion/notion' diff --git a/apps/sim/connectors/notion/notion.ts b/apps/sim/connectors/notion/notion.ts index fea1c0f0e69..fa24948695e 100644 --- a/apps/sim/connectors/notion/notion.ts +++ b/apps/sim/connectors/notion/notion.ts @@ -1,8 +1,8 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { isPlainRecord } from '@sim/utils/object' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' import { - fetchWithRetry, readBoundedHttpErrorPayload, VALIDATE_RETRY_OPTIONS, } from '@/lib/knowledge/documents/utils' diff --git a/apps/sim/connectors/onedrive/onedrive.test.ts b/apps/sim/connectors/onedrive/onedrive.test.ts index 7ce1f708fd7..47d83823da3 100644 --- a/apps/sim/connectors/onedrive/onedrive.test.ts +++ b/apps/sim/connectors/onedrive/onedrive.test.ts @@ -6,10 +6,12 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) vi.mock('@/lib/knowledge/documents/utils', () => ({ - fetchWithRetry: mockFetchWithRetry, readBoundedHttpErrorBody: async (response: Response) => response.text(), VALIDATE_RETRY_OPTIONS: {}, })) +vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => ({ + fetchWithRetry: mockFetchWithRetry, +})) vi.mock('@/components/icons', () => ({ MicrosoftOneDriveIcon: () => null })) import { onedriveConnector } from '@/connectors/onedrive/onedrive' diff --git a/apps/sim/connectors/onedrive/onedrive.ts b/apps/sim/connectors/onedrive/onedrive.ts index dced987dbaf..e095890078f 100644 --- a/apps/sim/connectors/onedrive/onedrive.ts +++ b/apps/sim/connectors/onedrive/onedrive.ts @@ -1,10 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' -import { - fetchWithRetry, - readBoundedHttpErrorBody, - VALIDATE_RETRY_OPTIONS, -} from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { readBoundedHttpErrorBody, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { onedriveConnectorMeta } from '@/connectors/onedrive/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { diff --git a/apps/sim/connectors/outlook/outlook.ts b/apps/sim/connectors/outlook/outlook.ts index c3a912363ba..0107d53bab9 100644 --- a/apps/sim/connectors/outlook/outlook.ts +++ b/apps/sim/connectors/outlook/outlook.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DEFAULT_MAX_CONVERSATIONS, outlookConnectorMeta } from '@/connectors/outlook/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { diff --git a/apps/sim/connectors/pagerduty/pagerduty.ts b/apps/sim/connectors/pagerduty/pagerduty.ts index f122f7d30ae..371cd27bf80 100644 --- a/apps/sim/connectors/pagerduty/pagerduty.ts +++ b/apps/sim/connectors/pagerduty/pagerduty.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { pagerdutyConnectorMeta } from '@/connectors/pagerduty/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { htmlToPlainText, joinTagArray, parseMultiValue, parseTagDate } from '@/connectors/utils' diff --git a/apps/sim/connectors/reddit/reddit.ts b/apps/sim/connectors/reddit/reddit.ts index cd83f7f59bb..79b7a779e85 100644 --- a/apps/sim/connectors/reddit/reddit.ts +++ b/apps/sim/connectors/reddit/reddit.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DEFAULT_MAX_POSTS, redditConnectorMeta } from '@/connectors/reddit/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { parseTagDate } from '@/connectors/utils' diff --git a/apps/sim/connectors/rootly/rootly.ts b/apps/sim/connectors/rootly/rootly.ts index a87524df9df..d3d1106a02c 100644 --- a/apps/sim/connectors/rootly/rootly.ts +++ b/apps/sim/connectors/rootly/rootly.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { rootlyConnectorMeta } from '@/connectors/rootly/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { joinTagArray, parseMultiValue, parseTagDate } from '@/connectors/utils' diff --git a/apps/sim/connectors/salesforce/salesforce.test.ts b/apps/sim/connectors/salesforce/salesforce.test.ts index 6bf56c4ceba..f101edefd2f 100644 --- a/apps/sim/connectors/salesforce/salesforce.test.ts +++ b/apps/sim/connectors/salesforce/salesforce.test.ts @@ -5,9 +5,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) -vi.mock('@/lib/knowledge/documents/utils', () => ({ +vi.mock('@/lib/knowledge/documents/utils', () => ({ VALIDATE_RETRY_OPTIONS: {} })) +vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => ({ fetchWithRetry: mockFetchWithRetry, - VALIDATE_RETRY_OPTIONS: {}, })) vi.mock('@/components/icons', () => ({ SalesforceIcon: () => null })) diff --git a/apps/sim/connectors/salesforce/salesforce.ts b/apps/sim/connectors/salesforce/salesforce.ts index 522e290d834..28581b2b328 100644 --- a/apps/sim/connectors/salesforce/salesforce.ts +++ b/apps/sim/connectors/salesforce/salesforce.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { SALESFORCE_LOGIN_HOSTS } from '@/lib/oauth/salesforce' import { salesforceConnectorMeta } from '@/connectors/salesforce/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' diff --git a/apps/sim/connectors/servicenow/servicenow.ts b/apps/sim/connectors/servicenow/servicenow.ts index bf223f89f21..d6f1b0cbd42 100644 --- a/apps/sim/connectors/servicenow/servicenow.ts +++ b/apps/sim/connectors/servicenow/servicenow.ts @@ -1,7 +1,8 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { validateServiceNowInstanceUrl } from '@/lib/core/security/input-validation' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { DEFAULT_MAX_ITEMS, servicenowConnectorMeta } from '@/connectors/servicenow/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { htmlToPlainText, parseTagDate } from '@/connectors/utils' diff --git a/apps/sim/connectors/sharepoint/sharepoint.test.ts b/apps/sim/connectors/sharepoint/sharepoint.test.ts index 9263246515b..7f3e1432182 100644 --- a/apps/sim/connectors/sharepoint/sharepoint.test.ts +++ b/apps/sim/connectors/sharepoint/sharepoint.test.ts @@ -6,10 +6,12 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) vi.mock('@/lib/knowledge/documents/utils', () => ({ - fetchWithRetry: mockFetchWithRetry, readBoundedHttpErrorBody: async (response: Response) => response.text(), VALIDATE_RETRY_OPTIONS: {}, })) +vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => ({ + fetchWithRetry: mockFetchWithRetry, +})) vi.mock('@/components/icons', () => ({ MicrosoftSharepointIcon: () => null })) import { diff --git a/apps/sim/connectors/sharepoint/sharepoint.ts b/apps/sim/connectors/sharepoint/sharepoint.ts index 27f21bd6fab..fa5d434b819 100644 --- a/apps/sim/connectors/sharepoint/sharepoint.ts +++ b/apps/sim/connectors/sharepoint/sharepoint.ts @@ -1,11 +1,8 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { isPlainRecord } from '@sim/utils/object' -import { - fetchWithRetry, - readBoundedHttpErrorBody, - VALIDATE_RETRY_OPTIONS, -} from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { readBoundedHttpErrorBody, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { sharepointConnectorMeta } from '@/connectors/sharepoint/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { diff --git a/apps/sim/connectors/slack/slack.ts b/apps/sim/connectors/slack/slack.ts index 22c2ace4b29..3db7efc1617 100644 --- a/apps/sim/connectors/slack/slack.ts +++ b/apps/sim/connectors/slack/slack.ts @@ -5,11 +5,8 @@ import { generateId } from '@sim/utils/id' import { isPlainRecord } from '@sim/utils/object' import { truncate } from '@sim/utils/string' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' -import { - fetchWithRetry, - isRateLimitError, - VALIDATE_RETRY_OPTIONS, -} from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { isRateLimitError, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { slackConversationTypes as conversationTypes, readSlackConversationSetting as readConversationSetting, diff --git a/apps/sim/connectors/trello/trello.ts b/apps/sim/connectors/trello/trello.ts index 9874824b278..9f4df81ff58 100644 --- a/apps/sim/connectors/trello/trello.ts +++ b/apps/sim/connectors/trello/trello.ts @@ -1,7 +1,8 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' import { env } from '@/lib/core/config/env' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { trelloConnectorMeta } from '@/connectors/trello/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { joinTagArray, parseMultiValue, parseTagDate } from '@/connectors/utils' diff --git a/apps/sim/connectors/typeform/typeform.ts b/apps/sim/connectors/typeform/typeform.ts index c9890b26595..7c4ce2f03dc 100644 --- a/apps/sim/connectors/typeform/typeform.ts +++ b/apps/sim/connectors/typeform/typeform.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import { typeformConnectorMeta } from '@/connectors/typeform/meta' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { parseTagDate } from '@/connectors/utils' diff --git a/apps/sim/connectors/utils.test.ts b/apps/sim/connectors/utils.test.ts index e28549c7262..b8640d29a75 100644 --- a/apps/sim/connectors/utils.test.ts +++ b/apps/sim/connectors/utils.test.ts @@ -25,10 +25,8 @@ vi.mock('@/components/icons', () => ({ RootlyIcon: () => null, AzureIcon: () => null, })) -vi.mock('@/lib/knowledge/documents/utils', () => ({ - fetchWithRetry: vi.fn(), - VALIDATE_RETRY_OPTIONS: {}, -})) +vi.mock('@/lib/knowledge/documents/utils', () => ({ VALIDATE_RETRY_OPTIONS: {} })) +vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => ({ fetchWithRetry: vi.fn() })) vi.mock('@/tools/jira/utils', () => ({ extractAdfText: vi.fn(), getJiraCloudId: vi.fn() })) vi.mock('@/tools/confluence/utils', () => ({ getConfluenceCloudId: vi.fn() })) vi.mock('@/tools/jsm/utils', () => ({ diff --git a/apps/sim/connectors/webflow/webflow.ts b/apps/sim/connectors/webflow/webflow.ts index fe96c3a6eff..427521bc8e3 100644 --- a/apps/sim/connectors/webflow/webflow.ts +++ b/apps/sim/connectors/webflow/webflow.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage, toError } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { htmlToPlainText, parseMultiValue, parseTagDate } from '@/connectors/utils' import { webflowConnectorMeta } from '@/connectors/webflow/meta' diff --git a/apps/sim/connectors/wordpress/wordpress.ts b/apps/sim/connectors/wordpress/wordpress.ts index 90747709cfc..fccf5d8e123 100644 --- a/apps/sim/connectors/wordpress/wordpress.ts +++ b/apps/sim/connectors/wordpress/wordpress.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { CONNECTOR_MAX_FILE_BYTES, diff --git a/apps/sim/connectors/workday/workday.ts b/apps/sim/connectors/workday/workday.ts index a19b9bf0ef6..210cfc1310d 100644 --- a/apps/sim/connectors/workday/workday.ts +++ b/apps/sim/connectors/workday/workday.ts @@ -1,8 +1,9 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { validatePathSegment, validateWorkdayTenantUrl } from '@/lib/core/security/input-validation' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' import type { RetryOptions } from '@/lib/knowledge/documents/utils' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { htmlToPlainText, diff --git a/apps/sim/connectors/x/x.ts b/apps/sim/connectors/x/x.ts index b94aaefb13d..8ca68fb5d50 100644 --- a/apps/sim/connectors/x/x.ts +++ b/apps/sim/connectors/x/x.ts @@ -1,7 +1,8 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { truncate } from '@sim/utils/string' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { parseMultiValue, parseTagDate } from '@/connectors/utils' import { DEFAULT_MAX_POSTS, xConnectorMeta } from '@/connectors/x/meta' diff --git a/apps/sim/connectors/youtube/youtube.ts b/apps/sim/connectors/youtube/youtube.ts index 4be5cbfa63a..c4e8d7bbd53 100644 --- a/apps/sim/connectors/youtube/youtube.ts +++ b/apps/sim/connectors/youtube/youtube.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { joinTagArray, parseTagDate } from '@/connectors/utils' import { youtubeConnectorMeta } from '@/connectors/youtube/meta' diff --git a/apps/sim/connectors/zoho-desk/zoho-desk.ts b/apps/sim/connectors/zoho-desk/zoho-desk.ts index c1b3c5d5eca..b18d55e07a2 100644 --- a/apps/sim/connectors/zoho-desk/zoho-desk.ts +++ b/apps/sim/connectors/zoho-desk/zoho-desk.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { htmlToPlainText, joinTagArray, parseMultiValue, parseTagDate } from '@/connectors/utils' import { diff --git a/apps/sim/connectors/zoom/zoom.test.ts b/apps/sim/connectors/zoom/zoom.test.ts index decac962516..b99297d3453 100644 --- a/apps/sim/connectors/zoom/zoom.test.ts +++ b/apps/sim/connectors/zoom/zoom.test.ts @@ -5,9 +5,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' const { mockFetchWithRetry } = vi.hoisted(() => ({ mockFetchWithRetry: vi.fn() })) -vi.mock('@/lib/knowledge/documents/utils', () => ({ +vi.mock('@/lib/knowledge/documents/utils', () => ({ VALIDATE_RETRY_OPTIONS: {} })) +vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => ({ fetchWithRetry: mockFetchWithRetry, - VALIDATE_RETRY_OPTIONS: {}, })) import { parseVtt, zoomConnector } from '@/connectors/zoom/zoom' diff --git a/apps/sim/connectors/zoom/zoom.ts b/apps/sim/connectors/zoom/zoom.ts index 0bba34df117..8dc960b3e0d 100644 --- a/apps/sim/connectors/zoom/zoom.ts +++ b/apps/sim/connectors/zoom/zoom.ts @@ -1,6 +1,7 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import { fetchWithRetry, VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { VALIDATE_RETRY_OPTIONS } from '@/lib/knowledge/documents/utils' import type { ConnectorConfig, ExternalDocument, ExternalDocumentList } from '@/connectors/types' import { CONNECTOR_MAX_FILE_BYTES, diff --git a/apps/sim/executor/handlers/workflow/workflow-handler.ts b/apps/sim/executor/handlers/workflow/workflow-handler.ts index 3eb08c62779..961606ac8a1 100644 --- a/apps/sim/executor/handlers/workflow/workflow-handler.ts +++ b/apps/sim/executor/handlers/workflow/workflow-handler.ts @@ -5,6 +5,7 @@ import { isRecordLike } from '@sim/utils/object' import type { Variable, WorkflowState } from '@sim/workflow-types/workflow' import { resolveBillingAttribution } from '@/lib/billing/core/billing-attribution' import { getExecutionDeadlineAt } from '@/lib/core/execution-limits' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { asOrchestrationError } from '@/lib/core/orchestration/types' import { getExecutionEnvironment } from '@/lib/environment/utils' import { buildNextCallChain, validateCallChain } from '@/lib/execution/call-chain' @@ -908,7 +909,10 @@ export class WorkflowBlockHandler implements BlockHandler { const startTime = performance.now() - const result = await subExecutor.execute(workflowId) + const executeChild = () => subExecutor.execute(workflowId) + const result = await (isCustomBlock + ? withResourceOutboundScope({ workspaceId: childWorkspaceId }, executeChild) + : executeChild()) const executionResult = this.toExecutionResult(result) const duration = performance.now() - startTime diff --git a/apps/sim/hooks/queries/organization-network.ts b/apps/sim/hooks/queries/organization-network.ts new file mode 100644 index 00000000000..12d34343cff --- /dev/null +++ b/apps/sim/hooks/queries/organization-network.ts @@ -0,0 +1,22 @@ +import { useQuery } from '@tanstack/react-query' +import { requestJson } from '@/lib/api/client/request' +import { getOrganizationNetworkContract } from '@/lib/api/contracts/organization-network' + +export const ORGANIZATION_NETWORK_STALE_TIME = 30_000 + +export const organizationNetworkKeys = { + all: ['organization-network'] as const, + details: () => [...organizationNetworkKeys.all, 'detail'] as const, + detail: (organizationId: string) => + [...organizationNetworkKeys.details(), organizationId] as const, +} + +export function useOrganizationNetwork(organizationId: string) { + return useQuery({ + queryKey: organizationNetworkKeys.detail(organizationId), + queryFn: ({ signal }) => + requestJson(getOrganizationNetworkContract, { params: { id: organizationId }, signal }), + enabled: Boolean(organizationId), + staleTime: ORGANIZATION_NETWORK_STALE_TIME, + }) +} diff --git a/apps/sim/lib/api/contracts/organization-network.ts b/apps/sim/lib/api/contracts/organization-network.ts new file mode 100644 index 00000000000..bd82581d40a --- /dev/null +++ b/apps/sim/lib/api/contracts/organization-network.ts @@ -0,0 +1,23 @@ +import { z } from 'zod' +import { organizationIdSchema } from '@/lib/api/contracts/primitives' +import { defineRouteContract } from '@/lib/api/contracts/types' +import { gatewayPublicMetadataSchema } from '@/lib/core/network/gateway-metadata' + +export const organizationNetworkSchema = z.discriminatedUnion('mode', [ + z.object({ mode: z.literal('direct') }), + z.object({ + mode: z.literal('gateway'), + publicIps: gatewayPublicMetadataSchema.shape.publicIps, + }), + z.object({ mode: z.literal('blocked') }), + z.object({ mode: z.literal('unavailable') }), +]) + +export const getOrganizationNetworkContract = defineRouteContract({ + method: 'GET', + path: '/api/organizations/[id]/network', + params: z.object({ id: organizationIdSchema }), + response: { mode: 'json', schema: organizationNetworkSchema }, +}) + +export type OrganizationNetwork = z.output diff --git a/apps/sim/lib/copilot/generated/docs-manifest.ts b/apps/sim/lib/copilot/generated/docs-manifest.ts index 69d6daa5aab..bced482121a 100644 --- a/apps/sim/lib/copilot/generated/docs-manifest.ts +++ b/apps/sim/lib/copilot/generated/docs-manifest.ts @@ -383,6 +383,7 @@ export const DOCS_MANIFEST: readonly string[] = [ 'platform/enterprise/scim.mdx', 'platform/enterprise/scim/entra.mdx', 'platform/enterprise/scim/okta.mdx', + 'platform/enterprise/security.mdx', 'platform/enterprise/self-hosted.mdx', 'platform/enterprise/session-policies.mdx', 'platform/enterprise/sso.mdx', diff --git a/apps/sim/lib/copilot/tool-executor/executor.ts b/apps/sim/lib/copilot/tool-executor/executor.ts index 7b0503f3bf7..e2d77c6315f 100644 --- a/apps/sim/lib/copilot/tool-executor/executor.ts +++ b/apps/sim/lib/copilot/tool-executor/executor.ts @@ -6,6 +6,7 @@ import { assertAssistantIntegrationCall, } from '@/lib/copilot/assistant/tool-policy' import { projectToolErrorMessageForCopilot } from '@/lib/copilot/request/tools/resolved-secret-result' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { DEFAULT_EXECUTION_TIMEOUT_MS } from '@/lib/execution/constants' import { recordSecretUsage } from '@/lib/secrets/usage/record' import { executeTool as executeAppTool } from '@/tools' @@ -96,71 +97,76 @@ export async function executeTool( } } - const normalizedParams = normalizeToolParams(toolId, params, context) - - const canUseRegisteredHandler = - isKnownTool(toolId) && (isSimExecuted(toolId) || usesHeadlessClientFallback) - if (!canUseRegisteredHandler) { - const appParams = buildAppToolParams(normalizedParams, context) - const options = { - ...(context.resolvedSecretTraceRegistry - ? { resolvedSecretTraceRegistry: context.resolvedSecretTraceRegistry } - : {}), - ...(context.abortSignal ? { signal: context.abortSignal } : {}), - operationContext: { - userId: context.userId, - workflowId: context.workflowId, - workspaceId: context.workspaceId, - executionId: context.executionId, - chatId: context.chatId, - toolCallId: context.toolCallId, - mcpBlockId: context.mcpBlockId, - executorDelegationOrigin: context.executorDelegationOrigin ?? { - subjectUserId: context.userId, + return withResourceOutboundScope(context, async () => { + const normalizedParams = normalizeToolParams(toolId, params, context) + + const canUseRegisteredHandler = + isKnownTool(toolId) && (isSimExecuted(toolId) || usesHeadlessClientFallback) + if (!canUseRegisteredHandler) { + const appParams = buildAppToolParams(normalizedParams, context) + const options = { + ...(context.resolvedSecretTraceRegistry + ? { resolvedSecretTraceRegistry: context.resolvedSecretTraceRegistry } + : {}), + ...(context.abortSignal ? { signal: context.abortSignal } : {}), + operationContext: { + userId: context.userId, workflowId: context.workflowId, - ...(context.executionId ? { executionId: context.executionId } : {}), + workspaceId: context.workspaceId, + executionId: context.executionId, + chatId: context.chatId, + toolCallId: context.toolCallId, + mcpBlockId: context.mcpBlockId, + executorDelegationOrigin: context.executorDelegationOrigin ?? { + subjectUserId: context.userId, + workflowId: context.workflowId, + ...(context.executionId ? { executionId: context.executionId } : {}), + }, + copilotToolExecution: context.copilotToolExecution, + copilotInteractionMode: context.copilotInteractionMode, + requestMode: context.requestMode, + billingAttribution: context.billingAttribution, + resolvedSecretTraceRegistry: context.resolvedSecretTraceRegistry, }, - copilotToolExecution: context.copilotToolExecution, - copilotInteractionMode: context.copilotInteractionMode, - requestMode: context.requestMode, - billingAttribution: context.billingAttribution, - resolvedSecretTraceRegistry: context.resolvedSecretTraceRegistry, - }, + } + try { + return await (Object.keys(options).length > 0 + ? executeAppTool(toolId, appParams, options) + : executeAppTool(toolId, appParams)) + } finally { + recordAppToolSecretUsage(context) + } } - try { - return await (Object.keys(options).length > 0 - ? executeAppTool(toolId, appParams, options) - : executeAppTool(toolId, appParams)) - } finally { - recordAppToolSecretUsage(context) - } - } - if (context.abortSignal?.aborted) { - logger.warn('Tool execution skipped: abort signal already set', { - toolId, - abortReason: context.abortSignal.reason ?? 'unknown', - }) - return { success: false, error: 'Execution aborted: abort signal was set before tool started' } - } + if (context.abortSignal?.aborted) { + logger.warn('Tool execution skipped: abort signal already set', { + toolId, + abortReason: context.abortSignal.reason ?? 'unknown', + }) + return { + success: false, + error: 'Execution aborted: abort signal was set before tool started', + } + } - const handler = handlerRegistry.get(toolId) - if (!handler) { - logger.warn('No handler registered for tool', { toolId }) - return { success: false, error: `No handler for tool: ${toolId}` } - } + const handler = handlerRegistry.get(toolId) + if (!handler) { + logger.warn('No handler registered for tool', { toolId }) + return { success: false, error: `No handler for tool: ${toolId}` } + } - try { - return await handler(normalizedParams, context) - } catch (error) { - const message = toError(error).message - logger.error('Tool execution failed', { - toolId, - error: projectToolErrorMessageForCopilot(message, context.resolvedSecretTraceRegistry), - abortSignalAborted: context.abortSignal?.aborted ?? false, - }) - return { success: false, error: message } - } + try { + return await handler(normalizedParams, context) + } catch (error) { + const message = toError(error).message + logger.error('Tool execution failed', { + toolId, + error: projectToolErrorMessageForCopilot(message, context.resolvedSecretTraceRegistry), + abortSignalAborted: context.abortSignal?.aborted ?? false, + }) + return { success: false, error: message } + } + }) } function normalizeToolParams( diff --git a/apps/sim/lib/copilot/tools/server/router.ts b/apps/sim/lib/copilot/tools/server/router.ts index 0c499a66e9e..173b6bc47e2 100644 --- a/apps/sim/lib/copilot/tools/server/router.ts +++ b/apps/sim/lib/copilot/tools/server/router.ts @@ -65,6 +65,7 @@ import { getCredentialsServerTool } from '@/lib/copilot/tools/server/user/get-cr import { setEnvironmentVariablesServerTool } from '@/lib/copilot/tools/server/user/set-environment-variables' import { editWorkflowServerTool } from '@/lib/copilot/tools/server/workflow/edit-workflow' import { queryLogsServerTool } from '@/lib/copilot/tools/server/workflow/query-logs' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { OrchestrationError } from '@/lib/core/orchestration/types' import { listCustomBlocksWithInputsForWorkspace } from '@/lib/workflows/custom-blocks/operations' import { withCustomBlockOverlay } from '@/blocks/custom/server-overlay' @@ -290,7 +291,9 @@ export async function routeExecution( const inner = run run = () => withCustomBlockOverlay(rows, inner) } - const result = await run() + const result = await (context?.workspaceId || context?.organizationId + ? withResourceOutboundScope(context, run) + : run()) // Validate output if tool declares a schema; otherwise fall back to the // generated JSON schema contract emitted from Go. diff --git a/apps/sim/lib/core/application/authorized-workspace-use-case.test.ts b/apps/sim/lib/core/application/authorized-workspace-use-case.test.ts index 5ead0766352..f9d1b64e6a5 100644 --- a/apps/sim/lib/core/application/authorized-workspace-use-case.test.ts +++ b/apps/sim/lib/core/application/authorized-workspace-use-case.test.ts @@ -10,11 +10,18 @@ import type { import { beforeEach, describe, expect, expectTypeOf, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ + routingEnabled: vi.fn(() => false), + resolveRoute: vi.fn(async () => ({ kind: 'direct' as const })), events: [] as string[], recordAudit: vi.fn(() => mocks.events.push('audit')), resolvePermission: vi.fn(), })) +vi.mock('@/lib/core/network/config.server', () => ({ + isOutboundRoutingEnabled: mocks.routingEnabled, + resolveOutboundRoute: mocks.resolveRoute, +})) + vi.mock('@sim/audit', () => ({ AuditAction: { FILE_UPDATED: 'file.updated' }, AuditResourceType: { FILE: 'file' }, @@ -33,6 +40,7 @@ vi.mock('@sim/platform-authz/workspace', () => ({ import { AuditAction, AuditResourceType } from '@sim/audit' import { defineAuthorizedWorkspaceUseCase, defineWorkspaceOperation } from '@/lib/core/application' +import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' import type { OrchestrationError } from '@/lib/core/orchestration/types' import { CREDENTIAL_GROUP_CREDENTIAL_USE_ACTION } from '@/lib/resource-policies/registry' @@ -100,10 +108,37 @@ const sessionPrincipal: SessionPrincipal = { describe('defineAuthorizedWorkspaceUseCase', () => { beforeEach(() => { vi.clearAllMocks() + mocks.routingEnabled.mockReturnValue(false) mocks.events.length = 0 mocks.resolvePermission.mockResolvedValue('write') }) + it('establishes outbound ownership after authorization and retains it through effects', async () => { + mocks.routingEnabled.mockReturnValue(true) + const useCase = defineAuthorizedWorkspaceUseCase({ + operation, + resolveContext: async () => canonicalContext, + authorizationOptions: {}, + async execute() { + expect(mocks.resolvePermission).toHaveBeenCalledOnce() + await resolveCurrentOutboundRoute() + return 'done' + }, + async afterSuccess() { + await resolveCurrentOutboundRoute() + }, + }) + await expect(useCase.execute({ principal: sessionPrincipal, input: {} })).resolves.toBe('done') + expect(mocks.resolveRoute.mock.calls).toEqual([['organization-1'], ['organization-1']]) + await resolveCurrentOutboundRoute() + expect(mocks.resolveRoute).toHaveBeenLastCalledWith(undefined) + + mocks.resolveRoute.mockClear() + mocks.resolvePermission.mockResolvedValue(null) + await expect(useCase.execute({ principal: sessionPrincipal, input: {} })).rejects.toThrow() + expect(mocks.resolveRoute).not.toHaveBeenCalled() + }) + it('narrows definition callbacks while keeping public execution principal-safe', async () => { const resolveContext = vi.fn( async ({ principal }: { principal: SessionPrincipal; input: TestInput }) => { diff --git a/apps/sim/lib/core/application/authorized-workspace-use-case.ts b/apps/sim/lib/core/application/authorized-workspace-use-case.ts index c3a8b2e1d3d..19ed9ff3965 100644 --- a/apps/sim/lib/core/application/authorized-workspace-use-case.ts +++ b/apps/sim/lib/core/application/authorized-workspace-use-case.ts @@ -12,6 +12,7 @@ import type { PrincipalForOperation, WorkspaceOperation, } from '@/lib/core/application/workspace-operation' +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' import type { OrchestrationRequestContext } from '@/lib/core/orchestration/types' import type { ResourcePolicyBinding } from '@/lib/resource-policies/registry' @@ -205,23 +206,25 @@ export function defineAuthorizedWorkspaceUseCase< async execute(args) { const executionContext = await authorizePhase(args) const { principal, context, request } = executionContext - const result = await definition.execute(executionContext) - const resultContext = { ...executionContext, result } - const projectedAudit = definition.projectAudit?.(resultContext) - if (projectedAudit !== undefined) { - const auditEntries = Array.isArray(projectedAudit) ? projectedAudit : [projectedAudit] - if (auditEntries.length > 0) { - recordProjectedUseCaseAuditEntries( - definition.operation, - context.workspaceId, - principal, - request, - auditEntries - ) + return runWithOutboundOrganization(context.workspaceOrganizationId, async () => { + const result = await definition.execute(executionContext) + const resultContext = { ...executionContext, result } + const projectedAudit = definition.projectAudit?.(resultContext) + if (projectedAudit !== undefined) { + const auditEntries = Array.isArray(projectedAudit) ? projectedAudit : [projectedAudit] + if (auditEntries.length > 0) { + recordProjectedUseCaseAuditEntries( + definition.operation, + context.workspaceId, + principal, + request, + auditEntries + ) + } } - } - await definition.afterSuccess?.(resultContext) - return result + await definition.afterSuccess?.(resultContext) + return result + }) }, } } diff --git a/apps/sim/lib/core/config/appconfig.test.ts b/apps/sim/lib/core/config/appconfig.test.ts index fb497fe1788..e5f797d665f 100644 --- a/apps/sim/lib/core/config/appconfig.test.ts +++ b/apps/sim/lib/core/config/appconfig.test.ts @@ -1,7 +1,7 @@ /** * @vitest-environment node */ -import { beforeEach, describe, expect, it, vi } from 'vitest' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const { mockSend } = vi.hoisted(() => ({ mockSend: vi.fn(), @@ -21,7 +21,7 @@ vi.mock('@aws-sdk/client-appconfigdata', () => ({ }, })) -import { fetchAppConfigProfile } from '@/lib/core/config/appconfig' +import { fetchAppConfigProfile, fetchAppConfigSnapshot } from '@/lib/core/config/appconfig' const encode = (value: unknown) => new TextEncoder().encode(JSON.stringify(value)) @@ -133,4 +133,154 @@ describe('fetchAppConfigProfile', () => { expect(b).toEqual({ x: 1 }) expect(mockSend.mock.calls.map(([c]) => c.__type)).toEqual(['start', 'get']) }) + + it('honors the server poll interval and serves warm values during one shared refresh', async () => { + vi.useFakeTimers() + vi.setSystemTime(100_000) + try { + mockSend.mockImplementation((command: { __type: string }) => + Promise.resolve( + command.__type === 'start' + ? { InitialConfigurationToken: 'token' } + : { + Configuration: encode({ revision: 'first' }), + NextPollConfigurationToken: 'next', + NextPollIntervalInSeconds: 60, + } + ) + ) + const ids = uniqueIds() + const parse = (value: unknown) => value + expect(await fetchAppConfigProfile(ids, parse)).toEqual({ revision: 'first' }) + vi.setSystemTime(130_001) + await fetchAppConfigProfile(ids, parse) + expect(mockSend).toHaveBeenCalledTimes(2) + + let finish: (value: unknown) => void = () => {} + mockSend.mockReturnValueOnce( + new Promise((resolve) => { + finish = resolve + }) + ) + vi.setSystemTime(161_000) + expect( + await Promise.all([fetchAppConfigProfile(ids, parse), fetchAppConfigProfile(ids, parse)]) + ).toEqual([{ revision: 'first' }, { revision: 'first' }]) + expect(mockSend).toHaveBeenCalledTimes(3) + finish({ Configuration: encode({ revision: 'second' }), NextPollConfigurationToken: 'next' }) + await vi.waitFor(async () => { + expect(await fetchAppConfigProfile(ids, parse)).toEqual({ revision: 'second' }) + }) + } finally { + vi.useRealTimers() + } + }) +}) + +describe('fetchAppConfigSnapshot freshness', () => { + beforeEach(() => { + vi.clearAllMocks() + vi.useFakeTimers() + vi.setSystemTime(100_000) + }) + afterEach(() => vi.useRealTimers()) + + it('evicts old profiles and requires fresh evidence when they are requested again', async () => { + mockSend.mockImplementation((command: { __type: string }) => + Promise.resolve( + command.__type === 'start' + ? { InitialConfigurationToken: 'token' } + : { Configuration: encode({ revision: 'valid' }), NextPollConfigurationToken: 'next' } + ) + ) + const oldest = uniqueIds() + await fetchAppConfigSnapshot(oldest, (value) => value) + const results = await Promise.all( + Array.from({ length: 65 }, () => fetchAppConfigSnapshot(uniqueIds(), (value) => value)) + ) + expect(results.every((result) => result.value !== null)).toBe(true) + mockSend.mockRejectedValueOnce(new Error('unavailable')) + expect(await fetchAppConfigSnapshot(oldest, (value) => value)).toEqual({ + value: null, + validatedAt: null, + }) + expect(mockSend.mock.calls.at(-1)?.[0].__type).toBe('start') + }) + + it('does not turn an empty first response or a cold failure into a valid snapshot', async () => { + mockSend.mockRejectedValueOnce(new Error('unavailable')) + expect(await fetchAppConfigSnapshot(uniqueIds(), (value) => value)).toEqual({ + value: null, + validatedAt: null, + }) + mockSend.mockImplementation((command: { __type: string }) => + Promise.resolve( + command.__type === 'start' + ? { InitialConfigurationToken: 'token' } + : { Configuration: new Uint8Array(), NextPollConfigurationToken: 'next' } + ) + ) + expect(await fetchAppConfigSnapshot(uniqueIds(), (value) => value)).toEqual({ + value: null, + validatedAt: null, + }) + }) + + it('does not renew an old snapshot after a rejected revision followed by unchanged polls', async () => { + let payload = encode({ revision: 'valid' }) + mockSend.mockImplementation((command: { __type: string }) => + Promise.resolve( + command.__type === 'start' + ? { InitialConfigurationToken: 'token' } + : { + Configuration: payload, + NextPollConfigurationToken: 'next', + NextPollIntervalInSeconds: 30, + } + ) + ) + const ids = uniqueIds() + const parse = (value: unknown) => value + const first = await fetchAppConfigSnapshot(ids, parse) + expect(first.validatedAt).toBe(100_000) + payload = new TextEncoder().encode('invalid json') + vi.setSystemTime(130_001) + expect(await fetchAppConfigSnapshot(ids, parse)).toEqual(first) + payload = new Uint8Array() + vi.setSystemTime(160_002) + expect(await fetchAppConfigSnapshot(ids, parse)).toEqual(first) + payload = encode({ revision: 'replacement' }) + vi.setSystemTime(190_003) + expect(await fetchAppConfigSnapshot(ids, parse)).toEqual({ + value: { revision: 'replacement' }, + validatedAt: 190_003, + }) + }) + + it('renews a validated unchanged revision and deduplicates due polls', async () => { + let payload = encode({ revision: 'valid' }) + mockSend.mockImplementation((command: { __type: string }) => + Promise.resolve( + command.__type === 'start' + ? { InitialConfigurationToken: 'token' } + : { + Configuration: payload, + NextPollConfigurationToken: 'next', + NextPollIntervalInSeconds: 30, + } + ) + ) + const ids = uniqueIds() + const parse = (value: unknown) => value + await fetchAppConfigSnapshot(ids, parse) + payload = new Uint8Array() + vi.setSystemTime(130_001) + const results = await Promise.all([ + fetchAppConfigSnapshot(ids, parse), + fetchAppConfigSnapshot(ids, parse), + ]) + expect(results[0].validatedAt).toBe(130_001) + expect(results[1]).toEqual(results[0]) + expect(mockSend.mock.calls).toHaveLength(3) + }) }) diff --git a/apps/sim/lib/core/config/appconfig.ts b/apps/sim/lib/core/config/appconfig.ts index 61aa093c375..8ebfc5eb954 100644 --- a/apps/sim/lib/core/config/appconfig.ts +++ b/apps/sim/lib/core/config/appconfig.ts @@ -6,6 +6,7 @@ import { } from '@aws-sdk/client-appconfigdata' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' +import { LRUCache } from 'lru-cache' import { getAwsCredentialsFromEnv } from '@/lib/core/config/aws' import { env } from '@/lib/core/config/env' @@ -22,16 +23,43 @@ export interface AppConfigProfileIdentifiers { interface CacheEntry { /** Last successfully parsed value, or `null` if the config is empty/unseeded. */ value: T | null - /** True once any poll has completed (success, empty payload, or error). */ - loaded: boolean /** Token for the next `GetLatestConfiguration` poll, rotated on each call. */ nextToken: string | undefined - expiresAt: number - /** In-flight poll, shared so concurrent callers don't each hit AppConfig. */ - inflight: Promise | null + validatedAt: number | null + remoteMatchesValue: boolean + strict: boolean } -const cache = new Map>() +export interface AppConfigSnapshot { + readonly value: T | null + readonly validatedAt: number | null +} + +interface PollContext { + ids: AppConfigProfileIdentifiers + parse: (json: unknown) => unknown + strict: boolean +} + +const cache = new LRUCache, PollContext>({ + max: 64, + ttl: DEFAULT_TTL_MS, + ttlResolution: 0, + ignoreFetchAbort: true, + /** Poll intervals and snapshot freshness share the same clock. */ + perf: { now: () => Date.now() }, + fetchMethod: async (_key, stale, { context, options }) => { + const entry = stale ?? { + value: null, + nextToken: undefined, + validatedAt: null, + remoteMatchesValue: false, + strict: context.strict, + } + options.ttl = await poll(context.ids, context.parse, entry) + return entry + }, +}) let client: AppConfigDataClient | null = null @@ -58,68 +86,69 @@ function cacheKey(ids: AppConfigProfileIdentifiers): string { * Run one AppConfig poll for `entry`: starts a session if no token is held, then * calls `GetLatestConfiguration`. An empty payload means "unchanged" (or an * unseeded profile) and the previous value is kept. Any error is logged and the - * last good value is retained. Marks the entry `loaded` on any outcome so callers - * never re-block on the cold path, and honors AppConfig's `NextPollInterval` so we + * last good value is retained. Returns AppConfig's `NextPollInterval` so we * don't poll faster than the server allows (which would throttle). */ async function poll( ids: AppConfigProfileIdentifiers, parse: (json: unknown) => T, entry: CacheEntry -): Promise { +): Promise { let response: GetLatestConfigurationCommandOutput try { const dataClient = getClient() if (!entry.nextToken) { + entry.remoteMatchesValue = false const session = await dataClient.send( new StartConfigurationSessionCommand({ ApplicationIdentifier: ids.application, EnvironmentIdentifier: ids.environment, ConfigurationProfileIdentifier: ids.profile, - }) + }), + { abortSignal: AbortSignal.timeout(5000) } ) entry.nextToken = session.InitialConfigurationToken } response = await dataClient.send( - new GetLatestConfigurationCommand({ ConfigurationToken: entry.nextToken }) + new GetLatestConfigurationCommand({ ConfigurationToken: entry.nextToken }), + { abortSignal: AbortSignal.timeout(5000) } ) entry.nextToken = response.NextPollConfigurationToken ?? entry.nextToken } catch (error) { - // Network/session failure: drop the token so the next attempt starts a fresh - // session (handles expired or invalid tokens). Mark loaded + back off so we - // serve the fallback and retry in the background rather than blocking every - // request during an outage. + /** A failed or expired session retries after backoff without renewing snapshot freshness. */ entry.nextToken = undefined - entry.expiresAt = Date.now() + DEFAULT_TTL_MS - entry.loaded = true logger.error('AppConfig fetch failed; serving last known value', { profile: cacheKey(ids), error: getErrorMessage(error), }) - return entry.value + return DEFAULT_TTL_MS } - // Parse outside the network try: a decode/parse error must NOT discard the - // already-rotated session token — the round trip succeeded, so the next poll - // can reuse it instead of opening a new session. Keep the last good value. + /** Decode failures retain the rotated session token and last validated value. */ try { if (response.Configuration && response.Configuration.length > 0) { + entry.remoteMatchesValue = false + if (entry.strict && response.Configuration.length > 1_048_576) { + throw new Error('Configuration exceeds the maximum size') + } const text = new TextDecoder().decode(response.Configuration) entry.value = parse(JSON.parse(text)) + entry.remoteMatchesValue = true + } + if (entry.remoteMatchesValue && entry.value !== null) { + entry.validatedAt = Date.now() } } catch (error) { logger.error('AppConfig response parse failed; serving last known value', { profile: cacheKey(ids), - error: getErrorMessage(error), + error: entry.strict ? 'Configuration rejected' : getErrorMessage(error), }) } const intervalMs = (response.NextPollIntervalInSeconds ?? 60) * 1000 - entry.expiresAt = Date.now() + Math.max(DEFAULT_TTL_MS, intervalMs) - entry.loaded = true - return entry.value + return Math.max(DEFAULT_TTL_MS, intervalMs) } /** @@ -137,31 +166,28 @@ export async function fetchAppConfigProfile( ids: AppConfigProfileIdentifiers, parse: (json: unknown) => T ): Promise { - const key = cacheKey(ids) - const entry = (cache.get(key) as CacheEntry | undefined) ?? { - value: null, - loaded: false, - nextToken: undefined, - expiresAt: 0, - inflight: null, - } - cache.set(key, entry) - - // Cold: never polled — await a single shared poll so concurrent callers don't - // each hit AppConfig (and don't race the rotating session token). - if (!entry.loaded) { - entry.inflight ??= poll(ids, parse, entry).finally(() => { - entry.inflight = null - }) - return entry.inflight - } - - // Warm but stale: serve cached value, refresh once in the background. - if (Date.now() >= entry.expiresAt && !entry.inflight) { - entry.inflight = poll(ids, parse, entry).finally(() => { - entry.inflight = null - }) - } + const entry = await cache.fetch(cacheKey(ids), { + context: { ids, parse, strict: false }, + allowStale: true, + }) + return (entry?.value ?? null) as T | null +} - return entry.value +/** + * Security-sensitive callers receive freshness evidence instead of an implicit fallback. + * Due polls are awaited and deduplicated. Rejected remote revisions cannot renew an old + * snapshot through subsequent unchanged responses. The caller owns its maximum stale age. + */ +export async function fetchAppConfigSnapshot( + ids: AppConfigProfileIdentifiers, + parse: (json: unknown) => T +): Promise> { + const entry = await cache.fetch(`strict:${cacheKey(ids)}`, { + context: { ids, parse, strict: true }, + allowStale: false, + }) + return Object.freeze({ + value: (entry?.value ?? null) as T | null, + validatedAt: entry?.validatedAt ?? null, + }) } diff --git a/apps/sim/lib/core/config/env.ts b/apps/sim/lib/core/config/env.ts index efece380bdb..72c47f9a279 100644 --- a/apps/sim/lib/core/config/env.ts +++ b/apps/sim/lib/core/config/env.ts @@ -107,6 +107,10 @@ export const env = createEnv({ skipValidation: true, server: { + OUTBOUND_ROUTING_SOURCE: z.enum(['env', 'appconfig']).optional(), + OUTBOUND_ROUTING_CONFIG: z.string().optional(), + OUTBOUND_GATEWAYS: z.string().optional(), + OUTBOUND_GATEWAY_CREDENTIALS: z.string().optional(), // Core Database & Authentication DATABASE_URL: z.string().url(), // Primary database connection string DATABASE_REPLICA_URL: z.string().url().optional(), // Read-replica connection string; opt-in reads fall back to the primary when unset diff --git a/apps/sim/lib/core/network/application/read-organization-network.test.ts b/apps/sim/lib/core/network/application/read-organization-network.test.ts new file mode 100644 index 00000000000..ed0a230acdb --- /dev/null +++ b/apps/sim/lib/core/network/application/read-organization-network.test.ts @@ -0,0 +1,105 @@ +/** @vitest-environment node */ +import type { Principal } from '@sim/auth/principal' +import { db } from '@sim/db' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + membership: vi.fn(), + section: vi.fn(), + route: vi.fn(), +})) +vi.mock('@/lib/permission-groups/resolve.server', () => ({ + getUserPermissionConfigForOrganization: vi.fn(async () => null), +})) +vi.mock('@/lib/settings/application/organization-section-access', () => ({ + authorizeOrganizationSettingsSection: mocks.section, +})) +vi.mock('@/lib/core/network/config.server', () => ({ resolveOutboundRoute: mocks.route })) + +import { readOrganizationNetwork } from '@/lib/core/network/application/read-organization-network' +import { OutboundRoutingError } from '@/lib/core/network/routing' + +const principal: Principal = { + kind: 'session', + userId: 'user_example', + sessionId: 'session_example', +} +const input = { organizationId: 'org_example' } + +beforeEach(() => { + vi.clearAllMocks() + mocks.membership.mockResolvedValue([{ role: 'admin' }]) + mocks.section.mockResolvedValue(true) + mocks.route.mockResolvedValue({ kind: 'direct' }) + const query = { from: vi.fn(), where: vi.fn(), limit: mocks.membership } + query.from.mockReturnValue(query) + query.where.mockReturnValue(query) + vi.mocked(db.select).mockReturnValue(query as ReturnType) +}) + +describe('organization network settings', () => { + it.each([[], [{ role: 'member' }]])('withholds routing from non-admins', async (rows) => { + mocks.membership.mockResolvedValue(rows) + await expect(readOrganizationNetwork.execute({ principal, input })).rejects.toThrow() + expect(mocks.route).not.toHaveBeenCalled() + expect(mocks.section).not.toHaveBeenCalled() + }) + + it('rejects workspace credentials before looking up the organization', async () => { + const workspacePrincipal: Principal = { + kind: 'workspace_api_key', + workspaceId: 'workspace_example', + apiKeyId: 'key_example', + } + await expect( + readOrganizationNetwork.execute({ principal: workspacePrincipal, input }) + ).rejects.toThrow() + expect(mocks.membership).not.toHaveBeenCalled() + expect(mocks.route).not.toHaveBeenCalled() + }) + + it('checks the target organization entitlement before reading its network', async () => { + mocks.section.mockResolvedValue(false) + await expect(readOrganizationNetwork.execute({ principal, input })).rejects.toThrow( + 'Network settings are not available' + ) + expect(mocks.route).not.toHaveBeenCalled() + expect(mocks.section).toHaveBeenCalledWith( + expect.objectContaining({ organizationId: 'org_example', section: 'security' }) + ) + }) + + it('projects only published addresses, never credentials or dial addresses', async () => { + mocks.route.mockResolvedValue({ + kind: 'gateway', + gateway: { + id: 'gateway_example', + url: 'https://private.example.invalid', + token: 'synthetic-secret', + ca: 'synthetic-ca', + publicIps: ['192.0.2.10', '192.0.2.20'], + }, + }) + await expect(readOrganizationNetwork.execute({ principal, input })).resolves.toEqual({ + mode: 'gateway', + publicIps: ['192.0.2.10', '192.0.2.20'], + }) + expect(mocks.route).toHaveBeenCalledWith('org_example') + }) + + it.each([ + ['ROUTE_BLOCKED', 'blocked'], + ['CONFIGURATION_UNAVAILABLE', 'unavailable'], + ['INVALID_CONFIGURATION', 'unavailable'], + ] as const)('reports %s without claiming direct fallback', async (code, mode) => { + mocks.route.mockRejectedValue(new OutboundRoutingError(code)) + await expect(readOrganizationNetwork.execute({ principal, input })).resolves.toEqual({ mode }) + }) + + it('propagates unexpected failures instead of treating them as default routing', async () => { + mocks.route.mockRejectedValue(new Error('unexpected failure')) + await expect(readOrganizationNetwork.execute({ principal, input })).rejects.toThrow( + 'unexpected failure' + ) + }) +}) diff --git a/apps/sim/lib/core/network/application/read-organization-network.ts b/apps/sim/lib/core/network/application/read-organization-network.ts new file mode 100644 index 00000000000..a51e80fc328 --- /dev/null +++ b/apps/sim/lib/core/network/application/read-organization-network.ts @@ -0,0 +1,55 @@ +import type { OperationUseCase } from '@/lib/core/application/operation' +import { authorizeOrganizationOperation } from '@/lib/core/application/organization-authorization' +import { defineOrganizationOperation } from '@/lib/core/application/organization-operation' +import { resolveOutboundRoute } from '@/lib/core/network/config.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { authorizeOrganizationSettingsSection } from '@/lib/settings/application/organization-section-access' + +export type OrganizationNetwork = + | { mode: 'direct' | 'blocked' | 'unavailable' } + | { mode: 'gateway'; publicIps: string[] } + +/** + * permission-group-exempt: Network settings are governed by organization administrator membership. + */ +export const readOrganizationNetworkOperation = defineOrganizationOperation({ + id: 'organization.network.read', + minimumRole: 'admin', + principalKinds: ['session'], + capability: 'none', +}) + +export const readOrganizationNetwork: OperationUseCase< + typeof readOrganizationNetworkOperation, + { organizationId: string }, + OrganizationNetwork +> = { + operation: readOrganizationNetworkOperation, + async execute({ principal, input }) { + const context = await authorizeOrganizationOperation( + principal, + readOrganizationNetworkOperation, + input + ) + if ( + !(await authorizeOrganizationSettingsSection({ + ...context, + section: 'security', + })) + ) { + throw new OrchestrationError('forbidden', 'Network settings are not available') + } + try { + const route = await resolveOutboundRoute(context.organizationId) + if (route.kind === 'direct') return { mode: 'direct' } + return { + mode: 'gateway', + publicIps: [...(route.gateway.publicIps ?? [])], + } + } catch (error) { + if (!(error instanceof OutboundRoutingError)) throw error + return { mode: error.code === 'ROUTE_BLOCKED' ? 'blocked' : 'unavailable' } + } + }, +} diff --git a/apps/sim/lib/core/network/config.server.test.ts b/apps/sim/lib/core/network/config.server.test.ts new file mode 100644 index 00000000000..325946e2a05 --- /dev/null +++ b/apps/sim/lib/core/network/config.server.test.ts @@ -0,0 +1,179 @@ +/** @vitest-environment node */ +import { describe, expect, it, vi } from 'vitest' +import { createOutboundRoutingReader } from '@/lib/core/network/config.server' + +const document = { + schemaVersion: 1, + revision: 'revision-1', + defaultRoute: { kind: 'direct' }, + organizations: { org_a: { kind: 'gateway', gatewayId: 'gateway_a' } }, +} +const catalog = { + gateway_a: { + organizationId: 'org_a', + url: 'https://proxy.example.invalid/', + credentialId: 'credential_a', + generation: 'generation-1', + }, +} +const credentials = { credential_a: { token: 'synthetic-test-token-0000000000000000' } } +const options = { + source: 'env', + configuration: JSON.stringify(document), + gateways: JSON.stringify(catalog), + credentials: JSON.stringify(credentials), +} +const dependencies = { now: () => 400_000, readSnapshot: vi.fn() } + +describe('outbound configuration', () => { + it('keeps unconfigured OSS deployments independent of AWS and organization context', async () => { + const readSnapshot = vi.fn() + const reader = createOutboundRoutingReader({}, { ...dependencies, readSnapshot }) + expect(reader.enabled).toBe(false) + expect(await reader.resolve(undefined)).toEqual({ kind: 'direct' }) + expect(readSnapshot).not.toHaveBeenCalled() + }) + + it('does not treat missing context as a personal workspace', async () => { + const reader = createOutboundRoutingReader(options, dependencies) + await expect(reader.resolve(undefined)).rejects.toThrow('MISSING_SCOPE') + expect(await reader.resolve(null)).toEqual({ kind: 'direct' }) + expect(await reader.resolve('org_a')).toMatchObject({ + kind: 'gateway', + gateway: { id: 'gateway_a' }, + }) + }) + + it('rejects assignments to another organization and gateway defaults', () => { + for (const overrides of [ + { defaultRoute: { kind: 'gateway', gatewayId: 'gateway_a' } }, + { organizations: { org_b: { kind: 'gateway', gatewayId: 'gateway_a' } } }, + ]) { + expect(() => + createOutboundRoutingReader( + { + ...options, + configuration: JSON.stringify({ ...document, ...overrides }), + }, + dependencies + ) + ).toThrow('INVALID_CONFIGURATION') + } + }) + + it('refuses to share an endpoint or credential between organizations', () => { + for (const gateway of [ + { ...catalog.gateway_a, organizationId: 'org_b' }, + { ...catalog.gateway_a, organizationId: 'org_b', url: 'https://second.invalid/' }, + ]) { + expect(() => + createOutboundRoutingReader( + { + ...options, + gateways: JSON.stringify({ ...catalog, gateway_b: gateway }), + }, + dependencies + ) + ).toThrow('INVALID_CONFIGURATION') + } + }) + + it('does not release a reserved organization when its assignment is removed', async () => { + const reader = createOutboundRoutingReader( + { + ...options, + configuration: JSON.stringify({ ...document, organizations: {} }), + }, + dependencies + ) + await expect(reader.resolve('org_a')).rejects.toThrow('ROUTE_BLOCKED') + expect(await reader.resolve('org_other')).toEqual({ kind: 'direct' }) + }) + + it('rejects an outbound IP published for two different owners', () => { + expect(() => + createOutboundRoutingReader( + { + ...options, + credentials: JSON.stringify({ ...credentials, credential_b: { token: 'b'.repeat(48) } }), + gateways: JSON.stringify({ + gateway_a: { ...catalog.gateway_a, publicIps: ['192.0.2.10'] }, + gateway_b: { + ...catalog.gateway_a, + organizationId: 'org_b', + url: 'https://second.invalid/', + credentialId: 'credential_b', + publicIps: ['192.0.2.10'], + }, + }), + }, + dependencies + ) + ).toThrow('INVALID_CONFIGURATION') + }) + + it.each([ + { ...options, source: undefined }, + { ...options, configuration: undefined }, + { ...options, gateways: '{}' }, + { ...options, credentials: '{}' }, + { + ...options, + gateways: JSON.stringify({ + gateway_a: { ...catalog.gateway_a, url: 'http://proxy.example.invalid/' }, + }), + }, + { + ...options, + gateways: JSON.stringify({ + gateway_a: { ...catalog.gateway_a, url: 'https://user:password@proxy.example.invalid/' }, + }), + }, + ])('rejects incomplete or unsafe operator configuration', (input) => { + expect(() => createOutboundRoutingReader(input, dependencies)).toThrow('INVALID_CONFIGURATION') + }) + + it('never falls back to direct routing on cold, stale or future snapshots', async () => { + for (const validatedAt of [null, 100_000, 500_000]) { + const reader = createOutboundRoutingReader( + { ...options, source: 'appconfig', configuration: undefined }, + { + ...dependencies, + readSnapshot: async (parse) => ({ value: parse(document), validatedAt }), + } + ) + await expect(reader.resolve('org_a')).rejects.toThrow('CONFIGURATION_UNAVAILABLE') + expect(await reader.resolve('org_other')).toEqual({ kind: 'direct' }) + expect(await reader.resolve(null)).toEqual({ kind: 'direct' }) + } + }) + + it('never reads AppConfig for organizations without a reservation', async () => { + const readSnapshot = vi.fn().mockRejectedValue(new Error('AppConfig unavailable')) + const reader = createOutboundRoutingReader( + { ...options, source: 'appconfig', configuration: undefined }, + { ...dependencies, readSnapshot } + ) + expect(await reader.resolve('org_other')).toEqual({ kind: 'direct' }) + expect(await reader.resolve(null)).toEqual({ kind: 'direct' }) + expect(readSnapshot).not.toHaveBeenCalled() + }) + + it('keeps a still-valid snapshot and re-resolves current routing for each operation', async () => { + let current: unknown = document + const reader = createOutboundRoutingReader( + { ...options, source: 'appconfig', configuration: undefined }, + { + ...dependencies, + readSnapshot: async (parse) => ({ value: parse(current), validatedAt: 350_000 }), + } + ) + expect(await reader.resolve('org_a')).toMatchObject({ kind: 'gateway' }) + current = { + ...document, + revision: 'revision-2', + organizations: { org_a: { kind: 'blocked' } }, + } + await expect(reader.resolve('org_a')).rejects.toThrow('ROUTE_BLOCKED') + }) +}) diff --git a/apps/sim/lib/core/network/config.server.ts b/apps/sim/lib/core/network/config.server.ts new file mode 100644 index 00000000000..2cf6c959fcf --- /dev/null +++ b/apps/sim/lib/core/network/config.server.ts @@ -0,0 +1,251 @@ +import { isIP } from 'node:net' +import { z } from 'zod' +import type { AppConfigSnapshot } from '@/lib/core/config/appconfig' +import { env } from '@/lib/core/config/env' +import { gatewayPublicMetadataSchema } from '@/lib/core/network/gateway-metadata' +import { + type OutboundRoutingConfig, + OutboundRoutingError, + parseOutboundJson, + parseOutboundRoutingConfig, + selectOutboundRoute, +} from '@/lib/core/network/routing' + +const MAX_STALE_MS = 300_000 +const keySchema = z + .string() + .min(1) + .max(128) + .regex(/^[a-zA-Z0-9_-]+$/) +const gatewaySchema = z.strictObject({ + organizationId: keySchema, + url: z.string().url().max(2048), + servername: z + .string() + .min(1) + .max(253) + .regex(/^[a-zA-Z0-9.-]+$/) + .optional(), + credentialId: keySchema, + generation: keySchema, + ...gatewayPublicMetadataSchema.partial().shape, +}) +const gatewaysSchema = z + .record(keySchema, gatewaySchema) + .refine((value) => Object.keys(value).length <= 100) +const credentialSchema = z.strictObject({ + token: z + .string() + .min(32) + .max(4096) + .regex(/^[a-zA-Z0-9_=-]+$/), + ca: z.string().min(1).max(65_536).optional(), +}) +const credentialsSchema = z + .record(keySchema, credentialSchema) + .refine((value) => Object.keys(value).length <= 100) + +export interface OutboundGateway { + readonly id: string + readonly organizationId: string + readonly url: string + readonly servername: string + readonly generation: string + readonly token: string + readonly ca?: string + readonly publicIps?: readonly string[] +} + +export type ResolvedOutboundRoute = + | { readonly kind: 'direct' } + | { + readonly kind: 'gateway' + readonly gateway: OutboundGateway + } + +interface OutboundConfigurationOptions { + source?: string + configuration?: string + gateways?: string + credentials?: string +} + +interface OutboundConfigurationDependencies { + readSnapshot( + parse: (value: unknown) => OutboundRoutingConfig + ): Promise> + now(): number +} + +/** A configured reader is immutable; failures cannot switch it to deployment defaults. */ +export function createOutboundRoutingReader( + options: OutboundConfigurationOptions, + dependencies: OutboundConfigurationDependencies +): { + enabled: boolean + resolve(organizationId: string | null | undefined): Promise +} { + if (options.source === undefined) { + if ( + options.configuration !== undefined || + options.gateways !== undefined || + options.credentials !== undefined + ) { + throw new OutboundRoutingError('INVALID_CONFIGURATION') + } + return { enabled: false, resolve: async () => ({ kind: 'direct' }) } + } + if (options.source !== 'env' && options.source !== 'appconfig') { + throw new OutboundRoutingError('INVALID_CONFIGURATION') + } + const parsedGateways = gatewaysSchema.safeParse(parseOutboundJson(options.gateways ?? '{}')) + const parsedCredentials = credentialsSchema.safeParse( + parseOutboundJson(options.credentials ?? '{}') + ) + if (!parsedGateways.success || !parsedCredentials.success) { + throw new OutboundRoutingError('INVALID_CONFIGURATION') + } + const gateways = new Map() + const managedOrganizations = new Set() + const endpointOwners = new Map() + const credentialOwners = new Map() + const addressOwners = new Map() + for (const [id, entry] of Object.entries(parsedGateways.data)) { + const url = new URL(entry.url) + const hostname = url.hostname.replace(/^\[|\]$/g, '') + if ( + url.protocol !== 'https:' || + url.username || + url.password || + url.pathname !== '/' || + url.search || + url.hash + ) { + throw new OutboundRoutingError('INVALID_CONFIGURATION') + } + const servername = entry.servername ?? hostname + if ( + isIP(servername) || + !servername || + !Object.hasOwn(parsedCredentials.data, entry.credentialId) + ) { + throw new OutboundRoutingError('INVALID_CONFIGURATION') + } + const credential = parsedCredentials.data[entry.credentialId] + for (const [owners, key] of [ + [endpointOwners, url.href], + [credentialOwners, credential.token], + ...(entry.publicIps ?? []).map((address) => [addressOwners, address] as const), + ] as const) { + const owner = owners.get(key) + if (owner !== undefined && owner !== entry.organizationId) { + throw new OutboundRoutingError('INVALID_CONFIGURATION') + } + owners.set(key, entry.organizationId) + } + managedOrganizations.add(entry.organizationId) + gateways.set( + id, + Object.freeze({ + id, + organizationId: entry.organizationId, + url: url.href, + servername, + generation: entry.generation, + ...credential, + ...(entry.publicIps ? { publicIps: Object.freeze([...new Set(entry.publicIps)]) } : {}), + }) + ) + } + const parse = (value: unknown): OutboundRoutingConfig => { + const config = parseOutboundRoutingConfig(value) + for (const [organizationId, route] of Object.entries(config.organizations)) { + if ( + route.kind === 'gateway' && + gateways.get(route.gatewayId)?.organizationId !== organizationId + ) { + throw new OutboundRoutingError('INVALID_CONFIGURATION') + } + } + return config + } + const staticConfig = + options.source === 'env' ? parse(parseOutboundJson(options.configuration ?? '')) : null + if (options.source === 'appconfig' && options.configuration !== undefined) { + throw new OutboundRoutingError('INVALID_CONFIGURATION') + } + return { + enabled: true, + async resolve(organizationId) { + if (organizationId === undefined) throw new OutboundRoutingError('MISSING_SCOPE') + if (organizationId === null || !managedOrganizations.has(organizationId)) { + return { kind: 'direct' } + } + let config = staticConfig + if (!config) { + const snapshot = await dependencies.readSnapshot(parse) + const age = + snapshot.validatedAt === null + ? Number.POSITIVE_INFINITY + : dependencies.now() - snapshot.validatedAt + if (!snapshot.value || age < 0 || age >= MAX_STALE_MS) { + throw new OutboundRoutingError('CONFIGURATION_UNAVAILABLE') + } + config = snapshot.value + } + /** Removing an assignment must not release a reserved organization onto shared egress. */ + if (!Object.hasOwn(config.organizations, organizationId)) { + throw new OutboundRoutingError('ROUTE_BLOCKED') + } + const route = selectOutboundRoute(config, organizationId) + if (route.kind === 'direct') return { kind: 'direct' } + if (route.kind === 'blocked') throw new OutboundRoutingError('ROUTE_BLOCKED') + const gateway = gateways.get(route.gatewayId) + if (!gateway) throw new OutboundRoutingError('INVALID_CONFIGURATION') + return { + kind: 'gateway', + gateway, + } + }, + } +} + +let reader: ReturnType | undefined + +function getReader() { + reader ??= createOutboundRoutingReader( + { + source: env.OUTBOUND_ROUTING_SOURCE, + configuration: env.OUTBOUND_ROUTING_CONFIG, + gateways: env.OUTBOUND_GATEWAYS, + credentials: env.OUTBOUND_GATEWAY_CREDENTIALS, + }, + { + now: Date.now, + async readSnapshot(parse) { + if (!env.APPCONFIG_APPLICATION || !env.APPCONFIG_ENVIRONMENT) { + throw new OutboundRoutingError('CONFIGURATION_UNAVAILABLE') + } + const { fetchAppConfigSnapshot } = await import('@/lib/core/config/appconfig') + return fetchAppConfigSnapshot( + { + application: env.APPCONFIG_APPLICATION, + environment: env.APPCONFIG_ENVIRONMENT, + profile: 'outbound-routing', + }, + parse + ) + }, + } + ) + return reader +} + +export function isOutboundRoutingEnabled(): boolean { + return getReader().enabled +} + +/** Resolved gateways contain credentials; never serialize them. */ +export function resolveOutboundRoute(organizationId: string | null | undefined) { + return getReader().resolve(organizationId) +} diff --git a/apps/sim/lib/core/network/context.server.test.ts b/apps/sim/lib/core/network/context.server.test.ts new file mode 100644 index 00000000000..f8dff63f90a --- /dev/null +++ b/apps/sim/lib/core/network/context.server.test.ts @@ -0,0 +1,46 @@ +/** @vitest-environment node */ +import { describe, expect, it, vi } from 'vitest' + +const { resolve } = vi.hoisted(() => ({ + resolve: vi.fn(async (organizationId: string | null | undefined) => ({ + kind: 'direct', + organizationId, + })), +})) +vi.mock('@/lib/core/network/config.server', () => ({ + resolveOutboundRoute: resolve, + isOutboundRoutingEnabled: () => true, +})) + +import { + resolveCurrentOutboundRoute, + runWithOutboundOrganization, +} from '@/lib/core/network/context.server' + +describe('outbound execution context', () => { + it('isolates interleaved organizations and restores the parent after nested calls', async () => { + let release: () => void = () => {} + const gate = new Promise((done) => { + release = done + }) + const first = runWithOutboundOrganization('org_a', async () => { + await gate + await resolveCurrentOutboundRoute() + await runWithOutboundOrganization('org_child', resolveCurrentOutboundRoute) + return resolveCurrentOutboundRoute() + }) + const second = runWithOutboundOrganization('org_b', async () => { + await resolveCurrentOutboundRoute() + release() + return resolveCurrentOutboundRoute() + }) + expect(await Promise.all([first, second])).toEqual([ + { kind: 'direct', organizationId: 'org_a' }, + { kind: 'direct', organizationId: 'org_b' }, + ]) + expect(await resolveCurrentOutboundRoute()).toEqual({ + kind: 'direct', + organizationId: undefined, + }) + }) +}) diff --git a/apps/sim/lib/core/network/context.server.ts b/apps/sim/lib/core/network/context.server.ts new file mode 100644 index 00000000000..c91ad01e1a3 --- /dev/null +++ b/apps/sim/lib/core/network/context.server.ts @@ -0,0 +1,32 @@ +import { AsyncLocalStorage } from 'node:async_hooks' +import { isOutboundRoutingEnabled, resolveOutboundRoute } from '@/lib/core/network/config.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' + +interface OutboundScope { + readonly organizationId: string | null +} + +const storage = new AsyncLocalStorage() + +/** + * Established by authorized operations or jobs after loading canonical ownership. + * Null means a verified personal workspace or platform operation. Never pass a requested organization. + */ +export function runWithOutboundOrganization(organizationId: string | null, run: () => T): T { + if (!isOutboundRoutingEnabled()) return run() + if (organizationId !== null && (typeof organizationId !== 'string' || !organizationId)) { + throw new OutboundRoutingError('MISSING_SCOPE') + } + return storage.run(Object.freeze({ organizationId }), run) +} + +/** Resolves current policy per operation, rather than freezing policy for a long-running job. */ +export function resolveCurrentOutboundRoute() { + return resolveOutboundRoute(storage.getStore()?.organizationId) +} + +/** Captures only the outbound scope for deferred callbacks; route policy is still read per call. */ +export function captureOutboundScope(): (run: () => T) => T { + const scope = storage.getStore() + return (run: () => T): T => (scope ? storage.run(scope, run) : storage.exit(run)) +} diff --git a/apps/sim/lib/core/network/fixtures/gateway-runtime.fixture.ts b/apps/sim/lib/core/network/fixtures/gateway-runtime.fixture.ts new file mode 100644 index 00000000000..7f27a8ef76c --- /dev/null +++ b/apps/sim/lib/core/network/fixtures/gateway-runtime.fixture.ts @@ -0,0 +1,54 @@ +/** Real runtime probe invoked by gateway.server.test.ts against its local TLS servers. */ +import { readFileSync } from 'node:fs' +import { request as undiciRequest } from 'undici/index.js' +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' +import { createGatewayDispatcher } from '@/lib/core/network/gateway.server' +import { secureFetchWithPinnedIP } from '@/lib/core/security/input-validation.server' + +const [proxyPort, originPort, certificatePath] = process.argv.slice(2) +if (!proxyPort || !originPort || !certificatePath) + throw new Error('Local fixture ports and certificate are required') +const gateway = { + id: 'synthetic', + organizationId: 'org_test', + url: `https://127.0.0.1:${proxyPort}`, + servername: 'gateway.invalid', + generation: 'test', + token: 'a'.repeat(48), + ca: readFileSync(certificatePath, 'utf8'), +} +const dispatcher = createGatewayDispatcher(gateway, { + profile: 'selfHostedService', + resolvedIP: '1.1.1.1', +}) +try { + const response = await undiciRequest('https://origin.invalid', { + dispatcher, + signal: AbortSignal.timeout(10_000), + }) + if (response.headers['x-via-proxy'] !== 'yes') throw new Error('Undici bypassed proxy') + if ((await response.body.text()) !== 'tls reached') throw new Error('Undici response mismatch') + + await runWithOutboundOrganization('org_test', async () => { + const pinned = await secureFetchWithPinnedIP('https://origin.invalid', '1.1.1.1', { + profile: 'selfHostedService', + timeout: 10_000, + }) + if (pinned.headers.get('x-via-proxy') !== 'yes') throw new Error('Pinned fetch bypassed proxy') + if ((await pinned.text()) !== 'tls reached') throw new Error('Pinned fetch response mismatch') + }) + + let rejected = false + try { + await undiciRequest('https://wrong.invalid', { + dispatcher, + headers: { host: 'origin.invalid' }, + signal: AbortSignal.timeout(10_000), + }) + } catch { + rejected = true + } + if (!rejected) throw new Error('Mismatched upstream certificate was accepted') +} finally { + await dispatcher.destroy() +} diff --git a/apps/sim/lib/core/network/gateway-metadata.ts b/apps/sim/lib/core/network/gateway-metadata.ts new file mode 100644 index 00000000000..d7b82b7f820 --- /dev/null +++ b/apps/sim/lib/core/network/gateway-metadata.ts @@ -0,0 +1,6 @@ +import { z } from 'zod' + +/** Public operator-published metadata; never includes transport or credential configuration. */ +export const gatewayPublicMetadataSchema = z.object({ + publicIps: z.array(z.ipv4()).max(16), +}) diff --git a/apps/sim/lib/core/network/gateway.server.test.ts b/apps/sim/lib/core/network/gateway.server.test.ts new file mode 100644 index 00000000000..bd746fabb59 --- /dev/null +++ b/apps/sim/lib/core/network/gateway.server.test.ts @@ -0,0 +1,339 @@ +/** @vitest-environment node */ +import { execFile } from 'node:child_process' +import { mkdtempSync, readFileSync, rmSync } from 'node:fs' +import { createServer as httpsServer } from 'node:https' +import { type AddressInfo, connect, type Socket } from 'node:net' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { getCACertificates, setDefaultCACertificates } from 'node:tls' +import { fileURLToPath } from 'node:url' +import { promisify } from 'node:util' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' +import type { OutboundGateway, ResolvedOutboundRoute } from '@/lib/core/network/config.server' + +const { state } = vi.hoisted(() => ({ + state: { gateways: new Map() }, +})) +vi.mock('@/lib/core/network/config.server', () => ({ + isOutboundRoutingEnabled: () => true, + async resolveOutboundRoute( + organizationId: string | null | undefined + ): Promise { + const gateway = organizationId ? state.gateways.get(organizationId) : undefined + if (!gateway || !organizationId) throw new Error('No configured route') + return { kind: 'gateway', gateway } + }, +})) + +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' +import { openGatewayTunnel, secureOutboundTunnel } from '@/lib/core/network/gateway.server' +import { + createPinnedFetchWithDispatcher, + secureFetchWithPinnedIP, +} from '@/lib/core/security/input-validation.server' + +const certificateDirectory = mkdtempSync(join(tmpdir(), 'gateway-tls-')) +const certificatePath = join(certificateDirectory, 'cert.pem') +const keyPath = join(certificateDirectory, 'key.pem') +let cert = '' +const sockets = new Set() +const admissions: Array<{ + token: string | undefined + destination: string | undefined + sni: string | undefined +}> = [] +let originRequests = 0 +let receivedHeaders: Record = {} +const proxiedPorts = new Set() +const secureOrigin = httpsServer(async (req, res) => { + res.setHeader('x-via-proxy', proxiedPorts.has(req.socket.remotePort ?? 0) ? 'yes' : 'no') + originRequests++ + receivedHeaders = req.headers + if (req.url === '/redirect') { + res.writeHead(302, { location: '/done' }) + res.end() + return + } + if (req.url === '/wait') return + const chunks: Buffer[] = [] + for await (const chunk of req) chunks.push(Buffer.from(chunk)) + res.end(req.method === 'POST' ? Buffer.concat(chunks) : 'tls reached') +}) +const proxy = httpsServer() +proxy.on('connect', (req, socket, head) => { + socket.on('error', () => {}) + const tlsSocket = req.socket as Socket & { servername?: string } + admissions.push({ + token: req.headers['proxy-authorization'], + destination: req.url, + sni: tlsSocket.servername, + }) + if (!req.headers['proxy-authorization']?.match(/^Bearer (alpha|bravo|a{48})$/)) { + socket.end('HTTP/1.1 407 Proxy Authentication Required\r\nContent-Length: 0\r\n\r\n') + return + } + if (req.url !== '1.1.1.1:443') { + socket.end('HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\n\r\n') + return + } + const upstream = connect(securePort, '127.0.0.1', () => { + if (upstream.localPort) proxiedPorts.add(upstream.localPort) + socket.write('HTTP/1.1 200 Connection Established\r\n\r\n') + if (head.length) upstream.write(head) + socket.pipe(upstream).pipe(socket) + }) + upstream.on('error', () => socket.destroy()) + socket.on('error', () => upstream.destroy()) + socket.on('close', () => upstream.destroy()) + sockets.add(upstream) + upstream.on('close', () => sockets.delete(upstream)) +}) +let securePort = 0 +const trust = getCACertificates('default') +beforeAll(async () => { + await promisify(execFile)('openssl', [ + 'req', + '-x509', + '-newkey', + 'rsa:2048', + '-sha256', + '-nodes', + '-keyout', + keyPath, + '-out', + certificatePath, + '-days', + '2', + '-subj', + '/CN=gateway.invalid', + '-addext', + 'subjectAltName=DNS:gateway.invalid,DNS:origin.invalid,IP:1.1.1.1', + '-addext', + 'extendedKeyUsage=serverAuth', + ]) + cert = readFileSync(certificatePath, 'utf8') + const key = readFileSync(keyPath, 'utf8') + for (const server of [secureOrigin, proxy]) server.setSecureContext({ cert, key }) + setDefaultCACertificates([...trust, cert]) + for (const server of [secureOrigin, proxy]) { + server.on('connection', (socket) => { + sockets.add(socket) + socket.on('close', () => sockets.delete(socket)) + }) + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)) + } + securePort = (secureOrigin.address() as AddressInfo).port + const proxyPort = (proxy.address() as AddressInfo).port + for (const [id, token] of [ + ['org_a', 'alpha'], + ['org_b', 'bravo'], + ]) + state.gateways.set(id, { + id, + organizationId: id, + token, + url: `https://127.0.0.1:${proxyPort}`, + servername: 'gateway.invalid', + generation: '1', + ca: cert, + }) +}) +afterAll(async () => { + for (const socket of sockets) socket.destroy() + await Promise.all( + [secureOrigin, proxy].map( + (server) => new Promise((resolve) => server.close(() => resolve())) + ) + ) + setDefaultCACertificates(trust) + rmSync(certificateDirectory, { recursive: true, force: true }) +}) +const url = () => 'https://origin.invalid' +const options = { profile: 'selfHostedService' as const } + +describe('organization gateways over real TLS CONNECT sockets', () => { + it('keeps an upgraded tunnel usable after five minutes', async () => { + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }) + let tunnel: Socket | undefined + try { + tunnel = await secureOutboundTunnel( + await openGatewayTunnel(state.gateways.get('org_a')!, '1.1.1.1', 443), + 'origin.invalid', + 443 + ) + await vi.advanceTimersByTimeAsync(6 * 60_000) + expect(tunnel.destroyed).toBe(false) + tunnel.write('GET /done HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n') + const chunks: Buffer[] = [] + for await (const chunk of tunnel) chunks.push(Buffer.from(chunk)) + expect(Buffer.concat(chunks).toString()).toContain('reached') + } finally { + tunnel?.destroy() + vi.useRealTimers() + } + }) + + it('pins guarded requests and keeps proxy credentials out of origin headers', async () => { + const response = await runWithOutboundOrganization('org_a', () => + secureFetchWithPinnedIP(url(), '1.1.1.1', { + ...options, + headers: { 'Proxy-Authorization': 'must-not-reach-origin' }, + }) + ) + expect(await response.text()).toBe('tls reached') + expect(admissions.at(-1)).toEqual({ + token: 'Bearer alpha', + destination: '1.1.1.1:443', + sni: 'gateway.invalid', + }) + expect(receivedHeaders['proxy-authorization']).toBeUndefined() + expect(receivedHeaders.host).toBe('origin.invalid') + }) + it('isolates concurrent requests to the same origin and rotates pools on credential generation change', async () => { + const transport = createPinnedFetchWithDispatcher('1.1.1.1', options) + const start = admissions.length + try { + expect( + await Promise.all( + ['org_a', 'org_b'].map((id) => + runWithOutboundOrganization(id, async () => (await transport.fetch(url())).text()) + ) + ) + ).toEqual(['tls reached', 'tls reached']) + expect( + admissions + .slice(start) + .map((item) => item.token) + .sort() + ).toEqual(['Bearer alpha', 'Bearer bravo']) + state.gateways.set('org_a', { ...state.gateways.get('org_a')!, generation: '2' }) + await runWithOutboundOrganization('org_a', async () => (await transport.fetch(url())).text()) + expect(admissions.length).toBe(start + 3) + } finally { + await transport.dispatcher.destroy() + } + }) + it('preserves upstream TLS identity on both transports and rejects the wrong hostname', async () => { + const transport = createPinnedFetchWithDispatcher('1.1.1.1', options) + try { + await runWithOutboundOrganization('org_a', async () => { + expect(await (await transport.fetch(url())).text()).toBe('tls reached') + expect(await (await secureFetchWithPinnedIP(url(), '1.1.1.1', options)).text()).toBe( + 'tls reached' + ) + await expect(transport.fetch('https://wrong.invalid')).rejects.toThrow() + await expect( + transport.fetch('https://wrong.invalid', { + headers: { host: 'origin.invalid' }, + }) + ).rejects.toThrow() + await expect( + secureFetchWithPinnedIP('https://wrong.invalid', '1.1.1.1', options) + ).rejects.toThrow() + }) + } finally { + await transport.dispatcher.destroy() + } + }) + it('preserves Request bodies, encodes multipart data, and honors manual redirects', async () => { + const transport = createPinnedFetchWithDispatcher('1.1.1.1', options) + try { + await runWithOutboundOrganization('org_a', async () => { + expect( + await ( + await transport.fetch(new Request(url(), { method: 'POST', body: 'request body' })) + ).text() + ).toBe('request body') + const form = new FormData() + form.set('field', 'multipart body') + expect( + await (await transport.fetch(url(), { method: 'POST', body: form })).text() + ).toContain('multipart body') + const redirect = await transport.fetch(`${url()}/redirect`, { redirect: 'manual' }) + expect(redirect.status).toBe(302) + await redirect.body?.cancel() + }) + } finally { + await transport.dispatcher.destroy() + } + }) + it('fails closed on denied proxy credentials without contacting the origin', async () => { + state.gateways.set('denied', { + ...state.gateways.get('org_a')!, + organizationId: 'denied', + token: 'denied', + }) + const transport = createPinnedFetchWithDispatcher('1.1.1.1', options) + const start = originRequests + try { + await runWithOutboundOrganization('denied', async () => { + await expect(transport.fetch(url())).rejects.toThrow() + await expect(secureFetchWithPinnedIP(url(), '1.1.1.1', options)).rejects.toThrow() + }) + expect(originRequests).toBe(start) + } finally { + await transport.dispatcher.destroy() + } + }) + it('routes and verifies TLS under the actual Bun runtime', async () => { + const start = admissions.length + await promisify(execFile)( + 'bun', + [ + '--no-env-file', + 'run', + fileURLToPath(new URL('./fixtures/gateway-runtime.fixture.ts', import.meta.url)), + String((proxy.address() as AddressInfo).port), + String(securePort), + certificatePath, + ], + { + timeout: 15_000, + env: { + ...process.env, + /** This local tunnel fixture must not inherit hosted-mode loopback restrictions. */ + NEXT_PUBLIC_APP_URL: 'http://localhost:3000', + NEXT_PUBLIC_FORCE_HOSTED: 'false', + NODE_EXTRA_CA_CERTS: certificatePath, + OUTBOUND_ROUTING_SOURCE: 'env', + OUTBOUND_ROUTING_CONFIG: JSON.stringify({ + schemaVersion: 1, + revision: 'test', + defaultRoute: { kind: 'blocked' }, + organizations: { org_test: { kind: 'gateway', gatewayId: 'synthetic' } }, + }), + OUTBOUND_GATEWAYS: JSON.stringify({ + synthetic: { + organizationId: 'org_test', + url: `https://127.0.0.1:${(proxy.address() as AddressInfo).port}`, + servername: 'gateway.invalid', + credentialId: 'synthetic', + generation: 'test', + }, + }), + OUTBOUND_GATEWAY_CREDENTIALS: JSON.stringify({ + synthetic: { token: 'a'.repeat(48), ca: cert }, + }), + }, + } + ) + expect(admissions.length).toBe(start + 3) + expect( + admissions.slice(start).every((entry) => entry.token === `Bearer ${'a'.repeat(48)}`) + ).toBe(true) + }) + + it('supports cancellation without a direct retry', async () => { + const transport = createPinnedFetchWithDispatcher('1.1.1.1', options) + const controller = new AbortController() + try { + const promise = runWithOutboundOrganization('org_a', () => + transport.fetch(`${url()}/wait`, { signal: controller.signal }) + ) + controller.abort() + await expect(promise).rejects.toThrow() + } finally { + await transport.dispatcher.destroy() + } + }) +}) diff --git a/apps/sim/lib/core/network/gateway.server.ts b/apps/sim/lib/core/network/gateway.server.ts new file mode 100644 index 00000000000..0c32542ae16 --- /dev/null +++ b/apps/sim/lib/core/network/gateway.server.ts @@ -0,0 +1,136 @@ +import { isIP, Socket } from 'node:net' +import { checkServerIdentity, connect as connectTls } from 'node:tls' +import { evaluateAddress, STRICT_EGRESS_POLICY } from '@sim/security/egress' +import { Agent, Client } from 'undici/index.js' +import type { OutboundGateway } from '@/lib/core/network/config.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import type { EgressProfile } from '@/lib/core/security/egress/profiles' +import { checkResolvedEgress, validateEgressUrl } from '@/lib/core/security/egress/validate' + +const CONNECT_TIMEOUT_MS = 10_000 + +/** The proxy CA never changes trust for the upstream service. */ +export async function secureOutboundTunnel( + socket: Socket, + hostname: string, + port: number +): Promise { + return new Promise((resolve, reject) => { + const tls = connectTls({ + socket, + host: hostname, + port, + servername: isIP(hostname) ? undefined : hostname, + checkServerIdentity: (_name, certificate) => checkServerIdentity(hostname, certificate), + rejectUnauthorized: true, + ALPNProtocols: ['http/1.1'], + }) + /** The inner TLS stream has no TCP descriptor; QoS must be applied to the outer socket. */ + if ('setTypeOfService' in socket && typeof socket.setTypeOfService === 'function') { + const setTypeOfService = socket.setTypeOfService.bind(socket) + Object.defineProperty(tls, 'setTypeOfService', { + value(tos: number) { + setTypeOfService(tos) + return tls + }, + }) + } + const timer = setTimeout( + () => tls.destroy(new OutboundRoutingError('GATEWAY_UNAVAILABLE')), + CONNECT_TIMEOUT_MS + ) + timer.unref() + tls.once('secureConnect', () => { + clearTimeout(timer) + resolve(tls) + }) + tls.once('error', () => { + clearTimeout(timer) + reject(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + }) + }) +} + +/** Uses the npm HTTP parser on both runtimes; Bun's Node HTTP shim cannot send CONNECT. */ +export async function openGatewayTunnel( + gateway: OutboundGateway, + address: string, + port: number +): Promise { + if ( + isIP(address) !== 4 || + port !== 443 || + !evaluateAddress(new URL(`https://${address}`), address, STRICT_EGRESS_POLICY).allowed + ) { + throw new OutboundRoutingError('UNSUPPORTED_TRANSPORT') + } + const authority = `${address}:443` + const client = new Client(gateway.url, { + allowH2: false, + maxHeaderSize: 16_384, + connectTimeout: CONNECT_TIMEOUT_MS, + connect: { servername: gateway.servername, ca: gateway.ca, rejectUnauthorized: true }, + }) + try { + const { statusCode, socket } = await client.connect({ + path: authority, + signal: AbortSignal.timeout(CONNECT_TIMEOUT_MS), + headers: { host: authority, 'proxy-authorization': `Bearer ${gateway.token}` }, + }) + if (statusCode !== 200 || !(socket instanceof Socket)) { + socket.destroy() + throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') + } + return socket + } catch { + throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') + } finally { + /** An upgraded socket is detached from the dispatcher and now belongs to its caller. */ + await client.destroy() + } +} + +/** A dispatcher belongs to one organization and gateway generation, never a process-wide default. */ +export function createGatewayDispatcher( + gateway: OutboundGateway, + options: { profile: EgressProfile; resolvedIP?: string; maxResponseSize?: number } +): Agent { + return new Agent({ + allowH2: false, + ...(options.maxResponseSize !== undefined ? { maxResponseSize: options.maxResponseSize } : {}), + connect(connection, callback) { + const port = Number(connection.port || (connection.protocol === 'https:' ? 443 : 80)) + const hostname = connection.hostname.replace(/^\[|\]$/g, '') + const authority = isIP(hostname) === 6 ? `[${hostname}]` : hostname + const url = new URL(`${connection.protocol}//${authority}:${port}`) + void (async () => { + if (connection.protocol !== 'https:' || port !== 443) { + throw new OutboundRoutingError('UNSUPPORTED_TRANSPORT') + } + let address = options.resolvedIP + if (address) { + if (!checkResolvedEgress(url, address, options.profile).allowed) { + throw new OutboundRoutingError('ROUTE_BLOCKED') + } + } else { + const result = await validateEgressUrl(url.href, 'url', options.profile, { + logDetails: false, + }) + if (!result.isValid) throw new OutboundRoutingError('ROUTE_BLOCKED') + address = result.resolvedIP + } + const socket = await openGatewayTunnel(gateway, address, port) + return secureOutboundTunnel(socket, hostname, port) + })().then( + (socket) => callback(null, socket), + (error) => + callback( + error instanceof OutboundRoutingError + ? error + : new OutboundRoutingError('GATEWAY_UNAVAILABLE'), + null + ) + ) + }, + }) +} diff --git a/apps/sim/lib/core/network/resource-scope.server.test.ts b/apps/sim/lib/core/network/resource-scope.server.test.ts new file mode 100644 index 00000000000..20126ad7ef1 --- /dev/null +++ b/apps/sim/lib/core/network/resource-scope.server.test.ts @@ -0,0 +1,100 @@ +/** @vitest-environment node */ +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + enabled: vi.fn(() => true), + workspace: vi.fn(), + route: vi.fn(async (organizationId: string | null | undefined) => ({ organizationId })), +})) +vi.mock('@/lib/core/network/config.server', () => ({ + isOutboundRoutingEnabled: mocks.enabled, + resolveOutboundRoute: mocks.route, +})) +vi.mock('@/lib/workspaces/application/workspace-context', () => ({ + loadWorkspaceApplicationContext: mocks.workspace, +})) + +import { + resolveCurrentOutboundRoute, + runWithOutboundOrganization, +} from '@/lib/core/network/context.server' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' + +describe('canonical resource outbound scope', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.enabled.mockReturnValue(true) + }) + + it.each(['publisher-org', null])( + 'uses current workspace ownership %s and restores its caller', + async (organizationId) => { + mocks.workspace.mockResolvedValue({ workspaceOrganizationId: organizationId }) + await runWithOutboundOrganization('caller-org', async () => { + expect( + await withResourceOutboundScope( + { workspaceId: 'source-workspace' }, + resolveCurrentOutboundRoute + ) + ).toEqual({ organizationId }) + expect(await resolveCurrentOutboundRoute()).toEqual({ organizationId: 'caller-org' }) + }) + expect(mocks.workspace).toHaveBeenCalledExactlyOnceWith('source-workspace', {}) + } + ) + + it('rejects a missing or archived workspace before executing provider work', async () => { + mocks.workspace.mockResolvedValue(null) + await expect( + withResourceOutboundScope({ workspaceId: 'removed' }, resolveCurrentOutboundRoute) + ).rejects.toMatchObject({ code: 'MISSING_SCOPE' }) + expect(mocks.route).not.toHaveBeenCalled() + }) + + it('uses canonical archived workspace ownership only when cleanup explicitly requests it', async () => { + mocks.workspace.mockImplementation(async (_id, options) => + options.includeArchived ? { workspaceOrganizationId: 'owner-org' } : null + ) + await expect( + withResourceOutboundScope({ workspaceId: 'archived' }, resolveCurrentOutboundRoute) + ).rejects.toMatchObject({ code: 'MISSING_SCOPE' }) + await expect( + withResourceOutboundScope({ workspaceId: 'archived' }, resolveCurrentOutboundRoute, { + includeArchived: true, + }) + ).resolves.toEqual({ organizationId: 'owner-org' }) + }) + + it('uses organization ownership without loading a workspace or retaining caller scope', async () => { + await runWithOutboundOrganization('caller-org', async () => { + expect( + await withResourceOutboundScope( + { workspaceId: null, organizationId: 'owner-org' }, + resolveCurrentOutboundRoute + ) + ).toEqual({ organizationId: 'owner-org' }) + expect(await resolveCurrentOutboundRoute()).toEqual({ organizationId: 'caller-org' }) + }) + expect(mocks.workspace).not.toHaveBeenCalled() + }) + + it('adds no workspace query when routing is unconfigured', async () => { + mocks.enabled.mockReturnValue(false) + const run = vi.fn(async () => 'done') + expect(await withResourceOutboundScope({ workspaceId: null }, run)).toBe('done') + expect(mocks.workspace).not.toHaveBeenCalled() + expect(run).toHaveBeenCalledOnce() + }) + + it('rejects ambiguous or missing ownership when routing is configured', async () => { + const run = vi.fn(async () => 'done') + await expect(withResourceOutboundScope({}, run)).rejects.toThrow( + 'Resource requires exactly one workspace or organization owner' + ) + await expect( + withResourceOutboundScope({ workspaceId: 'workspace', organizationId: 'org' }, run) + ).rejects.toThrow('Resource requires exactly one workspace or organization owner') + expect(run).not.toHaveBeenCalled() + expect(mocks.workspace).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/core/network/resource-scope.server.ts b/apps/sim/lib/core/network/resource-scope.server.ts new file mode 100644 index 00000000000..6694a353a2d --- /dev/null +++ b/apps/sim/lib/core/network/resource-scope.server.ts @@ -0,0 +1,19 @@ +import { isOutboundRoutingEnabled } from '@/lib/core/network/config.server' +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { type ResourceOwner, resourceScopeFromOwner } from '@/lib/core/resource-scope' +import { loadWorkspaceApplicationContext } from '@/lib/workspaces/application/workspace-context' + +/** Establishes routing after resource authorization, reloading current workspace ownership when needed. */ +export async function withResourceOutboundScope( + owner: ResourceOwner, + run: () => Promise, + options: { includeArchived?: boolean } = {} +): Promise { + if (!isOutboundRoutingEnabled()) return run() + const scope = resourceScopeFromOwner(owner) + if (scope.kind === 'organization') return runWithOutboundOrganization(scope.organizationId, run) + const workspace = await loadWorkspaceApplicationContext(scope.workspaceId, options) + if (!workspace) throw new OutboundRoutingError('MISSING_SCOPE') + return runWithOutboundOrganization(workspace.workspaceOrganizationId, run) +} diff --git a/apps/sim/lib/core/network/routing.test.ts b/apps/sim/lib/core/network/routing.test.ts new file mode 100644 index 00000000000..0324764837e --- /dev/null +++ b/apps/sim/lib/core/network/routing.test.ts @@ -0,0 +1,48 @@ +/** @vitest-environment node */ +import { describe, expect, it } from 'vitest' +import { + parseOutboundJson, + parseOutboundRoutingConfig, + selectOutboundRoute, +} from '@/lib/core/network/routing' + +const document = { + schemaVersion: 1, + revision: 'revision-1', + defaultRoute: { kind: 'direct' }, + organizations: { org_a: { kind: 'gateway', gatewayId: 'gateway_a' }, org_b: { kind: 'blocked' } }, +} + +describe('outbound routing policy', () => { + it('selects exact organization bindings, including a deliberate stop', () => { + const policy = parseOutboundRoutingConfig(document) + expect(selectOutboundRoute(policy, 'org_a')).toEqual({ + kind: 'gateway', + gatewayId: 'gateway_a', + }) + expect(selectOutboundRoute(policy, 'org_c')).toEqual({ kind: 'direct' }) + expect(selectOutboundRoute(policy, null)).toEqual({ kind: 'direct' }) + expect(() => selectOutboundRoute(policy, 'org_b')).toThrow('ROUTE_BLOCKED') + }) + + it('cannot be mutated or inherit a binding from Object.prototype', () => { + const policy = parseOutboundRoutingConfig(document) + expect(Object.isFrozen(policy.organizations.org_a)).toBe(true) + expect(Object.isFrozen(policy.organizations)).toBe(true) + expect(selectOutboundRoute(policy, 'constructor')).toEqual({ kind: 'direct' }) + }) + + it.each([ + { ...document, schemaVersion: 2 }, + { ...document, organizations: { org_a: { kind: 'gateway' } } }, + { ...document, defaultRoute: { kind: 'gateway', gatewayId: 'a', fallback: true } }, + { ...document, extra: true }, + ])('rejects malformed policy without exposing its content', (input) => { + expect(() => parseOutboundRoutingConfig(input)).toThrow('INVALID_CONFIGURATION') + }) + + it('bounds serialized configuration and does not expose JSON syntax errors', () => { + expect(() => parseOutboundJson('sensitive-value')).toThrow('INVALID_CONFIGURATION') + expect(() => parseOutboundJson(' '.repeat(1_048_577))).toThrow('INVALID_CONFIGURATION') + }) +}) diff --git a/apps/sim/lib/core/network/routing.ts b/apps/sim/lib/core/network/routing.ts new file mode 100644 index 00000000000..4b0d9275d46 --- /dev/null +++ b/apps/sim/lib/core/network/routing.ts @@ -0,0 +1,86 @@ +import { z } from 'zod' + +const MAX_CONFIG_BYTES = 1_048_576 +const identifier = z + .string() + .min(1) + .max(128) + .regex(/^[a-zA-Z0-9_-]+$/) +const routeSchema = z.discriminatedUnion('kind', [ + z.strictObject({ kind: z.literal('direct') }), + z.strictObject({ kind: z.literal('gateway'), gatewayId: identifier }), + z.strictObject({ kind: z.literal('blocked') }), +]) +const configSchema = z.strictObject({ + schemaVersion: z.literal(1), + revision: identifier, + defaultRoute: z.discriminatedUnion('kind', [ + z.strictObject({ kind: z.literal('direct') }), + z.strictObject({ kind: z.literal('blocked') }), + ]), + organizations: z + .record(identifier, routeSchema) + .refine((value) => Object.keys(value).length <= 10_000), +}) + +export type OutboundRoute = Readonly> +export interface OutboundRoutingConfig { + readonly schemaVersion: 1 + readonly revision: string + readonly defaultRoute: Exclude + readonly organizations: Readonly> +} + +export type OutboundRoutingErrorCode = + | 'CONFIGURATION_UNAVAILABLE' + | 'INVALID_CONFIGURATION' + | 'MISSING_SCOPE' + | 'ROUTE_BLOCKED' + | 'UNSUPPORTED_TRANSPORT' + | 'GATEWAY_UNAVAILABLE' + +/** Public error text never includes routing configuration, credentials or destination details. */ +export class OutboundRoutingError extends Error { + constructor(readonly code: OutboundRoutingErrorCode) { + super(`Outbound routing failed: ${code}`) + this.name = 'OutboundRoutingError' + } +} + +/** Bounds operator configuration before parsing and hides secret-bearing parser errors. */ +export function parseOutboundJson(value: string): unknown { + if (Buffer.byteLength(value, 'utf8') > MAX_CONFIG_BYTES) { + throw new OutboundRoutingError('INVALID_CONFIGURATION') + } + try { + return JSON.parse(value) + } catch { + throw new OutboundRoutingError('INVALID_CONFIGURATION') + } +} + +/** Produces an immutable routing snapshot without reading deployment state. */ +export function parseOutboundRoutingConfig(value: unknown): OutboundRoutingConfig { + const parsed = configSchema.safeParse(value) + if (!parsed.success) throw new OutboundRoutingError('INVALID_CONFIGURATION') + for (const route of Object.values(parsed.data.organizations)) Object.freeze(route) + Object.freeze(parsed.data.defaultRoute) + Object.freeze(parsed.data.organizations) + return Object.freeze(parsed.data) +} + +/** A null organization is a verified personal scope; undefined is never an implicit default. */ +export function selectOutboundRoute( + config: OutboundRoutingConfig, + organizationId: string | null +): OutboundRoute { + if (organizationId !== null && !identifier.safeParse(organizationId).success) { + throw new OutboundRoutingError('MISSING_SCOPE') + } + const route = + organizationId !== null && Object.hasOwn(config.organizations, organizationId) + ? config.organizations[organizationId] + : config.defaultRoute + if (route.kind === 'blocked') throw new OutboundRoutingError('ROUTE_BLOCKED') + return route +} diff --git a/apps/sim/lib/core/network/transport.server.test.ts b/apps/sim/lib/core/network/transport.server.test.ts new file mode 100644 index 00000000000..e659612d1d5 --- /dev/null +++ b/apps/sim/lib/core/network/transport.server.test.ts @@ -0,0 +1,109 @@ +/** @vitest-environment node */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { ResolvedOutboundRoute } from '@/lib/core/network/config.server' + +const { resolveRoute, createGateway } = vi.hoisted(() => ({ + resolveRoute: vi.fn<() => Promise>(), + createGateway: vi.fn(), +})) +vi.mock('@/lib/core/network/context.server', () => ({ resolveCurrentOutboundRoute: resolveRoute })) +vi.mock('@/lib/core/network/gateway.server', () => ({ createGatewayDispatcher: createGateway })) + +import { createOutboundTransport } from '@/lib/core/network/transport.server' + +const route = (organizationId = 'org_a', generation = 'v1'): ResolvedOutboundRoute => ({ + kind: 'gateway', + gateway: { + id: `gateway-${organizationId}`, + organizationId, + url: 'https://gateway.invalid', + servername: 'gateway.invalid', + generation, + token: 'synthetic', + }, +}) + +beforeEach(() => { + vi.clearAllMocks() + resolveRoute.mockResolvedValue(route()) + createGateway.mockImplementation(() => ({ + close: vi.fn(async () => {}), + destroy: vi.fn(async () => {}), + })) +}) + +describe('shared outbound transport ownership', () => { + it('rejects an explicit proxy when organization policy requires its gateway', async () => { + const owner = createOutboundTransport({ + profile: 'configuredEndpoint', + proxyUrl: 'http://proxy.invalid', + }) + try { + await expect(owner.selectDispatcher()).rejects.toThrow('UNSUPPORTED_TRANSPORT') + expect(createGateway).not.toHaveBeenCalled() + } finally { + await owner.destroy() + } + }) + + it('isolates organizations, reuses the gateway generation, and disposes pools', async () => { + const owner = createOutboundTransport({ profile: 'configuredEndpoint' }) + const first = await owner.selectDispatcher() + expect(await owner.selectDispatcher()).toBe(first) + resolveRoute.mockResolvedValue(route('org_b')) + expect(await owner.selectDispatcher()).not.toBe(first) + resolveRoute.mockResolvedValue(route('org_a', 'v1')) + expect(await owner.selectDispatcher()).toBe(first) + await owner.destroy() + for (const result of createGateway.mock.results) + expect(result.value.destroy).toHaveBeenCalledOnce() + await expect(owner.selectDispatcher()).rejects.toThrow('GATEWAY_UNAVAILABLE') + }) + + it('delegates explicit direct routes but propagates policy failure without a fallback', async () => { + const owner = createOutboundTransport({ profile: 'configuredEndpoint' }) + resolveRoute.mockResolvedValue({ kind: 'direct' }) + expect(await owner.selectDispatcher()).toBeNull() + resolveRoute.mockRejectedValue(new Error('policy unavailable')) + await expect(owner.selectDispatcher()).rejects.toThrow('policy unavailable') + expect(createGateway).not.toHaveBeenCalled() + }) + + it('refuses to create a pool when its owner is destroyed during policy resolution', async () => { + let resolve: (route: ResolvedOutboundRoute) => void = () => {} + resolveRoute.mockReturnValue( + new Promise((done) => { + resolve = done + }) + ) + const owner = createOutboundTransport({ profile: 'configuredEndpoint' }) + const pending = owner.selectDispatcher() + await owner.destroy() + resolve(route()) + await expect(pending).rejects.toThrow('GATEWAY_UNAVAILABLE') + expect(createGateway).not.toHaveBeenCalled() + }) + + it('bounds credential rotation per organization without blocking other organizations', async () => { + const drain: Array<() => void> = [] + createGateway.mockImplementation(() => ({ + close: vi.fn(() => new Promise((done) => drain.push(done))), + destroy: vi.fn(async () => {}), + })) + const owner = createOutboundTransport({ profile: 'configuredEndpoint' }) + try { + for (let index = 0; index < 3; index++) { + resolveRoute.mockResolvedValue(route('org_a', `v${index}`)) + expect(await owner.selectDispatcher()).not.toBeNull() + } + resolveRoute.mockResolvedValue(route('org_a', 'overflow')) + await expect(owner.selectDispatcher()).rejects.toThrow('GATEWAY_UNAVAILABLE') + expect(createGateway).toHaveBeenCalledTimes(3) + resolveRoute.mockResolvedValue(route('org_b')) + expect(await owner.selectDispatcher()).not.toBeNull() + } finally { + await owner.destroy() + for (const finish of drain) finish() + } + }) +}) diff --git a/apps/sim/lib/core/network/transport.server.ts b/apps/sim/lib/core/network/transport.server.ts new file mode 100644 index 00000000000..6f97b6707b3 --- /dev/null +++ b/apps/sim/lib/core/network/transport.server.ts @@ -0,0 +1,95 @@ +import { type Agent, type Dispatcher, request } from 'undici/index.js' +import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' +import { createGatewayDispatcher } from '@/lib/core/network/gateway.server' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import type { EgressProfile } from '@/lib/core/security/egress/profiles' + +interface OutboundTransportOptions { + profile: EgressProfile + resolvedIP?: string + maxResponseSize?: number + direct?: Agent + proxyUrl?: string +} + +interface OutboundTransportOwner { + /** Null delegates to the adapter's existing direct transport. */ + selectDispatcher(): Promise + close(): Promise + destroy(): Promise +} + +/** + * Owns routing and connection lifetimes for the shared HTTP helpers. Destination provenance + * and optional pinning are immutable for this owner; policy is resolved per operation. + */ +export function createOutboundTransport(options: OutboundTransportOptions): OutboundTransportOwner { + const pools = new Map() + const retired = new Map>() + let closed = false + const allPools = () => [ + ...(options.direct ? [options.direct] : []), + ...[...pools.values()].map((pool) => pool.agent), + ...[...retired.values()].flatMap((agents) => [...agents]), + ] + return { + async selectDispatcher() { + const route = await resolveCurrentOutboundRoute() + if (closed) throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') + if (route.kind === 'direct') return options.direct ?? null + if (options.proxyUrl) throw new OutboundRoutingError('UNSUPPORTED_TRANSPORT') + const owner = route.gateway.organizationId + const identity = JSON.stringify([route.gateway.id, route.gateway.generation]) + const current = pools.get(owner) + if (current?.identity === identity) return current.agent + const draining = retired.get(owner) ?? new Set() + if (draining.size >= 2) throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') + const agent = createGatewayDispatcher(route.gateway, options) + if (current) { + retired.set(owner, draining) + draining.add(current.agent) + void current.agent + .close() + .catch(() => {}) + .finally(() => { + draining.delete(current.agent) + if (draining.size === 0) retired.delete(owner) + }) + } + pools.set(owner, { identity, agent }) + return agent + }, + async close() { + closed = true + await Promise.all(allPools().map((pool) => pool.close())) + pools.clear() + retired.clear() + }, + async destroy() { + closed = true + await Promise.all(allPools().map((pool) => pool.destroy())) + pools.clear() + retired.clear() + }, + } +} + +type OutboundRequestOptions = Omit< + NonNullable[1]>, + 'headers' | 'dispatcher' +> & { + headers?: Record + dispatcher: Dispatcher +} + +/** + * The shared HTTP wire transport. Import the installed package explicitly: Bun's bare + * undici shim ignores dispatchers. Proxy credentials never become destination headers. + */ +export function requestWithOutboundDispatcher(url: string, options: OutboundRequestOptions) { + const headers = { ...options.headers } + for (const name of Object.keys(headers)) { + if (name.toLowerCase() === 'proxy-authorization') delete headers[name] + } + return request(url, { ...options, headers }) +} diff --git a/apps/sim/lib/core/security/guarded-request-fetch.server.test.ts b/apps/sim/lib/core/security/guarded-request-fetch.server.test.ts index 45cc68ff51f..e4ceb5fd380 100644 --- a/apps/sim/lib/core/security/guarded-request-fetch.server.test.ts +++ b/apps/sim/lib/core/security/guarded-request-fetch.server.test.ts @@ -54,6 +54,21 @@ describe('createSsrfGuardedFetchWithDispatcher (undici.request backed)', () => { vi.clearAllMocks() }) + it.each(['manual', 'error'] as const)( + 'checks the initial literal address with redirect mode %s', + async (redirect) => { + const transport = createSsrfGuardedFetchWithDispatcher({ profile: 'contentFetch' }) + try { + await expect(transport.fetch('https://127.0.0.1/', { redirect })).rejects.toThrow( + 'SSRF policy' + ) + expect(mockUndiciRequest).not.toHaveBeenCalled() + } finally { + await transport.dispatcher.destroy() + } + } + ) + it('constructs a Response with the reply status, headers, url, and a streaming body', async () => { mockUndiciRequest.mockResolvedValueOnce( undiciReply( @@ -81,11 +96,28 @@ describe('createSsrfGuardedFetchWithDispatcher (undici.request backed)', () => { expect(mockUndiciRequest).toHaveBeenCalledTimes(1) const [, options] = mockUndiciRequest.mock.calls[0] expect(options.method).toBe('POST') - expect(options.headers).toEqual({ 'content-type': 'application/json' }) + expect(options.headers).toEqual({ 'content-type': 'application/json', 'user-agent': 'undici' }) expect(options.body).toBe('{"jsonrpc":"2.0"}') expect(options.maxRedirections).toBeUndefined() }) + it('preserves an explicitly supplied User-Agent regardless of casing', async () => { + mockUndiciRequest.mockResolvedValueOnce(undiciReply(200, {}, byteStream('ok'))) + const transport = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }) + try { + const response = await transport.fetch('https://api.example.com/data', { + headers: { 'uSeR-aGeNt': 'custom-client/1.0' }, + }) + await response.text() + + expect(mockUndiciRequest.mock.calls[0][1].headers).toEqual({ + 'uSeR-aGeNt': 'custom-client/1.0', + }) + } finally { + await transport.dispatcher.destroy() + } + }) + it('follows a redirect through followRedirectsGuarded and reports the final url', async () => { mockUndiciRequest .mockResolvedValueOnce( @@ -102,6 +134,93 @@ describe('createSsrfGuardedFetchWithDispatcher (undici.request backed)', () => { expect(await response.text()).toBe('final-body') }) + it.each([ + [307, 'POST'], + [308, 'POST'], + [301, 'PUT'], + [302, 'PUT'], + ] as const)('replays a Request body through same-origin %s redirects', async (status, method) => { + const payloads: string[] = [] + mockUndiciRequest.mockImplementation(async (_url, options: { body: Buffer | Readable }) => { + const chunks: Buffer[] = [] + for await (const chunk of options.body instanceof Readable ? options.body : [options.body]) { + chunks.push(Buffer.from(chunk)) + } + payloads.push(Buffer.concat(chunks).toString()) + return payloads.length < 3 + ? undiciReply(status, { location: `/hop-${payloads.length}` }, byteStream('')) + : undiciReply(200, {}, byteStream('done')) + }) + const transport = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }) + try { + const response = await transport.fetch( + new Request('https://api.example.com/start', { + method, + headers: { 'content-type': 'application/json' }, + body: '{"payload":"replay"}', + }) + ) + expect(await response.text()).toBe('done') + expect(payloads).toEqual(Array(3).fill('{"payload":"replay"}')) + expect(response.redirected).toBe(true) + expect(mockUndiciRequest.mock.calls.map(([, options]) => options.method)).toEqual( + Array(3).fill(method) + ) + } finally { + await transport.dispatcher.destroy() + } + }) + + it.each(['manual', 'error'] as const)( + 'keeps Request bodies streaming in %s mode', + async (redirect) => { + const request = new Request('https://api.example.com/upload', { + method: 'POST', + redirect, + body: 'payload', + }) + const transport = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }) + mockUndiciRequest.mockImplementationOnce(async (_url, options: { body: Readable }) => { + expect(options.body).toBeInstanceOf(Readable) + const chunks: Buffer[] = [] + for await (const chunk of options.body) chunks.push(Buffer.from(chunk)) + expect(Buffer.concat(chunks).toString()).toBe('payload') + return undiciReply(200, {}, byteStream('done')) + }) + try { + expect(await (await transport.fetch(request)).text()).toBe('done') + } finally { + await transport.dispatcher.destroy() + } + } + ) + + it('does not read the Request body when init supplies a replacement', async () => { + const request = new Request('https://api.example.com/upload', { + method: 'POST', + body: 'original', + }) + const clone = vi.spyOn(request, 'clone') + const bodyOverride = vi.fn().mockReturnValueOnce('replacement').mockReturnValue(undefined) + mockUndiciRequest.mockResolvedValueOnce(undiciReply(200, {}, byteStream('done'))) + const transport = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }) + try { + const response = await transport.fetch(request, { + get body() { + return bodyOverride() + }, + }) + expect(await response.text()).toBe('done') + expect(mockUndiciRequest.mock.calls[0][1].body).toBe('replacement') + expect(request.bodyUsed).toBe(false) + expect(clone).not.toHaveBeenCalled() + expect(bodyOverride).toHaveBeenCalledTimes(1) + } finally { + await request.body?.cancel() + await transport.dispatcher.destroy() + } + }) + it('supports buffered reads (.json()) through the constructed body', async () => { mockUndiciRequest.mockResolvedValueOnce( undiciReply( @@ -128,7 +247,7 @@ describe('createSsrfGuardedFetchWithDispatcher (undici.request backed)', () => { }) const [, options] = mockUndiciRequest.mock.calls[0] - expect(options.headers).toEqual({ authorization: 'Bearer t' }) + expect(options.headers).toEqual({ authorization: 'Bearer t', 'user-agent': 'undici' }) expect(Buffer.isBuffer(options.body)).toBe(true) expect(Buffer.from(options.body).toString()).toBe('payload') }) @@ -217,16 +336,21 @@ describe('createSsrfGuardedFetchWithDispatcher (undici.request backed)', () => { await expect(response.text()).rejects.toThrow() }) - it('rejects the reader when the source is destroyed without an error (abort/reset)', async () => { - const source = new Readable({ read() {} }) // stays open, never pushes - mockUndiciRequest.mockResolvedValueOnce(undiciReply(200, {}, source)) - const { fetch } = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }) + it.each([undefined, 'gzip'])( + 'rejects the reader when the source is destroyed without an error (encoding: %s)', + async (encoding) => { + const source = new Readable({ read() {} }) // stays open, never pushes + mockUndiciRequest.mockResolvedValueOnce( + undiciReply(200, encoding ? { 'content-encoding': encoding } : {}, source) + ) + const { fetch } = createSsrfGuardedFetchWithDispatcher({ profile: 'configuredEndpoint' }) - const response = await fetch('https://mcp.example.com/hang', { method: 'GET' }) - const reader = response.body!.getReader() - const read = reader.read() - source.destroy() // no error argument — mirrors an aborted/reset socket + const response = await fetch('https://mcp.example.com/hang', { method: 'GET' }) + const reader = response.body!.getReader() + const read = reader.read() + source.destroy() // no error argument — mirrors an aborted/reset socket - await expect(read).rejects.toThrow(/closed before completing/) - }) + await expect(read).rejects.toThrow(/closed before completing/) + } + ) }) diff --git a/apps/sim/lib/core/security/input-validation.server.ts b/apps/sim/lib/core/security/input-validation.server.ts index 86891fa1e6e..24197066dd5 100644 --- a/apps/sim/lib/core/security/input-validation.server.ts +++ b/apps/sim/lib/core/security/input-validation.server.ts @@ -14,9 +14,14 @@ import { HttpsProxyAgent } from 'https-proxy-agent' import { Agent, type Dispatcher, + errors, type RequestInit as UndiciRequestInit, - request as undiciRequest, -} from 'undici' +} from 'undici/index.js' +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { + createOutboundTransport, + requestWithOutboundDispatcher, +} from '@/lib/core/network/transport.server' import { describeEgressDenial, type EgressProfile } from '@/lib/core/security/egress/profiles' import { checkEgressUrl, @@ -403,6 +408,7 @@ export interface SecureFetchResponse { } const DEFAULT_MAX_REDIRECTS = 5 +const DEFAULT_USER_AGENT = 'undici' /** * Fail-safe ceiling applied by {@link secureFetchWithPinnedIP} when the caller does not @@ -760,33 +766,29 @@ function contentEncodingDecoder( * `redirect: 'manual'`. */ async function undiciRequestAsResponse( - input: RequestInfo | URL, - init: RequestInit, - dispatcher: Dispatcher + url: string, + effectiveInit: UndiciRequestInit, + dispatcher: Dispatcher, + maxResponseSize?: number ): Promise { - let url: string - let effectiveInit = init as UndiciRequestInit - if (typeof Request !== 'undefined' && input instanceof Request) { - // A Request input carries its own method/headers/body/signal; lift them (explicit - // init fields win, per fetch semantics) so a guarded POST isn't downgraded to GET. - const bodyAllowed = input.method !== 'GET' && input.method !== 'HEAD' - effectiveInit = { - method: input.method, - headers: input.headers, - body: bodyAllowed ? await input.clone().arrayBuffer() : undefined, - signal: input.signal, - ...(init as UndiciRequestInit), - // double-cast-allowed: DOM RequestInit and undici RequestInit differ in TS but match at runtime - } as unknown as UndiciRequestInit - url = input.url - } else { - url = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url - } - const method = (effectiveInit.method ?? 'GET').toUpperCase() const canHaveBody = method !== 'GET' && method !== 'HEAD' const requestHeaders = toUndiciRequestHeaders(effectiveInit.headers) ?? {} - const requestBody = canHaveBody ? toUndiciRequestBody(effectiveInit.body) : undefined + if (!Object.keys(requestHeaders).some((name) => name.toLowerCase() === 'user-agent')) { + requestHeaders['user-agent'] = DEFAULT_USER_AGENT + } + let requestBody = canHaveBody ? toUndiciRequestBody(effectiveInit.body) : undefined + if ( + canHaveBody && + (effectiveInit.body instanceof FormData || effectiveInit.body instanceof Blob) + ) { + const encoded = new Request(url, { method, body: effectiveInit.body }) + if (!Object.keys(requestHeaders).some((key) => key.toLowerCase() === 'content-type')) { + const contentType = encoded.headers.get('content-type') + if (contentType) requestHeaders['content-type'] = contentType + } + requestBody = encoded.body ? toUndiciRequestBody(encoded.body) : undefined + } // fetch auto-adds a form content-type for a URLSearchParams body; preserve that parity // when the caller didn't set one (the MCP SDK does set it explicitly, but not every caller). if ( @@ -796,7 +798,7 @@ async function undiciRequestAsResponse( ) { requestHeaders['content-type'] = 'application/x-www-form-urlencoded;charset=UTF-8' } - const { statusCode, headers, body } = await undiciRequest(url, { + const { statusCode, headers, body } = await requestWithOutboundDispatcher(url, { method: method as Dispatcher.HttpMethod, headers: requestHeaders, body: requestBody, @@ -815,7 +817,8 @@ async function undiciRequestAsResponse( // Null-body statuses (204/205/304) can't carry a body; drain undici's (empty) stream so its // socket returns to the pool. Attach an error listener first so a socket reset mid-drain // surfaces as a handled event, not an unhandled 'error' that crashes the process. - const isNullBody = statusCode === 204 || statusCode === 205 || statusCode === 304 + const isNullBody = + method === 'HEAD' || statusCode === 204 || statusCode === 205 || statusCode === 304 if (isNullBody) { body.on('error', () => {}) body.resume() @@ -839,11 +842,33 @@ async function undiciRequestAsResponse( // `nodeReadableToWebStream` attaches its `error` listener synchronously, so wiring the pipe // AFTER it means a synchronous zlib error (e.g. a server mislabeling a non-gzip body as gzip) // is caught and rejects the reader instead of taking down the process. - const webBody = nodeReadableToWebStream(decoder ?? body) + let webBody = nodeReadableToWebStream(decoder ?? body) + if (decoder && maxResponseSize !== undefined && maxResponseSize >= 0) { + let decodedBytes = 0 + webBody = webBody.pipeThrough( + new TransformStream({ + transform(chunk, controller) { + decodedBytes += chunk.byteLength + if (decodedBytes > maxResponseSize) throw new errors.ResponseExceededMaxSizeError() + controller.enqueue(chunk) + }, + }) + ) + } if (decoder) { - body.once('error', (err) => decoder.destroy(err)) // forward maxResponseSize / socket reset - decoder.once('close', () => body.destroy()) // tear the source down so the socket can't leak + const signal = effectiveInit.signal + const onAbort = () => decoder.destroy(toError(signal?.reason ?? new Error('Aborted'))) + signal?.addEventListener('abort', onAbort, { once: true }) + body.once('error', (error) => decoder.destroy(error)) + body.once('close', () => { + if (!body.readableEnded) decoder.destroy(new Error('Response body closed before completing')) + }) + decoder.once('close', () => { + signal?.removeEventListener('abort', onAbort) + body.destroy() + }) body.pipe(decoder) + if (signal?.aborted) onAbort() } try { @@ -856,6 +881,7 @@ async function undiciRequestAsResponse( } catch (err) { // `new Response` rejects an out-of-range status (a 1xx undici shouldn't surface, but // defensively): destroy the source so its socket can't leak, then rethrow. + decoder?.destroy() body.destroy() throw err } @@ -874,25 +900,82 @@ async function liftFetchArgs( const target = typeof input === 'string' ? input : input instanceof URL ? input.href : input.url if (typeof Request !== 'undefined' && input instanceof Request) { const bodyAllowed = input.method !== 'GET' && input.method !== 'HEAD' - return { - target, - effectiveInit: { - method: input.method, - headers: input.headers, - body: bodyAllowed ? await input.clone().arrayBuffer() : undefined, - signal: input.signal, - // Carry the Request's redirect mode so the pinned fetch honors `manual`/`error` - // instead of defaulting a `Request({ redirect: 'manual' })` to `follow`. - redirect: input.redirect, - ...init, - }, + const effectiveInit: RequestInit = { + method: input.method, + headers: input.headers, + body: bodyAllowed ? input.body : undefined, + signal: input.signal, + // Carry the Request's redirect mode so the pinned fetch honors `manual`/`error` + // instead of defaulting a `Request({ redirect: 'manual' })` to `follow`. + redirect: input.redirect, + ...init, } + /** Request hides its original body source, so following redirects requires replayable bytes. */ + if ( + !Object.hasOwn(init ?? {}, 'body') && + effectiveInit.body && + (effectiveInit.redirect ?? 'follow') === 'follow' + ) { + effectiveInit.body = await input.clone().arrayBuffer() + } + return { target, effectiveInit } } return { target, effectiveInit: init ?? {} } } +export interface OutboundFetchDispatcher { + close(): Promise + destroy(): Promise +} + +/** Owns routing, redirect validation and connection pools for pinned and DNS-guarded fetches. */ +function createValidatedFetch( + direct: Agent, + options: { + profile: EgressProfile + resolvedIP?: string + maxResponseSize?: number + } +): { fetch: typeof fetch; dispatcher: OutboundFetchDispatcher } { + const dispatcher = createOutboundTransport({ ...options, direct }) + const rawFetch = async (url: string, init: UndiciRequestInit): Promise => { + const selected = await dispatcher.selectDispatcher() + if (!selected) throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') + return undiciRequestAsResponse(url, init, selected, options.maxResponseSize) + } + return { + dispatcher, + fetch: async (input: RequestInfo | URL, init?: RequestInit): Promise => { + const { target, effectiveInit } = await liftFetchArgs(input, init) + const mode = effectiveInit.redirect ?? 'follow' + // double-cast-allowed: DOM and Undici RequestInit represent the same wire request in this bridge + const undiciInit = effectiveInit as unknown as UndiciRequestInit + if (mode === 'follow') { + return followRedirectsGuarded( + rawFetch, + target, + undiciInit, + options.profile, + options.resolvedIP + ) + } + assertGuardedRedirectTarget(new URL(target), options.profile, options.resolvedIP) + const response = await rawFetch(target, undiciInit) + if ( + mode === 'error' && + isRedirectStatus(response.status) && + response.headers.has('location') + ) { + await response.body?.cancel().catch(() => {}) + throw new TypeError('Outbound fetch received an unexpected redirect') + } + return response + }, + } +} + /** - * SSRF-guarded `fetch` + its `Agent` for outbound requests to user-controlled + * SSRF-guarded `fetch` + its dispatcher for outbound requests to user-controlled * hosts: DNS resolves normally, and every socket connect validates the chosen * addresses via {@link createSsrfGuardedLookup}; redirects are followed manually * with per-hop validation (see {@link followRedirectsGuarded}) so IP-literal @@ -904,7 +987,7 @@ export function createSsrfGuardedFetchWithDispatcher(options: { maxResponseSize?: number }): { fetch: typeof fetch - dispatcher: Agent + dispatcher: OutboundFetchDispatcher } { const dispatcher = new Agent({ allowH2: false, @@ -912,22 +995,7 @@ export function createSsrfGuardedFetchWithDispatcher(options: { ...(options.maxResponseSize !== undefined ? { maxResponseSize: options.maxResponseSize } : {}), }) - const rawFetch = (url: string, init: UndiciRequestInit): Promise => - // double-cast-allowed: DOM RequestInit and undici RequestInit differ in TS but match at runtime - undiciRequestAsResponse(url, init as unknown as RequestInit, dispatcher) - - const guarded = async (input: RequestInfo | URL, init?: RequestInit): Promise => { - const { target, effectiveInit } = await liftFetchArgs(input, init) - return followRedirectsGuarded( - rawFetch, - target, - // double-cast-allowed: DOM RequestInit and undici RequestInit are structurally compatible at runtime but the TS types differ - effectiveInit as unknown as UndiciRequestInit, - options.profile - ) - } - - return { fetch: guarded, dispatcher } + return createValidatedFetch(dispatcher, options) } /** @@ -962,10 +1030,9 @@ export function createPinnedFetch( } /** - * Same as {@link createPinnedFetch} but also returns the underlying `Agent` so a - * caller with a defined connection lifetime (e.g. a long-lived MCP transport) can - * tear the Agent down on close instead of waiting for its idle timeout. Closing - * the Agent is what releases any pooled keep-alive / HTTP/2 sockets it holds. + * Like {@link createPinnedFetch}, with lifecycle controls for its direct and gateway + * connection pools. Callers with a defined connection lifetime close these pools + * when that lifetime ends. * * `maxResponseSize` caps the (decoded) response body in bytes and makes undici reject * with `UND_ERR_RES_EXCEEDED_MAX_SIZE` once exceeded — a DoS backstop for one-shot @@ -975,48 +1042,14 @@ export function createPinnedFetch( export function createPinnedFetchWithDispatcher( resolvedIP: string, options: { profile: EgressProfile; allowH2?: boolean; maxResponseSize?: number } -): { fetch: typeof fetch; dispatcher: Agent } { +): { fetch: typeof fetch; dispatcher: OutboundFetchDispatcher } { const dispatcher = new Agent({ allowH2: options.allowH2 ?? false, connect: { lookup: createPinnedLookup(resolvedIP) }, ...(options.maxResponseSize !== undefined ? { maxResponseSize: options.maxResponseSize } : {}), }) - const rawFetch = (url: string, init: UndiciRequestInit): Promise => - // double-cast-allowed: DOM RequestInit and undici RequestInit differ in TS but match at runtime - undiciRequestAsResponse(url, init as unknown as RequestInit, dispatcher) - - // Requests go through `undici.request` (not `undici.fetch`) because fetch's streaming - // `response.body` never delivers under the Bun runtime the server runs on — the same bug - // {@link createSsrfGuardedFetchWithDispatcher} works around. Redirects are handled here (not - // by a caller's wrapper — the pinned fetch is passed straight to provider/A2A SDKs), honoring - // the request's `redirect` mode: `manual`/`error` must NOT transparently follow (e.g. - // `detectMcpAuthType` inspects the 3xx to classify auth). The default `follow` uses - // {@link followRedirectsGuarded}, which drops headers on cross-origin hops (so a redirect - // can't disclose a provider `api-key` to another origin) and stamps the final `response.url`. - // Every hop still dispatches through the pinned `Agent` (its `connect.lookup` forces - // `resolvedIP`), so a redirect can't escape to another address. - const pinned = async (input: RequestInfo | URL, init?: RequestInit): Promise => { - const { target, effectiveInit } = await liftFetchArgs(input, init) - const mode = effectiveInit.redirect ?? 'follow' - // double-cast-allowed: DOM RequestInit and undici RequestInit are structurally compatible at runtime but the TS types differ - const undiciInit = effectiveInit as unknown as UndiciRequestInit - if (mode === 'manual') { - return rawFetch(target, undiciInit) - } - if (mode === 'error') { - const response = await rawFetch(target, undiciInit) - const location = response.headers.get('location') - if (response.status >= 300 && response.status < 400 && location) { - await response.body?.cancel().catch(() => {}) - throw new TypeError('Pinned fetch received an unexpected redirect (redirect: "error")') - } - return response - } - return followRedirectsGuarded(rawFetch, target, undiciInit, options.profile, resolvedIP) - } - - return { fetch: pinned, dispatcher } + return createValidatedFetch(dispatcher, { ...options, resolvedIP }) } /** @@ -1041,14 +1074,23 @@ export async function secureFetchWithPinnedIP( ? requestedMaxResponseBytes : DEFAULT_MAX_RESPONSE_BYTES + const transport = createOutboundTransport({ + profile: options.profile, + resolvedIP, + proxyUrl: options.proxyUrl, + }) + const outboundDispatcher = await transport.selectDispatcher() + return new Promise((resolve, reject) => { const parsed = new URL(url) const isHttps = parsed.protocol === 'https:' const defaultPort = isHttps ? 443 : 80 const port = parsed.port ? Number.parseInt(parsed.port, 10) : defaultPort - let agent: http.Agent - if (options.proxyUrl) { + let agent: http.Agent | undefined + if (outboundDispatcher) { + agent = undefined + } else if (options.proxyUrl) { // Proxy connection is already IP-pinned by validateAndPinProxyUrl; target-IP // pinning is intentionally bypassed (the proxy resolves the target). https // targets tunnel via CONNECT, http targets use absolute-URI forwarding. @@ -1060,6 +1102,9 @@ export async function secureFetchWithPinnedIP( } const { 'accept-encoding': _, ...sanitizedHeaders } = options.headers ?? {} + if (!Object.keys(sanitizedHeaders).some((name) => name.toLowerCase() === 'user-agent')) { + sanitizedHeaders['user-agent'] = DEFAULT_USER_AGENT + } const hasExplicitFraming = Object.keys(sanitizedHeaders).some((name) => { const header = name.toLowerCase() return header === 'content-length' || header === 'transfer-encoding' @@ -1081,8 +1126,14 @@ export async function secureFetchWithPinnedIP( timeout: options.timeout || 300000, } - const protocol = isHttps ? https : http - const req = protocol.request(requestOptions, (res) => { + let destroyRequest: () => void = () => {} + const onResponse = ( + res: Readable & { + statusCode?: number + headers: http.IncomingHttpHeaders + statusMessage?: string + } + ) => { const statusCode = res.statusCode || 0 const location = res.headers.location @@ -1212,6 +1263,7 @@ export async function secureFetchWithPinnedIP( const isBodylessResponse = (requestOptions.method || 'GET').toUpperCase() === 'HEAD' || statusCode === 204 || + statusCode === 205 || statusCode === 304 const contentLength = headersRecord['content-length'] if (contentLength && !isBodylessResponse) { @@ -1219,7 +1271,7 @@ export async function secureFetchWithPinnedIP( if (Number.isFinite(parsedLength) && parsedLength > maxResponseBytes) { cleanupAbort() res.destroy() - req.destroy() + destroyRequest() if (isRetryableHttpStatus(statusCode)) { settledResolve({ ok: false, @@ -1244,38 +1296,69 @@ export async function secureFetchWithPinnedIP( } } + const decoder = isBodylessResponse + ? null + : contentEncodingDecoder((headersRecord['content-encoding'] ?? '').toLowerCase().trim()) + const responseHeaders = decoder + ? stripHeaders(headersRecord, ['content-encoding', 'content-length']) + : headersRecord + let totalBytes = 0 - const nodeRes = res + let bodySettled = false + const nodeRes = decoder ?? res + const destroyTransport = destroyRequest + destroyRequest = () => { + nodeRes.destroy() + if (decoder) res.destroy() + destroyTransport() + } const body = new ReadableStream({ start(controller) { + const fail = (error: Error) => { + if (bodySettled) return + bodySettled = true + cleanupAbort() + controller.error(error) + destroyRequest() + } nodeRes.on('data', (chunk: Buffer) => { + if (bodySettled) return totalBytes += chunk.length if (totalBytes > maxResponseBytes) { - cleanupAbort() - controller.error( + fail( new PayloadSizeLimitError({ label: 'response body', maxBytes: maxResponseBytes, observedBytes: totalBytes, }) ) - nodeRes.destroy() return } controller.enqueue(new Uint8Array(chunk)) }) - nodeRes.on('end', () => { + nodeRes.once('end', () => { + if (bodySettled) return + bodySettled = true cleanupAbort() controller.close() }) - nodeRes.on('error', (err) => { - cleanupAbort() - controller.error(err) + nodeRes.once('error', fail) + nodeRes.once('close', () => { + if (!bodySettled) fail(new Error('Response body closed before completing')) }) + if (decoder) { + res.once('error', (error) => decoder.destroy(error)) + res.once('close', () => { + if (!res.readableEnded) + decoder.destroy(new Error('Response body closed before completing')) + }) + res.pipe(decoder) + } }, cancel() { + bodySettled = true cleanupAbort() - nodeRes.destroy() + destroyRequest() }, }) @@ -1300,7 +1383,7 @@ export async function secureFetchWithPinnedIP( ok: statusCode >= 200 && statusCode < 300, status: statusCode, statusText: res.statusMessage || '', - headers: new SecureFetchHeaders(headersRecord, setCookieArray), + headers: new SecureFetchHeaders(responseHeaders, setCookieArray), body, text: async () => (await readBodyAsBuffer()).toString('utf-8'), json: async () => JSON.parse((await readBodyAsBuffer()).toString('utf-8')), @@ -1309,7 +1392,7 @@ export async function secureFetchWithPinnedIP( return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength) as ArrayBuffer }, }) - }) + } let onAbort: (() => void) | null = null const cleanupAbort = () => { @@ -1326,29 +1409,66 @@ export async function secureFetchWithPinnedIP( reject(reason) } - req.on('error', (error) => { - settledReject(error) - }) - - req.on('timeout', () => { - req.destroy() - settledReject(new Error(`Request timed out after ${requestOptions.timeout}ms`)) - }) + let send: () => void + if (outboundDispatcher) { + const dispatcher = outboundDispatcher + const controller = new AbortController() + destroyRequest = () => { + controller.abort() + void transport.destroy() + } + send = () => { + void requestWithOutboundDispatcher(url, { + dispatcher, + method: (options.method || 'GET') as Dispatcher.HttpMethod, + headers: sanitizedHeaders, + body: options.body, + signal: AbortSignal.any([ + controller.signal, + AbortSignal.timeout(options.timeout || 300_000), + ]), + }) + .then(({ statusCode, headers, body }) => { + body.once('close', () => { + void transport.destroy() + }) + onResponse(Object.assign(body, { statusCode, headers })) + }) + .catch((error) => { + void transport.destroy() + settledReject(error) + }) + } + } else { + const protocol = isHttps ? https : http + const req = protocol.request(requestOptions, onResponse) + destroyRequest = () => { + req.destroy() + } + req.on('error', settledReject) + req.on('timeout', () => { + destroyRequest() + settledReject(new Error(`Request timed out after ${requestOptions.timeout}ms`)) + }) + send = () => { + req.end(options.body) + } + } if (options.signal) { if (options.signal.aborted) { - req.destroy() + destroyRequest() settledReject(options.signal.reason ?? new Error('Aborted')) return } onAbort = () => { - req.destroy() + destroyRequest() settledReject(options.signal?.reason ?? new Error('Aborted')) } options.signal.addEventListener('abort', onAbort, { once: true }) } - req.end(options.body) + send() }) } diff --git a/apps/sim/lib/core/security/pinned-fetch.server.test.ts b/apps/sim/lib/core/security/pinned-fetch.server.test.ts index ded5a698a08..a129ad9f999 100644 --- a/apps/sim/lib/core/security/pinned-fetch.server.test.ts +++ b/apps/sim/lib/core/security/pinned-fetch.server.test.ts @@ -109,7 +109,7 @@ describe('createPinnedFetch', () => { expect(url).toBe('https://myresource.openai.azure.com/openai/v1/responses') expect(options.dispatcher).toBeInstanceOf(mockAgent) expect(options.method).toBe('POST') - expect(options.headers).toEqual({ 'api-key': 'secret' }) + expect(options.headers).toEqual({ 'api-key': 'secret', 'user-agent': 'undici' }) expect(options.body).toBe('{}') expect(options.signal).toBe(controller.signal) }) @@ -159,7 +159,7 @@ describe('createPinnedFetch', () => { string > expect(secondHopHeaders['api-key']).toBeUndefined() - expect(Object.keys(secondHopHeaders)).toHaveLength(0) + expect(secondHopHeaders).toEqual({ 'user-agent': 'undici' }) expect(response.status).toBe(200) expect(response.url).toBe('https://other-origin.example/final') expect(response.redirected).toBe(true) diff --git a/apps/sim/lib/core/security/secure-fetch-request-framing.server.test.ts b/apps/sim/lib/core/security/secure-fetch-request-framing.server.test.ts index 3076de3590c..7d977c64e29 100644 --- a/apps/sim/lib/core/security/secure-fetch-request-framing.server.test.ts +++ b/apps/sim/lib/core/security/secure-fetch-request-framing.server.test.ts @@ -83,6 +83,18 @@ async function sendToLengthRequiredEndpoint( } describe('secureFetchWithPinnedIP request framing', () => { + it.each([ + { headers: undefined, expected: 'undici' }, + { headers: { 'uSeR-aGeNt': 'custom-client/1.0' }, expected: 'custom-client/1.0' }, + ])( + 'sends a default User-Agent and preserves an explicit one ($expected)', + async ({ headers, expected }) => { + const received = await sendToLengthRequiredEndpoint({ method: 'POST', headers }) + + expect(received.headers['user-agent']).toBe(expected) + } + ) + it.each(['POST', 'PUT', 'PATCH', 'DELETE'])( 'sends a UTF-8 %s body with its byte length', async (method) => { diff --git a/apps/sim/lib/core/security/secure-fetch-response-cap.server.test.ts b/apps/sim/lib/core/security/secure-fetch-response-cap.server.test.ts index ce7e2b7d6d6..ef44693ddc3 100644 --- a/apps/sim/lib/core/security/secure-fetch-response-cap.server.test.ts +++ b/apps/sim/lib/core/security/secure-fetch-response-cap.server.test.ts @@ -3,6 +3,9 @@ */ import http from 'node:http' import type { AddressInfo } from 'node:net' +import { Transform } from 'node:stream' +import zlib, { brotliCompressSync, deflateSync, gzipSync } from 'node:zlib' +import { Agent } from 'undici/index.js' import { afterEach, describe, expect, it, vi } from 'vitest' vi.mock('@sim/security/dns', () => ({ @@ -18,7 +21,9 @@ vi.mock('@/lib/core/config/env-flags', () => ({ getProxyUrl: () => undefined, })) +import * as networkTransport from '@/lib/core/network/transport.server' import { + createPinnedFetchWithDispatcher, DEFAULT_MAX_RESPONSE_BYTES, secureFetchWithPinnedIP, } from '@/lib/core/security/input-validation.server' @@ -26,7 +31,11 @@ import { const servers: http.Server[] = [] afterEach(() => { - for (const server of servers.splice(0)) server.close() + vi.restoreAllMocks() + for (const server of servers.splice(0)) { + server.closeAllConnections() + server.close() + } }) /** Starts a throwaway loopback server and returns its origin. */ @@ -38,6 +47,144 @@ async function startServer(handler: http.RequestListener): Promise { } describe('secureFetchWithPinnedIP response cap', () => { + it.each([ + { encoding: 'gzip', compress: gzipSync }, + { encoding: 'deflate', compress: deflateSync }, + { encoding: 'br', compress: brotliCompressSync }, + ])( + 'decodes $encoding JSON and removes encoded framing headers', + async ({ encoding, compress }) => { + const payload = { ok: true, message: 'compressed provider response' } + const encoded = compress(Buffer.from(JSON.stringify(payload))) + const origin = await startServer((_req, res) => { + res.writeHead(200, { + 'Content-Type': 'application/json', + 'Content-Encoding': encoding, + 'Content-Length': String(encoded.length), + }) + res.end(encoded) + }) + + const response = await secureFetchWithPinnedIP(origin, '127.0.0.1', { + profile: 'configuredEndpoint', + maxResponseBytes: 1024, + }) + + await expect(response.json()).resolves.toEqual(payload) + expect(response.headers.get('content-type')).toBe('application/json') + expect(response.headers.get('content-encoding')).toBeNull() + expect(response.headers.get('content-length')).toBeNull() + } + ) + + it('limits decoded bytes and closes an upstream still sending compressed content', async () => { + const closed = vi.fn() + const encoded = gzipSync(Buffer.alloc(64 * 1024, 0x41)) + expect(encoded.length).toBeLessThan(1024) + const origin = await startServer((_req, res) => { + res.once('close', closed) + res.writeHead(200, { 'Content-Encoding': 'gzip' }) + res.write(encoded) + }) + + const response = await secureFetchWithPinnedIP(origin, '127.0.0.1', { + profile: 'configuredEndpoint', + maxResponseBytes: 1024, + }) + + await expect(response.text()).rejects.toThrow(/response body/i) + await vi.waitFor(() => expect(closed).toHaveBeenCalledOnce()) + }) + + it('rejects malformed compression and closes the upstream connection', async () => { + const closed = vi.fn() + const origin = await startServer((_req, res) => { + res.once('close', closed) + res.writeHead(200, { 'Content-Encoding': 'gzip' }) + res.write('this is not gzip') + }) + + const response = await secureFetchWithPinnedIP(origin, '127.0.0.1', { + profile: 'configuredEndpoint', + }) + + await expect(response.text()).rejects.toThrow() + await vi.waitFor(() => expect(closed).toHaveBeenCalledOnce()) + }) + + it('rejects the decoded reader when the upstream resets before completing', async () => { + let upstream: http.ServerResponse | undefined + const origin = await startServer((_req, res) => { + upstream = res + res.writeHead(200, { 'Content-Encoding': 'gzip' }) + res.write(gzipSync(Buffer.from('payload')).subarray(0, 10)) + }) + + const response = await secureFetchWithPinnedIP(origin, '127.0.0.1', { + profile: 'configuredEndpoint', + }) + const body = response.text() + upstream!.destroy() + + await expect(body).rejects.toThrow(/aborted|closed before completing/) + }) + + it('rejects a compressed body read when its request is aborted', async () => { + const controller = new AbortController() + const closed = vi.fn() + const origin = await startServer((_req, res) => { + res.once('close', closed) + res.writeHead(200, { 'Content-Encoding': 'gzip' }) + res.write(gzipSync(Buffer.from('payload')).subarray(0, 10)) + }) + + const response = await secureFetchWithPinnedIP(origin, '127.0.0.1', { + profile: 'configuredEndpoint', + signal: controller.signal, + }) + const body = response.text() + controller.abort() + + await expect(body).rejects.toThrow() + await vi.waitFor(() => expect(closed).toHaveBeenCalledOnce()) + }) + + it('destroys the compressed source when the reader cancels', async () => { + const closed = vi.fn() + const origin = await startServer((_req, res) => { + res.once('close', closed) + res.writeHead(200, { 'Content-Encoding': 'gzip' }) + res.write(gzipSync(Buffer.from('payload'))) + }) + + const response = await secureFetchWithPinnedIP(origin, '127.0.0.1', { + profile: 'configuredEndpoint', + }) + await response.body!.cancel() + + await vi.waitFor(() => expect(closed).toHaveBeenCalledOnce()) + }) + + it.each([ + { method: 'HEAD', status: 200 }, + { method: 'GET', status: 204 }, + { method: 'GET', status: 205 }, + { method: 'GET', status: 304 }, + ])('does not decode a bodyless $method $status response', async ({ method, status }) => { + const origin = await startServer((_req, res) => { + res.writeHead(status, { 'Content-Encoding': 'gzip' }) + res.end() + }) + + const response = await secureFetchWithPinnedIP(origin, '127.0.0.1', { + profile: 'configuredEndpoint', + method, + }) + + await expect(response.text()).resolves.toBe('') + expect(response.headers.get('content-encoding')).toBe('gzip') + }) + it('rejects a body that exceeds an explicit cap instead of buffering it', async () => { const origin = await startServer((_req, res) => { res.writeHead(200, { 'Content-Type': 'application/octet-stream' }) @@ -112,3 +259,130 @@ describe('secureFetchWithPinnedIP response cap', () => { expect(response.status).toBe(304) }) }) + +describe('pinned fetch response decoding', () => { + it('returns a null body for HEAD even when metadata advertises gzip', async () => { + const origin = await startServer((_req, res) => { + res.writeHead(200, { 'Content-Encoding': 'gzip', 'Content-Length': '10000' }) + res.end() + }) + const transport = createPinnedFetchWithDispatcher('127.0.0.1', { + profile: 'configuredEndpoint', + maxResponseSize: 1024, + }) + try { + const response = await transport.fetch(origin, { method: 'HEAD' }) + + expect(response.body).toBeNull() + await expect(response.text()).resolves.toBe('') + expect(response.headers.get('content-encoding')).toBe('gzip') + expect(response.headers.get('content-length')).toBe('10000') + } finally { + await transport.dispatcher.destroy() + } + }) + + it('enforces maxResponseSize on decoded content and cancels its upstream', async () => { + const encoded = gzipSync(Buffer.alloc(64 * 1024, 0x41)) + const closed = vi.fn() + expect(encoded.length).toBeLessThan(1024) + const origin = await startServer((_req, res) => { + res.once('close', closed) + res.writeHead(200, { 'Content-Encoding': 'gzip' }) + res.write(encoded) + }) + const transport = createPinnedFetchWithDispatcher('127.0.0.1', { + profile: 'configuredEndpoint', + maxResponseSize: 1024, + }) + try { + const response = await transport.fetch(origin) + + await expect(response.text()).rejects.toMatchObject({ code: 'UND_ERR_RES_EXCEEDED_MAX_SIZE' }) + await vi.waitFor(() => expect(closed).toHaveBeenCalledOnce()) + } finally { + await transport.dispatcher.destroy() + } + }) + + it('still permits decoded responses when maxResponseSize is explicitly unbounded', async () => { + const payload = 'a'.repeat(64 * 1024) + const origin = await startServer((_req, res) => { + res.writeHead(200, { 'Content-Encoding': 'gzip' }) + res.end(gzipSync(payload)) + }) + const transport = createPinnedFetchWithDispatcher('127.0.0.1', { + profile: 'configuredEndpoint', + maxResponseSize: -1, + }) + try { + const response = await transport.fetch(origin) + await expect(response.text()).resolves.toBe(payload) + } finally { + await transport.dispatcher.destroy() + } + }) + + it.each(['bounded', 'guarded'] as const)( + 'keeps %s cancellation attached until decoding finishes', + async (mode) => { + const decoder = new Transform({ + transform(_chunk, _encoding, done) { + done() + }, + flush() {}, + }) + vi.spyOn(zlib, 'createGunzip').mockReturnValue(decoder as zlib.Gunzip) + const dispatcher = new Agent() + if (mode === 'bounded') { + vi.spyOn(networkTransport, 'createOutboundTransport').mockReturnValueOnce({ + selectDispatcher: async () => dispatcher, + close: () => dispatcher.close(), + destroy: () => dispatcher.destroy(), + }) + } + const pinned = + mode === 'guarded' + ? createPinnedFetchWithDispatcher('127.0.0.1', { profile: 'configuredEndpoint' }) + : undefined + const wireClosed = vi.fn() + const send = networkTransport.requestWithOutboundDispatcher + vi.spyOn(networkTransport, 'requestWithOutboundDispatcher').mockImplementationOnce( + async (...args) => { + const reply = await send(...args) + reply.body.once('close', wireClosed) + return reply + } + ) + const origin = await startServer((_req, res) => { + res.writeHead(200, { 'Content-Encoding': 'gzip' }) + res.end(gzipSync('payload')) + }) + const controller = new AbortController() + const removeListener = vi.spyOn(controller.signal, 'removeEventListener') + try { + const response = pinned + ? await pinned.fetch(origin, { signal: controller.signal }) + : await secureFetchWithPinnedIP(origin, '127.0.0.1', { + profile: 'configuredEndpoint', + signal: controller.signal, + }) + await vi.waitFor(() => expect(wireClosed).toHaveBeenCalledOnce()) + if (mode === 'bounded') { + expect(removeListener).not.toHaveBeenCalledWith('abort', expect.any(Function)) + } + const reading = response.text() + const reason = new Error('decoding cancelled') + controller.abort(reason) + + if (mode === 'guarded') await expect(reading).rejects.toBe(reason) + else await expect(reading).rejects.toThrow(/closed before completing/) + expect(decoder.destroyed).toBe(true) + } finally { + decoder.destroy() + await pinned?.dispatcher.destroy() + await dispatcher.destroy() + } + } + ) +}) diff --git a/apps/sim/lib/core/utils/fetch-deadline.ts b/apps/sim/lib/core/utils/fetch-deadline.ts index 006795a427a..f6643784a09 100644 --- a/apps/sim/lib/core/utils/fetch-deadline.ts +++ b/apps/sim/lib/core/utils/fetch-deadline.ts @@ -1,4 +1,4 @@ -import { Agent, type Dispatcher } from 'undici' +import { Agent, type Dispatcher } from 'undici/index.js' /** * Keeps the transport deadline from undercutting the application deadline. diff --git a/apps/sim/lib/credential-groups/application/public-enrollment.ts b/apps/sim/lib/credential-groups/application/public-enrollment.ts index 0c0f388e82b..a45f06f9135 100644 --- a/apps/sim/lib/credential-groups/application/public-enrollment.ts +++ b/apps/sim/lib/credential-groups/application/public-enrollment.ts @@ -2,6 +2,7 @@ import type { CredentialGroupEnrollmentPrincipal, Principal } from '@sim/auth/pr import { safeCompare } from '@sim/security/compare' import { sha256Hex } from '@sim/security/hash' import type { OperationUseCase } from '@/lib/core/application' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { OrchestrationError } from '@/lib/core/orchestration/types' import { resourceScopeFields, @@ -91,7 +92,7 @@ function defineAuthorizedCredentialGroupEnrollmentUseCase< }, async execute({ principal, input }) { const authorized = await authorize(principal, input) - return definition.execute(authorized) + return withResourceOutboundScope(authorized.context, () => definition.execute(authorized)) }, } } diff --git a/apps/sim/lib/credentials/application/organization-credentials.ts b/apps/sim/lib/credentials/application/organization-credentials.ts index 6117032c586..97ea3207545 100644 --- a/apps/sim/lib/credentials/application/organization-credentials.ts +++ b/apps/sim/lib/credentials/application/organization-credentials.ts @@ -17,6 +17,7 @@ import { } from '@/lib/core/application/organization-authorization' import { defineOrganizationOperation } from '@/lib/core/application/organization-operation' import { PrincipalKindAuthorizationError } from '@/lib/core/application/workspace-authorization' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { OrchestrationError } from '@/lib/core/orchestration/types' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import { throwCredentialMutationFailure } from '@/lib/credentials/application/credential-crud' @@ -216,9 +217,8 @@ export const createOrganizationCredential: OperationUseCase< ) requireAvailableOAuthCredentialProvider(catalog, input.providerId ?? '') } - const result = await createCredentialRecord( - { ...input, userId: context.userId }, - { authorizeWorkspace: false } + const result = await withResourceOutboundScope(context, () => + createCredentialRecord({ ...input, userId: context.userId }, { authorizeWorkspace: false }) ) if (!result.success) throwCredentialMutationFailure(result) if (!result.credential) throw new Error('Credential creation returned no credential') @@ -409,7 +409,9 @@ export const updateOrganizationCredential: OperationUseCase< (row.type === 'oauth' && row.createdBy !== context.userId) ) throw new OrchestrationError('not_found', 'Credential not found') - const result = await updateCredentialRecord({ ...input, credential: row }) + const result = await withResourceOutboundScope(row, () => + updateCredentialRecord({ ...input, credential: row }) + ) if (!result.success) throwCredentialMutationFailure(result) const updated = await getOrganizationCredential(input.organizationId, input.credentialId) if (!updated) throw new OrchestrationError('not_found', 'Credential not found') diff --git a/apps/sim/lib/data-drains/service.ts b/apps/sim/lib/data-drains/service.ts index 418d475137b..df5709af819 100644 --- a/apps/sim/lib/data-drains/service.ts +++ b/apps/sim/lib/data-drains/service.ts @@ -4,6 +4,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' import { eq } from 'drizzle-orm' +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' import { getDestination } from '@/lib/data-drains/destinations/registry' import { decryptCredentials } from '@/lib/data-drains/encryption' import { getSource } from '@/lib/data-drains/sources/registry' @@ -88,7 +89,8 @@ export async function runDrain( const credentials = destination.credentialsSchema.parse( await decryptCredentials(drain.destinationCredentials) ) - session = destination.openSession({ config, credentials }) + const activeSession = destination.openSession({ config, credentials }) + session = activeSession for await (const chunk of source.pages({ organizationId: drain.organizationId, @@ -99,19 +101,21 @@ export async function runDrain( const ndjson = `${chunk.map((row) => JSON.stringify(source.serialize(row))).join('\n')}\n` const body = Buffer.from(ndjson, 'utf8') - const result = await session.deliver({ - body, - contentType: 'application/x-ndjson', - metadata: { - drainId, - runId, - source: drain.source, - sequence, - rowCount: chunk.length, - runStartedAt: startedAt, - }, - signal, - }) + const result = await runWithOutboundOrganization(drain.organizationId, () => + activeSession.deliver({ + body, + contentType: 'application/x-ndjson', + metadata: { + drainId, + runId, + source: drain.source, + sequence, + rowCount: chunk.length, + runStartedAt: startedAt, + }, + signal, + }) + ) locators.push(result.locator) rowsExported += chunk.length diff --git a/apps/sim/lib/execution/files.ts b/apps/sim/lib/execution/files.ts index e23582aba18..d2d195681bd 100644 --- a/apps/sim/lib/execution/files.ts +++ b/apps/sim/lib/execution/files.ts @@ -1,4 +1,5 @@ import { createLogger } from '@sim/logger' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' import { TRIGGER_TYPES } from '@/lib/workflows/triggers/triggers' import type { InputFormatField } from '@/lib/workflows/types' @@ -57,7 +58,9 @@ export async function processExecutionFile( if (file.type === 'url' && file.data) { const { downloadFileFromUrl } = await import('@/lib/uploads/utils/file-utils.server') - const buffer = await downloadFileFromUrl(file.data, { userId }) + const buffer = await withResourceOutboundScope(executionContext, () => + downloadFileFromUrl(file.data, { userId }) + ) if (buffer.length > MAX_FILE_SIZE) { const fileSizeMB = (buffer.length / (1024 * 1024)).toFixed(2) diff --git a/apps/sim/lib/execution/isolated-vm-worker.cjs b/apps/sim/lib/execution/isolated-vm-worker.cjs index ac19d487b21..a5eeb2e6dd0 100644 --- a/apps/sim/lib/execution/isolated-vm-worker.cjs +++ b/apps/sim/lib/execution/isolated-vm-worker.cjs @@ -263,13 +263,16 @@ async function executeCode(request, executionId) { resolve(JSON.stringify({ error: 'Parent process disconnected' })) return } - sendIpcRequest({ type: 'fetch', fetchId, requestId, url, optionsJson }, (err) => { - const pending = pendingFetches.get(fetchId) - if (!pending) return - clearTimeout(pending.timeout) - pendingFetches.delete(fetchId) - pending.resolve(JSON.stringify({ error: `Fetch IPC send failed: ${err.message}` })) - }) + sendIpcRequest( + { type: 'fetch', fetchId, executionId, requestId, url, optionsJson }, + (err) => { + const pending = pendingFetches.get(fetchId) + if (!pending) return + clearTimeout(pending.timeout) + pendingFetches.delete(fetchId) + pending.resolve(JSON.stringify({ error: `Fetch IPC send failed: ${err.message}` })) + } + ) }) }) await jail.set('__fetchRef', fetchCallback) diff --git a/apps/sim/lib/execution/isolated-vm.test.ts b/apps/sim/lib/execution/isolated-vm.test.ts index 717f9ebaa2e..27dc37b469d 100644 --- a/apps/sim/lib/execution/isolated-vm.test.ts +++ b/apps/sim/lib/execution/isolated-vm.test.ts @@ -143,6 +143,7 @@ function createReadyFetchProxyProc(fetchMessage: { url: string; optionsJson?: st setImmediate(() => { proc.emit('message', { type: 'fetch', + executionId: currentExecutionId, fetchId: 1, requestId: msg.request?.requestId ?? 'fetch-test', url: fetchMessage.url, diff --git a/apps/sim/lib/execution/isolated-vm.ts b/apps/sim/lib/execution/isolated-vm.ts index 93bbf55fedb..d257c08c8fa 100644 --- a/apps/sim/lib/execution/isolated-vm.ts +++ b/apps/sim/lib/execution/isolated-vm.ts @@ -9,6 +9,7 @@ import { randomFloat } from '@sim/utils/random' import { env } from '@/lib/core/config/env' import { getConfiguredCacheProvider } from '@/lib/core/config/env-capabilities.server' import { getRedisClient } from '@/lib/core/config/redis' +import { captureOutboundScope } from '@/lib/core/network/context.server' import { type SecureFetchOptions, secureFetchWithValidation, @@ -160,6 +161,7 @@ const QUEUE_RETRY_DELAY_MS = 1000 const DISTRIBUTED_LEASE_GRACE_MS = 30000 interface PendingExecution { + runInOutboundScope: ReturnType resolve: (result: IsolatedVMExecutionResult) => void timeout: ReturnType ownerKey: string @@ -199,6 +201,7 @@ interface QueuedExecution { * against the queue-to-worker handoff. */ interface ExecutionState { + runInOutboundScope: ReturnType cancelled: boolean queueId?: number workerId?: number @@ -730,7 +733,7 @@ function handleBrokerMessage( } Promise.resolve() - .then(() => handler(args)) + .then(() => pending.runInOutboundScope(() => handler(args))) .then((resultValue) => { if (pending.cancelled) { sendResponse({ error: 'Execution cancelled' }) @@ -809,6 +812,18 @@ function handleWorkerMessage(workerId: number, message: unknown) { } if (msg.type === 'fetch') { + const pending = + typeof msg.executionId === 'number' + ? workerInfo?.pendingExecutions.get(msg.executionId) + : undefined + if (!pending || pending.cancelled) { + workerInfo?.process.send({ + type: 'fetchResponse', + fetchId: msg.fetchId, + response: JSON.stringify({ error: 'Execution no longer active' }), + }) + return + } const { fetchId, requestId, url, optionsJson } = msg as { fetchId: number requestId: string @@ -847,7 +862,8 @@ function handleWorkerMessage(workerId: number, message: unknown) { return } } - secureFetch(requestId, url, options) + pending + .runInOutboundScope(() => secureFetch(requestId, url, options)) .then((response) => { try { workerInfo?.process.send({ type: 'fetchResponse', fetchId, response }) @@ -1200,6 +1216,7 @@ function dispatchToWorker( }, req.timeoutMs + 1000) workerInfo.pendingExecutions.set(execId, { + runInOutboundScope: state.runInOutboundScope, resolve, timeout, ownerKey: ownerState.ownerKey, @@ -1472,7 +1489,7 @@ export async function executeInIsolatedVM( // An undetermined lease cannot reject the execution: the per-process pool and // the per-owner active/queued limits above still bound this work. - const state: ExecutionState = { cancelled: false } + const state: ExecutionState = { cancelled: false, runInOutboundScope: captureOutboundScope() } return new Promise((resolve) => { let abortListener: (() => void) | null = null diff --git a/apps/sim/lib/knowledge/application/authorized-knowledge-use-case.ts b/apps/sim/lib/knowledge/application/authorized-knowledge-use-case.ts index e2b2cba69f3..125907a75e5 100644 --- a/apps/sim/lib/knowledge/application/authorized-knowledge-use-case.ts +++ b/apps/sim/lib/knowledge/application/authorized-knowledge-use-case.ts @@ -8,6 +8,7 @@ import { type WorkspaceUseCaseAuditEntry, } from '@/lib/core/application' import { authorizeOrganizationOperation } from '@/lib/core/application/organization-authorization' +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' import { OrchestrationError, type OrchestrationRequestContext, @@ -230,11 +231,13 @@ export function defineAuthorizedKnowledgeUseCase< context: resolved.context, request, } - const result = await definition.execute(executionContext) - const resultContext = { ...executionContext, result } - recordOrganizationAudit(resultContext, resolved.context.organizationId) - await definition.afterSuccess?.(resultContext) - return result + return runWithOutboundOrganization(resolved.context.organizationId, async () => { + const result = await definition.execute(executionContext) + const resultContext = { ...executionContext, result } + recordOrganizationAudit(resultContext, resolved.context.organizationId) + await definition.afterSuccess?.(resultContext) + return result + }) }, } } diff --git a/apps/sim/lib/knowledge/connectors/external-group-sync.ts b/apps/sim/lib/knowledge/connectors/external-group-sync.ts index 230a5f92280..221c6802483 100644 --- a/apps/sim/lib/knowledge/connectors/external-group-sync.ts +++ b/apps/sim/lib/knowledge/connectors/external-group-sync.ts @@ -11,6 +11,7 @@ import { getErrorMessage, toError } from '@sim/utils/errors' import { chunkArray } from '@sim/utils/helpers' import { generateId } from '@sim/utils/id' import { and, eq, gt, inArray, isNull, lt, notInArray, or, sql } from 'drizzle-orm' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { resourceScopeColumns, resourceScopeFields, @@ -434,69 +435,77 @@ export async function refreshConnectorDirectory( return 'skipped' } - if ( - !( - await resolveKnowledgeAccessAvailability( - resourceScopeFields(resourceScopeFromOwner(connector)) - ) - ).sourceMirrored - ) { - return 'skipped' - } + return withResourceOutboundScope( + connector, + async (): Promise => { + if ( + !( + await resolveKnowledgeAccessAvailability( + resourceScopeFields(resourceScopeFromOwner(connector)) + ) + ).sourceMirrored + ) { + return 'skipped' + } - const connectorConfig = CONNECTOR_REGISTRY[connector.connectorType] - if (!connectorConfig?.openDirectory) return 'skipped' + const connectorConfig = CONNECTOR_REGISTRY[connector.connectorType] + if (!connectorConfig?.openDirectory) return 'skipped' - const credentialUserId = await resolveConnectorTokenUserId({ - credentialId: connector.credentialId, - ...resourceScopeFields(resourceScopeFromOwner(connector)), - fallbackUserId: connector.knowledgeBaseOwnerId, - }) - if (!credentialUserId) return 'unusable' + const credentialUserId = await resolveConnectorTokenUserId({ + credentialId: connector.credentialId, + ...resourceScopeFields(resourceScopeFromOwner(connector)), + fallbackUserId: connector.knowledgeBaseOwnerId, + }) + if (!credentialUserId) return 'unusable' - const sourceConfig = connector.sourceConfig as Record - const token = await resolveConnectorAccessToken({ - auth: connectorConfig.auth, - accessMode: 'admin', - connector, - userId: credentialUserId, - requestId, - sourceConfig, - }) - if (!token) return 'unusable' + const sourceConfig = connector.sourceConfig as Record + const token = await resolveConnectorAccessToken({ + auth: connectorConfig.auth, + accessMode: 'admin', + connector, + userId: credentialUserId, + requestId, + sourceConfig, + }) + if (!token) return 'unusable' - const recordError = async (lastSyncError: string | null) => { - await db - .update(knowledgeConnector) - .set({ lastSyncError, updatedAt: new Date() }) - .where( - and( - eq(knowledgeConnector.id, connector.id), - eq(knowledgeConnector.updatedAt, connector.updatedAt), - isNull(knowledgeConnector.syncLockToken), - isNull(knowledgeConnector.memberSyncLockToken), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - } - try { - const syncContext = syncContextForToken(token) - await connectorConfig.permissionConfig?.populateSyncContext(connector.id, syncContext) - const outcome = await refreshMirroredDirectory({ - ...resourceScopeFields(resourceScopeFromOwner(connector)), - connectorConfig, - sourceConfig, - syncContext, - accessToken: token.accessToken, - force: connector.lastSyncError?.startsWith(DIRECTORY_ERROR_PREFIX), - }) - if (outcome === 'refreshed' && connector.lastSyncError?.startsWith(DIRECTORY_ERROR_PREFIX)) { - await recordError(null) + const recordError = async (lastSyncError: string | null) => { + await db + .update(knowledgeConnector) + .set({ lastSyncError, updatedAt: new Date() }) + .where( + and( + eq(knowledgeConnector.id, connector.id), + eq(knowledgeConnector.updatedAt, connector.updatedAt), + isNull(knowledgeConnector.syncLockToken), + isNull(knowledgeConnector.memberSyncLockToken), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + } + try { + const syncContext = syncContextForToken(token) + await connectorConfig.permissionConfig?.populateSyncContext(connector.id, syncContext) + const outcome = await refreshMirroredDirectory({ + ...resourceScopeFields(resourceScopeFromOwner(connector)), + connectorConfig, + sourceConfig, + syncContext, + accessToken: token.accessToken, + force: connector.lastSyncError?.startsWith(DIRECTORY_ERROR_PREFIX), + }) + if ( + outcome === 'refreshed' && + connector.lastSyncError?.startsWith(DIRECTORY_ERROR_PREFIX) + ) { + await recordError(null) + } + return outcome + } catch (error) { + await recordError(getErrorMessage(error)) + throw error + } } - return outcome - } catch (error) { - await recordError(getErrorMessage(error)) - throw error - } + ) } diff --git a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts index 186e56e8d72..3357161f6d8 100644 --- a/apps/sim/lib/knowledge/connectors/member-sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/member-sync-engine.ts @@ -21,6 +21,7 @@ import { assertBillingAttributionSnapshot, type BillingAttributionSnapshot, } from '@/lib/billing/core/billing-attribution' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { resourceScopeFields, resourceScopeFromOwner } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' import { @@ -1713,544 +1714,547 @@ export async function executeMemberSync( userId: kbRow.userId, } - const runId = generateId() - const connector = await acquireMemberSyncLock(connectorId, runId, options.dispatchToken) - if (!connector) { - const [current] = await db - .select({ - status: knowledgeConnector.status, - memberSyncStatus: knowledgeConnector.memberSyncStatus, - memberSyncLockToken: knowledgeConnector.memberSyncLockToken, - syncLockToken: knowledgeConnector.syncLockToken, - }) - .from(knowledgeConnector) - .where(eq(knowledgeConnector.id, connectorId)) - .limit(1) - if ( - current?.memberSyncStatus === 'disabled' || - current?.syncLockToken || - (current && !MEMBER_LOCKABLE_CONNECTOR_STATUSES.some((status) => status === current.status)) - ) { - logger.info('Connector is not accepting member syncs, skipping', { - connectorId, - status: current.status, - }) - return skipped(result, 'connector_not_syncable') - } - if (options.dispatchToken && current?.memberSyncLockToken !== options.dispatchToken) { - logger.info('Member sync superseded by a newer dispatch, skipping', { connectorId }) - return skipped(result, 'dispatch_superseded') - } - logger.info('Member sync already in progress, skipping', { connectorId }) - return skipped(result, 'sync_in_progress') - } - - const runStartedAt = new Date() - const run: MemberSyncRun = { - connectorId, - knowledgeBaseId: connector.knowledgeBaseId, - ...resourceScopeFields(resourceScopeFromOwner(kbRow)), - runId, - runStartedAt, - deadlineAt: runStartedAt.getTime() + MEMBER_SYNC_SOFT_BUDGET_SECONDS * 1000, - result, - lease: createMemberSyncLease(connectorId, runId), - } - await insertMemberSyncLog(runId, connectorId, runStartedAt) - - try { - /** - * Where the feature is off — flag, plan, or a flag read that could not - * reach its source — nothing changes: readers already see no member-scoped - * document, and the run waits for the next schedule to look again. - */ - if (!(await isKnowledgeMemberAccessAvailable(run))) { - await deferMemberSync(run, connector.syncIntervalMinutes) - return { - ...skipped(result, 'connector_not_syncable'), - error: 'Per-member access is not available for this workspace', + return withResourceOutboundScope(kbOwner, async (): Promise => { + const runId = generateId() + const connector = await acquireMemberSyncLock(connectorId, runId, options.dispatchToken) + if (!connector) { + const [current] = await db + .select({ + status: knowledgeConnector.status, + memberSyncStatus: knowledgeConnector.memberSyncStatus, + memberSyncLockToken: knowledgeConnector.memberSyncLockToken, + syncLockToken: knowledgeConnector.syncLockToken, + }) + .from(knowledgeConnector) + .where(eq(knowledgeConnector.id, connectorId)) + .limit(1) + if ( + current?.memberSyncStatus === 'disabled' || + current?.syncLockToken || + (current && !MEMBER_LOCKABLE_CONNECTOR_STATUSES.some((status) => status === current.status)) + ) { + logger.info('Connector is not accepting member syncs, skipping', { + connectorId, + status: current.status, + }) + return skipped(result, 'connector_not_syncable') } - } - if (!connector.credentialGroupId || !connector.credentialGroupOptionId) { - await disableMemberSync(run, 'Connector is no longer attached to a Credential Group option') - return { - ...skipped(result, 'connector_not_syncable'), - error: 'Connector is no longer attached to a Credential Group option', + if (options.dispatchToken && current?.memberSyncLockToken !== options.dispatchToken) { + logger.info('Member sync superseded by a newer dispatch, skipping', { connectorId }) + return skipped(result, 'dispatch_superseded') } + logger.info('Member sync already in progress, skipping', { connectorId }) + return skipped(result, 'sync_in_progress') } - if (!connectorConfig.permissionScopedListing || connectorConfig.auth.mode !== 'oauth') { - throw new Error(`Connector ${connectorConfig.id} cannot sync per member`) - } - if (connector.credentialId && !connectorConfig.supportsSeparateContentCredential) { - throw new Error(`${connectorConfig.name} does not support a separate content credential`) - } - const binding = { - credentialGroupId: connector.credentialGroupId, - credentialGroupOptionId: connector.credentialGroupOptionId, - } - const sourceConfig = connector.sourceConfig as Record - if (connector.accessRewritePending && !(await finishPendingAccessRewrite(run))) { - /** The rewrite is not done, so nothing is listed yet; the next run picks it up at once. */ - result.membersRemaining = true - const landed = await completeMemberSync(run, connector.syncIntervalMinutes) - if (!landed) return skipped(result, 'sync_superseded') - logger.info('Member sync spent its budget hiding documents after a mode switch', { - connectorId, - runId, - }) - return result + const runStartedAt = new Date() + const run: MemberSyncRun = { + connectorId, + knowledgeBaseId: connector.knowledgeBaseId, + ...resourceScopeFields(resourceScopeFromOwner(kbRow)), + runId, + runStartedAt, + deadlineAt: runStartedAt.getTime() + MEMBER_SYNC_SOFT_BUDGET_SECONDS * 1000, + result, + lease: createMemberSyncLease(connectorId, runId), } + await insertMemberSyncLog(runId, connectorId, runStartedAt) - const contentDue = - Boolean(connector.listingCheckpoint) || - options.forceContentRefresh || - connector.accessRewritePending || - !connector.lastSyncAt || - connector.syncIntervalMinutes <= 0 || - runStartedAt.getTime() - connector.lastSyncAt.getTime() >= - connector.syncIntervalMinutes * 60_000 - if (connector.credentialId && options.forceContentRefresh) { - /** An interrupted explicit crawl stays due when its continuation no longer carries the force flag. */ - await withMemberLease(run, (tx) => - tx - .update(knowledgeConnector) - .set({ lastSyncAt: null, updatedAt: new Date() }) - .where(stillHoldsMemberSyncLock(connectorId, runId)) - ) - } - const serviceContent = connector.credentialId - ? contentDue - ? await syncDedicatedMemberContent({ - run, - connector, - connectorConfig, - sourceConfig, - kbOwner, - billingAttribution, - }) - : { complete: true } - : undefined - /** - * Anyone who joined the workspace since the last run is invited now, so - * membership grows on its own; the invitation is the only thing they need. - */ - const invited = run.workspaceId - ? await inviteWorkspaceMembersToCredentialGroup({ - workspaceId: run.workspaceId, - credentialGroupId: connector.credentialGroupId, - beforeBatch: run.lease.beatIfDue, - deadlineAt: run.deadlineAt, - }).catch((error) => { - logger.warn('Failed to invite new workspace members during a member run', { - connectorId, - error: getErrorMessage(error), + try { + /** + * Where the feature is off — flag, plan, or a flag read that could not + * reach its source — nothing changes: readers already see no member-scoped + * document, and the run waits for the next schedule to look again. + */ + if (!(await isKnowledgeMemberAccessAvailable(run))) { + await deferMemberSync(run, connector.syncIntervalMinutes) + return { + ...skipped(result, 'connector_not_syncable'), + error: 'Per-member access is not available for this workspace', + } + } + if (!connector.credentialGroupId || !connector.credentialGroupOptionId) { + await disableMemberSync(run, 'Connector is no longer attached to a Credential Group option') + return { + ...skipped(result, 'connector_not_syncable'), + error: 'Connector is no longer attached to a Credential Group option', + } + } + if (!connectorConfig.permissionScopedListing || connectorConfig.auth.mode !== 'oauth') { + throw new Error(`Connector ${connectorConfig.id} cannot sync per member`) + } + if (connector.credentialId && !connectorConfig.supportsSeparateContentCredential) { + throw new Error(`${connectorConfig.name} does not support a separate content credential`) + } + const binding = { + credentialGroupId: connector.credentialGroupId, + credentialGroupOptionId: connector.credentialGroupOptionId, + } + const sourceConfig = connector.sourceConfig as Record + + if (connector.accessRewritePending && !(await finishPendingAccessRewrite(run))) { + /** The rewrite is not done, so nothing is listed yet; the next run picks it up at once. */ + result.membersRemaining = true + const landed = await completeMemberSync(run, connector.syncIntervalMinutes) + if (!landed) return skipped(result, 'sync_superseded') + logger.info('Member sync spent its budget hiding documents after a mode switch', { + connectorId, + runId, + }) + return result + } + + const contentDue = + Boolean(connector.listingCheckpoint) || + options.forceContentRefresh || + connector.accessRewritePending || + !connector.lastSyncAt || + connector.syncIntervalMinutes <= 0 || + runStartedAt.getTime() - connector.lastSyncAt.getTime() >= + connector.syncIntervalMinutes * 60_000 + if (connector.credentialId && options.forceContentRefresh) { + /** An interrupted explicit crawl stays due when its continuation no longer carries the force flag. */ + await withMemberLease(run, (tx) => + tx + .update(knowledgeConnector) + .set({ lastSyncAt: null, updatedAt: new Date() }) + .where(stillHoldsMemberSyncLock(connectorId, runId)) + ) + } + const serviceContent = connector.credentialId + ? contentDue + ? await syncDedicatedMemberContent({ + run, + connector, + connectorConfig, + sourceConfig, + kbOwner, + billingAttribution, + }) + : { complete: true } + : undefined + /** + * Anyone who joined the workspace since the last run is invited now, so + * membership grows on its own; the invitation is the only thing they need. + */ + const invited = run.workspaceId + ? await inviteWorkspaceMembersToCredentialGroup({ + workspaceId: run.workspaceId, + credentialGroupId: connector.credentialGroupId, + beforeBatch: run.lease.beatIfDue, + deadlineAt: run.deadlineAt, + }).catch((error) => { + logger.warn('Failed to invite new workspace members during a member run', { + connectorId, + error: getErrorMessage(error), + }) + return null }) - return null + : null + if (invited && invited.invited > 0) { + logger.info('Invited new workspace members to the connector credential group', { + connectorId, + ...invited, }) - : null - if (invited && invited.invited > 0) { - logger.info('Invited new workspace members to the connector credential group', { - connectorId, - ...invited, - }) - } - if ( - !(await reconcileMembership( - run, - binding, - connector.directoryCheckpoint, - Boolean(options.forceContentRefresh) - )) - ) { - result.membersRemaining = true - if (!(await completeMemberSync(run, connector.syncIntervalMinutes))) - return skipped(result, 'sync_superseded') - return result - } + } + if ( + !(await reconcileMembership( + run, + binding, + connector.directoryCheckpoint, + Boolean(options.forceContentRefresh) + )) + ) { + result.membersRemaining = true + if (!(await completeMemberSync(run, connector.syncIntervalMinutes))) + return skipped(result, 'sync_superseded') + return result + } - const credentialIdByMemberId = new Map() - const tokens = createMemberTokenCache({ - run, - connectorConfig, - credentialIdByMemberId, - sourceConfig, - }) + const credentialIdByMemberId = new Map() + const tokens = createMemberTokenCache({ + run, + connectorConfig, + credentialIdByMemberId, + sourceConfig, + }) - while (Date.now() < run.deadlineAt) { - const member = await claimNextMember(run) - if (!member) break - result.membersClaimed += 1 - credentialIdByMemberId.clear() - credentialIdByMemberId.set(member.id, member.credentialId) - const syncContext: Record = { - syncRunId: runId, - memberId: member.id, - ...PER_MEMBER_LISTING_CONTEXT, - } - let contentFailures = false - const processPage = async ( - documents: ExternalDocument[], - checkpoint: ListingCheckpoint, - durableCheckpoint = true - ) => { - const externalIds = documents.map((item) => item.externalId) - const observeAttempted = async (attempted: ExternalDocument[]) => { - if (attempted.length === 0) return - const documentIds = [ - ...( - await loadDocumentIdsByExternalId( - connectorId, - attempted.map((item) => item.externalId) + while (Date.now() < run.deadlineAt) { + const member = await claimNextMember(run) + if (!member) break + result.membersClaimed += 1 + credentialIdByMemberId.clear() + credentialIdByMemberId.set(member.id, member.credentialId) + const syncContext: Record = { + syncRunId: runId, + memberId: member.id, + ...PER_MEMBER_LISTING_CONTEXT, + } + let contentFailures = false + const processPage = async ( + documents: ExternalDocument[], + checkpoint: ListingCheckpoint, + durableCheckpoint = true + ) => { + const externalIds = documents.map((item) => item.externalId) + const observeAttempted = async (attempted: ExternalDocument[]) => { + if (attempted.length === 0) return + const documentIds = [ + ...( + await loadDocumentIdsByExternalId( + connectorId, + attempted.map((item) => item.externalId) + ) + ).values(), + ] + await withMemberLease(run, async (tx) => { + result.observationsAdded += await recordMemberObservations( + tx, + member.id, + documentIds, + checkpoint.generationId ) - ).values(), - ] - await withMemberLease(run, async (tx) => { - result.observationsAdded += await recordMemberObservations( - tx, - member.id, - documentIds, - checkpoint.generationId - ) - await materializeDocumentAcls(connectorId, documentIds, tx) - if (durableCheckpoint && checkpoint.contentFailures) { - await tx - .update(knowledgeConnectorMember) - .set({ listingCheckpoint: checkpoint }) - .where(eq(knowledgeConnectorMember.id, member.id)) - } - if (!serviceContent) { - for (let offset = 0; offset < documentIds.length; offset += 500) { + await materializeDocumentAcls(connectorId, documentIds, tx) + if (durableCheckpoint && checkpoint.contentFailures) { await tx - .update(document) - .set({ sourceSeenAt: run.runStartedAt }) - .where( - and( - eq(document.connectorId, connectorId), - inArray(document.id, documentIds.slice(offset, offset + 500)) + .update(knowledgeConnectorMember) + .set({ listingCheckpoint: checkpoint }) + .where(eq(knowledgeConnectorMember.id, member.id)) + } + if (!serviceContent) { + for (let offset = 0; offset < documentIds.length; offset += 500) { + await tx + .update(document) + .set({ sourceSeenAt: run.runStartedAt }) + .where( + and( + eq(document.connectorId, connectorId), + inArray(document.id, documentIds.slice(offset, offset + 500)) + ) ) - ) + } } - } - }) - result.docsListed += attempted.length - } - if (!serviceContent) { - const corpus = await loadPageCorpus(connectorId, externalIds) - const pageState = createSyncRunState(result) - let rejectedCredentialError: Error | undefined - /** Commit sibling observations and failures before capacity pressure can end this page. */ - const persistAttempted = async (attempted: ExternalDocument[]) => { - if (rejectedCredentialError) throw rejectedCredentialError - if (attempted.some((item) => pageState.failedExternalIds.has(item.externalId))) { - await persistSourceDocumentFailures({ - knowledgeBaseId: connector.knowledgeBaseId, - connectorId, - connectorType: connector.connectorType, - documents: attempted, - failedExternalIds: pageState.failedExternalIds, - sourceFailures: pageState.sourceFailures, - priorByExternalId: corpus.priorByExternalId, - sourceConfig, - access: 'members', - lease: run.lease, - }) - checkpoint.contentFailures = true - contentFailures = true - result.listingIncomplete = true - } - await observeAttempted(attempted) + }) + result.docsListed += attempted.length } - const pendingOps = classifyListing({ - externalDocs: documents.filter((item) => { - const alreadyRead = corpus.priorByExternalId.get(item.externalId)?.sourceSeenAt - if ( - alreadyRead && - alreadyRead >= run.runStartedAt && - corpus.priorByExternalId.get(item.externalId)?.contentHash !== null - ) { - result.docsUnchanged += 1 - return false + if (!serviceContent) { + const corpus = await loadPageCorpus(connectorId, externalIds) + const pageState = createSyncRunState(result) + let rejectedCredentialError: Error | undefined + /** Commit sibling observations and failures before capacity pressure can end this page. */ + const persistAttempted = async (attempted: ExternalDocument[]) => { + if (rejectedCredentialError) throw rejectedCredentialError + if (attempted.some((item) => pageState.failedExternalIds.has(item.externalId))) { + await persistSourceDocumentFailures({ + knowledgeBaseId: connector.knowledgeBaseId, + connectorId, + connectorType: connector.connectorType, + documents: attempted, + failedExternalIds: pageState.failedExternalIds, + sourceFailures: pageState.sourceFailures, + priorByExternalId: corpus.priorByExternalId, + sourceConfig, + access: 'members', + lease: run.lease, + }) + checkpoint.contentFailures = true + contentFailures = true + result.listingIncomplete = true } - return true - }), - corpus, - forceRehydrate: false, - state: pageState, - }) - const pendingIds = new Set(pendingOps.map((op) => op.extDoc.externalId)) - await persistAttempted(documents.filter((item) => !pendingIds.has(item.externalId))) - const finished = await processDocOps({ - connectorId, - connector, - sourceConfig, - kbOwner, - billingAttribution, - pendingOps, - corpus, - forceRehydrate: false, - state: pageState, - hydration: { - concurrency: connectorConfig.contentConcurrency, - getDocument: async (externalId) => { - if (rejectedCredentialError) throw rejectedCredentialError - try { - return await connectorConfig.getDocument( - await tokens.get(member.id), - sourceConfig, - externalId, - syncContext - ) - } catch (error) { - if (connectorConfig.isCredentialInvalidError?.(error) === true) { - rejectedCredentialError = toError(error) - if (await tokens.reject(member.id)) - await recordMemberFailure( - run, - member, - error, - connector.syncIntervalMinutes, - true - ) - } - throw error + await observeAttempted(attempted) + } + const pendingOps = classifyListing({ + externalDocs: documents.filter((item) => { + const alreadyRead = corpus.priorByExternalId.get(item.externalId)?.sourceSeenAt + if ( + alreadyRead && + alreadyRead >= run.runStartedAt && + corpus.priorByExternalId.get(item.externalId)?.contentHash !== null + ) { + result.docsUnchanged += 1 + return false } + return true + }), + corpus, + forceRehydrate: false, + state: pageState, + }) + const pendingIds = new Set(pendingOps.map((op) => op.extDoc.externalId)) + await persistAttempted(documents.filter((item) => !pendingIds.has(item.externalId))) + const finished = await processDocOps({ + connectorId, + connector, + sourceConfig, + kbOwner, + billingAttribution, + pendingOps, + corpus, + forceRehydrate: false, + state: pageState, + hydration: { + concurrency: connectorConfig.contentConcurrency, + getDocument: async (externalId) => { + if (rejectedCredentialError) throw rejectedCredentialError + try { + return await connectorConfig.getDocument( + await tokens.get(member.id), + sourceConfig, + externalId, + syncContext + ) + } catch (error) { + if (connectorConfig.isCredentialInvalidError?.(error) === true) { + rejectedCredentialError = toError(error) + if (await tokens.reject(member.id)) + await recordMemberFailure( + run, + member, + error, + connector.syncIntervalMinutes, + true + ) + } + throw error + } + }, }, - }, - lease: run.lease, - documentAccess: 'members', - deadlineAt: durableCheckpoint ? run.deadlineAt : undefined, - onBatchComplete: async (attempted) => { - result.docsHydratedOnce += attempted.filter((item) => item.contentDeferred).length - await persistAttempted(attempted) - }, - }) - if (rejectedCredentialError) throw rejectedCredentialError - if (!finished) return false - } else { - await observeAttempted(documents) + lease: run.lease, + documentAccess: 'members', + deadlineAt: durableCheckpoint ? run.deadlineAt : undefined, + onBatchComplete: async (attempted) => { + result.docsHydratedOnce += attempted.filter((item) => item.contentDeferred).length + await persistAttempted(attempted) + }, + }) + if (rejectedCredentialError) throw rejectedCredentialError + if (!finished) return false + } else { + await observeAttempted(documents) + } } - } - const listed = await listForMember({ - run, - member, - connectorConfig, - sourceConfig, - tokens, - syncContext, - syncIntervalMinutes: connector.syncIntervalMinutes, - forceFull: Boolean( - serviceContent && - (result.docsAdded > 0 || - (connector.lastSyncAt && - (!member.memberSyncedThrough || member.memberSyncedThrough < connector.lastSyncAt))) - ), - processPage, - }) - if (listed.kind === 'failed') continue - if (listed.checkpoint?.contentFailures) result.listingIncomplete = true - if (listed.documents.length > 0) { - await processPage( - listed.documents, - beginListingCheckpoint({ - fingerprint: listingFingerprint({ connectorId, memberId: member.id }), - generationId: runId, - startedAt: listed.startedAt, - }), - false + const listed = await listForMember({ + run, + member, + connectorConfig, + sourceConfig, + tokens, + syncContext, + syncIntervalMinutes: connector.syncIntervalMinutes, + forceFull: Boolean( + serviceContent && + (result.docsAdded > 0 || + (connector.lastSyncAt && + (!member.memberSyncedThrough || + member.memberSyncedThrough < connector.lastSyncAt))) + ), + processPage, + }) + if (listed.kind === 'failed') continue + if (listed.checkpoint?.contentFailures) result.listingIncomplete = true + if (listed.documents.length > 0) { + await processPage( + listed.documents, + beginListingCheckpoint({ + fingerprint: listingFingerprint({ connectorId, memberId: member.id }), + generationId: runId, + startedAt: listed.startedAt, + }), + false + ) + } + const listedCount = listed.checkpoint?.listedCount ?? listed.documents.length + const suspect = + listed.mode === 'full' && + !listed.authoritative && + listed.complete && + classifySuspectListing(listedCount, member.lastListedCount ?? 0) !== null + const outcome: MemberListingOutcome = { + member, + mode: listed.mode, + listingStartedAt: listed.startedAt, + seenExternalIds: new Set(listed.documents.map((doc) => doc.externalId)), + removedExternalIds: listed.removedExternalIds, + listedCount, + complete: listed.complete, + resumable: listed.resumable, + suspect, + contentFailures: contentFailures || Boolean(listed.checkpoint?.contentFailures), + changeCursor: suspect ? undefined : listed.changeCursor, + checkpoint: listed.checkpoint, + observationRunId: listed.observationRunId, + } + const relevantIds = [...outcome.seenExternalIds, ...outcome.removedExternalIds] + const affected = await applyMemberListing( + run, + outcome, + await loadDocumentIdsByExternalId(connectorId, relevantIds), + connector.syncIntervalMinutes ) + await withMemberLease(run, (tx) => materializeDocumentAcls(connectorId, affected, tx)) } - const listedCount = listed.checkpoint?.listedCount ?? listed.documents.length - const suspect = - listed.mode === 'full' && - !listed.authoritative && - listed.complete && - classifySuspectListing(listedCount, member.lastListedCount ?? 0) !== null - const outcome: MemberListingOutcome = { - member, - mode: listed.mode, - listingStartedAt: listed.startedAt, - seenExternalIds: new Set(listed.documents.map((doc) => doc.externalId)), - removedExternalIds: listed.removedExternalIds, - listedCount, - complete: listed.complete, - resumable: listed.resumable, - suspect, - contentFailures: contentFailures || Boolean(listed.checkpoint?.contentFailures), - changeCursor: suspect ? undefined : listed.changeCursor, - checkpoint: listed.checkpoint, - observationRunId: listed.observationRunId, - } - const relevantIds = [...outcome.seenExternalIds, ...outcome.removedExternalIds] - const affected = await applyMemberListing( - run, - outcome, - await loadDocumentIdsByExternalId(connectorId, relevantIds), - connector.syncIntervalMinutes - ) - await withMemberLease(run, (tx) => materializeDocumentAcls(connectorId, affected, tx)) - } - /** A service-owned corpus outlives its last observer; only the content pass removes it. */ - if (!serviceContent) { - /** - * Nobody has completed a listing yet — a connector that just entered - * members mode, waiting for its first member to connect — so an - * unobserved document says nothing about access and must not be - * tombstoned, let alone purged a week later. - */ - const [listed] = await db - .select({ count: sql`count(*)::int` }) - .from(knowledgeConnectorMember) - .where( - and( - eq(knowledgeConnectorMember.connectorId, connectorId), - sql`${knowledgeConnectorMember.lastCompleteListingAt} IS NOT NULL` + /** A service-owned corpus outlives its last observer; only the content pass removes it. */ + if (!serviceContent) { + /** + * Nobody has completed a listing yet — a connector that just entered + * members mode, waiting for its first member to connect — so an + * unobserved document says nothing about access and must not be + * tombstoned, let alone purged a week later. + */ + const [listed] = await db + .select({ count: sql`count(*)::int` }) + .from(knowledgeConnectorMember) + .where( + and( + eq(knowledgeConnectorMember.connectorId, connectorId), + sql`${knowledgeConnectorMember.lastCompleteListingAt} IS NOT NULL` + ) ) - ) - const lifecycle = await applyMemberDocumentLifecycle({ + const lifecycle = await applyMemberDocumentLifecycle({ + connectorId, + knowledgeBaseId: connector.knowledgeBaseId, + runId, + lease: run.lease, + withLease: (fn) => withMemberLease(run, fn), + deadlineAt: run.deadlineAt, + allowRemoval: (listed?.count ?? 0) > 0, + }) + result.docsTombstoned = lifecycle.tombstoned + result.docsResurrected = lifecycle.resurrected + result.docsPurged = lifecycle.purged + result.docsDeleted = lifecycle.purged + result.membersRemaining = !lifecycle.finished + } + + await sweepStuckDocuments({ connectorId, knowledgeBaseId: connector.knowledgeBaseId, - runId, + syncStartedAt: runStartedAt, + retryCutoff: new Date(Date.now() - RETRY_WINDOW_DAYS * 24 * 60 * 60 * 1000), + billingAttribution, + result, lease: run.lease, - withLease: (fn) => withMemberLease(run, fn), - deadlineAt: run.deadlineAt, - allowRemoval: (listed?.count ?? 0) > 0, }) - result.docsTombstoned = lifecycle.tombstoned - result.docsResurrected = lifecycle.resurrected - result.docsPurged = lifecycle.purged - result.docsDeleted = lifecycle.purged - result.membersRemaining = !lifecycle.finished - } - await sweepStuckDocuments({ - connectorId, - knowledgeBaseId: connector.knowledgeBaseId, - syncStartedAt: runStartedAt, - retryCutoff: new Date(Date.now() - RETRY_WINDOW_DAYS * 24 * 60 * 60 * 1000), - billingAttribution, - result, - lease: run.lease, - }) - - result.membersRemaining = - result.membersRemaining || - serviceContent?.complete === false || - (await countDueMembers(run, binding)) > 0 - const landed = await completeMemberSync(run, connector.syncIntervalMinutes) - if (!landed) { - logger.warn( - 'Member sync result discarded — connector was reclaimed while this run was executing', - { - connectorId, - runId, - } - ) - return skipped(result, 'sync_superseded') - } - logger.info('Member sync completed', { connectorId, runId, ...result }) - return result - } catch (error) { - if (error instanceof SyncLockLostException) { - logger.warn('Member sync abandoned — lock was reclaimed while this run was executing', { - connectorId, - runId, - }) - return skipped(result, 'sync_superseded') - } - if (error instanceof ConnectorDeletedException) { - logger.info('Connector deleted during member sync', { connectorId }) - await failMemberSyncLog(runId, result, 'Connector deleted during sync').catch((logError) => - logger.error('Failed to record member sync failure', { - connectorId, - error: getErrorMessage(logError), - }) - ) - return skipped(result, 'connector_deleted_during_sync') - } - if (error instanceof MemberBindingGoneError) { - try { - await disableMemberSync(run, error.message) - } catch (disableError) { - if (!(disableError instanceof SyncLockLostException)) throw disableError - logger.warn('Member sync abandoned — lock was reclaimed before it could be disabled', { + result.membersRemaining = + result.membersRemaining || + serviceContent?.complete === false || + (await countDueMembers(run, binding)) > 0 + const landed = await completeMemberSync(run, connector.syncIntervalMinutes) + if (!landed) { + logger.warn( + 'Member sync result discarded — connector was reclaimed while this run was executing', + { + connectorId, + runId, + } + ) + return skipped(result, 'sync_superseded') + } + logger.info('Member sync completed', { connectorId, runId, ...result }) + return result + } catch (error) { + if (error instanceof SyncLockLostException) { + logger.warn('Member sync abandoned — lock was reclaimed while this run was executing', { connectorId, runId, }) return skipped(result, 'sync_superseded') } - return { ...skipped(result, 'connector_not_syncable'), error: error.message } - } + if (error instanceof ConnectorDeletedException) { + logger.info('Connector deleted during member sync', { connectorId }) + await failMemberSyncLog(runId, result, 'Connector deleted during sync').catch((logError) => + logger.error('Failed to record member sync failure', { + connectorId, + error: getErrorMessage(logError), + }) + ) + return skipped(result, 'connector_deleted_during_sync') + } + if (error instanceof MemberBindingGoneError) { + try { + await disableMemberSync(run, error.message) + } catch (disableError) { + if (!(disableError instanceof SyncLockLostException)) throw disableError + logger.warn('Member sync abandoned — lock was reclaimed before it could be disabled', { + connectorId, + runId, + }) + return skipped(result, 'sync_superseded') + } + return { ...skipped(result, 'connector_not_syncable'), error: error.message } + } - if (getConnectorSyncDeferral(error)) { - try { - result.deferred = await deferConnectorSync({ - connectorId, - knowledgeBaseId: connector.knowledgeBaseId, - runId, - lease: run.lease, - kind: 'member', - result, - error, - }) - result.listingIncomplete = true - logger.info('Member source sync deferred', { connectorId, ...result.deferred }) - return result - } catch (persistenceError) { - logger.error('Failed to persist member source deferral', { - connectorId, - error: - getConnectorFailureDiagnostic(persistenceError)?.message ?? - toError(persistenceError).message, - }) - result.error = 'Could not persist the member sync retry after provider deferral' - return result + if (getConnectorSyncDeferral(error)) { + try { + result.deferred = await deferConnectorSync({ + connectorId, + knowledgeBaseId: connector.knowledgeBaseId, + runId, + lease: run.lease, + kind: 'member', + result, + error, + }) + result.listingIncomplete = true + logger.info('Member source sync deferred', { connectorId, ...result.deferred }) + return result + } catch (persistenceError) { + logger.error('Failed to persist member source deferral', { + connectorId, + error: + getConnectorFailureDiagnostic(persistenceError)?.message ?? + toError(persistenceError).message, + }) + result.error = 'Could not persist the member sync retry after provider deferral' + return result + } } - } - const diagnostic = getConnectorFailureDiagnostic(error) - const errorMessage = diagnostic?.message ?? toError(error).message - const retryAfterMs = getRetryAfterMs(error) - logger.error('Member sync failed', { connectorId, runId, error: errorMessage, diagnostic }) - try { - await failMemberSyncLog(runId, result, errorMessage) - const failureUpdate = - error instanceof ConnectorSyncCapacityError - ? { - memberSyncStatus: 'error' as const, - lastMemberSyncError: errorMessage, - nextMemberSyncAt: null, - memberSyncConsecutiveFailures: connector.memberSyncConsecutiveFailures, - memberSyncLockToken: null, - memberSyncLockLeaseAt: null, - updatedAt: new Date(), - } - : buildMemberSyncFailureUpdate( - new Date(), - connector.memberSyncConsecutiveFailures, - errorMessage, - retryAfterMs - ) - const written = await db - .update(knowledgeConnector) - .set(failureUpdate) - .where(stillHoldsMemberSyncLock(connectorId, runId)) - .returning({ id: knowledgeConnector.id }) - if (written.length === 0) { - logger.warn('Member sync failure discarded — connector was reclaimed', { + const diagnostic = getConnectorFailureDiagnostic(error) + const errorMessage = diagnostic?.message ?? toError(error).message + const retryAfterMs = getRetryAfterMs(error) + logger.error('Member sync failed', { connectorId, runId, error: errorMessage, diagnostic }) + try { + await failMemberSyncLog(runId, result, errorMessage) + const failureUpdate = + error instanceof ConnectorSyncCapacityError + ? { + memberSyncStatus: 'error' as const, + lastMemberSyncError: errorMessage, + nextMemberSyncAt: null, + memberSyncConsecutiveFailures: connector.memberSyncConsecutiveFailures, + memberSyncLockToken: null, + memberSyncLockLeaseAt: null, + updatedAt: new Date(), + } + : buildMemberSyncFailureUpdate( + new Date(), + connector.memberSyncConsecutiveFailures, + errorMessage, + retryAfterMs + ) + const written = await db + .update(knowledgeConnector) + .set(failureUpdate) + .where(stillHoldsMemberSyncLock(connectorId, runId)) + .returning({ id: knowledgeConnector.id }) + if (written.length === 0) { + logger.warn('Member sync failure discarded — connector was reclaimed', { + connectorId, + runId, + }) + } + } catch (recoveryError) { + logger.error('Failed to record member sync failure', { connectorId, - runId, + error: + getConnectorFailureDiagnostic(recoveryError)?.message ?? toError(recoveryError).message, }) } - } catch (recoveryError) { - logger.error('Failed to record member sync failure', { - connectorId, - error: - getConnectorFailureDiagnostic(recoveryError)?.message ?? toError(recoveryError).message, - }) + result.error = errorMessage + return result } - result.error = errorMessage - return result - } + }) } diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.ts b/apps/sim/lib/knowledge/connectors/sync-engine.ts index d1e9ef65b85..78bec1bf562 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.ts @@ -15,6 +15,7 @@ import { assertBillingAttributionSnapshot, type BillingAttributionSnapshot, } from '@/lib/billing/core/billing-attribution' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { resourceScopeFields, resourceScopeFromOwner } from '@/lib/core/resource-scope' import { EMPTY_ACL } from '@/lib/knowledge/access/tokens' import { @@ -805,543 +806,551 @@ export async function executeSync( ) } assertBillingAttributionOwner(billingAttribution, kbOwner) - /** - * Identifies this run for the terminal writes. Generated before the CAS and - * written by it, so ownership is established atomically with the lock — and - * reused as the sync-log row id, which makes the connector row point at the - * run that holds it. - */ - const syncLogId = generateId() - - const lockResult = await db - .update(knowledgeConnector) - .set(buildSyncLockAcquisition(syncLogId, new Date())) - .where( - and( - inArray(knowledgeConnector.accessMode, CONTENT_ENGINE_ACCESS_MODES), - eq(knowledgeConnector.id, connectorId), - inArray(knowledgeConnector.status, LOCKABLE_CONNECTOR_STATUSES), - /** - * Proves this run is consuming the queue entry that was made for it. - * - * A task delayed past the lease is reclaimed and replaced, and the - * status check alone would let that stale task take the replacement's - * entry — running superseded options (a plain sync where the user had - * just asked for a full resync) while the replacement is turned away as - * `sync_in_progress`. Matching the token is the same discipline - * {@link holdsSyncLockToken} already applies to the `syncing` phase, - * extended to the phase before it. - */ - ...(options.dispatchToken - ? [eq(knowledgeConnector.syncLockToken, options.dispatchToken)] - : []), - isNull(knowledgeConnector.archivedAt), - isNull(knowledgeConnector.deletedAt) - ) - ) - .returning() - - if (lockResult.length === 0) { + return withResourceOutboundScope(kbOwner, async (): Promise => { /** - * Distinguishes the two ways the CAS can find no row. Costs one read on a - * path that already decided not to work, and the alternative is reporting a - * connector someone paused as a concurrency conflict. + * Identifies this run for the terminal writes. Generated before the CAS and + * written by it, so ownership is established atomically with the lock — and + * reused as the sync-log row id, which makes the connector row point at the + * run that holds it. */ - const [current] = await db - .select({ - status: knowledgeConnector.status, - syncLockToken: knowledgeConnector.syncLockToken, - }) - .from(knowledgeConnector) - .where(eq(knowledgeConnector.id, connectorId)) - .limit(1) + const syncLogId = generateId() - /** - * Status is checked before ownership because pausing a queued connector - * releases its token, so a mismatch is the *symptom* there and the status is - * the actual reason. Testing ownership first would report every - * pause-while-queued — the common case — as a superseded dispatch, losing - * the distinction this branch exists to draw. - */ - if (current?.status === 'paused' || current?.status === 'disabled') { - logger.info('Connector is not accepting syncs, skipping', { - connectorId, - status: current.status, - }) - return { ...result, skipReason: 'connector_not_syncable' } - } + const lockResult = await db + .update(knowledgeConnector) + .set(buildSyncLockAcquisition(syncLogId, new Date())) + .where( + and( + inArray(knowledgeConnector.accessMode, CONTENT_ENGINE_ACCESS_MODES), + eq(knowledgeConnector.id, connectorId), + inArray(knowledgeConnector.status, LOCKABLE_CONNECTOR_STATUSES), + /** + * Proves this run is consuming the queue entry that was made for it. + * + * A task delayed past the lease is reclaimed and replaced, and the + * status check alone would let that stale task take the replacement's + * entry — running superseded options (a plain sync where the user had + * just asked for a full resync) while the replacement is turned away as + * `sync_in_progress`. Matching the token is the same discipline + * {@link holdsSyncLockToken} already applies to the `syncing` phase, + * extended to the phase before it. + */ + ...(options.dispatchToken + ? [eq(knowledgeConnector.syncLockToken, options.dispatchToken)] + : []), + isNull(knowledgeConnector.archivedAt), + isNull(knowledgeConnector.deletedAt) + ) + ) + .returning() - if (options.dispatchToken && current?.syncLockToken !== options.dispatchToken) { - logger.info('Sync superseded by a newer dispatch, skipping', { connectorId }) - return { ...result, skipReason: 'dispatch_superseded' } - } + if (lockResult.length === 0) { + /** + * Distinguishes the two ways the CAS can find no row. Costs one read on a + * path that already decided not to work, and the alternative is reporting a + * connector someone paused as a concurrency conflict. + */ + const [current] = await db + .select({ + status: knowledgeConnector.status, + syncLockToken: knowledgeConnector.syncLockToken, + }) + .from(knowledgeConnector) + .where(eq(knowledgeConnector.id, connectorId)) + .limit(1) - logger.info('Sync already in progress, skipping', { connectorId }) - return { ...result, skipReason: 'sync_in_progress' } - } + /** + * Status is checked before ownership because pausing a queued connector + * releases its token, so a mismatch is the *symptom* there and the status is + * the actual reason. Testing ownership first would report every + * pause-while-queued — the common case — as a superseded dispatch, losing + * the distinction this branch exists to draw. + */ + if (current?.status === 'paused' || current?.status === 'disabled') { + logger.info('Connector is not accepting syncs, skipping', { + connectorId, + status: current.status, + }) + return { ...result, skipReason: 'connector_not_syncable' } + } - /** - * The row returned by the lock is the authoritative sync snapshot. A source update - * committed before the lock is included here; one attempted after it sees `syncing` - * and conflicts instead of letting this worker process stale configuration. - */ - const connector = lockResult[0] - /** The lock CAS only takes a content-engine row; this is the type's word for the same fact. */ - if (!isContentEngineAccessMode(connector.accessMode)) { - throw new Error(`Connector ${connectorId} left the content engine's modes while locked`) - } - const accessMode = connector.accessMode - const mirrored = mirrorsSourceAcls(connector.accessMode) - const sourceConfig = connector.sourceConfig as Record - const syncStartedAt = new Date() - const lease = createContentSyncLease(connectorId, syncLogId) - await db.insert(knowledgeConnectorSyncLog).values({ - id: syncLogId, - connectorId, - status: 'started', - startedAt: syncStartedAt, - }) + if (options.dispatchToken && current?.syncLockToken !== options.dispatchToken) { + logger.info('Sync superseded by a newer dispatch, skipping', { connectorId }) + return { ...result, skipReason: 'dispatch_superseded' } + } + + logger.info('Sync already in progress, skipping', { connectorId }) + return { ...result, skipReason: 'sync_in_progress' } + } - try { /** - * OAuth credentials are workspace-scoped and shared, so the member who authorized - * one is often not the knowledge base owner. Resolve the credential's own account - * owner — token reads are scoped to `account.userId`, so passing the KB owner - * resolves no token at all. Resolved once here rather than inside - * `resolveAccessToken` so per-page refreshes don't repeat the lookup. + * The row returned by the lock is the authoritative sync snapshot. A source update + * committed before the lock is included here; one attempted after it sees `syncing` + * and conflicts instead of letting this worker process stale configuration. */ - const credentialUserId = await resolveConnectorTokenUserId({ - credentialId: connector.credentialId, - ...resourceScopeFields(resourceScopeFromOwner(kbOwner)), - fallbackUserId: userId, - }) - if (!credentialUserId) { - throw new Error( - `Credential ${connector.credentialId} is not usable from workspace ${kbOwner.workspaceId} — reconnect the credential` - ) + const connector = lockResult[0] + /** The lock CAS only takes a content-engine row; this is the type's word for the same fact. */ + if (!isContentEngineAccessMode(connector.accessMode)) { + throw new Error(`Connector ${connectorId} left the content engine's modes while locked`) } + const accessMode = connector.accessMode + const mirrored = mirrorsSourceAcls(connector.accessMode) + const sourceConfig = connector.sourceConfig as Record + const syncStartedAt = new Date() + const lease = createContentSyncLease(connectorId, syncLogId) + await db.insert(knowledgeConnectorSyncLog).values({ + id: syncLogId, + connectorId, + status: 'started', + startedAt: syncStartedAt, + }) - let credentialToken = await resolveAccessToken( - connector, - connectorConfig, - credentialUserId, - sourceConfig, - accessMode - ) - /** Re-resolves the token for every OAuth call after the first, so a long run outlives a short-lived token. */ - const refreshOAuthToken = async (): Promise => { - if (connectorConfig.auth.mode === 'oauth') { - credentialToken = await resolveAccessToken( - connector, - connectorConfig, - credentialUserId, - sourceConfig, - accessMode + try { + /** + * OAuth credentials are workspace-scoped and shared, so the member who authorized + * one is often not the knowledge base owner. Resolve the credential's own account + * owner — token reads are scoped to `account.userId`, so passing the KB owner + * resolves no token at all. Resolved once here rather than inside + * `resolveAccessToken` so per-page refreshes don't repeat the lookup. + */ + const credentialUserId = await resolveConnectorTokenUserId({ + credentialId: connector.credentialId, + ...resourceScopeFields(resourceScopeFromOwner(kbOwner)), + fallbackUserId: userId, + }) + if (!credentialUserId) { + throw new Error( + `Credential ${connector.credentialId} is not usable from workspace ${kbOwner.workspaceId} — reconnect the credential` ) } - } - /** - * A credential that already knows its cloud id seeds the same `syncContext` - * slot the connector would otherwise memoise it into. Confluence discovers - * it by calling `accessible-resources` with a bearer token; an Atlassian - * service account holds an API token that cannot make that call, so for it - * the seed is the only source. Connectors need no service-account branch. - */ - const syncContext: Record = { - syncRunId: generateId(), - ...syncContextForToken(credentialToken), - /** Tells a connector to carry permissions with its listing; without it, none are read. */ - ...(mirrored ? { mirrorsSourceAcls: true } : {}), - } - if (mirrored) - await connectorConfig.permissionConfig?.populateSyncContext(connectorId, syncContext) + let credentialToken = await resolveAccessToken( + connector, + connectorConfig, + credentialUserId, + sourceConfig, + accessMode + ) + /** Re-resolves the token for every OAuth call after the first, so a long run outlives a short-lived token. */ + const refreshOAuthToken = async (): Promise => { + if (connectorConfig.auth.mode === 'oauth') { + credentialToken = await resolveAccessToken( + connector, + connectorConfig, + credentialUserId, + sourceConfig, + accessMode + ) + } + } + + /** + * A credential that already knows its cloud id seeds the same `syncContext` + * slot the connector would otherwise memoise it into. Confluence discovers + * it by calling `accessible-resources` with a bearer token; an Atlassian + * service account holds an API token that cannot make that call, so for it + * the seed is the only source. Connectors need no service-account branch. + */ + const syncContext: Record = { + syncRunId: generateId(), + ...syncContextForToken(credentialToken), + /** Tells a connector to carry permissions with its listing; without it, none are read. */ + ...(mirrored ? { mirrorsSourceAcls: true } : {}), + } + if (mirrored) + await connectorConfig.permissionConfig?.populateSyncContext(connectorId, syncContext) - // Shared cutoff for both the tombstone-retry bound below and the stuck-document - // retry near the end of this sync — same RETRY_WINDOW_DAYS window, one computation. - const retryCutoff = new Date(Date.now() - RETRY_WINDOW_DAYS * 24 * 60 * 60 * 1000) + // Shared cutoff for both the tombstone-retry bound below and the stuck-document + // retry near the end of this sync — same RETRY_WINDOW_DAYS window, one computation. + const retryCutoff = new Date(Date.now() - RETRY_WINDOW_DAYS * 24 * 60 * 60 * 1000) - /** - * Bounded to the same retry window as the stuck-document retry below: a - * document whose refresh keeps failing every sync (e.g. permanently - * oversized) would otherwise be a tombstone that never resolves, forcing a - * full listing — and its listing-time overhead — for this connector - * forever. Past the window, this connector stops forcing full syncs on its - * account; the document itself is unaffected and stays tombstoned either way. - * - * Known accepted trade-off: once past the window, a still-tombstoned - * document that's unchanged-but-genuinely-present at the source can only - * be resurrected by a full listing — and nothing here forces one anymore. - * On a connector that never runs a full sync again (persistent incremental - * syncMode, no manual full resync), that document stays correctly - * invisible (excluded everywhere by `isNull(deletedAt)`, so no - * search/billing/listing leakage) but unresolved indefinitely. This is - * deliberately not "fixed" by hard-deleting it after the window expires — - * that would delete a document we have no positive evidence is actually - * gone, reintroducing the exact risk this whole design exists to avoid. - */ - const hasTombstonedDocs = await db - .select({ id: document.id }) - .from(document) - .where( - and( - eq(document.connectorId, connectorId), - isNull(document.archivedAt), - or( - and(isNotNull(document.deletedAt), gt(document.deletedAt, retryCutoff)), - isNull(document.contentHash) + /** + * Bounded to the same retry window as the stuck-document retry below: a + * document whose refresh keeps failing every sync (e.g. permanently + * oversized) would otherwise be a tombstone that never resolves, forcing a + * full listing — and its listing-time overhead — for this connector + * forever. Past the window, this connector stops forcing full syncs on its + * account; the document itself is unaffected and stays tombstoned either way. + * + * Known accepted trade-off: once past the window, a still-tombstoned + * document that's unchanged-but-genuinely-present at the source can only + * be resurrected by a full listing — and nothing here forces one anymore. + * On a connector that never runs a full sync again (persistent incremental + * syncMode, no manual full resync), that document stays correctly + * invisible (excluded everywhere by `isNull(deletedAt)`, so no + * search/billing/listing leakage) but unresolved indefinitely. This is + * deliberately not "fixed" by hard-deleting it after the window expires — + * that would delete a document we have no positive evidence is actually + * gone, reintroducing the exact risk this whole design exists to avoid. + */ + const hasTombstonedDocs = await db + .select({ id: document.id }) + .from(document) + .where( + and( + eq(document.connectorId, connectorId), + isNull(document.archivedAt), + or( + and(isNotNull(document.deletedAt), gt(document.deletedAt, retryCutoff)), + isNull(document.contentHash) + ) ) ) - ) - .limit(1) - .then((rows) => rows.length > 0) - - /** - * Determine if this sync should be incremental. A `rehydrate` request forces a - * full listing too: re-hydration must see *every* document (a container page can - * be unchanged itself yet transclude a page that changed), and an incremental - * listing would omit those unchanged containers, so they'd never be re-fetched. - */ - const isIncremental = - !mirrored && - shouldRunIncrementalSync( - connectorConfig.supportsIncrementalSync, - connector.syncMode, - options?.fullSync, - options?.rehydrate, - hasTombstonedDocs, - connector.lastSyncAt - ) - const lastSyncAt = - isIncremental && connector.lastSyncAt ? new Date(connector.lastSyncAt) : undefined - - /** - * Re-hydrate and re-index connectors whose rendered content can drift without a - * hash change (transclusions) — see `ConnectorMeta.rehydrateOnFullSync`. Driven - * by the dedicated `rehydrate` request (the "Full resync" action) or implied by a - * true `fullSync`. It forces a full listing (above) and re-indexes unchanged - * deferred docs, but — unlike `fullSync` — it does NOT bypass any - * deletion-reconciliation safety guard. Incremental syncs of other connectors - * stay hash-gated. - */ - const forceRehydrate = Boolean( - (options?.rehydrate || options?.fullSync) && connectorConfig.rehydrateOnFullSync - ) + .limit(1) + .then((rows) => rows.length > 0) - let directoryRefreshed: Promise = Promise.resolve(undefined) - if (mirrored) { /** - * A switch into this mode hides every document before it flips, and one - * whose rewrite outgrew its request budget leaves the rest for the next - * run. It has to be finished *before this run lists anything*: the - * documents it did not reach are still readable by the whole workspace, - * and the completion write below clears the flag on the strength of this - * pass having left none under the mode the connector came from. The - * workspace-mode equivalent runs at completion instead, because restoring - * is safe to do last; hiding is not. + * Determine if this sync should be incremental. A `rehydrate` request forces a + * full listing too: re-hydration must see *every* document (a container page can + * be unchanged itself yet transclude a page that changed), and an incremental + * listing would omit those unchanged containers, so they'd never be re-fetched. */ - if (connector.accessRewritePending) { - await rewriteConnectorAcls(connectorId, EMPTY_ACL, { - beforeBatch: lease.beatIfDue, - lease, - }) - } + const isIncremental = + !mirrored && + shouldRunIncrementalSync( + connectorConfig.supportsIncrementalSync, + connector.syncMode, + options?.fullSync, + options?.rehydrate, + hasTombstonedDocs, + connector.lastSyncAt + ) + const lastSyncAt = + isIncremental && connector.lastSyncAt ? new Date(connector.lastSyncAt) : undefined + /** - * Started before the listing and awaited before the ACLs are written: a - * group grant this crawl writes must never point at membership nobody - * has read, and the scheduler's refresh is a cadence, not a guarantee. - * Observe failures immediately while allowing content ingestion to finish. - * The terminal sync write below still reports directory failures. + * Re-hydrate and re-index connectors whose rendered content can drift without a + * hash change (transclusions) — see `ConnectorMeta.rehydrateOnFullSync`. Driven + * by the dedicated `rehydrate` request (the "Full resync" action) or implied by a + * true `fullSync`. It forces a full listing (above) and re-indexes unchanged + * deferred docs, but — unlike `fullSync` — it does NOT bypass any + * deletion-reconciliation safety guard. Incremental syncs of other connectors + * stay hash-gated. */ - directoryRefreshed = refreshMirroredDirectory({ - ...resourceScopeFields(resourceScopeFromOwner(kbOwner)), + const forceRehydrate = Boolean( + (options?.rehydrate || options?.fullSync) && connectorConfig.rehydrateOnFullSync + ) + + let directoryRefreshed: Promise = Promise.resolve(undefined) + if (mirrored) { + /** + * A switch into this mode hides every document before it flips, and one + * whose rewrite outgrew its request budget leaves the rest for the next + * run. It has to be finished *before this run lists anything*: the + * documents it did not reach are still readable by the whole workspace, + * and the completion write below clears the flag on the strength of this + * pass having left none under the mode the connector came from. The + * workspace-mode equivalent runs at completion instead, because restoring + * is safe to do last; hiding is not. + */ + if (connector.accessRewritePending) { + await rewriteConnectorAcls(connectorId, EMPTY_ACL, { + beforeBatch: lease.beatIfDue, + lease, + }) + } + /** + * Started before the listing and awaited before the ACLs are written: a + * group grant this crawl writes must never point at membership nobody + * has read, and the scheduler's refresh is a cadence, not a guarantee. + * Observe failures immediately while allowing content ingestion to finish. + * The terminal sync write below still reports directory failures. + */ + directoryRefreshed = refreshMirroredDirectory({ + ...resourceScopeFields(resourceScopeFromOwner(kbOwner)), + connectorConfig, + sourceConfig, + syncContext, + accessToken: credentialToken.accessToken, + force: + Boolean(options.fullSync) || + connector.consecutiveFailures > 0 || + connector.lastSyncError?.startsWith(DIRECTORY_ERROR_PREFIX), + }).then(() => undefined, toError) + } + + const contentPass = await runConnectorContentPass({ + connectorId, + connector, connectorConfig, sourceConfig, syncContext, - accessToken: credentialToken.accessToken, - force: - Boolean(options.fullSync) || - connector.consecutiveFailures > 0 || - connector.lastSyncError?.startsWith(DIRECTORY_ERROR_PREFIX), - }).then(() => undefined, toError) - } - - const contentPass = await runConnectorContentPass({ - connectorId, - connector, - connectorConfig, - sourceConfig, - syncContext, - lastSyncAt, - kbOwner, - billingAttribution, - result, - forceRehydrate, - getAccessToken: async (pageNum) => { - if (pageNum > 0) await refreshOAuthToken() - return credentialToken.accessToken - }, - hydration: { - concurrency: connectorConfig.contentConcurrency, - beforeHydration: refreshOAuthToken, - getDocument: (externalId) => - connectorConfig.getDocument( - credentialToken.accessToken, - sourceConfig, - externalId, - syncContext - ), - }, - lease, - documentAccess: connector.accessMode, - runId: syncLogId, - leaseKind: 'content', - fingerprint: listingFingerprint({ - connectorType: connector.connectorType, - credentialId: connector.credentialId, - encryptedApiKey: connector.encryptedApiKey, - sourceConfig, - accessMode: connector.accessMode, - }), - fullSync: options.fullSync, - deadlineAt: syncStartedAt.getTime() + (CONNECTOR_SYNC_MAX_DURATION_SECONDS - 300) * 1000, - onPage: mirrored - ? async (externalDocs, generationStartedAt) => { - await directoryRefreshed - await applySourceMirroredAcls({ - connectorId, - connectorConfig, + lastSyncAt, + kbOwner, + billingAttribution, + result, + forceRehydrate, + getAccessToken: async (pageNum) => { + if (pageNum > 0) await refreshOAuthToken() + return credentialToken.accessToken + }, + hydration: { + concurrency: connectorConfig.contentConcurrency, + beforeHydration: refreshOAuthToken, + getDocument: (externalId) => + connectorConfig.getDocument( + credentialToken.accessToken, sourceConfig, - syncContext, - accessToken: credentialToken.accessToken, - externalDocs, - generationStartedAt, - ownedExternalIds: [], - lease, - }) - } - : undefined, - }) + externalId, + syncContext + ), + }, + lease, + documentAccess: connector.accessMode, + runId: syncLogId, + leaseKind: 'content', + fingerprint: listingFingerprint({ + connectorType: connector.connectorType, + credentialId: connector.credentialId, + encryptedApiKey: connector.encryptedApiKey, + sourceConfig, + accessMode: connector.accessMode, + }), + fullSync: options.fullSync, + deadlineAt: syncStartedAt.getTime() + (CONNECTOR_SYNC_MAX_DURATION_SECONDS - 300) * 1000, + onPage: mirrored + ? async (externalDocs, generationStartedAt) => { + await directoryRefreshed + await applySourceMirroredAcls({ + connectorId, + connectorConfig, + sourceConfig, + syncContext, + accessToken: credentialToken.accessToken, + externalDocs, + generationStartedAt, + ownedExternalIds: [], + lease, + }) + } + : undefined, + }) - result.listingIncomplete = isContentPassIncomplete(contentPass) - const reconciliationHoldNotice = contentPass.holdNotice - const directoryError = await directoryRefreshed - if (directoryError) throw directoryError + result.listingIncomplete = isContentPassIncomplete(contentPass) + const reconciliationHoldNotice = contentPass.holdNotice + const directoryError = await directoryRefreshed + if (directoryError) throw directoryError - const postBatchPresence = await checkSyncTargetPresence(connectorId, connector.knowledgeBaseId) - if (postBatchPresence.connectorDeleted) { - throw new ConnectorDeletedException(connectorId) - } - if (postBatchPresence.knowledgeBaseDeleted) { - throw new Error(`Knowledge base ${connector.knowledgeBaseId} was deleted during sync`) - } - - await sweepStuckDocuments({ - connectorId, - knowledgeBaseId: connector.knowledgeBaseId, - syncStartedAt, - retryCutoff, - billingAttribution, - result, - lease, - }) + const postBatchPresence = await checkSyncTargetPresence( + connectorId, + connector.knowledgeBaseId + ) + if (postBatchPresence.connectorDeleted) { + throw new ConnectorDeletedException(connectorId) + } + if (postBatchPresence.knowledgeBaseDeleted) { + throw new Error(`Knowledge base ${connector.knowledgeBaseId} was deleted during sync`) + } - const completionLanded = await completeSuccessfulSync( - connectorId, - connector.knowledgeBaseId, - syncLogId, - effectiveConnectorSyncIntervalMinutes(connector.accessMode, connector.syncIntervalMinutes), - result, - reconciliationHoldNotice, - contentPass - ) + await sweepStuckDocuments({ + connectorId, + knowledgeBaseId: connector.knowledgeBaseId, + syncStartedAt, + retryCutoff, + billingAttribution, + result, + lease, + }) - if (!completionLanded) { - logger.warn('Sync result discarded — connector was reclaimed while this run was executing', { + const completionLanded = await completeSuccessfulSync( connectorId, + connector.knowledgeBaseId, syncLogId, - ...result, - }) - return markSyncSuperseded(result) - } + effectiveConnectorSyncIntervalMinutes(connector.accessMode, connector.syncIntervalMinutes), + result, + reconciliationHoldNotice, + contentPass + ) - logger.info('Sync completed', { connectorId, ...result }) - return result - } catch (error) { - let connectorDeleted = error instanceof ConnectorDeletedException - if (error instanceof SyncLockLostException) { - /** A checkpoint can discover an archive before the next batch's presence check. */ - const [ownedArchive] = await db - .select({ - archivedAt: knowledgeConnector.archivedAt, - deletedAt: knowledgeConnector.deletedAt, - }) - .from(knowledgeConnector) - .where( - and( - holdsSyncLockToken(connectorId, syncLogId), - or(isNotNull(knowledgeConnector.archivedAt), isNotNull(knowledgeConnector.deletedAt)) - ) + if (!completionLanded) { + logger.warn( + 'Sync result discarded — connector was reclaimed while this run was executing', + { + connectorId, + syncLogId, + ...result, + } ) - .limit(1) - connectorDeleted = Boolean(ownedArchive?.archivedAt || ownedArchive?.deletedAt) - if (!connectorDeleted) { - /** A replacement-owned connector must receive no writes from this run. */ - logger.warn('Sync abandoned — lock was reclaimed while this run was executing', { - connectorId, - syncLogId, - ...result, - }) return markSyncSuperseded(result) } - } - - if (connectorDeleted) { - logger.info('Connector deleted during sync, cleaning up', { connectorId }) - try { - await releaseSyncLockOnDeletedConnector(connectorId, syncLogId) + logger.info('Sync completed', { connectorId, ...result }) + return result + } catch (error) { + let connectorDeleted = error instanceof ConnectorDeletedException + if (error instanceof SyncLockLostException) { + /** A checkpoint can discover an archive before the next batch's presence check. */ + const [ownedArchive] = await db + .select({ + archivedAt: knowledgeConnector.archivedAt, + deletedAt: knowledgeConnector.deletedAt, + }) + .from(knowledgeConnector) + .where( + and( + holdsSyncLockToken(connectorId, syncLogId), + or(isNotNull(knowledgeConnector.archivedAt), isNotNull(knowledgeConnector.deletedAt)) + ) + ) + .limit(1) + connectorDeleted = Boolean(ownedArchive?.archivedAt || ownedArchive?.deletedAt) + if (!connectorDeleted) { + /** A replacement-owned connector must receive no writes from this run. */ + logger.warn('Sync abandoned — lock was reclaimed while this run was executing', { + connectorId, + syncLogId, + ...result, + }) + return markSyncSuperseded(result) + } + } - /** - * Includes pending-removal tombstones. Page IDs so deleting a connector - * with a legacy corpus above the sync admission cap cannot materialize - * the entire corpus in the cleanup worker. - */ - let afterDocumentId: string | undefined - while (true) { - const connectorDocs = await db - .select({ id: document.id }) - .from(document) - .where( - and( - eq(document.connectorId, connectorId), - isNull(document.archivedAt), - afterDocumentId ? gt(document.id, afterDocumentId) : undefined + if (connectorDeleted) { + logger.info('Connector deleted during sync, cleaning up', { connectorId }) + + try { + await releaseSyncLockOnDeletedConnector(connectorId, syncLogId) + + /** + * Includes pending-removal tombstones. Page IDs so deleting a connector + * with a legacy corpus above the sync admission cap cannot materialize + * the entire corpus in the cleanup worker. + */ + let afterDocumentId: string | undefined + while (true) { + const connectorDocs = await db + .select({ id: document.id }) + .from(document) + .where( + and( + eq(document.connectorId, connectorId), + isNull(document.archivedAt), + afterDocumentId ? gt(document.id, afterDocumentId) : undefined + ) ) + .orderBy(asc(document.id)) + .limit(CONNECTOR_DELETION_CLEANUP_BATCH_SIZE) + if (connectorDocs.length === 0) break + + await hardDeleteDocuments( + connectorDocs.map((doc) => doc.id), + syncLogId, + connectorId ) - .orderBy(asc(document.id)) - .limit(CONNECTOR_DELETION_CLEANUP_BATCH_SIZE) - if (connectorDocs.length === 0) break + afterDocumentId = connectorDocs.at(-1)?.id + if (connectorDocs.length < CONNECTOR_DELETION_CLEANUP_BATCH_SIZE) break + } - await hardDeleteDocuments( - connectorDocs.map((doc) => doc.id), - syncLogId, - connectorId - ) - afterDocumentId = connectorDocs.at(-1)?.id - if (connectorDocs.length < CONNECTOR_DELETION_CLEANUP_BATCH_SIZE) break + await completeSyncLog(syncLogId, 'failed', result, { + errorMessage: 'Connector deleted during sync', + }) + } catch (cleanupError) { + logger.error('Failed to clean up after connector deletion', { + connectorId, + error: toError(cleanupError).message, + }) } - await completeSyncLog(syncLogId, 'failed', result, { - errorMessage: 'Connector deleted during sync', - }) - } catch (cleanupError) { - logger.error('Failed to clean up after connector deletion', { - connectorId, - error: toError(cleanupError).message, - }) + result.skipReason = 'connector_deleted_during_sync' + return result } - result.skipReason = 'connector_deleted_during_sync' - return result - } - - if (getConnectorSyncDeferral(error)) { - try { - result.deferred = await deferConnectorSync({ - connectorId, - knowledgeBaseId: connector.knowledgeBaseId, - runId: syncLogId, - lease, - kind: 'content', - result, - error, - }) - result.listingIncomplete = true - logger.info('Connector source sync deferred', { - connectorId, - ...result.deferred, - docsAdvanced: - result.docsAdded + result.docsUpdated + result.docsUnchanged + result.docsSkipped, - }) - return result - } catch (persistenceError) { - logger.error('Failed to persist connector source deferral', { - connectorId, - error: - getConnectorFailureDiagnostic(persistenceError)?.message ?? - toError(persistenceError).message, - }) - result.error = 'Could not persist the connector retry after provider deferral' - return result + if (getConnectorSyncDeferral(error)) { + try { + result.deferred = await deferConnectorSync({ + connectorId, + knowledgeBaseId: connector.knowledgeBaseId, + runId: syncLogId, + lease, + kind: 'content', + result, + error, + }) + result.listingIncomplete = true + logger.info('Connector source sync deferred', { + connectorId, + ...result.deferred, + docsAdvanced: + result.docsAdded + result.docsUpdated + result.docsUnchanged + result.docsSkipped, + }) + return result + } catch (persistenceError) { + logger.error('Failed to persist connector source deferral', { + connectorId, + error: + getConnectorFailureDiagnostic(persistenceError)?.message ?? + toError(persistenceError).message, + }) + result.error = 'Could not persist the connector retry after provider deferral' + return result + } } - } - const diagnostic = getConnectorFailureDiagnostic(error) - const errorMessage = diagnostic?.message ?? toError(error).message - const retryAfterMs = getRetryAfterMs(error) - const rateLimited = isRateLimitError(error) - logger.error('Sync failed', { - connectorId, - diagnostic, - error: errorMessage, - ...(retryAfterMs === undefined ? {} : { retryAfterMs }), - }) + const diagnostic = getConnectorFailureDiagnostic(error) + const errorMessage = diagnostic?.message ?? toError(error).message + const retryAfterMs = getRetryAfterMs(error) + const rateLimited = isRateLimitError(error) + logger.error('Sync failed', { + connectorId, + diagnostic, + error: errorMessage, + ...(retryAfterMs === undefined ? {} : { retryAfterMs }), + }) - try { - await completeSyncLog(syncLogId, 'failed', result, { errorMessage }) - - const failureUpdate = - error instanceof ConnectorSyncCapacityError - ? buildSyncCapacityUpdate(new Date(), connector.consecutiveFailures, errorMessage) - : rateLimited - ? buildSyncRateLimitUpdate( - new Date(), - connector.consecutiveFailures, - errorMessage, - retryAfterMs - ) - : buildSyncFailureUpdate( - new Date(), - connector.consecutiveFailures, - errorMessage, - retryAfterMs - ) + try { + await completeSyncLog(syncLogId, 'failed', result, { errorMessage }) + + const failureUpdate = + error instanceof ConnectorSyncCapacityError + ? buildSyncCapacityUpdate(new Date(), connector.consecutiveFailures, errorMessage) + : rateLimited + ? buildSyncRateLimitUpdate( + new Date(), + connector.consecutiveFailures, + errorMessage, + retryAfterMs + ) + : buildSyncFailureUpdate( + new Date(), + connector.consecutiveFailures, + errorMessage, + retryAfterMs + ) + + if (failureUpdate.status === 'disabled') { + logger.warn('Connector disabled after repeated failures', { + connectorId, + consecutiveFailures: failureUpdate.consecutiveFailures, + }) + } + + const failureWriteLanded = await writeTerminalConnectorState( + connectorId, + syncLogId, + failureUpdate + ) - if (failureUpdate.status === 'disabled') { - logger.warn('Connector disabled after repeated failures', { + /** + * Deliberately does NOT get {@link markSyncSuperseded}. `result.error` + * is set to the real failure cause below, so replacing it with lifecycle + * control flow would destroy the diagnostic. The supersession is carried + * by this log line instead. + */ + if (!failureWriteLanded) { + logger.warn( + 'Sync failure discarded — connector was reclaimed while this run was executing', + { connectorId, syncLogId, error: errorMessage } + ) + } + } catch (recoveryError) { + logger.error('Failed to record sync failure', { connectorId, - consecutiveFailures: failureUpdate.consecutiveFailures, + error: + getConnectorFailureDiagnostic(recoveryError)?.message ?? toError(recoveryError).message, }) } - const failureWriteLanded = await writeTerminalConnectorState( - connectorId, - syncLogId, - failureUpdate - ) - - /** - * Deliberately does NOT get {@link markSyncSuperseded}. `result.error` - * is set to the real failure cause below, so replacing it with lifecycle - * control flow would destroy the diagnostic. The supersession is carried - * by this log line instead. - */ - if (!failureWriteLanded) { - logger.warn( - 'Sync failure discarded — connector was reclaimed while this run was executing', - { connectorId, syncLogId, error: errorMessage } - ) - } - } catch (recoveryError) { - logger.error('Failed to record sync failure', { - connectorId, - error: - getConnectorFailureDiagnostic(recoveryError)?.message ?? toError(recoveryError).message, - }) + result.error = errorMessage + return result } - - result.error = errorMessage - return result - } + }) } diff --git a/apps/sim/lib/knowledge/documents/secure-fetch.server.test.ts b/apps/sim/lib/knowledge/documents/secure-fetch.server.test.ts new file mode 100644 index 00000000000..f9a1d28421d --- /dev/null +++ b/apps/sim/lib/knowledge/documents/secure-fetch.server.test.ts @@ -0,0 +1,79 @@ +/** @vitest-environment node */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const mocks = vi.hoisted(() => ({ + route: vi.fn(), + gatewayFetch: vi.fn(), + directFetch: vi.fn(), +})) +vi.mock('@/lib/core/network/context.server', () => ({ + resolveCurrentOutboundRoute: mocks.route, +})) +vi.mock('@/lib/core/security/input-validation.server', () => ({ + createSsrfGuardedFetchWithDispatcher: () => ({ fetch: mocks.gatewayFetch }), + secureFetchWithValidation: vi.fn(), +})) + +import { OutboundRoutingError } from '@/lib/core/network/routing' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' + +const url = 'https://api.example.invalid/items' +const noRetries = { maxRetries: 0, retryBudgetMs: 1_000 } + +beforeEach(() => { + vi.clearAllMocks() + vi.stubGlobal('fetch', mocks.directFetch) + mocks.route.mockResolvedValue({ kind: 'direct' }) + mocks.directFetch.mockImplementation(async () => new Response('direct')) + mocks.gatewayFetch.mockImplementation(async () => new Response('gateway')) +}) +afterEach(() => vi.unstubAllGlobals()) + +describe('connector request routing', () => { + it('preserves native fetch and request options for direct organizations', async () => { + const response = await fetchWithRetry( + url, + { method: 'POST', body: 'payload', headers: { authorization: 'Bearer test' } }, + noRetries + ) + expect(await response.text()).toBe('direct') + expect(mocks.directFetch).toHaveBeenCalledWith(url, { + method: 'POST', + body: 'payload', + headers: { authorization: 'Bearer test' }, + signal: expect.any(AbortSignal), + }) + expect(mocks.gatewayFetch).not.toHaveBeenCalled() + }) + + it('uses the validated gateway transport for a managed organization', async () => { + mocks.route.mockResolvedValue({ kind: 'gateway' }) + const response = await fetchWithRetry(url, {}, noRetries) + expect(await response.text()).toBe('gateway') + expect(mocks.directFetch).not.toHaveBeenCalled() + }) + + it('reads policy after provider admission and never bypasses a revoked route', async () => { + const fetcher = vi.fn( + async (input: RequestInfo | URL, init: RequestInit, transport: typeof fetch) => { + mocks.route.mockRejectedValue(new OutboundRoutingError('ROUTE_BLOCKED')) + return transport(input, init) + } + ) + await expect(fetchWithRetry(url, {}, { ...noRetries, fetcher })).rejects.toMatchObject({ + code: 'ROUTE_BLOCKED', + }) + expect(fetcher).toHaveBeenCalledOnce() + expect(mocks.directFetch).not.toHaveBeenCalled() + expect(mocks.gatewayFetch).not.toHaveBeenCalled() + }) + + it('does not fall back to native fetch after gateway failure', async () => { + mocks.route.mockResolvedValue({ kind: 'gateway' }) + mocks.gatewayFetch.mockRejectedValue(new OutboundRoutingError('GATEWAY_UNAVAILABLE')) + await expect(fetchWithRetry(url, {}, noRetries)).rejects.toMatchObject({ + code: 'GATEWAY_UNAVAILABLE', + }) + expect(mocks.directFetch).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/knowledge/documents/secure-fetch.server.ts b/apps/sim/lib/knowledge/documents/secure-fetch.server.ts index 24cd4814dc8..27302ee9442 100644 --- a/apps/sim/lib/knowledge/documents/secure-fetch.server.ts +++ b/apps/sim/lib/knowledge/documents/secure-fetch.server.ts @@ -1,4 +1,6 @@ +import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' import { + createSsrfGuardedFetchWithDispatcher, type SecureFetchOptions, type SecureFetchResponse, secureFetchWithValidation, @@ -57,3 +59,62 @@ export async function secureFetchWithRetry( return response }, retry) } + +const DEFAULT_FETCH_RETRY_BUDGET_MS = 150_000 +let connectorTransport: ReturnType | undefined + +/** + * Bounds requests and response bodies within one retry budget. + */ +export async function fetchWithRetry( + url: string, + options: RequestInit = {}, + retryOptions: RetryOptions = {} +): Promise { + const callerSignal = options.signal + ? retryOptions.signal + ? AbortSignal.any([options.signal, retryOptions.signal]) + : options.signal + : retryOptions.signal + + return retryWithExponentialBackoff( + async (signal, deadlineAt) => { + /** The fetch deadline stays active while callers consume the returned response body. */ + const requestSignal = AbortSignal.any([ + signal, + AbortSignal.timeout(Math.max(0, Math.ceil(deadlineAt - Date.now()))), + ]) + const transport: typeof fetch = async (input, init) => { + const route = await resolveCurrentOutboundRoute() + if (route.kind === 'direct') return fetch(input, init) + connectorTransport ??= createSsrfGuardedFetchWithDispatcher({ + profile: 'configuredEndpoint', + }) + return connectorTransport.fetch(input, init) + } + const init = { ...options, signal: requestSignal } + const response = retryOptions.fetcher + ? await retryOptions.fetcher(url, init, transport) + : await transport(url, init) + + if ( + !response.ok && + isRetryableError({ status: response.status, headers: response.headers }) + ) { + throw await createRetryableHttpError(response) + } + + return response + }, + { + ...retryOptions, + retryBudgetMs: retryOptions.retryBudgetMs ?? DEFAULT_FETCH_RETRY_BUDGET_MS, + maxRetryAfterMs: + retryOptions.maxRetryAfterMs ?? + retryOptions.retryBudgetMs ?? + retryOptions.maxDelayMs ?? + 30_000, + signal: callerSignal, + } + ) +} diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index 65eb4b14f95..188a5c0308d 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -53,6 +53,7 @@ import { resolveTriggerRegion } from '@/lib/core/async-jobs/region' import { env, envNumber } from '@/lib/core/config/env' import { getCostMultiplier, isTriggerDevEnabled } from '@/lib/core/config/env-flags' import { isInsideTriggerRun } from '@/lib/core/config/trigger-runtime' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { OrchestrationError } from '@/lib/core/orchestration/types' import type { ProviderCapacityDeferredError } from '@/lib/core/rate-limiter/provider-capacity-error' import { mapWithConcurrency } from '@/lib/core/utils/concurrency' @@ -1540,482 +1541,487 @@ export async function processDocumentAsync( const ctx = contextRows[0] processingFilename = ctx.filename - const persistedDocData = { - filename: ctx.filename, - fileUrl: ctx.fileUrl, - fileSize: ctx.fileSize, - mimeType: ctx.mimeType, - } + await withResourceOutboundScope(ctx, async () => { + const persistedDocData = { + filename: ctx.filename, + fileUrl: ctx.fileUrl, + fileSize: ctx.fileSize, + mimeType: ctx.mimeType, + } - /** - * Claiming is guarded by both completion status and queue generation. - * - * Without a status predicate this write was reachable for a finished - * document — a late or duplicate dispatch would flip `completed` back to - * `processing`, discard the pass that had already indexed and billed, and - * index it a second time. `pending`, `failed`, and `processing` remain - * claimable so a Trigger retry can recover if an earlier attempt threw - * before persisting its failure. Queued workers also match the exact stamp - * carried in their payload. A retry or recovery sweep re-stamps the row, so - * an older delayed quota continuation becomes a harmless no-op instead of - * stealing the newer pass. - */ - /** - * Queue acceptance can precede the parent's pending-state write. The published - * successor may adopt that exact processing generation; stamping its token - * fences the parent's delayed write and refunds admission at most once. - */ - const predecessor = - attemptContext?.processingQueueToken && attemptContext.processingPredecessorToken - ? and( - eq(document.processingStatus, 'processing'), - eq(document.processingQueueToken, attemptContext.processingPredecessorToken) + /** + * Claiming is guarded by both completion status and queue generation. + * + * Without a status predicate this write was reachable for a finished + * document — a late or duplicate dispatch would flip `completed` back to + * `processing`, discard the pass that had already indexed and billed, and + * index it a second time. `pending`, `failed`, and `processing` remain + * claimable so a Trigger retry can recover if an earlier attempt threw + * before persisting its failure. Queued workers also match the exact stamp + * carried in their payload. A retry or recovery sweep re-stamps the row, so + * an older delayed quota continuation becomes a harmless no-op instead of + * stealing the newer pass. + */ + /** + * Queue acceptance can precede the parent's pending-state write. The published + * successor may adopt that exact processing generation; stamping its token + * fences the parent's delayed write and refunds admission at most once. + */ + const predecessor = + attemptContext?.processingQueueToken && attemptContext.processingPredecessorToken + ? and( + eq(document.processingStatus, 'processing'), + eq(document.processingQueueToken, attemptContext.processingPredecessorToken) + ) + : undefined + const claimed = await db + .update(document) + .set({ + processingStatus: 'processing', + processingStartedAt, + processingDeferredUntil: null, + processingCompletedAt: null, + processingError: null, + ...(attemptContext?.processingQueueToken + ? { processingQueueToken: attemptContext.processingQueueToken } + : {}), + ...(predecessor && attemptContext?.refundPredecessorAdmission + ? { + processingAttempts: sql`CASE WHEN ${document.processingQueueToken} = ${attemptContext.processingPredecessorToken} THEN GREATEST(${document.processingAttempts} - 1, 0) ELSE ${document.processingAttempts} END`, + } + : {}), + }) + .where( + and( + eq(document.id, documentId), + inArray(document.processingStatus, ['pending', 'processing', 'failed']), + not(skippedDocumentCondition()), + ...(predecessor + ? [or(and(...queueGenerationConditions(attemptContext)), predecessor)] + : queueGenerationConditions(attemptContext)), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt) ) - : undefined - const claimed = await db - .update(document) - .set({ - processingStatus: 'processing', - processingStartedAt, - processingDeferredUntil: null, - processingCompletedAt: null, - processingError: null, - ...(attemptContext?.processingQueueToken - ? { processingQueueToken: attemptContext.processingQueueToken } - : {}), - ...(predecessor && attemptContext?.refundPredecessorAdmission - ? { - processingAttempts: sql`CASE WHEN ${document.processingQueueToken} = ${attemptContext.processingPredecessorToken} THEN GREATEST(${document.processingAttempts} - 1, 0) ELSE ${document.processingAttempts} END`, - } - : {}), - }) - .where( - and( - eq(document.id, documentId), - inArray(document.processingStatus, ['pending', 'processing', 'failed']), - not(skippedDocumentCondition()), - ...(predecessor - ? [or(and(...queueGenerationConditions(attemptContext)), predecessor)] - : queueGenerationConditions(attemptContext)), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt) ) - ) - .returning({ id: document.id }) + .returning({ id: document.id }) - if (claimed.length === 0) { - logger.info( - `[${documentId}] Skipping document processing: superseded, already active, completed, archived, or deleted` - ) - return - } + if (claimed.length === 0) { + logger.info( + `[${documentId}] Skipping document processing: superseded, already active, completed, archived, or deleted` + ) + return + } - attemptContext?.onClaimed?.() - - logger.info(`[${documentId}] Status updated to 'processing', starting document processor`) - - const rawConfig = ctx.chunkingConfig as { - maxSize?: number - minSize?: number - overlap?: number - strategy?: ChunkingStrategy - strategyOptions?: StrategyOptions - } | null - const kbConfig = { - maxSize: rawConfig?.maxSize ?? 1024, - minSize: rawConfig?.minSize ?? 100, - overlap: rawConfig?.overlap ?? 200, - } + attemptContext?.onClaimed?.() + + logger.info(`[${documentId}] Status updated to 'processing', starting document processor`) + + const rawConfig = ctx.chunkingConfig as { + maxSize?: number + minSize?: number + overlap?: number + strategy?: ChunkingStrategy + strategyOptions?: StrategyOptions + } | null + const kbConfig = { + maxSize: rawConfig?.maxSize ?? 1024, + minSize: rawConfig?.minSize ?? 100, + overlap: rawConfig?.overlap ?? 200, + } - const kbEmbedding: KbEmbeddingTarget = { - model: ctx.embeddingModel, - dimensions: toKbEmbeddingDimensions(ctx.embeddingDimension), - } - const kbEmbeddingModel = kbEmbedding.model - const queuedBillingContext = hasDocumentProcessingBillingScope(providedBillingContext) - ? assertDocumentProcessingBillingContext(providedBillingContext) - : undefined - const restoredBillingAttribution = - queuedBillingContext && queuedBillingContext.billingScope !== 'non-workspace' - ? queuedBillingContext.billingAttribution - : providedBillingContext && !queuedBillingContext - ? assertBillingAttributionSnapshot(providedBillingContext) - : undefined - if (queuedBillingContext?.billingScope === 'non-workspace') { - throw new Error('Document processing billing scope does not match knowledge base ownership') - } - if (!restoredBillingAttribution) { - throw new Error('Billing attribution is required for queued document processing') - } - const billingAttribution = restoredBillingAttribution - assertBillingAttributionOwner(billingAttribution, ctx) - const documentActorUserId = billingAttribution.actorUserId - - const usageGate = await checkIngestionUsageLimits(billingAttribution) - if (usageGate.isExceeded) { - logger.warn(`[${documentId}] Usage limit reached — skipping document indexing`) - throw new UsageLimitDocumentProcessingError( - usageGate.message ?? 'Usage limit exceeded. Please upgrade your plan to continue.' - ) - } - let billableEmbeddingTokens = 0 - let embeddingModelName = kbEmbeddingModel - let embeddingPricingId = kbEmbeddingModel + const kbEmbedding: KbEmbeddingTarget = { + model: ctx.embeddingModel, + dimensions: toKbEmbeddingDimensions(ctx.embeddingDimension), + } + const kbEmbeddingModel = kbEmbedding.model + const queuedBillingContext = hasDocumentProcessingBillingScope(providedBillingContext) + ? assertDocumentProcessingBillingContext(providedBillingContext) + : undefined + const restoredBillingAttribution = + queuedBillingContext && queuedBillingContext.billingScope !== 'non-workspace' + ? queuedBillingContext.billingAttribution + : providedBillingContext && !queuedBillingContext + ? assertBillingAttributionSnapshot(providedBillingContext) + : undefined + if (queuedBillingContext?.billingScope === 'non-workspace') { + throw new Error('Document processing billing scope does not match knowledge base ownership') + } + if (!restoredBillingAttribution) { + throw new Error('Billing attribution is required for queued document processing') + } + const billingAttribution = restoredBillingAttribution + assertBillingAttributionOwner(billingAttribution, ctx) + const documentActorUserId = billingAttribution.actorUserId + + const usageGate = await checkIngestionUsageLimits(billingAttribution) + if (usageGate.isExceeded) { + logger.warn(`[${documentId}] Usage limit reached — skipping document indexing`) + throw new UsageLimitDocumentProcessingError( + usageGate.message ?? 'Usage limit exceeded. Please upgrade your plan to continue.' + ) + } + let billableEmbeddingTokens = 0 + let embeddingModelName = kbEmbeddingModel + let embeddingPricingId = kbEmbeddingModel - const currentSourceFileProvenance = await loadCurrentSourceFileSecretProvenance({ - fileUrl: persistedDocData.fileUrl, - workspaceId: ctx.workspaceId, - }) - const documentSecretContext = await loadKnowledgeDocumentSecretRegistry( - documentId, - { - userId: documentActorUserId, - ...(ctx.workspaceId ? { workspaceId: ctx.workspaceId } : {}), - }, - currentSourceFileProvenance - ) + const currentSourceFileProvenance = await loadCurrentSourceFileSecretProvenance({ + fileUrl: persistedDocData.fileUrl, + workspaceId: ctx.workspaceId, + }) + const documentSecretContext = await loadKnowledgeDocumentSecretRegistry( + documentId, + { + userId: documentActorUserId, + ...(ctx.workspaceId ? { workspaceId: ctx.workspaceId } : {}), + }, + currentSourceFileProvenance + ) - let processingCommitted = false - const processingDeadlineAt = Math.min( - startTime + TIMEOUTS.OVERALL_PROCESSING - 15_000, - (attemptContext?.deadlineAt ?? Number.POSITIVE_INFINITY) - 15_000 - ) - await withTimeout( - (signal) => - runWithKnowledgeModelInputProvenance( - documentSecretContext.registry, - async () => { - await assertKnowledgeEmbeddingCapacity({ - ...kbEmbedding, - workspaceId: ctx.workspaceId, - signal, - }) - const processed = await processDocument( - persistedDocData.fileUrl, - persistedDocData.filename, - persistedDocData.mimeType, - kbConfig.maxSize, - kbConfig.overlap, - kbConfig.minSize, - { - ...sourceFileAccessFor(ctx.connectorId, documentActorUserId), + let processingCommitted = false + const processingDeadlineAt = Math.min( + startTime + TIMEOUTS.OVERALL_PROCESSING - 15_000, + (attemptContext?.deadlineAt ?? Number.POSITIVE_INFINITY) - 15_000 + ) + await withTimeout( + (signal) => + runWithKnowledgeModelInputProvenance( + documentSecretContext.registry, + async () => { + await assertKnowledgeEmbeddingCapacity({ + ...kbEmbedding, + workspaceId: ctx.workspaceId, signal, - processingDeadlineAt, - ...(indexingPassId - ? { ocrCheckpoint: { knowledgeBaseId, documentId, indexingPassId } } - : {}), - }, - ctx.workspaceId, - rawConfig?.strategy, - rawConfig?.strategyOptions - ) + }) + const processed = await processDocument( + persistedDocData.fileUrl, + persistedDocData.filename, + persistedDocData.mimeType, + kbConfig.maxSize, + kbConfig.overlap, + kbConfig.minSize, + { + ...sourceFileAccessFor(ctx.connectorId, documentActorUserId), + signal, + processingDeadlineAt, + ...(indexingPassId + ? { ocrCheckpoint: { knowledgeBaseId, documentId, indexingPassId } } + : {}), + }, + ctx.workspaceId, + rawConfig?.strategy, + rawConfig?.strategyOptions + ) - signal.throwIfAborted() - assertDocumentChunkCountWithinLimit(processed.chunks.length) + signal.throwIfAborted() + assertDocumentChunkCountWithinLimit(processed.chunks.length) - const now = new Date() + const now = new Date() - logger.info( - `[${documentId}] Document parsed successfully, generating embeddings for ${processed.chunks.length} chunks` - ) + logger.info( + `[${documentId}] Document parsed successfully, generating embeddings for ${processed.chunks.length} chunks` + ) - const chunkTexts = processed.chunks.map((chunk) => chunk.text) - const embeddingModelInfo = getEmbeddingModelInfo(kbEmbeddingModel) - const chunkTokenCounts: number[] = [] - for (let chunkIndex = 0; chunkIndex < chunkTexts.length; chunkIndex++) { - const tokenCount = estimateTokenCount( - chunkTexts[chunkIndex], - embeddingModelInfo.tokenizerProvider - ).count - chunkTokenCounts.push(tokenCount) - if (tokenCount > embeddingModelInfo.maxInputTokens) { - throw new PermanentDocumentProcessingError( - 'document_complexity_limit', - `Chunk ${chunkIndex + 1} contains ${tokenCount.toLocaleString()} estimated tokens, exceeding the ${embeddingModelInfo.maxInputTokens.toLocaleString()}-token limit for ${kbEmbeddingModel}. Reduce the knowledge-base chunk size and retry.` - ) + const chunkTexts = processed.chunks.map((chunk) => chunk.text) + const embeddingModelInfo = getEmbeddingModelInfo(kbEmbeddingModel) + const chunkTokenCounts: number[] = [] + for (let chunkIndex = 0; chunkIndex < chunkTexts.length; chunkIndex++) { + const tokenCount = estimateTokenCount( + chunkTexts[chunkIndex], + embeddingModelInfo.tokenizerProvider + ).count + chunkTokenCounts.push(tokenCount) + if (tokenCount > embeddingModelInfo.maxInputTokens) { + throw new PermanentDocumentProcessingError( + 'document_complexity_limit', + `Chunk ${chunkIndex + 1} contains ${tokenCount.toLocaleString()} estimated tokens, exceeding the ${embeddingModelInfo.maxInputTokens.toLocaleString()}-token limit for ${kbEmbeddingModel}. Reduce the knowledge-base chunk size and retry.` + ) + } } - } - const embeddings: number[][] = [] - const embeddingSourceHash = indexingPassId - ? sha256Hex(JSON.stringify(chunkTexts.map((text) => sha256Hex(text)))) - : undefined + const embeddings: number[][] = [] + const embeddingSourceHash = indexingPassId + ? sha256Hex(JSON.stringify(chunkTexts.map((text) => sha256Hex(text)))) + : undefined - if (chunkTexts.length > 0) { - const batchSize = LARGE_DOC_CONFIG.MAX_EMBEDDING_BATCH - const totalBatches = Math.ceil(chunkTexts.length / batchSize) + if (chunkTexts.length > 0) { + const batchSize = LARGE_DOC_CONFIG.MAX_EMBEDDING_BATCH + const totalBatches = Math.ceil(chunkTexts.length / batchSize) - logger.info(`[${documentId}] Generating embeddings in ${totalBatches} batches`) + logger.info(`[${documentId}] Generating embeddings in ${totalBatches} batches`) - for (let i = 0; i < chunkTexts.length; i += batchSize) { - signal.throwIfAborted() - const batch = chunkTexts.slice(i, i + batchSize) - const batchNum = Math.floor(i / batchSize) + 1 + for (let i = 0; i < chunkTexts.length; i += batchSize) { + signal.throwIfAborted() + const batch = chunkTexts.slice(i, i + batchSize) + const batchNum = Math.floor(i / batchSize) + 1 - logger.info( - `[${documentId}] Processing embedding batch ${batchNum}/${totalBatches}` - ) - const { - embeddings: batchEmbeddings, - billableTokens: batchBillableTokens, - modelName, - pricingId, - } = await generateEmbeddings( - batch, - kbEmbedding, - ctx.workspaceId, - signal, - indexingPassId && embeddingSourceHash - ? createEmbeddingCheckpoints({ - knowledgeBaseId, - documentId, - indexingPassId, - sourceHash: embeddingSourceHash, - batchOffset: i, - deadlineAt: processingDeadlineAt, - }) - : undefined - ) - for (const emb of batchEmbeddings) { - embeddings.push(emb) - } - billableEmbeddingTokens += batchBillableTokens - if (i === 0) { - embeddingModelName = modelName - embeddingPricingId = pricingId + logger.info( + `[${documentId}] Processing embedding batch ${batchNum}/${totalBatches}` + ) + const { + embeddings: batchEmbeddings, + billableTokens: batchBillableTokens, + modelName, + pricingId, + } = await generateEmbeddings( + batch, + kbEmbedding, + ctx.workspaceId, + signal, + indexingPassId && embeddingSourceHash + ? createEmbeddingCheckpoints({ + knowledgeBaseId, + documentId, + indexingPassId, + sourceHash: embeddingSourceHash, + batchOffset: i, + deadlineAt: processingDeadlineAt, + }) + : undefined + ) + for (const emb of batchEmbeddings) { + embeddings.push(emb) + } + billableEmbeddingTokens += batchBillableTokens + if (i === 0) { + embeddingModelName = modelName + embeddingPricingId = pricingId + } } } - } - if (embeddings.length !== processed.chunks.length) { - throw new Error( - `Embedding generation returned ${embeddings.length} vectors for ${processed.chunks.length} chunks` + if (embeddings.length !== processed.chunks.length) { + throw new Error( + `Embedding generation returned ${embeddings.length} vectors for ${processed.chunks.length} chunks` + ) + } + const documentTags = ctx + + logger.info( + `[${documentId}] Embeddings generated, creating embedding records with tags` ) - } - const documentTags = ctx - logger.info( - `[${documentId}] Embeddings generated, creating embedding records with tags` - ) + const chunkProvenances = processed.chunks.map((chunk) => + documentSecretContext.tracked + ? documentSecretContext.registry + ? durableSecretProvenanceFromRegistry( + documentSecretContext.registry, + chunk.text + ) + : EXACT_EMPTY_DURABLE_SECRET_PROVENANCE + : undefined + ) + const embeddingRecords = processed.chunks.map((chunk, chunkIndex) => ({ + id: generateId(), + knowledgeBaseId, + documentId, + chunkIndex, + chunkHash: sha256Hex(chunk.text), + content: chunk.text, + secretProvenanceVersion: chunkProvenances[chunkIndex] ? 1 : null, + contentLength: chunk.text.length, + tokenCount: chunkTokenCounts[chunkIndex], + ...embeddingVectorValues(kbEmbedding.dimensions, embeddings[chunkIndex]), + embeddingModel: kbEmbeddingModel, + startOffset: chunk.metadata.startIndex, + endOffset: chunk.metadata.endIndex, + tag1: documentTags.tag1, + tag2: documentTags.tag2, + tag3: documentTags.tag3, + tag4: documentTags.tag4, + tag5: documentTags.tag5, + tag6: documentTags.tag6, + tag7: documentTags.tag7, + number1: documentTags.number1, + number2: documentTags.number2, + number3: documentTags.number3, + number4: documentTags.number4, + number5: documentTags.number5, + date1: documentTags.date1, + date2: documentTags.date2, + boolean1: documentTags.boolean1, + boolean2: documentTags.boolean2, + boolean3: documentTags.boolean3, + createdAt: now, + updatedAt: now, + })) - const chunkProvenances = processed.chunks.map((chunk) => - documentSecretContext.tracked - ? documentSecretContext.registry - ? durableSecretProvenanceFromRegistry(documentSecretContext.registry, chunk.text) - : EXACT_EMPTY_DURABLE_SECRET_PROVENANCE - : undefined - ) - const embeddingRecords = processed.chunks.map((chunk, chunkIndex) => ({ - id: generateId(), - knowledgeBaseId, - documentId, - chunkIndex, - chunkHash: sha256Hex(chunk.text), - content: chunk.text, - secretProvenanceVersion: chunkProvenances[chunkIndex] ? 1 : null, - contentLength: chunk.text.length, - tokenCount: chunkTokenCounts[chunkIndex], - ...embeddingVectorValues(kbEmbedding.dimensions, embeddings[chunkIndex]), - embeddingModel: kbEmbeddingModel, - startOffset: chunk.metadata.startIndex, - endOffset: chunk.metadata.endIndex, - tag1: documentTags.tag1, - tag2: documentTags.tag2, - tag3: documentTags.tag3, - tag4: documentTags.tag4, - tag5: documentTags.tag5, - tag6: documentTags.tag6, - tag7: documentTags.tag7, - number1: documentTags.number1, - number2: documentTags.number2, - number3: documentTags.number3, - number4: documentTags.number4, - number5: documentTags.number5, - date1: documentTags.date1, - date2: documentTags.date2, - boolean1: documentTags.boolean1, - boolean2: documentTags.boolean2, - boolean3: documentTags.boolean3, - createdAt: now, - updatedAt: now, - })) - - signal.throwIfAborted() - processingCommitted = await db.transaction(async (tx) => { signal.throwIfAborted() - const activeDocument = await tx - .select({ id: document.id }) - .from(document) - .innerJoin(knowledgeBase, eq(document.knowledgeBaseId, knowledgeBase.id)) - .where( - and( - eq(document.id, documentId), - eq(document.processingStatus, 'processing'), - eq(document.processingStartedAt, processingStartedAt), - ...queueGenerationConditions(attemptContext), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt), - isNull(knowledgeBase.deletedAt) + processingCommitted = await db.transaction(async (tx) => { + signal.throwIfAborted() + const activeDocument = await tx + .select({ id: document.id }) + .from(document) + .innerJoin(knowledgeBase, eq(document.knowledgeBaseId, knowledgeBase.id)) + .where( + and( + eq(document.id, documentId), + eq(document.processingStatus, 'processing'), + eq(document.processingStartedAt, processingStartedAt), + ...queueGenerationConditions(attemptContext), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt), + isNull(knowledgeBase.deletedAt) + ) ) - ) - .for('update', { of: document }) - .limit(1) - - if (activeDocument.length === 0) { - return false - } - - if (embeddingRecords.length > 0) { - await tx.delete(embedding).where(eq(embedding.documentId, documentId)) + .for('update', { of: document }) + .limit(1) - const insertBatchSize = LARGE_DOC_CONFIG.MAX_CHUNKS_PER_BATCH - const batches: (typeof embeddingRecords)[] = [] - for (let i = 0; i < embeddingRecords.length; i += insertBatchSize) { - batches.push(embeddingRecords.slice(i, i + insertBatchSize)) + if (activeDocument.length === 0) { + return false } - logger.info(`[${documentId}] Inserting ${embeddingRecords.length} embeddings`) - for (const batch of batches) { - signal.throwIfAborted() - await tx.insert(embedding).values(batch) - } - const provenanceRecords = embeddingRecords.flatMap((record, index) => { - const provenance = chunkProvenances[index] - if (!provenance) return [] - return [ - { - embeddingId: record.id, - contentHash: record.chunkHash, - status: provenance.status, - entries: provenance.status === 'exact' ? [...provenance.entries] : [], - updatedAt: now, - }, - ] - }) - for (let i = 0; i < provenanceRecords.length; i += insertBatchSize) { - signal.throwIfAborted() - await tx - .insert(embeddingSecretProvenance) - .values(provenanceRecords.slice(i, i + insertBatchSize)) + if (embeddingRecords.length > 0) { + await tx.delete(embedding).where(eq(embedding.documentId, documentId)) + + const insertBatchSize = LARGE_DOC_CONFIG.MAX_CHUNKS_PER_BATCH + const batches: (typeof embeddingRecords)[] = [] + for (let i = 0; i < embeddingRecords.length; i += insertBatchSize) { + batches.push(embeddingRecords.slice(i, i + insertBatchSize)) + } + + logger.info(`[${documentId}] Inserting ${embeddingRecords.length} embeddings`) + for (const batch of batches) { + signal.throwIfAborted() + await tx.insert(embedding).values(batch) + } + const provenanceRecords = embeddingRecords.flatMap((record, index) => { + const provenance = chunkProvenances[index] + if (!provenance) return [] + return [ + { + embeddingId: record.id, + contentHash: record.chunkHash, + status: provenance.status, + entries: provenance.status === 'exact' ? [...provenance.entries] : [], + updatedAt: now, + }, + ] + }) + for (let i = 0; i < provenanceRecords.length; i += insertBatchSize) { + signal.throwIfAborted() + await tx + .insert(embeddingSecretProvenance) + .values(provenanceRecords.slice(i, i + insertBatchSize)) + } } - } - signal.throwIfAborted() - await tx - .update(document) - .set({ - chunkCount: processed.metadata.chunkCount, - tokenCount: processed.metadata.tokenCount, - characterCount: processed.metadata.characterCount, - processingStatus: 'completed', - processingCompletedAt: now, - processingError: null, - /** A completed pass restores the retry allowance for a future failure. */ - processingAttempts: 0, - processingQueueToken: null, - processingQueuedAt: null, - processingDeferredUntil: null, - }) - .where( - and( - eq(document.id, documentId), - eq(document.processingStatus, 'processing'), - eq(document.processingStartedAt, processingStartedAt), - ...queueGenerationConditions(attemptContext), - eq(document.userExcluded, false), - isNull(document.archivedAt), - isNull(document.deletedAt) + signal.throwIfAborted() + await tx + .update(document) + .set({ + chunkCount: processed.metadata.chunkCount, + tokenCount: processed.metadata.tokenCount, + characterCount: processed.metadata.characterCount, + processingStatus: 'completed', + processingCompletedAt: now, + processingError: null, + /** A completed pass restores the retry allowance for a future failure. */ + processingAttempts: 0, + processingQueueToken: null, + processingQueuedAt: null, + processingDeferredUntil: null, + }) + .where( + and( + eq(document.id, documentId), + eq(document.processingStatus, 'processing'), + eq(document.processingStartedAt, processingStartedAt), + ...queueGenerationConditions(attemptContext), + eq(document.userExcluded, false), + isNull(document.archivedAt), + isNull(document.deletedAt) + ) ) - ) - signal.throwIfAborted() - return true - }) - }, - { - opaqueInputSafe: - documentSecretContext.provenance.status === 'exact' && - documentSecretContext.provenance.entries.length === 0, - } - ), - Math.max(1, processingDeadlineAt - Date.now()), - 'Document processing', - attemptContext?.signal - ) + signal.throwIfAborted() + return true + }) + }, + { + opaqueInputSafe: + documentSecretContext.provenance.status === 'exact' && + documentSecretContext.provenance.entries.length === 0, + } + ), + Math.max(1, processingDeadlineAt - Date.now()), + 'Document processing', + attemptContext?.signal + ) - if (!processingCommitted) { - logger.info(`[${documentId}] Discarded output from an obsolete processing attempt`) - return - } + if (!processingCommitted) { + logger.info(`[${documentId}] Discarded output from an obsolete processing attempt`) + return + } - const processingTime = Date.now() - startTime - logger.info(`[${documentId}] Successfully processed document in ${processingTime}ms`) + const processingTime = Date.now() - startTime + logger.info(`[${documentId}] Successfully processed document in ${processingTime}ms`) - if (billableEmbeddingTokens > 0) { - try { - const costMultiplier = getCostMultiplier() - const { total: cost } = calculateCost( - embeddingPricingId, - billableEmbeddingTokens, - 0, - false, - costMultiplier - ) - if (cost > 0) { - /** - * Dedup identity for this embedding charge. `usage_log.event_key` is - * derived from `sourceReference` and guarded by a permanent unique - * index — usage_log rows are never pruned, there is no retention job - * — so the granularity has to separate two cases for all time: - * - * - A retry of the same pass must collapse. `knowledge-process-document` - * runs up to `KB_CONFIG_MAX_ATTEMPTS` attempts and the stale-document - * sweep can re-dispatch on top of that, so any per-attempt component - * (a `Date.now()` stamp, `processingStartedAt`) bills one indexing - * pass several times over. - * - A genuinely new pass must not collapse. A content change, a - * rehydrate, or a user-triggered reprocess pays a real embedding - * bill, and keying on `documentId` alone would suppress that charge - * permanently. - * - * `indexingPassId` is exactly that discriminator. Without one, the - * resolved pricing id is the safest fallback: it still collapses - * attempts and still re-bills a knowledge base whose embedding model - * changed. Token counts are deliberately left out — OCR-backed parsing - * is not bit-stable across attempts, so they would break the dedup - * they appear to sharpen. - */ - const usageSourceReference = [ - 'knowledge-document', - documentId, - indexingPassId ?? `model:${embeddingPricingId}`, - ].join(':') - await recordUsage({ - userId: documentActorUserId, - workspaceId: ctx.workspaceId ?? undefined, - ...toBillingContext(billingAttribution), - entries: [ - { - category: 'model', - source: 'knowledge-base', - description: embeddingModelName, - cost, - sourceReference: usageSourceReference, - metadata: { inputTokens: billableEmbeddingTokens, outputTokens: 0 }, - }, - ], - }) - await checkAndBillPayerOverageThreshold(billingAttribution.billingEntity) - } else { - logger.warn( - `[${documentId}] Embedding model "${embeddingModelName}" has no pricing entry — billing skipped`, - { billableEmbeddingTokens, embeddingModelName } + if (billableEmbeddingTokens > 0) { + try { + const costMultiplier = getCostMultiplier() + const { total: cost } = calculateCost( + embeddingPricingId, + billableEmbeddingTokens, + 0, + false, + costMultiplier ) + if (cost > 0) { + /** + * Dedup identity for this embedding charge. `usage_log.event_key` is + * derived from `sourceReference` and guarded by a permanent unique + * index — usage_log rows are never pruned, there is no retention job + * — so the granularity has to separate two cases for all time: + * + * - A retry of the same pass must collapse. `knowledge-process-document` + * runs up to `KB_CONFIG_MAX_ATTEMPTS` attempts and the stale-document + * sweep can re-dispatch on top of that, so any per-attempt component + * (a `Date.now()` stamp, `processingStartedAt`) bills one indexing + * pass several times over. + * - A genuinely new pass must not collapse. A content change, a + * rehydrate, or a user-triggered reprocess pays a real embedding + * bill, and keying on `documentId` alone would suppress that charge + * permanently. + * + * `indexingPassId` is exactly that discriminator. Without one, the + * resolved pricing id is the safest fallback: it still collapses + * attempts and still re-bills a knowledge base whose embedding model + * changed. Token counts are deliberately left out — OCR-backed parsing + * is not bit-stable across attempts, so they would break the dedup + * they appear to sharpen. + */ + const usageSourceReference = [ + 'knowledge-document', + documentId, + indexingPassId ?? `model:${embeddingPricingId}`, + ].join(':') + await recordUsage({ + userId: documentActorUserId, + workspaceId: ctx.workspaceId ?? undefined, + ...toBillingContext(billingAttribution), + entries: [ + { + category: 'model', + source: 'knowledge-base', + description: embeddingModelName, + cost, + sourceReference: usageSourceReference, + metadata: { inputTokens: billableEmbeddingTokens, outputTokens: 0 }, + }, + ], + }) + await checkAndBillPayerOverageThreshold(billingAttribution.billingEntity) + } else { + logger.warn( + `[${documentId}] Embedding model "${embeddingModelName}" has no pricing entry — billing skipped`, + { billableEmbeddingTokens, embeddingModelName } + ) + } + } catch (billingError) { + logger.error(`[${documentId}] Failed to record embedding usage`, { error: billingError }) } - } catch (billingError) { - logger.error(`[${documentId}] Failed to record embedding usage`, { error: billingError }) } - } + }) } catch (error) { const processingTime = Date.now() - startTime const embeddingQuotaExhausted = isEmbeddingQuotaExhaustion(error) diff --git a/apps/sim/lib/knowledge/documents/utils.test.ts b/apps/sim/lib/knowledge/documents/utils.test.ts index 36e742545a1..c840cecb406 100644 --- a/apps/sim/lib/knowledge/documents/utils.test.ts +++ b/apps/sim/lib/knowledge/documents/utils.test.ts @@ -14,9 +14,8 @@ vi.mock('@/lib/core/security/input-validation.server', () => ({ secureFetchWithValidation: mockSecureFetchWithValidation, })) -import { secureFetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' +import { fetchWithRetry, secureFetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' import { - fetchWithRetry, getRetryAfterMs, type HTTPError, hasRateLimitEvidence, diff --git a/apps/sim/lib/knowledge/documents/utils.ts b/apps/sim/lib/knowledge/documents/utils.ts index ce8a6fbb321..879525bf723 100644 --- a/apps/sim/lib/knowledge/documents/utils.ts +++ b/apps/sim/lib/knowledge/documents/utils.ts @@ -41,8 +41,12 @@ type RetryableError = | { status?: number; message?: string; headers?: HeaderReader } export interface RetryOptions { - /** Provider transport hooks run for every attempt, including retries and streamed responses. */ - fetcher?: typeof fetch + /** Admission hooks must call the supplied transport, which resolves routing after any wait. */ + fetcher?: ( + input: RequestInfo | URL, + init: RequestInit, + transport: typeof fetch + ) => Promise /** Cancels the current retry cycle, including waits between attempts. */ signal?: AbortSignal maxRetries?: number @@ -58,7 +62,6 @@ export interface RetryOptions { const MAX_HTTP_ERROR_DIAGNOSTIC_CHARS = 2000 const HTTP_ERROR_BODY_OMITTED = '[response body omitted]' -const DEFAULT_FETCH_RETRY_BUDGET_MS = 150_000 /** * Reads an upstream error body without allowing a provider or proxy error page @@ -603,51 +606,3 @@ export const VALIDATE_RETRY_OPTIONS: RetryOptions = { initialDelayMs: 1000, maxDelayMs: 10000, } - -/** - * Bounds requests and response bodies within one retry budget. - */ -export async function fetchWithRetry( - url: string, - options: RequestInit = {}, - retryOptions: RetryOptions = {} -): Promise { - const callerSignal = options.signal - ? retryOptions.signal - ? AbortSignal.any([options.signal, retryOptions.signal]) - : options.signal - : retryOptions.signal - - return retryWithExponentialBackoff( - async (signal, deadlineAt) => { - /** The fetch deadline stays active while callers consume the returned response body. */ - const requestSignal = AbortSignal.any([ - signal, - AbortSignal.timeout(Math.max(0, Math.ceil(deadlineAt - Date.now()))), - ]) - const response = await (retryOptions.fetcher ?? fetch)(url, { - ...options, - signal: requestSignal, - }) - - if ( - !response.ok && - isRetryableError({ status: response.status, headers: response.headers }) - ) { - throw await createRetryableHttpError(response) - } - - return response - }, - { - ...retryOptions, - retryBudgetMs: retryOptions.retryBudgetMs ?? DEFAULT_FETCH_RETRY_BUDGET_MS, - maxRetryAfterMs: - retryOptions.maxRetryAfterMs ?? - retryOptions.retryBudgetMs ?? - retryOptions.maxDelayMs ?? - 30_000, - signal: callerSignal, - } - ) -} diff --git a/apps/sim/lib/mcp/connection-manager.test.ts b/apps/sim/lib/mcp/connection-manager.test.ts index 37eea450396..0954cec05e3 100644 --- a/apps/sim/lib/mcp/connection-manager.test.ts +++ b/apps/sim/lib/mcp/connection-manager.test.ts @@ -36,7 +36,9 @@ const { mockGetOrCreateOauthRow, mockValidateMcpDomain, mockValidateMcpServerSsrf, + mockWithResourceOutboundScope, } = vi.hoisted(() => ({ + mockWithResourceOutboundScope: vi.fn(), MockMcpClientConstructor: vi.fn(), mockValidateMcpDomain: vi.fn(), mockValidateMcpServerSsrf: vi.fn(), @@ -45,6 +47,10 @@ const { mockGetOrCreateOauthRow: vi.fn(), })) +vi.mock('@/lib/core/network/resource-scope.server', () => ({ + withResourceOutboundScope: mockWithResourceOutboundScope, +})) + vi.mock('@/lib/mcp/pubsub', () => ({ mcpPubSub: { onToolsChanged: mockOnToolsChanged, @@ -84,6 +90,7 @@ describe('McpConnectionManager', () => { beforeEach(() => { vi.clearAllMocks() + mockWithResourceOutboundScope.mockImplementation((_owner, run) => run()) mockValidateMcpServerSsrf.mockResolvedValue('93.184.216.34') mockGetOrCreateOauthRow.mockResolvedValue({ id: 'oauth-row-1', @@ -111,6 +118,28 @@ describe('McpConnectionManager', () => { } describe('concurrent connect() guard', () => { + it('allows retrying a connection after workspace ownership resolution fails', async () => { + const connect = vi.fn().mockResolvedValue(undefined) + MockMcpClientConstructor.mockImplementation( + class { + connect = connect + disconnect = vi.fn().mockResolvedValue(undefined) + hasListChangedCapability = () => true + onClose = vi.fn() + } + ) + const mgr = createFreshManager() + const config = serverConfig('server-retry') + mockWithResourceOutboundScope.mockRejectedValueOnce(new Error('Workspace unavailable')) + + await expect(mgr.connect(config, 'user-1', 'ws-1')).rejects.toThrow('Workspace unavailable') + expect(MockMcpClientConstructor).not.toHaveBeenCalled() + await expect(mgr.connect(config, 'user-1', 'ws-1')).resolves.toEqual({ + supportsListChanged: true, + }) + expect(connect).toHaveBeenCalledOnce() + }) + it('creates only one client when two connect() calls race for the same serverId', async () => { const deferred = createDeferred() const instances: MockMcpClient[] = [] diff --git a/apps/sim/lib/mcp/connection-manager.ts b/apps/sim/lib/mcp/connection-manager.ts index ebaa67f96d0..5c64310dcba 100644 --- a/apps/sim/lib/mcp/connection-manager.ts +++ b/apps/sim/lib/mcp/connection-manager.ts @@ -1,3 +1,4 @@ +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' /** * MCP Connection Manager * @@ -142,98 +143,100 @@ export class McpConnectionManager { this.connectingServers.add(key) try { - const onToolsChanged: McpToolsChangedCallback = () => { - this.handleToolsChanged(key) - } + return await withResourceOutboundScope({ workspaceId }, async () => { + const onToolsChanged: McpToolsChangedCallback = () => { + this.handleToolsChanged(key) + } - let oauthCredentials: McpClientOptions['oauthCredentials'] - if (config.authType === 'oauth') { - const row = await getOrCreateOauthRow({ - mcpServerId: config.id, - userId, - workspaceId, - }) - if (!row.tokens) { - logger.info( - `[${config.name}] OAuth server has no workspace tokens — skipping persistent connection until authorized` - ) - return { supportsListChanged: false } + let oauthCredentials: McpClientOptions['oauthCredentials'] + if (config.authType === 'oauth') { + const row = await getOrCreateOauthRow({ + mcpServerId: config.id, + userId, + workspaceId, + }) + if (!row.tokens) { + logger.info( + `[${config.name}] OAuth server has no workspace tokens — skipping persistent connection until authorized` + ) + return { supportsListChanged: false } + } + oauthCredentials = { + credentialId: config.id, + initialProvider: new SimMcpOauthProvider({ + row, + preregistered: await loadPreregisteredClient(config.id), + }), + loadProvider: async () => { + const current = await getOrCreateOauthRow({ + mcpServerId: config.id, + userId, + workspaceId, + }) + if (!current.tokens) { + throw new McpOauthAuthorizationRequiredError(config.id, config.name) + } + const preregistered = await loadPreregisteredClient(config.id) + return new SimMcpOauthProvider({ row: current, preregistered }) + }, + } } - oauthCredentials = { - credentialId: config.id, - initialProvider: new SimMcpOauthProvider({ - row, - preregistered: await loadPreregisteredClient(config.id), - }), - loadProvider: async () => { - const current = await getOrCreateOauthRow({ - mcpServerId: config.id, - userId, - workspaceId, - }) - if (!current.tokens) { - throw new McpOauthAuthorizationRequiredError(config.id, config.name) - } - const preregistered = await loadPreregisteredClient(config.id) - return new SimMcpOauthProvider({ row: current, preregistered }) + + validateMcpDomain(config.url) + const resolvedIP = await validateMcpServerSsrf(config.url) + + const client = new McpClient({ + config, + securityPolicy: { + requireConsent: false, + auditLevel: 'basic', + maxToolExecutionsPerHour: 1000, }, + onToolsChanged, + resolvedIP: resolvedIP ?? undefined, + oauthCredentials, + }) + + try { + await withConnectTimeout(client, config.name) + } catch (error) { + logger.error(`[${config.name}] Failed to connect for persistent monitoring:`, error) + return { supportsListChanged: false } } - } - validateMcpDomain(config.url) - const resolvedIP = await validateMcpServerSsrf(config.url) - - const client = new McpClient({ - config, - securityPolicy: { - requireConsent: false, - auditLevel: 'basic', - maxToolExecutionsPerHour: 1000, - }, - onToolsChanged, - resolvedIP: resolvedIP ?? undefined, - oauthCredentials, - }) + const supportsListChanged = client.hasListChangedCapability() - try { - await withConnectTimeout(client, config.name) - } catch (error) { - logger.error(`[${config.name}] Failed to connect for persistent monitoring:`, error) - return { supportsListChanged: false } - } + if (!supportsListChanged) { + logger.info( + `[${config.name}] Server does not support listChanged — disconnecting (fallback to cache)` + ) + await client.disconnect() + return { supportsListChanged: false } + } - const supportsListChanged = client.hasListChangedCapability() + this.clearReconnectTimer(key) - if (!supportsListChanged) { - logger.info( - `[${config.name}] Server does not support listChanged — disconnecting (fallback to cache)` - ) - await client.disconnect() - return { supportsListChanged: false } - } + this.connections.set(key, client) + this.states.set(key, { + serverId: config.id, + serverName: config.name, + workspaceId, + userId, + connected: true, + supportsListChanged: true, + reconnectAttempts: 0, + lastActivity: Date.now(), + }) - this.clearReconnectTimer(key) + client.onClose(() => { + this.handleDisconnect(config, userId, workspaceId) + }) - this.connections.set(key, client) - this.states.set(key, { - serverId: config.id, - serverName: config.name, - workspaceId, - userId, - connected: true, - supportsListChanged: true, - reconnectAttempts: 0, - lastActivity: Date.now(), - }) + this.ensureIdleCheck() - client.onClose(() => { - this.handleDisconnect(config, userId, workspaceId) + logger.info(`[${config.name}] Persistent connection established (listChanged supported)`) + return { supportsListChanged: true } }) - - this.ensureIdleCheck() - - logger.info(`[${config.name}] Persistent connection established (listChanged supported)`) - return { supportsListChanged: true } } finally { this.connectingServers.delete(key) } diff --git a/apps/sim/lib/mcp/middleware.ts b/apps/sim/lib/mcp/middleware.ts index 88be4b04c39..a3bec4ce0dd 100644 --- a/apps/sim/lib/mcp/middleware.ts +++ b/apps/sim/lib/mcp/middleware.ts @@ -8,6 +8,7 @@ import { checkSessionOrInternalAuth, type AuthResult as HybridAuthResult, } from '@/lib/auth/hybrid' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { generateRequestId } from '@/lib/core/utils/request' import { assertContentLengthWithinLimit, @@ -328,7 +329,10 @@ export function withMcpAuth>( } try { - return await handler(request, (authResult as AuthResult).context, routeContext) + const context = (authResult as AuthResult).context + return await withResourceOutboundScope({ workspaceId: context.workspaceId }, () => + handler(request, context, routeContext) + ) } catch (error) { const bodyErrorResponse = mcpBodyReadErrorResponse(error, request) if (bodyErrorResponse) return bodyErrorResponse diff --git a/apps/sim/lib/mcp/pinned-fetch.ts b/apps/sim/lib/mcp/pinned-fetch.ts index a8360236156..a09d263299f 100644 --- a/apps/sim/lib/mcp/pinned-fetch.ts +++ b/apps/sim/lib/mcp/pinned-fetch.ts @@ -1,10 +1,10 @@ import type { FetchLike } from '@modelcontextprotocol/sdk/shared/transport.js' import { createLogger } from '@sim/logger' import { isPrivateIp } from '@sim/security/ssrf' -import type { Agent } from 'undici' import { createPinnedFetchWithDispatcher, createSsrfGuardedFetchWithDispatcher, + type OutboundFetchDispatcher, } from '@/lib/core/security/input-validation.server' import { MCP_EGRESS_PROFILE, @@ -277,7 +277,7 @@ async function bufferUnderDeadline(response: Response, signal: AbortSignal): Pro */ function releaseStreamOnSettle( response: Response, - dispatcher: Agent | undefined, + dispatcher: OutboundFetchDispatcher | undefined, signal: AbortSignal ): Response { if (!dispatcher || !response.body) { @@ -349,7 +349,7 @@ export function createSsrfGuardedMcpFetch( // Bound every phase — validation, request, body read — by the deadline + caller signal. const signal = init?.signal ? AbortSignal.any([init.signal, timeoutSignal]) : timeoutSignal // Per-request Agent must be torn down (finally): a one-shot leg never reuses its socket. - let dispatcher: Agent | undefined + let dispatcher: OutboundFetchDispatcher | undefined try { logger.info('OAuth guarded fetch: validating', { host }) const resolvedIP = await withDeadline(validateMcpServerSsrf(target, profile), signal) diff --git a/apps/sim/lib/selectors/application/execute-selector.ts b/apps/sim/lib/selectors/application/execute-selector.ts index 32ca55fb3b5..ca802667784 100644 --- a/apps/sim/lib/selectors/application/execute-selector.ts +++ b/apps/sim/lib/selectors/application/execute-selector.ts @@ -7,6 +7,7 @@ import { import { defineAuthorizedWorkspaceUseCase } from '@/lib/core/application' import type { OperationUseCase } from '@/lib/core/application/operation' import { requireOrganizationMembership } from '@/lib/core/application/organization-authorization' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { OrchestrationError } from '@/lib/core/orchestration/types' import { authorizePersonalSearchSetup } from '@/lib/knowledge/application/personal-search-account' import { type CredentialAuditRequest, recordCredentialAccess } from '@/lib/oauth/token-resolution' @@ -371,6 +372,7 @@ export const executeSelector: OperationUseCase< scope: args.input.scope, }) validateAuthorizedInput(args.input, context) - return executeAuthorizedSelector({ principal: args.principal, input: args.input, context }) + const executionArgs = { principal: args.principal, input: args.input, context } + return withResourceOutboundScope(context, () => executeAuthorizedSelector(executionArgs)) }, } diff --git a/apps/sim/lib/webhooks/polling/orchestrator.ts b/apps/sim/lib/webhooks/polling/orchestrator.ts index 6a7f553eb7d..d6cf75822b5 100644 --- a/apps/sim/lib/webhooks/polling/orchestrator.ts +++ b/apps/sim/lib/webhooks/polling/orchestrator.ts @@ -1,5 +1,6 @@ import { createLogger } from '@sim/logger' import { generateShortId } from '@sim/utils/id' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { getPollingHandler } from '@/lib/webhooks/polling/registry' import type { PollSummary } from '@/lib/webhooks/polling/types' import { fetchActiveWebhooks, runWithConcurrency } from '@/lib/webhooks/polling/utils' @@ -26,12 +27,14 @@ export async function pollProvider(providerName: string): Promise { activeWebhooks, async (entry) => { const requestId = generateShortId() - return handler.pollWebhook({ - webhookData: entry.webhook, - workflowData: entry.workflow, - requestId, - logger, - }) + return withResourceOutboundScope({ workspaceId: entry.workflow.workspaceId }, () => + handler.pollWebhook({ + webhookData: entry.webhook, + workflowData: entry.workflow, + requestId, + logger, + }) + ) }, logger ) diff --git a/apps/sim/lib/webhooks/polling/outlook.ts b/apps/sim/lib/webhooks/polling/outlook.ts index 4cedf69a4dd..7ee7e269233 100644 --- a/apps/sim/lib/webhooks/polling/outlook.ts +++ b/apps/sim/lib/webhooks/polling/outlook.ts @@ -2,7 +2,7 @@ import type { Logger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { htmlToText } from 'html-to-text' import { pollingIdempotency } from '@/lib/core/idempotency/service' -import { fetchWithRetry } from '@/lib/knowledge/documents/utils' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' import { getProviderConfig, type PollingProviderHandler, diff --git a/apps/sim/lib/webhooks/provider-subscriptions.ts b/apps/sim/lib/webhooks/provider-subscriptions.ts index 8cd221622b9..ddd7b11a0ee 100644 --- a/apps/sim/lib/webhooks/provider-subscriptions.ts +++ b/apps/sim/lib/webhooks/provider-subscriptions.ts @@ -2,6 +2,7 @@ import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { omit } from '@sim/utils/object' import type { NextRequest } from 'next/server' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { resolveBackgroundWebhookEnv, resolveWebhookProviderConfig, @@ -151,14 +152,15 @@ export async function createExternalWebhookSubscription( * outbox handler must not mint a provider resource it can no longer * durably record. */ - options.signal?.throwIfAborted() - - const result = await handler.createSubscription({ - webhook: { ...webhookData, providerConfig: resolvedProviderConfig }, - workflow, - userId, - requestId, - request, + const result = await withResourceOutboundScope({ workspaceId }, () => { + options.signal?.throwIfAborted() + return handler.createSubscription!({ + webhook: { ...webhookData, providerConfig: resolvedProviderConfig }, + workflow, + userId, + requestId, + request, + }) }) if (!result) { @@ -213,12 +215,18 @@ export async function cleanupExternalWebhook( { envVars } ) - await handler.deleteSubscription({ - webhook: resolvedWebhook, - workflow, - requestId, - strict: options.throwOnError, - }) + /** Workspace archival precedes provider cleanup; routing still uses its canonical owner. */ + await withResourceOutboundScope( + { workspaceId }, + () => + handler.deleteSubscription!({ + webhook: resolvedWebhook, + workflow, + requestId, + strict: options.throwOnError, + }), + { includeArchived: true } + ) } catch (error) { logger.warn(`[${requestId}] Error cleaning up external webhook (non-fatal)`, { provider, diff --git a/apps/sim/lib/workflows/executor/execution-core.ts b/apps/sim/lib/workflows/executor/execution-core.ts index a465c6b48e4..09fbb27c2d7 100644 --- a/apps/sim/lib/workflows/executor/execution-core.ts +++ b/apps/sim/lib/workflows/executor/execution-core.ts @@ -19,6 +19,8 @@ import { getTimeoutErrorMessage, isTimeoutAbortReason, } from '@/lib/core/execution-limits' +import { isOutboundRoutingEnabled } from '@/lib/core/network/config.server' +import { runWithOutboundOrganization } from '@/lib/core/network/context.server' import { withDatabaseReadRetry } from '@/lib/db/read-retry' import { getExecutionEnvironment } from '@/lib/environment/utils' import { clearExecutionCancellation } from '@/lib/execution/cancellation' @@ -29,6 +31,7 @@ import type { LoggingSession } from '@/lib/logs/execution/logging-session' import { redactLargeValueRefsInValue } from '@/lib/logs/execution/pii-large-values' import { redactObjectStrings } from '@/lib/logs/execution/pii-redaction' import { buildTraceSpans } from '@/lib/logs/execution/trace-spans/trace-spans' +import { resolveActiveWorkflowApplicationContext } from '@/lib/workflows/application/context' import { waitForChildRuns } from '@/lib/workflows/custom-blocks/child-execution' import { getCustomBlockRowsForWorkspace } from '@/lib/workflows/custom-blocks/operations' import { resolveStartBlockRunIdentity } from '@/lib/workflows/executor/start-run-identity' @@ -397,7 +400,13 @@ export async function executeWorkflowCore( label: 'getCustomBlockRowsForWorkspace', }) : [] - return withCustomBlockOverlay(rows, () => executeWorkflowCoreImpl(options)) + const execute = () => withCustomBlockOverlay(rows, () => executeWorkflowCoreImpl(options)) + if (!isOutboundRoutingEnabled()) return execute() + const context = await resolveActiveWorkflowApplicationContext({ + workflowId: options.snapshot.metadata.workflowId, + assertedWorkspaceId: workspaceId, + }) + return runWithOutboundOrganization(context.workspaceOrganizationId, execute) } async function executeWorkflowCoreImpl( diff --git a/apps/sim/lib/workspaces/admin-move-source-impact.ts b/apps/sim/lib/workspaces/admin-move-source-impact.ts index 687f89ec9ac..8a770c24d6e 100644 --- a/apps/sim/lib/workspaces/admin-move-source-impact.ts +++ b/apps/sim/lib/workspaces/admin-move-source-impact.ts @@ -65,7 +65,7 @@ const ENTERPRISE_GATED_SECTION_LABELS: Record ({ +vi.mock('@/lib/knowledge/documents/secure-fetch.server', () => ({ fetchWithRetry: (url: string, init: RequestInit) => fetch(url, init), })) diff --git a/apps/sim/tools/jira/utils.ts b/apps/sim/tools/jira/utils.ts index 545c9b32357..dd5820e93ea 100644 --- a/apps/sim/tools/jira/utils.ts +++ b/apps/sim/tools/jira/utils.ts @@ -1,7 +1,7 @@ import { createLogger } from '@sim/logger' import { resolveAtlassianCloudId } from '@/lib/atlassian/discovery' +import { fetchWithRetry } from '@/lib/knowledge/documents/secure-fetch.server' import type { RetryOptions } from '@/lib/knowledge/documents/utils' -import { fetchWithRetry } from '@/lib/knowledge/documents/utils' import { AttachmentDownloadBudget, rethrowAttachmentDownloadError, diff --git a/scripts/check-tool-registry-boundary.baseline.json b/scripts/check-tool-registry-boundary.baseline.json index af93cacb593..66d5fe7185f 100644 --- a/scripts/check-tool-registry-boundary.baseline.json +++ b/scripts/check-tool-registry-boundary.baseline.json @@ -402,16 +402,16 @@ "gateways": {} }, "app/workspace/[workspaceId]/settings/[section]/page.tsx": { - "modules": 2292, + "modules": 2348, "gateways": { - "apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx": 707, + "apps/sim/app/workspace/[workspaceId]/settings/[section]/settings.tsx": 739, "apps/sim/triggers/registry.ts": 485, - "apps/sim/lib/auth/index.ts": 366, + "apps/sim/lib/auth/index.ts": 378, "apps/sim/blocks/registry.ts": 354, - "apps/sim/lib/webhooks/providers/index.ts": 118, - "apps/sim/lib/webhooks/providers/registry.ts": 116, - "apps/sim/ee/access-control/components/access-control.tsx": 75, - "apps/sim/ee/access-control/components/group-detail.tsx": 73 + "apps/sim/lib/webhooks/providers/index.ts": 117, + "apps/sim/lib/webhooks/providers/registry.ts": 115, + "apps/sim/app/workspace/[workspaceId]/settings/components/recently-deleted/recently-deleted.tsx": 80, + "apps/sim/ee/access-control/components/access-control.tsx": 75 } }, "app/workspace/[workspaceId]/settings/billing/credit-usage/layout.tsx": { From 9fd04ad81469b3f048795e7310fd4c71e80f027f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sun, 13 Sep 2026 22:08:41 -0700 Subject: [PATCH 2/5] fix(network): reject missing gateway catalogs --- apps/sim/lib/core/network/config.server.test.ts | 17 +++++++++++++++++ apps/sim/lib/core/network/config.server.ts | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/apps/sim/lib/core/network/config.server.test.ts b/apps/sim/lib/core/network/config.server.test.ts index 325946e2a05..2cbe5959574 100644 --- a/apps/sim/lib/core/network/config.server.test.ts +++ b/apps/sim/lib/core/network/config.server.test.ts @@ -34,6 +34,23 @@ describe('outbound configuration', () => { expect(readSnapshot).not.toHaveBeenCalled() }) + it.each([undefined, '{}'])( + 'rejects missing AppConfig gateway catalog %s before routing', + (gateways) => { + expect(() => + createOutboundRoutingReader( + { + ...options, + source: 'appconfig', + configuration: undefined, + gateways, + }, + dependencies + ) + ).toThrow('INVALID_CONFIGURATION') + } + ) + it('does not treat missing context as a personal workspace', async () => { const reader = createOutboundRoutingReader(options, dependencies) await expect(reader.resolve(undefined)).rejects.toThrow('MISSING_SCOPE') diff --git a/apps/sim/lib/core/network/config.server.ts b/apps/sim/lib/core/network/config.server.ts index 2cf6c959fcf..d9f0f57529a 100644 --- a/apps/sim/lib/core/network/config.server.ts +++ b/apps/sim/lib/core/network/config.server.ts @@ -32,7 +32,7 @@ const gatewaySchema = z.strictObject({ }) const gatewaysSchema = z .record(keySchema, gatewaySchema) - .refine((value) => Object.keys(value).length <= 100) + .refine((value) => Object.keys(value).length > 0 && Object.keys(value).length <= 100) const credentialSchema = z.strictObject({ token: z .string() From 8d2ffb441dad3121a096672b0d9202dffb7e8c24 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sun, 13 Sep 2026 22:56:57 -0700 Subject: [PATCH 3/5] fix(network): preserve transport and background execution behavior --- .../docs/platform/enterprise/security.mdx | 2 + .../enrichment-capability-subject.test.ts | 32 +++ apps/sim/background/webhook-execution.test.ts | 20 ++ apps/sim/background/webhook-execution.ts | 206 +++++++++--------- .../background/workflow-column-execution.ts | 43 ++-- apps/sim/lib/copilot/tools/server/router.ts | 5 +- .../lib/core/network/config.server.test.ts | 23 +- apps/sim/lib/core/network/config.server.ts | 6 +- .../lib/core/network/context.server.test.ts | 25 +++ .../fixtures/gateway-runtime.fixture.ts | 1 - .../lib/core/network/gateway.server.test.ts | 7 +- apps/sim/lib/core/network/gateway.server.ts | 2 +- apps/sim/lib/core/network/routing.test.ts | 23 +- apps/sim/lib/core/network/routing.ts | 22 -- .../lib/core/network/transport.server.test.ts | 14 +- apps/sim/lib/core/network/transport.server.ts | 7 +- .../core/security/input-validation.server.ts | 11 +- .../secure-fetch-response-cap.server.test.ts | 38 ++++ apps/sim/lib/execution/isolated-vm.test.ts | 80 ++++++- 19 files changed, 367 insertions(+), 200 deletions(-) diff --git a/apps/docs/content/docs/platform/enterprise/security.mdx b/apps/docs/content/docs/platform/enterprise/security.mdx index 237aec5bb5b..0a67efba9cd 100644 --- a/apps/docs/content/docs/platform/enterprise/security.mdx +++ b/apps/docs/content/docs/platform/enterprise/security.mdx @@ -25,6 +25,8 @@ Dedicated routing supports public IPv4 destinations over HTTPS on port 443. It a Native database connections, AWS SDK integrations, remote sandbox traffic, and browser requests keep their existing network paths. Other provider SDKs and OAuth authorization or token refresh calls require separate coverage confirmation. Dedicated IPs do not change access permissions in connected services. +Google Drive, Fireflies, Google Workspace user and group discovery, and Atlassian OAuth site discovery currently use their existing network paths. + If dedicated routing is unavailable, affected requests fail instead of using shared IPs. Organizations without dedicated routing keep their existing behavior. ## Availability diff --git a/apps/sim/background/enrichment-capability-subject.test.ts b/apps/sim/background/enrichment-capability-subject.test.ts index 1f5361549a4..a89951e86fd 100644 --- a/apps/sim/background/enrichment-capability-subject.test.ts +++ b/apps/sim/background/enrichment-capability-subject.test.ts @@ -5,6 +5,10 @@ import { resetDbChainMock } from '@sim/testing' import { beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ + loadWorkspaceApplicationContext: vi.fn(), + resolveOutboundRoute: vi.fn(async (_organizationId: string | null | undefined) => ({ + kind: 'direct', + })), getTableById: vi.fn(), getRowById: vi.fn(), updateRow: vi.fn(), @@ -22,6 +26,14 @@ const mocks = vi.hoisted(() => ({ loadTableRowSecretProvenance: vi.fn(async () => ({ scope: null, entries: [] })), })) +vi.mock('@/lib/core/network/config.server', () => ({ + isOutboundRoutingEnabled: () => true, + resolveOutboundRoute: mocks.resolveOutboundRoute, +})) +vi.mock('@/lib/workspaces/application/workspace-context', () => ({ + loadWorkspaceApplicationContext: mocks.loadWorkspaceApplicationContext, +})) + vi.mock('@/lib/table/service', () => ({ getTableById: mocks.getTableById })) vi.mock('@/lib/table/rows/service', () => ({ getRowById: mocks.getRowById, @@ -69,6 +81,7 @@ vi.mock('@/lib/core/rate-limiter/rate-limiter', () => ({ }, })) +import { resolveCurrentOutboundRoute } from '@/lib/core/network/context.server' import { runRowCascadeLoop } from '@/background/workflow-column-execution' const GROUP = { @@ -143,6 +156,7 @@ describe('enrichment cell capability subject', () => { beforeEach(() => { vi.clearAllMocks() resetDbChainMock() + mocks.loadWorkspaceApplicationContext.mockResolvedValue({ workspaceOrganizationId: null }) mocks.getTableById.mockResolvedValue(TABLE) mocks.getRowById.mockResolvedValue({ id: 'row-1', @@ -162,6 +176,24 @@ describe('enrichment cell capability subject', () => { mocks.runEnrichment.mockResolvedValue({ result: {}, cost: 0, detail: {} }) }) + it.each(['org_reserved', 'org_other', null])( + 'restores the current workspace owner %s before running a queued enrichment', + async (organizationId) => { + mocks.loadWorkspaceApplicationContext.mockResolvedValue({ + workspaceOrganizationId: organizationId, + }) + mocks.runEnrichment.mockImplementationOnce(async () => { + await resolveCurrentOutboundRoute() + return { result: {}, cost: 0, detail: {} } + }) + + await runRowCascadeLoop(payload(null, 'billing-owner') as never) + + expect(mocks.resolveOutboundRoute).toHaveBeenCalledExactlyOnceWith(organizationId) + expect(mocks.loadWorkspaceApplicationContext).toHaveBeenCalledWith('workspace-1', {}) + } + ) + /** * A workspace-key write is actorless: nobody's permission group governs it, * and the billing owner beside it on the payload is a bystander. Handing that diff --git a/apps/sim/background/webhook-execution.test.ts b/apps/sim/background/webhook-execution.test.ts index 9fc447461f9..ec5340c21af 100644 --- a/apps/sim/background/webhook-execution.test.ts +++ b/apps/sim/background/webhook-execution.test.ts @@ -19,6 +19,7 @@ import { import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' const { + mockWithResourceOutboundScope, mockResolveWebhookRecordProviderConfig, mockExecuteWorkflowCore, mockWasExecutionFinalizedByCore, @@ -32,6 +33,7 @@ const { } = vi.hoisted(() => { const mockEnqueue = vi.fn() return { + mockWithResourceOutboundScope: vi.fn(), mockResolveWebhookRecordProviderConfig: vi.fn(), mockExecuteWorkflowCore: vi.fn(), mockWasExecutionFinalizedByCore: vi.fn(), @@ -62,6 +64,10 @@ const mockGetExecutionEnvironment = environmentUtilsMockFns.mockGetExecutionEnvi afterAll(resetEnvironmentUtilsMock) +vi.mock('@/lib/core/network/resource-scope.server', () => ({ + withResourceOutboundScope: mockWithResourceOutboundScope, +})) + vi.mock('@/lib/execution/preprocessing', () => executionPreprocessingMock) vi.mock('@/lib/logs/execution/logging-session', () => loggingSessionMock) @@ -285,6 +291,7 @@ describe('executeWebhookJob fault vs error handling', () => { projectDiagnosticError: loggingSessionMockFns.mockProjectDiagnosticError, } }) + mockWithResourceOutboundScope.mockReset().mockImplementation((_owner, run) => run()) mockRefreshExecutionSlotExpiry.mockReset().mockResolvedValue(true) mockReleaseExecutionSlot.mockReset().mockResolvedValue(undefined) mockGetProviderHandler.mockReturnValue({}) @@ -856,6 +863,19 @@ describe('executeWebhookJob fault vs error handling', () => { expect(mockExecuteWorkflowCore).not.toHaveBeenCalled() }) + it('requeues transient organization ownership failures before any workflow block starts', async () => { + mockWithResourceOutboundScope.mockRejectedValueOnce( + Object.assign(new Error('Connection terminated unexpectedly'), { code: 'ECONNRESET' }) + ) + + await expect(executeWebhookJob(payload)).resolves.toMatchObject({ + requeued: true, + }) + expect(mockExecuteWorkflowCore).not.toHaveBeenCalled() + expect(mockEnqueue).toHaveBeenCalledOnce() + expect(loggingSessionMockFns.mockSafeCompleteWithError).not.toHaveBeenCalled() + }) + it('requeues on retryable infrastructure errors thrown by setup reads', async () => { dbChainMockFns.limit.mockRejectedValueOnce( Object.assign(new Error('write CONNECT_TIMEOUT'), { code: 'CONNECT_TIMEOUT' }) diff --git a/apps/sim/background/webhook-execution.ts b/apps/sim/background/webhook-execution.ts index ff54b85dcd9..5240bc500c9 100644 --- a/apps/sim/background/webhook-execution.ts +++ b/apps/sim/background/webhook-execution.ts @@ -818,21 +818,21 @@ async function executeWebhookJobInternal( throw new Error(`Workflow ${payload.workflowId} has no associated workspace`) } - return withResourceOutboundScope({ workspaceId }, async () => { - const workflowVariables = (workflowRecord.variables as Record) || {} + const workflowVariables = (workflowRecord.variables as Record) || {} - let deploymentVersionId: string | undefined - /** - * Flipped immediately before `executeWorkflowCore` is invoked. While false, - * no block has run and no execution effect exists, so a retryable - * infrastructure error may be surfaced as a `RetryableSetupError` and the - * whole delivery safely re-attempted. Once true, errors are never - * reclassified as retryable — retrying after the executor started could - * double-run the workflow. - */ - let workflowCoreStarted = false + let deploymentVersionId: string | undefined + /** + * Flipped immediately before `executeWorkflowCore` is invoked. While false, + * no block has run and no execution effect exists, so a retryable + * infrastructure error may be surfaced as a `RetryableSetupError` and the + * whole delivery safely re-attempted. Once true, errors are never + * reclassified as retryable — retrying after the executor started could + * double-run the workflow. + */ + let workflowCoreStarted = false - try { + try { + return await withResourceOutboundScope({ workspaceId }, async () => { const workflowStatePromise = payload.deploymentVersionId ? loadWorkflowDeploymentVersionState( payload.workflowId, @@ -1175,105 +1175,105 @@ async function executeWebhookJobInternal( executedAt: new Date().toISOString(), provider: payload.provider, } - } catch (error: unknown) { - const errorMessage = toError(error).message - const errorStack = error instanceof Error ? error.stack : undefined + }) + } catch (error: unknown) { + const errorMessage = toError(error).message + const errorStack = error instanceof Error ? error.stack : undefined - /** - * Mirrors the schedule executor's setup boundary: an infrastructure error - * raised before the workflow core started left no execution effect, so it - * is surfaced as a `RetryableSetupError` — releasing the idempotency claim - * and, while attempts remain, requeueing without recording a terminal - * failed row for an attempt that will be retried. Exhausted retries fall - * through to normal failure handling but still throw typed so a provider - * redelivery is not rejected for a run that never happened. - */ - const retryableSetupCause = - !workflowCoreStarted && isRetryableInfrastructureError(error) - ? describeRetryableInfrastructureError(error) - : undefined - if (retryableSetupCause && hasRemainingWebhookInfraRetry(payload)) { - logger.warn(`[${requestId}] Retryable setup failure before webhook workflow started`, { - workflowId: payload.workflowId, - provider: payload.provider, - cause: retryableSetupCause, - }) - throw new RetryableSetupError(errorMessage, { cause: retryableSetupCause }) - } + /** + * Mirrors the schedule executor's setup boundary: an infrastructure error + * raised before the workflow core started left no execution effect, so it + * is surfaced as a `RetryableSetupError` — releasing the idempotency claim + * and, while attempts remain, requeueing without recording a terminal + * failed row for an attempt that will be retried. Exhausted retries fall + * through to normal failure handling but still throw typed so a provider + * redelivery is not rejected for a run that never happened. + */ + const retryableSetupCause = + !workflowCoreStarted && isRetryableInfrastructureError(error) + ? describeRetryableInfrastructureError(error) + : undefined + if (retryableSetupCause && hasRemainingWebhookInfraRetry(payload)) { + logger.warn(`[${requestId}] Retryable setup failure before webhook workflow started`, { + workflowId: payload.workflowId, + provider: payload.provider, + cause: retryableSetupCause, + }) + throw new RetryableSetupError(errorMessage, { cause: retryableSetupCause }) + } - logger.error( - `[${requestId}] Webhook execution failed`, - loggingSession.projectDiagnosticError(error, { - workflowId: payload.workflowId, - provider: payload.provider, - }) - ) + logger.error( + `[${requestId}] Webhook execution failed`, + loggingSession.projectDiagnosticError(error, { + workflowId: payload.workflowId, + provider: payload.provider, + }) + ) - // The finalized flag is set inside a fire-and-forget post-execution promise; await it so the - // signal is reliable and the failure is fully persisted before we decide fault vs error. - await loggingSession.waitForPostExecution() + // The finalized flag is set inside a fire-and-forget post-execution promise; await it so the + // signal is reliable and the failure is fully persisted before we decide fault vs error. + await loggingSession.waitForPostExecution() - // A failure inside workflow execution (block error, provider 4xx, missing required field, etc.) - // is finalized by core and already recorded in the execution logs. That is a user/workflow error, - // not a trigger.dev job fault — complete the run normally so we don't fire a false alert. Errors - // that were not finalized came from the webhook pipeline itself, so we re-throw to fault below. - if (wasExecutionFinalizedByCore(error, executionId)) { - return { - success: false, - workflowId: payload.workflowId, - executionId, - output: hasExecutionResult(error) ? error.executionResult.output : {}, - executedAt: new Date().toISOString(), - provider: payload.provider, - } + // A failure inside workflow execution (block error, provider 4xx, missing required field, etc.) + // is finalized by core and already recorded in the execution logs. That is a user/workflow error, + // not a trigger.dev job fault — complete the run normally so we don't fire a false alert. Errors + // that were not finalized came from the webhook pipeline itself, so we re-throw to fault below. + if (wasExecutionFinalizedByCore(error, executionId)) { + return { + success: false, + workflowId: payload.workflowId, + executionId, + output: hasExecutionResult(error) ? error.executionResult.output : {}, + executedAt: new Date().toISOString(), + provider: payload.provider, } + } - try { - await loggingSession.safeStart({ - userId: actorUserId, - actorUserId, - billingAttribution, - workspaceId, - variables: {}, - triggerData: { - isTest: false, - correlation, - }, - deploymentVersionId, - }) - - const executionResult = hasExecutionResult(error) - ? error.executionResult - : { - success: false, - output: {}, - logs: [], - } - const { traceSpans } = buildTraceSpans(executionResult) + try { + await loggingSession.safeStart({ + userId: actorUserId, + actorUserId, + billingAttribution, + workspaceId, + variables: {}, + triggerData: { + isTest: false, + correlation, + }, + deploymentVersionId, + }) - await loggingSession.safeCompleteWithError({ - endedAt: new Date().toISOString(), - totalDurationMs: 0, - error: { - message: errorMessage || 'Webhook execution failed', - stackTrace: errorStack, - }, - traceSpans, - executionState: executionResult.executionState, - }) - } catch (loggingError) { - logger.error( - `[${requestId}] Failed to complete logging session`, - loggingSession.projectDiagnosticError(loggingError) - ) - } + const executionResult = hasExecutionResult(error) + ? error.executionResult + : { + success: false, + output: {}, + logs: [], + } + const { traceSpans } = buildTraceSpans(executionResult) + + await loggingSession.safeCompleteWithError({ + endedAt: new Date().toISOString(), + totalDurationMs: 0, + error: { + message: errorMessage || 'Webhook execution failed', + stackTrace: errorStack, + }, + traceSpans, + executionState: executionResult.executionState, + }) + } catch (loggingError) { + logger.error( + `[${requestId}] Failed to complete logging session`, + loggingSession.projectDiagnosticError(loggingError) + ) + } - if (retryableSetupCause) { - throw new RetryableSetupError(errorMessage, { cause: retryableSetupCause }) - } - throw error + if (retryableSetupCause) { + throw new RetryableSetupError(errorMessage, { cause: retryableSetupCause }) } - }) + throw error + } } export const webhookExecution = task({ diff --git a/apps/sim/background/workflow-column-execution.ts b/apps/sim/background/workflow-column-execution.ts index 907765d65a8..ef8c5022cf7 100644 --- a/apps/sim/background/workflow-column-execution.ts +++ b/apps/sim/background/workflow-column-execution.ts @@ -23,6 +23,7 @@ import { isTimeoutAbortReason, type TimeoutAbortController, } from '@/lib/core/execution-limits' +import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { RateLimiter } from '@/lib/core/rate-limiter/rate-limiter' import { registerManualExecutionAborter, @@ -681,26 +682,28 @@ async function runWorkflowAndWriteTerminal( await enrichmentRegistry.importCrossingProvenance(inputProvenance, enrichInputs, { trusted: true, }) - const { result, cost, detail } = await runEnrichment(enrichment, enrichInputs, { - tableId, - rowId, - workspaceId, - /** - * The person who asked, not who pays. `triggeredByUserId` is an - * attribution: for a workspace-API-key run it names the workspace's - * billing owner, and running that bystander's tool denylist against - * an actorless request is wrong in both directions — it fails cells - * nobody meant to govern, and it skips the denylist for the person - * who actually triggered one. The governed subject is carried - * separately from the dispatch. `null` means no per-tool gate - * applies, which is the documented behavior for an actorless run — - * stated, because the field is required precisely so it cannot be - * skipped by omission. - */ - userId: payload.capabilityGovernedUserId ?? null, - signal: attemptSignal, - resolvedSecretTraceRegistry: enrichmentRegistry, - }) + const { result, cost, detail } = await withResourceOutboundScope({ workspaceId }, () => + runEnrichment(enrichment, enrichInputs, { + tableId, + rowId, + workspaceId, + /** + * The person who asked, not who pays. `triggeredByUserId` is an + * attribution: for a workspace-API-key run it names the workspace's + * billing owner, and running that bystander's tool denylist against + * an actorless request is wrong in both directions — it fails cells + * nobody meant to govern, and it skips the denylist for the person + * who actually triggered one. The governed subject is carried + * separately from the dispatch. `null` means no per-tool gate + * applies, which is the documented behavior for an actorless run — + * stated, because the field is required precisely so it cannot be + * skipped by omission. + */ + userId: payload.capabilityGovernedUserId ?? null, + signal: attemptSignal, + resolvedSecretTraceRegistry: enrichmentRegistry, + }) + ) // An abort during the cascade must not be recorded as a completed cell. if (attemptSignal.aborted) { diff --git a/apps/sim/lib/copilot/tools/server/router.ts b/apps/sim/lib/copilot/tools/server/router.ts index 173b6bc47e2..0c499a66e9e 100644 --- a/apps/sim/lib/copilot/tools/server/router.ts +++ b/apps/sim/lib/copilot/tools/server/router.ts @@ -65,7 +65,6 @@ import { getCredentialsServerTool } from '@/lib/copilot/tools/server/user/get-cr import { setEnvironmentVariablesServerTool } from '@/lib/copilot/tools/server/user/set-environment-variables' import { editWorkflowServerTool } from '@/lib/copilot/tools/server/workflow/edit-workflow' import { queryLogsServerTool } from '@/lib/copilot/tools/server/workflow/query-logs' -import { withResourceOutboundScope } from '@/lib/core/network/resource-scope.server' import { OrchestrationError } from '@/lib/core/orchestration/types' import { listCustomBlocksWithInputsForWorkspace } from '@/lib/workflows/custom-blocks/operations' import { withCustomBlockOverlay } from '@/blocks/custom/server-overlay' @@ -291,9 +290,7 @@ export async function routeExecution( const inner = run run = () => withCustomBlockOverlay(rows, inner) } - const result = await (context?.workspaceId || context?.organizationId - ? withResourceOutboundScope(context, run) - : run()) + const result = await run() // Validate output if tool declares a schema; otherwise fall back to the // generated JSON schema contract emitted from Go. diff --git a/apps/sim/lib/core/network/config.server.test.ts b/apps/sim/lib/core/network/config.server.test.ts index 2cbe5959574..945aff74e29 100644 --- a/apps/sim/lib/core/network/config.server.test.ts +++ b/apps/sim/lib/core/network/config.server.test.ts @@ -5,7 +5,6 @@ import { createOutboundRoutingReader } from '@/lib/core/network/config.server' const document = { schemaVersion: 1, revision: 'revision-1', - defaultRoute: { kind: 'direct' }, organizations: { org_a: { kind: 'gateway', gatewayId: 'gateway_a' } }, } const catalog = { @@ -13,7 +12,6 @@ const catalog = { organizationId: 'org_a', url: 'https://proxy.example.invalid/', credentialId: 'credential_a', - generation: 'generation-1', }, } const credentials = { credential_a: { token: 'synthetic-test-token-0000000000000000' } } @@ -107,6 +105,27 @@ describe('outbound configuration', () => { expect(await reader.resolve('org_other')).toEqual({ kind: 'direct' }) }) + it('uses explicit direct and blocked assignments only for reserved organizations', async () => { + for (const kind of ['direct', 'blocked']) { + const reader = createOutboundRoutingReader( + { + ...options, + configuration: JSON.stringify({ + ...document, + organizations: { org_a: { kind } }, + }), + }, + dependencies + ) + if (kind === 'direct') { + expect(await reader.resolve('org_a')).toEqual({ kind: 'direct' }) + } else { + await expect(reader.resolve('org_a')).rejects.toThrow('ROUTE_BLOCKED') + } + expect(await reader.resolve('constructor')).toEqual({ kind: 'direct' }) + } + }) + it('rejects an outbound IP published for two different owners', () => { expect(() => createOutboundRoutingReader( diff --git a/apps/sim/lib/core/network/config.server.ts b/apps/sim/lib/core/network/config.server.ts index d9f0f57529a..c04c56ddf9d 100644 --- a/apps/sim/lib/core/network/config.server.ts +++ b/apps/sim/lib/core/network/config.server.ts @@ -8,7 +8,6 @@ import { OutboundRoutingError, parseOutboundJson, parseOutboundRoutingConfig, - selectOutboundRoute, } from '@/lib/core/network/routing' const MAX_STALE_MS = 300_000 @@ -27,7 +26,6 @@ const gatewaySchema = z.strictObject({ .regex(/^[a-zA-Z0-9.-]+$/) .optional(), credentialId: keySchema, - generation: keySchema, ...gatewayPublicMetadataSchema.partial().shape, }) const gatewaysSchema = z @@ -50,7 +48,6 @@ export interface OutboundGateway { readonly organizationId: string readonly url: string readonly servername: string - readonly generation: string readonly token: string readonly ca?: string readonly publicIps?: readonly string[] @@ -151,7 +148,6 @@ export function createOutboundRoutingReader( organizationId: entry.organizationId, url: url.href, servername, - generation: entry.generation, ...credential, ...(entry.publicIps ? { publicIps: Object.freeze([...new Set(entry.publicIps)]) } : {}), }) @@ -197,7 +193,7 @@ export function createOutboundRoutingReader( if (!Object.hasOwn(config.organizations, organizationId)) { throw new OutboundRoutingError('ROUTE_BLOCKED') } - const route = selectOutboundRoute(config, organizationId) + const route = config.organizations[organizationId] if (route.kind === 'direct') return { kind: 'direct' } if (route.kind === 'blocked') throw new OutboundRoutingError('ROUTE_BLOCKED') const gateway = gateways.get(route.gatewayId) diff --git a/apps/sim/lib/core/network/context.server.test.ts b/apps/sim/lib/core/network/context.server.test.ts index f8dff63f90a..6c6b15c9c86 100644 --- a/apps/sim/lib/core/network/context.server.test.ts +++ b/apps/sim/lib/core/network/context.server.test.ts @@ -13,11 +13,36 @@ vi.mock('@/lib/core/network/config.server', () => ({ })) import { + captureOutboundScope, resolveCurrentOutboundRoute, runWithOutboundOrganization, } from '@/lib/core/network/context.server' describe('outbound execution context', () => { + it('restores captured ownership inside a callback invoked by another organization', async () => { + const captured = runWithOutboundOrganization('org_a', captureOutboundScope) + await runWithOutboundOrganization('org_b', async () => { + expect(await captured(resolveCurrentOutboundRoute)).toEqual({ + kind: 'direct', + organizationId: 'org_a', + }) + expect(await resolveCurrentOutboundRoute()).toEqual({ + kind: 'direct', + organizationId: 'org_b', + }) + }) + }) + + it('does not borrow ambient ownership when a callback captured no scope', async () => { + const captured = captureOutboundScope() + await runWithOutboundOrganization('org_a', async () => { + expect(await captured(resolveCurrentOutboundRoute)).toEqual({ + kind: 'direct', + organizationId: undefined, + }) + }) + }) + it('isolates interleaved organizations and restores the parent after nested calls', async () => { let release: () => void = () => {} const gate = new Promise((done) => { diff --git a/apps/sim/lib/core/network/fixtures/gateway-runtime.fixture.ts b/apps/sim/lib/core/network/fixtures/gateway-runtime.fixture.ts index 7f27a8ef76c..1a9130f5db5 100644 --- a/apps/sim/lib/core/network/fixtures/gateway-runtime.fixture.ts +++ b/apps/sim/lib/core/network/fixtures/gateway-runtime.fixture.ts @@ -13,7 +13,6 @@ const gateway = { organizationId: 'org_test', url: `https://127.0.0.1:${proxyPort}`, servername: 'gateway.invalid', - generation: 'test', token: 'a'.repeat(48), ca: readFileSync(certificatePath, 'utf8'), } diff --git a/apps/sim/lib/core/network/gateway.server.test.ts b/apps/sim/lib/core/network/gateway.server.test.ts index bd746fabb59..05dd39e51b6 100644 --- a/apps/sim/lib/core/network/gateway.server.test.ts +++ b/apps/sim/lib/core/network/gateway.server.test.ts @@ -134,7 +134,6 @@ beforeAll(async () => { token, url: `https://127.0.0.1:${proxyPort}`, servername: 'gateway.invalid', - generation: '1', ca: cert, }) }) @@ -189,7 +188,7 @@ describe('organization gateways over real TLS CONNECT sockets', () => { expect(receivedHeaders['proxy-authorization']).toBeUndefined() expect(receivedHeaders.host).toBe('origin.invalid') }) - it('isolates concurrent requests to the same origin and rotates pools on credential generation change', async () => { + it('isolates concurrent requests to the same origin and rotates pools when its gateway assignment changes', async () => { const transport = createPinnedFetchWithDispatcher('1.1.1.1', options) const start = admissions.length try { @@ -206,7 +205,7 @@ describe('organization gateways over real TLS CONNECT sockets', () => { .map((item) => item.token) .sort() ).toEqual(['Bearer alpha', 'Bearer bravo']) - state.gateways.set('org_a', { ...state.gateways.get('org_a')!, generation: '2' }) + state.gateways.set('org_a', { ...state.gateways.get('org_a')!, id: 'gateway-next' }) await runWithOutboundOrganization('org_a', async () => (await transport.fetch(url())).text()) expect(admissions.length).toBe(start + 3) } finally { @@ -299,7 +298,6 @@ describe('organization gateways over real TLS CONNECT sockets', () => { OUTBOUND_ROUTING_CONFIG: JSON.stringify({ schemaVersion: 1, revision: 'test', - defaultRoute: { kind: 'blocked' }, organizations: { org_test: { kind: 'gateway', gatewayId: 'synthetic' } }, }), OUTBOUND_GATEWAYS: JSON.stringify({ @@ -308,7 +306,6 @@ describe('organization gateways over real TLS CONNECT sockets', () => { url: `https://127.0.0.1:${(proxy.address() as AddressInfo).port}`, servername: 'gateway.invalid', credentialId: 'synthetic', - generation: 'test', }, }), OUTBOUND_GATEWAY_CREDENTIALS: JSON.stringify({ diff --git a/apps/sim/lib/core/network/gateway.server.ts b/apps/sim/lib/core/network/gateway.server.ts index 0c32542ae16..190f4ce8730 100644 --- a/apps/sim/lib/core/network/gateway.server.ts +++ b/apps/sim/lib/core/network/gateway.server.ts @@ -90,7 +90,7 @@ export async function openGatewayTunnel( } } -/** A dispatcher belongs to one organization and gateway generation, never a process-wide default. */ +/** A dispatcher belongs to one organization and gateway, never a process-wide default. */ export function createGatewayDispatcher( gateway: OutboundGateway, options: { profile: EgressProfile; resolvedIP?: string; maxResponseSize?: number } diff --git a/apps/sim/lib/core/network/routing.test.ts b/apps/sim/lib/core/network/routing.test.ts index 0324764837e..29e578989b8 100644 --- a/apps/sim/lib/core/network/routing.test.ts +++ b/apps/sim/lib/core/network/routing.test.ts @@ -1,41 +1,24 @@ /** @vitest-environment node */ import { describe, expect, it } from 'vitest' -import { - parseOutboundJson, - parseOutboundRoutingConfig, - selectOutboundRoute, -} from '@/lib/core/network/routing' +import { parseOutboundJson, parseOutboundRoutingConfig } from '@/lib/core/network/routing' const document = { schemaVersion: 1, revision: 'revision-1', - defaultRoute: { kind: 'direct' }, organizations: { org_a: { kind: 'gateway', gatewayId: 'gateway_a' }, org_b: { kind: 'blocked' } }, } describe('outbound routing policy', () => { - it('selects exact organization bindings, including a deliberate stop', () => { - const policy = parseOutboundRoutingConfig(document) - expect(selectOutboundRoute(policy, 'org_a')).toEqual({ - kind: 'gateway', - gatewayId: 'gateway_a', - }) - expect(selectOutboundRoute(policy, 'org_c')).toEqual({ kind: 'direct' }) - expect(selectOutboundRoute(policy, null)).toEqual({ kind: 'direct' }) - expect(() => selectOutboundRoute(policy, 'org_b')).toThrow('ROUTE_BLOCKED') - }) - - it('cannot be mutated or inherit a binding from Object.prototype', () => { + it('cannot be mutated', () => { const policy = parseOutboundRoutingConfig(document) expect(Object.isFrozen(policy.organizations.org_a)).toBe(true) expect(Object.isFrozen(policy.organizations)).toBe(true) - expect(selectOutboundRoute(policy, 'constructor')).toEqual({ kind: 'direct' }) }) it.each([ { ...document, schemaVersion: 2 }, { ...document, organizations: { org_a: { kind: 'gateway' } } }, - { ...document, defaultRoute: { kind: 'gateway', gatewayId: 'a', fallback: true } }, + { ...document, defaultRoute: { kind: 'blocked' } }, { ...document, extra: true }, ])('rejects malformed policy without exposing its content', (input) => { expect(() => parseOutboundRoutingConfig(input)).toThrow('INVALID_CONFIGURATION') diff --git a/apps/sim/lib/core/network/routing.ts b/apps/sim/lib/core/network/routing.ts index 4b0d9275d46..175a99c7fea 100644 --- a/apps/sim/lib/core/network/routing.ts +++ b/apps/sim/lib/core/network/routing.ts @@ -14,10 +14,6 @@ const routeSchema = z.discriminatedUnion('kind', [ const configSchema = z.strictObject({ schemaVersion: z.literal(1), revision: identifier, - defaultRoute: z.discriminatedUnion('kind', [ - z.strictObject({ kind: z.literal('direct') }), - z.strictObject({ kind: z.literal('blocked') }), - ]), organizations: z .record(identifier, routeSchema) .refine((value) => Object.keys(value).length <= 10_000), @@ -27,7 +23,6 @@ export type OutboundRoute = Readonly> export interface OutboundRoutingConfig { readonly schemaVersion: 1 readonly revision: string - readonly defaultRoute: Exclude readonly organizations: Readonly> } @@ -64,23 +59,6 @@ export function parseOutboundRoutingConfig(value: unknown): OutboundRoutingConfi const parsed = configSchema.safeParse(value) if (!parsed.success) throw new OutboundRoutingError('INVALID_CONFIGURATION') for (const route of Object.values(parsed.data.organizations)) Object.freeze(route) - Object.freeze(parsed.data.defaultRoute) Object.freeze(parsed.data.organizations) return Object.freeze(parsed.data) } - -/** A null organization is a verified personal scope; undefined is never an implicit default. */ -export function selectOutboundRoute( - config: OutboundRoutingConfig, - organizationId: string | null -): OutboundRoute { - if (organizationId !== null && !identifier.safeParse(organizationId).success) { - throw new OutboundRoutingError('MISSING_SCOPE') - } - const route = - organizationId !== null && Object.hasOwn(config.organizations, organizationId) - ? config.organizations[organizationId] - : config.defaultRoute - if (route.kind === 'blocked') throw new OutboundRoutingError('ROUTE_BLOCKED') - return route -} diff --git a/apps/sim/lib/core/network/transport.server.test.ts b/apps/sim/lib/core/network/transport.server.test.ts index e659612d1d5..ecd58fec4de 100644 --- a/apps/sim/lib/core/network/transport.server.test.ts +++ b/apps/sim/lib/core/network/transport.server.test.ts @@ -11,14 +11,16 @@ vi.mock('@/lib/core/network/gateway.server', () => ({ createGatewayDispatcher: c import { createOutboundTransport } from '@/lib/core/network/transport.server' -const route = (organizationId = 'org_a', generation = 'v1'): ResolvedOutboundRoute => ({ +const route = ( + organizationId = 'org_a', + gatewayId = `gateway-${organizationId}` +): ResolvedOutboundRoute => ({ kind: 'gateway', gateway: { - id: `gateway-${organizationId}`, + id: gatewayId, organizationId, url: 'https://gateway.invalid', servername: 'gateway.invalid', - generation, token: 'synthetic', }, }) @@ -46,13 +48,13 @@ describe('shared outbound transport ownership', () => { } }) - it('isolates organizations, reuses the gateway generation, and disposes pools', async () => { + it('isolates organizations, reuses gateway pools, and disposes pools', async () => { const owner = createOutboundTransport({ profile: 'configuredEndpoint' }) const first = await owner.selectDispatcher() expect(await owner.selectDispatcher()).toBe(first) resolveRoute.mockResolvedValue(route('org_b')) expect(await owner.selectDispatcher()).not.toBe(first) - resolveRoute.mockResolvedValue(route('org_a', 'v1')) + resolveRoute.mockResolvedValue(route('org_a')) expect(await owner.selectDispatcher()).toBe(first) await owner.destroy() for (const result of createGateway.mock.results) @@ -84,7 +86,7 @@ describe('shared outbound transport ownership', () => { expect(createGateway).not.toHaveBeenCalled() }) - it('bounds credential rotation per organization without blocking other organizations', async () => { + it('bounds gateway changes per organization without blocking other organizations', async () => { const drain: Array<() => void> = [] createGateway.mockImplementation(() => ({ close: vi.fn(() => new Promise((done) => drain.push(done))), diff --git a/apps/sim/lib/core/network/transport.server.ts b/apps/sim/lib/core/network/transport.server.ts index 6f97b6707b3..b4a6c630e7b 100644 --- a/apps/sim/lib/core/network/transport.server.ts +++ b/apps/sim/lib/core/network/transport.server.ts @@ -24,7 +24,7 @@ interface OutboundTransportOwner { * and optional pinning are immutable for this owner; policy is resolved per operation. */ export function createOutboundTransport(options: OutboundTransportOptions): OutboundTransportOwner { - const pools = new Map() + const pools = new Map() const retired = new Map>() let closed = false const allPools = () => [ @@ -39,9 +39,8 @@ export function createOutboundTransport(options: OutboundTransportOptions): Outb if (route.kind === 'direct') return options.direct ?? null if (options.proxyUrl) throw new OutboundRoutingError('UNSUPPORTED_TRANSPORT') const owner = route.gateway.organizationId - const identity = JSON.stringify([route.gateway.id, route.gateway.generation]) const current = pools.get(owner) - if (current?.identity === identity) return current.agent + if (current?.gatewayId === route.gateway.id) return current.agent const draining = retired.get(owner) ?? new Set() if (draining.size >= 2) throw new OutboundRoutingError('GATEWAY_UNAVAILABLE') const agent = createGatewayDispatcher(route.gateway, options) @@ -56,7 +55,7 @@ export function createOutboundTransport(options: OutboundTransportOptions): Outb if (draining.size === 0) retired.delete(owner) }) } - pools.set(owner, { identity, agent }) + pools.set(owner, { gatewayId: route.gateway.id, agent }) return agent }, async close() { diff --git a/apps/sim/lib/core/security/input-validation.server.ts b/apps/sim/lib/core/security/input-validation.server.ts index 24197066dd5..9c638d35fc1 100644 --- a/apps/sim/lib/core/security/input-validation.server.ts +++ b/apps/sim/lib/core/security/input-validation.server.ts @@ -1423,16 +1423,15 @@ export async function secureFetchWithPinnedIP( method: (options.method || 'GET') as Dispatcher.HttpMethod, headers: sanitizedHeaders, body: options.body, - signal: AbortSignal.any([ - controller.signal, - AbortSignal.timeout(options.timeout || 300_000), - ]), + headersTimeout: requestOptions.timeout, + bodyTimeout: requestOptions.timeout, + signal: controller.signal, }) - .then(({ statusCode, headers, body }) => { + .then(({ statusCode, statusText, headers, body }) => { body.once('close', () => { void transport.destroy() }) - onResponse(Object.assign(body, { statusCode, headers })) + onResponse(Object.assign(body, { statusCode, statusMessage: statusText, headers })) }) .catch((error) => { void transport.destroy() diff --git a/apps/sim/lib/core/security/secure-fetch-response-cap.server.test.ts b/apps/sim/lib/core/security/secure-fetch-response-cap.server.test.ts index ef44693ddc3..c86966085f3 100644 --- a/apps/sim/lib/core/security/secure-fetch-response-cap.server.test.ts +++ b/apps/sim/lib/core/security/secure-fetch-response-cap.server.test.ts @@ -47,6 +47,44 @@ async function startServer(handler: http.RequestListener): Promise { } describe('secureFetchWithPinnedIP response cap', () => { + it.each(['direct', 'gateway'])( + 'preserves the status phrase and active streaming past the %s inactivity timeout', + async (mode) => { + const dispatcher = new Agent() + if (mode === 'gateway') { + vi.spyOn(networkTransport, 'createOutboundTransport').mockReturnValue({ + selectDispatcher: async () => dispatcher, + close: () => dispatcher.close(), + destroy: () => dispatcher.destroy(), + }) + } + const origin = await startServer((_req, res) => { + res.writeHead(200, 'Synthetic status phrase') + res.write('start') + let chunks = 0 + const timer = setInterval(() => { + res.write('.') + if (++chunks === 25) { + clearInterval(timer) + res.end() + } + }, 50) + res.once('close', () => clearInterval(timer)) + }) + + try { + const response = await secureFetchWithPinnedIP(origin, '127.0.0.1', { + profile: 'configuredEndpoint', + timeout: 1000, + }) + expect(response.statusText).toBe('Synthetic status phrase') + await expect(response.text()).resolves.toBe(`start${'.'.repeat(25)}`) + } finally { + await dispatcher.destroy() + } + } + ) + it.each([ { encoding: 'gzip', compress: gzipSync }, { encoding: 'deflate', compress: deflateSync }, diff --git a/apps/sim/lib/execution/isolated-vm.test.ts b/apps/sim/lib/execution/isolated-vm.test.ts index 27dc37b469d..0511fd9fda5 100644 --- a/apps/sim/lib/execution/isolated-vm.test.ts +++ b/apps/sim/lib/execution/isolated-vm.test.ts @@ -172,7 +172,10 @@ function createReadyFetchProxyProc(fetchMessage: { url: string; optionsJson?: st return proc } -const { mockSpawn, mockExecSync, mockEnv } = vi.hoisted(() => ({ +const { mockSpawn, mockExecSync, mockEnv, mockResolveOutboundRoute } = vi.hoisted(() => ({ + mockResolveOutboundRoute: vi.fn(async (_organizationId: string | null | undefined) => ({ + kind: 'direct', + })), mockSpawn: vi.fn(), mockExecSync: vi.fn(() => Buffer.from('v23.11.0')), mockEnv: { @@ -200,6 +203,10 @@ const mockSecureFetch = inputValidationMockFns.mockSecureFetchWithValidation const mockGetRedisClient = redisConfigMockFns.mockGetRedisClient vi.mock('@/lib/core/security/input-validation.server', () => inputValidationMock) +vi.mock('@/lib/core/network/config.server', () => ({ + isOutboundRoutingEnabled: () => true, + resolveOutboundRoute: mockResolveOutboundRoute, +})) vi.mock('@/lib/core/config/env', () => ({ env: mockEnv, })) @@ -831,6 +838,77 @@ describe('isolated-vm scheduler', () => { expect(completionOrder).toEqual(['a-1', 'a-2', 'a-3', 'b-1', 'b-2']) }) + it('restores each execution owner when a reused worker emits outside its scope', async () => { + const { executeInIsolatedVM, spawnMock } = await loadExecutionModule({ + spawns: [ + () => { + const proc = createReadyFetchProxyProc({ url: 'https://example.com' }) + const emit = proc.emit.bind(proc) + proc.emit = (event, ...args: unknown[]) => { + const message = args[0] as { type?: string } | undefined + return event === 'message' && message?.type === 'fetch' + ? context.runWithOutboundOrganization('wrong-ambient', () => emit(event, ...args)) + : emit(event, ...args) + } + return proc + }, + ], + secureFetchImpl: async () => { + await context.resolveCurrentOutboundRoute() + return new Response('ok') + }, + }) + const context = await import('@/lib/core/network/context.server') + + for (const organizationId of ['org_a', 'org_b']) { + const result = await context.runWithOutboundOrganization(organizationId, () => + executeInIsolatedVM({ + code: 'return "fetch"', + params: {}, + envVars: {}, + contextVariables: {}, + timeoutMs: 1000, + requestId: organizationId, + }) + ) + expect(result.error).toBeUndefined() + } + expect(spawnMock).toHaveBeenCalledOnce() + expect(mockResolveOutboundRoute.mock.calls).toEqual([['org_a'], ['org_b']]) + }) + + it.each([undefined, -1])( + 'rejects fetch IPC with missing or inactive execution ID %s', + async (executionId) => { + const { executeInIsolatedVM, secureFetchMock } = await loadExecutionModule({ + spawns: [ + () => { + const proc = createReadyFetchProxyProc({ url: 'https://example.com' }) + const emit = proc.emit.bind(proc) + proc.emit = (event, ...args: unknown[]) => { + const message = args[0] as { type?: string } | undefined + if (event === 'message' && message?.type === 'fetch') { + return emit(event, { ...message, executionId }) + } + return emit(event, ...args) + } + return proc + }, + ], + }) + const result = await executeInIsolatedVM({ + code: 'return "fetch"', + params: {}, + envVars: {}, + contextVariables: {}, + timeoutMs: 1000, + requestId: 'inactive-fetch', + }) + expect(JSON.parse(String(result.result))).toEqual({ error: 'Execution no longer active' }) + expect(secureFetchMock).not.toHaveBeenCalled() + } + ) + it('rejects oversized fetch options payloads before outbound call', async () => { const { executeInIsolatedVM, secureFetchMock } = await loadExecutionModule({ envOverrides: { From 39a623ae29293eda25722a3e0e434f3148915896 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sun, 13 Sep 2026 22:59:53 -0700 Subject: [PATCH 4/5] test(network): remove unused runtime fixture argument --- .../lib/core/network/fixtures/gateway-runtime.fixture.ts | 6 +++--- apps/sim/lib/core/network/gateway.server.test.ts | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/apps/sim/lib/core/network/fixtures/gateway-runtime.fixture.ts b/apps/sim/lib/core/network/fixtures/gateway-runtime.fixture.ts index 1a9130f5db5..7551e2e72ed 100644 --- a/apps/sim/lib/core/network/fixtures/gateway-runtime.fixture.ts +++ b/apps/sim/lib/core/network/fixtures/gateway-runtime.fixture.ts @@ -5,9 +5,9 @@ import { runWithOutboundOrganization } from '@/lib/core/network/context.server' import { createGatewayDispatcher } from '@/lib/core/network/gateway.server' import { secureFetchWithPinnedIP } from '@/lib/core/security/input-validation.server' -const [proxyPort, originPort, certificatePath] = process.argv.slice(2) -if (!proxyPort || !originPort || !certificatePath) - throw new Error('Local fixture ports and certificate are required') +const [proxyPort, certificatePath] = process.argv.slice(2) +if (!proxyPort || !certificatePath) + throw new Error('Local fixture proxy port and certificate are required') const gateway = { id: 'synthetic', organizationId: 'org_test', diff --git a/apps/sim/lib/core/network/gateway.server.test.ts b/apps/sim/lib/core/network/gateway.server.test.ts index 05dd39e51b6..3ec317df477 100644 --- a/apps/sim/lib/core/network/gateway.server.test.ts +++ b/apps/sim/lib/core/network/gateway.server.test.ts @@ -283,7 +283,6 @@ describe('organization gateways over real TLS CONNECT sockets', () => { 'run', fileURLToPath(new URL('./fixtures/gateway-runtime.fixture.ts', import.meta.url)), String((proxy.address() as AddressInfo).port), - String(securePort), certificatePath, ], { From ded3a4cafbceac7866226c4b84a1edec4493ea2f Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sun, 13 Sep 2026 23:28:42 -0700 Subject: [PATCH 5/5] chore(network): keep routing result types private --- .../lib/core/network/application/read-organization-network.ts | 2 +- apps/sim/lib/core/network/routing.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/sim/lib/core/network/application/read-organization-network.ts b/apps/sim/lib/core/network/application/read-organization-network.ts index a51e80fc328..0ebca832dce 100644 --- a/apps/sim/lib/core/network/application/read-organization-network.ts +++ b/apps/sim/lib/core/network/application/read-organization-network.ts @@ -6,7 +6,7 @@ import { OutboundRoutingError } from '@/lib/core/network/routing' import { OrchestrationError } from '@/lib/core/orchestration/types' import { authorizeOrganizationSettingsSection } from '@/lib/settings/application/organization-section-access' -export type OrganizationNetwork = +type OrganizationNetwork = | { mode: 'direct' | 'blocked' | 'unavailable' } | { mode: 'gateway'; publicIps: string[] } diff --git a/apps/sim/lib/core/network/routing.ts b/apps/sim/lib/core/network/routing.ts index 175a99c7fea..121c9d94f82 100644 --- a/apps/sim/lib/core/network/routing.ts +++ b/apps/sim/lib/core/network/routing.ts @@ -19,14 +19,14 @@ const configSchema = z.strictObject({ .refine((value) => Object.keys(value).length <= 10_000), }) -export type OutboundRoute = Readonly> +type OutboundRoute = Readonly> export interface OutboundRoutingConfig { readonly schemaVersion: 1 readonly revision: string readonly organizations: Readonly> } -export type OutboundRoutingErrorCode = +type OutboundRoutingErrorCode = | 'CONFIGURATION_UNAVAILABLE' | 'INVALID_CONFIGURATION' | 'MISSING_SCOPE'