diff --git a/apps/docs/content/docs/platform/self-hosting/integrations-oauth.mdx b/apps/docs/content/docs/platform/self-hosting/integrations-oauth.mdx index c57b18f4991..26062069c3f 100644 --- a/apps/docs/content/docs/platform/self-hosting/integrations-oauth.mdx +++ b/apps/docs/content/docs/platform/self-hosting/integrations-oauth.mdx @@ -142,14 +142,15 @@ Replace `` with your configured public origin, such as `htt | Expire user authorization tokens | Enabled | | Request user authorization (OAuth) during installation | Disabled | | Enable Device Flow | Disabled | -| Post installation → Setup URL | Empty | +| Post installation → Setup URL | `/api/knowledge/github/setup/callback` | +| Post installation → Redirect on update | Enabled | | Webhook → Active | Disabled | -Authorization starts from Sim so the callback can finish the pending connection. The connector polls GitHub's API and does not need a webhook. +Authorization starts from Sim so the callback can finish the pending connection. The **Setup URL** returns installation approval to that same setup attempt. Keep **Request user authorization (OAuth) during installation** disabled: Sim starts account authorization when it is needed, before installation. See GitHub's [Setup URL guide](https://docs.github.com/en/apps/creating-github-apps/registering-a-github-app/about-the-setup-url). -GitHub App registration with the Redirect URI, expiring tokens enabled, and installation authorization, Device Flow, and webhooks disabled +For an existing deployment, deploy the application with the setup callback before changing the App registration. Use a separate GitHub App for each deployment origin, such as staging and production, so installation approval returns to the instance that started it. Keep the existing user authorization callback above unchanged. -*Example registration. Replace `sim.example.com` with your Sim domain.* +The connector polls GitHub's API and does not need a webhook. diff --git a/apps/docs/content/docs/search/github.mdx b/apps/docs/content/docs/search/github.mdx index 1b0ead15d2a..516e03cf2a6 100644 --- a/apps/docs/content/docs/search/github.mdx +++ b/apps/docs/content/docs/search/github.mdx @@ -7,7 +7,7 @@ import { Callout } from 'fumadocs-ui/components/callout' import { Step, Steps } from 'fumadocs-ui/components/steps' import { Image } from '@/components/ui/image' -GitHub Search indexes text files from repositories on `github.com`. An organization admin can install the GitHub App once and use it to index selected repositories. Each person connects their own GitHub account once to search the repositories they can access. Installing the App does not connect teammates or give them the installer's permissions. +GitHub Search indexes repository text files and supported PDF and Office documents on `github.com`. An organization admin can install the GitHub App once and use it to index selected repositories. Each person connects their own GitHub account once to search the repositories they can access. Installing the App does not connect teammates or give them the installer's permissions. Admin setup uses your organization's **Settings → Sources** page. Teammates connect from **Integrations** in the main sidebar. Installation indexing is available for organization Search. For workspace Search, use **Search → Add source** with member accounts or a dedicated user account. @@ -17,8 +17,6 @@ The repository must contain at least one commit. Each person needs a verified Gi On Sim Cloud, connect through the Sim Search GitHub App in the setup flow below. If you self-host Sim, a deployment administrator must first [configure GitHub Search](/platform/self-hosting/integrations-oauth#github-search). -The App needs read-only access to repository **Contents**, **Metadata**, and account **Email addresses**. No write permissions are needed. These are GitHub App permissions, not a list of OAuth scopes to paste into Sim. - To connect an installation for central indexing, you must be a Sim organization admin and either own the GitHub personal account or be an owner of the GitHub organization where the App is installed. You must also be able to read the repository you add. ## Add a repository @@ -33,35 +31,30 @@ Open **Settings → Sources → Add source** and select **GitHub**. This opens * -### Choose how to index - -In **Sync documents with**, choose **Connect GitHub App** to index through an installation: - -1. Select **Connect your GitHub account** if prompted. Finish authorization in the new tab, then return and select **Refresh**. - -Connect GitHub App dialog requiring a personal GitHub connection to verify manageable installations +### Connect GitHub -2. Select **Install GitHub App**. Choose your GitHub account or organization and the repositories to include. If it is already installed, check its repository selection. -3. Return to Sim and select **Refresh**. Choose the installation and select **Use installation**. Only installations on your own account or organizations you own are available. +Select **Connect GitHub** and approve the requested access on GitHub. If you need to install the App, choose your account or organization and the repositories it can read. Sim automatically returns you to the repository picker when approval finishes. Your open form stays in place. -GitHub App installation chooser with Install GitHub App, Refresh, and Use installation controls with an example organization +Add GitHub repository with one Connect GitHub action -The installation is now selected under **Sync documents with**. Reuse it when adding another repository in the same Sim organization. +Sim reuses an eligible installation when one is already available. If you have more than one, choose the GitHub account to use. Only installations on your own account or organizations you own are eligible. -Alternatively, leave **Connected members** selected to use members' accounts for indexing. To use a dedicated indexing account, select it under **Sync documents with**, or choose **Connect GitHub account** to add one. Each method still requires teammates to connect their own accounts for Search access. +When adding more repositories, your connected GitHub account is already selected. To connect a different organization, open the **GitHub** picker and select **Connect another organization**. Installing on all repositories does not automatically add all of them to Search. ### Choose what to index -Enter **Repository** as `owner/repo`. For installation indexing, it must belong to the installation's account and be included in the repositories granted to the App. Add one connection per repository; installing on all repositories does not automatically add them to Search. +Search the **Repository** picker and select one repository. It lists repositories available to the selected GitHub installation. Use **Add repository** again to add another. + +Searchable repository picker for the connected GitHub account Open **More options** if you need a different branch, path or extension filters, or metadata tags. | Field | What to enter | |---|---| -| Repository | `owner/repo`. Use **Add repository** for another repository. | +| Repository | Select a repository available to the connected installation. | | Branch | Optional. Leave blank to follow the repository's default branch. | | Path Filter | Optional prefix such as `docs/`. | | File Extensions | Optional comma-separated list, such as `.md, .txt, .mdx`. | @@ -73,34 +66,57 @@ Open **More options** if you need a different branch, path or extension filters, ### Connect your account -Invite teammates through **Settings → Members → Invite** using their Sim email addresses. They accept, then open **Integrations** in the main sidebar, select **Connect** for GitHub, and authorize the App. If GitHub is already connected, they do not need to connect again for each repository. For private repositories, both the person's account and the App installation must have access. GitHub also permits App user tokens to read public repositories without an installation; see [GitHub's permission rules](https://docs.github.com/en/apps/creating-github-apps/registering-a-github-app/choosing-permissions-for-a-github-app). +Teammates open **Integrations** in the main sidebar, select **Connect** beside **GitHub**, and authorize their own GitHub account. Members do not choose repositories or an indexing method. + +Member Integrations page with one GitHub Connect action -An App installation or dedicated indexing account can start syncing after the source is saved. With **Connected members**, indexing begins after someone connects. Each teammate still connects before searching. An existing GitHub connection in the same Sim organization is reused across its GitHub sources. +Each person connects once per Sim organization. When an admin adds another repository, Sim reuses existing member connections and checks each person's access automatically. No additional repository invitation is needed. The member's GitHub row shows their connection status; **Reconnect** appears if authorization needs renewal, and **Disconnect** remains available for their own account. -Open **Settings → Sources**, open **GitHub**, then open the source to inspect **Documents**, edit **Settings**, or review **Sync history**. +Connected member GitHub account reused across repositories -Use **Settings → Sources → People**, filter by **GitHub**, then select **Request connections** to send provider-specific connection requests. These requests do not grant organization membership. For another repository, add another source; members can also select **Integrations → GitHub → More → Add repository**. +The installation can begin indexing as soon as the admin saves a repository. For private repositories, both the person's account and the App installation must have access. GitHub also permits App user tokens to read public repositories without an installation; see [GitHub's permission rules](https://docs.github.com/en/apps/creating-github-apps/registering-a-github-app/choosing-permissions-for-a-github-app). + +Admins can use **Settings → Sources → People**, filter by **GitHub**, then select **Request connections** to request teammates' initial connections. These requests do not grant organization membership. +## Manage repositories + +Open **Settings → Sources → GitHub** to see the repositories and their sync status. Select **Add repository** to add another, or open an existing repository to inspect **Documents**, edit **Settings**, or review **Sync history**. + +Admin GitHub page listing configured repositories and their sync status + +The document filter separates **Failed** processing from intentionally **Skipped** files. Skipped entries explain why they were omitted and do not offer a processing retry. They are reconsidered when the source file changes. Empty files, unsupported binaries, oversized files, and links that do not resolve to files within the repository are skipped. Supported PDFs and Office documents use the shared knowledge-base parser. + +Skipped GitHub files with reasons and no Retry action + +A genuine source download failure remains **Failed** and is retried by scheduled connector sync. A processing failure with downloaded content can be retried from the document view. + +**Change connection** in a repository's Settings repairs its connection while keeping the same repository. The replacement connection must have access to that repository. To index a different repository, use **Add repository**. + ## How access is enforced GitHub App installation access supplies file contents for indexing. Each reader's own connected GitHub account determines which repository's indexed content they can search. Sim organization admins follow the same rule as other readers. For installation-indexed sources, Sim checks the installation's current status and verifies repository content access with the reader's GitHub account before returning results or opening indexed content. If GitHub cannot confirm access, that repository's content is withheld. Removing a person's repository access, disconnecting their account, or removing the repository from the App's access prevents subsequent reads once GitHub reflects the change. File edits still appear after background indexing. +This is an installation plus personal authorization flow. GitHub Search does not impersonate everyone in an email domain. [Google Drive delegation and GitLab administrator indexing](/search#choose-the-right-connection-method) use different supported identity and permission models. + ## Troubleshooting | Problem | Next step | |---|---| -| GitHub is missing or deactivated | Ask your Sim organization admin to add GitHub through **Settings → Sources → Add source**, or open its existing Sources row and select **Activate**. | -| GitHub says it is unavailable in this deployment | Check the [GitHub App configuration](/platform/self-hosting/integrations-oauth#github-search) with your deployment administrator, or contact Sim support on Sim Cloud. Activating the integration cannot fix missing deployment credentials. | +| GitHub is unavailable in Search | Ask your Sim organization admin to enable GitHub under **Settings → Sources**. For self-hosted Sim, also check the [GitHub App configuration](/platform/self-hosting/integrations-oauth#github-search). | | Repository cannot be read | Confirm the App is installed on that repository and your GitHub account has access. For SAML organizations, establish your GitHub SSO session before reconnecting. | | A teammate cannot authorize the App | Check the GitHub organization's App policies and required approvals. Self-hosted deployments must allow the teammate's account in their App visibility settings. | -| No eligible installations found | Finish connecting your own GitHub account, install the configured App on your account or an organization you own, then select **Refresh**. An installation of a different App or one you only have repository access to cannot be selected. | -| Repository is not accepted for an installation | Check `owner/repo`, the installation's account and repository selection, and your own access. Update the source's Repository field after a rename. After a transfer, add a source using an installation for the new owner. | -| Identity verification fails | Verify the email used by your Sim account in GitHub's email settings, then reconnect. A public profile email alone is insufficient. | +| GitHub approval did not finish | Select **Connect GitHub** again and complete approval. If GitHub requires owner approval, ask an organization owner to install the App first, then retry. Requesting an installation does not complete it. | +| No approval window opens | Allow pop-ups for Sim in your browser and select **Connect GitHub** again. You can cancel an unfinished connection without losing your repository form. | +| GitHub does not return to Sim after installation | For self-hosted Sim, verify the App’s **Setup URL** and **Redirect on update** settings. See [GitHub Search configuration](/platform/self-hosting/integrations-oauth#github-search). | +| Repository is missing or GitHub rejects it | Confirm the repository is included in the selected installation and reopen the picker. Update the source's Repository field after a rename. To index a different repository or one transferred to another owner, use **Add repository** with the appropriate connection. | +| Your GitHub email does not match | Add and verify the email used by your Sim account in GitHub's email settings, then reconnect. A verified secondary or private email is supported; a public profile email alone is insufficient. | +| GitHub does not allow Sim to read your email addresses | Ask the GitHub App administrator to enable read access to user email addresses, then reconnect. This is a user permission, separate from repository permissions. | +| GitHub is temporarily unavailable during authorization | Retry connecting after GitHub recovers or its rate limit resets. | | Authorization fails after installation | Return to Sim and start **Connect** there. Do not enable authorization during installation. | | Account authorization did not complete | Start the connection again from Sim. If it repeats, contact your organization admin or Sim support. For self-hosted Sim, check the [App callback and credentials](/platform/self-hosting/integrations-oauth#github-search). | | GitHub asks for a provider configuration update | An organization admin selects **Settings → Sources → More → Update sign-in settings**, then affected users reconnect GitHub. | @@ -109,5 +125,5 @@ For installation-indexed sources, Sim checks the installation's current status a | Empty repository returns an error | Add an initial commit, then sync again. GitHub does not return a file tree for an uninitialized repository. | - GitHub Search covers repository text files up to 100 MB, including symbolic links to files within the same repository. Path and extension filters apply to the link's path. Broken or external links, binaries, and submodules are not indexed. Issues, pull requests, separate wikis, GitHub Enterprise Server, and `ghe.com` domains are not supported by this connector. Personal access tokens remain available for general knowledge-base connectors, with that knowledge base's access rules. + GitHub Search covers repository text files and supported PDF, Word, Excel, PowerPoint, and OpenDocument files up to 100 MiB, including symbolic links to files within the same repository. Path and extension filters apply to the link's path. Broken or external links, unsupported binaries, and submodules are not indexed. Issues, pull requests, separate wikis, GitHub Enterprise Server, and `ghe.com` domains are not supported by this connector. Personal access tokens remain available for general knowledge-base connectors, with that knowledge base's access rules. diff --git a/apps/docs/public/static/search/github-app-callback.jpg b/apps/docs/public/static/search/github-app-callback.jpg deleted file mode 100644 index 5ae4a20392b..00000000000 Binary files a/apps/docs/public/static/search/github-app-callback.jpg and /dev/null differ diff --git a/apps/docs/public/static/search/github-connect.png b/apps/docs/public/static/search/github-connect.png new file mode 100644 index 00000000000..587d03437b0 Binary files /dev/null and b/apps/docs/public/static/search/github-connect.png differ diff --git a/apps/docs/public/static/search/github-installation-connect.jpg b/apps/docs/public/static/search/github-installation-connect.jpg deleted file mode 100644 index d64bff70f83..00000000000 Binary files a/apps/docs/public/static/search/github-installation-connect.jpg and /dev/null differ diff --git a/apps/docs/public/static/search/github-installation-select.jpg b/apps/docs/public/static/search/github-installation-select.jpg deleted file mode 100644 index d41a84f2d03..00000000000 Binary files a/apps/docs/public/static/search/github-installation-select.jpg and /dev/null differ diff --git a/apps/docs/public/static/search/github-member-connect.png b/apps/docs/public/static/search/github-member-connect.png new file mode 100644 index 00000000000..db15591bf94 Binary files /dev/null and b/apps/docs/public/static/search/github-member-connect.png differ diff --git a/apps/docs/public/static/search/github-member-connected.png b/apps/docs/public/static/search/github-member-connected.png new file mode 100644 index 00000000000..d39f5c128a2 Binary files /dev/null and b/apps/docs/public/static/search/github-member-connected.png differ diff --git a/apps/docs/public/static/search/github-repositories.png b/apps/docs/public/static/search/github-repositories.png new file mode 100644 index 00000000000..6ae32abc07b Binary files /dev/null and b/apps/docs/public/static/search/github-repositories.png differ diff --git a/apps/docs/public/static/search/github-repository-picker.png b/apps/docs/public/static/search/github-repository-picker.png new file mode 100644 index 00000000000..43e1a77bf6e Binary files /dev/null and b/apps/docs/public/static/search/github-repository-picker.png differ diff --git a/apps/docs/public/static/search/github-skipped-documents.png b/apps/docs/public/static/search/github-skipped-documents.png new file mode 100644 index 00000000000..e237cbec26f Binary files /dev/null and b/apps/docs/public/static/search/github-skipped-documents.png differ diff --git a/apps/sim/app/api/credential-groups/oauth-callback.test.ts b/apps/sim/app/api/credential-groups/oauth-callback.test.ts new file mode 100644 index 00000000000..4a15fcadf1a --- /dev/null +++ b/apps/sim/app/api/credential-groups/oauth-callback.test.ts @@ -0,0 +1,191 @@ +/** @vitest-environment node */ +import { NextRequest } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { CredentialGroupOAuthError } from '@/lib/credential-groups/provider-adapter' +import { OAuthIdentityVerificationError } from '@/lib/oauth/identity-error' + +const mocks = vi.hoisted(() => ({ + authenticate: vi.fn(), + completeOAuth: vi.fn(), + consumeAttempt: vi.fn(), + logError: vi.fn(), + completeSetupOAuth: vi.fn(), +})) + +vi.mock('@sim/logger', () => ({ + createLogger: () => ({ error: mocks.logError }), +})) +vi.mock('@/lib/knowledge/application/github-setup', () => ({ + completeGitHubSetupReaderOAuth: { execute: mocks.completeSetupOAuth }, +})) +vi.mock('@/lib/api/server/routes', () => ({ + internalSessionAuth: { + authenticate: async () => ({ kind: 'session', userId: 'admin', sessionId: 'browser' }), + }, +})) +vi.mock('@/lib/core/utils/urls', () => ({ getBaseUrl: () => 'https://sim.test' })) +vi.mock('@/lib/credential-groups/application/enrollment-auth', () => ({ + credentialGroupOAuthAttemptPrincipal: mocks.authenticate, +})) +vi.mock('@/lib/credential-groups/application/public-enrollment', () => ({ + completePublicCredentialGroupOAuth: { execute: mocks.completeOAuth }, +})) +vi.mock('@/lib/credential-groups/oauth-state', () => ({ + consumeCredentialGroupOAuthAttempt: mocks.consumeAttempt, +})) + +import { handleCredentialGroupOAuthCallback } from '@/app/api/credential-groups/oauth-callback' + +const completionId = '550e8400-e29b-41d4-a716-446655440000' +const attempt = { + provider: 'github-repositories', + invitationToken: 'invitation-token', + optionId: 'option-1', + returnTo: 'search', +} as const + +function completeCallback() { + return handleCredentialGroupOAuthCallback({ + request: new NextRequest( + 'https://sim.test/api/auth/oauth2/callback/github-repositories?state=cg_state&code=code-1' + ), + provider: 'github-repositories', + query: { state: 'cg_state', code: 'code-1' }, + limited: null, + }) +} + +describe('GitHub managed OAuth failure presentation', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.authenticate.mockResolvedValue({ kind: 'credential_group_enrollment' }) + }) + + describe.each([false, true])('completion redirect: %s', (completionRedirect) => { + it.each([ + { + failure: new OAuthIdentityVerificationError('email_mismatch', 'emails'), + status: 'github_email_mismatch', + }, + { + failure: new OAuthIdentityVerificationError('email_access_denied', 'emails', 403), + status: 'github_email_access_denied', + }, + { + failure: new OAuthIdentityVerificationError('provider_unavailable', 'profile', 503), + status: 'provider_unavailable', + }, + { + failure: new OAuthIdentityVerificationError('rate_limited', 'emails', 403), + status: 'rate_limited', + }, + { + failure: new OAuthIdentityVerificationError('invalid_response', 'emails'), + status: 'provider_unavailable', + }, + { + failure: new OAuthIdentityVerificationError('provider_rejected', 'profile', 401), + status: 'failed', + }, + ])('presents $status and logs only safe diagnostics', async ({ failure, status }) => { + mocks.consumeAttempt.mockResolvedValue({ ...attempt, completionRedirect, completionId }) + mocks.completeOAuth.mockRejectedValueOnce( + new CredentialGroupOAuthError('Private details: member@example.com ghu_token', 502, failure) + ) + const response = await completeCallback() + const location = new URL(response.headers.get('location')!, 'https://sim.test') + expect(location.searchParams.get('oauth')).toBe(status) + if (completionRedirect) { + expect(location.pathname).toBe('/credential-groups/complete') + expect(location.searchParams.get('completionId')).toBe(completionId) + } else { + expect(location.pathname).toBe('/credential-groups/enroll/invitation-token') + expect(location.searchParams.get('optionId')).toBe(attempt.optionId) + expect(location.searchParams.get('returnTo')).toBe('search') + } + expect(mocks.logError).toHaveBeenCalledExactlyOnceWith('Managed OAuth authorization failed', { + provider: 'github-repositories', + failure: status, + errorClass: 'credential_group_oauth', + statusCode: 502, + identityReason: failure.reason, + identityStage: failure.stage, + providerStatus: failure.httpStatus, + }) + expect(response.headers.get('location')).not.toContain('member@example.com') + expect(response.headers.get('location')).not.toContain('ghu_token') + }) + }) + + it('does not infer an identity failure from an untyped provider exception', async () => { + mocks.consumeAttempt.mockResolvedValue(attempt) + mocks.completeOAuth.mockRejectedValueOnce(new Error('member@example.com ghu_token')) + const response = await completeCallback() + expect(response.headers.get('location')).toContain('oauth=failed') + expect(mocks.logError).toHaveBeenCalledExactlyOnceWith('Managed OAuth authorization failed', { + provider: 'github-repositories', + failure: 'failed', + errorClass: 'unexpected', + }) + }) + + it('retains an application error classification without its private details', async () => { + mocks.consumeAttempt.mockResolvedValue(attempt) + mocks.completeOAuth.mockRejectedValueOnce( + new OrchestrationError('forbidden', 'Private details: member@example.com') + ) + await completeCallback() + expect(mocks.logError).toHaveBeenCalledExactlyOnceWith('Managed OAuth authorization failed', { + provider: 'github-repositories', + failure: 'failed', + errorClass: 'application', + applicationCode: 'forbidden', + statusCode: 403, + }) + }) +}) + +describe('GitHub installation setup OAuth return target', () => { + beforeEach(() => { + vi.clearAllMocks() + }) + it('resumes only the server-owned setup after the guarded OAuth completion', async () => { + mocks.consumeAttempt.mockResolvedValue({ + ...attempt, + returnTo: 'github-installation', + organizationId: 'organization', + completionId, + completionRedirect: true, + }) + mocks.completeSetupOAuth.mockResolvedValue({ credentialId: 'reader' }) + const response = await completeCallback() + const url = new URL(response.headers.get('location')!, 'https://sim.test') + expect(url.pathname).toBe('/api/knowledge/github/setup/continue') + expect(url.searchParams.get('organizationId')).toBe('organization') + expect(url.searchParams.get('setupId')).toBe(completionId) + expect(response.headers.get('referrer-policy')).toBe('no-referrer') + expect(mocks.completeSetupOAuth).toHaveBeenCalled() + expect(mocks.completeOAuth).not.toHaveBeenCalled() + }) + it('returns classified OAuth failure to setup status without accepting a client redirect URL', async () => { + mocks.consumeAttempt.mockResolvedValue({ + ...attempt, + returnTo: 'github-installation', + organizationId: 'organization', + completionId, + completionRedirect: true, + }) + const response = await handleCredentialGroupOAuthCallback({ + request: new NextRequest('https://sim.test/api/auth/oauth2/callback/github-repositories'), + provider: 'github-repositories', + query: { state: 'cg_state', error: 'access_denied' }, + limited: null, + }) + const url = new URL(response.headers.get('location')!, 'https://sim.test') + expect(url.origin).toBe('https://sim.test') + expect(url.pathname).toBe('/api/knowledge/github/setup/continue') + expect(url.searchParams.get('oauth')).toBe('denied') + expect(url.searchParams.get('setupId')).toBe(completionId) + }) +}) diff --git a/apps/sim/app/api/credential-groups/oauth-callback.ts b/apps/sim/app/api/credential-groups/oauth-callback.ts index aa565c8be62..1883ecdcb21 100644 --- a/apps/sim/app/api/credential-groups/oauth-callback.ts +++ b/apps/sim/app/api/credential-groups/oauth-callback.ts @@ -1,7 +1,9 @@ import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' -import type { NextRequest, NextResponse } from 'next/server' +import { type NextRequest, NextResponse } from 'next/server' import type { CredentialGroupOAuthCallbackQuery } from '@/lib/api/contracts/credential-groups' +import { internalSessionAuth } from '@/lib/api/server/routes' +import { asOrchestrationError, statusForOrchestrationError } from '@/lib/core/orchestration/types' import { credentialGroupOAuthAttemptPrincipal } from '@/lib/credential-groups/application/enrollment-auth' import { completePublicCredentialGroupOAuth } from '@/lib/credential-groups/application/public-enrollment' import { CredentialGroupOAuthStateVersionError } from '@/lib/credential-groups/oauth-attempt-version' @@ -10,8 +12,11 @@ import { consumeCredentialGroupOAuthAttempt } from '@/lib/credential-groups/oaut import { CredentialGroupInvitationUnavailableError, CredentialGroupOAuthError, + CredentialGroupProviderConfigurationError, } from '@/lib/credential-groups/provider-adapter' import type { CredentialGroupProvider } from '@/lib/credential-groups/providers' +import { completeGitHubSetupReaderOAuth } from '@/lib/knowledge/application/github-setup' +import { githubSetupContinueUrl } from '@/lib/knowledge/github-setup-urls' import { createCredentialGroupCompletionRedirect, createCredentialGroupEnrollmentRedirect, @@ -52,10 +57,26 @@ export async function handleCredentialGroupOAuthCallback({ const focus: Record = attempt.returnTo ? { optionId: attempt.optionId, returnTo: attempt.returnTo } : {} + const setupRedirect = (oauth?: CredentialGroupOAuthFailure) => + new NextResponse(null, { + status: 303, + headers: { + Location: githubSetupContinueUrl( + { organizationId: attempt.organizationId!, setupId: attempt.completionId! }, + oauth + ), + 'Cache-Control': 'no-store', + 'Referrer-Policy': 'no-referrer', + }, + }) + const installationSetup = + attempt.returnTo === 'github-installation' && attempt.organizationId && attempt.completionId const failureRedirect = (oauth: CredentialGroupOAuthFailure) => - attempt.completionRedirect - ? createCredentialGroupCompletionRedirect(oauth, attempt.completionId) - : createCredentialGroupEnrollmentRedirect(attempt.invitationToken, { ...focus, oauth }) + installationSetup + ? setupRedirect(oauth) + : attempt.completionRedirect + ? createCredentialGroupCompletionRedirect(oauth, attempt.completionId) + : createCredentialGroupEnrollmentRedirect(attempt.invitationToken, { ...focus, oauth }) if (limited) { return failureRedirect('rate_limited') } @@ -67,6 +88,11 @@ export async function handleCredentialGroupOAuthCallback({ } try { + if (installationSetup) { + const principal = await internalSessionAuth.authenticate() + await completeGitHubSetupReaderOAuth.execute({ principal, input: { attempt, code }, request }) + return setupRedirect() + } const principal = await credentialGroupOAuthAttemptPrincipal(attempt) await completePublicCredentialGroupOAuth.execute({ principal, @@ -80,11 +106,9 @@ export async function handleCredentialGroupOAuthCallback({ connected: attempt.optionId, }) } catch (error) { - logger.error('Managed OAuth authorization failed', { - provider, - error: getErrorMessage(error), - }) - const status = + const identityFailure = + error instanceof CredentialGroupOAuthError ? error.identityFailure : undefined + let status: CredentialGroupOAuthFailure = error instanceof CredentialGroupInvitationUnavailableError ? 'unavailable' : error instanceof CredentialGroupOAuthError && error.statusCode === 403 @@ -94,6 +118,52 @@ export async function handleCredentialGroupOAuthCallback({ : error instanceof CredentialGroupOAuthError && error.statusCode === 409 ? 'configuration_changed' : 'failed' + if (identityFailure) { + switch (identityFailure.reason) { + case 'email_mismatch': + status = provider === 'github-repositories' ? 'github_email_mismatch' : 'account_mismatch' + break + case 'email_access_denied': + status = + provider === 'github-repositories' + ? 'github_email_access_denied' + : 'permissions_required' + break + case 'rate_limited': + status = 'rate_limited' + break + case 'provider_unavailable': + case 'invalid_response': + status = 'provider_unavailable' + break + } + } + const applicationError = asOrchestrationError(error) + logger.error('Managed OAuth authorization failed', { + provider, + failure: status, + errorClass: + error instanceof CredentialGroupInvitationUnavailableError + ? 'invitation_unavailable' + : error instanceof CredentialGroupOAuthError + ? 'credential_group_oauth' + : error instanceof CredentialGroupProviderConfigurationError + ? 'provider_configuration' + : applicationError + ? 'application' + : 'unexpected', + ...(error instanceof CredentialGroupOAuthError && { statusCode: error.statusCode }), + ...(error instanceof CredentialGroupProviderConfigurationError && { statusCode: 503 }), + ...(applicationError && { + applicationCode: applicationError.code, + statusCode: statusForOrchestrationError(applicationError.code), + }), + ...(identityFailure && { + identityReason: identityFailure.reason, + identityStage: identityFailure.stage, + providerStatus: identityFailure.httpStatus, + }), + }) return failureRedirect(status) } } diff --git a/apps/sim/app/api/knowledge/github/setup/callback/route.ts b/apps/sim/app/api/knowledge/github/setup/callback/route.ts new file mode 100644 index 00000000000..c951aea0316 --- /dev/null +++ b/apps/sim/app/api/knowledge/github/setup/callback/route.ts @@ -0,0 +1,51 @@ +import type { SessionPrincipal } from '@sim/auth/principal' +import { NextResponse } from 'next/server' +import { completeGitHubSearchSetupContract } from '@/lib/api/contracts/knowledge/github-setup' +import { parseRequest } from '@/lib/api/server' +import { + InternalUnauthenticatedError, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { completeGitHubSearchSetup } from '@/lib/knowledge/application/github-setup' +import { createCredentialGroupCompletionRedirect } from '@/app/api/credential-groups/enrollment-redirect' + +/** GitHub's Setup URL carries untrusted installation IDs; the use case verifies ownership. */ +export const GET = withRouteHandler(async (request) => { + let principal: SessionPrincipal + try { + principal = await internalSessionAuth.authenticate() + } catch (error) { + if (error instanceof InternalUnauthenticatedError) + return NextResponse.json( + { error: 'Unauthorized' }, + { status: 401, headers: { 'Cache-Control': 'no-store' } } + ) + throw error + } + const limited = await internalRateLimits + .user({ bucketName: 'github-search-setup' }) + .enforce(request, principal) + if (limited) return limited + const parsed = await parseRequest(completeGitHubSearchSetupContract, request, {}) + if (!parsed.success) return parsed.response + try { + const { state, installation_id, setup_action } = parsed.data.query + const result = await completeGitHubSearchSetup.execute({ + principal, + input: { state, installationId: installation_id, setupAction: setup_action }, + request, + }) + return new NextResponse(null, { + status: 303, + headers: { + Location: result.url, + 'Cache-Control': 'no-store', + 'Referrer-Policy': 'no-referrer', + }, + }) + } catch { + return createCredentialGroupCompletionRedirect('unavailable') + } +}) diff --git a/apps/sim/app/api/knowledge/github/setup/continue/route.ts b/apps/sim/app/api/knowledge/github/setup/continue/route.ts new file mode 100644 index 00000000000..cdd918855de --- /dev/null +++ b/apps/sim/app/api/knowledge/github/setup/continue/route.ts @@ -0,0 +1,50 @@ +import type { SessionPrincipal } from '@sim/auth/principal' +import { NextResponse } from 'next/server' +import { continueGitHubSearchSetupContract } from '@/lib/api/contracts/knowledge/github-setup' +import { parseRequest } from '@/lib/api/server' +import { + InternalUnauthenticatedError, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { withRouteHandler } from '@/lib/core/utils/with-route-handler' +import { continueGitHubSearchSetup } from '@/lib/knowledge/application/github-setup' +import { createCredentialGroupCompletionRedirect } from '@/app/api/credential-groups/enrollment-redirect' + +/** OAuth completion resumes the existing setup attempt through current session authorization. */ +export const GET = withRouteHandler(async (request) => { + let principal: SessionPrincipal + try { + principal = await internalSessionAuth.authenticate() + } catch (error) { + if (error instanceof InternalUnauthenticatedError) + return NextResponse.json( + { error: 'Unauthorized' }, + { status: 401, headers: { 'Cache-Control': 'no-store' } } + ) + throw error + } + const limited = await internalRateLimits + .user({ bucketName: 'github-search-setup' }) + .enforce(request, principal) + if (limited) return limited + const parsed = await parseRequest(continueGitHubSearchSetupContract, request, {}) + if (!parsed.success) return parsed.response + try { + const result = await continueGitHubSearchSetup.execute({ + principal, + input: parsed.data.query, + request, + }) + return new NextResponse(null, { + status: 303, + headers: { + Location: result.url, + 'Cache-Control': 'no-store', + 'Referrer-Policy': 'no-referrer', + }, + }) + } catch { + return createCredentialGroupCompletionRedirect('unavailable', parsed.data.query.setupId) + } +}) diff --git a/apps/sim/app/api/knowledge/github/setup/redirects.test.ts b/apps/sim/app/api/knowledge/github/setup/redirects.test.ts new file mode 100644 index 00000000000..522fe7d4ef7 --- /dev/null +++ b/apps/sim/app/api/knowledge/github/setup/redirects.test.ts @@ -0,0 +1,94 @@ +/** @vitest-environment node */ +import { NextRequest, NextResponse } from 'next/server' +import { beforeEach, describe, expect, it, vi } from 'vitest' + +const m = vi.hoisted(() => ({ + authenticate: vi.fn(), + parse: vi.fn(), + limit: vi.fn(), + complete: vi.fn(), + resume: vi.fn(), +})) +vi.mock('@/lib/api/server', () => ({ parseRequest: m.parse })) +vi.mock('@/lib/api/server/routes', () => ({ + internalSessionAuth: { authenticate: m.authenticate }, + internalRateLimits: { user: () => ({ enforce: m.limit }) }, + InternalUnauthenticatedError: class extends Error {}, +})) +vi.mock('@/lib/knowledge/application/github-setup', () => ({ + completeGitHubSearchSetup: { execute: m.complete }, + continueGitHubSearchSetup: { execute: m.resume }, +})) + +import { InternalUnauthenticatedError } from '@/lib/api/server/routes' +import { GET as callback } from '@/app/api/knowledge/github/setup/callback/route' +import { GET as resume } from '@/app/api/knowledge/github/setup/continue/route' + +const principal = { kind: 'session', userId: 'admin', sessionId: 'browser' } +const setupId = '550e8400-e29b-41d4-a716-446655440000' +const state = '660e8400-e29b-41d4-a716-446655440000' +const completeUrl = `https://sim.example/credential-groups/complete?completionId=${setupId}` + +beforeEach(() => { + vi.clearAllMocks() + m.authenticate.mockResolvedValue(principal) + m.limit.mockResolvedValue(null) + m.complete.mockResolvedValue({ url: completeUrl }) + m.resume.mockResolvedValue({ url: 'https://github.com/apps/test/installations/new?state=opaque' }) +}) + +describe.each([ + { name: 'callback', handler: callback }, + { name: 'continue', handler: resume }, +])('GitHub setup $name ingress', ({ name, handler }) => { + const request = () => new NextRequest(`https://sim.example/api/knowledge/github/setup/${name}`) + it('rejects unauthenticated callers before parsing untrusted query fields', async () => { + m.authenticate.mockRejectedValueOnce(new InternalUnauthenticatedError()) + const response = await handler(request()) + expect(response.status).toBe(401) + expect(response.headers.get('cache-control')).toBe('no-store') + expect(m.parse).not.toHaveBeenCalled() + expect(m.complete).not.toHaveBeenCalled() + expect(m.resume).not.toHaveBeenCalled() + }) + it('applies rate admission before parsing', async () => { + m.limit.mockResolvedValueOnce(NextResponse.json({ error: 'Rate limited' }, { status: 429 })) + expect((await handler(request())).status).toBe(429) + expect(m.parse).not.toHaveBeenCalled() + }) + it('does not enter protected work after a malformed query', async () => { + m.parse.mockResolvedValueOnce({ + success: false, + response: NextResponse.json({ error: 'Invalid query' }, { status: 400 }), + }) + expect((await handler(request())).status).toBe(400) + expect(m.complete).not.toHaveBeenCalled() + expect(m.resume).not.toHaveBeenCalled() + }) +}) + +it('passes untrusted callback values to the authorized use case and redirects only to its result', async () => { + m.parse.mockResolvedValueOnce({ + success: true, + data: { query: { state, installation_id: '42', setup_action: 'install' } }, + }) + const response = await callback( + new NextRequest('https://sim.example/api/knowledge/github/setup/callback') + ) + expect(m.complete).toHaveBeenCalledWith( + expect.objectContaining({ + principal, + input: { state, installationId: '42', setupAction: 'install' }, + }) + ) + expect(response.status).toBe(303) + expect(response.headers.get('location')).toBe(completeUrl) + expect(response.headers.get('referrer-policy')).toBe('no-referrer') +}) + +it('keeps continuation scope and OAuth failure classification in the application input', async () => { + const query = { organizationId: 'org', setupId, oauth: 'denied' } + m.parse.mockResolvedValueOnce({ success: true, data: { query } }) + await resume(new NextRequest('https://sim.example/api/knowledge/github/setup/continue')) + expect(m.resume).toHaveBeenCalledWith(expect.objectContaining({ principal, input: query })) +}) diff --git a/apps/sim/app/api/knowledge/github/setup/route.ts b/apps/sim/app/api/knowledge/github/setup/route.ts new file mode 100644 index 00000000000..fa1b0f64517 --- /dev/null +++ b/apps/sim/app/api/knowledge/github/setup/route.ts @@ -0,0 +1,52 @@ +import { + cancelGitHubSearchSetupContract, + readGitHubSearchSetupContract, + startGitHubSearchSetupContract, +} from '@/lib/api/contracts/knowledge/github-setup' +import { + defineInternalJsonRoute, + internalOrchestrationErrorPolicy, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { + cancelGitHubSearchSetup, + readGitHubSearchSetup, + startGitHubSearchSetup, +} from '@/lib/knowledge/application/github-setup' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' + +export const POST = defineInternalJsonRoute({ + contract: startGitHubSearchSetupContract, + auth: internalSessionAuth, + operation: knowledgeOperations.startGitHubSetup, + rateLimit: internalRateLimits.user({ bucketName: 'github-search-setup' }), + errorPolicy: internalOrchestrationErrorPolicy, + mapInput: ({ body }) => body, + useCase: startGitHubSearchSetup, + present: (result) => ({ success: true, ...result }), + staticResponseHeaders: { 'Cache-Control': 'private, no-store' }, +}) + +export const GET = defineInternalJsonRoute({ + contract: readGitHubSearchSetupContract, + auth: internalSessionAuth, + operation: knowledgeOperations.readGitHubSetup, + rateLimit: internalRateLimits.user({ bucketName: 'github-search-setup-status' }), + errorPolicy: internalOrchestrationErrorPolicy, + mapInput: ({ query }) => query, + useCase: readGitHubSearchSetup, + present: (data) => ({ success: true, data }), + staticResponseHeaders: { 'Cache-Control': 'private, no-store' }, +}) + +export const DELETE = defineInternalJsonRoute({ + contract: cancelGitHubSearchSetupContract, + auth: internalSessionAuth, + operation: knowledgeOperations.cancelGitHubSetup, + rateLimit: internalRateLimits.user({ bucketName: 'github-search-setup' }), + errorPolicy: internalOrchestrationErrorPolicy, + mapInput: ({ body }) => body, + useCase: cancelGitHubSearchSetup, + staticResponseHeaders: { 'Cache-Control': 'private, no-store' }, +}) diff --git a/apps/sim/app/api/knowledge/github/setup/selection/route.ts b/apps/sim/app/api/knowledge/github/setup/selection/route.ts new file mode 100644 index 00000000000..a03b293c502 --- /dev/null +++ b/apps/sim/app/api/knowledge/github/setup/selection/route.ts @@ -0,0 +1,21 @@ +import { selectGitHubSearchSetupContract } from '@/lib/api/contracts/knowledge/github-setup' +import { + defineInternalJsonRoute, + internalOrchestrationErrorPolicy, + internalRateLimits, + internalSessionAuth, +} from '@/lib/api/server/routes' +import { selectGitHubSearchSetup } from '@/lib/knowledge/application/github-setup' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' + +export const POST = defineInternalJsonRoute({ + contract: selectGitHubSearchSetupContract, + auth: internalSessionAuth, + operation: knowledgeOperations.selectGitHubSetup, + rateLimit: internalRateLimits.user({ bucketName: 'github-search-setup' }), + errorPolicy: internalOrchestrationErrorPolicy, + mapInput: ({ body }) => body, + useCase: selectGitHubSearchSetup, + present: (result) => ({ success: true, ...result }), + staticResponseHeaders: { 'Cache-Control': 'private, no-store' }, +}) diff --git a/apps/sim/app/api/knowledge/migrated-routes.test.ts b/apps/sim/app/api/knowledge/migrated-routes.test.ts index b53310dd55b..4111d75993a 100644 --- a/apps/sim/app/api/knowledge/migrated-routes.test.ts +++ b/apps/sim/app/api/knowledge/migrated-routes.test.ts @@ -316,7 +316,7 @@ describe('migrated internal Knowledge routes', () => { documents: [ expect.objectContaining({ id: 'document-1', uploadedAt: '2026-01-01T00:00:00.000Z' }), ], - counts: { active: 1, excluded: 0, failed: 0 }, + counts: { active: 1, excluded: 0, failed: 0, skipped: 0 }, hasMore: false, }, }) diff --git a/apps/sim/app/credential-groups/enroll/[token]/page.test.tsx b/apps/sim/app/credential-groups/enroll/[token]/page.test.tsx index 970dcd47e6b..2ef014a73a6 100644 --- a/apps/sim/app/credential-groups/enroll/[token]/page.test.tsx +++ b/apps/sim/app/credential-groups/enroll/[token]/page.test.tsx @@ -306,6 +306,20 @@ describe('focused Search enrollment', () => { expect(oauthLinks()[0]?.getAttribute('href')).toContain('/site-two?returnTo=search') }) + it.each([ + ['github_email_mismatch', 'add and verify the email address'], + ['github_email_access_denied', 'Email addresses: Read-only permission'], + ['provider_unavailable', 'Try connecting again in a few minutes'], + ])( + 'renders recovery guidance for %s without treating it as a connection', + async (oauth, message) => { + await render({ returnTo: 'search', optionId: 'site-two', oauth }) + expect(document.querySelector('[role="status"]')?.textContent).toContain(message) + expect(oauthLinks()[0]?.textContent).toBe('Connect') + expect(document.body.textContent).toContain('Not connected') + } + ) + it('does not resolve enrollment metadata or trust a return workspace after authentication fails', async () => { mocks.authenticate.mockResolvedValue(null) await render({ returnTo: 'search', optionId: 'site-two', workspaceId: 'other-workspace' }) diff --git a/apps/sim/app/credential-groups/enroll/[token]/page.tsx b/apps/sim/app/credential-groups/enroll/[token]/page.tsx index 63423f76409..c819e8c122a 100644 --- a/apps/sim/app/credential-groups/enroll/[token]/page.tsx +++ b/apps/sim/app/credential-groups/enroll/[token]/page.tsx @@ -11,6 +11,7 @@ import { authenticateCredentialGroupEnrollment } from '@/lib/credential-groups/a import { readPublicCredentialGroupEnrollment } from '@/lib/credential-groups/application/public-enrollment' import { CredentialGroupEnrollmentError } from '@/lib/credential-groups/enrollments' import { getManagedMcpConnectorIcon } from '@/lib/credential-groups/managed-mcp-connector-icons' +import { CREDENTIAL_GROUP_OAUTH_FAILURE_MESSAGES } from '@/lib/credential-groups/oauth-completion' import { CredentialGroupProviderConfigurationError } from '@/lib/credential-groups/provider-adapter' import { getCredentialGroupProviderService } from '@/lib/credential-groups/providers' import { enforcePublicCredentialGroupIpRateLimit } from '@/lib/credential-groups/rate-limit' @@ -107,11 +108,11 @@ function UnavailableSearchConnection({ } const OAUTH_MESSAGES = { + ...CREDENTIAL_GROUP_OAUTH_FAILURE_MESSAGES, denied: 'Authorization was canceled. Nothing was connected.', account_mismatch: 'Choose the account matching the email address on this invitation.', permissions_required: 'All requested permissions are required to connect this account.', configuration_changed: 'This credential option changed. Reload the page and try again.', - rate_limited: 'Too many authorization attempts. Wait a few minutes and try again.', unavailable: 'Account authorization is temporarily unavailable. Please try again.', failed: 'Account authorization did not complete. Please try again.', } as const diff --git a/apps/sim/app/knowledge/github/setup/page.tsx b/apps/sim/app/knowledge/github/setup/page.tsx new file mode 100644 index 00000000000..4cfcc4876f7 --- /dev/null +++ b/apps/sim/app/knowledge/github/setup/page.tsx @@ -0,0 +1,30 @@ +import type { Metadata } from 'next' +import { gitHubSearchSetupScopeSchema } from '@/lib/api/contracts/knowledge/github-setup' +import { AuthHeader, AuthShell } from '@/app/(auth)/components' +import { GitHubSetup } from '@/app/knowledge/github/setup/setup' + +export const metadata: Metadata = { + title: 'Connect GitHub', + robots: { index: false, follow: false }, +} + +export default async function GitHubSetupPage({ + searchParams, +}: { + searchParams: Promise<{ organizationId?: string | string[]; setupId?: string | string[] }> +}) { + const { organizationId, setupId } = await searchParams + const scope = gitHubSearchSetupScopeSchema.safeParse({ organizationId, setupId }) + return ( + + {scope.success ? ( + + ) : ( + + )} + + ) +} diff --git a/apps/sim/app/knowledge/github/setup/setup.test.tsx b/apps/sim/app/knowledge/github/setup/setup.test.tsx new file mode 100644 index 00000000000..30e6696c027 --- /dev/null +++ b/apps/sim/app/knowledge/github/setup/setup.test.tsx @@ -0,0 +1,118 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import type { GitHubSearchSetupStatus } from '@/lib/api/contracts/knowledge/github-setup' + +const mocks = vi.hoisted(() => ({ + status: { status: 'pending' } as GitHubSearchSetupStatus, + select: vi.fn(), + complete: vi.fn(), + pending: false, +})) +vi.mock('@/hooks/queries/github-search-setup', () => ({ + useGitHubSearchSetup: () => ({ data: mocks.status, error: null }), + useSelectGitHubSearchSetup: () => ({ mutateAsync: mocks.select, isPending: mocks.pending }), + isGitHubSetupTerminalError: () => false, +})) +vi.mock('@/app/(auth)/components', () => ({ + AuthHeader: ({ title, description }: { title: string; description?: string }) => ( + <> +

{title}

+

{description}

+ + ), +})) +vi.mock('@/app/credential-groups/complete/completion-handoff', () => ({ + CredentialGroupCompletionHandoff: (props: { completionId: string }) => { + mocks.complete(props) + return null + }, +})) + +import { GitHubSetup } from '@/app/knowledge/github/setup/setup' + +const scope = { organizationId: 'org-1', setupId: 'c5b10b45-ffde-42a9-b6ae-39355a2abfe2' } +describe('GitHub account choice', () => { + let root: Root + let container: HTMLDivElement + beforeEach(() => { + vi.clearAllMocks() + mocks.pending = false + mocks.status = { + status: 'choosing', + installations: [ + { installationId: '1', accountId: '11', accountLogin: 'acme', accountType: 'Organization' }, + { + installationId: '2', + accountId: '22', + accountLogin: 'research', + accountType: 'Organization', + }, + ], + } + mocks.select.mockRejectedValue(new Error('GitHub is temporarily unavailable. Try again.')) + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) + act(() => root.render()) + }) + afterEach(() => { + act(() => root.unmount()) + container.remove() + }) + + it('connects the selected account directly and retains choices on provider failure', async () => { + const combobox = container.querySelector('[role="combobox"]')! + await act(async () => combobox.click()) + const account = Array.from(document.querySelectorAll('[role="option"]')).find( + (option) => option.textContent?.includes('research') + )! + expect(account).toBeDefined() + await act(async () => account.dispatchEvent(new MouseEvent('mousedown', { bubbles: true }))) + expect(mocks.select).toHaveBeenCalledWith({ + ...scope, + action: { kind: 'select', installationId: '2' }, + }) + expect(container.querySelector('[role="alert"]')?.textContent).toContain( + 'temporarily unavailable' + ) + expect(container.textContent).not.toContain('Use installation') + expect(container.textContent).not.toContain('Refresh') + expect(combobox).toBeEnabled() + }) + + it('offers installing into another organization without selecting an existing account', async () => { + const button = Array.from(container.querySelectorAll('button')).find((item) => + item.textContent?.includes('Connect another organization') + )! + await act(async () => button.click()) + expect(mocks.select).toHaveBeenCalledWith({ ...scope, action: { kind: 'install' } }) + }) + + it('shows progress and disables choices while connecting', () => { + mocks.pending = true + act(() => root.render()) + expect(container.querySelector('[role="combobox"]')).toHaveAttribute('aria-disabled', 'true') + expect(container.textContent).toContain('Connecting GitHub…') + }) + + it('only closes the handoff after an authoritative completion', () => { + expect(mocks.complete).not.toHaveBeenCalled() + mocks.status = { status: 'completed', credential: { id: 'connection-1', displayName: 'Acme' } } + act(() => root.render()) + expect(mocks.complete).toHaveBeenCalledWith({ completionId: scope.setupId }) + expect(container.textContent).toContain('GitHub connected') + }) + + it('removes account choices after the attempt expires', () => { + mocks.status = { status: 'expired' } + act(() => root.render()) + expect(container.querySelector('[role="combobox"]')).toBeNull() + expect(container.textContent).toContain('expired') + expect(mocks.complete).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/app/knowledge/github/setup/setup.tsx b/apps/sim/app/knowledge/github/setup/setup.tsx new file mode 100644 index 00000000000..57d40e49160 --- /dev/null +++ b/apps/sim/app/knowledge/github/setup/setup.tsx @@ -0,0 +1,85 @@ +'use client' + +import { useState } from 'react' +import { Chip, ChipCombobox } from '@sim/emcn' +import { getErrorMessage } from '@sim/utils/errors' +import type { + GitHubSearchSetupScope, + SelectGitHubSearchSetupBody, +} from '@/lib/api/contracts/knowledge/github-setup' +import { resolveGitHubSetupUrl } from '@/lib/knowledge/github-setup-navigation' +import { AuthHeader } from '@/app/(auth)/components' +import { CredentialGroupCompletionHandoff } from '@/app/credential-groups/complete/completion-handoff' +import { + isGitHubSetupTerminalError, + useGitHubSearchSetup, + useSelectGitHubSearchSetup, +} from '@/hooks/queries/github-search-setup' + +interface GitHubSetupProps { + scope: GitHubSearchSetupScope +} + +export function GitHubSetup({ scope }: GitHubSetupProps) { + const query = useGitHubSearchSetup(scope) + const { mutateAsync: select, isPending } = useSelectGitHubSearchSetup() + const [error, setError] = useState(null) + const result = query.data + + const choose = async (action: SelectGitHubSearchSetupBody['action']) => { + setError(null) + try { + const response = await select({ ...scope, action }) + window.location.replace(resolveGitHubSetupUrl(response.url, window.location.origin)) + } catch (failure) { + setError(getErrorMessage(failure, 'Could not connect GitHub. Try again.')) + } + } + + const failure = + (isGitHubSetupTerminalError(query.error) ? query.error?.message : null) ?? + (result?.status === 'failed' + ? result.error + : result?.status === 'expired' + ? 'This connection attempt expired. Close this window and connect GitHub again from Sim.' + : null) + + if (failure) return + if (result?.status === 'completed') { + return ( + <> + + + + ) + } + if (result?.status !== 'choosing') { + return + } + + return ( + <> + +
+ ({ + value: installation.installationId, + label: installation.accountLogin, + }))} + placeholder={isPending ? 'Connecting GitHub…' : 'Select an account'} + disabled={isPending} + onChange={(installationId) => void choose({ kind: 'select', installationId })} + /> + void choose({ kind: 'install' })}> + Connect another organization + + {error && ( +

+ {error} +

+ )} +
+ + ) +} diff --git a/apps/sim/app/o/[organizationId]/integrations/github-member-integration.tsx b/apps/sim/app/o/[organizationId]/integrations/github-member-integration.tsx new file mode 100644 index 00000000000..c356ca87c17 --- /dev/null +++ b/apps/sim/app/o/[organizationId]/integrations/github-member-integration.tsx @@ -0,0 +1,101 @@ +'use client' + +import { Chip, toast } from '@sim/emcn' +import type { OrganizationAccountConnectionResponse } from '@/lib/api/contracts/organization-accounts' +import { DisconnectAccountMenu } from '@/app/o/[organizationId]/integrations/disconnect-account-menu' +import { IntegrationTile } from '@/app/workspace/[workspaceId]/integrations/components/integrations-showcase' +import { SettingsResourceRow } from '@/app/workspace/[workspaceId]/settings/components/settings-resource-row' +import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' +import { + useConnectOrganizationAccount, + type useOrganizationAccounts, + useReconnectPersonalOrganizationAccount, +} from '@/hooks/queries/organization-accounts' + +interface GitHubMemberIntegrationProps { + organizationId: string + inventory: ReturnType + canConnect: boolean +} + +/** A member authorizes GitHub once, independently of the repositories added by admins. */ +export function GitHubMemberIntegration({ + organizationId, + inventory, + canConnect, +}: GitHubMemberIntegrationProps) { + const connect = useConnectOrganizationAccount() + const reconnect = useReconnectPersonalOrganizationAccount() + const accounts = + inventory.data?.viewerAccounts?.filter( + (account) => account.providerId === 'github-repositories' + ) ?? [] + const account = accounts.find((entry) => entry.status === 'needs_reauth') ?? accounts[0] + const option = + inventory.data?.credentialGroup?.status === 'active' + ? inventory.data.credentialGroup.options.find( + (entry) => entry.provider === 'github-repositories' && entry.status === 'active' + ) + : undefined + const loading = inventory.isPending && !inventory.data + const failed = inventory.isError + const meta = CONNECTOR_META_REGISTRY.github + const navigate = ({ authorizationUrl, invitationLink }: OrganizationAccountConnectionResponse) => + window.location.assign(authorizationUrl ?? invitationLink) + const onError = (error: Error) => toast.error(error.message) + const description = account + ? `${accounts.map((entry) => entry.displayName).join(', ')} · ${account.status === 'needs_reauth' ? 'Reconnect required' : 'Connected'}` + : loading + ? 'Loading connection…' + : failed + ? 'Could not load connection' + : option + ? 'Connect once to search the repositories your admin adds' + : 'An admin needs to reconnect GitHub' + + return ( + : undefined} + title='GitHub' + description={description} + trailing={ +
+ + {failed ? ( + void inventory.refetch()}> + {inventory.isFetching ? 'Retrying…' : 'Retry'} + + ) : account?.status === 'needs_reauth' && option?.id === account.optionId ? ( + + reconnect.mutate(account.credentialId, { onSuccess: navigate, onError }) + } + > + Reconnect + + ) : !account && !loading && canConnect && option ? ( + + connect.mutate( + { organizationId, optionId: option.id }, + { onSuccess: navigate, onError } + ) + } + > + Connect + + ) : null} +
+ } + /> + ) +} diff --git a/apps/sim/app/o/[organizationId]/integrations/integrations.test.tsx b/apps/sim/app/o/[organizationId]/integrations/integrations.test.tsx index 745b951f6ca..fa22c0f7467 100644 --- a/apps/sim/app/o/[organizationId]/integrations/integrations.test.tsx +++ b/apps/sim/app/o/[organizationId]/integrations/integrations.test.tsx @@ -5,6 +5,7 @@ import { NuqsTestingAdapter } from 'nuqs/adapters/testing' import { createRoot, type Root } from 'react-dom/client' import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' import type { SearchSourceSummary } from '@/lib/api/contracts/knowledge/connectors' +import type { OrganizationAccountConnectionResponse } from '@/lib/api/contracts/organization-accounts' import type { SearchConnector } from '@/lib/sim-search/connectors' const mocks = vi.hoisted(() => ({ @@ -24,6 +25,22 @@ const mocks = vi.hoisted(() => ({ request: vi.fn(), updateUrl: vi.fn(), setupConnector: null as SearchConnector | null, + organizationAccounts: vi.fn(), + connectOrganizationAccount: vi.fn(), + reconnectOrganizationAccount: vi.fn(), + refetchAccounts: vi.fn(), +})) +vi.mock('@/hooks/queries/organization-accounts', () => ({ + organizationAccountsKeys: { detail: (id: string) => ['organization-accounts', 'detail', id] }, + useOrganizationAccounts: mocks.organizationAccounts, + useConnectOrganizationAccount: () => ({ + mutate: mocks.connectOrganizationAccount, + isPending: false, + }), + useReconnectPersonalOrganizationAccount: () => ({ + mutate: mocks.reconnectOrganizationAccount, + isPending: false, + }), })) vi.mock('@/app/o/[organizationId]/integrations/slack-search-actions', () => ({ SlackSearchActions: () => Return to Slack, @@ -159,6 +176,12 @@ beforeEach(() => { vi.spyOn(toast, 'error').mockReturnValue('toast') vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) mocks.setupConnector = null + mocks.organizationAccounts.mockReturnValue({ + data: { credentialGroup: null, viewerAccounts: [] }, + isPending: false, + isError: false, + refetch: mocks.refetchAccounts, + }) rows = [memberSource] queryOverrides = {} mocks.context.mockReturnValue({ @@ -247,6 +270,237 @@ function menuItem(label: string) { )! } +function expectConnectionRedirect( + onSuccess: (response: OrganizationAccountConnectionResponse) => void, + authorizationUrl?: string +) { + const invitationLink = 'https://sim.test/credential-groups/enroll/fixture-token' + const assign = vi.fn() + const browserWindow = window + vi.stubGlobal('window', { location: { assign } }) + try { + onSuccess({ invitationLink, ...(authorizationUrl ? { authorizationUrl } : {}) }) + expect(assign).toHaveBeenCalledExactlyOnceWith(authorizationUrl ?? invitationLink) + } finally { + vi.stubGlobal('window', browserWindow) + } +} + +describe('GitHub member account inventory', () => { + const githubAccount = { + credentialId: 'github-account', + providerId: 'github-repositories', + groupId: 'accounts-group', + optionId: 'github-option', + displayName: 'My GitHub', + status: 'active' as const, + } + const githubGroup = { + id: 'accounts-group', + status: 'active', + options: [{ id: 'github-option', provider: 'github-repositories', status: 'active' }], + } + + beforeEach(() => { + rows = ['repo-one', 'repo-two'].map((connectorId) => ({ + ...memberSource, + connectorId, + connectorType: 'github', + sourceDescription: connectorId, + })) + mocks.overview.mockReturnValue({ + data: { providers: [{ connectorType: 'github' }] }, + isPending: false, + }) + mocks.integrations.mockReturnValue({ + data: [{ connectorType: 'github', approved: true }], + isPending: false, + }) + mocks.availability.mockReturnValue({ + integrationAvailability: new Map(), + oauthServiceAvailability: new Map([['github-repositories', true]]), + isIntegrationAvailabilityReady: true, + }) + mocks.organizationAccounts.mockReturnValue({ + data: { credentialGroup: githubGroup, viewerAccounts: [] }, + isPending: false, + isError: false, + refetch: mocks.refetchAccounts, + }) + }) + + it.each([ + undefined, + 'https://sim.test/api/credential-groups/enroll/fixture-token/oauth/github-option?returnTo=search', + ])('connects once through the account operation with compatible redirect %s', async (url) => { + await render() + expect(buttons('Connect')).toHaveLength(1) + expect(container.textContent).toContain('Connect once') + await act(async () => buttons('Connect')[0].click()) + expect(mocks.connectOrganizationAccount).toHaveBeenCalledExactlyOnceWith( + { organizationId: scope.organizationId, optionId: 'github-option' }, + expect.any(Object) + ) + expectConnectionRedirect(mocks.connectOrganizationAccount.mock.calls[0][1].onSuccess, url) + expect(mocks.sources).not.toHaveBeenCalled() + expect(mocks.connect).not.toHaveBeenCalled() + expect(mocks.connectSearchSource).not.toHaveBeenCalled() + expect(document.querySelector('[role="dialog"]')).toBeNull() + expect(container.textContent).not.toContain('repo-one') + }) + + it('keeps one account row when an admin adds another repository', async () => { + mocks.organizationAccounts.mockReturnValue({ + data: { credentialGroup: githubGroup, viewerAccounts: [githubAccount] }, + isPending: false, + }) + await render() + rows.push({ + ...rows[0], + connectorId: 'future-repository', + sourceDescription: 'future-repository', + }) + await render() + expect(document.querySelectorAll('[aria-label="GitHub integration actions"]')).toHaveLength(1) + expect(mocks.accountMenu).toHaveBeenLastCalledWith( + expect.objectContaining({ accounts: [githubAccount] }) + ) + expect(buttons('Connect')).toHaveLength(0) + expect(buttons('Reconnect')).toHaveLength(0) + expect(mocks.sources).not.toHaveBeenCalled() + expect(container.textContent).not.toContain('future-repository') + }) + + it('keeps an owned account visible before any repository source exists', async () => { + mocks.overview.mockReturnValue({ data: { providers: [] }, isPending: false }) + mocks.integrations.mockReturnValue({ data: [], isPending: false }) + mocks.organizationAccounts.mockReturnValue({ + data: { credentialGroup: githubGroup, viewerAccounts: [githubAccount] }, + isPending: false, + }) + await render('', ) + expect(container.textContent).toContain('My GitHub · Connected') + expect(document.querySelectorAll('[aria-label="GitHub integration actions"]')).toHaveLength(1) + expect(mocks.sources).not.toHaveBeenCalled() + expect(buttons('Connect')).toHaveLength(0) + expect(container.textContent).not.toContain('No integrations are available') + }) + + it.each(['group', 'option'] as const)( + 'keeps Disconnect but hides Reconnect when the canonical %s is disabled', + async (disabled) => { + const expired = { ...githubAccount, status: 'needs_reauth' } + mocks.organizationAccounts.mockReturnValue({ + data: { + credentialGroup: { + ...githubGroup, + status: disabled === 'group' ? 'disabled' : 'active', + options: [ + { ...githubGroup.options[0], status: disabled === 'option' ? 'disabled' : 'active' }, + ], + }, + viewerAccounts: [expired], + }, + isPending: false, + }) + await render() + expect(buttons('Reconnect')).toHaveLength(0) + expect(mocks.accountMenu).toHaveBeenLastCalledWith( + expect.objectContaining({ accounts: [expired] }) + ) + await openMenu('GitHub') + expect(menuItem('Disconnect github-account')).toBeDefined() + } + ) + + it.each([ + undefined, + 'https://sim.test/api/credential-groups/enroll/fixture-token/oauth/github-option?returnTo=accounts', + ])('allows personal reauthorization while Search is disabled with redirect %s', async (url) => { + mocks.overview.mockReturnValue({ data: { providers: [] }, isPending: false }) + mocks.integrations.mockReturnValue({ + data: [{ connectorType: 'github', approved: false }], + isPending: false, + }) + mocks.context.mockReturnValue({ + organization: { id: scope.organizationId }, + searchAccess: { memberScoped: false, sourceMirrored: false }, + }) + mocks.organizationAccounts.mockReturnValue({ + data: { + credentialGroup: githubGroup, + viewerAccounts: [{ ...githubAccount, status: 'needs_reauth' }], + }, + isPending: false, + }) + await render() + expect(buttons('Reconnect')).toHaveLength(1) + await act(async () => buttons('Reconnect')[0].click()) + expect(mocks.reconnectOrganizationAccount).toHaveBeenCalledExactlyOnceWith( + 'github-account', + expect.any(Object) + ) + expectConnectionRedirect(mocks.reconnectOrganizationAccount.mock.calls[0][1].onSuccess, url) + expect(mocks.connect).not.toHaveBeenCalled() + }) + + it('does not reconnect an account through a different active option', async () => { + mocks.organizationAccounts.mockReturnValue({ + data: { + credentialGroup: githubGroup, + viewerAccounts: [{ ...githubAccount, optionId: 'other-option', status: 'needs_reauth' }], + }, + isPending: false, + }) + await render() + expect(buttons('Reconnect')).toHaveLength(0) + expect(document.querySelector('[aria-label="GitHub integration actions"]')).not.toBeNull() + }) + + it.each(['pending', 'error'] as const)( + 'does not fall through to repository setup when the inventory is %s', + async (state) => { + mocks.organizationAccounts.mockReturnValue({ + data: undefined, + isPending: state === 'pending', + isError: state === 'error', + error: state === 'error' ? new Error('Could not load accounts') : null, + isFetching: false, + refetch: mocks.refetchAccounts, + }) + await render() + expect(container.textContent).toContain('GitHub') + expect(mocks.sources).not.toHaveBeenCalled() + expect(buttons('Connect')).toHaveLength(0) + expect(mocks.connectSearchSource).not.toHaveBeenCalled() + if (state === 'error') { + await act(async () => buttons('Retry')[0].click()) + expect(mocks.refetchAccounts).toHaveBeenCalledOnce() + } + } + ) + + it('retains legacy account management only after a successful response omits the inventory', async () => { + mocks.organizationAccounts.mockReturnValue({ + data: { credentialGroup: githubGroup }, + isPending: false, + isError: false, + }) + rows = rows.map((source) => ({ + ...source, + viewerMembership: 'connected', + viewerAccounts: [githubAccount], + })) + await render() + expect(mocks.sources).toHaveBeenCalledWith(scope, { connectorType: 'github', enabled: true }) + expect(document.querySelectorAll('[aria-label="GitHub integration actions"]')).toHaveLength(1) + expect(mocks.accountMenu).toHaveBeenLastCalledWith( + expect.objectContaining({ accounts: [githubAccount] }) + ) + expect(buttons('Connect')).toHaveLength(0) + }) +}) + describe('grouped member integrations', () => { it('renders one provider row and loads bounded pages per configured provider', async () => { mocks.overview.mockReturnValue({ diff --git a/apps/sim/app/o/[organizationId]/integrations/member-integrations-list.tsx b/apps/sim/app/o/[organizationId]/integrations/member-integrations-list.tsx index 636a0685b2d..3770b53d4cc 100644 --- a/apps/sim/app/o/[organizationId]/integrations/member-integrations-list.tsx +++ b/apps/sim/app/o/[organizationId]/integrations/member-integrations-list.tsx @@ -10,6 +10,7 @@ import { SEARCH_SOURCE_TYPES, type SearchConnector, } from '@/lib/sim-search/connectors' +import { GitHubMemberIntegration } from '@/app/o/[organizationId]/integrations/github-member-integration' import { MemberIntegrationRow } from '@/app/o/[organizationId]/integrations/member-integration-row' import { useOrganizationContext } from '@/app/o/[organizationId]/providers/organization-provider' import { SourceSetupModal } from '@/app/workspace/[workspaceId]/home/components/search-sources/source-setup-modal' @@ -19,7 +20,10 @@ import { } from '@/app/workspace/[workspaceId]/settings/components/settings-empty-state' import { RESOURCE_LIST_STACK } from '@/app/workspace/[workspaceId]/settings/components/settings-resource-row' import { useSearchSourceOverview, useSearchSources } from '@/hooks/queries/kb/connectors' -import { organizationAccountsKeys } from '@/hooks/queries/organization-accounts' +import { + organizationAccountsKeys, + useOrganizationAccounts, +} from '@/hooks/queries/organization-accounts' import { usePersonalSearchIntegrations } from '@/hooks/queries/personal-search-integrations' import { useSearchIntegrations } from '@/hooks/queries/search-integrations' import { searchSourceKeys } from '@/hooks/queries/utils/search-source-keys' @@ -40,6 +44,15 @@ export function MemberIntegrationsList({ const scope: ResourceScope = { kind: 'organization', organizationId: organization.id } const overview = useSearchSourceOverview(scope) const integrations = useSearchIntegrations(organization.id) + const organizationAccounts = useOrganizationAccounts(organization.id) + const githubAccounts = + organizationAccounts.data?.viewerAccounts?.filter( + (account) => account.providerId === 'github-repositories' + ) ?? [] + const usesGitHubInventory = + organizationAccounts.isPending || + organizationAccounts.isError || + organizationAccounts.data?.viewerAccounts !== undefined const slackInventory = usePersonalSearchIntegrations({ organizationId: organization.id, connectorType: 'slack', @@ -71,7 +84,10 @@ export function MemberIntegrationsList({ isIntegrationAvailabilityReady: availability.isIntegrationAvailabilityReady, }).members ) - return configured.has(type) || canCreate + const hasGitHubAccount = + type === 'github' && + (githubAccounts.length > 0 || organizationAccounts.isPending || organizationAccounts.isError) + return configured.has(type) || canCreate || hasGitHubAccount ? [{ type, meta, connector, canCreate, configured: configured.has(type) }] : [] }) @@ -79,21 +95,41 @@ export function MemberIntegrationsList({ const query = search.trim().toLowerCase() const showSlackSetupError = slackInventory.isError && approved.has('slack') && 'slack'.includes(query) - const visible = providers.filter((provider) => provider.meta.name.toLowerCase().includes(query)) + const visible = providers.filter( + (provider) => + provider.meta.name.toLowerCase().includes(query) || + (provider.type === 'github' && + githubAccounts.some((account) => account.displayName.toLowerCase().includes(query))) + ) + const githubProvider = visible.find((provider) => provider.type === 'github') + const githubRow = + githubProvider && usesGitHubInventory ? ( + + ) : null return ( <>
{failedQuery ? ( - void failedQuery.refetch()} - variant='inline' - /> + <> + void failedQuery.refetch()} + variant='inline' + /> + {githubRow} + ) : overview.isPending || integrations.isPending ? ( - Loading integrations… + <> + Loading integrations… + {githubRow} + ) : ( <> {showSlackSetupError && ( @@ -116,15 +152,23 @@ export function MemberIntegrationsList({ )} {providers.map((provider) => ( ))} {showEmpty && diff --git a/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.test.tsx b/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.test.tsx index 594914fb52f..4d77fb01677 100644 --- a/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.test.tsx +++ b/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.test.tsx @@ -66,6 +66,7 @@ vi.mock('@/connectors/registry', () => ({ auth: { mode: 'oauth', provider: 'google-calendar', adminCredentialType: 'service_account' }, }, slack: { name: 'Slack', auth: { mode: 'oauth', provider: 'slack' } }, + github: { name: 'GitHub', auth: { mode: 'oauth', provider: 'github-repositories' } }, gitlab: { name: 'GitLab', auth: { mode: 'apiKey' } }, }, })) @@ -221,6 +222,29 @@ describe('organization provider management', () => { }) } + it.each(['members', 'admin'])( + 'shows GitHub repositories without an account method label for %s access', + async (accessMode) => { + mocks.overview.mockReturnValue({ + data: { providers: [{ ...provider, connectorType: 'github', sourceCount: 1 }] }, + }) + mocks.sources.mockReturnValue({ + data: [ + { ...source, connectorType: 'github', sourceDescription: 'acme/platform', accessMode }, + ], + isPending: false, + }) + await render('github') + expect(container.textContent).toContain('acme/platform') + expect(container.textContent).toContain('Last synced') + expect(container.textContent).not.toContain('Member accounts') + expect(container.textContent).not.toContain('Admin or service account') + expect( + container.querySelector('a[href="/o/org-one/settings/integrations/sources/source-one"]') + ).not.toBeNull() + } + ) + it.each(['gmail', 'google_calendar', 'google_drive'])( 'offers configuration setup directly for %s without an Accounts or Advanced tab', async (connectorType) => { diff --git a/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.tsx b/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.tsx index 53a5ca6b797..ecf0c0f62b5 100644 --- a/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.tsx +++ b/apps/sim/app/o/[organizationId]/settings/integrations/providers/[connectorType]/provider-detail.tsx @@ -239,12 +239,14 @@ export function OrganizationProviderDetail({ connectorType }: OrganizationProvid key={source.connectorId} title={source.sourceDescription || meta.name} description={[ - source.accessMode === 'members' - ? 'Member accounts' - : meta.auth.mode === 'oauth' && - meta.auth.adminCredentialType === 'service_account' - ? 'Service account' - : 'Admin or service account', + connectorType === 'github' + ? null + : source.accessMode === 'members' + ? 'Member accounts' + : meta.auth.mode === 'oauth' && + meta.auth.adminCredentialType === 'service_account' + ? 'Service account' + : 'Admin or service account', !approved ? 'Deactivated' : !source.enabled @@ -260,7 +262,9 @@ export function OrganizationProviderDetail({ connectorType }: OrganizationProvid : source.lastSyncAt ? `Last synced ${format(new Date(source.lastSyncAt), 'MMM d, h:mm a')}` : 'Waiting for the first sync', - ].join(' · ')} + ] + .filter(Boolean) + .join(' · ')} href={organizationRoutes(organization.id).searchSource(source.connectorId)} clickLabel={`Open ${source.sourceDescription || meta.name}`} navigable diff --git a/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/search-params.ts b/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/search-params.ts index b4e2134e517..b4182204393 100644 --- a/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/search-params.ts +++ b/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/search-params.ts @@ -1,4 +1,5 @@ import { parseAsStringLiteral } from 'nuqs/server' +import { connectorDocumentFilterSchema } from '@/lib/api/contracts/knowledge/connectors' export const sourceViewParam = { key: 'view', @@ -7,7 +8,7 @@ export const sourceViewParam = { export const sourceDocumentFilterParam = { key: 'document-filter', - parser: parseAsStringLiteral(['active', 'excluded', 'failed']).withDefault('active'), + parser: parseAsStringLiteral(connectorDocumentFilterSchema.options).withDefault('active'), } as const export type SourceView = NonNullable> diff --git a/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/source-detail.test.tsx b/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/source-detail.test.tsx index e578d5900c8..0dbbe4341de 100644 --- a/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/source-detail.test.tsx +++ b/apps/sim/app/o/[organizationId]/settings/integrations/sources/[connectorId]/source-detail.test.tsx @@ -205,15 +205,20 @@ describe('organization source detail navigation', () => { ) }) - it('restores document search and status from the shared URL', async () => { - await render('?search=notes&document-filter=excluded') - expect(mocks.documents).toHaveBeenLastCalledWith( - expect.objectContaining({ search: 'notes', filter: 'excluded' }) - ) - expect(mocks.documents).toHaveBeenLastCalledWith( - expect.objectContaining({ searchControl: { value: 'notes', onChange: expect.any(Function) } }) - ) - }) + it.each(['excluded', 'failed', 'skipped'])( + 'restores document search and %s status from the shared URL', + async (filter) => { + await render(`?search=notes&document-filter=${filter}`) + expect(mocks.documents).toHaveBeenLastCalledWith( + expect.objectContaining({ search: 'notes', filter }) + ) + expect(mocks.documents).toHaveBeenLastCalledWith( + expect.objectContaining({ + searchControl: { value: 'notes', onChange: expect.any(Function) }, + }) + ) + } + ) it.each(['', '?view=history'])( 'shows a concise incomplete-update notice without provider details at %s', diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx index 1de2b420b96..c8e621a49ec 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/[documentId]/document.tsx @@ -20,6 +20,7 @@ import { truncate } from '@sim/utils/string' import { useParams, useRouter } from 'next/navigation' import { useQueryStates } from 'nuqs' import { EmptyState } from '@/components/empty-state/empty-state' +import { getDocumentIndexingStatus } from '@/lib/knowledge/documents/types' import type { ChunkData } from '@/lib/knowledge/types' import { formatTokenCount } from '@/lib/tokenization' import type { @@ -989,7 +990,8 @@ export function Document({ ) const hasDocumentData = documentData !== null - const processingStatus = documentData?.processingStatus + const processingStatus = documentData ? getDocumentIndexingStatus(documentData) : undefined + const processingError = documentData?.processingError const chunkRows: ResourceRow[] = useMemo(() => { /** @@ -1012,6 +1014,8 @@ export function Document({ {processingStatus === 'pending' && 'Document processing pending...'} {processingStatus === 'processing' && 'Document processing in progress...'} {processingStatus === 'failed' && 'Document processing failed'} + {processingStatus === 'skipped' && + (processingError ? `Skipped · ${processingError}` : 'Document skipped')} {!processingStatus && 'Document not ready'}
@@ -1056,7 +1060,7 @@ export function Document({ }, } }) - }, [isCompleted, hasDocumentData, processingStatus, displayChunks, searchQuery]) + }, [isCompleted, hasDocumentData, processingStatus, processingError, displayChunks, searchQuery]) const saveLabel = saveStatus === 'saving' diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx index 123316bd61c..61693dfadd0 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/base.tsx @@ -47,7 +47,11 @@ import { getFieldTypeForSlot, KNOWLEDGE_DOCUMENT_PROCESSING_STALE_THRESHOLD_MS, } from '@/lib/knowledge/constants' -import type { DocumentSortField, SortOrder } from '@/lib/knowledge/documents/types' +import { + type DocumentSortField, + getDocumentIndexingStatus, + type SortOrder, +} from '@/lib/knowledge/documents/types' import { type FilterFieldType, getOperatorsForFieldType } from '@/lib/knowledge/filters/types' import type { DocumentData } from '@/lib/knowledge/types' import { captureEvent } from '@/lib/posthog/client' @@ -175,7 +179,7 @@ const AnimatedLoader = ({ className }: { className?: string }) => ( ) const getStatusBadge = (doc: DocumentData) => { - switch (doc.processingStatus) { + switch (getDocumentIndexingStatus(doc)) { case 'pending': return ( @@ -188,6 +192,12 @@ const getStatusBadge = (doc: DocumentData) => { Processing ) + case 'skipped': + return ( + + Skipped + + ) case 'failed': return doc.processingError ? ( @@ -1558,7 +1568,8 @@ export function KnowledgeBase({ : undefined } onRetry={ - contextMenuDocument?.processingStatus === 'failed' && + contextMenuDocument && + getDocumentIndexingStatus(contextMenuDocument) === 'failed' && selectedDocumentCount === 1 && userPermissions.canEdit ? () => handleRetryDocument(contextMenuDocument.id) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.test.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.test.tsx index 2f8d57c4367..4c848f9bc01 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.test.tsx @@ -20,12 +20,13 @@ const mocks = vi.hoisted(() => ({ selectionLabels: undefined as SourceSelectionLabels | undefined, canonicalModes: {} as Record, isFieldPopulated: vi.fn(() => true), - credentials: [] as Pick[], + credentials: [] as (Pick & + Partial>)[], credentialsState: 'ready' as 'ready' | 'loading' | 'error', refetchCredentials: vi.fn(), oauthModal: vi.fn(), serviceAccountModal: vi.fn(), - githubInstallationModal: vi.fn(), + githubSetup: vi.fn(), serviceAccountTarget: null as ServiceAccountConnectTarget | null, memberAccess: true, mirroredAccess: true, @@ -181,24 +182,28 @@ vi.mock('@/app/workspace/[workspaceId]/knowledge/[id]/components/connector-confi return null }, })) -vi.mock('@/app/workspace/[workspaceId]/search/components/github-installation-modal', () => ({ - GitHubInstallationModal: (props: { - organizationId: string +vi.mock('@/hooks/use-github-installation-setup', () => ({ + useGitHubInstallationSetup: (props: { + organizationId?: string onConnected: (id: string) => void }) => { - mocks.githubInstallationModal(props) - return ( - - ) + mocks.githubSetup(props) + return { + pending: false, + error: null, + cancel: vi.fn(), + connect: () => { + mocks.credentials = [ + { + id: 'github-app-credential', + name: 'GitHub App: acme', + type: 'service_account', + provider: 'github-app-installation', + }, + ] + props.onConnected('github-app-credential') + }, + } }, })) vi.mock('@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields', () => ({ @@ -477,18 +482,22 @@ describe('Search methods requiring member identity', () => { initialAccessMode: 'members', scope: { kind: 'organization', organizationId: 'org-1' }, }) - expect(document.body.textContent).toContain('Sync documents with') - await act(async () => combobox('Connected members').click()) - const option = Array.from(document.querySelectorAll('[role="option"]')).find( - (node) => node.textContent?.trim() === 'Connect GitHub App' - ) - if (!option) throw new Error('Missing GitHub App option') - await act(async () => option.dispatchEvent(new MouseEvent('mousedown', { bubbles: true }))) - expect(mocks.githubInstallationModal).toHaveBeenCalledWith( + expect(document.body.textContent).not.toContain('Sync documents with') + expect(document.body.textContent).not.toContain('Connected members') + expect(button('Add repository')).toBeDisabled() + await act(async () => button('Connect GitHub').click()) + expect(mocks.githubSetup).toHaveBeenCalledWith( expect.objectContaining({ organizationId: 'org-1' }) ) - await act(async () => button('Use GitHub installation').click()) - expect(combobox('GitHub App: acme')).toBeDefined() + expect(document.body.textContent).not.toContain('Use installation') + expect(document.body.textContent).not.toContain('Refresh') + expect(document.body.textContent).toContain('GitHub App: acme') + expect( + configFieldsProps().connectorConfig.configFields.find((field) => field.id === 'repository') + ).toMatchObject({ + type: 'selector', + selectorKey: 'github.installationRepositories', + }) await act(async () => button('Add repository').click()) expect(mocks.create).toHaveBeenCalledWith( expect.objectContaining({ @@ -994,15 +1003,16 @@ describe('Account connection dropdown', () => { expect(mocks.oauthModal).not.toHaveBeenCalled() }) - it('offers GitHub indexing accounts directly without requiring a browsing credential', async () => { + it('keeps member and dedicated GitHub indexing choices in workspace Search', async () => { mocks.credentials = [] await render({ initialConnectorType: 'github', lockedAccessMode: 'members', - scope: { kind: 'organization', organizationId: 'org-1' }, + scope: { kind: 'workspace', workspaceId: 'workspace-1' }, setupDraftKey: 'github-members', }) expect(document.body.textContent).not.toContain('Account for browsing') + await act(async () => button('More options').click()) expect(document.body.textContent).toContain('Sync documents with') await act(async () => combobox('Connected members').click()) const option = Array.from(document.querySelectorAll('[role="option"]')).find( @@ -1015,7 +1025,7 @@ describe('Account connection dropdown', () => { expect(mocks.oauthModal).toHaveBeenLastCalledWith( expect.objectContaining({ open: true, - organizationId: 'org-1', + workspaceId: 'workspace-1', connectorType: 'github', sourceAccess: 'members', }) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx index 054740456b1..92da9801726 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/add-connector-modal/add-connector-modal.tsx @@ -27,6 +27,7 @@ import { getServiceAccountProviderForProviderId, type OAuthProvider, } from '@/lib/oauth' +import { GITHUB_INSTALLATION_PROVIDER_ID } from '@/lib/oauth/github-installation-types' import { getSearchConnectionLabels } from '@/lib/sim-search/connection-labels' import { getConnectorAccessAvailability } from '@/lib/sim-search/connectors' import { SIM_SEARCH_SYNC_INTERVAL_MINUTES } from '@/lib/sim-search/constants' @@ -53,7 +54,6 @@ import { } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/consts' import { useConnectorConfigFields } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields' import { useConnectorScope } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-scope' -import { GitHubInstallationModal } from '@/app/workspace/[workspaceId]/search/components/github-installation-modal' import { SettingsEmptyState, SettingsQueryErrorState, @@ -61,6 +61,7 @@ import { import { SettingsResourceRow } from '@/app/workspace/[workspaceId]/settings/components/settings-resource-row' import { withBrandIcon } from '@/blocks/brand-icon' import { getConnectorApiKeyConfig, isConnectorCredentialTypeAllowed } from '@/connectors/auth' +import { GitHubInstallationConnectionField } from '@/connectors/github/installation-connection-field' import { GitLabPermissionTabs, GitLabPermissionUploads, @@ -72,6 +73,7 @@ import { useCreateConnector } from '@/hooks/queries/kb/connectors' import { useOAuthCredentials } from '@/hooks/queries/oauth/oauth-credentials' import { useSourceAccounts } from '@/hooks/queries/source-accounts' import { useCredentialRefreshTriggers } from '@/hooks/use-credential-refresh-triggers' +import { useGitHubInstallationSetup } from '@/hooks/use-github-installation-setup' import { useOAuthReturnForKBConnectors } from '@/hooks/use-oauth-return' import { usePermissionConfig } from '@/hooks/use-permission-config' import { useConnectorSetupStore } from '@/stores/connector-setup/store' @@ -140,13 +142,15 @@ export function AddConnectorModal({ const [access, setAccess] = useState(() => ({ accessMode: initialConnectorAccessMode( initialType ? CONNECTOR_META_REGISTRY[initialType] : undefined, - lockedAccessMode ?? - (lockConnectorType ? initialAccessMode : draft?.accessMode) ?? - (isSearchIndex && initialAccessMode === 'workspace' - ? initialType && CONNECTOR_META_REGISTRY[initialType]?.auth.mode === 'apiKey' - ? 'admin' - : 'members' - : initialAccessMode) + canAdmin && isSearchIndex && initialType === 'github' && scope.kind === 'organization' + ? 'members' + : (lockedAccessMode ?? + (lockConnectorType ? initialAccessMode : draft?.accessMode) ?? + (isSearchIndex && initialAccessMode === 'workspace' + ? initialType && CONNECTOR_META_REGISTRY[initialType]?.auth.mode === 'apiKey' + ? 'admin' + : 'members' + : initialAccessMode)) ), })) const [disabledTagIds, setDisabledTagIds] = useState>( @@ -158,7 +162,6 @@ export function AddConnectorModal({ const [serviceAccountField, setServiceAccountField] = useState<'browsing' | 'content' | null>( null ) - const [showGitHubInstallationModal, setShowGitHubInstallationModal] = useState(false) const gitlabPermissions = useGitLabPermissionForm() const [apiKeyValue, setApiKeyValue] = useState('') @@ -175,6 +178,8 @@ export function AddConnectorModal({ const { mutate: createConnector, isPending: isCreating } = useCreateConnector() const connectorConfig = selectedType ? CONNECTOR_META_REGISTRY[selectedType] : null + const canSetUpGitHubInstallation = + canAdmin && isSearchIndex && selectedType === 'github' && scope.kind === 'organization' const docsUrl = isSearchIndex ? connectorConfig?.searchDocsUrl : undefined const setupGuideActions = docsUrl ? [ @@ -292,6 +297,17 @@ export function AddConnectorModal({ : credentials.length === 1 ? credentials[0].id : null + const installations = credentials.filter( + (credential) => credential.provider === GITHUB_INSTALLATION_PROVIDER_ID + ) + const installationCredentialId = contentCredentialId + ? (installations.find((credential) => credential.id === contentCredentialId)?.id ?? null) + : installations.length === 1 + ? installations[0].id + : null + const effectiveContentCredentialId = canSetUpGitHubInstallation + ? installationCredentialId + : contentCredentialId const { sourceConfig, @@ -315,10 +331,21 @@ export function AddConnectorModal({ initialSelectionLabels: draft?.selectionLabels, }) + const githubSetup = useGitHubInstallationSetup({ + organizationId: + canSetUpGitHubInstallation && scope.kind === 'organization' + ? scope.organizationId + : undefined, + onConnected: (credentialId) => { + if (credentialId !== installationCredentialId) handleFieldChange('repository', '') + setContentCredentialId(credentialId) + }, + }) + const indexingCredentialId = isApiKeyMode ? null : isMembersMode - ? contentCredentialId + ? effectiveContentCredentialId : effectiveCredentialId const indexingCredential = credentials.find( (credential) => credential.id === indexingCredentialId @@ -330,10 +357,11 @@ export function AddConnectorModal({ indexingCredential?.type === 'service_account') const showCredentialPicker = - !isMembersMode || - connectorConfig?.configFields.some( - (field) => field.type === 'selector' && isFieldVisible(field) - ) + !canSetUpGitHubInstallation && + (!isMembersMode || + connectorConfig?.configFields.some( + (field) => field.type === 'selector' && isFieldVisible(field) + )) const saveSetup = () => { if (!setupDraftKey) return @@ -343,7 +371,7 @@ export function AddConnectorModal({ canonicalModes, accessMode: access.accessMode, credentialId: effectiveCredentialId, - contentCredentialId, + contentCredentialId: effectiveContentCredentialId, disabledTagIds: Array.from(disabledTagIds), savedAt: Date.now(), }) @@ -367,10 +395,26 @@ export function AddConnectorModal({ ? { scope, accessMode: access.accessMode, - connectorConfig, + connectorConfig: canSetUpGitHubInstallation + ? { + ...connectorConfig, + configFields: connectorConfig.configFields.map((field) => + field.id === 'repository' + ? { + ...field, + type: 'selector', + selectorKey: 'github.installationRepositories', + placeholder: 'Select a repository', + } + : field + ), + } + : connectorConfig, sourceConfig, selectionLabels, - credentialId: effectiveCredentialId, + credentialId: canSetUpGitHubInstallation + ? installationCredentialId + : effectiveCredentialId, credentialType: credentials.find((item) => item.id === effectiveCredentialId)?.type, canonicalGroups, canonicalModes, @@ -380,10 +424,10 @@ export function AddConnectorModal({ } : null - const canSetUpGitHubInstallation = - canAdmin && isSearchIndex && selectedType === 'github' && scope.kind === 'organization' const contentCredentialField = - isMembersMode && connectorConfig?.supportsSeparateContentCredential ? ( + !canSetUpGitHubInstallation && + isMembersMode && + connectorConfig?.supportsSeparateContentCredential ? ( <> setShowGitHubInstallationModal(true), - }, - ] - : []), ]} isLoading={credentialsLoading} disabled={isCreating} @@ -481,7 +515,9 @@ export function AddConnectorModal({ const hasRequiredCredential = isApiKeyMode ? isApiKeyOptional || Boolean(apiKeyValue.trim()) - : isMembersMode || Boolean(effectiveCredentialId) + : canSetUpGitHubInstallation + ? Boolean(installationCredentialId) + : isMembersMode || Boolean(effectiveCredentialId) const hasSearchAccess = !isSearchIndex || Boolean( @@ -544,7 +580,7 @@ export function AddConnectorModal({ : isMembersMode ? { accessMode: 'members' as const, - credentialId: contentCredentialId ?? undefined, + credentialId: effectiveContentCredentialId ?? undefined, } : { accessMode: access.accessMode, credentialId: effectiveCredentialId! }), sourceConfig: finalSourceConfig, @@ -657,7 +693,8 @@ export function AddConnectorModal({ /> )} - {(!lockedAccessMode || slackSetupRequired) && + {!canSetUpGitHubInstallation && + (!lockedAccessMode || slackSetupRequired) && (memberAccessAvailable || mirroredAccessAvailable || slackSetupRequired) && ( ) : null} - {(!isSearchIndex || canSetUpGitHubInstallation) && contentCredentialField} + {canSetUpGitHubInstallation && ( + void refetchCredentials()} + onConnect={() => + void githubSetup.connect(installations.length ? 'install' : undefined) + } + connecting={githubSetup.pending} + connectionError={githubSetup.error} + onCancel={githubSetup.cancel} + onChange={(credentialId) => { + if (credentialId !== installationCredentialId) + handleFieldChange('repository', '') + setContentCredentialId(credentialId) + }} + disabled={isCreating} + /> + )} + {!isSearchIndex && contentCredentialField} {configFieldsProps && ( ))} @@ -957,20 +1016,6 @@ export function AddConnectorModal({ } /> )} - {showGitHubInstallationModal && - canSetUpGitHubInstallation && - isMembersMode && - scope.kind === 'organization' && ( - setShowGitHubInstallationModal(false)} - onConnected={(credentialId) => { - setContentCredentialId(credentialId) - setShowGitHubInstallationModal(false) - }} - /> - )} {showOAuthModal && connectorConfig && connectorConfig.auth.mode === 'oauth' && diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-documents/connector-documents.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-documents/connector-documents.tsx index ceb23481aa2..a63cf633a05 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-documents/connector-documents.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/connector-documents/connector-documents.tsx @@ -4,6 +4,7 @@ import { Chip, ChipDropdown, ChipInput, ChipLink, Skeleton } from '@sim/emcn' import { RefreshCw, Search, SquareArrowUpRight } from '@sim/emcn/icons' import type { ConnectorDocumentFilter } from '@/lib/api/contracts/knowledge/connectors' import type { ResourceScope } from '@/lib/core/resource-scope' +import { getDocumentIndexingStatus } from '@/lib/knowledge/documents/types' import { SettingsEmptyState, SettingsQueryErrorState, @@ -59,9 +60,10 @@ export function ConnectorDocuments({ const documents = (data?.pages.flatMap((page) => page.documents) ?? []).filter((document) => filter === 'excluded' ? document.userExcluded - : !document.userExcluded && (filter !== 'failed' || document.processingStatus === 'failed') + : !document.userExcluded && + (filter === 'active' || getDocumentIndexingStatus(document) === filter) ) - const counts = data?.pages[0]?.counts ?? { active: 0, excluded: 0, failed: 0 } + const counts = data?.pages[0]?.counts ?? { active: 0, excluded: 0, failed: 0, skipped: 0 } const visibleDocumentCount = counts[filter] const hasMoreVisibleDocuments = Boolean(hasNextPage && documents.length < visibleDocumentCount) @@ -92,7 +94,12 @@ export function ConnectorDocuments({ aria-label='Document status' value={filter} onChange={(value) => { - if (value === 'active' || value === 'excluded' || value === 'failed') + if ( + value === 'active' || + value === 'excluded' || + value === 'failed' || + value === 'skipped' + ) onFilterChange(value) }} matchTriggerWidth={false} @@ -103,6 +110,7 @@ export function ConnectorDocuments({ label: isLoading ? 'Excluded' : `Excluded (${counts.excluded})`, }, { value: 'failed', label: isLoading ? 'Failed' : `Failed (${counts.failed})` }, + { value: 'skipped', label: isLoading ? 'Skipped' : `Skipped (${counts.skipped})` }, ]} /> @@ -128,7 +136,9 @@ export function ConnectorDocuments({ ? 'No excluded documents' : filter === 'failed' ? 'No failed documents' - : 'No documents yet'} + : filter === 'skipped' + ? 'No skipped documents' + : 'No documents yet'} ) : ( documents.map((doc) => ( @@ -136,13 +146,17 @@ export function ConnectorDocuments({ key={doc.id} title={doc.filename} description={ - doc.processingStatus === 'failed' + getDocumentIndexingStatus(doc) === 'failed' ? 'Indexing failed' - : doc.processingStatus === 'pending' - ? 'Waiting to index' - : doc.processingStatus === 'processing' - ? 'Indexing' - : undefined + : getDocumentIndexingStatus(doc) === 'skipped' + ? doc.processingError + ? `Skipped · ${doc.processingError}` + : 'Skipped' + : doc.processingStatus === 'pending' + ? 'Waiting to index' + : doc.processingStatus === 'processing' + ? 'Indexing' + : undefined } trailing={
@@ -155,7 +169,7 @@ export function ConnectorDocuments({ aria-label={`Open ${doc.filename}`} /> )} - {doc.processingStatus === 'failed' && !doc.userExcluded && ( + {getDocumentIndexingStatus(doc) === 'failed' && !doc.userExcluded && ( { diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-documents-tab.test.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-documents-tab.test.tsx index b8c3e26b3ab..d748e7ff572 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-documents-tab.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-documents-tab.test.tsx @@ -50,6 +50,18 @@ function button(name: string) { if (!result) throw new Error(`Missing button ${name}`) return result } +function selectStatus(label: string) { + act(() => + document + .querySelector('[aria-label="Document status"]') + ?.dispatchEvent(new MouseEvent('pointerdown', { bubbles: true, button: 0 })) + ) + const option = [...document.querySelectorAll('[role="menuitem"]')].find((item) => + item.textContent?.includes(label) + ) + if (!option) throw new Error(`Missing status option ${label}`) + act(() => option.click()) +} beforeEach(() => { vi.clearAllMocks() for (const state of [mocks.excludeState, mocks.restoreState, mocks.retryState]) { @@ -69,16 +81,26 @@ beforeEach(() => { id: 'failed', filename: 'Handbook.txt', processingStatus: 'failed', + processingError: 'Source download failed', userExcluded: false, }, { id: 'ready', filename: 'Guide.txt', processingStatus: 'completed', + processingError: null, + userExcluded: false, + }, + { + id: 'skipped', + filename: 'logo.png', + processingStatus: 'failed', + processingOutcome: 'skipped', + processingError: 'Binary file was not indexed', userExcluded: false, }, ], - counts: { active: 3, excluded: 0, failed: 1 }, + counts: { active: 4, excluded: 0, failed: 1, skipped: 1 }, }, ], }, @@ -130,21 +152,46 @@ describe('connector document recovery', () => { }) it('requests failed documents from the server and hides healthy placeholder rows', () => { render() - act(() => - document - .querySelector('[aria-label="Document status"]') - ?.dispatchEvent(new MouseEvent('pointerdown', { bubbles: true, button: 0 })) - ) - const failed = [...document.querySelectorAll('[role="menuitem"]')].find((item) => - item.textContent?.includes('Failed (1)') - ) as HTMLElement - act(() => failed.click()) + selectStatus('Failed (1)') expect(mocks.query).toHaveBeenLastCalledWith('kb', 'connector', { filter: 'failed', search: undefined, }) expect(document.body.textContent).toContain('Handbook.txt') expect(document.body.textContent).not.toContain('Guide.txt') + expect(document.body.textContent).not.toContain('logo.png') + }) + it('keeps skipped documents included with their reason and reserves retries for real failures', () => { + render() + expect(document.body.textContent).toContain('Included (4)') + expect(document.body.textContent).toContain('logo.png') + expect(document.body.textContent).toContain('Skipped · Binary file was not indexed') + expect(document.body.textContent).toContain('Indexing failed') + expect( + [...document.querySelectorAll('button')].filter( + (node) => node.textContent?.trim() === 'Retry indexing' + ) + ).toHaveLength(1) + }) + it('requests skipped documents from the server without exposing failed rows or retries', () => { + render() + selectStatus('Skipped (1)') + expect(mocks.query).toHaveBeenLastCalledWith('kb', 'connector', { + filter: 'skipped', + search: undefined, + }) + expect(document.body.textContent).toContain('logo.png') + expect(document.body.textContent).toContain('Skipped · Binary file was not indexed') + expect(document.body.textContent).not.toContain('Handbook.txt') + expect(document.body.textContent).not.toContain('Guide.txt') + expect(document.body.textContent).not.toContain('Retry indexing') + expect(document.body.textContent).not.toContain('Load more documents') + act(() => button('Exclude').click()) + expect(mocks.exclude).toHaveBeenCalledWith({ + knowledgeBaseId: 'kb', + connectorId: 'connector', + documentIds: ['skipped'], + }) }) it.each([false, true])( 'clears prior action errors before retry (retry fails: %s)', diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.test.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.test.tsx index 16c81b1c5cf..d5776048408 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.test.tsx @@ -27,6 +27,8 @@ const mocks = vi.hoisted(() => ({ renderConfigFields: false, selectorOptions: vi.fn(), accessField: vi.fn(), + contentField: vi.fn(), + githubSetup: vi.fn(), })) vi.mock('next/navigation', () => ({ useParams: () => ({}) })) @@ -53,6 +55,17 @@ vi.mock('@/hooks/queries/oauth/oauth-credentials', () => ({ vi.mock('@/hooks/use-credential-refresh-triggers', () => ({ useCredentialRefreshTriggers: vi.fn(), })) +vi.mock('@/hooks/use-github-installation-setup', () => ({ + useGitHubInstallationSetup: (props: { onConnected: (credentialId: string) => void }) => { + mocks.githubSetup(props) + return { + pending: false, + error: null, + cancel: vi.fn(), + connect: () => props.onConnected('replacement-installation'), + } + }, +})) vi.mock( '@/app/workspace/[workspaceId]/integrations/components/connect-service-account-modal', () => ({ @@ -92,12 +105,16 @@ vi.mock( mocks.accessField(props) return null }, - ConnectorContentCredentialField: () => null, + ConnectorContentCredentialField: (props: unknown) => { + mocks.contentField(props) + return null + }, }) ) import { ConnectorSettingsFields } from '@/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields' import { confluenceConnectorMeta } from '@/connectors/confluence/meta' +import { githubConnectorMeta } from '@/connectors/github/meta' import { googleDriveConnectorMeta } from '@/connectors/google-drive/meta' function fieldProps(connectorConfig: ConnectorMeta): ConnectorSettingsFieldsProps { @@ -185,6 +202,138 @@ describe('connector settings service-account choices', () => { }) } + const installationProps = (): Partial => ({ + usesGitHubInstallation: true, + access: { accessMode: 'members' }, + credentialId: 'installation-1', + contentCredentialId: 'installation-1', + sourceConfig: { repository: 'acme/platform' }, + needsWorkspaceCredential: false, + accessDirty: false, + accessModeChanged: false, + accessComplete: true, + isFieldVisible: () => true, + }) + + it('shows the GitHub connection and preserves the repository rename field for installation sources', async () => { + mocks.credentials = [ + { + id: 'installation-1', + name: 'acme', + provider: 'github-app-installation', + type: 'service_account', + }, + ] + await render(githubConnectorMeta, installationProps()) + expect(container.textContent).toContain('GitHub') + expect(container.textContent).toContain('acme') + expect(mocks.contentField).not.toHaveBeenCalled() + expect(mocks.accessField).not.toHaveBeenCalled() + expect(container.textContent).not.toContain('Account for browsing') + expect(container.textContent).not.toContain('Keeps the current repository') + expect(mocks.configFields).toHaveBeenCalledWith( + expect.objectContaining({ + connectorConfig: githubConnectorMeta, + sourceConfig: { repository: 'acme/platform' }, + }) + ) + }) + + it('rotates only among GitHub installation connections through the existing access operation', async () => { + mocks.credentials = [ + { + id: 'installation-1', + name: 'acme', + provider: 'github-app-installation', + type: 'service_account', + }, + { + id: 'installation-2', + name: 'acme-backup', + provider: 'github-app-installation', + type: 'service_account', + }, + { id: 'legacy-reader', name: 'Personal GitHub', provider: 'github', type: 'oauth' }, + ] + const change = vi.fn() + const apply = vi.fn() + const reset = vi.fn() + const changeAccess = vi.fn() + await render(githubConnectorMeta, { ...installationProps(), onContentCredentialChange: change }) + await openAccountChoices() + expect(document.body.textContent).not.toContain('Personal GitHub') + await choose('acme-backup') + expect(change).toHaveBeenCalledWith('installation-2') + await render(githubConnectorMeta, { + ...installationProps(), + contentCredentialId: 'installation-2', + accessDirty: true, + onApplyAccess: apply, + onResetAccess: reset, + onAccessChange: changeAccess, + }) + expect(container.textContent).toContain( + 'Keeps the current repository. To add another repository, add a new source.' + ) + const applyButton = Array.from(container.querySelectorAll('button')).find( + (button) => button.textContent === 'Change connection' + )! + await act(async () => applyButton.click()) + expect(apply).toHaveBeenCalledOnce() + expect(changeAccess).not.toHaveBeenCalled() + const cancel = Array.from(container.querySelectorAll('button')).find( + (button) => button.textContent === 'Cancel' + )! + await act(async () => cancel.click()) + expect(reset).toHaveBeenCalledOnce() + }) + + it('offers GitHub recovery when the saved installation is unavailable', async () => { + const change = vi.fn() + await render(githubConnectorMeta, { ...installationProps(), onContentCredentialChange: change }) + const connect = Array.from(container.querySelectorAll('button')).find( + (button) => button.textContent === 'Connect GitHub' + )! + await act(async () => connect.click()) + expect(mocks.githubSetup).toHaveBeenCalledWith( + expect.objectContaining({ organizationId: 'org-1' }) + ) + expect(container.textContent).not.toContain('Refresh') + expect(change).toHaveBeenCalledWith('replacement-installation') + expect(mocks.accessField).not.toHaveBeenCalled() + expect(mocks.contentField).not.toHaveBeenCalled() + }) + + it('keeps the existing re-enable operation available without exposing access modes', async () => { + const apply = vi.fn() + await render(githubConnectorMeta, { + ...installationProps(), + canReenableMemberSync: true, + onApplyAccess: apply, + }) + const reenable = Array.from(container.querySelectorAll('button')).find( + (button) => button.textContent === 'Re-enable sync' + )! + await act(async () => reenable.click()) + expect(apply).toHaveBeenCalledOnce() + expect(mocks.accessField).not.toHaveBeenCalled() + }) + + it.each([ + { isSearchIndex: false }, + { scope: { kind: 'workspace' as const, workspaceId: 'workspace-1' } }, + ])( + 'keeps general knowledge-base and workspace settings on their original path: %j', + async (overrides) => { + await render(githubConnectorMeta, { ...installationProps(), ...overrides }) + expect(mocks.contentField).toHaveBeenCalled() + expect(mocks.accessField).toHaveBeenCalled() + expect(mocks.configFields).toHaveBeenCalledWith( + expect.objectContaining({ connectorConfig: githubConnectorMeta }) + ) + } + ) + it.each([true, false])( 'locks the sync method only for Search settings (%s)', async (isSearchIndex) => { @@ -195,6 +344,29 @@ describe('connector settings service-account choices', () => { } ) + it.each([null, 'dedicated-github-account'])( + 'preserves legacy GitHub member source settings with content account %s', + async (contentCredentialId) => { + await render(githubConnectorMeta, { + access: { accessMode: 'members' }, + contentCredentialId, + sourceConfig: { repository: 'team/docs' }, + needsWorkspaceCredential: false, + isFieldVisible: () => true, + }) + expect(mocks.contentField).toHaveBeenLastCalledWith( + expect.objectContaining({ credentialId: contentCredentialId }) + ) + expect(mocks.configFields).toHaveBeenCalledWith( + expect.objectContaining({ + sourceConfig: { repository: 'team/docs' }, + connectorConfig: githubConnectorMeta, + }) + ) + expect(document.body.textContent).not.toContain('Connect GitHub') + } + ) + it('keeps a failed availability check actionable before methods are known', async () => { const refetch = vi.fn() await render(confluenceConnectorMeta, { diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.tsx index bb82955401d..a149de2e6be 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/connector-settings-fields.tsx @@ -17,6 +17,7 @@ import { getServiceAccountProviderForProviderId, type OAuthProvider, } from '@/lib/oauth' +import { GITHUB_INSTALLATION_PROVIDER_ID } from '@/lib/oauth/github-installation-types' import type { SourceSelectionLabel, SourceSelectionLabels } from '@/lib/sim-search/source-identity' import { ConnectServiceAccountModal, @@ -42,6 +43,7 @@ import type { } from '@/app/workspace/[workspaceId]/knowledge/[id]/hooks/use-connector-config-fields' import { SettingsQueryErrorState } from '@/app/workspace/[workspaceId]/settings/components/settings-empty-state' import { isConnectorCredentialTypeAllowed } from '@/connectors/auth' +import { GitHubInstallationConnectionField } from '@/connectors/github/installation-connection-field' import { GitLabPermissionTabs, GitLabPermissionUploads, @@ -50,6 +52,7 @@ import type { GitLabPermissionForm } from '@/connectors/gitlab/permission-config import type { ConnectorConfigField, ConnectorMeta } from '@/connectors/types' import { useOAuthCredentials } from '@/hooks/queries/oauth/oauth-credentials' import { useCredentialRefreshTriggers } from '@/hooks/use-credential-refresh-triggers' +import { useGitHubInstallationSetup } from '@/hooks/use-github-installation-setup' const SWITCH_NOTICE: Record = { workspace: 'Every workspace member can read every synced document once the next sync completes.', @@ -68,6 +71,7 @@ export interface ConnectorSettingsFieldsProps { refetch: () => unknown } isSearchIndex: boolean + usesGitHubInstallation?: boolean connectorConfig: ConnectorMeta | null selectionLabels: SourceSelectionLabels sourceConfig: ConfigFieldMap @@ -113,6 +117,7 @@ export function ConnectorSettingsFields({ gitlabPermissions, availability, isSearchIndex, + usesGitHubInstallation = false, connectorConfig, sourceConfig, selectionLabels, @@ -154,6 +159,12 @@ export function ConnectorSettingsFields({ ? (getProviderIdFromServiceId(connectorConfig.auth.provider) as OAuthProvider) : null const syncsPerMember = access.accessMode === 'members' + const isGitHubInstallationSource = + usesGitHubInstallation && + isSearchIndex && + scope.kind === 'organization' && + connectorConfig?.id === 'github' && + syncsPerMember const requiresServiceAccount = Boolean( connectorConfig && !isConnectorCredentialTypeAllowed(connectorConfig.auth, access.accessMode, 'oauth') @@ -172,11 +183,20 @@ export function ConnectorSettingsFields({ serviceIcon: connectorConfig?.icon, }) const [showServiceAccountModal, setShowServiceAccountModal] = useState(false) + const githubSetup = useGitHubInstallationSetup({ + organizationId: + isGitHubInstallationSource && canAdmin && scope.kind === 'organization' + ? scope.organizationId + : undefined, + onConnected: onContentCredentialChange, + }) const [showMoreOptions, setShowMoreOptions] = useState(false) const isContentCredentialChange = accessDirty && !accessModeChanged const { data: rawCredentials = [], isLoading: credentialsLoading, + isFetching: credentialsFetching, + error: credentialsError, refetch: refetchCredentials, } = useOAuthCredentials(providerId ?? undefined, { enabled: (needsWorkspaceCredential || syncsPerMember) && Boolean(providerId), @@ -187,6 +207,9 @@ export function ConnectorSettingsFields({ const [browseCredentialId, setBrowseCredentialId] = useState(null) const selectorCredentialId = syncsPerMember ? browseCredentialId : credentialId const selectorCredential = rawCredentials.find((item) => item.id === selectorCredentialId) + const installations = rawCredentials.filter( + (credential) => credential.provider === GITHUB_INSTALLATION_PROVIDER_ID + ) const credentialOptions = useMemo( () => rawCredentials @@ -255,16 +278,60 @@ export function ConnectorSettingsFields({ /> )} - {syncsPerMember && connectorConfig?.supportsSeparateContentCredential && ( - + onRetry={() => void refetchCredentials()} + onConnect={() => void githubSetup.connect(installations.length ? 'install' : undefined)} + connecting={githubSetup.pending} + connectionError={githubSetup.error} + hint={ + accessDirty + ? 'Keeps the current repository. To add another repository, add a new source.' + : undefined + } + onCancel={githubSetup.cancel} + onChange={onContentCredentialChange} + > + {(accessDirty || canReenableMemberSync) && ( +
+ + {isSwitchingAccess + ? 'Updating…' + : canReenableMemberSync + ? 'Re-enable sync' + : 'Change connection'} + + {accessDirty && ( + + Cancel + + )} +
+ )} + )} - {connectorConfig && showAccessField && ( + {!isGitHubInstallationSource && + syncsPerMember && + connectorConfig?.supportsSeparateContentCredential && ( + + )} + {connectorConfig && showAccessField && !isGitHubInstallationSource && ( field.type === 'selector' && isFieldVisible(field) diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form.test.tsx b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form.test.tsx index d6559838abf..dea50c56b85 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form.test.tsx @@ -32,7 +32,7 @@ vi.mock('@/hooks/use-permission-config', () => ({ ['slack', { oauthAvailable: true, state: 'ready' }], ['slack_v2', { oauthAvailable: true, state: 'ready' }], ]), - oauthServiceAvailability: new Map(), + oauthServiceAvailability: new Map([['github-repositories', true]]), isIntegrationAvailabilityReady: true, isIntegrationAvailabilityFetching: false, integrationAvailabilityError: null, @@ -113,6 +113,23 @@ describe('shared connector settings form', () => { container.remove() }) + it.each([ + { sourceConfig: { repository: 'acme/platform', githubRepositoryId: '9010' }, expected: true }, + { sourceConfig: { repository: 'acme/platform' }, expected: false }, + ])( + 'identifies installation settings from the persisted repository binding: $expected', + ({ sourceConfig, expected }) => { + render( + connector({ connectorType: 'github', credentialId: 'installation-1', sourceConfig }), + 'github' + ) + expect(form.fieldsProps.usesGitHubInstallation).toBe(expected) + expect(form.fieldsProps.access.accessMode).toBe('members') + expect(form.fieldsProps.sourceConfig.repository).toBe('acme/platform') + expect(mocks.applyAccess).not.toHaveBeenCalled() + } + ) + it('treats persisted JSONB label key order as an unchanged draft', () => { render( connector({ @@ -309,6 +326,52 @@ describe('shared connector settings form', () => { expect(onSaved).not.toHaveBeenCalled() }) + it('preserves the GitHub repository and pending connection after an incompatible replacement is refused', () => { + const sourceConfig = { repository: 'acme/platform', branch: 'main' } + render( + connector({ + connectorType: 'github', + credentialId: 'installation-1', + sourceConfig: { ...sourceConfig, githubRepositoryId: '123' }, + }), + 'github-replacement' + ) + act(() => form.fieldsProps.onContentCredentialChange('installation-1')) + expect(form.fieldsProps.accessDirty).toBe(false) + expect(form.fieldsProps.sourceConfig).toMatchObject(sourceConfig) + expect(form.fieldsProps.usesGitHubInstallation).toBe(true) + + act(() => form.fieldsProps.onFieldChange('pathPrefix', 'docs/')) + act(() => form.fieldsProps.onContentCredentialChange('installation-2')) + expect(form.fieldsProps.sourceConfig).toMatchObject({ ...sourceConfig, pathPrefix: 'docs/' }) + expect(form.fieldsProps.accessDirty).toBe(true) + expect(form.canSave).toBe(false) + act(() => form.fieldsProps.onApplyAccess()) + expect(mocks.applyAccess).toHaveBeenCalledExactlyOnceWith( + { + knowledgeBaseId: 'kb-search', + connectorId: 'connector-1', + access: { accessMode: 'members', credentialId: 'installation-2' }, + }, + expect.any(Object) + ) + const message = + "This GitHub connection cannot access this source's repository. Choose a connection with access to the same repository, or add a new source for a different repository." + act(() => mocks.applyAccess.mock.calls[0][1].onError(new Error(message))) + expect(form.fieldsProps.error).toBe(message) + expect(form.fieldsProps.contentCredentialId).toBe('installation-2') + expect(form.fieldsProps.sourceConfig).toMatchObject({ ...sourceConfig, pathPrefix: 'docs/' }) + expect(form.fieldsProps.accessDirty).toBe(true) + expect(onSaved).not.toHaveBeenCalled() + expect(mocks.update).not.toHaveBeenCalled() + + act(() => form.fieldsProps.onResetAccess()) + expect(form.fieldsProps.contentCredentialId).toBe('installation-1') + expect(form.fieldsProps.accessDirty).toBe(false) + expect(form.fieldsProps.sourceConfig).toMatchObject({ ...sourceConfig, pathPrefix: 'docs/' }) + expect(form.canSave).toBe(true) + }) + it.each(['settingsPending', 'accessPending'] as const)( 'blocks page saving while %s is pending', (pending) => { diff --git a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form.ts b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form.ts index 15744d583ba..c764d2b0500 100644 --- a/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form.ts +++ b/apps/sim/app/workspace/[workspaceId]/knowledge/[id]/components/edit-connector-modal/use-connector-settings-form.ts @@ -462,6 +462,9 @@ export function useConnectorSettingsForm({ refetch: refetchIntegrationAvailability, }, isSearchIndex, + usesGitHubInstallation: + connector.connectorType === 'github' && + typeof connector.sourceConfig.githubRepositoryId === 'string', connectorConfig, sourceConfig, selectionLabels, diff --git a/apps/sim/app/workspace/[workspaceId]/search/components/github-installation-modal.test.tsx b/apps/sim/app/workspace/[workspaceId]/search/components/github-installation-modal.test.tsx deleted file mode 100644 index af00ad3f4ec..00000000000 --- a/apps/sim/app/workspace/[workspaceId]/search/components/github-installation-modal.test.tsx +++ /dev/null @@ -1,199 +0,0 @@ -/** - * @vitest-environment jsdom - */ -import { act } from 'react' -import { createRoot, type Root } from 'react-dom/client' -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' -import type { ListGitHubSearchInstallationsResponse } from '@/lib/api/contracts/knowledge/github-installations' - -const mocks = vi.hoisted(() => ({ - data: undefined as ListGitHubSearchInstallationsResponse | undefined, - error: null as Error | null, - fetching: false, - pending: false, - refetch: vi.fn(), - connectInstallation: vi.fn(), - ensureAccounts: vi.fn(), - connectAccount: vi.fn(), - onConnected: vi.fn(), -})) - -vi.mock('@/hooks/queries/github-search-installations', () => ({ - useGitHubSearchInstallations: () => ({ - data: mocks.data, - error: mocks.error, - isSuccess: Boolean(mocks.data) && !mocks.error, - isError: Boolean(mocks.error), - isFetching: mocks.fetching, - refetch: mocks.refetch, - }), - useConnectGitHubSearchInstallation: () => ({ - mutate: mocks.connectInstallation, - isPending: mocks.pending, - error: null, - }), -})) -vi.mock('@/hooks/queries/organization-accounts', () => ({ - useEnsureOrganizationAccounts: () => ({ - mutate: mocks.ensureAccounts, - isPending: false, - error: null, - }), - useConnectOrganizationAccount: () => ({ - mutate: mocks.connectAccount, - isPending: false, - error: null, - }), -})) - -import { GitHubInstallationModal } from '@/app/workspace/[workspaceId]/search/components/github-installation-modal' - -let root: Root -let container: HTMLDivElement - -async function render() { - await act(async () => { - root.render( - - ) - }) -} - -function button(label: string): HTMLButtonElement { - const match = Array.from(document.querySelectorAll('button')).find( - (node) => node.textContent?.trim() === label - ) - if (!match) throw new Error(`Missing button: ${label}`) - return match -} - -beforeEach(() => { - vi.clearAllMocks() - mocks.data = { - success: true, - available: true, - installUrl: 'https://github.com/apps/sim-search/installations/new', - needsUserConnection: false, - installations: [ - { - installationId: '123', - accountId: '456', - accountLogin: 'acme', - accountType: 'Organization', - }, - ], - } - mocks.error = null - mocks.fetching = false - mocks.pending = false - ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true - vi.stubGlobal( - 'ResizeObserver', - class { - observe() {} - unobserve() {} - disconnect() {} - } - ) - container = document.createElement('div') - document.body.appendChild(container) - root = createRoot(container) -}) - -afterEach(async () => { - await act(async () => root.unmount()) - container.remove() - vi.restoreAllMocks() -}) - -describe('GitHub installation setup', () => { - it('uses only a server-verified installation and returns its credential', async () => { - await render() - const installLink = document.querySelector('a[href*="installations/new"]') - expect(installLink?.href).toBe('https://github.com/apps/sim-search/installations/new') - expect(installLink?.target).toBe('_blank') - expect(installLink?.rel).toContain('noopener') - await act(async () => button('Use installation').click()) - expect(mocks.connectInstallation).toHaveBeenCalledWith( - { organizationId: 'org-1', installationId: '123' }, - expect.any(Object) - ) - const callbacks = mocks.connectInstallation.mock.calls[0][1] - callbacks.onSuccess({ success: true, credential: { id: 'credential-1', displayName: 'acme' } }) - expect(mocks.onConnected).toHaveBeenCalledWith('credential-1') - }) - - it('requires a connected personal account before selecting an installation', async () => { - mocks.data!.needsUserConnection = true - await render() - expect(button('Use installation').disabled).toBe(true) - expect(button('Connect your GitHub account')).toBeTruthy() - expect(document.querySelector('a[href*="installations/new"]')).toBeNull() - expect(document.querySelector('[role="combobox"]')).toBeNull() - }) - - it('opens the existing managed-account connection after preserving provider setup', async () => { - mocks.data!.needsUserConnection = true - const tab = { opener: {}, closed: false, location: { href: '' }, close: vi.fn() } - vi.spyOn(window, 'open').mockReturnValue(tab as unknown as Window) - await render() - await act(async () => button('Connect your GitHub account').click()) - expect(tab.opener).toBeNull() - expect(mocks.ensureAccounts).toHaveBeenCalledWith( - { - organizationId: 'org-1', - option: { provider: 'github-repositories', label: 'GitHub', required: false }, - }, - expect.any(Object) - ) - const setup = mocks.ensureAccounts.mock.calls[0][1] - await act(async () => - setup.onSuccess({ - credentialGroup: { - options: [ - { id: 'other-option', provider: 'confluence', status: 'active' }, - { id: 'github-option', provider: 'github-repositories', status: 'active' }, - ], - }, - }) - ) - expect(mocks.connectAccount).toHaveBeenCalledWith( - { organizationId: 'org-1', optionId: 'github-option' }, - expect.any(Object) - ) - await act(async () => - mocks.connectAccount.mock.calls[0][1].onSuccess({ - invitationLink: 'https://sim.ai/credential-groups/invite/test', - }) - ) - expect(tab.location.href).toBe('https://sim.ai/credential-groups/invite/test') - expect(document.body.textContent).toContain('Finish connecting your account in the other tab') - }) - - it('does not create an enrollment when the browser blocks the account tab', async () => { - mocks.data!.needsUserConnection = true - vi.spyOn(window, 'open').mockReturnValue(null) - await render() - await act(async () => button('Connect your GitHub account').click()) - expect(mocks.ensureAccounts).not.toHaveBeenCalled() - expect(document.body.textContent).toContain('Allow pop-ups') - }) - - it.each(['unavailable', 'error', 'refreshing', 'empty'] as const)( - 'refuses installation changes while %s', - async (state) => { - if (state === 'unavailable') mocks.data!.available = false - if (state === 'error') mocks.error = new Error('Installation lookup failed') - if (state === 'refreshing') mocks.fetching = true - if (state === 'empty') mocks.data!.installations = [] - await render() - expect(button('Use installation').disabled).toBe(true) - await act(async () => button('Use installation').click()) - expect(mocks.connectInstallation).not.toHaveBeenCalled() - } - ) -}) diff --git a/apps/sim/app/workspace/[workspaceId]/search/components/github-installation-modal.tsx b/apps/sim/app/workspace/[workspaceId]/search/components/github-installation-modal.tsx deleted file mode 100644 index a966ed0c85c..00000000000 --- a/apps/sim/app/workspace/[workspaceId]/search/components/github-installation-modal.tsx +++ /dev/null @@ -1,222 +0,0 @@ -'use client' - -import { useState } from 'react' -import { - Chip, - ChipCombobox, - ChipLink, - ChipModal, - ChipModalBody, - ChipModalError, - ChipModalField, - ChipModalFooter, - ChipModalHeader, -} from '@sim/emcn' -import { - SettingsEmptyState, - SettingsQueryErrorState, -} from '@/app/workspace/[workspaceId]/settings/components/settings-empty-state' -import { - useConnectGitHubSearchInstallation, - useGitHubSearchInstallations, -} from '@/hooks/queries/github-search-installations' -import { - useConnectOrganizationAccount, - useEnsureOrganizationAccounts, -} from '@/hooks/queries/organization-accounts' - -interface GitHubInstallationModalProps { - organizationId: string - onClose: () => void - onConnected: (credentialId: string) => void -} - -/** Installs a content account while preserving each reader's own GitHub authorization. */ -export function GitHubInstallationModal({ - organizationId, - onClose, - onConnected, -}: GitHubInstallationModalProps) { - const installations = useGitHubSearchInstallations(organizationId) - const connectInstallation = useConnectGitHubSearchInstallation() - const ensureAccounts = useEnsureOrganizationAccounts() - const connectAccount = useConnectOrganizationAccount() - const [installationId, setInstallationId] = useState(null) - const [waitingForAccount, setWaitingForAccount] = useState(false) - const [connectionError, setConnectionError] = useState(null) - const data = installations.isSuccess ? installations.data : undefined - const choices = data?.installations ?? [] - const selected = - choices.find((item) => item.installationId === installationId) ?? - (choices.length === 1 ? choices[0] : undefined) - const pending = - connectInstallation.isPending || ensureAccounts.isPending || connectAccount.isPending - const canConnect = - data?.available === true && - !data.needsUserConnection && - selected !== undefined && - !installations.isFetching && - !pending - - const connectGitHubAccount = () => { - const tab = window.open('about:blank', '_blank') - if (!tab) { - setConnectionError('Allow pop-ups for this site to connect your GitHub account.') - return - } - tab.opener = null - setConnectionError(null) - ensureAccounts.mutate( - { - organizationId, - option: { provider: 'github-repositories', label: 'GitHub', required: false }, - }, - { - onSuccess: ({ credentialGroup }) => { - const option = credentialGroup.options.find( - (item) => item.provider === 'github-repositories' && item.status === 'active' - ) - if (!option) { - tab.close() - setConnectionError('GitHub account setup is unavailable. Refresh and try again.') - return - } - if (tab.closed) return - connectAccount.mutate( - { organizationId, optionId: option.id }, - { - onSuccess: ({ invitationLink }) => { - if (tab.closed) return - tab.location.href = invitationLink - setWaitingForAccount(true) - }, - onError: () => tab.close(), - } - ) - }, - onError: () => tab.close(), - } - ) - } - - return ( - { - if (!open && !pending) onClose() - }} - > - Connect GitHub App - - - {(aria) => - installations.isError ? ( - void installations.refetch()} - variant='inline' - /> - ) : !data ? ( - Loading GitHub setup… - ) : !data.available ? ( - - GitHub App indexing is unavailable in this deployment. - - ) : data.needsUserConnection ? ( -
-

- {waitingForAccount - ? 'Finish connecting your account in the other tab, then refresh.' - : 'Connect your GitHub account to verify the installations you can manage.'} -

-
- - {waitingForAccount ? 'Open account connection' : 'Connect your GitHub account'} - - void installations.refetch()} - > - Refresh - -
-
- ) : ( -
- {choices.length > 0 ? ( - ({ - value: item.installationId, - label: item.accountLogin, - }))} - value={selected?.installationId} - onChange={setInstallationId} - placeholder='Select an installation' - disabled={pending || installations.isFetching} - /> - ) : ( - - No eligible installations found. - - )} -
- {data.installUrl && ( - - Install GitHub App - - )} - void installations.refetch()} - > - Refresh - -
-
- ) - } -
- - {connectionError ?? - ensureAccounts.error?.message ?? - connectAccount.error?.message ?? - connectInstallation.error?.message} - -
- { - if (!canConnect || !selected) return - connectInstallation.mutate( - { organizationId, installationId: selected.installationId }, - { onSuccess: ({ credential }) => onConnected(credential.id) } - ) - }, - }} - /> -
- ) -} diff --git a/apps/sim/app/workspace/[workspaceId]/search/components/search-source-setup.test.tsx b/apps/sim/app/workspace/[workspaceId]/search/components/search-source-setup.test.tsx index 1b54c95d232..3266b86d157 100644 --- a/apps/sim/app/workspace/[workspaceId]/search/components/search-source-setup.test.tsx +++ b/apps/sim/app/workspace/[workspaceId]/search/components/search-source-setup.test.tsx @@ -73,6 +73,14 @@ vi.mock('@/lib/auth/auth-client', () => ({ useSession: () => ({ data: { user: { id: mocks.userId } } }), })) vi.mock('@/hooks/use-oauth-return', () => ({ useOAuthReturnForKBConnectors: mocks.oauthReturn })) +vi.mock('@/hooks/use-github-installation-setup', () => ({ + useGitHubInstallationSetup: () => ({ + connect: vi.fn(), + cancel: vi.fn(), + pending: false, + error: null, + }), +})) vi.mock('@/hooks/use-permission-config', () => ({ usePermissionConfig: () => ({ integrationAvailability: new Map([ diff --git a/apps/sim/connectors/github/github.test.ts b/apps/sim/connectors/github/github.test.ts index 53eaa6594d9..32ec8e9f9b0 100644 --- a/apps/sim/connectors/github/github.test.ts +++ b/apps/sim/connectors/github/github.test.ts @@ -7,6 +7,7 @@ import { listingFingerprint, runResumableListing, } from '@/lib/knowledge/connectors/listing-checkpoint' +import { classifyExternalDoc } from '@/lib/knowledge/connectors/sync-primitives' import { githubConnector } from '@/connectors/github/github' import type { ExternalDocument } from '@/connectors/types' import { PER_MEMBER_LISTING_CONTEXT } from '@/connectors/utils' @@ -153,6 +154,7 @@ describe('githubConnector member listing', () => { content: '', contentDeferred: true, contentHash: 'git-sha:blob-sha', + skippedRetryPolicy: 'source-change', }) expect(fetchMock).toHaveBeenCalledTimes(1) expect(fetchMock.mock.calls[0]![1].headers.Authorization).toBe('Bearer member-token') @@ -486,6 +488,94 @@ describe('githubConnector.getDocument', () => { }) }) +describe('githubConnector content outcomes', () => { + afterEach(() => vi.unstubAllGlobals()) + + it.each([ + ['empty.txt', '', 'Empty file was not indexed'], + ['blank.txt', ' \n\t ', 'Empty file was not indexed'], + ['image.png', 'binary\0contents', 'Binary file was not indexed'], + ])( + 'records %s as a verified omission and reuses its unchanged hash', + async (path, body, reason) => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce(treeResponse([treeFile(path, 'blob-sha')])) + .mockResolvedValueOnce(new Response(body)) + vi.stubGlobal('fetch', fetchMock) + const context = {} + const listing = await githubConnector.listDocuments('token', source, undefined, context) + const skipped = await githubConnector.getDocument('token', source, path, context) + expect(skipped).toMatchObject({ + content: '', + contentDeferred: false, + contentHash: listing.documents[0].contentHash, + skippedReason: reason, + skippedExistingDisposition: 'replace', + }) + const existing = { id: 'document', contentHash: skipped!.contentHash, storageKey: null } + expect(classifyExternalDoc(listing.documents[0], existing)).toEqual({ type: 'unchanged' }) + expect( + classifyExternalDoc({ ...listing.documents[0], contentHash: 'git-sha:new-blob' }, existing) + ).toEqual({ type: 'update', existingId: 'document' }) + expect(classifyExternalDoc(listing.documents[0], { ...existing, contentHash: null })).toEqual( + { + type: 'update', + existingId: 'document', + } + ) + expect(fetchMock).toHaveBeenCalledTimes(2) + } + ) + + it('skips a known empty blob before downloading it', async () => { + const fetchMock = vi.fn().mockResolvedValueOnce(treeResponse([treeFile('empty.pdf', 'sha', 0)])) + vi.stubGlobal('fetch', fetchMock) + const context = {} + const listing = await githubConnector.listDocuments('token', source, undefined, context) + const document = await githubConnector.getDocument('token', source, 'empty.pdf', context) + expect(listing.documents[0]).toMatchObject({ + skippedReason: 'Empty file was not indexed', + skippedExistingDisposition: 'replace', + contentDeferred: false, + }) + expect(document?.skippedReason).toBe('Empty file was not indexed') + expect(document?.sourceFile).toBeUndefined() + expect(fetchMock).toHaveBeenCalledTimes(1) + }) + + it.each([ + ['report.pdf', 'application/pdf'], + ['report.docx', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'], + ])('hands the original %s bytes to the shared document processor', async (path, mimeType) => { + const bytes = Buffer.from([0x50, 0x4b, 0x00, 0x03, 0x04]) + const fetchMock = vi + .fn() + .mockResolvedValueOnce(treeResponse([treeFile(path, 'blob-sha', bytes.length)])) + .mockResolvedValueOnce(new Response(bytes)) + vi.stubGlobal('fetch', fetchMock) + const context = {} + const listing = await githubConnector.listDocuments('token', source, undefined, context) + const document = await githubConnector.getDocument('token', source, path, context) + expect(document).toMatchObject({ + content: '', + contentDeferred: false, + mimeType, + sourceFile: { bytes, fileName: path, mimeType }, + contentHash: listing.documents[0].contentHash, + }) + expect(document?.skippedReason).toBeUndefined() + expect( + classifyExternalDoc(listing.documents[0], { + id: 'previously-skipped', + contentHash: 'git-sha:blob-sha', + storageKey: null, + }) + ).toEqual({ type: 'update', existingId: 'previously-skipped' }) + expect(fetchMock).toHaveBeenCalledTimes(2) + }) +}) + describe('githubConnector symlinks', () => { afterEach(() => vi.unstubAllGlobals()) @@ -574,6 +664,43 @@ describe('githubConnector symlinks', () => { ]) }) + it('skips a directory link without requesting a directory as file content', async () => { + const directory = { ...treeFile('docs/folder'), mode: '040000', type: 'tree' } + const fetchMock = vi + .fn() + .mockResolvedValueOnce(treeResponse([link, directory])) + .mockResolvedValueOnce(new Response('folder')) + vi.stubGlobal('fetch', fetchMock) + await expect(githubConnector.getDocument('token', source, link.path)).resolves.toMatchObject({ + skippedReason: 'Symbolic link target is not a repository file', + skippedRetryPolicy: 'source-change', + skippedExistingDisposition: 'replace', + }) + expect(fetchMock).toHaveBeenCalledTimes(2) + }) + + it('routes a document symlink through the shared parser using its displayed file format', async () => { + const documentLink = { ...link, path: 'docs/report.pdf' } + const bytes = Buffer.from('%PDF-1.7\n\0binary document content') + const fetchMock = vi + .fn() + .mockResolvedValueOnce(treeResponse([documentLink, target])) + .mockResolvedValueOnce(new Response('target.md')) + .mockResolvedValueOnce(new Response(bytes)) + vi.stubGlobal('fetch', fetchMock) + const context = {} + const listing = await githubConnector.listDocuments('token', source, undefined, context) + const document = await githubConnector.getDocument('token', source, documentLink.path, context) + expect(document).toMatchObject({ + externalId: documentLink.path, + title: 'report.pdf', + sourceFile: { bytes, fileName: 'report.pdf', mimeType: 'application/pdf' }, + contentHash: listing.documents[0].contentHash, + }) + expect(document?.skippedReason).toBeUndefined() + expect(fetchMock).toHaveBeenCalledTimes(3) + }) + it('follows an in-repository link chain outside the configured listing prefix', async () => { const nextLink = { ...treeFile('intermediate.md', 'next-link-sha'), mode: '120000' } vi.stubGlobal( @@ -657,6 +784,7 @@ describe('githubConnector symlinks', () => { await expect(githubConnector.getDocument('token', source, link.path)).resolves.toMatchObject({ content: '', skippedReason: reason, + skippedExistingDisposition: 'replace', }) } }) diff --git a/apps/sim/connectors/github/github.ts b/apps/sim/connectors/github/github.ts index d1f71f951ae..f19991f8486 100644 --- a/apps/sim/connectors/github/github.ts +++ b/apps/sim/connectors/github/github.ts @@ -15,6 +15,7 @@ import { isPerMemberListing, markSkipped, parseTagDate, + pipelineParsedMimeType, readBodyWithLimit, sizeLimitSkipReason, stubOrSkipBySize, @@ -34,6 +35,8 @@ const GITHUB_API_URL = 'https://api.github.com' const BATCH_SIZE = 200 const GIT_SHA_PREFIX = 'git-sha:' const MAX_FILE_SIZE = CONNECTOR_MAX_FILE_BYTES +/** Rehydrates formats formerly skipped or decoded as plain text once through their real parser. */ +const SOURCE_FILE_HASH_SUFFIX = ':source-file-v1' const BINARY_SNIFF_BYTES = 8000 const MAX_SYMLINK_DEPTH = 40 const MAX_SYMLINK_TARGET_BYTES = 4096 @@ -43,6 +46,7 @@ const MAX_SYMLINK_TARGET_BYTES = 4096 * re-downloaded in full on every sync. */ const BINARY_SKIP_REASON = 'Binary file was not indexed' +const EMPTY_SKIP_REASON = 'Empty file was not indexed' /** * Heuristic binary detection: Git treats files containing a NUL byte in the @@ -260,14 +264,14 @@ async function fetchTree( return snapshot } -/** Streams a Git blob with the same binary and byte bounds used for ordinary files. */ -async function fetchBlobContent( +/** Keeps original bytes available to the shared parsers while bounding every blob read. */ +async function fetchBlobBytes( accessToken: string, owner: string, repo: string, sha: string, maxBytes = MAX_FILE_SIZE -): Promise { +): Promise { const url = `${GITHUB_API_URL}/repos/${owner}/${repo}/git/blobs/${encodeURIComponent(sha)}` const label = `git blob ${sha}` const response = await fetchWithRetry(url, { @@ -296,8 +300,7 @@ async function fetchBlobContent( if (!buffer) { throw new ConnectorFileTooLargeError(maxBytes) } - if (isBinaryBuffer(buffer)) return null - return decodeTextBuffer(buffer).text + return buffer } /** Resolves links within one snapshot; Contents can truncate dereferenced targets at 1 MiB. */ @@ -314,13 +317,21 @@ async function resolveSymlinkTarget( if (item.mode !== '120000') return item.type === 'blob' ? item : null if (visited.has(item.path) || (item.size ?? 0) > MAX_SYMLINK_TARGET_BYTES) return null visited.add(item.path) - let target: string | null + let targetBytes: Buffer try { - target = await fetchBlobContent(accessToken, owner, repo, item.sha, MAX_SYMLINK_TARGET_BYTES) + targetBytes = await fetchBlobBytes( + accessToken, + owner, + repo, + item.sha, + MAX_SYMLINK_TARGET_BYTES + ) } catch (error) { if (error instanceof ConnectorFileTooLargeError) return null throw error } + if (isBinaryBuffer(targetBytes)) return null + const target = decodeTextBuffer(targetBytes).text if (!target || posix.isAbsolute(target)) return null const targetPath = posix.normalize(posix.join(posix.dirname(item.path), target)) if (targetPath === '..' || targetPath.startsWith('../')) return null @@ -356,10 +367,13 @@ function treeItemToStub( title: item.path.split('/').pop() || item.path, content: '', contentDeferred: true, + skippedRetryPolicy: 'source-change', + /** Verified immutable omissions replace older content; fetch failures still throw. */ + skippedExistingDisposition: 'replace', mimeType: 'text/plain', sourceUrl: `https://github.com/${owner}/${repo}/blob/${branch.split('/').map(encodeURIComponent).join('/')}/${item.path.split('/').map(encodeURIComponent).join('/')}`, /** Contents dereferences symlinks but retains their SHA even when the target changes. */ - contentHash: `${GIT_SHA_PREFIX}${item.sha}${item.mode === '120000' ? `:${treeSha}` : ''}`, + contentHash: `${GIT_SHA_PREFIX}${item.sha}${item.mode === '120000' ? `:${treeSha}` : ''}${pipelineParsedMimeType(item.path) ? SOURCE_FILE_HASH_SUFFIX : ''}`, metadata: { path: item.path, sha: item.sha, @@ -464,13 +478,14 @@ export const githubConnector: ConnectorConfig = { batchSize: batch.length, }) - const documents = batch.map((item) => - stubOrSkipBySize( + const documents = batch.map((item) => { + const stub = stubOrSkipBySize( treeItemToStub(owner, repo, branch, item, snapshot.sha), item.size, MAX_FILE_SIZE ) - ) + return item.size === 0 ? markSkipped(stub, EMPTY_SKIP_REASON) : stub + }) const nextOffset = offset + BATCH_SIZE const hasMore = nextOffset < capped.length @@ -509,22 +524,32 @@ export const githubConnector: ConnectorConfig = { const size = target?.size ?? 0 const stub = treeItemToStub(owner, repo, branch, { ...treeItem, size }, snapshot.sha) if (!target) { - return { - ...markSkipped(stub, 'Symbolic link target is not a repository file'), - skippedExistingDisposition: 'replace', - } + return markSkipped(stub, 'Symbolic link target is not a repository file') } if (size > MAX_FILE_SIZE) return markSkipped(stub, sizeLimitSkipReason(MAX_FILE_SIZE)) + if (target.size === 0) return markSkipped(stub, EMPTY_SKIP_REASON) /** The immutable listed blob avoids ref drift and a redundant Contents API request. */ - let content: string | null + let bytes: Buffer try { - content = await fetchBlobContent(accessToken, owner, repo, target.sha) + bytes = await fetchBlobBytes(accessToken, owner, repo, target.sha) } catch (error) { if (error instanceof ConnectorFileTooLargeError) return markSkipped(stub, sizeLimitSkipReason(MAX_FILE_SIZE)) throw error } - if (content === null) return markSkipped(stub, BINARY_SKIP_REASON) + if (bytes.length === 0) return markSkipped(stub, EMPTY_SKIP_REASON) + const mimeType = pipelineParsedMimeType(stub.title) + if (mimeType) { + return { + ...stub, + contentDeferred: false, + mimeType, + sourceFile: { bytes, fileName: stub.title, mimeType }, + } + } + if (isBinaryBuffer(bytes)) return markSkipped(stub, BINARY_SKIP_REASON) + const content = decodeTextBuffer(bytes).text + if (!content.trim()) return markSkipped(stub, EMPTY_SKIP_REASON) return { ...stub, diff --git a/apps/sim/connectors/github/installation-connection-field.tsx b/apps/sim/connectors/github/installation-connection-field.tsx new file mode 100644 index 00000000000..f5f23f9324f --- /dev/null +++ b/apps/sim/connectors/github/installation-connection-field.tsx @@ -0,0 +1,86 @@ +import type { ReactNode } from 'react' +import { Chip, ChipCombobox, ChipModalError, ChipModalField } from '@sim/emcn' +import { Plus } from '@sim/emcn/icons' +import type { Credential } from '@/lib/oauth' +import { SettingsQueryErrorState } from '@/app/workspace/[workspaceId]/settings/components/settings-empty-state' + +interface GitHubInstallationConnectionFieldProps { + installations: Credential[] + credentialId: string | null + isLoading: boolean + isFetching: boolean + error: Error | null + disabled: boolean + onRetry: () => void + onConnect: () => void + onChange: (credentialId: string) => void + connecting?: boolean + connectionError?: string | null + hint?: string + onCancel?: () => void + children?: ReactNode +} + +export function GitHubInstallationConnectionField({ + installations, + credentialId, + isLoading, + isFetching, + error, + disabled, + onRetry, + onConnect, + onChange, + connecting = false, + connectionError, + hint, + onCancel, + children, +}: GitHubInstallationConnectionFieldProps) { + return ( + + {connecting ? ( +
+ Connecting GitHub… + + Cancel + +
+ ) : error && installations.length === 0 ? ( + + ) : installations.length > 0 ? ( + ({ + value: credential.id, + label: credential.name || 'GitHub', + })), + { + value: '__connect_github__', + label: 'Connect another organization', + icon: Plus, + onSelect: onConnect, + }, + ]} + value={credentialId ?? undefined} + onChange={onChange} + placeholder='Select GitHub connection' + disabled={disabled} + /> + ) : ( + + {isLoading ? 'Loading GitHub…' : 'Connect GitHub'} + + )} + {connectionError} + {children} +
+ ) +} diff --git a/apps/sim/hooks/queries/github-search-setup.test.tsx b/apps/sim/hooks/queries/github-search-setup.test.tsx new file mode 100644 index 00000000000..e94931c00d2 --- /dev/null +++ b/apps/sim/hooks/queries/github-search-setup.test.tsx @@ -0,0 +1,162 @@ +/** @vitest-environment jsdom */ +import { act } from 'react' +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { ApiClientError } from '@/lib/api/client/errors' +import { + type GitHubSearchSetupScope, + type GitHubSearchSetupStatus, + readGitHubSearchSetupContract, +} from '@/lib/api/contracts/knowledge/github-setup' + +const mocks = vi.hoisted(() => ({ request: vi.fn() })) +vi.mock('@/lib/api/client/request', () => ({ requestJson: mocks.request })) + +import { githubSearchSetupKeys, useGitHubSearchSetup } from '@/hooks/queries/github-search-setup' + +const SCOPE: GitHubSearchSetupScope = { + organizationId: 'org-1', + setupId: 'cbe393bf-eb93-4b61-8e53-ce1dd602ba77', +} + +interface ProbeProps { + scope?: GitHubSearchSetupScope +} + +describe('GitHub setup status queries', () => { + let root: Root + let client: QueryClient + let result: ReturnType + + function Probe({ scope }: ProbeProps) { + result = useGitHubSearchSetup(scope) + return null + } + + async function advance(milliseconds: number) { + await act(async () => { + await vi.advanceTimersByTimeAsync(milliseconds) + }) + await act(async () => { + await vi.advanceTimersByTimeAsync(1) + }) + } + + async function render(scope?: GitHubSearchSetupScope) { + await act(async () => + root.render( + + + + ) + ) + await advance(1) + } + + beforeEach(() => { + vi.useFakeTimers() + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) + mocks.request.mockReset() + mocks.request.mockResolvedValue({ success: true, data: { status: 'pending' } }) + client = new QueryClient({ defaultOptions: { queries: { retryDelay: 1 } } }) + root = createRoot(document.createElement('div')) + }) + + afterEach(async () => { + await act(async () => root.unmount()) + client.clear() + vi.useRealTimers() + vi.unstubAllGlobals() + }) + + it('does not request or poll without an active scope', async () => { + await render() + await advance(30_000) + expect(mocks.request).not.toHaveBeenCalled() + }) + + it('forwards the exact scope and aborts a read when its scope is replaced', async () => { + const signals: AbortSignal[] = [] + mocks.request.mockImplementation((_contract: unknown, input: { signal: AbortSignal }) => { + signals.push(input.signal) + return new Promise((_resolve, reject) => { + input.signal.addEventListener('abort', () => reject(new Error('Aborted')), { + once: true, + }) + }) + }) + await render(SCOPE) + expect(mocks.request).toHaveBeenCalledWith(readGitHubSearchSetupContract, { + query: SCOPE, + signal: expect.any(AbortSignal), + }) + expect(signals[0].aborted).toBe(false) + + const otherScope = { ...SCOPE, organizationId: 'org-2' } + await render(otherScope) + expect(signals[0].aborted).toBe(true) + expect(signals[1].aborted).toBe(false) + expect(mocks.request).toHaveBeenLastCalledWith(readGitHubSearchSetupContract, { + query: otherScope, + signal: signals[1], + }) + expect(client.getQueryData(githubSearchSetupKeys.detail(SCOPE))).toBeUndefined() + }) + + it.each([ + new TypeError('Failed to fetch'), + new ApiClientError({ status: 503, message: 'Temporarily unavailable', body: null }), + new ApiClientError({ status: 429, message: 'Try again later', body: null }), + ])('recovers a transient read failure: $message', async (failure) => { + mocks.request.mockRejectedValueOnce(failure) + await render(SCOPE) + await advance(5) + expect(result.data).toEqual({ status: 'pending' }) + expect(result.error).toBeNull() + expect(mocks.request).toHaveBeenCalledTimes(2) + + await advance(1500) + expect(mocks.request).toHaveBeenCalledTimes(3) + }) + + it('continues polling after transient errors exhaust the immediate retry budget', async () => { + const failure = new ApiClientError({ status: 503, message: 'Try again', body: null }) + mocks.request.mockRejectedValue(failure) + await render(SCOPE) + await advance(10) + expect(mocks.request).toHaveBeenCalledTimes(3) + expect(result.error).toBe(failure) + + mocks.request.mockResolvedValue({ success: true, data: { status: 'pending' } }) + await advance(5000) + expect(result.data).toEqual({ status: 'pending' }) + expect(result.error).toBeNull() + expect(mocks.request).toHaveBeenCalledTimes(4) + }) + + it('does not retry or poll after a terminal authorization denial', async () => { + const failure = new ApiClientError({ status: 403, message: 'Access denied', body: null }) + mocks.request.mockRejectedValue(failure) + await render(SCOPE) + await advance(30_000) + expect(result.error).toBe(failure) + expect(mocks.request).toHaveBeenCalledOnce() + }) + + it.each([ + { status: 'completed', credential: { id: 'installation-1', displayName: 'Acme' } }, + { status: 'failed', error: 'Choose a GitHub organization you own.' }, + { status: 'expired' }, + ])('stops polling after the server returns $status', async (terminalStatus) => { + await render(SCOPE) + expect(result.data).toEqual({ status: 'pending' }) + mocks.request.mockResolvedValue({ success: true, data: terminalStatus }) + await advance(1500) + expect(result.data).toEqual(terminalStatus) + expect(mocks.request).toHaveBeenCalledTimes(2) + + await advance(30_000) + expect(mocks.request).toHaveBeenCalledTimes(2) + }) +}) diff --git a/apps/sim/hooks/queries/github-search-setup.ts b/apps/sim/hooks/queries/github-search-setup.ts new file mode 100644 index 00000000000..2baf9dcd99a --- /dev/null +++ b/apps/sim/hooks/queries/github-search-setup.ts @@ -0,0 +1,77 @@ +'use client' + +import { useMutation, useQuery } from '@tanstack/react-query' +import { isApiClientError } from '@/lib/api/client/errors' +import { requestJson } from '@/lib/api/client/request' +import { + cancelGitHubSearchSetupContract, + type GitHubSearchSetupScope, + readGitHubSearchSetupContract, + type SelectGitHubSearchSetupBody, + type StartGitHubSearchSetupBody, + selectGitHubSearchSetupContract, + startGitHubSearchSetupContract, +} from '@/lib/api/contracts/knowledge/github-setup' + +export function isGitHubSetupTerminalError(error: unknown): boolean { + return ( + isApiClientError(error) && + error.status >= 400 && + error.status < 500 && + ![408, 429].includes(error.status) + ) +} + +export const githubSearchSetupKeys = { + all: ['github-search-setup'] as const, + details: () => [...githubSearchSetupKeys.all, 'detail'] as const, + detail: (scope?: GitHubSearchSetupScope) => + [ + ...githubSearchSetupKeys.details(), + scope?.organizationId ?? '', + scope?.setupId ?? '', + ] as const, +} + +export function useGitHubSearchSetup(scope?: GitHubSearchSetupScope) { + return useQuery({ + queryKey: githubSearchSetupKeys.detail(scope), + queryFn: async ({ signal }) => { + if (!scope) throw new Error('GitHub connection attempt is required') + return (await requestJson(readGitHubSearchSetupContract, { query: scope, signal })).data + }, + enabled: Boolean(scope), + staleTime: 0, + gcTime: 5 * 60_000, + retry: (failures, error) => !isGitHubSetupTerminalError(error) && failures < 2, + refetchInterval: (query) => + scope && + !isGitHubSetupTerminalError(query.state.error) && + (!query.state.data || ['pending', 'choosing'].includes(query.state.data.status)) + ? query.state.error + ? 5000 + : 1500 + : false, + }) +} + +export function useStartGitHubSearchSetup() { + return useMutation({ + mutationFn: (body: StartGitHubSearchSetupBody) => + requestJson(startGitHubSearchSetupContract, { body }), + }) +} + +export function useCancelGitHubSearchSetup() { + return useMutation({ + mutationFn: (body: GitHubSearchSetupScope) => + requestJson(cancelGitHubSearchSetupContract, { body }), + }) +} + +export function useSelectGitHubSearchSetup() { + return useMutation({ + mutationFn: (body: SelectGitHubSearchSetupBody) => + requestJson(selectGitHubSearchSetupContract, { body }), + }) +} diff --git a/apps/sim/hooks/queries/kb/connectors.ts b/apps/sim/hooks/queries/kb/connectors.ts index 807d908b116..f0c284635f2 100644 --- a/apps/sim/hooks/queries/kb/connectors.ts +++ b/apps/sim/hooks/queries/kb/connectors.ts @@ -500,7 +500,13 @@ export function useOrganizationSearchOverview( export function useSearchSources( owner?: string | ResourceScope, - options?: { enabled?: boolean; search?: string; mine?: boolean; connectorType?: string } + options?: { + enabled?: boolean + search?: string + mine?: boolean + connectorType?: string + excludeConnectorType?: string + } ) { const queryClient = useQueryClient() const scope = @@ -516,6 +522,9 @@ export function useSearchSources( search: options?.search?.trim().toLowerCase() ?? '', mine: options?.mine ?? false, ...(options?.connectorType?.trim() ? { connectorType: options.connectorType.trim() } : {}), + ...(options?.excludeConnectorType?.trim() + ? { excludeConnectorType: options.excludeConnectorType.trim() } + : {}), } const summary = useInfiniteQuery({ queryKey: searchSourceKeys.pages(scope, filters), diff --git a/apps/sim/hooks/queries/organization-accounts.ts b/apps/sim/hooks/queries/organization-accounts.ts index 7c96859612d..733c3f03927 100644 --- a/apps/sim/hooks/queries/organization-accounts.ts +++ b/apps/sim/hooks/queries/organization-accounts.ts @@ -28,6 +28,7 @@ import { type OrganizationAccountPeopleQuery, type RemoveOrganizationAccountMcpProviderParams, type ResendOrganizationAccountInvitationQuery, + reconnectPersonalOrganizationAccountContract, removeOrganizationAccountMcpProviderContract, resendOrganizationAccountInvitationContract, revokeOrganizationAccountEnrollmentContract, @@ -43,6 +44,13 @@ import { searchSourceKeys } from '@/hooks/queries/utils/search-source-keys' export const ORGANIZATION_ACCOUNTS_STALE_TIME = 30_000 +export function useReconnectPersonalOrganizationAccount() { + return useMutation({ + mutationFn: (credentialId: string) => + requestJson(reconnectPersonalOrganizationAccountContract, { params: { credentialId } }), + }) +} + /** Disconnects an owned grant; indexing and source setup do not gate this operation. */ export function useDisconnectPersonalOrganizationAccount(organizationId: string) { const queryClient = useQueryClient() diff --git a/apps/sim/hooks/queries/utils/search-source-keys.ts b/apps/sim/hooks/queries/utils/search-source-keys.ts index 17f69eb5683..ee508d3fb1b 100644 --- a/apps/sim/hooks/queries/utils/search-source-keys.ts +++ b/apps/sim/hooks/queries/utils/search-source-keys.ts @@ -12,7 +12,12 @@ export const searchSourceKeys = { ] as const, pages: ( scope: string | ResourceScope | undefined, - filters: { search: string; mine: boolean; connectorType?: string } + filters: { + search: string + mine: boolean + connectorType?: string + excludeConnectorType?: string + } ) => [...searchSourceKeys.list(scope), 'pages', filters] as const, overview: (scope?: string | ResourceScope) => [...searchSourceKeys.list(scope), 'overview'] as const, diff --git a/apps/sim/hooks/use-github-installation-setup-cache.test.tsx b/apps/sim/hooks/use-github-installation-setup-cache.test.tsx new file mode 100644 index 00000000000..b57c63752cd --- /dev/null +++ b/apps/sim/hooks/use-github-installation-setup-cache.test.tsx @@ -0,0 +1,178 @@ +/** @vitest-environment jsdom */ +import { act } from 'react' +import { QueryClient, QueryClientProvider } from '@tanstack/react-query' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { + readGitHubSearchSetupContract, + startGitHubSearchSetupContract, +} from '@/lib/api/contracts/knowledge/github-setup' +import { + listOrganizationCredentialsContract, + listOrganizationOAuthCredentialsContract, +} from '@/lib/api/contracts/organization-credentials' + +const mocks = vi.hoisted(() => ({ + request: vi.fn<(contract: unknown, input: RequestInput) => Promise>(), + connected: vi.fn(), +})) +vi.mock('@/lib/api/client/request', () => ({ requestJson: mocks.request })) +vi.mock('@/hooks/queries/credentials', () => ({ useWorkspaceCredential: vi.fn() })) + +import { oauthCredentialKeys, useOAuthCredentials } from '@/hooks/queries/oauth/oauth-credentials' +import { useGitHubInstallationSetup } from '@/hooks/use-github-installation-setup' + +interface RequestInput { + query?: { organizationId?: string; purpose?: string } +} + +describe('GitHub setup credential cache reconciliation', () => { + let root: Root + let client: QueryClient + let setup: ReturnType + let indexing: ReturnType + let browsing: ReturnType + let unrelated: ReturnType + let completed: boolean + let channels: Array<{ onmessage: ((event: MessageEvent) => void) | null }> + + function Probe() { + indexing = useOAuthCredentials('github-repositories', { organizationId: 'org-1' }) + browsing = useOAuthCredentials('github-repositories', { + organizationId: 'org-1', + purpose: 'browsing', + }) + unrelated = useOAuthCredentials('github-repositories', { organizationId: 'org-2' }) + setup = useGitHubInstallationSetup({ + organizationId: 'org-1', + onConnected: mocks.connected, + }) + return null + } + + async function flush() { + await act(async () => { + await vi.advanceTimersByTimeAsync(1) + }) + await act(async () => { + await vi.advanceTimersByTimeAsync(1) + }) + } + + function accountRequests(organizationId: string) { + return mocks.request.mock.calls.filter( + ([contract, input]: [unknown, RequestInput]) => + contract === listOrganizationCredentialsContract && + input.query?.organizationId === organizationId + ) + } + + beforeEach(() => { + vi.useFakeTimers() + vi.stubGlobal('IS_REACT_ACT_ENVIRONMENT', true) + mocks.request.mockReset() + mocks.connected.mockReset() + completed = false + channels = [] + vi.stubGlobal( + 'BroadcastChannel', + class { + onmessage = null + close = vi.fn() + constructor() { + channels.push(this) + } + } + ) + const tab = { opener: {}, location: { href: 'about:blank' }, focus: vi.fn(), close: vi.fn() } + vi.spyOn(window, 'open').mockReturnValue(tab as unknown as Window) + mocks.request.mockImplementation(async (contract: unknown, input: RequestInput) => { + if (contract === listOrganizationOAuthCredentialsContract) return { credentials: [] } + if (contract === listOrganizationCredentialsContract) { + const organizationId = input.query?.organizationId + return { + credentials: [ + { + id: `${organizationId}-existing`, + displayName: 'Existing installation', + providerId: 'github-app-installation', + }, + ...(completed && organizationId === 'org-1' + ? [ + { + id: 'installation-new', + displayName: 'New organization', + providerId: 'github-app-installation', + }, + ] + : []), + ], + } + } + if (contract === startGitHubSearchSetupContract) + return { success: true, url: 'https://github.com/apps/sim/installations/new' } + if (contract === readGitHubSearchSetupContract) + return { + success: true, + data: completed + ? { + status: 'completed', + credential: { id: 'installation-new', displayName: 'New organization' }, + } + : { status: 'pending' }, + } + throw new Error('Unexpected request') + }) + client = new QueryClient({ defaultOptions: { queries: { retry: false } } }) + root = createRoot(document.createElement('div')) + }) + + afterEach(async () => { + await act(async () => root.unmount()) + client.clear() + vi.useRealTimers() + vi.restoreAllMocks() + vi.unstubAllGlobals() + }) + + it('refreshes mounted Add and Settings lists on completion without a visibility event', async () => { + await act(async () => + root.render( + + + + ) + ) + await flush() + expect(indexing.data?.map(({ id }) => id)).toEqual(['org-1-existing']) + expect(browsing.data?.map(({ id }) => id)).toEqual(['org-1-existing']) + expect(accountRequests('org-1')).toHaveLength(2) + expect(accountRequests('org-2')).toHaveLength(1) + + await act(async () => setup.connect()) + await flush() + expect(setup.pending).toBe(true) + completed = true + await act(async () => { + channels[0].onmessage?.({ data: 'connected' } as MessageEvent) + }) + await flush() + await flush() + + expect(mocks.connected).toHaveBeenCalledExactlyOnceWith('installation-new') + expect(setup.pending).toBe(false) + expect(indexing.data?.map(({ id }) => id)).toEqual(['org-1-existing', 'installation-new']) + expect(browsing.data?.map(({ id }) => id)).toEqual(['org-1-existing', 'installation-new']) + expect(accountRequests('org-1')).toHaveLength(4) + expect(accountRequests('org-2')).toHaveLength(1) + expect(unrelated.data?.map(({ id }) => id)).toEqual(['org-2-existing']) + expect( + client.getQueryData(oauthCredentialKeys.list('github-repositories', '', '', 'org-1')) + ).toEqual(indexing.data) + expect( + client.getQueryData( + oauthCredentialKeys.list('github-repositories', '', '', 'org-1', 'browsing') + ) + ).toEqual(browsing.data) + }) +}) diff --git a/apps/sim/hooks/use-github-installation-setup.test.tsx b/apps/sim/hooks/use-github-installation-setup.test.tsx new file mode 100644 index 00000000000..688e036e2c8 --- /dev/null +++ b/apps/sim/hooks/use-github-installation-setup.test.tsx @@ -0,0 +1,224 @@ +/** + * @vitest-environment jsdom + */ +import { act } from 'react' +import { createRoot, type Root } from 'react-dom/client' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import { ApiClientError } from '@/lib/api/client/errors' +import type { GitHubSearchSetupStatus } from '@/lib/api/contracts/knowledge/github-setup' + +const mocks = vi.hoisted(() => ({ + start: vi.fn(), + cancel: vi.fn(), + refetch: vi.fn(), + invalidate: vi.fn(), + connected: vi.fn(), + status: undefined as GitHubSearchSetupStatus | undefined, + error: null as Error | null, +})) +const client = { invalidateQueries: mocks.invalidate } +vi.mock('@tanstack/react-query', () => ({ useQueryClient: () => client })) +vi.mock('@/hooks/queries/github-search-setup', () => ({ + isGitHubSetupTerminalError: (error: unknown) => + error instanceof ApiClientError && error.status === 403, + useStartGitHubSearchSetup: () => ({ mutateAsync: mocks.start, isPending: false }), + useCancelGitHubSearchSetup: () => ({ mutateAsync: mocks.cancel }), + useGitHubSearchSetup: () => ({ + data: mocks.status, + error: mocks.error, + refetch: mocks.refetch, + }), +})) + +import { useGitHubInstallationSetup } from '@/hooks/use-github-installation-setup' + +describe('GitHub installation setup handoff', () => { + let root: Root + let container: HTMLDivElement + let current: ReturnType + let channels: Array<{ + onmessage: ((event: MessageEvent) => void) | null + close: ReturnType + }> + let tab: { + opener: unknown + location: { href: string } + focus: ReturnType + close: ReturnType + } + function Probe({ organizationId = 'org-1' }: { organizationId?: string }) { + current = useGitHubInstallationSetup({ organizationId, onConnected: mocks.connected }) + return null + } + beforeEach(() => { + vi.clearAllMocks() + vi.useFakeTimers() + mocks.status = undefined + mocks.error = null + mocks.start.mockResolvedValue({ url: 'https://github.com/login/oauth/authorize' }) + mocks.cancel.mockResolvedValue({ success: true }) + channels = [] + vi.stubGlobal( + 'BroadcastChannel', + class { + onmessage = null + close = vi.fn() + constructor() { + channels.push(this) + } + } + ) + tab = { opener: {}, location: { href: 'about:blank' }, focus: vi.fn(), close: vi.fn() } + vi.spyOn(window, 'open').mockReturnValue(tab as unknown as Window) + ;(globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }).IS_REACT_ACT_ENVIRONMENT = true + container = document.createElement('div') + document.body.appendChild(container) + root = createRoot(container) + act(() => root.render()) + }) + afterEach(() => { + act(() => root.unmount()) + container.remove() + vi.useRealTimers() + vi.restoreAllMocks() + vi.unstubAllGlobals() + }) + + it('opens approval synchronously and completes only from the authorized server receipt', async () => { + await act(async () => current.connect()) + expect(tab.opener).toBeNull() + expect(tab.location.href).toBe('https://github.com/login/oauth/authorize') + expect(mocks.start).toHaveBeenCalledWith({ + organizationId: 'org-1', + setupId: expect.any(String), + }) + expect(current.pending).toBe(true) + act(() => channels[0].onmessage?.({ data: 'connected' } as MessageEvent)) + expect(mocks.refetch).toHaveBeenCalledOnce() + expect(mocks.connected).not.toHaveBeenCalled() + mocks.status = { + status: 'completed', + credential: { id: 'installation-1', displayName: 'Acme' }, + } + act(() => root.render()) + expect(current.pending).toBe(false) + expect(mocks.connected).toHaveBeenCalledExactlyOnceWith('installation-1') + expect(tab.close).toHaveBeenCalledOnce() + expect(mocks.invalidate).toHaveBeenCalledTimes(4) + expect(mocks.invalidate).toHaveBeenCalledWith({ + queryKey: ['oauthCredentials', 'list', 'github-repositories', '', '', 'org-1', 'browsing'], + }) + act(() => root.render()) + expect(mocks.connected).toHaveBeenCalledOnce() + }) + + it('keeps one approval attempt and focuses it when Connect is clicked again', async () => { + await act(async () => current.connect()) + await act(async () => current.connect()) + expect(window.open).toHaveBeenCalledOnce() + expect(mocks.start).toHaveBeenCalledOnce() + expect(tab.focus).toHaveBeenCalledOnce() + }) + + it('cancels without navigating a late start response or accepting its completion', async () => { + let resolve!: (value: { url: string }) => void + mocks.start.mockReturnValue( + new Promise((done) => { + resolve = done + }) + ) + let connecting!: Promise + act(() => { + connecting = current.connect() + }) + act(() => current.cancel()) + await act(async () => { + resolve({ url: 'https://github.com/apps/sim/installations/new' }) + await connecting + }) + expect(tab.location.href).toBe('about:blank') + expect(mocks.cancel).toHaveBeenCalledWith(mocks.start.mock.calls[0][0]) + mocks.status = { status: 'completed', credential: { id: 'cancelled', displayName: 'Acme' } } + act(() => root.render()) + expect(mocks.connected).not.toHaveBeenCalled() + expect(current.pending).toBe(false) + }) + + it('shows denied approval inline and allows a new attempt', async () => { + await act(async () => current.connect()) + act(() => channels[0].onmessage?.({ data: 'denied' } as MessageEvent)) + expect(current.pending).toBe(true) + expect(mocks.refetch).toHaveBeenCalledOnce() + mocks.status = { status: 'failed', error: 'Choose a GitHub organization you own.' } + act(() => root.render()) + expect(current.error).toBe('Choose a GitHub organization you own.') + expect(current.pending).toBe(false) + mocks.status = undefined + await act(async () => current.connect()) + expect(mocks.start).toHaveBeenCalledTimes(2) + expect(current.error).toBeNull() + }) + + it('cancels on organization change and ignores the previous receipt', async () => { + await act(async () => current.connect()) + mocks.status = { status: 'completed', credential: { id: 'wrong-org', displayName: 'Acme' } } + act(() => root.render()) + expect(mocks.cancel).toHaveBeenCalledWith(mocks.start.mock.calls[0][0]) + expect(mocks.connected).not.toHaveBeenCalled() + expect(current.pending).toBe(false) + }) + + it('keeps the approval open through transient polling failures but stops on lost access', async () => { + await act(async () => current.connect()) + mocks.error = new Error('Failed to fetch') + act(() => root.render()) + expect(current.pending).toBe(true) + expect(mocks.cancel).not.toHaveBeenCalled() + mocks.error = new ApiClientError({ + status: 403, + message: 'Organization access denied', + body: null, + }) + act(() => root.render()) + expect(current.pending).toBe(false) + expect(current.error).toBe('Organization access denied') + expect(mocks.cancel).toHaveBeenCalledOnce() + }) + + it('forwards an explicit new-organization request instead of reusing an existing installation', async () => { + await act(async () => current.connect('install')) + expect(mocks.start).toHaveBeenCalledWith({ + organizationId: 'org-1', + setupId: expect.any(String), + intent: 'install', + }) + }) + + it('bounds the attempt lifetime and cleans up its channel', async () => { + await act(async () => current.connect()) + act(() => vi.advanceTimersByTime(10 * 60_000)) + expect(current.pending).toBe(false) + expect(current.error).toContain('timed out') + expect(channels[0].close).toHaveBeenCalledOnce() + expect(mocks.cancel).toHaveBeenCalledOnce() + }) + + it('reports popup blocking before creating any server attempt', async () => { + vi.mocked(window.open).mockReturnValue(null) + await act(async () => current.connect()) + expect(mocks.start).not.toHaveBeenCalled() + expect(current.error).toContain('Allow pop-ups') + }) + + it.each([ + 'https://attacker.example/authorize', + 'javascript:alert(1)', + 'https://github.com@attacker.example', + ])('rejects an unexpected approval destination: %s', async (url) => { + mocks.start.mockResolvedValue({ url }) + await act(async () => current.connect()) + expect(tab.location.href).toBe('about:blank') + expect(current.error).toBe('GitHub returned an invalid setup URL') + expect(mocks.cancel).toHaveBeenCalledOnce() + }) +}) diff --git a/apps/sim/hooks/use-github-installation-setup.ts b/apps/sim/hooks/use-github-installation-setup.ts new file mode 100644 index 00000000000..762a6dd222d --- /dev/null +++ b/apps/sim/hooks/use-github-installation-setup.ts @@ -0,0 +1,172 @@ +'use client' + +import { useCallback, useEffect, useRef, useState } from 'react' +import { getErrorMessage } from '@sim/utils/errors' +import { generateId } from '@sim/utils/id' +import { useQueryClient } from '@tanstack/react-query' +import type { StartGitHubSearchSetupBody } from '@/lib/api/contracts/knowledge/github-setup' +import { + credentialGroupOAuthCompletionChannel, + isCredentialGroupOAuthFailure, +} from '@/lib/credential-groups/oauth-completion' +import { resolveGitHubSetupUrl } from '@/lib/knowledge/github-setup-navigation' +import { githubSearchInstallationKeys } from '@/hooks/queries/github-search-installations' +import { + isGitHubSetupTerminalError, + useCancelGitHubSearchSetup, + useGitHubSearchSetup, + useStartGitHubSearchSetup, +} from '@/hooks/queries/github-search-setup' +import { oauthCredentialKeys } from '@/hooks/queries/oauth/oauth-credentials' +import { organizationAccountsKeys } from '@/hooks/queries/organization-accounts' + +interface GitHubInstallationSetupProps { + organizationId?: string + onConnected: (credentialId: string) => void +} + +/** Keeps the source form in place; only an authorized server result completes setup. */ +export function useGitHubInstallationSetup({ + organizationId, + onConnected, +}: GitHubInstallationSetupProps) { + const active = useRef<{ setupId: string; tab: Window } | null>(null) + const callback = useRef(onConnected) + const [setupId, setSetupId] = useState() + const [error, setError] = useState(null) + const [previousOrganizationId, setPreviousOrganizationId] = useState(organizationId) + if (previousOrganizationId !== organizationId) { + setPreviousOrganizationId(organizationId) + setSetupId(undefined) + setError(null) + } + const client = useQueryClient() + const { mutateAsync: start, isPending: isStarting } = useStartGitHubSearchSetup() + const { mutateAsync: cancelSetup } = useCancelGitHubSearchSetup() + const scope = organizationId && setupId ? { organizationId, setupId } : undefined + const query = useGitHubSearchSetup(scope) + const { refetch } = query + + useEffect(() => { + callback.current = onConnected + }, [onConnected]) + + useEffect(() => { + return () => { + const attempt = active.current + active.current = null + attempt?.tab.close() + if (attempt && organizationId) + void cancelSetup({ organizationId, setupId: attempt.setupId }).catch(() => undefined) + } + }, [organizationId, cancelSetup]) + + useEffect(() => { + if (!setupId || !organizationId) return + const fail = (message: string) => { + if (active.current?.setupId !== setupId) return + active.current.tab.close() + active.current = null + setSetupId(undefined) + setError(message) + void cancelSetup({ organizationId, setupId }).catch(() => undefined) + } + const channel = new BroadcastChannel(credentialGroupOAuthCompletionChannel(setupId)) + channel.onmessage = ({ data }: MessageEvent) => { + if (isCredentialGroupOAuthFailure(data) || data === 'connected') void refetch() + } + const timer = window.setTimeout( + () => fail('GitHub connection timed out. Try connecting again.'), + 10 * 60_000 + ) + return () => { + channel.close() + window.clearTimeout(timer) + } + }, [organizationId, setupId, cancelSetup, refetch]) + + useEffect(() => { + if (!setupId || !organizationId || active.current?.setupId !== setupId) return + const result = query.data + if (result?.status === 'completed') { + active.current.tab.close() + active.current = null + setSetupId(undefined) + for (const purpose of [undefined, 'browsing'] as const) { + void client.invalidateQueries({ + queryKey: oauthCredentialKeys.list( + 'github-repositories', + '', + '', + organizationId, + purpose + ), + }) + } + void client.invalidateQueries({ queryKey: githubSearchInstallationKeys.list(organizationId) }) + void client.invalidateQueries({ queryKey: organizationAccountsKeys.detail(organizationId) }) + callback.current(result.credential.id) + } else if ( + isGitHubSetupTerminalError(query.error) || + result?.status === 'failed' || + result?.status === 'expired' + ) { + active.current.tab.close() + active.current = null + setSetupId(undefined) + setError( + query.error?.message ?? + (result?.status === 'failed' + ? result.error + : 'This GitHub connection attempt expired. Try connecting again.') + ) + void cancelSetup({ organizationId, setupId }).catch(() => undefined) + } + }, [query.data, query.error, setupId, organizationId, client, cancelSetup]) + + const connect = useCallback( + async (intent?: StartGitHubSearchSetupBody['intent']) => { + if (!organizationId) return + if (active.current) { + active.current.tab.focus() + return + } + const tab = window.open('about:blank', '_blank', 'width=600,height=700') + if (!tab) { + setError('Allow pop-ups for this site to connect GitHub, then try again.') + return + } + tab.opener = null + const id = generateId() + active.current = { setupId: id, tab } + setError(null) + try { + const result = await start({ organizationId, setupId: id, ...(intent ? { intent } : {}) }) + if (active.current?.setupId !== id) return + const url = resolveGitHubSetupUrl(result.url, window.location.origin) + setSetupId(id) + tab.location.href = url + } catch (failure) { + if (active.current?.setupId !== id) return + tab.close() + active.current = null + setSetupId(undefined) + setError(getErrorMessage(failure, 'Could not connect GitHub')) + void cancelSetup({ organizationId, setupId: id }).catch(() => undefined) + } + }, + [organizationId, start, cancelSetup] + ) + + const cancel = useCallback(() => { + const attempt = active.current + if (!attempt || !organizationId) return + active.current = null + attempt.tab.close() + setSetupId(undefined) + setError(null) + void cancelSetup({ organizationId, setupId: attempt.setupId }).catch(() => undefined) + }, [organizationId, cancelSetup]) + + return { connect, cancel, pending: isStarting || Boolean(setupId), error } +} diff --git a/apps/sim/lib/api/contracts/knowledge/connectors.test.ts b/apps/sim/lib/api/contracts/knowledge/connectors.test.ts index ed6870a25e9..5babbca4b6f 100644 --- a/apps/sim/lib/api/contracts/knowledge/connectors.test.ts +++ b/apps/sim/lib/api/contracts/knowledge/connectors.test.ts @@ -3,6 +3,7 @@ */ import { describe, expect, it } from 'vitest' import { + connectorDocumentDataSchema, connectorDocumentsQuerySchema, createConnectorBodySchema, updateConnectorAccessBodySchema, @@ -16,7 +17,7 @@ const base = { describe('connector document list contracts', () => { it('accepts each document set and trims filename search without interpreting wildcards', () => { - for (const filter of ['active', 'excluded', 'failed']) { + for (const filter of ['active', 'excluded', 'failed', 'skipped']) { expect( connectorDocumentsQuerySchema.parse({ filter, search: ' 50%_report ' }) ).toMatchObject({ @@ -26,6 +27,26 @@ describe('connector document list contracts', () => { } }) + it('adds skipped outcomes without requiring them from older document responses', () => { + const document = { + id: 'document', + filename: 'logo.png', + externalId: 'logo.png', + sourceUrl: null, + enabled: true, + userExcluded: false, + uploadedAt: '2026-01-01T00:00:00Z', + processingStatus: 'failed', + } + expect(connectorDocumentDataSchema.parse(document)).toMatchObject({ + processingStatus: 'failed', + processingOutcome: null, + }) + expect( + connectorDocumentDataSchema.parse({ ...document, processingOutcome: 'skipped' }) + ).toMatchObject({ processingStatus: 'failed', processingOutcome: 'skipped' }) + }) + it('preserves legacy flags when no document filter is supplied', () => { expect( connectorDocumentsQuerySchema.parse({ includeExcluded: 'true', failedOnly: 'false' }) diff --git a/apps/sim/lib/api/contracts/knowledge/connectors.ts b/apps/sim/lib/api/contracts/knowledge/connectors.ts index a84ae2f1ab3..da71e989b1b 100644 --- a/apps/sim/lib/api/contracts/knowledge/connectors.ts +++ b/apps/sim/lib/api/contracts/knowledge/connectors.ts @@ -75,7 +75,7 @@ export const deleteConnectorQuerySchema = z.object({ deleteDocuments: booleanQueryFlagSchema.optional().default(false), }) -export const connectorDocumentFilterSchema = z.enum(['active', 'excluded', 'failed']) +export const connectorDocumentFilterSchema = z.enum(['active', 'excluded', 'failed', 'skipped']) export type ConnectorDocumentFilter = z.output export const connectorDocumentsQuerySchema = z.object({ @@ -236,6 +236,8 @@ export const connectorDocumentDataSchema = z userExcluded: z.boolean(), uploadedAt: z.string(), processingStatus: z.string(), + processingOutcome: z.literal('skipped').nullable().default(null), + processingError: z.string().nullable().default(null), }) .passthrough() export type ConnectorDocumentData = z.output @@ -246,6 +248,7 @@ export const connectorDocumentsDataSchema = z.object({ active: z.number().int().nonnegative(), excluded: z.number().int().nonnegative(), failed: z.number().int().nonnegative().default(0), + skipped: z.number().int().nonnegative().default(0), }), hasMore: z.boolean().optional(), }) @@ -394,6 +397,12 @@ export const searchSourceCursorSchema = z.object({ export const listSearchSourcesQuerySchema = resourceOwnerSchema.safeExtend({ cursor: z.string().min(1).max(1024).optional(), connectorType: z.string().trim().min(1, 'connectorType cannot be empty').max(100).optional(), + excludeConnectorType: z + .string() + .trim() + .min(1, 'excludeConnectorType cannot be empty') + .max(100) + .optional(), search: z.string().trim().max(200).optional(), mine: booleanQueryFlagSchema.optional(), }) diff --git a/apps/sim/lib/api/contracts/knowledge/documents.test.ts b/apps/sim/lib/api/contracts/knowledge/documents.test.ts index 5c0510f5dc4..9a12967d460 100644 --- a/apps/sim/lib/api/contracts/knowledge/documents.test.ts +++ b/apps/sim/lib/api/contracts/knowledge/documents.test.ts @@ -2,12 +2,57 @@ * @vitest-environment node */ import { describe, expect, it } from 'vitest' +import { z } from 'zod' import { bulkCreateDocumentsBodySchema, + documentDataSchema, listKnowledgeDocumentsQuerySchema, parseDocumentTagFiltersParam, upsertDocumentBodySchema, } from '@/lib/api/contracts/knowledge/documents' +import { getDocumentIndexingStatus } from '@/lib/knowledge/documents/types' + +describe('document processing response compatibility', () => { + const document = { + id: 'document', + knowledgeBaseId: 'knowledge-base', + filename: 'logo.png', + fileUrl: '', + fileSize: 0, + mimeType: 'text/plain', + chunkCount: 0, + tokenCount: 0, + characterCount: 0, + processingStatus: 'failed', + enabled: true, + uploadedAt: '2026-01-01T00:00:00Z', + } + const previousSchema = documentDataSchema.omit({ processingOutcome: true }).extend({ + processingStatus: z.enum(['pending', 'processing', 'completed', 'failed']), + }) + + it('keeps skipped responses valid for the previous strict four-status reader', () => { + const response = documentDataSchema.parse({ ...document, processingOutcome: 'skipped' }) + expect(response.processingStatus).toBe('failed') + expect(getDocumentIndexingStatus(response)).toBe('skipped') + expect(previousSchema.parse(response)).toEqual(document) + }) + + it.each(['pending', 'processing', 'completed', 'failed'])( + 'accepts an older server’s %s response without inventing an outcome', + (processingStatus) => { + const response = documentDataSchema.parse({ ...document, processingStatus }) + expect(response.processingOutcome).toBeNull() + expect(getDocumentIndexingStatus(response)).toBe(processingStatus) + } + ) + + it('does not expand the stored-status wire enum to encode an indexing outcome', () => { + expect(documentDataSchema.safeParse({ ...document, processingStatus: 'skipped' }).success).toBe( + false + ) + }) +}) describe('listKnowledgeDocumentsQuerySchema.tagFilters', () => { it('keeps tagFilters a raw string (must NOT transform to an array)', () => { diff --git a/apps/sim/lib/api/contracts/knowledge/documents.ts b/apps/sim/lib/api/contracts/knowledge/documents.ts index 67dc7d8720f..a5219138b05 100644 --- a/apps/sim/lib/api/contracts/knowledge/documents.ts +++ b/apps/sim/lib/api/contracts/knowledge/documents.ts @@ -16,6 +16,7 @@ import { privateSecretProvenanceBundleSchema } from '@/lib/api/contracts/primiti import { defineRouteContract } from '@/lib/api/contracts/types' import { PRIVATE_SECRET_PROVENANCE_FIELD } from '@/lib/execution/private-tool-metadata' import { getFieldTypeForSlot, MAX_KNOWLEDGE_DOCUMENTS_PER_CREATE } from '@/lib/knowledge/constants' +import { DOCUMENT_PROCESSING_STATUSES } from '@/lib/knowledge/documents/types' import { getOperatorsForFieldType, isValidFilterValue } from '@/lib/knowledge/filters/types' import { knowledgeDocumentUploadMetadataSchema } from '@/lib/knowledge/upload-metadata' @@ -257,7 +258,8 @@ export const documentDataSchema = z chunkCount: z.number(), tokenCount: z.number(), characterCount: z.number(), - processingStatus: z.enum(['pending', 'processing', 'completed', 'failed']), + processingStatus: z.enum(DOCUMENT_PROCESSING_STATUSES), + processingOutcome: z.literal('skipped').nullable().default(null), /** When indexing was last dispatched to a worker, which precedes a worker starting it. */ processingQueuedAt: nullableWireDateSchema.optional(), processingStartedAt: nullableWireDateSchema.optional(), diff --git a/apps/sim/lib/api/contracts/knowledge/github-setup.ts b/apps/sim/lib/api/contracts/knowledge/github-setup.ts new file mode 100644 index 00000000000..dfd0d3c16e0 --- /dev/null +++ b/apps/sim/lib/api/contracts/knowledge/github-setup.ts @@ -0,0 +1,108 @@ +import { z } from 'zod' +import { + githubInstallationIdSchema, + githubSearchInstallationSchema, +} from '@/lib/api/contracts/knowledge/github-installations' +import { organizationIdSchema } from '@/lib/api/contracts/primitives' +import { defineRouteContract } from '@/lib/api/contracts/types' + +export const gitHubSearchSetupScopeSchema = z + .object({ + organizationId: organizationIdSchema, + setupId: z.string().uuid('GitHub setup ID must be a UUID'), + }) + .strict() +export type GitHubSearchSetupScope = z.input + +export const gitHubSearchSetupStatusSchema = z.discriminatedUnion('status', [ + z.object({ status: z.literal('pending') }), + z.object({ + status: z.literal('choosing'), + installations: z.array(githubSearchInstallationSchema).min(1).max(1000), + }), + z.object({ + status: z.literal('completed'), + credential: z.object({ + id: z.string().min(1).max(200), + displayName: z.string().min(1).max(500), + }), + }), + z.object({ status: z.literal('failed'), error: z.string().min(1).max(1000) }), + z.object({ status: z.literal('expired') }), +]) +export type GitHubSearchSetupStatus = z.output + +export const startGitHubSearchSetupResponseSchema = z.object({ + success: z.literal(true), + url: z.string().url().max(8192), +}) +export type StartGitHubSearchSetupResponse = z.output +export const startGitHubSearchSetupBodySchema = gitHubSearchSetupScopeSchema.extend({ + intent: z.literal('install').optional(), +}) +export type StartGitHubSearchSetupBody = z.input +export const startGitHubSearchSetupContract = defineRouteContract({ + method: 'POST', + path: '/api/knowledge/github/setup', + body: startGitHubSearchSetupBodySchema, + response: { mode: 'json', schema: startGitHubSearchSetupResponseSchema }, +}) + +export const readGitHubSearchSetupResponseSchema = z.object({ + success: z.literal(true), + data: gitHubSearchSetupStatusSchema, +}) +export type ReadGitHubSearchSetupResponse = z.output +export const readGitHubSearchSetupContract = defineRouteContract({ + method: 'GET', + path: '/api/knowledge/github/setup', + query: gitHubSearchSetupScopeSchema, + response: { mode: 'json', schema: readGitHubSearchSetupResponseSchema }, +}) + +export const cancelGitHubSearchSetupResponseSchema = z.object({ success: z.literal(true) }) +export type CancelGitHubSearchSetupResponse = z.output +export const cancelGitHubSearchSetupContract = defineRouteContract({ + method: 'DELETE', + path: '/api/knowledge/github/setup', + body: gitHubSearchSetupScopeSchema, + response: { mode: 'json', schema: cancelGitHubSearchSetupResponseSchema }, +}) + +export const selectGitHubSearchSetupBodySchema = gitHubSearchSetupScopeSchema.extend({ + action: z.discriminatedUnion('kind', [ + z.object({ kind: z.literal('select'), installationId: githubInstallationIdSchema }).strict(), + z.object({ kind: z.literal('install') }).strict(), + ]), +}) +export type SelectGitHubSearchSetupBody = z.input +export const selectGitHubSearchSetupContract = defineRouteContract({ + method: 'POST', + path: '/api/knowledge/github/setup/selection', + body: selectGitHubSearchSetupBodySchema, + response: { mode: 'json', schema: startGitHubSearchSetupResponseSchema }, +}) + +export const continueGitHubSearchSetupQuerySchema = gitHubSearchSetupScopeSchema.extend({ + oauth: z.string().min(1).max(100).optional(), +}) +export type ContinueGitHubSearchSetupQuery = z.input +export const continueGitHubSearchSetupContract = defineRouteContract({ + method: 'GET', + path: '/api/knowledge/github/setup/continue', + query: continueGitHubSearchSetupQuerySchema, + response: { mode: 'redirect' }, +}) + +export const completeGitHubSearchSetupQuerySchema = z.object({ + state: z.string().uuid('GitHub setup state must be a UUID'), + installation_id: githubInstallationIdSchema.optional(), + setup_action: z.string().min(1).max(40).optional(), +}) +export type CompleteGitHubSearchSetupQuery = z.input +export const completeGitHubSearchSetupContract = defineRouteContract({ + method: 'GET', + path: '/api/knowledge/github/setup/callback', + query: completeGitHubSearchSetupQuerySchema, + response: { mode: 'redirect' }, +}) diff --git a/apps/sim/lib/api/contracts/organization-accounts.test.ts b/apps/sim/lib/api/contracts/organization-accounts.test.ts index 6e9a19cfb03..6dd57746793 100644 --- a/apps/sim/lib/api/contracts/organization-accounts.test.ts +++ b/apps/sim/lib/api/contracts/organization-accounts.test.ts @@ -3,8 +3,39 @@ import { describe, expect, it } from 'vitest' import { addOrganizationAccountMcpProviderContract, listOrganizationAccountPeopleContract, + reconnectPersonalOrganizationAccountContract, + startOrganizationAccountConnectionContract, } from '@/lib/api/contracts/organization-accounts' +describe.each([ + startOrganizationAccountConnectionContract, + reconnectPersonalOrganizationAccountContract, +])('account connection response $path', (contract) => { + const invitationLink = 'https://sim.test/credential-groups/enroll/fixture-token' + const authorizationUrl = + 'https://sim.test/api/credential-groups/enroll/fixture-token/oauth/option-1?returnTo=search' + + it('accepts older enrollment-only responses and the additive direct OAuth URL', () => { + expect(contract.response.schema.parse({ invitationLink })).toEqual({ invitationLink }) + expect(contract.response.schema.parse({ invitationLink, authorizationUrl })).toEqual({ + invitationLink, + authorizationUrl, + }) + }) + + it('rejects malformed or oversized direct URLs', () => { + expect( + contract.response.schema.safeParse({ invitationLink, authorizationUrl: '/relative' }).success + ).toBe(false) + expect( + contract.response.schema.safeParse({ + invitationLink, + authorizationUrl: `https://sim.test/${'x'.repeat(8192)}`, + }).success + ).toBe(false) + }) +}) + describe('organization MCP provider creation contract', () => { const schema = addOrganizationAccountMcpProviderContract.body if (!schema) throw new Error('MCP provider creation requires a body contract') diff --git a/apps/sim/lib/api/contracts/organization-accounts.ts b/apps/sim/lib/api/contracts/organization-accounts.ts index def09b7e201..7b08459e082 100644 --- a/apps/sim/lib/api/contracts/organization-accounts.ts +++ b/apps/sim/lib/api/contracts/organization-accounts.ts @@ -19,6 +19,7 @@ import { defineRouteContract } from '@/lib/api/contracts/types' import { ORGANIZATION_ACCOUNT_INDEXING_SOURCE_LIMIT, ORGANIZATION_ACCOUNT_WORKSPACE_LIMIT, + ORGANIZATION_VIEWER_ACCOUNT_LIMIT, } from '@/lib/credential-groups/limits' const organizationAccountsParamsSchema = z.object({ id: organizationIdSchema }) @@ -41,6 +42,19 @@ export const getOrganizationAccountsContract = defineRouteContract({ availableProviders: z.array(credentialGroupProviderSchema), canManage: z.boolean(), indexingAvailable: z.boolean(), + viewerAccounts: z + .array( + z.object({ + credentialId: z.string().min(1).max(128), + displayName: z.string().max(512), + providerId: z.string().min(1).max(128), + groupId: z.string().min(1).max(128), + optionId: z.string().min(1).max(128), + status: z.enum(['active', 'needs_reauth']), + }) + ) + .max(ORGANIZATION_VIEWER_ACCOUNT_LIMIT) + .optional(), }), }, }) @@ -58,12 +72,20 @@ export const updateOrganizationAccountsContract = defineRouteContract({ body: updateCredentialGroupBodySchema, response: { mode: 'json', schema: organizationAccountsResponseSchema }, }) +export const organizationAccountConnectionResponseSchema = z.object({ + invitationLink: z.string().url(), + authorizationUrl: z.string().url().max(8192).optional(), +}) +export type OrganizationAccountConnectionResponse = z.output< + typeof organizationAccountConnectionResponseSchema +> + export const startOrganizationAccountConnectionContract = defineRouteContract({ method: 'POST', path: '/api/organizations/[id]/connected-accounts/connect', params: organizationAccountsParamsSchema, body: z.object({ optionId: z.string().min(1, 'Account option is required').max(128) }).strict(), - response: { mode: 'json', schema: z.object({ invitationLink: z.string().url() }) }, + response: { mode: 'json', schema: organizationAccountConnectionResponseSchema }, }) export const startOrganizationSlackConfigurationContract = defineRouteContract({ @@ -313,7 +335,7 @@ export const reconnectPersonalOrganizationAccountContract = defineRouteContract( method: 'POST', path: '/api/users/me/organization-accounts/[credentialId]/reconnect', params: z.object({ credentialId: z.string().min(1).max(128) }), - response: { mode: 'json', schema: z.object({ invitationLink: z.string().url() }) }, + response: { mode: 'json', schema: organizationAccountConnectionResponseSchema }, }) export const disconnectPersonalOrganizationAccountContract = defineRouteContract({ method: 'DELETE', diff --git a/apps/sim/lib/credential-groups/application/organization-access.test.ts b/apps/sim/lib/credential-groups/application/organization-access.test.ts index f86ea9f7790..32c4897be29 100644 --- a/apps/sim/lib/credential-groups/application/organization-access.test.ts +++ b/apps/sim/lib/credential-groups/application/organization-access.test.ts @@ -10,6 +10,8 @@ const mocks = vi.hoisted(() => ({ setup: vi.fn(), write: vi.fn(), policy: vi.fn(), + accountsGroup: vi.fn(), + invite: vi.fn(), })) vi.mock('@/lib/credential-groups/scoped-availability', () => ({ isScopedCredentialGroupsAvailable: mocks.available, @@ -25,14 +27,17 @@ vi.mock('@/lib/permission-groups/resolve.server', () => ({ })) vi.mock('@/lib/credential-groups/service', () => ({ ensureWorkspaceAccountsGroup: vi.fn(), - getOrganizationAccountsGroup: vi.fn(), + getOrganizationAccountsGroup: mocks.accountsGroup, updateCredentialGroup: vi.fn(), })) vi.mock('@/lib/credential-groups/provider-availability', () => ({ listConfiguredCredentialGroupProviders: vi.fn(), })) +vi.mock('@/lib/knowledge/access/availability', () => ({ + isKnowledgeMemberAccessAvailable: vi.fn().mockResolvedValue(true), +})) vi.mock('@/lib/credential-groups/self-enrollment', () => ({ - createViewerCredentialGroupEnrollment: vi.fn(), + createViewerCredentialGroupEnrollment: mocks.invite, })) vi.mock('@/lib/resource-policies/repository', () => ({ requireResourcePolicy: mocks.policy, @@ -40,10 +45,15 @@ vi.mock('@/lib/resource-policies/repository', () => ({ ResourcePolicyRevisionConflictError: class extends Error {}, })) +import { OrchestrationError } from '@/lib/core/orchestration/types' import { getOrganizationAccountWorkspaceAccess, updateOrganizationAccountWorkspaceAccess, } from '@/lib/credential-groups/application/organization-access' +import { + getOrganizationAccountsSettings, + startOrganizationAccountConnection, +} from '@/lib/credential-groups/application/organization-accounts' import { buildOrganizationAccountAccessPolicy } from '@/lib/credential-groups/application/workspace-access-policy' import { ResourcePolicyRevisionConflictError } from '@/lib/resource-policies/repository' @@ -84,6 +94,73 @@ describe('organization workspace sharing administration', () => { } ) + it('returns only the acting member account metadata for the canonical organization group, even when providers are disabled', async () => { + queueTableRows(schemaMock.member, [{ role: 'member' }]) + mocks.accountsGroup.mockResolvedValue({ id: 'group-1', status: 'disabled', options: [] }) + const account = { + credentialId: 'credential-1', + displayName: 'My GitHub', + providerId: 'github-repositories', + groupId: 'group-1', + optionId: 'option-1', + status: 'needs_reauth', + } + queueTableRows(schemaMock.credential, [account]) + const result = await getOrganizationAccountsSettings.execute({ + principal, + input: { organizationId: 'org-1' }, + }) + expect(result.viewerAccounts).toEqual([account]) + expect(result.canManage).toBe(false) + expect(eq).toHaveBeenCalledWith(schemaMock.credentialGroup.id, 'group-1') + expect(eq).toHaveBeenCalledWith(schemaMock.credentialGroupEnrollment.userId, 'admin-user') + expect(eq).toHaveBeenCalledWith(schemaMock.credential.organizationId, 'org-1') + }) + + it('starts an active account through the existing enrollment OAuth boundary', async () => { + queueTableRows(schemaMock.member, [{ role: 'member' }]) + mocks.accountsGroup.mockResolvedValue({ + id: 'group-1', + status: 'active', + options: [{ id: 'option-1', status: 'active' }], + }) + mocks.invite.mockResolvedValue({ + invitationLink: 'https://sim.test/credential-groups/enroll/fixture-token', + }) + await expect( + startOrganizationAccountConnection.execute({ + principal, + input: { organizationId: 'org-1', optionId: 'option-1' }, + }) + ).resolves.toEqual({ + invitationLink: + 'https://sim.test/credential-groups/enroll/fixture-token?optionId=option-1&returnTo=search', + authorizationUrl: + 'https://sim.test/api/credential-groups/enroll/fixture-token/oauth/option-1?returnTo=search', + }) + expect(mocks.invite).toHaveBeenCalledExactlyOnceWith({ + organizationId: 'org-1', + userId: 'admin-user', + credentialGroupId: 'group-1', + }) + }) + + it('does not issue a direct authorization link when enrollment access was revoked', async () => { + queueTableRows(schemaMock.member, [{ role: 'member' }]) + mocks.accountsGroup.mockResolvedValue({ + id: 'group-1', + status: 'active', + options: [{ id: 'option-1', status: 'active' }], + }) + mocks.invite.mockRejectedValue(new OrchestrationError('forbidden', 'Enrollment revoked')) + await expect( + startOrganizationAccountConnection.execute({ + principal, + input: { organizationId: 'org-1', optionId: 'option-1' }, + }) + ).rejects.toMatchObject({ code: 'forbidden' }) + }) + it('uses the routed org and checks every workspace before granting access', async () => { queueTableRows(schemaMock.member, [{ role: 'admin' }]) queueTableRows(schemaMock.workspace, [{ id: 'workspace-1' }]) diff --git a/apps/sim/lib/credential-groups/application/organization-accounts.ts b/apps/sim/lib/credential-groups/application/organization-accounts.ts index ccf1782f170..2eabfa67f15 100644 --- a/apps/sim/lib/credential-groups/application/organization-accounts.ts +++ b/apps/sim/lib/credential-groups/application/organization-accounts.ts @@ -1,4 +1,6 @@ import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' +import { credentialGroup as credentialGroupTable } from '@sim/db/schema' +import { eq } from 'drizzle-orm' import type { OperationUseCase } from '@/lib/core/application/operation' import { authorizeOrganizationOperation, @@ -11,6 +13,7 @@ import { import { OrchestrationError } from '@/lib/core/orchestration/types' import { validateUpdateCredentialGroupInput } from '@/lib/credential-groups/application/validation' import { loadScopedAccountsCredentialListContext } from '@/lib/credential-groups/credentials' +import { createCredentialGroupOAuthStartUrl } from '@/lib/credential-groups/enrollment-links' import { CredentialGroupEnrollmentError } from '@/lib/credential-groups/enrollments' import { ManagedMcpConnectorError } from '@/lib/credential-groups/managed-mcp-service' import { requireOrganizationAccountsSetup } from '@/lib/credential-groups/organization-setup' @@ -26,6 +29,7 @@ import type { CredentialGroupOptionInput, UpdateCredentialGroupInput, } from '@/lib/credential-groups/types' +import { listViewerOrganizationAccounts } from '@/lib/credential-groups/viewer-accounts' import { isKnowledgeMemberAccessAvailable } from '@/lib/knowledge/access/availability' export const organizationAccountOperations = { @@ -129,8 +133,16 @@ export function defineOrganizationAccountsUseCase< export const getOrganizationAccountsSettings = defineOrganizationAccountsUseCase({ operation: organizationAccountOperations.read, async execute({ context }) { + const credentialGroup = await getOrganizationAccountsGroup(context.organizationId) return { - credentialGroup: await getOrganizationAccountsGroup(context.organizationId), + credentialGroup, + viewerAccounts: credentialGroup + ? await listViewerOrganizationAccounts({ + organizationId: context.organizationId, + userId: context.userId, + matching: eq(credentialGroupTable.id, credentialGroup.id), + }) + : [], availableProviders: listConfiguredCredentialGroupProviders(), canManage: context.role === 'owner' || context.role === 'admin', indexingAvailable: await isKnowledgeMemberAccessAvailable({ @@ -219,6 +231,13 @@ export const startOrganizationAccountConnection = defineOrganizationAccountsUseC const url = new URL(invitationLink) url.searchParams.set('optionId', input.optionId) url.searchParams.set('returnTo', 'search') - return { invitationLink: url.toString() } + return { + invitationLink: url.toString(), + authorizationUrl: createCredentialGroupOAuthStartUrl({ + invitationLink, + optionId: input.optionId, + returnTo: 'search', + }), + } }, }) diff --git a/apps/sim/lib/credential-groups/application/personal-organization-accounts.test.ts b/apps/sim/lib/credential-groups/application/personal-organization-accounts.test.ts index 867ad51c839..df7d9fe70e3 100644 --- a/apps/sim/lib/credential-groups/application/personal-organization-accounts.test.ts +++ b/apps/sim/lib/credential-groups/application/personal-organization-accounts.test.ts @@ -54,7 +54,9 @@ describe('personal organization contributions', () => { vi.clearAllMocks() resetDbChainMock() mocks.available.mockResolvedValue(true) - mocks.invite.mockResolvedValue({ invitationLink: 'https://sim.test/enroll/token' }) + mocks.invite.mockResolvedValue({ + invitationLink: 'https://sim.test/credential-groups/enroll/fixture-token', + }) }) it('lists only the stable signed-in identity without requiring org membership', async () => { @@ -79,9 +81,21 @@ describe('personal organization contributions', () => { const url = new URL(result.invitationLink) expect(url.searchParams.get('optionId')).toBe('gmail-option') expect(url.searchParams.get('returnTo')).toBe('accounts') + expect(result.authorizationUrl).toBe( + 'https://sim.test/api/credential-groups/enroll/fixture-token/oauth/gmail-option?returnTo=accounts' + ) expect(dbChainMockFns.from).not.toHaveBeenCalledWith(schemaMock.member) }) + it('keeps the enrollment page for an MCP contribution', async () => { + queueTableRows(schemaMock.credential, [row]) + await expect( + reconnectPersonalOrganizationAccount.execute({ principal, input }) + ).resolves.toEqual({ + invitationLink: 'https://sim.test/credential-groups/enroll/fixture-token', + }) + }) + it('refuses disconnect of another contributor’s account', async () => { queueTableRows(schemaMock.credential, []) await expect( diff --git a/apps/sim/lib/credential-groups/application/personal-organization-accounts.ts b/apps/sim/lib/credential-groups/application/personal-organization-accounts.ts index ab7af64f16a..afd6096403a 100644 --- a/apps/sim/lib/credential-groups/application/personal-organization-accounts.ts +++ b/apps/sim/lib/credential-groups/application/personal-organization-accounts.ts @@ -14,6 +14,7 @@ import { and, asc, eq, gt, inArray, isNotNull } from 'drizzle-orm' import { defineOperation } from '@/lib/core/application' import { OrchestrationError } from '@/lib/core/orchestration/types' import { sameResourceScopeCondition } from '@/lib/core/resource-scope.server' +import { createCredentialGroupOAuthStartUrl } from '@/lib/credential-groups/enrollment-links' import { lockCredentialGroupEnrollmentLifecycle } from '@/lib/credential-groups/enrollments' import { isScopedCredentialGroupsAvailable } from '@/lib/credential-groups/scoped-availability' import { createViewerCredentialGroupEnrollment } from '@/lib/credential-groups/self-enrollment' @@ -152,7 +153,18 @@ export const reconnectPersonalOrganizationAccount = defineAuthorizedCredentialUs url.searchParams.set('optionId', account.optionId) url.searchParams.set('returnTo', 'accounts') } - return { invitationLink: url.toString() } + return { + invitationLink: url.toString(), + ...(account.type === 'managed_oauth' && account.optionId + ? { + authorizationUrl: createCredentialGroupOAuthStartUrl({ + invitationLink, + optionId: account.optionId, + returnTo: 'accounts', + }), + } + : {}), + } }, }) diff --git a/apps/sim/lib/credential-groups/enrollment-links.test.ts b/apps/sim/lib/credential-groups/enrollment-links.test.ts new file mode 100644 index 00000000000..69adab606af --- /dev/null +++ b/apps/sim/lib/credential-groups/enrollment-links.test.ts @@ -0,0 +1,51 @@ +/** @vitest-environment node */ +import { describe, expect, it } from 'vitest' +import { createCredentialGroupOAuthStartUrl } from '@/lib/credential-groups/enrollment-links' + +describe('credential group OAuth start links', () => { + it.each(['https://staging.sim.test', 'http://localhost:3015'])( + 'preserves the generated %s origin and encodes both path identifiers once', + (origin) => { + const token = 'fixture/token ?+%' + const optionId = 'option/with ?+%' + const url = new URL( + createCredentialGroupOAuthStartUrl({ + invitationLink: `${origin}/credential-groups/enroll/${encodeURIComponent(token)}?optionId=old&returnTo=accounts#old`, + optionId, + returnTo: 'search', + }) + ) + expect(url.origin).toBe(origin) + expect(url.pathname).toBe( + `/api/credential-groups/enroll/${encodeURIComponent(token)}/oauth/${encodeURIComponent(optionId)}` + ) + expect(url.search).toBe('?returnTo=search') + expect(url.hash).toBe('') + } + ) + + it('returns reconnects to account settings', () => { + expect( + createCredentialGroupOAuthStartUrl({ + invitationLink: 'https://sim.test/credential-groups/enroll/fixture-token', + optionId: 'option-1', + returnTo: 'accounts', + }) + ).toBe( + 'https://sim.test/api/credential-groups/enroll/fixture-token/oauth/option-1?returnTo=accounts' + ) + }) + + it.each(['/enroll/token', '/credential-groups/enroll/', '/credential-groups/enroll/token/extra'])( + 'rejects a noncanonical enrollment path %s', + (path) => { + expect(() => + createCredentialGroupOAuthStartUrl({ + invitationLink: `https://sim.test${path}`, + optionId: 'option-1', + returnTo: 'search', + }) + ).toThrow('Invalid credential group enrollment link') + } + ) +}) diff --git a/apps/sim/lib/credential-groups/enrollment-links.ts b/apps/sim/lib/credential-groups/enrollment-links.ts new file mode 100644 index 00000000000..1d7c76e2214 --- /dev/null +++ b/apps/sim/lib/credential-groups/enrollment-links.ts @@ -0,0 +1,16 @@ +/** Starts OAuth through the existing enrollment boundary, which owns authentication and PKCE. */ +export function createCredentialGroupOAuthStartUrl(input: { + invitationLink: string + optionId: string + returnTo: 'search' | 'accounts' +}): string { + const invitation = new URL(input.invitationLink) + const token = invitation.pathname.match(/^\/credential-groups\/enroll\/([^/]+)$/)?.[1] + if (!token) throw new Error('Invalid credential group enrollment link') + const url = new URL( + `/api/credential-groups/enroll/${encodeURIComponent(decodeURIComponent(token))}/oauth/${encodeURIComponent(input.optionId)}`, + invitation.origin + ) + url.searchParams.set('returnTo', input.returnTo) + return url.toString() +} diff --git a/apps/sim/lib/credential-groups/limits.ts b/apps/sim/lib/credential-groups/limits.ts index cad8c92b28b..60963f14139 100644 --- a/apps/sim/lib/credential-groups/limits.ts +++ b/apps/sim/lib/credential-groups/limits.ts @@ -1,4 +1,5 @@ export const CREDENTIAL_GROUP_MCP_SERVER_LIMIT = 50 +export const ORGANIZATION_VIEWER_ACCOUNT_LIMIT = 100 export const ORGANIZATION_ACCOUNT_WORKSPACE_LIMIT = 1000 export const CREDENTIAL_GROUP_WORKFLOW_ACCESS_LIMIT = 50 export const CREDENTIAL_GROUP_WORKFLOW_CATALOG_LIMIT = 500 diff --git a/apps/sim/lib/credential-groups/oauth-completion.ts b/apps/sim/lib/credential-groups/oauth-completion.ts index 31225358b61..1c732bdfc8c 100644 --- a/apps/sim/lib/credential-groups/oauth-completion.ts +++ b/apps/sim/lib/credential-groups/oauth-completion.ts @@ -4,9 +4,15 @@ export const CREDENTIAL_GROUP_OAUTH_FAILURE_MESSAGES = { expired: 'This connection attempt expired. Try connecting your account again.', denied: 'Authorization was canceled. Try connecting your account again.', account_mismatch: 'Choose the account matching your Sim email address.', + github_email_mismatch: + 'In GitHub Settings → Emails, add and verify the email address used for this Sim connection, then try again. A verified secondary email is supported.', + github_email_access_denied: + 'GitHub did not allow access to your email addresses. Ask an admin to check that the GitHub App has Email addresses: Read-only permission, then authorize the app again.', permissions_required: 'All requested permissions are required to connect this account.', configuration_changed: 'The connection settings changed. Try connecting your account again.', - rate_limited: 'Too many authorization attempts. Wait a few minutes and try again.', + rate_limited: 'Authorization is being rate limited. Wait a few minutes and try again.', + provider_unavailable: + 'The account provider could not verify your account right now. Try connecting again in a few minutes.', unavailable: 'This connection is unavailable. Try connecting your account again.', failed: 'Account authorization did not complete. Try connecting your account again.', } as const diff --git a/apps/sim/lib/credential-groups/oauth-state.ts b/apps/sim/lib/credential-groups/oauth-state.ts index edd588c1f37..97abfbb9663 100644 --- a/apps/sim/lib/credential-groups/oauth-state.ts +++ b/apps/sim/lib/credential-groups/oauth-state.ts @@ -44,7 +44,7 @@ interface StoredCredentialGroupOAuthAttempt { completionRedirect?: boolean connectionIntent?: CredentialGroupConnectionIntent completionId?: string - returnTo?: 'search' | 'accounts' + returnTo?: 'search' | 'accounts' | 'github-installation' nonceHash: string encryptedCodeVerifier?: string encryptedInvitationToken: string @@ -69,7 +69,7 @@ export interface CredentialGroupOAuthAttempt { completionRedirect?: boolean connectionIntent?: CredentialGroupConnectionIntent completionId?: string - returnTo?: 'search' | 'accounts' + returnTo?: 'search' | 'accounts' | 'github-installation' codeVerifier?: string invitationToken: string createdAt: number @@ -91,7 +91,7 @@ interface CreateCredentialGroupOAuthAttemptParams { completionRedirect?: boolean connectionIntent?: CredentialGroupConnectionIntent completionId?: string - returnTo?: 'search' | 'accounts' + returnTo?: 'search' | 'accounts' | 'github-installation' codeVerifier?: string invitationToken: string } @@ -147,7 +147,11 @@ function isStoredAttempt(value: unknown): value is StoredCredentialGroupOAuthAtt credentialGroupConnectionIntentSchema.safeParse(candidate.connectionIntent).success) && (candidate.returnTo === undefined || candidate.returnTo === 'search' || - candidate.returnTo === 'accounts') && + candidate.returnTo === 'accounts' || + (candidate.returnTo === 'github-installation' && + candidate.provider === 'github-repositories' && + typeof candidate.organizationId === 'string' && + typeof candidate.completionId === 'string')) && typeof candidate.nonceHash === 'string' && (candidate.encryptedCodeVerifier === undefined || typeof candidate.encryptedCodeVerifier === 'string') && @@ -166,6 +170,12 @@ export async function createCredentialGroupOAuthAttempt( ) { throw new Error('OAuth completion requires a valid correlation ID and completion redirect') } + if ( + params.returnTo === 'github-installation' && + (params.provider !== 'github-repositories' || !params.organizationId || !params.completionId) + ) { + throw new Error('GitHub installation OAuth requires an organization-bound setup attempt') + } if (params.connectionIntent) credentialGroupConnectionIntentSchema.parse(params.connectionIntent) const redis = requireRedis() const state = `${OAUTH_ATTEMPT_STATE_PREFIX}${generateId()}` diff --git a/apps/sim/lib/credential-groups/oauth.ts b/apps/sim/lib/credential-groups/oauth.ts index b905f373604..6699f6a7456 100644 --- a/apps/sim/lib/credential-groups/oauth.ts +++ b/apps/sim/lib/credential-groups/oauth.ts @@ -116,7 +116,7 @@ export async function startCredentialGroupOAuth( completionRedirect?: boolean connectionIntent?: CredentialGroupConnectionIntent completionId?: string - returnTo?: 'search' | 'accounts' + returnTo?: 'search' | 'accounts' | 'github-installation' } = {} ): Promise { if (!context.credentialOwnerId) throw new CredentialGroupInvitationUnavailableError() diff --git a/apps/sim/lib/credential-groups/provider-adapter.ts b/apps/sim/lib/credential-groups/provider-adapter.ts index 30cd9b36357..897bcf9282b 100644 --- a/apps/sim/lib/credential-groups/provider-adapter.ts +++ b/apps/sim/lib/credential-groups/provider-adapter.ts @@ -5,6 +5,7 @@ import type { CredentialGroupOAuthAttempt } from '@/lib/credential-groups/oauth- import type { CredentialGroupProvider } from '@/lib/credential-groups/providers' import type { DbOrTx } from '@/lib/db/types' import type { RefreshTokenResult } from '@/lib/oauth' +import type { OAuthIdentityVerificationError } from '@/lib/oauth/identity-error' export interface CredentialGroupProviderPolicy { provider: CredentialGroupProvider @@ -83,7 +84,8 @@ export class CredentialGroupProviderConfigurationError extends Error { export class CredentialGroupOAuthError extends Error { constructor( message: string, - readonly statusCode: 400 | 401 | 403 | 404 | 409 | 502 | 503 + readonly statusCode: 400 | 401 | 403 | 404 | 409 | 502 | 503, + readonly identityFailure?: OAuthIdentityVerificationError ) { super(message) this.name = 'CredentialGroupOAuthError' diff --git a/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts b/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts index f603f6def12..ac06e6f834b 100644 --- a/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts +++ b/apps/sim/lib/credential-groups/standard-oauth-provider.test.ts @@ -5,6 +5,7 @@ import { createHash } from 'node:crypto' import { beforeEach, describe, expect, it, vi } from 'vitest' import type { CredentialGroupOAuthContext } from '@/lib/credential-groups/enrollments' import type { CredentialGroupOAuthAttempt } from '@/lib/credential-groups/oauth-state' +import { OAuthIdentityVerificationError } from '@/lib/oauth/identity-error' const { mockGetToken, mockVerifyIdentity } = vi.hoisted(() => ({ mockGetToken: vi.fn(), @@ -237,6 +238,31 @@ describe('standard OAuth Credential Group provider', () => { ).rejects.toMatchObject({ statusCode: 403 }) }) + it.each([ + new OAuthIdentityVerificationError('email_mismatch', 'emails'), + new OAuthIdentityVerificationError('email_access_denied', 'emails', 403), + new OAuthIdentityVerificationError('provider_unavailable', 'profile', 503), + ])('preserves safe identity diagnostics through managed authorization: %s', async (failure) => { + mockVerifyIdentity.mockRejectedValueOnce(failure) + const context = buildContext() + const policy = await adapter.getPolicy(context.option, { + workspaceId: context.workspaceId, + credentialGroupId: context.credentialGroupId, + }) + await expect( + adapter.exchangeAndVerify({ + context, + attempt: buildAttempt(policy.scopeVersion), + code: 'code-1', + policy, + }) + ).rejects.toMatchObject({ + name: 'CredentialGroupOAuthError', + statusCode: 502, + identityFailure: failure, + }) + }) + it('uses the existing Atlassian callback and state-bound identity verification', async () => { const requiredScopes = ['read:me', 'read:jira-work', 'offline_access'] const context: CredentialGroupOAuthContext = { diff --git a/apps/sim/lib/credential-groups/standard-oauth-provider.ts b/apps/sim/lib/credential-groups/standard-oauth-provider.ts index 4a303538ec0..4492ad3812d 100644 --- a/apps/sim/lib/credential-groups/standard-oauth-provider.ts +++ b/apps/sim/lib/credential-groups/standard-oauth-provider.ts @@ -26,6 +26,7 @@ import { import type { CredentialGroupStandardOAuthProvider } from '@/lib/credential-groups/providers' import { getCredentialGroupProviderService } from '@/lib/credential-groups/providers' import { refreshOAuthToken } from '@/lib/oauth' +import { OAuthIdentityVerificationError } from '@/lib/oauth/identity-error' const OAUTH_DISCOVERY_TIMEOUT_MS = 10_000 const OAUTH_DISCOVERY_MAX_BYTES = 256 * 1024 @@ -332,10 +333,11 @@ export function createStandardOAuthCredentialGroupProviderAdapter( clientId: current.connector.clientId, expectedEmail: context.email, }) - } catch { + } catch (error) { throw new CredentialGroupOAuthError( `${service.name} returned an invalid identity token.`, - 502 + 502, + error instanceof OAuthIdentityVerificationError ? error : undefined ) } const nonceMatches = diff --git a/apps/sim/lib/credential-groups/viewer-accounts.ts b/apps/sim/lib/credential-groups/viewer-accounts.ts new file mode 100644 index 00000000000..00c99548a2d --- /dev/null +++ b/apps/sim/lib/credential-groups/viewer-accounts.ts @@ -0,0 +1,57 @@ +import { db } from '@sim/db' +import { credential, credentialGroup, credentialGroupEnrollment } from '@sim/db/schema' +import { and, eq, inArray, isNull, type SQL } from 'drizzle-orm' +import { resourceScopeCondition } from '@/lib/core/resource-scope.server' +import { ORGANIZATION_VIEWER_ACCOUNT_LIMIT } from '@/lib/credential-groups/limits' + +/** Own grant metadata remains manageable even while provider setup or enrollment is disabled. */ +export async function listViewerOrganizationAccounts(input: { + organizationId: string + userId: string + matching: SQL +}) { + const scope = { kind: 'organization', organizationId: input.organizationId } as const + const accounts = await db + .select({ + credentialId: credential.id, + displayName: credential.displayName, + status: credential.managedOauthStatus, + groupId: credentialGroup.id, + optionId: credential.credentialGroupOptionId, + providerId: credential.providerId, + }) + .from(credential) + .innerJoin( + credentialGroupEnrollment, + eq(credentialGroupEnrollment.id, credential.credentialGroupEnrollmentId) + ) + .innerJoin(credentialGroup, eq(credentialGroup.id, credentialGroupEnrollment.credentialGroupId)) + .where( + and( + resourceScopeCondition(credential, scope), + resourceScopeCondition(credentialGroup, scope), + eq(credentialGroupEnrollment.userId, input.userId), + eq(credential.type, 'managed_oauth'), + inArray(credential.managedOauthStatus, ['active', 'needs_reauth']), + isNull(credential.revokedAt), + input.matching + ) + ) + .limit(ORGANIZATION_VIEWER_ACCOUNT_LIMIT + 1) + if (accounts.length > ORGANIZATION_VIEWER_ACCOUNT_LIMIT) + throw new Error('Too many personal accounts for this organization') + return accounts.map((account) => { + if ( + !account.optionId || + !account.providerId || + (account.status !== 'active' && account.status !== 'needs_reauth') + ) + throw new Error('Invalid personal account metadata') + return { + ...account, + optionId: account.optionId, + providerId: account.providerId, + status: account.status, + } + }) +} diff --git a/apps/sim/lib/knowledge/__integration__/github-member.integration.ts b/apps/sim/lib/knowledge/__integration__/github-member.integration.ts index 31cd2ddc32e..133ebdce20d 100644 --- a/apps/sim/lib/knowledge/__integration__/github-member.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/github-member.integration.ts @@ -52,6 +52,7 @@ import { closeRedisConnection, getRedisClient } from '@/lib/core/config/redis' import { encryptSecret } from '@/lib/core/security/encryption' import { resetStorageMethod } from '@/lib/core/storage' import { compileCredentialGroupWorkflowAccessPolicy } from '@/lib/credential-groups/application/workflow-access-policy' +import { buildOrganizationAccountAccessPolicy } from '@/lib/credential-groups/application/workspace-access-policy' import { completeCredentialGroupEnrollment, getCredentialGroupOAuthContext, @@ -77,13 +78,17 @@ import { createKnowledgeAccessProvider } from '@/lib/knowledge/access/scope' import { subjectToken } from '@/lib/knowledge/access/tokens' import { KnowledgeDocumentNotReadyError } from '@/lib/knowledge/application/chunk-errors' import { listKnowledgeChunks } from '@/lib/knowledge/application/chunks' +import { + createKnowledgeConnector, + listKnowledgeConnectorDocuments, +} from '@/lib/knowledge/application/connectors' import { readKnowledgeDocument } from '@/lib/knowledge/application/documents' import { readIndexedKnowledgeDocument } from '@/lib/knowledge/application/read-indexed-document' import { searchKnowledge } from '@/lib/knowledge/application/search' import { readSearchSourceOverview } from '@/lib/knowledge/application/search-source-overview' import { listSearchSources } from '@/lib/knowledge/application/search-sources' import { grantKnowledgeConnectorCredentialAccess } from '@/lib/knowledge/connectors/member-access' -import { executeMemberSync } from '@/lib/knowledge/connectors/member-sync-engine' +import * as memberSyncEngine from '@/lib/knowledge/connectors/member-sync-engine' import { MEMBER_SUSPENDED_PURGE_DAYS, MEMBER_TOMBSTONE_PURGE_DAYS, @@ -120,6 +125,7 @@ interface RepositoryFixture { deniedStatus: 403 | 404 throttledReaders: Set throttledBlobReaders: Set + failedBlobs: Set truncated: boolean } @@ -186,6 +192,7 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { deniedStatus: 404, throttledReaders: new Set(), throttledBlobReaders: new Set(), + failedBlobs: new Set(), truncated: false, } repositories.set(name, value) @@ -250,18 +257,22 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { if (url.pathname === '/app/installations/42/access_tokens') { expect(request.method).toBe('POST') const body = await request.json() - expect(body).toMatchObject({ - permissions: { contents: 'read', metadata: 'read' }, - }) - expect(body.repository_ids).toHaveLength(1) - const repositoryId = body.repository_ids[0] + const contentToken = body.permissions.contents === 'read' + expect(body.permissions).toEqual( + contentToken ? { contents: 'read', metadata: 'read' } : { metadata: 'read' } + ) + if (contentToken) expect(body.repository_ids).toHaveLength(1) + else expect(body.repositories).toHaveLength(1) + const repositoryId = contentToken + ? body.repository_ids[0] + : repositories.get(body.repositories[0])?.id expect( [...repositories.values()].some( (repository) => repository.id === repositoryId && repository.installed ) ).toBe(true) return Response.json({ - token: `ghs_fixture_installation_${repositoryId}`, + token: `ghs_fixture_${contentToken ? 'installation' : 'metadata'}_${repositoryId}`, expires_at: new Date(Date.now() + 60 * 60_000).toISOString(), permissions: body.permissions, repositories: [{ id: repositoryId }], @@ -275,7 +286,10 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { return Response.json(installation()) } if (request.method !== 'GET') throw new Error(`Unexpected GitHub method: ${request.method}`) - const installationRepository = bearer.match(/^ghs_fixture_installation_(\d+)$/)?.[1] + const installationRepository = bearer.match( + /^ghs_fixture_(?:installation|metadata)_(\d+)$/ + )?.[1] + const metadataToken = bearer.startsWith('ghs_fixture_metadata_') const installationToken = Boolean(installationRepository) const member = enrolled.members.find((candidate) => [tokenFor(candidate.userId), `${tokenFor(candidate.userId)}_refreshed`].some( @@ -330,6 +344,13 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { private: !source.public, default_branch: source.defaultBranch, }) + expect(metadataToken).toBe(false) + if (match[2].startsWith('/branches/')) { + const branch = decodeURIComponent(match[2].slice('/branches/'.length)) + return branch === source.defaultBranch + ? Response.json({ name: branch, commit: { sha: shaFor(branch) }, protected: false }) + : Response.json({ message: 'Not Found' }, { status: 404 }) + } if (match[2].startsWith('/git/ref/heads/')) { referenceObserved?.(match[1]) if (source.stallRef) @@ -377,6 +398,12 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { { status: 403 } ) const sha = decodeURIComponent(match[2].slice('/git/blobs/'.length)) + if (source.failedBlobs.has(sha)) { + return Response.json( + { message: 'Fixture provider unavailable' }, + { status: 503, headers: { 'Retry-After': '3600' } } + ) + } const content = [...source.files.values(), ...source.symlinks.values()].find( (value) => shaFor(value) === sha ) @@ -547,6 +574,7 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { await db.delete(workspace).where(eq(workspace.id, ids.workspaceId)) await db.delete(organization).where(eq(organization.id, ids.organizationId)) await db.delete(user).where(inArray(user.id, [ids.aliceId, ids.bobId])) + vi.restoreAllMocks() vi.unstubAllGlobals() } }) @@ -593,7 +621,7 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { .update(knowledgeConnectorMember) .set({ nextAttemptAt: new Date(0) }) .where(eq(knowledgeConnectorMember.connectorId, connectorId)) - return executeMemberSync(connectorId, { + return memberSyncEngine.executeMemberSync(connectorId, { billingAttribution: billing, forceContentRefresh, }) @@ -644,7 +672,7 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { } } - it('indexes an organization installation once and denies live user, app, and org revocations before search or reads', async () => { + async function useOrganizationInstallation() { organizationSource = true Object.assign(env, { GITHUB_APP_ID: '1', @@ -663,6 +691,19 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { .update(credentialGroup) .set({ workspaceId: null, organizationId: ids.organizationId }) .where(eq(credentialGroup.id, enrolled.groupId)) + await db + .update(resourcePolicy) + .set({ + workspaceId: null, + organizationId: ids.organizationId, + document: buildOrganizationAccountAccessPolicy(enrolled.groupId, []), + }) + .where( + and( + eq(resourcePolicy.resourceType, 'credential_group'), + eq(resourcePolicy.resourceId, enrolled.groupId) + ) + ) await db .update(credential) .set({ workspaceId: null, organizationId: ids.organizationId }) @@ -719,6 +760,231 @@ describe('fixture-backed GitHub member search in PostgreSQL', () => { actorUserId: ids.aliceId, organizationId: ids.organizationId, }) + return installationCredentialId + } + + it('reuses connected organization members for a later installation source without another enrollment', async () => { + const installationCredentialId = await useOrganizationInstallation() + expect((await sync()).error).toBeUndefined() + const [shared] = await rows() + const enrollmentsBefore = await db + .select({ + id: credentialGroupEnrollment.id, + userId: credentialGroupEnrollment.userId, + status: credentialGroupEnrollment.status, + }) + .from(credentialGroupEnrollment) + .where(eq(credentialGroupEnrollment.credentialGroupId, enrolled.groupId)) + .orderBy(credentialGroupEnrollment.id) + const credentialsBefore = await db + .select({ + id: credential.id, + enrollmentId: credential.credentialGroupEnrollmentId, + subjectId: credential.providerSubjectId, + }) + .from(credential) + .where(eq(credential.credentialGroupOptionId, enrolled.optionId)) + .orderBy(credential.id) + const later = repository('later', [ids.aliceId]) + const input = { + knowledgeBaseId: ids.knowledgeBaseId, + assertedOrganizationId: ids.organizationId, + connectorType: 'github', + accessMode: 'members' as const, + credentialId: installationCredentialId, + sourceConfig: { repository: 'fixture/later' }, + syncIntervalMinutes: 0, + } + await expect( + createKnowledgeConnector.execute({ principal: actor(ids.bobId), input }) + ).rejects.toThrow() + const dispatchedSync = vi.spyOn(memberSyncEngine, 'executeMemberSync') + const { connector } = await createKnowledgeConnector.execute({ + principal: actor(ids.aliceId), + input, + }) + try { + expect(dispatchedSync).toHaveBeenCalledExactlyOnceWith(connector.id, expect.any(Object)) + expect((await dispatchedSync.mock.results[0].value).error).toBeUndefined() + } finally { + dispatchedSync.mockRestore() + } + const connectorId = connector.id + expect(connector).toMatchObject({ + credentialGroupId: enrolled.groupId, + credentialGroupOptionId: enrolled.optionId, + sourceConfig: { repository: 'fixture/later', githubRepositoryId: String(later.id) }, + }) + const [current] = await db + .select() + .from(knowledgeConnector) + .where(eq(knowledgeConnector.id, connectorId)) + expect(current).toMatchObject({ memberSyncStatus: 'idle' }) + expect(current.lastMemberSyncAt).not.toBeNull() + const memberships = await db + .select() + .from(knowledgeConnectorMember) + .where(eq(knowledgeConnectorMember.connectorId, connectorId)) + expect(memberships).toHaveLength(2) + expect(memberships.map((row) => row.credentialId).sort()).toEqual( + credentialsBefore.map((row) => row.id).sort() + ) + expect( + await db + .select({ + id: credentialGroupEnrollment.id, + userId: credentialGroupEnrollment.userId, + status: credentialGroupEnrollment.status, + }) + .from(credentialGroupEnrollment) + .where(eq(credentialGroupEnrollment.credentialGroupId, enrolled.groupId)) + .orderBy(credentialGroupEnrollment.id) + ).toEqual(enrollmentsBefore) + expect( + await db + .select({ + id: credential.id, + enrollmentId: credential.credentialGroupEnrollmentId, + subjectId: credential.providerSubjectId, + }) + .from(credential) + .where(eq(credential.credentialGroupOptionId, enrolled.optionId)) + .orderBy(credential.id) + ).toEqual(credentialsBefore) + const [indexed] = await rows(connectorId) + expect(await search(actor(ids.aliceId))).toEqual([shared.id, indexed.id].sort()) + expect(await search(actor(ids.bobId))).toEqual([shared.id]) + await assertAccess(actor(ids.aliceId), indexed, true) + await assertAccess(actor(ids.bobId), indexed, false) + const indexedRead = (userId: string) => + readIndexedKnowledgeDocument.execute({ + principal: actor(userId), + input: { + organizationId: ids.organizationId, + target: { kind: 'id', documentId: indexed.id }, + limit: 10, + resultSecretRegistry: new ResolvedSecretTraceRegistry(), + }, + }) + expect( + (await indexedRead(ids.aliceId)).chunks?.map((chunk) => chunk.content).join('\n') + ).toContain('Orion later') + await expect(indexedRead(ids.bobId)).rejects.toThrow('Document not found') + /** Organization cache bytes are internal; members read through the authorized Search operation. */ + await expect( + downloadFileFromUrl(indexed.fileUrl, { userId: ids.aliceId, knowledgeAccess: 'user' }) + ).rejects.toThrow('Access denied') + await expect( + downloadFileFromUrl(indexed.fileUrl, { userId: ids.bobId, knowledgeAccess: 'user' }) + ).rejects.toThrow('Access denied') + for (const userId of [ids.aliceId, ids.bobId]) { + const { sources } = await listSearchSources.execute({ + principal: actor(userId), + input: { organizationId: ids.organizationId, connectorId }, + }) + expect(sources).toMatchObject([ + { + connectorId, + viewerMembership: 'connected', + viewerDocumentCount: userId === ids.aliceId ? 1 : 0, + }, + ]) + } + expect( + requests + .filter((entry) => entry.path.startsWith('/repos/fixture/later/git/blobs/')) + .map((entry) => entry.userId) + ).toEqual(['installation']) + }) + + it('keeps actual skips, legacy skips, and provider failures distinct in authorized lists and source counts', async () => { + await useOrganizationInstallation() + const source = repositories.get('shared')! + source.readers.delete(ids.bobId) + source.files.set('empty.txt', '') + source.files.set('image.png', 'binary\0contents') + expect((await sync()).error).toBeUndefined() + const initial = await rows() + const empty = initial.find((row) => row.externalId === 'empty.txt')! + const binary = initial.find((row) => row.externalId === 'image.png')! + expect(empty).toMatchObject({ processingStatus: 'failed', storageKey: null }) + expect(binary).toMatchObject({ processingStatus: 'failed', storageKey: null }) + expect(empty.contentHash).not.toBeNull() + await db.update(document).set({ processingStatus: 'failed' }).where(eq(document.id, empty.id)) + const summary = async (userId: string) => + ( + await listSearchSources.execute({ + principal: actor(userId), + input: { organizationId: ids.organizationId, connectorId: enrolled.connectorId }, + }) + ).sources[0] + expect(await summary(ids.aliceId)).toMatchObject({ + viewerDocumentCount: 1, + viewerFailedDocumentCount: 0, + hasSyncError: false, + }) + source.files.set('unavailable.txt', 'Orion content whose blob cannot be fetched.') + source.failedBlobs.add(shaFor(source.files.get('unavailable.txt')!)) + await sync() + await db.update(document).set({ processingStatus: 'failed' }).where(eq(document.id, empty.id)) + const failed = (await rows()).find((row) => row.externalId === 'unavailable.txt')! + expect(failed).toMatchObject({ + processingStatus: 'failed', + storageKey: null, + contentHash: null, + }) + for (const userId of [ids.aliceId, ids.bobId]) { + const provider = createKnowledgeAccessProvider(actor(userId), { + organizationId: ids.organizationId, + knowledgeBaseIds: [ids.knowledgeBaseId], + }) + const listed = await getDocuments(ids.knowledgeBaseId, {}, 'github-skip-outcomes', provider) + expect(listed.pagination.total).toBe(userId === ids.aliceId ? 4 : 0) + if (userId === ids.aliceId) { + expect(listed.documents).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: empty.id, + processingStatus: 'failed', + processingOutcome: 'skipped', + }), + expect.objectContaining({ + id: binary.id, + processingStatus: 'failed', + processingOutcome: 'skipped', + }), + expect.objectContaining({ id: failed.id, processingStatus: 'failed' }), + ]) + ) + } + for (const filter of ['failed', 'skipped'] as const) { + const outcomes = await listKnowledgeConnectorDocuments.execute({ + principal: actor(userId), + input: { + connectorId: enrolled.connectorId, + knowledgeBaseId: ids.knowledgeBaseId, + filter, + }, + }) + expect(outcomes.counts).toMatchObject({ + failed: userId === ids.aliceId ? 1 : 0, + skipped: userId === ids.aliceId ? 2 : 0, + }) + expect(outcomes.documents.map((row) => row.id).sort()).toEqual( + userId === ids.aliceId + ? (filter === 'failed' ? [failed.id] : [empty.id, binary.id]).sort() + : [] + ) + } + expect(await summary(userId)).toMatchObject({ + viewerDocumentCount: userId === ids.aliceId ? 1 : 0, + viewerFailedDocumentCount: userId === ids.aliceId ? 1 : 0, + }) + } + }) + + it('indexes an organization installation once and denies live user, app, and org revocations before search or reads', async () => { + const installationCredentialId = await useOrganizationInstallation() const unrelatedSources = Array.from({ length: 105 }, () => generateId()) await db.insert(knowledgeConnector).values( unrelatedSources.map((id) => ({ diff --git a/apps/sim/lib/knowledge/__integration__/organization-search-overview.integration.ts b/apps/sim/lib/knowledge/__integration__/organization-search-overview.integration.ts index b93f9ca5709..1ec01b7aff5 100644 --- a/apps/sim/lib/knowledge/__integration__/organization-search-overview.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/organization-search-overview.integration.ts @@ -153,7 +153,14 @@ beforeEach(async () => { .where(eq(knowledgeConnectorMember.id, memberId)) await db .update(document) - .set({ processingStatus: 'completed', enabled: true, userExcluded: false }) + .set({ + processingStatus: 'completed', + enabled: true, + userExcluded: false, + contentHash: null, + storageKey: null, + fileUrl: 'https://fixture.test/private', + }) .where(eq(document.id, documentId)) }) @@ -276,9 +283,44 @@ describe('organization operational overview with real SQL', () => { }) expect(await provider('gmail')).toMatchObject({ status: 'active' }) }) - it('reports inaccessible processing failures without exposing documents, and ignores excluded failures', async () => { - await db.update(document).set({ processingStatus: 'failed' }).where(eq(document.id, documentId)) - expect(await provider('google_drive')).toMatchObject({ status: 'needs_attention' }) + it('ignores intentional skips while reporting inaccessible source and indexing failures', async () => { + await db + .update(document) + .set({ processingStatus: 'failed', contentHash: 'immutable-sha', fileUrl: '' }) + .where(eq(document.id, documentId)) + expect(await provider('google_drive')).toMatchObject({ + status: 'active', + issue: null, + isSyncing: false, + }) + await db + .update(knowledgeConnector) + .set({ lastSyncError: 'previous sync failed' }) + .where(eq(knowledgeConnector.id, driveId)) + expect(await provider('google_drive')).toMatchObject({ + status: 'needs_attention', + issue: 'sync_failed', + isSyncing: false, + }) + await db + .update(knowledgeConnector) + .set({ lastSyncError: null }) + .where(eq(knowledgeConnector.id, driveId)) + await db.update(document).set({ contentHash: null }).where(eq(document.id, documentId)) + expect(await provider('google_drive')).toMatchObject({ + status: 'needs_attention', + issue: 'document_indexing_failed', + isSyncing: false, + }) + await db + .update(document) + .set({ contentHash: 'immutable-sha', storageKey: 'fixture-retained-artifact' }) + .where(eq(document.id, documentId)) + expect(await provider('google_drive')).toMatchObject({ + status: 'needs_attention', + issue: 'document_indexing_failed', + isSyncing: false, + }) await db.update(document).set({ userExcluded: true }).where(eq(document.id, documentId)) expect(await provider('google_drive')).toMatchObject({ status: 'active' }) await db diff --git a/apps/sim/lib/knowledge/__integration__/search-source-progress.integration.ts b/apps/sim/lib/knowledge/__integration__/search-source-progress.integration.ts index 5a82344815a..8e1cc0d70b6 100644 --- a/apps/sim/lib/knowledge/__integration__/search-source-progress.integration.ts +++ b/apps/sim/lib/knowledge/__integration__/search-source-progress.integration.ts @@ -1,6 +1,7 @@ import { db } from '@sim/db' import { document, + embedding, knowledgeBase, knowledgeConnector, organization, @@ -9,16 +10,29 @@ import { workspace, } from '@sim/db/schema' import { generateId } from '@sim/utils/id' -import { and, eq } from 'drizzle-orm' -import { afterAll, beforeAll, describe, expect, it } from 'vitest' +import { and, eq, inArray } from 'drizzle-orm' +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest' import type { ConnectorDocumentFilter } from '@/lib/api/contracts/knowledge/connectors' +import * as embeddings from '@/lib/embeddings' import { createKnowledgeAclFixtureIds, seedKnowledgeAclFixture, } from '@/lib/knowledge/__integration__/seed-source-access-fixture' -import { listKnowledgeConnectorDocuments } from '@/lib/knowledge/application/connectors' +import { + deleteKnowledgeConnector, + listKnowledgeConnectorDocuments, +} from '@/lib/knowledge/application/connectors' +import { + listKnowledgeDocuments, + readKnowledgeDocument, + updateKnowledgeDocument, +} from '@/lib/knowledge/application/documents' import { readSearchSourceProgress } from '@/lib/knowledge/application/search-source-progress' import { listSearchSources } from '@/lib/knowledge/application/search-sources' +import { createContentSyncLease } from '@/lib/knowledge/connectors/sync-lock' +import { persistSkippedDocuments } from '@/lib/knowledge/connectors/sync-persistence' +import * as documentProcessor from '@/lib/knowledge/documents/document-processor' +import { processDocumentAsync, retryDocumentProcessing } from '@/lib/knowledge/documents/service' const ids = createKnowledgeAclFixtureIds() const alice = { kind: 'session' as const, userId: ids.aliceId, sessionId: 'fixture-alice' } @@ -207,14 +221,15 @@ describe('connector document filename search and document sets', () => { active: [failureId, ...planIds], excluded: [excludedId], failed: [failureId], + skipped: [], } - for (const filter of ['active', 'excluded', 'failed'] as const) { + for (const filter of ['active', 'excluded', 'failed', 'skipped'] as const) { const result = await listKnowledgeConnectorDocuments.execute({ principal: viewer, input: { ...scope, filter, search: ' nEeDle ' }, }) expect(result.documents.map((row) => row.id)).toEqual(expectedIds[filter]) - expect(result.counts).toEqual({ active: 3, excluded: 1, failed: 1 }) + expect(result.counts).toEqual({ active: 3, excluded: 1, failed: 1, skipped: 0 }) expect(result.hasMore).toBe(false) } }) @@ -244,7 +259,7 @@ describe('connector document filename search and document sets', () => { input: { ...scope, filter: 'active', search }, }) expect(result.documents.map((row) => row.id)).toEqual([id]) - expect(result.counts).toEqual({ active: 1, excluded: 0, failed: 0 }) + expect(result.counts).toEqual({ active: 1, excluded: 0, failed: 0, skipped: 0 }) } }) @@ -269,7 +284,7 @@ describe('connector document filename search and document sets', () => { input: { ...scope, filter: 'active', search: 'needle' }, }) expect(privateResult.documents.map((row) => row.id)).toEqual([privateId]) - expect(privateResult.counts).toEqual({ active: 1, excluded: 0, failed: 1 }) + expect(privateResult.counts).toEqual({ active: 1, excluded: 0, failed: 1, skipped: 0 }) await db .delete(permissions) @@ -284,3 +299,362 @@ describe('connector document filename search and document sets', () => { ).rejects.toThrow('Insufficient workspace permissions') }) }) + +describe('intentional skips and genuine failures across document reads', () => { + const fixture = createKnowledgeAclFixtureIds() + const viewer = { + kind: 'session' as const, + userId: fixture.aliceId, + sessionId: 'fixture-outcomes', + } + const otherViewer = { ...viewer, userId: fixture.bobId } + const legacySkipId = generateId() + const sourceFailureId = generateId() + const indexingFailureId = generateId() + const privateLegacySkipId = generateId() + const scope = { knowledgeBaseId: fixture.knowledgeBaseId, connectorId: fixture.connectorId } + const expectedCounts = { active: 4, excluded: 0, failed: 2, skipped: 2 } + const skipIds = [legacySkipId] + const failureIds = [sourceFailureId, indexingFailureId] + const privateIds = [privateLegacySkipId] + + beforeAll(async () => { + await seedKnowledgeAclFixture(fixture, { connectorType: 'google_drive' }) + await db + .update(knowledgeBase) + .set({ isSearchIndex: true }) + .where(eq(knowledgeBase.id, fixture.knowledgeBaseId)) + const rows: Array & { id: string; filename: string }> = [ + { + id: legacySkipId, + filename: 'outcome-a-legacy.png', + fileUrl: '', + uploadedAt: new Date('2026-01-01T00:00:00Z'), + }, + { + id: sourceFailureId, + filename: 'outcome-c-source.txt', + contentHash: null, + fileUrl: '', + processingError: 'Source download unavailable', + }, + { + id: indexingFailureId, + filename: 'outcome-d-indexing.txt', + storageKey: 'fixture-retained-artifact', + processingError: 'Embedding provider unavailable', + }, + { + id: privateLegacySkipId, + filename: 'outcome-e-private-legacy.png', + fileUrl: '', + acl: [`u:${fixture.bobId}@fixture.test`], + }, + ] + await db.insert(document).values( + rows.map((row) => ({ + knowledgeBaseId: fixture.knowledgeBaseId, + connectorId: fixture.connectorId, + externalId: row.id, + fileUrl: `https://fixture.test/${row.id}`, + fileSize: 10, + mimeType: 'text/plain', + processingStatus: 'failed', + contentHash: `git-sha:${row.id}`, + storageKey: null, + processingError: 'A source omission reason whose wording may change', + acl: [`u:${fixture.aliceId}@fixture.test`], + aclVerifiedAt: new Date(), + ...row, + })) + ) + for (const [externalId, title, userId, documentIds] of [ + ['current-skip', 'outcome-b-skipped.png', fixture.aliceId, skipIds], + ['private-current-skip', 'outcome-f-private-skipped.png', fixture.bobId, privateIds], + ] as const) { + const [persisted] = await persistSkippedDocuments( + fixture.knowledgeBaseId, + fixture.connectorId, + 'google_drive', + [ + { + type: 'skip', + extDoc: { + externalId, + title, + content: '', + mimeType: 'text/plain', + contentHash: `git-sha:${externalId}`, + skippedReason: 'Current worker intentionally omitted this file', + }, + }, + ], + undefined, + 'admin', + createContentSyncLease(fixture.connectorId, fixture.lockId) + ) + documentIds.push(persisted.documentId) + await db + .update(document) + .set({ acl: [`u:${userId}@fixture.test`], aclVerifiedAt: new Date() }) + .where(eq(document.id, persisted.documentId)) + } + await db + .update(knowledgeConnector) + .set({ status: 'active', syncLockToken: null }) + .where(eq(knowledgeConnector.id, fixture.connectorId)) + }) + + afterAll(async () => { + await db.delete(workspace).where(eq(workspace.id, fixture.workspaceId)) + await db.delete(organization).where(eq(organization.id, fixture.organizationId)) + await db.delete(user).where(eq(user.id, fixture.aliceId)) + await db.delete(user).where(eq(user.id, fixture.bobId)) + }) + + it('projects legacy and current skips before authorized filtering, counting, and pagination', async () => { + const expectedIds: Record = { + active: [...skipIds, ...failureIds], + excluded: [], + failed: failureIds, + skipped: skipIds, + } + for (const filter of ['active', 'excluded', 'failed', 'skipped'] as const) { + const result = await listKnowledgeConnectorDocuments.execute({ + principal: viewer, + input: { ...scope, filter, search: 'outcome-' }, + }) + expect(result.documents.map((row) => row.id)).toEqual(expectedIds[filter]) + expect(result.counts).toEqual(expectedCounts) + expect(result.hasMore).toBe(false) + for (const row of result.documents) { + expect(row.processingStatus).toBe('failed') + expect(row.processingOutcome).toBe(skipIds.includes(row.id) ? 'skipped' : null) + expect(row.processingError).toBeTruthy() + } + } + const pages = await Promise.all( + [0, 1].map((offset) => + listKnowledgeConnectorDocuments.execute({ + principal: viewer, + input: { ...scope, filter: 'skipped', limit: 1, offset }, + }) + ) + ) + expect(pages.flatMap((page) => page.documents.map((row) => row.id))).toEqual(skipIds) + expect(pages.map((page) => page.hasMore)).toEqual([true, false]) + expect(pages.map((page) => page.counts)).toEqual([expectedCounts, expectedCounts]) + const legacyFailures = await listKnowledgeConnectorDocuments.execute({ + principal: viewer, + input: { ...scope, failedOnly: true }, + }) + expect(legacyFailures.documents.map((row) => row.id)).toEqual(failureIds) + }) + + it('does not turn another viewer’s skips into indexing errors or expose their documents', async () => { + for (const [principal, failedCount] of [ + [viewer, 2], + [otherViewer, 0], + ] as const) { + const sources = await listSearchSources.execute({ + principal, + input: { workspaceId: fixture.workspaceId }, + }) + expect(sources.sources[0].viewerFailedDocumentCount).toBe(failedCount) + const progress = await readSearchSourceProgress.execute({ + principal, + input: { workspaceId: fixture.workspaceId, connectorIds: [fixture.connectorId] }, + }) + expect(progress.sources).toEqual([ + { + connectorId: fixture.connectorId, + isSyncing: false, + hasSyncError: false, + hasIndexingError: failedCount > 0, + }, + ]) + } + for (const filter of ['active', 'skipped', 'failed'] as const) { + const result = await listKnowledgeConnectorDocuments.execute({ + principal: otherViewer, + input: { ...scope, filter }, + }) + expect(result.documents.map((row) => row.id)).toEqual(filter === 'failed' ? [] : privateIds) + expect(result.counts).toEqual({ active: 2, excluded: 0, failed: 0, skipped: 2 }) + } + await expect( + readKnowledgeDocument.execute({ + principal: viewer, + input: { knowledgeBaseId: fixture.knowledgeBaseId, documentId: privateLegacySkipId }, + }) + ).rejects.toThrow('Document not found') + await expect( + readKnowledgeDocument.execute({ + principal: otherViewer, + input: { knowledgeBaseId: fixture.knowledgeBaseId, documentId: sourceFailureId }, + }) + ).rejects.toThrow('Document not found') + }) + + it('uses the same status for the regular knowledge base list and document detail', async () => { + const result = await listKnowledgeDocuments.execute({ + principal: viewer, + input: { knowledgeBaseId: fixture.knowledgeBaseId }, + }) + expect(result.documents.map((row) => row.id).sort()).toEqual([...skipIds, ...failureIds].sort()) + for (const row of result.documents) { + const expectedOutcome = skipIds.includes(row.id) ? 'skipped' : null + expect(row.processingStatus).toBe('failed') + expect(row.processingOutcome).toBe(expectedOutcome) + const detail = await readKnowledgeDocument.execute({ + principal: viewer, + input: { knowledgeBaseId: fixture.knowledgeBaseId, documentId: row.id }, + }) + expect(detail.document.processingStatus).toBe('failed') + expect(detail.document.processingOutcome).toBe(expectedOutcome) + expect(detail.document.processingError).toBe(row.processingError) + } + }) + + it('rejects skipped and source-failed retries without changing stored outcomes or embedding dispatch', async () => { + const embed = vi + .spyOn(embeddings, 'embedKnowledge') + .mockRejectedValue(new Error('Unexpected embedding dispatch')) + const retryIds = [...skipIds, sourceFailureId] + const before = await db.select().from(document).where(inArray(document.id, retryIds)) + try { + for (const documentId of retryIds) { + await expect( + updateKnowledgeDocument.execute({ + principal: viewer, + input: { knowledgeBaseId: fixture.knowledgeBaseId, documentId, retryProcessing: true }, + }) + ).rejects.toThrow( + documentId === sourceFailureId ? 'Sync the connector' : 'intentionally skipped' + ) + } + for (const documentId of skipIds) { + const stored = before.find((row) => row.id === documentId)! + const result = await retryDocumentProcessing( + fixture.knowledgeBaseId, + documentId, + stored, + 'fixture-skipped-retry', + undefined + ) + expect(result).toMatchObject({ success: false, status: 'skipped' }) + expect(result.message).toContain('intentionally skipped') + } + const after = await db.select().from(document).where(inArray(document.id, retryIds)) + expect(after.sort((left, right) => left.id.localeCompare(right.id))).toEqual( + before.sort((left, right) => left.id.localeCompare(right.id)) + ) + expect( + await db.select().from(embedding).where(inArray(embedding.documentId, retryIds)) + ).toEqual([]) + expect(embed).not.toHaveBeenCalled() + } finally { + embed.mockRestore() + } + }) + + it.each([false, true])( + 'preserves legacy and current skips against delayed tokenless workers (missing context: %s)', + async (missingContext) => { + const processor = vi + .spyOn(documentProcessor, 'processDocument') + .mockRejectedValue(new Error('Unexpected skipped document processing')) + const embed = vi + .spyOn(embeddings, 'embedKnowledge') + .mockRejectedValue(new Error('Unexpected embedding dispatch')) + const before = await db.select().from(document).where(inArray(document.id, skipIds)) + try { + if (missingContext) { + await db + .update(knowledgeBase) + .set({ deletedAt: new Date() }) + .where(eq(knowledgeBase.id, fixture.knowledgeBaseId)) + } + for (const row of before) { + await processDocumentAsync( + fixture.knowledgeBaseId, + row.id, + row, + {}, + undefined, + 'fixture-delayed-tokenless-worker', + { chargedAtDispatch: false } + ) + } + const after = await db.select().from(document).where(inArray(document.id, skipIds)) + expect(after.sort((left, right) => left.id.localeCompare(right.id))).toEqual( + before.sort((left, right) => left.id.localeCompare(right.id)) + ) + expect(processor).not.toHaveBeenCalled() + expect(embed).not.toHaveBeenCalled() + } finally { + if (missingContext) { + await db + .update(knowledgeBase) + .set({ deletedAt: null }) + .where(eq(knowledgeBase.id, fixture.knowledgeBaseId)) + } + processor.mockRestore() + embed.mockRestore() + } + } + ) + + it('preserves skip outcomes and real failures when a workspace source is removed with documents kept', async () => { + await expect( + deleteKnowledgeConnector.execute({ + principal: viewer, + input: { ...scope, deleteDocuments: false }, + }) + ).rejects.toThrow('cannot be kept') + await db + .update(knowledgeBase) + .set({ isSearchIndex: false }) + .where(eq(knowledgeBase.id, fixture.knowledgeBaseId)) + await db + .update(knowledgeConnector) + .set({ accessMode: 'workspace' }) + .where(eq(knowledgeConnector.id, fixture.connectorId)) + /** Workspace syncs write workspace ACLs; detached rows must not retain mirrored grants. */ + await db + .update(document) + .set({ acl: ['ws'], aclRequirements: [] }) + .where(eq(document.connectorId, fixture.connectorId)) + const result = await deleteKnowledgeConnector.execute({ + principal: viewer, + input: { ...scope, deleteDocuments: false }, + }) + expect(result).toMatchObject({ documentsDeleted: 0, documentsKept: 6 }) + const retained = await db + .select() + .from(document) + .where(eq(document.knowledgeBaseId, fixture.knowledgeBaseId)) + expect(retained).toHaveLength(6) + for (const row of retained) { + expect(row.connectorId).toBeNull() + expect(row.acl).toEqual(['ws']) + expect(row.processingStatus).toBe('failed') + } + for (const documentId of skipIds) { + for (const principal of [viewer, otherViewer]) { + const detail = await readKnowledgeDocument.execute({ + principal, + input: { knowledgeBaseId: fixture.knowledgeBaseId, documentId }, + }) + expect(detail.document.processingStatus).toBe('failed') + expect(detail.document.processingOutcome).toBe('skipped') + } + await expect( + updateKnowledgeDocument.execute({ + principal: viewer, + input: { knowledgeBaseId: fixture.knowledgeBaseId, documentId, retryProcessing: true }, + }) + ).rejects.toThrow('intentionally skipped') + } + }) +}) diff --git a/apps/sim/lib/knowledge/application/connector-access.test.ts b/apps/sim/lib/knowledge/application/connector-access.test.ts index 0e0358850c4..d80c90158a6 100644 --- a/apps/sim/lib/knowledge/application/connector-access.test.ts +++ b/apps/sim/lib/knowledge/application/connector-access.test.ts @@ -21,6 +21,11 @@ const mocks = vi.hoisted(() => ({ sourceAccess: vi.fn(), oauthContext: vi.fn(), startOAuth: vi.fn(), + authorizeOrganization: vi.fn(), + credential: vi.fn(), + decrypt: vi.fn(), + installationBinding: vi.fn(), + repository: vi.fn(), })) vi.mock('@sim/audit', () => ({ AuditAction: {}, AuditResourceType: {}, recordAudit: vi.fn() })) @@ -32,6 +37,26 @@ vi.mock('@sim/platform-authz/workspace', () => ({ vi.mock('@/lib/permission-groups/resolve.server', () => ({ getUserPermissionConfig: async () => null, })) +vi.mock('@/lib/core/application/organization-authorization', () => ({ + authorizeOrganizationOperation: mocks.authorizeOrganization, +})) +vi.mock('@/lib/knowledge/application/connector-credential', () => ({ + requireConnectorCredential: mocks.credential, +})) +vi.mock('@/lib/core/security/encryption', () => ({ decryptSecret: mocks.decrypt })) +vi.mock('@/lib/oauth/github-installation', () => ({ + GitHubInstallationError: class extends Error { + constructor( + message: string, + readonly status?: number, + readonly operation?: string + ) { + super(message) + } + }, + parseGitHubInstallationBinding: mocks.installationBinding, + resolveGitHubInstallationRepository: mocks.repository, +})) vi.mock('@/lib/knowledge/application/contexts', () => ({ resolveActiveKnowledgeConnectorContext: mocks.context, })) @@ -80,10 +105,12 @@ vi.mock('@/lib/knowledge/connectors/member-provisioning', () => ({ provisionKnowledgeConnectorMembersBinding: mocks.provision, })) +import { OrchestrationError } from '@/lib/core/orchestration/types' import { startKnowledgeConnectorMemberEnrollment, updateKnowledgeConnectorAccess, } from '@/lib/knowledge/application/connector-access' +import { GitHubInstallationError } from '@/lib/oauth/github-installation' const principal = { kind: 'session' as const, userId: 'admin', sessionId: 'session' } const input = { knowledgeBaseId: 'kb', connectorId: 'source', assertedWorkspaceId: 'workspace' } @@ -131,6 +158,155 @@ beforeEach(() => { mocks.sourceAccess.mockResolvedValue(undefined) mocks.oauthContext.mockResolvedValue({ credentialOwnerId: 'admin', option: { id: 'option' } }) mocks.startOAuth.mockResolvedValue('https://provider.example.test/authorize') + mocks.authorizeOrganization.mockResolvedValue({ + organizationId: 'org', + userId: 'admin', + role: 'admin', + }) +}) + +describe('GitHub installation connection replacement', () => { + const sourceConfig = { repository: 'acme/platform', githubRepositoryId: '123', branch: 'main' } + const replacementInput = { + knowledgeBaseId: 'kb', + connectorId: 'source', + accessMode: 'members' as const, + credentialId: 'replacement-installation', + } + + beforeEach(() => { + mocks.context.mockResolvedValue({ + organizationId: 'org', + knowledgeBaseId: 'kb', + connectorId: 'source', + knowledgeBase: { organizationId: 'org', id: 'kb', name: 'Search', isSearchIndex: true }, + }) + mocks.connector.mockResolvedValue({ + ...row, + connectorType: 'github', + accessMode: 'members', + credentialId: 'previous-installation', + sourceConfig, + }) + mocks.meta.mockReturnValue({ + name: 'GitHub', + search: true, + auth: { mode: 'oauth', provider: 'github-repositories' }, + supportsSeparateContentCredential: true, + }) + mocks.credential.mockResolvedValue({ + id: replacementInput.credentialId, + providerId: 'github-app-installation', + organizationId: 'org', + workspaceId: null, + type: 'service_account', + revokedAt: null, + encryptedServiceAccountKey: 'encrypted-binding', + providerSubjectId: '42', + providerTenantId: '7', + }) + mocks.decrypt.mockResolvedValue({ decrypted: '{}' }) + mocks.installationBinding.mockReturnValue({ installationId: '42', accountId: '7' }) + mocks.repository.mockResolvedValue({ id: '123', fullName: 'acme/platform' }) + mocks.binding.mockImplementation(async ({ sourceConfig }) => ({ + credentialGroupId: 'group', + credentialGroupOptionId: 'github-members', + sourceConfig, + })) + }) + + it.each([ + { status: 404, operation: 'repository' as const }, + { status: 422, operation: 'repository-token' as const }, + ])( + 'rejects an incompatible installation with actionable validation ($status)', + async ({ status, operation }) => { + mocks.repository.mockRejectedValue( + new GitHubInstallationError('Provider detail', status, operation) + ) + await expect( + updateKnowledgeConnectorAccess.execute({ principal, input: replacementInput }) + ).rejects.toMatchObject({ + code: 'validation', + message: + "This GitHub connection cannot access this source's repository. Choose a connection with access to the same repository, or add a new source for a different repository.", + }) + expect(mocks.repository).toHaveBeenCalledExactlyOnceWith( + { installationId: '42', accountId: '7' }, + 'acme/platform' + ) + expect(mocks.update).not.toHaveBeenCalled() + expect(mocks.binding).not.toHaveBeenCalled() + expect(mocks.token).not.toHaveBeenCalled() + } + ) + + it('rejects a recreated repository at the same path before changing the binding', async () => { + mocks.repository.mockResolvedValue({ id: '999', fullName: 'acme/platform' }) + await expect( + updateKnowledgeConnectorAccess.execute({ principal, input: replacementInput }) + ).rejects.toMatchObject({ + code: 'validation', + message: 'Create a new source to index a different GitHub repository', + }) + expect(mocks.update).not.toHaveBeenCalled() + }) + + it('passes a verified replacement for the same repository to the atomic access update', async () => { + await updateKnowledgeConnectorAccess.execute({ principal, input: replacementInput }) + expect(mocks.authorizeOrganization).toHaveBeenCalledWith( + principal, + expect.objectContaining({ minimumRole: 'admin' }), + expect.objectContaining({ organizationId: 'org' }) + ) + expect(mocks.credential).toHaveBeenCalledWith( + expect.objectContaining({ + principal, + credentialId: replacementInput.credentialId, + scope: { kind: 'organization', organizationId: 'org' }, + }) + ) + expect(mocks.update).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + userId: 'admin', + connectorId: 'source', + knowledgeBase: { id: 'kb', name: 'Search', organizationId: 'org' }, + target: { + accessMode: 'members', + credentialId: replacementInput.credentialId, + binding: { + credentialGroupId: 'group', + credentialGroupOptionId: 'github-members', + sourceConfig, + }, + }, + }) + ) + }) + + it('rechecks organization administration before reading the installation or mutating the source', async () => { + mocks.authorizeOrganization.mockRejectedValue( + new OrchestrationError('forbidden', 'Organization administrator access is required') + ) + await expect( + updateKnowledgeConnectorAccess.execute({ principal, input: replacementInput }) + ).rejects.toMatchObject({ code: 'forbidden' }) + expect(mocks.credential).not.toHaveBeenCalled() + expect(mocks.repository).not.toHaveBeenCalled() + expect(mocks.update).not.toHaveBeenCalled() + }) + + it.each([403, 429, 503])( + 'preserves a real provider failure (%s) without changing the binding', + async (status) => { + const error = new GitHubInstallationError('Provider unavailable', status, 'repository') + mocks.repository.mockRejectedValue(error) + await expect( + updateKnowledgeConnectorAccess.execute({ principal, input: replacementInput }) + ).rejects.toBe(error) + expect(mocks.update).not.toHaveBeenCalled() + } + ) }) describe('source member enrollment', () => { diff --git a/apps/sim/lib/knowledge/application/connectors.test.ts b/apps/sim/lib/knowledge/application/connectors.test.ts index 337d4c34d28..1c60570d056 100644 --- a/apps/sim/lib/knowledge/application/connectors.test.ts +++ b/apps/sim/lib/knowledge/application/connectors.test.ts @@ -4,7 +4,7 @@ import { document, knowledgeConnector, member } from '@sim/db/schema' import { dbChainMockFns, queueTableRows, resetDbChainMock } from '@sim/testing' -import { afterAll, beforeEach, describe, expect, it, vi } from 'vitest' +import { afterAll, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' const mocks = vi.hoisted(() => ({ resolveKnowledgeBase: vi.fn(), @@ -151,7 +151,9 @@ vi.mock('@/connectors/registry.server', () => ({ }, })) +import { internalOrchestrationErrorPolicy } from '@/lib/api/server/routes/internal-json-route' import { OrchestrationError } from '@/lib/core/orchestration/types' +import * as encryption from '@/lib/core/security/encryption' import { createApprovedSearchSource, createKnowledgeConnector, @@ -165,6 +167,7 @@ import { validateConnectorSourceConfig, } from '@/lib/knowledge/application/connectors' import { MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_SEARCH_LENGTH } from '@/lib/knowledge/constants' +import * as githubInstallation from '@/lib/oauth/github-installation' import { capabilityRefusal } from '@/lib/permission-groups/capability-assertions' import { DEFAULT_PERMISSION_GROUP_CONFIG } from '@/lib/permission-groups/fields' import { confluenceConnectorMeta } from '@/connectors/confluence/meta' @@ -926,7 +929,7 @@ describe('knowledge connector application use cases', () => { mocks.resolveConnector.mockResolvedValueOnce(sameWorkspaceContext) queueTableRows(document, [{ value: 5 }]) queueTableRows(document, [{ value: 2 }]) - queueTableRows(document, [{ value: 1 }]) + queueTableRows(document, [{ failed: 1, skipped: 3 }]) queueTableRows(document, [ { id: 'document-3', filename: 'c.txt', userExcluded: false }, { id: 'document-4', filename: 'd.txt', userExcluded: true }, @@ -949,7 +952,7 @@ describe('knowledge connector application use cases', () => { { id: 'document-3', filename: 'c.txt', userExcluded: false }, { id: 'document-4', filename: 'd.txt', userExcluded: true }, ], - counts: { active: 5, excluded: 2, failed: 1 }, + counts: { active: 5, excluded: 2, failed: 1, skipped: 3 }, hasMore: false, offset: 2, limit: 2, @@ -1651,3 +1654,190 @@ describe('organization connector credential authorization', () => { expect(mocks.validateConnectorConfig).not.toHaveBeenCalled() }) }) + +describe('GitHub installation source rejection at the application boundary', () => { + const principal = { kind: 'session', userId: 'org-admin', sessionId: 'session' } as const + const sourceConfig = { repository: 'example/private', githubRepositoryId: '123' } + const credential = { + id: 'installation-credential', + organizationId: 'org', + workspaceId: null, + providerId: 'github-app-installation', + type: 'service_account', + encryptedServiceAccountKey: 'encrypted-binding', + providerSubjectId: '42', + providerTenantId: '7', + revokedAt: null, + } + const connector = { + id: 'source', + connectorType: 'github', + credentialId: credential.id, + accessMode: 'members' as const, + sourceConfig, + encryptedApiKey: null, + } + const createInput = { + knowledgeBaseId: 'org-index', + assertedOrganizationId: 'org', + connectorType: 'github', + credentialId: credential.id, + accessMode: 'members' as const, + sourceConfig, + syncIntervalMinutes: 60, + } + const validationMessage = + 'Check that the repository is included in the selected GitHub App installation, then retry.' + + beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + queueTableRows(member, [{ role: 'admin' }]) + queueTableRows(member, [{ role: 'admin' }]) + const context = { + organizationId: 'org', + knowledgeBaseId: 'org-index', + knowledgeBase: { id: 'org-index', name: 'Search', isSearchIndex: true }, + } + mocks.resolveKnowledgeBase.mockResolvedValue(context) + mocks.resolveConnector.mockResolvedValue({ ...context, connectorId: connector.id, connector }) + mocks.getUserPermissionConfig.mockResolvedValue(null) + mocks.authorizeOrganizationCredentialUse.mockResolvedValue({ credential }) + mocks.resolveTokenIdentity.mockResolvedValue({ kind: 'service_account' }) + mocks.resolveTokenBundle.mockResolvedValue({ accessToken: 'repository-token' }) + mocks.resolveMembersBinding.mockResolvedValue({ + credentialGroupId: 'group', + credentialGroupOptionId: 'option', + organizationId: 'org', + sourceConfig, + }) + vi.spyOn(encryption, 'decryptSecret').mockResolvedValue({ + decrypted: JSON.stringify({ + type: 'github_app_installation', + version: 1, + appId: '1', + appClientId: 'app-client', + installationId: '42', + accountId: '7', + accountType: 'Organization', + accountLogin: 'example', + repositorySelection: 'selected', + }), + }) + vi.spyOn(githubInstallation, 'resolveGitHubInstallationRepository').mockResolvedValue({ + id: '123', + fullName: sourceConfig.repository, + defaultBranch: 'main', + }) + mocks.createConnector.mockImplementation( + async (input: { resolveAccessToken(id: string): Promise }) => { + await input.resolveAccessToken(credential.id) + throw new Error('Unexpected connector persistence') + } + ) + mocks.updateConnector.mockImplementation( + async (input: { + prepareSourceConfig( + currentConnector: typeof connector, + config: typeof sourceConfig + ): Promise + validateSourceConfig( + currentConnector: typeof connector, + config: typeof sourceConfig + ): Promise + }) => { + const prepared = await input.prepareSourceConfig(connector, sourceConfig) + await input.validateSourceConfig(connector, prepared) + throw new Error('Unexpected connector persistence') + } + ) + }) + + afterEach(() => { + vi.restoreAllMocks() + resetDbChainMock() + }) + + it.each([ + [422, 'repository-token'], + [404, 'repository'], + ] as const)( + 'returns a safe validation response for GitHub %s during %s', + async (status, operation) => { + vi.mocked(githubInstallation.resolveGitHubInstallationRepository).mockRejectedValueOnce( + new githubInstallation.GitHubInstallationError( + 'private provider payload', + status, + operation + ) + ) + const error = await createKnowledgeConnector + .execute({ principal, input: createInput }) + .catch((error: unknown) => error) + expect(error).toBeInstanceOf(OrchestrationError) + expect(internalOrchestrationErrorPolicy.project(error)).toMatchObject({ + status: 400, + body: { error: validationMessage }, + }) + expect(mocks.authorizeOrganizationCredentialUse).toHaveBeenCalledWith( + expect.objectContaining({ principal, organizationId: 'org', credentialId: credential.id }) + ) + expect(mocks.createConnector).not.toHaveBeenCalled() + expect(mocks.recordAudit).not.toHaveBeenCalled() + } + ) + + it.each(['create', 'update'] as const)( + 'classifies a repository token rejected during authorized %s validation', + async (operation) => { + mocks.resolveTokenBundle.mockRejectedValueOnce( + new githubInstallation.GitHubInstallationError( + 'private token response', + 422, + 'repository-token' + ) + ) + const result = + operation === 'create' + ? createKnowledgeConnector.execute({ principal, input: createInput }) + : updateKnowledgeConnector.execute({ + principal, + input: { + knowledgeBaseId: 'org-index', + connectorId: connector.id, + updates: { sourceConfig }, + }, + }) + await expect(result).rejects.toMatchObject({ code: 'validation', message: validationMessage }) + expect(mocks.resolveTokenBundle).toHaveBeenCalledOnce() + expect(mocks.recordAudit).not.toHaveBeenCalled() + } + ) + + it.each([403, 429, 503])( + 'preserves GitHub %s and its operation for retry classification', + async (status) => { + const failure = new githubInstallation.GitHubInstallationError( + 'GitHub provider request failed', + status, + 'repository-token' + ) + mocks.resolveTokenBundle.mockRejectedValueOnce(failure) + await expect( + createKnowledgeConnector.execute({ principal, input: createInput }) + ).rejects.toBe(failure) + expect(internalOrchestrationErrorPolicy.project(failure)).toBeNull() + expect(mocks.recordAudit).not.toHaveBeenCalled() + } + ) + + it('preserves network failures during authorized source creation', async () => { + const failure = new TypeError('Network request failed') + mocks.resolveTokenBundle.mockRejectedValueOnce(failure) + await expect(createKnowledgeConnector.execute({ principal, input: createInput })).rejects.toBe( + failure + ) + expect(internalOrchestrationErrorPolicy.project(failure)).toBeNull() + expect(mocks.recordAudit).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/knowledge/application/connectors.ts b/apps/sim/lib/knowledge/application/connectors.ts index dbe6747d077..f4deff13f95 100644 --- a/apps/sim/lib/knowledge/application/connectors.ts +++ b/apps/sim/lib/knowledge/application/connectors.ts @@ -43,6 +43,7 @@ import { resolveActiveKnowledgeResourceContext, resolveKnowledgeWorkspaceContext, } from '@/lib/knowledge/application/contexts' +import { rethrowGitHubInstallationSourceError } from '@/lib/knowledge/application/github-installation-error' import { prepareGitHubInstallationSource } from '@/lib/knowledge/application/github-installation-source' import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { @@ -79,6 +80,11 @@ import { MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_PAGE_SIZE, MAX_KNOWLEDGE_CONNECTOR_DOCUMENT_SEARCH_LENGTH, } from '@/lib/knowledge/constants' +import { + documentProcessingOutcomeSelection, + failedDocumentCondition, + skippedDocumentCondition, +} from '@/lib/knowledge/documents/processing-status' import { type ResolvedMembersBinding, resolveKnowledgeConnectorMembersBinding, @@ -322,7 +328,7 @@ export async function resolveConnectorCredentialAccessToken(input: { userId: identity.kind === 'oauth' ? identity.userId : input.actingUserId, requestId: input.requestId, sourceConfig: input.sourceConfig, - }) + }).catch(rethrowGitHubInstallationSourceError) return resolved } @@ -420,7 +426,7 @@ export async function validateConnectorSourceConfig(input: { userId: tokenUserId, requestId: input.requestId, sourceConfig: input.sourceConfig, - }) + }).catch(rethrowGitHubInstallationSourceError) if (!resolved) { return { message: 'Failed to refresh access token. Please reconnect your account.', @@ -1268,6 +1274,8 @@ const connectorDocumentSelection = { userExcluded: document.userExcluded, uploadedAt: document.uploadedAt, processingStatus: document.processingStatus, + processingOutcome: documentProcessingOutcomeSelection(), + processingError: document.processingError, } export const listKnowledgeConnectorDocuments = defineAuthorizedKnowledgeUseCase({ @@ -1316,7 +1324,7 @@ export const listKnowledgeConnectorDocuments = defineAuthorizedKnowledgeUseCase( ? sql`${document.filename} ILIKE ${`%${escapeLikePattern(search)}%`} ESCAPE '\\'` : undefined, ] as const - const [[activeCount], excludedCountRows, [failedCount]] = await Promise.all([ + const [[activeCount], excludedCountRows, [outcomeCounts]] = await Promise.all([ db .select({ value: count() }) .from(document) @@ -1328,15 +1336,12 @@ export const listKnowledgeConnectorDocuments = defineAuthorizedKnowledgeUseCase( .where(and(...baseConditions, eq(document.userExcluded, true))) : Promise.resolve([{ value: 0 }]), db - .select({ value: count() }) + .select({ + failed: sql`count(*) FILTER (WHERE ${failedDocumentCondition()})::int`, + skipped: sql`count(*) FILTER (WHERE ${skippedDocumentCondition()})::int`, + }) .from(document) - .where( - and( - ...baseConditions, - eq(document.userExcluded, false), - eq(document.processingStatus, 'failed') - ) - ), + .where(and(...baseConditions, eq(document.userExcluded, false))), ]) const excludedCount = excludedCountRows[0] const rows = await db @@ -1346,7 +1351,8 @@ export const listKnowledgeConnectorDocuments = defineAuthorizedKnowledgeUseCase( and( ...baseConditions, filter ? eq(document.userExcluded, filter === 'excluded') : undefined, - filter === 'failed' ? eq(document.processingStatus, 'failed') : undefined + filter === 'failed' ? failedDocumentCondition() : undefined, + filter === 'skipped' ? skippedDocumentCondition() : undefined ) ) .orderBy(asc(document.userExcluded), asc(document.filename), asc(document.id)) @@ -1359,7 +1365,8 @@ export const listKnowledgeConnectorDocuments = defineAuthorizedKnowledgeUseCase( counts: { active: activeCount?.value ?? 0, excluded: excludedCount?.value ?? 0, - failed: failedCount?.value ?? 0, + failed: outcomeCounts?.failed ?? 0, + skipped: outcomeCounts?.skipped ?? 0, }, hasMore, offset, diff --git a/apps/sim/lib/knowledge/application/github-installation-error.ts b/apps/sim/lib/knowledge/application/github-installation-error.ts new file mode 100644 index 00000000000..bad83142274 --- /dev/null +++ b/apps/sim/lib/knowledge/application/github-installation-error.ts @@ -0,0 +1,22 @@ +import { OrchestrationError } from '@/lib/core/orchestration/types' +import { GitHubInstallationError } from '@/lib/oauth/github-installation' + +/** Maps repository selection refusals while preserving provider failures for their caller. */ +export function rethrowGitHubInstallationSourceError( + error: unknown, + { existingSource = false }: { existingSource?: boolean } = {} +): never { + if ( + error instanceof GitHubInstallationError && + ((error.operation === 'repository-token' && error.status === 422) || + (error.operation === 'repository' && error.status === 404)) + ) { + throw new OrchestrationError( + 'validation', + existingSource + ? "This GitHub connection cannot access this source's repository. Choose a connection with access to the same repository, or add a new source for a different repository." + : 'Check that the repository is included in the selected GitHub App installation, then retry.' + ) + } + throw error +} diff --git a/apps/sim/lib/knowledge/application/github-installation-source.test.ts b/apps/sim/lib/knowledge/application/github-installation-source.test.ts index 2c7666faf04..f68d58c33b8 100644 --- a/apps/sim/lib/knowledge/application/github-installation-source.test.ts +++ b/apps/sim/lib/knowledge/application/github-installation-source.test.ts @@ -12,6 +12,7 @@ vi.mock('@/lib/knowledge/application/connector-credential', () => ({ })) vi.mock('@/lib/core/security/encryption', () => ({ decryptSecret: m.decrypt })) vi.mock('@/lib/oauth/github-installation', () => ({ + GitHubInstallationError: class extends Error {}, parseGitHubInstallationBinding: m.parse, resolveGitHubInstallationRepository: m.repository, })) diff --git a/apps/sim/lib/knowledge/application/github-installation-source.ts b/apps/sim/lib/knowledge/application/github-installation-source.ts index 8ee2daa57b4..788216aac32 100644 --- a/apps/sim/lib/knowledge/application/github-installation-source.ts +++ b/apps/sim/lib/knowledge/application/github-installation-source.ts @@ -3,6 +3,7 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' import { resourceScopeFromOwner } from '@/lib/core/resource-scope' import { decryptSecret } from '@/lib/core/security/encryption' import { requireConnectorCredential } from '@/lib/knowledge/application/connector-credential' +import { rethrowGitHubInstallationSourceError } from '@/lib/knowledge/application/github-installation-error' import { parseGitHubInstallationBinding, resolveGitHubInstallationRepository, @@ -80,7 +81,9 @@ export async function prepareGitHubInstallationSource( binding.accountId !== contentCredential.providerTenantId ) throw new OrchestrationError('validation', 'Reconnect this GitHub installation before using it') - const resolved = await resolveGitHubInstallationRepository(binding, repository.trim()) + const resolved = await resolveGitHubInstallationRepository(binding, repository.trim()).catch( + (error) => rethrowGitHubInstallationSourceError(error, { existingSource: wasInstallation }) + ) if (wasInstallation && input.previousConfig?.githubRepositoryId !== resolved.id) throw new OrchestrationError( 'validation', diff --git a/apps/sim/lib/knowledge/application/github-installations.test.ts b/apps/sim/lib/knowledge/application/github-installations.test.ts index 645180d446e..8e6355ffc27 100644 --- a/apps/sim/lib/knowledge/application/github-installations.test.ts +++ b/apps/sim/lib/knowledge/application/github-installations.test.ts @@ -33,11 +33,30 @@ vi.mock('@/lib/permission-groups/resolve.server', () => ({ getUserPermissionConfigForOrganization: async () => null, })) vi.mock('@/lib/oauth/github-installation', () => ({ + GitHubInstallationError: class extends Error { + constructor( + message: string, + readonly status?: number + ) { + super(message) + } + }, getGitHubInstallationConfiguration: m.configuration, listUserAdminGitHubInstallations: m.list, verifyGitHubInstallationBinding: m.verify, })) -vi.mock('@/lib/credentials/managed-oauth', () => ({ resolveManagedOAuthToken: m.token })) +vi.mock('@/lib/credentials/managed-oauth', () => ({ + resolveManagedOAuthToken: m.token, + ManagedOAuthCredentialError: class extends Error { + constructor( + readonly code: string, + message: string, + readonly statusCode: number + ) { + super(message) + } + }, +})) vi.mock('@/lib/credential-groups/provider-registry', () => ({ getCredentialGroupProviderAdapter: () => ({ getPolicy: async () => ({ authorizationAppId: 'current-app', scopeVersion: 1 }), @@ -45,10 +64,12 @@ vi.mock('@/lib/credential-groups/provider-registry', () => ({ })) vi.mock('@/lib/core/security/encryption', () => ({ encryptSecret: m.encrypt })) +import { ManagedOAuthCredentialError } from '@/lib/credentials/managed-oauth' import { connectGitHubSearchInstallation, listGitHubSearchInstallations, } from '@/lib/knowledge/application/github-installations' +import { GitHubInstallationError } from '@/lib/oauth/github-installation' const principal = { kind: 'session', userId: 'admin', sessionId: 'session' } as const const input = { organizationId: 'org', installationId: '42' } @@ -142,6 +163,56 @@ describe('GitHub Search installation application operations', () => { }) expect(m.list).toHaveBeenCalledWith('ghu_reader', { signal }) }) + it('reenters reader OAuth when GitHub revoked a token still recorded as active', async () => { + setupReader() + m.list.mockRejectedValueOnce(new GitHubInstallationError('Bad credentials', 401)) + await expect( + listGitHubSearchInstallations.execute({ principal, input }) + ).resolves.toMatchObject({ + needsUserConnection: true, + installations: [], + }) + }) + it('reenters reader OAuth when token refresh detects a revoked grant', async () => { + setupReader() + m.token.mockRejectedValueOnce( + new ManagedOAuthCredentialError( + 'MANAGED_CREDENTIAL_NEEDS_REAUTH', + 'Refresh grant revoked', + 401 + ) + ) + await expect( + listGitHubSearchInstallations.execute({ principal, input }) + ).resolves.toMatchObject({ + needsUserConnection: true, + installations: [], + }) + expect(m.list).not.toHaveBeenCalled() + }) + it('preserves provider infrastructure failures during discovery', async () => { + setupReader() + const error = new GitHubInstallationError('Provider unavailable', 503) + m.list.mockRejectedValueOnce(error) + await expect(listGitHubSearchInstallations.execute({ principal, input })).rejects.toBe(error) + }) + it('preserves transient token refresh failures instead of requesting OAuth', async () => { + setupReader() + const error = new ManagedOAuthCredentialError( + 'MANAGED_CREDENTIAL_REFRESH_FAILED', + 'Transient refresh error', + 502 + ) + m.token.mockRejectedValueOnce(error) + await expect(listGitHubSearchInstallations.execute({ principal, input })).rejects.toBe(error) + }) + it('does not reclassify App JWT authorization failures during installation verification', async () => { + setupReader() + const error = new GitHubInstallationError('App JWT rejected', 401) + m.verify.mockRejectedValueOnce(error) + await expect(connect()).rejects.toBe(error) + expect(m.encrypt).not.toHaveBeenCalled() + }) it('reverifies GitHub admin authority before persisting an installation', async () => { setupReader() m.verify.mockRejectedValue(new Error('GitHub administrator access required')) @@ -155,7 +226,7 @@ describe('GitHub Search installation application operations', () => { const result = await connect() expect(result).toMatchObject({ created: true, - credential: { displayName: 'GitHub App · example' }, + credential: { displayName: 'example' }, }) expect(m.verify).toHaveBeenCalledWith('ghu_reader', '42', { signal: undefined }) expect(m.encrypt).toHaveBeenCalledWith(JSON.stringify(binding)) diff --git a/apps/sim/lib/knowledge/application/github-installations.ts b/apps/sim/lib/knowledge/application/github-installations.ts index 917e9569cfb..7add172b1c2 100644 --- a/apps/sim/lib/knowledge/application/github-installations.ts +++ b/apps/sim/lib/knowledge/application/github-installations.ts @@ -13,18 +13,25 @@ import { OrchestrationError } from '@/lib/core/orchestration/types' import { encryptSecret } from '@/lib/core/security/encryption' import { LIVE_ENROLLMENT_STATUSES } from '@/lib/credential-groups/credentials' import { getCredentialGroupProviderAdapter } from '@/lib/credential-groups/provider-registry' -import { resolveManagedOAuthToken } from '@/lib/credentials/managed-oauth' +import { + ManagedOAuthCredentialError, + resolveManagedOAuthToken, +} from '@/lib/credentials/managed-oauth' import type { DbOrTx } from '@/lib/db/types' import { requireOrganizationSearchAvailable } from '@/lib/knowledge/access/availability' import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' import { resolveKnowledgeOrganizationContext } from '@/lib/knowledge/application/contexts' import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { + GitHubInstallationError, getGitHubInstallationConfiguration, listUserAdminGitHubInstallations, verifyGitHubInstallationBinding, } from '@/lib/oauth/github-installation' -import { GITHUB_INSTALLATION_PROVIDER_ID } from '@/lib/oauth/github-installation-types' +import { + GITHUB_INSTALLATION_PROVIDER_ID, + type GitHubInstallationSummary, +} from '@/lib/oauth/github-installation-types' interface InstallationInput { organizationId: string @@ -36,7 +43,11 @@ interface ConnectInstallationInput extends InstallationInput { } /** Selects only the acting person's live, organization-bound GitHub connection. */ -async function findReaderCredential(executor: DbOrTx, organizationId: string, userId: string) { +export async function findGitHubSearchReaderCredential( + executor: DbOrTx, + organizationId: string, + userId: string +) { const policy = await getCredentialGroupProviderAdapter('github-repositories').getPolicy( undefined, { organizationId } @@ -99,18 +110,31 @@ export const listGitHubSearchInstallations = defineAuthorizedKnowledgeUseCase({ await requireOrganizationSearchAvailable(context.organizationId) const configuration = getGitHubInstallationConfiguration() const reader = configuration.configured - ? await findReaderCredential(db, context.organizationId, principal.userId) + ? await findGitHubSearchReaderCredential(db, context.organizationId, principal.userId) : null - const installations = reader - ? await listUserAdminGitHubInstallations( + let needsUserConnection = configuration.configured && !reader + let installations: GitHubInstallationSummary[] = [] + if (reader) { + try { + installations = await listUserAdminGitHubInstallations( (await readerToken(context.organizationId, reader.id)).accessToken, { signal: input.signal } ) - : [] + } catch (error) { + /** Only reader-token discovery can request reauthorization; App JWT failures stay errors. */ + if ( + (error instanceof GitHubInstallationError && error.status === 401) || + (error instanceof ManagedOAuthCredentialError && + error.code === 'MANAGED_CREDENTIAL_NEEDS_REAUTH') + ) + needsUserConnection = true + else throw error + } + } return { available: configuration.configured, installUrl: configuration.installUrl, - needsUserConnection: configuration.configured && !reader, + needsUserConnection, installations, } }, @@ -127,7 +151,11 @@ export const connectGitHubSearchInstallation = defineAuthorizedKnowledgeUseCase( 'validation', 'GitHub App installation indexing is not configured for this environment' ) - const reader = await findReaderCredential(db, context.organizationId, principal.userId) + const reader = await findGitHubSearchReaderCredential( + db, + context.organizationId, + principal.userId + ) if (!reader) throw new OrchestrationError( 'validation', @@ -168,7 +196,11 @@ export const connectGitHubSearchInstallation = defineAuthorizedKnowledgeUseCase( ) .for('update') .limit(1) - const current = await findReaderCredential(tx, context.organizationId, principal.userId) + const current = await findGitHubSearchReaderCredential( + tx, + context.organizationId, + principal.userId + ) if ( current?.id !== reader.id || current.authorizationAppId !== reader.authorizationAppId || @@ -194,7 +226,7 @@ export const connectGitHubSearchInstallation = defineAuthorizedKnowledgeUseCase( .limit(1) const id = existing?.id ?? generateId() const now = new Date() - const displayName = `GitHub App · ${binding.accountLogin}` + const displayName = binding.accountLogin const values = { displayName, encryptedServiceAccountKey: encrypted, diff --git a/apps/sim/lib/knowledge/application/github-setup.test.ts b/apps/sim/lib/knowledge/application/github-setup.test.ts new file mode 100644 index 00000000000..bd1d9939517 --- /dev/null +++ b/apps/sim/lib/knowledge/application/github-setup.test.ts @@ -0,0 +1,608 @@ +/** @vitest-environment node */ +import { db } from '@sim/db' +import { credential, member, user } from '@sim/db/schema' +import { queueTableRows, resetDbChainMock } from '@sim/testing' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +const m = vi.hoisted(() => { + const values = new Map() + return { + values, + get: vi.fn(async (key: string) => { + const entry = values.get(key) + return entry && entry.expires > Date.now() ? entry.value : null + }), + set: vi.fn( + async (key: string, value: string, _unit: string, ttl: number, condition: string) => { + const entry = values.get(key) + const exists = entry && entry.expires > Date.now() + if ((condition === 'NX' && exists) || (condition === 'XX' && !exists)) return null + values.set(key, { value, expires: Date.now() + ttl }) + return 'OK' + } + ), + eval: vi.fn( + async ( + _script: string, + _count: number, + key: string, + nextValue: string, + expected: string, + ttl: number + ) => { + const entry = values.get(key) + if (!entry || entry.expires <= Date.now()) return 0 + const current = JSON.parse(entry.value) as Record + const next = JSON.parse(nextValue) as Record + if ( + current.phase !== expected || + ['organizationId', 'userId', 'sessionId', 'setupId', 'createdAt'].some( + (field) => current[field] !== next[field] + ) + ) + return 0 + values.set(key, { value: nextValue, expires: Date.now() + ttl }) + return 1 + } + ), + list: vi.fn(), + connect: vi.fn(), + reader: vi.fn(), + receipt: vi.fn(), + provision: vi.fn(), + enrollment: vi.fn(), + oauthContext: vi.fn(), + oauthStart: vi.fn(), + oauthComplete: vi.fn(), + } +}) +vi.mock('@/lib/core/config/redis', () => ({ + getRedisClient: () => ({ get: m.get, set: m.set, eval: m.eval }), +})) +vi.mock('@/lib/core/utils/urls', () => ({ getBaseUrl: () => 'https://sim.example' })) +vi.mock('@/lib/knowledge/application/contexts', () => ({ + resolveKnowledgeOrganizationContext: async ({ organizationId }: { organizationId: string }) => ({ + organizationId, + workspaceId: undefined, + }), +})) +vi.mock('@/lib/permission-groups/resolve.server', () => ({ + getUserPermissionConfigForOrganization: async () => null, +})) +vi.mock('@/lib/knowledge/application/github-installations', () => ({ + listGitHubSearchInstallations: { execute: m.list }, + connectGitHubSearchInstallation: { execute: m.connect }, + findGitHubSearchReaderCredential: m.reader, +})) +vi.mock('@/lib/knowledge/connectors/member-provisioning', () => ({ + provisionKnowledgeConnectorMembersBinding: m.provision, +})) +vi.mock('@/lib/credential-groups/self-enrollment', () => ({ + createViewerCredentialGroupEnrollment: m.enrollment, +})) +vi.mock('@/lib/credential-groups/enrollments', () => ({ + getCredentialGroupOAuthContextForEnrollment: m.oauthContext, +})) +vi.mock('@/lib/credential-groups/oauth', () => ({ startCredentialGroupOAuth: m.oauthStart })) +vi.mock('@/lib/credential-groups/application/public-enrollment', () => ({ + completePublicCredentialGroupOAuth: { execute: m.oauthComplete }, +})) +vi.mock('@/lib/credential-groups/search-connection-completion', () => ({ + readSearchConnectionCompletion: m.receipt, +})) +vi.mock('@/connectors/registry', () => ({ + CONNECTOR_META_REGISTRY: { + github: { name: 'GitHub', auth: { mode: 'oauth', provider: 'github-repositories' } }, + }, +})) +vi.mock('@/lib/credentials/managed-oauth', () => ({ + ManagedOAuthCredentialError: class extends Error {}, +})) +vi.mock('@/lib/oauth/github-installation', () => ({ + GitHubInstallationError: class extends Error {}, + getGitHubInstallationConfiguration: () => ({ + configured: true, + installUrl: 'https://github.com/apps/test-search/installations/new', + }), +})) + +import { + cancelGitHubSearchSetup, + completeGitHubSearchSetup, + completeGitHubSetupReaderOAuth, + continueGitHubSearchSetup, + readGitHubSearchSetup, + selectGitHubSearchSetup, + startGitHubSearchSetup, +} from '@/lib/knowledge/application/github-setup' +import { + GITHUB_SETUP_TTL_MS, + readGitHubSetupAttempt, + saveGitHubSetupAttempt, +} from '@/lib/knowledge/github-setup-state' + +const principal = { kind: 'session', userId: 'admin', sessionId: 'browser-1' } as const +const input = { organizationId: 'organization', setupId: '550e8400-e29b-41d4-a716-446655440000' } +const scope = { ...input, userId: principal.userId, sessionId: principal.sessionId } +const installation = { + installationId: '42', + accountId: '7', + accountLogin: 'example', + accountType: 'Organization' as const, +} +const secondInstallation = { + ...installation, + installationId: '43', + accountId: '8', + accountLogin: 'another', +} +const reader = { id: 'reader' } +const resultCredential = { id: 'installation-credential', displayName: 'example' } +function admin(role = 'admin') { + queueTableRows(member, [{ role }]) +} +async function start(intent?: 'install') { + admin() + return startGitHubSearchSetup.execute({ + principal, + input: { ...input, ...(intent ? { intent } : {}) }, + }) +} +async function status() { + admin() + return readGitHubSearchSetup.execute({ principal, input }) +} +async function cancel() { + admin() + return cancelGitHubSearchSetup.execute({ principal, input }) +} +async function callback(state: string, installationId = '42') { + admin() + return completeGitHubSearchSetup.execute({ + principal, + input: { state, installationId, setupAction: 'install' }, + }) +} +async function resume() { + admin() + return continueGitHubSearchSetup.execute({ principal, input }) +} +const oauthAttempt = { + state: 'cg_fixture', + provider: 'github-repositories' as const, + userId: principal.userId, + organizationId: input.organizationId, + email: 'admin@example.test', + credentialGroupId: 'group', + enrollmentId: 'enrollment', + optionId: 'option', + authorizationAppId: 'app', + scopeVersion: 1, + nonceHash: 'hash', + requiredScopes: [], + redirectUri: 'https://sim.example/api/auth/oauth2/callback/github-repositories', + completionRedirect: true, + completionId: input.setupId, + returnTo: 'github-installation' as const, + invitationToken: 'fixture-invitation', + createdAt: 0, +} + +beforeEach(() => { + vi.clearAllMocks() + resetDbChainMock() + m.values.clear() + m.list.mockResolvedValue({ available: true, needsUserConnection: false, installations: [] }) + m.connect.mockResolvedValue({ credential: resultCredential, created: true }) + m.reader.mockResolvedValue(reader) + m.receipt.mockResolvedValue(null) + m.provision.mockResolvedValue({ credentialGroupId: 'group', credentialGroupOptionId: 'option' }) + m.enrollment.mockResolvedValue({ + enrollment: { id: 'enrollment', email: 'admin@example.test' }, + invitationLink: 'https://sim.example/credential-groups/enroll/fixture-invitation', + }) + m.oauthContext.mockResolvedValue({ fixture: 'oauth-context' }) + m.oauthStart.mockResolvedValue('https://github.com/login/oauth/authorize?state=cg_fixture') + m.oauthComplete.mockResolvedValue({ credentialId: 'reader' }) +}) +afterEach(() => { + vi.useRealTimers() + resetDbChainMock() +}) + +describe('GitHub setup lifecycle', () => { + it.each(['member', 'missing'])( + 'requires current organization admin access before state or provider work: %s', + async (role) => { + queueTableRows(member, role === 'missing' ? [] : [{ role }]) + await expect(startGitHubSearchSetup.execute({ principal, input })).rejects.toMatchObject({ + code: role === 'missing' ? 'not_found' : 'forbidden', + }) + expect(m.set).not.toHaveBeenCalled() + expect(m.list).not.toHaveBeenCalled() + } + ) + it('refuses non-session principals before state lookup', async () => { + await expect( + startGitHubSearchSetup.execute({ + principal: { kind: 'personal_api_key', userId: 'admin', keyId: 'key' }, + input, + }) + ).rejects.toMatchObject({ code: 'forbidden' }) + expect(m.get).not.toHaveBeenCalled() + }) + it('issues opaque state, verifies installation binding through the existing operation, and makes refresh idempotent', async () => { + const { url } = await start() + const state = new URL(url).searchParams.get('state')! + expect(state).not.toBe(input.setupId) + expect(url).not.toContain(input.organizationId) + expect(new URL(url).pathname).toBe('/apps/test-search/installations/new') + await callback(state) + expect(m.connect).toHaveBeenCalledWith( + expect.objectContaining({ + principal, + input: expect.objectContaining({ + organizationId: input.organizationId, + installationId: '42', + }), + }) + ) + await expect(status()).resolves.toEqual({ status: 'completed', credential: resultCredential }) + await callback(state) + expect(m.connect).toHaveBeenCalledTimes(1) + }) + it('reuses a sole eligible installation without forcing GitHub configuration', async () => { + m.list.mockResolvedValue({ + available: true, + needsUserConnection: false, + installations: [installation], + }) + const { url } = await start() + expect(new URL(url).pathname).toBe('/credential-groups/complete') + expect(new URL(url).searchParams.get('completionId')).toBe(input.setupId) + await expect(status()).resolves.toEqual({ status: 'completed', credential: resultCredential }) + }) + it('preserves explicit install intent even when one installation exists', async () => { + m.list.mockResolvedValue({ + available: true, + needsUserConnection: false, + installations: [installation], + }) + const { url } = await start('install') + expect(new URL(url).hostname).toBe('github.com') + expect(m.connect).not.toHaveBeenCalled() + }) + it('offers multiple installations using bounded metadata and rejects choices not offered', async () => { + m.list.mockResolvedValue({ + available: true, + needsUserConnection: false, + installations: [{ ...installation, appClientId: 'private-extra' }, secondInstallation], + }) + expect(new URL((await start()).url).pathname).toBe('/knowledge/github/setup') + await expect(status()).resolves.toEqual({ + status: 'choosing', + installations: [installation, secondInstallation], + }) + admin() + await expect( + selectGitHubSearchSetup.execute({ + principal, + input: { ...input, action: { kind: 'select', installationId: '999' } }, + }) + ).rejects.toMatchObject({ code: 'validation' }) + expect(m.connect).not.toHaveBeenCalled() + admin() + await selectGitHubSearchSetup.execute({ + principal, + input: { ...input, action: { kind: 'select', installationId: '43' } }, + }) + expect(m.connect).toHaveBeenCalledWith( + expect.objectContaining({ input: expect.objectContaining({ installationId: '43' }) }) + ) + }) + it('status reads never poll GitHub or return OAuth/state material', async () => { + await start() + m.list.mockClear() + await expect(status()).resolves.toEqual({ status: 'pending' }) + expect(m.list).not.toHaveBeenCalled() + expect(m.reader).not.toHaveBeenCalled() + }) + it.each([{ userId: 'other-user' }, { sessionId: 'other-browser' }])( + 'cannot read or consume another initiating session: %s', + async (change) => { + const state = new URL((await start()).url).searchParams.get('state')! + const other = { ...principal, ...change } + admin() + await expect(readGitHubSearchSetup.execute({ principal: other, input })).resolves.toEqual({ + status: 'expired', + }) + await expect( + completeGitHubSearchSetup.execute({ + principal: other, + input: { state, installationId: '42' }, + }) + ).rejects.toMatchObject({ code: 'validation' }) + expect(m.connect).not.toHaveBeenCalled() + await callback(state) + } + ) + it('keeps organization scopes isolated even when the correlation ID is known', async () => { + await start() + admin() + await expect( + readGitHubSearchSetup.execute({ + principal, + input: { ...input, organizationId: 'another-org' }, + }) + ).resolves.toEqual({ status: 'expired' }) + }) + it('cancellation prevents late callbacks and repeated begin with the same ID', async () => { + const state = new URL((await start()).url).searchParams.get('state')! + await cancel() + await expect(callback(state)).rejects.toMatchObject({ code: 'validation' }) + await expect(start()).rejects.toMatchObject({ code: 'validation' }) + await expect(status()).resolves.toEqual({ status: 'expired' }) + expect(m.connect).not.toHaveBeenCalled() + }) + it('a cancellation arriving before start leaves a tombstone', async () => { + await cancel() + await expect(start()).rejects.toMatchObject({ code: 'validation' }) + expect(m.list).not.toHaveBeenCalled() + }) + it('cancellation after committed completion preserves the result', async () => { + const state = new URL((await start()).url).searchParams.get('state')! + await callback(state) + await cancel() + await expect(status()).resolves.toEqual({ status: 'completed', credential: resultCredential }) + }) + it('expires without a sliding refresh window', async () => { + vi.useFakeTimers() + const { url } = await start() + vi.advanceTimersByTime(GITHUB_SETUP_TTL_MS - 1) + await expect(status()).resolves.toEqual({ status: 'pending' }) + vi.advanceTimersByTime(1) + await expect(status()).resolves.toEqual({ status: 'expired' }) + await expect(callback(new URL(url).searchParams.get('state')!)).rejects.toMatchObject({ + code: 'validation', + }) + expect(m.connect).not.toHaveBeenCalled() + }) + it('rechecks current administrator access on the callback', async () => { + const state = new URL((await start()).url).searchParams.get('state')! + admin('member') + await expect( + completeGitHubSearchSetup.execute({ principal, input: { state, installationId: '42' } }) + ).rejects.toMatchObject({ code: 'forbidden' }) + expect(m.connect).not.toHaveBeenCalled() + }) + it('records a safe failure and never exposes provider response bodies', async () => { + const state = new URL((await start()).url).searchParams.get('state')! + m.connect.mockRejectedValueOnce(new Error('secret_token fixture@example.test provider-body')) + await callback(state) + await expect(status()).resolves.toEqual({ + status: 'failed', + error: 'GitHub setup could not finish. Try again.', + }) + expect(JSON.stringify([...m.values.values()])).not.toContain('secret_token') + }) + it('an installation request awaiting GitHub approval cannot create a credential', async () => { + const state = new URL((await start()).url).searchParams.get('state')! + admin() + await completeGitHubSearchSetup.execute({ principal, input: { state, setupAction: 'request' } }) + expect(m.connect).not.toHaveBeenCalled() + await expect(status()).resolves.toMatchObject({ status: 'failed' }) + }) +}) + +describe('GitHub setup reader OAuth continuation', () => { + async function startOAuth(intent?: 'install') { + m.list.mockResolvedValueOnce({ available: true, needsUserConnection: true, installations: [] }) + queueTableRows(credential, []) + return start(intent) + } + it('starts the existing OAuth boundary and requires its receipt plus a current reader to continue', async () => { + await startOAuth() + expect(m.oauthStart).toHaveBeenCalledWith({ fixture: 'oauth-context' }, 'fixture-invitation', { + completionRedirect: true, + completionId: input.setupId, + returnTo: 'github-installation', + connectionIntent: { kind: 'create' }, + }) + await expect(resume()).rejects.toMatchObject({ code: 'forbidden' }) + m.receipt.mockResolvedValue('wrong-reader') + await expect(resume()).rejects.toMatchObject({ code: 'forbidden' }) + m.receipt.mockResolvedValue(reader.id) + expect(new URL((await resume()).url).hostname).toBe('github.com') + }) + it('preserves explicit install intent after OAuth with existing installations', async () => { + await startOAuth('install') + m.receipt.mockResolvedValue(reader.id) + m.list.mockResolvedValue({ + available: true, + needsUserConnection: false, + installations: [installation], + }) + expect(new URL((await resume()).url).hostname).toBe('github.com') + expect(m.connect).not.toHaveBeenCalled() + }) + it('a revoked enrollment stays denied before provider authorization', async () => { + m.enrollment.mockRejectedValueOnce(new Error('revoked')) + await startOAuth() + expect(m.oauthStart).not.toHaveBeenCalled() + await expect(status()).resolves.toMatchObject({ status: 'failed' }) + }) + it('requires the exact OAuth attempt and verified current user before completing the reader grant', async () => { + await startOAuth() + admin() + await expect( + completeGitHubSetupReaderOAuth.execute({ + principal, + input: { attempt: { ...oauthAttempt, state: 'wrong-state' }, code: 'fixture-code' }, + }) + ).rejects.toMatchObject({ code: 'validation' }) + expect(m.oauthComplete).not.toHaveBeenCalled() + admin() + queueTableRows(user, [{ emailVerified: false }]) + await expect( + completeGitHubSetupReaderOAuth.execute({ + principal, + input: { attempt: oauthAttempt, code: 'fixture-code' }, + }) + ).rejects.toMatchObject({ code: 'forbidden' }) + admin() + queueTableRows(user, [{ emailVerified: true }]) + await completeGitHubSetupReaderOAuth.execute({ + principal, + input: { attempt: oauthAttempt, code: 'fixture-code' }, + }) + expect(m.oauthComplete).toHaveBeenCalledWith( + expect.objectContaining({ + principal: expect.objectContaining({ + kind: 'credential_group_enrollment', + userId: 'admin', + organizationId: 'organization', + enrollmentId: 'enrollment', + }), + }) + ) + }) + it('cancellation prevents the intermediate OAuth grant as well as installation', async () => { + await startOAuth() + await cancel() + admin() + await expect( + completeGitHubSetupReaderOAuth.execute({ + principal, + input: { attempt: oauthAttempt, code: 'fixture-code' }, + }) + ).rejects.toMatchObject({ code: 'validation' }) + expect(m.oauthComplete).not.toHaveBeenCalled() + }) + it('maps OAuth failure classifications into the authoritative setup status', async () => { + await startOAuth() + admin() + await continueGitHubSearchSetup.execute({ + principal, + input: { ...input, oauth: 'github_email_mismatch' }, + }) + await expect(status()).resolves.toMatchObject({ + status: 'failed', + error: expect.stringContaining('verified secondary email'), + }) + expect(m.connect).not.toHaveBeenCalled() + }) +}) + +function deferred() { + let resolve!: (value: T) => void + const promise = new Promise((complete) => { + resolve = complete + }) + return { promise, resolve } +} + +describe('GitHub setup atomic claims', () => { + it('never wraps nested authorized operations in a global-pool transaction', async () => { + m.list.mockImplementationOnce(async () => { + expect(db.transaction).not.toHaveBeenCalled() + return { available: true, needsUserConnection: false, installations: [installation] } + }) + m.connect.mockImplementationOnce(async () => { + expect(db.transaction).not.toHaveBeenCalled() + return { credential: resultCredential } + }) + await start() + }) + it('cancellation during discovery prevents the final grant claim', async () => { + const entered = deferred() + const listed = deferred<{ + available: boolean + needsUserConnection: boolean + installations: (typeof installation)[] + }>() + m.list.mockImplementationOnce(() => { + entered.resolve() + return listed.promise + }) + const pending = start() + await entered.promise + await cancel() + listed.resolve({ available: true, needsUserConnection: false, installations: [installation] }) + await expect(pending).rejects.toMatchObject({ code: 'conflict' }) + expect(m.connect).not.toHaveBeenCalled() + await expect(status()).resolves.toEqual({ status: 'expired' }) + }) + it('a claimed final grant survives cancellation and concurrent callback replay without a duplicate grant', async () => { + const state = new URL((await start()).url).searchParams.get('state')! + const entered = deferred() + const connected = deferred<{ credential: typeof resultCredential }>() + m.connect.mockImplementationOnce(() => { + entered.resolve() + return connected.promise + }) + const pending = callback(state) + await entered.promise + await cancel() + await callback(state) + expect(m.connect).toHaveBeenCalledTimes(1) + connected.resolve({ credential: resultCredential }) + await pending + await expect(status()).resolves.toEqual({ status: 'completed', credential: resultCredential }) + }) + it('cancellation during a reader grant prevents subsequent installation continuation', async () => { + m.list.mockResolvedValueOnce({ available: true, needsUserConnection: true, installations: [] }) + queueTableRows(credential, []) + await start() + const entered = deferred() + const authorized = deferred<{ credentialId: string }>() + m.oauthComplete.mockImplementationOnce(() => { + entered.resolve() + return authorized.promise + }) + admin() + queueTableRows(user, [{ emailVerified: true }]) + const pending = completeGitHubSetupReaderOAuth.execute({ + principal, + input: { attempt: oauthAttempt, code: 'fixture-code' }, + }) + await entered.promise + await cancel() + authorized.resolve({ credentialId: 'reader' }) + await pending + m.receipt.mockResolvedValue('reader') + await expect(resume()).rejects.toMatchObject({ code: 'validation' }) + expect(m.connect).not.toHaveBeenCalled() + await expect(status()).resolves.toEqual({ status: 'expired' }) + }) + it('an interrupted claim is not replayed and expires into a recoverable new attempt', async () => { + vi.useFakeTimers() + const state = new URL((await start()).url).searchParams.get('state')! + const attempt = (await readGitHubSetupAttempt(scope))! + await saveGitHubSetupAttempt( + { ...scope, createdAt: attempt.createdAt, phase: 'connecting', installationId: '42', state }, + 'installing' + ) + await callback(state) + await cancel() + expect(m.connect).not.toHaveBeenCalled() + vi.advanceTimersByTime(GITHUB_SETUP_TTL_MS) + await expect(status()).resolves.toEqual({ status: 'expired' }) + m.list.mockResolvedValue({ + available: true, + needsUserConnection: false, + installations: [installation], + }) + admin() + await startGitHubSearchSetup.execute({ + principal, + input: { ...input, setupId: '660e8400-e29b-41d4-a716-446655440000' }, + }) + expect(m.connect).toHaveBeenCalledTimes(1) + }) + it('fails closed when Redis refuses the transition before the grant', async () => { + const state = new URL((await start()).url).searchParams.get('state')! + m.eval.mockRejectedValueOnce(new Error('Redis unavailable')) + await expect(callback(state)).rejects.toThrow('Redis unavailable') + expect(m.connect).not.toHaveBeenCalled() + }) +}) diff --git a/apps/sim/lib/knowledge/application/github-setup.ts b/apps/sim/lib/knowledge/application/github-setup.ts new file mode 100644 index 00000000000..9fdc061c52b --- /dev/null +++ b/apps/sim/lib/knowledge/application/github-setup.ts @@ -0,0 +1,515 @@ +import type { SessionPrincipal } from '@sim/auth/principal' +import { db } from '@sim/db' +import { credential, user } from '@sim/db/schema' +import { createLogger } from '@sim/logger' +import { sha256Hex } from '@sim/security/hash' +import { and, eq } from 'drizzle-orm' +import { asOrchestrationError, OrchestrationError } from '@/lib/core/orchestration/types' +import { completePublicCredentialGroupOAuth } from '@/lib/credential-groups/application/public-enrollment' +import { getCredentialGroupOAuthContextForEnrollment } from '@/lib/credential-groups/enrollments' +import { startCredentialGroupOAuth } from '@/lib/credential-groups/oauth' +import { + CREDENTIAL_GROUP_OAUTH_FAILURE_MESSAGES, + isCredentialGroupOAuthFailure, +} from '@/lib/credential-groups/oauth-completion' +import type { CredentialGroupOAuthAttempt } from '@/lib/credential-groups/oauth-state' +import { readSearchConnectionCompletion } from '@/lib/credential-groups/search-connection-completion' +import { createViewerCredentialGroupEnrollment } from '@/lib/credential-groups/self-enrollment' +import { ManagedOAuthCredentialError } from '@/lib/credentials/managed-oauth' +import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/authorized-knowledge-use-case' +import { resolveKnowledgeOrganizationContext } from '@/lib/knowledge/application/contexts' +import { + connectGitHubSearchInstallation, + findGitHubSearchReaderCredential, + listGitHubSearchInstallations, +} from '@/lib/knowledge/application/github-installations' +import { knowledgeOperations } from '@/lib/knowledge/application/operations' +import { provisionKnowledgeConnectorMembersBinding } from '@/lib/knowledge/connectors/member-provisioning' +import { + expiredGitHubSetupError, + type GitHubSetupAttempt, + type GitHubSetupScope, + githubSetupScope, + issueGitHubSetupCallback, + readGitHubSetupAttempt, + resolveGitHubSetupCallback, + saveGitHubSetupAttempt, +} from '@/lib/knowledge/github-setup-state' +import { githubSetupCompletionUrl, githubSetupPageUrl } from '@/lib/knowledge/github-setup-urls' +import { + GitHubInstallationError, + getGitHubInstallationConfiguration, +} from '@/lib/oauth/github-installation' +import { CONNECTOR_META_REGISTRY } from '@/connectors/registry' + +const logger = createLogger('GitHubSearchSetup') +interface SetupInput { + organizationId: string + setupId: string +} +interface SetupSelectionInput extends SetupInput { + action: { kind: 'select'; installationId: string } | { kind: 'install' } +} +interface SetupCallbackInput { + state: string + installationId?: string + setupAction?: string +} + +function safeSetupError(error: unknown): string { + if (error instanceof ManagedOAuthCredentialError) + return 'Reconnect your GitHub account, then start setup again.' + if (error instanceof GitHubInstallationError) + return error.status === 403 + ? 'Choose a GitHub account or organization you own and allow the required app permissions.' + : 'GitHub could not finish setup right now. Try again in a few minutes.' + const classified = asOrchestrationError(error) + if (classified && ['validation', 'forbidden', 'conflict'].includes(classified.code)) + return classified.message + return 'GitHub setup could not finish. Try again.' +} + +function installationUrl(state: string) { + const { installUrl } = getGitHubInstallationConfiguration() + if (!installUrl) throw new OrchestrationError('validation', 'GitHub App setup is unavailable.') + const url = new URL(installUrl) + url.searchParams.set('state', state) + return url.toString() +} + +function currentUrl(attempt: GitHubSetupAttempt): string { + switch (attempt.phase) { + case 'oauth': + return attempt.url + case 'authorizing': + return githubSetupPageUrl(attempt) + case 'installing': + return installationUrl(attempt.state) + case 'completed': + return githubSetupCompletionUrl(attempt.setupId) + case 'failed': + return githubSetupCompletionUrl(attempt.setupId, 'failed') + case 'cancelled': + throw expiredGitHubSetupError() + default: + return githubSetupPageUrl(attempt) + } +} + +async function requireAttempt(scope: GitHubSetupScope) { + const attempt = await readGitHubSetupAttempt(scope) + if (!attempt || attempt.phase === 'cancelled') throw expiredGitHubSetupError() + return attempt +} + +function base(attempt: GitHubSetupAttempt) { + return { + ...githubSetupScope(attempt), + createdAt: attempt.createdAt, + ...(attempt.intent ? { intent: attempt.intent } : {}), + } +} + +async function failAttempt( + attempt: GitHubSetupAttempt, + error: unknown, + stage: string = attempt.phase +) { + logger.warn('GitHub setup failed', { + stage, + errorType: + error instanceof TypeError + ? 'type_error' + : error instanceof SyntaxError + ? 'syntax_error' + : error instanceof Error + ? 'error' + : 'unknown', + ...(error instanceof Error && { fingerprint: sha256Hex(error.message).slice(0, 12) }), + failure: + asOrchestrationError(error)?.code ?? + (error instanceof GitHubInstallationError ? 'provider' : 'unexpected'), + ...(error instanceof GitHubInstallationError && { providerStatus: error.status }), + }) + await saveGitHubSetupAttempt( + { ...base(attempt), phase: 'failed', error: safeSetupError(error) }, + attempt.phase + ) + return { url: githubSetupCompletionUrl(attempt.setupId, 'failed') } +} + +async function connectAttempt( + principal: SessionPrincipal, + attempt: GitHubSetupAttempt, + installationId: string +) { + const connecting: GitHubSetupAttempt = { + ...base(attempt), + phase: 'connecting', + installationId, + ...('state' in attempt && attempt.state ? { state: attempt.state } : {}), + } + await saveGitHubSetupAttempt(connecting, attempt.phase) + let result + try { + result = await connectGitHubSearchInstallation.execute({ + principal, + input: { + organizationId: attempt.organizationId, + installationId, + signal: AbortSignal.timeout(60_000), + }, + }) + } catch (error) { + return failAttempt(connecting, error) + } + /** An interrupted claim expires; a fresh attempt reuses the idempotent installation binding. */ + await saveGitHubSetupAttempt( + { + ...base(attempt), + phase: 'completed', + credential: result.credential, + }, + 'connecting' + ) + return { url: githubSetupCompletionUrl(attempt.setupId) } +} + +async function installAttempt(attempt: GitHubSetupAttempt) { + const state = await issueGitHubSetupCallback(attempt) + await saveGitHubSetupAttempt({ ...base(attempt), phase: 'installing', state }, attempt.phase) + return { url: installationUrl(state) } +} + +async function discoverInstallations(principal: SessionPrincipal, attempt: GitHubSetupAttempt) { + const result = await listGitHubSearchInstallations.execute({ + principal, + input: { organizationId: attempt.organizationId, signal: AbortSignal.timeout(60_000) }, + }) + if (!result.available) + throw new OrchestrationError('validation', 'GitHub App setup is unavailable.') + if (result.needsUserConnection) return null + if (attempt.intent === 'install' || result.installations.length === 0) + return installAttempt(attempt) + if (result.installations.length === 1) + return connectAttempt(principal, attempt, result.installations[0]!.installationId) + await saveGitHubSetupAttempt( + { + ...base(attempt), + phase: 'choosing', + installations: result.installations.map( + ({ installationId, accountId, accountLogin, accountType }) => ({ + installationId, + accountId, + accountLogin, + accountType, + }) + ), + }, + attempt.phase + ) + return { url: githubSetupPageUrl(attempt) } +} + +async function authorizeReader(attempt: GitHubSetupAttempt) { + const binding = await provisionKnowledgeConnectorMembersBinding({ + organizationId: attempt.organizationId, + userId: attempt.userId, + connectorMeta: CONNECTOR_META_REGISTRY.github!, + }) + const { enrollment, invitationLink } = await createViewerCredentialGroupEnrollment({ + organizationId: attempt.organizationId, + userId: attempt.userId, + credentialGroupId: binding.credentialGroupId, + }) + const token = new URL(invitationLink).pathname.split('/').at(-1) + if (!token) throw new Error('Account enrollment did not return an invitation token') + const context = await getCredentialGroupOAuthContextForEnrollment( + { + organizationId: attempt.organizationId, + credentialGroupId: binding.credentialGroupId, + enrollmentId: enrollment.id, + email: enrollment.email, + userId: attempt.userId, + }, + binding.credentialGroupOptionId + ) + if (!context) + throw new OrchestrationError('forbidden', 'This account connection is no longer available.') + const [existing] = await db + .select({ id: credential.id }) + .from(credential) + .where( + and( + eq(credential.organizationId, attempt.organizationId), + eq(credential.type, 'managed_oauth'), + eq(credential.credentialGroupEnrollmentId, enrollment.id), + eq(credential.credentialGroupOptionId, binding.credentialGroupOptionId) + ) + ) + .limit(1) + const url = await startCredentialGroupOAuth(context, token, { + completionRedirect: true, + completionId: attempt.setupId, + returnTo: 'github-installation', + connectionIntent: existing + ? { kind: 'reconnect', credentialId: existing.id } + : { kind: 'create' }, + }) + await saveGitHubSetupAttempt({ ...base(attempt), phase: 'oauth', url }, attempt.phase) + return { url } +} + +export const startGitHubSearchSetup = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.startGitHubSetup, + resolveContext: ({ input }: { input: SetupInput & { intent?: 'install' } }) => + resolveKnowledgeOrganizationContext(input), + async execute({ principal, input }) { + const scope = { + organizationId: input.organizationId, + setupId: input.setupId, + userId: principal.userId, + sessionId: principal.sessionId, + } + const existing = await readGitHubSetupAttempt(scope) + if (existing && existing.phase !== 'starting') return { url: currentUrl(existing) } + if (await readSearchConnectionCompletion({ ...scope, completionId: input.setupId })) + throw new OrchestrationError('conflict', 'Start a new GitHub setup attempt.') + const attempt: GitHubSetupAttempt = existing ?? { + ...scope, + phase: 'starting', + createdAt: Date.now(), + ...(input.intent ? { intent: input.intent } : {}), + } + if (!existing) await saveGitHubSetupAttempt(attempt, 'new') + let stage = 'discovery' + try { + const discovered = await discoverInstallations(principal, attempt) + if (discovered) return discovered + stage = 'reader_authorization' + return await authorizeReader(attempt) + } catch (error) { + return failAttempt(attempt, error, stage) + } + }, +}) + +export const readGitHubSearchSetup = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.readGitHubSetup, + resolveContext: ({ input }: { input: SetupInput }) => resolveKnowledgeOrganizationContext(input), + async execute({ principal, input }) { + const attempt = await readGitHubSetupAttempt({ + ...input, + userId: principal.userId, + sessionId: principal.sessionId, + }) + if (!attempt || attempt.phase === 'cancelled') return { status: 'expired' as const } + if (attempt.phase === 'completed') + return { status: 'completed' as const, credential: attempt.credential } + if (attempt.phase === 'failed') return { status: 'failed' as const, error: attempt.error } + if (attempt.phase === 'choosing') + return { status: 'choosing' as const, installations: attempt.installations } + return { status: 'pending' as const } + }, +}) + +export const cancelGitHubSearchSetup = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.cancelGitHubSetup, + resolveContext: ({ input }: { input: SetupInput }) => resolveKnowledgeOrganizationContext(input), + async execute({ principal, input }) { + const scope = { ...input, userId: principal.userId, sessionId: principal.sessionId } + for (let retry = 0; retry < 3; retry++) { + const attempt = await readGitHubSetupAttempt(scope) + if (attempt?.phase === 'completed' || attempt?.phase === 'connecting') break + try { + await saveGitHubSetupAttempt( + attempt + ? { ...base(attempt), phase: 'cancelled' } + : { ...scope, createdAt: Date.now(), phase: 'cancelled' }, + attempt?.phase ?? 'new' + ) + break + } catch (error) { + if (asOrchestrationError(error)?.code !== 'conflict' || retry === 2) throw error + } + } + return { success: true as const } + }, +}) + +export const continueGitHubSearchSetup = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.continueGitHubSetup, + resolveContext: ({ input }: { input: SetupInput & { oauth?: string } }) => + resolveKnowledgeOrganizationContext(input), + async execute({ principal, input }) { + const scope = { + organizationId: input.organizationId, + setupId: input.setupId, + userId: principal.userId, + sessionId: principal.sessionId, + } + const attempt = await requireAttempt(scope) + if (attempt.phase !== 'oauth' && attempt.phase !== 'authorizing') + return { url: currentUrl(attempt) } + if (isCredentialGroupOAuthFailure(input.oauth)) { + await saveGitHubSetupAttempt( + { + ...base(attempt), + phase: 'failed', + error: CREDENTIAL_GROUP_OAUTH_FAILURE_MESSAGES[input.oauth], + }, + attempt.phase + ) + return { url: githubSetupCompletionUrl(input.setupId, input.oauth) } + } + const receipt = await readSearchConnectionCompletion({ + ...scope, + completionId: input.setupId, + }) + const reader = await findGitHubSearchReaderCredential( + db, + input.organizationId, + principal.userId + ) + if (!receipt || reader?.id !== receipt) + throw new OrchestrationError( + 'forbidden', + 'Complete GitHub authorization before continuing setup.' + ) + try { + const result = await discoverInstallations(principal, attempt) + if (!result) + throw new OrchestrationError('forbidden', 'Your GitHub connection is no longer available.') + return result + } catch (error) { + return failAttempt(attempt, error) + } + }, +}) + +export const selectGitHubSearchSetup = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.selectGitHubSetup, + resolveContext: ({ input }: { input: SetupSelectionInput }) => + resolveKnowledgeOrganizationContext(input), + async execute({ principal, input }) { + const scope = { + organizationId: input.organizationId, + setupId: input.setupId, + userId: principal.userId, + sessionId: principal.sessionId, + } + const attempt = await requireAttempt(scope) + if (attempt.phase === 'completed') return { url: currentUrl(attempt) } + if (attempt.phase === 'connecting') return { url: currentUrl(attempt) } + if (attempt.phase !== 'choosing') + throw new OrchestrationError('conflict', 'Refresh GitHub setup before making a selection.') + if (input.action.kind === 'install') return installAttempt(attempt) + const { installationId } = input.action + if ( + !attempt.installations.some((installation) => installation.installationId === installationId) + ) + throw new OrchestrationError( + 'validation', + 'Choose one of the available GitHub installations.' + ) + return connectAttempt(principal, attempt, installationId) + }, +}) + +export const completeGitHubSearchSetup = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.completeGitHubSetup, + async resolveContext({ + principal, + input, + }: { + principal: SessionPrincipal + input: SetupCallbackInput + }) { + const scope = await resolveGitHubSetupCallback(input.state, principal) + return { ...(await resolveKnowledgeOrganizationContext(scope)), setupScope: scope } + }, + async execute({ principal, input, context }) { + const attempt = await requireAttempt(context.setupScope) + if (attempt.phase === 'completed' || attempt.phase === 'failed') + return { url: currentUrl(attempt) } + if ( + (attempt.phase !== 'installing' && attempt.phase !== 'connecting') || + attempt.state !== input.state + ) + throw expiredGitHubSetupError() + if (attempt.phase === 'connecting') { + if (attempt.installationId !== input.installationId) throw expiredGitHubSetupError() + return { url: currentUrl(attempt) } + } + if ( + !input.installationId || + (input.setupAction && !['install', 'update'].includes(input.setupAction)) + ) + return failAttempt( + attempt, + new OrchestrationError( + 'validation', + 'GitHub installation was not completed. Ask a GitHub organization owner to install the app, then try again.' + ) + ) + return connectAttempt(principal, attempt, input.installationId) + }, +}) + +/** An installation setup keeps its admin authority and cancellation boundary during reader OAuth. */ +export const completeGitHubSetupReaderOAuth = defineAuthorizedKnowledgeUseCase({ + operation: knowledgeOperations.completeGitHubSetupOAuth, + resolveContext: ({ + principal, + input, + }: { + principal: SessionPrincipal + input: { attempt: CredentialGroupOAuthAttempt; code: string } + }) => { + const { attempt } = input + if ( + attempt.provider !== 'github-repositories' || + attempt.returnTo !== 'github-installation' || + !attempt.organizationId || + !attempt.completionId || + attempt.userId !== principal.userId + ) + throw expiredGitHubSetupError() + return resolveKnowledgeOrganizationContext({ organizationId: attempt.organizationId }) + }, + async execute({ principal, input, context, request }) { + const scope = { + organizationId: context.organizationId, + setupId: input.attempt.completionId!, + userId: principal.userId, + sessionId: principal.sessionId, + } + const setup = await requireAttempt(scope) + if ( + setup.phase !== 'oauth' || + new URL(setup.url).searchParams.get('state') !== input.attempt.state + ) + throw expiredGitHubSetupError() + const [viewer] = await db + .select({ emailVerified: user.emailVerified }) + .from(user) + .where(eq(user.id, principal.userId)) + .limit(1) + if (!viewer?.emailVerified) + throw new OrchestrationError('forbidden', 'Verify your Sim email before connecting GitHub.') + await saveGitHubSetupAttempt({ ...base(setup), phase: 'authorizing', url: setup.url }, 'oauth') + return completePublicCredentialGroupOAuth.execute({ + principal: { + kind: 'credential_group_enrollment', + userId: principal.userId, + organizationId: scope.organizationId, + credentialGroupId: input.attempt.credentialGroupId, + enrollmentId: input.attempt.enrollmentId, + email: input.attempt.email, + invitationTokenHash: sha256Hex(input.attempt.invitationToken), + }, + input, + request, + }) + }, +}) diff --git a/apps/sim/lib/knowledge/application/operations.test.ts b/apps/sim/lib/knowledge/application/operations.test.ts index a2d8f28daaf..0dabc3ef7fc 100644 --- a/apps/sim/lib/knowledge/application/operations.test.ts +++ b/apps/sim/lib/knowledge/application/operations.test.ts @@ -16,6 +16,13 @@ describe('knowledge operation registry', () => { it('defines unique stable semantic operation IDs', () => { const ids = Object.values(knowledgeOperations).map((operation) => operation.id) expect(ids).toEqual([ + 'knowledge.github.setup.oauth.complete', + 'knowledge.github.setup.start', + 'knowledge.github.setup.read', + 'knowledge.github.setup.cancel', + 'knowledge.github.setup.continue', + 'knowledge.github.setup.select', + 'knowledge.github.setup.complete', 'knowledge.github.installations.list', 'knowledge.github.installations.connect', 'knowledge.slack.prepare', diff --git a/apps/sim/lib/knowledge/application/operations.ts b/apps/sim/lib/knowledge/application/operations.ts index 87d98e6e53c..09f8225cb62 100644 --- a/apps/sim/lib/knowledge/application/operations.ts +++ b/apps/sim/lib/knowledge/application/operations.ts @@ -102,6 +102,69 @@ const HUMAN_COPILOT_AND_EXECUTOR_PRINCIPAL_POLICY = { } as const export const knowledgeOperations = { + completeGitHubSetupOAuth: defineKnowledgeOperation( + defineWorkspaceOperation({ + id: 'knowledge.github.setup.oauth.complete', + minimumRole: 'admin', + workspaceApiKey: 'deny', + capability: 'knowledge.use', + principalKinds: ['session'], + }) + ), + startGitHubSetup: defineKnowledgeOperation( + defineWorkspaceOperation({ + id: 'knowledge.github.setup.start', + minimumRole: 'admin', + workspaceApiKey: 'deny', + capability: 'knowledge.use', + principalKinds: ['session'], + }) + ), + readGitHubSetup: defineKnowledgeOperation( + defineWorkspaceOperation({ + id: 'knowledge.github.setup.read', + minimumRole: 'admin', + workspaceApiKey: 'deny', + capability: 'knowledge.use', + principalKinds: ['session'], + }) + ), + cancelGitHubSetup: defineKnowledgeOperation( + defineWorkspaceOperation({ + id: 'knowledge.github.setup.cancel', + minimumRole: 'admin', + workspaceApiKey: 'deny', + capability: 'knowledge.use', + principalKinds: ['session'], + }) + ), + continueGitHubSetup: defineKnowledgeOperation( + defineWorkspaceOperation({ + id: 'knowledge.github.setup.continue', + minimumRole: 'admin', + workspaceApiKey: 'deny', + capability: 'knowledge.use', + principalKinds: ['session'], + }) + ), + selectGitHubSetup: defineKnowledgeOperation( + defineWorkspaceOperation({ + id: 'knowledge.github.setup.select', + minimumRole: 'admin', + workspaceApiKey: 'deny', + capability: 'knowledge.use', + principalKinds: ['session'], + }) + ), + completeGitHubSetup: defineKnowledgeOperation( + defineWorkspaceOperation({ + id: 'knowledge.github.setup.complete', + minimumRole: 'admin', + workspaceApiKey: 'deny', + capability: 'knowledge.use', + principalKinds: ['session'], + }) + ), listGitHubInstallations: defineKnowledgeOperation( defineWorkspaceOperation({ id: 'knowledge.github.installations.list', diff --git a/apps/sim/lib/knowledge/application/organization-search-overview.ts b/apps/sim/lib/knowledge/application/organization-search-overview.ts index eb58334a776..97ebe9209cc 100644 --- a/apps/sim/lib/knowledge/application/organization-search-overview.ts +++ b/apps/sim/lib/knowledge/application/organization-search-overview.ts @@ -8,7 +8,7 @@ import { knowledgeConnectorSyncLog, organizationSearchIntegration, } from '@sim/db/schema' -import { and, eq, exists, inArray, isNotNull, isNull, sql } from 'drizzle-orm' +import { and, eq, exists, inArray, isNotNull, isNull, type SQL, sql } from 'drizzle-orm' import { OrchestrationError } from '@/lib/core/orchestration/types' import { resolveKnowledgeAccessAvailability } from '@/lib/knowledge/access/availability' import { SOURCE_ACL_MAX_AGE_MS } from '@/lib/knowledge/access/freshness' @@ -17,6 +17,7 @@ import { resolveKnowledgeOwnerContext } from '@/lib/knowledge/application/contex import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { SOURCE_CONTENT_ERROR } from '@/lib/knowledge/connectors/sync-limits' import { MAX_SEARCH_SOURCE_PROVIDER_TYPES } from '@/lib/knowledge/constants' +import { failedDocumentCondition } from '@/lib/knowledge/documents/processing-status' import { canConnectWithDefaults, SEARCH_SOURCE_TYPES } from '@/lib/sim-search/connectors' interface OrganizationSearchOverviewInput { @@ -136,7 +137,7 @@ export const readOrganizationSearchOverview = defineAuthorizedKnowledgeUseCase({ ) ) ) - const hasDocumentsInState = (statuses: string[]) => + const hasDocumentsInState = (condition: SQL) => exists( db .select({ id: document.id }) @@ -149,7 +150,7 @@ export const readOrganizationSearchOverview = defineAuthorizedKnowledgeUseCase({ eq(document.userExcluded, false), isNull(document.archivedAt), isNull(document.deletedAt), - inArray(document.processingStatus, statuses) + condition ) ) ) @@ -194,7 +195,7 @@ export const readOrganizationSearchOverview = defineAuthorizedKnowledgeUseCase({ hasError: sql`bool_or(NOT ${paused} AND ( ${knowledgeConnector.status} = 'error' OR ${knowledgeConnector.lastSyncError} IS NOT NULL - OR ${hasDocumentsInState(['failed'])} + OR ${hasDocumentsInState(failedDocumentCondition())} OR (${knowledgeConnector.accessMode} = 'admin' AND ${latestCentralRunHasError}) OR (${knowledgeConnector.accessMode} = 'members' AND ( ${knowledgeConnector.memberSyncStatus} = 'error' @@ -203,12 +204,12 @@ export const readOrganizationSearchOverview = defineAuthorizedKnowledgeUseCase({ )) ))`, hasAccountError: sql`bool_or(NOT ${paused} AND ${knowledgeConnector.accessMode} = 'members' AND ${hasMemberError})`, - hasDocumentError: sql`bool_or(NOT ${paused} AND ${hasDocumentsInState(['failed'])})`, + hasDocumentError: sql`bool_or(NOT ${paused} AND ${hasDocumentsInState(failedDocumentCondition())})`, hasIndexing: sql`bool_or(NOT ${paused} AND (${knowledgeConnector.accessMode} <> 'members' OR ${hasActiveMembers} OR ${knowledgeConnector.credentialId} IS NOT NULL) AND ( ${knowledgeConnector.status} IN ('pending', 'syncing') - OR ${continuing} OR ${hasDocumentsInState(['pending', 'processing'])} + OR ${continuing} OR ${hasDocumentsInState(inArray(document.processingStatus, ['pending', 'processing']))} OR (${knowledgeConnector.accessMode} = 'members' AND ( ${knowledgeConnector.memberSyncStatus} IN ('pending', 'running') OR ${hasMemberFirstListing} )) diff --git a/apps/sim/lib/knowledge/application/search-source-progress.ts b/apps/sim/lib/knowledge/application/search-source-progress.ts index 5543c3ec974..25e30956d0b 100644 --- a/apps/sim/lib/knowledge/application/search-source-progress.ts +++ b/apps/sim/lib/knowledge/application/search-source-progress.ts @@ -1,6 +1,6 @@ import { db } from '@sim/db' import { document, knowledgeBase, knowledgeConnector } from '@sim/db/schema' -import { and, eq, exists, inArray, isNull, sql } from 'drizzle-orm' +import { and, eq, exists, inArray, isNull, type SQL, sql } from 'drizzle-orm' import { OrchestrationError } from '@/lib/core/orchestration/types' import { type ResourceOwner, resourceScopeFromOwner } from '@/lib/core/resource-scope' import { resourceScopeCondition } from '@/lib/core/resource-scope.server' @@ -10,6 +10,7 @@ import { defineAuthorizedKnowledgeUseCase } from '@/lib/knowledge/application/au import { resolveKnowledgeOwnerContext } from '@/lib/knowledge/application/contexts' import { knowledgeOperations } from '@/lib/knowledge/application/operations' import { MAX_SEARCH_SOURCE_PROGRESS_ITEMS } from '@/lib/knowledge/constants' +import { failedDocumentCondition } from '@/lib/knowledge/documents/processing-status' import { searchIntegrationAccessCondition } from '@/lib/knowledge/search/integration-policy' interface ReadSearchSourceProgressInput extends ResourceOwner { @@ -34,7 +35,7 @@ export const readSearchSourceProgress = defineAuthorizedKnowledgeUseCase({ const access = await createKnowledgeAccessProvider(principal, context).getForConnectors( input.connectorIds ) - const hasDocumentsInState = (statuses: string[]) => + const hasDocumentsInState = (condition: SQL) => sql`${exists( db .select({ id: document.id }) @@ -43,7 +44,7 @@ export const readSearchSourceProgress = defineAuthorizedKnowledgeUseCase({ and( eq(document.knowledgeBaseId, knowledgeConnector.knowledgeBaseId), eq(document.connectorId, knowledgeConnector.id), - inArray(document.processingStatus, statuses), + condition, eq(document.enabled, true), eq(document.userExcluded, false), isNull(document.archivedAt), @@ -60,8 +61,10 @@ export const readSearchSourceProgress = defineAuthorizedKnowledgeUseCase({ memberSyncStatus: knowledgeConnector.memberSyncStatus, hasRetainedSyncError: sql`${knowledgeConnector.lastSyncError} IS NOT NULL`, approved: sql`${searchIntegrationAccessCondition()}`, - isIndexing: hasDocumentsInState(['pending', 'processing']), - hasIndexingError: hasDocumentsInState(['failed']), + isIndexing: hasDocumentsInState( + inArray(document.processingStatus, ['pending', 'processing']) + ), + hasIndexingError: hasDocumentsInState(failedDocumentCondition()), }) .from(knowledgeConnector) .innerJoin(knowledgeBase, eq(knowledgeBase.id, knowledgeConnector.knowledgeBaseId)) diff --git a/apps/sim/lib/knowledge/application/search-sources.test.ts b/apps/sim/lib/knowledge/application/search-sources.test.ts index d806ccb77b5..91081980cce 100644 --- a/apps/sim/lib/knowledge/application/search-sources.test.ts +++ b/apps/sim/lib/knowledge/application/search-sources.test.ts @@ -646,7 +646,23 @@ describe('bounded Search source pagination', () => { ]) }) - it.each(['filter', 'provider', 'viewer', 'scope'] as const)( + it('excludes an account-level provider before paginating repository sources', async () => { + seed([source('drive')]) + await listSearchSources.execute({ + principal, + input: { ...input, excludeConnectorType: 'github' }, + }) + expect(dbChainMockFns.where.mock.calls).toContainEqual([ + expect.objectContaining({ + type: 'and', + conditions: expect.arrayContaining([ + { type: 'ne', left: knowledgeConnector.connectorType, right: 'github' }, + ]), + }), + ]) + }) + + it.each(['filter', 'provider', 'excluded-provider', 'viewer', 'scope'] as const)( 'rejects a cursor replayed under a different %s before reading sources', async (change) => { seed(rows(26)) @@ -666,6 +682,7 @@ describe('bounded Search source pagination', () => { cursor: first.nextCursor!, ...(change === 'filter' ? { mine: true } : {}), ...(change === 'provider' ? { connectorType: 'gmail' } : {}), + ...(change === 'excluded-provider' ? { excludeConnectorType: 'github' } : {}), }, }) ).rejects.toMatchObject({ code: 'validation' }) diff --git a/apps/sim/lib/knowledge/application/search-sources.ts b/apps/sim/lib/knowledge/application/search-sources.ts index 8f33d05cfe1..1d6129acac5 100644 --- a/apps/sim/lib/knowledge/application/search-sources.ts +++ b/apps/sim/lib/knowledge/application/search-sources.ts @@ -1,7 +1,7 @@ import { requirePrincipalSubjectUserId } from '@sim/auth/principal' import { db } from '@sim/db' import { document, embedding, knowledgeBase, knowledgeConnector, user } from '@sim/db/schema' -import { and, desc, eq, exists, inArray, isNull, lt, or, sql } from 'drizzle-orm' +import { and, desc, eq, exists, inArray, isNull, lt, ne, or, sql } from 'drizzle-orm' import { listSearchSourcesContract, searchSourceCursorSchema, @@ -22,6 +22,7 @@ import { SEARCH_SOURCE_CANDIDATE_PAGE_SIZE, SEARCH_SOURCE_PAGE_SIZE, } from '@/lib/knowledge/constants' +import { failedDocumentCondition } from '@/lib/knowledge/documents/processing-status' import { listOrganizationSearchApprovals } from '@/lib/knowledge/search/integration-policy' import { describeSearchSource } from '@/lib/sim-search/source-identity' import { getConnectorMeta } from '@/connectors/registry' @@ -30,6 +31,7 @@ export interface ListSearchSourcesInput extends ResourceOwner { cursor?: string connectorId?: string connectorType?: string + excludeConnectorType?: string search?: string mine?: boolean } @@ -43,12 +45,14 @@ export const listSearchSources = defineAuthorizedKnowledgeUseCase({ const userId = requirePrincipalSubjectUserId(principal) const search = input.search?.trim().toLowerCase() ?? '' const connectorType = input.connectorType?.trim() + const excludeConnectorType = input.excludeConnectorType?.trim() const cursorScope = cursorScopeKey(cursorRoute(listSearchSourcesContract), { workspaceId: context.workspaceId, organizationId: context.organizationId, userId: userId, search, connectorType: connectorType ?? '', + ...(excludeConnectorType ? { excludeConnectorType } : {}), connectorId: input.connectorId ?? '', mine: input.mine === true, order: 'newest', @@ -95,6 +99,9 @@ export const listSearchSources = defineAuthorizedKnowledgeUseCase({ isNull(knowledgeConnector.archivedAt), isNull(knowledgeConnector.deletedAt), connectorType ? eq(knowledgeConnector.connectorType, connectorType) : undefined, + excludeConnectorType + ? ne(knowledgeConnector.connectorType, excludeConnectorType) + : undefined, input.connectorId ? eq(knowledgeConnector.id, input.connectorId) : undefined, cursor ? or( @@ -179,7 +186,7 @@ export const listSearchSources = defineAuthorizedKnowledgeUseCase({ .where(and(eq(embedding.documentId, document.id), eq(embedding.enabled, true))) )} )::int`, - failedCount: sql`count(*) FILTER (WHERE ${document.processingStatus} = 'failed')::int`, + failedCount: sql`count(*) FILTER (WHERE ${failedDocumentCondition()})::int`, isIndexing: sql`bool_or(${document.processingStatus} IN ('pending', 'processing'))`, }) .from(document) diff --git a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts index e4b2c551769..dc11c0d278d 100644 --- a/apps/sim/lib/knowledge/connectors/sync-engine.test.ts +++ b/apps/sim/lib/knowledge/connectors/sync-engine.test.ts @@ -276,7 +276,7 @@ describe('resolveTagMapping', () => { describe('classifyExternalDoc', () => { const base = { content: 'hello', contentDeferred: false, contentHash: 'h1' } - it('records a new skipped file as a failed row', async () => { + it('records a new skipped file as an intentional exclusion', async () => { const { classifyExternalDoc } = await import('@/lib/knowledge/connectors/sync-primitives') expect( classifyExternalDoc({ ...base, content: '', skippedReason: 'too big' }, undefined) diff --git a/apps/sim/lib/knowledge/connectors/sync-persistence.ts b/apps/sim/lib/knowledge/connectors/sync-persistence.ts index 0dbdbe4935b..3598bd78e8f 100644 --- a/apps/sim/lib/knowledge/connectors/sync-persistence.ts +++ b/apps/sim/lib/knowledge/connectors/sync-persistence.ts @@ -258,7 +258,7 @@ export interface KnowledgeBaseOwner { userId: string } -/** Builds a content-less `failed` document row for a skipped (e.g. oversized) file. */ +/** Builds a content-less document row for an intentional source exclusion. */ function buildSkippedDocumentRow( knowledgeBaseId: string, connectorId: string, @@ -295,7 +295,7 @@ function buildSkippedDocumentRow( } /** - * Records source files that were intentionally not indexed as content-less `failed` + * Records source files that were intentionally not indexed as content-less * documents. New rows are inserted in bulk; authoritative skips replace stale rows. * This keeps the files visible in the knowledge base UI — with `processingError` * explaining why — instead of silently dropping them. The rows have no storage key, diff --git a/apps/sim/lib/knowledge/connectors/viewer-source-accounts.ts b/apps/sim/lib/knowledge/connectors/viewer-source-accounts.ts index 2c9041e4acb..924dd0f386b 100644 --- a/apps/sim/lib/knowledge/connectors/viewer-source-accounts.ts +++ b/apps/sim/lib/knowledge/connectors/viewer-source-accounts.ts @@ -1,8 +1,6 @@ -import { db } from '@sim/db' -import { credential, credentialGroup, credentialGroupEnrollment } from '@sim/db/schema' -import { and, eq, inArray, isNull, or } from 'drizzle-orm' -import { resourceScopeCondition } from '@/lib/core/resource-scope.server' -import { SEARCH_SOURCE_CANDIDATE_PAGE_SIZE } from '@/lib/knowledge/constants' +import { credential, credentialGroup } from '@sim/db/schema' +import { and, eq, or } from 'drizzle-orm' +import { listViewerOrganizationAccounts } from '@/lib/credential-groups/viewer-accounts' import { getConnectorMeta } from '@/connectors/registry' interface ViewerSourceAccount { @@ -52,36 +50,11 @@ export async function resolveViewerSourceAccounts(input: { }) const result = new Map() if (!matches.length) return result - const scope = { kind: 'organization', organizationId: input.organizationId } as const - const accounts = await db - .select({ - credentialId: credential.id, - displayName: credential.displayName, - status: credential.managedOauthStatus, - groupId: credentialGroup.id, - optionId: credential.credentialGroupOptionId, - providerId: credential.providerId, - }) - .from(credential) - .innerJoin( - credentialGroupEnrollment, - eq(credentialGroupEnrollment.id, credential.credentialGroupEnrollmentId) - ) - .innerJoin(credentialGroup, eq(credentialGroup.id, credentialGroupEnrollment.credentialGroupId)) - .where( - and( - resourceScopeCondition(credential, scope), - resourceScopeCondition(credentialGroup, scope), - eq(credentialGroupEnrollment.userId, input.userId), - eq(credential.type, 'managed_oauth'), - inArray(credential.managedOauthStatus, ['active', 'needs_reauth']), - isNull(credential.revokedAt), - or(...matches) - ) - ) - .limit(SEARCH_SOURCE_CANDIDATE_PAGE_SIZE + 1) - if (accounts.length > SEARCH_SOURCE_CANDIDATE_PAGE_SIZE) - throw new Error('Too many personal accounts for the source page') + const accounts = await listViewerOrganizationAccounts({ + organizationId: input.organizationId, + userId: input.userId, + matching: or(...matches)!, + }) for (const { source, providerId } of bindings) { const own = accounts.filter((account) => source.accessMode === 'members' diff --git a/apps/sim/lib/knowledge/documents/processing-outbox-handler.test.ts b/apps/sim/lib/knowledge/documents/processing-outbox-handler.test.ts index 08669fa0a65..4ea7dc6b006 100644 --- a/apps/sim/lib/knowledge/documents/processing-outbox-handler.test.ts +++ b/apps/sim/lib/knowledge/documents/processing-outbox-handler.test.ts @@ -153,16 +153,17 @@ describe('knowledge document processing outbox handler', () => { ) }) - it.each([null, { ...DOCUMENT, processingStatus: 'completed' }])( - 'completes without redispatch when the document is absent or completed', - async (document) => { - mocks.getKnowledgeDocument.mockResolvedValueOnce(document) + it.each([ + null, + { ...DOCUMENT, processingStatus: 'completed' }, + { ...DOCUMENT, processingStatus: 'failed', processingOutcome: 'skipped' }, + ])('completes without redispatch when the document is absent or terminal', async (document) => { + mocks.getKnowledgeDocument.mockResolvedValueOnce(document) - await handler()(PAYLOAD, createContext()) + await handler()(PAYLOAD, createContext()) - expect(mocks.processDocumentsWithQueue).not.toHaveBeenCalled() - } - ) + expect(mocks.processDocumentsWithQueue).not.toHaveBeenCalled() + }) it('keeps the event retryable while an earlier processing attempt is active', async () => { const processingStartedAt = new Date() diff --git a/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts b/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts index 97691adacaf..e1ee59af466 100644 --- a/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts +++ b/apps/sim/lib/knowledge/documents/processing-outbox-handler.ts @@ -108,7 +108,12 @@ const processKnowledgeDocument: OutboxHandler = async (rawPayload, cont payload.documentId, SYSTEM_ACCESS_SCOPE ) - if (!document || document.processingStatus === 'completed') return + if ( + !document || + document.processingStatus === 'completed' || + document.processingOutcome === 'skipped' + ) + return if (document.processingStatus === 'processing') { const reclaimed = await reclaimStaleDocumentProcessingClaim({ knowledgeBaseId: payload.knowledgeBaseId, diff --git a/apps/sim/lib/knowledge/documents/processing-status.test.ts b/apps/sim/lib/knowledge/documents/processing-status.test.ts new file mode 100644 index 00000000000..db907f8da06 --- /dev/null +++ b/apps/sim/lib/knowledge/documents/processing-status.test.ts @@ -0,0 +1,44 @@ +/** @vitest-environment node */ +import { describe, expect, it } from 'vitest' +import { getDocumentProcessingOutcome } from '@/lib/knowledge/documents/processing-status' +import { DOCUMENT_PROCESSING_STATUSES } from '@/lib/knowledge/documents/types' + +describe('document processing outcomes', () => { + const placeholder = { + processingStatus: 'failed', + externalId: 'source-file', + storageKey: null, + contentHash: 'known-version', + fileUrl: '', + } + + it('recognizes a legacy intentional skip without reading error text or contacting its source', () => { + expect(getDocumentProcessingOutcome(placeholder)).toBe('skipped') + }) + + it.each([ + { ...placeholder, contentHash: null }, + { ...placeholder, storageKey: 'kb/stored.txt' }, + { ...placeholder, externalId: null }, + { ...placeholder, fileUrl: 'https://fixture.test/stored.txt' }, + ])('preserves real source, indexing, and upload failures: %j', (row) => { + expect(getDocumentProcessingOutcome(row)).toBeNull() + }) + + it.each(DOCUMENT_PROCESSING_STATUSES)( + 'preserves a stored %s outcome with an artifact', + (status) => { + expect( + getDocumentProcessingOutcome({ + ...placeholder, + processingStatus: status, + storageKey: 'kb/stored.txt', + }) + ).toBeNull() + } + ) + + it('does not infer a source omission from an unknown state', () => { + expect(getDocumentProcessingOutcome({ ...placeholder, processingStatus: 'invalid' })).toBeNull() + }) +}) diff --git a/apps/sim/lib/knowledge/documents/processing-status.ts b/apps/sim/lib/knowledge/documents/processing-status.ts new file mode 100644 index 00000000000..b3fdb30579b --- /dev/null +++ b/apps/sim/lib/knowledge/documents/processing-status.ts @@ -0,0 +1,52 @@ +import { document } from '@sim/db/schema' +import { and, eq, isNotNull, isNull, not, sql } from 'drizzle-orm' +import type { DocumentProcessingOutcome } from '@/lib/knowledge/documents/types' + +interface StoredDocumentProcessingState { + processingStatus: string + externalId: string | null + storageKey: string | null + contentHash: string | null + fileUrl: string +} + +/** + * Source omissions keep the legacy failed storage state for mixed-version app/worker + * compatibility. Their immutable source identity and version, with no stored or remote + * artifact, distinguish them from download failures (no version) and indexing failures + * (an artifact). Source identity survives disconnecting a connector with documents kept. + * The additive outcome lets current readers distinguish skips without rewriting old rows. + */ +export function getDocumentProcessingOutcome( + row: StoredDocumentProcessingState +): DocumentProcessingOutcome { + if ( + row.processingStatus === 'failed' && + row.externalId !== null && + row.storageKey === null && + row.fileUrl === '' && + row.contentHash !== null + ) { + return 'skipped' + } + return null +} + +/** SQL counterpart used for paging and aggregate counts before document rows are loaded. */ +export function skippedDocumentCondition() { + return and( + eq(document.processingStatus, 'failed'), + isNotNull(document.externalId), + isNull(document.storageKey), + eq(document.fileUrl, ''), + isNotNull(document.contentHash) + )! +} + +export function failedDocumentCondition() { + return and(eq(document.processingStatus, 'failed'), not(skippedDocumentCondition()))! +} + +export function documentProcessingOutcomeSelection() { + return sql`CASE WHEN ${skippedDocumentCondition()} THEN 'skipped' ELSE NULL END` +} diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index baa3a7f4309..d3dea4c4ecc 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -24,7 +24,7 @@ import { isNotNull, isNull, lt, - ne, + not, or, type SQL, sql, @@ -116,6 +116,11 @@ import { ProviderCapacityContinuationExhaustedError, } from '@/lib/knowledge/documents/processing-provider-deferral' import { scheduleDocumentProcessingQuotaContinuation } from '@/lib/knowledge/documents/processing-quota-continuation' +import { + documentProcessingOutcomeSelection, + getDocumentProcessingOutcome, + skippedDocumentCondition, +} from '@/lib/knowledge/documents/processing-status' import { DOCUMENT_PROCESSING_STALE_THRESHOLD_MS } from '@/lib/knowledge/documents/processing-timeouts.server' import { enqueueKnowledgeStorageCleanup, @@ -129,7 +134,10 @@ import { type TagFilterCondition, } from '@/lib/knowledge/documents/tag-filter' import { + type DocumentProcessingOutcome, + type DocumentProcessingStatus, type DocumentSortField, + isDocumentProcessingStatus, MAX_PROCESSING_ATTEMPTS, QUEUED_DISPATCH_GRACE_MS, type SortOrder, @@ -1494,7 +1502,8 @@ export async function processDocumentAsync( .where( and( eq(document.id, documentId), - ne(document.processingStatus, 'completed'), + inArray(document.processingStatus, ['pending', 'processing', 'failed']), + not(skippedDocumentCondition()), ...queueGenerationConditions(attemptContext), eq(document.userExcluded, false), isNull(document.archivedAt), @@ -1558,7 +1567,8 @@ export async function processDocumentAsync( .where( and( eq(document.id, documentId), - ne(document.processingStatus, 'completed'), + inArray(document.processingStatus, ['pending', 'processing', 'failed']), + not(skippedDocumentCondition()), ...(predecessor ? [or(and(...queueGenerationConditions(attemptContext)), predecessor)] : queueGenerationConditions(attemptContext)), @@ -2472,7 +2482,8 @@ export async function getDocuments( chunkCount: number tokenCount: number characterCount: number - processingStatus: 'pending' | 'processing' | 'completed' | 'failed' + processingStatus: DocumentProcessingStatus + processingOutcome: DocumentProcessingOutcome processingStartedAt: Date | null processingCompletedAt: Date | null processingError: string | null @@ -2579,6 +2590,7 @@ export async function getDocuments( tokenCount: document.tokenCount, characterCount: document.characterCount, processingStatus: document.processingStatus, + processingOutcome: documentProcessingOutcomeSelection(), processingStartedAt: document.processingStartedAt, processingCompletedAt: document.processingCompletedAt, processingError: document.processingError, @@ -2693,7 +2705,8 @@ export async function getDocuments( chunkCount: doc.chunkCount, tokenCount: doc.tokenCount, characterCount: doc.characterCount, - processingStatus: doc.processingStatus as 'pending' | 'processing' | 'completed' | 'failed', + processingStatus: doc.processingStatus as DocumentProcessingStatus, + processingOutcome: doc.processingOutcome, processingStartedAt: doc.processingStartedAt, processingCompletedAt: doc.processingCompletedAt, processingError: doc.processingError, @@ -2731,6 +2744,7 @@ export async function getDocuments( export type ActiveKnowledgeDocument = typeof document.$inferSelect & { connectorType: string | null + processingOutcome: DocumentProcessingOutcome } /** @@ -2747,6 +2761,7 @@ export async function getKnowledgeDocument( const [row] = await db .select({ ...getTableColumns(document), + processingOutcome: documentProcessingOutcomeSelection(), connectorType: knowledgeConnector.connectorType, }) .from(document) @@ -2775,6 +2790,7 @@ export async function getKnowledgeDocumentById( const [row] = await db .select({ ...getTableColumns(document), + processingOutcome: documentProcessingOutcomeSelection(), connectorType: knowledgeConnector.connectorType, }) .from(document) @@ -3065,7 +3081,7 @@ export async function getDocumentByUploadId( knowledgeBaseId: string ): Promise< | (Omit>, 'processingStatus'> & { - processingStatus: 'pending' | 'processing' | 'completed' | 'failed' + processingStatus: DocumentProcessingStatus }) | null > { @@ -3090,6 +3106,7 @@ export async function getDocumentByUploadId( tag6: document.tag6, tag7: document.tag7, processingStatus: document.processingStatus, + processingOutcome: documentProcessingOutcomeSelection(), }) .from(document) .where( @@ -3102,12 +3119,7 @@ export async function getDocumentByUploadId( .limit(1) if (!existing) return null const processingStatus = existing.processingStatus - if ( - processingStatus !== 'pending' && - processingStatus !== 'processing' && - processingStatus !== 'completed' && - processingStatus !== 'failed' - ) { + if (!isDocumentProcessingStatus(processingStatus)) { throw new Error(`Document ${existing.id} has invalid processing status`) } return { ...existing, processingStatus } @@ -3373,6 +3385,7 @@ export async function retryDocumentProcessing( and( eq(document.id, documentId), or(isNull(document.connectorId), isNotNull(document.contentHash)), + not(skippedDocumentCondition()), or( inArray(document.processingStatus, ['completed', 'failed']), and( @@ -3399,6 +3412,25 @@ export async function retryDocumentProcessing( }) if (!requeued) { + const [skipped] = await db + .select({ id: document.id }) + .from(document) + .where( + and( + eq(document.id, documentId), + eq(document.knowledgeBaseId, knowledgeBaseId), + skippedDocumentCondition() + ) + ) + .limit(1) + if (skipped) { + return { + success: false, + status: 'skipped', + message: + 'This source file was intentionally skipped. Sync the connector after changing the source.', + } + } const [sourceFailure] = await db .select({ id: document.id }) .from(document) @@ -3515,7 +3547,8 @@ export async function updateDocument( chunkCount: number tokenCount: number characterCount: number - processingStatus: 'pending' | 'processing' | 'completed' | 'failed' + processingStatus: DocumentProcessingStatus + processingOutcome: DocumentProcessingOutcome processingStartedAt: Date | null processingCompletedAt: Date | null processingError: string | null @@ -3546,7 +3579,7 @@ export async function updateDocument( chunkCount: number tokenCount: number characterCount: number - processingStatus: 'pending' | 'processing' | 'completed' | 'failed' + processingStatus: DocumentProcessingStatus processingError: string | null processingStartedAt: Date | null processingCompletedAt: Date | null @@ -3709,7 +3742,8 @@ export async function updateDocument( chunkCount: doc.chunkCount, tokenCount: doc.tokenCount, characterCount: doc.characterCount, - processingStatus: doc.processingStatus as 'pending' | 'processing' | 'completed' | 'failed', + processingStatus: doc.processingStatus as DocumentProcessingStatus, + processingOutcome: getDocumentProcessingOutcome(doc), processingStartedAt: doc.processingStartedAt, processingCompletedAt: doc.processingCompletedAt, processingError: doc.processingError, diff --git a/apps/sim/lib/knowledge/documents/types.ts b/apps/sim/lib/knowledge/documents/types.ts index eca8991a467..3b0bd225f99 100644 --- a/apps/sim/lib/knowledge/documents/types.ts +++ b/apps/sim/lib/knowledge/documents/types.ts @@ -95,6 +95,17 @@ export const DOCUMENT_PROCESSING_STATUSES = [ export type DocumentProcessingStatus = (typeof DOCUMENT_PROCESSING_STATUSES)[number] +/** An intentional source omission, separate from the backward-compatible processing state. */ +export type DocumentProcessingOutcome = 'skipped' | null + +/** Displays an intentional source outcome while accepting responses from older servers. */ +export function getDocumentIndexingStatus(document: { + processingStatus: string + processingOutcome?: DocumentProcessingOutcome +}): string { + return document.processingOutcome ?? document.processingStatus +} + /** * Narrows a stored `processing_status` onto the union. * diff --git a/apps/sim/lib/knowledge/github-setup-navigation.ts b/apps/sim/lib/knowledge/github-setup-navigation.ts new file mode 100644 index 00000000000..07e0a73d46a --- /dev/null +++ b/apps/sim/lib/knowledge/github-setup-navigation.ts @@ -0,0 +1,12 @@ +/** Setup can navigate only within Sim or to GitHub's approval pages. */ +export function resolveGitHubSetupUrl(value: string, origin: string): string { + const url = new URL(value, origin) + if ( + url.username || + url.password || + (url.origin !== origin && !(url.protocol === 'https:' && url.hostname === 'github.com')) + ) { + throw new Error('GitHub returned an invalid setup URL') + } + return url.href +} diff --git a/apps/sim/lib/knowledge/github-setup-state.ts b/apps/sim/lib/knowledge/github-setup-state.ts new file mode 100644 index 00000000000..deeff458dbd --- /dev/null +++ b/apps/sim/lib/knowledge/github-setup-state.ts @@ -0,0 +1,171 @@ +import type { SessionPrincipal } from '@sim/auth/principal' +import { sha256Hex } from '@sim/security/hash' +import { generateId } from '@sim/utils/id' +import { z } from 'zod' +import { getRedisClient } from '@/lib/core/config/redis' +import { OrchestrationError } from '@/lib/core/orchestration/types' + +export const GITHUB_SETUP_TTL_MS = 10 * 60 * 1000 +const scopeSchema = z + .object({ + organizationId: z.string().min(1).max(200), + setupId: z.string().uuid(), + userId: z.string().min(1).max(200), + sessionId: z.string().min(1).max(200), + }) + .strict() +export type GitHubSetupScope = z.infer + +const installationSchema = z + .object({ + installationId: z + .string() + .regex(/^[1-9]\d*$/) + .max(32), + accountId: z + .string() + .regex(/^[1-9]\d*$/) + .max(32), + accountLogin: z.string().min(1).max(100), + accountType: z.enum(['User', 'Organization']), + }) + .strict() +const baseSchema = scopeSchema.extend({ + createdAt: z.number().int().nonnegative(), + intent: z.literal('install').optional(), +}) +const attemptSchema = z.discriminatedUnion('phase', [ + baseSchema.extend({ phase: z.literal('starting') }), + baseSchema.extend({ phase: z.literal('oauth'), url: z.string().url().max(8192) }), + baseSchema.extend({ phase: z.literal('authorizing'), url: z.string().url().max(8192) }), + baseSchema.extend({ + phase: z.literal('choosing'), + installations: z.array(installationSchema).min(1).max(1000), + }), + baseSchema.extend({ phase: z.literal('installing'), state: z.string().uuid() }), + baseSchema.extend({ + phase: z.literal('connecting'), + installationId: z + .string() + .regex(/^[1-9]\d*$/) + .max(32), + state: z.string().uuid().optional(), + }), + baseSchema.extend({ + phase: z.literal('completed'), + credential: z + .object({ id: z.string().min(1).max(200), displayName: z.string().min(1).max(500) }) + .strict(), + }), + baseSchema.extend({ phase: z.literal('failed'), error: z.string().min(1).max(1000) }), + baseSchema.extend({ phase: z.literal('cancelled') }), +]) +export type GitHubSetupAttempt = z.infer +export type GitHubSetupPhase = GitHubSetupAttempt['phase'] + +function redis() { + const client = getRedisClient() + if (!client) throw new Error('GitHub setup requires Redis') + return client +} + +function attemptKey(scope: GitHubSetupScope) { + return `github-search:setup:${sha256Hex(JSON.stringify([scope.organizationId, scope.userId, scope.setupId]))}` +} + +function callbackKey(state: string) { + return `github-search:setup-callback:${sha256Hex(state)}` +} + +export function expiredGitHubSetupError() { + return new OrchestrationError('validation', 'GitHub setup expired or was canceled. Start again.') +} + +/** Retains the original expiry across refreshes and retries. */ +function remainingTtl(createdAt: number) { + const remaining = createdAt + GITHUB_SETUP_TTL_MS - Date.now() + if (createdAt > Date.now() || remaining <= 0) throw expiredGitHubSetupError() + return remaining +} + +export function githubSetupScope(attempt: GitHubSetupScope): GitHubSetupScope { + return scopeSchema.parse({ + organizationId: attempt.organizationId, + setupId: attempt.setupId, + userId: attempt.userId, + sessionId: attempt.sessionId, + }) +} + +const TRANSITION = ` +local raw = redis.call('GET', KEYS[1]) +if not raw then return 0 end +local current = cjson.decode(raw) +local next = cjson.decode(ARGV[1]) +if current.phase ~= ARGV[2] or current.userId ~= next.userId or current.sessionId ~= next.sessionId or current.organizationId ~= next.organizationId or current.setupId ~= next.setupId or current.createdAt ~= next.createdAt then return 0 end +redis.call('SET', KEYS[1], ARGV[1], 'PX', ARGV[3]) +return 1 +` + +/** Atomically claims a lifecycle transition; cancellation cannot be overwritten by stale work. */ +export async function saveGitHubSetupAttempt( + attempt: GitHubSetupAttempt, + expected: GitHubSetupPhase | 'new' +) { + const key = attemptKey(attempt) + const value = JSON.stringify(attemptSchema.parse(attempt)) + const ttl = remainingTtl(attempt.createdAt) + const saved = + expected === 'new' + ? (await redis().set(key, value, 'PX', ttl, 'NX')) === 'OK' + : (await redis().eval(TRANSITION, 1, key, value, expected, ttl)) === 1 + if (!saved) + throw new OrchestrationError( + 'conflict', + 'GitHub setup changed. Refresh to continue or start again.' + ) +} + +export async function readGitHubSetupAttempt( + scope: GitHubSetupScope +): Promise { + const raw = await redis().get(attemptKey(scope)) + if (!raw) return null + const attempt = attemptSchema.parse(JSON.parse(raw)) + if ( + attempt.organizationId !== scope.organizationId || + attempt.userId !== scope.userId || + attempt.setupId !== scope.setupId || + attempt.sessionId !== scope.sessionId || + attempt.createdAt > Date.now() || + Date.now() - attempt.createdAt >= GITHUB_SETUP_TTL_MS + ) + return null + return attempt +} + +/** Only a random, server-issued nonce is sent to GitHub; the owner remains in Redis. */ +export async function issueGitHubSetupCallback(attempt: GitHubSetupAttempt) { + const state = generateId() + const stored = await redis().set( + callbackKey(state), + JSON.stringify(githubSetupScope(attempt)), + 'PX', + remainingTtl(attempt.createdAt), + 'NX' + ) + if (stored !== 'OK') throw new Error('Could not create GitHub setup state') + return state +} + +/** A different browser session cannot consume or discover an attempt through its callback. */ +export async function resolveGitHubSetupCallback(state: string, principal: SessionPrincipal) { + const raw = await redis().get(callbackKey(state)) + if (!raw) throw expiredGitHubSetupError() + const scope = scopeSchema.parse(JSON.parse(raw)) + if (scope.userId !== principal.userId || scope.sessionId !== principal.sessionId) + throw expiredGitHubSetupError() + const attempt = await readGitHubSetupAttempt(scope) + if (!attempt) throw expiredGitHubSetupError() + return scope +} diff --git a/apps/sim/lib/knowledge/github-setup-urls.test.ts b/apps/sim/lib/knowledge/github-setup-urls.test.ts new file mode 100644 index 00000000000..97aa5587390 --- /dev/null +++ b/apps/sim/lib/knowledge/github-setup-urls.test.ts @@ -0,0 +1,64 @@ +/** @vitest-environment node */ +import { describe, expect, it, vi } from 'vitest' +import { + completeGitHubSearchSetupQuerySchema, + gitHubSearchSetupScopeSchema, + selectGitHubSearchSetupBodySchema, + startGitHubSearchSetupBodySchema, +} from '@/lib/api/contracts/knowledge/github-setup' + +vi.mock('@/lib/core/utils/urls', () => ({ getBaseUrl: () => 'https://sim.example' })) + +import { + githubSetupCompletionUrl, + githubSetupContinueUrl, + githubSetupPageUrl, +} from '@/lib/knowledge/github-setup-urls' + +const scope = { + organizationId: 'org/with?special&characters', + setupId: '550e8400-e29b-41d4-a716-446655440000', +} +describe('GitHub setup URLs and contracts', () => { + it.each([githubSetupPageUrl, githubSetupContinueUrl])( + 'uses the configured origin and encodes scope as values', + (build) => { + const url = new URL(build(scope)) + expect(url.origin).toBe('https://sim.example') + expect(url.searchParams.get('organizationId')).toBe(scope.organizationId) + expect(url.searchParams.get('setupId')).toBe(scope.setupId) + expect(url.hash).toBe('') + } + ) + it('correlates the generic completion page without installation IDs or provider state', () => { + const url = new URL(githubSetupCompletionUrl(scope.setupId, 'denied')) + expect(url.pathname).toBe('/credential-groups/complete') + expect([...url.searchParams]).toEqual([ + ['completionId', scope.setupId], + ['oauth', 'denied'], + ]) + }) + it('admits explicit install intent only on start and accepts no client redirect target', () => { + const valid = { organizationId: 'org', setupId: scope.setupId } + expect( + startGitHubSearchSetupBodySchema.safeParse({ ...valid, intent: 'install' }).success + ).toBe(true) + expect(gitHubSearchSetupScopeSchema.safeParse({ ...valid, intent: 'install' }).success).toBe( + false + ) + expect( + startGitHubSearchSetupBodySchema.safeParse({ ...valid, returnTo: 'https://evil.example' }) + .success + ).toBe(false) + expect( + selectGitHubSearchSetupBodySchema.safeParse({ + ...valid, + action: { kind: 'select', installationId: '0' }, + }).success + ).toBe(false) + expect( + completeGitHubSearchSetupQuerySchema.safeParse({ state: 'guessed', installation_id: '42' }) + .success + ).toBe(false) + }) +}) diff --git a/apps/sim/lib/knowledge/github-setup-urls.ts b/apps/sim/lib/knowledge/github-setup-urls.ts new file mode 100644 index 00000000000..7d6b721aa44 --- /dev/null +++ b/apps/sim/lib/knowledge/github-setup-urls.ts @@ -0,0 +1,27 @@ +import { getBaseUrl } from '@/lib/core/utils/urls' +import type { CredentialGroupOAuthFailure } from '@/lib/credential-groups/oauth-completion' + +export function githubSetupPageUrl(scope: { organizationId: string; setupId: string }) { + const url = new URL('/knowledge/github/setup', getBaseUrl()) + url.searchParams.set('organizationId', scope.organizationId) + url.searchParams.set('setupId', scope.setupId) + return url.toString() +} + +export function githubSetupContinueUrl( + scope: { organizationId: string; setupId: string }, + oauth?: CredentialGroupOAuthFailure +) { + const url = new URL('/api/knowledge/github/setup/continue', getBaseUrl()) + url.searchParams.set('organizationId', scope.organizationId) + url.searchParams.set('setupId', scope.setupId) + if (oauth) url.searchParams.set('oauth', oauth) + return url.toString() +} + +export function githubSetupCompletionUrl(setupId: string, oauth?: CredentialGroupOAuthFailure) { + const url = new URL('/credential-groups/complete', getBaseUrl()) + url.searchParams.set('completionId', setupId) + if (oauth) url.searchParams.set('oauth', oauth) + return url.toString() +} diff --git a/apps/sim/lib/knowledge/orchestration/documents.ts b/apps/sim/lib/knowledge/orchestration/documents.ts index 7416255d29b..4436933ecfd 100644 --- a/apps/sim/lib/knowledge/orchestration/documents.ts +++ b/apps/sim/lib/knowledge/orchestration/documents.ts @@ -17,6 +17,10 @@ import { retryDocumentProcessing, updateDocument, } from '@/lib/knowledge/documents/service' +import type { + DocumentProcessingOutcome, + DocumentProcessingStatus, +} from '@/lib/knowledge/documents/types' import { auditActorFields, classifyKnowledgeFailure, @@ -65,7 +69,7 @@ export type CreatedKnowledgeDocument = Omit< 'processingStatus' > & { /** New documents are pending; idempotent completion may return a later persisted state. */ - processingStatus: 'pending' | 'processing' | 'completed' | 'failed' + processingStatus: DocumentProcessingStatus } export interface PerformUploadKnowledgeDocumentParams extends KnowledgeOperationContext { @@ -546,6 +550,7 @@ export interface PerformRetryKnowledgeDocumentParams { fileSize: number mimeType: string processingStatus: string + processingOutcome?: DocumentProcessingOutcome connectorId?: string | null contentHash?: string | null } @@ -560,6 +565,13 @@ export async function performRetryKnowledgeDocumentProcessing( const { knowledgeBaseId, document, billingAttribution } = params const requestId = params.requestId ?? generateRequestId() + if (document.processingOutcome === 'skipped') { + return fail( + 'This source file was intentionally skipped. Sync the connector after changing the source.', + 'validation' + ) + } + if (document.connectorId && document.contentHash === null) { return fail( 'Source content could not be downloaded. Sync the connector to retry.', @@ -603,7 +615,7 @@ export async function performRetryKnowledgeDocumentProcessing( // got off the ground leaves a dead document, and reporting that as success // paints the UI green over it. if (!result.success) { - return fail(result.message, 'internal') + return fail(result.message, result.status === 'skipped' ? 'validation' : 'internal') } return { success: true, status: result.status, message: result.message } } catch (error) { diff --git a/apps/sim/lib/knowledge/types.ts b/apps/sim/lib/knowledge/types.ts index 18ddcc53c01..1aab5c59142 100644 --- a/apps/sim/lib/knowledge/types.ts +++ b/apps/sim/lib/knowledge/types.ts @@ -1,4 +1,8 @@ import type { ChunkingStrategy, StrategyOptions } from '@/lib/chunkers/types' +import type { + DocumentProcessingOutcome, + DocumentProcessingStatus, +} from '@/lib/knowledge/documents/types' import type { KbEmbeddingDimensions } from '@/lib/knowledge/embedding-models' /** @@ -142,7 +146,8 @@ export interface DocumentData { chunkCount: number tokenCount: number characterCount: number - processingStatus: 'pending' | 'processing' | 'completed' | 'failed' + processingStatus: DocumentProcessingStatus + processingOutcome?: DocumentProcessingOutcome processingStartedAt?: string | null processingCompletedAt?: string | null processingError?: string | null diff --git a/apps/sim/lib/oauth/github-installation.test.ts b/apps/sim/lib/oauth/github-installation.test.ts index 4267804d545..c49835e6514 100644 --- a/apps/sim/lib/oauth/github-installation.test.ts +++ b/apps/sim/lib/oauth/github-installation.test.ts @@ -6,6 +6,7 @@ import { assertGitHubInstallationActive, assertGitHubInstallationRepositoryActive, getGitHubInstallationConfiguration, + listGitHubInstallationRepositories, listUserAdminGitHubInstallations, parseGitHubInstallationBinding, resolveGitHubInstallationAccessToken, @@ -91,6 +92,92 @@ function mockDiscovery( } describe('GitHub installation setup', () => { + it('reports the failing token operation without including provider response contents', async () => { + fetchMock + .mockResolvedValueOnce(json(installation)) + .mockResolvedValueOnce(json({ message: 'sensitive provider detail' }, 422)) + await expect( + resolveGitHubInstallationAccessToken(binding, { repositoryId: '101' }) + ).rejects.toMatchObject({ + operation: 'repository-token', + status: 422, + message: + 'Check that the repository is included in the selected GitHub App installation, then retry.', + }) + }) + it('lists one metadata-only repository page with continuation and account checks', async () => { + fetchMock + .mockResolvedValueOnce(json(installation)) + .mockResolvedValueOnce(json(tokenResponse(101, false))) + .mockResolvedValueOnce( + json({ + total_count: 201, + repositories: Array.from({ length: 100 }, (_, i) => ({ + id: 101 + i, + full_name: `team/repo-${i}`, + owner: { id: 11 }, + default_branch: 'main', + })), + }) + ) + const controller = new AbortController() + const result = await listGitHubInstallationRepositories(binding, { + page: 2, + signal: controller.signal, + }) + expect(result.repositories).toHaveLength(100) + expect(result.repositories[0]).toEqual({ id: '101', fullName: 'team/repo-0' }) + expect(result.hasMore).toBe(true) + expect(fetchMock).toHaveBeenCalledTimes(3) + expect(JSON.parse(String(fetchMock.mock.calls[1][1]?.body))).toEqual({ + permissions: { metadata: 'read' }, + }) + expect(fetchMock.mock.calls[2][0]).toBe( + 'https://api.github.com/installation/repositories?per_page=100&page=2' + ) + controller.abort() + expect(fetchMock.mock.calls[2][1]?.signal?.aborted).toBe(true) + }) + + it.each([0, 101, 1.5, Number.NaN])( + 'rejects an invalid repository page %s before provider reads', + async (page) => { + await expect(listGitHubInstallationRepositories(binding, { page })).rejects.toThrow( + 'page is invalid' + ) + expect(fetchMock).not.toHaveBeenCalled() + } + ) + + it('refuses a suspended installation before minting a listing token', async () => { + fetchMock.mockResolvedValueOnce(json({ ...installation, suspended_at: '2026-01-01T00:00:00Z' })) + await expect(listGitHubInstallationRepositories(binding)).rejects.toThrow('unavailable') + expect(fetchMock).toHaveBeenCalledOnce() + }) + + it('rejects content permissions in repository browsing tokens', async () => { + fetchMock.mockResolvedValueOnce(json(installation)).mockResolvedValueOnce(json(tokenResponse())) + await expect(listGitHubInstallationRepositories(binding)).rejects.toThrow() + expect(fetchMock).toHaveBeenCalledTimes(2) + }) + + it('rejects repositories from another installation account', async () => { + fetchMock + .mockResolvedValueOnce(json(installation)) + .mockResolvedValueOnce(json(tokenResponse(101, false))) + .mockResolvedValueOnce( + json({ + total_count: 1, + repositories: [ + { id: 101, full_name: 'other/repo', owner: { id: 12 }, default_branch: 'main' }, + ], + }) + ) + await expect(listGitHubInstallationRepositories(binding)).rejects.toThrow( + 'another GitHub installation account' + ) + }) + it('rechecks repository installation selection without caching a previous success', async () => { fetchMock .mockResolvedValueOnce(json(installation)) diff --git a/apps/sim/lib/oauth/github-installation.ts b/apps/sim/lib/oauth/github-installation.ts index 1e3c7dfe720..270ba839567 100644 --- a/apps/sim/lib/oauth/github-installation.ts +++ b/apps/sim/lib/oauth/github-installation.ts @@ -64,13 +64,24 @@ const repositorySchema = z.object({ export class GitHubInstallationError extends Error { constructor( message: string, - readonly status?: number + readonly status?: number, + readonly operation?: GitHubInstallationOperation ) { super(message) this.name = 'GitHubInstallationError' } } +type GitHubInstallationOperation = + | 'user' + | 'memberships' + | 'installations' + | 'installation' + | 'repository' + | 'repository-list' + | 'repository-token' + | 'listing-token' + function readConfiguration() { const appId = env.GITHUB_APP_ID?.trim() const clientId = env.GITHUB_APP_CLIENT_ID?.trim() @@ -143,7 +154,8 @@ async function request( path: string, token: string, signal: AbortSignal, - body?: unknown + body?: unknown, + operation: GitHubInstallationOperation = 'installation' ): Promise { const requestSignal = AbortSignal.any([signal, AbortSignal.timeout(REQUEST_TIMEOUT_MS)]) const response = await fetch(`${API_URL}${path}`, { @@ -162,8 +174,11 @@ async function request( if (!response.ok) { await response.body?.cancel() throw new GitHubInstallationError( - `GitHub installation request failed with HTTP ${response.status}`, - response.status + operation === 'repository-token' && response.status === 422 + ? 'Check that the repository is included in the selected GitHub App installation, then retry.' + : `GitHub ${operation} request failed with HTTP ${response.status}`, + response.status, + operation ) } return readResponseJsonWithLimit(response, { @@ -214,7 +229,7 @@ async function adminAccountIds(userAccessToken: string, signal: AbortSignal) { throw new GitHubInstallationError('Connect your GitHub account before choosing an installation') const user = z .object({ id: apiIdSchema, type: z.literal('User') }) - .parse(await request('/user', userAccessToken, signal)) + .parse(await request('/user', userAccessToken, signal, undefined, 'user')) const organizations = new Set() const membershipsSchema = z .array( @@ -231,7 +246,9 @@ async function adminAccountIds(userAccessToken: string, signal: AbortSignal) { await request( `/user/memberships/orgs?state=active&per_page=${PAGE_SIZE}&page=${page}`, userAccessToken, - signal + signal, + undefined, + 'memberships' ) ) for (const membership of memberships) { @@ -271,7 +288,9 @@ export async function listUserAdminGitHubInstallations( await request( `/user/installations?per_page=${PAGE_SIZE}&page=${page}`, userAccessToken, - signal + signal, + undefined, + 'installations' ) ) for (const installation of data.installations) { @@ -407,7 +426,8 @@ async function mintToken( ...(repositoryId ? { repository_ids: [Number(repositoryId)] } : { repositories: [repositoryName] }), - } + }, + 'repository-token' ) ) const expiresAt = Date.parse(response.expires_at) @@ -449,7 +469,13 @@ export async function resolveGitHubInstallationRepository( await assertGitHubInstallationActive(binding, { signal }) const token = await mintToken(binding, signal, undefined, repo) const resolved = repositorySchema.parse( - await request(`/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`, token, signal) + await request( + `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}`, + token, + signal, + undefined, + 'repository' + ) ) if (String(resolved.owner.id) !== binding.accountId) throw new GitHubInstallationError('Repository belongs to another GitHub installation account') @@ -460,6 +486,60 @@ export async function resolveGitHubInstallationRepository( } } +/** Browses one bounded page with metadata-only access; content tokens remain repository-scoped. */ +export async function listGitHubInstallationRepositories( + binding: GitHubInstallationBinding, + options: RequestOptions & { page?: number } = {} +) { + const page = options.page ?? 1 + if (!Number.isSafeInteger(page) || page < 1 || page > 100) + throw new GitHubInstallationError('GitHub repository page is invalid') + const signal = operationSignal(options) + await assertGitHubInstallationActive(binding, { signal }) + const configuration = requireConfiguration() + const token = z + .object({ + token: z.string().min(1).max(1024), + expires_at: z.iso.datetime(), + permissions: z.object({ metadata: z.literal('read') }).strict(), + }) + .parse( + await request( + `/app/installations/${binding.installationId}/access_tokens`, + createAppJwt(configuration), + signal, + { permissions: { metadata: 'read' } }, + 'listing-token' + ) + ) + const expiresAt = Date.parse(token.expires_at) + if (expiresAt <= Date.now() || expiresAt > Date.now() + 65 * 60_000) + throw new GitHubInstallationError('GitHub returned an invalid installation token expiration') + const result = z + .object({ + total_count: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), + repositories: z.array(repositorySchema).max(PAGE_SIZE), + }) + .parse( + await request( + `/installation/repositories?per_page=${PAGE_SIZE}&page=${page}`, + token.token, + signal, + undefined, + 'repository-list' + ) + ) + if (result.repositories.some((repository) => String(repository.owner.id) !== binding.accountId)) + throw new GitHubInstallationError('Repository belongs to another GitHub installation account') + return { + repositories: result.repositories.map((repository) => ({ + id: String(repository.id), + fullName: repository.full_name, + })), + hasMore: result.repositories.length === PAGE_SIZE && page * PAGE_SIZE < result.total_count, + } +} + /** Mints contents access for exactly one source repository; generic unscoped token reads are refused. */ export async function resolveGitHubInstallationAccessToken( binding: GitHubInstallationBinding, diff --git a/apps/sim/lib/oauth/github-repositories.test.ts b/apps/sim/lib/oauth/github-repositories.test.ts index 610e7006877..c2820879cc1 100644 --- a/apps/sim/lib/oauth/github-repositories.test.ts +++ b/apps/sim/lib/oauth/github-repositories.test.ts @@ -8,6 +8,7 @@ import { parseGitHubRepositoriesTokenResponse, verifyGitHubRepositoriesIdentity, } from '@/lib/oauth/github-repositories' +import { OAuthIdentityVerificationError } from '@/lib/oauth/identity-error' import { refreshOAuthToken } from '@/lib/oauth/oauth' const tokenResponse = { @@ -188,27 +189,144 @@ describe('GitHub identity verification', () => { ) }) - it.each([[primary], [{ ...work, verified: false }], []])( - 'refuses absent or unverified invited email %j', - async (emails) => { + it.each([{ emails: [primary] }, { emails: [{ ...work, verified: false }] }, { emails: [] }])( + 'refuses absent or unverified invited email $emails', + async ({ emails }) => { vi.stubGlobal( 'fetch', vi.fn().mockResolvedValueOnce(response(user)).mockResolvedValueOnce(response(emails)) ) - await expect(verifyGitHubRepositoriesIdentity('ghu_access', work.email)).rejects.toThrow() + await expect( + verifyGitHubRepositoriesIdentity('ghu_access', work.email) + ).rejects.toMatchObject({ + name: 'OAuthIdentityVerificationError', + reason: 'email_mismatch', + stage: 'emails', + }) } ) it('rejects a bot identity', async () => { const fetchMock = vi.fn().mockResolvedValue(response({ ...user, type: 'Bot' })) vi.stubGlobal('fetch', fetchMock) - await expect(verifyGitHubRepositoriesIdentity('ghu_access')).rejects.toThrow() + await expect(verifyGitHubRepositoriesIdentity('ghu_access')).rejects.toMatchObject({ + reason: 'invalid_response', + stage: 'profile', + }) expect(fetchMock).toHaveBeenCalledTimes(1) }) it.each([401, 403])('fails closed when GitHub rejects identity with %i', async (status) => { vi.stubGlobal('fetch', vi.fn().mockResolvedValue(response({}, status))) - await expect(verifyGitHubRepositoriesIdentity('ghu_access')).rejects.toThrow(`HTTP ${status}`) + await expect(verifyGitHubRepositoriesIdentity('ghu_access')).rejects.toMatchObject({ + reason: 'provider_rejected', + stage: 'profile', + httpStatus: status, + }) + }) + + it('distinguishes denied email-read permission from a verified email mismatch', async () => { + vi.stubGlobal( + 'fetch', + vi + .fn() + .mockResolvedValueOnce(response(user)) + .mockResolvedValueOnce(response({ message: 'Resource not accessible by integration' }, 403)) + ) + await expect(verifyGitHubRepositoriesIdentity('ghu_access', work.email)).rejects.toMatchObject({ + reason: 'email_access_denied', + stage: 'emails', + httpStatus: 403, + }) + }) + + it.each([ + { status: 429, headers: {}, message: 'Too many requests' }, + { status: 403, headers: { 'x-ratelimit-remaining': '0' }, message: 'Forbidden' }, + { status: 403, headers: { 'retry-after': '60' }, message: 'Forbidden' }, + { status: 403, headers: {}, message: 'You have exceeded a secondary rate limit.' }, + { status: 403, headers: {}, message: 'You have triggered an abuse detection mechanism.' }, + ])( + 'does not misdiagnose provider rate limiting as missing email permission: %j', + async (error) => { + vi.stubGlobal( + 'fetch', + vi + .fn() + .mockResolvedValueOnce(response(user)) + .mockResolvedValueOnce( + new Response(JSON.stringify({ message: error.message }), { + status: error.status, + headers: error.headers, + }) + ) + ) + await expect( + verifyGitHubRepositoriesIdentity('ghu_access', work.email) + ).rejects.toMatchObject({ + reason: 'rate_limited', + stage: 'emails', + httpStatus: error.status, + }) + } + ) + + it('reports GitHub service failure without retaining its response body', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(response({ message: work.email }, 503))) + const failure = await verifyGitHubRepositoriesIdentity('ghu_access', work.email).catch( + (error: unknown) => error + ) + expect(failure).toBeInstanceOf(OAuthIdentityVerificationError) + expect(failure).toMatchObject({ + reason: 'provider_unavailable', + stage: 'profile', + httpStatus: 503, + }) + expect(JSON.stringify(failure)).not.toContain(work.email) + }) + + it('sanitizes network failures instead of retaining a transport error', async () => { + vi.stubGlobal('fetch', vi.fn().mockRejectedValue(new Error('transport failed ghu_access'))) + const failure = await verifyGitHubRepositoriesIdentity('ghu_access', work.email).catch( + (error: unknown) => error + ) + expect(failure).toMatchObject({ reason: 'provider_unavailable', stage: 'profile' }) + expect(String(failure)).not.toContain('ghu_access') + expect(failure).not.toHaveProperty('cause') + }) + + it('distinguishes an invalid email response from a verified email mismatch', async () => { + vi.stubGlobal( + 'fetch', + vi + .fn() + .mockResolvedValueOnce(response(user)) + .mockResolvedValueOnce( + response([ + { + email: work.email, + primary: false, + }, + ]) + ) + ) + await expect(verifyGitHubRepositoriesIdentity('ghu_access', work.email)).rejects.toMatchObject({ + reason: 'invalid_response', + stage: 'emails', + }) + }) + + it('does not claim a mismatch when the bounded email scan cannot finish', async () => { + const fetchMock = vi.fn().mockResolvedValueOnce(response(user)) + for (let page = 0; page < 10; page++) { + fetchMock.mockResolvedValueOnce(response(Array.from({ length: 100 }, () => primary))) + } + vi.stubGlobal('fetch', fetchMock) + await expect(verifyGitHubRepositoriesIdentity('ghu_access', work.email)).rejects.toMatchObject({ + reason: 'invalid_response', + stage: 'emails', + }) + expect(fetchMock).toHaveBeenCalledTimes(11) }) }) diff --git a/apps/sim/lib/oauth/github-repositories.ts b/apps/sim/lib/oauth/github-repositories.ts index 5c3ce80c12c..93c9a6361e5 100644 --- a/apps/sim/lib/oauth/github-repositories.ts +++ b/apps/sim/lib/oauth/github-repositories.ts @@ -3,6 +3,10 @@ import { getOAuth2Tokens, type OAuth2Tokens } from 'better-auth/oauth2' import type { GenericOAuthConfig } from 'better-auth/plugins' import { z } from 'zod' import { readResponseJsonWithLimit } from '@/lib/core/utils/stream-limits' +import { + OAuthIdentityVerificationError, + type OAuthIdentityVerificationStage, +} from '@/lib/oauth/identity-error' export const GITHUB_REPOSITORIES_PROVIDER_ID = 'github-repositories' export const GITHUB_AUTHORIZATION_URL = 'https://github.com/login/oauth/authorize' @@ -47,7 +51,7 @@ export async function verifyGitHubRepositoriesIdentity( expectedEmail?: string ) { if (!accessToken.startsWith('ghu_')) { - throw new Error('Connect with a GitHub App user authorization') + throw new OAuthIdentityVerificationError('provider_rejected', 'token') } const headers = { Accept: 'application/vnd.github+json', @@ -55,29 +59,76 @@ export async function verifyGitHubRepositoriesIdentity( 'X-GitHub-Api-Version': '2022-11-28', 'User-Agent': 'Sim', } - async function get(path: string): Promise { + async function get(path: string, stage: OAuthIdentityVerificationStage): Promise { const signal = AbortSignal.timeout(REQUEST_TIMEOUT_MS) - const response = await fetch(`${API_BASE_URL}${path}`, { - headers, - signal, - redirect: 'error', - }) + let response: Response + try { + response = await fetch(`${API_BASE_URL}${path}`, { + headers, + signal, + redirect: 'error', + }) + } catch { + throw new OAuthIdentityVerificationError('provider_unavailable', stage) + } if (!response.ok) { - await response.body?.cancel() - throw new Error(`GitHub identity request failed with HTTP ${response.status}`) + let rateLimited = + response.status === 429 || + (response.status === 403 && + (response.headers.get('x-ratelimit-remaining') === '0' || + response.headers.has('retry-after'))) + if (response.status === 403 && !rateLimited) { + const body = await readResponseJsonWithLimit(response, { + maxBytes: RESPONSE_MAX_BYTES, + signal, + label: 'GitHub identity error response', + }).catch(() => null) + rateLimited = + typeof body === 'object' && + body !== null && + 'message' in body && + typeof body.message === 'string' && + /rate limit|abuse detection/i.test(body.message) + } else { + await response.body?.cancel().catch(() => undefined) + } + const reason = rateLimited + ? 'rate_limited' + : response.status >= 500 + ? 'provider_unavailable' + : response.status === 403 && stage === 'emails' + ? 'email_access_denied' + : 'provider_rejected' + throw new OAuthIdentityVerificationError(reason, stage, response.status) + } + try { + return await readResponseJsonWithLimit(response, { + maxBytes: RESPONSE_MAX_BYTES, + signal, + label: 'GitHub identity response', + }) + } catch { + throw new OAuthIdentityVerificationError( + signal.aborted ? 'provider_unavailable' : 'invalid_response', + stage, + response.status + ) } - return readResponseJsonWithLimit(response, { - maxBytes: RESPONSE_MAX_BYTES, - signal, - label: 'GitHub identity response', - }) } - const user = userSchema.parse(await get('/user')) + const parsedUser = userSchema.safeParse(await get('/user', 'profile')) + if (!parsedUser.success) { + throw new OAuthIdentityVerificationError('invalid_response', 'profile') + } + const user = parsedUser.data const normalizedEmail = expectedEmail?.trim().toLowerCase() for (let page = 1; page <= MAX_EMAIL_PAGES; page++) { - const emails = emailsSchema.parse( - await get(`/user/emails?per_page=${EMAIL_PAGE_SIZE}&page=${page}`) + const parsedEmails = emailsSchema.safeParse( + await get(`/user/emails?per_page=${EMAIL_PAGE_SIZE}&page=${page}`, 'emails') ) + if (!parsedEmails.success) { + throw new OAuthIdentityVerificationError('invalid_response', 'emails') + } + const emails = parsedEmails.data const matching = emails.find( (entry) => entry.verified && @@ -94,9 +145,11 @@ export async function verifyGitHubRepositoriesIdentity( grantedScopes: [], } } - if (emails.length < EMAIL_PAGE_SIZE) break + if (emails.length < EMAIL_PAGE_SIZE) { + throw new OAuthIdentityVerificationError('email_mismatch', 'emails') + } } - throw new Error('GitHub did not verify the required email address') + throw new OAuthIdentityVerificationError('invalid_response', 'emails') } interface GitHubRepositoriesProviderParams { diff --git a/apps/sim/lib/oauth/identity-error.ts b/apps/sim/lib/oauth/identity-error.ts new file mode 100644 index 00000000000..8fbac5f2a1c --- /dev/null +++ b/apps/sim/lib/oauth/identity-error.ts @@ -0,0 +1,21 @@ +export type OAuthIdentityFailureReason = + | 'email_mismatch' + | 'email_access_denied' + | 'provider_rejected' + | 'rate_limited' + | 'provider_unavailable' + | 'invalid_response' + +export type OAuthIdentityVerificationStage = 'token' | 'profile' | 'emails' + +/** Carries safe diagnostics without retaining provider bodies, tokens, or email addresses. */ +export class OAuthIdentityVerificationError extends Error { + constructor( + readonly reason: OAuthIdentityFailureReason, + readonly stage: OAuthIdentityVerificationStage, + readonly httpStatus?: number + ) { + super(`OAuth identity verification failed: ${reason}`) + this.name = 'OAuthIdentityVerificationError' + } +} diff --git a/apps/sim/lib/oauth/types.ts b/apps/sim/lib/oauth/types.ts index d2bc4cacce3..b97bc4fd9e8 100644 --- a/apps/sim/lib/oauth/types.ts +++ b/apps/sim/lib/oauth/types.ts @@ -33,6 +33,7 @@ export const SLACK_CUSTOM_BOT_SECRET_TYPE = 'slack_custom_bot' as const export type OAuthProvider = | 'github-repositories' + | 'github-app-installation' | 'google' | 'google-email' | 'google-drive' diff --git a/apps/sim/lib/selectors/application/execute-selector.test.ts b/apps/sim/lib/selectors/application/execute-selector.test.ts index a0aa015b5b8..e0f2a9e0625 100644 --- a/apps/sim/lib/selectors/application/execute-selector.test.ts +++ b/apps/sim/lib/selectors/application/execute-selector.test.ts @@ -166,19 +166,38 @@ describe('executeSelector', () => { ]) }) - it('keeps generic organization browsing admin-only', async () => { - mocks.requireOrganizationMembership.mockRejectedValueOnce(new Error('Admin required')) - await expect( - execute({ scope: { kind: 'organization', organizationId: 'org-1' } }) - ).rejects.toThrow('Admin required') - expect(mocks.requireOrganizationMembership).toHaveBeenCalledWith( - principal, - 'org-1', - 'admin', - 'knowledge.use' + it.each(['gmail.labels', 'github.installationRepositories'])( + 'keeps generic organization browsing admin-only for %s', + async (selectorKey) => { + mocks.requireOrganizationMembership.mockRejectedValueOnce(new Error('Admin required')) + await expect( + execute({ selectorKey, scope: { kind: 'organization', organizationId: 'org-1' } }) + ).rejects.toThrow('Admin required') + expect(mocks.requireOrganizationMembership).toHaveBeenCalledWith( + principal, + 'org-1', + 'admin', + 'knowledge.use' + ) + expect(mocks.authorizePersonalSearch).not.toHaveBeenCalled() + expect(mocks.getAttachment).not.toHaveBeenCalled() + } + ) + + it('refuses organization GitHub installation browsing from workspace scope before resolving credentials', async () => { + mocks.resolveScope.mockResolvedValue({ + workspaceId: 'workspace-1', + workspaceOrganizationId: null, + allowPersonalApiKeys: true, + selectorKey: 'github.installationRepositories', + selectorManifest: getSelectorManifestEntry('github.installationRepositories'), + selectorScope: scope, + }) + await expect(execute({ selectorKey: 'github.installationRepositories' })).rejects.toThrow( + 'Context unavailable' ) - expect(mocks.authorizePersonalSearch).not.toHaveBeenCalled() - expect(mocks.getAttachment).not.toHaveBeenCalled() + expect(mocks.authorizeCredential).not.toHaveBeenCalled() + expect(mocks.executeAttachment).not.toHaveBeenCalled() }) it('rejects a personal setup marker outside its approved provider selector and organization scope', async () => { diff --git a/apps/sim/lib/selectors/manifest.test.ts b/apps/sim/lib/selectors/manifest.test.ts index e9cb015bf5f..f8a41558c4b 100644 --- a/apps/sim/lib/selectors/manifest.test.ts +++ b/apps/sim/lib/selectors/manifest.test.ts @@ -9,8 +9,8 @@ describe('selector manifest', () => { const count = (classification: (typeof classifications)[number]) => classifications.filter((value) => value === classification).length - expect(Object.keys(selectorManifest)).toHaveLength(97) - expect(count('provider-server')).toBe(84) + expect(Object.keys(selectorManifest)).toHaveLength(98) + expect(count('provider-server')).toBe(85) expect(count('internal-server')).toBe(12) expect(count('local')).toBe(1) expect(classifications).not.toContain('provider-legacy') @@ -36,7 +36,7 @@ describe('selector manifest', () => { const rawConnectionKeys = providerKeys.filter( (key) => !serverSelectorRegistry[key as keyof typeof serverSelectorRegistry].credential ) - expect(providerKeys).toHaveLength(84) + expect(providerKeys).toHaveLength(85) expect(rawConnectionKeys.sort()).toEqual([ 'cloudwatch.logGroups', 'cloudwatch.logStreams', diff --git a/apps/sim/lib/selectors/manifest.ts b/apps/sim/lib/selectors/manifest.ts index b6596873305..6d65ad50ea5 100644 --- a/apps/sim/lib/selectors/manifest.ts +++ b/apps/sim/lib/selectors/manifest.ts @@ -147,6 +147,10 @@ export const selectorManifest = { detail: true, }), 'gmail.labels': providerSelector(['impersonateUserEmail']), + 'github.installationRepositories': { + ...providerSelector([], { listMode: 'paginated', detail: true, unknownDetail: true }), + scopeKinds: ['organization'], + }, 'google.calendar': providerSelector(['impersonateUserEmail'], { listMode: 'paginated', detail: true, diff --git a/apps/sim/lib/selectors/server/providers/github.test.ts b/apps/sim/lib/selectors/server/providers/github.test.ts new file mode 100644 index 00000000000..2785a57a37f --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/github.test.ts @@ -0,0 +1,128 @@ +/** @vitest-environment node */ +import { beforeEach, describe, expect, it, vi } from 'vitest' +import type { ExecuteServerSelectorArgs } from '@/lib/selectors/server/types' + +const mocks = vi.hoisted(() => ({ + authorize: vi.fn(), + decrypt: vi.fn(), + parseBinding: vi.fn(), + list: vi.fn(), + resolve: vi.fn(), +})) +vi.mock('@/lib/core/security/encryption', () => ({ decryptSecret: mocks.decrypt })) +vi.mock('@/lib/credentials/application/organization-credentials', () => ({ + authorizeOrganizationCredentialUse: mocks.authorize, +})) +vi.mock('@/lib/oauth/github-installation', () => ({ + GitHubInstallationError: class extends Error {}, + parseGitHubInstallationBinding: mocks.parseBinding, + listGitHubInstallationRepositories: mocks.list, + resolveGitHubInstallationRepository: mocks.resolve, +})) + +import { githubSelectorAttachments } from '@/lib/selectors/server/providers/github' + +const principal = { kind: 'session', userId: 'admin', sessionId: 'session' } as const +const binding = { installationId: '21', accountId: '11' } +const row = { + organizationId: 'org-1', + workspaceId: null, + type: 'service_account', + providerSubjectId: '21', + providerTenantId: '11', + encryptedServiceAccountKey: 'encrypted', +} +function args(): ExecuteServerSelectorArgs { + return { + selectorKey: 'github.installationRepositories', + principal, + requesterUserId: principal.userId, + scope: { kind: 'organization', organizationId: 'org-1' }, + organizationId: 'org-1', + context: { oauthCredential: 'credential-1' }, + credential: { + suppliedId: 'credential-1', + providerId: 'github-app-installation', + organization: { principal, organizationId: 'org-1' }, + }, + request: { kind: 'list' }, + references: new Map(), + protectedValues: { add: vi.fn(), contains: () => false, containsExceptExact: () => false }, + } +} +const execute = (input: ExecuteServerSelectorArgs) => + githubSelectorAttachments['github.installationRepositories'].execute(input) + +describe('GitHub installation repository selector', () => { + beforeEach(() => { + vi.clearAllMocks() + mocks.authorize.mockResolvedValue({ credential: row }) + mocks.decrypt.mockResolvedValue({ decrypted: '{}' }) + mocks.parseBinding.mockReturnValue(binding) + mocks.list.mockResolvedValue({ + repositories: [{ id: '101', fullName: 'team/repo' }], + hasMore: true, + }) + }) + it('rechecks the acting admin connection and passes only bounded repository options', async () => { + const input = args() + input.request = { kind: 'list', cursor: '2' } + input.signal = new AbortController().signal + expect(await execute(input)).toEqual({ + kind: 'list', + items: [{ id: 'team/repo', label: 'team/repo' }], + nextCursor: '3', + }) + expect(mocks.authorize).toHaveBeenCalledWith( + expect.objectContaining({ + principal, + organizationId: 'org-1', + credentialId: 'credential-1', + expectedProviderId: 'github-app-installation', + }) + ) + expect(mocks.list).toHaveBeenCalledWith(binding, { page: 2, signal: input.signal }) + }) + it.each(['0', '101', '2.5', 'https://evil.example', '02'])( + 'refuses cursor %s before reading secrets', + async (cursor) => { + const input = args() + input.request = { kind: 'list', cursor } + await expect(execute(input)).rejects.toThrow('Context unavailable') + expect(mocks.authorize).not.toHaveBeenCalled() + expect(mocks.list).not.toHaveBeenCalled() + } + ) + it('refuses workspace and mismatched organization scope', async () => { + await expect( + execute({ ...args(), scope: { kind: 'workspace', workspaceId: 'ws-1' } }) + ).rejects.toThrow('Context unavailable') + await expect(execute({ ...args(), organizationId: 'org-2' })).rejects.toThrow( + 'Context unavailable' + ) + expect(mocks.authorize).not.toHaveBeenCalled() + }) + it('never reads provider data after credential authorization fails', async () => { + mocks.authorize.mockRejectedValue(new Error('Forbidden')) + await expect(execute(args())).rejects.toThrow('Forbidden') + expect(mocks.decrypt).not.toHaveBeenCalled() + expect(mocks.list).not.toHaveBeenCalled() + }) + it.each([ + { ...row, organizationId: 'other' }, + { ...row, workspaceId: 'ws' }, + { ...row, providerSubjectId: '22' }, + { ...row, providerTenantId: '12' }, + ])('refuses a stale or mismatched stored binding %#', async (credential) => { + mocks.authorize.mockResolvedValue({ credential }) + await expect(execute(args())).rejects.toThrow('Connection unavailable') + expect(mocks.list).not.toHaveBeenCalled() + }) + it('bounds the last repository page explicitly', async () => { + const input = args() + input.request = { kind: 'list', cursor: '100' } + expect(await execute(input)).toMatchObject({ + diagnostics: { truncated: { reason: 'provider-cap', pages: 100 } }, + }) + }) +}) diff --git a/apps/sim/lib/selectors/server/providers/github.ts b/apps/sim/lib/selectors/server/providers/github.ts new file mode 100644 index 00000000000..f6007104014 --- /dev/null +++ b/apps/sim/lib/selectors/server/providers/github.ts @@ -0,0 +1,100 @@ +import { decryptSecret } from '@/lib/core/security/encryption' +import { authorizeOrganizationCredentialUse } from '@/lib/credentials/application/organization-credentials' +import { + GitHubInstallationError, + listGitHubInstallationRepositories, + parseGitHubInstallationBinding, + resolveGitHubInstallationRepository, +} from '@/lib/oauth/github-installation' +import { GITHUB_INSTALLATION_PROVIDER_ID } from '@/lib/oauth/github-installation-types' +import { parseGitHubRepository } from '@/lib/oauth/github-repository' +import { + SelectorConnectionUnavailableError, + SelectorContextUnavailableError, +} from '@/lib/selectors/server/errors' +import { + detailSelectorResult, + listSelectorResult, + type ServerSelectorAttachmentMap, +} from '@/lib/selectors/server/types' + +export const githubSelectorAttachments = { + 'github.installationRepositories': { + credential: { kind: 'stored', field: 'oauthCredential', serviceIds: ['github-repositories'] }, + destination: 'fixed', + async execute(args) { + if ( + args.scope.kind !== 'organization' || + args.workspaceId || + args.organizationId !== args.scope.organizationId || + !args.credential?.organization || + args.credential.providerId !== GITHUB_INSTALLATION_PROVIDER_ID + ) + throw new SelectorContextUnavailableError() + const page = args.request.kind === 'list' ? Number(args.request.cursor ?? '1') : 1 + if ( + !Number.isSafeInteger(page) || + page < 1 || + page > 100 || + (args.request.kind === 'list' && + args.request.cursor !== undefined && + String(page) !== args.request.cursor) + ) + throw new SelectorContextUnavailableError() + const { credential } = await authorizeOrganizationCredentialUse({ + ...args.credential.organization, + credentialId: args.credential.suppliedId, + expectedProviderId: GITHUB_INSTALLATION_PROVIDER_ID, + requestId: 'selector-execution', + purpose: 'browsing', + }) + if ( + credential.type !== 'service_account' || + credential.organizationId !== args.scope.organizationId || + credential.workspaceId !== null || + !credential.encryptedServiceAccountKey || + credential.encryptedServiceAccountKey.length > 16_384 + ) + throw new SelectorConnectionUnavailableError() + const { decrypted } = await decryptSecret(credential.encryptedServiceAccountKey) + args.protectedValues.add(decrypted) + const binding = parseGitHubInstallationBinding(JSON.parse(decrypted)) + if ( + binding.installationId !== credential.providerSubjectId || + binding.accountId !== credential.providerTenantId + ) + throw new SelectorConnectionUnavailableError() + try { + if (args.request.kind === 'detail') { + try { + parseGitHubRepository(args.request.id) + } catch { + return detailSelectorResult(null) + } + const repository = await resolveGitHubInstallationRepository(binding, args.request.id, { + signal: args.signal, + }) + return detailSelectorResult({ id: repository.fullName, label: repository.fullName }) + } + const result = await listGitHubInstallationRepositories(binding, { + page, + signal: args.signal, + }) + return listSelectorResult( + result.repositories.map((repository) => ({ + id: repository.fullName, + label: repository.fullName, + })), + result.hasMore && page < 100 ? String(page + 1) : undefined, + result.hasMore && page === 100 + ? { truncated: { reason: 'provider-cap', pages: 100 } } + : undefined + ) + } catch (error) { + if (args.signal?.aborted) throw error + if (error instanceof GitHubInstallationError) throw new SelectorConnectionUnavailableError() + throw error + } + }, + }, +} satisfies ServerSelectorAttachmentMap<'github.installationRepositories'> diff --git a/apps/sim/lib/selectors/server/registry.ts b/apps/sim/lib/selectors/server/registry.ts index 4e067cc778e..09441960e09 100644 --- a/apps/sim/lib/selectors/server/registry.ts +++ b/apps/sim/lib/selectors/server/registry.ts @@ -9,6 +9,7 @@ import { calcomSelectorAttachments } from '@/lib/selectors/server/providers/calc import { clickupSelectorAttachments } from '@/lib/selectors/server/providers/clickup' import { cloudWatchSelectorAttachments } from '@/lib/selectors/server/providers/cloudwatch' import { confluenceSelectorAttachments } from '@/lib/selectors/server/providers/confluence' +import { githubSelectorAttachments } from '@/lib/selectors/server/providers/github' import { googleSelectorAttachments } from '@/lib/selectors/server/providers/google' import { harmonicSelectorAttachments } from '@/lib/selectors/server/providers/harmonic' import { hubspotSelectorAttachments } from '@/lib/selectors/server/providers/hubspot' @@ -46,6 +47,7 @@ export const serverSelectorRegistry = { ...cloudWatchSelectorAttachments, ...confluenceSelectorAttachments, ...googleSelectorAttachments, + ...githubSelectorAttachments, ...harmonicSelectorAttachments, ...hubspotSelectorAttachments, ...imapSelectorAttachments,