fix(file-search): align agent content reads with indexed lines

Author: icecrasher321

apps/docs/content/docs/integrations/file.mdx

@@ -55,7 +55,7 @@ Read workspace file objects from selected files, canonical workspace file IDs, o
 
 ### File Get Content
 
-Extract the text content of workspace files selected directly, identified by canonical file ID, or collected from one or more workspace folders.
+Extract workspace file text using the same parser mode as File Search. Use the returned fileId and offset/limit to read the matching line and surrounding context. For documents and spreadsheets, line numbers refer to extracted text, not page numbers or worksheet row numbers.
 
 #### Input
 
@@ -390,4 +390,6 @@ Existing parser safeguards also apply to complete extraction. PDFs allow at most
 Updates are indexed asynchronously. `pendingFiles` and `failedFiles` indicate revisions that are not yet searchable; a new revision becomes searchable only when its full index is ready. An empty result proves absence only within the searched scope when `complete` is true and `skippedFiles` is zero.
 
 Regex is evaluated against complete logical lines, including long lines, and cannot span line breaks. Returned lines may use a shortened preview. The result limit (up to 200 lines) and a 10-second query deadline limit an individual request, not the amount of text indexed. An expensive query fails explicitly instead of returning an apparently complete subset; narrow its literal text or folder scope and retry.
+
+Search returns one result per matching logical line with `fileId`, 1-based `lineNumber`, and `text` (a bounded preview for long lines). An Agent can use **Search** to locate content, then **Get Content** with the returned `fileId`, `offset` near `lineNumber`, and a small `limit` to read surrounding context. These operations use the same complete-text parser mode. Line numbers for documents and spreadsheets refer to extracted text, not page numbers or worksheet row numbers. Re-run search if the file changes between calls.
 {/* MANUAL-CONTENT-END */}

apps/sim/lib/internal/file/operations.test.ts

@@ -3,8 +3,14 @@
  */
 import { createMockRequest, hybridAuthMockFns } from '@sim/testing'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
+import * as XLSX from 'xlsx'
 import { OrchestrationError } from '@/lib/core/orchestration/types'
+import { isSupportedFileType, parseBuffer } from '@/lib/file-parsers'
+import { CsvParser } from '@/lib/file-parsers/csv-parser'
+import { FileParserError } from '@/lib/file-parsers/errors'
+import { XlsxParser } from '@/lib/file-parsers/xlsx-parser'
 import { MAX_FOLDER_PATH_SEGMENTS } from '@/lib/folders/paths'
+import { extractIndexText } from '@/lib/workspace-files/search/extract'
 
 const {
   mockAssertActiveWorkspaceAccess,
@@ -1064,6 +1070,78 @@ describe('file manage operations', () => {
     })
   })
 
+  it.each(['csv', 'xlsx'])('reads the same late %s line that search indexed', async (extension) => {
+    let buffer: Buffer
+    if (extension === 'csv') {
+      buffer = Buffer.from(`name,name\n${'first,second\n'.repeat(1001)}tail,needle\n`)
+    } else {
+      const workbook = XLSX.utils.book_new()
+      XLSX.utils.book_append_sheet(
+        workbook,
+        {
+          A1: { t: 's', v: 'header' },
+          ZZ1001: { t: 's', v: 'tail needle' },
+          '!ref': 'A1:ZZ1001',
+        },
+        'Data'
+      )
+      buffer = XLSX.write(workbook, { type: 'buffer', bookType: 'xlsx' })
+    }
+    const parser = extension === 'csv' ? new CsvParser() : new XlsxParser()
+    const parse = (
+      bytes: Buffer,
+      _extension: string,
+      options?: import('@/lib/file-parsers/types').FileParseOptions
+    ) => parser.parseBuffer(bytes, options)
+    vi.mocked(isSupportedFileType).mockReturnValueOnce(true).mockReturnValueOnce(true)
+    vi.mocked(parseBuffer).mockImplementationOnce(parse).mockImplementationOnce(parse)
+    const indexed = await extractIndexText(
+      { kind: 'stored', buffer },
+      `data.${extension}`,
+      new AbortController().signal
+    )
+    const lines = indexed!.text.split('\n')
+    const lineNumber = lines.findIndex((line) => line.includes('needle')) + 1
+    expect(lineNumber).toBeGreaterThan(0)
+    mockGetWorkspaceFile.mockResolvedValueOnce({
+      ...workspaceFile('file-1'),
+      name: `data.${extension}`,
+    })
+    mockDownloadServableFileFromStorage.mockResolvedValueOnce({ buffer })
+    const response = await POST(
+      createMockRequest('POST', {
+        operation: 'content',
+        workspaceId: 'workspace-1',
+        fileId: 'file-1',
+        offset: lineNumber,
+        limit: 1,
+      })
+    )
+    expect(response.status).toBe(200)
+    expect(await response.json()).toMatchObject({
+      data: {
+        contents: [lines[lineNumber - 1]],
+        lineRanges: [{ offset: lineNumber, lineCount: 1, totalLinesExact: true }],
+      },
+    })
+  })
+  it('does not bypass complete extraction limits with a raw-text fallback', async () => {
+    vi.mocked(isSupportedFileType).mockReturnValueOnce(true)
+    vi.mocked(parseBuffer).mockRejectedValueOnce(
+      new FileParserError('complexity_limit', 'too large')
+    )
+    const response = await POST(
+      createMockRequest('POST', {
+        operation: 'content',
+        workspaceId: 'workspace-1',
+        fileId: 'file-1',
+        offset: 1,
+        limit: 1,
+      })
+    )
+    expect(response.status).toBe(413)
+  })
+
   it('returns a scoped, deduplicated union of exact canonical file provenance', async () => {
     mockGetBoundWorkspaceFileSecretProvenance.mockImplementation(
       async (_workspaceId: string, identity: { fileId: string }) =>

apps/sim/lib/internal/file/operations.ts

@@ -27,7 +27,8 @@ import {
   RESOLVED_SECRET_PROVENANCE_METADATA_V1,
   requestsPrivateToolMetadata,
 } from '@/lib/execution/private-tool-metadata'
-import { isSupportedFileType, parseBuffer } from '@/lib/file-parsers'
+import { isSupportedFileType } from '@/lib/file-parsers'
+import { getFileParserErrorCode } from '@/lib/file-parsers/errors'
 import { buildFolderPath, parseFolderPath, ROOT_FOLDER_PATH } from '@/lib/folders/paths'
 import type { FolderIdScope } from '@/lib/folders/scope'
 import { collectFolderDepths } from '@/lib/folders/subtree'
@@ -93,14 +94,15 @@ import {
 } from '@/lib/workspace-files/application/workspace-file-folders'
 import { selectDirectoryEntries } from '@/lib/workspace-files/directory-listing'
 import type { WorkspaceFileContentEdit } from '@/lib/workspace-files/edit-content'
-import { countLines, detectLineEnding } from '@/lib/workspace-files/edit-content'
 import { toWorkspaceFileFolderPathView } from '@/lib/workspace-files/folder-display-path'
 import { resolveFolderIdsForPaths } from '@/lib/workspace-files/folder-path-selection'
 import {
   MAX_WORKSPACE_FILE_BULK_AFFECTED_ITEMS,
   MAX_ZIP_DOWNLOAD_FILES,
 } from '@/lib/workspace-files/limits'
 import { MAX_WORKSPACE_FILE_CONTENT_BYTES } from '@/lib/workspace-files/orchestration'
+import { parseWorkspaceFileText } from '@/lib/workspace-files/text-extraction'
+import { type FileTextLineRange, sliceFileTextLines } from '@/lib/workspace-files/text-lines'
 import { isWorkspaceAccessDeniedError } from '@/lib/workspaces/permissions/utils'
 import type { UserFile } from '@/executor/types'
 import {
@@ -375,48 +377,6 @@ interface ArchiveEntry {
 
 const isLikelyTextBuffer = (buffer: Buffer): boolean => isUtf8(buffer) && !buffer.includes(0)
 
-/** What a caller needs to tell a file that ended from a window that ran out. */
-interface FileContentLineRange {
-  offset: number
-  lineCount: number
-  totalLines: number
-  /** False when extraction was truncated, so `totalLines` is not the file's end. */
-  totalLinesExact: boolean
-}
-
-/**
- * Narrows extracted text to a line window.
- *
- * Reported alongside the text rather than inferred from it: without
- * `totalLines` a caller cannot distinguish a file that ended from a window
- * that stopped early, which is the same absent-versus-unknown confusion the
- * search index carries.
- */
-function sliceTextLines(
-  text: string,
-  offset: number | undefined,
-  limit: number | undefined,
-  truncatedExtraction: boolean
-): { text: string; range?: FileContentLineRange } {
-  if (offset === undefined && limit === undefined) return { text }
-
-  /* Counted the same way insert accepts them; see {@link countLines}. */
-  const effective = text.split(/\r\n|\n/).slice(0, countLines(text))
-  const start = Math.max((offset ?? 1) - 1, 0)
-  const window = effective.slice(start, limit === undefined ? undefined : start + limit)
-
-  return {
-    /* Rejoined with the text's own ending, so the window stays usable verbatim as an edit's search text. */
-    text: window.join(detectLineEnding(text)),
-    range: {
-      offset: start + 1,
-      lineCount: window.length,
-      totalLines: effective.length,
-      totalLinesExact: !truncatedExtraction,
-    },
-  }
-}
-
 /**
  * Download a stored file and extract its text content. Parseable types (PDF, DOCX,
  * CSV, etc.) go through the shared file-parsers; other UTF-8 files are returned as
@@ -452,7 +412,10 @@ const extractUserFileTextContent = async (
   const extension = getFileExtension(userFile.name)
   if (extension && isSupportedFileType(extension)) {
     try {
-      const result = await parseBuffer(buffer, extension)
+      const result = await parseWorkspaceFileText(buffer, extension, {
+        maxTextBytes: MAX_GET_CONTENT_FILE_BYTES,
+        signal: context.signal,
+      })
       if (result.metadata?.degraded === true) {
         /** Scraped or placeholder output is a failure, not the file's content. */
         throw new Error(result.metadata.warning ?? 'Parser returned degraded output')
@@ -463,6 +426,14 @@ const extractUserFileTextContent = async (
         contributingFiles,
       }
     } catch (error) {
+      context.signal?.throwIfAborted()
+      if (isPayloadSizeLimitError(error)) throw error
+      if (getFileParserErrorCode(error) === 'complexity_limit') {
+        throw new OrchestrationError(
+          'payload_too_large',
+          'File exceeds complete text extraction limits'
+        )
+      }
       logger.warn('Falling back to raw text after parser failure', {
         name: userFile.name,
         error: getErrorMessage(error, 'Unknown error'),
@@ -1191,7 +1162,7 @@ export async function executeFileManageOperation(
         const provenanceSources: FileContentProvenanceSource[] = [...sources]
 
         const contents: string[] = []
-        const lineRanges: FileContentLineRange[] = []
+        const lineRanges: FileTextLineRange[] = []
         let totalBytes = 0
         for (const source of sources) {
           signal?.throwIfAborted()
@@ -1215,7 +1186,7 @@ export async function executeFileManageOperation(
             )
             for (const renderedSource of renderedSources) provenanceSources.push(renderedSource)
           }
-          const { text: content, range } = sliceTextLines(
+          const { text: content, lineRange: range } = sliceFileTextLines(
             extracted.text,
             body.offset,
             body.limit,

apps/sim/lib/workspace-files/application/read-workspace-file-text.test.ts

@@ -3,6 +3,7 @@
  */
 import type { Principal } from '@sim/auth/principal'
 import { beforeEach, describe, expect, it, vi } from 'vitest'
+import { FileParserError } from '@/lib/file-parsers/errors'
 
 const mocks = vi.hoisted(() => ({
   getFile: vi.fn(),
@@ -124,6 +125,29 @@ describe('readWorkspaceFileText', () => {
     ).rejects.toMatchObject({ code: 'conflict' })
   })
 
+  it('uses the complete text representation before selecting a line window', async () => {
+    mocks.parseBuffer.mockResolvedValueOnce({ content: 'header\ntail needle\n', metadata: {} })
+    const result = await readWorkspaceFileText.execute({
+      principal: principals[0],
+      input: input({ offset: 2, limit: 1 }),
+    })
+    expect(mocks.parseBuffer).toHaveBeenCalledWith(expect.any(Buffer), 'txt', {
+      contentMode: 'complete',
+      pdfTextMode: 'complete',
+      maxTextBytes: 25 * 1024 * 1024,
+      signal: undefined,
+    })
+    expect(result.text).toBe('tail needle')
+    expect(result.lineRange).toMatchObject({ offset: 2, lineCount: 1, totalLines: 2 })
+  })
+  it('reports complete extraction complexity limits as payload limits', async () => {
+    mocks.parseBuffer.mockRejectedValueOnce(
+      new FileParserError('complexity_limit', 'expanded text budget')
+    )
+    await expect(
+      readWorkspaceFileText.execute({ principal: principals[0], input: input() })
+    ).rejects.toMatchObject({ code: 'payload_too_large' })
+  })
   it('denies a principal below the read role', async () => {
     mocks.resolvePermission.mockResolvedValue(null)
 

apps/sim/lib/workspace-files/application/read-workspace-file-text.ts

@@ -1,7 +1,9 @@
 import { getErrorMessage } from '@sim/utils/errors'
 import type { AuthorizedWorkspaceUseCaseContext } from '@/lib/core/application'
 import { OrchestrationError } from '@/lib/core/orchestration/types'
-import { isSupportedFileType, parseBuffer } from '@/lib/file-parsers'
+import { isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits'
+import { isSupportedFileType } from '@/lib/file-parsers'
+import { getFileParserErrorCode } from '@/lib/file-parsers/errors'
 import {
   type ActiveWorkspaceFileContext,
   fetchWorkspaceFileBuffer,
@@ -18,7 +20,8 @@ import { defineAuthorizedWorkspaceFileUseCase } from '@/lib/workspace-files/appl
 import { fileOperations } from '@/lib/workspace-files/application/operations'
 import { resolveRenderedWorkspaceArtifact } from '@/lib/workspace-files/application/resolve-rendered-workspace-artifact'
 import { resolveActiveWorkspaceFileContext } from '@/lib/workspace-files/application/workspace-file-context'
-import { countLines, detectLineEnding } from '@/lib/workspace-files/edit-content'
+import { parseWorkspaceFileText } from '@/lib/workspace-files/text-extraction'
+import { sliceFileTextLines } from '@/lib/workspace-files/text-lines'
 
 export interface ReadWorkspaceFileTextInput {
   fileId: string
@@ -118,7 +121,12 @@ async function executeReadWorkspaceFileText({
   const metadata = parsed.metadata ?? {}
 
   const truncated = metadata.truncated === true
-  const { text, lineRange } = sliceTextLines(parsed.content, input.offset, input.limit, truncated)
+  const { text, lineRange } = sliceFileTextLines(
+    parsed.content,
+    input.offset,
+    input.limit,
+    truncated
+  )
 
   return {
     file,
@@ -131,63 +139,6 @@ async function executeReadWorkspaceFileText({
   }
 }
 
-/**
- * Narrows extracted text to a line window.
- *
- * `totalLines` travels with it because without it a caller cannot tell a file
- * that ended from a window that stopped early, and would either stop reading
- * too soon or keep asking for lines that do not exist. Lines are counted the
- * way {@link countLines} counts them, so the numbers here name the same lines
- * that search reports and that an insert will accept.
- *
- * `totalLinesExact` is false when the parser stopped early: the count then
- * describes only the part that was extracted, and reporting it as the file's
- * end would tell a caller it had read everything. The separate flag is what
- * keeps `totalLines` useful in the ordinary case without lying in this one.
- *
- * The window is rejoined with the line ending the text already used, so a
- * ranged read of a CRLF file stays usable verbatim as exact search text for an edit.
- */
-function sliceTextLines(
-  text: string,
-  offset: number | undefined,
-  limit: number | undefined,
-  truncatedExtraction: boolean
-): {
-  text: string
-  lineRange?: {
-    offset: number
-    lineCount: number
-    totalLines: number
-    totalLinesExact: boolean
-  }
-} {
-  if (offset === undefined && limit === undefined) return { text }
-
-  const totalLines = countLines(text)
-  const eol = detectLineEnding(text)
-  const lines = text.split(/\r\n|\n/).slice(0, totalLines)
-  const start = Math.max((offset ?? 1) - 1, 0)
-  const window = lines.slice(start, limit === undefined ? undefined : start + limit)
-
-  return {
-    text: window.join(eol),
-    lineRange: {
-      offset: start + 1,
-      lineCount: window.length,
-      totalLines,
-      totalLinesExact: !truncatedExtraction,
-    },
-  }
-}
-
-/**
- * Extracts a workspace file's text.
- *
- * Runs on `files.read_content` unchanged: extracting text reads exactly the
- * bytes that operation already authorizes, and turning them into text grants
- * no further reach. No audit is projected, matching the existing content read.
- */
 /**
  * Turns stored bytes into text without ever answering `500`.
  *
@@ -208,15 +159,30 @@ async function parseFileText(content: Buffer, extension: string, fileName: strin
     return { content: '', metadata: {} }
   }
   try {
-    return await parseBuffer(content, extension)
+    return await parseWorkspaceFileText(content, extension, {
+      maxTextBytes: MAX_TEXT_EXTRACTION_BYTES,
+    })
   } catch (error) {
+    if (isPayloadSizeLimitError(error) || getFileParserErrorCode(error) === 'complexity_limit') {
+      throw new OrchestrationError(
+        'payload_too_large',
+        `"${fileName}" exceeds complete text extraction limits`
+      )
+    }
     throw new OrchestrationError(
       'conflict',
       `"${fileName}" could not be read as text: ${getErrorMessage(error, 'the stored bytes could not be parsed')}`
     )
   }
 }
 
+/**
+ * Extracts a workspace file's text.
+ *
+ * Runs on `files.read_content` unchanged: extracting text reads exactly the
+ * bytes that operation already authorizes, and turning them into text grants
+ * no further reach. No audit is projected, matching the existing content read.
+ */
 export const readWorkspaceFileText = defineAuthorizedWorkspaceFileUseCase({
   operation: fileOperations.readContent,
   resolveContext: ({ input }) => resolveActiveWorkspaceFileContext(input),