From 389ecbd017b4b6e499688a2584fa2b5a6c87b2aa Mon Sep 17 00:00:00 2001 From: Christopher Pruijsen Date: Fri, 11 Sep 2026 13:19:31 +0100 Subject: [PATCH] Honor TokenReader subrange bounds in peek and backtrack. Embedded readers could observe tokens outside their TokenSequence window because peekToken, peekPreviousTokenKind, and backtrackToMarker ignored the start/end indexes that the other peek methods already enforce. Fixes #481 --- ...x-tokenreader-bounds_2026-09-11-13-00.json | 10 +++ tsdoc/src/parser/TokenReader.ts | 10 ++- .../src/parser/__tests__/TokenReader.test.ts | 84 +++++++++++++++++++ 3 files changed, 103 insertions(+), 1 deletion(-) create mode 100644 common/changes/@microsoft/tsdoc/fix-tokenreader-bounds_2026-09-11-13-00.json create mode 100644 tsdoc/src/parser/__tests__/TokenReader.test.ts diff --git a/common/changes/@microsoft/tsdoc/fix-tokenreader-bounds_2026-09-11-13-00.json b/common/changes/@microsoft/tsdoc/fix-tokenreader-bounds_2026-09-11-13-00.json new file mode 100644 index 00000000..0a634174 --- /dev/null +++ b/common/changes/@microsoft/tsdoc/fix-tokenreader-bounds_2026-09-11-13-00.json @@ -0,0 +1,10 @@ +{ + "changes": [ + { + "packageName": "@microsoft/tsdoc", + "comment": "Fix TokenReader peek and backtrack methods so they honor an embedded token sequence's bounds.", + "type": "patch" + } + ], + "packageName": "@microsoft/tsdoc" +} diff --git a/tsdoc/src/parser/TokenReader.ts b/tsdoc/src/parser/TokenReader.ts index 770a06e7..45c8177a 100644 --- a/tsdoc/src/parser/TokenReader.ts +++ b/tsdoc/src/parser/TokenReader.ts @@ -120,6 +120,10 @@ export class TokenReader { * consuming anything. */ public peekToken(): Token { + if (this._currentIndex >= this._readerEndIndex) { + // Always return a real Token, matching peekTokenKind() === EndOfInput. + return this.tokens[this.tokens.length - 1]; + } return this.tokens[this._currentIndex]; } @@ -180,7 +184,7 @@ export class TokenReader { * Returns the kind of the token immediately before the current token. */ public peekPreviousTokenKind(): TokenKind { - if (this._currentIndex === 0) { + if (this._currentIndex === this._readerStartIndex) { return TokenKind.EndOfInput; } return this.tokens[this._currentIndex - 1].kind; @@ -201,6 +205,10 @@ export class TokenReader { // If this happens, it's a parser bug throw new Error('The marker has expired'); } + if (marker < this._readerStartIndex) { + // If this happens, it's a parser bug + throw new Error('The marker is outside the TokenReader range'); + } this._currentIndex = marker; if (marker < this._accumulatedStartIndex) { diff --git a/tsdoc/src/parser/__tests__/TokenReader.test.ts b/tsdoc/src/parser/__tests__/TokenReader.test.ts new file mode 100644 index 00000000..59d6b7bf --- /dev/null +++ b/tsdoc/src/parser/__tests__/TokenReader.test.ts @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license. +// See LICENSE in the project root for license information. + +import { TSDocParser } from '../TSDocParser'; +import { type Token, TokenKind } from '../Token'; +import { TokenReader } from '../TokenReader'; +import { TokenSequence } from '../TokenSequence'; +import type { ParserContext } from '../ParserContext'; + +function parseComment(buffer: string): ParserContext { + const tsdocParser: TSDocParser = new TSDocParser(); + return tsdocParser.parseString(buffer); +} + +function indexOfTokenText(parserContext: ParserContext, text: string): number { + const tokens: ReadonlyArray = parserContext.tokens; + for (let i: number = 0; i < tokens.length; ++i) { + if (tokens[i].toString() === text) { + return i; + } + } + throw new Error('Token not found: ' + JSON.stringify(text)); +} + +function createEmbeddedReader( + parserContext: ParserContext, + startIndex: number, + endIndex: number +): TokenReader { + return new TokenReader( + parserContext, + new TokenSequence({ + parserContext: parserContext, + startIndex: startIndex, + endIndex: endIndex + }) + ); +} + +test('peekPreviousTokenKind does not leak the token before an embedded window', () => { + const parserContext: ParserContext = parseComment('/** prefix {@link Dest} suffix */'); + const destIndex: number = indexOfTokenText(parserContext, 'Dest'); + const tokenReader: TokenReader = createEmbeddedReader(parserContext, destIndex, destIndex + 1); + + expect(parserContext.tokens[destIndex - 1].kind).toEqual(TokenKind.Spacing); + expect(tokenReader.peekPreviousTokenKind()).toEqual(TokenKind.EndOfInput); +}); + +test('peekToken at the end of an embedded window is EndOfInput, not the following token', () => { + const parserContext: ParserContext = parseComment('/** prefix {@link Dest} suffix */'); + const destIndex: number = indexOfTokenText(parserContext, 'Dest'); + const tokenReader: TokenReader = createEmbeddedReader(parserContext, destIndex, destIndex + 1); + + expect(tokenReader.readToken().toString()).toEqual('Dest'); + expect(tokenReader.peekTokenKind()).toEqual(TokenKind.EndOfInput); + + expect(parserContext.tokens[destIndex + 1].kind).toEqual(TokenKind.RightCurlyBracket); + expect(tokenReader.peekToken()).toBeDefined(); + expect(tokenReader.peekToken().kind).toEqual(TokenKind.EndOfInput); +}); + +test('peekToken for an empty TokenSequence is EndOfInput, not tokens[0]', () => { + const parserContext: ParserContext = parseComment('/** prefix {@link Dest} suffix */'); + const tokenReader: TokenReader = new TokenReader(parserContext, TokenSequence.createEmpty(parserContext)); + + expect(parserContext.tokens[0].toString()).toEqual('prefix'); + expect(tokenReader.peekTokenKind()).toEqual(TokenKind.EndOfInput); + expect(tokenReader.peekToken().kind).toEqual(TokenKind.EndOfInput); +}); + +test('backtrackToMarker rejects a marker before the embedded start', () => { + const parserContext: ParserContext = parseComment('/** prefix {@link Dest} suffix */'); + const destIndex: number = indexOfTokenText(parserContext, 'Dest'); + const tokenReader: TokenReader = createEmbeddedReader(parserContext, destIndex, destIndex + 1); + + tokenReader.readToken(); + + expect(() => { + tokenReader.backtrackToMarker(destIndex - 1); + }).toThrowError('The marker is outside the TokenReader range'); + + tokenReader.backtrackToMarker(destIndex); + expect(tokenReader.peekToken().toString()).toEqual('Dest'); +});