diff --git a/.github/workflows/test-pr.yaml b/.github/workflows/test-pr.yaml index d497d9b1c..a9e9566da 100644 --- a/.github/workflows/test-pr.yaml +++ b/.github/workflows/test-pr.yaml @@ -38,6 +38,7 @@ jobs: - php - scala3,schema-scala3 - elixir,schema-elixir,graphql-elixir + - comment-injection-treesitter,comment-injection-typescript,comment-injection-typescript-zod,comment-injection-typescript-effect-schema # Partially working # - schema-typescript # TODO Unify with typescript once fixed @@ -68,6 +69,9 @@ jobs: # - fixture: objective-c # segfault on compiled test cmd # runs-on: macos-latest + # comment-injection-objective-c also hits that runtime + # segfault (the generated .m compiles but ./test crashes + # with SIGSEGV), so it stays out of CI too. name: ${{ matrix.fixture }} steps: diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..f0ba64eaa --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,64 @@ +# AGENTS.md + +Notes for coding agents working in this repository. + +## Environment + +- This is a TypeScript/npm monorepo using npm workspaces. +- Prefer the Node version from `.nvmrc` (`nvm use`; currently Node 22.14.0). `package.json` requires Node >= 18.12.0. +- Install dependencies with `npm ci`. + +## Build and run + +- Build everything with: + + ```bash + npm run build + ``` + + This runs `npm run clean`, builds all workspaces that have a `build` script, then runs the root `tsc`. + +- After building, the CLI entry point is `dist/index.js`, for example: + + ```bash + node dist/index.js --version + node dist/index.js --help + ``` + +- For live rebuild/re-run while developing renderer output, use `npm start -- ""`. + +## Tests + +- The test runner is `script/test`, exposed as: + + ```bash + npm test + ``` + +- The full suite runs all fixtures and needs external language toolchains for many targets (`dotnet`, Java/Maven, Go, Rust, Python/mypy, PHP, Ruby, Kotlin, Scala, Elixir, etc.). On a machine without those tools, plain `npm test` will fail when it reaches the first missing toolchain. + +- For local focused testing, use fixture filters. Fixture names are in `test/languages.ts` and `test/fixtures.ts`; comma-separated fixture groups are supported: + + ```bash + QUICKTEST=true FIXTURE=javascript npm test + QUICKTEST=true FIXTURE=typescript npm test -- test/inputs/json/samples/pokedex.json + CPUs=2 QUICKTEST=true FIXTURE=javascript npm test + ``` + + `QUICKTEST=true` skips the large miscellaneous JSON input set. Extra arguments after `--` are sample files or directories to run. + +- GitHub Actions uses the same pattern, e.g. `QUICKTEST=true FIXTURE=${{ matrix.fixture }} npm test`, after installing toolchain dependencies for each fixture group in `.github/workflows/test-pr.yaml`. + +## Validation performed + +The following commands were run successfully in this workspace: + +```bash +npm ci +npm run build +node dist/index.js --version +CPUs=2 QUICKTEST=true FIXTURE=javascript npm test +QUICKTEST=true FIXTURE=typescript npm test -- test/inputs/json/samples/pokedex.json +``` + +Also observed: `npm test` without fixture filters started the full 70-fixture suite and failed on this machine because `dotnet` is not installed. `npm run lint` currently fails because ESLint cannot find a configuration file. diff --git a/COMMENT-INJECTION.md b/COMMENT-INJECTION.md new file mode 100644 index 000000000..2f96d814e --- /dev/null +++ b/COMMENT-INJECTION.md @@ -0,0 +1,79 @@ +# JSON Schema comment injection + +## TypeScript reproduction + +Added a focused schema fixture: + +- `test/inputs/schema/comment-injection.schema` +- `test/inputs/schema/comment-injection.1.json` + +The schema puts comment-closing text in both an object `description` and a property `description`: + +- `*/` and `/*` for C-style block comments (the opener matters for Kotlin and Scala 3, whose block comments nest) +- `{-` and `-}` for Elm/Haskell comments +- `"""` on its own line for Python/Elixir docstrings/heredocs +- ` &
` for C# XML doc comments +- `\r}`, `\u0085}`, `\u2028}`, and `\u2029}` for line-comment outputs whose lexers treat CR, NEL, LINE SEPARATOR, or PARAGRAPH SEPARATOR as line terminators +- property descriptions ending in `\`, `"`, and `"""` for docstring/heredoc delimiter boundaries and C-family line splicing + +Run: + +```bash +CPUs=1 QUICKTEST=true FIXTURE=schema-typescript npm test -- test/inputs/schema/comment-injection.schema +``` + +Expected result: the generated TypeScript validates `comment-injection.1.json` and prints equivalent JSON. + +Before escaping was added, this test failed before validation because `TopLevel.ts` was syntactically invalid; the schema `description` escaped the generated `/** ... */` comment. + +## Schema fields that can reach comments + +In JSON Schema input, `packages/quicktype-core/src/attributes/Description.ts` collects `description` into type and property-description attributes. Renderers then emit those attributes as documentation comments. + +Observed comment sinks: + +- schema/type `description` on objects/classes +- property `description` for object properties/fields +- `description` on enum schemas +- `description` on union schemas and other named types when a renderer emits docs for those named types + +`title` is different in the inspected path: JSON Schema input uses it for type/top-level naming, not as raw documentation text. The generated JSON Schema renderer can output JSON `title` fields, but those are JSON string values, not source comments. + +Other non-schema inputs can also supply descriptions or leading comments, but the reproduction here is limited to JSON Schema `description`. + +## Potentially affected outputs and triggers + +Outputs are affected when raw schema descriptions are placed into comments/docstrings without escaping that target's comment delimiter or line terminators. The shared fix now normalizes description line endings and escapes delimiter text at comment-emission time. + +- C-style doc comments `/** ... */`: TypeScript, Flow, JavaScript when descriptions are emitted, Java, C (`cjson`), C++, PHP, Kotlin, Scala 3, Smithy4s. Trigger with `*/`; escaped as `* /`. The opener `/*` is escaped as `/ *`, too, because Kotlin and Scala 3 nest block comments, so an unmatched opener would swallow the rest of the file. +- Elm/Haskell doc comments `{-| ... -}`: Elm, Haskell. Trigger with `{-` or `-}`; escaped as `{ -` and `- }`. +- Triple-quoted docstrings/heredocs: Python, Elixir. These are string literals, so three separate escapes apply: `"""` is escaped as `\"\"\"`, backslashes are doubled (otherwise a description ending in `\` swallows the first quote of the closing delimiter), and a line-ending unescaped `"` directly before an inline closing `"""` is escaped as `\"` (otherwise four quotes in a row leave a stray quote after the closing delimiter). +- XML doc comments `/// `: C#. Trigger with ``, `<`, or `&`; escaped as XML entities (`<`, `>`, `&`) in both the regular and `--density dense` paths. +- Line comments (`//`, `///`): C#, Go, Rust, Ruby, Swift, Objective-C, Dart, Pike, Crystal, and enum comments in TypeScript-Zod/TypeScript-Effect-Schema. Trigger with a carriage return (`\r`) — or U+0085/U+2028/U+2029, which JavaScript and C# lexers also treat as line terminators — when descriptions are split only on `\n`; fixed by normalizing all of them to `\n` before comment emission. +- C-family line splicing: C, C++, and Objective-C splice a backslash-newline into one line even inside comments, so a description ending in `\` pulls the next generated line into the comment (and trips `-Wcomment` under `-Werror`). Fixed by appending `.` after a comment line's trailing backslash in those renderers. + +Plain JavaScript output did not emit the tested schema descriptions into model comments, so the object/property reproduction does not affect it the same way. TypeScript-Zod and TypeScript-Effect-Schema also did not emit the object/property descriptions from `comment-injection.schema`; they only emitted the enum description from `comment-injection-enum.schema`. + +The tree-sitter fixture includes Go, Rust, and Ruby even though their grammars did not reproduce a syntax break with the CR line-comment payload; this keeps parser coverage in place for those generated outputs and future payload/escaping changes. + +Still not covered by the tree-sitter fixture: Objective-C, Crystal, and Elm. Objective-C's available tree-sitter grammar reports baseline errors on generated `.m` output; Crystal and Elm have usable grammars but are not included in this parser-coverage pass. + +## Test cases added + +- `test/inputs/schema/comment-injection.schema` covers object and property descriptions, including both Elm/Haskell nested-comment delimiters, XML doc metacharacters, Unicode line terminators, and descriptions ending in `\`, `"`, and `"""`. +- `test/inputs/schema/comment-injection-enum.schema` covers enum descriptions via an enum-valued property. +- `test/inputs/schema/comment-injection-nested-comment.schema` specifically covers unmatched nested-comment openers (`{-` and `/*`) in object and property descriptions. +- `test/inputs/schema/comment-injection-enum-nested-comment.schema` is the enum-description variant of the nested-comment payload; the tree-sitter fixture substitutes it for the enum-only targets so each sample run does distinct work. + +The existing `JSONSchemaFixture` instances pick these samples up for schema-based language tests. Additional narrow `comment-injection-*` fixtures cover affected outputs that did not already have full schema fixtures: Objective-C uses all four samples; TypeScript-Zod and TypeScript-Effect-Schema use the two enum-description samples. The `comment-injection-objective-c` fixture is registered but not run in CI: like the full `objective-c` fixture, the generated `.m` compiles cleanly with clang but the compiled `./test` binary segfaults at runtime (SIGSEGV) on the sample JSON — a pre-existing Objective-C harness issue unrelated to comment injection. It can be run locally on macOS. + +A parser-only fixture, `comment-injection-treesitter`, generates all configured targets and parses them with tree-sitter WASM grammars. It currently covers TypeScript, TypeScript-Zod, TypeScript-Effect-Schema, Swift, C#, Java, Dart, C (`cjson`), C++, PHP, Kotlin, Go, Pike, Rust, Ruby, Python, Elixir, Scala 3, and Haskell. It is intentionally one fixture/test that loops over all configured languages and reports all parse failures together. The Swift, Dart, Kotlin, Pike, and Elixir grammars are vendored under `test/tree-sitter-wasms` to avoid npm peer/native dependency issues in CI. Loaded grammars are cached per WASM path across targets and samples. + +Two grammar quirks require per-target workarounds in the fixture: + +- tree-sitter-dart wrongly parses a block-comment opener inside a line comment (`// /*`) as an ERROR, whereas real Dart runs line comments to the end of the line; the Dart target neutralizes `/*` inside line comments before parsing. +- tree-sitter-c parses preprocessor conditionals structurally and cannot match the `extern "C" {` brace across the two `#ifdef __cplusplus` blocks that `cjson` output uses as a guard, reporting a spurious `MISSING #endif` even for benign output; the cjson target strips those two guard blocks (which only wrap the file) before parsing, so a real injection that leaves a MISSING node is still caught. + +The fixture also scans the raw generated bytes for injection classes the grammars can't detect (their lexers are more lenient than the real compilers): no output may contain a surviving U+0085/U+2028/U+2029 line terminator, and per-target `forbiddenSubstrings` catch markup that parses as comment text but is dangerous downstream (C# rejects the raw ` &
` payload, verifying the XML entity escaping). + +These are regression tests and should pass with the shared comment escaping/sanitization in place. diff --git a/package-lock.json b/package-lock.json index 06837b485..0fad90016 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "quicktype", - "version": "23.2.0", + "version": "23.3.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "quicktype", - "version": "23.2.0", + "version": "23.3.0", "license": "Apache-2.0", "workspaces": [ "./packages/quicktype-core", @@ -56,8 +56,21 @@ "promise-timeout": "^1.3.0", "semver": "^7.5.4", "shelljs": "^0.8.5", + "tree-sitter-c": "^0.24.1", + "tree-sitter-c-sharp": "^0.23.5", + "tree-sitter-cpp": "^0.23.4", + "tree-sitter-go": "^0.25.0", + "tree-sitter-haskell": "^0.23.1", + "tree-sitter-java": "^0.23.5", + "tree-sitter-php": "^0.24.2", + "tree-sitter-python": "^0.25.0", + "tree-sitter-ruby": "^0.23.1", + "tree-sitter-rust": "^0.24.0", + "tree-sitter-scala": "^0.24.0", + "tree-sitter-typescript": "^0.23.2", "ts-node": "^10.9.2", - "watch": "^1.0.2" + "watch": "^1.0.2", + "web-tree-sitter": "^0.26.9" }, "engines": { "node": ">=18.12.0" @@ -692,6 +705,7 @@ "version": "22.14.1", "resolved": "https://registry.npmjs.org/@types/node/-/node-22.14.1.tgz", "integrity": "sha512-u0HuPQwe/dHrItgHHpmw3N2fYCR6x4ivMNbPHRkBVP4CvN+kiRrKHWk3i8tXiO/joPwXLMYvF9TTF0eqgHIuOw==", + "dev": true, "license": "MIT", "dependencies": { "undici-types": "~6.21.0" @@ -4146,6 +4160,18 @@ } } }, + "node_modules/node-gyp-build": { + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", + "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==", + "dev": true, + "license": "MIT", + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, "node_modules/node-persist": { "version": "4.0.1", "dev": true, @@ -5338,6 +5364,418 @@ "node": ">=14" } }, + "node_modules/tree-sitter-c": { + "version": "0.24.1", + "resolved": "https://registry.npmjs.org/tree-sitter-c/-/tree-sitter-c-0.24.1.tgz", + "integrity": "sha512-lkYwWN3SRecpvaeqmFKkuPNR3ZbtnvHU+4XAEEkJdrp3JfSp2pBrhXOtvfsENUneye76g889Y0ddF2DM0gEDpA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^8.3.1", + "node-gyp-build": "^4.8.4" + }, + "peerDependencies": { + "tree-sitter": "^0.22.4" + }, + "peerDependenciesMeta": { + "tree-sitter": { + "optional": true + } + } + }, + "node_modules/tree-sitter-c-sharp": { + "version": "0.23.5", + "resolved": "https://registry.npmjs.org/tree-sitter-c-sharp/-/tree-sitter-c-sharp-0.23.5.tgz", + "integrity": "sha512-xJGOeXPMmld0nES5+080N/06yY6LQi+KWGWV4LfZaZe6srJPtUtfhIbRSN7EZN6IaauzW28v6W4QHFwmeUW6HQ==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^8.2.2", + "node-gyp-build": "^4.8.4" + }, + "peerDependencies": { + "tree-sitter": "^0.25.0" + }, + "peerDependenciesMeta": { + "tree-sitter": { + "optional": true + } + } + }, + "node_modules/tree-sitter-c-sharp/node_modules/node-addon-api": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.9.0.tgz", + "integrity": "sha512-ekZMeaaIzSQTSpr7X2X3iJM7lTzgnx8ahAG9pJfT/7+14mlEM8ZYQ9cgCDvSSRbReFK0oHli3WrZdCiRsgAT9Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, + "node_modules/tree-sitter-c/node_modules/node-addon-api": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.9.0.tgz", + "integrity": "sha512-ekZMeaaIzSQTSpr7X2X3iJM7lTzgnx8ahAG9pJfT/7+14mlEM8ZYQ9cgCDvSSRbReFK0oHli3WrZdCiRsgAT9Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, + "node_modules/tree-sitter-cpp": { + "version": "0.23.4", + "resolved": "https://registry.npmjs.org/tree-sitter-cpp/-/tree-sitter-cpp-0.23.4.tgz", + "integrity": "sha512-qR5qUDyhZ5jJ6V8/umiBxokRbe89bCGmcq/dk94wI4kN86qfdV8k0GHIUEKaqWgcu42wKal5E97LKpLeVW8sKw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^8.2.1", + "node-gyp-build": "^4.8.2", + "tree-sitter-c": "^0.23.1" + }, + "peerDependencies": { + "tree-sitter": "^0.21.1" + }, + "peerDependenciesMeta": { + "tree-sitter": { + "optional": true + } + } + }, + "node_modules/tree-sitter-cpp/node_modules/node-addon-api": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.9.0.tgz", + "integrity": "sha512-ekZMeaaIzSQTSpr7X2X3iJM7lTzgnx8ahAG9pJfT/7+14mlEM8ZYQ9cgCDvSSRbReFK0oHli3WrZdCiRsgAT9Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, + "node_modules/tree-sitter-cpp/node_modules/tree-sitter-c": { + "version": "0.23.6", + "resolved": "https://registry.npmjs.org/tree-sitter-c/-/tree-sitter-c-0.23.6.tgz", + "integrity": "sha512-0dxXKznVyUA0s6PjNolJNs2yF87O5aL538A/eR6njA5oqX3C3vH4vnx3QdOKwuUdpKEcFdHuiDpRKLLCA/tjvQ==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^8.3.0", + "node-gyp-build": "^4.8.4" + }, + "peerDependencies": { + "tree-sitter": "^0.22.1" + }, + "peerDependenciesMeta": { + "tree-sitter": { + "optional": true + } + } + }, + "node_modules/tree-sitter-go": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/tree-sitter-go/-/tree-sitter-go-0.25.0.tgz", + "integrity": "sha512-APBc/Dq3xz/e35Xpkhb1blu5UgW+2E3RyGWawZSCNcbGwa7jhSQPS8KsUupuzBla8PCo8+lz9W/JDJjmfRa2tw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^8.3.1", + "node-gyp-build": "^4.8.4" + }, + "peerDependencies": { + "tree-sitter": "^0.25.0" + }, + "peerDependenciesMeta": { + "tree-sitter": { + "optional": true + } + } + }, + "node_modules/tree-sitter-go/node_modules/node-addon-api": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.9.0.tgz", + "integrity": "sha512-ekZMeaaIzSQTSpr7X2X3iJM7lTzgnx8ahAG9pJfT/7+14mlEM8ZYQ9cgCDvSSRbReFK0oHli3WrZdCiRsgAT9Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, + "node_modules/tree-sitter-haskell": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/tree-sitter-haskell/-/tree-sitter-haskell-0.23.1.tgz", + "integrity": "sha512-qG4CYhejveu9DLMLEGBz/n9/TTeGSFLC6wniwOgG6m8/v7Dng8qR0ob0EVG7+XH+9WiOxohpGA23EhceWuxY4w==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^8.2.2", + "node-gyp-build": "^4.8.2" + }, + "peerDependencies": { + "tree-sitter": "^0.21.1" + }, + "peerDependenciesMeta": { + "tree-sitter": { + "optional": true + } + } + }, + "node_modules/tree-sitter-haskell/node_modules/node-addon-api": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.9.0.tgz", + "integrity": "sha512-ekZMeaaIzSQTSpr7X2X3iJM7lTzgnx8ahAG9pJfT/7+14mlEM8ZYQ9cgCDvSSRbReFK0oHli3WrZdCiRsgAT9Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, + "node_modules/tree-sitter-java": { + "version": "0.23.5", + "resolved": "https://registry.npmjs.org/tree-sitter-java/-/tree-sitter-java-0.23.5.tgz", + "integrity": "sha512-Yju7oQ0Xx7GcUT01mUglPP+bYfvqjNCGdxqigTnew9nLGoII42PNVP3bHrYeMxswiCRM0yubWmN5qk+zsg0zMA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^8.2.2", + "node-gyp-build": "^4.8.2" + }, + "peerDependencies": { + "tree-sitter": "^0.21.1" + }, + "peerDependenciesMeta": { + "tree-sitter": { + "optional": true + } + } + }, + "node_modules/tree-sitter-java/node_modules/node-addon-api": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.9.0.tgz", + "integrity": "sha512-ekZMeaaIzSQTSpr7X2X3iJM7lTzgnx8ahAG9pJfT/7+14mlEM8ZYQ9cgCDvSSRbReFK0oHli3WrZdCiRsgAT9Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, + "node_modules/tree-sitter-javascript": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/tree-sitter-javascript/-/tree-sitter-javascript-0.23.1.tgz", + "integrity": "sha512-/bnhbrTD9frUYHQTiYnPcxyHORIw157ERBa6dqzaKxvR/x3PC4Yzd+D1pZIMS6zNg2v3a8BZ0oK7jHqsQo9fWA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^8.2.2", + "node-gyp-build": "^4.8.2" + }, + "peerDependencies": { + "tree-sitter": "^0.21.1" + }, + "peerDependenciesMeta": { + "tree-sitter": { + "optional": true + } + } + }, + "node_modules/tree-sitter-javascript/node_modules/node-addon-api": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.9.0.tgz", + "integrity": "sha512-ekZMeaaIzSQTSpr7X2X3iJM7lTzgnx8ahAG9pJfT/7+14mlEM8ZYQ9cgCDvSSRbReFK0oHli3WrZdCiRsgAT9Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, + "node_modules/tree-sitter-php": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/tree-sitter-php/-/tree-sitter-php-0.24.2.tgz", + "integrity": "sha512-zwgAePc/HozNaWOOfwRAA+3p8yhuehRw8Fb7vn5qd2XjiIc93uJPryDTMYTSjBRjVIUg/KY6pM3rRzs8dSwKfw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^8.2.2", + "node-gyp-build": "^4.8.2" + }, + "peerDependencies": { + "tree-sitter": "^0.22.4" + }, + "peerDependenciesMeta": { + "tree-sitter": { + "optional": true + } + } + }, + "node_modules/tree-sitter-php/node_modules/node-addon-api": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.9.0.tgz", + "integrity": "sha512-ekZMeaaIzSQTSpr7X2X3iJM7lTzgnx8ahAG9pJfT/7+14mlEM8ZYQ9cgCDvSSRbReFK0oHli3WrZdCiRsgAT9Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, + "node_modules/tree-sitter-python": { + "version": "0.25.0", + "resolved": "https://registry.npmjs.org/tree-sitter-python/-/tree-sitter-python-0.25.0.tgz", + "integrity": "sha512-eCmJx6zQa35GxaCtQD+wXHOhYqBxEL+bp71W/s3fcDMu06MrtzkVXR437dRrCrbrDbyLuUDJpAgycs7ncngLXw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^8.5.0", + "node-gyp-build": "^4.8.4" + }, + "peerDependencies": { + "tree-sitter": "^0.25.0" + }, + "peerDependenciesMeta": { + "tree-sitter": { + "optional": true + } + } + }, + "node_modules/tree-sitter-python/node_modules/node-addon-api": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.9.0.tgz", + "integrity": "sha512-ekZMeaaIzSQTSpr7X2X3iJM7lTzgnx8ahAG9pJfT/7+14mlEM8ZYQ9cgCDvSSRbReFK0oHli3WrZdCiRsgAT9Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, + "node_modules/tree-sitter-ruby": { + "version": "0.23.1", + "resolved": "https://registry.npmjs.org/tree-sitter-ruby/-/tree-sitter-ruby-0.23.1.tgz", + "integrity": "sha512-d9/RXgWjR6HanN7wTYhS5bpBQLz1VkH048Vm3CodPGyJVnamXMGb8oEhDypVCBq4QnHui9sTXuJBBP3WtCw5RA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^8.2.2", + "node-gyp-build": "^4.8.2" + }, + "peerDependencies": { + "tree-sitter": "^0.21.1" + }, + "peerDependenciesMeta": { + "tree-sitter": { + "optional": true + } + } + }, + "node_modules/tree-sitter-ruby/node_modules/node-addon-api": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.9.0.tgz", + "integrity": "sha512-ekZMeaaIzSQTSpr7X2X3iJM7lTzgnx8ahAG9pJfT/7+14mlEM8ZYQ9cgCDvSSRbReFK0oHli3WrZdCiRsgAT9Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, + "node_modules/tree-sitter-rust": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/tree-sitter-rust/-/tree-sitter-rust-0.24.0.tgz", + "integrity": "sha512-NWemUDf629Tfc90Y0Z55zuwPCAHkLxWnMf2RznYu4iBkkrQl2o/CHGB7Cr52TyN5F1DAx8FmUnDtCy9iUkXZEQ==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^8.2.2", + "node-gyp-build": "^4.8.4" + }, + "peerDependencies": { + "tree-sitter": "^0.22.1" + }, + "peerDependenciesMeta": { + "tree-sitter": { + "optional": true + } + } + }, + "node_modules/tree-sitter-rust/node_modules/node-addon-api": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.9.0.tgz", + "integrity": "sha512-ekZMeaaIzSQTSpr7X2X3iJM7lTzgnx8ahAG9pJfT/7+14mlEM8ZYQ9cgCDvSSRbReFK0oHli3WrZdCiRsgAT9Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, + "node_modules/tree-sitter-scala": { + "version": "0.24.0", + "resolved": "https://registry.npmjs.org/tree-sitter-scala/-/tree-sitter-scala-0.24.0.tgz", + "integrity": "sha512-vkMuAUrBZ1zZz2XcGDQk18Kz73JkpgaeXzbNVobPke0G35sd9jH32aUxG6OLRKM7et0TbsfqkWf4DeJoGk4K1g==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^8.2.2", + "node-gyp-build": "^4.8.2" + }, + "peerDependencies": { + "tree-sitter": "^0.21.1" + }, + "peerDependenciesMeta": { + "tree-sitter": { + "optional": true + } + } + }, + "node_modules/tree-sitter-scala/node_modules/node-addon-api": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.9.0.tgz", + "integrity": "sha512-ekZMeaaIzSQTSpr7X2X3iJM7lTzgnx8ahAG9pJfT/7+14mlEM8ZYQ9cgCDvSSRbReFK0oHli3WrZdCiRsgAT9Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, + "node_modules/tree-sitter-typescript": { + "version": "0.23.2", + "resolved": "https://registry.npmjs.org/tree-sitter-typescript/-/tree-sitter-typescript-0.23.2.tgz", + "integrity": "sha512-e04JUUKxTT53/x3Uq1zIL45DoYKVfHH4CZqwgZhPg5qYROl5nQjV+85ruFzFGZxu+QeFVbRTPDRnqL9UbU4VeA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-addon-api": "^8.2.2", + "node-gyp-build": "^4.8.2", + "tree-sitter-javascript": "^0.23.1" + }, + "peerDependencies": { + "tree-sitter": "^0.21.0" + }, + "peerDependenciesMeta": { + "tree-sitter": { + "optional": true + } + } + }, + "node_modules/tree-sitter-typescript/node_modules/node-addon-api": { + "version": "8.9.0", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.9.0.tgz", + "integrity": "sha512-ekZMeaaIzSQTSpr7X2X3iJM7lTzgnx8ahAG9pJfT/7+14mlEM8ZYQ9cgCDvSSRbReFK0oHli3WrZdCiRsgAT9Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, "node_modules/ts-api-utils": { "version": "1.3.0", "dev": true, @@ -5481,6 +5919,7 @@ "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, "license": "MIT" }, "node_modules/unicode-properties": { @@ -5584,6 +6023,13 @@ "node": ">=0.1.95" } }, + "node_modules/web-tree-sitter": { + "version": "0.26.9", + "resolved": "https://registry.npmjs.org/web-tree-sitter/-/web-tree-sitter-0.26.9.tgz", + "integrity": "sha512-YJwSHANl6XFgeEjB8nitgj0qZYt5gkIesJ4w2srS2wcLB4GUa4xcOkM0YaMsU6WNR53YVIkDSY7Ej4pf3IXtCA==", + "dev": true, + "license": "MIT" + }, "node_modules/webidl-conversions": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", @@ -5902,7 +6348,7 @@ } }, "packages/quicktype-core": { - "version": "18.0.15", + "version": "23.3.0", "license": "Apache-2.0", "dependencies": { "@glideapps/ts-necessities": "2.2.3", @@ -5942,7 +6388,7 @@ } }, "packages/quicktype-graphql-input": { - "version": "18.0.15", + "version": "23.3.0", "license": "Apache-2.0", "dependencies": { "collection-utils": "^1.0.1", @@ -5956,7 +6402,7 @@ } }, "packages/quicktype-typescript-input": { - "version": "18.0.15", + "version": "23.3.0", "license": "Apache-2.0", "dependencies": { "@mark.probst/typescript-json-schema": "0.55.0", @@ -6051,7 +6497,7 @@ }, "packages/quicktype-vscode": { "name": "quicktype", - "version": "23.0.122", + "version": "23.3.0", "devDependencies": { "@types/mocha": "^10.0.6", "@types/node": "~22.14.0", diff --git a/package.json b/package.json index fb19d8882..84a6fd7cb 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "quicktype", - "version": "23.2.0", + "version": "23.3.0", "license": "Apache-2.0", "main": "dist/index.js", "types": "dist/index.d.ts", @@ -63,8 +63,21 @@ "promise-timeout": "^1.3.0", "semver": "^7.5.4", "shelljs": "^0.8.5", + "tree-sitter-c": "^0.24.1", + "tree-sitter-c-sharp": "^0.23.5", + "tree-sitter-cpp": "^0.23.4", + "tree-sitter-go": "^0.25.0", + "tree-sitter-haskell": "^0.23.1", + "tree-sitter-java": "^0.23.5", + "tree-sitter-php": "^0.24.2", + "tree-sitter-python": "^0.25.0", + "tree-sitter-ruby": "^0.23.1", + "tree-sitter-rust": "^0.24.0", + "tree-sitter-scala": "^0.24.0", + "tree-sitter-typescript": "^0.23.2", "ts-node": "^10.9.2", - "watch": "^1.0.2" + "watch": "^1.0.2", + "web-tree-sitter": "^0.26.9" }, "overrides": { "cross-fetch": { diff --git a/packages/quicktype-core/package.json b/packages/quicktype-core/package.json index 769199b79..83fda0471 100644 --- a/packages/quicktype-core/package.json +++ b/packages/quicktype-core/package.json @@ -1,6 +1,6 @@ { "name": "quicktype-core", - "version": "18.0.15", + "version": "23.3.0", "description": "The quicktype engine as a library", "license": "Apache-2.0", "main": "dist/index.js", diff --git a/packages/quicktype-core/src/ConvenienceRenderer.ts b/packages/quicktype-core/src/ConvenienceRenderer.ts index 41ae7e4dd..bbc99243f 100644 --- a/packages/quicktype-core/src/ConvenienceRenderer.ts +++ b/packages/quicktype-core/src/ConvenienceRenderer.ts @@ -97,7 +97,13 @@ function splitDescription( descriptions: Iterable | undefined, ): string[] | undefined { if (descriptions === undefined) return undefined; - const description = Array.from(descriptions).join("\n\n").trim(); + // U+0085, U+2028 and U+2029 count as line terminators in some + // target languages (JavaScript, C#), so they must not survive + // into single-line comments. + const description = Array.from(descriptions) + .join("\n\n") + .replace(/\r\n?|[\u0085\u2028\u2029]/g, "\n") + .trim(); if (description === "") return undefined; return wordWrap(description) .split("\n") @@ -1166,6 +1172,14 @@ export abstract class ConvenienceRenderer extends Renderer { afterComment, }: CommentOptions = {}, ): void { + const replacements = this.commentLineEscapes({ + lineStart, + firstLineStart, + lineEnd, + beforeComment, + afterComment, + }); + if (beforeComment !== undefined) { this.emitLine(beforeComment); } @@ -1174,15 +1188,20 @@ export abstract class ConvenienceRenderer extends Renderer { for (const line of lines) { let start = first ? firstLineStart : lineStart; first = false; + const escapedLine = this.escapeCommentLine( + line, + replacements, + lineEnd, + ); - if (this.sourcelikeToString(line) === "") { + if (this.sourcelikeToString(escapedLine) === "") { start = trimEnd(start); } if (lineEnd) { - this.emitLine(start, line, lineEnd); + this.emitLine(start, escapedLine, lineEnd); } else { - this.emitLine(start, line); + this.emitLine(start, escapedLine); } } @@ -1197,6 +1216,77 @@ export abstract class ConvenienceRenderer extends Renderer { this.emitDescriptionBlock(description); } + private commentLineEscapes( + options: Required> & + Pick, + ): ReadonlyArray { + const delimiters = [ + options.lineStart, + options.firstLineStart, + options.lineEnd, + options.beforeComment, + options.afterComment, + ]; + const containsDelimiter = (delimiter: string): boolean => + delimiters.some((part) => part?.includes(delimiter) ?? false); + + const replacements: Array = []; + if (containsDelimiter("/*") || containsDelimiter("*/")) { + // The opener must be escaped, too: some languages (Kotlin, + // Scala) nest block comments, so a lone `/*` would reopen a + // comment that the closing delimiter can't terminate. + replacements.push(["/*", "/ *"], ["*/", "* /"]); + } + if (containsDelimiter("{-") || containsDelimiter("-}")) { + replacements.push(["{-", "{ -"], ["-}", "- }"]); + } + if (containsDelimiter('"""')) { + // Triple-quoted comments (Python docstrings, Elixir + // moduledocs) are string literals, so backslashes are escape + // characters — in particular a trailing backslash would + // swallow the first quote of the closing delimiter. + replacements.push(["\\", "\\\\"], ['"""', '\\"\\"\\"']); + } + return replacements; + } + + // C-family lexers splice a backslash-newline into one line even + // inside comments, which would pull the following line of + // generated code into an attacker-controlled comment. + protected get commentLinesSpliceOnBackslash(): boolean { + return false; + } + + private escapeCommentLine( + line: Sourcelike, + replacements: ReadonlyArray, + lineEnd?: string, + ): Sourcelike { + if (replacements.length === 0 && !this.commentLinesSpliceOnBackslash) { + return line; + } + + let escaped = replacements.reduce( + (result, [unsafe, safe]) => result.split(unsafe).join(safe), + this.sourcelikeToString(line), + ); + // A quote at the end of the line would merge with a closing + // `"""` emitted right after it. The quote needs escaping iff + // it's preceded by an even number of backslashes. + if (lineEnd !== undefined && lineEnd.startsWith('"')) { + const trailing = /(\\*)"$/.exec(escaped); + if (trailing !== null && trailing[1].length % 2 === 0) { + escaped = `${escaped.slice(0, -1)}\\"`; + } + } + + if (this.commentLinesSpliceOnBackslash && escaped.endsWith("\\")) { + escaped = `${escaped}.`; + } + + return escaped; + } + protected emitDescriptionBlock(lines: Sourcelike[]): void { this.emitCommentLines(lines); } diff --git a/packages/quicktype-core/src/language/CJSON/CJSONRenderer.ts b/packages/quicktype-core/src/language/CJSON/CJSONRenderer.ts index 936a0baa4..1e20fa94c 100644 --- a/packages/quicktype-core/src/language/CJSON/CJSONRenderer.ts +++ b/packages/quicktype-core/src/language/CJSON/CJSONRenderer.ts @@ -5704,6 +5704,10 @@ export class CJSONRenderer extends ConvenienceRenderer { ); } + protected get commentLinesSpliceOnBackslash(): boolean { + return true; + } + /** * Emit description block * @param lines: description block lines diff --git a/packages/quicktype-core/src/language/CPlusPlus/CPlusPlusRenderer.ts b/packages/quicktype-core/src/language/CPlusPlus/CPlusPlusRenderer.ts index 4013ebc7c..0d49aecda 100644 --- a/packages/quicktype-core/src/language/CPlusPlus/CPlusPlusRenderer.ts +++ b/packages/quicktype-core/src/language/CPlusPlus/CPlusPlusRenderer.ts @@ -586,6 +586,10 @@ export class CPlusPlusRenderer extends ConvenienceRenderer { return kind === "class"; } + protected get commentLinesSpliceOnBackslash(): boolean { + return true; + } + protected emitDescriptionBlock(lines: Sourcelike[]): void { this.emitCommentLines(lines, { lineStart: " * ", diff --git a/packages/quicktype-core/src/language/CSharp/CSharpRenderer.ts b/packages/quicktype-core/src/language/CSharp/CSharpRenderer.ts index c8f16738f..83a49e752 100644 --- a/packages/quicktype-core/src/language/CSharp/CSharpRenderer.ts +++ b/packages/quicktype-core/src/language/CSharp/CSharpRenderer.ts @@ -256,11 +256,19 @@ export class CSharpRenderer extends ConvenienceRenderer { } protected emitDescriptionBlock(lines: Sourcelike[]): void { + // Doc comments are XML, so anything that could be mistaken + // for markup must be escaped. + const escapedLines = lines.map((line) => + this.sourcelikeToString(line) + .replace(/&/g, "&") + .replace(//g, ">"), + ); const start = "/// "; if (this._csOptions.dense) { - this.emitLine(start, lines.join("; "), ""); + this.emitLine(start, escapedLines.join("; "), ""); } else { - this.emitCommentLines(lines, { + this.emitCommentLines(escapedLines, { lineStart: "/// ", beforeComment: start, afterComment: "/// ", diff --git a/packages/quicktype-core/src/language/Objective-C/ObjectiveCRenderer.ts b/packages/quicktype-core/src/language/Objective-C/ObjectiveCRenderer.ts index 5d0917db5..170554c86 100644 --- a/packages/quicktype-core/src/language/Objective-C/ObjectiveCRenderer.ts +++ b/packages/quicktype-core/src/language/Objective-C/ObjectiveCRenderer.ts @@ -135,6 +135,10 @@ export class ObjectiveCRenderer extends ConvenienceRenderer { return type; } + protected get commentLinesSpliceOnBackslash(): boolean { + return true; + } + protected emitDescriptionBlock(lines: Sourcelike[]): void { this.emitCommentLines(lines, { lineStart: "/// " }); } diff --git a/packages/quicktype-core/src/language/Php/PhpRenderer.ts b/packages/quicktype-core/src/language/Php/PhpRenderer.ts index 8dfb3c5e6..97deb5782 100644 --- a/packages/quicktype-core/src/language/Php/PhpRenderer.ts +++ b/packages/quicktype-core/src/language/Php/PhpRenderer.ts @@ -228,6 +228,19 @@ export class PhpRenderer extends ConvenienceRenderer { }); } + private emitDocBlockDescription(desc: string[] | undefined): void { + if (desc === undefined) { + this.emitLine("/**"); + return; + } + + this.emitCommentLines(desc, { + lineStart: " * ", + beforeComment: "/**", + }); + this.emitLine(" *"); + } + public emitBlock(line: Sourcelike, f: () => void): void { this.emitLine(line, " {"); this.indent(f); @@ -824,11 +837,7 @@ export class PhpRenderer extends ConvenienceRenderer { _name: Name, desc?: string[], ): void { - this.emitLine("/**"); - if (desc !== undefined) { - this.emitLine(" * ", desc); - this.emitLine(" *"); - } + this.emitDocBlockDescription(desc); // this.emitLine(" * @param ", this.phpType(false, p.type, false, "", "|null")); this.emitLine( @@ -867,11 +876,7 @@ export class PhpRenderer extends ConvenienceRenderer { name: Name, desc?: string[], ): void { - this.emitLine("/**"); - if (desc !== undefined) { - this.emitLine(" * ", desc); - this.emitLine(" *"); - } + this.emitDocBlockDescription(desc); this.emitLine(" * @throws Exception"); this.emitLine(" * @return ", this.phpConvertType(className, p.type)); @@ -921,11 +926,7 @@ export class PhpRenderer extends ConvenienceRenderer { name: Name, desc?: string[], ): void { - this.emitLine("/**"); - if (desc !== undefined) { - this.emitLine(" * ", desc); - this.emitLine(" *"); - } + this.emitDocBlockDescription(desc); this.emitLine( " * @param ", @@ -957,11 +958,7 @@ export class PhpRenderer extends ConvenienceRenderer { desc?: string[], ): void { if (this._options.withGet) { - this.emitLine("/**"); - if (desc !== undefined) { - this.emitLine(" * ", desc); - this.emitLine(" *"); - } + this.emitDocBlockDescription(desc); if (!this._options.fastGet) { this.emitLine(" * @throws Exception"); @@ -1013,11 +1010,7 @@ export class PhpRenderer extends ConvenienceRenderer { desc?: string[], ): void { if (this._options.withSet) { - this.emitLine("/**"); - if (desc !== undefined) { - this.emitLine(" * ", desc); - this.emitLine(" *"); - } + this.emitDocBlockDescription(desc); this.emitLine( " * @param ", @@ -1054,11 +1047,7 @@ export class PhpRenderer extends ConvenienceRenderer { idx: number, ): void { if (this._options.withGet) { - this.emitLine("/**"); - if (desc !== undefined) { - this.emitLine(" * ", desc); - this.emitLine(" *"); - } + this.emitDocBlockDescription(desc); const rendered = this.phpType(false, p.type); this.emitLine(" * @return ", rendered); diff --git a/packages/quicktype-core/src/language/Python/PythonRenderer.ts b/packages/quicktype-core/src/language/Python/PythonRenderer.ts index fd7bb84e8..0bf0e5652 100644 --- a/packages/quicktype-core/src/language/Python/PythonRenderer.ts +++ b/packages/quicktype-core/src/language/Python/PythonRenderer.ts @@ -13,7 +13,7 @@ import { import { type Name, type Namer, funPrefixNamer } from "../../Naming"; import type { RenderContext } from "../../Renderer"; import type { OptionValues } from "../../RendererOptions"; -import { type Sourcelike, modifySource } from "../../Source"; +import type { Sourcelike } from "../../Source"; import { stringEscape } from "../../support/Strings"; import { defined, panic } from "../../support/Support"; import type { TargetLanguage } from "../../TargetLanguage"; @@ -88,15 +88,8 @@ export class PythonRenderer extends ConvenienceRenderer { protected emitDescriptionBlock(lines: Sourcelike[]): void { if (lines.length === 1) { - const docstring = modifySource((content) => { - if (content.endsWith('"')) { - return content.slice(0, -1) + '\\"'; - } - - return content; - }, lines[0]); this.emitComments([ - { customLines: [docstring], lineStart: '"""', lineEnd: '"""' }, + { customLines: lines, lineStart: '"""', lineEnd: '"""' }, ]); } else { this.emitCommentLines(lines, { diff --git a/packages/quicktype-graphql-input/package.json b/packages/quicktype-graphql-input/package.json index 6a95cf3dd..7c7fdd255 100644 --- a/packages/quicktype-graphql-input/package.json +++ b/packages/quicktype-graphql-input/package.json @@ -1,6 +1,6 @@ { "name": "quicktype-graphql-input", - "version": "18.0.15", + "version": "23.3.0", "package": "Package for using GraphQL as an input language to quicktype", "license": "Apache-2.0", "main": "dist/index.js", diff --git a/packages/quicktype-typescript-input/package.json b/packages/quicktype-typescript-input/package.json index ee5b0442a..2d8c59732 100644 --- a/packages/quicktype-typescript-input/package.json +++ b/packages/quicktype-typescript-input/package.json @@ -1,6 +1,6 @@ { "name": "quicktype-typescript-input", - "version": "18.0.15", + "version": "23.3.0", "description": "Package for using TypeScript as an input language to quicktype", "license": "Apache-2.0", "main": "dist/index.js", diff --git a/packages/quicktype-vscode/package.json b/packages/quicktype-vscode/package.json index 702830d32..1a413cbfb 100644 --- a/packages/quicktype-vscode/package.json +++ b/packages/quicktype-vscode/package.json @@ -2,7 +2,7 @@ "name": "quicktype", "displayName": "Paste JSON as Code", "description": "Copy JSON, paste as Go, TypeScript, C#, C++ and more.", - "version": "23.0.122", + "version": "23.3.0", "publisher": "quicktype", "icon": "media/icon.png", "galleryBanner": { diff --git a/test/fixtures.ts b/test/fixtures.ts index d4eb8f2fd..a26ee3f16 100644 --- a/test/fixtures.ts +++ b/test/fixtures.ts @@ -818,6 +818,241 @@ class JSONSchemaFixture extends LanguageFixture { } } +type TreeSitterTarget = { + displayName: string; + language: languages.Language; + output: string; + wasmModule: string; + extensions: string[]; + schema: string; + allowMissingNodes?: boolean; + // Rewrite the generated source before parsing, to work around + // false positives in a buggy tree-sitter grammar. + preprocessSource?: (source: string) => string; + // Raw substrings that must not survive into the generated output. + // Used for injection classes the tree-sitter grammars can't catch, + // because their lexers are more lenient than the real compiler + // (e.g. XML metacharacters in C# doc comments, which parse fine as + // comment text but break the XML doc tooling). + forbiddenSubstrings?: string[]; +}; + +// Line terminators that a language's real lexer honors but which are +// not `\n`, so they must never survive into a generated single-line +// comment. The tree-sitter grammars treat them as ordinary +// characters, so this is checked by content scan rather than parsing. +const unicodeLineTerminators: ReadonlyArray = [ + ["U+0085 NEL", 0x0085], + ["U+2028 LINE SEPARATOR", 0x2028], + ["U+2029 PARAGRAPH SEPARATOR", 0x2029], +]; + +type TreeSitterContentFailure = { + language: string; + filename: string; + forbidden: string; +}; + +type TreeSitterParseProblem = { + type: string; + startPosition: unknown; + endPosition: unknown; +}; + +type TreeSitterParseFailure = { + language: string; + filename: string; + problems: TreeSitterParseProblem[]; +}; + +const commentInjectionSchema = "test/inputs/schema/comment-injection.schema"; +const commentInjectionEnumSchema = + "test/inputs/schema/comment-injection-enum.schema"; +const commentInjectionNestedCommentSchema = + "test/inputs/schema/comment-injection-nested-comment.schema"; +const commentInjectionEnumNestedCommentSchema = + "test/inputs/schema/comment-injection-enum-nested-comment.schema"; +const treeSitterWasm = (filename: string): string => + path.join(__dirname, "tree-sitter-wasms", filename); + +const commentInjectionTreeSitterTargets: TreeSitterTarget[] = [ + { + displayName: "typescript", + language: languages.TypeScriptLanguage, + output: "TopLevel.ts", + wasmModule: "tree-sitter-typescript/tree-sitter-typescript.wasm", + extensions: [".ts"], + schema: commentInjectionSchema, + }, + { + displayName: "typescript-zod", + language: languages.TypeScriptZodLanguage, + output: "TopLevel.ts", + wasmModule: "tree-sitter-typescript/tree-sitter-typescript.wasm", + extensions: [".ts"], + schema: commentInjectionEnumSchema, + }, + { + displayName: "typescript-effect-schema", + language: languages.TypeScriptEffectSchemaLanguage, + output: "TopLevel.ts", + wasmModule: "tree-sitter-typescript/tree-sitter-typescript.wasm", + extensions: [".ts"], + schema: commentInjectionEnumSchema, + }, + { + displayName: "swift", + language: languages.SwiftLanguage, + output: "quicktype.swift", + wasmModule: treeSitterWasm("tree-sitter-swift.wasm"), + extensions: [".swift"], + schema: commentInjectionSchema, + }, + { + displayName: "csharp", + language: languages.CSharpLanguage, + output: "QuickType.cs", + wasmModule: "tree-sitter-c-sharp/tree-sitter-c_sharp.wasm", + extensions: [".cs"], + schema: commentInjectionSchema, + // C# doc comments are XML; the payload's markup must be entity + // escaped. tree-sitter parses it as comment text either way, + // so the escaping is verified by content scan. + forbiddenSubstrings: [" &
", " &
"], + }, + { + displayName: "java", + language: languages.JavaLanguage, + output: "TopLevel.java", + wasmModule: "tree-sitter-java/tree-sitter-java.wasm", + extensions: [".java"], + schema: commentInjectionSchema, + }, + { + displayName: "dart", + language: languages.DartLanguage, + output: "TopLevel.dart", + wasmModule: treeSitterWasm("tree-sitter-dart.wasm"), + extensions: [".dart"], + schema: commentInjectionSchema, + // tree-sitter-dart wrongly treats a block-comment opener inside + // a line comment as opening a block comment (`// /*` parses as + // an ERROR), whereas real Dart runs line comments to the end of + // the line. Neutralize openers in comment text before parsing. + preprocessSource: (source) => + source.replace(/\/\/[^\n]*/g, (comment) => + comment.replace(/\/\*/g, "/ *"), + ), + }, + { + displayName: "cjson", + language: languages.CJSONLanguage, + output: "TopLevel.c", + wasmModule: "tree-sitter-c/tree-sitter-c.wasm", + extensions: [".c", ".h"], + schema: commentInjectionSchema, + // tree-sitter-c parses `#ifdef`/`#endif` as real structure and + // can't match the `extern "C" {` brace across the two separate + // `#ifdef __cplusplus` guard blocks, so it reports a spurious + // MISSING `#endif` even for benign output. Strip the guard + // blocks (they only wrap the file) so a real injection that + // leaves a MISSING node is still caught. + preprocessSource: (source) => + source.replace( + /#ifdef __cplusplus\n(?:extern "C" \{|\})\n#endif\n/g, + "", + ), + }, + { + displayName: "cplusplus", + language: languages.CPlusPlusLanguage, + output: "TopLevel.cpp", + wasmModule: "tree-sitter-cpp/tree-sitter-cpp.wasm", + extensions: [".cpp", ".hpp", ".h"], + schema: commentInjectionSchema, + }, + { + displayName: "php", + language: languages.PHPLanguage, + output: "TopLevel.php", + wasmModule: "tree-sitter-php/tree-sitter-php.wasm", + extensions: [".php"], + schema: commentInjectionSchema, + }, + { + displayName: "kotlin", + language: languages.KotlinLanguage, + output: "TopLevel.kt", + wasmModule: treeSitterWasm("tree-sitter-kotlin.wasm"), + extensions: [".kt"], + schema: commentInjectionSchema, + }, + { + displayName: "go", + language: languages.GoLanguage, + output: "quicktype.go", + wasmModule: "tree-sitter-go/tree-sitter-go.wasm", + extensions: [".go"], + schema: commentInjectionSchema, + }, + { + displayName: "pike", + language: languages.PikeLanguage, + output: "TopLevel.pmod", + wasmModule: treeSitterWasm("tree-sitter-pike.wasm"), + extensions: [".pmod"], + schema: commentInjectionSchema, + }, + { + displayName: "rust", + language: languages.RustLanguage, + output: "module_under_test.rs", + wasmModule: "tree-sitter-rust/tree-sitter-rust.wasm", + extensions: [".rs"], + schema: commentInjectionSchema, + }, + { + displayName: "ruby", + language: languages.RubyLanguage, + output: "TopLevel.rb", + wasmModule: "tree-sitter-ruby/tree-sitter-ruby.wasm", + extensions: [".rb"], + schema: commentInjectionSchema, + }, + { + displayName: "python", + language: languages.PythonLanguage, + output: "quicktype.py", + wasmModule: "tree-sitter-python/tree-sitter-python.wasm", + extensions: [".py"], + schema: commentInjectionSchema, + }, + { + displayName: "elixir", + language: languages.ElixirLanguage, + output: "QuickType.ex", + wasmModule: treeSitterWasm("tree-sitter-elixir.wasm"), + extensions: [".ex"], + schema: commentInjectionSchema, + }, + { + displayName: "scala3", + language: languages.Scala3Language, + output: "TopLevel.scala", + wasmModule: "tree-sitter-scala/tree-sitter-scala.wasm", + extensions: [".scala"], + schema: commentInjectionSchema, + }, + { + displayName: "haskell", + language: languages.HaskellLanguage, + output: "QuickType.hs", + wasmModule: "tree-sitter-haskell/tree-sitter-haskell.wasm", + extensions: [".hs"], + schema: commentInjectionSchema, + }, +]; + function graphQLSchemaFilename(baseName: string): string { const baseMatch = baseName.match(/(.*\D)\d+$/); if (baseMatch === null) { @@ -829,6 +1064,280 @@ function graphQLSchemaFilename(baseName: string): string { return `${baseMatch[1]}.gqlschema`; } +class CommentInjectionSchemaFixture extends JSONSchemaFixture { + constructor( + language: languages.Language, + private readonly _samples: string[] = [ + "test/inputs/schema/comment-injection.schema", + "test/inputs/schema/comment-injection-enum.schema", + "test/inputs/schema/comment-injection-nested-comment.schema", + "test/inputs/schema/comment-injection-enum-nested-comment.schema", + ], + ) { + super(language, `comment-injection-${language.name}`); + } + + runForName(name: string): boolean { + return this.name === name || name === "comment-injection"; + } + + getSamples(sources: string[]): { priority: Sample[]; others: Sample[] } { + return samplesFromSources(sources, this._samples, [], "schema"); + } +} + +function collectFilesWithExtensions( + directory: string, + extensions: string[], +): string[] { + const result: string[] = []; + for (const entry of fs.readdirSync(directory)) { + const fullPath = path.join(directory, entry); + const stat = fs.statSync(fullPath); + if (stat.isDirectory()) { + result.push(...collectFilesWithExtensions(fullPath, extensions)); + } else if (extensions.includes(path.extname(entry))) { + result.push(fullPath); + } + } + return result; +} + +class CommentInjectionTreeSitterFixture extends Fixture { + name = "comment-injection-treesitter"; + + constructor() { + super(languages.TypeScriptLanguage); + } + + runForName(name: string): boolean { + return this.name === name; + } + + async setup(): Promise { + return; + } + + getSamples(sources: string[]): { priority: Sample[]; others: Sample[] } { + const commentInjectionSamples = [ + commentInjectionSchema, + commentInjectionNestedCommentSchema, + ]; + const makeSample = (schema: string): Sample => ({ + path: schema, + additionalRendererOptions: {}, + saveOutput: false, + }); + if (sources.length === 0) { + return { + priority: commentInjectionSamples.map(makeSample), + others: [], + }; + } + + const sourcePaths = _.flatMap(sources, (source) => + fs.existsSync(source) && fs.lstatSync(source).isDirectory() + ? testsInDir(source, "schema") + : [source], + ); + const selected = commentInjectionSamples.filter((schema) => + sourcePaths.some( + (source) => path.basename(source) === path.basename(schema), + ), + ); + return { priority: selected.map(makeSample), others: [] }; + } + + private readonly _treeSitterLanguages = new Map(); + + private async loadTreeSitterLanguage( + TreeSitter: any, + wasmModule: string, + ): Promise { + const wasmPath = require.resolve(wasmModule); + let language = this._treeSitterLanguages.get(wasmPath); + if (language === undefined) { + language = await TreeSitter.Language.load(wasmPath); + this._treeSitterLanguages.set(wasmPath, language); + } + + return language; + } + + private async parseGeneratedFiles( + TreeSitter: any, + target: TreeSitterTarget, + generatedFiles: string[], + ): Promise { + const parser = new TreeSitter.Parser(); + const language = await this.loadTreeSitterLanguage( + TreeSitter, + target.wasmModule, + ); + parser.setLanguage(language); + + const failures: TreeSitterParseFailure[] = []; + + for (const filename of generatedFiles) { + let source = fs.readFileSync(filename, "utf8"); + if (target.preprocessSource !== undefined) { + source = target.preprocessSource(source); + } + + const tree = parser.parse(source); + const problems: TreeSitterParseProblem[] = []; + + function visit(node: any): void { + if ( + node.type === "ERROR" || + (node.isMissing && !target.allowMissingNodes) + ) { + problems.push({ + type: node.isMissing + ? `MISSING ${node.type}` + : node.type, + startPosition: node.startPosition, + endPosition: node.endPosition, + }); + } + + for (let i = 0; i < node.childCount; i++) { + visit(node.child(i)); + } + } + + visit(tree.rootNode); + + if (problems.length > 0) { + failures.push({ + language: target.displayName, + filename, + problems: problems.slice(0, 10), + }); + } + } + + return failures; + } + + // Injection classes the tree-sitter grammars can't detect, because + // their lexers are more lenient than the real compiler: Unicode + // line terminators surviving into a comment, and per-target markup + // that parses as comment text but is dangerous downstream. These + // are verified by scanning the generated bytes directly. + private scanGeneratedFiles( + target: TreeSitterTarget, + generatedFiles: string[], + ): TreeSitterContentFailure[] { + const failures: TreeSitterContentFailure[] = []; + for (const filename of generatedFiles) { + const source = fs.readFileSync(filename, "utf8"); + for (const [name, codePoint] of unicodeLineTerminators) { + if (source.includes(String.fromCodePoint(codePoint))) { + failures.push({ + language: target.displayName, + filename, + forbidden: name, + }); + } + } + + for (const forbidden of target.forbiddenSubstrings ?? []) { + if (source.includes(forbidden)) { + failures.push({ + language: target.displayName, + filename, + forbidden: JSON.stringify(forbidden), + }); + } + } + } + + return failures; + } + + async runWithSample( + sample: Sample, + index: number, + total: number, + ): Promise { + const cwd = this.getRunDirectory(); + const message = this.runMessageStart(sample, index, total, cwd, false); + mkdirs(cwd); + + const TreeSitter = require("web-tree-sitter"); + await TreeSitter.Parser.init(); + + const repoRoot = process.cwd(); + let parsedFileCount = 0; + const failures: TreeSitterParseFailure[] = []; + const contentFailures: TreeSitterContentFailure[] = []; + await inDir(cwd, async () => { + for (const target of commentInjectionTreeSitterTargets) { + const outputDir = path.join(target.displayName); + mkdirs(outputDir); + const targetLanguage = { + ...target.language, + output: path.join(outputDir, target.output), + }; + // Enum targets can't render the regular schemas, so they + // get the enum variant of whichever sample is running. + const schema = + target.schema === commentInjectionEnumSchema + ? sample.path === commentInjectionNestedCommentSchema + ? commentInjectionEnumNestedCommentSchema + : commentInjectionEnumSchema + : sample.path; + await quicktypeForLanguage( + targetLanguage, + path.join(repoRoot, schema), + "schema", + false, + {}, + ); + + const generatedFiles = collectFilesWithExtensions( + outputDir, + target.extensions, + ); + if (generatedFiles.length === 0) { + failWith("No generated files to parse", { + language: target.displayName, + outputDir, + extensions: target.extensions, + }); + } + + failures.push( + ...(await this.parseGeneratedFiles( + TreeSitter, + target, + generatedFiles, + )), + ); + contentFailures.push( + ...this.scanGeneratedFiles(target, generatedFiles), + ); + parsedFileCount += generatedFiles.length; + } + }); + + if (failures.length > 0) { + failWith("tree-sitter parse found syntax errors", { + failures, + }); + } + + if (contentFailures.length > 0) { + failWith("generated output contains forbidden comment content", { + contentFailures, + }); + } + + this.runMessageEnd(message, parsedFileCount); + } +} + class GraphQLFixture extends LanguageFixture { constructor( language: languages.Language, @@ -1095,6 +1604,20 @@ export const allFixtures: Fixture[] = [ new JSONSchemaFixture(languages.PikeLanguage), new JSONSchemaFixture(languages.HaskellLanguage), new JSONSchemaFixture(languages.ElixirLanguage), + new CommentInjectionSchemaFixture(languages.TypeScriptLanguage), + new CommentInjectionSchemaFixture(languages.ObjectiveCLanguage), + new CommentInjectionSchemaFixture(languages.TypeScriptZodLanguage, [ + "test/inputs/schema/comment-injection-enum.schema", + "test/inputs/schema/comment-injection-enum-nested-comment.schema", + ]), + new CommentInjectionSchemaFixture( + languages.TypeScriptEffectSchemaLanguage, + [ + "test/inputs/schema/comment-injection-enum.schema", + "test/inputs/schema/comment-injection-enum-nested-comment.schema", + ], + ), + new CommentInjectionTreeSitterFixture(), // FIXME: Why are we missing so many language with GraphQL? new GraphQLFixture(languages.CSharpLanguage), new GraphQLFixture(languages.JavaLanguage), diff --git a/test/inputs/schema/comment-injection-enum-nested-comment.1.json b/test/inputs/schema/comment-injection-enum-nested-comment.1.json new file mode 100644 index 000000000..0e32666c2 --- /dev/null +++ b/test/inputs/schema/comment-injection-enum-nested-comment.1.json @@ -0,0 +1,3 @@ +{ + "kind": "ok" +} diff --git a/test/inputs/schema/comment-injection-enum-nested-comment.schema b/test/inputs/schema/comment-injection-enum-nested-comment.schema new file mode 100644 index 000000000..4e1e3ce3b --- /dev/null +++ b/test/inputs/schema/comment-injection-enum-nested-comment.schema @@ -0,0 +1,12 @@ +{ + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": ["ok"], + "description": "Nested enum block comment openers:\n{-\n/*\nThis text must remain inside the generated enum documentation comment." + } + }, + "required": ["kind"], + "additionalProperties": false +} diff --git a/test/inputs/schema/comment-injection-enum.1.json b/test/inputs/schema/comment-injection-enum.1.json new file mode 100644 index 000000000..0e32666c2 --- /dev/null +++ b/test/inputs/schema/comment-injection-enum.1.json @@ -0,0 +1,3 @@ +{ + "kind": "ok" +} diff --git a/test/inputs/schema/comment-injection-enum.schema b/test/inputs/schema/comment-injection-enum.schema new file mode 100644 index 000000000..eed5c117b --- /dev/null +++ b/test/inputs/schema/comment-injection-enum.schema @@ -0,0 +1,12 @@ +{ + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": ["ok"], + "description": "Comment delimiters:\n*/\n{-\n-}\n\"\"\"\n}\r}" + } + }, + "required": ["kind"], + "additionalProperties": false +} diff --git a/test/inputs/schema/comment-injection-nested-comment.1.json b/test/inputs/schema/comment-injection-nested-comment.1.json new file mode 100644 index 000000000..6f7f858be --- /dev/null +++ b/test/inputs/schema/comment-injection-nested-comment.1.json @@ -0,0 +1,3 @@ +{ + "value": "ok" +} diff --git a/test/inputs/schema/comment-injection-nested-comment.schema b/test/inputs/schema/comment-injection-nested-comment.schema new file mode 100644 index 000000000..4cb37abc4 --- /dev/null +++ b/test/inputs/schema/comment-injection-nested-comment.schema @@ -0,0 +1,12 @@ +{ + "type": "object", + "description": "Nested block comment openers:\n{-\n/*\nThis text must remain inside the generated documentation comment.", + "properties": { + "value": { + "type": "string", + "description": "Nested property block comment openers:\n{-\n/*\nThis text must remain inside the generated property documentation comment." + } + }, + "required": ["value"], + "additionalProperties": false +} diff --git a/test/inputs/schema/comment-injection.1.json b/test/inputs/schema/comment-injection.1.json new file mode 100644 index 000000000..6f7f858be --- /dev/null +++ b/test/inputs/schema/comment-injection.1.json @@ -0,0 +1,3 @@ +{ + "value": "ok" +} diff --git a/test/inputs/schema/comment-injection.schema b/test/inputs/schema/comment-injection.schema new file mode 100644 index 000000000..dcfba15c0 --- /dev/null +++ b/test/inputs/schema/comment-injection.schema @@ -0,0 +1,24 @@ +{ + "type": "object", + "description": "Comment delimiters:\n*/\n/*\n{-\n-}\n\"\"\"\n &
\n}\r}\u0085}\u2028}\u2029}", + "properties": { + "value": { + "type": "string", + "description": "Property delimiters:\n*/\n/*\n{-\n-}\n\"\"\"\n &
\n}\r}\u0085}\u2028}\u2029}" + }, + "trailingBackslash": { + "type": "string", + "description": "Ends with a backslash \\" + }, + "trailingQuote": { + "type": "string", + "description": "Ends with a quote \"" + }, + "trailingTripleQuote": { + "type": "string", + "description": "Ends with a triple quote \"\"\"" + } + }, + "required": ["value"], + "additionalProperties": false +} diff --git a/test/languages.ts b/test/languages.ts index 9573f3e57..db1a41fb1 100644 --- a/test/languages.ts +++ b/test/languages.ts @@ -93,9 +93,9 @@ export const CSharpLanguageSystemTextJson: Language = { name: "csharp", base: "test/fixtures/csharp-SystemTextJson", // https://github.com/dotnet/cli/issues/1582 - setupCommand: "dotnet restore --no-cache", + setupCommand: "dotnet restore -p:CheckEolTargetFramework=false --no-cache", runCommand(sample: string) { - return `dotnet run -- "${sample}"`; + return `dotnet run -p:CheckEolTargetFramework=false -- "${sample}"`; }, diffViaSchema: true, skipDiffViaSchema: ["34702.json", "437e7.json"], diff --git a/test/tree-sitter-wasms/tree-sitter-dart.wasm b/test/tree-sitter-wasms/tree-sitter-dart.wasm new file mode 100644 index 000000000..bad21c5dd Binary files /dev/null and b/test/tree-sitter-wasms/tree-sitter-dart.wasm differ diff --git a/test/tree-sitter-wasms/tree-sitter-elixir.wasm b/test/tree-sitter-wasms/tree-sitter-elixir.wasm new file mode 100644 index 000000000..128c34191 Binary files /dev/null and b/test/tree-sitter-wasms/tree-sitter-elixir.wasm differ diff --git a/test/tree-sitter-wasms/tree-sitter-kotlin.wasm b/test/tree-sitter-wasms/tree-sitter-kotlin.wasm new file mode 100644 index 000000000..e95090c99 Binary files /dev/null and b/test/tree-sitter-wasms/tree-sitter-kotlin.wasm differ diff --git a/test/tree-sitter-wasms/tree-sitter-pike.wasm b/test/tree-sitter-wasms/tree-sitter-pike.wasm new file mode 100644 index 000000000..37e77ff2b Binary files /dev/null and b/test/tree-sitter-wasms/tree-sitter-pike.wasm differ diff --git a/test/tree-sitter-wasms/tree-sitter-swift.wasm b/test/tree-sitter-wasms/tree-sitter-swift.wasm new file mode 100644 index 000000000..210e2c886 Binary files /dev/null and b/test/tree-sitter-wasms/tree-sitter-swift.wasm differ