From f123f2da9242fd4b305e5c24f9464a728ac3b9d0 Mon Sep 17 00:00:00 2001 From: Nima Akbarzadeh Date: Sat, 12 Sep 2026 00:29:20 +0200 Subject: [PATCH] Release 1.2.0: fix streaming edge cases, linear-time scanner, packaging Behaviour-preserving release. Every input 1.1.0 parsed successfully parses to the same value; tests/test_compat_1_1_0.py runs the frozen 1.1.0 parser next to the new one over every prefix of a corpus, and tests/fuzz_compat_1_1_0.py does the same for random documents. Fixed - Exponent numbers (1e5, 2.5E-3) inside an incomplete container raised. - Strict mode returned "" for an unterminated string ending in an incomplete escape, losing already-streamed text (issue #8). - Strict mode returned a lone surrogate when cut inside an escaped surrogate pair. - JSON5 mode raised on partial literals and exponents, and mis-handled a comment that had only streamed its first '/'. - JSON5 string decoding depended on whether the optional json5 package was installed. - bytes input crashed the fallback parser; a leading BOM raised. - JSONParser.strict / on_extra_token / last_parse_reminding are readable and assignable again; 0.x method names are callable again. Changed - Index-based scanner: parse time is linear in input size (900 KB partial document: ~1 s -> ~60 ms). - Literals that are not a prefix of true/false/null now raise like json.loads instead of being accepted. - _JSON5Parser subclasses _JSONParser instead of duplicating it. - pyproject.toml, requires-python >= 3.8, classifiers, py.typed, full type hints; CI on 3.8-3.14 with and without json5. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/tests.yml | 8 +- CHANGELOG.md | 77 +++++ MANIFEST.in | 3 + README.md | 31 +- partialjson/__init__.py | 11 +- partialjson/json5_parser.py | 537 ++++++++++++++--------------------- partialjson/json_parser.py | 460 ++++++++++++++++++++---------- partialjson/py.typed | 0 pyproject.toml | 61 ++++ setup.py | 35 --- tests/fuzz_compat_1_1_0.py | 81 ++++++ tests/legacy_1_1_0_parser.py | 236 +++++++++++++++ tests/test_compat_1_1_0.py | 127 +++++++++ tests/test_json5.py | 7 +- tests/test_parser.py | 5 +- tests/test_regressions.py | 251 ++++++++++++++++ 16 files changed, 1411 insertions(+), 519 deletions(-) create mode 100644 MANIFEST.in create mode 100644 partialjson/py.typed create mode 100644 pyproject.toml delete mode 100644 setup.py create mode 100644 tests/fuzz_compat_1_1_0.py create mode 100644 tests/legacy_1_1_0_parser.py create mode 100644 tests/test_compat_1_1_0.py create mode 100644 tests/test_regressions.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 348536e..8594bc3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -20,5 +20,9 @@ jobs: python -m pip install --upgrade pip pip install -e . pip install -r requirements-dev.txt - - name: Run tests + - name: Run tests (without optional json5 dependency) run: pytest -q + - name: Run tests (with optional json5 dependency) + run: | + pip install -e '.[json5]' + pytest -q diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a9f55d..691289a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,83 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [1.2.0] - 2026-09-12 + +No API changes. Every input that 1.1.0 parsed successfully parses to the same +value in 1.2.0, verified by `tests/test_compat_1_1_0.py`, which runs the frozen +1.1.0 parser next to the current one over every prefix of a corpus of documents. + +### Fixed + +- Numbers with an exponent (`1e5`, `2.5E-3`) inside an incomplete array or + object raised `JSONDecodeError`. They now parse; an exponent that has not + received its digits yet (`1e`, `1e-`) is dropped until it is complete. +- In strict mode an unterminated string whose tail was an incomplete escape + (`"foo\`, `"foo\u00`) returned `""`, discarding text that had already + streamed. It now returns `"foo"`; only the unfinished escape is held back + (issue #8). +- In strict mode a string cut between the two halves of a surrogate pair + (`"\ud83d`, half of an emoji) returned a lone surrogate, which raises + `UnicodeEncodeError` as soon as it is encoded. The high half is now held + back until its partner arrives. +- The JSON5 parser raised on partial literals (`{"a": tr`, `[fals`, `[Inf`) + and on exponent numbers, and treated a comment that had only streamed its + first `/` as an unknown token. It now behaves like the JSON parser. +- JSON5 string decoding no longer depends on whether the optional `json5` + package is installed; the same escapes (`\x41`, `\'`, line continuations, + surrogate pairs) decode the same way either way. +- `bytes` and `bytearray` input, which `json.loads` accepts, no longer crash + the fallback parser with `AttributeError`. A chunk that ends in the middle + of a multi-byte UTF-8 character drops the incomplete bytes. +- A leading UTF-8 byte-order mark no longer causes a `JSONDecodeError`. +- `JSONParser.strict`, `.on_extra_token` and `.last_parse_reminding` are + readable and assignable again (assigning `strict` on a 1.x parser was + silently ignored), and the 0.x method names `parse_string`, `parse_number`, + `parse_array`, `parse_object`, `parse_true`, `parse_false`, `parse_null` + and `parse_space` are callable again. + +### Changed + +- The scanner works on string indexes instead of re-slicing the input at + every token, so a parse is linear in the input size. A 900 KB partial + document went from about 1 s to about 60 ms per `parse()` call. +- A literal that is not a prefix of `true`/`false`/`null` (for example + `[trap]`, which 1.1.0 returned as `[True]`) now raises, matching + `json.loads`. Prefixes such as `[t`, `[tru` still parse. +- `_JSON5Parser` is now a subclass of the JSON parser instead of a copy of it. +- Packaging moved to `pyproject.toml` with `requires-python >= 3.8`, + classifiers and a `py.typed` marker; the package is fully type-annotated. +- CI runs on Python 3.8 through 3.14, with and without the optional `json5` + dependency. + +## [1.1.0] - 2026-02-20 + +### Added + +- JSON5 support: comments, unquoted keys, single-quoted strings, hex numbers, + `Infinity`/`NaN`, trailing commas. Available through + `create_json5_parser()` or `JSONParser(json5_enabled=True)`; install + `partialjson[json5]` for the optional `json5` fast path (issue #9). +- `create_json_parser()` factory. + +## [1.0.0] - 2026-02 + +### Added + +- `CITATION.cff`, `CODE_OF_CONDUCT.md`, `CONTRIBUTING.md`, JOSS paper draft, + GitHub Actions test workflow. + +### Changed + +- `JSONParser` became a thin facade over an internal implementation class. + +## [0.1.0] - 2025-01-28 + +### Fixed + +- Incomplete escape sequences (`"\`, `"\u12`) at the end of a streamed + string no longer raise (issue #8). + ## [0.0.8] - 2024-08-03 ### Added diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..f346d38 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +include LICENSE README.md CHANGELOG.md CITATION.cff +graft tests +global-exclude __pycache__ *.py[cod] diff --git a/README.md b/README.md index 92eb0dc..0a32bab 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,30 @@ print(parser.parse(incomplete_json5)) # {'name': 'Demo', 'version': 1.0, 'items': [1, 2, 3]} ``` -Install the optional `json5` dependency for full JSON5 support: `pip install partialjson[json5]` +The optional `json5` dependency speeds up parsing of complete JSON5 documents: `pip install partialjson[json5]`. Partial documents parse the same way with or without it. + +### What you get while a string is still streaming + +Text that has already arrived is returned; only what cannot be decided yet is held back. With `strict=True` (the default) escapes are decoded and an unfinished escape or half an emoji is dropped until it is complete: + +```python +parser.parse('{"msg": "caf\\u00') # {'msg': 'caf'} +parser.parse('{"msg": "caf\\u00e9"') # {'msg': 'café'} +parser.parse('{"msg": "hi \\ud83d"') # {'msg': 'hi '} +parser.parse('{"msg": "hi \\ud83d\\ude00"') # {'msg': 'hi 😀'} +``` + +With `strict=False` the raw text of an unfinished string is returned untouched, backslashes included. + +### Extra tokens + +If the input contains a complete value followed by more text, the value is returned and the callback passed as `on_extra_token` is called with the input, the value and the leftover text. The default callback prints to stdout; pass `on_extra_token=None` to silence it, or read `parser.last_parse_reminding` afterwards. + +```python +parser = JSONParser(on_extra_token=None) +parser.parse('{"a": 1} trailing') # {'a': 1} +parser.last_parse_reminding # ' trailing' +``` ### Installation @@ -65,11 +88,13 @@ Also can be found on [pypi](https://pypi.org/project/partialjson/) ## Testing ```bash -pip install -e . +pip install -e '.[json5]' pip install -r requirements-dev.txt pytest -q ``` +`tests/test_compat_1_1_0.py` runs the frozen 1.1.0 parser next to the current one over every prefix of a corpus of documents, so behaviour changes for existing users show up as test failures. + ## Citation If you use this software, please cite it using the metadata in `CITATION.cff`. @@ -88,7 +113,7 @@ Please refer to each project's style and contribution guidelines for submitting 1. **Fork** the repo on GitHub 2. **Clone** the project to your own machine -3. **Update the Version** inside **init**.py +3. **Update the Version** inside `partialjson/__init__.py` and add a `CHANGELOG.md` entry 4. **Commit** changes to your own branch 5. **Push** your work back up to your fork 6. Submit a **Pull request** so that we can review your changes diff --git a/partialjson/__init__.py b/partialjson/__init__.py index 3d3c395..19e5b38 100644 --- a/partialjson/__init__.py +++ b/partialjson/__init__.py @@ -1,13 +1,14 @@ """ Partial Json. -Parsing ChatGPT JSON stream response — Partial and incomplete JSON parser python library for OpenAI +Parse partial and incomplete JSON, such as a streaming LLM response, without +crashing: ``JSONParser().parse('{"a": [1, 2')`` returns ``{"a": [1, 2]}``. """ -from .json_parser import JSONParser, create_json_parser from .json5_parser import create_json5_parser +from .json_parser import JSONParser, create_json_parser -__version__ = "1.1.0" +__version__ = "1.2.0" __author__ = "Nima Akbarzadeh" __author_email__ = "iw4p@protonmail.com" __license__ = "MIT" @@ -16,8 +17,8 @@ PYPI_SIMPLE_ENDPOINT: str = "https://pypi.org/project/partialjson" __all__ = [ + "PYPI_SIMPLE_ENDPOINT", "JSONParser", - "create_json_parser", "create_json5_parser", - "PYPI_SIMPLE_ENDPOINT", + "create_json_parser", ] diff --git a/partialjson/json5_parser.py b/partialjson/json5_parser.py index c044fc5..ab50e77 100644 --- a/partialjson/json5_parser.py +++ b/partialjson/json5_parser.py @@ -1,346 +1,235 @@ -"""JSON5 parser - extends JSON with comments, unquoted keys, single quotes, etc.""" -import json -import re +"""JSON5 parser - extends JSON with comments, unquoted keys, single quotes, etc. +Built on top of the JSON scanner in ``json_parser``; only the JSON5-specific +pieces (whitespace and comments, identifiers, extra string escapes, hex and +signed numbers, ``Infinity``/``NaN``, case-insensitive literals) are overridden. +""" +import json +from types import ModuleType +from typing import Any, ClassVar, FrozenSet, Optional, Tuple + +from .json_parser import ( + _HEX, + _NO_KEY, + OnExtraToken, + ScanResult, + _default_on_extra_token, + _is_high_surrogate, + _is_low_surrogate, + _JSONParser, +) + +json5: Optional[ModuleType] try: import json5 -except ImportError: +except ImportError: # pragma: no cover - exercised via monkeypatching in tests json5 = None +__all__ = ["_default_on_extra_token", "create_json5_parser"] -def create_json5_parser(strict=True, on_extra_token=None): +_JSON5_WHITESPACE = "\v\f\u00A0\u2028\u2029\uFEFF" +_LINE_TERMINATORS = "\n\r\u2028\u2029" +_SIMPLE_ESCAPES = { + "b": "\b", + "f": "\f", + "n": "\n", + "r": "\r", + "t": "\t", + "v": "\v", + "0": "\0", +} + + +def create_json5_parser( + strict: bool = True, on_extra_token: Optional[OnExtraToken] = None +) -> "_JSON5Parser": """Create a JSON5 parser.""" return _JSON5Parser(strict=strict, on_extra_token=on_extra_token) -def _default_on_extra_token(text, data, reminding): - print("Parsed JSON with extra tokens:", {"text": text, "data": data, "reminding": reminding}) - - -_INCOMPLETE_ESCAPE_REGEX = re.compile(r"^\\(?:u[0-9a-fA-F]{0,3}|x[0-9a-fA-F]{0,1})?$") -_JSON5_WHITESPACE = "\v\f\u00A0\u2028\u2029\uFEFF" - - -class _JSON5Parser: +def _decode_json5_string(content: str) -> str: + """Decode the body of a JSON5 string literal (quotes already removed).""" + out = [] + i = 0 + n = len(content) + while i < n: + c = content[i] + if c != "\\": + out.append(c) + i += 1 + continue + if i + 1 >= n: + raise ValueError("incomplete escape") + esc = content[i + 1] + if esc == "u": + hex4 = content[i + 2 : i + 6] + if len(hex4) < 4 or any(h not in _HEX for h in hex4): + raise ValueError("bad \\u escape") + code = int(hex4, 16) + i += 6 + if _is_high_surrogate(hex4) and content[i : i + 2] == "\\u": + low = content[i + 2 : i + 6] + if len(low) == 4 and all(h in _HEX for h in low) and _is_low_surrogate(low): + code = 0x10000 + ((code - 0xD800) << 10) + (int(low, 16) - 0xDC00) + i += 6 + out.append(chr(code)) + elif esc == "x": + hex2 = content[i + 2 : i + 4] + if len(hex2) < 2 or any(h not in _HEX for h in hex2): + raise ValueError("bad \\x escape") + out.append(chr(int(hex2, 16))) + i += 4 + elif esc == "\r": + i += 3 if content[i + 2 : i + 3] == "\n" else 2 + elif esc in _LINE_TERMINATORS: + i += 2 # line continuation + elif esc in _SIMPLE_ESCAPES: + out.append(_SIMPLE_ESCAPES[esc]) + i += 2 + else: + out.append(esc) # \' \" \\ \/ and any other escaped character + i += 2 + return "".join(out) + + +class _JSON5Parser(_JSONParser): """JSON5 parser with comments, unquoted keys, single quotes, hex, Infinity, etc.""" - def __init__(self, strict=True, on_extra_token=None): - self.strict = strict - self.on_extra_token = on_extra_token or _default_on_extra_token - self.last_parse_reminding = None - self._parsers = self._build_parsers() - - def _build_parsers(self): - parsers = { - " ": self._parse_space, - "\r": self._parse_space, - "\n": self._parse_space, - "\t": self._parse_space, - "[": self._parse_array, - "{": self._parse_object, - '"': self._parse_string, - "'": self._parse_string, - "t": self._parse_true, - "f": self._parse_false, - "n": self._parse_null, - "/": self._parse_space, - "+": self._parse_number, - "I": self._parse_number, - "N": self._parse_n_literal, - "T": self._parse_true, - "F": self._parse_false, - } - for c in _JSON5_WHITESPACE: - parsers[c] = self._parse_space - for c in "0123456789.-": - parsers[c] = self._parse_number - return parsers + _VALUE_START: ClassVar[FrozenSet[str]] = frozenset("[{\"'tfnTFNI+0123456789.-") - def parse(self, s): - if len(s) >= 1: - if json5: - try: - return json5.loads(s) - except (json.JSONDecodeError, ValueError) as e: - data, reminding = self.parse_any(s, e) - self.last_parse_reminding = reminding - if self.on_extra_token and reminding: - self.on_extra_token(s, data, reminding) - return data - data, reminding = self.parse_any(s, json.JSONDecodeError("", "", 0)) - self.last_parse_reminding = reminding - if self.on_extra_token and reminding: - self.on_extra_token(s, data, reminding) - return data - return json.loads("{}") + # ------------------------------------------------------------ fast path - def parse_any(self, s, e): - if not s: - raise e - while s and self._is_space_or_comment_start(s): - s = self._parse_space(s, e) - if not s: - return None, "" - parser = self._parsers.get(s[0]) - if not parser: - raise e - return parser(s, e) - - def _is_space_or_comment_start(self, s): - if not s: - return False - c = s[0] - if c.isspace() or c in _JSON5_WHITESPACE: - return True - if s.startswith("//") or s.startswith("/*"): - return True - return False - - def _parse_space(self, s, e): - i = 0 - while i < len(s): - if s[i].isspace() or s[i] in _JSON5_WHITESPACE: + def _loads(self, s: str) -> Any: + try: + return json.loads(s) + except (json.JSONDecodeError, ValueError): + if json5 is None: + raise + return json5.loads(s) + + # ------------------------------------------------- whitespace & comments + + def _is_space(self, s: str, i: int) -> bool: + c = s[i] + return c.isspace() or c in _JSON5_WHITESPACE + + def _skip_space(self, s: str, i: int) -> int: + n = len(s) + while i < n: + c = s[i] + if c.isspace() or c in _JSON5_WHITESPACE: i += 1 - elif s[i : i + 2] == "//": - i += 2 - while i < len(s) and s[i] not in "\n\r\u2028\u2029": - i += 1 - elif s[i : i + 2] == "/*": - i += 2 - end = s.find("*/", i) - if end == -1: - return "" - i = end + 2 - else: - break - return s[i:] - - def _parse_array(self, s, e): - s = s[1:] - acc = [] - while True: - while s and self._is_space_or_comment_start(s): - s = self._parse_space(s, e) - if not s: - break - if s[0] == "]": - s = s[1:] - break - res, s = self.parse_any(s, e) - acc.append(res) - while s and self._is_space_or_comment_start(s): - s = self._parse_space(s, e) - if s and s.startswith(","): - s = s[1:] - return acc, s - - def _parse_object(self, s, e): - s = s[1:] - acc = {} - while True: - while s and self._is_space_or_comment_start(s): - s = self._parse_space(s, e) - if not s: - break - if s[0] == "}": - s = s[1:] - break - if s[0] not in '"\'': - key, s = self._parse_identifier(s, e) - if not key: - while s and self._is_space_or_comment_start(s): - s = self._parse_space(s, e) - if s and s[0] == "}": - s = s[1:] + elif c == "/": + if i + 1 >= n: + return n # a comment that has only streamed its first '/' + nxt = s[i + 1] + if nxt == "/": + i += 2 + while i < n and s[i] not in _LINE_TERMINATORS: + i += 1 + elif nxt == "*": + end = s.find("*/", i + 2) + if end == -1: + return n # unterminated block comment swallows the rest + i = end + 2 + else: break else: - key, s = self.parse_any(s, e) - while s and self._is_space_or_comment_start(s): - s = self._parse_space(s, e) - if not s or s[0] == "}": - if key is not None: - acc[key] = None - if s and s[0] == "}": - s = s[1:] - break - if s[0] != ":": - if key is not None: - acc[key] = None - break - s = s[1:] - while s and self._is_space_or_comment_start(s): - s = self._parse_space(s, e) - if not s or s[0] in ",}": - acc[key] = None - if s and s.startswith(","): - s = s[1:] - elif s and s.startswith("}"): - s = s[1:] - break - while s and self._is_space_or_comment_start(s): - s = self._parse_space(s, e) - if s and ( - s[0] in self._parsers - or s[0] in "/+IN" - or s[0] in _JSON5_WHITESPACE - ): - value, s = self.parse_any(s, e) - acc[key] = value - else: - if key is not None: - acc[key] = None break - while s and self._is_space_or_comment_start(s): - s = self._parse_space(s, e) - if s and s.startswith(","): - s = s[1:] - return acc, s - - def _parse_identifier(self, s, e): - i = 0 - while i < len(s) and (s[i].isalnum() or s[i] in "_$"): - i += 1 - return s[:i], s[i:] - - def _parse_string(self, s, e): - quote = s[0] - end = 1 - while end < len(s): - if s[end] == "\\": - end += 2 - continue - if s[end] == quote: - break - end += 1 - - if end >= len(s): - content = s[1:] - if not self.strict: - return content, "" - if _INCOMPLETE_ESCAPE_REGEX.match(content): - return "", "" - try: - if quote == "'": - return content, "" - return json.loads(f'"{content}"'), "" - except json.JSONDecodeError: - return "", "" - - str_val = s[: end + 1] - remainder = s[end + 1 :] - - if json5: - try: - return json5.loads(str_val), remainder - except Exception: - pass - - decoded = str_val[1:-1] - decoded = re.sub(r"\\\n", "", decoded) - decoded = re.sub(r"\\\r\n", "", decoded) - - def replace_hex(match): - return chr(int(match.group(1), 16)) - - decoded = re.sub(r"\\x([0-9a-fA-F]{2})", replace_hex, decoded) - - if quote == "'": - decoded = decoded.replace('"', '\\"').replace("\\'", "'") - try: - return json.loads(f'"{decoded}"'), remainder - except Exception: - return decoded, remainder - if "\\x" in decoded or "\\\n" in str_val or "\\\r" in str_val: - return decoded, remainder + return i + + # --------------------------------------------------------------- values + + def _scan_value(self, s: str, i: int, e: BaseException) -> ScanResult: + c = s[i] + if c == "'": + return self._scan_string(s, i, e) + if c in "tT": + return self._scan_literal(s, i, "true", True, e) + if c in "fF": + return self._scan_literal(s, i, "false", False, e) + if c == "n": + return self._scan_literal(s, i, "null", None, e) + if c == "N": + return self._scan_n_literal(s, i, e) + if c in "+I": + return self._scan_number(s, i, e) + return super()._scan_value(s, i, e) + + def _scan_key(self, s: str, i: int, e: BaseException) -> ScanResult: + if s[i] in "\"'": + return self._scan_string(s, i, e) + n = len(s) + j = i + while j < n and (s[j].isalnum() or s[j] in "_$"): + j += 1 + if j == i: + return _NO_KEY, i + return s[i:j], j + + # -------------------------------------------------------------- strings + + def _scan_extra_escape(self, s: str, j: int, n: int) -> Tuple[bool, int]: + esc = s[j + 1] + if esc == "x": + hex2 = s[j + 2 : j + 4] + if all(h in _HEX for h in hex2): + if len(hex2) < 2: + return True, 0 # \x or \xA at the end of the input + return False, 4 + return False, 2 + if esc == "\r" and s[j + 2 : j + 3] == "\n": + return False, 3 + return False, 2 + + def _decode_incomplete(self, quote: str, content: str) -> Any: try: - return json.loads(str_val), remainder - except Exception: - return decoded, remainder - - def _parse_number(self, s, e): - if s.startswith(("-0x", "-0X")): - i = 3 - while i < len(s) and s[i] in "0123456789abcdefABCDEF": - i += 1 - num_str = s[1:i] - remainder = s[i:] - if len(num_str) <= 2: - return s[:3], "" - return -int(num_str, 16), remainder - if s.startswith(("+0x", "+0X")): - i = 3 - while i < len(s) and s[i] in "0123456789abcdefABCDEF": - i += 1 - num_str = s[1:i] - remainder = s[i:] - if len(num_str) <= 2: - return s[:3], "" - return int(num_str, 16), remainder - if s.startswith(("0x", "0X")): - i = 2 - while i < len(s) and s[i] in "0123456789abcdefABCDEF": - i += 1 - num_str = s[:i] - remainder = s[i:] - if len(num_str) <= 2: - return num_str, "" - return int(num_str, 16), remainder - - for literal, val in [("Infinity", float("inf")), ("NaN", float("nan"))]: - if s.startswith(literal): - return val, s[len(literal) :] - if s.startswith("+" + literal): - return val, s[len(literal) + 1 :] - if s.startswith("-" + literal): - return -val if literal == "Infinity" else val, s[len(literal) + 1 :] - - if s.startswith(".") and len(s) > 1 and s[1].isdigit(): - i = 1 - while i < len(s) and s[i].isdigit(): - i += 1 - num_str = s[:i] - return float(num_str), s[i:] - - if s.startswith("+"): - res, remainder = self._parse_number(s[1:], e) - return res, remainder + return _decode_json5_string(content) + except ValueError: + return "" - i = 0 - while i < len(s) and s[i] in "0123456789.-": - i += 1 - num_str = s[:i] - s = s[i:] - if not num_str or num_str == "-" or num_str == ".": - return num_str, "" + def _decode_complete(self, literal: str, quote: str) -> Any: try: - if num_str.endswith("."): - num = int(num_str[:-1]) - else: - num = ( - float(num_str) - if "." in num_str or "e" in num_str or "E" in num_str - else int(num_str) - ) + return _decode_json5_string(literal[1:-1]) except ValueError: - raise e - return num, s - - def _parse_n_literal(self, s, e): - if s.lower().startswith("nan"): - return self._parse_number(s, e) - return self._parse_null(s, e) - - def _parse_true(self, s, e): - if s.lower().startswith("true"): - return True, s[4:] - raise e - - def _parse_false(self, s, e): - if s.lower().startswith("false"): - return False, s[5:] - raise e - - def _parse_null(self, s, e): - if s.lower().startswith("null"): - return None, s[4:] - raise e + return literal[1:-1] + + # -------------------------------------------------------------- numbers + + def _scan_number(self, s: str, i: int, e: BaseException) -> ScanResult: + n = len(s) + sign = 1 + j = i + if s[j] in "+-": + sign = -1 if s[j] == "-" else 1 + j += 1 + if s[j : j + 2] in ("0x", "0X"): + k = j + 2 + while k < n and s[k] in _HEX: + k += 1 + if k == j + 2: + return s[i:k], n # "0x" with no digits yet + return sign * int(s[j + 2 : k], 16), k + for word, value in (("Infinity", float("inf")), ("NaN", float("nan"))): + k = self._literal_matches(s, j, word) + if k and (k == len(word) or j + k >= n): + return sign * value, j + k + if s[i] == "+": + if j >= n: + return "+", n + return super()._scan_number(s, j, e) + return super()._scan_number(s, i, e) + + # ------------------------------------------------------------- literals + + def _literal_matches(self, s: str, i: int, word: str) -> int: + n = len(s) + k = 0 + while i + k < n and k < len(word) and s[i + k].lower() == word[k].lower(): + k += 1 + return k + + def _scan_n_literal(self, s: str, i: int, e: BaseException) -> ScanResult: + if s[i + 1 : i + 2].lower() == "a": + return self._scan_number(s, i, e) + return self._scan_literal(s, i, "null", None, e) diff --git a/partialjson/json_parser.py b/partialjson/json_parser.py index 2a015ed..e340b96 100644 --- a/partialjson/json_parser.py +++ b/partialjson/json_parser.py @@ -1,50 +1,66 @@ -"""Pure JSON parser - no JSON5 extensions.""" +"""Pure JSON parser - no JSON5 extensions. + +The scanner works on string indexes instead of slicing the input on every +token, so a parse is linear in the size of the input. Behaviour is kept +identical to the 1.1.0 release except for the fixes listed in CHANGELOG.md. +""" import json -import re +from typing import Any, Callable, ClassVar, Dict, FrozenSet, Optional, Tuple, Union + +OnExtraToken = Callable[[str, Any, str], None] +ScanResult = Tuple[Any, int] +_HEX = "0123456789abcdefABCDEF" +_NUMBER_CHARS = "0123456789.-+eE" +_NO_KEY = object() # returned by _scan_key when no key could be read -def create_json_parser(strict=True, on_extra_token=None): + +def create_json_parser( + strict: bool = True, on_extra_token: Optional[OnExtraToken] = None +) -> "_JSONParser": """Create a JSON parser (no JSON5 extensions).""" return _JSONParser(strict=strict, on_extra_token=on_extra_token) -def _default_on_extra_token(text, data, reminding): +def _default_on_extra_token(text: str, data: Any, reminding: str) -> None: print("Parsed JSON with extra tokens:", {"text": text, "data": data, "reminding": reminding}) -_INCOMPLETE_ESCAPE_REGEX = re.compile(r"^\\(?:u[0-9a-fA-F]{0,3}|x[0-9a-fA-F]{0,1})?$") +def _is_high_surrogate(hex4: str) -> bool: + code = int(hex4, 16) + return 0xD800 <= code <= 0xDBFF + + +def _is_low_surrogate(hex4: str) -> bool: + code = int(hex4, 16) + return 0xDC00 <= code <= 0xDFFF class _JSONParser: """Internal JSON-only parser implementation.""" - def __init__(self, strict=True, on_extra_token=None): + # Characters that may legally start a value in this dialect. Used by the + # object parser to decide whether an unexpected character ends the parse. + _VALUE_START: ClassVar[FrozenSet[str]] = frozenset('[{"tfn0123456789.-') + + def __init__( + self, strict: bool = True, on_extra_token: Optional[OnExtraToken] = None + ) -> None: self.strict = strict - self.on_extra_token = on_extra_token or _default_on_extra_token - self.last_parse_reminding = None - self._parsers = self._build_parsers() - - def _build_parsers(self): - parsers = { - " ": self._parse_space, - "\r": self._parse_space, - "\n": self._parse_space, - "\t": self._parse_space, - "[": self._parse_array, - "{": self._parse_object, - '"': self._parse_string, - "t": self._parse_true, - "f": self._parse_false, - "n": self._parse_null, - } - for c in "0123456789.-": - parsers[c] = self._parse_number - return parsers - - def parse(self, s): + self.on_extra_token: Optional[OnExtraToken] = on_extra_token or _default_on_extra_token + self.last_parse_reminding: Optional[str] = None + + # ------------------------------------------------------------------ public + + def parse(self, s: Union[str, bytes, bytearray]) -> Any: + """Parse ``s``, returning as much of the document as is available.""" + if isinstance(s, (bytes, bytearray)): + s = self._decode_bytes(s) + if s[:1] == "\ufeff": + s = s[1:] if len(s) >= 1: try: - return json.loads(s) + return self._loads(s) except (json.JSONDecodeError, ValueError) as e: data, reminding = self.parse_any(s, e) self.last_parse_reminding = reminding @@ -53,152 +69,280 @@ def parse(self, s): return data return json.loads("{}") - def parse_any(self, s, e): + def parse_any(self, s: str, e: BaseException) -> Tuple[Any, str]: + """Parse one value from the start of ``s``. + + Returns ``(value, remaining_text)``. Raises ``e`` when ``s`` is empty + or does not start with anything that looks like JSON. + """ if not s: raise e - while s and s[0].isspace(): - s = self._parse_space(s, e) - if not s: + i = self._skip_space(s, 0) + if i >= len(s): return None, "" - parser = self._parsers.get(s[0]) - if not parser: - raise e - return parser(s, e) + value, i = self._scan_value(s, i, e) + return value, s[i:] + + # 0.x compatible entry points. Each takes the text and the exception to + # raise on invalid input, and returns ``(value, remaining_text)``. + + def parse_space(self, s: str, e: BaseException) -> str: + return s[self._skip_space(s, 0) :] + + def parse_array(self, s: str, e: BaseException) -> Tuple[Any, str]: + value, i = self._scan_array(s, 0, e) + return value, s[i:] + + def parse_object(self, s: str, e: BaseException) -> Tuple[Any, str]: + value, i = self._scan_object(s, 0, e) + return value, s[i:] + + def parse_string(self, s: str, e: BaseException) -> Tuple[Any, str]: + value, i = self._scan_string(s, 0, e) + return value, s[i:] + + def parse_number(self, s: str, e: BaseException) -> Tuple[Any, str]: + value, i = self._scan_number(s, 0, e) + return value, s[i:] + + def parse_true(self, s: str, e: BaseException) -> Tuple[Any, str]: + value, i = self._scan_literal(s, 0, "true", True, e) + return value, s[i:] + + def parse_false(self, s: str, e: BaseException) -> Tuple[Any, str]: + value, i = self._scan_literal(s, 0, "false", False, e) + return value, s[i:] + + def parse_null(self, s: str, e: BaseException) -> Tuple[Any, str]: + value, i = self._scan_literal(s, 0, "null", None, e) + return value, s[i:] + + # --------------------------------------------------------------- internals + + @staticmethod + def _decode_bytes(s: Union[bytes, bytearray]) -> str: + # A streamed chunk may end in the middle of a multi-byte character; + # drop the incomplete tail, it will be complete in the next chunk. + return bytes(s).decode("utf-8", errors="ignore") + + def _loads(self, s: str) -> Any: + """Fast path for complete documents.""" + return json.loads(s) + + def _is_space(self, s: str, i: int) -> bool: + return s[i].isspace() - def _parse_space(self, s, e): - i = 0 - while i < len(s) and s[i].isspace(): + def _skip_space(self, s: str, i: int) -> int: + n = len(s) + while i < n and self._is_space(s, i): i += 1 - return s[i:] + return i - def _parse_array(self, s, e): - s = s[1:] + def _can_start_value(self, s: str, i: int) -> bool: + return s[i] in self._VALUE_START + + def _scan_value(self, s: str, i: int, e: BaseException) -> ScanResult: + """Dispatch on the character at ``s[i]``; whitespace must be skipped.""" + c = s[i] + if c == "[": + return self._scan_array(s, i, e) + if c == "{": + return self._scan_object(s, i, e) + if c == '"': + return self._scan_string(s, i, e) + if c == "t": + return self._scan_literal(s, i, "true", True, e) + if c == "f": + return self._scan_literal(s, i, "false", False, e) + if c == "n": + return self._scan_literal(s, i, "null", None, e) + if c in "0123456789.-": + return self._scan_number(s, i, e) + raise e + + def _scan_array(self, s: str, i: int, e: BaseException) -> ScanResult: + n = len(s) + i += 1 # '[' acc = [] while True: - while s and s[0].isspace(): - s = self._parse_space(s, e) - if not s: + i = self._skip_space(s, i) + if i >= n: break - if s[0] == "]": - s = s[1:] + if s[i] == "]": + i += 1 break - res, s = self.parse_any(s, e) - acc.append(res) - while s and s[0].isspace(): - s = self._parse_space(s, e) - if s and s.startswith(","): - s = s[1:] - return acc, s - - def _parse_object(self, s, e): - s = s[1:] - acc = {} + value, i = self._scan_value(s, i, e) + acc.append(value) + i = self._skip_space(s, i) + if i < n and s[i] == ",": + i += 1 + return acc, i + + def _scan_key(self, s: str, i: int, e: BaseException) -> ScanResult: + return self._scan_value(s, i, e) + + def _scan_object(self, s: str, i: int, e: BaseException) -> ScanResult: + n = len(s) + i += 1 # '{' + acc: Dict[Any, Any] = {} while True: - while s and s[0].isspace(): - s = self._parse_space(s, e) - if not s: + i = self._skip_space(s, i) + if i >= n: + break + if s[i] == "}": + i += 1 break - if s[0] == "}": - s = s[1:] + key, i = self._scan_key(s, i, e) + if key is _NO_KEY: + i = self._skip_space(s, i) + if i < n and s[i] == "}": + i += 1 break - key, s = self.parse_any(s, e) - while s and s[0].isspace(): - s = self._parse_space(s, e) - if not s or s[0] == "}": + i = self._skip_space(s, i) + if i >= n or s[i] == "}": if key is not None: acc[key] = None - if s and s[0] == "}": - s = s[1:] + if i < n: + i += 1 break - if s[0] != ":": + if s[i] != ":": if key is not None: acc[key] = None break - s = s[1:] - while s and s[0].isspace(): - s = self._parse_space(s, e) - if not s or s[0] in ",}": + i += 1 # ':' + i = self._skip_space(s, i) + if i >= n or s[i] in ",}": acc[key] = None - if s and s.startswith(","): - s = s[1:] - elif s and s.startswith("}"): - s = s[1:] + if i < n: + i += 1 break - if s and s[0] in self._parsers: - value, s = self.parse_any(s, e) + if self._can_start_value(s, i): + value, i = self._scan_value(s, i, e) acc[key] = value else: if key is not None: acc[key] = None break - while s and s[0].isspace(): - s = self._parse_space(s, e) - if s and s.startswith(","): - s = s[1:] - return acc, s - - def _parse_string(self, s, e): - quote = s[0] - end = 1 - while end < len(s): - if s[end] == "\\": - end += 2 + i = self._skip_space(s, i) + if i < n and s[i] == ",": + i += 1 + return acc, i + + def _scan_string(self, s: str, i: int, e: BaseException) -> ScanResult: + """Scan a string literal starting at the quote ``s[i]``. + + For an unterminated string the trailing incomplete escape sequence + (``\\``, ``\\u``, ``\\u1``, ...) and a dangling high surrogate escape + are dropped, since their final value is not known yet; everything + before them is returned. + """ + n = len(s) + quote = s[i] + start = i + 1 + j = start + cut = -1 # where an incomplete escape begins, if the string is cut there + pending_high = -1 # start index of a high-surrogate escape awaiting its pair + while j < n: + c = s[j] + if c == "\\": + if j + 1 >= n: + cut = j + break + esc = s[j + 1] + if esc == "u": + hex4 = s[j + 2 : j + 6] + if len(hex4) < 4 or any(h not in _HEX for h in hex4): + if all(h in _HEX for h in hex4) and j + 6 > n: + cut = j # incomplete \uXXXX at end of input + break + # Malformed escape: let the decoder report it. + pending_high = -1 + j += 2 + continue + if pending_high != -1 and _is_low_surrogate(hex4): + pending_high = -1 + elif _is_high_surrogate(hex4): + pending_high = j + else: + pending_high = -1 + j += 6 + continue + incomplete, length = self._scan_extra_escape(s, j, n) + if incomplete: + cut = j + break + pending_high = -1 + j += length continue - if s[end] == quote: + if c == quote: break - end += 1 + pending_high = -1 + j += 1 - if end >= len(s): - content = s[1:] + if j >= n or cut != -1: + # Unterminated string. if not self.strict: - return content, "" - if _INCOMPLETE_ESCAPE_REGEX.match(content): - return "", "" - try: - return json.loads(f'"{content}"'), "" - except json.JSONDecodeError: - return "", "" + return s[start:], n + end = cut if cut != -1 else n + if pending_high != -1 and pending_high + 6 == end: + end = pending_high + return self._decode_incomplete(quote, s[start:end]), n - str_val = s[: end + 1] - remainder = s[end + 1 :] + return self._decode_complete(s[i : j + 1], quote), j + 1 + + def _scan_extra_escape(self, s: str, j: int, n: int) -> Tuple[bool, int]: + """Hook for dialects with extra escapes. Returns (incomplete, length).""" + return False, 2 + + def _decode_incomplete(self, quote: str, content: str) -> Any: + try: + return json.loads('"' + content + '"') + except json.JSONDecodeError: + return "" + + def _decode_complete(self, literal: str, quote: str) -> Any: if not self.strict: - return str_val[1:-1], remainder - return json.loads(str_val), remainder + return literal[1:-1] + return json.loads(literal) - def _parse_number(self, s, e): - i = 0 - while i < len(s) and s[i] in "0123456789.-": - i += 1 - num_str = s[:i] - s = s[i:] + def _scan_number(self, s: str, i: int, e: BaseException) -> ScanResult: + n = len(s) + j = i + while j < n and s[j] in _NUMBER_CHARS: + j += 1 + num_str = s[i:j] + if j >= n: + # An exponent that has not received its digits yet is dropped. + stripped = num_str.rstrip("+-") + if stripped.endswith(("e", "E")): + num_str = stripped[:-1] if not num_str or num_str == "-" or num_str == ".": - return num_str, "" + return num_str, n try: if num_str.endswith("."): - num = int(num_str[:-1]) + num: Any = int(num_str[:-1]) + elif "." in num_str or "e" in num_str or "E" in num_str: + num = float(num_str) else: - num = ( - float(num_str) - if "." in num_str or "e" in num_str or "E" in num_str - else int(num_str) - ) + num = int(num_str) except ValueError: - raise e - return num, s + raise e from None + return num, j - def _parse_true(self, s, e): - if s.startswith("t") or s.startswith("T"): - return True, s[4:] - raise e - - def _parse_false(self, s, e): - if s.startswith("f") or s.startswith("F"): - return False, s[5:] - raise e + def _literal_matches(self, s: str, i: int, word: str) -> int: + """Length of the prefix of ``word`` present at ``s[i:]``.""" + n = len(s) + k = 0 + while i + k < n and k < len(word) and s[i + k] == word[k]: + k += 1 + return k - def _parse_null(self, s, e): - if s.startswith("n"): - return None, s[4:] + def _scan_literal( + self, s: str, i: int, word: str, value: Any, e: BaseException + ) -> ScanResult: + k = self._literal_matches(s, i, word) + if k == len(word) or i + k >= len(s): + return value, i + k raise e @@ -206,28 +350,54 @@ def _parse_null(self, s, e): class JSONParser: """JSON parser. Use create_json_parser() or create_json5_parser() for new code.""" - def __init__(self, strict=True, json5_enabled=False, on_extra_token=None): + _impl: _JSONParser + json5_enabled: bool + + def __init__( + self, + strict: bool = True, + json5_enabled: bool = False, + on_extra_token: Optional[OnExtraToken] = None, + ) -> None: if json5_enabled: from .json5_parser import create_json5_parser - self._impl = create_json5_parser(strict=strict, on_extra_token=on_extra_token) + impl: _JSONParser = create_json5_parser(strict=strict, on_extra_token=on_extra_token) else: - self._impl = create_json_parser(strict=strict, on_extra_token=on_extra_token) + impl = create_json_parser(strict=strict, on_extra_token=on_extra_token) + object.__setattr__(self, "_impl", impl) + object.__setattr__(self, "json5_enabled", json5_enabled) - def parse(self, s): + def parse(self, s: Union[str, bytes, bytearray]) -> Any: return self._impl.parse(s) - def parse_any(self, s, e): + def parse_any(self, s: str, e: BaseException) -> Tuple[Any, str]: return self._impl.parse_any(s, e) @property - def last_parse_reminding(self): - return getattr(self._impl, "last_parse_reminding", None) + def strict(self) -> bool: + return self._impl.strict + + @strict.setter + def strict(self, value: bool) -> None: + self._impl.strict = value + + @property + def last_parse_reminding(self) -> Optional[str]: + return self._impl.last_parse_reminding + + @last_parse_reminding.setter + def last_parse_reminding(self, value: Optional[str]) -> None: + self._impl.last_parse_reminding = value @property - def on_extra_token(self): - return getattr(self._impl, "on_extra_token", None) + def on_extra_token(self) -> Optional[OnExtraToken]: + return self._impl.on_extra_token @on_extra_token.setter - def on_extra_token(self, value): + def on_extra_token(self, value: Optional[OnExtraToken]) -> None: self._impl.on_extra_token = value + + def __getattr__(self, name: str) -> Any: + # Everything else (parse_string, parse_number, ...) lives on the impl. + return getattr(self._impl, name) diff --git a/partialjson/py.typed b/partialjson/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..1ef7f0d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,61 @@ +[build-system] +requires = ["setuptools>=61"] +build-backend = "setuptools.build_meta" + +[project] +name = "partialjson" +dynamic = ["version"] +description = "Parse incomplete or partial JSON, e.g. from a streaming LLM response" +readme = "README.md" +license = { text = "MIT" } +authors = [{ name = "Nima Akbarzadeh", email = "iw4p@protonmail.com" }] +requires-python = ">=3.8" +keywords = ["json", "partial", "incomplete", "streaming", "llm", "openai", "json5"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: Text Processing", + "Typing :: Typed", +] + +[project.optional-dependencies] +json5 = ["json5"] + +[project.urls] +Homepage = "https://github.com/iw4p/partialjson" +Repository = "https://github.com/iw4p/partialjson" +Changelog = "https://github.com/iw4p/partialjson/blob/main/CHANGELOG.md" +Issues = "https://github.com/iw4p/partialjson/issues" + +[tool.setuptools] +packages = ["partialjson"] + +[tool.setuptools.package-data] +partialjson = ["py.typed"] + +[tool.setuptools.dynamic] +version = { attr = "partialjson.__version__" } + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.ruff] +target-version = "py38" +line-length = 100 +extend-exclude = ["tests/legacy_1_1_0_parser.py"] + +[tool.ruff.lint] +select = ["E", "F", "I", "B", "UP", "RUF"] +ignore = ["UP006", "UP007", "UP035", "UP045"] # keep typing.* spellings for 3.8 runtime evaluation diff --git a/setup.py b/setup.py deleted file mode 100644 index 8007c3b..0000000 --- a/setup.py +++ /dev/null @@ -1,35 +0,0 @@ -import os.path -import pathlib -import re - -from setuptools import setup - -PROJECT_NAME = "partialjson" -# The directory containing this file -HERE = pathlib.Path(__file__).parent - -# The text of the README file -README = (HERE / "README.md").read_text() - - -def get_property(prop): - result = re.search( - r'{}\s*=\s*[\'"]([^\'"]*)[\'"]'.format(prop), - open(os.path.join(PROJECT_NAME, "__init__.py")).read(), - ) - return result.group(1) - - -setup( - name="partialjson", - version=get_property("__version__"), - description="Parse incomplete or partial json", - long_description=README, - long_description_content_type="text/markdown", - url=get_property("__url__"), - author=get_property("__author__"), - author_email=get_property("__author_email__"), - license=get_property("__license__"), - packages=["partialjson"], - extras_require={"json5": ["json5"]}, -) diff --git a/tests/fuzz_compat_1_1_0.py b/tests/fuzz_compat_1_1_0.py new file mode 100644 index 0000000..8a0188a --- /dev/null +++ b/tests/fuzz_compat_1_1_0.py @@ -0,0 +1,81 @@ +"""Randomised version of test_compat_1_1_0: generate documents, parse every +prefix with the frozen 1.1.0 parser and the current one, report differences +that are not one of the documented fixes. + +Not collected by pytest. Run it by hand: + + python tests/fuzz_compat_1_1_0.py [seed] [documents] +""" +import json +import os +import random +import sys + +sys.path.insert(0, os.path.dirname(__file__)) +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from legacy_1_1_0_parser import LegacyJSONParser +from test_compat_1_1_0 import _ends_in_incomplete_escape, _run + +from partialjson import JSONParser + +WORDS = [ + "a", "name", "x y", "", "tab\there", 'quote"q', "back\\slash", "nl\nline", + "é", "😀", " ", "true", "null", "1e5", "/", "\x01", "ሴ", "🎉x", +] +NUMBERS = [0, 1, -1, 42, 3.14, -2.5, 1e5, 2.5e-3, -1.5e10, 1e300, 0.001, 12345678901234567890, 7.0] + + +def random_string(): + return "".join(random.choice(WORDS) for _ in range(random.randint(0, 3))) + + +def random_value(depth=0): + r = random.random() + if depth > 3 or r < 0.35: + leaves = [ + random_string, + lambda: random.choice(NUMBERS), + lambda: True, + lambda: False, + lambda: None, + ] + return random.choice(leaves)() + if r < 0.7: + return [random_value(depth + 1) for _ in range(random.randint(0, 4))] + return {random_string(): random_value(depth + 1) for _ in range(random.randint(0, 4))} + + +def main(seed, count): + random.seed(seed) + unexplained = [] + prefixes = 0 + for _ in range(count): + doc = json.dumps( + random_value(), + ensure_ascii=random.random() < 0.5, + indent=random.choice([None, None, 1]), + ) + for strict in (True, False): + legacy, current = LegacyJSONParser(strict=strict), JSONParser(strict=strict) + for cut in range(1, len(doc) + 1): + prefix = doc[:cut] + prefixes += 1 + old, new = _run(legacy, prefix), _run(current, prefix) + if old == new: + continue + if old[0] == "raised" and new[0] == "ok": + continue + if strict and _ends_in_incomplete_escape(prefix): + continue + unexplained.append((strict, prefix, old, new)) + print(f"documents={count} prefixes={prefixes} unexplained={len(unexplained)}") + for item in unexplained[:10]: + print(repr(item)) + return 1 if unexplained else 0 + + +if __name__ == "__main__": + seed = int(sys.argv[1]) if len(sys.argv) > 1 else 0 + count = int(sys.argv[2]) if len(sys.argv) > 2 else 500 + sys.exit(main(seed, count)) diff --git a/tests/legacy_1_1_0_parser.py b/tests/legacy_1_1_0_parser.py new file mode 100644 index 0000000..9c44262 --- /dev/null +++ b/tests/legacy_1_1_0_parser.py @@ -0,0 +1,236 @@ +"""Verbatim copy of partialjson/json_parser.py as released in 1.1.0. + +Used only as a behavioural oracle by tests/test_compat_1_1_0.py. Do not edit. +""" + +import json +import re + + +def create_json_parser(strict=True, on_extra_token=None): + """Create a JSON parser (no JSON5 extensions).""" + return _JSONParser(strict=strict, on_extra_token=on_extra_token) + + +def _default_on_extra_token(text, data, reminding): + print("Parsed JSON with extra tokens:", {"text": text, "data": data, "reminding": reminding}) + + +_INCOMPLETE_ESCAPE_REGEX = re.compile(r"^\\(?:u[0-9a-fA-F]{0,3}|x[0-9a-fA-F]{0,1})?$") + + +class _JSONParser: + """Internal JSON-only parser implementation.""" + + def __init__(self, strict=True, on_extra_token=None): + self.strict = strict + self.on_extra_token = on_extra_token or _default_on_extra_token + self.last_parse_reminding = None + self._parsers = self._build_parsers() + + def _build_parsers(self): + parsers = { + " ": self._parse_space, + "\r": self._parse_space, + "\n": self._parse_space, + "\t": self._parse_space, + "[": self._parse_array, + "{": self._parse_object, + '"': self._parse_string, + "t": self._parse_true, + "f": self._parse_false, + "n": self._parse_null, + } + for c in "0123456789.-": + parsers[c] = self._parse_number + return parsers + + def parse(self, s): + if len(s) >= 1: + try: + return json.loads(s) + except (json.JSONDecodeError, ValueError) as e: + data, reminding = self.parse_any(s, e) + self.last_parse_reminding = reminding + if self.on_extra_token and reminding: + self.on_extra_token(s, data, reminding) + return data + return json.loads("{}") + + def parse_any(self, s, e): + if not s: + raise e + while s and s[0].isspace(): + s = self._parse_space(s, e) + if not s: + return None, "" + parser = self._parsers.get(s[0]) + if not parser: + raise e + return parser(s, e) + + def _parse_space(self, s, e): + i = 0 + while i < len(s) and s[i].isspace(): + i += 1 + return s[i:] + + def _parse_array(self, s, e): + s = s[1:] + acc = [] + while True: + while s and s[0].isspace(): + s = self._parse_space(s, e) + if not s: + break + if s[0] == "]": + s = s[1:] + break + res, s = self.parse_any(s, e) + acc.append(res) + while s and s[0].isspace(): + s = self._parse_space(s, e) + if s and s.startswith(","): + s = s[1:] + return acc, s + + def _parse_object(self, s, e): + s = s[1:] + acc = {} + while True: + while s and s[0].isspace(): + s = self._parse_space(s, e) + if not s: + break + if s[0] == "}": + s = s[1:] + break + key, s = self.parse_any(s, e) + while s and s[0].isspace(): + s = self._parse_space(s, e) + if not s or s[0] == "}": + if key is not None: + acc[key] = None + if s and s[0] == "}": + s = s[1:] + break + if s[0] != ":": + if key is not None: + acc[key] = None + break + s = s[1:] + while s and s[0].isspace(): + s = self._parse_space(s, e) + if not s or s[0] in ",}": + acc[key] = None + if s and s.startswith(","): + s = s[1:] + elif s and s.startswith("}"): + s = s[1:] + break + if s and s[0] in self._parsers: + value, s = self.parse_any(s, e) + acc[key] = value + else: + if key is not None: + acc[key] = None + break + while s and s[0].isspace(): + s = self._parse_space(s, e) + if s and s.startswith(","): + s = s[1:] + return acc, s + + def _parse_string(self, s, e): + quote = s[0] + end = 1 + while end < len(s): + if s[end] == "\\": + end += 2 + continue + if s[end] == quote: + break + end += 1 + + if end >= len(s): + content = s[1:] + if not self.strict: + return content, "" + if _INCOMPLETE_ESCAPE_REGEX.match(content): + return "", "" + try: + return json.loads(f'"{content}"'), "" + except json.JSONDecodeError: + return "", "" + + str_val = s[: end + 1] + remainder = s[end + 1 :] + if not self.strict: + return str_val[1:-1], remainder + return json.loads(str_val), remainder + + def _parse_number(self, s, e): + i = 0 + while i < len(s) and s[i] in "0123456789.-": + i += 1 + num_str = s[:i] + s = s[i:] + if not num_str or num_str == "-" or num_str == ".": + return num_str, "" + try: + if num_str.endswith("."): + num = int(num_str[:-1]) + else: + num = ( + float(num_str) + if "." in num_str or "e" in num_str or "E" in num_str + else int(num_str) + ) + except ValueError: + raise e + return num, s + + def _parse_true(self, s, e): + if s.startswith("t") or s.startswith("T"): + return True, s[4:] + raise e + + def _parse_false(self, s, e): + if s.startswith("f") or s.startswith("F"): + return False, s[5:] + raise e + + def _parse_null(self, s, e): + if s.startswith("n"): + return None, s[4:] + raise e + + +# Backward compatibility +class LegacyJSONParser: + """JSON parser. Use create_json_parser() or create_json5_parser() for new code.""" + + def __init__(self, strict=True, json5_enabled=False, on_extra_token=None): + if json5_enabled: + + self._impl = create_json5_parser(strict=strict, on_extra_token=on_extra_token) + else: + self._impl = create_json_parser(strict=strict, on_extra_token=on_extra_token) + + def parse(self, s): + return self._impl.parse(s) + + def parse_any(self, s, e): + return self._impl.parse_any(s, e) + + @property + def last_parse_reminding(self): + return getattr(self._impl, "last_parse_reminding", None) + + @property + def on_extra_token(self): + return getattr(self._impl, "on_extra_token", None) + + @on_extra_token.setter + def on_extra_token(self, value): + self._impl.on_extra_token = value diff --git a/tests/test_compat_1_1_0.py b/tests/test_compat_1_1_0.py new file mode 100644 index 0000000..0481688 --- /dev/null +++ b/tests/test_compat_1_1_0.py @@ -0,0 +1,127 @@ +"""Behavioural compatibility with the 1.1.0 release. + +Every prefix of every document in the corpus is parsed with the frozen 1.1.0 +parser (tests/legacy_1_1_0_parser.py) and with the current one. The results +must be identical, except where 1.1.0 was demonstrably wrong: + +* 1.1.0 raised on a prefix of a valid document (numbers with exponents). +* strict mode: the prefix ends inside an incomplete escape sequence or on a + lone high surrogate, where 1.1.0 discarded the whole string. +""" +import contextlib +import io +import json +import os +import sys + +import pytest + +sys.path.insert(0, os.path.dirname(__file__)) +from legacy_1_1_0_parser import LegacyJSONParser + +from partialjson import JSONParser + +HERE = os.path.dirname(__file__) +_HEX = set("0123456789abcdefABCDEF") + +CORPUS = [ + '{"name": "John Doe", "age": 30, "is_student": false, "courses": ["Math", "Science"]}', + '{"a": {"b": [1, {"c": "d", "e": [true, false, null]}], "f": -12.5, "g": 0}}', + '{"text": "line1\\nline2\\ttab \\"quoted\\" back\\\\slash \\/ slash", "n": 1}', + '{"emoji": "\\ud83d\\ude00 smile \\u00e9\\u20ac", "raw": "😀 é €", "k": "\\u0041"}', + '{"nums": [1e5, -2.5E-3, 0.5, 1.0e+2, 10, -0, 3.14159], "big": 12345678901234567890}', + '[[], {}, [[]], [{}], "", " ", "\\\\", "\\"", 0, -1, 1.5, true, false, null]', + '{"": "", "a": "", "b": " ", "c": "\\u0000", "d": "\\b\\f\\r"}', + ' {\n "spaced" : [ 1 , 2 , 3 ] ,\n "obj" : { "x" : "y" }\n} \n', + '"a top level string with \\u00fcml\\u00e4ut and \\ud83c\\udf89"', + '[1, [2, [3, [4, [5, [6, [7, [8, [9, [10]]]]]]]]]]', + '{"a": 1, "a": 2, "b": {"a": 3}}', + "12345", + "-0.001e-10", + "true", + "null", + '{"servlet": [{"servlet-name": "cofaxCDS", "init-param": {"configGlossary:installationAt": ' + '"Philadelphia, PA", "useJSP": false, "cachePackageTagsTrack": 200}}], ' + '"taglib": {"taglib-uri": "cofax.tld"}}', +] + + +def _run(parser, text): + with contextlib.redirect_stdout(io.StringIO()): + try: + return ("ok", parser.parse(text)) + except Exception as ex: + return ("raised", type(ex).__name__) + + +def _ends_in_incomplete_escape(prefix): + """True if ``prefix`` ends inside an unterminated string whose tail is an + incomplete escape (``\\``, ``\\u``, ``\\uAB``...) or a lone high surrogate.""" + i = 0 + n = len(prefix) + in_string = False + last_escape = None # (start, kind) + while i < n: + c = prefix[i] + if not in_string: + if c == '"': + in_string = True + last_escape = None + i += 1 + continue + if c == "\\": + if i + 1 >= n: + return True + if prefix[i + 1] == "u": + hex4 = prefix[i + 2 : i + 6] + if len(hex4) < 4 and all(h in _HEX for h in hex4): + return True + code = int(hex4, 16) + if 0xD800 <= code <= 0xDBFF: + last_escape = (i, "high") + elif last_escape and last_escape[1] == "high" and 0xDC00 <= code <= 0xDFFF: + last_escape = None + else: + last_escape = None + i += 6 + continue + last_escape = None + i += 2 + continue + if c == '"': + in_string = False + last_escape = None + else: + last_escape = None + i += 1 + return in_string and last_escape is not None and last_escape[0] + 6 == n + + +def _sameness(a, b): + # NaN never compares equal to itself; the corpus has none, so plain == works. + return a == b + + +@pytest.mark.parametrize("strict", [True, False], ids=["strict", "non_strict"]) +@pytest.mark.parametrize("doc", CORPUS, ids=range(len(CORPUS))) +def test_every_prefix_matches_1_1_0(doc, strict): + legacy = LegacyJSONParser(strict=strict) + current = JSONParser(strict=strict) + unexplained = [] + for cut in range(1, len(doc) + 1): + prefix = doc[:cut] + old = _run(legacy, prefix) + new = _run(current, prefix) + if _sameness(old, new): + continue + if old[0] == "raised" and new[0] == "ok": + continue # 1.1.0 crashed on a prefix of valid JSON; fixed + if strict and _ends_in_incomplete_escape(prefix): + continue # 1.1.0 threw the whole string away; fixed + unexplained.append((prefix, old, new)) + assert not unexplained, json.dumps(unexplained[:5], indent=2, ensure_ascii=False) + + +def test_corpus_is_valid_json(): + for doc in CORPUS: + json.loads(doc) diff --git a/tests/test_json5.py b/tests/test_json5.py index f8478e2..6592aa7 100644 --- a/tests/test_json5.py +++ b/tests/test_json5.py @@ -1,7 +1,8 @@ -import pytest import math + from partialjson.json_parser import JSONParser + def test_json5_comments(): parser = JSONParser(json5_enabled=True) assert parser.parse("{// comment\n\"a\": 1}") == {"a": 1} @@ -28,8 +29,8 @@ def test_json5_multi_line_strings(): def test_json5_hex_numbers(): parser = JSONParser(json5_enabled=True) assert parser.parse("0x1f") == 31 - assert parser.parse("-0x10") == -16 # Note: JSON5 spec says hex can have optional sign - # Actually checking spec: "Hexadecimal numbers ... may be prefixed with an optional plus or minus sign" + # JSON5: hexadecimal numbers may be prefixed with an optional plus or minus sign. + assert parser.parse("-0x10") == -16 assert parser.parse("0XFF") == 255 def test_json5_special_numbers(): diff --git a/tests/test_parser.py b/tests/test_parser.py index 54fd7cb..17cc02e 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -1,4 +1,5 @@ import pytest + from partialjson.json_parser import JSONParser @@ -14,7 +15,7 @@ def test_numbers_and_floats(): def test_invalid_number_raises_error(): parser = JSONParser(strict=True) - with pytest.raises(Exception): + with pytest.raises(ValueError): parser.parse("1.2.3.4") @@ -73,7 +74,7 @@ def test_spaces_and_extra_tokens(): def test_invalid_input_raises_error(): parser = JSONParser(strict=True) - with pytest.raises(Exception): + with pytest.raises(ValueError): parser.parse(":atom") diff --git a/tests/test_regressions.py b/tests/test_regressions.py new file mode 100644 index 0000000..4ffbdbf --- /dev/null +++ b/tests/test_regressions.py @@ -0,0 +1,251 @@ +"""Regression tests for the fixes shipped in 1.2.0.""" +import contextlib +import io +import math + +import pytest + +import partialjson +from partialjson import JSONParser, create_json5_parser, create_json_parser, json5_parser + + +@pytest.fixture(params=["json", "json5", "json5-no-lib"]) +def parser(request, monkeypatch): + """The JSON parser, and the JSON5 parser both with and without the + optional ``json5`` package, all of which must accept plain JSON.""" + if request.param == "json": + return JSONParser() + if request.param == "json5-no-lib": + monkeypatch.setattr(json5_parser, "json5", None) + return JSONParser(json5_enabled=True) + + +# --- numbers with exponents used to raise inside a container ----------------- + +@pytest.mark.parametrize( + "text, expected", + [ + ("[1e5", [100000.0]), + ('{"a": 1e-7', {"a": 1e-7}), + ("[1.5e10, 2", [1.5e10, 2]), + ("[-1.5E+3]", [-1500.0]), + ("[1e", [1]), + ("[1e-", [1]), + ("[1.5E+", [1.5]), + ("[2.5e3, 1e", [2500.0, 1]), + ("1e", 1), + ], +) +def test_exponent_numbers(parser, text, expected): + assert parser.parse(text) == expected + + +def test_incomplete_number_keeps_legacy_shape(): + parser = JSONParser() + assert parser.parse("-") == "-" + assert parser.parse("[1, -") == [1, "-"] + assert parser.parse("[1.") == [1] + with pytest.raises(ValueError): + parser.parse("[1.5.") # invalid, and 1.1.0 raised here as well + + +# --- incomplete escapes no longer throw away the rest of the string ---------- + +@pytest.mark.parametrize( + "text, expected", + [ + ('{"a":"foo\\', {"a": "foo"}), + ('{"a":"foo\\u', {"a": "foo"}), + ('{"a":"foo\\u00', {"a": "foo"}), + ('{"a":"foo\\u00e', {"a": "foo"}), + ('{"a":"foo\\u00e9', {"a": "fooé"}), + ('{"a":"foo\\u00e9\\', {"a": "fooé"}), + ('"foo\\', "foo"), + ('{"a":"\\', {"a": ""}), + ('{"a":"\\u', {"a": ""}), + ('{"a":"\\u123', {"a": ""}), + ('{"a":"\\u1234', {"a": "ሴ"}), + ('{"a":"foo\\"', {"a": 'foo"'}), + ('{"a":"esc\\\\', {"a": "esc\\"}), # complete escaped backslash is kept + ('{"a":"esc\\\\\\', {"a": "esc\\"}), # ...but a trailing lone one is not + ], +) +def test_incomplete_escape_keeps_prefix(parser, text, expected): + assert parser.parse(text) == expected + + +@pytest.mark.parametrize( + "text, expected", + [ + ('{"a":"hi \\ud83d', {"a": "hi "}), + ('{"a":"hi \\ud83d\\', {"a": "hi "}), + ('{"a":"hi \\ud83d\\ude0', {"a": "hi "}), + ('{"a":"hi \\ud83d\\ude00', {"a": "hi 😀"}), + ('{"a":"hi \\ud83d\\ude00 x', {"a": "hi 😀 x"}), + ('{"a":"\\ud83c\\udf89\\ud83d', {"a": "🎉"}), + ], +) +def test_surrogate_pairs_are_held_back_until_complete(parser, text, expected): + result = parser.parse(text) + assert result == expected + result["a"].encode("utf-8") # never hands out a lone surrogate + + +def test_non_strict_incomplete_string_is_still_raw(): + parser = JSONParser(strict=False) + assert parser.parse('{"a":"foo\\u00') == {"a": "foo\\u00"} + assert parser.parse('{"a":"foo\\') == {"a": "foo\\"} + assert parser.parse('"A\\nB') == "A\\nB" + + +# --- literals --------------------------------------------------------------- + +@pytest.mark.parametrize( + "text, expected", + [ + ("[t", [True]), + ("[tru", [True]), + ("[fals", [False]), + ('{"a": nul', {"a": None}), + ("[1,t", [1, True]), + ], +) +def test_partial_literals(parser, text, expected): + assert parser.parse(text) == expected + + +def test_literal_prefix_no_longer_swallows_following_tokens(): + parser = JSONParser() + with pytest.raises(ValueError): + parser.parse("[trap]") + with pytest.raises(ValueError): + parser.parse("[t,1]") + + +def test_json5_case_insensitive_partial_literals(): + parser = create_json5_parser() + assert parser.parse("[T") == [True] + assert parser.parse("[Fal") == [False] + assert parser.parse("[NU") == [None] + assert parser.parse("[Na") == [math.nan] or math.isnan(parser.parse("[Na")[0]) + assert parser.parse("[Inf") == [math.inf] + assert parser.parse("[-Inf") == [-math.inf] + assert parser.parse("{a: 0x") == {"a": "0x"} + assert parser.parse("{a: 0x1F") == {"a": 31} + assert parser.parse("{a: +") == {"a": "+"} + + +# --- JSON5 comments and strings --------------------------------------------- + +def test_json5_incomplete_comment_at_end_of_stream(): + parser = create_json5_parser() + assert parser.parse('{"a": 1, /') == {"a": 1} + assert parser.parse('{"a": 1, //') == {"a": 1} + assert parser.parse('{"a": 1, // comment') == {"a": 1} + assert parser.parse('{"a": 1, /* comment') == {"a": 1} + assert parser.parse('{"a": 1, /* c */ "b": 2') == {"a": 1, "b": 2} + + +@pytest.mark.parametrize("has_lib", [True, False]) +def test_json5_string_decoding_is_independent_of_optional_dependency(monkeypatch, has_lib): + if not has_lib: + monkeypatch.setattr(json5_parser, "json5", None) + parser = create_json5_parser() + assert parser.parse("{a: 'it\\'s \\x41\\u0042', b: 'x\\\ny'") == {"a": "it's AB", "b": "xy"} + assert parser.parse("{a: 'it\\'s \\x4") == {"a": "it's "} + assert parser.parse("{a: 'it\\'s \\x41") == {"a": "it's A"} + assert parser.parse("{a: '\\ud83d\\ude00'") == {"a": "😀"} + assert parser.parse("{a: 1, b: 'x\\u2028y'}") == {"a": 1, "b": "x" + chr(0x2028) + "y"} + assert parser.parse('{"a": "\\/"}') == {"a": "/"} + + +def test_json5_plain_json_fast_path_without_lib(monkeypatch): + monkeypatch.setattr(json5_parser, "json5", None) + parser = create_json5_parser() + assert parser.parse('{"a": [1, 2, {"b": null}]}') == {"a": [1, 2, {"b": None}]} + assert parser.last_parse_reminding is None + + +# --- input types ------------------------------------------------------------- + +def test_bytes_input(parser): + assert parser.parse(b'{"a": 1}') == {"a": 1} + assert parser.parse(b'{"a": "caf\xc3\xa9", "b": [1') == {"a": "café", "b": [1]} + assert parser.parse(b'{"a": "caf\xc3') == {"a": "caf"} # split multi-byte char + assert parser.parse(bytearray(b"[1, 2")) == [1, 2] + assert parser.parse(b"") == {} + + +def test_bom_is_ignored(parser): + assert parser.parse('\ufeff{"a": 1}') == {"a": 1} + assert parser.parse('\ufeff{"a": 1') == {"a": 1} + assert parser.parse(b'\xef\xbb\xbf{"a": 1') == {"a": 1} + + +# --- facade / 0.x API surface ---------------------------------------------- + +def test_facade_exposes_and_forwards_settings(): + parser = JSONParser() + assert parser.strict is True + assert parser.json5_enabled is False + parser.strict = False + assert parser.parse('{"a":"x\\u00') == {"a": "x\\u00"} + calls = [] + parser.on_extra_token = lambda text, data, reminding: calls.append(reminding) + assert parser.parse("[1] x") == [1] + assert calls == [" x"] + assert parser.last_parse_reminding == " x" + parser.on_extra_token = None + assert parser.parse("[1] y") == [1] + assert JSONParser(json5_enabled=True).json5_enabled is True + + +def test_0x_method_names_still_callable(): + parser = JSONParser() + e = ValueError("x") + assert parser.parse_string('"abc" rest', e) == ("abc", " rest") + assert parser.parse_string('"abc', e) == ("abc", "") + assert parser.parse_number("12.5, 3", e) == (12.5, ", 3") + assert parser.parse_array("[1, 2] z", e) == ([1, 2], " z") + assert parser.parse_object('{"a": 1} z', e) == ({"a": 1}, " z") + assert parser.parse_true("true,", e) == (True, ",") + assert parser.parse_false("fa", e) == (False, "") + assert parser.parse_null("null]", e) == (None, "]") + assert parser.parse_space(" x", e) == "x" + assert parser.parse_any(" [1] ", e) == ([1], " ") + with pytest.raises(AttributeError): + _ = parser.does_not_exist + + +def test_default_on_extra_token_still_prints(): + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + JSONParser().parse("[1] x") + assert "Parsed JSON with extra tokens" in buf.getvalue() + + +def test_public_names(): + assert set(partialjson.__all__) >= {"JSONParser", "create_json_parser", "create_json5_parser"} + assert isinstance(create_json_parser(), partialjson.json_parser._JSONParser) + + +# --- performance guard ------------------------------------------------------- + +def test_large_partial_document_is_linear(): + import json + import time + + def build(n): + items = [{"id": i, "name": f"item {i}", "tags": ["a", "b"]} for i in range(n)] + return json.dumps(items)[:-5] + + def timed(text): + t = time.perf_counter() + parser.parse(text) + return time.perf_counter() - t + + parser = JSONParser() + t_small = timed(build(2000)) + t_large = timed(build(16000)) + # 8x the input should cost roughly 8x, not 64x. Allow generous slack. + assert t_large < t_small * 30, (t_small, t_large)