From c16cce86989279b47f093ca2b72bced697e78634 Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Fri, 3 Jul 2026 21:39:52 -0700 Subject: [PATCH 1/7] Add pluggable regex support --- CMakeLists.txt | 39 ++++ include/jsonata/Functions.h | 19 +- include/jsonata/IRegex.h | 78 +++++++ include/jsonata/Jsonata.h | 14 +- include/jsonata/Parser.h | 3 +- include/jsonata/StdRegex.h | 41 ++++ include/jsonata/Tokenizer.h | 8 +- include/jsonata/utils/Signature.h | 1 + src/jsonata/Functions.cpp | 358 ++++++++++++------------------ src/jsonata/Jsonata.cpp | 47 ++-- src/jsonata/Parser.cpp | 5 +- src/jsonata/StdRegex.cpp | 94 ++++++++ src/jsonata/Tokenizer.cpp | 19 +- src/jsonata/utils/Signature.cpp | 2 +- test/RE2Engine.cpp | 117 ++++++++++ test/RE2Engine.h | 46 ++++ test/RE2EngineTest.cpp | 90 ++++++++ 17 files changed, 711 insertions(+), 270 deletions(-) create mode 100644 include/jsonata/IRegex.h create mode 100644 include/jsonata/StdRegex.h create mode 100644 src/jsonata/StdRegex.cpp create mode 100644 test/RE2Engine.cpp create mode 100644 test/RE2Engine.h create mode 100644 test/RE2EngineTest.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 1032787..ac40e2b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -201,6 +201,45 @@ if(JSONATA_BUILD_TESTS) endif() endif() endif() + + # Optional: demonstrates that jsonata-cpp's pluggable regex engine + # (IRegex, see include/jsonata/IRegex.h) can be backed by Google's RE2 + # instead of std::regex. Off by default. RE2 must already be available + # via find_package (e.g. installed through vcpkg/conan/brew) -- it is + # not fetched automatically like nlohmann_json, since current RE2 + # releases also require Abseil, making a FetchContent fallback a much + # heavier addition than an optional test warrants. + option(JSONATA_BUILD_RE2_TEST "Build the optional RE2 regex engine test (requires re2 to be installed)" OFF) + + if(JSONATA_BUILD_RE2_TEST) + find_package(re2 CONFIG REQUIRED) + + add_executable(jsonata_re2_test test/RE2Engine.cpp test/RE2EngineTest.cpp) + target_link_libraries(jsonata_re2_test + jsonata + re2::re2 + gtest_main + gtest + Threads::Threads + ) + # re2::re2 pulls in Abseil, and Abseil's ABI (e.g. absl::string_view's + # layout) depends on which copy of its headers get resolved first. + # Other PUBLIC dependencies (e.g. a system-wide nlohmann_json found + # via find_package) can inject unrelated include directories that + # happen to also contain a differently-configured Abseil, silently + # shadowing the one re2 was actually built against and causing link + # errors. Force re2/Abseil's own include dirs to be searched first. + target_include_directories(jsonata_re2_test BEFORE PRIVATE + $ + ) + + if(TARGET gtest) + include(GoogleTest) + gtest_discover_tests(jsonata_re2_test + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + ) + endif() + endif() endif() # Installation rules diff --git a/include/jsonata/Functions.h b/include/jsonata/Functions.h index 3ed243f..cbfb229 100644 --- a/include/jsonata/Functions.h +++ b/include/jsonata/Functions.h @@ -27,6 +27,7 @@ #include #include +#include "jsonata/IRegex.h" #include "Utils.h" namespace jsonata { @@ -66,7 +67,8 @@ class Functions { static std::optional trim(const std::string& str); static std::any split(const std::string& str, const std::string& separator, int64_t limit = -1); - static std::any split(const std::string& str, const std::regex& pattern, + static std::any split(const std::string& str, + const std::shared_ptr& pattern, int64_t limit = -1); static std::optional join(const Utils::JList& arr, const std::string& separator = ""); @@ -173,9 +175,10 @@ class Functions { static int64_t millis(); // Regex functions - static Utils::JList evaluateMatcher(const std::regex& pattern, + static Utils::JList evaluateMatcher(const std::shared_ptr& pattern, const std::string& str); - static Utils::JList match(const std::string& str, const std::regex& pattern, + static Utils::JList match(const std::string& str, + const std::shared_ptr& pattern, int64_t limit = -1); // Lambda detection @@ -225,16 +228,18 @@ class Functions { static bool isNumericString(const std::string& str); static std::string safeReplacement(const std::string& replacement); static std::string safeReplaceAll(const std::string& str, - const std::regex& pattern, + const std::shared_ptr& pattern, const std::string& replacement); static std::string safeReplaceFirst(const std::string& str, - const std::regex& pattern, + const std::shared_ptr& pattern, const std::string& replacement); static std::string safeReplaceAllFn(const std::string& str, - const std::regex& pattern, + const std::shared_ptr& pattern, const std::any& func); + static std::string expandReplacement(const std::string& replacement, + const RegexMatch& match); static nlohmann::ordered_map toJsonataMatch( - const std::smatch& match); + const RegexMatch& match); static std::string encodeURI(const std::string& uri); static std::string leftPad(const std::string& str, int64_t size, const std::string& padStr = " "); diff --git a/include/jsonata/IRegex.h b/include/jsonata/IRegex.h new file mode 100644 index 0000000..381ab43 --- /dev/null +++ b/include/jsonata/IRegex.h @@ -0,0 +1,78 @@ +/** + * jsonata-cpp is the JSONata C++ reference port + * + * Copyright Robert Yokota + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include + +namespace jsonata { + +// A single match, engine-agnostic (the equivalent of std::smatch, but not +// tied to std::regex so alternative engines such as RE2 can produce it too). +struct RegexMatch { + std::string text; + size_t position = 0; + size_t length = 0; + // groups[0] is capture group 1, matching std::smatch's [1] indexing + // offset by the implicit full-match group 0. + std::vector> groups; +}; + +// Flags parsed from a JSONata regex literal's /pattern/flags suffix. Engines +// translate these into their own native flag representation. +struct RegexFlags { + bool caseInsensitive = false; + bool multiline = false; +}; + +// Engine-agnostic compiled regex. JSONata regex literals (and patterns given +// as plain strings to functions like $match/$replace/$split) are compiled +// into this interface, so the evaluator never depends on std::regex +// directly. The default engine wraps std::regex (see StdRegex.h); a custom +// engine (e.g. RE2) can be injected via Jsonata's constructor. +class IRegex { + public: + virtual ~IRegex() = default; + + // True if the pattern matches anywhere in str. Backs $contains. + virtual bool test(const std::string& str) const = 0; + + // The first match, if any. Backs $replace's first/limited-match paths. + virtual std::optional findFirst(const std::string& str) const = 0; + + // All non-overlapping matches, in order. Backs $match and $contains. + virtual std::vector findAll(const std::string& str) const = 0; + + // Splits str on every match, returning the segments between matches + // (including empty leading/trailing segments). Backs $split. + virtual std::vector split(const std::string& str) const = 0; +}; + +// Compiles a pattern into an IRegex. Used both for JSONata regex literals +// and for patterns supplied as plain strings at runtime. +using RegexEngine = std::function( + const std::string& pattern, RegexFlags flags)>; + +// The built-in std::regex-backed engine; this is jsonata-cpp's default and +// preserves its pre-existing regex behavior exactly. +RegexEngine defaultRegexEngine(); + +} // namespace jsonata diff --git a/include/jsonata/Jsonata.h b/include/jsonata/Jsonata.h index 68db659..50a30b3 100644 --- a/include/jsonata/Jsonata.h +++ b/include/jsonata/Jsonata.h @@ -34,6 +34,7 @@ #include #include +#include "jsonata/IRegex.h" #include "jsonata/Parser.h" // Forward declarations @@ -124,8 +125,9 @@ class JFunction { class Jsonata { public: // Constructors - Jsonata(); - Jsonata(const std::string& jsonataExpression); + explicit Jsonata(RegexEngine regexEngine = defaultRegexEngine()); + Jsonata(const std::string& jsonataExpression, + RegexEngine regexEngine = defaultRegexEngine()); Jsonata(const Jsonata& other); // Copy constructor for per-thread instances // Main evaluation methods (ordered JSON variants) @@ -150,8 +152,13 @@ class Jsonata { // Environment access std::shared_ptr getEnvironment() const; + // Regex engine used to compile JSONata regex literals and dynamic + // string patterns (e.g. for $match/$replace/$split). + RegexEngine getRegexEngine() const { return regexEngine_; } + // Factory methods - static Jsonata jsonata(const std::string& expression); + static Jsonata jsonata(const std::string& expression, + RegexEngine regexEngine = defaultRegexEngine()); // Instance methods (matching Java reference) std::shared_ptr createFrame(); @@ -193,6 +200,7 @@ class Jsonata { static void clearPerThreadInstance(); std::shared_ptr environment_; + RegexEngine regexEngine_ = defaultRegexEngine(); static thread_local std::shared_ptr currentParser_; static thread_local std::any tls_input_; static thread_local std::shared_ptr tls_environment_; diff --git a/include/jsonata/Parser.h b/include/jsonata/Parser.h index 32716ad..fc77c1d 100644 --- a/include/jsonata/Parser.h +++ b/include/jsonata/Parser.h @@ -369,7 +369,8 @@ class Parser { // Main parsing method std::shared_ptr parse(const std::string& source, - bool recover = false); + bool recover = false, + RegexEngine regexEngine = defaultRegexEngine()); // Get parsing errors const std::vector>& getErrors() const { diff --git a/include/jsonata/StdRegex.h b/include/jsonata/StdRegex.h new file mode 100644 index 0000000..bc959ea --- /dev/null +++ b/include/jsonata/StdRegex.h @@ -0,0 +1,41 @@ +/** + * jsonata-cpp is the JSONata C++ reference port + * + * Copyright Robert Yokota + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +#include "jsonata/IRegex.h" + +namespace jsonata { + +// Default IRegex implementation, backed by std::regex (ECMAScript grammar). +class StdRegex : public IRegex { + public: + StdRegex(const std::string& pattern, RegexFlags flags); + + bool test(const std::string& str) const override; + std::optional findFirst(const std::string& str) const override; + std::vector findAll(const std::string& str) const override; + std::vector split(const std::string& str) const override; + + private: + std::regex regex_; + static RegexMatch toRegexMatch(const std::smatch& m); +}; + +} // namespace jsonata diff --git a/include/jsonata/Tokenizer.h b/include/jsonata/Tokenizer.h index 351f9c1..9e831ff 100644 --- a/include/jsonata/Tokenizer.h +++ b/include/jsonata/Tokenizer.h @@ -32,6 +32,8 @@ #include #include +#include "jsonata/IRegex.h" + namespace jsonata { class Tokenizer { @@ -63,10 +65,12 @@ class Tokenizer { size_t position_; size_t length_; int64_t depth_; + RegexEngine regexEngine_; public: // Constructor - explicit Tokenizer(const std::string& path); + explicit Tokenizer(const std::string& path, + RegexEngine regexEngine = defaultRegexEngine()); // Main tokenization method std::unique_ptr next(bool prefix = false); @@ -80,7 +84,7 @@ class Tokenizer { std::unique_ptr create(const std::string& type, const std::any& value); bool isClosingSlash(size_t position) const; - std::regex scanRegex(); + std::shared_ptr scanRegex(); // Codepoint access methods (like Java charAt) int32_t charAt(size_t index) const; diff --git a/include/jsonata/utils/Signature.h b/include/jsonata/utils/Signature.h index f9cdefb..b01346b 100644 --- a/include/jsonata/utils/Signature.h +++ b/include/jsonata/utils/Signature.h @@ -48,6 +48,7 @@ #include // Include Utils to resolve JList +#include "../IRegex.h" #include "../Utils.h" namespace jsonata { diff --git a/src/jsonata/Functions.cpp b/src/jsonata/Functions.cpp index 2a6bd98..cef9877 100644 --- a/src/jsonata/Functions.cpp +++ b/src/jsonata/Functions.cpp @@ -628,7 +628,8 @@ std::any Functions::split(const std::string& str, const std::string& separator, return result; } -std::any Functions::split(const std::string& str, const std::regex& pattern, +std::any Functions::split(const std::string& str, + const std::shared_ptr& pattern, int64_t limit) { Utils::JList result; @@ -637,16 +638,10 @@ std::any Functions::split(const std::string& str, const std::regex& pattern, return result; } - // Use std::sregex_token_iterator to split with regex (equivalent to Java's - // Pattern.split with -1) The -1 parameter to sregex_token_iterator gives us - // the non-matching parts (like Java's split with trailing empties - // preserved) - std::sregex_token_iterator iter(str.begin(), str.end(), pattern, -1); - std::sregex_token_iterator end; - - // Java logic: split completely first, then truncate if needed - for (; iter != end; ++iter) { - result.push_back(std::string(*iter)); + // pattern->split gives us the non-matching parts (like Java's split with + // trailing empties preserved, equivalent to Pattern.split with -1) + for (const auto& part : pattern->split(str)) { + result.push_back(part); } // Java reference: if limit is specified and less than result size, truncate @@ -1099,17 +1094,20 @@ Functions::getFunctionRegistry() { // implementation) if (isString(args[1])) { auto pattern = std::any_cast(args[1]); + auto* jsonataInstance = Jsonata::getCurrentInstance(); + if (!jsonataInstance) return std::any(); try { - std::regex regex_pattern(pattern); - auto result = match(str, regex_pattern, limit); + auto regexPattern = + jsonataInstance->getRegexEngine()(pattern, RegexFlags{}); + auto result = match(str, regexPattern, limit); return std::any(result); - } catch (const std::regex_error&) { + } catch (const std::exception&) { return std::any(); } } else { // Check if it's a regex object try { - auto regex = std::any_cast(args[1]); + auto regex = std::any_cast>(args[1]); auto result = match(str, regex, limit); return std::any(result); } catch (const std::bad_any_cast&) { @@ -1205,7 +1203,7 @@ Functions::getFunctionRegistry() { } else { // Check if it's a regex object try { - auto regex = std::any_cast(args[1]); + auto regex = std::any_cast>(args[1]); return split(str, regex, limit); } catch (const std::bad_any_cast&) { // Not a regex object, check if it's a function @@ -2473,7 +2471,7 @@ void Functions::stringifyInternal(std::ostringstream& os, const std::any& arg, // reference exactly Java: if (arg instanceof Symbol) { return; } - // outputs nothing (empty string) return; - } else if (arg.type() == typeid(std::regex)) { + } else if (arg.type() == typeid(std::shared_ptr)) { return; } else { throw JException("T0411", 0, @@ -2678,12 +2676,11 @@ bool Functions::contains(const std::string& str, const std::any& token) { return str.find(searchStr) != std::string::npos; } // Java lines 696-701: else if (token instanceof Pattern) - else if (token.type() == typeid(std::regex)) { - auto regex = std::any_cast(token); + else if (token.type() == typeid(std::shared_ptr)) { + auto regex = std::any_cast>(token); // Java lines 697-701: var matches = evaluateMatcher((Pattern)token, // str); result = !matches.isEmpty(); - auto matches = evaluateMatcher(regex, str); - return !matches.empty(); + return regex->test(str); } // Check if it's a regex object (stored as map with "type" = "regex") else { @@ -2712,19 +2709,10 @@ std::optional Functions::replace(const std::string& str, // Check if replacement is a function if (isLambda(replacement)) { // Handle function-based replacement - if (pattern.type() == typeid(std::regex)) { - auto regex = std::any_cast(pattern); + if (pattern.type() == typeid(std::shared_ptr)) { + auto regex = std::any_cast>(pattern); // Use thread-local instance (matching Java implementation) return safeReplaceAllFn(str, regex, replacement); - } else { - // Check if it's a regex object (stored as map with "type" = - // "regex") - try { - auto regex = std::any_cast(pattern); - return safeReplaceAllFn(str, regex, replacement); - } catch (const std::bad_any_cast&) { - // Not a regex object - } } // For string patterns with function replacement, would need more // complex logic @@ -2743,9 +2731,9 @@ std::optional Functions::replace(const std::string& str, replaceStr = *replacementStr; } - // Handle regex patterns (direct std::regex or regex-object map) - if (pattern.type() == typeid(std::regex)) { - std::regex regex = std::any_cast(pattern); + // Handle regex patterns + if (pattern.type() == typeid(std::shared_ptr)) { + auto regex = std::any_cast>(pattern); if (limit == -1) { // No limit specified - replace all occurrences (Java default @@ -2812,13 +2800,15 @@ std::optional Functions::replace(const std::string& str, // String replacement) Compile the string pattern as a regex and apply // safeReplaceFirst repeatedly std::string result = str; + auto* jsonataInstance = Jsonata::getCurrentInstance(); try { - std::regex regex(searchStr); + if (!jsonataInstance) throw std::runtime_error("no current instance"); + auto regex = jsonataInstance->getRegexEngine()(searchStr, RegexFlags{}); for (int64_t i = 0; i < limit; i++) { result = safeReplaceFirst(result, regex, replaceStr); } return result; - } catch (const std::regex_error&) { + } catch (const std::exception&) { // If pattern is not a valid regex, fall back to literal first-only // replacements for (int64_t i = 0; i < limit; i++) { @@ -5000,23 +4990,20 @@ int64_t Functions::millis() { .count(); } -Utils::JList Functions::evaluateMatcher(const std::regex& pattern, +Utils::JList Functions::evaluateMatcher(const std::shared_ptr& pattern, const std::string& str) { Utils::JList matches = Utils::createSequence(); - std::sregex_iterator iter(str.begin(), str.end(), pattern); - std::sregex_iterator end; - for (; iter != end; ++iter) { - const std::smatch& smatch = *iter; + for (const auto& regexMatch : pattern->findAll(str)) { nlohmann::ordered_map match; - match["match"] = smatch.str(); - match["index"] = static_cast(smatch.position()); + match["match"] = regexMatch.text; + match["index"] = static_cast(regexMatch.position); // Collect the groups starting from group 1 (excluding full match) - // matching Java implementation Utils::JList groups; - for (size_t g = 1; g < smatch.size(); ++g) { - groups.push_back(smatch[g].str()); + for (const auto& group : regexMatch.groups) { + groups.push_back(group.value_or("")); } match["groups"] = groups; matches.push_back(match); @@ -5025,7 +5012,8 @@ Utils::JList Functions::evaluateMatcher(const std::regex& pattern, return matches; } -Utils::JList Functions::match(const std::string& str, const std::regex& pattern, +Utils::JList Functions::match(const std::string& str, + const std::shared_ptr& pattern, int64_t limit) { auto matches = evaluateMatcher(pattern, str); if (limit > 0 && matches.size() > static_cast(limit)) { @@ -5147,112 +5135,106 @@ std::string Functions::safeReplacement(const std::string& replacement) { return result; } +std::string Functions::expandReplacement(const std::string& repl, + const RegexMatch& match) { + // Manual implementation to match Java semantics for $-expansion and + // handling of ambiguous group references like $18 and $123. Shared by + // safeReplaceAll and safeReplaceFirst. + std::string out; + out.reserve(repl.size()); + // max capturing group index (exclude 0, the full match) + const int64_t maxIndex = static_cast(match.groups.size()); + + for (size_t i = 0; i < repl.size(); ++i) { + char c = repl[i]; + if (c != '$') { + out.push_back(c); + continue; + } + + if (i + 1 >= repl.size()) { + out.push_back('$'); + continue; + } + + char next = repl[i + 1]; + if (next == '$') { + out.push_back('$'); + i += 1; + continue; + } + + if (std::isdigit(static_cast(next))) { + // Parse all following digits to form the number + size_t j = i + 1; + while (j < repl.size() && + std::isdigit(static_cast(repl[j]))) { + ++j; + } + // digits are [i+1, j) + int64_t chosenIdx = -1; + size_t chosenLen = 0; + for (size_t k = (j - (i + 1)); k >= 1; --k) { + int64_t idx = std::stoi(repl.substr(i + 1, k)); + if (idx <= maxIndex && idx >= 0) { + chosenIdx = idx; + chosenLen = k; + break; + } + if (k == 1) break; // prevent underflow of size_t + } + if (chosenIdx >= 0) { + out += chosenIdx == 0 ? match.text + : match.groups[chosenIdx - 1].value_or(""); + // Append leftover digits as literals (e.g., $123 with + // 12 valid -> append '3') + out.append(repl.substr(i + 1 + chosenLen, + j - (i + 1 + chosenLen))); + i = j - 1; + continue; + } else { + // No valid group index; consume first digit + // (interpreted as invalid $) and emit nothing for + // the group, leaving remaining digits literal. + if (j > i + 1) { + // skip first digit + i = i + 1; // loop will i++ again, effectively + // skipping this digit + } + // Do not append '$' + continue; + } + } + + // '$' followed by non-digit: treat as literal '$' + out.push_back('$'); + // do not consume the next char here; it will be processed in + // next iteration + } + + return out; +} + std::string Functions::safeReplaceAll(const std::string& str, - const std::regex& pattern, + const std::shared_ptr& pattern, const std::string& replacement) { - // Manual implementation to match Java semantics for $-expansion and - // handling of ambiguous group references like $18 and $123. std::string result; result.reserve(str.size()); - - std::sregex_iterator it(str.begin(), str.end(), pattern); - std::sregex_iterator end; + std::string prepared = safeReplacement(replacement); size_t lastEnd = 0; - for (; it != end; ++it) { - const std::smatch& m = *it; + for (const auto& m : pattern->findAll(str)) { // Guard against zero-length matches to avoid infinite loops - if (m.length() == 0) { + if (m.length == 0) { // Append the remainder and stop (aligns with override behavior) result.append(str.substr(lastEnd)); return result; } // Text before match - result.append( - str.substr(lastEnd, static_cast(m.position()) - lastEnd)); - - // Expand replacement against this match - auto expand = [&](const std::string& repl, - const std::smatch& match) -> std::string { - std::string out; - out.reserve(repl.size()); - const int64_t maxIndex = static_cast(match.size()) - - 1; // max capturing group index (exclude 0) - - for (size_t i = 0; i < repl.size(); ++i) { - char c = repl[i]; - if (c != '$') { - out.push_back(c); - continue; - } - - if (i + 1 >= repl.size()) { - out.push_back('$'); - continue; - } - - char next = repl[i + 1]; - if (next == '$') { - out.push_back('$'); - i += 1; - continue; - } - - if (std::isdigit(static_cast(next))) { - // Parse all following digits to form the number - size_t j = i + 1; - while (j < repl.size() && - std::isdigit(static_cast(repl[j]))) { - ++j; - } - // digits are [i+1, j) - int64_t chosenIdx = -1; - size_t chosenLen = 0; - for (size_t k = (j - (i + 1)); k >= 1; --k) { - int64_t idx = std::stoi(repl.substr(i + 1, k)); - if (idx <= maxIndex && idx >= 0) { - chosenIdx = idx; - chosenLen = k; - break; - } - if (k == 1) break; // prevent underflow of size_t - } - if (chosenIdx >= 0) { - out += match.str(chosenIdx); - // Append leftover digits as literals (e.g., $123 with - // 12 valid -> append '3') - out.append(repl.substr(i + 1 + chosenLen, - j - (i + 1 + chosenLen))); - i = j - 1; - continue; - } else { - // No valid group index; consume first digit - // (interpreted as invalid $) and emit nothing for - // the group, leaving remaining digits literal. - if (j > i + 1) { - // skip first digit - i = i + 1; // loop will i++ again, effectively - // skipping this digit - } - // Do not append '$' - continue; - } - } - - // '$' followed by non-digit: treat as literal '$' - out.push_back('$'); - // do not consume the next char here; it will be processed in - // next iteration - } - - return out; - }; - - std::string prepared = safeReplacement(replacement); - result += expand(prepared, m); - - lastEnd = static_cast(m.position() + m.length()); + result.append(str.substr(lastEnd, m.position - lastEnd)); + result += expandReplacement(prepared, m); + lastEnd = m.position + m.length; } // Trailing remainder @@ -5261,101 +5243,35 @@ std::string Functions::safeReplaceAll(const std::string& str, } std::string Functions::safeReplaceFirst(const std::string& str, - const std::regex& pattern, + const std::shared_ptr& pattern, const std::string& replacement) { - // Manual first-only replace using same expansion as safeReplaceAll - std::smatch m; - if (!std::regex_search(str, m, pattern)) { + auto m = pattern->findFirst(str); + if (!m) { return str; } // Guard against zero-length match - if (m.length() == 0) { + if (m->length == 0) { return str; } - auto expand = [&](const std::string& repl, - const std::smatch& match) -> std::string { - std::string out; - out.reserve(repl.size()); - const int64_t maxIndex = static_cast(match.size()) - - 1; // max capturing group index (exclude 0) - - for (size_t i = 0; i < repl.size(); ++i) { - char c = repl[i]; - if (c != '$') { - out.push_back(c); - continue; - } - if (i + 1 >= repl.size()) { - out.push_back('$'); - continue; - } - char next = repl[i + 1]; - if (next == '$') { - out.push_back('$'); - i += 1; - continue; - } - if (std::isdigit(static_cast(next))) { - size_t j = i + 1; - while (j < repl.size() && - std::isdigit(static_cast(repl[j]))) - ++j; - int64_t chosenIdx = -1; - size_t chosenLen = 0; - for (size_t k = (j - (i + 1)); k >= 1; --k) { - int64_t idx = std::stoi(repl.substr(i + 1, k)); - if (idx <= maxIndex && idx >= 0) { - chosenIdx = idx; - chosenLen = k; - break; - } - if (k == 1) break; - } - if (chosenIdx >= 0) { - out += match.str(chosenIdx); - out.append(repl.substr(i + 1 + chosenLen, - j - (i + 1 + chosenLen))); - i = j - 1; - continue; - } else { - // consume first digit of the invalid group reference and - // emit nothing - if (j > i + 1) { - i = i + 1; - } - continue; - } - } - out.push_back('$'); - } - return out; - }; - std::string prepared = safeReplacement(replacement); std::string res; res.reserve(str.size()); - res.append(str.substr(0, static_cast(m.position()))); - res += expand(prepared, m); - res.append(str.substr(static_cast(m.position() + m.length()))); + res.append(str.substr(0, m->position)); + res += expandReplacement(prepared, *m); + res.append(str.substr(m->position + m->length)); return res; } std::string Functions::safeReplaceAllFn(const std::string& str, - const std::regex& pattern, + const std::shared_ptr& pattern, const std::any& func) { // Following Java implementation: Functions.java lines 844-859 - std::string result = str; - std::sregex_iterator matchIter(str.begin(), str.end(), pattern); - std::sregex_iterator endIter; - size_t lastPos = 0; std::string finalResult; - for (; matchIter != endIter; ++matchIter) { - const std::smatch& match = *matchIter; - + for (const auto& match : pattern->findAll(str)) { try { // Convert match to Jsonata format (equivalent to Java's // toJsonataMatch) @@ -5382,10 +5298,10 @@ std::string Functions::safeReplaceAllFn(const std::string& str, std::any_cast(functionResult); // Add text before match - finalResult += str.substr(lastPos, match.position() - lastPos); + finalResult += str.substr(lastPos, match.position - lastPos); // Add replacement finalResult += replacementStr; - lastPos = match.position() + match.length(); + lastPos = match.position + match.length; } else { // Java line 852: return null for non-string results // This will be caught by the replace function and converted to @@ -5410,16 +5326,16 @@ std::string Functions::safeReplaceAllFn(const std::string& str, } nlohmann::ordered_map Functions::toJsonataMatch( - const std::smatch& match) { + const RegexMatch& match) { nlohmann::ordered_map result; - result["match"] = match.str(); + result["match"] = match.text; // Based on test expectations: groups array starts from first capture group // (excluding full match) This matches the behavior expected by test case034 Utils::JList groups; - for (size_t i = 1; i < match.size(); ++i) { - groups.push_back(match[i].str()); + for (const auto& group : match.groups) { + groups.push_back(group.value_or("")); } result["groups"] = groups; diff --git a/src/jsonata/Jsonata.cpp b/src/jsonata/Jsonata.cpp index fae86a0..8ad4074 100644 --- a/src/jsonata/Jsonata.cpp +++ b/src/jsonata/Jsonata.cpp @@ -118,7 +118,7 @@ void Frame::setEvaluateExitCallback(ExitCallback callback) { } // Jsonata implementation -Jsonata::Jsonata() { +Jsonata::Jsonata(RegexEngine regexEngine) : regexEngine_(regexEngine) { parser_ = std::make_unique(); currentInstance_ = this; // Initialize environment like Java does: environment = @@ -137,14 +137,14 @@ std::shared_ptr Jsonata::createFrame( return std::make_shared(enclosingEnvironment); } -Jsonata Jsonata::jsonata(const std::string& expression) { - Jsonata instance(expression); +Jsonata Jsonata::jsonata(const std::string& expression, RegexEngine regexEngine) { + Jsonata instance(expression, regexEngine); // Parse and store the expression return instance; } std::shared_ptr Jsonata::parse(const std::string& expression) { - return parser_->parse(expression); + return parser_->parse(expression, false, regexEngine_); } std::any Jsonata::evaluate(std::shared_ptr expr, @@ -690,8 +690,8 @@ std::any Jsonata::evaluateRegex(std::shared_ptr expr, } try { - // The tokenizer stores std::regex objects in the value - return std::any_cast(expr->value); + // The tokenizer stores compiled IRegex objects in the value + return std::any_cast>(expr->value); } catch (const std::bad_any_cast&) { throw JException("T0410", expr->position, "Invalid regex value"); } @@ -2418,7 +2418,8 @@ Jsonata* Jsonata::getPerThreadInstance() { tls_environment_.reset(); } -Jsonata::Jsonata(const std::string& jsonataExpression) { +Jsonata::Jsonata(const std::string& jsonataExpression, RegexEngine regexEngine) + : regexEngine_(regexEngine) { currentInstance_ = this; // Initialize environment like Java does: environment = // createFrame(staticFrame) @@ -2426,7 +2427,7 @@ Jsonata::Jsonata(const std::string& jsonataExpression) { // Parse the expression Parser parser; - expression_ = parser.parse(jsonataExpression); + expression_ = parser.parse(jsonataExpression, false, regexEngine_); } Jsonata::Jsonata(const Jsonata& other) { @@ -2437,6 +2438,7 @@ Jsonata::Jsonata(const Jsonata& other) { expression_ = other.expression_; environment_ = other.environment_; timestamp_ = other.timestamp_; + regexEngine_ = other.regexEngine_; } nlohmann::ordered_json Jsonata::evaluate(const nlohmann::ordered_json& input) { @@ -2655,21 +2657,20 @@ std::any Jsonata::apply(const std::any& proc, const Utils::JList& args, struct RegexState { std::shared_ptr str; - std::shared_ptr regex; - std::sregex_iterator it; - std::sregex_iterator end; + std::vector matches; + size_t index = 0; }; static std::any regexClosure(std::shared_ptr state) { - if (state->it == state->end) return std::any{}; - auto match = *state->it; - ++(state->it); + if (state->index >= state->matches.size()) return std::any{}; + const RegexMatch& match = state->matches[state->index]; + ++(state->index); nlohmann::ordered_map result; - result["match"] = std::string(match.str()); - result["start"] = static_cast(match.position()); - result["end"] = static_cast(match.position() + match.length()); + result["match"] = match.text; + result["start"] = static_cast(match.position); + result["end"] = static_cast(match.position + match.length); Utils::JList groups; - groups.push_back(std::string(match.str())); + groups.push_back(match.text); result["groups"] = std::any(groups); JFunction nextFn; nextFn.implementation = [state](const Utils::JList&, const std::any&, std::shared_ptr) -> std::any { @@ -2703,17 +2704,15 @@ std::any Jsonata::applyInner(const std::any& proc, const Utils::JList& args, // Check if it's a regex pattern (Java lines 1757-1766) // Java: } else if (proc instanceof Pattern) { - if (proc.type() == typeid(std::regex)) { - auto regex = std::any_cast(proc); + if (proc.type() == typeid(std::shared_ptr)) { + auto regex = std::any_cast>(proc); Utils::JList results; for (const auto& arg : validatedArgs) { if (arg.has_value() && arg.type() == typeid(std::string)) { auto state = std::make_shared(); state->str = std::make_shared(std::any_cast(arg)); - state->regex = std::make_shared(regex); - state->it = std::sregex_iterator(state->str->begin(), state->str->end(), *state->regex); - state->end = std::sregex_iterator(); + state->matches = regex->findAll(*state->str); results.push_back(regexClosure(state)); } } @@ -3743,7 +3742,7 @@ bool Jsonata::isFunctionLike(const std::any& o) const { } // Check if it's a regex pattern (C++ equivalent of "o instanceof Pattern") - if (o.has_value() && o.type() == typeid(std::regex)) { + if (o.has_value() && o.type() == typeid(std::shared_ptr)) { return true; } diff --git a/src/jsonata/Parser.cpp b/src/jsonata/Parser.cpp index c85c977..df752c8 100644 --- a/src/jsonata/Parser.cpp +++ b/src/jsonata/Parser.cpp @@ -1308,10 +1308,11 @@ void Parser::registerSymbol(std::shared_ptr symbol) { } std::shared_ptr Parser::parse(const std::string& source, - bool recover) { + bool recover, + RegexEngine regexEngine) { source_ = source; recover_ = recover; - lexer_ = std::make_unique(source); + lexer_ = std::make_unique(source, regexEngine); // Start parsing (Java: advance() defaults to prefix=false) advance(""); diff --git a/src/jsonata/StdRegex.cpp b/src/jsonata/StdRegex.cpp new file mode 100644 index 0000000..6d3db92 --- /dev/null +++ b/src/jsonata/StdRegex.cpp @@ -0,0 +1,94 @@ +/** + * jsonata-cpp is the JSONata C++ reference port + * + * Copyright Robert Yokota + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "jsonata/StdRegex.h" + +namespace jsonata { + +namespace { + +std::regex_constants::syntax_option_type toSyntaxOptions(RegexFlags flags) { + auto options = std::regex_constants::ECMAScript; + if (flags.caseInsensitive) { + options |= std::regex_constants::icase; + } + // Note: the 'm' (multiline) flag is intentionally not translated here, + // preserving jsonata-cpp's pre-existing std::regex behavior exactly. + // RegexFlags.multiline is still populated for custom engines that do + // want to honor it. + return options; +} + +} // namespace + +StdRegex::StdRegex(const std::string& pattern, RegexFlags flags) + : regex_(pattern, toSyntaxOptions(flags)) {} + +RegexMatch StdRegex::toRegexMatch(const std::smatch& m) { + RegexMatch result; + result.text = m.str(); + result.position = static_cast(m.position()); + result.length = static_cast(m.length()); + for (size_t g = 1; g < m.size(); ++g) { + if (m[g].matched) { + result.groups.push_back(m[g].str()); + } else { + result.groups.push_back(std::nullopt); + } + } + return result; +} + +bool StdRegex::test(const std::string& str) const { + return std::regex_search(str, regex_); +} + +std::optional StdRegex::findFirst(const std::string& str) const { + std::smatch m; + if (!std::regex_search(str, m, regex_)) { + return std::nullopt; + } + return toRegexMatch(m); +} + +std::vector StdRegex::findAll(const std::string& str) const { + std::vector results; + std::sregex_iterator it(str.begin(), str.end(), regex_); + std::sregex_iterator end; + for (; it != end; ++it) { + results.push_back(toRegexMatch(*it)); + } + return results; +} + +std::vector StdRegex::split(const std::string& str) const { + std::vector results; + std::sregex_token_iterator it(str.begin(), str.end(), regex_, -1); + std::sregex_token_iterator end; + for (; it != end; ++it) { + results.push_back(*it); + } + return results; +} + +RegexEngine defaultRegexEngine() { + return [](const std::string& pattern, RegexFlags flags) -> std::shared_ptr { + return std::make_shared(pattern, flags); + }; +} + +} // namespace jsonata diff --git a/src/jsonata/Tokenizer.cpp b/src/jsonata/Tokenizer.cpp index 3aec3e9..737f077 100644 --- a/src/jsonata/Tokenizer.cpp +++ b/src/jsonata/Tokenizer.cpp @@ -53,8 +53,8 @@ std::unordered_map Tokenizer::createEscapes() { {"f", "\f"}, {"n", "\n"}, {"r", "\r"}, {"t", "\t"}}; } -Tokenizer::Tokenizer(const std::string& path) - : path_(path), position_(0), depth_(0) { +Tokenizer::Tokenizer(const std::string& path, RegexEngine regexEngine) + : path_(path), position_(0), depth_(0), regexEngine_(regexEngine) { // Pre-compute codepoints and byte offsets for O(1) charAt access auto it = path_.begin(); while (it != path_.end()) { @@ -109,7 +109,7 @@ bool Tokenizer::isClosingSlash(size_t position) const { return false; } -std::regex Tokenizer::scanRegex() { +std::shared_ptr Tokenizer::scanRegex() { // The prefix '/' will have been previously scanned. Find the end of the // regex. Search for closing '/' ignoring any that are escaped, or within // brackets (matches Java logic exactly) @@ -140,16 +140,17 @@ std::regex Tokenizer::scanRegex() { } flags = substring(start, position_); - // Convert flags to std::regex_constants - std::regex_constants::syntax_option_type regexFlags = - std::regex_constants::ECMAScript; + RegexFlags regexFlags; if (flags.find('i') != std::string::npos) { - regexFlags |= std::regex_constants::icase; + regexFlags.caseInsensitive = true; + } + if (flags.find('m') != std::string::npos) { + regexFlags.multiline = true; } try { - return std::regex(pattern, regexFlags); - } catch (const std::regex_error& e) { + return regexEngine_(pattern, regexFlags); + } catch (const std::exception& e) { throw JException("S0301", static_cast(start), pattern); } } diff --git a/src/jsonata/utils/Signature.cpp b/src/jsonata/utils/Signature.cpp index 711c0d8..fd985d6 100644 --- a/src/jsonata/utils/Signature.cpp +++ b/src/jsonata/utils/Signature.cpp @@ -136,7 +136,7 @@ std::string Signature::checkObjectType(const std::any &value) { } catch (const std::bad_any_cast &) { // Try shared_ptr - check if it's a regex object try { - auto regex = std::any_cast(value); + auto regex = std::any_cast>(value); return "f"; } catch (const std::bad_any_cast &) { return ""; diff --git a/test/RE2Engine.cpp b/test/RE2Engine.cpp new file mode 100644 index 0000000..1a922ff --- /dev/null +++ b/test/RE2Engine.cpp @@ -0,0 +1,117 @@ +/** + * jsonata-cpp is the JSONata C++ reference port + * + * Copyright Robert Yokota + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "RE2Engine.h" + +#include + +using jsonata::IRegex; +using jsonata::RegexEngine; +using jsonata::RegexFlags; +using jsonata::RegexMatch; + +namespace { + +RE2::Options toRE2Options(RegexFlags flags) { + RE2::Options options; + options.set_case_sensitive(!flags.caseInsensitive); + // RE2's one_line defaults to false (^/$ match at line boundaries); set it + // true when the JSONata pattern did NOT request the 'm' flag, so ^/$ + // only match the start/end of the whole string. + options.set_one_line(!flags.multiline); + options.set_log_errors(false); + return options; +} + +} // namespace + +RE2Regex::RE2Regex(const std::string& pattern, RegexFlags flags) + : re_(pattern, toRE2Options(flags)) { + if (!re_.ok()) { + throw std::runtime_error("RE2 compile error: " + re_.error()); + } +} + +RegexMatch RE2Regex::toRegexMatch(const re2::StringPiece& input, + const std::vector& submatch, + int numGroups) { + RegexMatch result; + result.text.assign(submatch[0].data(), submatch[0].size()); + result.position = static_cast(submatch[0].data() - input.data()); + result.length = submatch[0].size(); + for (int g = 1; g <= numGroups; ++g) { + if (submatch[g].data() == nullptr) { + result.groups.push_back(std::nullopt); + } else { + result.groups.push_back( + std::string(submatch[g].data(), submatch[g].size())); + } + } + return result; +} + +bool RE2Regex::test(const std::string& str) const { + return RE2::PartialMatch(str, re_); +} + +std::optional RE2Regex::findFirst(const std::string& str) const { + int numGroups = re_.NumberOfCapturingGroups(); + std::vector submatch(numGroups + 1); + re2::StringPiece input(str); + if (!re_.Match(input, 0, str.size(), RE2::UNANCHORED, submatch.data(), + numGroups + 1)) { + return std::nullopt; + } + return toRegexMatch(input, submatch, numGroups); +} + +std::vector RE2Regex::findAll(const std::string& str) const { + std::vector results; + int numGroups = re_.NumberOfCapturingGroups(); + std::vector submatch(numGroups + 1); + re2::StringPiece input(str); + + size_t startpos = 0; + while (startpos <= str.size()) { + if (!re_.Match(input, startpos, str.size(), RE2::UNANCHORED, + submatch.data(), numGroups + 1)) { + break; + } + RegexMatch match = toRegexMatch(input, submatch, numGroups); + // Guard against zero-length matches to avoid an infinite loop. + startpos = match.position + (match.length == 0 ? 1 : match.length); + results.push_back(std::move(match)); + } + return results; +} + +std::vector RE2Regex::split(const std::string& str) const { + std::vector result; + size_t lastEnd = 0; + for (const auto& match : findAll(str)) { + result.push_back(str.substr(lastEnd, match.position - lastEnd)); + lastEnd = match.position + match.length; + } + result.push_back(str.substr(lastEnd)); + return result; +} + +RegexEngine re2RegexEngine() { + return [](const std::string& pattern, RegexFlags flags) -> std::shared_ptr { + return std::make_shared(pattern, flags); + }; +} diff --git a/test/RE2Engine.h b/test/RE2Engine.h new file mode 100644 index 0000000..e84d372 --- /dev/null +++ b/test/RE2Engine.h @@ -0,0 +1,46 @@ +/** + * jsonata-cpp is the JSONata C++ reference port + * + * Copyright Robert Yokota + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +#include "jsonata/IRegex.h" + +// Demonstration IRegex implementation backed by Google's RE2, showing that +// jsonata-cpp's regex engine hook (see IRegex.h/StdRegex.h) is not tied to +// std::regex. Lives under test/ since it's only built for the optional RE2 +// test target (JSONATA_BUILD_RE2_TEST) and isn't part of the jsonata library. +class RE2Regex : public jsonata::IRegex { + public: + RE2Regex(const std::string& pattern, jsonata::RegexFlags flags); + + bool test(const std::string& str) const override; + std::optional findFirst(const std::string& str) const override; + std::vector findAll(const std::string& str) const override; + std::vector split(const std::string& str) const override; + + private: + RE2 re_; + static jsonata::RegexMatch toRegexMatch( + const re2::StringPiece& input, + const std::vector& submatch, int numGroups); +}; + +// Factory for use with Jsonata's constructor, e.g. +// Jsonata("/foo/i", re2RegexEngine()). +jsonata::RegexEngine re2RegexEngine(); diff --git a/test/RE2EngineTest.cpp b/test/RE2EngineTest.cpp new file mode 100644 index 0000000..55b6afc --- /dev/null +++ b/test/RE2EngineTest.cpp @@ -0,0 +1,90 @@ +/** + * jsonata-cpp is the JSONata C++ reference port + * + * Copyright Robert Yokota + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "RE2Engine.h" + +using jsonata::Jsonata; + +TEST(RE2EngineTest, Match) { + // A single match is returned as a bare object, not wrapped in an array + // (standard JSONata singleton-sequence unwrapping). + Jsonata expr("$match(\"hello world\", /o w/)", re2RegexEngine()); + auto result = expr.evaluate(nullptr); + ASSERT_TRUE(result.is_object()); + EXPECT_EQ(result["match"], "o w"); + EXPECT_EQ(result["index"], 4); +} + +TEST(RE2EngineTest, MatchCaseInsensitiveFlag) { + Jsonata expr("$match(\"HELLO\", /hello/i)", re2RegexEngine()); + auto result = expr.evaluate(nullptr); + ASSERT_TRUE(result.is_object()); + EXPECT_EQ(result["match"], "HELLO"); +} + +TEST(RE2EngineTest, Contains) { + Jsonata expr("$contains(\"hello\", /ell/)", re2RegexEngine()); + auto result = expr.evaluate(nullptr); + ASSERT_TRUE(result.is_boolean()); + EXPECT_TRUE(result.get()); +} + +TEST(RE2EngineTest, ReplaceWithString) { + Jsonata expr("$replace(\"abc123def\", /[0-9]+/, \"#\")", re2RegexEngine()); + auto result = expr.evaluate(nullptr); + EXPECT_EQ(result, "abc#def"); +} + +TEST(RE2EngineTest, ReplaceWithFunction) { + Jsonata expr( + "$replace(\"abc123\", /[0-9]+/, function($m) { $m.match & \"!\" })", + re2RegexEngine()); + auto result = expr.evaluate(nullptr); + EXPECT_EQ(result, "abc123!"); +} + +TEST(RE2EngineTest, Split) { + Jsonata expr("$split(\"a1b2c3\", /[0-9]/)", re2RegexEngine()); + auto result = expr.evaluate(nullptr); + ASSERT_TRUE(result.is_array()); + std::vector values; + for (const auto& v : result) values.push_back(v.get()); + EXPECT_EQ(values, (std::vector{"a", "b", "c", ""})); +} + +TEST(RE2EngineTest, RejectsBackreferences) { + // Proves RE2 is actually compiling the pattern rather than silently + // falling back to std::regex: backreferences can't run in RE2's + // guaranteed-linear-time engine, so this must fail when the regex + // literal is compiled (during expression construction). + EXPECT_THROW( + { Jsonata expr("$match(\"xaay\", /(a)\\1/)", re2RegexEngine()); }, + std::exception); +} + +TEST(RE2EngineTest, DefaultEngineStillStdRegex) { + // No engine argument -> falls back to the default std::regex-backed + // engine, which *does* support backreferences. Confirms the default + // path is unaffected. + Jsonata expr("$match(\"xaay\", /(a)\\1/)"); + auto result = expr.evaluate(nullptr); + ASSERT_TRUE(result.is_object()); + EXPECT_EQ(result["match"], "aa"); +} From d8a074c7cc787aeab69c083f5f91448ce963a600 Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Fri, 3 Jul 2026 22:15:02 -0700 Subject: [PATCH 2/7] copilot feedback --- include/jsonata/IRegex.h | 10 ++++++++-- include/jsonata/StdRegex.h | 2 +- src/jsonata/Functions.cpp | 12 +++++++----- src/jsonata/Jsonata.cpp | 27 +++++++++++++++------------ src/jsonata/StdRegex.cpp | 20 +++++++++++++++++--- src/jsonata/Tokenizer.cpp | 3 ++- test/RE2Engine.cpp | 27 ++++++++++++--------------- test/RE2Engine.h | 3 ++- 8 files changed, 64 insertions(+), 40 deletions(-) diff --git a/include/jsonata/IRegex.h b/include/jsonata/IRegex.h index 381ab43..19c7682 100644 --- a/include/jsonata/IRegex.h +++ b/include/jsonata/IRegex.h @@ -55,8 +55,14 @@ class IRegex { // True if the pattern matches anywhere in str. Backs $contains. virtual bool test(const std::string& str) const = 0; - // The first match, if any. Backs $replace's first/limited-match paths. - virtual std::optional findFirst(const std::string& str) const = 0; + // The first match starting the search no earlier than byte offset `pos`. + // Backs $replace's first/limited-match paths, and lazy one-at-a-time + // iteration when a regex literal is invoked as a function -- callers + // that only want the first few matches of a large input should call + // this repeatedly (advancing `pos` past each match) rather than use + // findAll, which materializes every match up front. + virtual std::optional findFirst(const std::string& str, + size_t pos = 0) const = 0; // All non-overlapping matches, in order. Backs $match and $contains. virtual std::vector findAll(const std::string& str) const = 0; diff --git a/include/jsonata/StdRegex.h b/include/jsonata/StdRegex.h index bc959ea..6ddf082 100644 --- a/include/jsonata/StdRegex.h +++ b/include/jsonata/StdRegex.h @@ -29,7 +29,7 @@ class StdRegex : public IRegex { StdRegex(const std::string& pattern, RegexFlags flags); bool test(const std::string& str) const override; - std::optional findFirst(const std::string& str) const override; + std::optional findFirst(const std::string& str, size_t pos) const override; std::vector findAll(const std::string& str) const override; std::vector split(const std::string& str) const override; diff --git a/src/jsonata/Functions.cpp b/src/jsonata/Functions.cpp index cef9877..91e860b 100644 --- a/src/jsonata/Functions.cpp +++ b/src/jsonata/Functions.cpp @@ -1095,10 +1095,11 @@ Functions::getFunctionRegistry() { if (isString(args[1])) { auto pattern = std::any_cast(args[1]); auto* jsonataInstance = Jsonata::getCurrentInstance(); - if (!jsonataInstance) return std::any(); + RegexEngine engine = jsonataInstance + ? jsonataInstance->getRegexEngine() + : defaultRegexEngine(); try { - auto regexPattern = - jsonataInstance->getRegexEngine()(pattern, RegexFlags{}); + auto regexPattern = engine(pattern, RegexFlags{}); auto result = match(str, regexPattern, limit); return std::any(result); } catch (const std::exception&) { @@ -2801,9 +2802,10 @@ std::optional Functions::replace(const std::string& str, // safeReplaceFirst repeatedly std::string result = str; auto* jsonataInstance = Jsonata::getCurrentInstance(); + RegexEngine engine = + jsonataInstance ? jsonataInstance->getRegexEngine() : defaultRegexEngine(); try { - if (!jsonataInstance) throw std::runtime_error("no current instance"); - auto regex = jsonataInstance->getRegexEngine()(searchStr, RegexFlags{}); + auto regex = engine(searchStr, RegexFlags{}); for (int64_t i = 0; i < limit; i++) { result = safeReplaceFirst(result, regex, replaceStr); } diff --git a/src/jsonata/Jsonata.cpp b/src/jsonata/Jsonata.cpp index 8ad4074..5aa33ef 100644 --- a/src/jsonata/Jsonata.cpp +++ b/src/jsonata/Jsonata.cpp @@ -2430,7 +2430,7 @@ Jsonata::Jsonata(const std::string& jsonataExpression, RegexEngine regexEngine) expression_ = parser.parse(jsonataExpression, false, regexEngine_); } -Jsonata::Jsonata(const Jsonata& other) { +Jsonata::Jsonata(const Jsonata& other) : regexEngine_(other.regexEngine_) { // Copy constructor matching Java implementation: Jsonata(Jsonata other) // this.ast = other.ast; // this.environment = other.environment; @@ -2438,7 +2438,6 @@ Jsonata::Jsonata(const Jsonata& other) { expression_ = other.expression_; environment_ = other.environment_; timestamp_ = other.timestamp_; - regexEngine_ = other.regexEngine_; } nlohmann::ordered_json Jsonata::evaluate(const nlohmann::ordered_json& input) { @@ -2657,20 +2656,24 @@ std::any Jsonata::apply(const std::any& proc, const Utils::JList& args, struct RegexState { std::shared_ptr str; - std::vector matches; - size_t index = 0; + std::shared_ptr regex; + size_t pos = 0; }; static std::any regexClosure(std::shared_ptr state) { - if (state->index >= state->matches.size()) return std::any{}; - const RegexMatch& match = state->matches[state->index]; - ++(state->index); + // Searches lazily, one match at a time, from state->pos -- unlike + // IRegex::findAll, this never scans further into the string than the + // caller actually consumes via .next(). + auto match = state->regex->findFirst(*state->str, state->pos); + if (!match) return std::any{}; + state->pos = match->position + (match->length == 0 ? 1 : match->length); + nlohmann::ordered_map result; - result["match"] = match.text; - result["start"] = static_cast(match.position); - result["end"] = static_cast(match.position + match.length); + result["match"] = match->text; + result["start"] = static_cast(match->position); + result["end"] = static_cast(match->position + match->length); Utils::JList groups; - groups.push_back(match.text); + groups.push_back(match->text); result["groups"] = std::any(groups); JFunction nextFn; nextFn.implementation = [state](const Utils::JList&, const std::any&, std::shared_ptr) -> std::any { @@ -2712,7 +2715,7 @@ std::any Jsonata::applyInner(const std::any& proc, const Utils::JList& args, if (arg.has_value() && arg.type() == typeid(std::string)) { auto state = std::make_shared(); state->str = std::make_shared(std::any_cast(arg)); - state->matches = regex->findAll(*state->str); + state->regex = regex; results.push_back(regexClosure(state)); } } diff --git a/src/jsonata/StdRegex.cpp b/src/jsonata/StdRegex.cpp index 6d3db92..6c36f7d 100644 --- a/src/jsonata/StdRegex.cpp +++ b/src/jsonata/StdRegex.cpp @@ -57,12 +57,26 @@ bool StdRegex::test(const std::string& str) const { return std::regex_search(str, regex_); } -std::optional StdRegex::findFirst(const std::string& str) const { +std::optional StdRegex::findFirst(const std::string& str, size_t pos) const { + if (pos > str.size()) { + return std::nullopt; + } std::smatch m; - if (!std::regex_search(str, m, regex_)) { + auto begin = str.cbegin() + static_cast(pos); + // When resuming from a non-zero offset, tell std::regex there is a real + // preceding character (match_prev_avail) and that `begin` isn't the true + // start of input (match_not_bol), so ^ and word-boundary constructs are + // evaluated relative to the whole string, not to `pos` -- matching what + // std::sregex_iterator does internally between successive matches. + auto flags = pos == 0 ? std::regex_constants::match_default + : std::regex_constants::match_not_bol | + std::regex_constants::match_prev_avail; + if (!std::regex_search(begin, str.cend(), m, regex_, flags)) { return std::nullopt; } - return toRegexMatch(m); + RegexMatch result = toRegexMatch(m); + result.position += pos; + return result; } std::vector StdRegex::findAll(const std::string& str) const { diff --git a/src/jsonata/Tokenizer.cpp b/src/jsonata/Tokenizer.cpp index 737f077..ad3a116 100644 --- a/src/jsonata/Tokenizer.cpp +++ b/src/jsonata/Tokenizer.cpp @@ -121,6 +121,7 @@ std::shared_ptr Tokenizer::scanRegex() { int32_t currentChar = charAt(position_); if (isClosingSlash(position_)) { // end of regex found + size_t patternStart = start; pattern = substring(start, position_); if (pattern.empty()) { throw JException("S0301", static_cast(position_)); @@ -151,7 +152,7 @@ std::shared_ptr Tokenizer::scanRegex() { try { return regexEngine_(pattern, regexFlags); } catch (const std::exception& e) { - throw JException("S0301", static_cast(start), pattern); + throw JException("S0301", static_cast(patternStart), pattern); } } diff --git a/test/RE2Engine.cpp b/test/RE2Engine.cpp index 1a922ff..7b7d27b 100644 --- a/test/RE2Engine.cpp +++ b/test/RE2Engine.cpp @@ -68,11 +68,17 @@ bool RE2Regex::test(const std::string& str) const { return RE2::PartialMatch(str, re_); } -std::optional RE2Regex::findFirst(const std::string& str) const { +std::optional RE2Regex::findFirst(const std::string& str, size_t pos) const { + if (pos > str.size()) { + return std::nullopt; + } int numGroups = re_.NumberOfCapturingGroups(); std::vector submatch(numGroups + 1); re2::StringPiece input(str); - if (!re_.Match(input, 0, str.size(), RE2::UNANCHORED, submatch.data(), + // RE2::Match takes startpos against the full `input`, so ^ still only + // matches true position 0 -- no special anchoring flags needed, unlike + // std::regex when searching a sub-range. + if (!re_.Match(input, pos, str.size(), RE2::UNANCHORED, submatch.data(), numGroups + 1)) { return std::nullopt; } @@ -81,20 +87,11 @@ std::optional RE2Regex::findFirst(const std::string& str) const { std::vector RE2Regex::findAll(const std::string& str) const { std::vector results; - int numGroups = re_.NumberOfCapturingGroups(); - std::vector submatch(numGroups + 1); - re2::StringPiece input(str); - - size_t startpos = 0; - while (startpos <= str.size()) { - if (!re_.Match(input, startpos, str.size(), RE2::UNANCHORED, - submatch.data(), numGroups + 1)) { - break; - } - RegexMatch match = toRegexMatch(input, submatch, numGroups); + size_t pos = 0; + while (auto match = findFirst(str, pos)) { // Guard against zero-length matches to avoid an infinite loop. - startpos = match.position + (match.length == 0 ? 1 : match.length); - results.push_back(std::move(match)); + pos = match->position + (match->length == 0 ? 1 : match->length); + results.push_back(std::move(*match)); } return results; } diff --git a/test/RE2Engine.h b/test/RE2Engine.h index e84d372..987d829 100644 --- a/test/RE2Engine.h +++ b/test/RE2Engine.h @@ -30,7 +30,8 @@ class RE2Regex : public jsonata::IRegex { RE2Regex(const std::string& pattern, jsonata::RegexFlags flags); bool test(const std::string& str) const override; - std::optional findFirst(const std::string& str) const override; + std::optional findFirst(const std::string& str, + size_t pos) const override; std::vector findAll(const std::string& str) const override; std::vector split(const std::string& str) const override; From 35ece885310f48213d1c69d13155a4ac3f1679ff Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Fri, 3 Jul 2026 22:33:07 -0700 Subject: [PATCH 3/7] copilot feedback --- src/jsonata/Functions.cpp | 31 +++++++++++++++++++++++-------- src/jsonata/utils/Signature.cpp | 4 ++-- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/jsonata/Functions.cpp b/src/jsonata/Functions.cpp index 91e860b..71e80c0 100644 --- a/src/jsonata/Functions.cpp +++ b/src/jsonata/Functions.cpp @@ -2680,10 +2680,12 @@ bool Functions::contains(const std::string& str, const std::any& token) { else if (token.type() == typeid(std::shared_ptr)) { auto regex = std::any_cast>(token); // Java lines 697-701: var matches = evaluateMatcher((Pattern)token, - // str); result = !matches.isEmpty(); + // str); result = !matches.isEmpty(); (IRegex::test is equivalent, + // without materializing the match list) return regex->test(str); } - // Check if it's a regex object (stored as map with "type" = "regex") + // Fallback: token is neither a string nor a regex; stringify it and + // do a literal substring search. else { // Java line 703: else throw new Error("unknown type to match: // "+token); For C++, fall back to string conversion as fallback @@ -5224,19 +5226,22 @@ std::string Functions::safeReplaceAll(const std::string& str, result.reserve(str.size()); std::string prepared = safeReplacement(replacement); + // Stream matches one at a time via findFirst rather than materializing + // all of them up front with findAll -- avoids holding every match in + // memory simultaneously for inputs with many matches. size_t lastEnd = 0; - for (const auto& m : pattern->findAll(str)) { + while (auto m = pattern->findFirst(str, lastEnd)) { // Guard against zero-length matches to avoid infinite loops - if (m.length == 0) { + if (m->length == 0) { // Append the remainder and stop (aligns with override behavior) result.append(str.substr(lastEnd)); return result; } // Text before match - result.append(str.substr(lastEnd, m.position - lastEnd)); - result += expandReplacement(prepared, m); - lastEnd = m.position + m.length; + result.append(str.substr(lastEnd, m->position - lastEnd)); + result += expandReplacement(prepared, *m); + lastEnd = m->position + m->length; } // Trailing remainder @@ -5270,10 +5275,20 @@ std::string Functions::safeReplaceAllFn(const std::string& str, const std::shared_ptr& pattern, const std::any& func) { // Following Java implementation: Functions.java lines 844-859 + // Stream matches one at a time via findFirst rather than materializing + // all of them up front with findAll. size_t lastPos = 0; + size_t searchPos = 0; std::string finalResult; - for (const auto& match : pattern->findAll(str)) { + while (auto matchOpt = pattern->findFirst(str, searchPos)) { + const RegexMatch& match = *matchOpt; + // Advance the search position past zero-length matches (matching + // IRegex::findAll's own behavior) so we don't loop forever + // re-finding the same empty match; `lastPos` (output bookkeeping) + // is left as-is for a zero-length match, unchanged from before. + searchPos = match.position + (match.length == 0 ? 1 : match.length); + try { // Convert match to Jsonata format (equivalent to Java's // toJsonataMatch) diff --git a/src/jsonata/utils/Signature.cpp b/src/jsonata/utils/Signature.cpp index fd985d6..91e79dd 100644 --- a/src/jsonata/utils/Signature.cpp +++ b/src/jsonata/utils/Signature.cpp @@ -134,9 +134,9 @@ std::string Signature::checkObjectType(const std::any &value) { std::any_cast>(value); return "o"; } catch (const std::bad_any_cast &) { - // Try shared_ptr - check if it's a regex object + // Check if it's a regex object try { - auto regex = std::any_cast>(value); + std::any_cast>(value); return "f"; } catch (const std::bad_any_cast &) { return ""; From 78c8e7a1098485cc04dde02b3e36882a1e591505 Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Fri, 3 Jul 2026 23:01:42 -0700 Subject: [PATCH 4/7] copilot feedback --- src/jsonata/Functions.cpp | 3 +-- test/RE2Engine.cpp | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/jsonata/Functions.cpp b/src/jsonata/Functions.cpp index 71e80c0..e517bcd 100644 --- a/src/jsonata/Functions.cpp +++ b/src/jsonata/Functions.cpp @@ -5178,14 +5178,13 @@ std::string Functions::expandReplacement(const std::string& repl, // digits are [i+1, j) int64_t chosenIdx = -1; size_t chosenLen = 0; - for (size_t k = (j - (i + 1)); k >= 1; --k) { + for (size_t k = (j - (i + 1)); k > 0; --k) { int64_t idx = std::stoi(repl.substr(i + 1, k)); if (idx <= maxIndex && idx >= 0) { chosenIdx = idx; chosenLen = k; break; } - if (k == 1) break; // prevent underflow of size_t } if (chosenIdx >= 0) { out += chosenIdx == 0 ? match.text diff --git a/test/RE2Engine.cpp b/test/RE2Engine.cpp index 7b7d27b..a429987 100644 --- a/test/RE2Engine.cpp +++ b/test/RE2Engine.cpp @@ -18,6 +18,7 @@ #include "RE2Engine.h" #include +#include using jsonata::IRegex; using jsonata::RegexEngine; From da814c12183e7be0c84a4a8096581c851faaca96 Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Fri, 3 Jul 2026 23:15:19 -0700 Subject: [PATCH 5/7] copilot feedback --- src/jsonata/Functions.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/jsonata/Functions.cpp b/src/jsonata/Functions.cpp index e517bcd..1e28da4 100644 --- a/src/jsonata/Functions.cpp +++ b/src/jsonata/Functions.cpp @@ -5179,7 +5179,14 @@ std::string Functions::expandReplacement(const std::string& repl, int64_t chosenIdx = -1; size_t chosenLen = 0; for (size_t k = (j - (i + 1)); k > 0; --k) { - int64_t idx = std::stoi(repl.substr(i + 1, k)); + int64_t idx; + try { + idx = std::stoll(repl.substr(i + 1, k)); + } catch (const std::exception&) { + // Too large to represent (e.g. "$999999999999999999999"); + // treat like any other out-of-range group reference. + idx = -1; + } if (idx <= maxIndex && idx >= 0) { chosenIdx = idx; chosenLen = k; From 5ea946e278e5e619e05c90a432b90c4ec0e101ff Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Sat, 4 Jul 2026 10:02:25 -0700 Subject: [PATCH 6/7] copilot feedback --- include/jsonata/IRegex.h | 2 +- test/StringTest.cpp | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/include/jsonata/IRegex.h b/include/jsonata/IRegex.h index 19c7682..fbe5445 100644 --- a/include/jsonata/IRegex.h +++ b/include/jsonata/IRegex.h @@ -64,7 +64,7 @@ class IRegex { virtual std::optional findFirst(const std::string& str, size_t pos = 0) const = 0; - // All non-overlapping matches, in order. Backs $match and $contains. + // All non-overlapping matches, in order. Backs $match. virtual std::vector findAll(const std::string& str) const = 0; // Splits str on every match, returning the segments between matches diff --git a/test/StringTest.cpp b/test/StringTest.cpp index 04958da..67e2e17 100644 --- a/test/StringTest.cpp +++ b/test/StringTest.cpp @@ -175,6 +175,41 @@ TEST_F(StringTest, evalTest) { EXPECT_EQ(result.get(), "AAA"); } +TEST_F(StringTest, evalSeesEnclosingVariableBindingTest) { + // $eval's dynamically-parsed expression must see variables bound in + // the enclosing scope (here, via an in-expression := assignment), not + // just a static top-level environment (regression test). + Jsonata expr("($x := 5; $eval(\"$x + 1\"))"); + auto result = expr.evaluate(nullptr); + ASSERT_TRUE(result.is_number()); + EXPECT_EQ(result.get(), 6); +} + +TEST_F(StringTest, evalSeesExplicitTopLevelBindingsTest) { + // Same as above, but for bindings passed via evaluate()'s bindings + // argument rather than an in-expression assignment. + Jsonata expr("$eval(\"$x\")"); + auto bindingFrame = expr.createFrame(); + bindingFrame->bind("x", int64_t(42)); + auto result = expr.evaluate(nullptr, bindingFrame); + ASSERT_TRUE(result.is_number()); + EXPECT_EQ(result.get(), 42); +} + +TEST_F(StringTest, evalUnaffectedBySiblingArgumentScopeTest) { + // $eval's second (focus) argument is evaluated before its own body + // runs, and here contains a nested block with its own environment. + // Confirms evaluating that sibling argument doesn't leave the tracked + // "current" environment pointing at the inner block's scope, which + // would otherwise cause $eval to resolve $x (from the outer scope) as + // undefined instead of 5. (jsonata-python had this bug; jsonata-cpp + // does not, verified as a regression test.) + Jsonata expr("($x := 5; $eval(\"$x\", (($y := 1; $y))))"); + auto result = expr.evaluate(nullptr); + ASSERT_TRUE(result.is_number()); + EXPECT_EQ(result.get(), 5); +} + TEST_F(StringTest, regexTest) { auto input = makeObject({{"foo", 1}, {"bar", 2}}); auto result = Jsonata("($matcher := $eval('/^' & 'foo' & '/i'); $.$spread()[$.$keys() ~> $matcher])").evaluate(input); From dc1c4ebf69e8aac5859f4574be7c9ad21c8c3089 Mon Sep 17 00:00:00 2001 From: Robert Yokota Date: Sat, 4 Jul 2026 17:11:22 -0700 Subject: [PATCH 7/7] Add timeout and stack guardrails --- README.md | 47 +++++++++++++++++++++++++++++ include/jsonata/Jsonata.h | 27 ++++++++++++----- include/jsonata/Timebox.h | 15 ++++++---- src/jsonata/Functions.cpp | 7 +++-- src/jsonata/JException.cpp | 4 +++ src/jsonata/Jsonata.cpp | 26 +++++++++++----- src/jsonata/Timebox.cpp | 34 +++++++++------------ test/StringTest.cpp | 61 ++++++++++++++++++++++++++++++++++++++ 8 files changed, 179 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 82db649..5d93325 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,53 @@ int main() { } ``` +## Guardrails + +JSONata is Turing-complete, so it's possible to write expressions that loop forever or exhaust memory. If you evaluate +untrusted expressions, configure these guardrails (see the JS reference implementation's +[guardrails docs](https://docs.jsonata.org/guardrails) for more background): + +- **Stack overflow** — the `stack` argument caps the depth of the eval-apply cycle. Exceeding it raises a `JException` + with error `D1011`. +- **Excessive execution time** — the `timeout` argument (in milliseconds) catches tail-recursive infinite loops that + `stack` can't. Exceeding it raises `D1012`. +- **Rogue regular expressions** — the `regexEngine` argument lets you swap in a linear-time engine (e.g. + [RE2](https://github.com/google/re2)) to protect against [ReDoS](https://en.wikipedia.org/wiki/ReDoS), since the + `timeout` guardrail can't interrupt a regex match in progress. + +To swap in a linear-time regex engine, you can wrap RE2 with the `IRegex` interface. +A complete, working example lives in `test/RE2Engine.h`/`.cpp`; the shape is: + +```cpp +#include +#include + +class RE2Regex : public jsonata::IRegex { + public: + RE2Regex(const std::string& pattern, jsonata::RegexFlags flags); + bool test(const std::string& str) const override; + std::optional findFirst(const std::string& str, size_t pos) const override; + std::vector findAll(const std::string& str) const override; + std::vector split(const std::string& str) const override; + private: + RE2 re_; +}; + +jsonata::RegexEngine re2RegexEngine() { + return [](const std::string& pattern, jsonata::RegexFlags flags) { + return std::make_shared(pattern, flags); + }; +} +``` + +```cpp +#include + +jsonata::Jsonata expr("", re2RegexEngine(), + /*timeout=*/1000, /*stack=*/500); +auto result = expr.evaluate(data); +``` + ## Running Tests This project uses the repository of the reference implementation as a submodule. This allows referencing the current version of the unit tests. To clone this repository, run: diff --git a/include/jsonata/Jsonata.h b/include/jsonata/Jsonata.h index 50a30b3..798b958 100644 --- a/include/jsonata/Jsonata.h +++ b/include/jsonata/Jsonata.h @@ -66,8 +66,8 @@ class Frame { std::shared_ptr parent_; nlohmann::ordered_map bindings_; std::chrono::time_point timestamp_; - int64_t timeout_; - int64_t recursionDepth_; + std::optional timeout_; + std::optional recursionDepth_; EntryCallback entryCallback_; ExitCallback exitCallback_; std::unique_ptr timebox_; @@ -84,8 +84,10 @@ class Frame { void bind(const std::string& name, const std::any& value); std::any lookup(const std::string& name) const; - // Runtime bounds - void setRuntimeBounds(int64_t timeout, int64_t maxRecursionDepth); + // Runtime bounds. Either bound may be left unset (std::nullopt) to leave + // that guardrail disabled. + void setRuntimeBounds(std::optional timeout, + std::optional maxRecursionDepth); // Evaluation callbacks void setEvaluateEntryCallback(EntryCallback callback); @@ -125,9 +127,18 @@ class JFunction { class Jsonata { public: // Constructors - explicit Jsonata(RegexEngine regexEngine = defaultRegexEngine()); + // + // timeout/stack configure the optional evaluation guardrails (D1012/D1011): + // timeout is a wall-clock limit in milliseconds, stack limits the depth of + // the eval-apply cycle. Either may be left unset (std::nullopt) to leave + // that guardrail disabled. + explicit Jsonata(RegexEngine regexEngine = defaultRegexEngine(), + std::optional timeout = std::nullopt, + std::optional stack = std::nullopt); Jsonata(const std::string& jsonataExpression, - RegexEngine regexEngine = defaultRegexEngine()); + RegexEngine regexEngine = defaultRegexEngine(), + std::optional timeout = std::nullopt, + std::optional stack = std::nullopt); Jsonata(const Jsonata& other); // Copy constructor for per-thread instances // Main evaluation methods (ordered JSON variants) @@ -158,7 +169,9 @@ class Jsonata { // Factory methods static Jsonata jsonata(const std::string& expression, - RegexEngine regexEngine = defaultRegexEngine()); + RegexEngine regexEngine = defaultRegexEngine(), + std::optional timeout = std::nullopt, + std::optional stack = std::nullopt); // Instance methods (matching Java reference) std::shared_ptr createFrame(); diff --git a/include/jsonata/Timebox.h b/include/jsonata/Timebox.h index 1d16221..9f622df 100644 --- a/include/jsonata/Timebox.h +++ b/include/jsonata/Timebox.h @@ -21,6 +21,7 @@ #include #include #include +#include namespace jsonata { @@ -33,8 +34,8 @@ class Frame; */ class Timebox { private: - int64_t timeout_; // timeout in milliseconds (default: 5000ms) - int64_t maxDepth_; // max recursion depth (default: 100) + std::optional timeout_; // timeout in milliseconds (unset: no limit) + std::optional maxDepth_; // max recursion depth (unset: no limit) std::chrono::steady_clock::time_point startTime_; int64_t depth_; @@ -50,13 +51,15 @@ class Timebox { /** * Protect the process from a runaway expression - * with custom timeout and max depth + * with custom timeout and/or max depth. Either bound may be left unset + * (std::nullopt) to leave that guardrail disabled. */ - Timebox(Frame& expr, int64_t timeout, int64_t maxDepth); + Timebox(Frame& expr, std::optional timeout, + std::optional maxDepth); // Getters - int64_t getTimeout() const { return timeout_; } - int64_t getMaxDepth() const { return maxDepth_; } + std::optional getTimeout() const { return timeout_; } + std::optional getMaxDepth() const { return maxDepth_; } int64_t getCurrentDepth() const { return depth_; } // Runtime check method diff --git a/src/jsonata/Functions.cpp b/src/jsonata/Functions.cpp index 1e28da4..74559c1 100644 --- a/src/jsonata/Functions.cpp +++ b/src/jsonata/Functions.cpp @@ -4938,8 +4938,11 @@ std::any Functions::functionEval(const std::string& expr, // that doesn't get overwritten) auto savedInstance = Jsonata::getCurrentInstance(); - // This creates NEW instance just to parse the expression - Jsonata astInstance(expr); + // This creates NEW instance just to parse the expression. Use the + // enclosing instance's regex engine so regex literals inside the + // eval'd expression are compiled consistently with the rest of the + // enclosing expression (e.g. RE2 instead of std::regex). + Jsonata astInstance(expr, currentInstance->getRegexEngine()); auto expressionAst = astInstance.getExpression(); // Restore the original current instance immediately after parsing diff --git a/src/jsonata/JException.cpp b/src/jsonata/JException.cpp index d44192d..3bfbcb9 100644 --- a/src/jsonata/JException.cpp +++ b/src/jsonata/JException.cpp @@ -129,6 +129,10 @@ std::unordered_map errorCodes = { {"D1002", "Cannot negate a non-numeric value: {{value}}"}, {"D1004", "Regular expression matches zero length string"}, {"D1009", "Multiple key definitions evaluate to same key: {{value}}"}, + {"D1011", + "Stack overflow. Check for non-terminating recursive function. " + "Consider rewriting as tail-recursive"}, + {"D1012", "Evaluation timeout after {{value}} milliseconds. Check for infinite loop"}, {"D2005", "The left side of := must be a variable name (start with $)"}, // defunct // - diff --git a/src/jsonata/Jsonata.cpp b/src/jsonata/Jsonata.cpp index 5aa33ef..8037bc8 100644 --- a/src/jsonata/Jsonata.cpp +++ b/src/jsonata/Jsonata.cpp @@ -75,12 +75,12 @@ void Jsonata::initializeBuiltinFunctions(std::shared_ptr frame) { } // Frame implementation -Frame::Frame() : parent_(nullptr), timeout_(0), recursionDepth_(0) { +Frame::Frame() : parent_(nullptr) { timestamp_ = std::chrono::steady_clock::now(); } Frame::Frame(std::shared_ptr enclosingEnvironment) - : parent_(enclosingEnvironment), timeout_(0), recursionDepth_(0) { + : parent_(enclosingEnvironment) { timestamp_ = std::chrono::steady_clock::now(); } @@ -99,7 +99,8 @@ std::any Frame::lookup(const std::string& name) const { return std::any{}; // null } -void Frame::setRuntimeBounds(int64_t timeout, int64_t maxRecursionDepth) { +void Frame::setRuntimeBounds(std::optional timeout, + std::optional maxRecursionDepth) { timeout_ = timeout; recursionDepth_ = maxRecursionDepth; // Create Timebox to handle recursion depth checking (Java reference logic) @@ -118,12 +119,17 @@ void Frame::setEvaluateExitCallback(ExitCallback callback) { } // Jsonata implementation -Jsonata::Jsonata(RegexEngine regexEngine) : regexEngine_(regexEngine) { +Jsonata::Jsonata(RegexEngine regexEngine, std::optional timeout, + std::optional stack) + : regexEngine_(regexEngine) { parser_ = std::make_unique(); currentInstance_ = this; // Initialize environment like Java does: environment = // createFrame(staticFrame) environment_ = createFrame(getStaticFrame()); + if (timeout.has_value() || stack.has_value()) { + environment_->setRuntimeBounds(timeout, stack); + } initializeErrorCodes(); } @@ -137,8 +143,10 @@ std::shared_ptr Jsonata::createFrame( return std::make_shared(enclosingEnvironment); } -Jsonata Jsonata::jsonata(const std::string& expression, RegexEngine regexEngine) { - Jsonata instance(expression, regexEngine); +Jsonata Jsonata::jsonata(const std::string& expression, RegexEngine regexEngine, + std::optional timeout, + std::optional stack) { + Jsonata instance(expression, regexEngine, timeout, stack); // Parse and store the expression return instance; } @@ -2418,12 +2426,16 @@ Jsonata* Jsonata::getPerThreadInstance() { tls_environment_.reset(); } -Jsonata::Jsonata(const std::string& jsonataExpression, RegexEngine regexEngine) +Jsonata::Jsonata(const std::string& jsonataExpression, RegexEngine regexEngine, + std::optional timeout, std::optional stack) : regexEngine_(regexEngine) { currentInstance_ = this; // Initialize environment like Java does: environment = // createFrame(staticFrame) environment_ = createFrame(getStaticFrame()); + if (timeout.has_value() || stack.has_value()) { + environment_->setRuntimeBounds(timeout, stack); + } // Parse the expression Parser parser; diff --git a/src/jsonata/Timebox.cpp b/src/jsonata/Timebox.cpp index cdd2ad1..5310398 100644 --- a/src/jsonata/Timebox.cpp +++ b/src/jsonata/Timebox.cpp @@ -18,8 +18,6 @@ */ #include "jsonata/Timebox.h" -#include - #include "jsonata/JException.h" #include "jsonata/Jsonata.h" #include "jsonata/Parser.h" @@ -28,7 +26,8 @@ namespace jsonata { Timebox::Timebox(Frame& expr) : Timebox(expr, 5000L, 100) {} -Timebox::Timebox(Frame& expr, int64_t timeout, int64_t maxDepth) +Timebox::Timebox(Frame& expr, std::optional timeout, + std::optional maxDepth) : timeout_(timeout), maxDepth_(maxDepth), depth_(0) { startTime_ = std::chrono::steady_clock::now(); initialize(expr); @@ -72,26 +71,21 @@ void Timebox::onEvaluateExit() { } void Timebox::checkRunnaway() { - // Check stack depth (Java reference: Timebox.java lines 64-70) - if (depth_ > maxDepth_) { - std::ostringstream oss; - oss << "Stack overflow error: Check for non-terminating recursive " - "function. " - << "Consider rewriting as tail-recursive. Depth=" << depth_ - << " max=" << maxDepth_; - // Java reference expects "U1001" error code for recursion limit - throw JException("U1001", -1); + // Check stack depth + if (maxDepth_.has_value() && depth_ > *maxDepth_) { + throw JException("D1011", -1); } - // Check timeout (Java reference: Timebox.java lines 71-78) - auto currentTime = std::chrono::steady_clock::now(); - auto elapsed = std::chrono::duration_cast( - currentTime - startTime_) - .count(); + // Check timeout + if (timeout_.has_value()) { + auto currentTime = std::chrono::steady_clock::now(); + auto elapsed = std::chrono::duration_cast( + currentTime - startTime_) + .count(); - if (elapsed > timeout_) { - // Java reference expects "U1001" error code for timeout as well - throw JException("U1001", -1); + if (elapsed > *timeout_) { + throw JException("D1012", -1, *timeout_); + } } } diff --git a/test/StringTest.cpp b/test/StringTest.cpp index 67e2e17..d34997a 100644 --- a/test/StringTest.cpp +++ b/test/StringTest.cpp @@ -210,6 +210,67 @@ TEST_F(StringTest, evalUnaffectedBySiblingArgumentScopeTest) { EXPECT_EQ(result.get(), 5); } +TEST_F(StringTest, stackGuardrailStopsNonTailRecursionTest) { + // Ackermann is not tail-recursive, so its eval-apply depth grows with + // each call. A small `stack` bound should trip before it completes. + std::string ackExpr = + "($ack := function($m, $n) { $m = 0 ? $n + 1 : ($n = 0 ? " + "$ack($m - 1, 1) : $ack($m - 1, $ack($m, $n - 1))) }; $ack(4, 3))"; + Jsonata expr(ackExpr, defaultRegexEngine(), std::nullopt, 50); + try { + expr.evaluate(nullptr); + FAIL() << "Expected JException D1011"; + } catch (const JException& e) { + EXPECT_EQ(e.getError(), "D1011"); + } +} + +TEST_F(StringTest, stackGuardrailAllowsBoundedRecursionTest) { + // A stack bound that's high enough for the actual recursion depth + // should not interfere with a normal, terminating evaluation. + std::string ackExpr = + "($ack := function($m, $n) { $m = 0 ? $n + 1 : ($n = 0 ? " + "$ack($m - 1, 1) : $ack($m - 1, $ack($m, $n - 1))) }; $ack(3, 4))"; + Jsonata expr(ackExpr, defaultRegexEngine(), std::nullopt, 1000); + auto result = expr.evaluate(nullptr); + ASSERT_TRUE(result.is_number()); + EXPECT_EQ(result.get(), 125); +} + +TEST_F(StringTest, timeoutGuardrailStopsInfiniteTailRecursionTest) { + // Tail recursion doesn't grow the eval-apply depth, so only the + // `timeout` guardrail (not `stack`) can catch this infinite loop. + Jsonata expr("($f := function($n) { $f($n + 1) }; $f(0))", + defaultRegexEngine(), 100, std::nullopt); + try { + expr.evaluate(nullptr); + FAIL() << "Expected JException D1012"; + } catch (const JException& e) { + EXPECT_EQ(e.getError(), "D1012"); + } +} + +TEST_F(StringTest, stackGuardrailPropagatesThroughEvalTest) { + // $eval() reuses the enclosing instance's environment directly, so a + // `stack` bound configured on the outer expression must also apply to + // dynamically-evaluated code (regression test for guardrail bypass). + std::string ackExpr = + "($ack := function($m, $n) { $m = 0 ? $n + 1 : ($n = 0 ? " + "$ack($m - 1, 1) : $ack($m - 1, $ack($m, $n - 1))) }; " + "$ack(4, 3))"; + Jsonata expr("$eval(\"" + ackExpr + "\")", defaultRegexEngine(), + std::nullopt, 50); + // Unlike jsonata-java (which wraps every $eval-time error, guardrail or + // not, into a generic D3121), jsonata-cpp's functionEval re-throws + // JException as-is, so the original D1011 code survives here. + try { + expr.evaluate(nullptr); + FAIL() << "Expected JException D1011"; + } catch (const JException& e) { + EXPECT_EQ(e.getError(), "D1011"); + } +} + TEST_F(StringTest, regexTest) { auto input = makeObject({{"foo", 1}, {"bar", 2}}); auto result = Jsonata("($matcher := $eval('/^' & 'foo' & '/i'); $.$spread()[$.$keys() ~> $matcher])").evaluate(input);