diff --git a/CMakeLists.txt b/CMakeLists.txt index 1032787..ac40e2b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -201,6 +201,45 @@ if(JSONATA_BUILD_TESTS) endif() endif() endif() + + # Optional: demonstrates that jsonata-cpp's pluggable regex engine + # (IRegex, see include/jsonata/IRegex.h) can be backed by Google's RE2 + # instead of std::regex. Off by default. RE2 must already be available + # via find_package (e.g. installed through vcpkg/conan/brew) -- it is + # not fetched automatically like nlohmann_json, since current RE2 + # releases also require Abseil, making a FetchContent fallback a much + # heavier addition than an optional test warrants. + option(JSONATA_BUILD_RE2_TEST "Build the optional RE2 regex engine test (requires re2 to be installed)" OFF) + + if(JSONATA_BUILD_RE2_TEST) + find_package(re2 CONFIG REQUIRED) + + add_executable(jsonata_re2_test test/RE2Engine.cpp test/RE2EngineTest.cpp) + target_link_libraries(jsonata_re2_test + jsonata + re2::re2 + gtest_main + gtest + Threads::Threads + ) + # re2::re2 pulls in Abseil, and Abseil's ABI (e.g. absl::string_view's + # layout) depends on which copy of its headers get resolved first. + # Other PUBLIC dependencies (e.g. a system-wide nlohmann_json found + # via find_package) can inject unrelated include directories that + # happen to also contain a differently-configured Abseil, silently + # shadowing the one re2 was actually built against and causing link + # errors. Force re2/Abseil's own include dirs to be searched first. + target_include_directories(jsonata_re2_test BEFORE PRIVATE + $ + ) + + if(TARGET gtest) + include(GoogleTest) + gtest_discover_tests(jsonata_re2_test + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + ) + endif() + endif() endif() # Installation rules diff --git a/README.md b/README.md index 82db649..5d93325 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,53 @@ int main() { } ``` +## Guardrails + +JSONata is Turing-complete, so it's possible to write expressions that loop forever or exhaust memory. If you evaluate +untrusted expressions, configure these guardrails (see the JS reference implementation's +[guardrails docs](https://docs.jsonata.org/guardrails) for more background): + +- **Stack overflow** — the `stack` argument caps the depth of the eval-apply cycle. Exceeding it raises a `JException` + with error `D1011`. +- **Excessive execution time** — the `timeout` argument (in milliseconds) catches tail-recursive infinite loops that + `stack` can't. Exceeding it raises `D1012`. +- **Rogue regular expressions** — the `regexEngine` argument lets you swap in a linear-time engine (e.g. + [RE2](https://github.com/google/re2)) to protect against [ReDoS](https://en.wikipedia.org/wiki/ReDoS), since the + `timeout` guardrail can't interrupt a regex match in progress. + +To swap in a linear-time regex engine, you can wrap RE2 with the `IRegex` interface. +A complete, working example lives in `test/RE2Engine.h`/`.cpp`; the shape is: + +```cpp +#include +#include + +class RE2Regex : public jsonata::IRegex { + public: + RE2Regex(const std::string& pattern, jsonata::RegexFlags flags); + bool test(const std::string& str) const override; + std::optional findFirst(const std::string& str, size_t pos) const override; + std::vector findAll(const std::string& str) const override; + std::vector split(const std::string& str) const override; + private: + RE2 re_; +}; + +jsonata::RegexEngine re2RegexEngine() { + return [](const std::string& pattern, jsonata::RegexFlags flags) { + return std::make_shared(pattern, flags); + }; +} +``` + +```cpp +#include + +jsonata::Jsonata expr("", re2RegexEngine(), + /*timeout=*/1000, /*stack=*/500); +auto result = expr.evaluate(data); +``` + ## Running Tests This project uses the repository of the reference implementation as a submodule. This allows referencing the current version of the unit tests. To clone this repository, run: diff --git a/include/jsonata/Functions.h b/include/jsonata/Functions.h index 3ed243f..cbfb229 100644 --- a/include/jsonata/Functions.h +++ b/include/jsonata/Functions.h @@ -27,6 +27,7 @@ #include #include +#include "jsonata/IRegex.h" #include "Utils.h" namespace jsonata { @@ -66,7 +67,8 @@ class Functions { static std::optional trim(const std::string& str); static std::any split(const std::string& str, const std::string& separator, int64_t limit = -1); - static std::any split(const std::string& str, const std::regex& pattern, + static std::any split(const std::string& str, + const std::shared_ptr& pattern, int64_t limit = -1); static std::optional join(const Utils::JList& arr, const std::string& separator = ""); @@ -173,9 +175,10 @@ class Functions { static int64_t millis(); // Regex functions - static Utils::JList evaluateMatcher(const std::regex& pattern, + static Utils::JList evaluateMatcher(const std::shared_ptr& pattern, const std::string& str); - static Utils::JList match(const std::string& str, const std::regex& pattern, + static Utils::JList match(const std::string& str, + const std::shared_ptr& pattern, int64_t limit = -1); // Lambda detection @@ -225,16 +228,18 @@ class Functions { static bool isNumericString(const std::string& str); static std::string safeReplacement(const std::string& replacement); static std::string safeReplaceAll(const std::string& str, - const std::regex& pattern, + const std::shared_ptr& pattern, const std::string& replacement); static std::string safeReplaceFirst(const std::string& str, - const std::regex& pattern, + const std::shared_ptr& pattern, const std::string& replacement); static std::string safeReplaceAllFn(const std::string& str, - const std::regex& pattern, + const std::shared_ptr& pattern, const std::any& func); + static std::string expandReplacement(const std::string& replacement, + const RegexMatch& match); static nlohmann::ordered_map toJsonataMatch( - const std::smatch& match); + const RegexMatch& match); static std::string encodeURI(const std::string& uri); static std::string leftPad(const std::string& str, int64_t size, const std::string& padStr = " "); diff --git a/include/jsonata/IRegex.h b/include/jsonata/IRegex.h new file mode 100644 index 0000000..fbe5445 --- /dev/null +++ b/include/jsonata/IRegex.h @@ -0,0 +1,84 @@ +/** + * jsonata-cpp is the JSONata C++ reference port + * + * Copyright Robert Yokota + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include +#include +#include +#include +#include + +namespace jsonata { + +// A single match, engine-agnostic (the equivalent of std::smatch, but not +// tied to std::regex so alternative engines such as RE2 can produce it too). +struct RegexMatch { + std::string text; + size_t position = 0; + size_t length = 0; + // groups[0] is capture group 1, matching std::smatch's [1] indexing + // offset by the implicit full-match group 0. + std::vector> groups; +}; + +// Flags parsed from a JSONata regex literal's /pattern/flags suffix. Engines +// translate these into their own native flag representation. +struct RegexFlags { + bool caseInsensitive = false; + bool multiline = false; +}; + +// Engine-agnostic compiled regex. JSONata regex literals (and patterns given +// as plain strings to functions like $match/$replace/$split) are compiled +// into this interface, so the evaluator never depends on std::regex +// directly. The default engine wraps std::regex (see StdRegex.h); a custom +// engine (e.g. RE2) can be injected via Jsonata's constructor. +class IRegex { + public: + virtual ~IRegex() = default; + + // True if the pattern matches anywhere in str. Backs $contains. + virtual bool test(const std::string& str) const = 0; + + // The first match starting the search no earlier than byte offset `pos`. + // Backs $replace's first/limited-match paths, and lazy one-at-a-time + // iteration when a regex literal is invoked as a function -- callers + // that only want the first few matches of a large input should call + // this repeatedly (advancing `pos` past each match) rather than use + // findAll, which materializes every match up front. + virtual std::optional findFirst(const std::string& str, + size_t pos = 0) const = 0; + + // All non-overlapping matches, in order. Backs $match. + virtual std::vector findAll(const std::string& str) const = 0; + + // Splits str on every match, returning the segments between matches + // (including empty leading/trailing segments). Backs $split. + virtual std::vector split(const std::string& str) const = 0; +}; + +// Compiles a pattern into an IRegex. Used both for JSONata regex literals +// and for patterns supplied as plain strings at runtime. +using RegexEngine = std::function( + const std::string& pattern, RegexFlags flags)>; + +// The built-in std::regex-backed engine; this is jsonata-cpp's default and +// preserves its pre-existing regex behavior exactly. +RegexEngine defaultRegexEngine(); + +} // namespace jsonata diff --git a/include/jsonata/Jsonata.h b/include/jsonata/Jsonata.h index 68db659..798b958 100644 --- a/include/jsonata/Jsonata.h +++ b/include/jsonata/Jsonata.h @@ -34,6 +34,7 @@ #include #include +#include "jsonata/IRegex.h" #include "jsonata/Parser.h" // Forward declarations @@ -65,8 +66,8 @@ class Frame { std::shared_ptr parent_; nlohmann::ordered_map bindings_; std::chrono::time_point timestamp_; - int64_t timeout_; - int64_t recursionDepth_; + std::optional timeout_; + std::optional recursionDepth_; EntryCallback entryCallback_; ExitCallback exitCallback_; std::unique_ptr timebox_; @@ -83,8 +84,10 @@ class Frame { void bind(const std::string& name, const std::any& value); std::any lookup(const std::string& name) const; - // Runtime bounds - void setRuntimeBounds(int64_t timeout, int64_t maxRecursionDepth); + // Runtime bounds. Either bound may be left unset (std::nullopt) to leave + // that guardrail disabled. + void setRuntimeBounds(std::optional timeout, + std::optional maxRecursionDepth); // Evaluation callbacks void setEvaluateEntryCallback(EntryCallback callback); @@ -124,8 +127,18 @@ class JFunction { class Jsonata { public: // Constructors - Jsonata(); - Jsonata(const std::string& jsonataExpression); + // + // timeout/stack configure the optional evaluation guardrails (D1012/D1011): + // timeout is a wall-clock limit in milliseconds, stack limits the depth of + // the eval-apply cycle. Either may be left unset (std::nullopt) to leave + // that guardrail disabled. + explicit Jsonata(RegexEngine regexEngine = defaultRegexEngine(), + std::optional timeout = std::nullopt, + std::optional stack = std::nullopt); + Jsonata(const std::string& jsonataExpression, + RegexEngine regexEngine = defaultRegexEngine(), + std::optional timeout = std::nullopt, + std::optional stack = std::nullopt); Jsonata(const Jsonata& other); // Copy constructor for per-thread instances // Main evaluation methods (ordered JSON variants) @@ -150,8 +163,15 @@ class Jsonata { // Environment access std::shared_ptr getEnvironment() const; + // Regex engine used to compile JSONata regex literals and dynamic + // string patterns (e.g. for $match/$replace/$split). + RegexEngine getRegexEngine() const { return regexEngine_; } + // Factory methods - static Jsonata jsonata(const std::string& expression); + static Jsonata jsonata(const std::string& expression, + RegexEngine regexEngine = defaultRegexEngine(), + std::optional timeout = std::nullopt, + std::optional stack = std::nullopt); // Instance methods (matching Java reference) std::shared_ptr createFrame(); @@ -193,6 +213,7 @@ class Jsonata { static void clearPerThreadInstance(); std::shared_ptr environment_; + RegexEngine regexEngine_ = defaultRegexEngine(); static thread_local std::shared_ptr currentParser_; static thread_local std::any tls_input_; static thread_local std::shared_ptr tls_environment_; diff --git a/include/jsonata/Parser.h b/include/jsonata/Parser.h index 32716ad..fc77c1d 100644 --- a/include/jsonata/Parser.h +++ b/include/jsonata/Parser.h @@ -369,7 +369,8 @@ class Parser { // Main parsing method std::shared_ptr parse(const std::string& source, - bool recover = false); + bool recover = false, + RegexEngine regexEngine = defaultRegexEngine()); // Get parsing errors const std::vector>& getErrors() const { diff --git a/include/jsonata/StdRegex.h b/include/jsonata/StdRegex.h new file mode 100644 index 0000000..6ddf082 --- /dev/null +++ b/include/jsonata/StdRegex.h @@ -0,0 +1,41 @@ +/** + * jsonata-cpp is the JSONata C++ reference port + * + * Copyright Robert Yokota + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +#include "jsonata/IRegex.h" + +namespace jsonata { + +// Default IRegex implementation, backed by std::regex (ECMAScript grammar). +class StdRegex : public IRegex { + public: + StdRegex(const std::string& pattern, RegexFlags flags); + + bool test(const std::string& str) const override; + std::optional findFirst(const std::string& str, size_t pos) const override; + std::vector findAll(const std::string& str) const override; + std::vector split(const std::string& str) const override; + + private: + std::regex regex_; + static RegexMatch toRegexMatch(const std::smatch& m); +}; + +} // namespace jsonata diff --git a/include/jsonata/Timebox.h b/include/jsonata/Timebox.h index 1d16221..9f622df 100644 --- a/include/jsonata/Timebox.h +++ b/include/jsonata/Timebox.h @@ -21,6 +21,7 @@ #include #include #include +#include namespace jsonata { @@ -33,8 +34,8 @@ class Frame; */ class Timebox { private: - int64_t timeout_; // timeout in milliseconds (default: 5000ms) - int64_t maxDepth_; // max recursion depth (default: 100) + std::optional timeout_; // timeout in milliseconds (unset: no limit) + std::optional maxDepth_; // max recursion depth (unset: no limit) std::chrono::steady_clock::time_point startTime_; int64_t depth_; @@ -50,13 +51,15 @@ class Timebox { /** * Protect the process from a runaway expression - * with custom timeout and max depth + * with custom timeout and/or max depth. Either bound may be left unset + * (std::nullopt) to leave that guardrail disabled. */ - Timebox(Frame& expr, int64_t timeout, int64_t maxDepth); + Timebox(Frame& expr, std::optional timeout, + std::optional maxDepth); // Getters - int64_t getTimeout() const { return timeout_; } - int64_t getMaxDepth() const { return maxDepth_; } + std::optional getTimeout() const { return timeout_; } + std::optional getMaxDepth() const { return maxDepth_; } int64_t getCurrentDepth() const { return depth_; } // Runtime check method diff --git a/include/jsonata/Tokenizer.h b/include/jsonata/Tokenizer.h index 351f9c1..9e831ff 100644 --- a/include/jsonata/Tokenizer.h +++ b/include/jsonata/Tokenizer.h @@ -32,6 +32,8 @@ #include #include +#include "jsonata/IRegex.h" + namespace jsonata { class Tokenizer { @@ -63,10 +65,12 @@ class Tokenizer { size_t position_; size_t length_; int64_t depth_; + RegexEngine regexEngine_; public: // Constructor - explicit Tokenizer(const std::string& path); + explicit Tokenizer(const std::string& path, + RegexEngine regexEngine = defaultRegexEngine()); // Main tokenization method std::unique_ptr next(bool prefix = false); @@ -80,7 +84,7 @@ class Tokenizer { std::unique_ptr create(const std::string& type, const std::any& value); bool isClosingSlash(size_t position) const; - std::regex scanRegex(); + std::shared_ptr scanRegex(); // Codepoint access methods (like Java charAt) int32_t charAt(size_t index) const; diff --git a/include/jsonata/utils/Signature.h b/include/jsonata/utils/Signature.h index f9cdefb..b01346b 100644 --- a/include/jsonata/utils/Signature.h +++ b/include/jsonata/utils/Signature.h @@ -48,6 +48,7 @@ #include // Include Utils to resolve JList +#include "../IRegex.h" #include "../Utils.h" namespace jsonata { diff --git a/src/jsonata/Functions.cpp b/src/jsonata/Functions.cpp index 2a6bd98..74559c1 100644 --- a/src/jsonata/Functions.cpp +++ b/src/jsonata/Functions.cpp @@ -628,7 +628,8 @@ std::any Functions::split(const std::string& str, const std::string& separator, return result; } -std::any Functions::split(const std::string& str, const std::regex& pattern, +std::any Functions::split(const std::string& str, + const std::shared_ptr& pattern, int64_t limit) { Utils::JList result; @@ -637,16 +638,10 @@ std::any Functions::split(const std::string& str, const std::regex& pattern, return result; } - // Use std::sregex_token_iterator to split with regex (equivalent to Java's - // Pattern.split with -1) The -1 parameter to sregex_token_iterator gives us - // the non-matching parts (like Java's split with trailing empties - // preserved) - std::sregex_token_iterator iter(str.begin(), str.end(), pattern, -1); - std::sregex_token_iterator end; - - // Java logic: split completely first, then truncate if needed - for (; iter != end; ++iter) { - result.push_back(std::string(*iter)); + // pattern->split gives us the non-matching parts (like Java's split with + // trailing empties preserved, equivalent to Pattern.split with -1) + for (const auto& part : pattern->split(str)) { + result.push_back(part); } // Java reference: if limit is specified and less than result size, truncate @@ -1099,17 +1094,21 @@ Functions::getFunctionRegistry() { // implementation) if (isString(args[1])) { auto pattern = std::any_cast(args[1]); + auto* jsonataInstance = Jsonata::getCurrentInstance(); + RegexEngine engine = jsonataInstance + ? jsonataInstance->getRegexEngine() + : defaultRegexEngine(); try { - std::regex regex_pattern(pattern); - auto result = match(str, regex_pattern, limit); + auto regexPattern = engine(pattern, RegexFlags{}); + auto result = match(str, regexPattern, limit); return std::any(result); - } catch (const std::regex_error&) { + } catch (const std::exception&) { return std::any(); } } else { // Check if it's a regex object try { - auto regex = std::any_cast(args[1]); + auto regex = std::any_cast>(args[1]); auto result = match(str, regex, limit); return std::any(result); } catch (const std::bad_any_cast&) { @@ -1205,7 +1204,7 @@ Functions::getFunctionRegistry() { } else { // Check if it's a regex object try { - auto regex = std::any_cast(args[1]); + auto regex = std::any_cast>(args[1]); return split(str, regex, limit); } catch (const std::bad_any_cast&) { // Not a regex object, check if it's a function @@ -2473,7 +2472,7 @@ void Functions::stringifyInternal(std::ostringstream& os, const std::any& arg, // reference exactly Java: if (arg instanceof Symbol) { return; } - // outputs nothing (empty string) return; - } else if (arg.type() == typeid(std::regex)) { + } else if (arg.type() == typeid(std::shared_ptr)) { return; } else { throw JException("T0411", 0, @@ -2678,14 +2677,15 @@ bool Functions::contains(const std::string& str, const std::any& token) { return str.find(searchStr) != std::string::npos; } // Java lines 696-701: else if (token instanceof Pattern) - else if (token.type() == typeid(std::regex)) { - auto regex = std::any_cast(token); + else if (token.type() == typeid(std::shared_ptr)) { + auto regex = std::any_cast>(token); // Java lines 697-701: var matches = evaluateMatcher((Pattern)token, - // str); result = !matches.isEmpty(); - auto matches = evaluateMatcher(regex, str); - return !matches.empty(); + // str); result = !matches.isEmpty(); (IRegex::test is equivalent, + // without materializing the match list) + return regex->test(str); } - // Check if it's a regex object (stored as map with "type" = "regex") + // Fallback: token is neither a string nor a regex; stringify it and + // do a literal substring search. else { // Java line 703: else throw new Error("unknown type to match: // "+token); For C++, fall back to string conversion as fallback @@ -2712,19 +2712,10 @@ std::optional Functions::replace(const std::string& str, // Check if replacement is a function if (isLambda(replacement)) { // Handle function-based replacement - if (pattern.type() == typeid(std::regex)) { - auto regex = std::any_cast(pattern); + if (pattern.type() == typeid(std::shared_ptr)) { + auto regex = std::any_cast>(pattern); // Use thread-local instance (matching Java implementation) return safeReplaceAllFn(str, regex, replacement); - } else { - // Check if it's a regex object (stored as map with "type" = - // "regex") - try { - auto regex = std::any_cast(pattern); - return safeReplaceAllFn(str, regex, replacement); - } catch (const std::bad_any_cast&) { - // Not a regex object - } } // For string patterns with function replacement, would need more // complex logic @@ -2743,9 +2734,9 @@ std::optional Functions::replace(const std::string& str, replaceStr = *replacementStr; } - // Handle regex patterns (direct std::regex or regex-object map) - if (pattern.type() == typeid(std::regex)) { - std::regex regex = std::any_cast(pattern); + // Handle regex patterns + if (pattern.type() == typeid(std::shared_ptr)) { + auto regex = std::any_cast>(pattern); if (limit == -1) { // No limit specified - replace all occurrences (Java default @@ -2812,13 +2803,16 @@ std::optional Functions::replace(const std::string& str, // String replacement) Compile the string pattern as a regex and apply // safeReplaceFirst repeatedly std::string result = str; + auto* jsonataInstance = Jsonata::getCurrentInstance(); + RegexEngine engine = + jsonataInstance ? jsonataInstance->getRegexEngine() : defaultRegexEngine(); try { - std::regex regex(searchStr); + auto regex = engine(searchStr, RegexFlags{}); for (int64_t i = 0; i < limit; i++) { result = safeReplaceFirst(result, regex, replaceStr); } return result; - } catch (const std::regex_error&) { + } catch (const std::exception&) { // If pattern is not a valid regex, fall back to literal first-only // replacements for (int64_t i = 0; i < limit; i++) { @@ -4944,8 +4938,11 @@ std::any Functions::functionEval(const std::string& expr, // that doesn't get overwritten) auto savedInstance = Jsonata::getCurrentInstance(); - // This creates NEW instance just to parse the expression - Jsonata astInstance(expr); + // This creates NEW instance just to parse the expression. Use the + // enclosing instance's regex engine so regex literals inside the + // eval'd expression are compiled consistently with the rest of the + // enclosing expression (e.g. RE2 instead of std::regex). + Jsonata astInstance(expr, currentInstance->getRegexEngine()); auto expressionAst = astInstance.getExpression(); // Restore the original current instance immediately after parsing @@ -5000,23 +4997,20 @@ int64_t Functions::millis() { .count(); } -Utils::JList Functions::evaluateMatcher(const std::regex& pattern, +Utils::JList Functions::evaluateMatcher(const std::shared_ptr& pattern, const std::string& str) { Utils::JList matches = Utils::createSequence(); - std::sregex_iterator iter(str.begin(), str.end(), pattern); - std::sregex_iterator end; - for (; iter != end; ++iter) { - const std::smatch& smatch = *iter; + for (const auto& regexMatch : pattern->findAll(str)) { nlohmann::ordered_map match; - match["match"] = smatch.str(); - match["index"] = static_cast(smatch.position()); + match["match"] = regexMatch.text; + match["index"] = static_cast(regexMatch.position); // Collect the groups starting from group 1 (excluding full match) - // matching Java implementation Utils::JList groups; - for (size_t g = 1; g < smatch.size(); ++g) { - groups.push_back(smatch[g].str()); + for (const auto& group : regexMatch.groups) { + groups.push_back(group.value_or("")); } match["groups"] = groups; matches.push_back(match); @@ -5025,7 +5019,8 @@ Utils::JList Functions::evaluateMatcher(const std::regex& pattern, return matches; } -Utils::JList Functions::match(const std::string& str, const std::regex& pattern, +Utils::JList Functions::match(const std::string& str, + const std::shared_ptr& pattern, int64_t limit) { auto matches = evaluateMatcher(pattern, str); if (limit > 0 && matches.size() > static_cast(limit)) { @@ -5147,112 +5142,115 @@ std::string Functions::safeReplacement(const std::string& replacement) { return result; } -std::string Functions::safeReplaceAll(const std::string& str, - const std::regex& pattern, - const std::string& replacement) { +std::string Functions::expandReplacement(const std::string& repl, + const RegexMatch& match) { // Manual implementation to match Java semantics for $-expansion and - // handling of ambiguous group references like $18 and $123. - std::string result; - result.reserve(str.size()); + // handling of ambiguous group references like $18 and $123. Shared by + // safeReplaceAll and safeReplaceFirst. + std::string out; + out.reserve(repl.size()); + // max capturing group index (exclude 0, the full match) + const int64_t maxIndex = static_cast(match.groups.size()); + + for (size_t i = 0; i < repl.size(); ++i) { + char c = repl[i]; + if (c != '$') { + out.push_back(c); + continue; + } - std::sregex_iterator it(str.begin(), str.end(), pattern); - std::sregex_iterator end; + if (i + 1 >= repl.size()) { + out.push_back('$'); + continue; + } - size_t lastEnd = 0; - for (; it != end; ++it) { - const std::smatch& m = *it; - // Guard against zero-length matches to avoid infinite loops - if (m.length() == 0) { - // Append the remainder and stop (aligns with override behavior) - result.append(str.substr(lastEnd)); - return result; + char next = repl[i + 1]; + if (next == '$') { + out.push_back('$'); + i += 1; + continue; } - // Text before match - result.append( - str.substr(lastEnd, static_cast(m.position()) - lastEnd)); - - // Expand replacement against this match - auto expand = [&](const std::string& repl, - const std::smatch& match) -> std::string { - std::string out; - out.reserve(repl.size()); - const int64_t maxIndex = static_cast(match.size()) - - 1; // max capturing group index (exclude 0) - - for (size_t i = 0; i < repl.size(); ++i) { - char c = repl[i]; - if (c != '$') { - out.push_back(c); - continue; + if (std::isdigit(static_cast(next))) { + // Parse all following digits to form the number + size_t j = i + 1; + while (j < repl.size() && + std::isdigit(static_cast(repl[j]))) { + ++j; + } + // digits are [i+1, j) + int64_t chosenIdx = -1; + size_t chosenLen = 0; + for (size_t k = (j - (i + 1)); k > 0; --k) { + int64_t idx; + try { + idx = std::stoll(repl.substr(i + 1, k)); + } catch (const std::exception&) { + // Too large to represent (e.g. "$999999999999999999999"); + // treat like any other out-of-range group reference. + idx = -1; } - - if (i + 1 >= repl.size()) { - out.push_back('$'); - continue; + if (idx <= maxIndex && idx >= 0) { + chosenIdx = idx; + chosenLen = k; + break; } - - char next = repl[i + 1]; - if (next == '$') { - out.push_back('$'); - i += 1; - continue; + } + if (chosenIdx >= 0) { + out += chosenIdx == 0 ? match.text + : match.groups[chosenIdx - 1].value_or(""); + // Append leftover digits as literals (e.g., $123 with + // 12 valid -> append '3') + out.append(repl.substr(i + 1 + chosenLen, + j - (i + 1 + chosenLen))); + i = j - 1; + continue; + } else { + // No valid group index; consume first digit + // (interpreted as invalid $) and emit nothing for + // the group, leaving remaining digits literal. + if (j > i + 1) { + // skip first digit + i = i + 1; // loop will i++ again, effectively + // skipping this digit } + // Do not append '$' + continue; + } + } - if (std::isdigit(static_cast(next))) { - // Parse all following digits to form the number - size_t j = i + 1; - while (j < repl.size() && - std::isdigit(static_cast(repl[j]))) { - ++j; - } - // digits are [i+1, j) - int64_t chosenIdx = -1; - size_t chosenLen = 0; - for (size_t k = (j - (i + 1)); k >= 1; --k) { - int64_t idx = std::stoi(repl.substr(i + 1, k)); - if (idx <= maxIndex && idx >= 0) { - chosenIdx = idx; - chosenLen = k; - break; - } - if (k == 1) break; // prevent underflow of size_t - } - if (chosenIdx >= 0) { - out += match.str(chosenIdx); - // Append leftover digits as literals (e.g., $123 with - // 12 valid -> append '3') - out.append(repl.substr(i + 1 + chosenLen, - j - (i + 1 + chosenLen))); - i = j - 1; - continue; - } else { - // No valid group index; consume first digit - // (interpreted as invalid $) and emit nothing for - // the group, leaving remaining digits literal. - if (j > i + 1) { - // skip first digit - i = i + 1; // loop will i++ again, effectively - // skipping this digit - } - // Do not append '$' - continue; - } - } + // '$' followed by non-digit: treat as literal '$' + out.push_back('$'); + // do not consume the next char here; it will be processed in + // next iteration + } - // '$' followed by non-digit: treat as literal '$' - out.push_back('$'); - // do not consume the next char here; it will be processed in - // next iteration - } + return out; +} - return out; - }; +std::string Functions::safeReplaceAll(const std::string& str, + const std::shared_ptr& pattern, + const std::string& replacement) { + std::string result; + result.reserve(str.size()); + std::string prepared = safeReplacement(replacement); - std::string prepared = safeReplacement(replacement); - result += expand(prepared, m); + // Stream matches one at a time via findFirst rather than materializing + // all of them up front with findAll -- avoids holding every match in + // memory simultaneously for inputs with many matches. + size_t lastEnd = 0; + while (auto m = pattern->findFirst(str, lastEnd)) { + // Guard against zero-length matches to avoid infinite loops + if (m->length == 0) { + // Append the remainder and stop (aligns with override behavior) + result.append(str.substr(lastEnd)); + return result; + } - lastEnd = static_cast(m.position() + m.length()); + // Text before match + result.append(str.substr(lastEnd, m->position - lastEnd)); + result += expandReplacement(prepared, *m); + lastEnd = m->position + m->length; } // Trailing remainder @@ -5261,100 +5259,44 @@ std::string Functions::safeReplaceAll(const std::string& str, } std::string Functions::safeReplaceFirst(const std::string& str, - const std::regex& pattern, + const std::shared_ptr& pattern, const std::string& replacement) { - // Manual first-only replace using same expansion as safeReplaceAll - std::smatch m; - if (!std::regex_search(str, m, pattern)) { + auto m = pattern->findFirst(str); + if (!m) { return str; } // Guard against zero-length match - if (m.length() == 0) { + if (m->length == 0) { return str; } - auto expand = [&](const std::string& repl, - const std::smatch& match) -> std::string { - std::string out; - out.reserve(repl.size()); - const int64_t maxIndex = static_cast(match.size()) - - 1; // max capturing group index (exclude 0) - - for (size_t i = 0; i < repl.size(); ++i) { - char c = repl[i]; - if (c != '$') { - out.push_back(c); - continue; - } - if (i + 1 >= repl.size()) { - out.push_back('$'); - continue; - } - char next = repl[i + 1]; - if (next == '$') { - out.push_back('$'); - i += 1; - continue; - } - if (std::isdigit(static_cast(next))) { - size_t j = i + 1; - while (j < repl.size() && - std::isdigit(static_cast(repl[j]))) - ++j; - int64_t chosenIdx = -1; - size_t chosenLen = 0; - for (size_t k = (j - (i + 1)); k >= 1; --k) { - int64_t idx = std::stoi(repl.substr(i + 1, k)); - if (idx <= maxIndex && idx >= 0) { - chosenIdx = idx; - chosenLen = k; - break; - } - if (k == 1) break; - } - if (chosenIdx >= 0) { - out += match.str(chosenIdx); - out.append(repl.substr(i + 1 + chosenLen, - j - (i + 1 + chosenLen))); - i = j - 1; - continue; - } else { - // consume first digit of the invalid group reference and - // emit nothing - if (j > i + 1) { - i = i + 1; - } - continue; - } - } - out.push_back('$'); - } - return out; - }; - std::string prepared = safeReplacement(replacement); std::string res; res.reserve(str.size()); - res.append(str.substr(0, static_cast(m.position()))); - res += expand(prepared, m); - res.append(str.substr(static_cast(m.position() + m.length()))); + res.append(str.substr(0, m->position)); + res += expandReplacement(prepared, *m); + res.append(str.substr(m->position + m->length)); return res; } std::string Functions::safeReplaceAllFn(const std::string& str, - const std::regex& pattern, + const std::shared_ptr& pattern, const std::any& func) { // Following Java implementation: Functions.java lines 844-859 - std::string result = str; - std::sregex_iterator matchIter(str.begin(), str.end(), pattern); - std::sregex_iterator endIter; - + // Stream matches one at a time via findFirst rather than materializing + // all of them up front with findAll. size_t lastPos = 0; + size_t searchPos = 0; std::string finalResult; - for (; matchIter != endIter; ++matchIter) { - const std::smatch& match = *matchIter; + while (auto matchOpt = pattern->findFirst(str, searchPos)) { + const RegexMatch& match = *matchOpt; + // Advance the search position past zero-length matches (matching + // IRegex::findAll's own behavior) so we don't loop forever + // re-finding the same empty match; `lastPos` (output bookkeeping) + // is left as-is for a zero-length match, unchanged from before. + searchPos = match.position + (match.length == 0 ? 1 : match.length); try { // Convert match to Jsonata format (equivalent to Java's @@ -5382,10 +5324,10 @@ std::string Functions::safeReplaceAllFn(const std::string& str, std::any_cast(functionResult); // Add text before match - finalResult += str.substr(lastPos, match.position() - lastPos); + finalResult += str.substr(lastPos, match.position - lastPos); // Add replacement finalResult += replacementStr; - lastPos = match.position() + match.length(); + lastPos = match.position + match.length; } else { // Java line 852: return null for non-string results // This will be caught by the replace function and converted to @@ -5410,16 +5352,16 @@ std::string Functions::safeReplaceAllFn(const std::string& str, } nlohmann::ordered_map Functions::toJsonataMatch( - const std::smatch& match) { + const RegexMatch& match) { nlohmann::ordered_map result; - result["match"] = match.str(); + result["match"] = match.text; // Based on test expectations: groups array starts from first capture group // (excluding full match) This matches the behavior expected by test case034 Utils::JList groups; - for (size_t i = 1; i < match.size(); ++i) { - groups.push_back(match[i].str()); + for (const auto& group : match.groups) { + groups.push_back(group.value_or("")); } result["groups"] = groups; diff --git a/src/jsonata/JException.cpp b/src/jsonata/JException.cpp index d44192d..3bfbcb9 100644 --- a/src/jsonata/JException.cpp +++ b/src/jsonata/JException.cpp @@ -129,6 +129,10 @@ std::unordered_map errorCodes = { {"D1002", "Cannot negate a non-numeric value: {{value}}"}, {"D1004", "Regular expression matches zero length string"}, {"D1009", "Multiple key definitions evaluate to same key: {{value}}"}, + {"D1011", + "Stack overflow. Check for non-terminating recursive function. " + "Consider rewriting as tail-recursive"}, + {"D1012", "Evaluation timeout after {{value}} milliseconds. Check for infinite loop"}, {"D2005", "The left side of := must be a variable name (start with $)"}, // defunct // - diff --git a/src/jsonata/Jsonata.cpp b/src/jsonata/Jsonata.cpp index fae86a0..8037bc8 100644 --- a/src/jsonata/Jsonata.cpp +++ b/src/jsonata/Jsonata.cpp @@ -75,12 +75,12 @@ void Jsonata::initializeBuiltinFunctions(std::shared_ptr frame) { } // Frame implementation -Frame::Frame() : parent_(nullptr), timeout_(0), recursionDepth_(0) { +Frame::Frame() : parent_(nullptr) { timestamp_ = std::chrono::steady_clock::now(); } Frame::Frame(std::shared_ptr enclosingEnvironment) - : parent_(enclosingEnvironment), timeout_(0), recursionDepth_(0) { + : parent_(enclosingEnvironment) { timestamp_ = std::chrono::steady_clock::now(); } @@ -99,7 +99,8 @@ std::any Frame::lookup(const std::string& name) const { return std::any{}; // null } -void Frame::setRuntimeBounds(int64_t timeout, int64_t maxRecursionDepth) { +void Frame::setRuntimeBounds(std::optional timeout, + std::optional maxRecursionDepth) { timeout_ = timeout; recursionDepth_ = maxRecursionDepth; // Create Timebox to handle recursion depth checking (Java reference logic) @@ -118,12 +119,17 @@ void Frame::setEvaluateExitCallback(ExitCallback callback) { } // Jsonata implementation -Jsonata::Jsonata() { +Jsonata::Jsonata(RegexEngine regexEngine, std::optional timeout, + std::optional stack) + : regexEngine_(regexEngine) { parser_ = std::make_unique(); currentInstance_ = this; // Initialize environment like Java does: environment = // createFrame(staticFrame) environment_ = createFrame(getStaticFrame()); + if (timeout.has_value() || stack.has_value()) { + environment_->setRuntimeBounds(timeout, stack); + } initializeErrorCodes(); } @@ -137,14 +143,16 @@ std::shared_ptr Jsonata::createFrame( return std::make_shared(enclosingEnvironment); } -Jsonata Jsonata::jsonata(const std::string& expression) { - Jsonata instance(expression); +Jsonata Jsonata::jsonata(const std::string& expression, RegexEngine regexEngine, + std::optional timeout, + std::optional stack) { + Jsonata instance(expression, regexEngine, timeout, stack); // Parse and store the expression return instance; } std::shared_ptr Jsonata::parse(const std::string& expression) { - return parser_->parse(expression); + return parser_->parse(expression, false, regexEngine_); } std::any Jsonata::evaluate(std::shared_ptr expr, @@ -690,8 +698,8 @@ std::any Jsonata::evaluateRegex(std::shared_ptr expr, } try { - // The tokenizer stores std::regex objects in the value - return std::any_cast(expr->value); + // The tokenizer stores compiled IRegex objects in the value + return std::any_cast>(expr->value); } catch (const std::bad_any_cast&) { throw JException("T0410", expr->position, "Invalid regex value"); } @@ -2418,18 +2426,23 @@ Jsonata* Jsonata::getPerThreadInstance() { tls_environment_.reset(); } -Jsonata::Jsonata(const std::string& jsonataExpression) { +Jsonata::Jsonata(const std::string& jsonataExpression, RegexEngine regexEngine, + std::optional timeout, std::optional stack) + : regexEngine_(regexEngine) { currentInstance_ = this; // Initialize environment like Java does: environment = // createFrame(staticFrame) environment_ = createFrame(getStaticFrame()); + if (timeout.has_value() || stack.has_value()) { + environment_->setRuntimeBounds(timeout, stack); + } // Parse the expression Parser parser; - expression_ = parser.parse(jsonataExpression); + expression_ = parser.parse(jsonataExpression, false, regexEngine_); } -Jsonata::Jsonata(const Jsonata& other) { +Jsonata::Jsonata(const Jsonata& other) : regexEngine_(other.regexEngine_) { // Copy constructor matching Java implementation: Jsonata(Jsonata other) // this.ast = other.ast; // this.environment = other.environment; @@ -2655,21 +2668,24 @@ std::any Jsonata::apply(const std::any& proc, const Utils::JList& args, struct RegexState { std::shared_ptr str; - std::shared_ptr regex; - std::sregex_iterator it; - std::sregex_iterator end; + std::shared_ptr regex; + size_t pos = 0; }; static std::any regexClosure(std::shared_ptr state) { - if (state->it == state->end) return std::any{}; - auto match = *state->it; - ++(state->it); + // Searches lazily, one match at a time, from state->pos -- unlike + // IRegex::findAll, this never scans further into the string than the + // caller actually consumes via .next(). + auto match = state->regex->findFirst(*state->str, state->pos); + if (!match) return std::any{}; + state->pos = match->position + (match->length == 0 ? 1 : match->length); + nlohmann::ordered_map result; - result["match"] = std::string(match.str()); - result["start"] = static_cast(match.position()); - result["end"] = static_cast(match.position() + match.length()); + result["match"] = match->text; + result["start"] = static_cast(match->position); + result["end"] = static_cast(match->position + match->length); Utils::JList groups; - groups.push_back(std::string(match.str())); + groups.push_back(match->text); result["groups"] = std::any(groups); JFunction nextFn; nextFn.implementation = [state](const Utils::JList&, const std::any&, std::shared_ptr) -> std::any { @@ -2703,17 +2719,15 @@ std::any Jsonata::applyInner(const std::any& proc, const Utils::JList& args, // Check if it's a regex pattern (Java lines 1757-1766) // Java: } else if (proc instanceof Pattern) { - if (proc.type() == typeid(std::regex)) { - auto regex = std::any_cast(proc); + if (proc.type() == typeid(std::shared_ptr)) { + auto regex = std::any_cast>(proc); Utils::JList results; for (const auto& arg : validatedArgs) { if (arg.has_value() && arg.type() == typeid(std::string)) { auto state = std::make_shared(); state->str = std::make_shared(std::any_cast(arg)); - state->regex = std::make_shared(regex); - state->it = std::sregex_iterator(state->str->begin(), state->str->end(), *state->regex); - state->end = std::sregex_iterator(); + state->regex = regex; results.push_back(regexClosure(state)); } } @@ -3743,7 +3757,7 @@ bool Jsonata::isFunctionLike(const std::any& o) const { } // Check if it's a regex pattern (C++ equivalent of "o instanceof Pattern") - if (o.has_value() && o.type() == typeid(std::regex)) { + if (o.has_value() && o.type() == typeid(std::shared_ptr)) { return true; } diff --git a/src/jsonata/Parser.cpp b/src/jsonata/Parser.cpp index c85c977..df752c8 100644 --- a/src/jsonata/Parser.cpp +++ b/src/jsonata/Parser.cpp @@ -1308,10 +1308,11 @@ void Parser::registerSymbol(std::shared_ptr symbol) { } std::shared_ptr Parser::parse(const std::string& source, - bool recover) { + bool recover, + RegexEngine regexEngine) { source_ = source; recover_ = recover; - lexer_ = std::make_unique(source); + lexer_ = std::make_unique(source, regexEngine); // Start parsing (Java: advance() defaults to prefix=false) advance(""); diff --git a/src/jsonata/StdRegex.cpp b/src/jsonata/StdRegex.cpp new file mode 100644 index 0000000..6c36f7d --- /dev/null +++ b/src/jsonata/StdRegex.cpp @@ -0,0 +1,108 @@ +/** + * jsonata-cpp is the JSONata C++ reference port + * + * Copyright Robert Yokota + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "jsonata/StdRegex.h" + +namespace jsonata { + +namespace { + +std::regex_constants::syntax_option_type toSyntaxOptions(RegexFlags flags) { + auto options = std::regex_constants::ECMAScript; + if (flags.caseInsensitive) { + options |= std::regex_constants::icase; + } + // Note: the 'm' (multiline) flag is intentionally not translated here, + // preserving jsonata-cpp's pre-existing std::regex behavior exactly. + // RegexFlags.multiline is still populated for custom engines that do + // want to honor it. + return options; +} + +} // namespace + +StdRegex::StdRegex(const std::string& pattern, RegexFlags flags) + : regex_(pattern, toSyntaxOptions(flags)) {} + +RegexMatch StdRegex::toRegexMatch(const std::smatch& m) { + RegexMatch result; + result.text = m.str(); + result.position = static_cast(m.position()); + result.length = static_cast(m.length()); + for (size_t g = 1; g < m.size(); ++g) { + if (m[g].matched) { + result.groups.push_back(m[g].str()); + } else { + result.groups.push_back(std::nullopt); + } + } + return result; +} + +bool StdRegex::test(const std::string& str) const { + return std::regex_search(str, regex_); +} + +std::optional StdRegex::findFirst(const std::string& str, size_t pos) const { + if (pos > str.size()) { + return std::nullopt; + } + std::smatch m; + auto begin = str.cbegin() + static_cast(pos); + // When resuming from a non-zero offset, tell std::regex there is a real + // preceding character (match_prev_avail) and that `begin` isn't the true + // start of input (match_not_bol), so ^ and word-boundary constructs are + // evaluated relative to the whole string, not to `pos` -- matching what + // std::sregex_iterator does internally between successive matches. + auto flags = pos == 0 ? std::regex_constants::match_default + : std::regex_constants::match_not_bol | + std::regex_constants::match_prev_avail; + if (!std::regex_search(begin, str.cend(), m, regex_, flags)) { + return std::nullopt; + } + RegexMatch result = toRegexMatch(m); + result.position += pos; + return result; +} + +std::vector StdRegex::findAll(const std::string& str) const { + std::vector results; + std::sregex_iterator it(str.begin(), str.end(), regex_); + std::sregex_iterator end; + for (; it != end; ++it) { + results.push_back(toRegexMatch(*it)); + } + return results; +} + +std::vector StdRegex::split(const std::string& str) const { + std::vector results; + std::sregex_token_iterator it(str.begin(), str.end(), regex_, -1); + std::sregex_token_iterator end; + for (; it != end; ++it) { + results.push_back(*it); + } + return results; +} + +RegexEngine defaultRegexEngine() { + return [](const std::string& pattern, RegexFlags flags) -> std::shared_ptr { + return std::make_shared(pattern, flags); + }; +} + +} // namespace jsonata diff --git a/src/jsonata/Timebox.cpp b/src/jsonata/Timebox.cpp index cdd2ad1..5310398 100644 --- a/src/jsonata/Timebox.cpp +++ b/src/jsonata/Timebox.cpp @@ -18,8 +18,6 @@ */ #include "jsonata/Timebox.h" -#include - #include "jsonata/JException.h" #include "jsonata/Jsonata.h" #include "jsonata/Parser.h" @@ -28,7 +26,8 @@ namespace jsonata { Timebox::Timebox(Frame& expr) : Timebox(expr, 5000L, 100) {} -Timebox::Timebox(Frame& expr, int64_t timeout, int64_t maxDepth) +Timebox::Timebox(Frame& expr, std::optional timeout, + std::optional maxDepth) : timeout_(timeout), maxDepth_(maxDepth), depth_(0) { startTime_ = std::chrono::steady_clock::now(); initialize(expr); @@ -72,26 +71,21 @@ void Timebox::onEvaluateExit() { } void Timebox::checkRunnaway() { - // Check stack depth (Java reference: Timebox.java lines 64-70) - if (depth_ > maxDepth_) { - std::ostringstream oss; - oss << "Stack overflow error: Check for non-terminating recursive " - "function. " - << "Consider rewriting as tail-recursive. Depth=" << depth_ - << " max=" << maxDepth_; - // Java reference expects "U1001" error code for recursion limit - throw JException("U1001", -1); + // Check stack depth + if (maxDepth_.has_value() && depth_ > *maxDepth_) { + throw JException("D1011", -1); } - // Check timeout (Java reference: Timebox.java lines 71-78) - auto currentTime = std::chrono::steady_clock::now(); - auto elapsed = std::chrono::duration_cast( - currentTime - startTime_) - .count(); + // Check timeout + if (timeout_.has_value()) { + auto currentTime = std::chrono::steady_clock::now(); + auto elapsed = std::chrono::duration_cast( + currentTime - startTime_) + .count(); - if (elapsed > timeout_) { - // Java reference expects "U1001" error code for timeout as well - throw JException("U1001", -1); + if (elapsed > *timeout_) { + throw JException("D1012", -1, *timeout_); + } } } diff --git a/src/jsonata/Tokenizer.cpp b/src/jsonata/Tokenizer.cpp index 3aec3e9..ad3a116 100644 --- a/src/jsonata/Tokenizer.cpp +++ b/src/jsonata/Tokenizer.cpp @@ -53,8 +53,8 @@ std::unordered_map Tokenizer::createEscapes() { {"f", "\f"}, {"n", "\n"}, {"r", "\r"}, {"t", "\t"}}; } -Tokenizer::Tokenizer(const std::string& path) - : path_(path), position_(0), depth_(0) { +Tokenizer::Tokenizer(const std::string& path, RegexEngine regexEngine) + : path_(path), position_(0), depth_(0), regexEngine_(regexEngine) { // Pre-compute codepoints and byte offsets for O(1) charAt access auto it = path_.begin(); while (it != path_.end()) { @@ -109,7 +109,7 @@ bool Tokenizer::isClosingSlash(size_t position) const { return false; } -std::regex Tokenizer::scanRegex() { +std::shared_ptr Tokenizer::scanRegex() { // The prefix '/' will have been previously scanned. Find the end of the // regex. Search for closing '/' ignoring any that are escaped, or within // brackets (matches Java logic exactly) @@ -121,6 +121,7 @@ std::regex Tokenizer::scanRegex() { int32_t currentChar = charAt(position_); if (isClosingSlash(position_)) { // end of regex found + size_t patternStart = start; pattern = substring(start, position_); if (pattern.empty()) { throw JException("S0301", static_cast(position_)); @@ -140,17 +141,18 @@ std::regex Tokenizer::scanRegex() { } flags = substring(start, position_); - // Convert flags to std::regex_constants - std::regex_constants::syntax_option_type regexFlags = - std::regex_constants::ECMAScript; + RegexFlags regexFlags; if (flags.find('i') != std::string::npos) { - regexFlags |= std::regex_constants::icase; + regexFlags.caseInsensitive = true; + } + if (flags.find('m') != std::string::npos) { + regexFlags.multiline = true; } try { - return std::regex(pattern, regexFlags); - } catch (const std::regex_error& e) { - throw JException("S0301", static_cast(start), pattern); + return regexEngine_(pattern, regexFlags); + } catch (const std::exception& e) { + throw JException("S0301", static_cast(patternStart), pattern); } } diff --git a/src/jsonata/utils/Signature.cpp b/src/jsonata/utils/Signature.cpp index 711c0d8..91e79dd 100644 --- a/src/jsonata/utils/Signature.cpp +++ b/src/jsonata/utils/Signature.cpp @@ -134,9 +134,9 @@ std::string Signature::checkObjectType(const std::any &value) { std::any_cast>(value); return "o"; } catch (const std::bad_any_cast &) { - // Try shared_ptr - check if it's a regex object + // Check if it's a regex object try { - auto regex = std::any_cast(value); + std::any_cast>(value); return "f"; } catch (const std::bad_any_cast &) { return ""; diff --git a/test/RE2Engine.cpp b/test/RE2Engine.cpp new file mode 100644 index 0000000..a429987 --- /dev/null +++ b/test/RE2Engine.cpp @@ -0,0 +1,115 @@ +/** + * jsonata-cpp is the JSONata C++ reference port + * + * Copyright Robert Yokota + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "RE2Engine.h" + +#include +#include + +using jsonata::IRegex; +using jsonata::RegexEngine; +using jsonata::RegexFlags; +using jsonata::RegexMatch; + +namespace { + +RE2::Options toRE2Options(RegexFlags flags) { + RE2::Options options; + options.set_case_sensitive(!flags.caseInsensitive); + // RE2's one_line defaults to false (^/$ match at line boundaries); set it + // true when the JSONata pattern did NOT request the 'm' flag, so ^/$ + // only match the start/end of the whole string. + options.set_one_line(!flags.multiline); + options.set_log_errors(false); + return options; +} + +} // namespace + +RE2Regex::RE2Regex(const std::string& pattern, RegexFlags flags) + : re_(pattern, toRE2Options(flags)) { + if (!re_.ok()) { + throw std::runtime_error("RE2 compile error: " + re_.error()); + } +} + +RegexMatch RE2Regex::toRegexMatch(const re2::StringPiece& input, + const std::vector& submatch, + int numGroups) { + RegexMatch result; + result.text.assign(submatch[0].data(), submatch[0].size()); + result.position = static_cast(submatch[0].data() - input.data()); + result.length = submatch[0].size(); + for (int g = 1; g <= numGroups; ++g) { + if (submatch[g].data() == nullptr) { + result.groups.push_back(std::nullopt); + } else { + result.groups.push_back( + std::string(submatch[g].data(), submatch[g].size())); + } + } + return result; +} + +bool RE2Regex::test(const std::string& str) const { + return RE2::PartialMatch(str, re_); +} + +std::optional RE2Regex::findFirst(const std::string& str, size_t pos) const { + if (pos > str.size()) { + return std::nullopt; + } + int numGroups = re_.NumberOfCapturingGroups(); + std::vector submatch(numGroups + 1); + re2::StringPiece input(str); + // RE2::Match takes startpos against the full `input`, so ^ still only + // matches true position 0 -- no special anchoring flags needed, unlike + // std::regex when searching a sub-range. + if (!re_.Match(input, pos, str.size(), RE2::UNANCHORED, submatch.data(), + numGroups + 1)) { + return std::nullopt; + } + return toRegexMatch(input, submatch, numGroups); +} + +std::vector RE2Regex::findAll(const std::string& str) const { + std::vector results; + size_t pos = 0; + while (auto match = findFirst(str, pos)) { + // Guard against zero-length matches to avoid an infinite loop. + pos = match->position + (match->length == 0 ? 1 : match->length); + results.push_back(std::move(*match)); + } + return results; +} + +std::vector RE2Regex::split(const std::string& str) const { + std::vector result; + size_t lastEnd = 0; + for (const auto& match : findAll(str)) { + result.push_back(str.substr(lastEnd, match.position - lastEnd)); + lastEnd = match.position + match.length; + } + result.push_back(str.substr(lastEnd)); + return result; +} + +RegexEngine re2RegexEngine() { + return [](const std::string& pattern, RegexFlags flags) -> std::shared_ptr { + return std::make_shared(pattern, flags); + }; +} diff --git a/test/RE2Engine.h b/test/RE2Engine.h new file mode 100644 index 0000000..987d829 --- /dev/null +++ b/test/RE2Engine.h @@ -0,0 +1,47 @@ +/** + * jsonata-cpp is the JSONata C++ reference port + * + * Copyright Robert Yokota + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#pragma once + +#include + +#include "jsonata/IRegex.h" + +// Demonstration IRegex implementation backed by Google's RE2, showing that +// jsonata-cpp's regex engine hook (see IRegex.h/StdRegex.h) is not tied to +// std::regex. Lives under test/ since it's only built for the optional RE2 +// test target (JSONATA_BUILD_RE2_TEST) and isn't part of the jsonata library. +class RE2Regex : public jsonata::IRegex { + public: + RE2Regex(const std::string& pattern, jsonata::RegexFlags flags); + + bool test(const std::string& str) const override; + std::optional findFirst(const std::string& str, + size_t pos) const override; + std::vector findAll(const std::string& str) const override; + std::vector split(const std::string& str) const override; + + private: + RE2 re_; + static jsonata::RegexMatch toRegexMatch( + const re2::StringPiece& input, + const std::vector& submatch, int numGroups); +}; + +// Factory for use with Jsonata's constructor, e.g. +// Jsonata("/foo/i", re2RegexEngine()). +jsonata::RegexEngine re2RegexEngine(); diff --git a/test/RE2EngineTest.cpp b/test/RE2EngineTest.cpp new file mode 100644 index 0000000..55b6afc --- /dev/null +++ b/test/RE2EngineTest.cpp @@ -0,0 +1,90 @@ +/** + * jsonata-cpp is the JSONata C++ reference port + * + * Copyright Robert Yokota + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include + +#include "RE2Engine.h" + +using jsonata::Jsonata; + +TEST(RE2EngineTest, Match) { + // A single match is returned as a bare object, not wrapped in an array + // (standard JSONata singleton-sequence unwrapping). + Jsonata expr("$match(\"hello world\", /o w/)", re2RegexEngine()); + auto result = expr.evaluate(nullptr); + ASSERT_TRUE(result.is_object()); + EXPECT_EQ(result["match"], "o w"); + EXPECT_EQ(result["index"], 4); +} + +TEST(RE2EngineTest, MatchCaseInsensitiveFlag) { + Jsonata expr("$match(\"HELLO\", /hello/i)", re2RegexEngine()); + auto result = expr.evaluate(nullptr); + ASSERT_TRUE(result.is_object()); + EXPECT_EQ(result["match"], "HELLO"); +} + +TEST(RE2EngineTest, Contains) { + Jsonata expr("$contains(\"hello\", /ell/)", re2RegexEngine()); + auto result = expr.evaluate(nullptr); + ASSERT_TRUE(result.is_boolean()); + EXPECT_TRUE(result.get()); +} + +TEST(RE2EngineTest, ReplaceWithString) { + Jsonata expr("$replace(\"abc123def\", /[0-9]+/, \"#\")", re2RegexEngine()); + auto result = expr.evaluate(nullptr); + EXPECT_EQ(result, "abc#def"); +} + +TEST(RE2EngineTest, ReplaceWithFunction) { + Jsonata expr( + "$replace(\"abc123\", /[0-9]+/, function($m) { $m.match & \"!\" })", + re2RegexEngine()); + auto result = expr.evaluate(nullptr); + EXPECT_EQ(result, "abc123!"); +} + +TEST(RE2EngineTest, Split) { + Jsonata expr("$split(\"a1b2c3\", /[0-9]/)", re2RegexEngine()); + auto result = expr.evaluate(nullptr); + ASSERT_TRUE(result.is_array()); + std::vector values; + for (const auto& v : result) values.push_back(v.get()); + EXPECT_EQ(values, (std::vector{"a", "b", "c", ""})); +} + +TEST(RE2EngineTest, RejectsBackreferences) { + // Proves RE2 is actually compiling the pattern rather than silently + // falling back to std::regex: backreferences can't run in RE2's + // guaranteed-linear-time engine, so this must fail when the regex + // literal is compiled (during expression construction). + EXPECT_THROW( + { Jsonata expr("$match(\"xaay\", /(a)\\1/)", re2RegexEngine()); }, + std::exception); +} + +TEST(RE2EngineTest, DefaultEngineStillStdRegex) { + // No engine argument -> falls back to the default std::regex-backed + // engine, which *does* support backreferences. Confirms the default + // path is unaffected. + Jsonata expr("$match(\"xaay\", /(a)\\1/)"); + auto result = expr.evaluate(nullptr); + ASSERT_TRUE(result.is_object()); + EXPECT_EQ(result["match"], "aa"); +} diff --git a/test/StringTest.cpp b/test/StringTest.cpp index 04958da..d34997a 100644 --- a/test/StringTest.cpp +++ b/test/StringTest.cpp @@ -175,6 +175,102 @@ TEST_F(StringTest, evalTest) { EXPECT_EQ(result.get(), "AAA"); } +TEST_F(StringTest, evalSeesEnclosingVariableBindingTest) { + // $eval's dynamically-parsed expression must see variables bound in + // the enclosing scope (here, via an in-expression := assignment), not + // just a static top-level environment (regression test). + Jsonata expr("($x := 5; $eval(\"$x + 1\"))"); + auto result = expr.evaluate(nullptr); + ASSERT_TRUE(result.is_number()); + EXPECT_EQ(result.get(), 6); +} + +TEST_F(StringTest, evalSeesExplicitTopLevelBindingsTest) { + // Same as above, but for bindings passed via evaluate()'s bindings + // argument rather than an in-expression assignment. + Jsonata expr("$eval(\"$x\")"); + auto bindingFrame = expr.createFrame(); + bindingFrame->bind("x", int64_t(42)); + auto result = expr.evaluate(nullptr, bindingFrame); + ASSERT_TRUE(result.is_number()); + EXPECT_EQ(result.get(), 42); +} + +TEST_F(StringTest, evalUnaffectedBySiblingArgumentScopeTest) { + // $eval's second (focus) argument is evaluated before its own body + // runs, and here contains a nested block with its own environment. + // Confirms evaluating that sibling argument doesn't leave the tracked + // "current" environment pointing at the inner block's scope, which + // would otherwise cause $eval to resolve $x (from the outer scope) as + // undefined instead of 5. (jsonata-python had this bug; jsonata-cpp + // does not, verified as a regression test.) + Jsonata expr("($x := 5; $eval(\"$x\", (($y := 1; $y))))"); + auto result = expr.evaluate(nullptr); + ASSERT_TRUE(result.is_number()); + EXPECT_EQ(result.get(), 5); +} + +TEST_F(StringTest, stackGuardrailStopsNonTailRecursionTest) { + // Ackermann is not tail-recursive, so its eval-apply depth grows with + // each call. A small `stack` bound should trip before it completes. + std::string ackExpr = + "($ack := function($m, $n) { $m = 0 ? $n + 1 : ($n = 0 ? " + "$ack($m - 1, 1) : $ack($m - 1, $ack($m, $n - 1))) }; $ack(4, 3))"; + Jsonata expr(ackExpr, defaultRegexEngine(), std::nullopt, 50); + try { + expr.evaluate(nullptr); + FAIL() << "Expected JException D1011"; + } catch (const JException& e) { + EXPECT_EQ(e.getError(), "D1011"); + } +} + +TEST_F(StringTest, stackGuardrailAllowsBoundedRecursionTest) { + // A stack bound that's high enough for the actual recursion depth + // should not interfere with a normal, terminating evaluation. + std::string ackExpr = + "($ack := function($m, $n) { $m = 0 ? $n + 1 : ($n = 0 ? " + "$ack($m - 1, 1) : $ack($m - 1, $ack($m, $n - 1))) }; $ack(3, 4))"; + Jsonata expr(ackExpr, defaultRegexEngine(), std::nullopt, 1000); + auto result = expr.evaluate(nullptr); + ASSERT_TRUE(result.is_number()); + EXPECT_EQ(result.get(), 125); +} + +TEST_F(StringTest, timeoutGuardrailStopsInfiniteTailRecursionTest) { + // Tail recursion doesn't grow the eval-apply depth, so only the + // `timeout` guardrail (not `stack`) can catch this infinite loop. + Jsonata expr("($f := function($n) { $f($n + 1) }; $f(0))", + defaultRegexEngine(), 100, std::nullopt); + try { + expr.evaluate(nullptr); + FAIL() << "Expected JException D1012"; + } catch (const JException& e) { + EXPECT_EQ(e.getError(), "D1012"); + } +} + +TEST_F(StringTest, stackGuardrailPropagatesThroughEvalTest) { + // $eval() reuses the enclosing instance's environment directly, so a + // `stack` bound configured on the outer expression must also apply to + // dynamically-evaluated code (regression test for guardrail bypass). + std::string ackExpr = + "($ack := function($m, $n) { $m = 0 ? $n + 1 : ($n = 0 ? " + "$ack($m - 1, 1) : $ack($m - 1, $ack($m, $n - 1))) }; " + "$ack(4, 3))"; + Jsonata expr("$eval(\"" + ackExpr + "\")", defaultRegexEngine(), + std::nullopt, 50); + // Unlike jsonata-java (which wraps every $eval-time error, guardrail or + // not, into a generic D3121), jsonata-cpp's functionEval re-throws + // JException as-is, so the original D1011 code survives here. + try { + expr.evaluate(nullptr); + FAIL() << "Expected JException D1011"; + } catch (const JException& e) { + EXPECT_EQ(e.getError(), "D1011"); + } +} + TEST_F(StringTest, regexTest) { auto input = makeObject({{"foo", 1}, {"bar", 2}}); auto result = Jsonata("($matcher := $eval('/^' & 'foo' & '/i'); $.$spread()[$.$keys() ~> $matcher])").evaluate(input);