From 56edafac1a587ba90fbee755694e374186289604 Mon Sep 17 00:00:00 2001 From: viktorbeck98 Date: Mon, 12 Oct 2026 13:06:35 +0200 Subject: [PATCH 1/2] fix: JsonParser crash without dict config, EventID 0 without message A JsonParserConfig object (or the default config) left the parser without a content parser, so the first log with a message field raised AttributeError. It now gets a MatcherParser without templates. Logs without a message field got EventID 0, the ID of the first template. They now get EventID -1 and the template , like any log that matches no template. Closes #357 Co-Authored-By: Claude Opus 5.5 --- src/detectmatelibrary/parsers/json_parser.py | 5 +- tests/test_parsers/test_json_parser.py | 57 ++++++++++++++++++-- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/src/detectmatelibrary/parsers/json_parser.py b/src/detectmatelibrary/parsers/json_parser.py index f3d9d412..02387898 100644 --- a/src/detectmatelibrary/parsers/json_parser.py +++ b/src/detectmatelibrary/parsers/json_parser.py @@ -74,6 +74,9 @@ def __init__( ) self.content_parser = MatcherParser(config=content_parser_config) config = cfg_dict + else: + # a config object cannot define the content parser: use one without templates + self.content_parser = MatcherParser() super().__init__(name=name, config=config) self.config: JsonParserConfig @@ -86,7 +89,7 @@ def parse(self, input_: schemas.LogSchema, output_: schemas.ParserSchema) -> Non timestamp = self.time_extractor.extract(obj=log, delete=True) content = self.content_extractor.extract(obj=log, delete=True) - parsed = {"EventTemplate": "", "Params": [], "EventId": 0} + parsed = {"EventTemplate": "", "Params": [], "EventId": -1} # if the json also contains a message field, parse it for template and parameters if content: log_ = schemas.LogSchema({"log": content}) diff --git a/tests/test_parsers/test_json_parser.py b/tests/test_parsers/test_json_parser.py index 35964071..35e7e4fa 100644 --- a/tests/test_parsers/test_json_parser.py +++ b/tests/test_parsers/test_json_parser.py @@ -60,10 +60,10 @@ def test_parse_json_without_message(self): output = schemas.ParserSchema() parser.parse(input_log, output) - # When no message field, template should be empty - assert output.template == "" + # When no message field, no template is found + assert output.template == "" assert len(output.variables) == 0 - assert output.EventID == 0 + assert output.EventID == -1 # But other fields should be present assert "level" in output.logFormatVariables @@ -164,6 +164,57 @@ def test_parse_with_content_parser(self): assert len(output.variables) > 0 assert "level" in output.logFormatVariables + def test_parse_message_with_config_object(self): + """A message parsed with a config object matches no template.""" + config = JsonParserConfig() + parser = JsonParser(name="TestParser", config=config) + + json_log = { + "time": "2023-11-18 10:30:00", + "message": "hello world" + } + + input_log = schemas.LogSchema({ + "logID": "1", + "log": json.dumps(json_log) + }) + + output = schemas.ParserSchema() + parser.parse(input_log, output) + + assert output.template == "" + assert output.EventID == -1 + + def test_no_message_differs_from_first_template(self): + """Logs without message do not share the first template's EventID.""" + config_dict = { + "parsers": { + "TestParser": { + "auto_config": True, + "method_type": "json_parser", + }, + "JsonMatcherParser": { + "auto_config": True, + "method_type": "matcher_parser", + "path_templates": TEST_TEMPLATES + } + } + } + parser = JsonParser(name="TestParser", config=config_dict) + + json_logs = [ + {"message": "pid=9699 uid=0 auid=4294967295 ses=4294967295 msg='op=PAM:accounting acct=\"root\""}, + {"level": "INFO"}, + ] + + event_ids = [] + for json_log in json_logs: + output = schemas.ParserSchema() + parser.parse(schemas.LogSchema({"logID": "1", "log": json.dumps(json_log)}), output) + event_ids.append(output.EventID) + + assert event_ids == [0, -1] + def test_custom_field_names(self): """Test with custom timestamp and message field names.""" config = JsonParserConfig(timestamp_name="timestamp", content_name="msg") From 2dc4c288c39b58c1032db3f96c3dccd31a6175d1 Mon Sep 17 00:00:00 2001 From: viktorbeck98 Date: Thu, 15 Oct 2026 16:40:40 +0200 Subject: [PATCH 2/2] JsonParser: eventID -2 without message field, warn when no templates A log without a message field now gets eventID -2 and an empty template, so it no longer shares -1 with a message that matches no template (and no longer triggers the template_not_found rule). A parser whose content parser has no templates logs one warning at the first message. Co-Authored-By: Claude Opus 5.5 --- docs/parsers/json_parser.md | 10 ++++ src/detectmatelibrary/parsers/json_parser.py | 13 ++++- tests/test_parsers/test_json_parser.py | 50 +++++++++++++++++--- 3 files changed, 66 insertions(+), 7 deletions(-) diff --git a/docs/parsers/json_parser.md b/docs/parsers/json_parser.md index 7f3b1ccc..cca796e4 100644 --- a/docs/parsers/json_parser.md +++ b/docs/parsers/json_parser.md @@ -23,6 +23,16 @@ With the configuration file below, the `message` field is also matched against t --8<-- "docs/examples/parsers/json_parser.py:dict-based" ``` +## Event IDs + +| Log | Event ID | Template | +|-----|----------|----------| +| `message` matches a template | that template's ID | the template | +| `message` matches no template | `-1` | `` | +| no `message` field | `-2` | empty | + +If the content parser has no templates (for example with the defaults, as in the first example), every message gets `-1`. The parser then logs a warning at the first message. + ## Configuration file The configuration used by the second example above (the first one runs with all defaults). It sets only what this use case needs; every other parameter keeps its default (see [Configuration arguments](#configuration-arguments)). diff --git a/src/detectmatelibrary/parsers/json_parser.py b/src/detectmatelibrary/parsers/json_parser.py index e500013f..c6519b04 100644 --- a/src/detectmatelibrary/parsers/json_parser.py +++ b/src/detectmatelibrary/parsers/json_parser.py @@ -4,6 +4,7 @@ ) from detectmatelibrary.common.parser import CoreParser, CoreParserConfig from detectmatelibrary.utils.key_extractor import KeyExtractor +from detectmatelibrary.tools.logging import logger from detectmatelibrary import schemas from collections.abc import Mapping @@ -11,6 +12,9 @@ import json from pydantic import Field +# eventID of a log without a message field (-1 is a message that matches no template) +NO_MESSAGE_EVENT_ID = -2 + def iter_flatten(obj: dict[str, Any], sep: str = ".") -> Iterable[tuple[str, Any]]: """Iteratively flattens a nested dict/list JSON-like object. Yields @@ -79,6 +83,7 @@ def __init__( self.content_parser = MatcherParser() super().__init__(name=name, config=config) self.config: JsonParserConfig + self._warn_no_templates = self.content_parser.config.path_templates is None self.time_extractor = KeyExtractor(key_substr=config.timestamp_name) self.content_extractor = KeyExtractor(key_substr=config.content_name) @@ -89,9 +94,15 @@ def parse(self, input_: schemas.LogSchema, output_: schemas.ParserSchema) -> Non timestamp = self.time_extractor.extract(obj=log, delete=True) content = self.content_extractor.extract(obj=log, delete=True) - parsed = {"EventTemplate": "", "Params": [], "EventId": -1} + parsed = {"EventTemplate": "", "Params": [], "EventId": NO_MESSAGE_EVENT_ID} # if the json also contains a message field, parse it for template and parameters if content: + if self._warn_no_templates: + logger.warning( + f"<<{self.name}>> the content parser has no templates, so every message gets " + "eventID -1. Define the content parser with path_templates in a config dict." + ) + self._warn_no_templates = False log_ = schemas.LogSchema({"log": content}) parsed_content = self.content_parser.process(log_) parsed["EventTemplate"] = parsed_content["template"] # type: ignore diff --git a/tests/test_parsers/test_json_parser.py b/tests/test_parsers/test_json_parser.py index 72f3660e..eb6a8e26 100644 --- a/tests/test_parsers/test_json_parser.py +++ b/tests/test_parsers/test_json_parser.py @@ -1,4 +1,5 @@ import json +import logging from detectmatelibrary.parsers.json_parser import ( JsonParser, JsonParserConfig, @@ -60,10 +61,10 @@ def test_parse_json_without_message(self): output = schemas.ParserSchema() parser.parse(input_log, output) - # When no message field, no template is found - assert output.template == "" + # When no message field, the template is empty and the eventID is -2 + assert output.template == "" assert len(output.variables) == 0 - assert output.eventID == -1 + assert output.eventID == -2 # But other fields should be present assert "level" in output.headerVariables @@ -185,8 +186,9 @@ def test_parse_message_with_config_object(self): assert output.template == "" assert output.eventID == -1 - def test_no_message_differs_from_first_template(self): - """Logs without message do not share the first template's EventID.""" + def test_no_message_differs_from_first_template_and_no_match(self): + """Logs without message share neither the first template's eventID nor + the eventID of a message that matches no template.""" config_dict = { "parsers": { "TestParser": { @@ -204,6 +206,7 @@ def test_no_message_differs_from_first_template(self): json_logs = [ {"message": "pid=9699 uid=0 auid=4294967295 ses=4294967295 msg='op=PAM:accounting acct=\"root\""}, + {"message": "matches no template"}, {"level": "INFO"}, ] @@ -213,7 +216,42 @@ def test_no_message_differs_from_first_template(self): parser.parse(schemas.LogSchema({"logID": "1", "log": json.dumps(json_log)}), output) event_ids.append(output.eventID) - assert event_ids == [0, -1] + assert event_ids == [0, -1, -2] + + def test_warns_once_when_message_meets_no_templates(self, caplog): + """A parser without templates warns once, at the first message.""" + parser = JsonParser(name="TestParser", config=JsonParserConfig()) + + def parse(json_log): + parser.parse(schemas.LogSchema({"log": json.dumps(json_log)}), schemas.ParserSchema()) + + with caplog.at_level(logging.WARNING): + parse({"level": "INFO"}) + assert not caplog.records + parse({"message": "hello"}) + parse({"message": "world"}) + + warnings = [r for r in caplog.records if "no templates" in r.message] + assert len(warnings) == 1 + + def test_no_warning_with_templates(self, caplog): + """A parser with templates does not warn about missing templates.""" + config_dict = { + "parsers": { + "TestParser": {"auto_config": True, "method_type": "json_parser"}, + "JsonMatcherParser": { + "auto_config": True, + "method_type": "matcher_parser", + "path_templates": TEST_TEMPLATES + } + } + } + parser = JsonParser(name="TestParser", config=config_dict) + + with caplog.at_level(logging.WARNING): + parser.parse(schemas.LogSchema({"log": json.dumps({"message": "hello"})}), schemas.ParserSchema()) + + assert not [r for r in caplog.records if "no templates" in r.message] def test_custom_field_names(self): """Test with custom timestamp and message field names."""