diff --git a/docs/parsers/json_parser.md b/docs/parsers/json_parser.md index 7f3b1ccc..cca796e4 100644 --- a/docs/parsers/json_parser.md +++ b/docs/parsers/json_parser.md @@ -23,6 +23,16 @@ With the configuration file below, the `message` field is also matched against t --8<-- "docs/examples/parsers/json_parser.py:dict-based" ``` +## Event IDs + +| Log | Event ID | Template | +|-----|----------|----------| +| `message` matches a template | that template's ID | the template | +| `message` matches no template | `-1` | `` | +| no `message` field | `-2` | empty | + +If the content parser has no templates (for example with the defaults, as in the first example), every message gets `-1`. The parser then logs a warning at the first message. + ## Configuration file The configuration used by the second example above (the first one runs with all defaults). It sets only what this use case needs; every other parameter keeps its default (see [Configuration arguments](#configuration-arguments)). diff --git a/src/detectmatelibrary/parsers/json_parser.py b/src/detectmatelibrary/parsers/json_parser.py index 22bc5987..c6519b04 100644 --- a/src/detectmatelibrary/parsers/json_parser.py +++ b/src/detectmatelibrary/parsers/json_parser.py @@ -4,6 +4,7 @@ ) from detectmatelibrary.common.parser import CoreParser, CoreParserConfig from detectmatelibrary.utils.key_extractor import KeyExtractor +from detectmatelibrary.tools.logging import logger from detectmatelibrary import schemas from collections.abc import Mapping @@ -11,6 +12,9 @@ import json from pydantic import Field +# eventID of a log without a message field (-1 is a message that matches no template) +NO_MESSAGE_EVENT_ID = -2 + def iter_flatten(obj: dict[str, Any], sep: str = ".") -> Iterable[tuple[str, Any]]: """Iteratively flattens a nested dict/list JSON-like object. Yields @@ -74,8 +78,12 @@ def __init__( ) self.content_parser = MatcherParser(config=content_parser_config) config = cfg_dict + else: + # a config object cannot define the content parser: use one without templates + self.content_parser = MatcherParser() super().__init__(name=name, config=config) self.config: JsonParserConfig + self._warn_no_templates = self.content_parser.config.path_templates is None self.time_extractor = KeyExtractor(key_substr=config.timestamp_name) self.content_extractor = KeyExtractor(key_substr=config.content_name) @@ -86,9 +94,15 @@ def parse(self, input_: schemas.LogSchema, output_: schemas.ParserSchema) -> Non timestamp = self.time_extractor.extract(obj=log, delete=True) content = self.content_extractor.extract(obj=log, delete=True) - parsed = {"EventTemplate": "", "Params": [], "EventId": 0} + parsed = {"EventTemplate": "", "Params": [], "EventId": NO_MESSAGE_EVENT_ID} # if the json also contains a message field, parse it for template and parameters if content: + if self._warn_no_templates: + logger.warning( + f"<<{self.name}>> the content parser has no templates, so every message gets " + "eventID -1. Define the content parser with path_templates in a config dict." + ) + self._warn_no_templates = False log_ = schemas.LogSchema({"log": content}) parsed_content = self.content_parser.process(log_) parsed["EventTemplate"] = parsed_content["template"] # type: ignore diff --git a/tests/test_parsers/test_json_parser.py b/tests/test_parsers/test_json_parser.py index 73299688..eb6a8e26 100644 --- a/tests/test_parsers/test_json_parser.py +++ b/tests/test_parsers/test_json_parser.py @@ -1,4 +1,5 @@ import json +import logging from detectmatelibrary.parsers.json_parser import ( JsonParser, JsonParserConfig, @@ -60,10 +61,10 @@ def test_parse_json_without_message(self): output = schemas.ParserSchema() parser.parse(input_log, output) - # When no message field, template should be empty + # When no message field, the template is empty and the eventID is -2 assert output.template == "" assert len(output.variables) == 0 - assert output.eventID == 0 + assert output.eventID == -2 # But other fields should be present assert "level" in output.headerVariables @@ -164,6 +165,94 @@ def test_parse_with_content_parser(self): assert len(output.variables) > 0 assert "level" in output.headerVariables + def test_parse_message_with_config_object(self): + """A message parsed with a config object matches no template.""" + config = JsonParserConfig() + parser = JsonParser(name="TestParser", config=config) + + json_log = { + "time": "2023-11-18 10:30:00", + "message": "hello world" + } + + input_log = schemas.LogSchema({ + "logID": "1", + "log": json.dumps(json_log) + }) + + output = schemas.ParserSchema() + parser.parse(input_log, output) + + assert output.template == "" + assert output.eventID == -1 + + def test_no_message_differs_from_first_template_and_no_match(self): + """Logs without message share neither the first template's eventID nor + the eventID of a message that matches no template.""" + config_dict = { + "parsers": { + "TestParser": { + "auto_config": True, + "method_type": "json_parser", + }, + "JsonMatcherParser": { + "auto_config": True, + "method_type": "matcher_parser", + "path_templates": TEST_TEMPLATES + } + } + } + parser = JsonParser(name="TestParser", config=config_dict) + + json_logs = [ + {"message": "pid=9699 uid=0 auid=4294967295 ses=4294967295 msg='op=PAM:accounting acct=\"root\""}, + {"message": "matches no template"}, + {"level": "INFO"}, + ] + + event_ids = [] + for json_log in json_logs: + output = schemas.ParserSchema() + parser.parse(schemas.LogSchema({"logID": "1", "log": json.dumps(json_log)}), output) + event_ids.append(output.eventID) + + assert event_ids == [0, -1, -2] + + def test_warns_once_when_message_meets_no_templates(self, caplog): + """A parser without templates warns once, at the first message.""" + parser = JsonParser(name="TestParser", config=JsonParserConfig()) + + def parse(json_log): + parser.parse(schemas.LogSchema({"log": json.dumps(json_log)}), schemas.ParserSchema()) + + with caplog.at_level(logging.WARNING): + parse({"level": "INFO"}) + assert not caplog.records + parse({"message": "hello"}) + parse({"message": "world"}) + + warnings = [r for r in caplog.records if "no templates" in r.message] + assert len(warnings) == 1 + + def test_no_warning_with_templates(self, caplog): + """A parser with templates does not warn about missing templates.""" + config_dict = { + "parsers": { + "TestParser": {"auto_config": True, "method_type": "json_parser"}, + "JsonMatcherParser": { + "auto_config": True, + "method_type": "matcher_parser", + "path_templates": TEST_TEMPLATES + } + } + } + parser = JsonParser(name="TestParser", config=config_dict) + + with caplog.at_level(logging.WARNING): + parser.parse(schemas.LogSchema({"log": json.dumps({"message": "hello"})}), schemas.ParserSchema()) + + assert not [r for r in caplog.records if "no templates" in r.message] + def test_custom_field_names(self): """Test with custom timestamp and message field names.""" config = JsonParserConfig(timestamp_name="timestamp", content_name="msg")