From 8662d24cb8513162902c57c242ae30870e345fae Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 19 Oct 2026 08:24:37 +0200 Subject: [PATCH 01/23] add pipeline first draft and compress schema name --- .../alert_aggregators/basic_concat.py | 4 +- .../detectors/random_detector.py | 8 +- src/detectmatelibrary/pipeline.py | 95 +++++++++++++++++++ .../utils/data_normalizer.py | 6 +- .../test_basic_concat.py | 6 +- .../test_variable_detector.py | 4 +- tests/test_common/test_core.py | 28 +++--- .../test_common/test_core_alert_aggregator.py | 22 ++--- tests/test_common/test_core_detector.py | 22 ++--- tests/test_common/test_core_parser.py | 16 ++-- tests/test_common/test_extract_timestamp.py | 18 ++-- tests/test_common/test_train_buffer.py | 18 ++-- .../test_bigram_frequency_detector.py | 38 ++++---- tests/test_detectors/test_charset_detector.py | 38 ++++---- tests/test_detectors/test_deeplog_detector.py | 8 +- .../test_event_sequence_detector.py | 34 +++---- tests/test_detectors/test_logbert_detector.py | 8 +- .../test_detectors/test_mismatch_warnings.py | 6 +- .../test_detectors/test_new_event_detector.py | 24 ++--- .../test_new_value_combo_detector.py | 72 +++++++------- .../test_detectors/test_new_value_detector.py | 30 +++--- tests/test_detectors/test_random_detector.py | 8 +- tests/test_detectors/test_rule_detector.py | 40 ++++---- .../test_value_range_detector.py | 48 +++++----- tests/test_helper/test_from_to.py | 18 ++-- tests/test_parsers/test_json_parser.py | 26 ++--- tests/test_parsers/test_logbatcher_parser.py | 12 +-- .../test_time_dependent_stability.py | 4 +- tests/test_pipelines/test_dummy_methods.py | 12 +-- 29 files changed, 384 insertions(+), 289 deletions(-) create mode 100644 src/detectmatelibrary/pipeline.py diff --git a/docs/examples/alert_aggregators/basic_concat.py b/docs/examples/alert_aggregators/basic_concat.py index 27cb326e..b816e99b 100644 --- a/docs/examples/alert_aggregators/basic_concat.py +++ b/docs/examples/alert_aggregators/basic_concat.py @@ -2,7 +2,7 @@ # --8<-- [start:example_1] from detectmatelibrary.alert_aggregation.basic_concat import BasicConcatAggregation -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc aggregations_config = { @@ -17,7 +17,7 @@ } -input_ = schemas.DetectorSchema({ +input_ = sc.DetectorSchema({ "detectorID": "1", "detectorType": "dummy", "alertID": "2", diff --git a/src/detectmatelibrary/detectors/random_detector.py b/src/detectmatelibrary/detectors/random_detector.py index 3ad33ad4..44c05056 100644 --- a/src/detectmatelibrary/detectors/random_detector.py +++ b/src/detectmatelibrary/detectors/random_detector.py @@ -4,7 +4,7 @@ from detectmatelibrary.utils.data_buffer import BufferMode -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc from typing_extensions import override from typing import List @@ -34,15 +34,15 @@ def __init__( self.config: RandomDetectorConfig @override - def train(self, input_: List[schemas.ParserSchema] | schemas.ParserSchema) -> None: + def train(self, input_: List[sc.ParserSchema] | sc.ParserSchema) -> None: """Training is not applicable for RandomDetector.""" return @override def detect( self, - input_: schemas.ParserSchema, # type: ignore - output_: schemas.DetectorSchema, + input_: sc.ParserSchema, # type: ignore + output_: sc.DetectorSchema, ) -> bool: """Detect anomalies randomly in the input data.""" overall_score = 0.0 diff --git a/src/detectmatelibrary/pipeline.py b/src/detectmatelibrary/pipeline.py new file mode 100644 index 00000000..c230f102 --- /dev/null +++ b/src/detectmatelibrary/pipeline.py @@ -0,0 +1,95 @@ + +from detectmatelibrary.common.core import CoreComponent, CoreConfig +from detectmatelibrary.common.detector import CoreDetectorConfig +from detectmatelibrary.common.parser import CoreParserConfig + +from detectmatelibrary.utils.data_buffer import ArgsBuffer, BufferMode + +import detectmatelibrary.schemas as sc + +from typing import Any, Optional, overload, override +from pydantic import Field + + +class PipelineConfig(CoreConfig): + component_type: str = Field(default="main_pipeline", description="<$IGNORE$>") + method_type: str = Field(default="pipeline", description="<$IGNORE$>") + + do_parsing: bool = Field(default=True, description="Run parser in the pipeline.") + do_aggregation: bool = Field(default=True, description="Run aggregator in the pipeline.") + + parser: CoreParserConfig | dict[str, Any] | None = Field( + default=None, description="Parser configuration use, if do_parsing=False keep empty." + ) + detectors: list[CoreDetectorConfig | dict[str, Any]] | None = Field( + default=None, description="Detectors configuration use." + ) + aggregators: CoreParserConfig | dict[str, Any] | None = Field( + default=None, description="Aggregator configuration use, if do_aggregation=False keep empty." + ) + + +def _select_input(config: PipelineConfig) -> type[sc.BaseSchema]: + return sc.LogSchema if config.do_parsing else sc.ParserSchema + + +def _select_output(config: PipelineConfig) -> type[sc.BaseSchema]: + return sc.DetectorSchema if config.do_aggregation else sc.AggregateSchema + + +class Pipeline(CoreComponent): + def __init__( + self, + name: str = "MainPipeline", + buffer_mode: BufferMode = BufferMode.WINDOW, + buffer_size: Optional[int] = 1, + config: PipelineConfig | dict[str, Any] = PipelineConfig(), + ) -> None: + if isinstance(config, dict): + config = PipelineConfig.from_dict(config, name) + + super().__init__( + name=name, + type_=config.method_type, + config=config, + args_buffer=ArgsBuffer(mode=buffer_mode, size=buffer_size), + input_schema=_select_input(config), + output_schema=_select_output(config), + ) + self.config: PipelineConfig + + @overload # type: ignore + @override + def run( + self, + input_: list[sc.LogSchema] | sc.LogSchema, + output_: list[sc.DetectorSchema] | sc.DetectorSchema + ) -> bool: + pass + + @overload + @override + def run(self, input_: list[sc.LogSchema] | sc.LogSchema, output_: sc.AggregateSchema) -> bool: + pass + + @overload + @override + def run( + self, + input_: list[sc.ParserSchema] | sc.ParserSchema, + output_: list[sc.DetectorSchema] | sc.DetectorSchema + ) -> bool: + pass + + @overload + @override + def run(self, input_: list[sc.ParserSchema] | sc.ParserSchema, output_: sc.AggregateSchema) -> bool: + pass + + @override + def run( + self, + input_: list[sc.LogSchema] | sc.LogSchema | list[sc.ParserSchema] | sc.ParserSchema, + output_: list[sc.DetectorSchema] | sc.DetectorSchema | sc.AggregateSchema + ) -> bool: + return True diff --git a/src/detectmatelibrary/utils/data_normalizer.py b/src/detectmatelibrary/utils/data_normalizer.py index 167ec481..a79a225d 100644 --- a/src/detectmatelibrary/utils/data_normalizer.py +++ b/src/detectmatelibrary/utils/data_normalizer.py @@ -1,4 +1,4 @@ -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc from typing import Any import pandas as pd @@ -13,8 +13,8 @@ class DataNormalizer: def __init__( self, data: pd.DataFrame | dict[str, Any] | list[Any], - input_schema: schemas.BaseSchema, - output_schema: schemas.BaseSchema + input_schema: sc.BaseSchema, + output_schema: sc.BaseSchema ) -> None: self.data = data self.input_schema = input_schema diff --git a/tests/test_alert_aggregators/test_basic_concat.py b/tests/test_alert_aggregators/test_basic_concat.py index 08d60c60..17a26087 100644 --- a/tests/test_alert_aggregators/test_basic_concat.py +++ b/tests/test_alert_aggregators/test_basic_concat.py @@ -1,7 +1,7 @@ from detectmatelibrary.alert_aggregation.basic_concat import BasicConcatAggregation from detectmatelibrary.schemas import AggregateSchema from detectmatelibrary.utils.aux import time_test_mode -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc from datetime import datetime @@ -27,7 +27,7 @@ } -data_global = [schemas.DetectorSchema({ +data_global = [sc.DetectorSchema({ "detectorID": str(i), "detectorType": "dummy", "alertID": str(i), @@ -40,7 +40,7 @@ "alertDetails": {"99 problems": "but logs aint one"} }) for i in range(1, 4)] -data_global += [schemas.DetectorSchema({ +data_global += [sc.DetectorSchema({ "detectorID": "1", "detectorType": "dummy", "alertID": "1", diff --git a/tests/test_base_detectors/test_variable_detector.py b/tests/test_base_detectors/test_variable_detector.py index af0f7d86..52eca4d0 100644 --- a/tests/test_base_detectors/test_variable_detector.py +++ b/tests/test_base_detectors/test_variable_detector.py @@ -1,6 +1,6 @@ from detectmatelibrary.base_detectors import VariableDetector -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc import os @@ -28,7 +28,7 @@ def train(detector: VariableDetector, *users) -> None: for u in users: - detector.train(schemas.ParserSchema({ + detector.train(sc.ParserSchema({ "parserType": "t", "eventID": 1, "template": "t", "variables": [u], "logID": "1", "parsedLogID": "1", "parserID": "p", "log": "l", "headerVariables": {}, diff --git a/tests/test_common/test_core.py b/tests/test_common/test_core.py index e4142e6f..914ee2ab 100644 --- a/tests/test_common/test_core.py +++ b/tests/test_common/test_core.py @@ -6,7 +6,7 @@ from detectmatelibrary.utils.data_buffer import ArgsBuffer, BufferMode import detectmatelibrary.schemas._op as op_schemas -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc import logging import os @@ -41,8 +41,8 @@ class MockConfigWithTraining(CoreConfig): } -def _make_log(i: int) -> schemas.LogSchema: - return schemas.LogSchema({ +def _make_log(i: int) -> sc.LogSchema: + return sc.LogSchema({ "__version__": "2.0.0", "logID": str(i), "logSource": "test", @@ -60,7 +60,7 @@ def __init__( self, name: str, config: MockConfigWithTraining = MockConfigWithTraining() ) -> None: super().__init__( - name=name, type_="Dummy", config=config, input_schema=schemas.LogSchema + name=name, type_="Dummy", config=config, input_schema=sc.LogSchema ) self.train_data = [] @@ -83,7 +83,7 @@ def __init__( self, name: str, config: MockConfigWithConfigure = MockConfigWithConfigure() ) -> None: super().__init__( - name=name, type_="Dummy", config=config, input_schema=schemas.LogSchema + name=name, type_="Dummy", config=config, input_schema=sc.LogSchema ) self.configure_data: list = [] self.set_configuration_called: int = 0 @@ -101,7 +101,7 @@ def run(self, input_, output_) -> bool: class MockComponentWithConfigureAndTraining(CoreComponent): def __init__(self, name: str, config: CoreConfig = CoreConfig()) -> None: super().__init__( - name=name, type_="Dummy", config=config, input_schema=schemas.LogSchema + name=name, type_="Dummy", config=config, input_schema=sc.LogSchema ) self.configure_data: list = [] self.train_data: list = [] @@ -240,7 +240,7 @@ def test_training(self) -> None: for i in range(10): component.process( - schemas.LogSchema({ + sc.LogSchema({ "__version__": "2.0.0", "logID": str(i), "logSource": "test", @@ -250,7 +250,7 @@ def test_training(self) -> None: assert len(component.train_data) == component.fitlogic.train_state.data_used for i, log in enumerate(component.train_data): - expected = schemas.LogSchema({ + expected = sc.LogSchema({ "__version__": "2.0.0", "logID": str(i), "logSource": "test", @@ -266,7 +266,7 @@ def test_training_use_config_data(self) -> None: for i in range(10): component.process( - schemas.LogSchema({ + sc.LogSchema({ "__version__": "2.0.0", "logID": str(i), "logSource": "test", @@ -277,7 +277,7 @@ def test_training_use_config_data(self) -> None: total += component.fitlogic.config_state.total_need_data assert len(component.train_data) == total for i, log in enumerate(component.train_data): - expected = schemas.LogSchema({ + expected = sc.LogSchema({ "__version__": "2.0.0", "logID": str(i), "logSource": "test", @@ -292,7 +292,7 @@ def test_training_force_stop(self) -> None: if i == 2: component.update_state("stop_training") component.process( - schemas.LogSchema({ + sc.LogSchema({ "__version__": "2.0.0", "logID": str(i), "logSource": "test", @@ -308,7 +308,7 @@ def test_training_keep_training(self) -> None: for i in range(10): component.process( - schemas.LogSchema({ + sc.LogSchema({ "__version__": "2.0.0", "logID": str(i), "logSource": "test", @@ -477,7 +477,7 @@ class MockConfigWithPostTrain(CoreConfig): class MockComponentWithPostTrain(CoreComponent): def __init__(self, name: str, config: CoreConfig = MockConfigWithPostTrain()) -> None: super().__init__( - name=name, type_="Dummy", config=config, input_schema=schemas.LogSchema + name=name, type_="Dummy", config=config, input_schema=sc.LogSchema ) self.post_train_called: int = 0 @@ -612,7 +612,7 @@ def test_has_no_saver(self): class MockWindowComponent(CoreComponent): def __init__(self, name: str, config: CoreConfig) -> None: super().__init__( - name=name, type_="Dummy", config=config, input_schema=schemas.LogSchema, + name=name, type_="Dummy", config=config, input_schema=sc.LogSchema, args_buffer=ArgsBuffer(mode=BufferMode.WINDOW, size=2), ) self.train_data: list = [] diff --git a/tests/test_common/test_core_alert_aggregator.py b/tests/test_common/test_core_alert_aggregator.py index 88d09fdf..99cef585 100644 --- a/tests/test_common/test_core_alert_aggregator.py +++ b/tests/test_common/test_core_alert_aggregator.py @@ -3,7 +3,7 @@ ) from detectmatelibrary.utils.aux import time_test_mode -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc import pydantic import pytest @@ -81,8 +81,8 @@ def test_initialize_default(self) -> None: assert isinstance(detector, CoreAlertAggregator) assert detector.name == "TestAAG" assert isinstance(detector.config, CoreAlertAggregatorConfig) - assert detector.input_schema == schemas.DetectorSchema - assert detector.output_schema == schemas.AggregateSchema + assert detector.input_schema == sc.DetectorSchema + assert detector.output_schema == sc.AggregateSchema def test_incorrect_config_type(self) -> None: dummy_config2 = { @@ -100,13 +100,13 @@ def test_incorrect_config_type(self) -> None: def test_process_correct_input_schema(self) -> None: detector = MockupAlertAggregator(name="TestAAG", config=dummy_config) - data = schemas.DetectorSchema(dummy_schema).serialize() + data = sc.DetectorSchema(dummy_schema).serialize() result = detector.process(data) # no error should be produced assert isinstance(result, bytes) # and result should be bytes def test_process_input_schema_not_serialized(self) -> None: alert_aggregator = MockupAlertAggregator(name="TestAAG", config=MockupConfig()) - expected_result = schemas.AggregateSchema({ + expected_result = sc.AggregateSchema({ "__version__": "2.0.0", "detectorIDs": ["1"], "detectorTypes": ["dummy"], @@ -117,13 +117,13 @@ def test_process_input_schema_not_serialized(self) -> None: "logTimestamps": [-1], "alertDetails": {} }) - data = schemas.DetectorSchema(dummy_schema) + data = sc.DetectorSchema(dummy_schema) result = alert_aggregator.process(data) assert result == expected_result, f"result -> {result}" def test_process_input_schema_not_serialized_window_3(self) -> None: alert_aggregator = MockupAlertAggregator(name="TestDetector", config=MockupConfig(), buffer_size=3) - expected_result = schemas.AggregateSchema({ + expected_result = sc.AggregateSchema({ "__version__": "2.0.0", "detectorIDs": ["1", "1", "1"], "detectorTypes": ["dummy", "dummy", "dummy"], @@ -134,7 +134,7 @@ def test_process_input_schema_not_serialized_window_3(self) -> None: "logTimestamps": [-1, -1, -1], "alertDetails": {} }) - data = schemas.DetectorSchema(dummy_schema) + data = sc.DetectorSchema(dummy_schema) assert alert_aggregator.process(data) is None assert alert_aggregator.process(data) is None @@ -146,7 +146,7 @@ def test_process_input_schema_not_serialized_buffer_3(self) -> None: alert_aggregator = MockupAlertAggregatorBuffer( name="TestDetector", config=MockupConfig(), buffer_size=3 ) - expected_result = schemas.AggregateSchema({ + expected_result = sc.AggregateSchema({ "__version__": "2.0.0", "detectorIDs": ["1", "1", "1"], "detectorTypes": ["dummy", "dummy", "dummy"], @@ -157,7 +157,7 @@ def test_process_input_schema_not_serialized_buffer_3(self) -> None: "logTimestamps": [-1, -1, -1], "alertDetails": {} }) - data = schemas.DetectorSchema(dummy_schema) + data = sc.DetectorSchema(dummy_schema) assert alert_aggregator.process(data) is None assert alert_aggregator.process(data) is None @@ -167,7 +167,7 @@ def test_process_input_schema_not_serialized_buffer_3(self) -> None: def test_none_detector(self) -> None: alert_aggregator = NoneMockupAlertAggregator(name="TestDetector", config=MockupConfig()) - data = schemas.DetectorSchema(dummy_schema) + data = sc.DetectorSchema(dummy_schema) assert alert_aggregator.process(data) is None assert alert_aggregator.process(data) is not None diff --git a/tests/test_common/test_core_detector.py b/tests/test_common/test_core_detector.py index 20dfe73b..3ca12189 100644 --- a/tests/test_common/test_core_detector.py +++ b/tests/test_common/test_core_detector.py @@ -1,7 +1,7 @@ from detectmatelibrary.common.detector import CoreDetectorConfig, BufferMode from detectmatelibrary.common.detector import CoreDetector from detectmatelibrary.utils.aux import time_test_mode -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc import pydantic import pytest @@ -103,8 +103,8 @@ def test_initialize_default(self) -> None: assert isinstance(detector, CoreDetector) assert detector.name == "TestDetector" assert isinstance(detector.config, CoreDetectorConfig) - assert detector.input_schema == schemas.ParserSchema - assert detector.output_schema == schemas.DetectorSchema + assert detector.input_schema == sc.ParserSchema + assert detector.output_schema == sc.DetectorSchema def test_incorrect_config_type(self) -> None: dummy_config2 = { @@ -122,13 +122,13 @@ def test_incorrect_config_type(self) -> None: def test_process_correct_input_schema(self) -> None: detector = MockupDetector(name="TestDetector", config=dummy_config) - data = schemas.ParserSchema(dummy_schema).serialize() + data = sc.ParserSchema(dummy_schema).serialize() result = detector.process(data) # no error should be produced assert isinstance(result, bytes) # and result should be bytes def test_process_input_schema_not_serialized(self) -> None: detector = MockupDetector(name="TestDetector", config=MockupConfig()) - expected_result = schemas.DetectorSchema({ + expected_result = sc.DetectorSchema({ "__version__": "2.0.0", "detectorID": "TestDetector", "detectorType": "core_detector", @@ -140,13 +140,13 @@ def test_process_input_schema_not_serialized(self) -> None: "logTimestamps": [12121], "receivedTimestamp": 0, }) - data = schemas.ParserSchema(dummy_schema) + data = sc.ParserSchema(dummy_schema) result = detector.process(data) assert result == expected_result, f"result -> {result}" def test_process_input_schema_not_serialized_window_3(self) -> None: detector = MockupDetector_window(name="TestDetector", config=MockupConfig()) - expected_result = schemas.DetectorSchema({ + expected_result = sc.DetectorSchema({ "__version__": "2.0.0", "detectorID": "TestDetector", "detectorType": "core_detector", @@ -158,7 +158,7 @@ def test_process_input_schema_not_serialized_window_3(self) -> None: "logTimestamps": [12121, 12121, 12121], "receivedTimestamp": 0 }) - data = schemas.ParserSchema(dummy_schema) + data = sc.ParserSchema(dummy_schema) assert detector.process(data) is None assert detector.process(data) is None @@ -168,7 +168,7 @@ def test_process_input_schema_not_serialized_window_3(self) -> None: def test_process_input_schema_not_serialized_buffer_3(self) -> None: detector = MockupDetector_window(name="TestDetector01", config=MockupConfig()) - expected_result = schemas.DetectorSchema({ + expected_result = sc.DetectorSchema({ "__version__": "2.0.0", "detectorID": "TestDetector01", "detectorType": "core_detector", @@ -180,7 +180,7 @@ def test_process_input_schema_not_serialized_buffer_3(self) -> None: "logTimestamps": [12121, 12121, 12121], "receivedTimestamp": 0, }) - data = schemas.ParserSchema(dummy_schema) + data = sc.ParserSchema(dummy_schema) assert detector.process(data) is None assert detector.process(data) is None @@ -190,7 +190,7 @@ def test_process_input_schema_not_serialized_buffer_3(self) -> None: def test_none_detector(self) -> None: detector = NoneMockupDetector(name="TestDetector", config=MockupConfig()) - data = schemas.ParserSchema(dummy_schema) + data = sc.ParserSchema(dummy_schema) assert detector.process(data) is None assert detector.process(data) is not None diff --git a/tests/test_common/test_core_parser.py b/tests/test_common/test_core_parser.py index c28117dd..3d4cc05b 100644 --- a/tests/test_common/test_core_parser.py +++ b/tests/test_common/test_core_parser.py @@ -1,6 +1,6 @@ from detectmatelibrary.common.parser import CoreParser, CoreParserConfig, get_format_variables from detectmatelibrary.utils.aux import time_test_mode -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc import pydantic import pytest @@ -64,8 +64,8 @@ def test_initialize_default(self) -> None: assert isinstance(parser, CoreParser) assert parser.name == "TestParser" assert isinstance(parser.config, CoreParserConfig) - assert parser.input_schema == schemas.LogSchema - assert parser.output_schema == schemas.ParserSchema + assert parser.input_schema == sc.LogSchema + assert parser.output_schema == sc.ParserSchema def test_incorrect_config_type(self) -> None: invalid_args = { @@ -82,13 +82,13 @@ def test_incorrect_config_type(self) -> None: def test_process_correct_input_schema(self) -> None: parser = MockupParser(name="TestParser", config=default_args) - data = schemas.LogSchema({"logID": "1", "log": "This is a log."}).serialize() + data = sc.LogSchema({"logID": "1", "log": "This is a log."}).serialize() result = parser.process(data) # no error should be produced assert isinstance(result, bytes) # and result should be bytes def test_process_correct_input_schema_not_serialize(self) -> None: parser = MockupParser(name="TestParser", config=MockupConfig()) - expected_result = schemas.ParserSchema({ + expected_result = sc.ParserSchema({ "__version__": "2.0.0", "parserType": "core_parser", "parserID": "TestParser", @@ -103,14 +103,14 @@ def test_process_correct_input_schema_not_serialize(self) -> None: expected_result.variables = ["a", "b"] expected_result.headerVariables["Time"] = "0" - data = schemas.LogSchema({"logID": "1", "log": "This is a log."}) + data = sc.LogSchema({"logID": "1", "log": "This is a log."}) result = parser.process(data) assert result == expected_result, f"results {result} and expected {expected_result}" def test_process_ids(self) -> None: parser = MockupParser(name="TestParser", config=MockupConfig()) - data = schemas.LogSchema({ + data = sc.LogSchema({ "logID": "1", "log": "This is a log.", "logSource": "", "hostname": "" }) @@ -122,7 +122,7 @@ def test_process_ids(self) -> None: def test_none_parser(self) -> None: parser = NoneMockupParser(name="TestParser", config=MockupConfig()) - data = schemas.LogSchema({ + data = sc.LogSchema({ "logID": "1", "log": "This is a log.", "logSource": "", "hostname": "" }) diff --git a/tests/test_common/test_extract_timestamp.py b/tests/test_common/test_extract_timestamp.py index a775a621..f441851e 100644 --- a/tests/test_common/test_extract_timestamp.py +++ b/tests/test_common/test_extract_timestamp.py @@ -1,5 +1,5 @@ from detectmatelibrary.common.detector import _extract_timestamp -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc class TestCoreDetector: @@ -50,7 +50,7 @@ def test_various_time_formats(self) -> None: ("Wednesday, March 4, 2026 14:18:00", EXPECTED_UTC), ] for time_str, expected in test_cases: - schema = schemas.ParserSchema({**dummy_schema, "headerVariables": {"Time": time_str}}) + schema = sc.ParserSchema({**dummy_schema, "headerVariables": {"Time": time_str}}) result = _extract_timestamp(schema) assert result == [expected], ( f"Format '{time_str}': expected [{expected}], got {result}" @@ -59,9 +59,9 @@ def test_various_time_formats(self) -> None: def test_microsecond_epoch_fits_the_schema(self) -> None: """Issue #271: a microsecond epoch folded only once landed in milliseconds and overflowed the int32 timestamp fields.""" - schema = schemas.DetectorSchema({ + schema = sc.DetectorSchema({ "logTimestamps": _extract_timestamp( - schemas.ParserSchema({"headerVariables": {"Time": "1643114452000000"}}) + sc.ParserSchema({"headerVariables": {"Time": "1643114452000000"}}) ), }) assert str(schema) # rebuilds the protobuf -- used to raise ValueError @@ -71,11 +71,11 @@ def test_timestamp_fields_hold_more_than_int32(self) -> None: """Timestamps are int64, so they survive 2038 (and a stray ms value).""" for schema, field, value in [ - (schemas.ParserSchema(), "receivedTimestamp", 2**31), - (schemas.ParserSchema(), "parsedTimestamp", 2**31), - (schemas.DetectorSchema(), "detectedTimestamp", 2**31), - (schemas.DetectorSchema(), "receivedTimestamp", 1643114452000), - (schemas.AggregateSchema(), "aggregatedTimestamp", 2**31), + (sc.ParserSchema(), "receivedTimestamp", 2**31), + (sc.ParserSchema(), "parsedTimestamp", 2**31), + (sc.DetectorSchema(), "detectedTimestamp", 2**31), + (sc.DetectorSchema(), "receivedTimestamp", 1643114452000), + (sc.AggregateSchema(), "aggregatedTimestamp", 2**31), ]: schema[field] = value assert getattr(schema.get_schema(), field) == value diff --git a/tests/test_common/test_train_buffer.py b/tests/test_common/test_train_buffer.py index 4f1fa4c2..665e49ae 100644 --- a/tests/test_common/test_train_buffer.py +++ b/tests/test_common/test_train_buffer.py @@ -5,19 +5,19 @@ import pytest -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc from detectmatelibrary.common._core_op._train_buffer import TrainBuffer from detectmatelibrary.utils.data_buffer import ArgsBuffer, BufferMode, DataBuffer _MOD = "detectmatelibrary.common._core_op._train_buffer" -def _log(i: int) -> schemas.BaseSchema: +def _log(i: int) -> sc.BaseSchema: # .copy() gives the BaseSchema that CoreComponent actually buffers - return schemas.LogSchema({"logID": str(i), "logSource": "test", "hostname": "h"}).copy() + return sc.LogSchema({"logID": str(i), "logSource": "test", "hostname": "h"}).copy() -def _windows(n: int, size: int, start: int = 0) -> list[list[schemas.BaseSchema]]: +def _windows(n: int, size: int, start: int = 0) -> list[list[sc.BaseSchema]]: """Sliding windows over logs start..start+n-1, as a WINDOW-mode CoreComponent buffers them.""" data_buffer = DataBuffer(ArgsBuffer(BufferMode.WINDOW, size=size)) @@ -26,13 +26,13 @@ def _windows(n: int, size: int, start: int = 0) -> list[list[schemas.BaseSchema] def _count_serialize(monkeypatch) -> dict[str, int]: calls = {"n": 0} - real = schemas.BaseSchema.serialize + real = sc.BaseSchema.serialize def counting(self): calls["n"] += 1 return real(self) - monkeypatch.setattr(schemas.BaseSchema, "serialize", counting) + monkeypatch.setattr(sc.BaseSchema, "serialize", counting) return calls @@ -46,7 +46,7 @@ def _ids(records) -> list[str]: def _buffer(tmp_path: Path, max_records: int, window: int | None = None, name: str = "c") -> TrainBuffer: return TrainBuffer( - schemas.LogSchema, max_records=max_records, spill_dir=str(tmp_path), window=window, name=name + sc.LogSchema, max_records=max_records, spill_dir=str(tmp_path), window=window, name=name ) @@ -73,7 +73,7 @@ def test_spill_replays_all_records_in_order(self, tmp_path): assert len(list(run.glob("*.parquet"))) == 3 out = list(buf) assert _ids(out) == [str(i) for i in range(10)] - assert all(r.schema_class is schemas.LogSchema().schema_class for r in out) + assert all(r.schema_class is sc.LogSchema().schema_class for r in out) assert _run_dirs(tmp_path) == [] def test_spilled_batches_replay_in_order(self, tmp_path): @@ -126,7 +126,7 @@ def records(start): def test_default_dir_is_the_system_temp_dir(self, tmp_path, monkeypatch): monkeypatch.setattr(tempfile, "tempdir", str(tmp_path)) - buf = TrainBuffer(schemas.LogSchema, max_records=1, name="c") + buf = TrainBuffer(sc.LogSchema, max_records=1, name="c") buf.add(_log(0)) assert len(_run_dirs(tmp_path)) == 1 assert _ids(buf) == ["0"] diff --git a/tests/test_detectors/test_bigram_frequency_detector.py b/tests/test_detectors/test_bigram_frequency_detector.py index 25690a27..e52a6b78 100644 --- a/tests/test_detectors/test_bigram_frequency_detector.py +++ b/tests/test_detectors/test_bigram_frequency_detector.py @@ -17,7 +17,7 @@ from detectmatelibrary.constants import GLOBAL_EVENT_ID from detectmatelibrary.parsers.template_matcher import MatcherParser from detectmatelibrary.helper.from_to import From -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc from detectmatelibrary.utils.aux import time_test_mode from tests.test_data import AUDIT_LOG, AUDIT_TEMPLATES, TRAIN_UNTIL # Set time test mode for consistent timestamps @@ -107,8 +107,8 @@ def test_default_initialization(self): assert detector.name == "BigramFrequencyDetector" assert hasattr(detector, 'config') assert detector.data_buffer.mode == BufferMode.NO_BUF - assert detector.input_schema == schemas.ParserSchema - assert detector.output_schema == schemas.DetectorSchema + assert detector.input_schema == sc.ParserSchema + assert detector.output_schema == sc.DetectorSchema assert hasattr(detector, 'persistency') def test_custom_config_initialization(self): @@ -129,7 +129,7 @@ def test_train_multiple_values(self): # Train with multiple values (only event 1 should be tracked per config) for event in range(3): for level in ["INFO", "WARNING", "ERROR"]: - parser_data = schemas.ParserSchema({ + parser_data = sc.ParserSchema({ "parserType": "test", "eventID": event, "template": "test template", @@ -161,7 +161,7 @@ def test_detect_known_value_no_alert(self): detector = BigramFrequencyDetector(config=config, name="MultipleDetector") # Train with a value - train_data = schemas.ParserSchema({ + train_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -175,7 +175,7 @@ def test_detect_known_value_no_alert(self): detector.train(train_data) # Detect with the same value - test_data = schemas.ParserSchema({ + test_data = sc.ParserSchema({ "parserType": "test", "eventID": 12, "template": "test template", @@ -186,7 +186,7 @@ def test_detect_known_value_no_alert(self): "log": "test log message", "headerVariables": {"level": "CRITICAL"} }) - output = schemas.DetectorSchema() + output = sc.DetectorSchema() result = detector.detect(test_data, output) @@ -197,7 +197,7 @@ def test_detect_known_value_alert(self): detector = BigramFrequencyDetector(config=config, name="MultipleDetector") # Train with a value - train_data = schemas.ParserSchema({ + train_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -211,7 +211,7 @@ def test_detect_known_value_alert(self): detector.train(train_data) # Detect with the same value - test_data = schemas.ParserSchema({ + test_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -222,7 +222,7 @@ def test_detect_known_value_alert(self): "log": "test log message", "headerVariables": {"level": "CRITICAL"} }) - output = schemas.DetectorSchema() + output = sc.DetectorSchema() result = detector.detect(test_data, output) @@ -271,7 +271,7 @@ def test_audit_log_anomalies(self): detected_ids: set[str] = set() for log in logs[TRAIN_UNTIL:]: - output = schemas.DetectorSchema() + output = sc.DetectorSchema() if detector.detect(log, output_=output): detected_ids.add(log["logID"]) @@ -314,7 +314,7 @@ def test_audit_log_anomalie_fed(self): detected_ids: set[str] = set() for log in logs[TRAIN_UNTIL:]: - output = schemas.DetectorSchema() + output = sc.DetectorSchema() if detector1.detect(log, output_=output): detected_ids.add(log["logID"]) @@ -389,7 +389,7 @@ def test_global_instance_detects_new_type(self): detected_ids: set[str] = set() for log in logs[TRAIN_UNTIL:]: - output = schemas.DetectorSchema() + output = sc.DetectorSchema() if detector.detect(log, output_=output): assert all(key.startswith("Global -") for key in output["alertDetails"]) detected_ids.add(log["logID"]) @@ -417,7 +417,7 @@ def test_set_configuration_preserves_persist(self): def _parser_data(event_id, level, var_value): - return schemas.ParserSchema({ + return sc.ParserSchema({ "parserType": "test", "eventID": event_id, "template": "test template", @@ -478,7 +478,7 @@ def test_detect_helper_survives_missing_variable(self): detector.train(_parser_data(1, "INFO", "abc")) # Detect with a parser line that lacks the 'test' variable - test_data = schemas.ParserSchema({ + test_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -489,7 +489,7 @@ def test_detect_helper_survives_missing_variable(self): "log": "test log message", "headerVariables": {"level": "INFO"}, }) - output = schemas.DetectorSchema() + output = sc.DetectorSchema() # Must not raise TypeError result = detector.detect(test_data, output) assert isinstance(result, bool) @@ -504,7 +504,7 @@ def test_default_freqs_false_does_not_consult_fallback(self): # Detect a typical English-looking value. Without fallback, sparse # per-var table → low prob → alert expected. test_data = _parser_data(1, "INFO", "the") - output = schemas.DetectorSchema() + output = sc.DetectorSchema() result = detector.detect(test_data, output) assert result is True @@ -535,7 +535,7 @@ def test_default_freqs_true_consults_fallback(self): # Detect a common English word — fallback should produce non-zero probs test_data = _parser_data(1, "INFO", "the") - output = schemas.DetectorSchema() + output = sc.DetectorSchema() result = detector.detect(test_data, output) assert result is False @@ -625,7 +625,7 @@ def test_two_variables_have_independent_freq_tables(self): } } detector = BigramFrequencyDetector(config=cfg, name="TwoVarDetector") - parser_data = schemas.ParserSchema({ + parser_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", diff --git a/tests/test_detectors/test_charset_detector.py b/tests/test_detectors/test_charset_detector.py index e4c3fcba..0a49b03c 100644 --- a/tests/test_detectors/test_charset_detector.py +++ b/tests/test_detectors/test_charset_detector.py @@ -15,7 +15,7 @@ from detectmatelibrary.constants import GLOBAL_EVENT_ID from detectmatelibrary.parsers.template_matcher import MatcherParser from detectmatelibrary.helper.from_to import From -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc from detectmatelibrary.utils.aux import time_test_mode from tests.test_data import AUDIT_LOG, AUDIT_TEMPLATES, TRAIN_UNTIL @@ -74,8 +74,8 @@ def test_default_initialization(self): assert detector.name == "CharsetDetector" assert hasattr(detector, 'config') assert detector.data_buffer.mode == BufferMode.NO_BUF - assert detector.input_schema == schemas.ParserSchema - assert detector.output_schema == schemas.DetectorSchema + assert detector.input_schema == sc.ParserSchema + assert detector.output_schema == sc.DetectorSchema assert hasattr(detector, 'persistency') def test_custom_config_initialization(self): @@ -120,7 +120,7 @@ def test_train_multiple_values(self): # Train with multiple values (only event 1 should be tracked per config) for event in range(3): for level in ["INFO", "WARNING", "ERROR"]: - parser_data = schemas.ParserSchema({ + parser_data = sc.ParserSchema({ "parserType": "test", "eventID": event, "template": "test template", @@ -151,7 +151,7 @@ def test_detect_known_value_no_alert(self): detector = CharsetDetector(config=config, name="MultipleDetector") # Train with a value - train_data = schemas.ParserSchema({ + train_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -165,7 +165,7 @@ def test_detect_known_value_no_alert(self): detector.train(train_data) # Detect with the same value - test_data = schemas.ParserSchema({ + test_data = sc.ParserSchema({ "parserType": "test", "eventID": 12, "template": "test template", @@ -176,7 +176,7 @@ def test_detect_known_value_no_alert(self): "log": "test log message", "headerVariables": {"level": "CRITICAL"} }) - output = schemas.DetectorSchema() + output = sc.DetectorSchema() result = detector.detect(test_data, output) @@ -187,7 +187,7 @@ def test_detect_known_value_alert(self): detector = CharsetDetector(config=config, name="MultipleDetector") # Train with a value - train_data = schemas.ParserSchema({ + train_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -201,7 +201,7 @@ def test_detect_known_value_alert(self): detector.train(train_data) # Detect with the same value - test_data = schemas.ParserSchema({ + test_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -212,7 +212,7 @@ def test_detect_known_value_alert(self): "log": "test log message", "headerVariables": {"level": "CRITICAL"} }) - output = schemas.DetectorSchema() + output = sc.DetectorSchema() result = detector.detect(test_data, output) @@ -241,7 +241,7 @@ def test_detect_unknown_chars_reported_per_variable(self): } detector = CharsetDetector(config=cfg, name="Single") - train = schemas.ParserSchema({ + train = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "t", "variables": ["abc"], "logID": "1", "parsedLogID": "1", "parserID": "p", "log": "l", "headerVariables": {}, @@ -249,22 +249,22 @@ def test_detect_unknown_chars_reported_per_variable(self): detector.train(train) # All known chars - ok = schemas.ParserSchema({ + ok = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "t", "variables": ["cba"], "logID": "2", "parsedLogID": "2", "parserID": "p", "log": "l", "headerVariables": {}, }) - out = schemas.DetectorSchema() + out = sc.DetectorSchema() assert not detector.detect(ok, out) assert out.score == 0.0 # Unknown chars 'x' and 'y' - bad = schemas.ParserSchema({ + bad = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "t", "variables": ["axy"], "logID": "3", "parsedLogID": "3", "parserID": "p", "log": "l", "headerVariables": {}, }) - out = schemas.DetectorSchema() + out = sc.DetectorSchema() assert detector.detect(bad, out) assert out.score == 1.0 assert any("'x'" in msg and "'y'" in msg for msg in out["alertDetails"].values()) @@ -307,7 +307,7 @@ def test_audit_log_anomalies(self): detected_ids: set[str] = set() for log in logs[TRAIN_UNTIL:]: - output = schemas.DetectorSchema() + output = sc.DetectorSchema() if detector.detect(log, output_=output): detected_ids.add(log["logID"]) @@ -335,7 +335,7 @@ def test_audit_log_anomalies_to_binary(self): detected_ids: set[str] = set() for log in logs[TRAIN_UNTIL:]: - output = schemas.DetectorSchema() + output = sc.DetectorSchema() if detector2.detect(log, output_=output): detected_ids.add(log["logID"]) @@ -443,7 +443,7 @@ def test_global_instance_detects_new_type(self): detected_ids: set[str] = set() for log in logs[TRAIN_UNTIL:]: - output = schemas.DetectorSchema() + output = sc.DetectorSchema() if detector.detect(log, output_=output): assert all(key.startswith("Global -") for key in output["alertDetails"]) detected_ids.add(log["logID"]) @@ -459,7 +459,7 @@ def test_persist_flag_survives_set_configuration(self): # Feed configure() with a couple of stable-variable samples for _ in range(5): - sample = schemas.ParserSchema({ + sample = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "t", "variables": ["abc"], "logID": "x", "parsedLogID": "x", "parserID": "p", "log": "l", diff --git a/tests/test_detectors/test_deeplog_detector.py b/tests/test_detectors/test_deeplog_detector.py index ae64cc2b..5d8ebc08 100644 --- a/tests/test_detectors/test_deeplog_detector.py +++ b/tests/test_detectors/test_deeplog_detector.py @@ -10,7 +10,7 @@ from tests.test_data import AUDIT_LOG, AUDIT_TEMPLATES, TRAIN_UNTIL -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc import pytest @@ -35,7 +35,7 @@ def test_end2end_autoconfig(self) -> None: for j in range(2): for i in [1, 2, 3, 4, 5, 1, 2]: - deeplog.process(schemas.ParserSchema({"eventID": i})) + deeplog.process(sc.ParserSchema({"eventID": i})) if j == 0: assert deeplog.get_state() == "Configuring" @@ -44,7 +44,7 @@ def test_end2end_autoconfig(self) -> None: for _ in range(2): for i in [1, 2, 3, 4, 5, 1, 2]: - deeplog.process(schemas.ParserSchema({"eventID": i})) + deeplog.process(sc.ParserSchema({"eventID": i})) assert deeplog.get_state() == "Default" @pytest.mark.ignored @@ -79,7 +79,7 @@ def test_end2end_no_autoconfig(self) -> None: for j in range(2): for i in [1, 2, 3, 4, 5, 1, 2]: - deeplog.process(schemas.ParserSchema({"eventID": i})) + deeplog.process(sc.ParserSchema({"eventID": i})) if j == 0: assert deeplog.get_state() == "Training" diff --git a/tests/test_detectors/test_event_sequence_detector.py b/tests/test_detectors/test_event_sequence_detector.py index aab1629b..6c2c6a8a 100644 --- a/tests/test_detectors/test_event_sequence_detector.py +++ b/tests/test_detectors/test_event_sequence_detector.py @@ -17,7 +17,7 @@ ) from detectmatelibrary.parsers.template_matcher import MatcherParser from detectmatelibrary.helper.from_to import From -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc from detectmatelibrary.base_detectors import PersistConfig from detectmatelibrary.common._core_op._fit_logic import EnumState from detectmatelibrary.utils.aux import time_test_mode @@ -48,7 +48,7 @@ def _make_schema(event_id, template="test template", log_id="1", log_format_variables=None): - return schemas.ParserSchema({ + return sc.ParserSchema({ "parserType": "test", "eventID": event_id, "template": template, @@ -70,8 +70,8 @@ def test_default_initialization(self): assert detector.name == "EventSequenceDetector" assert hasattr(detector, "config") assert detector.data_buffer.mode == BufferMode.NO_BUF - assert detector.input_schema == schemas.ParserSchema - assert detector.output_schema == schemas.DetectorSchema + assert detector.input_schema == sc.ParserSchema + assert detector.output_schema == sc.DetectorSchema assert hasattr(detector, "persistency") # unconfigured until auto-config picks a length assert detector.config.fixed_window_size is None @@ -121,8 +121,8 @@ def test_detect_known_sequence_no_alert(self): detector.train(_make_schema(event_id)) # The detection window is independent of the training one, so prime it # with a 3 before appending the 1 that completes the known (3, 1) - detector.detect(_make_schema(3, log_id="6"), schemas.DetectorSchema()) - output = schemas.DetectorSchema() + detector.detect(_make_schema(3, log_id="6"), sc.DetectorSchema()) + output = sc.DetectorSchema() result = detector.detect(_make_schema(1, log_id="7"), output) assert not result @@ -135,8 +135,8 @@ def test_detect_unknown_sequence_alert(self): detector.train(_make_schema(event_id)) # (3, 9) was never seen during training - detector.detect(_make_schema(3, log_id="6"), schemas.DetectorSchema()) - output = schemas.DetectorSchema() + detector.detect(_make_schema(3, log_id="6"), sc.DetectorSchema()) + output = sc.DetectorSchema() result = detector.detect(_make_schema(9, log_id="7"), output) assert result @@ -147,7 +147,7 @@ def test_detect_below_window_length_no_alert(self): detector = EventSequenceDetector(config=config, name="MultipleDetector") # First event with an empty detector: window not yet full, can't be an anomaly - output = schemas.DetectorSchema() + output = sc.DetectorSchema() result = detector.detect(_make_schema(1), output) assert not result @@ -161,7 +161,7 @@ def test_reset_window_clears_state(self): detector = EventSequenceDetector(config=config, name="MultipleDetector") detector.train(_make_schema(1)) - detector.detect(_make_schema(1), schemas.DetectorSchema()) + detector.detect(_make_schema(1), sc.DetectorSchema()) assert len(detector._train_window) == 1 assert len(detector._detect_window) == 1 @@ -223,7 +223,7 @@ def test_audit_log_anomalies(self): detected_ids: set[str] = set() for log in logs[TRAIN_UNTIL:]: - output = schemas.DetectorSchema() + output = sc.DetectorSchema() if detector.detect(log, output_=output): detected_ids.add(log["logID"]) @@ -266,7 +266,7 @@ def test_audit_log_anomalie_fed(self): detected_ids: set[str] = set() for log in logs[TRAIN_UNTIL:]: - output = schemas.DetectorSchema() + output = sc.DetectorSchema() if detector1.detect(log, output_=output): detected_ids.add(log["logID"]) @@ -305,7 +305,7 @@ def test_audit_log_anomalie_binary(self): detected_ids: set[str] = set() for log in logs[TRAIN_UNTIL:]: - output = schemas.DetectorSchema() + output = sc.DetectorSchema() if detector1.detect(log, output_=output): detected_ids.add(log["logID"]) @@ -458,7 +458,7 @@ def test_no_stable_window_size_generates_no_instance(self): for event_id in [4, 5, 6, 7]: detector.train(_make_schema(event_id)) assert detector.get_known_sequences() == set() - assert not detector.detect(_make_schema(9), schemas.DetectorSchema()) + assert not detector.detect(_make_schema(9), sc.DetectorSchema()) def test_unstable_stream_generates_no_instance(self): """The other route to 'no stable candidate': every window fills and @@ -488,7 +488,7 @@ def test_unstable_stream_generates_no_instance(self): for event_id in [1, 2, 3, 1, 2, 3]: detector.train(_make_schema(event_id)) assert detector.get_known_sequences() == set() - assert not detector.detect(_make_schema(9), schemas.DetectorSchema()) + assert not detector.detect(_make_schema(9), sc.DetectorSchema()) def test_auto_config_off_without_fixed_window_size_is_inert(self): detector = EventSequenceDetector( @@ -500,7 +500,7 @@ def test_auto_config_off_without_fixed_window_size_is_inert(self): detector.train(_make_schema(event_id)) assert detector.get_known_sequences() == set() - assert not detector.detect(_make_schema(9), schemas.DetectorSchema()) + assert not detector.detect(_make_schema(9), sc.DetectorSchema()) def test_auto_config_off_stays_inert_through_a_configure_phase(self): """A configure window alone must not pick a length: auto_config=False @@ -598,7 +598,7 @@ def test_reload_adopts_persisted_sequence_length(self): # the known cycle must not alert under the restored model alerted = [ - det2.detect(_make_schema(event_id), schemas.DetectorSchema()) + det2.detect(_make_schema(event_id), sc.DetectorSchema()) for event_id in [1, 2, 3, 1, 2, 3] ] assert not any(alerted) diff --git a/tests/test_detectors/test_logbert_detector.py b/tests/test_detectors/test_logbert_detector.py index c5c5cc68..1b2859cc 100644 --- a/tests/test_detectors/test_logbert_detector.py +++ b/tests/test_detectors/test_logbert_detector.py @@ -10,7 +10,7 @@ from tests.test_data import AUDIT_LOG, AUDIT_TEMPLATES, TRAIN_UNTIL -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc import pytest @@ -35,7 +35,7 @@ def test_end2end_autoconfig(self) -> None: for j in range(2): for i in [1, 2, 3, 4, 5, 1, 2]: - logbert.process(schemas.ParserSchema({"eventID": i})) + logbert.process(sc.ParserSchema({"eventID": i})) if j == 0: assert logbert.get_state() == "Configuring" @@ -44,7 +44,7 @@ def test_end2end_autoconfig(self) -> None: for _ in range(2): for i in [1, 2, 3, 4, 5, 1, 2]: - logbert.process(schemas.ParserSchema({"eventID": i})) + logbert.process(sc.ParserSchema({"eventID": i})) assert logbert.get_state() == "Default" @pytest.mark.ignored @@ -84,7 +84,7 @@ def test_end2end_no_autoconfig(self) -> None: for j in range(2): for i in [1, 2, 3, 4, 5, 1, 2]: - logbert.process(schemas.ParserSchema({"eventID": i})) + logbert.process(sc.ParserSchema({"eventID": i})) if j == 0: assert logbert.get_state() == "Training" diff --git a/tests/test_detectors/test_mismatch_warnings.py b/tests/test_detectors/test_mismatch_warnings.py index 438efc43..a2aec88e 100644 --- a/tests/test_detectors/test_mismatch_warnings.py +++ b/tests/test_detectors/test_mismatch_warnings.py @@ -9,7 +9,7 @@ import logging import pytest -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc from detectmatelibrary.detectors.new_value_detector import NewValueDetector from detectmatelibrary.detectors.new_value_combo_detector import NewValueComboDetector @@ -18,8 +18,8 @@ def _make_parser_schema( event_id: int, variables: list, log_format_variables: dict, -) -> schemas.ParserSchema: - return schemas.ParserSchema({ +) -> sc.ParserSchema: + return sc.ParserSchema({ "parserType": "test", "eventID": event_id, "template": "test template", diff --git a/tests/test_detectors/test_new_event_detector.py b/tests/test_detectors/test_new_event_detector.py index 342cd1de..ca02ccbe 100644 --- a/tests/test_detectors/test_new_event_detector.py +++ b/tests/test_detectors/test_new_event_detector.py @@ -12,7 +12,7 @@ BufferMode from detectmatelibrary.parsers.template_matcher import MatcherParser from detectmatelibrary.helper.from_to import From -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc from detectmatelibrary.utils.aux import time_test_mode from detectmatelibrary.common._core_op._fit_logic import EnumState from detectmatelibrary.constants import GLOBAL_EVENT_ID @@ -55,8 +55,8 @@ def test_default_initialization(self): assert detector.name == "NewEventDetector" assert hasattr(detector, 'config') assert detector.data_buffer.mode == BufferMode.NO_BUF - assert detector.input_schema == schemas.ParserSchema - assert detector.output_schema == schemas.DetectorSchema + assert detector.input_schema == sc.ParserSchema + assert detector.output_schema == sc.DetectorSchema assert hasattr(detector, 'persistency') def test_custom_config_initialization(self): @@ -76,7 +76,7 @@ def test_train_multiple_event_ids(self): detector = NewEventDetector(config=config, name="MultipleDetector") event_ids = {0, 3, 8, 9} for event in event_ids: - parser_data = schemas.ParserSchema({ + parser_data = sc.ParserSchema({ "parserType": "test", "eventID": event, "template": "test template", @@ -101,7 +101,7 @@ def test_detect_known_event_id_no_alert(self): detector = NewEventDetector(config=config, name="MultipleDetector") # Train with an event_id - train_data = schemas.ParserSchema({ + train_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -115,7 +115,7 @@ def test_detect_known_event_id_no_alert(self): detector.train(train_data) # Detect with the same event_id - test_data = schemas.ParserSchema({ + test_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -126,7 +126,7 @@ def test_detect_known_event_id_no_alert(self): "log": "test log message", "headerVariables": {"level": "CRITICAL"} }) - output = schemas.DetectorSchema() + output = sc.DetectorSchema() result = detector.detect(test_data, output) @@ -171,7 +171,7 @@ def test_audit_log_anomalies(self): detected_ids: set[str] = set() for i, log in enumerate(logs[TRAIN_UNTIL:]): - output = schemas.DetectorSchema() + output = sc.DetectorSchema() if detector.detect(log, output_=output): detected_ids.add(log["logID"]) @@ -204,7 +204,7 @@ def test_audit_log_anomalie_fed(self): detected_ids: set[str] = set() for log in logs[TRAIN_UNTIL:]: - output = schemas.DetectorSchema() + output = sc.DetectorSchema() if detector1.detect(log, output_=output): detected_ids.add(log["logID"]) @@ -229,7 +229,7 @@ def test_audit_log_anomalie_binary(self): detected_ids: set[str] = set() for log in logs[TRAIN_UNTIL:]: - output = schemas.DetectorSchema() + output = sc.DetectorSchema() if detector2.detect(log, output_=output): detected_ids.add(log["logID"]) @@ -303,7 +303,7 @@ def test_global_instance_detects_new_type(self): detected_ids: set[str] = set() for log in logs[TRAIN_UNTIL:]: - output = schemas.DetectorSchema() + output = sc.DetectorSchema() if detector.detect(log, output_=output): assert all(key.startswith("Global -") for key in output["alertDetails"]) detected_ids.add(log["logID"]) @@ -315,7 +315,7 @@ class TestNewEventDetectorFederationCleanup: def test_finalize_federation_removes_the_slow_table(self, tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) # the slow store writes .temp/ CSVs here detector = NewEventDetector(config=NewEventDetectorConfig(allow_fed=True)) - detector.train(schemas.ParserSchema({"eventID": 1, "template": "test template"})) + detector.train(sc.ParserSchema({"eventID": 1, "template": "test template"})) assert (tmp_path / ".temp").exists() detector.finalize_federation() diff --git a/tests/test_detectors/test_new_value_combo_detector.py b/tests/test_detectors/test_new_value_combo_detector.py index 64eba628..c7729a96 100644 --- a/tests/test_detectors/test_new_value_combo_detector.py +++ b/tests/test_detectors/test_new_value_combo_detector.py @@ -11,7 +11,7 @@ from detectmatelibrary.common._config import generate_detector_config from detectmatelibrary.parsers.template_matcher import MatcherParser from detectmatelibrary.helper.from_to import From -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc from detectmatelibrary.utils.aux import time_test_mode from tests.test_data import AUDIT_LOG, AUDIT_TEMPLATES, TRAIN_UNTIL @@ -71,8 +71,8 @@ def test_default_initialization(self): assert detector.name == "NewValueComboDetector" assert detector.data_buffer.mode == BufferMode.NO_BUF - assert detector.input_schema == schemas.ParserSchema - assert detector.output_schema == schemas.DetectorSchema + assert detector.input_schema == sc.ParserSchema + assert detector.output_schema == sc.DetectorSchema def test_custom_config_initialization(self): detector = NewValueComboDetector(name="CustomInit", config=config) @@ -89,7 +89,7 @@ def test_train_multiple_values(self): # Train with multiple values for event in range(3): for level in ["INFO", "WARNING", "ERROR"]: - parser_data = schemas.ParserSchema({ + parser_data = sc.ParserSchema({ "parserType": "test", "eventID": event, "template": "test template", @@ -113,7 +113,7 @@ def test_detect_known_value_no_alert(self): detector = NewValueComboDetector(config=config, name="MultipleDetector") # Train with a value - train_data = schemas.ParserSchema({ + train_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -126,7 +126,7 @@ def test_detect_known_value_no_alert(self): }) detector.train(train_data) - train_data = schemas.ParserSchema({ + train_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -140,7 +140,7 @@ def test_detect_known_value_no_alert(self): detector.train(train_data) # Detect with the same value - test_data = schemas.ParserSchema({ + test_data = sc.ParserSchema({ "parserType": "test", "eventID": 12, "template": "test template", @@ -151,7 +151,7 @@ def test_detect_known_value_no_alert(self): "log": "test log message", "headerVariables": {"level": "CRITICAL"} }) - output = schemas.DetectorSchema() + output = sc.DetectorSchema() result = detector.detect(test_data, output) @@ -162,7 +162,7 @@ def test_detect_known_value_alert(self): detector = NewValueComboDetector(config=config, name="MultipleDetector") # Train with a value - train_data = schemas.ParserSchema({ + train_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -175,7 +175,7 @@ def test_detect_known_value_alert(self): }) detector.train(train_data) - train_data = schemas.ParserSchema({ + train_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -189,7 +189,7 @@ def test_detect_known_value_alert(self): detector.train(train_data) # Detect with an unknown combination (pos 1 has a new value) - test_data = schemas.ParserSchema({ + test_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -200,7 +200,7 @@ def test_detect_known_value_alert(self): "log": "test log message", "headerVariables": {"level": "CRITICAL"} }) - output = schemas.DetectorSchema() + output = sc.DetectorSchema() result = detector.detect(test_data, output) @@ -253,7 +253,7 @@ def test_configure_method_ingests_events(self): # Configure with sample events for event_id in [1, 2, 3]: - parser_data = schemas.ParserSchema({ + parser_data = sc.ParserSchema({ "parserType": "test", "eventID": event_id, "template": f"Template {event_id}", @@ -279,7 +279,7 @@ def test_set_configuration_updates_config(self): # Configure with events that have varying stability for i in range(10): - parser_data = schemas.ParserSchema({ + parser_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "Template 1", @@ -307,7 +307,7 @@ def test_configuration_workflow(self): training_data = [] for i in range(20): for event_id in [1, 2]: - parser_data = schemas.ParserSchema({ + parser_data = sc.ParserSchema({ "parserType": "test", "eventID": event_id, "template": f"Template {event_id}", @@ -329,7 +329,7 @@ def test_configuration_workflow(self): detector.train(data) # Step 4: Verify detector can detect anomalies - test_data = schemas.ParserSchema({ + test_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "Template 1", @@ -340,7 +340,7 @@ def test_configuration_workflow(self): "log": "test log", "headerVariables": {"level": "INFO"} }) - output = schemas.DetectorSchema() + output = sc.DetectorSchema() # Should detect anomaly due to new combination result = detector.detect(test_data, output) @@ -352,7 +352,7 @@ def test_set_configuration_with_combo_size(self): # Configure with multiple variable events for i in range(15): - parser_data = schemas.ParserSchema({ + parser_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "Multi-var template", @@ -377,7 +377,7 @@ def test_configuration_with_no_stable_variables(self): # Configure with highly variable data for i in range(10): - parser_data = schemas.ParserSchema({ + parser_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "Variable template", @@ -404,7 +404,7 @@ class TestNewValueComboDetectorEndToEnd: @staticmethod def _make_schema(event_id, template, variables, log_format_variables, log_id="1"): - return schemas.ParserSchema({ + return sc.ParserSchema({ "parserType": "test", "eventID": event_id, "template": template, @@ -457,7 +457,7 @@ def test_configure_train_detect_anomalous_combo(self): ["MALICIOUS_USER", "MALICIOUS_IP"], {"username": "MALICIOUS_USER", "src_ip": "MALICIOUS_IP"}, ) - output = schemas.DetectorSchema() + output = sc.DetectorSchema() result = detector.detect(anomalous, output) assert result is True @@ -501,7 +501,7 @@ def test_configure_train_detect_benign_combo(self): ["user1", "192.168.1.100"], {"username": "user1", "src_ip": "192.168.1.100"}, ) - output = schemas.DetectorSchema() + output = sc.DetectorSchema() result = detector.detect(benign, output) assert result is False @@ -547,7 +547,7 @@ def test_configure_only_selects_stable_event_types(self): ["hacker", "/etc/shadow"], {"username": "hacker", "filepath": "/etc/shadow"}, ) - output = schemas.DetectorSchema() + output = sc.DetectorSchema() result = detector.detect(file_event, output) assert result is False @@ -583,7 +583,7 @@ def test_audit_log_anomalies(self): detected_ids: set[str] = set() for log in logs[TRAIN_UNTIL:]: - output = schemas.DetectorSchema() + output = sc.DetectorSchema() if detector.detect(log, output_=output): detected_ids.add(log["logID"]) @@ -610,7 +610,7 @@ def test_audit_log_anomalies_to_binary(self): detected_ids: set[str] = set() for log in logs[TRAIN_UNTIL:]: - output = schemas.DetectorSchema() + output = sc.DetectorSchema() if detector2.detect(log, output_=output): detected_ids.add(log["logID"]) @@ -644,7 +644,7 @@ def test_audit_log_anomalie_fed(self): detected_ids: set[str] = set() for log in logs[TRAIN_UNTIL:]: - output = schemas.DetectorSchema() + output = sc.DetectorSchema() if detector1.detect(log, output_=output): detected_ids.add(log["logID"]) @@ -670,7 +670,7 @@ def test_classification_fields_survive_set_configuration(self): assert detector.config.auto_config is True # the default (set_configuration-first) path for i in range(10): - parser_data = schemas.ParserSchema({ + parser_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "Template 1", @@ -696,7 +696,7 @@ def _configured(classification: ClassificationMethods) -> NewValueComboDetector: ) detector = NewValueComboDetector(config=cfg, name="NewValueComboDetector") for i in range(10): - detector.configure(schemas.ParserSchema({ + detector.configure(sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "Template 1", @@ -753,7 +753,7 @@ def _records(classification=None): for i in range(12): # var_0 and var_1 both cycle over 3 values -> both STABLE in pass 1, # so pass 2 gets a ("var_0", "var_1") combo tracker to look at. - detector.configure(schemas.ParserSchema({ + detector.configure(sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "Template 1", @@ -837,8 +837,8 @@ class TestNewValueComboDetectorConfigureSpill: afterwards.""" @staticmethod - def _record(i: int) -> schemas.ParserSchema: - return schemas.ParserSchema({ + def _record(i: int) -> sc.ParserSchema: + return sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "Template 1", @@ -913,7 +913,7 @@ class TestNewValueComboDetectorAllCombos: @staticmethod def _record(variables, log_format_variables=None, event_id=1, log_id="1"): - return schemas.ParserSchema({ + return sc.ParserSchema({ "parserType": "test", "eventID": event_id, "template": "Template 1", @@ -959,7 +959,7 @@ def test_unseen_value_pair_in_a_later_combo_alerts(self): detector.train(self._record(["b", "y"], {"level": "ERROR"})) # ("a", "x") is known to the first combo; ("a", "ERROR") is new to the second - output = schemas.DetectorSchema() + output = sc.DetectorSchema() assert detector.detect(self._record(["a", "x"], {"level": "ERROR"}), output) assert output.score == 1.0 assert list(output.alertDetails) == ["EventID 1 - ('var_0', 'level')"] @@ -971,7 +971,7 @@ def test_every_combo_is_trained(self): trackers = detector.persistency.get_events_data()[1].get_data() assert list(trackers) == [("var_0", "var_1"), ("var_0", "level")] assert not detector.detect( - self._record(["a", "x"], {"level": "INFO"}), schemas.DetectorSchema() + self._record(["a", "x"], {"level": "INFO"}), sc.DetectorSchema() ) def test_combo_with_a_missing_variable_is_skipped(self): @@ -984,7 +984,7 @@ def test_combo_with_a_missing_variable_is_skipped(self): assert list(trackers) == [("var_0", "var_1"), ("var_0", "level")] assert trackers[("var_0", "level")].unique_set == {("a", "INFO")} - output = schemas.DetectorSchema() + output = sc.DetectorSchema() assert not detector.detect(self._record(["b", "y"]), output) assert not detector.detect(self._record(["new"]), output) @@ -1000,7 +1000,7 @@ def test_every_combo_survives_federation(self, tmp_path, monkeypatch): assert trackers[("var_0", "var_1")].unique_set == {("a", "x"), ("b", "y")} assert trackers[("var_0", "level")].unique_set == {("a", "INFO"), ("b", "ERROR")} assert received.detect( - self._record(["a", "x"], {"level": "ERROR"}), schemas.DetectorSchema() + self._record(["a", "x"], {"level": "ERROR"}), sc.DetectorSchema() ) @staticmethod diff --git a/tests/test_detectors/test_new_value_detector.py b/tests/test_detectors/test_new_value_detector.py index 09a1d78c..acb39231 100644 --- a/tests/test_detectors/test_new_value_detector.py +++ b/tests/test_detectors/test_new_value_detector.py @@ -14,7 +14,7 @@ from detectmatelibrary.constants import GLOBAL_EVENT_ID from detectmatelibrary.parsers.template_matcher import MatcherParser from detectmatelibrary.helper.from_to import From -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc from detectmatelibrary.utils.aux import time_test_mode from tests.test_data import AUDIT_LOG, AUDIT_TEMPLATES, TRAIN_UNTIL @@ -75,8 +75,8 @@ def test_default_initialization(self): assert detector.name == "NewValueDetector" assert hasattr(detector, 'config') assert detector.data_buffer.mode == BufferMode.NO_BUF - assert detector.input_schema == schemas.ParserSchema - assert detector.output_schema == schemas.DetectorSchema + assert detector.input_schema == sc.ParserSchema + assert detector.output_schema == sc.DetectorSchema assert hasattr(detector, 'persistency') def test_custom_config_initialization(self): @@ -97,7 +97,7 @@ def test_train_multiple_values(self): # Train with multiple values (only event 1 should be tracked per config) for event in range(3): for level in ["INFO", "WARNING", "ERROR"]: - parser_data = schemas.ParserSchema({ + parser_data = sc.ParserSchema({ "parserType": "test", "eventID": event, "template": "test template", @@ -129,7 +129,7 @@ def test_detect_known_value_no_alert(self): detector = NewValueDetector(config=config, name="MultipleDetector") # Train with a value - train_data = schemas.ParserSchema({ + train_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -143,7 +143,7 @@ def test_detect_known_value_no_alert(self): detector.train(train_data) # Detect with the same value - test_data = schemas.ParserSchema({ + test_data = sc.ParserSchema({ "parserType": "test", "eventID": 12, "template": "test template", @@ -154,7 +154,7 @@ def test_detect_known_value_no_alert(self): "log": "test log message", "headerVariables": {"level": "CRITICAL"} }) - output = schemas.DetectorSchema() + output = sc.DetectorSchema() result = detector.detect(test_data, output) @@ -165,7 +165,7 @@ def test_detect_known_value_alert(self): detector = NewValueDetector(config=config, name="MultipleDetector") # Train with a value - train_data = schemas.ParserSchema({ + train_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -179,7 +179,7 @@ def test_detect_known_value_alert(self): detector.train(train_data) # Detect with the same value - test_data = schemas.ParserSchema({ + test_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -190,7 +190,7 @@ def test_detect_known_value_alert(self): "log": "test log message", "headerVariables": {"level": "CRITICAL"} }) - output = schemas.DetectorSchema() + output = sc.DetectorSchema() result = detector.detect(test_data, output) @@ -235,7 +235,7 @@ def test_audit_log_anomalies(self): detected_ids: set[str] = set() for log in logs[TRAIN_UNTIL:]: - output = schemas.DetectorSchema() + output = sc.DetectorSchema() if detector.detect(log, output_=output): detected_ids.add(log["logID"]) @@ -263,7 +263,7 @@ def test_audit_log_anomalies_to_binary(self): detected_ids: set[str] = set() for log in logs[TRAIN_UNTIL:]: - output = schemas.DetectorSchema() + output = sc.DetectorSchema() if detector2.detect(log, output_=output): detected_ids.add(log["logID"]) @@ -297,7 +297,7 @@ def test_audit_log_anomalie_fed(self): detected_ids: set[str] = set() for log in logs[TRAIN_UNTIL:]: - output = schemas.DetectorSchema() + output = sc.DetectorSchema() if detector1.detect(log, output_=output): detected_ids.add(log["logID"]) @@ -347,7 +347,7 @@ def test_auto_config_off_keeps_explicit_events_through_configure_phase(self): explicit_events = detector.config.events.model_dump() for i in range(20): - detector.process(schemas.ParserSchema({ + detector.process(sc.ParserSchema({ "parserType": "test", "eventID": [1, 2, 3][i % 3], "template": "test template", @@ -398,7 +398,7 @@ def test_global_instance_detects_new_type(self): detected_ids: set[str] = set() for log in logs[TRAIN_UNTIL:]: - output = schemas.DetectorSchema() + output = sc.DetectorSchema() if detector.detect(log, output_=output): assert all(key.startswith("Global -") for key in output["alertDetails"]) detected_ids.add(log["logID"]) diff --git a/tests/test_detectors/test_random_detector.py b/tests/test_detectors/test_random_detector.py index 471dab0d..af12e915 100644 --- a/tests/test_detectors/test_random_detector.py +++ b/tests/test_detectors/test_random_detector.py @@ -1,5 +1,5 @@ from detectmatelibrary.detectors.random_detector import RandomDetector -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc from detectmatelibrary.utils.aux import time_test_mode from copy import deepcopy @@ -49,7 +49,7 @@ class TestRandomDetectorEdgeCases: def test_random_seed_consistency(self): detector = RandomDetector(name="TestDetector", config=deepcopy(config)) - parser_data = schemas.ParserSchema({ + parser_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -60,13 +60,13 @@ def test_random_seed_consistency(self): "log": "test log message", "headerVariables": {"timestamp": "123456"} }) - detector_output = schemas.DetectorSchema() + detector_output = sc.DetectorSchema() # Multiple calls should return consistent results when mocked result1 = detector.detect(parser_data, detector_output) # Reset output for second call - detector_output = schemas.DetectorSchema() + detector_output = sc.DetectorSchema() result2 = detector.detect(parser_data, detector_output) assert result1 == result2 diff --git a/tests/test_detectors/test_rule_detector.py b/tests/test_detectors/test_rule_detector.py index 122e2664..7c34de29 100644 --- a/tests/test_detectors/test_rule_detector.py +++ b/tests/test_detectors/test_rule_detector.py @@ -1,62 +1,62 @@ import detectmatelibrary.detectors.rule_detector as rd -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc import pytest class TestCaseRules: def test_template_not_found(self) -> None: - parsed_log = schemas.ParserSchema({"eventID": -1}) + parsed_log = sc.ParserSchema({"eventID": -1}) alert, msg = rd.template_not_found(parsed_log) assert alert assert "No template found by parser" == msg - parsed_log = schemas.ParserSchema({"eventID": 2}) + parsed_log = sc.ParserSchema({"eventID": 2}) alert, _ = rd.template_not_found(parsed_log) assert not alert def test_find_keyword(self) -> None: keywords = ["hello", "ciao"] - parsed_log = schemas.ParserSchema({"log": "hello world"}) + parsed_log = sc.ParserSchema({"log": "hello world"}) alert, msg = rd.find_keyword(parsed_log, keywords) assert alert assert "Found word 'hello' in the logs" == msg - parsed_log = schemas.ParserSchema({"log": "ciao world"}) + parsed_log = sc.ParserSchema({"log": "ciao world"}) alert, msg = rd.find_keyword(parsed_log, keywords) assert alert assert "Found word 'ciao' in the logs" == msg - parsed_log = schemas.ParserSchema({"log": "world"}) + parsed_log = sc.ParserSchema({"log": "world"}) alert, _ = rd.find_keyword(parsed_log, keywords) assert not alert def test_find_exceptions(self) -> None: - parsed_log = schemas.ParserSchema({"log": "hello Exception"}) + parsed_log = sc.ParserSchema({"log": "hello Exception"}) alert, msg = rd.exceptions(parsed_log) assert alert assert "Found word 'exception' in the logs" == msg - parsed_log = schemas.ParserSchema({"log": "world"}) + parsed_log = sc.ParserSchema({"log": "world"}) alert, _ = rd.exceptions(parsed_log) assert not alert def test_error_log(self) -> None: - parsed_log = schemas.ParserSchema( + parsed_log = sc.ParserSchema( {"headerVariables": {"Level": "Error"}} ) alert, msg = rd.error_log(parsed_log) assert alert assert "Error found" == msg - parsed_log = schemas.ParserSchema( + parsed_log = sc.ParserSchema( {"headerVariables": {"Level": "Info"}} ) alert, _ = rd.error_log(parsed_log) assert not alert - parsed_log = schemas.ParserSchema({"log": "hello world"}) + parsed_log = sc.ParserSchema({"log": "hello world"}) alert, _ = rd.error_log(parsed_log) assert not alert @@ -65,9 +65,9 @@ class TestCaseRuleDetector: def test_run_all_rules(self) -> None: rule_detector = rd.RuleDetector() - assert rule_detector.process(schemas.ParserSchema()) is None + assert rule_detector.process(sc.ParserSchema()) is None - alert1 = rule_detector.process(schemas.ParserSchema( + alert1 = rule_detector.process(sc.ParserSchema( { "eventID": -1, "log": "fail exception", @@ -85,9 +85,9 @@ def test_run_all_rules(self) -> None: def test_all_rules_serialize(self) -> None: rule_detector = rd.RuleDetector() - assert rule_detector.process(schemas.ParserSchema().serialize()) is None + assert rule_detector.process(sc.ParserSchema().serialize()) is None - alert1 = rule_detector.process(schemas.ParserSchema( + alert1 = rule_detector.process(sc.ParserSchema( { "eventID": -1, "log": "fail exception", @@ -96,7 +96,7 @@ def test_all_rules_serialize(self) -> None: ).serialize()) assert alert1 is not None - (alert := schemas.DetectorSchema()).deserialize(alert1) + (alert := sc.DetectorSchema()).deserialize(alert1) assert alert["alertDetails"] == { "R001 - TemplateNotFound": "No template found by parser", "R003 - CheckForExceptions": "Found word 'exception' in the logs", @@ -122,9 +122,9 @@ def test_run_single_rule(self) -> None: } ) - assert rule_detector.process(schemas.ParserSchema()) is None + assert rule_detector.process(sc.ParserSchema()) is None - alert1 = rule_detector.process(schemas.ParserSchema( + alert1 = rule_detector.process(sc.ParserSchema( { "eventID": -1, "log": "fail exception", @@ -155,8 +155,8 @@ def test_run_single_rule_with_args(self) -> None: } ) - assert rule_detector.process(schemas.ParserSchema({"log": "ciao"})) is None - assert rule_detector.process(schemas.ParserSchema({"log": "hi"})) is not None + assert rule_detector.process(sc.ParserSchema({"log": "ciao"})) is None + assert rule_detector.process(sc.ParserSchema({"log": "hi"})) is not None def test_rule_not_found(self) -> None: diff --git a/tests/test_detectors/test_value_range_detector.py b/tests/test_detectors/test_value_range_detector.py index b9b62d4f..74e4586c 100644 --- a/tests/test_detectors/test_value_range_detector.py +++ b/tests/test_detectors/test_value_range_detector.py @@ -21,7 +21,7 @@ from detectmatelibrary.constants import GLOBAL_EVENT_ID from detectmatelibrary.parsers.template_matcher import MatcherParser from detectmatelibrary.helper.from_to import From -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc from detectmatelibrary.utils.aux import time_test_mode from detectmatelibrary.tools.logging import logger from tests.test_data import AUDIT_LOG, AUDIT_TEMPLATES, TRAIN_UNTIL @@ -75,8 +75,8 @@ def test_default_initialization(self): assert detector.name == "ValueRangeDetector" assert hasattr(detector, 'config') assert detector.data_buffer.mode == BufferMode.NO_BUF - assert detector.input_schema == schemas.ParserSchema - assert detector.output_schema == schemas.DetectorSchema + assert detector.input_schema == sc.ParserSchema + assert detector.output_schema == sc.DetectorSchema assert hasattr(detector, 'persistency') def test_custom_config_initialization(self): @@ -130,7 +130,7 @@ def test_train_multiple_values(self): if event == 1: min_val = min(min_val, value) max_val = max(max_val, value) - parser_data = schemas.ParserSchema({ + parser_data = sc.ParserSchema({ "parserType": "test", "eventID": event, "template": "test template", @@ -148,7 +148,7 @@ def test_train_multiple_values(self): if event == 1: min_val = min(min_val, value) max_val = max(max_val, value) - parser_data = schemas.ParserSchema({ + parser_data = sc.ParserSchema({ "parserType": "test", "eventID": event, "template": "test template", @@ -175,7 +175,7 @@ def test_train_detect_non_numeric_raises(self): detector = ValueRangeDetector(config=config, name="CustomInit") # Train with multiple values (the minimum and maximum value should be captured) - parser_data = schemas.ParserSchema({ + parser_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -188,7 +188,7 @@ def test_train_detect_non_numeric_raises(self): }) with pytest.raises(ValueError): detector.train(parser_data) - normal_data = schemas.ParserSchema({ + normal_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -201,7 +201,7 @@ def test_train_detect_non_numeric_raises(self): }) detector.train(normal_data) with pytest.raises(ValueError): - output = schemas.DetectorSchema() + output = sc.DetectorSchema() detector.detect(parser_data, output) def test_train_detect_non_numeric_ignore(self): @@ -209,7 +209,7 @@ def test_train_detect_non_numeric_ignore(self): detector = ValueRangeDetector(config=config, name="MultipleDetector") # Train with multiple values (the minimum and maximum value should be captured) - parser_data = schemas.ParserSchema({ + parser_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -221,7 +221,7 @@ def test_train_detect_non_numeric_ignore(self): "headerVariables": {} }) detector.train(parser_data) - normal_data = schemas.ParserSchema({ + normal_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -233,7 +233,7 @@ def test_train_detect_non_numeric_ignore(self): "headerVariables": {} }) detector.train(normal_data) - output = schemas.DetectorSchema() + output = sc.DetectorSchema() detector.detect(parser_data, output) @@ -245,7 +245,7 @@ def test_detect_learned_value_range_no_alert(self): # Train with values for val in ["1", "5000", "2130"]: - train_data = schemas.ParserSchema({ + train_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -259,7 +259,7 @@ def test_detect_learned_value_range_no_alert(self): detector.train(train_data) # Detect with the same value - test_data = schemas.ParserSchema({ + test_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -270,7 +270,7 @@ def test_detect_learned_value_range_no_alert(self): "log": "test log message", "headerVariables": {"level": "CRITICAL"} }) - output = schemas.DetectorSchema() + output = sc.DetectorSchema() result = detector.detect(test_data, output) @@ -282,7 +282,7 @@ def test_detect_known_value_ranges_alert(self): # Train with values for val in ["1", "5000", "2130"]: - train_data = schemas.ParserSchema({ + train_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -296,7 +296,7 @@ def test_detect_known_value_ranges_alert(self): detector.train(train_data) # Detect with the different value - test_data = schemas.ParserSchema({ + test_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -307,7 +307,7 @@ def test_detect_known_value_ranges_alert(self): "log": "test log message", "headerVariables": {"level": "CRITICAL"} }) - output = schemas.DetectorSchema() + output = sc.DetectorSchema() result = detector.detect(test_data, output) @@ -315,7 +315,7 @@ def test_detect_known_value_ranges_alert(self): assert output.score == 1.0 # Detect with the different value - test_data = schemas.ParserSchema({ + test_data = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "test template", @@ -326,7 +326,7 @@ def test_detect_known_value_ranges_alert(self): "log": "test log message", "headerVariables": {"level": "CRITICAL"} }) - output = schemas.DetectorSchema() + output = sc.DetectorSchema() result = detector.detect(test_data, output) @@ -373,7 +373,7 @@ def test_audit_log_anomalies(self): detected_ids: set[str] = set() for log in logs[TRAIN_UNTIL:]: - output = schemas.DetectorSchema() + output = sc.DetectorSchema() if detector.detect(log, output_=output): detected_ids.add(log["logID"]) @@ -403,7 +403,7 @@ def test_audit_log_anomalies_to_binary(self): detected_ids: set[str] = set() for log in logs[TRAIN_UNTIL:]: - output = schemas.DetectorSchema() + output = sc.DetectorSchema() if detector2.detect(log, output_=output): detected_ids.add(log["logID"]) @@ -437,7 +437,7 @@ def test_audit_log_anomalie_fed(self): detected_ids: set[str] = set() for log in logs[TRAIN_UNTIL:]: - output = schemas.DetectorSchema() + output = sc.DetectorSchema() if detector1.detect(log, output_=output): detected_ids.add(log["logID"]) @@ -514,7 +514,7 @@ def test_global_instance_detects_new_type(self): detected_ids: set[str] = set() for log in logs[TRAIN_UNTIL:]: - output = schemas.DetectorSchema() + output = sc.DetectorSchema() if detector.detect(log, output_=output): assert all(key.startswith("Global -") for key in output["alertDetails"]) detected_ids.add(log["logID"]) @@ -544,7 +544,7 @@ def test_set_configuration_preserves_persist(self): # Feed configure() with a couple of stable-variable samples for _ in range(5): - sample = schemas.ParserSchema({ + sample = sc.ParserSchema({ "parserType": "test", "eventID": 1, "template": "t", "variables": ["123"], "logID": "x", "parsedLogID": "x", "parserID": "p", "log": "l", diff --git a/tests/test_helper/test_from_to.py b/tests/test_helper/test_from_to.py index 9a20ce2f..7195c112 100644 --- a/tests/test_helper/test_from_to.py +++ b/tests/test_helper/test_from_to.py @@ -3,7 +3,7 @@ from detectmatelibrary._testutils.dummy_parser import DummyParser from tests.test_data import AUDIT_TEMPLATES, DUMMY_TXT_PATH, DUMMY_TXT_PATH2, DUMMY_JSON_PATH, \ DUMMY_JSON_PATH2, DUMMY_YAML_PATH, DUMMY_YAML_PATH2 -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc import detectmateperformance as dmp import polars as pl import json @@ -128,7 +128,7 @@ def test_fromlog_no_process(self): log = next(From.log(parser, in_path=AUDIT_TEMPLATES, do_process=False)) assert log.log == expected_log - assert isinstance(log, schemas.LogSchema) + assert isinstance(log, sc.LogSchema) def test_fromlog(self): parser = DummyParser() @@ -136,7 +136,7 @@ def test_fromlog(self): log = next(From.log(parser, in_path=AUDIT_TEMPLATES, do_process=True)) assert log.log == expected_log - assert isinstance(log, schemas.ParserSchema) + assert isinstance(log, sc.ParserSchema) @remove_files def test_frombinary(self): @@ -179,7 +179,7 @@ def test_frompolars(self): gen = From.polars(DummyDetector(), df=table, do_process=False) parsed1 = next(gen) - schema1 = schemas.ParserSchema({ + schema1 = sc.ParserSchema({ "log": "hello there", "variables": ["a", "b"], "template": "hello <*>", @@ -191,7 +191,7 @@ def test_frompolars(self): assert parsed1["logID"] == "0" parsed2 = next(gen) - schema2 = schemas.ParserSchema({ + schema2 = sc.ParserSchema({ "log": "general kenobi", "variables": ["c", "d"], "template": "<*> kenobi", @@ -213,7 +213,7 @@ def test_frompolars_lazy(self): gen = From.polars(DummyDetector(), df=table, do_process=False) parsed1 = next(gen) - schema1 = schemas.ParserSchema({ + schema1 = sc.ParserSchema({ "log": "hello there", "variables": ["a", "b"], "template": "hello <*>", @@ -225,7 +225,7 @@ def test_frompolars_lazy(self): assert parsed1["logID"] == "0" parsed2 = next(gen) - schema2 = schemas.ParserSchema({ + schema2 = sc.ParserSchema({ "log": "general kenobi", "variables": ["c", "d"], "template": "<*> kenobi", @@ -250,7 +250,7 @@ def test_frompolars_rename(self): gen = From.polars(DummyDetector(), df=table, do_process=False, renames=renames) parsed1 = next(gen) - schema1 = schemas.ParserSchema({ + schema1 = sc.ParserSchema({ "log": "hello there", "variables": ["a", "b"], "template": "hello <*>", @@ -274,7 +274,7 @@ def test_frompolars_rename_vars(self): gen = From.polars(DummyDetector(), df=table, do_process=False, renames=renames) parsed1 = next(gen) - schema1 = schemas.ParserSchema({ + schema1 = sc.ParserSchema({ "log": "hello there", "variables": ["a", "b"], "template": "hello <*>", diff --git a/tests/test_parsers/test_json_parser.py b/tests/test_parsers/test_json_parser.py index 73299688..82573cf7 100644 --- a/tests/test_parsers/test_json_parser.py +++ b/tests/test_parsers/test_json_parser.py @@ -4,7 +4,7 @@ JsonParserConfig, flatten_dict ) -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc from tests.test_data import TEST_TEMPLATES @@ -52,12 +52,12 @@ def test_parse_json_without_message(self): "error_code": 404 } - input_log = schemas.LogSchema({ + input_log = sc.LogSchema({ "logID": "1", "log": json.dumps(json_log) }) - output = schemas.ParserSchema() + output = sc.ParserSchema() parser.parse(input_log, output) # When no message field, template should be empty @@ -87,12 +87,12 @@ def test_parse_nested_json(self): } } - input_log = schemas.LogSchema({ + input_log = sc.LogSchema({ "logID": "1", "log": json.dumps(json_log) }) - output = schemas.ParserSchema() + output = sc.ParserSchema() parser.parse(input_log, output) # Check flattened structure @@ -112,12 +112,12 @@ def test_parse_json_with_array(self): "events": ["login", "update", "logout"] } - input_log = schemas.LogSchema({ + input_log = sc.LogSchema({ "logID": "1", "log": json.dumps(json_log) }) - output = schemas.ParserSchema() + output = sc.ParserSchema() parser.parse(input_log, output) # Check array is flattened @@ -151,12 +151,12 @@ def test_parse_with_content_parser(self): "level": "INFO" } - input_log = schemas.LogSchema({ + input_log = sc.LogSchema({ "logID": "1", "log": json.dumps(json_log) }) - output = schemas.ParserSchema() + output = sc.ParserSchema() parser.parse(input_log, output) # The content parser should have parsed the message @@ -174,12 +174,12 @@ def test_custom_field_names(self): "severity": "high" } - input_log = schemas.LogSchema({ + input_log = sc.LogSchema({ "logID": "1", "log": json.dumps(json_log) }) - output = schemas.ParserSchema() + output = sc.ParserSchema() parser.parse(input_log, output) # Timestamp should be extracted and Time should be in headerVariables @@ -198,7 +198,7 @@ def test_process_method(self): "status": "ok" } - input_log = schemas.LogSchema({ + input_log = sc.LogSchema({ "logID": "1", "log": json.dumps(json_log) }) @@ -206,7 +206,7 @@ def test_process_method(self): # Test the full process method result = parser.process(input_log) - assert isinstance(result, schemas.ParserSchema) + assert isinstance(result, sc.ParserSchema) assert "Time" in result.headerVariables assert "status" in result.headerVariables assert result.headerVariables["status"] == "ok" diff --git a/tests/test_parsers/test_logbatcher_parser.py b/tests/test_parsers/test_logbatcher_parser.py index 2f289e73..abac37b0 100644 --- a/tests/test_parsers/test_logbatcher_parser.py +++ b/tests/test_parsers/test_logbatcher_parser.py @@ -8,7 +8,7 @@ import pytest -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc from detectmatelibrary.common.parser import CoreParser from detectmatelibrary.parsers.logbatcher import LogBatcherParser, LogBatcherParserConfig from detectmatelibrary.utils.aux import time_test_mode @@ -45,7 +45,7 @@ def test_config_method_type(self): class TestLogBatcherParserParse: def test_template_extracted(self): parser = _make_parser() - log_schema = schemas.LogSchema({"logID": "1", "log": LOG}) + log_schema = sc.LogSchema({"logID": "1", "log": LOG}) result = parser.process(log_schema) @@ -53,7 +53,7 @@ def test_template_extracted(self): def test_variables_extracted(self): parser = _make_parser() - log_schema = schemas.LogSchema({"logID": "1", "log": LOG}) + log_schema = sc.LogSchema({"logID": "1", "log": LOG}) result = parser.process(log_schema) @@ -62,7 +62,7 @@ def test_variables_extracted(self): def test_event_id_is_int(self): parser = _make_parser() - log_schema = schemas.LogSchema({"logID": "1", "log": LOG}) + log_schema = sc.LogSchema({"logID": "1", "log": LOG}) result = parser.process(log_schema) @@ -72,11 +72,11 @@ def test_second_call_hits_cache(self): """Second identical log must not trigger a new LLM call.""" parser = _make_parser() - log_schema1 = schemas.LogSchema({"logID": "1", "log": LOG}) + log_schema1 = sc.LogSchema({"logID": "1", "log": LOG}) parser.process(log_schema1) llm_call_count = parser._llm_parser.chat.call_count - log_schema2 = schemas.LogSchema({"logID": "2", "log": LOG}) + log_schema2 = sc.LogSchema({"logID": "2", "log": LOG}) parser.process(log_schema2) assert parser._llm_parser.chat.call_count == llm_call_count diff --git a/tests/test_persistency/test_time_dependent_stability.py b/tests/test_persistency/test_time_dependent_stability.py index 3c8ad7cf..ab67f1f0 100644 --- a/tests/test_persistency/test_time_dependent_stability.py +++ b/tests/test_persistency/test_time_dependent_stability.py @@ -3,7 +3,7 @@ import logging import math -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc from detectmatelibrary.detectors.charset_detector import CharsetDetector, CharsetDetectorConfig from detectmatelibrary.base_detectors import StabilityAutoConfigParams, VariableAutoConfigParams from detectmatelibrary.base_detectors._stability import TimestampReader @@ -386,7 +386,7 @@ def test_event_tracker_detector_backed_round_trip(self): def _parser_record(ts_value, event_id=1): - return schemas.ParserSchema({ + return sc.ParserSchema({ "parserType": "test", "eventID": event_id, "template": "test template", diff --git a/tests/test_pipelines/test_dummy_methods.py b/tests/test_pipelines/test_dummy_methods.py index 13db8128..2a6e44ed 100644 --- a/tests/test_pipelines/test_dummy_methods.py +++ b/tests/test_pipelines/test_dummy_methods.py @@ -1,7 +1,7 @@ from detectmatelibrary._testutils.dummy_detector import DummyDetector, DummyDetectorConfig from detectmatelibrary._testutils.dummy_parser import DummyParser, DummyParserConfig -import detectmatelibrary.schemas as schemas +import detectmatelibrary.schemas as sc default_args = { @@ -32,8 +32,8 @@ def test_initialize_default(self) -> None: def test_run_detect_method(self) -> None: detector = DummyDetector() - data = schemas.ParserSchema({"log": "test log"}) - output = schemas.DetectorSchema() + data = sc.ParserSchema({"log": "test log"}) + output = sc.DetectorSchema() result = detector.detect(data, output) @@ -54,8 +54,8 @@ def test_parser_initialize_default(self) -> None: def test_run_parse_method(self) -> None: parser = DummyParser() - input_data = schemas.LogSchema({"log": "test log"}) - output_data = schemas.ParserSchema() + input_data = sc.LogSchema({"log": "test log"}) + output_data = sc.ParserSchema() parser.parse(input_data, output_data) @@ -66,7 +66,7 @@ def test_pipeline(test) -> None: parser = DummyParser() detector = DummyDetector() - log_input = schemas.LogSchema({"log": "test log", "logID": "1"}) + log_input = sc.LogSchema({"log": "test log", "logID": "1"}) parsed_log = parser.process(log_input) while (result := detector.process(parsed_log)) is None: From 7a0511e15b33030d402c99cc09cc5f3083589658 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 19 Oct 2026 08:34:40 +0200 Subject: [PATCH 02/23] add correct typing in the config arguments --- .../base_detectors/deeplearning_detector.py | 2 +- src/detectmatelibrary/base_detectors/variable_detector.py | 2 +- src/detectmatelibrary/common/alert_aggregator.py | 6 +++--- src/detectmatelibrary/common/detector.py | 6 +++--- src/detectmatelibrary/common/parser.py | 8 ++++---- .../detectors/bigram_frequency_detector.py | 2 +- src/detectmatelibrary/detectors/charset_detector.py | 2 +- .../detectors/event_sequence_detector.py | 4 ++-- src/detectmatelibrary/detectors/new_event_detector.py | 3 ++- .../detectors/new_value_combo_detector.py | 2 +- src/detectmatelibrary/detectors/new_value_detector.py | 2 +- src/detectmatelibrary/detectors/random_detector.py | 4 ++-- src/detectmatelibrary/detectors/value_range_detector.py | 2 +- 13 files changed, 23 insertions(+), 22 deletions(-) diff --git a/src/detectmatelibrary/base_detectors/deeplearning_detector.py b/src/detectmatelibrary/base_detectors/deeplearning_detector.py index f00beeee..798551ba 100644 --- a/src/detectmatelibrary/base_detectors/deeplearning_detector.py +++ b/src/detectmatelibrary/base_detectors/deeplearning_detector.py @@ -51,7 +51,7 @@ def __init__( self, model_cls: DeepModel, name: str = "CoreDetector", - config: DeepLearningDetectorConfig = DeepLearningDetectorConfig() + config: DeepLearningDetectorConfig | dict[str, Any] = DeepLearningDetectorConfig() ) -> None: if isinstance(config, dict): diff --git a/src/detectmatelibrary/base_detectors/variable_detector.py b/src/detectmatelibrary/base_detectors/variable_detector.py index 33c63d38..e8bd13cb 100644 --- a/src/detectmatelibrary/base_detectors/variable_detector.py +++ b/src/detectmatelibrary/base_detectors/variable_detector.py @@ -125,7 +125,7 @@ class VariableDetector(TrackerDetector, VariableHooks): _time_handler: TimeFormatHandler = _core_time_handler def __init__( - self, name: str, config: VariableDetectorConfig = VariableDetectorConfig() + self, name: str, config: VariableDetectorConfig | dict[str, Any] = VariableDetectorConfig() ) -> None: if isinstance(config, dict): config = VariableDetectorConfig.from_dict(config, name) diff --git a/src/detectmatelibrary/common/alert_aggregator.py b/src/detectmatelibrary/common/alert_aggregator.py index 28045ead..a7a3c326 100644 --- a/src/detectmatelibrary/common/alert_aggregator.py +++ b/src/detectmatelibrary/common/alert_aggregator.py @@ -41,15 +41,15 @@ def __init__( name: str = "CoreAlertAggregator", buffer_mode: BufferMode = BufferMode.WINDOW, buffer_size: Optional[int] = 1, - config: Optional[CoreAlertAggregatorConfig | dict[str, Any]] = CoreAlertAggregatorConfig(), + config: CoreAlertAggregatorConfig | dict[str, Any] = CoreAlertAggregatorConfig(), ) -> None: if isinstance(config, dict): config = CoreAlertAggregatorConfig.from_dict(config, name) super().__init__( name=name, - type_=config.method_type, # type: ignore - config=config, # type: ignore + type_=config.method_type, + config=config, args_buffer=ArgsBuffer(mode=buffer_mode, size=buffer_size), input_schema=DetectorSchema, output_schema=AggregateSchema, diff --git a/src/detectmatelibrary/common/detector.py b/src/detectmatelibrary/common/detector.py index ac956f99..621d0875 100644 --- a/src/detectmatelibrary/common/detector.py +++ b/src/detectmatelibrary/common/detector.py @@ -60,15 +60,15 @@ def __init__( name: str = "CoreDetector", buffer_mode: BufferMode = BufferMode.NO_BUF, buffer_size: Optional[int] = None, - config: Optional[CoreDetectorConfig | dict[str, Any]] = CoreDetectorConfig(), + config: CoreDetectorConfig | dict[str, Any] = CoreDetectorConfig(), ) -> None: if isinstance(config, dict): config = CoreDetectorConfig.from_dict(config, name) super().__init__( name=name, - type_=config.component_type, # type: ignore - config=config, # type: ignore + type_=config.component_type, + config=config, args_buffer=ArgsBuffer(mode=buffer_mode, size=buffer_size), input_schema=ParserSchema, output_schema=DetectorSchema, diff --git a/src/detectmatelibrary/common/parser.py b/src/detectmatelibrary/common/parser.py index adae332e..e805cc8b 100644 --- a/src/detectmatelibrary/common/parser.py +++ b/src/detectmatelibrary/common/parser.py @@ -6,7 +6,7 @@ from detectmatelibrary.utils.aux import get_timestamp from detectmatelibrary import schemas -from typing import Any, Optional, cast +from typing import Any, cast from pydantic import model_validator import re from pydantic import Field @@ -44,15 +44,15 @@ class CoreParser(CoreComponent): def __init__( self, name: str = "CoreParser", - config: Optional[CoreParserConfig | dict[str, Any]] = CoreParserConfig(), + config: CoreParserConfig | dict[str, Any] = CoreParserConfig(), ): if isinstance(config, dict): config = CoreParserConfig.from_dict(config, name) super().__init__( name=name, - type_=config.method_type, # type: ignore - config=config, # type: ignore + type_=config.method_type, + config=config, args_buffer=ArgsBuffer(mode=BufferMode.NO_BUF, size=None), input_schema=schemas.LogSchema, output_schema=schemas.ParserSchema, diff --git a/src/detectmatelibrary/detectors/bigram_frequency_detector.py b/src/detectmatelibrary/detectors/bigram_frequency_detector.py index 4ee0ccaf..f90bb6a3 100644 --- a/src/detectmatelibrary/detectors/bigram_frequency_detector.py +++ b/src/detectmatelibrary/detectors/bigram_frequency_detector.py @@ -73,7 +73,7 @@ class BigramFrequencyDetector(VariableDetector): def __init__( self, name: str = "BigramFrequencyDetector", - config: BigramFrequencyDetectorConfig = BigramFrequencyDetectorConfig(), + config: BigramFrequencyDetectorConfig | dict[str, Any] = BigramFrequencyDetectorConfig(), ) -> None: if isinstance(config, dict): config = BigramFrequencyDetectorConfig.from_dict(config, name) diff --git a/src/detectmatelibrary/detectors/charset_detector.py b/src/detectmatelibrary/detectors/charset_detector.py index c88e5262..5ceb010c 100644 --- a/src/detectmatelibrary/detectors/charset_detector.py +++ b/src/detectmatelibrary/detectors/charset_detector.py @@ -17,7 +17,7 @@ class CharsetDetector(VariableDetector): def __init__( self, name: str = "CharsetDetector", - config: CharsetDetectorConfig = CharsetDetectorConfig(), + config: CharsetDetectorConfig | dict[str, Any] = CharsetDetectorConfig(), ) -> None: if isinstance(config, dict): config = CharsetDetectorConfig.from_dict(config, name) diff --git a/src/detectmatelibrary/detectors/event_sequence_detector.py b/src/detectmatelibrary/detectors/event_sequence_detector.py index fde8f390..e6093418 100644 --- a/src/detectmatelibrary/detectors/event_sequence_detector.py +++ b/src/detectmatelibrary/detectors/event_sequence_detector.py @@ -9,7 +9,7 @@ from collections import deque -from typing_extensions import override +from typing_extensions import override, Any from pydantic import Field, model_validator @@ -71,7 +71,7 @@ class EventSequenceDetector(TrackerDetector): def __init__( self, name: str = "EventSequenceDetector", - config: EventSequenceDetectorConfig = EventSequenceDetectorConfig(), + config: EventSequenceDetectorConfig | dict[str, Any] = EventSequenceDetectorConfig(), ) -> None: if isinstance(config, dict): config = EventSequenceDetectorConfig.from_dict(config, name) diff --git a/src/detectmatelibrary/detectors/new_event_detector.py b/src/detectmatelibrary/detectors/new_event_detector.py index 85ee74d1..541317d3 100644 --- a/src/detectmatelibrary/detectors/new_event_detector.py +++ b/src/detectmatelibrary/detectors/new_event_detector.py @@ -10,6 +10,7 @@ from detectmatelibrary.schemas import ParserSchema, DetectorSchema from pydantic import Field +from typing import Any class NewEventDetectorConfig(TrackerDetectorConfig): @@ -24,7 +25,7 @@ class NewEventDetector(TrackerDetector): def __init__( self, name: str = "NewEventDetector", - config: NewEventDetectorConfig = NewEventDetectorConfig() + config: NewEventDetectorConfig | dict[str, Any] = NewEventDetectorConfig() ) -> None: if isinstance(config, dict): diff --git a/src/detectmatelibrary/detectors/new_value_combo_detector.py b/src/detectmatelibrary/detectors/new_value_combo_detector.py index cb492e38..e9024f94 100644 --- a/src/detectmatelibrary/detectors/new_value_combo_detector.py +++ b/src/detectmatelibrary/detectors/new_value_combo_detector.py @@ -91,7 +91,7 @@ class NewValueComboDetector(VariableDetector): def __init__( self, name: str = "NewValueComboDetector", - config: NewValueComboDetectorConfig = NewValueComboDetectorConfig(), + config: NewValueComboDetectorConfig | dict[str, Any] = NewValueComboDetectorConfig(), ) -> None: if isinstance(config, dict): config = NewValueComboDetectorConfig.from_dict(config, name) diff --git a/src/detectmatelibrary/detectors/new_value_detector.py b/src/detectmatelibrary/detectors/new_value_detector.py index 2eed4f55..0931074b 100644 --- a/src/detectmatelibrary/detectors/new_value_detector.py +++ b/src/detectmatelibrary/detectors/new_value_detector.py @@ -17,7 +17,7 @@ class NewValueDetector(VariableDetector): def __init__( self, name: str = "NewValueDetector", - config: NewValueDetectorConfig = NewValueDetectorConfig(), + config: NewValueDetectorConfig | dict[str, Any] = NewValueDetectorConfig(), ) -> None: if isinstance(config, dict): config = NewValueDetectorConfig.from_dict(config, name) diff --git a/src/detectmatelibrary/detectors/random_detector.py b/src/detectmatelibrary/detectors/random_detector.py index 44c05056..1dfd5198 100644 --- a/src/detectmatelibrary/detectors/random_detector.py +++ b/src/detectmatelibrary/detectors/random_detector.py @@ -7,7 +7,7 @@ import detectmatelibrary.schemas as sc from typing_extensions import override -from typing import List +from typing import List, Any from pydantic import Field import numpy as np @@ -26,7 +26,7 @@ class RandomDetector(CoreDetector): def __init__( self, name: str = "RandomDetector", - config: RandomDetectorConfig = RandomDetectorConfig(), + config: RandomDetectorConfig | dict[str, Any] = RandomDetectorConfig(), ) -> None: if isinstance(config, dict): config = RandomDetectorConfig.from_dict(config, name) diff --git a/src/detectmatelibrary/detectors/value_range_detector.py b/src/detectmatelibrary/detectors/value_range_detector.py index 5fda5192..09675eea 100644 --- a/src/detectmatelibrary/detectors/value_range_detector.py +++ b/src/detectmatelibrary/detectors/value_range_detector.py @@ -26,7 +26,7 @@ class ValueRangeDetector(VariableDetector): def __init__( self, name: str = "ValueRangeDetector", - config: ValueRangeDetectorConfig = ValueRangeDetectorConfig(), + config: ValueRangeDetectorConfig | dict[str, Any] = ValueRangeDetectorConfig(), ) -> None: if isinstance(config, dict): config = ValueRangeDetectorConfig.from_dict(config, name) From f53061a3069dbc783f2bef52321dce4dd5cc5f85 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 19 Oct 2026 09:34:13 +0200 Subject: [PATCH 03/23] find configs in ipy --- src/detectmatelibrary/__init__.py | 2 ++ .../alert_aggregation/__init__.py | 6 +++++ .../alert_aggregation/basic_concat.py | 8 +++--- src/detectmatelibrary/helper/i_pick_you.py | 26 +++++++++++++++++++ src/detectmatelibrary/parsers/__init__.py | 23 ++++++++++++++++ tests/test_helper/test_ipickyou.py | 21 +++++++++++++++ 6 files changed, 82 insertions(+), 4 deletions(-) create mode 100644 src/detectmatelibrary/alert_aggregation/__init__.py create mode 100644 src/detectmatelibrary/helper/i_pick_you.py create mode 100644 tests/test_helper/test_ipickyou.py diff --git a/src/detectmatelibrary/__init__.py b/src/detectmatelibrary/__init__.py index 3cfe8053..15479df2 100644 --- a/src/detectmatelibrary/__init__.py +++ b/src/detectmatelibrary/__init__.py @@ -1,4 +1,5 @@ from . import common +from . import alert_aggregation from . import detectors from . import parsers from . import schemas @@ -10,6 +11,7 @@ "common", "detectors", "parsers", + "alert_aggregation", "schemas", "utils", "__authors__", diff --git a/src/detectmatelibrary/alert_aggregation/__init__.py b/src/detectmatelibrary/alert_aggregation/__init__.py new file mode 100644 index 00000000..aeb32921 --- /dev/null +++ b/src/detectmatelibrary/alert_aggregation/__init__.py @@ -0,0 +1,6 @@ +from .basic_concat import BasicConcatAggregation, BasicConcatAggregationConfig + +__all__ = [ + "BasicConcatAggregation", + "BasicConcatAggregationConfig", +] diff --git a/src/detectmatelibrary/alert_aggregation/basic_concat.py b/src/detectmatelibrary/alert_aggregation/basic_concat.py index 2ca1ec32..a5701f11 100644 --- a/src/detectmatelibrary/alert_aggregation/basic_concat.py +++ b/src/detectmatelibrary/alert_aggregation/basic_concat.py @@ -1,6 +1,6 @@ from detectmatelibrary.common.alert_aggregator import (CoreAlertAggregatorConfig, CoreAlertAggregator) from detectmatelibrary.schemas import DetectorSchema, AggregateSchema -from typing import Any, Optional +from typing import Any class BasicConcatAggregationConfig(CoreAlertAggregatorConfig): @@ -13,12 +13,12 @@ class BasicConcatAggregation(CoreAlertAggregator): def __init__( self, name: str = "BasicConcatAggregator", - config: Optional[BasicConcatAggregationConfig | dict[str, Any]] = BasicConcatAggregationConfig(), + config: BasicConcatAggregationConfig | dict[str, Any] = BasicConcatAggregationConfig(), ) -> None: if isinstance(config, dict): config = BasicConcatAggregationConfig.from_dict(config, name) - buffer_size: int = config.buffer_size # type: ignore - self.deduplicate_values: bool = config.deduplicate_values # type: ignore + buffer_size: int = config.buffer_size + self.deduplicate_values: bool = config.deduplicate_values super().__init__(name=name, buffer_size=buffer_size, config=config) def aggregate_alerts( diff --git a/src/detectmatelibrary/helper/i_pick_you.py b/src/detectmatelibrary/helper/i_pick_you.py new file mode 100644 index 00000000..cc94442b --- /dev/null +++ b/src/detectmatelibrary/helper/i_pick_you.py @@ -0,0 +1,26 @@ +from detectmatelibrary.common.core import CoreConfig +from detectmatelibrary import detectors, parsers, alert_aggregation + +from typing import Optional + + +__types = { + "parsers": parsers, + "detectors": detectors, + "alert_aggregators": alert_aggregation +} + + +def find_config_from_name( + method_type: str, component_type: str +) -> Optional[type[CoreConfig]]: + + for obj in (dir(module := __types[component_type])): + try: + if isinstance(conf := getattr(module, obj)(), CoreConfig): + if conf.method_type == method_type: + return getattr(module, obj) # type: ignore + except TypeError: + pass + + return None diff --git a/src/detectmatelibrary/parsers/__init__.py b/src/detectmatelibrary/parsers/__init__.py index e69de29b..889c74ca 100644 --- a/src/detectmatelibrary/parsers/__init__.py +++ b/src/detectmatelibrary/parsers/__init__.py @@ -0,0 +1,23 @@ +from .drain import DrainParser, DrainConfig +from .autoparser import AutoParser, AutoParserConfig +from .json_parser import JsonParser, JsonParserConfig +from .tree_matcher import TemplateCppTreeMatcher, TemplateCppTreeMatcherConfig +# from .logbatcher import LogBatcherParser, LogBatcherParserConfig +# TODO: logbacther does not pass the mypy +from .template_matcher import MatcherParser, MatcherParserConfig + + +__all__ = [ + "DrainParser", + "DrainConfig", + "AutoParser", + "AutoParserConfig", + "JsonParser", + "JsonParserConfig", + "TemplateCppTreeMatcher", + "TemplateCppTreeMatcherConfig", + # "LogBatcherParser", + # "LogBatcherParserConfig", + "MatcherParser", + "MatcherParserConfig", +] diff --git a/tests/test_helper/test_ipickyou.py b/tests/test_helper/test_ipickyou.py new file mode 100644 index 00000000..25baa98c --- /dev/null +++ b/tests/test_helper/test_ipickyou.py @@ -0,0 +1,21 @@ +from detectmatelibrary.alert_aggregation.basic_concat import BasicConcatAggregationConfig +from detectmatelibrary.detectors.random_detector import RandomDetectorConfig +from detectmatelibrary.parsers.drain import DrainConfig + +import detectmatelibrary.helper.i_pick_you as ipy + + +class TestFindFromName: + def test_get_configs(self): + assert (config := ipy.find_config_from_name("drain_parser", "parsers")) is not None + assert isinstance(config(), DrainConfig) + + assert (config := ipy.find_config_from_name("random_detector", "detectors")) is not None + assert isinstance(config(), RandomDetectorConfig) + + assert (config := ipy.find_config_from_name( + "basic_concat_aggregator", "alert_aggregators" + )) is not None + assert isinstance(config(), BasicConcatAggregationConfig) + + assert ipy.find_config_from_name("aux", "alert_aggregators") is None From aaac6d204acead37855629d7cf97cfc472f1e12d Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 19 Oct 2026 09:54:10 +0200 Subject: [PATCH 04/23] add to find component instead of the other --- src/detectmatelibrary/helper/i_pick_you.py | 19 +++++++++++++------ tests/test_helper/test_ipickyou.py | 20 ++++++++++---------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/src/detectmatelibrary/helper/i_pick_you.py b/src/detectmatelibrary/helper/i_pick_you.py index cc94442b..4ecab212 100644 --- a/src/detectmatelibrary/helper/i_pick_you.py +++ b/src/detectmatelibrary/helper/i_pick_you.py @@ -1,5 +1,6 @@ -from detectmatelibrary.common.core import CoreConfig +from detectmatelibrary.common.core import CoreConfig, CoreComponent from detectmatelibrary import detectors, parsers, alert_aggregation +from detectmatelibrary.schemas import BaseSchema from typing import Optional @@ -11,15 +12,21 @@ } -def find_config_from_name( +def find_component_from_type( method_type: str, component_type: str -) -> Optional[type[CoreConfig]]: +) -> Optional[type[CoreComponent[BaseSchema, BaseSchema]]]: - for obj in (dir(module := __types[component_type])): + for obj in dir(module := __types[component_type]): try: - if isinstance(conf := getattr(module, obj)(), CoreConfig): + if isinstance(conf := getattr(module, obj)(), CoreConfig): # Look for config if conf.method_type == method_type: - return getattr(module, obj) # type: ignore + for comp in dir(__types[component_type]): # Look for Component + try: + if isinstance(comp_ := getattr(module, comp)(), CoreComponent): + if type(comp_.config) == type(conf): # noqa: E721 + return getattr(module, comp) # type: ignore + except TypeError: + pass except TypeError: pass diff --git a/tests/test_helper/test_ipickyou.py b/tests/test_helper/test_ipickyou.py index 25baa98c..c7b24b1c 100644 --- a/tests/test_helper/test_ipickyou.py +++ b/tests/test_helper/test_ipickyou.py @@ -1,21 +1,21 @@ -from detectmatelibrary.alert_aggregation.basic_concat import BasicConcatAggregationConfig -from detectmatelibrary.detectors.random_detector import RandomDetectorConfig -from detectmatelibrary.parsers.drain import DrainConfig +from detectmatelibrary.alert_aggregation.basic_concat import BasicConcatAggregation +from detectmatelibrary.detectors.random_detector import RandomDetector +from detectmatelibrary.parsers.drain import DrainParser import detectmatelibrary.helper.i_pick_you as ipy class TestFindFromName: def test_get_configs(self): - assert (config := ipy.find_config_from_name("drain_parser", "parsers")) is not None - assert isinstance(config(), DrainConfig) + assert (config := ipy.find_component_from_type("drain_parser", "parsers")) is not None + assert isinstance(config(), DrainParser) - assert (config := ipy.find_config_from_name("random_detector", "detectors")) is not None - assert isinstance(config(), RandomDetectorConfig) + assert (config := ipy.find_component_from_type("random_detector", "detectors")) is not None + assert isinstance(config(), RandomDetector) - assert (config := ipy.find_config_from_name( + assert (config := ipy.find_component_from_type( "basic_concat_aggregator", "alert_aggregators" )) is not None - assert isinstance(config(), BasicConcatAggregationConfig) + assert isinstance(config(), BasicConcatAggregation) - assert ipy.find_config_from_name("aux", "alert_aggregators") is None + assert ipy.find_component_from_type("aux", "alert_aggregators") is None From 9874b21234fe6326573c281928d61e69879da3b0 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 19 Oct 2026 10:07:08 +0200 Subject: [PATCH 05/23] preparing tests --- src/detectmatelibrary/common/core.py | 5 ++++- src/detectmatelibrary/helper/i_pick_you.py | 19 +++++++++++++++++-- tests/test_helper/test_ipickyou.py | 13 +++++++++---- 3 files changed, 30 insertions(+), 7 deletions(-) diff --git a/src/detectmatelibrary/common/core.py b/src/detectmatelibrary/common/core.py index 3aec5409..5b7d59bd 100644 --- a/src/detectmatelibrary/common/core.py +++ b/src/detectmatelibrary/common/core.py @@ -69,11 +69,14 @@ def __init__( self, name: str, type_: str = "Core", - config: CoreConfig = CoreConfig(), + config: CoreConfig | dict[str, Any] = CoreConfig(), args_buffer: ArgsBuffer = ArgsBuffer(BufferMode.NO_BUF), input_schema: type[BaseSchema] = BaseSchema, output_schema: type[BaseSchema] = BaseSchema, ) -> None: + if isinstance(config, dict): + config = CoreConfig.from_dict(config, name) + Component.__init__(self, name=name, type_=type_, config=config) FedOperations.__init__(self) self.config: CoreConfig diff --git a/src/detectmatelibrary/helper/i_pick_you.py b/src/detectmatelibrary/helper/i_pick_you.py index 4ecab212..8c8a43d2 100644 --- a/src/detectmatelibrary/helper/i_pick_you.py +++ b/src/detectmatelibrary/helper/i_pick_you.py @@ -2,7 +2,7 @@ from detectmatelibrary import detectors, parsers, alert_aggregation from detectmatelibrary.schemas import BaseSchema -from typing import Optional +from typing import Optional, Any __types = { @@ -12,7 +12,7 @@ } -def find_component_from_type( +def _find_component_from_type( method_type: str, component_type: str ) -> Optional[type[CoreComponent[BaseSchema, BaseSchema]]]: @@ -31,3 +31,18 @@ def find_component_from_type( pass return None + + +def pickComponent(config: CoreConfig | dict[str, Any], component_type: str) -> CoreComponent: + """Component, I pick you!!!""" + + if isinstance(config, CoreConfig): + method_type = config.method_type + else: + method_type = config[method_type] + + component = _find_component_from_type(method_type, component_type=component_type) + if component is None: + raise ValueError(f"{method_type} form {component_type} not found.") + + return component(name="picked_component", config=config) diff --git a/tests/test_helper/test_ipickyou.py b/tests/test_helper/test_ipickyou.py index c7b24b1c..a2c0085f 100644 --- a/tests/test_helper/test_ipickyou.py +++ b/tests/test_helper/test_ipickyou.py @@ -7,15 +7,20 @@ class TestFindFromName: def test_get_configs(self): - assert (config := ipy.find_component_from_type("drain_parser", "parsers")) is not None + assert (config := ipy._find_component_from_type("drain_parser", "parsers")) is not None assert isinstance(config(), DrainParser) - assert (config := ipy.find_component_from_type("random_detector", "detectors")) is not None + assert (config := ipy._find_component_from_type("random_detector", "detectors")) is not None assert isinstance(config(), RandomDetector) - assert (config := ipy.find_component_from_type( + assert (config := ipy._find_component_from_type( "basic_concat_aggregator", "alert_aggregators" )) is not None assert isinstance(config(), BasicConcatAggregation) - assert ipy.find_component_from_type("aux", "alert_aggregators") is None + assert ipy._find_component_from_type("aux", "alert_aggregators") is None + + +class TestPickComponets: + def test_pick_component(self): + pass From ff92f40330bdabd2f98708d986c98b3c4c1c543e Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 19 Oct 2026 10:24:49 +0200 Subject: [PATCH 06/23] add optional name in pick component --- src/detectmatelibrary/helper/i_pick_you.py | 21 +++++++--- tests/test_helper/test_ipickyou.py | 49 ++++++++++++++++++---- 2 files changed, 57 insertions(+), 13 deletions(-) diff --git a/src/detectmatelibrary/helper/i_pick_you.py b/src/detectmatelibrary/helper/i_pick_you.py index 8c8a43d2..6cedf188 100644 --- a/src/detectmatelibrary/helper/i_pick_you.py +++ b/src/detectmatelibrary/helper/i_pick_you.py @@ -4,7 +4,7 @@ from typing import Optional, Any - +# --------------------- Auxiliar __types = { "parsers": parsers, "detectors": detectors, @@ -33,16 +33,27 @@ def _find_component_from_type( return None -def pickComponent(config: CoreConfig | dict[str, Any], component_type: str) -> CoreComponent: - """Component, I pick you!!!""" +# --------------------------------- Main methods +def pickComponent( + config: CoreConfig | dict[str, Any], component_type: str, name: Optional[str] = None +) -> CoreComponent: + """Component, I pick you!!! + + If name=None, it will check in the config + """ if isinstance(config, CoreConfig): method_type = config.method_type + name = name if name is not None else "pickedComponent" + else: - method_type = config[method_type] + if name is None: + name = list(config[component_type].keys())[0] + + method_type = config[component_type][name]["method_type"] component = _find_component_from_type(method_type, component_type=component_type) if component is None: raise ValueError(f"{method_type} form {component_type} not found.") - return component(name="picked_component", config=config) + return component(name=name, config=config) diff --git a/tests/test_helper/test_ipickyou.py b/tests/test_helper/test_ipickyou.py index a2c0085f..f6b54910 100644 --- a/tests/test_helper/test_ipickyou.py +++ b/tests/test_helper/test_ipickyou.py @@ -1,26 +1,59 @@ from detectmatelibrary.alert_aggregation.basic_concat import BasicConcatAggregation from detectmatelibrary.detectors.random_detector import RandomDetector -from detectmatelibrary.parsers.drain import DrainParser +from detectmatelibrary.parsers.drain import DrainParser, DrainConfig import detectmatelibrary.helper.i_pick_you as ipy class TestFindFromName: def test_get_configs(self): - assert (config := ipy._find_component_from_type("drain_parser", "parsers")) is not None - assert isinstance(config(), DrainParser) + assert (comp := ipy._find_component_from_type("drain_parser", "parsers")) is not None + assert isinstance(comp(), DrainParser) - assert (config := ipy._find_component_from_type("random_detector", "detectors")) is not None - assert isinstance(config(), RandomDetector) + assert (comp := ipy._find_component_from_type("random_detector", "detectors")) is not None + assert isinstance(comp(), RandomDetector) - assert (config := ipy._find_component_from_type( + assert (comp := ipy._find_component_from_type( "basic_concat_aggregator", "alert_aggregators" )) is not None - assert isinstance(config(), BasicConcatAggregation) + assert isinstance(comp(), BasicConcatAggregation) assert ipy._find_component_from_type("aux", "alert_aggregators") is None class TestPickComponets: + def test_pick_component_dict(self): + config = { + "detectors": { + "TestDetector": {"method_type": "random_detector"} + } + } + comp = ipy.pickComponent( + name="TestDetector", config=config, component_type="detectors" + ) + + assert isinstance(comp, RandomDetector) + + def test_pick_component_dict_no_name(self): + config = { + "detectors": { + "TestDetector": {"method_type": "random_detector"} + } + } + comp = ipy.pickComponent(config=config, component_type="detectors") + + assert isinstance(comp, RandomDetector) + def test_pick_component(self): - pass + config = DrainConfig() + comp = ipy.pickComponent( + name="Drain", config=config, component_type="parsers" + ) + + assert isinstance(comp, DrainParser) + + def test_pick_component_no_name(self): + config = DrainConfig() + comp = ipy.pickComponent(config=config, component_type="parsers") + + assert isinstance(comp, DrainParser) From c92dbabe4344b53a94981b0c0bcba18ac561ca71 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 19 Oct 2026 10:35:26 +0200 Subject: [PATCH 07/23] add pickcomponents --- src/detectmatelibrary/helper/i_pick_you.py | 15 ++++++++++ tests/test_helper/test_ipickyou.py | 32 +++++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/src/detectmatelibrary/helper/i_pick_you.py b/src/detectmatelibrary/helper/i_pick_you.py index 6cedf188..42ef1c07 100644 --- a/src/detectmatelibrary/helper/i_pick_you.py +++ b/src/detectmatelibrary/helper/i_pick_you.py @@ -57,3 +57,18 @@ def pickComponent( raise ValueError(f"{method_type} form {component_type} not found.") return component(name=name, config=config) + + +def pickComponents( + configs: list[CoreConfig | dict[str, Any]], + component_type: str, + names: Optional[list[str | None]] = None +) -> list[CoreComponent]: + + if names is None: + names = [None for _ in configs] + + return [ + pickComponent(config=config, component_type=component_type, name=name) + for config, name in zip(configs, names) + ] diff --git a/tests/test_helper/test_ipickyou.py b/tests/test_helper/test_ipickyou.py index f6b54910..be995eb0 100644 --- a/tests/test_helper/test_ipickyou.py +++ b/tests/test_helper/test_ipickyou.py @@ -1,5 +1,5 @@ from detectmatelibrary.alert_aggregation.basic_concat import BasicConcatAggregation -from detectmatelibrary.detectors.random_detector import RandomDetector +from detectmatelibrary.detectors.random_detector import RandomDetector, RandomDetectorConfig from detectmatelibrary.parsers.drain import DrainParser, DrainConfig import detectmatelibrary.helper.i_pick_you as ipy @@ -57,3 +57,33 @@ def test_pick_component_no_name(self): comp = ipy.pickComponent(config=config, component_type="parsers") assert isinstance(comp, DrainParser) + + def test_pick_multiple_components(self): + configs = [ + { + "detectors": { + "a": {"method_type": "random_detector"} + } + }, + RandomDetectorConfig(), + ] + names = ["a", "b"] + + for component in ipy.pickComponents( + configs=configs, names=names, component_type="detectors" + ): + assert isinstance(component, RandomDetector) + + def test_pick_multiple_components_no_names(self): + configs = [ + { + "detectors": { + "a": {"method_type": "random_detector"} + } + }, + RandomDetectorConfig(), + ] + for component in ipy.pickComponents( + configs=configs, component_type="detectors" + ): + assert isinstance(component, RandomDetector) From 65d826da9e616b64bcd43818050ee83ab4ff9257 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 19 Oct 2026 11:24:08 +0200 Subject: [PATCH 08/23] draft pipeline full implemented --- src/detectmatelibrary/pipeline.py | 106 ++++++++++++++++++++++-------- 1 file changed, 79 insertions(+), 27 deletions(-) diff --git a/src/detectmatelibrary/pipeline.py b/src/detectmatelibrary/pipeline.py index c230f102..c69c41da 100644 --- a/src/detectmatelibrary/pipeline.py +++ b/src/detectmatelibrary/pipeline.py @@ -1,13 +1,15 @@ from detectmatelibrary.common.core import CoreComponent, CoreConfig -from detectmatelibrary.common.detector import CoreDetectorConfig -from detectmatelibrary.common.parser import CoreParserConfig +from detectmatelibrary.common.alert_aggregator import CoreAlertAggregator, CoreAlertAggregatorConfig +from detectmatelibrary.common.detector import CoreDetector, CoreDetectorConfig +from detectmatelibrary.common.parser import CoreParser, CoreParserConfig from detectmatelibrary.utils.data_buffer import ArgsBuffer, BufferMode +import detectmatelibrary.helper.i_pick_you as ipy import detectmatelibrary.schemas as sc -from typing import Any, Optional, overload, override +from typing import Any, overload, Optional, override from pydantic import Field @@ -24,7 +26,7 @@ class PipelineConfig(CoreConfig): detectors: list[CoreDetectorConfig | dict[str, Any]] | None = Field( default=None, description="Detectors configuration use." ) - aggregators: CoreParserConfig | dict[str, Any] | None = Field( + aggregator: CoreAlertAggregatorConfig | dict[str, Any] | None = Field( default=None, description="Aggregator configuration use, if do_aggregation=False keep empty." ) @@ -37,12 +39,43 @@ def _select_output(config: PipelineConfig) -> type[sc.BaseSchema]: return sc.DetectorSchema if config.do_aggregation else sc.AggregateSchema +def _run_pipeline( + config: PipelineConfig, + input_: sc.ParserSchema | bytes | sc.LogSchema, + parser: Optional[CoreParser], + detectors: list[CoreDetector], + alert_aggreator: Optional[CoreAlertAggregator], +) -> list[sc.DetectorSchema] | list[sc.AggregateSchema]: + + input_ = input_ if isinstance(input_, list) else [input_] # type: ignore + if config.do_parsing: + aux = [] + for in_ in input_: # type: ignore + if (out_ := parser.process(in_)) is not None: # type: ignore + aux.append(out_) + input_ = aux # type: ignore + + output_ = [] + for in_ in input_: # type: ignore + for detector in detectors: + if (out_ := detector.process(in_)) is not None: # type: ignore + output_.append(out_) + + if not config.do_aggregation: + return output_ # type: ignore + + last_output_ = [] + for in_ in output_: + if (out_ := alert_aggreator.process(in_)) is not None: # type: ignore + last_output_.append(out_) + + return last_output_ # type: ignore + + class Pipeline(CoreComponent): def __init__( self, name: str = "MainPipeline", - buffer_mode: BufferMode = BufferMode.WINDOW, - buffer_size: Optional[int] = 1, config: PipelineConfig | dict[str, Any] = PipelineConfig(), ) -> None: if isinstance(config, dict): @@ -52,44 +85,63 @@ def __init__( name=name, type_=config.method_type, config=config, - args_buffer=ArgsBuffer(mode=buffer_mode, size=buffer_size), + args_buffer=ArgsBuffer(mode=BufferMode.NO_BUF, size=None), input_schema=_select_input(config), output_schema=_select_output(config), ) self.config: PipelineConfig - @overload # type: ignore + self.parser: CoreParser | None = ipy.pickComponent( # type: ignore + self.config.parser, component_type="parsers" + ) if self.config.do_parsing else None + + self.detectors: list[CoreDetector] = ipy.pickComponents( # type: ignore + self.config.parser, component_type="detectors" + ) + + self.alert_aggregator: CoreAlertAggregator | None = ipy.pickComponent( # type: ignore + self.config.aggregator, component_type="alert_aggregation" + ) if self.config.do_aggregation else None + + @overload # type: ignore @override - def run( - self, - input_: list[sc.LogSchema] | sc.LogSchema, - output_: list[sc.DetectorSchema] | sc.DetectorSchema - ) -> bool: + def process(self, data: sc.LogSchema | bytes) -> sc.AggregateSchema | bytes | None: # type: ignore pass - @overload @override - def run(self, input_: list[sc.LogSchema] | sc.LogSchema, output_: sc.AggregateSchema) -> bool: + @overload + def process( # type: ignore + self, data: sc.LogSchema | bytes + ) -> sc.DetectorSchema | bytes | None | list[sc.DetectorSchema] | list[bytes]: pass @overload @override - def run( - self, - input_: list[sc.ParserSchema] | sc.ParserSchema, - output_: list[sc.DetectorSchema] | sc.DetectorSchema - ) -> bool: + def process(self, data: sc.ParserSchema | bytes) -> sc.AggregateSchema | bytes | None: pass @overload @override - def run(self, input_: list[sc.ParserSchema] | sc.ParserSchema, output_: sc.AggregateSchema) -> bool: + def process( # type: ignore + self, data: sc.ParserSchema | bytes + ) -> sc.DetectorSchema | bytes | None | list[sc.DetectorSchema] | list[bytes]: pass @override - def run( - self, - input_: list[sc.LogSchema] | sc.LogSchema | list[sc.ParserSchema] | sc.ParserSchema, - output_: list[sc.DetectorSchema] | sc.DetectorSchema | sc.AggregateSchema - ) -> bool: - return True + def process( + self, data: sc.ParserSchema | bytes | sc.LogSchema + ) -> sc.DetectorSchema | bytes | None | list[sc.DetectorSchema] | list[bytes] | sc.AggregateSchema: + + out_ = _run_pipeline( + self.config, + input_=data, + parser=self.parser, + detectors=self.detectors, + alert_aggreator=self.alert_aggregator + ) + + if len(out_) == 0: + return None + if len(out_) == 1: + return out_[0] + return out_ # type: ignore From 034dbd6e31442477f9fd639740dd62a59ee1a6bc Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 19 Oct 2026 11:53:43 +0200 Subject: [PATCH 09/23] add pipeline first tests --- tests/test_pipelines/test_main_pipeline.py | 74 ++++++++++++++++++++++ 1 file changed, 74 insertions(+) create mode 100644 tests/test_pipelines/test_main_pipeline.py diff --git a/tests/test_pipelines/test_main_pipeline.py b/tests/test_pipelines/test_main_pipeline.py new file mode 100644 index 00000000..fc47c2e9 --- /dev/null +++ b/tests/test_pipelines/test_main_pipeline.py @@ -0,0 +1,74 @@ + +from detectmatelibrary.pipeline import PipelineConfig, _run_pipeline + +from detectmatelibrary._testutils.dummy_parser import DummyParser +from detectmatelibrary._testutils.dummy_detector import DummyDetectorConfig +from detectmatelibrary._testutils.dummy_alert_aggregator import ( + DummyAlertAggregator, DummyAlertAggregatorConfig +) + +import detectmatelibrary.schemas as sc + +from detectmatelibrary.common.detector import CoreDetector + + +class AlwayAlert(CoreDetector): + """A dummy detector for testing purposes.""" + + def __init__( + self, + name: str = "DummyDetector", + config: DummyDetectorConfig = DummyDetectorConfig() + ) -> None: + + super().__init__(name=name, config=config) + + def detect( + self, + input_: list[sc.ParserSchema] | sc.ParserSchema, + output_: sc.DetectorSchema + ) -> bool: + output_["description"] = "Dummy detection process" + + output_["score"] = 1.0 + output_["alertDetails"]["type"] = "Anomaly detected by DummyDetector" + return True + + +log_schema = {"logID": "1", "log": "hello"} + + +class TestRunPipeline: + def test_run_parser2detectors(self) -> None: + out_ = _run_pipeline( + config=PipelineConfig(do_parsing=True, do_aggregation=False), + input_=log_schema, + parser=DummyParser(), + detectors=[AlwayAlert(), AlwayAlert()], + alert_aggreator=None + ) + + assert isinstance(out_, list) + assert isinstance(out_[0], sc.DetectorSchema) + assert len(out_) == 2 + + def test_run_parser2aggregator(self) -> None: + out_ = _run_pipeline( + config=PipelineConfig(do_parsing=True, do_aggregation=True), + input_=log_schema, + parser=DummyParser(), + detectors=[AlwayAlert(), AlwayAlert()], + alert_aggreator=DummyAlertAggregator( + config=DummyAlertAggregatorConfig(buffer_size=2) + ) + ) + + assert isinstance(out_, list) + assert isinstance(out_[0], sc.AggregateSchema) + assert len(out_) == 1 + + def test_run_detectors_only(self) -> None: + pass + + def test_run_detectors2aggregator(self) -> None: + pass From 254c062203f50dd8fb338e3e6040dae61f739564 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 19 Oct 2026 12:08:38 +0200 Subject: [PATCH 10/23] add all pipeline tests --- tests/test_pipelines/test_main_pipeline.py | 41 ++++++++++++++++++++-- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/tests/test_pipelines/test_main_pipeline.py b/tests/test_pipelines/test_main_pipeline.py index fc47c2e9..2249b22f 100644 --- a/tests/test_pipelines/test_main_pipeline.py +++ b/tests/test_pipelines/test_main_pipeline.py @@ -35,7 +35,20 @@ def detect( return True -log_schema = {"logID": "1", "log": "hello"} +log_schema = sc.LogSchema({"logID": "1", "log": "hello"}) + +parser_schema = sc.ParserSchema({ + "parserType": "a", + "parserID": "ciao", + "logID": "1", + "eventID": 1, + "template": "a", + "parsedLogID": "as2", + "log": "hello", + "receivedTimestamp": 1, + "parsedTimestamp": 2, + "headerVariables": {"Time": "-1"} +}) class TestRunPipeline: @@ -68,7 +81,29 @@ def test_run_parser2aggregator(self) -> None: assert len(out_) == 1 def test_run_detectors_only(self) -> None: - pass + out_ = _run_pipeline( + config=PipelineConfig(do_parsing=False, do_aggregation=False), + input_=parser_schema, + parser=None, + detectors=[AlwayAlert(), AlwayAlert()], + alert_aggreator=None + ) + + assert isinstance(out_, list) + assert isinstance(out_[0], sc.DetectorSchema) + assert len(out_) == 2 def test_run_detectors2aggregator(self) -> None: - pass + out_ = _run_pipeline( + config=PipelineConfig(do_parsing=False, do_aggregation=True), + input_=parser_schema, + parser=None, + detectors=[AlwayAlert(), AlwayAlert()], + alert_aggreator=DummyAlertAggregator( + config=DummyAlertAggregatorConfig(buffer_size=2) + ) + ) + + assert isinstance(out_, list) + assert isinstance(out_[0], sc.AggregateSchema) + assert len(out_) == 1 From 4f5f52cead0efcde15617d19a544999e7a5044a7 Mon Sep 17 00:00:00 2001 From: "angre.garcia-gomez@ait.ac.at" Date: Mon, 19 Oct 2026 13:18:31 +0200 Subject: [PATCH 11/23] add full pipeline test --- .../alert_aggregation/__init__.py | 3 ++ .../alert_aggregation/basic_concat.py | 4 +- .../alert_aggregation/logid_count.py | 4 +- src/detectmatelibrary/pipeline.py | 15 ++++-- .../test_common/test_core_alert_aggregator.py | 12 ++--- tests/test_pipelines/test_main_pipeline.py | 50 ++++++++++++++++++- 6 files changed, 72 insertions(+), 16 deletions(-) diff --git a/src/detectmatelibrary/alert_aggregation/__init__.py b/src/detectmatelibrary/alert_aggregation/__init__.py index aeb32921..d1b82f94 100644 --- a/src/detectmatelibrary/alert_aggregation/__init__.py +++ b/src/detectmatelibrary/alert_aggregation/__init__.py @@ -1,6 +1,9 @@ from .basic_concat import BasicConcatAggregation, BasicConcatAggregationConfig +from .logid_count import LogIDCountAggegation, LogIDCountAggegationConfig __all__ = [ "BasicConcatAggregation", "BasicConcatAggregationConfig", + "LogIDCountAggegation", + "LogIDCountAggegationConfig", ] diff --git a/src/detectmatelibrary/alert_aggregation/basic_concat.py b/src/detectmatelibrary/alert_aggregation/basic_concat.py index 708c7335..b03c3644 100644 --- a/src/detectmatelibrary/alert_aggregation/basic_concat.py +++ b/src/detectmatelibrary/alert_aggregation/basic_concat.py @@ -4,7 +4,7 @@ from detectmatelibrary.schemas import DetectorSchema, AggregateSchema -from typing import Any, Optional +from typing import Any from pydantic import Field @@ -26,7 +26,7 @@ def __init__( if isinstance(config, dict): config = BasicConcatAggregationConfig.from_dict(config, name) - buffer_size: int = config.buffer_size # type: ignore + buffer_size: int = config.buffer_size super().__init__(name=name, buffer_size=buffer_size, config=config) def aggregate_alerts( diff --git a/src/detectmatelibrary/alert_aggregation/logid_count.py b/src/detectmatelibrary/alert_aggregation/logid_count.py index deea9db2..335aeabd 100644 --- a/src/detectmatelibrary/alert_aggregation/logid_count.py +++ b/src/detectmatelibrary/alert_aggregation/logid_count.py @@ -5,7 +5,7 @@ from detectmatelibrary.schemas import DetectorSchema, AggregateSchema from detectmatelibrary.utils.data_buffer import BufferMode -from typing import Any, Optional +from typing import Any from functools import lru_cache from pydantic import Field import polars as pl @@ -66,7 +66,7 @@ class LogIDCountAggegation(CoreAlertAggregator): def __init__( self, name: str = "LogIDCountAggregator", - config: Optional[LogIDCountAggegationConfig | dict[str, Any]] = LogIDCountAggegationConfig(), + config: LogIDCountAggegationConfig | dict[str, Any] = LogIDCountAggegationConfig(), ) -> None: if isinstance(config, dict): config = LogIDCountAggegationConfig.from_dict(config, name) diff --git a/src/detectmatelibrary/pipeline.py b/src/detectmatelibrary/pipeline.py index c69c41da..679a12e7 100644 --- a/src/detectmatelibrary/pipeline.py +++ b/src/detectmatelibrary/pipeline.py @@ -1,5 +1,6 @@ from detectmatelibrary.common.core import CoreComponent, CoreConfig + from detectmatelibrary.common.alert_aggregator import CoreAlertAggregator, CoreAlertAggregatorConfig from detectmatelibrary.common.detector import CoreDetector, CoreDetectorConfig from detectmatelibrary.common.parser import CoreParser, CoreParserConfig @@ -39,6 +40,10 @@ def _select_output(config: PipelineConfig) -> type[sc.BaseSchema]: return sc.DetectorSchema if config.do_aggregation else sc.AggregateSchema +def _check_output(in_: sc.BaseSchema, method: CoreComponent) -> Optional[bytes | sc.BaseSchema]: + return method.process(in_) + + def _run_pipeline( config: PipelineConfig, input_: sc.ParserSchema | bytes | sc.LogSchema, @@ -51,14 +56,14 @@ def _run_pipeline( if config.do_parsing: aux = [] for in_ in input_: # type: ignore - if (out_ := parser.process(in_)) is not None: # type: ignore + if (out_ := _check_output(in_, parser)) is not None: # type: ignore aux.append(out_) input_ = aux # type: ignore output_ = [] for in_ in input_: # type: ignore for detector in detectors: - if (out_ := detector.process(in_)) is not None: # type: ignore + if (out_ := _check_output(in_, detector)) is not None: # type: ignore output_.append(out_) if not config.do_aggregation: @@ -66,7 +71,7 @@ def _run_pipeline( last_output_ = [] for in_ in output_: - if (out_ := alert_aggreator.process(in_)) is not None: # type: ignore + if (out_ := _check_output(in_, alert_aggreator)) is not None: # type: ignore last_output_.append(out_) return last_output_ # type: ignore @@ -96,11 +101,11 @@ def __init__( ) if self.config.do_parsing else None self.detectors: list[CoreDetector] = ipy.pickComponents( # type: ignore - self.config.parser, component_type="detectors" + self.config.detectors, component_type="detectors" ) self.alert_aggregator: CoreAlertAggregator | None = ipy.pickComponent( # type: ignore - self.config.aggregator, component_type="alert_aggregation" + self.config.aggregator, component_type="alert_aggregators" ) if self.config.do_aggregation else None @overload # type: ignore diff --git a/tests/test_common/test_core_alert_aggregator.py b/tests/test_common/test_core_alert_aggregator.py index 7ac26e1e..0ab17c09 100644 --- a/tests/test_common/test_core_alert_aggregator.py +++ b/tests/test_common/test_core_alert_aggregator.py @@ -77,7 +77,7 @@ def aggregate_alerts(self, input_, output_): class TestAuxiliar: def test_get_field_single(self) -> None: expected = {"dummy"} - input_ = schemas.DetectorSchema(dummy_schema) + input_ = sc.DetectorSchema(dummy_schema) assert len(field := _extract_field(input_, "detectorType")) == 1 assert expected == set(field) @@ -88,7 +88,7 @@ def test_get_field_multiple(self) -> None: dummy_schema2 = dummy_schema.copy() dummy_schema2["detectorType"] = "dummy2" input_ = [dummy_schema, dummy_schema2] - input_ = [schemas.DetectorSchema(in_) for in_ in input_] + input_ = [sc.DetectorSchema(in_) for in_ in input_] assert len(field := _extract_field(input_, "detectorType")) == 2 assert expected == set(field) @@ -100,7 +100,7 @@ def test_get_logID_timestamp_single(self) -> None: dummy_schema2 = dummy_schema.copy() dummy_schema2["logIDs"] = ["logID1", "logID2", "logID3", "logID1"] dummy_schema2["logTimestamps"] = [-1, 0, 1, 3] - input_ = schemas.DetectorSchema(dummy_schema2) + input_ = sc.DetectorSchema(dummy_schema2) logids, times = _extract_list_fields( input_, field1="logIDs", field2="logTimestamps" @@ -117,7 +117,7 @@ def test_get_logID_timestamp_multiple(self) -> None: dummy_schema2["logIDs"] = ["logID2", "logID3", "logID1"] dummy_schema2["logTimestamps"] = [0, 1, 3] input_ = [dummy_schema, dummy_schema2] - input_ = [schemas.DetectorSchema(in_) for in_ in input_] + input_ = [sc.DetectorSchema(in_) for in_ in input_] logids, times = _extract_list_fields( input_, field1="logIDs", field2="logTimestamps" @@ -176,7 +176,7 @@ def test_process_input_schema_not_serialized(self) -> None: def test_process_input_schema_not_serialized_window_3(self) -> None: alert_aggregator = MockupAlertAggregator(name="TestDetector", config=MockupConfig(), buffer_size=3) - expected_result = schemas.AggregateSchema({ + expected_result = sc.AggregateSchema({ "__version__": "1.0.0", "detectorIDs": ["1"], "detectorTypes": ["dummy"], @@ -199,7 +199,7 @@ def test_process_input_schema_not_serialized_buffer_3(self) -> None: alert_aggregator = MockupAlertAggregatorBuffer( name="TestDetector", config=MockupConfig(), buffer_size=3 ) - expected_result = schemas.AggregateSchema({ + expected_result = sc.AggregateSchema({ "__version__": "1.0.0", "detectorIDs": ["1"], "detectorTypes": ["dummy"], diff --git a/tests/test_pipelines/test_main_pipeline.py b/tests/test_pipelines/test_main_pipeline.py index 2249b22f..c120c233 100644 --- a/tests/test_pipelines/test_main_pipeline.py +++ b/tests/test_pipelines/test_main_pipeline.py @@ -1,5 +1,6 @@ +from tests.test_data import AUDIT_LOG, AUDIT_TEMPLATES, TRAIN_UNTIL -from detectmatelibrary.pipeline import PipelineConfig, _run_pipeline +from detectmatelibrary.pipeline import PipelineConfig, _run_pipeline, Pipeline from detectmatelibrary._testutils.dummy_parser import DummyParser from detectmatelibrary._testutils.dummy_detector import DummyDetectorConfig @@ -7,10 +8,20 @@ DummyAlertAggregator, DummyAlertAggregatorConfig ) +from detectmatelibrary.helper.from_to import From import detectmatelibrary.schemas as sc from detectmatelibrary.common.detector import CoreDetector +from detectmatelibrary.detectors.ecvc_detector import ECVCDetectorConfig +from detectmatelibrary.detectors.charset_detector import CharsetDetectorConfig + +from detectmatelibrary.parsers.tree_matcher import TemplateCppTreeMatcherConfig + +from detectmatelibrary.alert_aggregation.logid_count import LogIDCountAggegationConfig + +import pytest + class AlwayAlert(CoreDetector): """A dummy detector for testing purposes.""" @@ -107,3 +118,40 @@ def test_run_detectors2aggregator(self) -> None: assert isinstance(out_, list) assert isinstance(out_[0], sc.AggregateSchema) assert len(out_) == 1 + + +class TestPipeline: + @pytest.mark.ignored + def test_full_pipeline(self) -> None: + config = PipelineConfig( + do_parsing=True, + do_aggregation=True, + parser=TemplateCppTreeMatcherConfig( + path_templates=AUDIT_TEMPLATES, + log_format="type= msg=audit(