Pipeline to run library benchmarks

docs/examples/alert_aggregators/basic_concat.py

@@ -2,15 +2,15 @@
 # --8<-- [start:example_1]
 from detectmatelibrary.alert_aggregation.basic_concat import BasicConcatAggregation
 
-import detectmatelibrary.schemas as schemas
+import detectmatelibrary.schemas as sc
 
 import yaml
 
 
 with open("docs/examples/alert_aggregators/basic_concat.yaml") as f:
     config = yaml.safe_load(f)
 
-input_ = schemas.DetectorSchema({
+input_ = sc.DetectorSchema({
     "detectorID": "1",
     "detectorType": "dummy",
     "alertID": "2",

docs/examples/others/pipeline.py

@@ -0,0 +1,46 @@
+# --8<-- [start:example_1]
+from detectmatelibrary.alert_aggregation.logid_count import LogIDCountAggegationConfig
+
+from detectmatelibrary.parsers.tree_matcher import TemplateCppTreeMatcherConfig
+
+from detectmatelibrary.detectors.charset_detector import CharsetDetectorConfig
+
+from detectmatelibrary.pipeline import PipelineConfig, Pipeline
+import detectmatelibrary.schemas as schemas
+
+config = PipelineConfig(
+        do_parsing=True,  # Use the parser
+        do_aggregation=True,  # Use the aggregator
+        parser=TemplateCppTreeMatcherConfig(
+            path_templates="docs/examples/data/tutorial_templates.txt",
+            log_format="type=<Type> msg=audit(<Time>): <Content>"
+        ),
+        n_workers=1,  # You can add parallelization for the detectors
+        detectors=[
+            {
+                "detectors": {
+                    "ECVCDetector": {
+                        "method_type": "ecvc_detector",
+                        "data_use_configure": 100,
+                        "window_size": 10,
+                        "seed": 0,
+                        "validation_per": 0.,
+                        "threshold_method": "mean"
+                    }
+                }
+            },
+            CharsetDetectorConfig(
+                data_use_configure=100,
+                data_use_training=1,
+                auto_config=True,
+            )
+        ],
+        aggregator=LogIDCountAggegationConfig(max_count=1)
+    )
+
+pipeline = Pipeline(config=config)
+pipeline.process(schemas.LogSchema({
+    "logID": "0",
+    "log": "pid=9699 uid=0 auid=4294967295 ses=4294967295 msg='op=PAM:accounting acct=\"root\"'",
+}))
+# --8<-- [end:example_1]

docs/examples/others/select_component.py

@@ -0,0 +1,22 @@
+
+# --8<-- [start:pickyou]
+from detectmatelibrary.detectors.random_detector import RandomDetectorConfig
+
+import detectmatelibrary.helper.select_component as selc
+
+configs = [
+    {
+        "detectors": {
+            "a": {"method_type": "random_detector"}
+        }
+    },
+    RandomDetectorConfig(),
+]
+names = ["a", "b"]
+
+components = selc.selectComponents(
+    configs=configs, names=names, component_type="detectors"
+)
+print(len(components))  # 2
+
+# --8<-- [end:pickyou]

docs/helper/select_component.md

@@ -0,0 +1,26 @@
+# Select Component
+
+Methods that make it easy to choose components solely through configuration settings.
+
+```python
+def selectComponent(
+    config: CoreConfig | dict[str, Any],
+    component_type: str,
+    name: Optional[str] = None
+) -> CoreComponent:
+    """Select a single component"""
+
+
+def selectComponents(
+    configs: list[CoreConfig | dict[str, Any]],
+    component_type: str,
+    names: Optional[list[str | None]] = None
+) -> list[CoreComponent]:
+    """Select multiple components"""
+```
+
+## Usage
+
+```python
+--8<-- "docs/examples/others/select_component.py:pickyou"
+```

docs/index.md

@@ -27,6 +27,7 @@ Here you can get the full scope.
 * [Parsers](parsers.md)
 * [Detectors](detectors.md)
 * [Alert Aggregation](alert_aggregator.md)
+* [Pipelines](pipeline.md)
 * [Federation](federation.md)
 
 ## How to contribute

docs/pipeline.md

@@ -0,0 +1,18 @@
+# Pipelines
+
+Pipelines that enable running the various components within the library, simulating the final behavior delivered by the service.
+
+
+## Main pipeline
+
+The main pipeline functions like a standard inference pipeline. To initialize the entire system, the user only needs to provide either a configuration dictionary or the configuration classes themselves. Since the pipeline inherits from [CoreComponent](advanced/overall_architecture.md), all CoreComponent methods are also available in the pipeline class.
+
+If `do_parsing=False`, the pipeline will take already-parsed logs as input, and the parsing step will be skipped. Likewise, if `do_aggregation=False`, the pipeline will output the detectors’ alerts directly, and the alert aggregation step will be skipped.
+
+The pipeline allows you to pass either a Config class instance or the dictionary used to initialize that Config class as arguments to the various components. For the detectors list, both methods can be combined, as shown in the example.
+
+### Example
+
+```python
+--8<-- "docs/examples/others/pipeline.py:example_1"
+```

mkdocs.yml

@@ -43,6 +43,7 @@ nav:
       - Parsers: parsers.md
       - Detectors: detectors.md
       - Alert Aggregation: alert_aggregator.md
+      - Pipelines: pipeline.md
     - Parser types:
       - 00_JSON Parser: parsers/json_parser.md
       - 01_Template Matcher: parsers/template_matcher.md
@@ -69,6 +70,7 @@ nav:
       - 01_LogID Count: alert_aggregators/logid_count.md
     - Helpers:
       - FromTo: helper/from_to.md
+      - Select Component: helper/select_component.md
     - Auxiliary:
       - Persistency: auxiliar/persistency.md
       - Data buffer: auxiliar/input_buffer.md

src/detectmatelibrary/__init__.py

@@ -1,4 +1,5 @@
 from . import common
+from . import alert_aggregation
 from . import detectors
 from . import parsers
 from . import schemas
@@ -10,6 +11,7 @@
     "common",
     "detectors",
     "parsers",
+    "alert_aggregation",
     "schemas",
     "utils",
     "__authors__",

src/detectmatelibrary/_testutils/dummy_alert_aggregator.py

@@ -1,6 +1,6 @@
 from detectmatelibrary.common.alert_aggregator import (CoreAlertAggregatorConfig, CoreAlertAggregator)
 from detectmatelibrary.schemas import DetectorSchema, AggregateSchema
-from typing import Any, Optional
+from typing import Any
 
 
 class DummyAlertAggregatorConfig(CoreAlertAggregatorConfig):
@@ -12,11 +12,11 @@ class DummyAlertAggregator(CoreAlertAggregator):
     def __init__(
         self,
         name: str = "DummyAlertAggregator",
-        config: Optional[DummyAlertAggregatorConfig | dict[str, Any]] = DummyAlertAggregatorConfig(),
+        config: DummyAlertAggregatorConfig | dict[str, Any] = DummyAlertAggregatorConfig(),
     ) -> None:
         if isinstance(config, dict):
             config = DummyAlertAggregatorConfig.from_dict(config, name)
-        buffer_size: int = config.buffer_size  # type: ignore
+        buffer_size: int = config.buffer_size
         super().__init__(name=name, buffer_size=buffer_size, config=config)
 
     def aggregate_alerts(

src/detectmatelibrary/alert_aggregation/__init__.py

@@ -0,0 +1,9 @@
+from .basic_concat import BasicConcatAggregation, BasicConcatAggregationConfig
+from .logid_count import LogIDCountAggegation, LogIDCountAggegationConfig
+
+__all__ = [
+    "BasicConcatAggregation",
+    "BasicConcatAggregationConfig",
+    "LogIDCountAggegation",
+    "LogIDCountAggegationConfig",
+]

src/detectmatelibrary/alert_aggregation/basic_concat.py

@@ -4,7 +4,7 @@
 
 from detectmatelibrary.schemas import DetectorSchema, AggregateSchema
 
-from typing import Any, Optional
+from typing import Any
 from pydantic import Field
 
 
@@ -21,12 +21,12 @@ class BasicConcatAggregation(CoreAlertAggregator):
     def __init__(
         self,
         name: str = "BasicConcatAggregator",
-        config: Optional[BasicConcatAggregationConfig | dict[str, Any]] = BasicConcatAggregationConfig(),
+        config: BasicConcatAggregationConfig | dict[str, Any] = BasicConcatAggregationConfig(),
     ) -> None:
         if isinstance(config, dict):
             config = BasicConcatAggregationConfig.from_dict(config, name)
 
-        buffer_size: int = config.buffer_size  # type: ignore
+        buffer_size: int = config.buffer_size
         super().__init__(name=name, buffer_size=buffer_size, config=config)
 
     def aggregate_alerts(

src/detectmatelibrary/alert_aggregation/logid_count.py

@@ -5,7 +5,7 @@
 from detectmatelibrary.schemas import DetectorSchema, AggregateSchema
 from detectmatelibrary.utils.data_buffer import BufferMode
 
-from typing import Any, Optional
+from typing import Any
 from functools import lru_cache
 from pydantic import Field
 import polars as pl
@@ -66,7 +66,7 @@ class LogIDCountAggegation(CoreAlertAggregator):
     def __init__(
         self,
         name: str = "LogIDCountAggregator",
-        config: Optional[LogIDCountAggegationConfig | dict[str, Any]] = LogIDCountAggegationConfig(),
+        config: LogIDCountAggegationConfig | dict[str, Any] = LogIDCountAggegationConfig(),
     ) -> None:
         if isinstance(config, dict):
             config = LogIDCountAggegationConfig.from_dict(config, name)

src/detectmatelibrary/base_detectors/deeplearning_detector.py

@@ -51,7 +51,7 @@ def __init__(
         self,
         model_cls: DeepModel,
         name: str = "CoreDetector",
-        config: DeepLearningDetectorConfig = DeepLearningDetectorConfig()
+        config: DeepLearningDetectorConfig | dict[str, Any] = DeepLearningDetectorConfig()
     ) -> None:
 
         if isinstance(config, dict):

src/detectmatelibrary/base_detectors/variable_detector.py

@@ -125,7 +125,7 @@ class VariableDetector(TrackerDetector, VariableHooks):
     _time_handler: TimeFormatHandler = _core_time_handler
 
     def __init__(
-        self, name: str, config: VariableDetectorConfig = VariableDetectorConfig()
+        self, name: str, config: VariableDetectorConfig | dict[str, Any] = VariableDetectorConfig()
     ) -> None:
         if isinstance(config, dict):
             config = VariableDetectorConfig.from_dict(config, name)

src/detectmatelibrary/common/alert_aggregator.py

@@ -50,15 +50,15 @@ def __init__(
         name: str = "CoreAlertAggregator",
         buffer_mode: BufferMode = BufferMode.WINDOW,
         buffer_size: Optional[int] = 1,
-        config: Optional[CoreAlertAggregatorConfig | dict[str, Any]] = CoreAlertAggregatorConfig(),
+        config: CoreAlertAggregatorConfig | dict[str, Any] = CoreAlertAggregatorConfig(),
     ) -> None:
         if isinstance(config, dict):
             config = CoreAlertAggregatorConfig.from_dict(config, name)
 
         super().__init__(
             name=name,
-            type_=config.method_type,  # type: ignore
-            config=config,  # type: ignore
+            type_=config.method_type,
+            config=config,
             args_buffer=ArgsBuffer(mode=buffer_mode, size=buffer_size),
             input_schema=DetectorSchema,
             output_schema=AggregateSchema,

src/detectmatelibrary/common/core.py

@@ -69,11 +69,14 @@ def __init__(
         self,
         name: str,
         type_: str = "Core",
-        config: CoreConfig = CoreConfig(),
+        config: CoreConfig | dict[str, Any] = CoreConfig(),
         args_buffer: ArgsBuffer = ArgsBuffer(BufferMode.NO_BUF),
         input_schema: type[BaseSchema] = BaseSchema,
         output_schema: type[BaseSchema] = BaseSchema,
     ) -> None:
+        if isinstance(config, dict):
+            config = CoreConfig.from_dict(config, name)
+
         Component.__init__(self, name=name, type_=type_, config=config)
         FedOperations.__init__(self)
         self.config: CoreConfig

src/detectmatelibrary/common/detector.py

@@ -60,15 +60,15 @@ def __init__(
         name: str = "CoreDetector",
         buffer_mode: BufferMode = BufferMode.NO_BUF,
         buffer_size: Optional[int] = None,
-        config: Optional[CoreDetectorConfig | dict[str, Any]] = CoreDetectorConfig(),
+        config: CoreDetectorConfig | dict[str, Any] = CoreDetectorConfig(),
     ) -> None:
         if isinstance(config, dict):
             config = CoreDetectorConfig.from_dict(config, name)
 
         super().__init__(
             name=name,
-            type_=config.component_type,  # type: ignore
-            config=config,  # type: ignore
+            type_=config.component_type,
+            config=config,
             args_buffer=ArgsBuffer(mode=buffer_mode, size=buffer_size),
             input_schema=ParserSchema,
             output_schema=DetectorSchema,

src/detectmatelibrary/common/parser.py

@@ -6,7 +6,7 @@
 from detectmatelibrary.utils.aux import get_timestamp
 from detectmatelibrary import schemas
 
-from typing import Any, Optional, cast
+from typing import Any, cast
 from pydantic import model_validator
 import re
 from pydantic import Field
@@ -44,15 +44,15 @@ class CoreParser(CoreComponent):
     def __init__(
         self,
         name: str = "CoreParser",
-        config: Optional[CoreParserConfig | dict[str, Any]] = CoreParserConfig(),
+        config: CoreParserConfig | dict[str, Any] = CoreParserConfig(),
     ):
         if isinstance(config, dict):
             config = CoreParserConfig.from_dict(config, name)
 
         super().__init__(
             name=name,
-            type_=config.method_type,  # type: ignore
-            config=config,  # type: ignore
+            type_=config.method_type,
+            config=config,
             args_buffer=ArgsBuffer(mode=BufferMode.NO_BUF, size=None),
             input_schema=schemas.LogSchema,
             output_schema=schemas.ParserSchema,

src/detectmatelibrary/detectors/bigram_frequency_detector.py

@@ -73,7 +73,7 @@ class BigramFrequencyDetector(VariableDetector):
     def __init__(
         self,
         name: str = "BigramFrequencyDetector",
-        config: BigramFrequencyDetectorConfig = BigramFrequencyDetectorConfig(),
+        config: BigramFrequencyDetectorConfig | dict[str, Any] = BigramFrequencyDetectorConfig(),
     ) -> None:
         if isinstance(config, dict):
             config = BigramFrequencyDetectorConfig.from_dict(config, name)

src/detectmatelibrary/detectors/charset_detector.py

@@ -17,7 +17,7 @@ class CharsetDetector(VariableDetector):
     def __init__(
         self,
         name: str = "CharsetDetector",
-        config: CharsetDetectorConfig = CharsetDetectorConfig(),
+        config: CharsetDetectorConfig | dict[str, Any] = CharsetDetectorConfig(),
     ) -> None:
         if isinstance(config, dict):
             config = CharsetDetectorConfig.from_dict(config, name)

src/detectmatelibrary/detectors/event_sequence_detector.py

@@ -9,7 +9,7 @@
 
 from collections import deque
 
-from typing_extensions import override
+from typing_extensions import override, Any
 from pydantic import Field, model_validator
 
 
@@ -71,7 +71,7 @@ class EventSequenceDetector(TrackerDetector):
     def __init__(
         self,
         name: str = "EventSequenceDetector",
-        config: EventSequenceDetectorConfig = EventSequenceDetectorConfig(),
+        config: EventSequenceDetectorConfig | dict[str, Any] = EventSequenceDetectorConfig(),
     ) -> None:
         if isinstance(config, dict):
             config = EventSequenceDetectorConfig.from_dict(config, name)

src/detectmatelibrary/detectors/new_event_detector.py

@@ -10,6 +10,7 @@
 from detectmatelibrary.schemas import ParserSchema, DetectorSchema
 
 from pydantic import Field
+from typing import Any
 
 
 class NewEventDetectorConfig(TrackerDetectorConfig):
@@ -24,7 +25,7 @@ class NewEventDetector(TrackerDetector):
     def __init__(
         self,
         name: str = "NewEventDetector",
-        config: NewEventDetectorConfig = NewEventDetectorConfig()
+        config: NewEventDetectorConfig | dict[str, Any] = NewEventDetectorConfig()
     ) -> None:
 
         if isinstance(config, dict):