From 453e0cd5e6b0ba3791b383944aed45037ba4221f Mon Sep 17 00:00:00 2001 From: Markus Opolka Date: Mon, 18 May 2026 11:11:36 +0200 Subject: [PATCH 1/3] Support regex in include-exclude labels --- README.md | 2 ++ cmd/alert.go | 40 +++++++++++++++++++++++++++++----------- cmd/alert_test.go | 30 ++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 52d5694..b8ffac4 100644 --- a/README.md +++ b/README.md @@ -231,6 +231,8 @@ OK - 1 Alerts: 0 Firing - 0 Pending - 1 Inactive \_[OK] [ApacheDown] is inactive ``` +Values support regular expressions (e.g. `severity=warn.+`). + #### Checking watchdog alerts In Prometheus a "watchdog" or "dead man's switch" is an alert that is always firing to ensure alerting pipeline is working. The `-W, --watchdog` flag can be used to flip/negate the exit state of the plugin for these kind of alerts: diff --git a/cmd/alert.go b/cmd/alert.go index 6ee2f17..be01b51 100644 --- a/cmd/alert.go +++ b/cmd/alert.go @@ -120,7 +120,11 @@ inactive = 0`, } } - labelsMatchedInclude := matchesLabel(rl.AlertingRule.Labels, cliAlertConfig.IncludeLabels) + labelsMatchedInclude, regexErr := matchesLabel(rl.AlertingRule.Labels, cliAlertConfig.IncludeLabels) + + if regexErr != nil { + check.ExitRaw(check.Unknown, "Invalid regular expression provided:", regexErr.Error()) + } if len(cliAlertConfig.IncludeLabels) > 0 && !labelsMatchedInclude { // If the alert labels don't match here we can skip it. @@ -143,7 +147,11 @@ inactive = 0`, continue } - labelsMatchedExclude := matchesLabel(rl.AlertingRule.Labels, cliAlertConfig.ExcludeLabels) + labelsMatchedExclude, regexErr := matchesLabel(rl.AlertingRule.Labels, cliAlertConfig.ExcludeLabels) + + if regexErr != nil { + check.ExitRaw(check.Unknown, "Invalid regular expression provided:", regexErr.Error()) + } if len(cliAlertConfig.ExcludeLabels) > 0 && labelsMatchedExclude { // If the alert labels matches here we can skip it. @@ -261,12 +269,12 @@ func init() { fs.StringArrayVar(&cliAlertConfig.IncludeLabels, "include-label", []string{}, "The label of one or more specific alerts to include. "+ - "\nThis parameter can be repeated e.g.: '--include-label prio=high --include-label another=example'"+ + "\nThis parameter can be repeated e.g.: '--include-label prio=high --include-label another=example'. Supports regex for values"+ "\nNote that repeated --include-label are combined using a union.") fs.StringArrayVar(&cliAlertConfig.ExcludeLabels, "exclude-label", []string{}, "The label of one or more specific alerts to exclude."+ - "\nThis parameter can be repeated e.g.: '--exclude-label prio=high --exclude-label another=example'") + "\nThis parameter can be repeated e.g.: '--exclude-label prio=high --exclude-label another=example'. Supports regex for values") fs.BoolVarP(&cliAlertConfig.ProblemsOnly, "problems", "P", false, "Display only alerts which status is not inactive/OK. Note that in combination with the --name flag this might result in no alerts being displayed") @@ -314,22 +322,32 @@ func matches(input string, regexToExclude []string) (bool, error) { } // Matches a list of labels against a list of labels -func matchesLabel(labels model.LabelSet, labelsToMatch []string) bool { +func matchesLabel(labels model.LabelSet, labelsToMatch []string) (bool, error) { for _, lb := range labelsToMatch { - kv := strings.SplitN(lb, "=", 2) + expectedLabelSet := strings.SplitN(lb, "=", 2) - if len(kv) != 2 { + if len(expectedLabelSet) != 2 { continue } + // Do we have a value for the expected key? + actualValue, ok := labels[model.LabelName(expectedLabelSet[0])] - key, value := model.LabelName(kv[0]), model.LabelValue(kv[1]) + if !ok { + return false, nil + } - if val, ok := labels[key]; ok && val == value { - return true + re, err := regexp.Compile(expectedLabelSet[1]) + if err != nil { + return false, err + } + + // Does the values match the expected label regex? + if re.MatchString(string(actualValue)) { + return true, nil } } - return false + return false, nil } // negateStatus turns an OK state into critical and a warning/critical state into OK diff --git a/cmd/alert_test.go b/cmd/alert_test.go index b915c62..6dda8c6 100644 --- a/cmd/alert_test.go +++ b/cmd/alert_test.go @@ -269,6 +269,20 @@ exit status 2 \_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning"} |total=1 firing=0 pending=1 inactive=0 +exit status 1 +`, + }, + { + name: "alert-exclude-label-regex", + server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(loadTestdata(alertTestDataSet1)) + })), + args: []string{"run", "../main.go", "alert", "--exclude-label", "severity=crit.*"}, + expected: `[WARNING] - 1 Alerts: 0 Firing - 1 Pending - 0 Inactive +\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning"} +|total=1 firing=0 pending=1 inactive=0 + exit status 1 `, }, @@ -285,6 +299,22 @@ exit status 1 \_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"} |total=3 firing=1 pending=1 inactive=1 +exit status 2 +`, + }, + { + name: "alert-include-label-multiple-regex", + server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(loadTestdata(alertTestDataSet1)) + })), + args: []string{"run", "../main.go", "alert", "--include-label", "team=data.+", "--include-label", "severity=critical"}, + expected: `[CRITICAL] - 3 Alerts: 1 Firing - 1 Pending - 1 Inactive +\_ [OK] [HostOutOfMemory] is inactive +\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning"} +\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"} +|total=3 firing=1 pending=1 inactive=1 + exit status 2 `, }, From 4d70136334d6f6456eaef48f7812b4686b6909ed Mon Sep 17 00:00:00 2001 From: Markus Opolka Date: Wed, 3 Jun 2026 09:07:26 +0200 Subject: [PATCH 2/3] Extend testdata --- testdata/alertmanager/alert.rules | 28 ++++++- testdata/docker-compose.yml | 4 + testdata/prometheus.yml | 19 +++++ testdata/unittest/alertDataset5.json | 107 +++++++++++++++++++++++++++ 4 files changed, 157 insertions(+), 1 deletion(-) create mode 100644 testdata/unittest/alertDataset5.json diff --git a/testdata/alertmanager/alert.rules b/testdata/alertmanager/alert.rules index 3509dc0..d11d3be 100644 --- a/testdata/alertmanager/alert.rules +++ b/testdata/alertmanager/alert.rules @@ -15,6 +15,8 @@ groups: labels: severity: critical icingaState: warning + example: foobar + team: monitoring annotations: summary: Prometheus target missing (instance {{ $labels.instance }}) description: "A Prometheus target has disappeared. An exporter might be crashed.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" @@ -25,6 +27,8 @@ groups: labels: severity: low icingaState: warning + example: barfoo + team: database annotations: summary: Prometheus AlertManager job missing (instance {{ $labels.instance }}) description: "A Prometheus AlertManager job has disappeared\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" @@ -34,15 +38,37 @@ groups: for: 2m labels: severity: warning + team: linux annotations: summary: Host out of memory (instance {{ $labels.instance }}) description: "Node memory is filling up (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - alert: HostHighCpuLoad - expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[2m])) * 100) > 80 + expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[2m])) * 100) > 5 for: 0m labels: severity: extreme + team: awesome annotations: summary: Host high CPU load (instance {{ $labels.instance }}) description: "CPU load is > 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: HostMemoryIsUnderutilized + expr: min_over_time(node_memory_MemFree_bytes[1w]) > node_memory_MemTotal_bytes * .8 + for: 0m + labels: + team: linux + severity: info + annotations: + summary: Host Memory is underutilized (instance {{ $labels.instance }}) + description: "Node memory usage is < 20% for 1 week. Consider reducing memory space. (instance {{ $labels.instance }})\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: BlackboxSlowProbe + expr: probe_duration_seconds > 0 + for: 0m + labels: + severity: warning + team: network + annotations: + summary: Blackbox slow probe (instance {{ $labels.instance }}) + description: "Blackbox probe took more than 1s to complete\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" diff --git a/testdata/docker-compose.yml b/testdata/docker-compose.yml index f0cf523..3208310 100644 --- a/testdata/docker-compose.yml +++ b/testdata/docker-compose.yml @@ -17,6 +17,10 @@ services: image: docker.io/prom/node-exporter ports: - '9100:9100' + blackbox-exporter: + image: docker.io/prom/blackbox-exporter + ports: + - '9115:9115' alertmanager: image: docker.io/prom/alertmanager diff --git a/testdata/prometheus.yml b/testdata/prometheus.yml index 0104fcb..a5c57b5 100644 --- a/testdata/prometheus.yml +++ b/testdata/prometheus.yml @@ -16,6 +16,25 @@ scrape_configs: static_configs: - targets: ['node-exporter:9100'] + - job_name: 'blackbox' + metrics_path: /probe + params: + module: [http_2xx] + static_configs: + - targets: + - http://alertmanager:9093 + - http://prometheus:9090 + - http://blackbox-exporter:9115 + relabel_configs: + - source_labels: [__address__] + target_label: __param_target + - source_labels: [__param_target] + target_label: instance + - source_labels: [__param_target] + target_label: name + - target_label: __address__ + replacement: blackbox-exporter:9115 + alerting: alertmanagers: - static_configs: diff --git a/testdata/unittest/alertDataset5.json b/testdata/unittest/alertDataset5.json new file mode 100644 index 0000000..0859428 --- /dev/null +++ b/testdata/unittest/alertDataset5.json @@ -0,0 +1,107 @@ +{ + "status": "success", + "data": { + "groups": [ + { + "name": "Examples", + "file": "alerts.yaml", + "rules": [ + { + "state": "firing", + "name": "PrometheusTargetMissing", + "query": "up == 0", + "duration": 0, + "keepFiringFor": 0, + "labels": { + "example": "foobar", + "icingaState": "warning", + "severity": "critical", + "team": "monitoring" + }, + "annotations": { + "description": "A Prometheus target has disappeared.", + "summary": "Prometheus target missing" + }, + "alerts": [], + "health": "ok", + "evaluationTime": 0.000096631, + "lastEvaluation": "2026-06-02T08:52:53.929002951Z", + "type": "alerting" + }, + { + "state": "firing", + "name": "ContainerKilled", + "query": "example", + "duration": 17280000, + "labels": { + "severity": "warning", + "team": "k8s" + }, + "annotations": { + "description": "A container has disappeared\n VALUE = {{ $value }}\n LABELS = {{ $labels }}", + "summary": "Container killed (instance {{ $labels.instance }})" + }, + "alerts": [ + { + "labels": { + "alertname": "ContainerKilled", + "instance": "example:8888", + "job": "cadvisor", + "name": "nodered", + "severity": "warning" + }, + "annotations": { + "description": "Container killed (instance example:8088)", + "summary": "Job: [cadvisor] on Instance: [example:8088] is firing - value: 1234" + }, + "state": "firing", + "activeAt": "2022-11-21T10:38:35.373483748Z", + "value": "123.4" + }, + { + "labels": { + "alertname": "ContainerKilled", + "instance": "example:8888", + "job": "cadvisor", + "name": "foobar", + "severity": "warning" + }, + "annotations": { + "description": "Container killed (instance foobar:8088)", + "summary": "Job: [cadvisor] on Instance: [foobar:8088] is firing - value: 1234" + }, + "state": "firing", + "activeAt": "2022-11-21T10:38:35.373483748Z", + "value": "123.4" + }, + { + "labels": { + "alertname": "ContainerKilled", + "instance": "example:8888", + "job": "cadvisor", + "name": "mosquitto", + "severity": "warning" + }, + "annotations": { + "description": "Container killed (instance example:8088)", + "summary": "Job: [cadvisor] on Instance: [example:8088] is firing - value: 1234" + }, + "state": "firing", + "activeAt": "2022-11-21T10:38:35.373483748Z", + "value": "123.4" + } + ], + "health": "ok", + "evaluationTime": 0.002909617, + "lastEvaluation": "2022-11-24T14:08:25.375220595Z", + "type": "alerting" + } + ], + "interval": 10, + "limit": 0, + "evaluationTime": 0.003046259, + "lastEvaluation": "2022-11-24T14:08:25.375096825Z" + } + ] + } +} From 4e9a4b928d7c97c41f82cb9df43e8f835084cac8 Mon Sep 17 00:00:00 2001 From: Markus Opolka Date: Wed, 3 Jun 2026 09:37:41 +0200 Subject: [PATCH 3/3] Use alert labels when evaluating inclusion exclusion of alerts --- cmd/alert.go | 35 +++++++++++----- cmd/alert_test.go | 63 ++++++++++++++++++---------- testdata/unittest/alertDataset1.json | 3 ++ 3 files changed, 67 insertions(+), 34 deletions(-) diff --git a/cmd/alert.go b/cmd/alert.go index be01b51..9286572 100644 --- a/cmd/alert.go +++ b/cmd/alert.go @@ -120,17 +120,6 @@ inactive = 0`, } } - labelsMatchedInclude, regexErr := matchesLabel(rl.AlertingRule.Labels, cliAlertConfig.IncludeLabels) - - if regexErr != nil { - check.ExitRaw(check.Unknown, "Invalid regular expression provided:", regexErr.Error()) - } - - if len(cliAlertConfig.IncludeLabels) > 0 && !labelsMatchedInclude { - // If the alert labels don't match here we can skip it. - continue - } - // Skip inactive alerts if flag is set if len(rl.AlertingRule.Alerts) == 0 && cliAlertConfig.ProblemsOnly { continue @@ -147,6 +136,7 @@ inactive = 0`, continue } + // Check if the alert group should be excluded labelsMatchedExclude, regexErr := matchesLabel(rl.AlertingRule.Labels, cliAlertConfig.ExcludeLabels) if regexErr != nil { @@ -199,6 +189,28 @@ inactive = 0`, counterFiring++ } + labelsMatchedInclude, regexErr := matchesLabel(alert.Labels, cliAlertConfig.IncludeLabels) + + if regexErr != nil { + check.ExitRaw(check.Unknown, "Invalid regular expression provided:", regexErr.Error()) + } + + if len(cliAlertConfig.IncludeLabels) > 0 && !labelsMatchedInclude { + // If the alert labels don't match here we can skip it. + continue + } + + labelsMatchedExclude, regexErr := matchesLabel(alert.Labels, cliAlertConfig.ExcludeLabels) + + if regexErr != nil { + check.ExitRaw(check.Unknown, "Invalid regular expression provided:", regexErr.Error()) + } + + if len(cliAlertConfig.ExcludeLabels) > 0 && labelsMatchedExclude { + // If the alert labels matches here we can skip it. + continue + } + sc := result.NewPartialResult() rlStatus := rl.GetStatus(cliAlertConfig.StateLabelKey) @@ -329,6 +341,7 @@ func matchesLabel(labels model.LabelSet, labelsToMatch []string) (bool, error) { if len(expectedLabelSet) != 2 { continue } + // Do we have a value for the expected key? actualValue, ok := labels[model.LabelName(expectedLabelSet[0])] diff --git a/cmd/alert_test.go b/cmd/alert_test.go index 6dda8c6..edd11fb 100644 --- a/cmd/alert_test.go +++ b/cmd/alert_test.go @@ -39,6 +39,8 @@ func TestAlertCmd(t *testing.T) { alertTestDataSet4 := "../testdata/unittest/alertDataset4.json" + alertTestDataSet5 := "../testdata/unittest/alertDataset5.json" + tests := []AlertTest{ { name: "alert-none", @@ -85,8 +87,8 @@ func TestAlertCmd(t *testing.T) { args: []string{"run", "../main.go", "alert"}, expected: `[CRITICAL] - 3 Alerts: 1 Firing - 1 Pending - 1 Inactive \_ [OK] [HostOutOfMemory] is inactive -\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning"} -\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"} +\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning","team":"database"} +\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","icinga":"ok","instance":"https://localhost:443","job":"blackbox","severity":"critical","team":"network"} |total=3 firing=1 pending=1 inactive=1 exit status 2 @@ -100,8 +102,8 @@ exit status 2 })), args: []string{"run", "../main.go", "alert", "--problems"}, expected: `[CRITICAL] - 2 Alerts: 1 Firing - 1 Pending - 0 Inactive -\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning"} -\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"} +\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning","team":"database"} +\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","icinga":"ok","instance":"https://localhost:443","job":"blackbox","severity":"critical","team":"network"} |total=2 firing=1 pending=1 inactive=0 exit status 2 @@ -115,7 +117,7 @@ exit status 2 })), args: []string{"run", "../main.go", "alert", "--problems", "-g", "TLS"}, expected: `[CRITICAL] - 1 Alerts: 1 Firing - 0 Pending - 0 Inactive -\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"} +\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","icinga":"ok","instance":"https://localhost:443","job":"blackbox","severity":"critical","team":"network"} |total=1 firing=1 pending=0 inactive=0 exit status 2 @@ -129,8 +131,8 @@ exit status 2 })), args: []string{"run", "../main.go", "alert", "--problems", "-g", "SQL", "-g", "TLS"}, expected: `[CRITICAL] - 2 Alerts: 1 Firing - 1 Pending - 0 Inactive -\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning"} -\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"} +\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning","team":"database"} +\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","icinga":"ok","instance":"https://localhost:443","job":"blackbox","severity":"critical","team":"network"} |total=2 firing=1 pending=1 inactive=0 exit status 2 @@ -144,7 +146,7 @@ exit status 2 })), args: []string{"run", "../main.go", "alert", "--problems", "--exclude-alert", "Sql.*DeniedRate"}, expected: `[CRITICAL] - 1 Alerts: 1 Firing - 0 Pending - 0 Inactive -\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"} +\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","icinga":"ok","instance":"https://localhost:443","job":"blackbox","severity":"critical","team":"network"} |total=1 firing=1 pending=0 inactive=0 exit status 2 @@ -196,7 +198,7 @@ exit status 3 args: []string{"run", "../main.go", "alert", "--name", "HostOutOfMemory", "--name", "BlackboxTLS"}, expected: `[CRITICAL] - 2 Alerts: 1 Firing - 0 Pending - 1 Inactive \_ [OK] [HostOutOfMemory] is inactive -\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"} +\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","icinga":"ok","instance":"https://localhost:443","job":"blackbox","severity":"critical","team":"network"} |total=2 firing=1 pending=0 inactive=1 exit status 2 @@ -210,7 +212,7 @@ exit status 2 })), args: []string{"run", "../main.go", "alert", "--name", "HostOutOfMemory", "--name", "BlackboxTLS", "--problems"}, expected: `[CRITICAL] - 1 Alerts: 1 Firing - 0 Pending - 0 Inactive -\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"} +\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","icinga":"ok","instance":"https://localhost:443","job":"blackbox","severity":"critical","team":"network"} |total=1 firing=1 pending=0 inactive=0 exit status 2 @@ -250,10 +252,10 @@ exit status 2 w.Write(loadTestdata(alertTestDataSet1)) })), args: []string{"run", "../main.go", "alert", "--include-label", "severity=critical"}, - expected: `[CRITICAL] - 2 Alerts: 1 Firing - 0 Pending - 1 Inactive + expected: `[CRITICAL] - 3 Alerts: 1 Firing - 1 Pending - 1 Inactive \_ [OK] [HostOutOfMemory] is inactive -\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"} -|total=2 firing=1 pending=0 inactive=1 +\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","icinga":"ok","instance":"https://localhost:443","job":"blackbox","severity":"critical","team":"network"} +|total=3 firing=1 pending=1 inactive=1 exit status 2 `, @@ -266,7 +268,7 @@ exit status 2 })), args: []string{"run", "../main.go", "alert", "--exclude-label", "severity=critical"}, expected: `[WARNING] - 1 Alerts: 0 Firing - 1 Pending - 0 Inactive -\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning"} +\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning","team":"database"} |total=1 firing=0 pending=1 inactive=0 exit status 1 @@ -280,7 +282,7 @@ exit status 1 })), args: []string{"run", "../main.go", "alert", "--exclude-label", "severity=crit.*"}, expected: `[WARNING] - 1 Alerts: 0 Firing - 1 Pending - 0 Inactive -\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning"} +\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning","team":"database"} |total=1 firing=0 pending=1 inactive=0 exit status 1 @@ -295,8 +297,8 @@ exit status 1 args: []string{"run", "../main.go", "alert", "--include-label", "team=database", "--include-label", "severity=critical"}, expected: `[CRITICAL] - 3 Alerts: 1 Firing - 1 Pending - 1 Inactive \_ [OK] [HostOutOfMemory] is inactive -\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning"} -\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"} +\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning","team":"database"} +\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","icinga":"ok","instance":"https://localhost:443","job":"blackbox","severity":"critical","team":"network"} |total=3 firing=1 pending=1 inactive=1 exit status 2 @@ -311,8 +313,8 @@ exit status 2 args: []string{"run", "../main.go", "alert", "--include-label", "team=data.+", "--include-label", "severity=critical"}, expected: `[CRITICAL] - 3 Alerts: 1 Firing - 1 Pending - 1 Inactive \_ [OK] [HostOutOfMemory] is inactive -\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning"} -\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"} +\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning","team":"database"} +\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","icinga":"ok","instance":"https://localhost:443","job":"blackbox","severity":"critical","team":"network"} |total=3 firing=1 pending=1 inactive=1 exit status 2 @@ -327,8 +329,8 @@ exit status 2 args: []string{"run", "../main.go", "alert", "--include-label", "severity=warning", "--include-label", "severity=critical"}, expected: `[CRITICAL] - 3 Alerts: 1 Firing - 1 Pending - 1 Inactive \_ [OK] [HostOutOfMemory] is inactive -\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning"} -\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"} +\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning","team":"database"} +\_ [CRITICAL] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","icinga":"ok","instance":"https://localhost:443","job":"blackbox","severity":"critical","team":"network"} |total=3 firing=1 pending=1 inactive=1 exit status 2 @@ -352,11 +354,26 @@ exit status 2 args: []string{"run", "../main.go", "alert", "--label-key-state=icinga"}, expected: `[WARNING] - 3 Alerts: 1 Firing - 1 Pending - 1 Inactive \_ [OK] [HostOutOfMemory] is inactive -\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning"} -\_ [OK] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","instance":"https://localhost:443","job":"blackbox","severity":"critical"} +\_ [WARNING] [SqlAccessDeniedRate] - Job: [mysql] on Instance: [localhost] is pending - value: 0.40 - {"alertname":"SqlAccessDeniedRate","instance":"localhost","job":"mysql","severity":"warning","team":"database"} +\_ [OK] [BlackboxTLS] - Job: [blackbox] on Instance: [https://localhost:443] is firing - value: -6065338.00 - {"alertname":"TLS","icinga":"ok","instance":"https://localhost:443","job":"blackbox","severity":"critical","team":"network"} |total=3 firing=1 pending=1 inactive=1 exit status 1 +`, + }, + { + name: "alert-include-with-name-and-regex", + server: httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(loadTestdata(alertTestDataSet5)) + })), + args: []string{"run", "../main.go", "alert", "--name", "ContainerKilled", "--include-label", "name=(mosquitto|nodered)"}, + expected: `[CRITICAL] - 3 Alerts: 3 Firing - 0 Pending - 0 Inactive +\_ [CRITICAL] [ContainerKilled] - Job: [cadvisor] on Instance: [example:8888] is firing - value: 123.40 - {"alertname":"ContainerKilled","instance":"example:8888","job":"cadvisor","name":"nodered","severity":"warning"} +\_ [CRITICAL] [ContainerKilled] - Job: [cadvisor] on Instance: [example:8888] is firing - value: 123.40 - {"alertname":"ContainerKilled","instance":"example:8888","job":"cadvisor","name":"mosquitto","severity":"warning"} +|total=3 firing=3 pending=0 inactive=0 + +exit status 2 `, }, } diff --git a/testdata/unittest/alertDataset1.json b/testdata/unittest/alertDataset1.json index 54a346b..fe20397 100644 --- a/testdata/unittest/alertDataset1.json +++ b/testdata/unittest/alertDataset1.json @@ -54,6 +54,7 @@ "alertname": "SqlAccessDeniedRate", "instance": "localhost", "job": "mysql", + "team": "database", "severity": "warning" }, "annotations": { @@ -100,6 +101,8 @@ "alertname": "TLS", "instance": "https://localhost:443", "job": "blackbox", + "icinga": "ok", + "team": "network", "severity": "critical" }, "annotations": {