knowledge_base: add failing job examples

knowledge_base/failing_job_examples/.gitignore

@@ -0,0 +1,2 @@
+.databricks/
+__pycache__/

knowledge_base/failing_job_examples/README.md

@@ -0,0 +1,42 @@
+# Failing job examples
+
+This bundle provides small Lakeflow Jobs that fail for known, deterministic reasons. Use them to test alerting, monitoring, troubleshooting, and automated remediation workflows without changing production jobs.
+
+Every job is expected to finish in the `FAILED` state. The bundle has no schedules, configured retries, or persistent data writes.
+
+The target workspace must support serverless Jobs compute.
+
+## Included failures
+
+| Resource key | Failure type | Expected error |
+| --- | --- | --- |
+| `schema_drift_failure` | An upstream `amount` column changed from numeric to string | `Schema drift detected: expected amount to be numeric, found string` |
+| `missing_input_failure` | A required table does not exist | Spark `TABLE_OR_VIEW_NOT_FOUND` |
+| `invalid_configuration_failure` | Invalid batch size and missing checkpoint path | `Invalid job configuration: batch_size must be greater than zero; checkpoint_path is required in incremental mode` |
+
+## Run the examples
+
+The default target uses development mode, so deployed job names are prefixed with your user name.
+
+```bash
+databricks bundle validate
+databricks bundle deploy
+```
+
+Run each example separately:
+
+```bash
+databricks bundle run schema_drift_failure
+databricks bundle run missing_input_failure
+databricks bundle run invalid_configuration_failure
+```
+
+Each `bundle run` command returns a non-zero exit code after the job reaches its intentional failure. Open the run URL printed by the CLI to inspect the task output and stack trace.
+
+## Clean up
+
+After testing, remove the deployed development resources:
+
+```bash
+databricks bundle destroy
+```

knowledge_base/failing_job_examples/databricks.yml

@@ -0,0 +1,10 @@
+bundle:
+  name: failing_job_examples
+
+include:
+  - resources/*.job.yml
+
+targets:
+  dev:
+    default: true
+    mode: development

knowledge_base/failing_job_examples/resources/invalid_configuration.job.yml

@@ -0,0 +1,14 @@
+resources:
+  jobs:
+    invalid_configuration_failure:
+      name: failing_job_example_invalid_configuration
+      description: Intentionally fails because required application settings are invalid.
+      tags:
+        example: intentional-failure
+        failure_type: invalid-configuration
+      max_concurrent_runs: 1
+      tasks:
+        - task_key: validate_configuration
+          max_retries: 0
+          notebook_task:
+            notebook_path: ../src/invalid_configuration.py

knowledge_base/failing_job_examples/resources/missing_input.job.yml

@@ -0,0 +1,14 @@
+resources:
+  jobs:
+    missing_input_failure:
+      name: failing_job_example_missing_input
+      description: Intentionally fails while reading an input table that does not exist.
+      tags:
+        example: intentional-failure
+        failure_type: missing-input
+      max_concurrent_runs: 1
+      tasks:
+        - task_key: read_missing_input
+          max_retries: 0
+          notebook_task:
+            notebook_path: ../src/missing_input.py

knowledge_base/failing_job_examples/resources/schema_drift.job.yml

@@ -0,0 +1,14 @@
+resources:
+  jobs:
+    schema_drift_failure:
+      name: failing_job_example_schema_drift
+      description: Intentionally fails when an upstream column changes from numeric to string.
+      tags:
+        example: intentional-failure
+        failure_type: schema-drift
+      max_concurrent_runs: 1
+      tasks:
+        - task_key: validate_transaction_schema
+          max_retries: 0
+          notebook_task:
+            notebook_path: ../src/schema_drift.py

knowledge_base/failing_job_examples/src/invalid_configuration.py

@@ -0,0 +1,15 @@
+# Databricks notebook source
+config = {
+    "batch_size": 0,
+    "checkpoint_path": "",
+    "mode": "incremental",
+}
+
+errors = []
+if config["batch_size"] <= 0:
+    errors.append("batch_size must be greater than zero")
+if config["mode"] == "incremental" and not config["checkpoint_path"]:
+    errors.append("checkpoint_path is required in incremental mode")
+
+if errors:
+    raise ValueError(f"Invalid job configuration: {'; '.join(errors)}")

knowledge_base/failing_job_examples/src/missing_input.py

@@ -0,0 +1,11 @@
+# Databricks notebook source
+from pyspark.sql import SparkSession
+
+
+spark = SparkSession.builder.getOrCreate()
+missing_view = "__failing_job_examples_missing_orders_7f3f2a9c__"
+spark.catalog.dropGlobalTempView(missing_view)
+missing_table = f"global_temp.{missing_view}"
+
+print(f"Reading required input table: {missing_table}")
+spark.read.table(missing_table).count()

knowledge_base/failing_job_examples/src/schema_drift.py

@@ -0,0 +1,17 @@
+# Databricks notebook source
+from pyspark.sql.types import StringType, StructField, StructType
+
+
+schema = StructType(
+    [
+        StructField("transaction_id", StringType(), False),
+        StructField("amount", StringType(), False),
+    ]
+)
+amount_type = schema["amount"].dataType.simpleString()
+numeric_types = {"byte", "short", "int", "bigint", "float", "double"}
+
+if amount_type not in numeric_types and not amount_type.startswith("decimal"):
+    raise TypeError(
+        f"Schema drift detected: expected amount to be numeric, found {amount_type}"
+    )