From a3c40aa16a1a6d14bf38c3ae1712abd717367fd6 Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Fri, 4 Sep 2026 14:49:57 +0200 Subject: [PATCH 1/3] knowledge_base: add failing job examples --- .../failing_job_examples/.gitignore | 2 + knowledge_base/failing_job_examples/README.md | 44 +++++++++++++++++++ .../failing_job_examples/databricks.yml | 10 +++++ .../resources/data_quality.job.yml | 14 ++++++ .../resources/invalid_configuration.job.yml | 14 ++++++ .../resources/missing_input.job.yml | 14 ++++++ .../resources/schema_drift.job.yml | 14 ++++++ .../failing_job_examples/src/data_quality.py | 40 +++++++++++++++++ .../src/invalid_configuration.py | 15 +++++++ .../failing_job_examples/src/missing_input.py | 11 +++++ .../failing_job_examples/src/schema_drift.py | 17 +++++++ 11 files changed, 195 insertions(+) create mode 100644 knowledge_base/failing_job_examples/.gitignore create mode 100644 knowledge_base/failing_job_examples/README.md create mode 100644 knowledge_base/failing_job_examples/databricks.yml create mode 100644 knowledge_base/failing_job_examples/resources/data_quality.job.yml create mode 100644 knowledge_base/failing_job_examples/resources/invalid_configuration.job.yml create mode 100644 knowledge_base/failing_job_examples/resources/missing_input.job.yml create mode 100644 knowledge_base/failing_job_examples/resources/schema_drift.job.yml create mode 100644 knowledge_base/failing_job_examples/src/data_quality.py create mode 100644 knowledge_base/failing_job_examples/src/invalid_configuration.py create mode 100644 knowledge_base/failing_job_examples/src/missing_input.py create mode 100644 knowledge_base/failing_job_examples/src/schema_drift.py diff --git a/knowledge_base/failing_job_examples/.gitignore b/knowledge_base/failing_job_examples/.gitignore new file mode 100644 index 0000000..ee6e40f --- /dev/null +++ b/knowledge_base/failing_job_examples/.gitignore @@ -0,0 +1,2 @@ +.databricks/ +__pycache__/ diff --git a/knowledge_base/failing_job_examples/README.md b/knowledge_base/failing_job_examples/README.md new file mode 100644 index 0000000..e549161 --- /dev/null +++ b/knowledge_base/failing_job_examples/README.md @@ -0,0 +1,44 @@ +# Failing job examples + +This bundle provides small Lakeflow Jobs that fail for known, deterministic reasons. Use them to test alerting, monitoring, troubleshooting, and automated remediation workflows without changing production jobs. + +Every job is expected to finish in the `FAILED` state. The bundle has no schedules, configured retries, or persistent data writes. + +The target workspace must support serverless Jobs compute. + +## Included failures + +| Resource key | Failure type | Expected error | +| --- | --- | --- | +| `schema_drift_failure` | An upstream `amount` column changed from numeric to string | `Schema drift detected: expected amount to be numeric, found string` | +| `data_quality_failure` | Duplicate IDs, a null customer, and a non-positive amount | `Data quality checks failed: duplicate_transaction_ids=1, null_customers=1, non_positive_amounts=1` | +| `missing_input_failure` | A required table does not exist | Spark `TABLE_OR_VIEW_NOT_FOUND` | +| `invalid_configuration_failure` | Invalid batch size and missing checkpoint path | `Invalid job configuration: batch_size must be greater than zero; checkpoint_path is required in incremental mode` | + +## Run the examples + +The default target uses development mode, so deployed job names are prefixed with your user name. + +```bash +databricks bundle validate --strict +databricks bundle deploy +``` + +Run each example separately: + +```bash +databricks bundle run schema_drift_failure +databricks bundle run data_quality_failure +databricks bundle run missing_input_failure +databricks bundle run invalid_configuration_failure +``` + +Each `bundle run` command returns a non-zero exit code after the job reaches its intentional failure. Open the run URL printed by the CLI to inspect the task output and stack trace. + +## Clean up + +After testing, remove the deployed development resources: + +```bash +databricks bundle destroy +``` diff --git a/knowledge_base/failing_job_examples/databricks.yml b/knowledge_base/failing_job_examples/databricks.yml new file mode 100644 index 0000000..0dc1840 --- /dev/null +++ b/knowledge_base/failing_job_examples/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: failing_job_examples + +include: + - resources/*.job.yml + +targets: + dev: + default: true + mode: development diff --git a/knowledge_base/failing_job_examples/resources/data_quality.job.yml b/knowledge_base/failing_job_examples/resources/data_quality.job.yml new file mode 100644 index 0000000..165232f --- /dev/null +++ b/knowledge_base/failing_job_examples/resources/data_quality.job.yml @@ -0,0 +1,14 @@ +resources: + jobs: + data_quality_failure: + name: failing_job_example_data_quality + description: Intentionally fails after detecting invalid transaction records. + tags: + example: intentional-failure + failure_type: data-quality + max_concurrent_runs: 1 + tasks: + - task_key: validate_transactions + max_retries: 0 + notebook_task: + notebook_path: ../src/data_quality.py diff --git a/knowledge_base/failing_job_examples/resources/invalid_configuration.job.yml b/knowledge_base/failing_job_examples/resources/invalid_configuration.job.yml new file mode 100644 index 0000000..9f90660 --- /dev/null +++ b/knowledge_base/failing_job_examples/resources/invalid_configuration.job.yml @@ -0,0 +1,14 @@ +resources: + jobs: + invalid_configuration_failure: + name: failing_job_example_invalid_configuration + description: Intentionally fails because required application settings are invalid. + tags: + example: intentional-failure + failure_type: invalid-configuration + max_concurrent_runs: 1 + tasks: + - task_key: validate_configuration + max_retries: 0 + notebook_task: + notebook_path: ../src/invalid_configuration.py diff --git a/knowledge_base/failing_job_examples/resources/missing_input.job.yml b/knowledge_base/failing_job_examples/resources/missing_input.job.yml new file mode 100644 index 0000000..7bf95ca --- /dev/null +++ b/knowledge_base/failing_job_examples/resources/missing_input.job.yml @@ -0,0 +1,14 @@ +resources: + jobs: + missing_input_failure: + name: failing_job_example_missing_input + description: Intentionally fails while reading an input table that does not exist. + tags: + example: intentional-failure + failure_type: missing-input + max_concurrent_runs: 1 + tasks: + - task_key: read_missing_input + max_retries: 0 + notebook_task: + notebook_path: ../src/missing_input.py diff --git a/knowledge_base/failing_job_examples/resources/schema_drift.job.yml b/knowledge_base/failing_job_examples/resources/schema_drift.job.yml new file mode 100644 index 0000000..3a07766 --- /dev/null +++ b/knowledge_base/failing_job_examples/resources/schema_drift.job.yml @@ -0,0 +1,14 @@ +resources: + jobs: + schema_drift_failure: + name: failing_job_example_schema_drift + description: Intentionally fails when an upstream column changes from numeric to string. + tags: + example: intentional-failure + failure_type: schema-drift + max_concurrent_runs: 1 + tasks: + - task_key: validate_transaction_schema + max_retries: 0 + notebook_task: + notebook_path: ../src/schema_drift.py diff --git a/knowledge_base/failing_job_examples/src/data_quality.py b/knowledge_base/failing_job_examples/src/data_quality.py new file mode 100644 index 0000000..587646e --- /dev/null +++ b/knowledge_base/failing_job_examples/src/data_quality.py @@ -0,0 +1,40 @@ +# Databricks notebook source +from pyspark.sql import SparkSession, functions as F +from pyspark.sql.types import DoubleType, StringType, StructField, StructType + + +spark = SparkSession.builder.getOrCreate() +schema = StructType( + [ + StructField("transaction_id", StringType(), False), + StructField("customer", StringType(), True), + StructField("amount", DoubleType(), False), + ] +) + +transactions = spark.createDataFrame( + [ + ("txn-001", "Acme", 150.0), + ("txn-001", "Acme", 150.0), + ("txn-002", None, 75.0), + ("txn-003", "Globex", -10.0), + ], + schema, +) + +duplicate_ids = ( + transactions.groupBy("transaction_id").count().where(F.col("count") > 1).count() +) +null_customers = transactions.where(F.col("customer").isNull()).count() +non_positive_amounts = transactions.where(F.col("amount") <= 0).count() + +failures = { + "duplicate_transaction_ids": duplicate_ids, + "null_customers": null_customers, + "non_positive_amounts": non_positive_amounts, +} +failed_checks = {name: count for name, count in failures.items() if count > 0} + +if failed_checks: + details = ", ".join(f"{name}={count}" for name, count in failed_checks.items()) + raise ValueError(f"Data quality checks failed: {details}") diff --git a/knowledge_base/failing_job_examples/src/invalid_configuration.py b/knowledge_base/failing_job_examples/src/invalid_configuration.py new file mode 100644 index 0000000..ed35bda --- /dev/null +++ b/knowledge_base/failing_job_examples/src/invalid_configuration.py @@ -0,0 +1,15 @@ +# Databricks notebook source +config = { + "batch_size": 0, + "checkpoint_path": "", + "mode": "incremental", +} + +errors = [] +if config["batch_size"] <= 0: + errors.append("batch_size must be greater than zero") +if config["mode"] == "incremental" and not config["checkpoint_path"]: + errors.append("checkpoint_path is required in incremental mode") + +if errors: + raise ValueError(f"Invalid job configuration: {'; '.join(errors)}") diff --git a/knowledge_base/failing_job_examples/src/missing_input.py b/knowledge_base/failing_job_examples/src/missing_input.py new file mode 100644 index 0000000..cd0f185 --- /dev/null +++ b/knowledge_base/failing_job_examples/src/missing_input.py @@ -0,0 +1,11 @@ +# Databricks notebook source +from pyspark.sql import SparkSession + + +spark = SparkSession.builder.getOrCreate() +missing_view = "__failing_job_examples_missing_orders_7f3f2a9c__" +spark.catalog.dropGlobalTempView(missing_view) +missing_table = f"global_temp.{missing_view}" + +print(f"Reading required input table: {missing_table}") +spark.read.table(missing_table).count() diff --git a/knowledge_base/failing_job_examples/src/schema_drift.py b/knowledge_base/failing_job_examples/src/schema_drift.py new file mode 100644 index 0000000..469da34 --- /dev/null +++ b/knowledge_base/failing_job_examples/src/schema_drift.py @@ -0,0 +1,17 @@ +# Databricks notebook source +from pyspark.sql.types import StringType, StructField, StructType + + +schema = StructType( + [ + StructField("transaction_id", StringType(), False), + StructField("amount", StringType(), False), + ] +) +amount_type = schema["amount"].dataType.simpleString() +numeric_types = {"byte", "short", "int", "bigint", "float", "double"} + +if amount_type not in numeric_types and not amount_type.startswith("decimal"): + raise TypeError( + f"Schema drift detected: expected amount to be numeric, found {amount_type}" + ) From 1330079d8c1b9fdfc85a412e74607601a00ed7fb Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Fri, 4 Sep 2026 14:58:40 +0200 Subject: [PATCH 2/3] docs: simplify failing jobs validation command --- knowledge_base/failing_job_examples/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/knowledge_base/failing_job_examples/README.md b/knowledge_base/failing_job_examples/README.md index e549161..31c2412 100644 --- a/knowledge_base/failing_job_examples/README.md +++ b/knowledge_base/failing_job_examples/README.md @@ -20,7 +20,7 @@ The target workspace must support serverless Jobs compute. The default target uses development mode, so deployed job names are prefixed with your user name. ```bash -databricks bundle validate --strict +databricks bundle validate databricks bundle deploy ``` From 8d1078f779beb36a6d84c57265a1af7782e173ba Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Fri, 4 Sep 2026 15:00:57 +0200 Subject: [PATCH 3/3] knowledge_base: remove data quality failure example --- knowledge_base/failing_job_examples/README.md | 2 - .../resources/data_quality.job.yml | 14 ------- .../failing_job_examples/src/data_quality.py | 40 ------------------- 3 files changed, 56 deletions(-) delete mode 100644 knowledge_base/failing_job_examples/resources/data_quality.job.yml delete mode 100644 knowledge_base/failing_job_examples/src/data_quality.py diff --git a/knowledge_base/failing_job_examples/README.md b/knowledge_base/failing_job_examples/README.md index 31c2412..0ddd78b 100644 --- a/knowledge_base/failing_job_examples/README.md +++ b/knowledge_base/failing_job_examples/README.md @@ -11,7 +11,6 @@ The target workspace must support serverless Jobs compute. | Resource key | Failure type | Expected error | | --- | --- | --- | | `schema_drift_failure` | An upstream `amount` column changed from numeric to string | `Schema drift detected: expected amount to be numeric, found string` | -| `data_quality_failure` | Duplicate IDs, a null customer, and a non-positive amount | `Data quality checks failed: duplicate_transaction_ids=1, null_customers=1, non_positive_amounts=1` | | `missing_input_failure` | A required table does not exist | Spark `TABLE_OR_VIEW_NOT_FOUND` | | `invalid_configuration_failure` | Invalid batch size and missing checkpoint path | `Invalid job configuration: batch_size must be greater than zero; checkpoint_path is required in incremental mode` | @@ -28,7 +27,6 @@ Run each example separately: ```bash databricks bundle run schema_drift_failure -databricks bundle run data_quality_failure databricks bundle run missing_input_failure databricks bundle run invalid_configuration_failure ``` diff --git a/knowledge_base/failing_job_examples/resources/data_quality.job.yml b/knowledge_base/failing_job_examples/resources/data_quality.job.yml deleted file mode 100644 index 165232f..0000000 --- a/knowledge_base/failing_job_examples/resources/data_quality.job.yml +++ /dev/null @@ -1,14 +0,0 @@ -resources: - jobs: - data_quality_failure: - name: failing_job_example_data_quality - description: Intentionally fails after detecting invalid transaction records. - tags: - example: intentional-failure - failure_type: data-quality - max_concurrent_runs: 1 - tasks: - - task_key: validate_transactions - max_retries: 0 - notebook_task: - notebook_path: ../src/data_quality.py diff --git a/knowledge_base/failing_job_examples/src/data_quality.py b/knowledge_base/failing_job_examples/src/data_quality.py deleted file mode 100644 index 587646e..0000000 --- a/knowledge_base/failing_job_examples/src/data_quality.py +++ /dev/null @@ -1,40 +0,0 @@ -# Databricks notebook source -from pyspark.sql import SparkSession, functions as F -from pyspark.sql.types import DoubleType, StringType, StructField, StructType - - -spark = SparkSession.builder.getOrCreate() -schema = StructType( - [ - StructField("transaction_id", StringType(), False), - StructField("customer", StringType(), True), - StructField("amount", DoubleType(), False), - ] -) - -transactions = spark.createDataFrame( - [ - ("txn-001", "Acme", 150.0), - ("txn-001", "Acme", 150.0), - ("txn-002", None, 75.0), - ("txn-003", "Globex", -10.0), - ], - schema, -) - -duplicate_ids = ( - transactions.groupBy("transaction_id").count().where(F.col("count") > 1).count() -) -null_customers = transactions.where(F.col("customer").isNull()).count() -non_positive_amounts = transactions.where(F.col("amount") <= 0).count() - -failures = { - "duplicate_transaction_ids": duplicate_ids, - "null_customers": null_customers, - "non_positive_amounts": non_positive_amounts, -} -failed_checks = {name: count for name, count in failures.items() if count > 0} - -if failed_checks: - details = ", ".join(f"{name}={count}" for name, count in failed_checks.items()) - raise ValueError(f"Data quality checks failed: {details}")