diff --git a/knowledge_base/failing_job_examples/.gitignore b/knowledge_base/failing_job_examples/.gitignore new file mode 100644 index 0000000..ee6e40f --- /dev/null +++ b/knowledge_base/failing_job_examples/.gitignore @@ -0,0 +1,2 @@ +.databricks/ +__pycache__/ diff --git a/knowledge_base/failing_job_examples/README.md b/knowledge_base/failing_job_examples/README.md new file mode 100644 index 0000000..0ddd78b --- /dev/null +++ b/knowledge_base/failing_job_examples/README.md @@ -0,0 +1,42 @@ +# Failing job examples + +This bundle provides small Lakeflow Jobs that fail for known, deterministic reasons. Use them to test alerting, monitoring, troubleshooting, and automated remediation workflows without changing production jobs. + +Every job is expected to finish in the `FAILED` state. The bundle has no schedules, configured retries, or persistent data writes. + +The target workspace must support serverless Jobs compute. + +## Included failures + +| Resource key | Failure type | Expected error | +| --- | --- | --- | +| `schema_drift_failure` | An upstream `amount` column changed from numeric to string | `Schema drift detected: expected amount to be numeric, found string` | +| `missing_input_failure` | A required table does not exist | Spark `TABLE_OR_VIEW_NOT_FOUND` | +| `invalid_configuration_failure` | Invalid batch size and missing checkpoint path | `Invalid job configuration: batch_size must be greater than zero; checkpoint_path is required in incremental mode` | + +## Run the examples + +The default target uses development mode, so deployed job names are prefixed with your user name. + +```bash +databricks bundle validate +databricks bundle deploy +``` + +Run each example separately: + +```bash +databricks bundle run schema_drift_failure +databricks bundle run missing_input_failure +databricks bundle run invalid_configuration_failure +``` + +Each `bundle run` command returns a non-zero exit code after the job reaches its intentional failure. Open the run URL printed by the CLI to inspect the task output and stack trace. + +## Clean up + +After testing, remove the deployed development resources: + +```bash +databricks bundle destroy +``` diff --git a/knowledge_base/failing_job_examples/databricks.yml b/knowledge_base/failing_job_examples/databricks.yml new file mode 100644 index 0000000..0dc1840 --- /dev/null +++ b/knowledge_base/failing_job_examples/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: failing_job_examples + +include: + - resources/*.job.yml + +targets: + dev: + default: true + mode: development diff --git a/knowledge_base/failing_job_examples/resources/invalid_configuration.job.yml b/knowledge_base/failing_job_examples/resources/invalid_configuration.job.yml new file mode 100644 index 0000000..9f90660 --- /dev/null +++ b/knowledge_base/failing_job_examples/resources/invalid_configuration.job.yml @@ -0,0 +1,14 @@ +resources: + jobs: + invalid_configuration_failure: + name: failing_job_example_invalid_configuration + description: Intentionally fails because required application settings are invalid. + tags: + example: intentional-failure + failure_type: invalid-configuration + max_concurrent_runs: 1 + tasks: + - task_key: validate_configuration + max_retries: 0 + notebook_task: + notebook_path: ../src/invalid_configuration.py diff --git a/knowledge_base/failing_job_examples/resources/missing_input.job.yml b/knowledge_base/failing_job_examples/resources/missing_input.job.yml new file mode 100644 index 0000000..7bf95ca --- /dev/null +++ b/knowledge_base/failing_job_examples/resources/missing_input.job.yml @@ -0,0 +1,14 @@ +resources: + jobs: + missing_input_failure: + name: failing_job_example_missing_input + description: Intentionally fails while reading an input table that does not exist. + tags: + example: intentional-failure + failure_type: missing-input + max_concurrent_runs: 1 + tasks: + - task_key: read_missing_input + max_retries: 0 + notebook_task: + notebook_path: ../src/missing_input.py diff --git a/knowledge_base/failing_job_examples/resources/schema_drift.job.yml b/knowledge_base/failing_job_examples/resources/schema_drift.job.yml new file mode 100644 index 0000000..3a07766 --- /dev/null +++ b/knowledge_base/failing_job_examples/resources/schema_drift.job.yml @@ -0,0 +1,14 @@ +resources: + jobs: + schema_drift_failure: + name: failing_job_example_schema_drift + description: Intentionally fails when an upstream column changes from numeric to string. + tags: + example: intentional-failure + failure_type: schema-drift + max_concurrent_runs: 1 + tasks: + - task_key: validate_transaction_schema + max_retries: 0 + notebook_task: + notebook_path: ../src/schema_drift.py diff --git a/knowledge_base/failing_job_examples/src/invalid_configuration.py b/knowledge_base/failing_job_examples/src/invalid_configuration.py new file mode 100644 index 0000000..ed35bda --- /dev/null +++ b/knowledge_base/failing_job_examples/src/invalid_configuration.py @@ -0,0 +1,15 @@ +# Databricks notebook source +config = { + "batch_size": 0, + "checkpoint_path": "", + "mode": "incremental", +} + +errors = [] +if config["batch_size"] <= 0: + errors.append("batch_size must be greater than zero") +if config["mode"] == "incremental" and not config["checkpoint_path"]: + errors.append("checkpoint_path is required in incremental mode") + +if errors: + raise ValueError(f"Invalid job configuration: {'; '.join(errors)}") diff --git a/knowledge_base/failing_job_examples/src/missing_input.py b/knowledge_base/failing_job_examples/src/missing_input.py new file mode 100644 index 0000000..cd0f185 --- /dev/null +++ b/knowledge_base/failing_job_examples/src/missing_input.py @@ -0,0 +1,11 @@ +# Databricks notebook source +from pyspark.sql import SparkSession + + +spark = SparkSession.builder.getOrCreate() +missing_view = "__failing_job_examples_missing_orders_7f3f2a9c__" +spark.catalog.dropGlobalTempView(missing_view) +missing_table = f"global_temp.{missing_view}" + +print(f"Reading required input table: {missing_table}") +spark.read.table(missing_table).count() diff --git a/knowledge_base/failing_job_examples/src/schema_drift.py b/knowledge_base/failing_job_examples/src/schema_drift.py new file mode 100644 index 0000000..469da34 --- /dev/null +++ b/knowledge_base/failing_job_examples/src/schema_drift.py @@ -0,0 +1,17 @@ +# Databricks notebook source +from pyspark.sql.types import StringType, StructField, StructType + + +schema = StructType( + [ + StructField("transaction_id", StringType(), False), + StructField("amount", StringType(), False), + ] +) +amount_type = schema["amount"].dataType.simpleString() +numeric_types = {"byte", "short", "int", "bigint", "float", "double"} + +if amount_type not in numeric_types and not amount_type.startswith("decimal"): + raise TypeError( + f"Schema drift detected: expected amount to be numeric, found {amount_type}" + )