Skip to content

fix(estimator): do not enable debugger by default #6188

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zhaoqizqwang wants to merge 2 commits into
base: master-v2
Choose a base branch
from
Open

fix(estimator): do not enable debugger by default #6188

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dacc4da
fix(estimator): Do not attach DebuggerHookConfig or active ProfilerCo…
zhaoqizqwang Aug 17, 2026
19b138e
fix unit tests
zhaoqizqwang Aug 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 10 additions & 8 deletions src/sagemaker/estimator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -790,10 +790,9 @@ def __init__(
sagemaker_session=sagemaker_session,
)
# If customer passes True from either direct_input or sagemaker_config, we will
# create a default hook config as an empty dict which will later be populated
# with default s3_output_path from _prepare_debugger_for_training function
# create a default hook config populated with the default s3_output_path.
if self.debugger_hook_config is True:
self.debugger_hook_config = {}
self.debugger_hook_config = DebuggerHookConfig()

self.tensorboard_output_config = tensorboard_output_config

Expand Down Expand Up @@ -1197,8 +1196,11 @@ def _validate_and_set_debugger_configs(self):
)

if region_supports_debugger:
if self.debugger_hook_config in [None, {}]:
self.debugger_hook_config = DebuggerHookConfig(s3_output_path=self.output_path)
# Debugger is opt-in: do NOT auto-attach a DebuggerHookConfig when the customer
# did not request one (unset -> None). The hook is still created upstream in
# _prepare_debugger_for_training when debugger_rules are set, or when the customer
# explicitly passes debugger_hook_config=True / a DebuggerHookConfig instance.
pass
else:
if self.debugger_hook_config is not False and self.debugger_hook_config:
# when user set debugger config in a unsupported region
Expand Down Expand Up @@ -1245,11 +1247,11 @@ def _prepare_profiler_for_training(self):
if self.profiler_rules:
raise RuntimeError("ProfilerRule cannot be set when disable_profiler is True.")
elif _region_supports_profiler(self.sagemaker_session.boto_region_name):
if self.profiler_config is None:
self.profiler_config = ProfilerConfig(s3_output_path=self.output_path)
# Profiler is opt-in: do NOT auto-attach an active ProfilerConfig by default.
# If profiler_config is left unset it is emitted as disabled below.
if self.rules is None or (self.rules and not self.profiler_rules):
self.profiler_rules = []
if self.profiler_config.profile_params:
if self.profiler_config and self.profiler_config.profile_params:
self.profiler_rules.append(
get_default_profiler_processing_job(
instance_type=self.profiler_config.profile_params.instanceType,
Expand Down
7 changes: 1 addition & 6 deletions tests/unit/sagemaker/huggingface/test_estimator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,13 +148,8 @@ def _create_train_job(version, base_framework_version):
"retry_strategy": None,
"experiment_config": None,
"enable_network_isolation": False,
"debugger_hook_config": {
"CollectionConfigurations": [],
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
},
"profiler_config": {
"DisableProfiler": False,
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
"DisableProfiler": True,
},
}

Expand Down
9 changes: 1 addition & 8 deletions tests/unit/sagemaker/tensorflow/test_estimator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,17 +146,10 @@ def _create_train_job(tf_version, horovod=False, ps=False, py_version="py2", smd
"enable_network_isolation": False,
"experiment_config": None,
"profiler_config": {
"DisableProfiler": False,
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
"DisableProfiler": True,
},
}

if not ps:
conf["debugger_hook_config"] = {
"CollectionConfigurations": [],
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
}

return conf


Expand Down
7 changes: 1 addition & 6 deletions tests/unit/sagemaker/training_compiler/test_huggingface_pytorch_compiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,13 +150,8 @@ def _create_train_job(
"environment": None,
"retry_strategy": None,
"experiment_config": EXPERIMENT_CONFIG,
"debugger_hook_config": {
"CollectionConfigurations": [],
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
},
"profiler_config": {
"DisableProfiler": False,
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
"DisableProfiler": True,
},
}

Expand Down
7 changes: 1 addition & 6 deletions tests/unit/sagemaker/training_compiler/test_huggingface_tensorflow_compiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,13 +147,8 @@ def _create_train_job(
"retry_strategy": None,
"experiment_config": EXPERIMENT_CONFIG,
"enable_network_isolation": False,
"debugger_hook_config": {
"CollectionConfigurations": [],
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
},
"profiler_config": {
"DisableProfiler": False,
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
"DisableProfiler": True,
},
}

Expand Down
7 changes: 1 addition & 6 deletions tests/unit/sagemaker/training_compiler/test_pytorch_compiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,13 +146,8 @@ def _create_train_job(
"retry_strategy": None,
"experiment_config": EXPERIMENT_CONFIG,
"enable_network_isolation": False,
"debugger_hook_config": {
"CollectionConfigurations": [],
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
},
"profiler_config": {
"DisableProfiler": False,
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
"DisableProfiler": True,
},
}

Expand Down
7 changes: 1 addition & 6 deletions tests/unit/sagemaker/training_compiler/test_tensorflow_compiler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,13 +152,8 @@ def _create_train_job(framework_version, instance_type, training_compiler_config
"retry_strategy": None,
"experiment_config": EXPERIMENT_CONFIG,
"enable_network_isolation": False,
"debugger_hook_config": {
"CollectionConfigurations": [],
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
},
"profiler_config": {
"DisableProfiler": False,
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
"DisableProfiler": True,
},
}

Expand Down
16 changes: 0 additions & 16 deletions tests/unit/sagemaker/workflow/test_step_collections.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -767,10 +767,6 @@ def test_register_model_with_model_repack_with_estimator(
"TrainingImage": MODEL_REPACKING_IMAGE_URI,
"TrainingInputMode": "File",
},
"DebugHookConfig": {
"CollectionConfigurations": [],
"S3OutputPath": f"s3://{BUCKET}/",
},
"ProfilerConfig": {"DisableProfiler": True},
"HyperParameters": {
"inference_script": '"inference.py"',
Expand Down Expand Up @@ -896,10 +892,6 @@ def test_register_model_with_model_repack_with_model(model, model_metrics, drift
"TrainingImage": MODEL_REPACKING_IMAGE_URI,
"TrainingInputMode": "File",
},
"DebugHookConfig": {
"CollectionConfigurations": [],
"S3OutputPath": f"s3://{BUCKET}/",
},
"ProfilerConfig": {"DisableProfiler": True},
"HyperParameters": {
"inference_script": '"inference.py"',
Expand Down Expand Up @@ -1027,10 +1019,6 @@ def test_register_model_with_model_repack_with_pipeline_model(
"TrainingImage": MODEL_REPACKING_IMAGE_URI,
"TrainingInputMode": "File",
},
"DebugHookConfig": {
"CollectionConfigurations": [],
"S3OutputPath": f"s3://{BUCKET}/",
},
"ProfilerConfig": {"DisableProfiler": True},
"HyperParameters": {
"dependencies": "null",
Expand Down Expand Up @@ -1254,10 +1242,6 @@ def test_estimator_transformer_with_model_repack_with_estimator(estimator, sourc
"sagemaker_region": '"us-west-2"',
},
"VpcConfig": {"Subnets": ["abc", "def"], "SecurityGroupIds": ["123", "456"]},
"DebugHookConfig": {
"S3OutputPath": "s3://my-bucket/",
"CollectionConfigurations": [],
},
}
elif request_dict["Type"] == "Model":
assert request_dict["Name"] == "EstimatorTransformerStepCreateModelStep"
Expand Down
6 changes: 1 addition & 5 deletions tests/unit/sagemaker/workflow/test_steps.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -337,10 +337,6 @@ def test_training_step_base_estimator(sagemaker_session):
},
"RoleArn": ROLE,
"StoppingCondition": {"MaxRuntimeInSeconds": 86400},
"DebugHookConfig": {
"S3OutputPath": {"Std:Join": {"On": "/", "Values": ["s3:/", "a", "b"]}},
"CollectionConfigurations": [],
},
"ProfilerConfig": {
"DisableProfiler": False,
"ProfilingIntervalInMilliseconds": 500,
Expand Down Expand Up @@ -453,7 +449,7 @@ def test_training_step_tensorflow(sagemaker_session):
"sagemaker_instance_type": {"Get": "Parameters.InstanceType"},
"sagemaker_distributed_dataparallel_custom_mpi_options": '""',
},
"ProfilerConfig": {"DisableProfiler": False, "S3OutputPath": "s3://my-bucket/"},
"ProfilerConfig": {"DisableProfiler": True},
},
"CacheConfig": {"Enabled": True, "ExpireAfter": "PT1H"},
}
Expand Down
2 changes: 0 additions & 2 deletions tests/unit/sagemaker/workflow/test_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,6 @@ def test_repack_model_step(estimator):
"DependsOn": ["TestStep"],
"Arguments": {
"AlgorithmSpecification": {"TrainingInputMode": "File"},
"DebugHookConfig": {"CollectionConfigurations": [], "S3OutputPath": "s3://my-bucket/"},
"InputDataConfig": [
{
"ChannelName": "training",
Expand Down Expand Up @@ -176,7 +175,6 @@ def test_repack_model_step_with_source_dir(estimator, source_dir):
"Type": "Training",
"Arguments": {
"AlgorithmSpecification": {"TrainingInputMode": "File"},
"DebugHookConfig": {"CollectionConfigurations": [], "S3OutputPath": "s3://my-bucket/"},
"InputDataConfig": [
{
"ChannelName": "training",
Expand Down
7 changes: 1 addition & 6 deletions tests/unit/test_chainer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,13 +154,8 @@ def _create_train_job(version, py_version):
"enable_network_isolation": False,
"environment": None,
"experiment_config": None,
"debugger_hook_config": {
"CollectionConfigurations": [],
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
},
"profiler_config": {
"DisableProfiler": False,
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
"DisableProfiler": True,
},
}

Expand Down
35 changes: 15 additions & 20 deletions tests/unit/test_estimator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -693,7 +693,7 @@ def test_estimator_with_debugger_hook_config_provided_as_bool_from_direct_input(
base_job_name="base_job_name",
debugger_hook_config=True,
)
assert estimator.debugger_hook_config == {}
assert isinstance(estimator.debugger_hook_config, DebuggerHookConfig)


def test_estimator_with_debugger_hook_config_provided_as_dict_from_direct_input(
Expand Down Expand Up @@ -1067,8 +1067,7 @@ def test_framework_with_debugger_and_built_in_rule(sagemaker_session):
],
}
assert args["profiler_config"] == {
"DisableProfiler": False,
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
"DisableProfiler": True,
}


Expand Down Expand Up @@ -1226,14 +1225,10 @@ def test_framework_without_debugger_and_profiler(time, sagemaker_session):
f.fit("s3://mydata")
sagemaker_session.train.assert_called_once()
_, args = sagemaker_session.train.call_args
assert args["debugger_hook_config"] == {
"CollectionConfigurations": [],
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
}
assert "debugger_hook_config" not in args or args["debugger_hook_config"] is None
assert "debugger_rule_configs" not in args
assert args["profiler_config"] == {
"DisableProfiler": False,
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
"DisableProfiler": True,
}


Expand Down Expand Up @@ -1296,8 +1291,7 @@ def test_framework_with_debugger_and_profiler_rules(sagemaker_session):
],
}
assert args["profiler_config"] == {
"DisableProfiler": False,
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
"DisableProfiler": True,
}
assert args["profiler_rule_configs"] == [
{
Expand Down Expand Up @@ -1332,8 +1326,7 @@ def test_framework_with_only_profiler_rule_specified(sagemaker_session):
sagemaker_session.train.assert_called_once()
_, args = sagemaker_session.train.call_args
assert args["profiler_config"] == {
"DisableProfiler": False,
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
"DisableProfiler": True,
}
assert args["profiler_rule_configs"] == [
{
Expand Down Expand Up @@ -2601,7 +2594,7 @@ def test_fit_verify_job_name(strftime, sagemaker_session):
@pytest.mark.parametrize(
"debugger_hook_config_direct_input, sagemaker_config, expected_debugger_hook_config_output",
[
(None, None, S3_OUTPUT_PATH_FROM_SESSION_S3_DEFAULT_CONFIG),
(None, None, None),
(True, None, S3_OUTPUT_PATH_FROM_SESSION_S3_DEFAULT_CONFIG),
(False, None, False),
(HOOK_CONFIG, None, HOOK_CONFIG.s3_output_path),
Expand Down Expand Up @@ -2665,6 +2658,8 @@ def test_prepare_for_training_for_debugger_hook_config_value_combinations(

if expected_debugger_hook_config_output is False:
assert fw.debugger_hook_config == expected_debugger_hook_config_output
elif expected_debugger_hook_config_output is None:
assert fw.debugger_hook_config is None
else:
assert fw.debugger_hook_config.s3_output_path == expected_debugger_hook_config_output

Expand Down Expand Up @@ -3522,7 +3517,7 @@ def test_unsupported_type_in_dict():
"input_config": None,
"input_mode": "File",
"output_config": {"S3OutputPath": OUTPUT_PATH},
"profiler_config": {"DisableProfiler": False, "S3OutputPath": OUTPUT_PATH},
"profiler_config": {"DisableProfiler": True},
"resource_config": {
"InstanceCount": INSTANCE_COUNT,
"InstanceType": INSTANCE_TYPE,
Expand Down Expand Up @@ -3802,7 +3797,7 @@ def test_generic_to_fit_no_input(time, sagemaker_session):

args.pop("job_name")
args.pop("role")
args.pop("debugger_hook_config")
args.pop("debugger_hook_config", None)

assert args == NO_INPUT_TRAIN_CALL

Expand All @@ -3827,7 +3822,7 @@ def test_generic_to_fit_no_hps(time, sagemaker_session):

args.pop("job_name")
args.pop("role")
args.pop("debugger_hook_config")
args.pop("debugger_hook_config", None)

assert args == BASE_TRAIN_CALL

Expand All @@ -3854,7 +3849,7 @@ def test_generic_to_fit_with_hps(time, sagemaker_session):

args.pop("job_name")
args.pop("role")
args.pop("debugger_hook_config")
args.pop("debugger_hook_config", None)

assert args == HP_TRAIN_CALL

Expand Down Expand Up @@ -3887,7 +3882,7 @@ def test_generic_to_fit_with_experiment_config(time, sagemaker_session):

args.pop("job_name")
args.pop("role")
args.pop("debugger_hook_config")
args.pop("debugger_hook_config", None)

assert args == EXP_TRAIN_CALL

Expand Down Expand Up @@ -4041,7 +4036,7 @@ def test_generic_to_deploy(time, sagemaker_session):

args.pop("job_name")
args.pop("role")
args.pop("debugger_hook_config")
args.pop("debugger_hook_config", None)

assert args == HP_TRAIN_CALL

Expand Down
7 changes: 1 addition & 6 deletions tests/unit/test_mxnet.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,13 +170,8 @@ def _get_train_args(job_name):
"retry_strategy": None,
"experiment_config": None,
"enable_network_isolation": False,
"debugger_hook_config": {
"CollectionConfigurations": [],
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
},
"profiler_config": {
"DisableProfiler": False,
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
"DisableProfiler": True,
},
}

Expand Down
7 changes: 1 addition & 6 deletions tests/unit/test_pytorch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,13 +187,8 @@ def _create_train_job(version, py_version):
"retry_strategy": None,
"experiment_config": None,
"enable_network_isolation": False,
"debugger_hook_config": {
"CollectionConfigurations": [],
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
},
"profiler_config": {
"DisableProfiler": False,
"S3OutputPath": "s3://{}/".format(BUCKET_NAME),
"DisableProfiler": True,
},
}

Expand Down
Loading
Loading