fix(integ): update debugger/profiler for opt-in defaults

src/sagemaker/estimator.py

@@ -790,10 +790,9 @@ def __init__(
             sagemaker_session=sagemaker_session,
         )
         # If customer passes True from either direct_input or sagemaker_config, we will
-        # create a default hook config as an empty dict which will later be populated
-        # with default s3_output_path from _prepare_debugger_for_training function
+        # create a default hook config populated with the default s3_output_path.
         if self.debugger_hook_config is True:
-            self.debugger_hook_config = {}
+            self.debugger_hook_config = DebuggerHookConfig()
 
         self.tensorboard_output_config = tensorboard_output_config
 
@@ -1197,8 +1196,11 @@ def _validate_and_set_debugger_configs(self):
         )
 
         if region_supports_debugger:
-            if self.debugger_hook_config in [None, {}]:
-                self.debugger_hook_config = DebuggerHookConfig(s3_output_path=self.output_path)
+            # Debugger is opt-in: do NOT auto-attach a DebuggerHookConfig when the customer
+            # did not request one (unset -> None). The hook is still created upstream in
+            # _prepare_debugger_for_training when debugger_rules are set, or when the customer
+            # explicitly passes debugger_hook_config=True / a DebuggerHookConfig instance.
+            pass
         else:
             if self.debugger_hook_config is not False and self.debugger_hook_config:
                 # when user set debugger config in a unsupported region
@@ -1245,11 +1247,15 @@ def _prepare_profiler_for_training(self):
             if self.profiler_rules:
                 raise RuntimeError("ProfilerRule cannot be set when disable_profiler is True.")
         elif _region_supports_profiler(self.sagemaker_session.boto_region_name):
-            if self.profiler_config is None:
+            # Profiler is opt-in: do NOT auto-attach an active ProfilerConfig by default.
+            # If profiler_config is left unset it is emitted as disabled below.
+            # Passing a ProfilerRule is itself an opt-in, so give it a config to attach to;
+            # otherwise the request would carry a profiler rule with profiling disabled.
+            if self.profiler_rules and self.profiler_config is None:
                 self.profiler_config = ProfilerConfig(s3_output_path=self.output_path)
             if self.rules is None or (self.rules and not self.profiler_rules):
                 self.profiler_rules = []
-                if self.profiler_config.profile_params:
+                if self.profiler_config and self.profiler_config.profile_params:
                     self.profiler_rules.append(
                         get_default_profiler_processing_job(
                             instance_type=self.profiler_config.profile_params.instanceType,

tests/integ/sagemaker/workflow/test_training_steps.py

@@ -24,6 +24,7 @@
 from sagemaker import TrainingInput, get_execution_role, utils, image_uris
 from sagemaker.debugger import (
     DebuggerHookConfig,
+    ProfilerConfig,
     Rule,
     rule_configs,
 )
@@ -100,6 +101,8 @@ def test_training_job_with_debugger_and_profiler(
         sagemaker_session=sagemaker_session,
         rules=rules,
         debugger_hook_config=debugger_hook_config,
+        # Profiling is opt-in, so request it explicitly.
+        profiler_config=ProfilerConfig(system_monitor_interval_millis=500),
         # TODO: remove base_job_name once we merge
         # https://github.com/aws/sagemaker-python-sdk/pull/3158/files
         base_job_name="TestJob",

tests/integ/test_debugger.py

@@ -435,6 +435,7 @@ def test_debug_hook_disabled_with_checkpointing(
             distribution={"smdistributed": {"dataparallel": {"enabled": True}}},
             checkpoint_local_path="/opt/ml/checkpoints",
             checkpoint_s3_uri=os.path.join(s3_output_path, "checkpoints"),
+            debugger_hook_config=True,
         )
         pt._prepare_for_training()
         # Debug Hook should be disabled
@@ -455,6 +456,7 @@ def test_debug_hook_disabled_with_checkpointing(
             distribution={"smdistributed": {"modelparallel": {"enabled": True}}},
             checkpoint_local_path="/opt/ml/checkpoints",
             checkpoint_s3_uri=os.path.join(s3_output_path, "checkpoints"),
+            debugger_hook_config=True,
         )
         tf._prepare_for_training()
         # Debug Hook should be disabled
@@ -472,6 +474,7 @@ def test_debug_hook_disabled_with_checkpointing(
             instance_type="ml.p3.16xlarge",
             sagemaker_session=sagemaker_session,
             # Training using SMDataParallel Distributed Training Framework
+            debugger_hook_config=True,
         )
         xg._prepare_for_training()
         # Debug Hook should be enabled

tests/integ/test_profiler.py

@@ -74,6 +74,8 @@ def test_mxnet_with_default_profiler_config_and_profiler_rule(
             instance_count=1,
             instance_type=cpu_instance_type,
             sagemaker_session=sagemaker_session,
+            # Profiling is opt-in, so request it explicitly.
+            profiler_config=ProfilerConfig(system_monitor_interval_millis=500),
         )
 
         train_input = mx.sagemaker_session.upload_data(
@@ -422,6 +424,9 @@ def test_mxnet_profiling_with_disable_debugger_hook(
             instance_type=cpu_instance_type,
             sagemaker_session=sagemaker_session,
             debugger_hook_config=False,
+            # Profiling is opt-in, so request it explicitly. The point of this test is that
+            # disabling the debugger hook must not disable profiling.
+            profiler_config=ProfilerConfig(system_monitor_interval_millis=500),
         )
 
         train_input = mx.sagemaker_session.upload_data(

tests/unit/sagemaker/huggingface/test_estimator.py

@@ -148,13 +148,8 @@ def _create_train_job(version, base_framework_version):
         "retry_strategy": None,
         "experiment_config": None,
         "enable_network_isolation": False,
-        "debugger_hook_config": {
-            "CollectionConfigurations": [],
-            "S3OutputPath": "s3://{}/".format(BUCKET_NAME),
-        },
         "profiler_config": {
-            "DisableProfiler": False,
-            "S3OutputPath": "s3://{}/".format(BUCKET_NAME),
+            "DisableProfiler": True,
         },
     }
 

tests/unit/sagemaker/tensorflow/test_estimator.py

@@ -146,17 +146,10 @@ def _create_train_job(tf_version, horovod=False, ps=False, py_version="py2", smd
         "enable_network_isolation": False,
         "experiment_config": None,
         "profiler_config": {
-            "DisableProfiler": False,
-            "S3OutputPath": "s3://{}/".format(BUCKET_NAME),
+            "DisableProfiler": True,
         },
     }
 
-    if not ps:
-        conf["debugger_hook_config"] = {
-            "CollectionConfigurations": [],
-            "S3OutputPath": "s3://{}/".format(BUCKET_NAME),
-        }
-
     return conf
 
 

tests/unit/sagemaker/training_compiler/test_huggingface_pytorch_compiler.py

@@ -150,13 +150,8 @@ def _create_train_job(
         "environment": None,
         "retry_strategy": None,
         "experiment_config": EXPERIMENT_CONFIG,
-        "debugger_hook_config": {
-            "CollectionConfigurations": [],
-            "S3OutputPath": "s3://{}/".format(BUCKET_NAME),
-        },
         "profiler_config": {
-            "DisableProfiler": False,
-            "S3OutputPath": "s3://{}/".format(BUCKET_NAME),
+            "DisableProfiler": True,
         },
     }
 

tests/unit/sagemaker/training_compiler/test_huggingface_tensorflow_compiler.py

@@ -147,13 +147,8 @@ def _create_train_job(
         "retry_strategy": None,
         "experiment_config": EXPERIMENT_CONFIG,
         "enable_network_isolation": False,
-        "debugger_hook_config": {
-            "CollectionConfigurations": [],
-            "S3OutputPath": "s3://{}/".format(BUCKET_NAME),
-        },
         "profiler_config": {
-            "DisableProfiler": False,
-            "S3OutputPath": "s3://{}/".format(BUCKET_NAME),
+            "DisableProfiler": True,
         },
     }
 

tests/unit/sagemaker/training_compiler/test_pytorch_compiler.py

@@ -146,13 +146,8 @@ def _create_train_job(
         "retry_strategy": None,
         "experiment_config": EXPERIMENT_CONFIG,
         "enable_network_isolation": False,
-        "debugger_hook_config": {
-            "CollectionConfigurations": [],
-            "S3OutputPath": "s3://{}/".format(BUCKET_NAME),
-        },
         "profiler_config": {
-            "DisableProfiler": False,
-            "S3OutputPath": "s3://{}/".format(BUCKET_NAME),
+            "DisableProfiler": True,
         },
     }
 

tests/unit/sagemaker/training_compiler/test_tensorflow_compiler.py

@@ -152,13 +152,8 @@ def _create_train_job(framework_version, instance_type, training_compiler_config
         "retry_strategy": None,
         "experiment_config": EXPERIMENT_CONFIG,
         "enable_network_isolation": False,
-        "debugger_hook_config": {
-            "CollectionConfigurations": [],
-            "S3OutputPath": "s3://{}/".format(BUCKET_NAME),
-        },
         "profiler_config": {
-            "DisableProfiler": False,
-            "S3OutputPath": "s3://{}/".format(BUCKET_NAME),
+            "DisableProfiler": True,
         },
     }
 

tests/unit/sagemaker/workflow/test_step_collections.py

@@ -767,10 +767,6 @@ def test_register_model_with_model_repack_with_estimator(
                         "TrainingImage": MODEL_REPACKING_IMAGE_URI,
                         "TrainingInputMode": "File",
                     },
-                    "DebugHookConfig": {
-                        "CollectionConfigurations": [],
-                        "S3OutputPath": f"s3://{BUCKET}/",
-                    },
                     "ProfilerConfig": {"DisableProfiler": True},
                     "HyperParameters": {
                         "inference_script": '"inference.py"',
@@ -896,10 +892,6 @@ def test_register_model_with_model_repack_with_model(model, model_metrics, drift
                         "TrainingImage": MODEL_REPACKING_IMAGE_URI,
                         "TrainingInputMode": "File",
                     },
-                    "DebugHookConfig": {
-                        "CollectionConfigurations": [],
-                        "S3OutputPath": f"s3://{BUCKET}/",
-                    },
                     "ProfilerConfig": {"DisableProfiler": True},
                     "HyperParameters": {
                         "inference_script": '"inference.py"',
@@ -1027,10 +1019,6 @@ def test_register_model_with_model_repack_with_pipeline_model(
                         "TrainingImage": MODEL_REPACKING_IMAGE_URI,
                         "TrainingInputMode": "File",
                     },
-                    "DebugHookConfig": {
-                        "CollectionConfigurations": [],
-                        "S3OutputPath": f"s3://{BUCKET}/",
-                    },
                     "ProfilerConfig": {"DisableProfiler": True},
                     "HyperParameters": {
                         "dependencies": "null",
@@ -1254,10 +1242,6 @@ def test_estimator_transformer_with_model_repack_with_estimator(estimator, sourc
                     "sagemaker_region": '"us-west-2"',
                 },
                 "VpcConfig": {"Subnets": ["abc", "def"], "SecurityGroupIds": ["123", "456"]},
-                "DebugHookConfig": {
-                    "S3OutputPath": "s3://my-bucket/",
-                    "CollectionConfigurations": [],
-                },
             }
         elif request_dict["Type"] == "Model":
             assert request_dict["Name"] == "EstimatorTransformerStepCreateModelStep"

tests/unit/sagemaker/workflow/test_steps.py

@@ -337,10 +337,6 @@ def test_training_step_base_estimator(sagemaker_session):
             },
             "RoleArn": ROLE,
             "StoppingCondition": {"MaxRuntimeInSeconds": 86400},
-            "DebugHookConfig": {
-                "S3OutputPath": {"Std:Join": {"On": "/", "Values": ["s3:/", "a", "b"]}},
-                "CollectionConfigurations": [],
-            },
             "ProfilerConfig": {
                 "DisableProfiler": False,
                 "ProfilingIntervalInMilliseconds": 500,
@@ -453,7 +449,7 @@ def test_training_step_tensorflow(sagemaker_session):
                 "sagemaker_instance_type": {"Get": "Parameters.InstanceType"},
                 "sagemaker_distributed_dataparallel_custom_mpi_options": '""',
             },
-            "ProfilerConfig": {"DisableProfiler": False, "S3OutputPath": "s3://my-bucket/"},
+            "ProfilerConfig": {"DisableProfiler": True},
         },
         "CacheConfig": {"Enabled": True, "ExpireAfter": "PT1H"},
     }

tests/unit/sagemaker/workflow/test_utils.py

@@ -94,7 +94,6 @@ def test_repack_model_step(estimator):
         "DependsOn": ["TestStep"],
         "Arguments": {
             "AlgorithmSpecification": {"TrainingInputMode": "File"},
-            "DebugHookConfig": {"CollectionConfigurations": [], "S3OutputPath": "s3://my-bucket/"},
             "InputDataConfig": [
                 {
                     "ChannelName": "training",
@@ -176,7 +175,6 @@ def test_repack_model_step_with_source_dir(estimator, source_dir):
         "Type": "Training",
         "Arguments": {
             "AlgorithmSpecification": {"TrainingInputMode": "File"},
-            "DebugHookConfig": {"CollectionConfigurations": [], "S3OutputPath": "s3://my-bucket/"},
             "InputDataConfig": [
                 {
                     "ChannelName": "training",

tests/unit/test_chainer.py

@@ -154,13 +154,8 @@ def _create_train_job(version, py_version):
         "enable_network_isolation": False,
         "environment": None,
         "experiment_config": None,
-        "debugger_hook_config": {
-            "CollectionConfigurations": [],
-            "S3OutputPath": "s3://{}/".format(BUCKET_NAME),
-        },
         "profiler_config": {
-            "DisableProfiler": False,
-            "S3OutputPath": "s3://{}/".format(BUCKET_NAME),
+            "DisableProfiler": True,
         },
     }