diff --git a/src/ezautoml/__version__.py b/src/ezautoml/__version__.py index 06aa5fa..a5f830a 100644 --- a/src/ezautoml/__version__.py +++ b/src/ezautoml/__version__.py @@ -1,2 +1 @@ - -__version__= "0.7.1" \ No newline at end of file +__version__ = "0.7.1" diff --git a/src/ezautoml/cli.py b/src/ezautoml/cli.py index 4d2cd9a..f1e2d2e 100644 --- a/src/ezautoml/cli.py +++ b/src/ezautoml/cli.py @@ -18,8 +18,10 @@ from ezautoml.__version__ import __version__ import warnings + warnings.filterwarnings("ignore", category=UserWarning) + def parse_args(): """ Parses the command-line arguments. @@ -27,34 +29,73 @@ def parse_args(): parser = argparse.ArgumentParser( prog="ezautoml", description="A Democratized, lightweight and modern framework for Python Automated Machine Learning.", - epilog="For more info, visit: https://github.com/eZWALT/eZAutoML" + epilog="For more info, visit: https://github.com/eZWALT/eZAutoML", ) # Required arguments - parser.add_argument("--dataset", type=str, required=True, help="Path to the dataset file (CSV)") - parser.add_argument("--target", type=str, required=True, help="The target column name for prediction") parser.add_argument( - "--task", - choices=["classification", "regression", "c", "r"], - required=True, - help="Task type: 'classification', 'regression', 'c' for classification, or 'r' for regression" + "--dataset", type=str, required=True, help="Path to the dataset file (CSV)" + ) + parser.add_argument( + "--target", + type=str, + required=True, + help="The target column name for prediction", + ) + parser.add_argument( + "--task", + choices=["classification", "regression", "c", "r"], + required=True, + help="Task type: 'classification', 'regression', 'c' for classification, or 'r' for regression", ) # Optional arguments - parser.add_argument("--models", type=str, default="lgbm,xgb,rf", help="Comma-separated list of models to use (e.g., lr,rf,xgb). Use initials!") - parser.add_argument("--search", choices=["random", "optuna"], default="random", help="Optimization algorithm to perform") - parser.add_argument("--trials", type=int, default=10, help="Maximum number of trials inside an optimization algorithm") - parser.add_argument("--output", type=str, default=".", help="Directory to save the output models/results") - parser.add_argument("--save", action="store_true", help="Directory to save the output models/results") - parser.add_argument("--verbose", action="store_true", help="Increase logging verbosity") - parser.add_argument("--version", action="version", version=f"eZAutoML {__version__}", help="Show the current version") + parser.add_argument( + "--models", + type=str, + default="lgbm,xgb,rf", + help="Comma-separated list of models to use (e.g., lr,rf,xgb). Use initials!", + ) + parser.add_argument( + "--search", + choices=["random", "optuna"], + default="random", + help="Optimization algorithm to perform", + ) + parser.add_argument( + "--trials", + type=int, + default=10, + help="Maximum number of trials inside an optimization algorithm", + ) + parser.add_argument( + "--output", + type=str, + default=".", + help="Directory to save the output models/results", + ) + parser.add_argument( + "--save", + action="store_true", + help="Directory to save the output models/results", + ) + parser.add_argument( + "--verbose", action="store_true", help="Increase logging verbosity" + ) + parser.add_argument( + "--version", + action="version", + version=f"eZAutoML {__version__}", + help="Show the current version", + ) return parser.parse_args() + def sanitize_feature_names(df): """ Sanitizes column names by replacing non-alphanumeric characters with underscores. """ - sanitized_columns = [re.sub(r'[^0-9a-zA-Z_]', '_', col) for col in df.columns] + sanitized_columns = [re.sub(r"[^0-9a-zA-Z_]", "_", col) for col in df.columns] df.columns = sanitized_columns return df @@ -75,6 +116,7 @@ def load_and_prepare_data(dataset_path, target_column, task_type): return X, y + def get_task_type_and_metrics(task): """ Returns the appropriate TaskType and metrics based on the task (classification or regression). @@ -84,7 +126,7 @@ def get_task_type_and_metrics(task): "c": "classification", "r": "regression", "classification": "classification", - "regression": "regression" + "regression": "regression", } task = task_mapping.get(task, task) # Get the corresponding full task name @@ -94,22 +136,28 @@ def get_task_type_and_metrics(task): metrics = MetricSet( { "accuracy": Metric(name="accuracy", fn=accuracy_score, minimize=False), - "f1_score": Metric(name="f1_score", fn=f1_score, minimize=False, default_kwargs={"average": "macro"}) + "f1_score": Metric( + name="f1_score", + fn=f1_score, + minimize=False, + default_kwargs={"average": "macro"}, + ), }, - primary_metric_name="f1_score" + primary_metric_name="f1_score", ) elif task == "regression": task_type = TaskType.REGRESSION metrics = MetricSet( { "mse": Metric(name="mse", fn=mean_squared_error, minimize=True), - "r2": Metric(name="r2", fn=r2_score, minimize=False) + "r2": Metric(name="r2", fn=r2_score, minimize=False), }, - primary_metric_name="mse" + primary_metric_name="mse", ) return task_type, metrics + def select_optimizer(search_strategy): """ Returns the optimizer class based on the search strategy. @@ -117,17 +165,23 @@ def select_optimizer(search_strategy): """ if search_strategy == "optuna": log_wip_message() # Log the WIP message - raise SystemExit("Optuna optimizer is currently a Work In Progress. The process will be terminated.") # Terminate the process + raise SystemExit( + "Optuna optimizer is currently a Work In Progress. The process will be terminated." + ) # Terminate the process else: optimizer_cls = RandomSearchOptimizer return optimizer_cls + def log_wip_message(): """ Logs a Work In Progress (WIP) message for the Optuna optimizer. """ - logger.warning("Optuna optimizer is WIP (Work in Progress). Please proceed with caution.") + logger.warning( + "Optuna optimizer is WIP (Work in Progress). Please proceed with caution." + ) + def save_results(ezautoml, output_dir): """ @@ -138,6 +192,7 @@ def save_results(ezautoml, output_dir): ezautoml.history.to_json(os.path.join(output_dir, "history.json")) ezautoml.history.to_csv(os.path.join(output_dir, "history_summary.csv")) + def main(): args = parse_args() @@ -145,7 +200,7 @@ def main(): "c": "classification", "r": "regression", "classification": "classification", - "regression": "regression" + "regression": "regression", } task = task_mapping.get(args.task, args.task) @@ -155,7 +210,9 @@ def main(): if X is None or y is None: return - search_space_file = "classification_space" if task == "classification" else "regression_space" + search_space_file = ( + "classification_space" if task == "classification" else "regression_space" + ) search_space = SearchSpace.from_builtin(search_space_file) optimizer_cls = select_optimizer(args.search) @@ -166,7 +223,7 @@ def main(): metrics=metrics, optimizer_cls=optimizer_cls, max_trials=args.trials, - verbose=args.verbose + verbose=args.verbose, ) ezautoml.fit(X, y) @@ -182,5 +239,6 @@ def main(): if args.save: save_results(ezautoml, args.output) + if __name__ == "__main__": main() diff --git a/src/ezautoml/data/loader.py b/src/ezautoml/data/loader.py index 0239789..cb3b743 100644 --- a/src/ezautoml/data/loader.py +++ b/src/ezautoml/data/loader.py @@ -17,9 +17,9 @@ from sklearn.impute import SimpleImputer from skimage.transform import resize -import torch +import torch from torch.utils import data -import torch.nn as nn +import torch.nn as nn import torch.nn.functional as F from torchvision.datasets import MNIST, FashionMNIST, CIFAR10 @@ -57,14 +57,16 @@ class DatasetLoader: - def __init__(self, local_path: str = "data", metadata_path: str = "./data/metadata.json"): + def __init__( + self, local_path: str = "data", metadata_path: str = "./data/metadata.json" + ): self.local_path = local_path self.dataset_groups = { "builtin": ["breast_cancer"], "local": None, # Discovered dynamically "medmnist": [ "pathmnist", - #"chestmnist", + # "chestmnist", "octmnist", "pneumoniamnist", "breastmnist", @@ -84,25 +86,25 @@ def __init__(self, local_path: str = "data", metadata_path: str = "./data/metada ], } self.no_scale_datasets = { - "mnist", - "fashion_mnist", - "cifar10", - "pathmnist", - "chestmnist", - "octmnist", - "pneumoniamnist", - "breastmnist", - "bloodmnist", - "tissuemnist", - "organamnist", - "organcmnist", - "organsmnist", - "organmnist3d", - "nodulmnist3d", - "adrenalmnist3d", - "fracturemnist3d", - "vesselmnist3d", - "synapsemnist3d", + "mnist", + "fashion_mnist", + "cifar10", + "pathmnist", + "chestmnist", + "octmnist", + "pneumoniamnist", + "breastmnist", + "bloodmnist", + "tissuemnist", + "organamnist", + "organcmnist", + "organsmnist", + "organmnist3d", + "nodulmnist3d", + "adrenalmnist3d", + "fracturemnist3d", + "vesselmnist3d", + "synapsemnist3d", } self.datasets = {} # Populated via method logger.info("DatasetLoader initialized.") @@ -116,19 +118,20 @@ def _smart_read_csv(self, file): # Detect delimiter: semicolon or comma delimiter = ";" if sample.count(";") > sample.count(",") else "," - + # Detect decimal: comma or dot decimal = "," if "," in sample and "." not in sample else "." # Read the CSV while ensuring that no column is treated as the index - df = pd.read_csv(file, delimiter=delimiter, decimal=decimal, index_col=False) - + df = pd.read_csv( + file, delimiter=delimiter, decimal=decimal, index_col=False + ) + return df except Exception as e: logger.error(f"Error reading {file}: {e}") return None - def _load_local_datasets(self): datasets = {} logger.info("Searching for local CSV datasets...") @@ -147,7 +150,9 @@ def _load_local_datasets(self): raise EnvironmentError("Could not read CSV.") if filename not in target_columns: - raise ValueError(f"Target column for {filename} not found in metadata!") + raise ValueError( + f"Target column for {filename} not found in metadata!" + ) target_col = target_columns[filename] X = df.drop(columns=[target_col]) @@ -155,7 +160,7 @@ def _load_local_datasets(self): X, y = self._clean_and_preprocess_local(X, y) datasets[filename] = (X, y) - + except Exception as e: logger.warning(f"Skipping {file}: {repr(e)}") @@ -164,23 +169,24 @@ def _load_local_datasets(self): def _convert_categoricals(self, X, y): X = X.copy() - + # Convert categorical features in X for col in X.select_dtypes(include="object").columns: X[col] = LabelEncoder().fit_transform(X[col]) - + # If y is numeric (not categorical), encode based on unique values if np.issubdtype(y.dtype, np.number): # Use np.unique to get the unique values and map them to a range starting from 0 unique_values = np.unique(y) - y = np.searchsorted(unique_values, y) # This maps the original values to new ones - + y = np.searchsorted( + unique_values, y + ) # This maps the original values to new ones + else: # For categorical y, use LabelEncoder y = LabelEncoder().fit_transform(y) - - return X, y + return X, y def _clean_and_preprocess_local(self, X, y): for col in X.columns: @@ -206,9 +212,9 @@ def _clean_and_preprocess_local(self, X, y): if X[col].isnull().any(): imputer = SimpleImputer(strategy="most_frequent") X[col] = imputer.fit_transform(X[[col]]).ravel() - + return self._convert_categoricals(X, y) - + def _load_builtin_datasets(self): logger.info("Loading built-in datasets...") return { @@ -262,14 +268,26 @@ def _load_medmnist(self, name): dataset = dataset_cls(root="data", split="train", download=True) # Check if `targets` or `labels` is available - if hasattr(dataset, 'targets'): - y = dataset.targets.numpy() if isinstance(dataset.targets, torch.Tensor) else np.array(dataset.targets) - elif hasattr(dataset, 'labels'): - y = dataset.labels.numpy() if isinstance(dataset.labels, torch.Tensor) else np.array(dataset.labels) + if hasattr(dataset, "targets"): + y = ( + dataset.targets.numpy() + if isinstance(dataset.targets, torch.Tensor) + else np.array(dataset.targets) + ) + elif hasattr(dataset, "labels"): + y = ( + dataset.labels.numpy() + if isinstance(dataset.labels, torch.Tensor) + else np.array(dataset.labels) + ) else: - raise AttributeError(f"Dataset '{name}' has no 'targets' or 'labels' attribute.") + raise AttributeError( + f"Dataset '{name}' has no 'targets' or 'labels' attribute." + ) - X = torch.stack([transforms.ToTensor()(img[0]).flatten() for img in dataset]).numpy() + X = torch.stack( + [transforms.ToTensor()(img[0]).flatten() for img in dataset] + ).numpy() return X, y @@ -363,7 +381,9 @@ def load_selected_datasets(self, groups=None, names=None): # Do not scale image-based datasets :) for name, (X, y) in selected.items(): try: - processed[name] = self._preprocess_data(X, y, scale=(name not in self.no_scale_datasets)) + processed[name] = self._preprocess_data( + X, y, scale=(name not in self.no_scale_datasets) + ) except Exception as e: logger.error(f"Error preprocessing {name}: {e}") @@ -373,11 +393,11 @@ def load_selected_datasets(self, groups=None, names=None): def get_datasets(self): return self.datasets - + def load_user_datasets(self, file_paths: list[str], metadata: dict[str, str]): """ Load user-provided datasets from CSV files with minimal tabular ML treatment. - + :param file_paths: list of CSV file paths :param metadata: dict of filename -> target_column :return: dict of {filename: (X, y)} @@ -401,7 +421,9 @@ def load_user_datasets(self, file_paths: list[str], metadata: dict[str, str]): X, y = self._clean_and_preprocess_local(X, y) user_datasets[filename] = (X, y) - logger.success(f"Loaded user dataset: {filename} | X: {X.shape} | y: {y.shape}") + logger.success( + f"Loaded user dataset: {filename} | X: {X.shape} | y: {y.shape}" + ) except Exception as e: logger.warning(f"Skipping {filename}: {repr(e)}") @@ -409,18 +431,17 @@ def load_user_datasets(self, file_paths: list[str], metadata: dict[str, str]): return user_datasets - def test_main_loading(): loader = DatasetLoader(local_path="data") # Example usage: datasets = loader.load_selected_datasets( groups=[ - #"builtin", - #"medmnist3d", - #"medmnist", - "local", - #"torchvision" + # "builtin", + # "medmnist3d", + # "medmnist", + "local", + # "torchvision" ] ) @@ -431,6 +452,7 @@ def test_main_loading(): logger.info(f" ➤ Y type: {type(Y)}") logger.info(f" ➤ Y dtype: {Y.dtype}") + def test_user_custom_data(): user_files = ["/home/wtroiani/lol1.csv", "/home/wtroiani/lol2.csv"] user_metadata = { @@ -438,7 +460,9 @@ def test_user_custom_data(): "lol2.csv": "smoking", } - loader = DatasetLoader(local_path="../../data", metadata_path="../../data/metadata.json") + loader = DatasetLoader( + local_path="../../data", metadata_path="../../data/metadata.json" + ) user_datasets = loader.load_user_datasets(user_files, user_metadata) for name, (X, y) in user_datasets.items(): @@ -447,4 +471,3 @@ def test_user_custom_data(): if __name__ == "__main__": test_user_custom_data() - \ No newline at end of file diff --git a/src/ezautoml/data/preprocess.py b/src/ezautoml/data/preprocess.py index 355706e..0635eba 100644 --- a/src/ezautoml/data/preprocess.py +++ b/src/ezautoml/data/preprocess.py @@ -4,6 +4,7 @@ from sklearn.impute import SimpleImputer from sklearn.model_selection import train_test_split + def prepare_data(df, target_column, scale=True, task_type="classification"): """ Preprocess the given DataFrame: missing value handling, categorical encoding, @@ -20,7 +21,7 @@ def prepare_data(df, target_column, scale=True, task_type="classification"): - y: Processed target array. - target_encoder: LabelEncoder for classification tasks (None otherwise). """ - + # 1. Separate features and target X = df.drop(columns=[target_column]) y = df[target_column] @@ -38,7 +39,9 @@ def prepare_data(df, target_column, scale=True, task_type="classification"): for col in cat_cols: if X[col].isnull().any(): - X[col] = SimpleImputer(strategy="most_frequent").fit_transform(X[[col]]).ravel() + X[col] = ( + SimpleImputer(strategy="most_frequent").fit_transform(X[[col]]).ravel() + ) # 4. Encode categorical features for col in cat_cols: @@ -59,4 +62,4 @@ def prepare_data(df, target_column, scale=True, task_type="classification"): target_encoder = LabelEncoder() y = target_encoder.fit_transform(y) - return X, y, target_encoder \ No newline at end of file + return X, y, target_encoder diff --git a/src/ezautoml/evaluation/evaluator.py b/src/ezautoml/evaluation/evaluator.py index 99e68fb..398cfa7 100644 --- a/src/ezautoml/evaluation/evaluator.py +++ b/src/ezautoml/evaluation/evaluator.py @@ -6,10 +6,11 @@ @dataclass class Evaluation: """Class to store evaluation results and perform comparisons.""" + results: Dict[str, float] metric_set: MetricSet - def compare(self, other: 'Evaluation') -> Dict[str, str]: + def compare(self, other: "Evaluation") -> Dict[str, str]: """Compare this evaluation with another evaluation.""" comparison = {} for metric_name in self.results: @@ -21,7 +22,9 @@ def compare(self, other: 'Evaluation') -> Dict[str, str]: continue # Compare the current result with the challenger result - improvement = self.metric_set[metric_name].is_improvement(current_value, challenger_value) + improvement = self.metric_set[metric_name].is_improvement( + current_value, challenger_value + ) comparison[metric_name] = improvement.value return comparison @@ -34,11 +37,13 @@ def __str__(self) -> str: class Evaluator: """Class responsible for evaluating predictions.""" - + def __init__(self, metric_set: MetricSet): self.metric_set = metric_set - - def evaluate(self, ground_truth: 'ArrayLike', predictions: 'ArrayLike') -> Evaluation: + + def evaluate( + self, ground_truth: "ArrayLike", predictions: "ArrayLike" + ) -> Evaluation: """Evaluate predictions using the metrics in the MetricSet.""" results = { metric_name: metric.evaluate(ground_truth, predictions) @@ -52,14 +57,16 @@ def evaluate(self, ground_truth: 'ArrayLike', predictions: 'ArrayLike') -> Evalu if __name__ == "__main__": import numpy as np from sklearn.metrics import accuracy_score, mean_squared_error, f1_score - + # Define a set of metrics - metrics = MetricSet(metrics={ - "accuracy": Metric(name="accuracy", fn=accuracy_score, minimize=False), - "mse": Metric(name="mse", fn=mean_squared_error, minimize=True), - "f1_score": Metric(name="f1_score", fn=f1_score, minimize=False) - }, - primary_metric_name="accuracy") + metrics = MetricSet( + metrics={ + "accuracy": Metric(name="accuracy", fn=accuracy_score, minimize=False), + "mse": Metric(name="mse", fn=mean_squared_error, minimize=True), + "f1_score": Metric(name="f1_score", fn=f1_score, minimize=False), + }, + primary_metric_name="accuracy", + ) # Create an evaluator instance evaluator = Evaluator(metric_set=metrics) @@ -67,7 +74,7 @@ def evaluate(self, ground_truth: 'ArrayLike', predictions: 'ArrayLike') -> Evalu # True and predicted values y_true = np.array([1, 0, 1, 1, 0]) y_pred_good = np.array([1, 0, 1, 1, 0]) # Good predictions - y_pred_bad = np.array([0, 0, 0, 0, 0]) # Bad predictions + y_pred_bad = np.array([0, 0, 0, 0, 0]) # Bad predictions # Evaluate good predictions evaluation_good = evaluator.evaluate(y_true, y_pred_good) diff --git a/src/ezautoml/evaluation/metric.py b/src/ezautoml/evaluation/metric.py index 2eb99f2..bd05d36 100644 --- a/src/ezautoml/evaluation/metric.py +++ b/src/ezautoml/evaluation/metric.py @@ -9,6 +9,7 @@ # Author: Walter J.T.V # # ===----------------------------------------------------------------------===# + class Comparison(str, Enum): BETTER = "better" WORSE = "worse" @@ -26,14 +27,19 @@ class Metric: def evaluate(self, *args, **kwargs) -> float: if self.fn is None: raise ValueError(f"Metric '{self.name}' has no function attached.") - all_kwargs = {**self.default_kwargs, **kwargs} # Merge default and call-time kwargs + all_kwargs = { + **self.default_kwargs, + **kwargs, + } # Merge default and call-time kwargs return self.fn(*args, **all_kwargs) def is_improvement(self, current: float, challenger: float) -> Comparison: """Compares the current value with the challenger value.""" if current == challenger: return Comparison.EQUAL - if (challenger < current and self.minimize) or (challenger > current and not self.minimize): + if (challenger < current and self.minimize) or ( + challenger > current and not self.minimize + ): return Comparison.BETTER return Comparison.WORSE @@ -55,8 +61,9 @@ def worst(self) -> float: @dataclass(frozen=True) class MetricSet: """A collection of multiple metrics, organized as a set.""" + metrics: Dict[str, Metric] = field(default_factory=dict) - primary_metric_name: str = "accuracy" + primary_metric_name: str = "accuracy" def __getitem__(self, key: str) -> Metric: return self.metrics[key] @@ -66,7 +73,7 @@ def __iter__(self): def __len__(self): return len(self.metrics) - + def items(self): return self.metrics.items() @@ -78,8 +85,7 @@ def get_worst_values(self) -> Dict[str, float]: @property def primary(self) -> Metric: - return self.metrics[self.primary_metric_name] - + return self.metrics[self.primary_metric_name] if __name__ == "__main__": @@ -87,12 +93,18 @@ def primary(self) -> Metric: from sklearn.metrics import accuracy_score, mean_squared_error, f1_score import numpy as np - metrics = MetricSet({ - "accuracy": Metric(name="accuracy", fn=accuracy_score, minimize=False), - "mse": Metric(name="mse", fn=mean_squared_error, minimize=True), - "f1_score": Metric(name="f1_score", fn=lambda y_true, y_pred: f1_score(y_true, y_pred, average='binary'), minimize=False) - }, - primary_metric_name="accuracy") + metrics = MetricSet( + { + "accuracy": Metric(name="accuracy", fn=accuracy_score, minimize=False), + "mse": Metric(name="mse", fn=mean_squared_error, minimize=True), + "f1_score": Metric( + name="f1_score", + fn=lambda y_true, y_pred: f1_score(y_true, y_pred, average="binary"), + minimize=False, + ), + }, + primary_metric_name="accuracy", + ) # True and predicted values y_true = np.array([1, 0, 1, 1, 0]) @@ -107,4 +119,4 @@ def primary(self) -> Metric: print(f"{name}:") print(f" Good Score = {score_good:.4f}") print(f" Bad Score = {score_bad:.4f}") - print(f" Improvement = {improvement.value}") \ No newline at end of file + print(f" Improvement = {improvement.value}") diff --git a/src/ezautoml/evaluation/task.py b/src/ezautoml/evaluation/task.py index c8cc298..b5c16dd 100644 --- a/src/ezautoml/evaluation/task.py +++ b/src/ezautoml/evaluation/task.py @@ -1,5 +1,6 @@ from enum import Enum + class TaskType(Enum): CLASSIFICATION = "classification" REGRESSION = "regression" diff --git a/src/ezautoml/model.py b/src/ezautoml/model.py index 01a1927..12db66f 100644 --- a/src/ezautoml/model.py +++ b/src/ezautoml/model.py @@ -15,12 +15,15 @@ import contextlib import os + + @contextlib.contextmanager def suppress_output(): - with open(os.devnull, 'w') as fnull: + with open(os.devnull, "w") as fnull: with contextlib.redirect_stdout(fnull), contextlib.redirect_stderr(fnull): yield + class eZAutoML: def __init__( self, @@ -31,7 +34,7 @@ def __init__( max_trials=100, max_time=3600, seed=42, - verbose=True + verbose=True, ): self.verbose = verbose self.search_space = search_space @@ -40,8 +43,8 @@ def __init__( self.max_trials = max_trials self.max_time = max_time self.seed = seed - self.task = task - + self.task = task + assert self.task.value == search_space.task.value self.history = History() @@ -54,7 +57,9 @@ def __init__( def fit(self, X, y): """Run optimization using Random Search.""" if self.verbose: - self.console.print("[bold green]Starting optimization...", style="bold green") + self.console.print( + "[bold green]Starting optimization...", style="bold green" + ) # Initialize the optimizer optimizer = self.optimizer_cls( @@ -62,14 +67,14 @@ def fit(self, X, y): metrics=self.metrics, max_trials=self.max_trials, max_time=self.max_time, - seed=self.seed + seed=self.seed, ) start_time = time.time() primary_metric = self.metrics.primary primary_name = primary_metric.name primary_fn = primary_metric.fn - best_score = float('inf') if primary_metric.minimize else float('-inf') + best_score = float("inf") if primary_metric.minimize else float("-inf") best_model_config = None # Optimization loop @@ -105,7 +110,7 @@ def fit(self, X, y): model_name=config.model.name, optimizer_name=optimizer.__class__.__name__, evaluation=evaluation, - duration=duration + duration=duration, ) self.history.add(trial) @@ -114,11 +119,13 @@ def fit(self, X, y): if self.verbose: self.console.print( f"[Trial {len(self.history.trials)}] {primary_name}={score:.4f} in {duration:.2f}s - {model_name}", - style="dim" + style="dim", ) # Keep best based on minimize flag - if (primary_metric.minimize and score < best_score) or (not primary_metric.minimize and score > best_score): + if (primary_metric.minimize and score < best_score) or ( + not primary_metric.minimize and score > best_score + ): best_score = score best_model_config = config @@ -136,13 +143,14 @@ def fit(self, X, y): ) self.best_config = best_model_config - self.best_model = best_model_config.model.instantiate(best_model_config.model_params) + self.best_model = best_model_config.model.instantiate( + best_model_config.model_params + ) self.best_model.fit(X, y) else: if self.verbose: self.console.print("[bold red]No valid pipeline found.[/bold red]") - def predict(self, X): """Make predictions using the fitted model.""" if self.best_model is None: @@ -162,20 +170,27 @@ def test(self, X_test, y_test): raise RuntimeError("Model not fitted. Call fit() first.") # Make predictions and evaluate - predictions = self.predict(X_test) + predictions = self.predict(X_test) primary_name = self.metrics.primary_metric_name - test_score = self.metrics.primary.fn (y_test, predictions, **self.metrics.primary.default_kwargs) - + test_score = self.metrics.primary.fn( + y_test, predictions, **self.metrics.primary.default_kwargs + ) + if self.verbose: - self.console.print(f"[bold blue]Test {primary_name}:[/bold blue] {test_score:.4f}") + self.console.print( + f"[bold blue]Test {primary_name}:[/bold blue] {test_score:.4f}" + ) return test_score def summary(self, k=5): minimize_map = {self.metrics.primary.name: self.metrics.primary.minimize} - return self.history.summary(k=k, metrics=[self.metrics.primary.name], minimize_map=minimize_map) - + return self.history.summary( + k=k, metrics=[self.metrics.primary.name], minimize_map=minimize_map + ) + # TODO: Add Serialization method for models for a big save function (model + output results) + # --- Main Function --- import time from sklearn.ensemble import RandomForestClassifier @@ -192,21 +207,23 @@ def summary(self, k=5): from ezautoml.optimization.optimizers.random_search import RandomSearchOptimizer from ezautoml.data.loader import DatasetLoader + def main(): - # --- Load dataset --- + # --- Load dataset --- loader = DatasetLoader( - local_path="../../data", - metadata_path="../../data/metadata.json" + local_path="../../data", metadata_path="../../data/metadata.json" ) datasets = loader.load_selected_datasets(groups=["local", "builtin", "torchvision"]) X, y = datasets["dota"] # Replace with any available dataset - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42) + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.3, random_state=42 + ) # Define metrics and evaluator metrics = MetricSet( {"accuracy": Metric(name="accuracy", fn=accuracy_score, minimize=False)}, - primary_metric_name="accuracy" + primary_metric_name="accuracy", ) - + search_space = SearchSpace.from_yaml("classification_space.yaml") # Initialize eZAutoML @@ -216,7 +233,7 @@ def main(): metrics=metrics, max_trials=5, max_time=600, # 10 minutes - seed=42 + seed=42, ) # Fit model @@ -225,15 +242,14 @@ def main(): test_accuracy = ezautoml.test(X_test, y_test) # Show best trial summary ezautoml.summary(k=5) - - + + def main2(): from sklearn.metrics import mean_squared_error, r2_score # --- Load dataset --- loader = DatasetLoader( - local_path="../../data", - metadata_path="../../data/metadata.json" + local_path="../../data", metadata_path="../../data/metadata.json" ) datasets = loader.load_selected_datasets(groups=["local", "builtin", "torchvision"]) @@ -241,15 +257,17 @@ def main2(): raise ValueError("Dataset 'wine.csv' not found in loaded datasets.") X, y = datasets["wine.csv"] - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42) + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.3, random_state=42 + ) # Define regression metrics metrics = MetricSet( { "mse": Metric(name="mse", fn=mean_squared_error, minimize=True), - "r2": Metric(name="r2", fn=r2_score, minimize=False) + "r2": Metric(name="r2", fn=r2_score, minimize=False), }, - primary_metric_name="mse" + primary_metric_name="mse", ) # Load regression search space @@ -262,7 +280,7 @@ def main2(): metrics=metrics, max_trials=100, max_time=600, - seed=42 + seed=42, ) # Fit, test, summarize diff --git a/src/ezautoml/optimization/optimizer.py b/src/ezautoml/optimization/optimizer.py index ad628a4..07b576c 100644 --- a/src/ezautoml/optimization/optimizer.py +++ b/src/ezautoml/optimization/optimizer.py @@ -7,6 +7,7 @@ from ezautoml.space.search_point import SearchPoint from ezautoml.evaluation.metric import MetricSet + class Optimizer(ABC): """Abstract base optimizer for CASH: model selection + hyperparameter optimization. @@ -20,7 +21,7 @@ def __init__( max_trials: int, max_time: int, # in seconds seed: Optional[int] = None, - verbose: bool = False + verbose: bool = False, ) -> None: self.metrics = metrics self.space = space @@ -80,7 +81,7 @@ def create( max_trials: int, max_time: int, seed: Optional[int] = None, - ) -> 'Optimizer': + ) -> "Optimizer": return cls( metrics=metrics, space=space, diff --git a/src/ezautoml/optimization/optimizers/optuna.py b/src/ezautoml/optimization/optimizers/optuna.py index 819a7ee..e1605fa 100644 --- a/src/ezautoml/optimization/optimizers/optuna.py +++ b/src/ezautoml/optimization/optimizers/optuna.py @@ -12,9 +12,6 @@ import numpy as np - - - class OptunaOptimizer(Optimizer): """Optuna optimization strategy for CASH (model selection + hyperparameter tuning).""" @@ -60,10 +57,15 @@ def tell(self, report: SearchPoint) -> None: # Extract the result from the report, assuming `result` contains an `Evaluation` object if report.result: - score = report.result.results["accuracy"] # Assuming 'accuracy' is one of the keys in the results + score = report.result.results[ + "accuracy" + ] # Assuming 'accuracy' is one of the keys in the results # Check if any trials are completed - if len(self.study.trials) == 0 or all(trial.state != optuna.trial.TrialState.COMPLETE for trial in self.study.trials): + if len(self.study.trials) == 0 or all( + trial.state != optuna.trial.TrialState.COMPLETE + for trial in self.study.trials + ): # Add the first trial (initial trial) if no trials are completed yet trial = optuna.trial.FrozenTrial( number=self.trial_count, @@ -77,7 +79,9 @@ def tell(self, report: SearchPoint) -> None: else: # Check if the best trial has been completed if self.study.best_trial.state == optuna.trial.TrialState.COMPLETE: - comparison = self.metrics.primary.is_improvement(self.study.best_value, score) + comparison = self.metrics.primary.is_improvement( + self.study.best_value, score + ) # If new trial is better, add it if comparison == Comparison.BETTER: @@ -93,7 +97,9 @@ def tell(self, report: SearchPoint) -> None: logger.info(f"New best trial found with score {score}.") else: if self.verbose: - logger.info(f"Trial did not improve. Current best: {self.study.best_value}") + logger.info( + f"Trial did not improve. Current best: {self.study.best_value}" + ) else: # If no trials are fully completed, just add the current trial trial = optuna.trial.FrozenTrial( @@ -109,7 +115,6 @@ def tell(self, report: SearchPoint) -> None: self.trial_count += 1 - def ask(self, n: int = 1) -> Union[SearchPoint, List[SearchPoint]]: """Sample new candidate configurations from the search space.""" if self.stop_optimization(): @@ -130,7 +135,9 @@ def ask(self, n: int = 1) -> Union[SearchPoint, List[SearchPoint]]: # TODO APPLY DATA PROCESSORS # Split the data into training and validation sets - X_train, X_val, y_train, y_val = train_test_split(self.X, self.y, test_size=0.3, random_state=self.seed) + X_train, X_val, y_train, y_val = train_test_split( + self.X, self.y, test_size=0.3, random_state=self.seed + ) # Train the model model.fit(X_train, y_train) @@ -146,7 +153,9 @@ def ask(self, n: int = 1) -> Union[SearchPoint, List[SearchPoint]]: trials.append(config) if self.verbose: - logger.info(f"[ASK] Sampled configuration: {config} - Evaluation: {evaluation_score}") + logger.info( + f"[ASK] Sampled configuration: {config} - Evaluation: {evaluation_score}" + ) return trials if n > 1 else trials[0] @@ -166,7 +175,9 @@ def objective(self, trial: optuna.Trial): model = config.model.instantiate(config.model_params) # Split the data - X_train, X_val, y_train, y_val = train_test_split(self.X, self.y, test_size=0.3, random_state=self.seed) + X_train, X_val, y_train, y_val = train_test_split( + self.X, self.y, test_size=0.3, random_state=self.seed + ) # Train the model model.fit(X_train, y_train) @@ -181,6 +192,7 @@ def optimize(self): """Run the optimization using the Optuna study.""" self.study.optimize(self.objective, n_trials=self.max_trials) + if __name__ == "__main__": import time import random @@ -201,23 +213,29 @@ def optimize(self): from ezautoml.evaluation.task import TaskType from ezautoml.results.trial import Trial from ezautoml.results.history import History - + from ezautoml.data.loader import DatasetLoader # 1. Initialize DatasetLoader - loader = DatasetLoader(local_path="../../data", metadata_path="../../data/metadata.json") - datasets = loader.load_selected_datasets(groups=["local", "builtin", "torchvision"]) # Load datasets + loader = DatasetLoader( + local_path="../../data", metadata_path="../../data/metadata.json" + ) + datasets = loader.load_selected_datasets( + groups=["local", "builtin", "torchvision"] + ) # Load datasets # 2. Select a dataset (e.g., breast cancer dataset) X, y = datasets["breast_cancer"] # Adjust depending on the dataset you want to use # 3. Split the dataset into train/test - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42) + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.3, random_state=42 + ) # 4. Define metrics and evaluator metrics = MetricSet( {"accuracy": Metric(name="accuracy", fn=accuracy_score, minimize=False)}, - primary_metric_name="accuracy" + primary_metric_name="accuracy", ) evaluator = Evaluator(metric_set=metrics) @@ -257,8 +275,7 @@ def optimize(self): # 7. Define search space and optimizer search_space = SearchSpace( - models=[rf_component, dt_component, lr_component], - task="classification" + models=[rf_component, dt_component, lr_component], task="classification" ) optimizer = OptunaOptimizer( @@ -269,7 +286,7 @@ def optimize(self): max_trials=10, max_time=3600, seed=42, - verbose=True + verbose=True, ) # 8. Initialize trial history @@ -302,7 +319,7 @@ def optimize(self): model_name=trial_config.model.name, optimizer_name="RandomSearch", evaluation=evaluation, - duration=duration + duration=duration, ) history.add(trial) diff --git a/src/ezautoml/optimization/optimizers/random_search.py b/src/ezautoml/optimization/optimizers/random_search.py index d7653ec..212d9a0 100644 --- a/src/ezautoml/optimization/optimizers/random_search.py +++ b/src/ezautoml/optimization/optimizers/random_search.py @@ -6,6 +6,7 @@ from typing import List, Optional, Union import time + class RandomSearchOptimizer(Optimizer): """Random search strategy for CASH (model selection + hyperparameter tuning).""" @@ -81,23 +82,29 @@ def get_best_trial(self) -> Optional[SearchPoint]: from ezautoml.evaluation.task import TaskType from ezautoml.results.trial import Trial from ezautoml.results.history import History - + from ezautoml.data.loader import DatasetLoader # Initialize DatasetLoader - loader = DatasetLoader(local_path="../../data", metadata_path="../../data/metadata.json") - datasets = loader.load_selected_datasets(groups=["local", "builtin", "torchvision"]) # Load datasets + loader = DatasetLoader( + local_path="../../data", metadata_path="../../data/metadata.json" + ) + datasets = loader.load_selected_datasets( + groups=["local", "builtin", "torchvision"] + ) # Load datasets # Select a dataset (for example, load the breast cancer dataset) X, y = datasets["breast_cancer"] # Adjust depending on the dataset you want to use # Split into train/test - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42) + X_train, X_test, y_train, y_test = train_test_split( + X, y, test_size=0.3, random_state=42 + ) # Define metrics and evaluator metrics = MetricSet( {"accuracy": Metric(name="accuracy", fn=accuracy_score, minimize=False)}, - primary_metric_name="accuracy" + primary_metric_name="accuracy", ) evaluator = Evaluator(metric_set=metrics) @@ -137,16 +144,11 @@ def get_best_trial(self) -> Optional[SearchPoint]: # Define search space and optimizer search_space = SearchSpace( - models=[rf_component, dt_component, lr_component], - task="classification" + models=[rf_component, dt_component, lr_component], task="classification" ) optimizer = RandomSearchOptimizer( - space=search_space, - metrics=metrics, - max_trials=100, - max_time=3600, - seed=42 + space=search_space, metrics=metrics, max_trials=100, max_time=3600, seed=42 ) # Initialize trial history @@ -179,9 +181,9 @@ def get_best_trial(self) -> Optional[SearchPoint]: model_name=trial_config.model.name, optimizer_name="RandomSearch", evaluation=evaluation, - duration=duration + duration=duration, ) history.add(trial) # Print summary of the best trials - history.summary(k=50, metrics=["accuracy"]) \ No newline at end of file + history.summary(k=50, metrics=["accuracy"]) diff --git a/src/ezautoml/parsers/optuna.py b/src/ezautoml/parsers/optuna.py index c9777fc..3f75a81 100644 --- a/src/ezautoml/parsers/optuna.py +++ b/src/ezautoml/parsers/optuna.py @@ -1,13 +1,14 @@ -import optuna +import optuna from ezautoml.space.search_space import SearchSpace from ezautoml.space.search_point import SearchPoint from ezautoml.space.component import Component from ezautoml.space.space import Integer, Categorical, Real + class OptunaParser: """Parse the search space and convert it into Optuna-compatible trials.""" - + def __init__(self, search_space: SearchSpace): self.search_space = search_space @@ -26,10 +27,12 @@ def parse_hyperparameters(self, model_config: Component, trial: optuna.Trial): def convert_to_search_point(self, trial: optuna.Trial) -> SearchPoint: """Convert an Optuna trial to a SearchPoint.""" - + # Select model configuration based on trial's suggested model index - model_config = self.search_space.models[trial.suggest_int("model", 0, len(self.search_space.models) - 1)] - + model_config = self.search_space.models[ + trial.suggest_int("model", 0, len(self.search_space.models) - 1) + ] + # Parse the hyperparameters for the model model_params = self.parse_hyperparameters(model_config, trial) @@ -44,4 +47,4 @@ def convert_to_search_point(self, trial: optuna.Trial) -> SearchPoint: feature_params_list=[], # Feature parameters ) - return config \ No newline at end of file + return config diff --git a/src/ezautoml/registry.py b/src/ezautoml/registry.py index 23ea302..5c07fbf 100644 --- a/src/ezautoml/registry.py +++ b/src/ezautoml/registry.py @@ -1,4 +1,4 @@ -# Global Constructor Registry to serialize/deserialize safely +# Global Constructor Registry to serialize/deserialize safely from dataclasses import dataclass, field from typing import Callable, Dict @@ -11,6 +11,7 @@ # Author: Walter J.T.V # # ===----------------------------------------------------------------------===# + @dataclass class ConstructorRegistry: registry: Dict[str, Callable] = field(default_factory=dict) @@ -35,8 +36,8 @@ def has(self, name: str) -> bool: def list(self): """Returns a list of all registered constructor names.""" return list(self.registry) - - + + ############################################################################### ############################################################################### ################### Instantiate registry structure ############################ @@ -44,31 +45,29 @@ def list(self): ############################################################################### from sklearn.ensemble import ( - RandomForestClassifier, RandomForestRegressor, - GradientBoostingClassifier, GradientBoostingRegressor, - AdaBoostClassifier, AdaBoostRegressor, - BaggingClassifier, BaggingRegressor, - ExtraTreesClassifier, ExtraTreesRegressor + RandomForestClassifier, + RandomForestRegressor, + GradientBoostingClassifier, + GradientBoostingRegressor, + AdaBoostClassifier, + AdaBoostRegressor, + BaggingClassifier, + BaggingRegressor, + ExtraTreesClassifier, + ExtraTreesRegressor, ) from sklearn.linear_model import ( - LogisticRegression, Ridge, Lasso, ElasticNet, - LinearRegression -) -from sklearn.svm import ( - SVC, SVR -) -from sklearn.neighbors import ( - KNeighborsClassifier, KNeighborsRegressor -) -from sklearn.tree import ( - DecisionTreeClassifier, DecisionTreeRegressor -) -from sklearn.naive_bayes import ( - GaussianNB, MultinomialNB -) -from sklearn.preprocessing import ( - StandardScaler, MinMaxScaler, RobustScaler + LogisticRegression, + Ridge, + Lasso, + ElasticNet, + LinearRegression, ) +from sklearn.svm import SVC, SVR +from sklearn.neighbors import KNeighborsClassifier, KNeighborsRegressor +from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor +from sklearn.naive_bayes import GaussianNB, MultinomialNB +from sklearn.preprocessing import StandardScaler, MinMaxScaler, RobustScaler from sklearn.decomposition import PCA from sklearn.cluster import KMeans @@ -82,7 +81,7 @@ def list(self): class NoFeatureEngineering: def fit(self, X, y=None): return self - + def transform(self, X): return X @@ -95,19 +94,21 @@ def fit(self, X, y=None): def transform(self, X): return X + # NoDataProcessing: Does nothing, just returns the input class NoDataProcessing: def fit(self, X, y=None): return self - + def transform(self, X): return X + # NoOptimizationAlgSelection: A placeholder class to simulate no optimization algorithm selection class NoOptimizationAlgSelection: def fit(self, X, y=None): return self - + def transform(self, X): return X @@ -122,7 +123,7 @@ def transform(self, X): RandomForestClassifier, GradientBoostingClassifier, LogisticRegression, - SVC, + SVC, KNeighborsClassifier, DecisionTreeClassifier, GaussianNB, @@ -133,7 +134,6 @@ def transform(self, X): XGBClassifier, LGBMClassifier, CatBoostClassifier, - # ----------------------------------------------------- # 2. Regression Models (Expanded) # ----------------------------------------------------- @@ -152,27 +152,23 @@ def transform(self, X): ExtraTreesRegressor, # Bagging-based model LGBMRegressor, CatBoostRegressor, - # ----------------------------------------------------- # 3. Feature processing components (Top 5) # ----------------------------------------------------- KMeans, PCA, - # ----------------------------------------------------- # 4. Data processing components (Top 5) # ----------------------------------------------------- StandardScaler, MinMaxScaler, RobustScaler, - # ----------------------------------------------------- # 5. Null components # ----------------------------------------------------- NoFeatureEngineering, - NoDataProcessing, + NoDataProcessing, NoFeatureProcessing, - NoOptimizationAlgSelection - + NoOptimizationAlgSelection, ]: constructor_registry.register(constructor) diff --git a/src/ezautoml/results/history.py b/src/ezautoml/results/history.py index 37be6f2..dda3428 100644 --- a/src/ezautoml/results/history.py +++ b/src/ezautoml/results/history.py @@ -10,6 +10,7 @@ from ezautoml.results.trial import Trial from ezautoml.evaluation.evaluator import Evaluation + def to_json_serializable(obj): """Convert a dataclass or object to a JSON-serializable format.""" if is_dataclass(obj): @@ -21,6 +22,7 @@ def to_json_serializable(obj): else: return obj + class History: def __init__(self): self.trials: List[Trial] = [] @@ -31,23 +33,37 @@ def add(self, trial: Trial): def best(self, metric: str = "accuracy", minimize: bool = False) -> Optional[Trial]: """Return the best trial based on the given metric and minimize/maximize flag.""" - valid_trials = [t for t in self.trials if t.evaluation and metric in t.evaluation.results] + valid_trials = [ + t for t in self.trials if t.evaluation and metric in t.evaluation.results + ] if not valid_trials: return None - return min(valid_trials, key=lambda t: t.evaluation.results[metric]) if minimize \ + return ( + min(valid_trials, key=lambda t: t.evaluation.results[metric]) + if minimize else max(valid_trials, key=lambda t: t.evaluation.results[metric]) + ) - def top_k(self, k: int = 5, metric: str = "accuracy", minimize: bool = False) -> List[Trial]: + def top_k( + self, k: int = 5, metric: str = "accuracy", minimize: bool = False + ) -> List[Trial]: """Return the top k trials based on the given metric, considering minimize or maximize.""" valid_trials = [t for t in self.trials if metric in t.evaluation.results] return sorted( valid_trials, - key=lambda t: t.evaluation.results.get(metric, float('inf') if minimize else float('-inf')), - reverse=not minimize + key=lambda t: t.evaluation.results.get( + metric, float("inf") if minimize else float("-inf") + ), + reverse=not minimize, )[:k] - def summary(self, k: int = 10, metrics: List[str] = ["accuracy"], minimize_map: Optional[dict] = None): + def summary( + self, + k: int = 10, + metrics: List[str] = ["accuracy"], + minimize_map: Optional[dict] = None, + ): """Pretty print the top k trials with rich, correctly handling minimize/maximize logic.""" console = Console() table = Table(title=f"Top {k} Trials", show_lines=True) @@ -68,8 +84,16 @@ def summary(self, k: int = 10, metrics: List[str] = ["accuracy"], minimize_map: for metric in metrics: table.add_column(metric.capitalize(), justify="center") - for i, trial in enumerate(self.top_k(k, metric=primary_metric, minimize=minimize), start=1): - row = [str(i), str(trial.seed), trial.model_name, trial.optimizer_name, f"{trial.duration:.2f}"] + for i, trial in enumerate( + self.top_k(k, metric=primary_metric, minimize=minimize), start=1 + ): + row = [ + str(i), + str(trial.seed), + trial.model_name, + trial.optimizer_name, + f"{trial.duration:.2f}", + ] for metric in metrics: score = trial.evaluation.results.get(metric, "N/A") if isinstance(score, float): @@ -81,13 +105,14 @@ def summary(self, k: int = 10, metrics: List[str] = ["accuracy"], minimize_map: console.print(table) - def to_csv(self, filepath: str): """Save the trial history to a CSV file.""" if not self.trials: return - fieldnames = ["seed", "model", "optimizer", "duration"] + list(self.trials[0].evaluation.results.keys()) + fieldnames = ["seed", "model", "optimizer", "duration"] + list( + self.trials[0].evaluation.results.keys() + ) with open(filepath, mode="w", newline="") as f: writer = csv.DictWriter(f, fieldnames=fieldnames) @@ -97,51 +122,50 @@ def to_csv(self, filepath: str): "seed": trial.seed, "model": trial.model_name, "optimizer": trial.optimizer_name, - "duration": trial.duration + "duration": trial.duration, } row.update(trial.evaluation.results) writer.writerow(row) - + def _trial_to_dict(self, trial): return { "seed": trial.seed, "model_name": trial.model_name, "optimizer_name": trial.optimizer_name, "duration": trial.duration, - "evaluation": { - "results": trial.evaluation.results - } + "evaluation": {"results": trial.evaluation.results}, } - + def to_json(self, filepath: str): """Save the entire history (trials, evaluations) to a JSON file.""" - with open(filepath, 'w') as f: - json.dump([self._trial_to_dict(trial) for trial in self.trials], f, indent=4) + with open(filepath, "w") as f: + json.dump( + [self._trial_to_dict(trial) for trial in self.trials], f, indent=4 + ) + @classmethod def from_json(cls, filepath: str) -> "History": """Load history from a JSON file.""" - with open(filepath, 'r') as f: + with open(filepath, "r") as f: data = json.load(f) - + history = cls() for trial_data in data: # Deserialize the trial and evaluation evaluation = Evaluation( metric_set=MetricSet({}), # Assuming MetricSet is initialized properly - results=trial_data['evaluation'] + results=trial_data["evaluation"], ) trial = Trial( - seed=trial_data['seed'], - model_name=trial_data['model_name'], - optimizer_name=trial_data['optimizer_name'], + seed=trial_data["seed"], + model_name=trial_data["model_name"], + optimizer_name=trial_data["optimizer_name"], evaluation=evaluation, - duration=trial_data['duration'] + duration=trial_data["duration"], ) history.add(trial) return history - - if __name__ == "__main__": @@ -151,8 +175,14 @@ def from_json(cls, filepath: str) -> "History": from ezautoml.results.history import History # Define accuracy metric (maximize) - accuracy_metric = Metric(name="accuracy", fn=lambda y_true, y_pred: sum(y_true == y_pred) / len(y_true), minimize=False) - metric_set = MetricSet(metrics={"accuracy": accuracy_metric}, primary_metric_name="accuracy") + accuracy_metric = Metric( + name="accuracy", + fn=lambda y_true, y_pred: sum(y_true == y_pred) / len(y_true), + minimize=False, + ) + metric_set = MetricSet( + metrics={"accuracy": accuracy_metric}, primary_metric_name="accuracy" + ) # Dummy function to create trials def make_trial(seed, acc): @@ -162,7 +192,7 @@ def make_trial(seed, acc): model_name=f"Model_{seed}", optimizer_name="Optuna", evaluation=eval, - duration=0.01 * seed + duration=0.01 * seed, ) # Create History and add trials @@ -178,8 +208,10 @@ def make_trial(seed, acc): # Show best trial best_trial = history.best(metric="accuracy", minimize=False) if best_trial: - print(f"\nBest Trial: Model={best_trial.model_name}, Accuracy={best_trial.evaluation.results['accuracy']:.4f}") + print( + f"\nBest Trial: Model={best_trial.model_name}, Accuracy={best_trial.evaluation.results['accuracy']:.4f}" + ) # Export history - #history.to_json("history.json") - #history.to_csv("history_summary.csv") + # history.to_json("history.json") + # history.to_csv("history_summary.csv") diff --git a/src/ezautoml/results/trial.py b/src/ezautoml/results/trial.py index c331045..9cf17f6 100644 --- a/src/ezautoml/results/trial.py +++ b/src/ezautoml/results/trial.py @@ -24,10 +24,14 @@ def print_summary(self) -> None: table.add_row("Seed", str(self.seed)) table.add_row("Model", self.model_name) table.add_row("Optimizer", self.optimizer_name) - table.add_row("Evaluation", str(self.evaluation)) # Uses __str__ from Evaluation + table.add_row( + "Evaluation", str(self.evaluation) + ) # Uses __str__ from Evaluation table.add_row("Duration", f"{self.duration:.2f} seconds") - panel = Panel(table, title=f"Trial Summary (Seed: {self.seed})", title_align="left") + panel = Panel( + table, title=f"Trial Summary (Seed: {self.seed})", title_align="left" + ) Console().print(panel) def to_dict(self) -> Dict[str, Any]: @@ -48,7 +52,7 @@ def __repr__(self) -> str: # Create a dummy evaluation object dummy_results = {"accuracy": 0.912, "f1_score": 0.880} - dummy_metric_set = MetricSet(metrics={}, primary_metric_name="accuracy") + dummy_metric_set = MetricSet(metrics={}, primary_metric_name="accuracy") evaluation = Evaluation(results=dummy_results, metric_set=dummy_metric_set) # Create and display the Trial @@ -57,8 +61,8 @@ def __repr__(self) -> str: model_name="ResNet50", optimizer_name="Adam", evaluation=evaluation, - duration=420.3 + duration=420.3, ) - print(str(trial)) # Pretty summary + print(str(trial)) # Pretty summary trial.print_summary() # Rich terminal panel diff --git a/src/ezautoml/scripts/generate_spaces.py b/src/ezautoml/scripts/generate_spaces.py index cd43f41..3972e30 100644 --- a/src/ezautoml/scripts/generate_spaces.py +++ b/src/ezautoml/scripts/generate_spaces.py @@ -2,7 +2,7 @@ from ezautoml.evaluation.task import TaskType from ezautoml.space.component import Component, Tag from ezautoml.space.search_space import SearchSpace -from ezautoml.space.hyperparam import Hyperparam +from ezautoml.space.hyperparam import Hyperparam from ezautoml.space.space import Integer, Real, Categorical from ezautoml.registry import constructor_registry @@ -11,19 +11,35 @@ # Define models by task # ----------------------------- classification_model_names = [ - "RandomForestClassifier", "GradientBoostingClassifier", "LogisticRegression", - "KNeighborsClassifier", "DecisionTreeClassifier", "GaussianNB", - "AdaBoostClassifier", "BaggingClassifier", "ExtraTreesClassifier", - "XGBClassifier", "LGBMClassifier", - #"SVC" + "RandomForestClassifier", + "GradientBoostingClassifier", + "LogisticRegression", + "KNeighborsClassifier", + "DecisionTreeClassifier", + "GaussianNB", + "AdaBoostClassifier", + "BaggingClassifier", + "ExtraTreesClassifier", + "XGBClassifier", + "LGBMClassifier", + # "SVC" ] regression_model_names = [ - "RandomForestRegressor", "GradientBoostingRegressor", "Ridge", "Lasso", - "ElasticNet", "LinearRegression", "KNeighborsRegressor", - "DecisionTreeRegressor", "XGBRegressor", "AdaBoostRegressor", - "BaggingRegressor", "ExtraTreesRegressor", "LGBMRegressor", - #"SVR", + "RandomForestRegressor", + "GradientBoostingRegressor", + "Ridge", + "Lasso", + "ElasticNet", + "LinearRegression", + "KNeighborsRegressor", + "DecisionTreeRegressor", + "XGBRegressor", + "AdaBoostRegressor", + "BaggingRegressor", + "ExtraTreesRegressor", + "LGBMRegressor", + # "SVR", ] # ----------------------------- @@ -33,9 +49,15 @@ ("no_data_proc", "NoDataProcessing", TaskType.BOTH, Tag.DATA_PROCESSING), ("no_feat_proc", "NoFeatureProcessing", TaskType.BOTH, Tag.FEATURE_PROCESSING), ("no_feat_eng", "NoFeatureEngineering", TaskType.BOTH, Tag.FEATURE_ENGINEERING), - ("no_opt_alg", "NoOptimizationAlgSelection", TaskType.BOTH, Tag.OPTIMIZATION_ALGORITHM_SELECTION) + ( + "no_opt_alg", + "NoOptimizationAlgSelection", + TaskType.BOTH, + Tag.OPTIMIZATION_ALGORITHM_SELECTION, + ), ] + # ----------------------------- # Helper functions to get registered components # ----------------------------- @@ -53,28 +75,30 @@ def get_registered_components(model_names, task): Hyperparam("n_estimators", Integer(10, 1000)), Hyperparam("max_depth", Integer(1, 50)), Hyperparam("min_samples_split", Integer(2, 20)), - Hyperparam("min_samples_leaf", Integer(1, 10)) + Hyperparam("min_samples_leaf", Integer(1, 10)), ] boosting_common = [ Hyperparam("n_estimators", Integer(10, 1000)), Hyperparam("learning_rate", Real(0.01, 0.5)), - Hyperparam("max_depth", Integer(1, 50)) + Hyperparam("max_depth", Integer(1, 50)), ] bagging_common = [ Hyperparam("n_estimators", Integer(10, 100)), Hyperparam("max_samples", Real(0.1, 1.0)), - Hyperparam("max_features", Real(0.1, 1.0)) + Hyperparam("max_features", Real(0.1, 1.0)), ] if name in ["RandomForestClassifier", "RandomForestRegressor"]: - hyperparams = rf_tree_common + [Hyperparam("max_features", Categorical(["sqrt", "log2", None]))] + hyperparams = rf_tree_common + [ + Hyperparam("max_features", Categorical(["sqrt", "log2", None])) + ] elif name in ["GradientBoostingClassifier", "GradientBoostingRegressor"]: hyperparams = boosting_common + [Hyperparam("subsample", Real(0.5, 1.0))] elif name == "LogisticRegression": hyperparams = [ Hyperparam("C", Real(1e-4, 100.0)), Hyperparam("max_iter", Integer(100, 1000)), - Hyperparam("penalty", Categorical(["l2"])) + Hyperparam("penalty", Categorical(["l2"])), ] elif name in ["Ridge"]: hyperparams = [ @@ -83,13 +107,13 @@ def get_registered_components(model_names, task): elif name == "Lasso": hyperparams = [ Hyperparam("alpha", Real(1e-4, 10.0)), - Hyperparam("max_iter", Integer(100, 1000)) + Hyperparam("max_iter", Integer(100, 1000)), ] elif name == "ElasticNet": hyperparams = [ Hyperparam("alpha", Real(1e-4, 10.0)), Hyperparam("l1_ratio", Real(0.0, 1.0)), - Hyperparam("max_iter", Integer(100, 1000)) + Hyperparam("max_iter", Integer(100, 1000)), ] elif name == "LinearRegression": hyperparams = [] @@ -112,14 +136,26 @@ def get_registered_components(model_names, task): Hyperparam("n_neighbors", Integer(1, 20)), Hyperparam("weights", Categorical(["uniform", "distance"])), Hyperparam("leaf_size", Integer(10, 100)), - Hyperparam("p", Integer(1, 2)) + Hyperparam("p", Integer(1, 2)), ] elif name in ["DecisionTreeClassifier", "DecisionTreeRegressor"]: hyperparams = [ - Hyperparam("criterion", Categorical(["gini", "entropy", "log_loss"] if "Classifier" in name else ["squared_error", "friedman_mse", "absolute_error", "poisson"])), + Hyperparam( + "criterion", + Categorical( + ["gini", "entropy", "log_loss"] + if "Classifier" in name + else [ + "squared_error", + "friedman_mse", + "absolute_error", + "poisson", + ] + ), + ), Hyperparam("max_depth", Integer(1, 50)), Hyperparam("min_samples_split", Integer(2, 20)), - Hyperparam("min_samples_leaf", Integer(1, 10)) + Hyperparam("min_samples_leaf", Integer(1, 10)), ] elif name == "GaussianNB": hyperparams = [] @@ -155,17 +191,19 @@ def get_registered_components(model_names, task): Hyperparam("colsample_bytree", Real(0.5, 1.0)), # feature fraction Hyperparam("reg_alpha", Real(0.0, 10.0)), # L1 regularization Hyperparam("reg_lambda", Real(0.0, 10.0)), # L2 regularization - ] + ] else: hyperparams = [] - components.append(Component( - name=name, - constructor=constructor, - task=task, - tag=Tag.MODEL_SELECTION, - hyperparams=hyperparams - )) + components.append( + Component( + name=name, + constructor=constructor, + task=task, + tag=Tag.MODEL_SELECTION, + hyperparams=hyperparams, + ) + ) return components @@ -175,34 +213,43 @@ def get_null_components(): for name, registry_name, task, tag in null_components: if constructor_registry.has(registry_name): constructor = constructor_registry.get(registry_name) - components.append(Component(name=name, constructor=constructor, task=task, tag=tag)) + components.append( + Component(name=name, constructor=constructor, task=task, tag=tag) + ) return components + # ----------------------------- # Build model, data, and feature components for each task # ----------------------------- -classification_models = get_registered_components(classification_model_names, TaskType.CLASSIFICATION) -regression_models = get_registered_components(regression_model_names, TaskType.REGRESSION) +classification_models = get_registered_components( + classification_model_names, TaskType.CLASSIFICATION +) +regression_models = get_registered_components( + regression_model_names, TaskType.REGRESSION +) # **Only include ONE null component for each task** data_processors = [get_null_components()[0]] # Only the NoDataProcessing component -feature_processors = [get_null_components()[1]] # Only the NoFeatureProcessing component +feature_processors = [ + get_null_components()[1] +] # Only the NoFeatureProcessing component # ----------------------------- # Build search spaces with models and hyperparameters # ----------------------------- classification_space = SearchSpace( - models=classification_models, # Only the classification models with hyperparameters - data_processors=data_processors, # Just one data processor - feature_processors=feature_processors, # Just one feature processor - task=TaskType.CLASSIFICATION + models=classification_models, # Only the classification models with hyperparameters + data_processors=data_processors, # Just one data processor + feature_processors=feature_processors, # Just one feature processor + task=TaskType.CLASSIFICATION, ) regression_space = SearchSpace( - models=regression_models, # Only the regression models with hyperparameters - data_processors=data_processors, # Just one data processor - feature_processors=feature_processors, # Just one feature processor - task=TaskType.REGRESSION + models=regression_models, # Only the regression models with hyperparameters + data_processors=data_processors, # Just one data processor + feature_processors=feature_processors, # Just one feature processor + task=TaskType.REGRESSION, ) @@ -212,4 +259,6 @@ def get_null_components(): if serialize: regression_space.to_yaml(path="./ezautoml/resources/spaces/regression_space.yaml") - classification_space.to_yaml(path="./ezautoml/resources/spaces/classification_space.yaml") + classification_space.to_yaml( + path="./ezautoml/resources/spaces/classification_space.yaml" + ) diff --git a/src/ezautoml/space/component.py b/src/ezautoml/space/component.py index 41c8af2..44c4d35 100644 --- a/src/ezautoml/space/component.py +++ b/src/ezautoml/space/component.py @@ -7,7 +7,7 @@ from ezautoml.evaluation.task import TaskType -from ezautoml.space.space import * +from ezautoml.space.space import * from ezautoml.space.hyperparam import Hyperparam from ezautoml.registry import constructor_registry @@ -24,6 +24,7 @@ # To avoid doing subclasses for the moment # TODO: use anything other than MODEL_SELECTION + # These are the common 5 steps to automate in AutoML class Tag(Enum): MODEL_SELECTION = "model_selection" @@ -31,7 +32,8 @@ class Tag(Enum): FEATURE_PROCESSING = "feature_processing" DATA_PROCESSING = "data_processing" OPTIMIZATION_ALGORITHM_SELECTION = "optimization_algorithm_selection" - + + class Component: def __init__( self, @@ -40,15 +42,17 @@ def __init__( tag: Tag, hyperparams: List[Hyperparam] = None, task: TaskType = TaskType.BOTH, - validate_interface: bool = True + validate_interface: bool = True, ): if not callable(constructor): raise ValueError(f"Constructor must be callable, got {constructor}") if not constructor_registry.has(constructor.__name__): logger.info(constructor_registry) - raise ValueError(f"Constructor '{constructor.__name__}' is not registered in constructor_registry.") - + raise ValueError( + f"Constructor '{constructor.__name__}' is not registered in constructor_registry." + ) + self.name = name self.constructor = constructor self.hyperparams = hyperparams or [] @@ -64,12 +68,18 @@ def _validate_interface(self): if self.tag == Tag.MODEL_SELECTION: required_methods = ["fit", "predict"] # optionally "predict_proba" - elif self.tag in [Tag.DATA_PROCESSING, Tag.FEATURE_PROCESSING, Tag.FEATURE_ENGINEERING]: + elif self.tag in [ + Tag.DATA_PROCESSING, + Tag.FEATURE_PROCESSING, + Tag.FEATURE_ENGINEERING, + ]: required_methods = ["fit", "transform"] elif self.tag == Tag.OPTIMIZATION_ALGORITHM_SELECTION: required_methods = [] - missing = [method for method in required_methods if not hasattr(instance, method)] + missing = [ + method for method in required_methods if not hasattr(instance, method) + ] if missing: raise TypeError( f"Component '{self.name}' of tag '{self.tag.name}' is missing required methods: {missing}" @@ -100,7 +110,13 @@ def from_dict(cls, data: dict): hyperparams = [Hyperparam.from_dict(hp) for hp in data.get("hyperparams", [])] task = TaskType(data["task"]) tag = Tag(data.get("tag", Tag.MODEL_SELECTION.value)) - sus = cls(data["name"], constructor=constructor,tag=tag, hyperparams=hyperparams, task=task) + sus = cls( + data["name"], + constructor=constructor, + tag=tag, + hyperparams=hyperparams, + task=task, + ) return sus def __str__(self): @@ -112,7 +128,8 @@ def __str__(self): f"constructor='{self.constructor.__name__}', " f"hyperparams=[{', '.join(hyperparam_strs)}])" ) - + + if __name__ == "__main__": from sklearn.ensemble import RandomForestClassifier from sklearn.linear_model import LogisticRegression @@ -122,23 +139,40 @@ def __str__(self): # Define hyperparameters for RandomForest rf_params = [ Hyperparam("n_estimators", Integer(50, 150)), - Hyperparam("max_depth", Integer(5, 20)) + Hyperparam("max_depth", Integer(5, 20)), ] # Define hyperparameters for LogisticRegression lr_params = [ Hyperparam("C", Real(0.01, 10)), - Hyperparam("penalty", Categorical(["l2", "none"])) + Hyperparam("penalty", Categorical(["l2", "none"])), ] # Model components - rf_component = Component("RandomForest", tag=Tag.MODEL_SELECTION, constructor=RandomForestClassifier,hyperparams=rf_params) - lr_component = Component("LogisticRegression",tag=Tag.MODEL_SELECTION, constructor=LogisticRegression, hyperparams=lr_params) + rf_component = Component( + "RandomForest", + tag=Tag.MODEL_SELECTION, + constructor=RandomForestClassifier, + hyperparams=rf_params, + ) + lr_component = Component( + "LogisticRegression", + tag=Tag.MODEL_SELECTION, + constructor=LogisticRegression, + hyperparams=lr_params, + ) # Feature processors - pca_component = Component("PCA",tag=Tag.FEATURE_PROCESSING, constructor=PCA, hyperparams=[Hyperparam("n_components", Real(0.1, 0.95))]) + pca_component = Component( + "PCA", + tag=Tag.FEATURE_PROCESSING, + constructor=PCA, + hyperparams=[Hyperparam("n_components", Real(0.1, 0.95))], + ) # Data processors - scaler_component = Component("StandardScaler", tag=Tag.DATA_PROCESSING, constructor=StandardScaler) + scaler_component = Component( + "StandardScaler", tag=Tag.DATA_PROCESSING, constructor=StandardScaler + ) # Manually simulate a SearchSpace sampling all_models = [rf_component, lr_component] diff --git a/src/ezautoml/space/hyperparam.py b/src/ezautoml/space/hyperparam.py index 2fbe691..34292ef 100644 --- a/src/ezautoml/space/hyperparam.py +++ b/src/ezautoml/space/hyperparam.py @@ -6,15 +6,18 @@ # Space # # # # This abstract class defines ranges for hyperparameters of different types: # -# Integer numbers (Natural, Integer), Real and Categorical values which can be# +# Integer numbers (Natural, Integer), Real and Categorical values which can be# # used to define the whole program search space # # Author: Walter J.T.V # # ===----------------------------------------------------------------------===# + class Hyperparam: def __init__(self, name: str, space: Space): self.name = name - self.space = space # Space defines the range (could be Categorical, Integer, or Real) + self.space = ( + space # Space defines the range (could be Categorical, Integer, or Real) + ) def sample(self) -> Union[str, int, float]: """Sample a value from the hyperparameter space.""" @@ -22,28 +25,26 @@ def sample(self) -> Union[str, int, float]: def to_dict(self) -> dict: """Serialize a hyperparameter to a dictionary, using the space's to_dict.""" - return { - 'name': self.name, - 'space': self.space.to_dict() - } + return {"name": self.name, "space": self.space.to_dict()} @classmethod - def from_dict(cls, data: dict) -> 'Hyperparam': - space_type = data['space']['type'] - space_data = data['space'] - if space_type == 'Integer': - return cls(data['name'], Integer(space_data['low'], space_data['high'])) - elif space_type == 'Real': - return cls(data['name'], Real(space_data['low'], space_data['high'])) - elif space_type == 'Categorical': - return cls(data['name'], Categorical(space_data['categories'])) + def from_dict(cls, data: dict) -> "Hyperparam": + space_type = data["space"]["type"] + space_data = data["space"] + if space_type == "Integer": + return cls(data["name"], Integer(space_data["low"], space_data["high"])) + elif space_type == "Real": + return cls(data["name"], Real(space_data["low"], space_data["high"])) + elif space_type == "Categorical": + return cls(data["name"], Categorical(space_data["categories"])) return None - + def __str__(self): return f"Hyperparam(name={self.name}, space={self.space})" - - + # Example of defining a simple search space + + if __name__ == "__main__": # Define some hyperparameters hyperparameters = [ @@ -66,4 +67,4 @@ def sample_search_space(hyperparameters): # Serialize the hyperparameters to a dictionary hyperparam_dicts = [hp.to_dict() for hp in hyperparameters] - print("Serialized Hyperparameters:", hyperparam_dicts) \ No newline at end of file + print("Serialized Hyperparameters:", hyperparam_dicts) diff --git a/src/ezautoml/space/search_point.py b/src/ezautoml/space/search_point.py index 73359ef..ec2c025 100644 --- a/src/ezautoml/space/search_point.py +++ b/src/ezautoml/space/search_point.py @@ -1,9 +1,7 @@ - - from typing import Dict, Any, List, Optional import yaml from ezautoml.space.component import Component -from ezautoml.results.trial import Trial +from ezautoml.results.trial import Trial from ezautoml.space.hyperparam import Hyperparam # ===----------------------------------------------------------------------===# @@ -18,7 +16,7 @@ class SearchPoint: def __init__( self, model: Component, - model_params: Dict[str,Hyperparam], + model_params: Dict[str, Hyperparam], data_processors: Optional[List[Component]] = None, data_params_list: Optional[List[Dict[str, Any]]] = None, feature_processors: Optional[List[Component]] = None, @@ -31,10 +29,12 @@ def __init__( self.data_params_list = data_params_list or [{} for _ in self.data_processors] self.feature_processors = feature_processors or [] - self.feature_params_list = feature_params_list or [{} for _ in self.feature_processors] + self.feature_params_list = feature_params_list or [ + {} for _ in self.feature_processors + ] # stores the evaluation result - self.result: Optional[Trial] = None + self.result: Optional[Trial] = None def instantiate_pipeline(self): """ @@ -64,8 +64,10 @@ def describe(self) -> Dict[str, Any]: ], "feature_processors": [ {"name": proc.name, "params": params} - for proc, params in zip(self.feature_processors, self.feature_params_list) - ] + for proc, params in zip( + self.feature_processors, self.feature_params_list + ) + ], } def to_dict(self) -> Dict[str, Any]: @@ -79,7 +81,7 @@ def to_yaml(self, path: str) -> None: yaml.dump(self.to_dict(), f) @staticmethod - def from_yaml(path: str, components: List[Component]) -> 'SearchPoint': + def from_yaml(path: str, components: List[Component]) -> "SearchPoint": with open(path, "r") as f: data = yaml.safe_load(f) @@ -89,11 +91,17 @@ def find_component(name): model = find_component(data["model"]) model_params = data["model_params"] - data_processors = [find_component(dp["name"]) for dp in data.get("data_processors", [])] + data_processors = [ + find_component(dp["name"]) for dp in data.get("data_processors", []) + ] data_params_list = [dp["params"] for dp in data.get("data_processors", [])] - feature_processors = [find_component(fp["name"]) for fp in data.get("feature_processors", [])] - feature_params_list = [fp["params"] for fp in data.get("feature_processors", [])] + feature_processors = [ + find_component(fp["name"]) for fp in data.get("feature_processors", []) + ] + feature_params_list = [ + fp["params"] for fp in data.get("feature_processors", []) + ] sp = SearchPoint( model=model, @@ -101,7 +109,7 @@ def find_component(name): data_processors=data_processors, data_params_list=data_params_list, feature_processors=feature_processors, - feature_params_list=feature_params_list + feature_params_list=feature_params_list, ) if "result" in data: @@ -109,11 +117,11 @@ def find_component(name): return sp - def __str__(self): desc = self.describe() return yaml.dump(desc, sort_keys=False) + if __name__ == "__main__": import yaml from sklearn.ensemble import RandomForestClassifier @@ -133,7 +141,7 @@ def __str__(self): [ Hyperparam("n_estimators", Integer(10, 100)), Hyperparam("max_depth", Integer(3, 10)), - ] + ], ) scaler = Component("StandardScaler", StandardScaler, []) @@ -143,7 +151,7 @@ def __str__(self): PCA, [ Hyperparam("n_components", Real(0.5, 0.99)), - ] + ], ) # Sample hyperparameters @@ -167,7 +175,7 @@ def __str__(self): model_name=model.name, optimizer_name="RandomSearch", evaluation={"accuracy": 0.87, "f1_score": 0.84}, - duration=12.3 + duration=12.3, ) # Serialize to YAML @@ -183,4 +191,4 @@ def __str__(self): if loaded.result: print("\nRestored Trial:") - print(loaded.result) \ No newline at end of file + print(loaded.result) diff --git a/src/ezautoml/space/search_space.py b/src/ezautoml/space/search_space.py index b9f63e2..00716fd 100644 --- a/src/ezautoml/space/search_space.py +++ b/src/ezautoml/space/search_space.py @@ -1,5 +1,3 @@ - - # ===----------------------------------------------------------------------===# # Search Space # # # @@ -18,6 +16,7 @@ from ezautoml.evaluation.task import TaskType from ezautoml.space.search_point import SearchPoint from ezautoml.space.component import Component + # Default search spaces serialized import ezautoml.resources.spaces as spaces @@ -32,11 +31,16 @@ def __init__( ): if not models: raise ValueError("SearchSpace must include at least one model.") - + # Validate unique component names - all_names = [c.name for c in models + (data_processors or []) + (feature_processors or [])] + all_names = [ + c.name + for c in models + (data_processors or []) + (feature_processors or []) + ] if len(all_names) != len(set(all_names)): - raise ValueError("Component names must be unique across all component lists.") + raise ValueError( + "Component names must be unique across all component lists." + ) self.models = models self.data_processors = data_processors or [] @@ -54,9 +58,13 @@ def sample(self, seed: Optional[int] = None) -> SearchPoint: data_proc_list = [] data_params_list = [] if self.data_processors: - compatible_data = [d for d in self.data_processors if d.is_compatible(self.task)] + compatible_data = [ + d for d in self.data_processors if d.is_compatible(self.task) + ] if not compatible_data: - raise ValueError(f"No compatible data processors found for task: {self.task}") + raise ValueError( + f"No compatible data processors found for task: {self.task}" + ) selected_data_proc = rng.choice(compatible_data) data_proc_list = [selected_data_proc] data_params_list = [selected_data_proc.sample_params()] @@ -64,9 +72,13 @@ def sample(self, seed: Optional[int] = None) -> SearchPoint: feat_proc_list = [] feat_params_list = [] if self.feature_processors: - compatible_feat = [f for f in self.feature_processors if f.is_compatible(self.task)] + compatible_feat = [ + f for f in self.feature_processors if f.is_compatible(self.task) + ] if not compatible_feat: - raise ValueError(f"No compatible feature processors found for task: {self.task}") + raise ValueError( + f"No compatible feature processors found for task: {self.task}" + ) selected_feat_proc = rng.choice(compatible_feat) feat_proc_list = [selected_feat_proc] feat_params_list = [selected_feat_proc.sample_params()] @@ -98,21 +110,24 @@ def to_yaml(self, path: str) -> None: yaml.dump(full_dict, f) @staticmethod - def from_yaml(path: str) -> 'SearchSpace': + def from_yaml(path: str) -> "SearchSpace": with open(path, "r") as f: data = yaml.safe_load(f) models = [Component.from_dict(d) for d in data["models"]] data_procs = [Component.from_dict(d) for d in data.get("data_processors", [])] - feat_procs = [Component.from_dict(d) for d in data.get("feature_processors", [])] + feat_procs = [ + Component.from_dict(d) for d in data.get("feature_processors", []) + ] task = TaskType[data["task"].upper()] return SearchSpace(models, data_procs, feat_procs, task) - + @staticmethod - def from_builtin(name: str) -> 'SearchSpace': + def from_builtin(name: str) -> "SearchSpace": """Load a built-in YAML search space by name.""" import ezautoml.resources.spaces as spaces + # Using importlib.resources.files() to read the YAML file from package resources with importlib.resources.files(spaces).joinpath(f"{name}.yaml").open("r") as f: data = yaml.safe_load(f) @@ -120,16 +135,26 @@ def from_builtin(name: str) -> 'SearchSpace': # Parse components from the YAML data models = [Component.from_dict(d) for d in data["models"]] data_procs = [Component.from_dict(d) for d in data.get("data_processors", [])] - feat_procs = [Component.from_dict(d) for d in data.get("feature_processors", [])] + feat_procs = [ + Component.from_dict(d) for d in data.get("feature_processors", []) + ] task = TaskType[data["task"].upper()] # Return the SearchSpace instance return SearchSpace(models, data_procs, feat_procs, task) def __str__(self): - models_str = ', '.join([str(model) for model in self.models]) - data_processors_str = ', '.join([str(dp) for dp in self.data_processors]) if self.data_processors else "None" - feature_processors_str = ', '.join([str(fp) for fp in self.feature_processors]) if self.feature_processors else "None" + models_str = ", ".join([str(model) for model in self.models]) + data_processors_str = ( + ", ".join([str(dp) for dp in self.data_processors]) + if self.data_processors + else "None" + ) + feature_processors_str = ( + ", ".join([str(fp) for fp in self.feature_processors]) + if self.feature_processors + else "None" + ) return ( f"SearchSpace(task={self.task.name}, " f"models=[{models_str}], " @@ -137,9 +162,10 @@ def __str__(self): f"feature_processors=[{feature_processors_str}])" ) + if __name__ == "__main__": # Load a built-in search space by name search_space = SearchSpace.from_builtin("regression_space") # Print out the loaded search space - print(f"Loaded SearchSpace: {search_space}") \ No newline at end of file + print(f"Loaded SearchSpace: {search_space}") diff --git a/src/ezautoml/space/space.py b/src/ezautoml/space/space.py index 24f66c9..5c05918 100644 --- a/src/ezautoml/space/space.py +++ b/src/ezautoml/space/space.py @@ -17,7 +17,7 @@ class Space(ABC): def sample(self): """Sample a value from the space.""" pass - + @abstractmethod def to_dict(self): """Serialize the space to a dictionary.""" @@ -26,9 +26,12 @@ def to_dict(self): def __repr__(self): return f"{self.__class__.__name__}()" + class Categorical(Space): def __init__(self, categories): - assert isinstance(categories, list) and len(categories) > 0, "Must provide a non-empty list of categories." + assert ( + isinstance(categories, list) and len(categories) > 0 + ), "Must provide a non-empty list of categories." self.categories = categories def contains(self, value): @@ -39,17 +42,17 @@ def sample(self) -> str: def to_dict(self): """Serialize the Categorical space to a dictionary.""" - return { - 'type': 'Categorical', - 'categories': self.categories - } + return {"type": "Categorical", "categories": self.categories} def __str__(self): return f"Categorical({self.categories})" + class Integer(Space): def __init__(self, low, high): - assert isinstance(low, int) and isinstance(high, int), "Bounds must be integers." + assert isinstance(low, int) and isinstance( + high, int + ), "Bounds must be integers." assert low <= high, f"Invalid Integer bounds: low={low}, high={high}" self.low = low self.high = high @@ -59,18 +62,17 @@ def sample(self) -> int: def to_dict(self): """Serialize the Integer space to a dictionary.""" - return { - 'type': 'Integer', - 'low': self.low, - 'high': self.high - } + return {"type": "Integer", "low": self.low, "high": self.high} def __str__(self): return f"Integer({self.low}, {self.high})" + class Real(Space): def __init__(self, low, high): - assert isinstance(low, (int, float)) and isinstance(high, (int, float)), "Bounds must be numeric." + assert isinstance(low, (int, float)) and isinstance( + high, (int, float) + ), "Bounds must be numeric." assert low <= high, f"Invalid Real bounds: low={low}, high={high}" self.low = low self.high = high @@ -80,11 +82,7 @@ def sample(self) -> float: def to_dict(self): """Serialize the Real space to a dictionary.""" - return { - 'type': 'Real', - 'low': self.low, - 'high': self.high - } + return {"type": "Real", "low": self.low, "high": self.high} def __str__(self): return f"Real({self.low}, {self.high})" @@ -107,6 +105,5 @@ def sample_search_space(search_space): sampled_params.append(space.sample()) return sampled_params - sampled_point = sample_search_space(search_space) print(sampled_point)