From 29c03b10d3c0648021437f9383fe9a9ac3ff6f72 Mon Sep 17 00:00:00 2001 From: MonikaDvorackova Date: Tue, 20 Oct 2026 13:50:12 +0200 Subject: [PATCH 1/2] perf: lazily import evaluate package components --- benchmarks/import_time.py | 30 ++++++++++ src/evaluate/__init__.py | 116 ++++++++++++++++++++++++++++++------- tests/test_lazy_imports.py | 51 ++++++++++++++++ 3 files changed, 175 insertions(+), 22 deletions(-) create mode 100644 benchmarks/import_time.py create mode 100644 tests/test_lazy_imports.py diff --git a/benchmarks/import_time.py b/benchmarks/import_time.py new file mode 100644 index 000000000..1a28bc91c --- /dev/null +++ b/benchmarks/import_time.py @@ -0,0 +1,30 @@ +import statistics +import subprocess +import sys +import time + +STATEMENTS = [ + "import evaluate", + "from evaluate import load", + "from evaluate import EvaluationSuite", + "from evaluate import evaluator", +] + +for statement in STATEMENTS: + measurements = [] + + for _ in range(7): + start = time.perf_counter() + subprocess.run( + [sys.executable, "-c", statement], + check=True, + stdout=subprocess.DEVNULL, + ) + measurements.append(time.perf_counter() - start) + + print( + f"{statement:40} " + f"median={statistics.median(measurements):.3f}s " + f"min={min(measurements):.3f}s " + f"max={max(measurements):.3f}s" + ) diff --git a/src/evaluate/__init__.py b/src/evaluate/__init__.py index a8c25bd92..5979f4401 100644 --- a/src/evaluate/__init__.py +++ b/src/evaluate/__init__.py @@ -26,26 +26,98 @@ del version -from .evaluation_suite import EvaluationSuite -from .evaluator import ( - AudioClassificationEvaluator, - AutomaticSpeechRecognitionEvaluator, - Evaluator, - ImageClassificationEvaluator, - QuestionAnsweringEvaluator, - SummarizationEvaluator, - Text2TextGenerationEvaluator, - TextClassificationEvaluator, - TextGenerationEvaluator, - TokenClassificationEvaluator, - TranslationEvaluator, - evaluator, + +import sys +from importlib import import_module +from types import ModuleType + +_LAZY_IMPORTS = { + "AudioClassificationEvaluator": "evaluator", + "AutomaticSpeechRecognitionEvaluator": "evaluator", + "CombinedEvaluations": "module", + "Comparison": "module", + "ComparisonInfo": "info", + "EvaluationModule": "module", + "EvaluationModuleInfo": "info", + "EvaluationSuite": "evaluation_suite", + "Evaluator": "evaluator", + "ImageClassificationEvaluator": "evaluator", + "Measurement": "module", + "MeasurementInfo": "info", + "Metric": "module", + "MetricInfo": "info", + "QuestionAnsweringEvaluator": "evaluator", + "SummarizationEvaluator": "evaluator", + "Text2TextGenerationEvaluator": "evaluator", + "TextClassificationEvaluator": "evaluator", + "TextGenerationEvaluator": "evaluator", + "TokenClassificationEvaluator": "evaluator", + "TranslationEvaluator": "evaluator", + "combine": "module", + "disable_progress_bar": "utils.logging", + "enable_progress_bar": "utils.logging", + "infer_gradio_input_types": "utils.gradio", + "inspect_evaluation_module": "inspect", + "is_progress_bar_enabled": "utils.logging", + "json_to_string_type": "utils.gradio", + "launch_gradio_widget": "utils.gradio", + "list_evaluation_modules": "inspect", + "load": "loading", + "parse_gradio_data": "utils.gradio", + "parse_readme": "utils.gradio", + "parse_test_cases": "utils.gradio", + "push_to_hub": "hub", + "save": "saving", +} + + +_LAZY_MODULES = { + "config": "config", + "evaluation_suite": "evaluation_suite", + "gradio": "utils.gradio", + "hub": "hub", + "info": "info", + "inspect": "inspect", + "loading": "loading", + "logging": "utils.logging", + "module": "module", + "naming": "naming", + "saving": "saving", + "utils": "utils", +} + +__all__ = sorted( + set(_LAZY_IMPORTS) | set(_LAZY_MODULES) | {"evaluator", "SCRIPTS_VERSION"} ) -from .hub import push_to_hub -from .info import ComparisonInfo, EvaluationModuleInfo, MeasurementInfo, MetricInfo -from .inspect import inspect_evaluation_module, list_evaluation_modules -from .loading import load -from .module import CombinedEvaluations, Comparison, EvaluationModule, Measurement, Metric, combine -from .saving import save -from .utils import * -from .utils import gradio, logging + + +def __getattr__(name): + if name in _LAZY_IMPORTS: + module = import_module("." + _LAZY_IMPORTS[name], __name__) + value = getattr(module, name) + globals()[name] = value + return value + if name in _LAZY_MODULES: + module = import_module("." + _LAZY_MODULES[name], __name__) + globals()[name] = module + return module + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + +def __dir__(): + return sorted(set(globals()) | set(_LAZY_IMPORTS)) + + +class _EvaluateModule(ModuleType): + def __getattribute__(self, name): + if name == "evaluator": + namespace = ModuleType.__getattribute__(self, "__dict__") + value = namespace.get("evaluator") + if isinstance(value, ModuleType) or value is None: + value = import_module(".evaluator", __name__).evaluator + namespace["evaluator"] = value + return value + return ModuleType.__getattribute__(self, name) + + +sys.modules[__name__].__class__ = _EvaluateModule diff --git a/tests/test_lazy_imports.py b/tests/test_lazy_imports.py new file mode 100644 index 000000000..818049673 --- /dev/null +++ b/tests/test_lazy_imports.py @@ -0,0 +1,51 @@ +import subprocess +import sys + + +def run_python(code): + return subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + + +def test_package_import_does_not_load_heavy_dependencies(): + output = run_python( + "import evaluate, sys; " + "print(any(name in sys.modules for name in " + "('datasets', 'pandas', 'pyarrow', 'transformers', 'torch')))" + ) + assert output == "False" + + +def test_public_api_exports(): + output = run_python( + "import evaluate; " + "assert callable(evaluate.load); " + "assert callable(evaluate.evaluator); " + "assert isinstance(evaluate.EvaluationSuite, type); " + "print('OK')" + ) + assert output == "OK" + + +def test_evaluator_remains_callable_after_submodule_import(): + output = run_python( + "import evaluate, importlib; " + "importlib.import_module('evaluate.evaluator'); " + "print(callable(evaluate.evaluator))" + ) + assert output == "True" + + +def test_wildcard_import(): + output = run_python( + "from evaluate import *; " + "assert callable(load); " + "assert callable(evaluator); " + "assert isinstance(EvaluationSuite, type); " + "print('OK')" + ) + assert output == "OK" From b6c113657a122b885b4b8ed05b7f4a9ed413b480 Mon Sep 17 00:00:00 2001 From: MonikaDvorackova Date: Wed, 21 Oct 2026 21:52:13 +0200 Subject: [PATCH 2/2] test: cover legacy public API and wildcard imports --- tests/test_lazy_imports.py | 68 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/tests/test_lazy_imports.py b/tests/test_lazy_imports.py index 818049673..8b80bba36 100644 --- a/tests/test_lazy_imports.py +++ b/tests/test_lazy_imports.py @@ -49,3 +49,71 @@ def test_wildcard_import(): "print('OK')" ) assert output == "OK" + + +def test_legacy_public_exports(): + import evaluate + + legacy_names = { + "AudioClassificationEvaluator", + "AutomaticSpeechRecognitionEvaluator", + "CombinedEvaluations", + "Comparison", + "ComparisonInfo", + "EvaluationModule", + "EvaluationModuleInfo", + "EvaluationSuite", + "Evaluator", + "ImageClassificationEvaluator", + "Measurement", + "MeasurementInfo", + "Metric", + "MetricInfo", + "QuestionAnsweringEvaluator", + "SummarizationEvaluator", + "Text2TextGenerationEvaluator", + "TextClassificationEvaluator", + "TextGenerationEvaluator", + "TokenClassificationEvaluator", + "TranslationEvaluator", + "combine", + "config", + "disable_progress_bar", + "enable_progress_bar", + "evaluation_suite", + "evaluator", + "gradio", + "hub", + "infer_gradio_input_types", + "info", + "inspect", + "inspect_evaluation_module", + "is_progress_bar_enabled", + "json_to_string_type", + "launch_gradio_widget", + "list_evaluation_modules", + "load", + "loading", + "logging", + "module", + "naming", + "parse_gradio_data", + "parse_readme", + "parse_test_cases", + "push_to_hub", + "save", + "saving", + "utils", + } + + assert all(hasattr(evaluate, name) for name in legacy_names) + + +def test_wildcard_exports_match_all(): + import evaluate + + namespace = {} + exec("from evaluate import *", namespace) + + exported = set(namespace) - {"__builtins__"} + assert exported == set(evaluate.__all__)