Speed up evaluate imports with lazy loading

benchmarks/import_time.py

@@ -0,0 +1,30 @@
+import statistics
+import subprocess
+import sys
+import time
+
+STATEMENTS = [
+    "import evaluate",
+    "from evaluate import load",
+    "from evaluate import EvaluationSuite",
+    "from evaluate import evaluator",
+]
+
+for statement in STATEMENTS:
+    measurements = []
+
+    for _ in range(7):
+        start = time.perf_counter()
+        subprocess.run(
+            [sys.executable, "-c", statement],
+            check=True,
+            stdout=subprocess.DEVNULL,
+        )
+        measurements.append(time.perf_counter() - start)
+
+    print(
+        f"{statement:40} "
+        f"median={statistics.median(measurements):.3f}s "
+        f"min={min(measurements):.3f}s "
+        f"max={max(measurements):.3f}s"
+    )

src/evaluate/__init__.py

@@ -26,26 +26,98 @@
 
 del version
 
-from .evaluation_suite import EvaluationSuite
-from .evaluator import (
-    AudioClassificationEvaluator,
-    AutomaticSpeechRecognitionEvaluator,
-    Evaluator,
-    ImageClassificationEvaluator,
-    QuestionAnsweringEvaluator,
-    SummarizationEvaluator,
-    Text2TextGenerationEvaluator,
-    TextClassificationEvaluator,
-    TextGenerationEvaluator,
-    TokenClassificationEvaluator,
-    TranslationEvaluator,
-    evaluator,
+
+import sys
+from importlib import import_module
+from types import ModuleType
+
+_LAZY_IMPORTS = {
+    "AudioClassificationEvaluator": "evaluator",
+    "AutomaticSpeechRecognitionEvaluator": "evaluator",
+    "CombinedEvaluations": "module",
+    "Comparison": "module",
+    "ComparisonInfo": "info",
+    "EvaluationModule": "module",
+    "EvaluationModuleInfo": "info",
+    "EvaluationSuite": "evaluation_suite",
+    "Evaluator": "evaluator",
+    "ImageClassificationEvaluator": "evaluator",
+    "Measurement": "module",
+    "MeasurementInfo": "info",
+    "Metric": "module",
+    "MetricInfo": "info",
+    "QuestionAnsweringEvaluator": "evaluator",
+    "SummarizationEvaluator": "evaluator",
+    "Text2TextGenerationEvaluator": "evaluator",
+    "TextClassificationEvaluator": "evaluator",
+    "TextGenerationEvaluator": "evaluator",
+    "TokenClassificationEvaluator": "evaluator",
+    "TranslationEvaluator": "evaluator",
+    "combine": "module",
+    "disable_progress_bar": "utils.logging",
+    "enable_progress_bar": "utils.logging",
+    "infer_gradio_input_types": "utils.gradio",
+    "inspect_evaluation_module": "inspect",
+    "is_progress_bar_enabled": "utils.logging",
+    "json_to_string_type": "utils.gradio",
+    "launch_gradio_widget": "utils.gradio",
+    "list_evaluation_modules": "inspect",
+    "load": "loading",
+    "parse_gradio_data": "utils.gradio",
+    "parse_readme": "utils.gradio",
+    "parse_test_cases": "utils.gradio",
+    "push_to_hub": "hub",
+    "save": "saving",
+}
+
+
+_LAZY_MODULES = {
+    "config": "config",
+    "evaluation_suite": "evaluation_suite",
+    "gradio": "utils.gradio",
+    "hub": "hub",
+    "info": "info",
+    "inspect": "inspect",
+    "loading": "loading",
+    "logging": "utils.logging",
+    "module": "module",
+    "naming": "naming",
+    "saving": "saving",
+    "utils": "utils",
+}
+
+__all__ = sorted(
+    set(_LAZY_IMPORTS) | set(_LAZY_MODULES) | {"evaluator", "SCRIPTS_VERSION"}
 )
-from .hub import push_to_hub
-from .info import ComparisonInfo, EvaluationModuleInfo, MeasurementInfo, MetricInfo
-from .inspect import inspect_evaluation_module, list_evaluation_modules
-from .loading import load
-from .module import CombinedEvaluations, Comparison, EvaluationModule, Measurement, Metric, combine
-from .saving import save
-from .utils import *
-from .utils import gradio, logging
+
+
+def __getattr__(name):
+    if name in _LAZY_IMPORTS:
+        module = import_module("." + _LAZY_IMPORTS[name], __name__)
+        value = getattr(module, name)
+        globals()[name] = value
+        return value
+    if name in _LAZY_MODULES:
+        module = import_module("." + _LAZY_MODULES[name], __name__)
+        globals()[name] = module
+        return module
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+
+
+def __dir__():
+    return sorted(set(globals()) | set(_LAZY_IMPORTS))
+
+
+class _EvaluateModule(ModuleType):
+    def __getattribute__(self, name):
+        if name == "evaluator":
+            namespace = ModuleType.__getattribute__(self, "__dict__")
+            value = namespace.get("evaluator")
+            if isinstance(value, ModuleType) or value is None:
+                value = import_module(".evaluator", __name__).evaluator
+                namespace["evaluator"] = value
+            return value
+        return ModuleType.__getattribute__(self, name)
+
+
+sys.modules[__name__].__class__ = _EvaluateModule

tests/test_lazy_imports.py

@@ -0,0 +1,119 @@
+import subprocess
+import sys
+
+
+def run_python(code):
+    return subprocess.run(
+        [sys.executable, "-c", code],
+        capture_output=True,
+        text=True,
+        check=True,
+    ).stdout.strip()
+
+
+def test_package_import_does_not_load_heavy_dependencies():
+    output = run_python(
+        "import evaluate, sys; "
+        "print(any(name in sys.modules for name in "
+        "('datasets', 'pandas', 'pyarrow', 'transformers', 'torch')))"
+    )
+    assert output == "False"
+
+
+def test_public_api_exports():
+    output = run_python(
+        "import evaluate; "
+        "assert callable(evaluate.load); "
+        "assert callable(evaluate.evaluator); "
+        "assert isinstance(evaluate.EvaluationSuite, type); "
+        "print('OK')"
+    )
+    assert output == "OK"
+
+
+def test_evaluator_remains_callable_after_submodule_import():
+    output = run_python(
+        "import evaluate, importlib; "
+        "importlib.import_module('evaluate.evaluator'); "
+        "print(callable(evaluate.evaluator))"
+    )
+    assert output == "True"
+
+
+def test_wildcard_import():
+    output = run_python(
+        "from evaluate import *; "
+        "assert callable(load); "
+        "assert callable(evaluator); "
+        "assert isinstance(EvaluationSuite, type); "
+        "print('OK')"
+    )
+    assert output == "OK"
+
+
+def test_legacy_public_exports():
+    import evaluate
+
+    legacy_names = {
+        "AudioClassificationEvaluator",
+        "AutomaticSpeechRecognitionEvaluator",
+        "CombinedEvaluations",
+        "Comparison",
+        "ComparisonInfo",
+        "EvaluationModule",
+        "EvaluationModuleInfo",
+        "EvaluationSuite",
+        "Evaluator",
+        "ImageClassificationEvaluator",
+        "Measurement",
+        "MeasurementInfo",
+        "Metric",
+        "MetricInfo",
+        "QuestionAnsweringEvaluator",
+        "SummarizationEvaluator",
+        "Text2TextGenerationEvaluator",
+        "TextClassificationEvaluator",
+        "TextGenerationEvaluator",
+        "TokenClassificationEvaluator",
+        "TranslationEvaluator",
+        "combine",
+        "config",
+        "disable_progress_bar",
+        "enable_progress_bar",
+        "evaluation_suite",
+        "evaluator",
+        "gradio",
+        "hub",
+        "infer_gradio_input_types",
+        "info",
+        "inspect",
+        "inspect_evaluation_module",
+        "is_progress_bar_enabled",
+        "json_to_string_type",
+        "launch_gradio_widget",
+        "list_evaluation_modules",
+        "load",
+        "loading",
+        "logging",
+        "module",
+        "naming",
+        "parse_gradio_data",
+        "parse_readme",
+        "parse_test_cases",
+        "push_to_hub",
+        "save",
+        "saving",
+        "utils",
+    }
+
+    assert all(hasattr(evaluate, name) for name in legacy_names)
+
+
+def test_wildcard_exports_match_all():
+    import evaluate
+
+    namespace = {}
+    exec("from evaluate import *", namespace)
+
+    exported = set(namespace) - {"__builtins__"}
+    assert exported == set(evaluate.__all__)