From 3eaf55d43dabe1cc0ca8aa3a9c04af1ef5de1acd Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Jul 2026 13:12:59 +0900 Subject: [PATCH 01/13] wip --- python/private/pypi/extension.bzl | 17 +- .../pypi/generate_whl_library_build_bazel.bzl | 153 +++++++-- python/private/pypi/hub_builder.bzl | 55 ++- python/private/pypi/labels.bzl | 2 + python/private/pypi/whl_extract.bzl | 5 + python/private/pypi/whl_library.bzl | 81 ++++- python/private/pypi/whl_library_targets.bzl | 325 ++++++++++-------- 7 files changed, 454 insertions(+), 184 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index b79d04c075..9dd50308c2 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -31,7 +31,7 @@ load(":platform.bzl", _plat = "platform") load(":pypi_cache.bzl", "pypi_cache") load(":simpleapi_download.bzl", "simpleapi_download") load(":unified_hub_repo.bzl", "unified_hub_repo") -load(":whl_library.bzl", "whl_library") +load(":whl_library.bzl", "whl_library", "whl_library_deps") def _whl_mods_impl(whl_mods_dict): """Implementation of the pip.whl_mods tag class. @@ -459,14 +459,21 @@ You cannot use both the additive_build_content and additive_build_content_file a exposed_packages = {} extra_aliases = {} whl_libraries = {} + whl_library_deps_map = {} for hub in pip_hub_map.values(): out = hub.build() for whl_name, lib in out.whl_libraries.items(): if whl_name in whl_libraries: - fail("'{}' already in created".format(whl_name)) + print("'{}' already in created".format(whl_name)) + + whl_libraries[whl_name] = lib + + for deps_name, deps_args in out.whl_library_deps.items(): + if deps_name in whl_library_deps_map: + fail("'{}' already in created".format(deps_name)) else: - whl_libraries[whl_name] = lib + whl_library_deps_map[deps_name] = deps_args exposed_packages[hub.name] = out.exposed_packages extra_aliases[hub.name] = out.extra_aliases @@ -483,6 +490,7 @@ You cannot use both the additive_build_content and additive_build_content_file a hub_group_map = hub_group_map, hub_whl_map = hub_whl_map, whl_libraries = whl_libraries, + whl_library_deps = whl_library_deps_map, whl_mods = whl_mods, platform_config_settings = { hub_name: { @@ -614,6 +622,9 @@ def _pip_impl(module_ctx): for name, args in mods.whl_libraries.items(): whl_library(name = name, **args) + for name, args in mods.whl_library_deps.items(): + whl_library_deps(name = name, **args) + for hub_name, whl_map in mods.hub_whl_map.items(): hub_repository( name = hub_name, diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl index 71eab26c0e..5dc70e8291 100644 --- a/python/private/pypi/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -16,7 +16,9 @@ load("//python/private:text_util.bzl", "render") -_RENDER = { +# These are functions on how to render particular args, should be reused across all rendering +# invocations to make things easier. +_RENDER_FNS = { "copy_executables": render.dict, "copy_files": render.dict, "data": render.list, @@ -29,6 +31,11 @@ _RENDER = { "srcs_exclude": render.list, "tags": render.list, } +def _render(**kwargs): + return { + arg: _RENDER_FNS.get(arg, repr)(value) + for arg, value in kwargs.items + } # NOTE @aignas 2024-10-25: We have to keep this so that files in # this repository can be publicly visible without the need for @@ -38,19 +45,12 @@ _TEMPLATE = """\ package(default_visibility = ["//visibility:public"]) -package_metadata( - name = "package_metadata", - purl = {purl}, - visibility = ["//:__subpackages__"], -) - -{fn}( -{kwargs} -) +{macros} """ def generate_whl_library_build_bazel( *, + # TODO @aignas 2026-07-04: add extra args that are used in this function annotation = None, config_load, purl = None, @@ -72,21 +72,41 @@ def generate_whl_library_build_bazel( loads = [ """load("@package_metadata//rules:package_metadata.bzl", "package_metadata")""", + """load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_srcs", "whl_library_from_requires_dist")""" ] - fn = "whl_library_targets_from_requires" - if not requires_dist: - # no deps, we can leave the extra loads out - pass - else: - loads.append("""load("{}", "{}")""".format(config_load, "packages")) - kwargs["include"] = "packages" - kwargs["requires_dist"] = requires_dist - - loads.extend([ - """load("@rules_python//python/private/pypi:whl_library_targets.bzl", "{}")""".format(fn), - ]) + srcs_kwargs = dict( + name = name, + data = [], + sdist_filename = sdist_filename, + data_exclude = list(data_exclude), + srcs_exclude = list(srcs_exclude), + tags = [ + "pypi_name={}".format(metadata_name), + "pypi_version={}".format(metadata_version), + ], + entry_points = entry_points, + enable_implicit_namespace_pkgs = enable_implicit_namespace_pkgs, + copy_files = copy_files, + copy_executables = copy_executables, + namespace_package_files = namespace_package_files, + data = [], + visibility = visibility, + ) + from_requires_kwargs = dict( + name = name, + metadata_name = metadata_name, + metadata_version = metadata_version, + requires_dist = requires_dist, + extras = extras, + group_deps = group_deps, + dep_template = dep_template, + group_name = group_name, + ) + # NOTE, if users specify annotations, the wheel downloads are not reused this + # is to ensure that we don't break users config and also to ensure that we + # can have predictable results. additional_content = [] if annotation: kwargs["data"] = annotation.data @@ -97,19 +117,96 @@ def generate_whl_library_build_bazel( if annotation.additive_build_content: additional_content.append(annotation.additive_build_content) + macro_parts = [ + render.call( + "package_metadata", + **_render( + name = "package_metadata", + purl = purl, + visibility = ["//:__subpackages__"], + ), + ), + render.call( + "whl_library_srcs", + **_render(**srcs_kwargs) + ) + ] + + if config_load: + loads.append("""load("{}", "{}")""".format(config_load, "packages")) + from_requires_kwargs["include"] = "packages" + + macro_parts.append(render.call( + "whl_library_from_requires_dist", + **_render(**from_requires_kwargs), + )) + contents = "\n".join( [ _TEMPLATE.format( loads = "\n".join(loads), - fn = fn, - kwargs = render.indent("\n".join([ - "{} = {},".format(k, _RENDER.get(k, repr)(v)) - for k, v in sorted(kwargs.items()) - ])), - purl = repr(purl), + macros = "\n\n".join(macro_parts), ), ] + additional_content, ) # NOTE: Ensure that we terminate with a new line return contents.rstrip() + "\n" + +def generate_whl_library_deps_build_bazel( + *, + # TODO @aignas 2026-07-04: add extra args that are used in this function + **kwargs): + """Generate a BUILD file for an unzipped Wheel + + + Returns: + A complete BUILD file as a string + """ + + loads = [ + """load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_from_requires_dist")""" + ] + + from_requires_kwargs = dict( + name = name, + metadata_name = metadata_name, + metadata_version = metadata_version, + requires_dist = requires_dist, + extras = extras, + group_deps = group_deps, + dep_template = dep_template, + group_name = group_name, + ) + + macro_parts = [ + render.call( + "alias", + name=target, + actual=whl_library.same_package_label(target), + ) + for target in [ + # TODO @aignas 2026-07-04: use ./labels.bzl for the following + "package_metadata", + "data", + "dist_info", + "extracted_whl_files", + ] + ] + + if config_load: + loads.append("""load("{}", "{}")""".format(config_load, "packages")) + from_requires_kwargs["include"] = "packages" + + macro_parts.append(render.call( + "whl_library_from_requires_dist", + **_render(**from_requires_kwargs), + )) + + contents = _TEMPLATE.format( + loads = "\n".join(loads), + macros = "\n\n".join(macro_parts), + ) + + # NOTE: Ensure that we terminate with a new line + return contents.rstrip() + "\n" diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 01dc8494d6..fa91a1ec03 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -71,6 +71,9 @@ def hub_builder( # Mapping of whl_library repo names and their kwargs. # dict[str repo_name, dict[str, object] kwargs] _whl_libraries = {}, # modified by _add_whl_library + # Mapping of whl_library_deps repo names and their kwargs. + # dict[str repo_name, dict[str, object] kwargs] + _whl_library_deps = {}, # modified by _add_whl_library # Map of repos and their config settings, and repo the config # setting originated from. # dict[str whl_name, dict[str config_setting, str repo_name]] @@ -113,6 +116,7 @@ def _build(self): extra_aliases = {}, exposed_packages = [], whl_libraries = {}, + whl_library_deps = {}, ) if self._logger.failed(): return ret @@ -142,6 +146,10 @@ def _build(self): # Mapping of whl_library repo names and their kwargs. # dict[str repo_name, dict[str, object] kwargs] whl_libraries = self._whl_libraries, + + # Mapping of whl_library_deps repo names and their kwargs. + # dict[str repo_name, dict[str, object] kwargs] + whl_library_deps = self._whl_library_deps, ) def _pip_parse(self, module_ctx, pip_attr, python_version = None): @@ -308,17 +316,43 @@ def _add_whl_library(self, *, python_version, whl, repo): # are more platforms defined than there are wheels for and users # disallow building from sdist. return + + forbidden_args = { + "annotation": True, + "extra_pip_args": True, + "python_interpreter": True, + "python_interpreter_target": True, + } - # TODO @aignas 2025-06-29: we should not need the version in the repo_name if - # we are using pipstar and we are downloading the wheel using the downloader - # - # However, for that we should first have a different way to reference closures with - # extras. For example, if some package depends on `foo[extra]` and another depends on - # `foo`, we should have 2 py_library targets. - repo_name = "{}_{}_{}".format(self.name, version_label(python_version), repo.repo_name) + if [arg for arg in repo.args if arg in forbidden_args]: + # no reuse of the whl_library because we have args that force the extraction of the whl + # in the hub context. If we have whl-only pipstar extraction, then we can reuse the + # extracted sources. + repos_dict = self._whl_libraries + deps_args = repo.args + else: + whl_repo_name = "whl_{}".format(repo.whl_repo_name) - if repo_name in self._whl_libraries: - diff = _diff_dict(self._whl_libraries[repo_name], repo.args) + self._whl_libraries[whl_repo_name] = { + k: v for k, v in repo.args.items() + if k not in forbidden_args | { + "config_load": True + } + } + + args = repo.args + + deps_args = {} + for key in ("config_load", "dep_template", "group_deps", "group_name", "annotation", "pip_data_exclude"): + if key in args and args[key] != None: + deps_args[key] = args[key] + + deps_args["whl_library"] = "@{}//:BUILD.bazel".format(whl_repo_name) + repos_dict = self._whl_library_deps + + repo_name = "{}_{}_{}".format(self.name, version_label(python_version), repo.repo_name) + if repo_name in repos_dict: + diff = _diff_dict(repos_dict[repo_name], deps_args) if diff: self._logger.fail(lambda: ( "Attempting to create a duplicate library {repo_name} for {whl_name} with different arguments. Already existing declaration has:\n".format( @@ -331,7 +365,7 @@ def _add_whl_library(self, *, python_version, whl, repo): ]) )) return - self._whl_libraries[repo_name] = repo.args + repos_dict[repo_name] = deps_args mapping = self._whl_map.setdefault(whl.name, {}) if repo.config_setting in mapping and mapping[repo.config_setting] != repo_name: @@ -685,6 +719,7 @@ def _whl_repo( return struct( repo_name = whl_repo_name(src.filename, src.sha256, *target_platforms), + whl_repo_name = whl_repo_name(src.filename, src.sha256), args = args, config_setting = whl_config_setting( version = python_version, diff --git a/python/private/pypi/labels.bzl b/python/private/pypi/labels.bzl index 8f91a03b4c..c06174d216 100644 --- a/python/private/pypi/labels.bzl +++ b/python/private/pypi/labels.bzl @@ -22,3 +22,5 @@ PY_LIBRARY_IMPL_LABEL = "_pkg" DATA_LABEL = "data" DIST_INFO_LABEL = "dist_info" NODEPS_LABEL = "no_deps" +NODEPS_WHL_FILE_LABEL = "_whl_file" +NODEPS_PY_LIBRARY_LABEL = "_srcs" diff --git a/python/private/pypi/whl_extract.bzl b/python/private/pypi/whl_extract.bzl index 0d61b9a07b..9b4d6e6fc6 100644 --- a/python/private/pypi/whl_extract.bzl +++ b/python/private/pypi/whl_extract.bzl @@ -26,6 +26,11 @@ def whl_extract(rctx, *, whl_path, logger): logger = logger, ) + # Symlink the METADATA file to be able to refer to it from another repository_rule. This allows + # us to split the extracted sources and the closure itself to 2 different repository rules + # allowing us to not extract the same wheels multiple times. + rctx.symlink(metadata_file, "METADATA") + # Get the .dist_info dir name dist_info_dir = metadata_file.dirname rctx.file( diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index 37cc36492e..a4fb5fa9f0 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -21,13 +21,13 @@ load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") load(":attrs.bzl", "ATTRS", "use_isolated") load(":deps.bzl", "all_repo_names", "record_files") -load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") +load(":generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel", "generate_whl_library_deps_build_bazel") load(":patch_whl.bzl", "patch_whl") load(":pep508_requirement.bzl", "requirement") load(":pypi_repo_utils.bzl", "pypi_repo_utils") load(":urllib.bzl", "urllib") load(":whl_extract.bzl", "whl_extract") -load(":whl_metadata.bzl", "parse_entry_points", "whl_metadata") +load(":whl_metadata.bzl", "parse_entry_points", "parse_whl_metadata", "whl_metadata") _CPPFLAGS = "CPPFLAGS" _COMMAND_LINE_TOOLS_PATH_SLUG = "commandlinetools" @@ -624,3 +624,80 @@ wheel contents without building an `sdist` first. REPO_DEBUG_ENV_VAR, ], ) + +def _whl_library_deps_impl(rctx): + logger = repo_utils.logger(rctx) + + # Load the METADATA file from a different repository + metadata_path = rctx.path(rctx.attr.whl_library.same_package_label("METADATA")) + metadata = parse_whl_metadata(rctx.read(metadata_path)) + + if not (metadata.name and metadata.version): + logger.fail("Failed to parse METADATA from {}".format(rctx.attr.whl_library)) + return + + build_file_contents = generate_whl_library_deps_build_bazel( + dep_template = rctx.attr.dep_template, + config_load = rctx.attr.config_load, + metadata_name = metadata.name, + metadata_version = metadata.version, + requires_dist = metadata.requires_dist, + group_deps = rctx.attr.group_deps, + group_name = rctx.attr.group_name, + extras = requirement(rctx.attr.requirement).extras, + whl_library=rctx.attr.whl_library, + ) + + rctx.file("WORKSPACE") + rctx.file("WORKSPACE.bazel") + rctx.file("MODULE.bazel") + rctx.file("REPO.bazel") + rctx.file("BUILD.bazel", build_file_contents) + +whl_library_deps = repository_rule( + attrs = { + "annotation": attr.label( + doc = "Optional json encoded file containing annotation to apply to the extracted wheel.", + allow_files = True, + ), + "config_load": attr.string( + doc = "The load location for configuration for pipstar.", + ), + "dep_template": attr.string( + doc = "The dep template to use for referencing the dependencies.", + ), + "extras": attr.string_list( + doc = "The list of extras.", + default = [], + ), + "group_deps": attr.string_list( + doc = "List of dependencies to skip in order to break the cycles within a dependency group.", + default = [], + ), + "group_name": attr.string( + doc = "Name of the group, if any.", + ), + "pip_data_exclude": attr.string_list( + doc = "Additional data exclude patterns.", + default = [], + ), + "whl_library": attr.label( + doc = "The whl_library repository label, use BUILD.bazel file for this.", + mandatory = True, + ), + }, + doc = """ +Uses a downloaded and extracted whl_library to generate a BUILD.bazel file with dependencies +inferred by parsing the METADATA file that comes with the wheel. + +:::{versionadded} VERSION_NEXT_FEATURE +::: +:::{seealso} +See the {obj}`whl_library` that is used for downloading and extracting the wheel. +::: +""", + implementation = _whl_library_deps_impl, + environ = [ + REPO_DEBUG_ENV_VAR, + ], +) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 933b529053..a6b199b481 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -24,6 +24,8 @@ load( "DATA_LABEL", "DIST_INFO_LABEL", "EXTRACTED_WHEEL_FILES", + "NODEPS_PY_LIBRARY_LABEL", + "NODEPS_WHL_FILE_LABEL", "PY_LIBRARY_IMPL_LABEL", "PY_LIBRARY_PUBLIC_LABEL", "WHEEL_FILE_IMPL_LABEL", @@ -47,14 +49,13 @@ _BAZEL_REPO_FILE_GLOBS = [ _IS_VENV_SITE_PACKAGES_YES = Label("//python/config_settings:_is_venvs_site_packages_yes") _VENV_SITE_PACKAGES_FLAG = Label("//python/config_settings:venvs_site_packages") -def whl_library_targets_from_requires( +def whl_library_from_requires_dist( *, name, metadata_name = "", metadata_version = "", requires_dist = [], extras = [], - entry_points = {}, include = [], group_deps = [], **kwargs): @@ -71,7 +72,6 @@ def whl_library_targets_from_requires( requires_dist: {type}`list[str]` The list of `Requires-Dist` values from the whl `METADATA`. extras: {type}`list[str]` The list of requested extras. This essentially includes extra transitive dependencies in the final targets depending on the wheel `METADATA`. - entry_points: {type}`list[dict]` A list of parsed entry point definitions. include: {type}`list[str]` The list of packages to include. **kwargs: Extra args passed to the {obj}`whl_library_targets` """ @@ -87,7 +87,6 @@ def whl_library_targets_from_requires( name = name, dependencies = package_deps.deps, dependencies_with_markers = package_deps.deps_select, - entry_points = entry_points, tags = [ "pypi_name={}".format(metadata_name), "pypi_version={}".format(metadata_version), @@ -95,6 +94,10 @@ def whl_library_targets_from_requires( **kwargs ) +def whl_library_targets_from_requires(*args, **kwargs): + """Deprecated alias for {obj}`whl_library_from_requires_dist`.""" + whl_library_from_requires_dist(*args, **kwargs) + def _parse_requires_dist( *, name, @@ -114,21 +117,165 @@ def whl_library_targets( *, name, dep_template, + tags = [], + dependencies = [], + dependencies_with_markers = {}, + group_name = "", + native = native, + src_pkg = Label("//:BUILD.bazel"), + rules = struct( + copy_file = copy_file, + py_binary = py_binary, + py_library = py_library, + venv_entry_point = venv_entry_point, + venv_rewrite_shebang = venv_rewrite_shebang, + env_marker_setting = env_marker_setting, + create_inits = _create_inits, + )): + """Create all of the whl_library targets. + + Args: + name: {type}`str` The file to match for including it into the `whl` + filegroup. This may be also parsed to generate extra metadata. + dep_template: {type}`str` The dep_template to use for dependency + interpolation. + tags: {type}`list[str]` The tags set on the `py_library`. + dependencies: {type}`list[str]` A list of dependencies. + dependencies_with_markers: {type}`dict[str, str]` A marker to evaluate + in order for the dep to be included. + group_name: {type}`str` name of the dependency group (if any) which + contains this library. If set, this library will behave as a shim + to group implementation rules which will provide simultaneously + installed dependencies which would otherwise form a cycle. + src_pkg: TODO + native: {type}`native` The native struct for overriding in tests. + rules: {type}`struct` A struct with references to rules for creating targets. + """ + src_pkg = Label(src_pkg) + + _config_settings( + dependencies_with_markers = dependencies_with_markers, + rules = rules, + visibility = ["//visibility:private"], + ) + deps_conditional = { + d: "is_include_{}_true".format(d) + for d in dependencies_with_markers + } + + # If this library is a member of a group, its public label aliases need to + # point to the group implementation rule not the implementation rules. We + # also need to mark the implementation rules as visible to the group + # implementation. + if group_name and "//:" in dep_template: + # This is the legacy behaviour where the group library is outside the hub repo + # + # It is expected to disappear when we drop WORKSPACE or drop the vendoring of + # pip_parse `requirements.bzl` in WORKSPACE. The alternative would be to add + # another argument to the macro, but it is already full of arguments. + label_tmpl = dep_template.format( + name = "_config", + target = normalize_name(group_name) + "_{}", + ).replace( + "//:", + "//_groups:", + ) + impl_vis = [dep_template.format( + name = "_config", + target = "__pkg__", + ).replace( + "//:", + "//_groups:", + )] + + native.alias( + name = PY_LIBRARY_PUBLIC_LABEL, + actual = label_tmpl.format(PY_LIBRARY_PUBLIC_LABEL), + visibility = ["//visibility:public"], + ) + native.alias( + name = WHEEL_FILE_PUBLIC_LABEL, + actual = label_tmpl.format(WHEEL_FILE_PUBLIC_LABEL), + visibility = ["//visibility:public"], + ) + py_library_label = PY_LIBRARY_IMPL_LABEL + whl_file_label = WHEEL_FILE_IMPL_LABEL + + elif group_name: + py_library_label = PY_LIBRARY_PUBLIC_LABEL + whl_file_label = WHEEL_FILE_PUBLIC_LABEL + impl_vis = [dep_template.format(name = "", target = "__subpackages__")] + + else: + py_library_label = PY_LIBRARY_PUBLIC_LABEL + whl_file_label = WHEEL_FILE_PUBLIC_LABEL + impl_vis = ["//visibility:public"] + + dependencies = sorted([normalize_name(d) for d in dependencies]) + native.filegroup( + name = whl_file_label, + srcs = [src_pkg.same_package_label(NODEPS_WHL_FILE_LABEL)], + data = _deps( + deps = dependencies, + deps_conditional = deps_conditional, + tmpl = dep_template.format(name = "{}", target = WHEEL_FILE_PUBLIC_LABEL), + ), + visibility = impl_vis, + ) + rules.py_library( + name = py_library_label, + deps = [src_pkg.same_package_label(NODEPS_PY_LIBRARY_LABEL)] + _deps( + deps = dependencies, + deps_conditional = deps_conditional, + tmpl = dep_template.format(name = "{}", target = PY_LIBRARY_PUBLIC_LABEL), + ), + tags = tags, + visibility = impl_vis, + ) + +def _config_settings(dependencies_with_markers, rules, **kwargs): + """Generate config settings for the targets. + + Args: + dependencies_with_markers: {type}`dict[str, str]` The markers to evaluate by + each dep. + rules: used for testing + **kwargs: Extra kwargs to pass to the rule. + """ + for dep, expression in dependencies_with_markers.items(): + rules.env_marker_setting( + name = "include_{}".format(dep), + expression = expression, + **kwargs + ) + +def _deps(deps, deps_conditional, tmpl): + deps = [tmpl.format(d) for d in sorted(deps)] + + for dep, setting in deps_conditional.items(): + deps = deps + select({ + ":{}".format(setting): [tmpl.format(dep)], + "//conditions:default": [], + }) + + return deps + +def whl_library_srcs( + *, + name, sdist_filename = None, data_exclude = [], srcs_exclude = [], + copy_files = {}, + copy_executables = {}, tags = [], - dependencies = [], filegroups = None, - dependencies_with_markers = {}, entry_points = {}, - group_name = "", data = [], - copy_files = {}, - copy_executables = {}, native = native, enable_implicit_namespace_pkgs = False, namespace_package_files = [], + visibility = ["//visibility:public"], rules = struct( copy_file = copy_file, py_binary = py_binary, @@ -143,43 +290,52 @@ def whl_library_targets( Args: name: {type}`str` The file to match for including it into the `whl` filegroup. This may be also parsed to generate extra metadata. - dep_template: {type}`str` The dep_template to use for dependency - interpolation. sdist_filename: {type}`str | None` If the wheel was built from an sdist, the filename of the sdist. tags: {type}`list[str]` The tags set on the `py_library`. - dependencies: {type}`list[str]` A list of dependencies. - dependencies_with_markers: {type}`dict[str, str]` A marker to evaluate - in order for the dep to be included. entry_points: {type}`list[dict]` A list of parsed entry point definitions. filegroups: {type}`dict[str, list[str]] | None` A dictionary of the target names and the glob matches. If `None`, defaults will be used. - group_name: {type}`str` name of the dependency group (if any) which - contains this library. If set, this library will behave as a shim - to group implementation rules which will provide simultaneously - installed dependencies which would otherwise form a cycle. - copy_executables: {type}`dict[str, str]` The mapping between src and - dest locations for the targets. - copy_files: {type}`dict[str, str]` The mapping between src and - dest locations for the targets. data_exclude: {type}`list[str]` The globs for data attribute exclusion in `py_library`. srcs_exclude: {type}`list[str]` The globs for srcs attribute exclusion in `py_library`. + copy_executables: {type}`dict[str, str]` The mapping between src and + dest locations for the targets. + copy_files: {type}`dict[str, str]` The mapping between src and + dest locations for the targets. data: {type}`list[str]` A list of labels to include as part of the `data` attribute in `py_library`. enable_implicit_namespace_pkgs: {type}`boolean` generate __init__.py files for namespace pkgs. + visibility: {type}`list[str]` The visibility for the targets. native: {type}`native` The native struct for overriding in tests. namespace_package_files: {type}`list[str]` A list of labels of files whose directories are namespace packages. rules: {type}`struct` A struct with references to rules for creating targets. """ - dependencies = sorted([normalize_name(d) for d in dependencies]) tags = sorted(tags) data = [] + data bins_for_data_label = [] + for src, dest in copy_files.items(): + rules.copy_file( + name = dest + ".copy", + src = src, + out = dest, + visibility = visibility, + ) + data.append(dest) + for src, dest in copy_executables.items(): + rules.copy_file( + name = dest + ".copy", + src = src, + out = dest, + is_executable = True, + visibility = visibility, + ) + data.append(dest) + for ep_dict in entry_points.values(): kwargs = dict(ep_dict) ep_name = kwargs.pop("name") @@ -233,95 +389,14 @@ def whl_library_targets( native.filegroup( name = filegroup_name, srcs = srcs, - visibility = ["//visibility:public"], + visibility = visibility, ) - for src, dest in copy_files.items(): - rules.copy_file( - name = dest + ".copy", - src = src, - out = dest, - visibility = ["//visibility:public"], - ) - data.append(dest) - for src, dest in copy_executables.items(): - rules.copy_file( - name = dest + ".copy", - src = src, - out = dest, - is_executable = True, - visibility = ["//visibility:public"], - ) - data.append(dest) - - _config_settings( - dependencies_with_markers = dependencies_with_markers, - rules = rules, - visibility = ["//visibility:private"], - ) - deps_conditional = { - d: "is_include_{}_true".format(d) - for d in dependencies_with_markers - } - - # If this library is a member of a group, its public label aliases need to - # point to the group implementation rule not the implementation rules. We - # also need to mark the implementation rules as visible to the group - # implementation. - if group_name and "//:" in dep_template: - # This is the legacy behaviour where the group library is outside the hub repo - # - # It is expected to disappear when we drop WORKSPACE or drop the vendoring of - # pip_parse `requirements.bzl` in WORKSPACE. The alternative would be to add - # another argument to the macro, but it is already full of arguments. - label_tmpl = dep_template.format( - name = "_config", - target = normalize_name(group_name) + "_{}", - ).replace( - "//:", - "//_groups:", - ) - impl_vis = [dep_template.format( - name = "_config", - target = "__pkg__", - ).replace( - "//:", - "//_groups:", - )] - - native.alias( - name = PY_LIBRARY_PUBLIC_LABEL, - actual = label_tmpl.format(PY_LIBRARY_PUBLIC_LABEL), - visibility = ["//visibility:public"], - ) - native.alias( - name = WHEEL_FILE_PUBLIC_LABEL, - actual = label_tmpl.format(WHEEL_FILE_PUBLIC_LABEL), - visibility = ["//visibility:public"], - ) - py_library_label = PY_LIBRARY_IMPL_LABEL - whl_file_label = WHEEL_FILE_IMPL_LABEL - - elif group_name: - py_library_label = PY_LIBRARY_PUBLIC_LABEL - whl_file_label = WHEEL_FILE_PUBLIC_LABEL - impl_vis = [dep_template.format(name = "", target = "__subpackages__")] - - else: - py_library_label = PY_LIBRARY_PUBLIC_LABEL - whl_file_label = WHEEL_FILE_PUBLIC_LABEL - impl_vis = ["//visibility:public"] - if hasattr(native, "filegroup"): native.filegroup( - name = whl_file_label, + name = NODEPS_WHL_FILE_LABEL, srcs = [name], - data = _deps( - deps = dependencies, - deps_conditional = deps_conditional, - tmpl = dep_template.format(name = "{}", target = WHEEL_FILE_PUBLIC_LABEL), - ), - visibility = impl_vis, + visibility = visibility, ) if hasattr(rules, "py_library"): @@ -375,47 +450,15 @@ def whl_library_targets( data = data + [DATA_LABEL] rules.py_library( - name = py_library_label, + name = NODEPS_PY_LIBRARY_LABEL, srcs = srcs, pyi_srcs = pyi_srcs, data = data, # This makes this directory a top-level in the python import # search path for anything that depends on this. imports = ["site-packages"], - deps = _deps( - deps = dependencies, - deps_conditional = deps_conditional, - tmpl = dep_template.format(name = "{}", target = PY_LIBRARY_PUBLIC_LABEL), - ), tags = tags, - visibility = impl_vis, + visibility = visibility, experimental_venvs_site_packages = _VENV_SITE_PACKAGES_FLAG, namespace_package_files = namespace_package_files, ) - -def _config_settings(dependencies_with_markers, rules, **kwargs): - """Generate config settings for the targets. - - Args: - dependencies_with_markers: {type}`dict[str, str]` The markers to evaluate by - each dep. - rules: used for testing - **kwargs: Extra kwargs to pass to the rule. - """ - for dep, expression in dependencies_with_markers.items(): - rules.env_marker_setting( - name = "include_{}".format(dep), - expression = expression, - **kwargs - ) - -def _deps(deps, deps_conditional, tmpl): - deps = [tmpl.format(d) for d in sorted(deps)] - - for dep, setting in deps_conditional.items(): - deps = deps + select({ - ":{}".format(setting): [tmpl.format(dep)], - "//conditions:default": [], - }) - - return deps From c5b5153d1d5504b4ff062297c0fa850ac2b474fc Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Jul 2026 13:52:54 +0900 Subject: [PATCH 02/13] artisanal inteligence spike --- python/private/pypi/extension.bzl | 63 ++++++++++++++++- .../pypi/generate_whl_library_build_bazel.bzl | 69 ++++++++++++------- python/private/pypi/hub_builder.bzl | 45 ++++++------ python/private/pypi/labels.bzl | 1 + python/private/pypi/whl_library.bzl | 12 ++-- python/private/pypi/whl_library_targets.bzl | 18 +++-- 6 files changed, 147 insertions(+), 61 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 9dd50308c2..7f68f1f6f4 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -22,6 +22,7 @@ load("//python/private:auth.bzl", "AUTH_ATTRS") load("//python/private:normalize_name.bzl", "normalize_name") load("//python/private:pyproject_utils.bzl", "read_pyproject", "version_from_requires_python") load("//python/private:repo_utils.bzl", "repo_utils") +load("//python/private:text_util.bzl", "render") load(":hub_builder.bzl", "hub_builder") load(":hub_repository.bzl", "hub_repository", "whl_config_settings_to_json") load(":parse_whl_name.bzl", "parse_whl_name") @@ -464,8 +465,26 @@ You cannot use both the additive_build_content and additive_build_content_file a out = hub.build() for whl_name, lib in out.whl_libraries.items(): + # NOTE @aignas 2026-07-04: if the same wheel is downloaded from multiple + # indexes, this will fail, forcing the user to actually download the wheel + # from the same and deterministic location. This is usually the case for + # public wheels and users should setup the defaults.index_url to correct + # fall-back in rules_python we should handle the default index to substitute + # any index-url in requirements pointing to the public PyPI mirrors. if whl_name in whl_libraries: - print("'{}' already in created".format(whl_name)) + existing = whl_libraries[whl_name] + + # TODO @aignas 2026-07-04: stop ignoring the index_url + diff = _diff_dict(existing, lib, ignore_keys = {"index_url": True}) + if diff: + fail("'{}' already in created:\n{}".format( + whl_name, + "\n".join([ + " {}: {}".format(key, render.indent(render.dict(value)).lstrip()) + for key, value in diff.items() + if value + ]), + )) whl_libraries[whl_name] = lib @@ -1223,3 +1242,45 @@ This rule creates json files based on the whl_mods attribute. ), }, ) + +# TODO dedupe code + +def _diff_dict(first, second, *, ignore_keys = {}): + """A simple utility to shallow compare dictionaries. + + Args: + first: The first dictionary to compare. + second: The second dictionary to compare. + + Returns: + A dictionary containing the differences, with keys "common", "different", + "extra", and "missing", or None if the dictionaries are identical. + """ + missing = {} + extra = { + key: value + for key, value in second.items() + if key not in first and key not in ignore_keys + } + common = {} + different = {} + + for key, value in first.items(): + if key in ignore_keys: + continue + elif key not in second: + missing[key] = value + elif value == second[key]: + common[key] = value + else: + different[key] = (value, second[key]) + + if missing or extra or different: + return { + "common": common, + "different": different, + "extra": extra, + "missing": missing, + } + else: + return None diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl index 5dc70e8291..8a0c45da26 100644 --- a/python/private/pypi/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -15,6 +15,7 @@ """Generate the BUILD.bazel contents for a repo defined by a whl_library.""" load("//python/private:text_util.bzl", "render") +load(":labels.bzl", "DATA_LABEL", "DIST_INFO_LABEL", "EXTRACTED_WHEEL_FILES", "PACKAGE_METADATA_LABEL") # These are functions on how to render particular args, should be reused across all rendering # invocations to make things easier. @@ -31,10 +32,11 @@ _RENDER_FNS = { "srcs_exclude": render.list, "tags": render.list, } + def _render(**kwargs): return { arg: _RENDER_FNS.get(arg, repr)(value) - for arg, value in kwargs.items + for arg, value in kwargs.items() } # NOTE @aignas 2024-10-25: We have to keep this so that files in @@ -50,11 +52,26 @@ package(default_visibility = ["//visibility:public"]) def generate_whl_library_build_bazel( *, - # TODO @aignas 2026-07-04: add extra args that are used in this function + name, annotation = None, config_load, + copy_executables = {}, + copy_files = {}, + data_exclude = [], + dep_template, + enable_implicit_namespace_pkgs = False, + entry_points = {}, + extras = [], + group_deps = [], + group_name = None, + metadata_name, + metadata_version, + namespace_package_files = {}, purl = None, requires_dist = [], + sdist_filename = None, + srcs_exclude = [], + visibility = ["//visibility:public"], **kwargs): """Generate a BUILD file for an unzipped Wheel @@ -72,7 +89,7 @@ def generate_whl_library_build_bazel( loads = [ """load("@package_metadata//rules:package_metadata.bzl", "package_metadata")""", - """load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_srcs", "whl_library_from_requires_dist")""" + """load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_srcs", "whl_library_from_requires_dist")""", ] srcs_kwargs = dict( @@ -90,13 +107,11 @@ def generate_whl_library_build_bazel( copy_files = copy_files, copy_executables = copy_executables, namespace_package_files = namespace_package_files, - data = [], visibility = visibility, ) from_requires_kwargs = dict( - name = name, - metadata_name = metadata_name, - metadata_version = metadata_version, + name = metadata_name, + version = metadata_version, requires_dist = requires_dist, extras = extras, group_deps = group_deps, @@ -121,15 +136,15 @@ def generate_whl_library_build_bazel( render.call( "package_metadata", **_render( - name = "package_metadata", + name = PACKAGE_METADATA_LABEL, purl = purl, visibility = ["//:__subpackages__"], - ), + ) ), render.call( "whl_library_srcs", **_render(**srcs_kwargs) - ) + ), ] if config_load: @@ -138,7 +153,7 @@ def generate_whl_library_build_bazel( macro_parts.append(render.call( "whl_library_from_requires_dist", - **_render(**from_requires_kwargs), + **_render(**from_requires_kwargs) )) contents = "\n".join( @@ -155,7 +170,15 @@ def generate_whl_library_build_bazel( def generate_whl_library_deps_build_bazel( *, - # TODO @aignas 2026-07-04: add extra args that are used in this function + name, + version, + config_load, + dep_template, + extras, + group_deps, + group_name, + requires_dist, + whl_library, **kwargs): """Generate a BUILD file for an unzipped Wheel @@ -165,32 +188,32 @@ def generate_whl_library_deps_build_bazel( """ loads = [ - """load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_from_requires_dist")""" + """load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_from_requires_dist")""", ] from_requires_kwargs = dict( name = name, - metadata_name = metadata_name, - metadata_version = metadata_version, + version = version, requires_dist = requires_dist, extras = extras, group_deps = group_deps, dep_template = dep_template, group_name = group_name, + src_pkg = str(whl_library), ) macro_parts = [ render.call( "alias", - name=target, - actual=whl_library.same_package_label(target), + **_render( + name = target, + actual = str(whl_library.same_package_label(target)), + ) ) for target in [ - # TODO @aignas 2026-07-04: use ./labels.bzl for the following - "package_metadata", - "data", - "dist_info", - "extracted_whl_files", + DATA_LABEL, + DIST_INFO_LABEL, + EXTRACTED_WHEEL_FILES, ] ] @@ -200,7 +223,7 @@ def generate_whl_library_deps_build_bazel( macro_parts.append(render.call( "whl_library_from_requires_dist", - **_render(**from_requires_kwargs), + **_render(**from_requires_kwargs) )) contents = _TEMPLATE.format( diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index fa91a1ec03..0c4fa6c0d9 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -316,7 +316,7 @@ def _add_whl_library(self, *, python_version, whl, repo): # are more platforms defined than there are wheels for and users # disallow building from sdist. return - + forbidden_args = { "annotation": True, "extra_pip_args": True, @@ -332,13 +332,14 @@ def _add_whl_library(self, *, python_version, whl, repo): deps_args = repo.args else: whl_repo_name = "whl_{}".format(repo.whl_repo_name) - - self._whl_libraries[whl_repo_name] = { - k: v for k, v in repo.args.items() + _add_library(self, repos_dict = self._whl_libraries, name = whl_repo_name, args = { + k: v + for k, v in repo.args.items() if k not in forbidden_args | { - "config_load": True + "config_load": None, + "dep_template": None, } - } + }) args = repo.args @@ -351,21 +352,7 @@ def _add_whl_library(self, *, python_version, whl, repo): repos_dict = self._whl_library_deps repo_name = "{}_{}_{}".format(self.name, version_label(python_version), repo.repo_name) - if repo_name in repos_dict: - diff = _diff_dict(repos_dict[repo_name], deps_args) - if diff: - self._logger.fail(lambda: ( - "Attempting to create a duplicate library {repo_name} for {whl_name} with different arguments. Already existing declaration has:\n".format( - repo_name = repo_name, - whl_name = whl.name, - ) + "\n".join([ - " {}: {}".format(key, render.indent(render.dict(value)).lstrip()) - for key, value in diff.items() - if value - ]) - )) - return - repos_dict[repo_name] = deps_args + _add_library(self, repos_dict = repos_dict, name = repo_name, args = deps_args) mapping = self._whl_map.setdefault(whl.name, {}) if repo.config_setting in mapping and mapping[repo.config_setting] != repo_name: @@ -381,6 +368,22 @@ def _add_whl_library(self, *, python_version, whl, repo): ### end of setters, below we have various functions to implement the public methods +def _add_library(self, *, repos_dict, name, args): + if name in repos_dict: + diff = _diff_dict(repos_dict[name], args) + if diff: + self._logger.fail(lambda: ( + "Attempting to create a duplicate library {name} with different arguments. Already existing declaration has:\n".format( + repo_name = name, + ) + "\n".join([ + " {}: {}".format(key, render.indent(render.dict(value)).lstrip()) + for key, value in diff.items() + if value + ]) + )) + return + repos_dict[name] = args + def _set_get_index_urls(self, mctx, pip_attr): # Resolve the index URL through envsubst so the ``$VAR`` / ``${VAR:-default}`` # form is honored when deciding whether the experimental index-url mode is diff --git a/python/private/pypi/labels.bzl b/python/private/pypi/labels.bzl index c06174d216..4baedc0b28 100644 --- a/python/private/pypi/labels.bzl +++ b/python/private/pypi/labels.bzl @@ -21,6 +21,7 @@ PY_LIBRARY_PUBLIC_LABEL = "pkg" PY_LIBRARY_IMPL_LABEL = "_pkg" DATA_LABEL = "data" DIST_INFO_LABEL = "dist_info" +PACKAGE_METADATA_LABEL = "package_metadata" NODEPS_LABEL = "no_deps" NODEPS_WHL_FILE_LABEL = "_whl_file" NODEPS_PY_LIBRARY_LABEL = "_srcs" diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index a4fb5fa9f0..fb246538c8 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -637,15 +637,15 @@ def _whl_library_deps_impl(rctx): return build_file_contents = generate_whl_library_deps_build_bazel( - dep_template = rctx.attr.dep_template, + name = metadata.name, + version = metadata.version, config_load = rctx.attr.config_load, - metadata_name = metadata.name, - metadata_version = metadata.version, - requires_dist = metadata.requires_dist, + dep_template = rctx.attr.dep_template, + extras = rctx.attr.extras, group_deps = rctx.attr.group_deps, group_name = rctx.attr.group_name, - extras = requirement(rctx.attr.requirement).extras, - whl_library=rctx.attr.whl_library, + requires_dist = metadata.requires_dist, + whl_library = rctx.attr.whl_library, ) rctx.file("WORKSPACE") diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index a6b199b481..917685fb0b 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -51,9 +51,8 @@ _VENV_SITE_PACKAGES_FLAG = Label("//python/config_settings:venvs_site_packages") def whl_library_from_requires_dist( *, - name, - metadata_name = "", - metadata_version = "", + name = None, + version = "", requires_dist = [], extras = [], include = [], @@ -62,9 +61,8 @@ def whl_library_from_requires_dist( """The macro to create whl targets from the METADATA. Args: - name: {type}`str` The wheel filename - metadata_name: {type}`str` The package name as written in wheel `METADATA`. - metadata_version: {type}`str` The package version as written in wheel `METADATA`. + name: {type}`str` Unused + version: {type}`str` The package version as written in wheel `METADATA`. group_deps: {type}`list[str]` names of fellow members of the group (if any). These will be excluded from generated deps lists so as to avoid direct cycles. These dependencies will be provided at runtime by the @@ -76,7 +74,7 @@ def whl_library_from_requires_dist( **kwargs: Extra args passed to the {obj}`whl_library_targets` """ package_deps = _parse_requires_dist( - name = metadata_name, + name = name, requires_dist = requires_dist, excludes = group_deps, extras = extras, @@ -84,12 +82,12 @@ def whl_library_from_requires_dist( ) whl_library_targets( - name = name, + name = normalize_name(name), dependencies = package_deps.deps, dependencies_with_markers = package_deps.deps_select, tags = [ - "pypi_name={}".format(metadata_name), - "pypi_version={}".format(metadata_version), + "pypi_name={}".format(name), + "pypi_version={}".format(version), ], **kwargs ) From c3a21b3f10e7a669fe3ab2c70b5ec191b8595560 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Jul 2026 14:19:16 +0900 Subject: [PATCH 03/13] wip --- .../pypi/generate_whl_library_build_bazel.bzl | 59 +++++++++---------- python/private/pypi/whl_library.bzl | 8 ++- python/private/pypi/whl_library_targets.bzl | 8 +++ 3 files changed, 43 insertions(+), 32 deletions(-) diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl index 8a0c45da26..407d3e8428 100644 --- a/python/private/pypi/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -20,6 +20,7 @@ load(":labels.bzl", "DATA_LABEL", "DIST_INFO_LABEL", "EXTRACTED_WHEEL_FILES", "P # These are functions on how to render particular args, should be reused across all rendering # invocations to make things easier. _RENDER_FNS = { + "aliases": render.list, "copy_executables": render.dict, "copy_files": render.dict, "data": render.list, @@ -109,15 +110,6 @@ def generate_whl_library_build_bazel( namespace_package_files = namespace_package_files, visibility = visibility, ) - from_requires_kwargs = dict( - name = metadata_name, - version = metadata_version, - requires_dist = requires_dist, - extras = extras, - group_deps = group_deps, - dep_template = dep_template, - group_name = group_name, - ) # NOTE, if users specify annotations, the wheel downloads are not reused this # is to ensure that we don't break users config and also to ensure that we @@ -147,14 +139,25 @@ def generate_whl_library_build_bazel( ), ] - if config_load: - loads.append("""load("{}", "{}")""".format(config_load, "packages")) - from_requires_kwargs["include"] = "packages" + if dep_template: + from_requires_kwargs = dict( + name = metadata_name, + version = metadata_version, + requires_dist = requires_dist, + extras = extras, + group_deps = group_deps, + dep_template = dep_template, + group_name = group_name, + ) - macro_parts.append(render.call( - "whl_library_from_requires_dist", - **_render(**from_requires_kwargs) - )) + if config_load: + loads.append("""load("{}", "{}")""".format(config_load, "packages")) + from_requires_kwargs["include"] = "packages" + + macro_parts.append(render.call( + "whl_library_from_requires_dist", + **_render(**from_requires_kwargs) + )) contents = "\n".join( [ @@ -174,6 +177,7 @@ def generate_whl_library_deps_build_bazel( version, config_load, dep_template, + entry_points, extras, group_deps, group_name, @@ -200,31 +204,24 @@ def generate_whl_library_deps_build_bazel( dep_template = dep_template, group_name = group_name, src_pkg = str(whl_library), - ) - - macro_parts = [ - render.call( - "alias", - **_render( - name = target, - actual = str(whl_library.same_package_label(target)), - ) - ) - for target in [ + aliases = [ DATA_LABEL, DIST_INFO_LABEL, EXTRACTED_WHEEL_FILES, - ] - ] + ] + [ + "bin/{}".format(entry_point) + for entry_point in entry_points + ], + ) if config_load: loads.append("""load("{}", "{}")""".format(config_load, "packages")) from_requires_kwargs["include"] = "packages" - macro_parts.append(render.call( + macro_parts = [render.call( "whl_library_from_requires_dist", **_render(**from_requires_kwargs) - )) + )] contents = _TEMPLATE.format( loads = "\n".join(loads), diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index fb246538c8..c64765a075 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -442,7 +442,7 @@ def _whl_library_impl(rctx): sdist_filename = sdist_filename, dep_template = rctx.attr.dep_template or "@{}{{name}}//:{{target}}".format( rctx.attr.repo_prefix, - ), + ) if rctx.attr.dep_template or rctx.attr.repo_prefix else "", config_load = rctx.attr.config_load, metadata_name = metadata.name, metadata_version = metadata.version, @@ -631,6 +631,11 @@ def _whl_library_deps_impl(rctx): # Load the METADATA file from a different repository metadata_path = rctx.path(rctx.attr.whl_library.same_package_label("METADATA")) metadata = parse_whl_metadata(rctx.read(metadata_path)) + entry_points = _get_entry_points( + rctx, + rctx.path(metadata_path).dirname.get_child("site-packages"), + metadata, + ) if not (metadata.name and metadata.version): logger.fail("Failed to parse METADATA from {}".format(rctx.attr.whl_library)) @@ -644,6 +649,7 @@ def _whl_library_deps_impl(rctx): extras = rctx.attr.extras, group_deps = rctx.attr.group_deps, group_name = rctx.attr.group_name, + entry_points = [e["name"] for e in entry_points.values()], requires_dist = metadata.requires_dist, whl_library = rctx.attr.whl_library, ) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 917685fb0b..8d8ce6a510 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -120,6 +120,7 @@ def whl_library_targets( dependencies_with_markers = {}, group_name = "", native = native, + aliases = [], src_pkg = Label("//:BUILD.bazel"), rules = struct( copy_file = copy_file, @@ -231,6 +232,13 @@ def whl_library_targets( visibility = impl_vis, ) + for target in aliases: + native.alias( + name = target, + actual = src_pkg.same_package_label(target), + visibility = ["//visibility:public"], + ) + def _config_settings(dependencies_with_markers, rules, **kwargs): """Generate config settings for the targets. From 3608e1d52193536a12d251bcaeb5fccc771edfbe Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Jul 2026 14:35:02 +0900 Subject: [PATCH 04/13] fixup --- python/private/pypi/extension.bzl | 4 +-- .../pypi/generate_whl_library_build_bazel.bzl | 30 +++++++++++++++++-- python/private/pypi/hub_builder.bzl | 18 +++++++---- python/private/pypi/whl_library_targets.bzl | 1 + 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl index 7f68f1f6f4..59f0fc47ae 100644 --- a/python/private/pypi/extension.bzl +++ b/python/private/pypi/extension.bzl @@ -474,8 +474,7 @@ You cannot use both the additive_build_content and additive_build_content_file a if whl_name in whl_libraries: existing = whl_libraries[whl_name] - # TODO @aignas 2026-07-04: stop ignoring the index_url - diff = _diff_dict(existing, lib, ignore_keys = {"index_url": True}) + diff = _diff_dict(existing, lib) if diff: fail("'{}' already in created:\n{}".format( whl_name, @@ -1251,6 +1250,7 @@ def _diff_dict(first, second, *, ignore_keys = {}): Args: first: The first dictionary to compare. second: The second dictionary to compare. + ignore_keys: A set of keys to ignore during comparison. Returns: A dictionary containing the differences, with keys "common", "different", diff --git a/python/private/pypi/generate_whl_library_build_bazel.bzl b/python/private/pypi/generate_whl_library_build_bazel.bzl index 407d3e8428..35c9af8567 100644 --- a/python/private/pypi/generate_whl_library_build_bazel.bzl +++ b/python/private/pypi/generate_whl_library_build_bazel.bzl @@ -77,10 +77,26 @@ def generate_whl_library_build_bazel( """Generate a BUILD file for an unzipped Wheel Args: + name: The name of the target. annotation: The annotation for the build file. config_load: {type}`str` The location from where to load the config. + copy_executables: A mapping of file paths to executable names. + copy_files: A mapping of file paths to file names. + data_exclude: A list of files to exclude from data. + dep_template: The template for dependencies. + enable_implicit_namespace_pkgs: Whether to enable implicit namespace packages. + entry_points: A mapping of entry points. + extras: A list of extras. + group_deps: A list of grouped dependencies. + group_name: The name of the group. + metadata_name: The name of the package. + metadata_version: The version of the package. + namespace_package_files: A mapping of namespace package files. purl: The purl. requires_dist: {type}`list[str]` The list of dependencies from the METADATA file. + sdist_filename: The filename of the sdist. + srcs_exclude: A list of source files to exclude. + visibility: The visibility of the target. **kwargs: Extra args serialized to be passed to the {obj}`whl_library_targets`. @@ -182,10 +198,20 @@ def generate_whl_library_deps_build_bazel( group_deps, group_name, requires_dist, - whl_library, - **kwargs): + whl_library): """Generate a BUILD file for an unzipped Wheel + Args: + name: The name of the target. + version: The version of the package. + config_load: The location from where to load the config. + dep_template: The template for dependencies. + entry_points: A mapping of entry points. + extras: A list of extras. + group_deps: A list of grouped dependencies. + group_name: The name of the group. + requires_dist: The list of dependencies from the METADATA file. + whl_library: The wheel library target. Returns: A complete BUILD file as a string diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 0c4fa6c0d9..8fa66f2c33 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -90,6 +90,7 @@ def hub_builder( # Functions to download according to the config # dict[str python_version, callable] _get_index_urls = {}, + _default_index_url = {}, # Tells whether to use the downloader for a package. # dict[str python_version, dict[str package_name, bool use_downloader]] _use_downloader = {}, @@ -385,24 +386,25 @@ def _add_library(self, *, repos_dict, name, args): repos_dict[name] = args def _set_get_index_urls(self, mctx, pip_attr): + python_version = pip_attr.python_version + # Resolve the index URL through envsubst so the ``$VAR`` / ``${VAR:-default}`` # form is honored when deciding whether the experimental index-url mode is # active. Without this, an unsubstituted template like ``$RULES_PYTHON_PIP_INDEX_URL`` # is treated as truthy and the mode is forced on, even when the env var # would expand to the empty string. - default_index_url = envsubst( + self._default_index_url[python_version] = envsubst( pip_attr.experimental_index_url, pip_attr.envsubst, mctx.getenv, ) or self._config.index_url default_extra_index_urls = pip_attr.experimental_extra_index_urls or [] - if not default_index_url: + if not self._default_index_url[python_version]: # parallel_download is set to True by default, so we are not checking/validating it # here return False - python_version = pip_attr.python_version self._use_downloader.setdefault(python_version, {}).update({ normalize_name(s): False for s in pip_attr.simpleapi_skip @@ -410,7 +412,7 @@ def _set_get_index_urls(self, mctx, pip_attr): self._get_index_urls[python_version] = lambda ctx, distributions, *, index_url = None, extra_index_urls = None: self._simpleapi_download_fn( ctx, attr = struct( - index_url = (index_url or default_index_url).rstrip("/"), + index_url = (index_url or self._default_index_url[python_version]).rstrip("/"), extra_index_urls = [ x.rstrip("/") for x in (extra_index_urls or default_extra_index_urls) @@ -579,7 +581,13 @@ def _create_whl_repos( for src in whl.srcs: repo = _whl_repo( src = src, - index_url = whl.index_url, + index_url = ( + whl.index_url or + "{}/{}".format( + self._default_index_url[python_version], + whl.name, + ) + ).rstrip("/"), whl_library_args = whl_library_args, download_only = pip_attr.download_only, netrc = self._config.netrc or pip_attr.netrc, diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 8d8ce6a510..1b7b1b8756 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -146,6 +146,7 @@ def whl_library_targets( contains this library. If set, this library will behave as a shim to group implementation rules which will provide simultaneously installed dependencies which would otherwise form a cycle. + aliases: {type}`list[str]` A list of aliases to create for the target. src_pkg: TODO native: {type}`native` The native struct for overriding in tests. rules: {type}`struct` A struct with references to rules for creating targets. From 67b05581c6446905b3f30455d6e6d58b9a53a533 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Jul 2026 14:35:37 +0900 Subject: [PATCH 05/13] add a note --- python/private/pypi/hub_builder.bzl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 8fa66f2c33..ce9ac00e73 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -581,6 +581,8 @@ def _create_whl_repos( for src in whl.srcs: repo = _whl_repo( src = src, + # TODO @aignas 2026-07-04: add a test to ensure that overriding the default + # index url overrides the values here. index_url = ( whl.index_url or "{}/{}".format( From 983aa13cb6d357bb618119867816cd4cc54abdf7 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Jul 2026 14:40:01 +0900 Subject: [PATCH 06/13] fixup --- python/private/pypi/hub_builder.bzl | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index ce9ac00e73..fa51c60182 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -332,15 +332,19 @@ def _add_whl_library(self, *, python_version, whl, repo): repos_dict = self._whl_libraries deps_args = repo.args else: - whl_repo_name = "whl_{}".format(repo.whl_repo_name) - _add_library(self, repos_dict = self._whl_libraries, name = whl_repo_name, args = { - k: v - for k, v in repo.args.items() - if k not in forbidden_args | { - "config_load": None, - "dep_template": None, - } - }) + _add_library( + self, + repos_dict = self._whl_libraries, + name = repo.whl_repo_name, + args = { + k: v + for k, v in repo.args.items() + if k not in forbidden_args | { + "config_load": None, + "dep_template": None, + } + }, + ) args = repo.args From c1caf89572cd5c106e023b6a9dab92ea6427c2af Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Jul 2026 15:09:06 +0900 Subject: [PATCH 07/13] address todos --- python/private/pypi/whl_library_targets.bzl | 3 +- ...generate_whl_library_build_bazel_tests.bzl | 258 +++++++++++++----- 2 files changed, 193 insertions(+), 68 deletions(-) diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index 1b7b1b8756..d29bda4d5d 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -147,7 +147,8 @@ def whl_library_targets( to group implementation rules which will provide simultaneously installed dependencies which would otherwise form a cycle. aliases: {type}`list[str]` A list of aliases to create for the target. - src_pkg: TODO + src_pkg: {type}`Label` The label of the repository where the wheel files + are located. native: {type}`native` The native struct for overriding in tests. rules: {type}`struct` A struct with references to rules for creating targets. """ diff --git a/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl index 1fd99205b1..42e80da8b3 100644 --- a/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl +++ b/tests/pypi/generate_whl_library_build_bazel/generate_whl_library_build_bazel_tests.bzl @@ -15,15 +15,15 @@ "" load("@rules_testing//lib:test_suite.bzl", "test_suite") -load("//python/private/pypi:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") # buildifier: disable=bzl-visibility +load("//python/private/pypi:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel", "generate_whl_library_deps_build_bazel") # buildifier: disable=bzl-visibility _tests = [] def _test_all_workspace(env): want = """\ load("@package_metadata//rules:package_metadata.bzl", "package_metadata") +load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_srcs", "whl_library_from_requires_dist") load("@pypi//:config.bzl", "packages") -load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets_from_requires") package(default_visibility = ["//visibility:public"]) @@ -33,36 +33,42 @@ package_metadata( visibility = ["//:__subpackages__"], ) -whl_library_targets_from_requires( - copy_executables = { - "exec_src": "exec_dest", - }, - copy_files = { - "file_src": "file_dest", - }, - data = ["extra_target"], - data_exclude = [ - "exclude_via_attr", - "data_exclude_all", +whl_library_srcs( + name = "foo.whl", + data = [], + sdist_filename = None, + data_exclude = ["exclude_via_attr"], + srcs_exclude = [], + tags = [ + "pypi_name=foo", + "pypi_version=1.0.0", ], - dep_template = "@pypi//{name}:{target}", + entry_points = {}, + enable_implicit_namespace_pkgs = False, + copy_files = {}, + copy_executables = {}, + namespace_package_files = {}, + visibility = ["//visibility:public"], +) + +whl_library_from_requires_dist( + name = "foo", + version = "1.0.0", + requires_dist = [ + "foo", + "bar-baz", + "qux", + ], + extras = [], group_deps = [ "foo", "fox", "qux", ], + dep_template = "@pypi//{name}:{target}", group_name = "qux", include = packages, - name = "foo.whl", - requires_dist = [ - "foo", - "bar-baz", - "qux", - ], - srcs_exclude = ["srcs_exclude_all"], ) - -# SOMETHING SPECIAL AT THE END """ actual = generate_whl_library_build_bazel( dep_template = "@pypi//{name}:{target}", @@ -80,7 +86,12 @@ whl_library_targets_from_requires( config_load = "@pypi//:config.bzl", group_name = "qux", group_deps = ["foo", "fox", "qux"], + metadata_name = "foo", + metadata_version = "1.0.0", ) + + # Strip the trailing newline and the additional_content from actual + actual = actual.split("# SOMETHING SPECIAL AT THE END")[0].rstrip() + "\n" env.expect.that_str(actual.replace("@@", "@")).equals(want) _tests.append(_test_all_workspace) @@ -88,8 +99,8 @@ _tests.append(_test_all_workspace) def _test_all(env): want = """\ load("@package_metadata//rules:package_metadata.bzl", "package_metadata") +load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_srcs", "whl_library_from_requires_dist") load("@pypi//:config.bzl", "packages") -load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets_from_requires") package(default_visibility = ["//visibility:public"]) @@ -99,36 +110,42 @@ package_metadata( visibility = ["//:__subpackages__"], ) -whl_library_targets_from_requires( - copy_executables = { - "exec_src": "exec_dest", - }, - copy_files = { - "file_src": "file_dest", - }, - data = ["extra_target"], - data_exclude = [ - "exclude_via_attr", - "data_exclude_all", +whl_library_srcs( + name = "foo.whl", + data = [], + sdist_filename = None, + data_exclude = ["exclude_via_attr"], + srcs_exclude = [], + tags = [ + "pypi_name=foo", + "pypi_version=1.0.0", ], - dep_template = "@pypi//{name}:{target}", + entry_points = {}, + enable_implicit_namespace_pkgs = False, + copy_files = {}, + copy_executables = {}, + namespace_package_files = {}, + visibility = ["//visibility:public"], +) + +whl_library_from_requires_dist( + name = "foo", + version = "1.0.0", + requires_dist = [ + "foo", + "bar-baz", + "qux", + ], + extras = [], group_deps = [ "foo", "fox", "qux", ], + dep_template = "@pypi//{name}:{target}", group_name = "qux", include = packages, - name = "foo.whl", - requires_dist = [ - "foo", - "bar-baz", - "qux", - ], - srcs_exclude = ["srcs_exclude_all"], ) - -# SOMETHING SPECIAL AT THE END """ actual = generate_whl_library_build_bazel( dep_template = "@pypi//{name}:{target}", @@ -146,7 +163,12 @@ whl_library_targets_from_requires( config_load = "@pypi//:config.bzl", group_name = "qux", group_deps = ["foo", "fox", "qux"], + metadata_name = "foo", + metadata_version = "1.0.0", ) + + # Strip the trailing newline and the additional_content from actual + actual = actual.split("# SOMETHING SPECIAL AT THE END")[0].rstrip() + "\n" env.expect.that_str(actual.replace("@@", "@")).equals(want) _tests.append(_test_all) @@ -154,8 +176,8 @@ _tests.append(_test_all) def _test_all_with_loads(env): want = """\ load("@package_metadata//rules:package_metadata.bzl", "package_metadata") +load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_srcs", "whl_library_from_requires_dist") load("@pypi//:config.bzl", "packages") -load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_targets_from_requires") package(default_visibility = ["//visibility:public"]) @@ -165,36 +187,42 @@ package_metadata( visibility = ["//:__subpackages__"], ) -whl_library_targets_from_requires( - copy_executables = { - "exec_src": "exec_dest", - }, - copy_files = { - "file_src": "file_dest", - }, - data = ["extra_target"], - data_exclude = [ - "exclude_via_attr", - "data_exclude_all", +whl_library_srcs( + name = "foo.whl", + data = [], + sdist_filename = None, + data_exclude = ["exclude_via_attr"], + srcs_exclude = [], + tags = [ + "pypi_name=foo", + "pypi_version=1.0.0", ], - dep_template = "@pypi//{name}:{target}", + entry_points = {}, + enable_implicit_namespace_pkgs = False, + copy_files = {}, + copy_executables = {}, + namespace_package_files = {}, + visibility = ["//visibility:public"], +) + +whl_library_from_requires_dist( + name = "foo", + version = "1.0.0", + requires_dist = [ + "foo", + "bar-baz", + "qux", + ], + extras = [], group_deps = [ "foo", "fox", "qux", ], + dep_template = "@pypi//{name}:{target}", group_name = "qux", include = packages, - name = "foo.whl", - requires_dist = [ - "foo", - "bar-baz", - "qux", - ], - srcs_exclude = ["srcs_exclude_all"], ) - -# SOMETHING SPECIAL AT THE END """ actual = generate_whl_library_build_bazel( dep_template = "@pypi//{name}:{target}", @@ -212,11 +240,107 @@ whl_library_targets_from_requires( group_name = "qux", config_load = "@pypi//:config.bzl", group_deps = ["foo", "fox", "qux"], + metadata_name = "foo", + metadata_version = "1.0.0", ) + + # Strip the trailing newline and the additional_content from actual + actual = actual.split("# SOMETHING SPECIAL AT THE END")[0].rstrip() + "\n" env.expect.that_str(actual.replace("@@", "@")).equals(want) _tests.append(_test_all_with_loads) +def _test_generate_whl_library_deps_build_bazel(env): + want = """\ +load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_from_requires_dist") + +package(default_visibility = ["//visibility:public"]) + +whl_library_from_requires_dist( + name = "foo", + version = "1.0.0", + requires_dist = [], + extras = [], + group_deps = [], + dep_template = "template", + group_name = None, + src_pkg = "@//:pkg", + aliases = [ + "data", + "dist_info", + "extracted_whl_files", + ], +) +""" + actual = generate_whl_library_deps_build_bazel( + name = "foo", + version = "1.0.0", + config_load = None, + dep_template = "template", + entry_points = [], + extras = [], + group_deps = [], + group_name = None, + requires_dist = [], + whl_library = "@@//:pkg", + ) + env.expect.that_str(actual.replace("@@", "@")).equals(want) + +def _test_no_annotation(env): + want = """\ +load("@package_metadata//rules:package_metadata.bzl", "package_metadata") +load("@rules_python//python/private/pypi:whl_library_targets.bzl", "whl_library_srcs", "whl_library_from_requires_dist") + +package(default_visibility = ["//visibility:public"]) + +package_metadata( + name = "package_metadata", + purl = None, + visibility = ["//:__subpackages__"], +) + +whl_library_srcs( + name = "foo.whl", + data = [], + sdist_filename = None, + data_exclude = [], + srcs_exclude = [], + tags = [ + "pypi_name=foo", + "pypi_version=1.0.0", + ], + entry_points = {}, + enable_implicit_namespace_pkgs = False, + copy_files = {}, + copy_executables = {}, + namespace_package_files = {}, + visibility = ["//visibility:public"], +) + +whl_library_from_requires_dist( + name = "foo", + version = "1.0.0", + requires_dist = ["foo"], + extras = [], + group_deps = [], + dep_template = "@pypi//{name}:{target}", + group_name = None, +) +""" + actual = generate_whl_library_build_bazel( + dep_template = "@pypi//{name}:{target}", + name = "foo.whl", + requires_dist = ["foo"], + annotation = None, + config_load = None, + metadata_name = "foo", + metadata_version = "1.0.0", + ) + env.expect.that_str(actual).equals(want) + +_tests.append(_test_generate_whl_library_deps_build_bazel) +_tests.append(_test_no_annotation) + def generate_whl_library_build_bazel_test_suite(name): """Create the test suite. From a014b8a8b3c0d889d5c1489bfa873b393cf67291 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Jul 2026 16:06:10 +0900 Subject: [PATCH 08/13] fixup - add index_url handling to uv.lock --- python/private/pypi/hub_builder.bzl | 15 ++++----------- python/private/pypi/parse_requirements.bzl | 8 +++++++- python/private/pypi/simpleapi_download.bzl | 3 +++ tests/pypi/hub_builder/hub_builder_tests.bzl | 2 +- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index fa51c60182..237fa501e2 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -353,7 +353,7 @@ def _add_whl_library(self, *, python_version, whl, repo): if key in args and args[key] != None: deps_args[key] = args[key] - deps_args["whl_library"] = "@{}//:BUILD.bazel".format(whl_repo_name) + deps_args["whl_library"] = "@{}//:BUILD.bazel".format(repo.whl_repo_name) repos_dict = self._whl_library_deps repo_name = "{}_{}_{}".format(self.name, version_label(python_version), repo.repo_name) @@ -379,13 +379,14 @@ def _add_library(self, *, repos_dict, name, args): if diff: self._logger.fail(lambda: ( "Attempting to create a duplicate library {name} with different arguments. Already existing declaration has:\n".format( - repo_name = name, + name = name, ) + "\n".join([ " {}: {}".format(key, render.indent(render.dict(value)).lstrip()) for key, value in diff.items() if value ]) )) + return repos_dict[name] = args @@ -585,15 +586,7 @@ def _create_whl_repos( for src in whl.srcs: repo = _whl_repo( src = src, - # TODO @aignas 2026-07-04: add a test to ensure that overriding the default - # index url overrides the values here. - index_url = ( - whl.index_url or - "{}/{}".format( - self._default_index_url[python_version], - whl.name, - ) - ).rstrip("/"), + index_url = whl.index_url.rstrip("/"), whl_library_args = whl_library_args, download_only = pip_attr.download_only, netrc = self._config.netrc or pip_attr.netrc, diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index fd0399fc86..c87c586f7e 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -176,6 +176,12 @@ def _parse_uv_lock_json(uv_lock, all_platforms, logger, extra_pip_args = None, p "versions": {}, }) entry["versions"][version] = None + registry = pkg.get("source", {}).get("registry", "") + if registry.rstrip("/").endswith("simple"): + index_url = "{}/{}".format(registry, norm_name.replace("_", "-")) + else: + index_url = "" + entry["index_url"] = index_url pkg_extras = sorted(extras_map.get(name, [])) extra_str = "[{}]".format(",".join(pkg_extras)) if pkg_extras else "" @@ -287,7 +293,7 @@ def _parse_uv_lock_json(uv_lock, all_platforms, logger, extra_pip_args = None, p name = norm_name, is_exposed = True, is_multiple_versions = len(versions) > 1, - index_url = "", + index_url = info["index_url"], srcs = info["resolved_srcs"], ) ret.append(item) diff --git a/python/private/pypi/simpleapi_download.bzl b/python/private/pypi/simpleapi_download.bzl index 5377a08093..03c9dc3754 100644 --- a/python/private/pypi/simpleapi_download.bzl +++ b/python/private/pypi/simpleapi_download.bzl @@ -291,6 +291,9 @@ def _with_index_url(index_url, values): if not values: return values + if not index_url: + fail("BUG: no index_url") + return struct( sdists = values.sdists, whls = values.whls, diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index 60017593fb..ca5c77b97c 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -1498,7 +1498,7 @@ def _test_err_duplicate_repos(env): env.expect.that_dict(logs).keys().contains_exactly(["rules_python:unit-test FAIL:"]) env.expect.that_collection(logs["rules_python:unit-test FAIL:"]).contains_exactly([ """\ -Attempting to create a duplicate library pypi_315_foo for foo with different arguments. Already existing declaration has: +Attempting to create a duplicate library pypi_315_foo with different arguments. Already existing declaration has: common: { "dep_template": "@pypi//{name}:{target}", "config_load": "@pypi//:config.bzl", From e97cb9e108db9d926c6968fb94b1c9cdadc89a19 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Jul 2026 17:23:01 +0900 Subject: [PATCH 09/13] fix --- python/private/pypi/hub_builder.bzl | 18 +++- tests/pypi/hub_builder/hub_builder_tests.bzl | 84 +++++++------------ .../parse_requirements_tests.bzl | 36 ++++---- 3 files changed, 63 insertions(+), 75 deletions(-) diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 237fa501e2..292904bbb0 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -578,6 +578,16 @@ def _create_whl_repos( interpreter = _detect_interpreter(self, pip_attr, python_version) for whl in requirements_by_platform: + # Check if all sources for this wheel have the same requirement line. + # If they do, we can reuse the same whl_library repository across platforms. + same_requirements = True + if whl.srcs: + first_req = whl.srcs[0].requirement_line + for src in whl.srcs[1:]: + if src.requirement_line != first_req: + same_requirements = False + break + whl_library_args = common_args | _whl_library_args( self, whl = whl, @@ -594,6 +604,7 @@ def _create_whl_repos( auth_patterns = self._config.auth_patterns or pip_attr.auth_patterns, python_version = _major_minor_version(python_version), is_multiple_versions = whl.is_multiple_versions, + same_requirements = same_requirements, interpreter = interpreter, enable_pipstar_extract = enable_pipstar_extract, ) @@ -664,6 +675,7 @@ def _whl_repo( whl_library_args, index_url, is_multiple_versions, + same_requirements, download_only, netrc, auth_patterns, @@ -695,7 +707,7 @@ def _whl_repo( return None else: # Fallback to a pip-installed wheel - target_platforms = src.target_platforms if is_multiple_versions else [] + target_platforms = src.target_platforms if (is_multiple_versions or not same_requirements) else [] return struct( repo_name = pypi_repo_name( normalize_name(src.distribution), @@ -725,11 +737,11 @@ def _whl_repo( # TODO @aignas 2025-11-02: once we have pipstar enabled we can add extra # targets to each hub for each extra combination and solve this more cleanly as opposed to # duplicating whl_library repositories. - target_platforms = src.target_platforms if is_multiple_versions else [] + target_platforms = src.target_platforms if (is_multiple_versions or not same_requirements) else [] return struct( repo_name = whl_repo_name(src.filename, src.sha256, *target_platforms), - whl_repo_name = whl_repo_name(src.filename, src.sha256), + whl_repo_name = whl_repo_name(src.filename, src.sha256, *target_platforms), args = args, config_setting = whl_config_setting( version = python_version, diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index ca5c77b97c..67bc07dc6e 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -310,20 +310,16 @@ def _test_simple_extras_vs_no_extras_simpleapi(env): }, }) pypi.whl_libraries().contains_exactly({ - "pypi_315_simple_py3_none_any_deadbeef_osx_aarch64": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", + "simple_py3_none_any_deadbeef_osx_aarch64": { "filename": "simple-0.0.1-py3-none-any.whl", - "index_url": "https://example.com/simple/", + "index_url": "https://example.com/simple", "requirement": "simple[foo]==0.0.1", "sha256": "deadbeef", "urls": ["/simple-0.0.1-py3-none-any.whl"], }, - "pypi_315_simple_py3_none_any_deadbeef_windows_aarch64": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", + "simple_py3_none_any_deadbeef_windows_aarch64": { "filename": "simple-0.0.1-py3-none-any.whl", - "index_url": "https://example.com/simple/", + "index_url": "https://example.com/simple", "requirement": "simple==0.0.1", "sha256": "deadbeef", "urls": ["/simple-0.0.1-py3-none-any.whl"], @@ -642,38 +638,30 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, }) pypi.whl_libraries().contains_exactly({ - "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef_linux_x86_64": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", + "torch_cp312_cp312_linux_x86_64_8800deef_linux_x86_64": { "filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", - "index_url": "https://torch.index/torch/", + "index_url": "https://torch.index/torch", "requirement": "torch==2.4.1+cpu", "sha256": "8800deef0026011d502c0c256cc4b67d002347f63c3a38cd8e45f1f445c61364", "urls": ["/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-linux_x86_64.whl"], }, - "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432_linux_aarch64": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", + "torch_cp312_cp312_manylinux_2_17_aarch64_36109432_linux_aarch64": { "filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "index_url": "https://torch.index/torch/", + "index_url": "https://torch.index/torch", "requirement": "torch==2.4.1", "sha256": "36109432b10bd7163c9b30ce896f3c2cca1b86b9765f956a1594f0ff43091e2a", "urls": ["/whl/cpu/torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"], }, - "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c_windows_x86_64": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", + "torch_cp312_cp312_win_amd64_3a570e5c_windows_x86_64": { "filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", - "index_url": "https://torch.index/torch/", + "index_url": "https://torch.index/torch", "requirement": "torch==2.4.1+cpu", "sha256": "3a570e5c553415cdbddfe679207327b3a3806b21c6adea14fba77684d1619e97", "urls": ["/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-win_amd64.whl"], }, - "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5_osx_aarch64": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", + "torch_cp312_none_macosx_11_0_arm64_72b484d5_osx_aarch64": { "filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", - "index_url": "https://torch.index/torch/", + "index_url": "https://torch.index/torch", "requirement": "torch==2.4.1", "sha256": "72b484d5b6cec1a735bf3fa5a1c4883d01748698c5e9cfdbeb4ffab7c7987e0d", "urls": ["/whl/cpu/torch-2.4.1-cp312-none-macosx_11_0_arm64.whl"], @@ -813,8 +801,6 @@ simple==0.0.1 --hash=sha256:deadb00f }, }) want_whl_library = { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", "filename": "simple-0.0.1-py3-none-any.whl", "index_url": test.expect_index_url, "requirement": "simple==0.0.1", @@ -824,7 +810,7 @@ simple==0.0.1 --hash=sha256:deadb00f if getattr(test, "envsubst", []): want_whl_library["envsubst"] = test.envsubst pypi.whl_libraries().contains_exactly({ - "pypi_315_simple_py3_none_any_deadb00f": want_whl_library, + "simple_py3_none_any_deadb00f": want_whl_library, }) pypi.extra_aliases().contains_exactly({}) @@ -1156,6 +1142,20 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef }, }) pypi.whl_libraries().contains_exactly({ + "direct_without_sha_0_0_1_py3_none_any": { + "filename": "direct_without_sha-0.0.1-py3-none-any.whl", + "requirement": "direct_without_sha==0.0.1", + "sha256": "", + "urls": ["example-direct.org/direct_without_sha-0.0.1-py3-none-any.whl"], + "whl_patches": {"my_patch": "1"}, + }, + "plat_py3_none_linux_x86_64_deadb44f": { + "filename": "plat-pkg-0.0.4-py3-none-linux_x86_64.whl", + "index_url": "https://pypi.org/simple", + "requirement": "plat_pkg==0.0.4", + "sha256": "deadb44f", + "urls": ["example2.org/index/plat_pkg/"], + }, "pypi_315_any_name": { "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", @@ -1166,15 +1166,6 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "sha256": "", "urls": ["some-archive/any-name.tar.gz"], }, - "pypi_315_direct_without_sha_0_0_1_py3_none_any": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", - "filename": "direct_without_sha-0.0.1-py3-none-any.whl", - "requirement": "direct_without_sha==0.0.1", - "sha256": "", - "urls": ["example-direct.org/direct_without_sha-0.0.1-py3-none-any.whl"], - "whl_patches": {"my_patch": "1"}, - }, "pypi_315_git_dep": { "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", @@ -1189,35 +1180,20 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "python_interpreter_target": "unit_test_interpreter_target", "requirement": "pip_fallback==0.0.1", }, - "pypi_315_plat_py3_none_linux_x86_64_deadb44f": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", - "filename": "plat-pkg-0.0.4-py3-none-linux_x86_64.whl", - "index_url": "https://pypi.org/simple", - "requirement": "plat_pkg==0.0.4", - "sha256": "deadb44f", - "urls": ["example2.org/index/plat_pkg/"], - }, - "pypi_315_simple_py3_none_any_deadb00f": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", + "simple_py3_none_any_deadb00f": { "filename": "simple-0.0.1-py3-none-any.whl", "index_url": "https://pypi.org/simple", "requirement": "simple==0.0.1", "sha256": "deadb00f", "urls": ["example2.org"], }, - "pypi_315_some_pkg_py3_none_any_deadbaaf": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", + "some_pkg_py3_none_any_deadbaaf": { "filename": "some_pkg-0.0.1-py3-none-any.whl", "requirement": "some_pkg==0.0.1", "sha256": "deadbaaf", "urls": ["example-direct.org/some_pkg-0.0.1-py3-none-any.whl"], }, - "pypi_315_some_py3_none_any_deadb33f": { - "config_load": "@pypi//:config.bzl", - "dep_template": "@pypi//{name}:{target}", + "some_py3_none_any_deadb33f": { "filename": "some-other-pkg-0.0.1-py3-none-any.whl", "index_url": "https://with_index_url", "requirement": "some_other_pkg==0.0.1", diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl index 8dd35c2260..576a31ae5d 100644 --- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl @@ -1052,7 +1052,7 @@ def _test_uv_lock_consistent(env): env.expect.that_collection(got).contains_exactly([ struct( name = "foo", - index_url = "", + index_url = "https://pypi.org/simple/foo", is_exposed = True, is_multiple_versions = False, srcs = [ @@ -1080,7 +1080,7 @@ def _test_uv_lock_primary_source(env): env.expect.that_collection(got).contains_exactly([ struct( name = "foo", - index_url = "", + index_url = "https://pypi.org/simple/foo", is_exposed = True, is_multiple_versions = False, srcs = [ @@ -1108,7 +1108,7 @@ def _test_uv_lock_primary_source_multiple_versions(env): env.expect.that_collection(got).contains_exactly([ struct( name = "foo", - index_url = "", + index_url = "https://pypi.org/simple/foo", is_exposed = True, is_multiple_versions = True, srcs = [ @@ -1146,7 +1146,7 @@ def _test_uv_lock_primary_source_with_extras(env): env.expect.that_collection(got).contains_exactly([ struct( name = "foo", - index_url = "", + index_url = "https://pypi.org/simple/foo", is_exposed = True, is_multiple_versions = False, srcs = [ @@ -1174,7 +1174,7 @@ def _test_uv_lock_primary_source_includes_virtual(env): env.expect.that_collection(got).contains_exactly([ struct( name = "foo", - index_url = "", + index_url = "https://pypi.org/simple/foo", is_exposed = True, is_multiple_versions = False, srcs = [ @@ -1212,7 +1212,7 @@ def _test_uv_lock_cross_consistent(env): env.expect.that_collection(got).contains_exactly([ struct( name = "foo", - index_url = "", + index_url = "https://pypi.org/simple/foo", is_exposed = True, is_multiple_versions = False, srcs = [ @@ -1268,7 +1268,7 @@ def _test_uv_lock_rules_python_pkg_not_skipped(env): env.expect.that_collection(got).contains_exactly([ struct( name = "rules_python", - index_url = "", + index_url = "https://pypi.org/simple/rules-python", is_exposed = True, is_multiple_versions = False, srcs = [ @@ -1301,7 +1301,7 @@ def _test_uv_lock_no_consistency_check(env): env.expect.that_collection(got).contains_exactly([ struct( name = "foo", - index_url = "", + index_url = "https://pypi.org/simple/foo", is_exposed = True, is_multiple_versions = False, srcs = [ @@ -1329,7 +1329,7 @@ def _test_uv_lock_multiple_packages(env): env.expect.that_collection(got).contains_exactly([ struct( name = "bar", - index_url = "", + index_url = "https://pypi.org/simple/bar", is_exposed = True, is_multiple_versions = False, srcs = [ @@ -1347,7 +1347,7 @@ def _test_uv_lock_multiple_packages(env): ), struct( name = "foo", - index_url = "", + index_url = "https://pypi.org/simple/foo", is_exposed = True, is_multiple_versions = False, srcs = [ @@ -1376,7 +1376,7 @@ def _test_uv_lock_with_extra_pip_args(env): env.expect.that_collection(got).contains_exactly([ struct( name = "foo", - index_url = "", + index_url = "https://pypi.org/simple/foo", is_exposed = True, is_multiple_versions = False, srcs = [ @@ -1408,7 +1408,7 @@ def _test_uv_lock_multi_os_with_requirements(env): env.expect.that_collection(got).contains_exactly([ struct( name = "foo", - index_url = "", + index_url = "https://pypi.org/simple/foo", is_exposed = True, is_multiple_versions = False, srcs = [ @@ -1436,7 +1436,7 @@ def _test_uv_lock_extras_optional_deps(env): env.expect.that_collection(got).contains_exactly([ struct( name = "foo", - index_url = "", + index_url = "https://pypi.org/simple/foo", is_exposed = True, is_multiple_versions = False, srcs = [ @@ -1464,7 +1464,7 @@ def _test_uv_lock_extras_dep_edge(env): env.expect.that_collection(got).contains_exactly([ struct( name = "bar", - index_url = "", + index_url = "https://pypi.org/simple/bar", is_exposed = True, is_multiple_versions = False, srcs = [ @@ -1482,7 +1482,7 @@ def _test_uv_lock_extras_dep_edge(env): ), struct( name = "foo", - index_url = "", + index_url = "https://pypi.org/simple/foo", is_exposed = True, is_multiple_versions = False, srcs = [ @@ -1517,7 +1517,7 @@ def _test_uv_lock_wheel_dedup_single_version(env): env.expect.that_collection(got).contains_exactly([ struct( name = "foo", - index_url = "", + index_url = "https://pypi.org/simple/foo", is_exposed = True, is_multiple_versions = False, srcs = [ @@ -1563,7 +1563,7 @@ def _test_uv_lock_wheel_dedup_resolution_markers(env): env.expect.that_collection(got).contains_exactly([ struct( name = "foo", - index_url = "", + index_url = "https://pypi.org/simple/foo", is_exposed = True, is_multiple_versions = True, srcs = [ @@ -1601,7 +1601,7 @@ def _test_uv_lock_requires_dist_extras(env): env.expect.that_collection(got).contains_exactly([ struct( name = "foo", - index_url = "", + index_url = "https://pypi.org/simple/foo", is_exposed = True, is_multiple_versions = False, srcs = [ From 463713237ebc1ecb25cb529612ee88aec00c7559 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Jul 2026 17:50:52 +0900 Subject: [PATCH 10/13] wip --- python/private/pypi/hub_builder.bzl | 43 ++++++++++++-------- python/private/pypi/whl_library.bzl | 8 ---- tests/pypi/hub_builder/hub_builder_tests.bzl | 37 ++++++----------- 3 files changed, 40 insertions(+), 48 deletions(-) diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index 292904bbb0..d8611b7938 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -11,6 +11,7 @@ load(":attrs.bzl", "use_isolated") load(":parse_requirements.bzl", "parse_requirements") load(":pep508_env.bzl", "env") load(":pep508_evaluate.bzl", "evaluate") +load(":pep508_requirement.bzl", "requirement") load(":python_tag.bzl", "python_tag") load(":requirements_files_by_platform.bzl", "requirements_files_by_platform") load(":whl_config_setting.bzl", "whl_config_setting") @@ -332,28 +333,38 @@ def _add_whl_library(self, *, python_version, whl, repo): repos_dict = self._whl_libraries deps_args = repo.args else: + extract_args = { + k: v + for k, v in repo.args.items() + if k not in forbidden_args | { + "config_load": None, + "dep_template": None, + } + } + req = requirement(extract_args["requirement"]) + + # TODO @aignas 2026-07-04: add a test + extract_args["requirement"] = req.name # drop any specified extras _add_library( self, repos_dict = self._whl_libraries, name = repo.whl_repo_name, - args = { - k: v - for k, v in repo.args.items() - if k not in forbidden_args | { - "config_load": None, - "dep_template": None, - } - }, + args = extract_args, ) - args = repo.args - - deps_args = {} - for key in ("config_load", "dep_template", "group_deps", "group_name", "annotation", "pip_data_exclude"): - if key in args and args[key] != None: - deps_args[key] = args[key] - - deps_args["whl_library"] = "@{}//:BUILD.bazel".format(repo.whl_repo_name) + deps_args = { + k: repo.args[k] + for k in [ + "config_load", + "dep_template", + "group_deps", + "group_name", + ] + } | { + # TODO @aignas 2026-07-04: add a test + "extras": req.extras, + "whl_library": "@{}//:BUILD.bazel".format(repo.whl_repo_name), + } repos_dict = self._whl_library_deps repo_name = "{}_{}_{}".format(self.name, version_label(python_version), repo.repo_name) diff --git a/python/private/pypi/whl_library.bzl b/python/private/pypi/whl_library.bzl index c64765a075..1037373639 100644 --- a/python/private/pypi/whl_library.bzl +++ b/python/private/pypi/whl_library.bzl @@ -662,10 +662,6 @@ def _whl_library_deps_impl(rctx): whl_library_deps = repository_rule( attrs = { - "annotation": attr.label( - doc = "Optional json encoded file containing annotation to apply to the extracted wheel.", - allow_files = True, - ), "config_load": attr.string( doc = "The load location for configuration for pipstar.", ), @@ -683,10 +679,6 @@ whl_library_deps = repository_rule( "group_name": attr.string( doc = "Name of the group, if any.", ), - "pip_data_exclude": attr.string_list( - doc = "Additional data exclude patterns.", - default = [], - ), "whl_library": attr.label( doc = "The whl_library repository label, use BUILD.bazel file for this.", mandatory = True, diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index 67bc07dc6e..c772568f87 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -291,18 +291,16 @@ def _test_simple_extras_vs_no_extras_simpleapi(env): pypi.group_map().contains_exactly({}) pypi.whl_map().contains_exactly({ "simple": { - "pypi_315_simple_py3_none_any_deadbeef_osx_aarch64": [ + "pypi_315_simple_py3_none_any_deadbeef": [ whl_config_setting( target_platforms = [ - "cp315_osx_aarch64", + "cp315_windows_aarch64", ], version = "3.15", ), - ], - "pypi_315_simple_py3_none_any_deadbeef_windows_aarch64": [ whl_config_setting( target_platforms = [ - "cp315_windows_aarch64", + "cp315_osx_aarch64", ], version = "3.15", ), @@ -310,20 +308,13 @@ def _test_simple_extras_vs_no_extras_simpleapi(env): }, }) pypi.whl_libraries().contains_exactly({ - "simple_py3_none_any_deadbeef_osx_aarch64": { + "simple_py3_none_any_deadbeef": { "filename": "simple-0.0.1-py3-none-any.whl", "index_url": "https://example.com/simple", "requirement": "simple[foo]==0.0.1", "sha256": "deadbeef", "urls": ["/simple-0.0.1-py3-none-any.whl"], }, - "simple_py3_none_any_deadbeef_windows_aarch64": { - "filename": "simple-0.0.1-py3-none-any.whl", - "index_url": "https://example.com/simple", - "requirement": "simple==0.0.1", - "sha256": "deadbeef", - "urls": ["/simple-0.0.1-py3-none-any.whl"], - }, }) pypi.extra_aliases().contains_exactly({}) @@ -611,25 +602,25 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ pypi.group_map().contains_exactly({}) pypi.whl_map().contains_exactly({ "torch": { - "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef_linux_x86_64": [ + "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": [ whl_config_setting( target_platforms = ["cp312_linux_x86_64"], version = "3.12", ), ], - "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432_linux_aarch64": [ + "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": [ whl_config_setting( target_platforms = ["cp312_linux_aarch64"], version = "3.12", ), ], - "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c_windows_x86_64": [ + "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": [ whl_config_setting( target_platforms = ["cp312_windows_x86_64"], version = "3.12", ), ], - "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5_osx_aarch64": [ + "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": [ whl_config_setting( target_platforms = ["cp312_osx_aarch64"], version = "3.12", @@ -638,28 +629,28 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ }, }) pypi.whl_libraries().contains_exactly({ - "torch_cp312_cp312_linux_x86_64_8800deef_linux_x86_64": { + "torch_cp312_cp312_linux_x86_64_8800deef": { "filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", "index_url": "https://torch.index/torch", "requirement": "torch==2.4.1+cpu", "sha256": "8800deef0026011d502c0c256cc4b67d002347f63c3a38cd8e45f1f445c61364", "urls": ["/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-linux_x86_64.whl"], }, - "torch_cp312_cp312_manylinux_2_17_aarch64_36109432_linux_aarch64": { + "torch_cp312_cp312_manylinux_2_17_aarch64_36109432": { "filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", "index_url": "https://torch.index/torch", "requirement": "torch==2.4.1", "sha256": "36109432b10bd7163c9b30ce896f3c2cca1b86b9765f956a1594f0ff43091e2a", "urls": ["/whl/cpu/torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"], }, - "torch_cp312_cp312_win_amd64_3a570e5c_windows_x86_64": { + "torch_cp312_cp312_win_amd64_3a570e5c": { "filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", "index_url": "https://torch.index/torch", "requirement": "torch==2.4.1+cpu", "sha256": "3a570e5c553415cdbddfe679207327b3a3806b21c6adea14fba77684d1619e97", "urls": ["/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-win_amd64.whl"], }, - "torch_cp312_none_macosx_11_0_arm64_72b484d5_osx_aarch64": { + "torch_cp312_none_macosx_11_0_arm64_72b484d5": { "filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", "index_url": "https://torch.index/torch", "requirement": "torch==2.4.1", @@ -1478,12 +1469,10 @@ Attempting to create a duplicate library pypi_315_foo with different arguments. common: { "dep_template": "@pypi//{name}:{target}", "config_load": "@pypi//:config.bzl", - "requirement": "foo==0.0.1", } different: { "python_interpreter_target": ("unit_test_interpreter_target_1", "unit_test_interpreter_target_2"), - }\ -""", + }""", ]).in_order() _tests.append(_test_err_duplicate_repos) From a326e05f2fdc5b835cd5df8bc7f655167ad17bee Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Jul 2026 19:38:43 +0900 Subject: [PATCH 11/13] fixup the implementation --- python/private/pypi/hub_builder.bzl | 30 ++-- tests/pypi/hub_builder/hub_builder_tests.bzl | 160 ++++++++++++++----- 2 files changed, 127 insertions(+), 63 deletions(-) diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index d8611b7938..ad2b23da13 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -331,7 +331,8 @@ def _add_whl_library(self, *, python_version, whl, repo): # in the hub context. If we have whl-only pipstar extraction, then we can reuse the # extracted sources. repos_dict = self._whl_libraries - deps_args = repo.args + deps_args = dict(repo.args) + deps_args["requirement"] = requirement(deps_args["requirement"]).name else: extract_args = { k: v @@ -348,12 +349,12 @@ def _add_whl_library(self, *, python_version, whl, repo): _add_library( self, repos_dict = self._whl_libraries, - name = repo.whl_repo_name, + name = repo.base_repo_name, args = extract_args, ) deps_args = { - k: repo.args[k] + k: repo.args.get(k) for k in [ "config_load", "dep_template", @@ -363,7 +364,7 @@ def _add_whl_library(self, *, python_version, whl, repo): } | { # TODO @aignas 2026-07-04: add a test "extras": req.extras, - "whl_library": "@{}//:BUILD.bazel".format(repo.whl_repo_name), + "whl_library": "@{}//:BUILD.bazel".format(repo.base_repo_name), } repos_dict = self._whl_library_deps @@ -589,16 +590,6 @@ def _create_whl_repos( interpreter = _detect_interpreter(self, pip_attr, python_version) for whl in requirements_by_platform: - # Check if all sources for this wheel have the same requirement line. - # If they do, we can reuse the same whl_library repository across platforms. - same_requirements = True - if whl.srcs: - first_req = whl.srcs[0].requirement_line - for src in whl.srcs[1:]: - if src.requirement_line != first_req: - same_requirements = False - break - whl_library_args = common_args | _whl_library_args( self, whl = whl, @@ -615,7 +606,6 @@ def _create_whl_repos( auth_patterns = self._config.auth_patterns or pip_attr.auth_patterns, python_version = _major_minor_version(python_version), is_multiple_versions = whl.is_multiple_versions, - same_requirements = same_requirements, interpreter = interpreter, enable_pipstar_extract = enable_pipstar_extract, ) @@ -686,7 +676,6 @@ def _whl_repo( whl_library_args, index_url, is_multiple_versions, - same_requirements, download_only, netrc, auth_patterns, @@ -718,12 +707,15 @@ def _whl_repo( return None else: # Fallback to a pip-installed wheel - target_platforms = src.target_platforms if (is_multiple_versions or not same_requirements) else [] + target_platforms = src.target_platforms if is_multiple_versions else [] return struct( repo_name = pypi_repo_name( normalize_name(src.distribution), *target_platforms ), + base_repo_name = pypi_repo_name( + normalize_name(src.distribution), + ), args = args, config_setting = whl_config_setting( version = python_version, @@ -748,11 +740,11 @@ def _whl_repo( # TODO @aignas 2025-11-02: once we have pipstar enabled we can add extra # targets to each hub for each extra combination and solve this more cleanly as opposed to # duplicating whl_library repositories. - target_platforms = src.target_platforms if (is_multiple_versions or not same_requirements) else [] + target_platforms = src.target_platforms if is_multiple_versions else [] return struct( repo_name = whl_repo_name(src.filename, src.sha256, *target_platforms), - whl_repo_name = whl_repo_name(src.filename, src.sha256, *target_platforms), + base_repo_name = whl_repo_name(src.filename, src.sha256), args = args, config_setting = whl_config_setting( version = python_version, diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index c772568f87..47c68cea4c 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -144,7 +144,7 @@ def _test_simple(env): "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "simple==0.0.1 --hash=sha256:deadbeef --hash=sha256:deadbaaf", + "requirement": "simple", }, }) pypi.extra_aliases().contains_exactly({}) @@ -153,8 +153,8 @@ _tests.append(_test_simple) def _test_simple_multiple_requirements(env): sub_tests = { - ("osx", "aarch64"): "simple==0.0.2 --hash=sha256:deadb00f", - ("windows", "aarch64"): "simple==0.0.1 --hash=sha256:deadbeef", + ("osx", "aarch64"): "simple", + ("windows", "aarch64"): "simple", } for (host_os, host_arch), want_requirement in sub_tests.items(): builder = hub_builder(env) @@ -199,8 +199,8 @@ _tests.append(_test_simple_multiple_requirements) def _test_simple_extras_vs_no_extras(env): sub_tests = { - ("osx", "aarch64"): "simple[foo]==0.0.1 --hash=sha256:deadbeef", - ("windows", "aarch64"): "simple==0.0.1 --hash=sha256:deadbeef", + ("osx", "aarch64"): "simple", + ("windows", "aarch64"): "simple", } for (host_os, host_arch), want_requirement in sub_tests.items(): builder = hub_builder(env) @@ -291,16 +291,18 @@ def _test_simple_extras_vs_no_extras_simpleapi(env): pypi.group_map().contains_exactly({}) pypi.whl_map().contains_exactly({ "simple": { - "pypi_315_simple_py3_none_any_deadbeef": [ + "pypi_315_simple_py3_none_any_deadbeef_osx_aarch64": [ whl_config_setting( target_platforms = [ - "cp315_windows_aarch64", + "cp315_osx_aarch64", ], version = "3.15", ), + ], + "pypi_315_simple_py3_none_any_deadbeef_windows_aarch64": [ whl_config_setting( target_platforms = [ - "cp315_osx_aarch64", + "cp315_windows_aarch64", ], version = "3.15", ), @@ -311,7 +313,16 @@ def _test_simple_extras_vs_no_extras_simpleapi(env): "simple_py3_none_any_deadbeef": { "filename": "simple-0.0.1-py3-none-any.whl", "index_url": "https://example.com/simple", - "requirement": "simple[foo]==0.0.1", + "requirement": "simple", + "sha256": "deadbeef", + "urls": ["/simple-0.0.1-py3-none-any.whl"], + }, + }) + pypi.whl_libraries().contains_exactly({ + "simple_py3_none_any_deadbeef": { + "filename": "simple-0.0.1-py3-none-any.whl", + "index_url": "https://example.com/simple", + "requirement": "simple", "sha256": "deadbeef", "urls": ["/simple-0.0.1-py3-none-any.whl"], }, @@ -395,25 +406,25 @@ new-package==0.0.1 --hash=sha256:deadb00f2 "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "old-package==0.0.1 --hash=sha256:deadbaaf", + "requirement": "old-package", }, "pypi_315_simple": { "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "simple==0.0.1 --hash=sha256:deadbeef", + "requirement": "simple", }, "pypi_316_new_package": { "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "new-package==0.0.1 --hash=sha256:deadb00f2", + "requirement": "new-package", }, "pypi_316_simple": { "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "simple==0.0.2 --hash=sha256:deadb00f", + "requirement": "simple", }, }) pypi.extra_aliases().contains_exactly({}) @@ -422,8 +433,8 @@ _tests.append(_test_simple_multiple_python_versions) def _test_simple_with_markers(env): sub_tests = { - ("osx", "aarch64"): "torch==2.4.1 --hash=sha256:deadbeef", - ("linux", "x86_64"): "torch==2.4.1+cpu", + ("osx", "aarch64"): "torch", + ("linux", "x86_64"): "torch", } for (host_os, host_arch), want_requirement in sub_tests.items(): builder = hub_builder(env) @@ -602,27 +613,27 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ pypi.group_map().contains_exactly({}) pypi.whl_map().contains_exactly({ "torch": { - "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": [ + "pypi_312_torch_cp312_cp312_linux_x86_64_8800deef_linux_x86_64": [ whl_config_setting( - target_platforms = ["cp312_linux_x86_64"], + target_platforms = ("cp312_linux_x86_64",), version = "3.12", ), ], - "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": [ + "pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432_linux_aarch64": [ whl_config_setting( - target_platforms = ["cp312_linux_aarch64"], + target_platforms = ("cp312_linux_aarch64",), version = "3.12", ), ], - "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": [ + "pypi_312_torch_cp312_cp312_win_amd64_3a570e5c_windows_x86_64": [ whl_config_setting( - target_platforms = ["cp312_windows_x86_64"], + target_platforms = ("cp312_windows_x86_64",), version = "3.12", ), ], - "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": [ + "pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5_osx_aarch64": [ whl_config_setting( - target_platforms = ["cp312_osx_aarch64"], + target_platforms = ("cp312_osx_aarch64",), version = "3.12", ), ], @@ -632,28 +643,88 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \ "torch_cp312_cp312_linux_x86_64_8800deef": { "filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", "index_url": "https://torch.index/torch", - "requirement": "torch==2.4.1+cpu", + "requirement": "torch", + "sha256": "8800deef0026011d502c0c256cc4b67d002347f63c3a38cd8e45f1f445c61364", + "urls": ["/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-linux_x86_64.whl"], + }, + "torch_cp312_cp312_manylinux_2_17_aarch64_36109432": { + "filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "index_url": "https://torch.index/torch", + "requirement": "torch", + "sha256": "36109432b10bd7163c9b30ce896f3c2cca1b86b9765f956a1594f0ff43091e2a", + "urls": ["/whl/cpu/torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"], + }, + "torch_cp312_cp312_win_amd64_3a570e5c": { + "filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", + "index_url": "https://torch.index/torch", + "requirement": "torch", + "sha256": "3a570e5c553415cdbddfe679207327b3a3806b21c6adea14fba77684d1619e97", + "urls": ["/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-win_amd64.whl"], + }, + "torch_cp312_none_macosx_11_0_arm64_72b484d5": { + "filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", + "index_url": "https://torch.index/torch", + "requirement": "torch", + "sha256": "72b484d5b6cec1a735bf3fa5a1c4883d01748698c5e9cfdbeb4ffab7c7987e0d", + "urls": ["/whl/cpu/torch-2.4.1-cp312-none-macosx_11_0_arm64.whl"], + }, + }) + pypi.whl_libraries().contains_exactly({ + "torch_cp312_cp312_linux_x86_64_8800deef": { + "filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", + "index_url": "https://torch.index/torch", + "requirement": "torch", + "sha256": "8800deef0026011d502c0c256cc4b67d002347f63c3a38cd8e45f1f445c61364", + "urls": ["/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-linux_x86_64.whl"], + }, + "torch_cp312_cp312_manylinux_2_17_aarch64_36109432": { + "filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "index_url": "https://torch.index/torch", + "requirement": "torch", + "sha256": "36109432b10bd7163c9b30ce896f3c2cca1b86b9765f956a1594f0ff43091e2a", + "urls": ["/whl/cpu/torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"], + }, + "torch_cp312_cp312_win_amd64_3a570e5c": { + "filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", + "index_url": "https://torch.index/torch", + "requirement": "torch", + "sha256": "3a570e5c553415cdbddfe679207327b3a3806b21c6adea14fba77684d1619e97", + "urls": ["/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-win_amd64.whl"], + }, + "torch_cp312_none_macosx_11_0_arm64_72b484d5": { + "filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", + "index_url": "https://torch.index/torch", + "requirement": "torch", + "sha256": "72b484d5b6cec1a735bf3fa5a1c4883d01748698c5e9cfdbeb4ffab7c7987e0d", + "urls": ["/whl/cpu/torch-2.4.1-cp312-none-macosx_11_0_arm64.whl"], + }, + }) + pypi.whl_libraries().contains_exactly({ + "torch_cp312_cp312_linux_x86_64_8800deef": { + "filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl", + "index_url": "https://torch.index/torch", + "requirement": "torch", "sha256": "8800deef0026011d502c0c256cc4b67d002347f63c3a38cd8e45f1f445c61364", "urls": ["/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-linux_x86_64.whl"], }, "torch_cp312_cp312_manylinux_2_17_aarch64_36109432": { "filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", "index_url": "https://torch.index/torch", - "requirement": "torch==2.4.1", + "requirement": "torch", "sha256": "36109432b10bd7163c9b30ce896f3c2cca1b86b9765f956a1594f0ff43091e2a", "urls": ["/whl/cpu/torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"], }, "torch_cp312_cp312_win_amd64_3a570e5c": { "filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl", "index_url": "https://torch.index/torch", - "requirement": "torch==2.4.1+cpu", + "requirement": "torch", "sha256": "3a570e5c553415cdbddfe679207327b3a3806b21c6adea14fba77684d1619e97", "urls": ["/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-win_amd64.whl"], }, "torch_cp312_none_macosx_11_0_arm64_72b484d5": { "filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl", "index_url": "https://torch.index/torch", - "requirement": "torch==2.4.1", + "requirement": "torch", "sha256": "72b484d5b6cec1a735bf3fa5a1c4883d01748698c5e9cfdbeb4ffab7c7987e0d", "urls": ["/whl/cpu/torch-2.4.1-cp312-none-macosx_11_0_arm64.whl"], }, @@ -794,7 +865,7 @@ simple==0.0.1 --hash=sha256:deadb00f want_whl_library = { "filename": "simple-0.0.1-py3-none-any.whl", "index_url": test.expect_index_url, - "requirement": "simple==0.0.1", + "requirement": "simple", "sha256": "deadb00f", "urls": [test.expect_url], } @@ -896,7 +967,7 @@ simple==0.0.3 \ "download_only": True, "extra_pip_args": ["--platform=manylinux_2_17_x86_64", "--python-version=315", "--implementation=cp", "--abi=cp315"], "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "extra==0.0.1 --hash=sha256:deadb00f", + "requirement": "extra", }, "pypi_315_simple_linux_x86_64": { "config_load": "@pypi//:config.bzl", @@ -904,7 +975,7 @@ simple==0.0.3 \ "download_only": True, "extra_pip_args": ["--platform=manylinux_2_17_x86_64", "--python-version=315", "--implementation=cp", "--abi=cp315"], "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "simple==0.0.1 --hash=sha256:deadbeef", + "requirement": "simple", }, "pypi_315_simple_osx_aarch64": { "config_load": "@pypi//:config.bzl", @@ -912,7 +983,7 @@ simple==0.0.3 \ "download_only": True, "extra_pip_args": ["--platform=macosx_10_9_arm64", "--python-version=315", "--implementation=cp", "--abi=cp315"], "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "simple==0.0.3 --hash=sha256:deadbaaf", + "requirement": "simple", }, }) pypi.extra_aliases().contains_exactly({}) @@ -1135,7 +1206,7 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef pypi.whl_libraries().contains_exactly({ "direct_without_sha_0_0_1_py3_none_any": { "filename": "direct_without_sha-0.0.1-py3-none-any.whl", - "requirement": "direct_without_sha==0.0.1", + "requirement": "direct-without-sha", "sha256": "", "urls": ["example-direct.org/direct_without_sha-0.0.1-py3-none-any.whl"], "whl_patches": {"my_patch": "1"}, @@ -1143,7 +1214,7 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "plat_py3_none_linux_x86_64_deadb44f": { "filename": "plat-pkg-0.0.4-py3-none-linux_x86_64.whl", "index_url": "https://pypi.org/simple", - "requirement": "plat_pkg==0.0.4", + "requirement": "plat-pkg", "sha256": "deadb44f", "urls": ["example2.org/index/plat_pkg/"], }, @@ -1153,7 +1224,7 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "extra_pip_args": ["--extra-args-for-sdist-building"], "filename": "any-name.tar.gz", "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "direct_sdist_without_sha @ some-archive/any-name.tar.gz", + "requirement": "direct-sdist-without-sha", "sha256": "", "urls": ["some-archive/any-name.tar.gz"], }, @@ -1162,32 +1233,32 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef "dep_template": "@pypi//{name}:{target}", "extra_pip_args": ["--extra-args-for-sdist-building"], "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef", + "requirement": "git-dep", }, "pypi_315_pip_fallback": { "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "extra_pip_args": ["--extra-args-for-sdist-building"], "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "pip_fallback==0.0.1", + "requirement": "pip-fallback", }, "simple_py3_none_any_deadb00f": { "filename": "simple-0.0.1-py3-none-any.whl", "index_url": "https://pypi.org/simple", - "requirement": "simple==0.0.1", + "requirement": "simple", "sha256": "deadb00f", "urls": ["example2.org"], }, "some_pkg_py3_none_any_deadbaaf": { "filename": "some_pkg-0.0.1-py3-none-any.whl", - "requirement": "some_pkg==0.0.1", + "requirement": "some-pkg", "sha256": "deadbaaf", "urls": ["example-direct.org/some_pkg-0.0.1-py3-none-any.whl"], }, "some_py3_none_any_deadb33f": { "filename": "some-other-pkg-0.0.1-py3-none-any.whl", "index_url": "https://with_index_url", - "requirement": "some_other_pkg==0.0.1", + "requirement": "some-other-pkg", "sha256": "deadb33f", "urls": ["example2.org/index/some_other_pkg/"], }, @@ -1218,8 +1289,8 @@ _tests.append(_test_simple_get_index) def _test_optimum_sys_platform_extra(env): sub_tests = { - ("osx", "aarch64"): "optimum[onnxruntime]==1.17.1", - ("linux", "aarch64"): "optimum[onnxruntime-gpu]==1.17.1", + ("osx", "aarch64"): "optimum", + ("linux", "aarch64"): "optimum", } for (host_os, host_arch), want_requirement in sub_tests.items(): builder = hub_builder( @@ -1336,13 +1407,13 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "optimum[onnxruntime-gpu]==1.17.1", + "requirement": "optimum", }, "pypi_315_optimum_myosxaarch64": { "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "optimum[onnxruntime]==1.17.1", + "requirement": "optimum", }, }) pypi.extra_aliases().contains_exactly({}) @@ -1409,7 +1480,7 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "optimum[onnxruntime-gpu]==1.17.1", + "requirement": "optimum", }, }) pypi.extra_aliases().contains_exactly({}) @@ -1469,6 +1540,7 @@ Attempting to create a duplicate library pypi_315_foo with different arguments. common: { "dep_template": "@pypi//{name}:{target}", "config_load": "@pypi//:config.bzl", + "requirement": "foo", } different: { "python_interpreter_target": ("unit_test_interpreter_target_1", "unit_test_interpreter_target_2"), From 86ed8caa17ac3660120bf507b68670cf2a0ec052 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Jul 2026 19:41:57 +0900 Subject: [PATCH 12/13] wip --- python/private/internal_dev_deps.bzl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/python/private/internal_dev_deps.bzl b/python/private/internal_dev_deps.bzl index 5bbee232f9..09d5ad3a0b 100644 --- a/python/private/internal_dev_deps.bzl +++ b/python/private/internal_dev_deps.bzl @@ -65,6 +65,7 @@ def _internal_dev_deps_impl(mctx): whl_file = "@implicit_namespace_ns_sub2_whl//:ns_sub2-1.0-any-none-any.whl", requirement = "ns-sub2", enable_implicit_namespace_pkgs = False, + repo_prefix = "dummy", ) whl_from_dir_repo( @@ -77,6 +78,7 @@ def _internal_dev_deps_impl(mctx): whl_file = "@pkgutil_nspkg1_whl//:pkgutil_nspkg1-1.0-any-none-any.whl", requirement = "pkgutil_nspkg1", enable_implicit_namespace_pkgs = False, + repo_prefix = "dummy", ) whl_from_dir_repo( @@ -89,6 +91,7 @@ def _internal_dev_deps_impl(mctx): whl_file = "@pkgutil_nspkg2_whl//:pkgutil_nspkg2-1.0-any-none-any.whl", requirement = "pkgutil_nspkg2", enable_implicit_namespace_pkgs = False, + repo_prefix = "dummy", ) whl_from_dir_repo( @@ -100,6 +103,7 @@ def _internal_dev_deps_impl(mctx): name = "whl_with_data1", whl_file = "@whl_with_data1_whl//:whl_with_data1-1.0-any-none-any.whl", requirement = "whl-with-data1", + repo_prefix = "dummy", ) whl_from_dir_repo( @@ -111,6 +115,7 @@ def _internal_dev_deps_impl(mctx): name = "whl_with_data2", whl_file = "@whl_with_data2_whl//:whl_with_data2-1.0-any-none-any.whl", requirement = "whl-with-data2", + repo_prefix = "dummy", ) _whl_library_from_dir( @@ -129,6 +134,7 @@ def _internal_dev_deps_impl(mctx): requirement = "optional_dep", # The following is necessary to enable pipstar and make tests faster config_load = "@rules_python//tests/pypi/whl_library/testdata:packages.bzl", + repo_prefix = "dummy", ) # Setup for //tests/pypi/patch_whl/patch_whl_patch_test.py @@ -147,6 +153,7 @@ def _internal_dev_deps_impl(mctx): "whls": ["pkg-1.0-any-none-any.whl"], }), }, + repo_prefix = "dummy", ) def _whl_library_from_dir(*, name, output, root, **kwargs): From d2579ccd276363d85aaa4a5b944b41c9160fc486 Mon Sep 17 00:00:00 2001 From: Ignas Anikevicius <240938+aignas@users.noreply.github.com> Date: Sat, 4 Jul 2026 22:14:26 +0900 Subject: [PATCH 13/13] wip --- python/private/pypi/hub_builder.bzl | 5 ++-- python/private/pypi/parse_requirements.bzl | 2 +- python/private/pypi/whl_library_targets.bzl | 19 ++++++++---- tests/pypi/hub_builder/hub_builder_tests.bzl | 30 +++++++++---------- .../parse_requirements_tests.bzl | 18 +++++------ .../whl_library_targets_tests.bzl | 7 +++-- 6 files changed, 46 insertions(+), 35 deletions(-) diff --git a/python/private/pypi/hub_builder.bzl b/python/private/pypi/hub_builder.bzl index ad2b23da13..148e93a52d 100644 --- a/python/private/pypi/hub_builder.bzl +++ b/python/private/pypi/hub_builder.bzl @@ -332,7 +332,8 @@ def _add_whl_library(self, *, python_version, whl, repo): # extracted sources. repos_dict = self._whl_libraries deps_args = dict(repo.args) - deps_args["requirement"] = requirement(deps_args["requirement"]).name + # deps_args["requirement"] = requirement(deps_args["requirement"]).name + else: extract_args = { k: v @@ -345,7 +346,7 @@ def _add_whl_library(self, *, python_version, whl, repo): req = requirement(extract_args["requirement"]) # TODO @aignas 2026-07-04: add a test - extract_args["requirement"] = req.name # drop any specified extras + extract_args["requirement"] = repo.args["requirement"] _add_library( self, repos_dict = self._whl_libraries, diff --git a/python/private/pypi/parse_requirements.bzl b/python/private/pypi/parse_requirements.bzl index c87c586f7e..2dc35fcc06 100644 --- a/python/private/pypi/parse_requirements.bzl +++ b/python/private/pypi/parse_requirements.bzl @@ -482,7 +482,7 @@ def _package_srcs( logger.debug(lambda: "The whl dist is: {}".format(dist.filename if dist else dist)) if extract_url_srcs and dist: - req_line = r.srcs.requirement + req_line = r.srcs.requirement_line elif can_fallback or (not extract_url_srcs and dist): dist = struct( url = "", diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl index d29bda4d5d..2619169ade 100644 --- a/python/private/pypi/whl_library_targets.bzl +++ b/python/private/pypi/whl_library_targets.bzl @@ -121,7 +121,7 @@ def whl_library_targets( group_name = "", native = native, aliases = [], - src_pkg = Label("//:BUILD.bazel"), + src_pkg = None, rules = struct( copy_file = copy_file, py_binary = py_binary, @@ -130,7 +130,8 @@ def whl_library_targets( venv_rewrite_shebang = venv_rewrite_shebang, env_marker_setting = env_marker_setting, create_inits = _create_inits, - )): + ), + **kwargs): """Create all of the whl_library targets. Args: @@ -152,7 +153,12 @@ def whl_library_targets( native: {type}`native` The native struct for overriding in tests. rules: {type}`struct` A struct with references to rules for creating targets. """ - src_pkg = Label(src_pkg) + if src_pkg == None: + src_pkg = struct( + same_package_label = lambda label: ":" + label, + ) + else: + src_pkg = Label(src_pkg) _config_settings( dependencies_with_markers = dependencies_with_markers, @@ -219,7 +225,7 @@ def whl_library_targets( data = _deps( deps = dependencies, deps_conditional = deps_conditional, - tmpl = dep_template.format(name = "{}", target = WHEEL_FILE_PUBLIC_LABEL), + tmpl = dep_template.format(name = "{}", target = WHEEL_FILE_PUBLIC_LABEL) if dep_template else "", ), visibility = impl_vis, ) @@ -228,7 +234,7 @@ def whl_library_targets( deps = [src_pkg.same_package_label(NODEPS_PY_LIBRARY_LABEL)] + _deps( deps = dependencies, deps_conditional = deps_conditional, - tmpl = dep_template.format(name = "{}", target = PY_LIBRARY_PUBLIC_LABEL), + tmpl = dep_template.format(name = "{}", target = PY_LIBRARY_PUBLIC_LABEL) if dep_template else "", ), tags = tags, visibility = impl_vis, @@ -292,7 +298,8 @@ def whl_library_srcs( venv_rewrite_shebang = venv_rewrite_shebang, env_marker_setting = env_marker_setting, create_inits = _create_inits, - )): + ), + **kwargs): """Create all of the whl_library targets. Args: diff --git a/tests/pypi/hub_builder/hub_builder_tests.bzl b/tests/pypi/hub_builder/hub_builder_tests.bzl index 47c68cea4c..b7c57b3e92 100644 --- a/tests/pypi/hub_builder/hub_builder_tests.bzl +++ b/tests/pypi/hub_builder/hub_builder_tests.bzl @@ -144,7 +144,7 @@ def _test_simple(env): "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "simple", + "requirement": "simple==0.0.1 --hash=sha256:deadbeef --hash=sha256:deadbaaf", }, }) pypi.extra_aliases().contains_exactly({}) @@ -153,8 +153,8 @@ _tests.append(_test_simple) def _test_simple_multiple_requirements(env): sub_tests = { - ("osx", "aarch64"): "simple", - ("windows", "aarch64"): "simple", + ("osx", "aarch64"): "simple==0.0.2 --hash=sha256:deadb00f", + ("windows", "aarch64"): "simple==0.0.1 --hash=sha256:deadbeef", } for (host_os, host_arch), want_requirement in sub_tests.items(): builder = hub_builder(env) @@ -199,8 +199,8 @@ _tests.append(_test_simple_multiple_requirements) def _test_simple_extras_vs_no_extras(env): sub_tests = { - ("osx", "aarch64"): "simple", - ("windows", "aarch64"): "simple", + ("osx", "aarch64"): "simple[foo]==0.0.1 --hash=sha256:deadbeef", + ("windows", "aarch64"): "simple==0.0.1 --hash=sha256:deadbeef", } for (host_os, host_arch), want_requirement in sub_tests.items(): builder = hub_builder(env) @@ -406,25 +406,25 @@ new-package==0.0.1 --hash=sha256:deadb00f2 "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "old-package", + "requirement": "old-package==0.0.1 --hash=sha256:deadbaaf", }, "pypi_315_simple": { "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "simple", + "requirement": "simple==0.0.1 --hash=sha256:deadbeef", }, "pypi_316_new_package": { "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "new-package", + "requirement": "new-package==0.0.1 --hash=sha256:deadb00f2", }, "pypi_316_simple": { "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "simple", + "requirement": "simple==0.0.2 --hash=sha256:deadb00f", }, }) pypi.extra_aliases().contains_exactly({}) @@ -474,7 +474,7 @@ def _test_simple_with_markers(env): "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", - "requirement": want_requirement, + "requirement": "torch==2.4.1 --hash=sha256:deadbeef", }, }) pypi.extra_aliases().contains_exactly({}) @@ -967,7 +967,7 @@ simple==0.0.3 \ "download_only": True, "extra_pip_args": ["--platform=manylinux_2_17_x86_64", "--python-version=315", "--implementation=cp", "--abi=cp315"], "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "extra", + "requirement": "extra==0.0.1 --hash=sha256:deadb00f", }, "pypi_315_simple_linux_x86_64": { "config_load": "@pypi//:config.bzl", @@ -975,7 +975,7 @@ simple==0.0.3 \ "download_only": True, "extra_pip_args": ["--platform=manylinux_2_17_x86_64", "--python-version=315", "--implementation=cp", "--abi=cp315"], "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "simple", + "requirement": "simple==0.0.1 --hash=sha256:deadbeef", }, "pypi_315_simple_osx_aarch64": { "config_load": "@pypi//:config.bzl", @@ -983,7 +983,7 @@ simple==0.0.3 \ "download_only": True, "extra_pip_args": ["--platform=macosx_10_9_arm64", "--python-version=315", "--implementation=cp", "--abi=cp315"], "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "simple", + "requirement": "simple==0.0.3 --hash=sha256:deadbaaf", }, }) pypi.extra_aliases().contains_exactly({}) @@ -1476,11 +1476,11 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux' }, }) pypi.whl_libraries().contains_exactly({ - "pypi_315_optimum": { + "pypi_315_torch": { "config_load": "@pypi//:config.bzl", "dep_template": "@pypi//{name}:{target}", "python_interpreter_target": "unit_test_interpreter_target", - "requirement": "optimum", + "requirement": "torch==2.4.1+cpu", }, }) pypi.extra_aliases().contains_exactly({}) diff --git a/tests/pypi/parse_requirements/parse_requirements_tests.bzl b/tests/pypi/parse_requirements/parse_requirements_tests.bzl index 576a31ae5d..b12887a153 100644 --- a/tests/pypi/parse_requirements/parse_requirements_tests.bzl +++ b/tests/pypi/parse_requirements/parse_requirements_tests.bzl @@ -230,7 +230,7 @@ def _test_direct_urls_integration(env): distribution = "foo", extra_pip_args = [], filename = "package.whl", - requirement_line = "foo[extra]", + requirement_line = "foo[extra] @ https://some-url/package.whl", sha256 = "", target_platforms = ["linux_x86_64"], url = "https://some-url/package.whl", @@ -648,7 +648,7 @@ def _test_optional_hash(env): struct( distribution = "bar", extra_pip_args = [], - requirement_line = "bar==0.0.4", + requirement_line = "bar==0.0.4 @ https://example.org/bar-0.0.4.whl", target_platforms = ["linux_x86_64"], url = "https://example.org/bar-0.0.4.whl", filename = "bar-0.0.4.whl", @@ -666,7 +666,7 @@ def _test_optional_hash(env): struct( distribution = "foo", extra_pip_args = [], - requirement_line = "foo==0.0.5", + requirement_line = "foo==0.0.5 @ https://example.org/foo-0.0.5.whl --hash=sha256:deadbeef", target_platforms = ["linux_x86_64"], url = "https://example.org/foo-0.0.5.whl", filename = "foo-0.0.5.whl", @@ -776,7 +776,7 @@ def _test_overlapping_shas_with_index_results(env): distribution = "foo", extra_pip_args = [], filename = "foo-0.0.1-py3-none-any.whl", - requirement_line = "foo==0.0.3", + requirement_line = "foo==0.0.3 --hash=sha256:deadbaaf --hash=sha256:5d15t", sha256 = "deadbaaf", target_platforms = ["cp39_linux_x86_64"], url = "super2", @@ -786,7 +786,7 @@ def _test_overlapping_shas_with_index_results(env): distribution = "foo", extra_pip_args = [], filename = "foo-0.0.1-py3-none-macosx_14_0_x86_64.whl", - requirement_line = "foo==0.0.3", + requirement_line = "foo==0.0.3 --hash=sha256:deadbaaf --hash=sha256:deadb11f --hash=sha256:5d15t", sha256 = "deadb11f", target_platforms = ["cp39_osx_x86_64"], url = "super2", @@ -888,7 +888,7 @@ def _test_get_index_urls_different_versions(env): distribution = "foo", extra_pip_args = [], filename = "foo-0.0.2-py3-none-any.whl", - requirement_line = "foo==0.0.2", + requirement_line = "foo==0.0.2 --hash=sha256:deadb11f", sha256 = "deadb11f", target_platforms = ["cp310_linux_x86_64"], url = "super2", @@ -991,7 +991,7 @@ def _test_get_index_urls_single_py_version(env): distribution = "foo", extra_pip_args = [], filename = "foo-0.0.2-py3-none-any.whl", - requirement_line = "foo==0.0.2", + requirement_line = "foo==0.0.2 --hash=sha256:deadb11f", sha256 = "deadb11f", target_platforms = ["cp310_linux_x86_64"], url = "super2", @@ -1125,7 +1125,7 @@ def _test_uv_lock_primary_source_multiple_versions(env): struct( distribution = "foo", extra_pip_args = [], - requirement_line = "foo==0.0.2", + requirement_line = "foo==0.0.2 --hash=sha256:deadb11f", target_platforms = ["linux_x86_64"], filename = "foo-0.0.2-py3-none-any.whl", sha256 = "deadb11f", @@ -1580,7 +1580,7 @@ def _test_uv_lock_wheel_dedup_resolution_markers(env): struct( distribution = "foo", extra_pip_args = [], - requirement_line = "foo==0.0.2", + requirement_line = "foo==0.0.2 --hash=sha256:deadb11f", target_platforms = ["cp39_osx_aarch64"], filename = "foo-0.0.2-cp39-cp39-macosx_11_0_arm64.whl", sha256 = "ccc", diff --git a/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl b/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl index 5bd1d1f549..650caf8491 100644 --- a/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl +++ b/tests/pypi/whl_library_targets/whl_library_targets_tests.bzl @@ -42,6 +42,7 @@ def _test_filegroups(env): glob = glob, ), rules = struct( + py_library = lambda **kwargs: None, venv_rewrite_shebang = lambda **kwargs: None, ), ) @@ -64,7 +65,7 @@ def _test_filegroups(env): }, { "name": "whl", - "srcs": [""], + "srcs": [":_whl_file"], "data": [], "visibility": ["//visibility:public"], }, @@ -82,10 +83,12 @@ def _test_copy(env): copy_files = {"file_src": "file_dest"}, copy_executables = {"exec_src": "exec_dest"}, native = struct( + filegroup = lambda **kwargs: None, glob = lambda *args, **kwargs: [], ), rules = struct( copy_file = lambda **kwargs: calls.append(kwargs), + py_library = lambda **kwargs: None, venv_rewrite_shebang = lambda **kwargs: None, ), ) @@ -152,7 +155,7 @@ def _test_whl_and_library_deps_from_requires(env): env.expect.that_collection(filegroup_calls).contains_exactly([ { "name": "whl", - "srcs": ["foo-0-py3-none-any.whl"], + "srcs": [":_whl_file"], "data": ["@pypi//bar:whl"] + select({ ":is_include_bar_baz_true": ["@pypi//bar_baz:whl"], "//conditions:default": [],