diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py index 8e42cca7e..625d0100d 100644 --- a/src/buildstream/_frontend/cli.py +++ b/src/buildstream/_frontend/cli.py @@ -1694,3 +1694,124 @@ def artifact_delete(app, artifacts, deps): """Remove artifacts from the local cache""" with app.initialized(): app.stream.artifact_delete(artifacts, selection=deps) + + +############################################################# +# Buildtree Commands # +############################################################# +@cli.group(short_help="Manipulate cached buildtree.") +def buildtree(): + """Manipulate cached buildtree""" + + +##################################################################### +# Buildtree Checkout Command # +##################################################################### +@buildtree.command(name="checkout", short_help="Checkout contents of a buildtree") +@click.option("--buildroot", is_flag=True, help="Export full buildroot instead buildtree.") +@click.option("--force", "-f", is_flag=True, help="Allow files to be overwritten") +@click.option("--hardlinks", is_flag=True, help="Checkout hardlinks instead of copying if possible") +@click.option( + "--tar", + default=None, + metavar="LOCATION", + type=click.Path(), + help="Create a tarball from the artifact contents instead " + "of a file tree. If LOCATION is '-', the tarball " + "will be dumped to the standard output.", +) +@click.option( + "--compression", + default=None, + type=click.Choice(["gz", "xz", "bz2"]), + help="The compression option of the tarball created.", +) +@click.option( + "--directory", default=None, type=click.Path(file_okay=False), help="The directory to checkout the artifact to" +) +@click.option( + "--artifact-remote", + "artifact_remotes", + type=RemoteSpecType(RemoteSpecPurpose.PULL), + multiple=True, + help="A remote for downloading artifacts", +) +@click.option( + "--ignore-project-artifact-remotes", + is_flag=True, + help="Ignore remote artifact cache servers recommended by projects", +) +@click.argument("target", required=False, type=click.Path(readable=False)) +@click.pass_obj +def buildtree_checkout( + app, + buildroot, + force, + hardlinks, + tar, + compression, + directory, + artifact_remotes, + ignore_project_artifact_remotes, + target, +): + """Checkout buildtree + + When this command is executed from a workspace directory, the default + is to checkout the artifact of the workspace element. + """ + from .. import utils + + if hardlinks and tar: + click.echo("ERROR: options --hardlinks and --tar conflict", err=True) + sys.exit(-1) + + if tar and directory: + click.echo("ERROR: options --directory and --tar conflict", err=True) + sys.exit(-1) + + if not tar: + if compression: + click.echo("ERROR: --compression can only be provided if --tar is provided", err=True) + sys.exit(-1) + else: + location = tar + try: + inferred_compression = utils._get_compression(tar) + except UtilError as e: + click.echo("ERROR: Invalid file extension given with '--tar': {}".format(e), err=True) + sys.exit(-1) + if compression and inferred_compression != "" and inferred_compression != compression: + click.echo( + "WARNING: File extension and compression differ." + "File extension has been overridden by --compression", + err=True, + ) + if not compression: + compression = inferred_compression + + with app.initialized(): + if not target: + target = app.stream.get_default_target() + if not target: + raise AppError('Missing argument "ELEMENT".') + + if not tar: + if directory is None: + location = os.path.abspath(os.path.join(os.getcwd(), target)) + if location[-4:] == ".bst": + location = location[:-4] + else: + location = directory + + app.stream.buildtree_checkout( + target, + location=location, + buildroot=buildroot, + force=force, + hardlinks=hardlinks, + compression=compression, + tar=bool(tar), + artifact_remotes=artifact_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, + ) diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index d326b9532..71be9dde0 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -916,6 +916,84 @@ def artifact_delete(self, targets, *, selection=_PipelineSelection.NONE): if not ref_removed: self._context.messenger.info("No artifacts were removed") + # buildtree_checkout() + # + # Checkout target buildtree artifact to the specified location + # + # Args: + # target: Target to checkout + # location: Location to checkout the artifact to + # force: Whether files can be overwritten if necessary + # hardlinks: Whether checking out files hardlinked to + # their artifacts is acceptable + # tar: If true, a tarball from the artifact contents will + # be created, otherwise the file tree of the artifact + # will be placed at the given location. If true and + # location is '-', the tarball will be dumped on the + # standard output. + # artifact_remotes: Artifact cache remotes specified on the commmand line + # ignore_project_artifact_remotes: Whether to ignore artifact remotes specified by projects + # + def buildtree_checkout( + self, + target: str, + *, + location: Optional[str] = None, + buildroot: bool = False, + force: bool = False, + hardlinks: bool = False, + compression: str = "", + tar: bool = False, + artifact_remotes: Iterable[RemoteSpec] = (), + ignore_project_artifact_remotes: bool = False, + ): + elements = self._load( + (target,), + selection=_PipelineSelection.NONE, + load_artifacts=True, + attempt_artifact_metadata=True, + connect_artifact_cache=True, + artifact_remotes=artifact_remotes, + ignore_project_artifact_remotes=ignore_project_artifact_remotes, + ) + + assert len(elements) == 1 + element: Element = elements[0] + + self._check_location_writable(location, force=force, tar=tar) + + self.query_cache([element]) + self._pull_missing_artifacts([element]) + + if buildroot: + # If the user requested the buildroot but no buildroot is available, fail. + if not element._buildroot_exists(): + raise StreamError("No buildroot artifact available", reason="missing-buildroot") + else: + # If the user requested the buildtree but no buildtree is available, + # suggest checking out the buildroot if one is available. + # Otherwise, fail if neither is available. + if not element._buildtree_exists(): + if element._buildroot_exists(): + raise StreamError( + "No buildtree artifact available, but a buildroot is available. " + "Use the --buildroot option to check out the buildroot.", + reason="missing-buildtree-buildroot-exists", + ) + raise StreamError( + "Neither a buildtree nor a buildroot artifact is available.", + reason="missing-buildtree-and-buildroot", + ) + + try: + artifact = element._get_artifact() + virdir = artifact.get_buildroot() if buildroot else artifact.get_buildtree() + self._export_artifact(tar, location, compression, element, hardlinks, virdir) + except BstError as e: + raise StreamError( + "Error while exporting buildtree artifacts" ": '{}'".format(e), detail=e.detail, reason=e.reason + ) from e + # source_checkout() # # Checkout sources of the target element to the specified location diff --git a/tests/integration/buildtree.py b/tests/integration/buildtree.py new file mode 100644 index 000000000..847003da2 --- /dev/null +++ b/tests/integration/buildtree.py @@ -0,0 +1,216 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Pylint doesn't play well with fixtures and dependency injection from pytest +# pylint: disable=redefined-outer-name + +import os +import tarfile +import shutil + +import pytest + +from buildstream._testing import cli, cli_integration, Cli # pylint: disable=unused-import +from buildstream.exceptions import ErrorDomain +from buildstream._testing._utils.site import HAVE_SANDBOX + +pytestmark = pytest.mark.integration + + +DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") + + +# +# Verify fail cases when checkout buildtree +# +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +def test_buildtree_checkout_fail(cli, datafiles): + project = str(datafiles) + element_name = "build-shell/buildtree.bst" + checkout = os.path.join(cli.directory, "checkout") + tar = os.path.join(cli.directory, "buildtree-checkout.tar") + + res = cli.run(project=project, args=["buildtree", "checkout", "--directory", checkout, element_name]) + res.assert_main_error(ErrorDomain.STREAM, "missing-buildtree-and-buildroot") + + res = cli.run( + project=project, args=["buildtree", "checkout", "--buildroot", "--directory", checkout, element_name] + ) + res.assert_main_error(ErrorDomain.STREAM, "missing-buildroot") + + res = cli.run(project=project, args=["--cache-buildtrees", "never", "build", element_name]) + res.assert_success() + + res = cli.run(project=project, args=["buildtree", "checkout", "--directory", checkout, element_name]) + res.assert_main_error(ErrorDomain.STREAM, "missing-buildtree-and-buildroot") + + res = cli.run( + project=project, args=["buildtree", "checkout", "--compression", "gz", "--directory", checkout, element_name] + ) + assert res.exit_code != 0 + assert "ERROR: --compression can only be provided if --tar is provided" in res.stderr + + res = cli.run(project=project, args=["buildtree", "checkout", "--tar", tar, "--directory", checkout, element_name]) + assert res.exit_code != 0 + assert "ERROR: options --directory and --tar conflict" in res.stderr + + res = cli.run(project=project, args=["buildtree", "checkout", "--tar", tar, "--hardlinks", element_name]) + assert res.exit_code != 0 + assert "ERROR: options --hardlinks and --tar conflict" in res.stderr + + +# +# Verify checkout buildtree +# +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +def test_buildtree_checkout(cli, datafiles): + project = str(datafiles) + element_name = "build-shell/buildtree.bst" + checkout = os.path.join(cli.directory, "checkout") + tar = os.path.join(cli.directory, "buildtree-checkout.tar") + + # Build only once with cached buildtree + res = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) + res.assert_success() + + # verify checkout buildtree only + res = cli.run(project=project, args=["buildtree", "checkout", "--directory", checkout, element_name]) + res.assert_success() + + expect_buildtree = ["test"] + assert expect_buildtree == os.listdir(checkout) + shutil.rmtree(checkout) + + # verify checkout buildtree only tar + res = cli.run(project=project, args=["buildtree", "checkout", "--tar", tar, element_name]) + res.assert_success() + + assert os.path.exists(tar) + with tarfile.open(tar) as tf: + expected_content = [os.path.join(".", expect) for expect in expect_buildtree] + tar_members = [f.name for f in tf] + assert tar_members == expected_content + os.remove(tar) + + # verify checkout buildtree only tar in different compression formats + for compression in ["gz", "xz", "bz2"]: + res = cli.run( + project=project, + args=["buildtree", "checkout", "--tar", "-", "--compression", compression, element_name], + binary_capture=True, + ) + res.assert_success() + + with open(tar, "wb") as f: + f.write(res.output) + + with tarfile.open(tar, "r:" + compression) as tf: + expected_content = [os.path.join(".", expect) for expect in expect_buildtree] + tar_members = [f.name for f in tf] + assert tar_members == expected_content + + os.remove(tar) + + # verify checkout whole buildroot + res = cli.run( + project=project, args=["buildtree", "checkout", "--buildroot", "--directory", checkout, element_name] + ) + res.assert_success() + + expect_buildroot = [ + "lib", + "media", + "proc", + "usr", + "home", + "buildstream-install", + "dev", + "var", + "sys", + "bin", + "run", + "buildstream", + "tmp", + "sbin", + "etc", + "mnt", + "srv", + "root", + ] + assert expect_buildroot == os.listdir(checkout) + + # verify checkout whole buildroot with force flag + res = cli.run(project=project, args=["buildtree", "checkout", "--force", "--directory", checkout, element_name]) + res.assert_success() + + assert sorted([*expect_buildtree, *expect_buildroot]) == sorted(os.listdir(checkout)) + + shutil.rmtree(checkout) + + # verify checkout whole buildroot with hardlinks + res = cli.run( + project=project, + args=["buildtree", "checkout", "--buildroot", "--hardlinks", "--directory", checkout, element_name], + ) + res.assert_success() + + expect_buildroot = [ + "lib", + "media", + "proc", + "usr", + "home", + "buildstream-install", + "dev", + "var", + "sys", + "bin", + "run", + "buildstream", + "tmp", + "sbin", + "etc", + "mnt", + "srv", + "root", + ] + assert expect_buildroot == os.listdir(checkout) + shutil.rmtree(checkout) + + +# +# Verify checkout buildtree on non buildable element +# +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox") +def test_buildtree_checkout_non_buildable(cli, datafiles): + project = str(datafiles) + element_name = "build-shell/script.bst" + checkout = os.path.join(cli.directory, "checkout") + + # Build only once with cached buildtree + res = cli.run(project=project, args=["--cache-buildtrees", "always", "build", element_name]) + res.assert_success() + + # verify checkout buildtree only + res = cli.run(project=project, args=["buildtree", "checkout", "--directory", checkout, element_name]) + res.assert_main_error(ErrorDomain.STREAM, "missing-buildtree-buildroot-exists") + + # verify checkout buildtree only + res = cli.run( + project=project, args=["buildtree", "checkout", "--buildroot", "--directory", checkout, element_name] + ) + res.assert_success()