Synchronise master with upstream

doc/source/admin/image-building.rst

@@ -741,6 +741,29 @@ variables that will be picked up from the user env:
 Also these variables could be overwritten using ``--build-args``, which have
 precedence.
 
+Docker BuildKit
+---------------
+
+When using ``--engine docker``, ``kolla-build`` builds images via
+``docker buildx build`` (Docker BuildKit) by default. This requires the
+``docker-buildx-plugin`` package to be installed.
+
+To disable BuildKit and fall back to the legacy docker-py SDK, set
+``buildkit = False`` in ``kolla-build.conf`` or pass ``--nobuildkit`` on the
+command line.
+
+To use a specific buildx builder instance (e.g. a ``docker-container`` or
+remote driver), pass ``--buildkit-builder``:
+
+.. code-block:: console
+
+   kolla-build --buildkit-builder mybuilder
+
+.. note::
+
+   ``--buildkit`` and ``--squash`` are mutually exclusive. Use one or the
+   other.
+
 Cross-compiling
 ---------------
 
@@ -756,7 +779,11 @@ To build ``ARM`` images on ``x86_64`` platform, pass the ``--base-arch`` and
 
 .. note::
 
-   To make this work on x86_64 platform you can use tools like: `qemu-user-static
+   Cross-compilation is natively handled by Docker BuildKit; using BuildKit
+   (the default) is recommended for multi-platform builds.
+
+   To make this work on x86_64 platform with the docker-py based builder
+   (``--nobuildkit``) you can use tools like: `qemu-user-static
    <https://github.com/multiarch/qemu-user-static>`_ or `binfmt
    <https://github.com/tonistiigi/binfmt>`_.
 

docker/base/Dockerfile.j2

@@ -138,6 +138,7 @@ RUN {{ macros.install_packages(base_centos_yum_repo_packages | customizable("cen
         'procps-ng',
         'python3',
         'python3-pip',
+        'python3-pyyaml',
         'socat',
         'sudo',
         'tar',
@@ -230,6 +231,7 @@ COPY apt_preferences /etc/apt/preferences.d/kolla-custom
    'procps',
    'python3',
    'python3-pip',
+   'python3-yaml',
    'socat',
    'sudo',
    'tgt',

docker/base/set_configs.py

@@ -23,6 +23,7 @@
 import shutil
 import stat
 import sys
+import yaml
 
 
 # TODO(rhallisey): add docstring.
@@ -298,13 +299,33 @@ def validate_source(data):
 
 def load_config():
     def load_from_file():
-        config_file = '/var/lib/kolla/config_files/config.json'
-        LOG.info("Loading config file at %s", config_file)
+        yaml_config_file = '/var/lib/kolla/config_files/config.yaml'
+        json_config_file = '/var/lib/kolla/config_files/config.json'
 
-        # Attempt to read config file
-        with open(config_file) as f:
+        if os.path.exists(yaml_config_file):
+            config_file = yaml_config_file
+            LOG.info("Loading config file at %s", config_file)
             try:
-                return json.load(f)
+                with open(config_file) as f:
+                    config = yaml.safe_load(f)
+
+                    if config is None:
+                        raise InvalidConfig(
+                            "Invalid yaml file found at %s: "
+                            "file is empty" % config_file)
+                    return config
+            except yaml.YAMLError:
+                raise InvalidConfig(
+                    "Invalid yaml file found at %s" % config_file)
+            except IOError as e:
+                raise InvalidConfig(
+                    "Could not read file %s: %r" % (config_file, e))
+        else:
+            config_file = json_config_file
+            LOG.info("Loading config file at %s", config_file)
+            try:
+                with open(config_file) as f:
+                    return json.load(f)
             except ValueError:
                 raise InvalidConfig(
                     "Invalid json file found at %s" % config_file)
@@ -631,7 +652,7 @@ def execute_config_check(config):
     """
     state = get_defaults_state()
 
-    # Build a set of all current destination paths from config.json
+    # Build a set of all current destination paths from the config
     # If the destination is a directory, we append the
     # basename of the source
     current_dests = {
@@ -642,7 +663,7 @@ def execute_config_check(config):
     }
 
     # Detect any paths that are present in the state file but
-    # missing from config.json.
+    # missing from the config file.
     # These would be either restored (if state[dest] has a backup)
     # or removed (if dest is null)
     removed_dests = [
@@ -653,7 +674,7 @@ def execute_config_check(config):
     if removed_dests:
         raise StateMismatch(
             f"The following config files are tracked in state but missing "
-            f"from config.json. "
+            f"from the config file. "
             f"They would be restored or removed: {sorted(removed_dests)}"
         )
 

docker/gnocchi/gnocchi-base/Dockerfile.j2

@@ -18,7 +18,6 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
         'python3-rados',
     ] %}
 
-RUN mkdir -p /var/www/cgi-bin/gnocchi
 {% elif base_package_type == 'deb' %}
 
     {% set gnocchi_base_packages = [

docker/horizon/Dockerfile.j2

@@ -46,13 +46,9 @@ COPY extend_start.sh /usr/local/bin/kolla_extend_start
 # NOTE(kevko): This dance with local settings python paths below is needed
 # because we are using different distros with different python version and we need to
 # know to which path symlink should point to.
-# NOTE(bbezak): pin setuptools for pip build isolation due to
-# https://bugs.launchpad.net/horizon/+bug/2141293
 RUN ln -s horizon-source/* horizon \
     && {{ macros.upper_constraints_remove("horizon") }} \
-    && echo "setuptools<82" > /tmp/horizon-build-constraints.txt \
-    && PIP_BUILD_CONSTRAINT=/tmp/horizon-build-constraints.txt \
-       {{ macros.install_pip(horizon_pip_packages | customizable("pip_packages")) }} \
+    && {{ macros.install_pip(horizon_pip_packages | customizable("pip_packages")) }} \
     && mkdir -p /etc/openstack-dashboard \
     && cp -r /horizon/openstack_dashboard/conf/* /etc/openstack-dashboard/ \
     && cp /horizon/openstack_dashboard/local/local_settings.py.example /etc/openstack-dashboard/local_settings.py \

docker/masakari/masakari-base/Dockerfile.j2

@@ -33,9 +33,8 @@ COPY extend_start.sh /usr/local/bin/kolla_extend_start
 
 RUN ln -s masakari-base-source/* masakari \
     && {{ macros.install_pip(masakari_base_pip_packages | customizable("pip_packages")) }} \
-    && mkdir -p /etc/masakari /var/www/cgi-bin/masakari \
+    && mkdir -p /etc/masakari \
     && cp -r /masakari/etc/masakari/* /etc/masakari/ \
-    && chmod 755 /var/www/cgi-bin/masakari \
     && touch /usr/local/bin/kolla_masakari_extend_start \
     && chmod 644 /usr/local/bin/kolla_extend_start /usr/local/bin/kolla_masakari_extend_start
 

docker/nova/nova-libvirt/Dockerfile.j2

@@ -20,6 +20,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
         'libguestfs',
         'libvirt-client',
         'libvirt-daemon',
+        'libvirt-daemon-log',
         'libvirt-daemon-config-nwfilter',
         'libvirt-daemon-driver-nwfilter',
         'libvirt-daemon-driver-nodedev',
@@ -75,6 +76,7 @@ LABEL maintainer="{{ maintainer }}" name="{{ image_name }}" build-date="{{ build
 
     {% if base_distro in ['debian'] %}
         {% set nova_libvirt_packages = nova_libvirt_packages + [
+            'libvirt-daemon-log',
             'usermode'
         ] %}
     {% endif %}
@@ -89,7 +91,8 @@ RUN rm -f /etc/libvirt/qemu/networks/default.xml /etc/libvirt/qemu/networks/auto
 {% endif %}
 
 COPY extend_start.sh /usr/local/bin/kolla_extend_start
-RUN chmod 644 /usr/local/bin/kolla_extend_start
+COPY kolla_nova_libvirt_start.sh /usr/local/bin/kolla_nova_libvirt_start
+RUN chmod 644 /usr/local/bin/kolla_extend_start && chmod 744 /usr/local/bin/kolla_nova_libvirt_start
 
 {{ macros.kolla_patch_sources() }}
 

docker/nova/nova-libvirt/kolla_nova_libvirt_start.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -euo pipefail
+
+# NOTE(mikal): Start virtlogd as a sidecar. We need this for
+# nova instance console logs to rotate. Otherwise they will
+# consume unbounded amounts of disk.
+#
+# No exec here: libvirtd will be the "main" process.
+/usr/sbin/virtlogd &
+
+# Now hand over PID 1's direct child role to libvirtd.
+exec /usr/sbin/libvirtd --listen

kolla/common/config.py

@@ -171,6 +171,14 @@
                help='The Docker namespace name'),
     cfg.StrOpt('network_mode', default='host',
                help='The network mode for Docker build. Example: host'),
+    cfg.BoolOpt('buildkit', default=True,
+                help='Use Docker BuildKit (docker buildx build) when building '
+                     'images. Requires the docker-buildx-plugin to be '
+                     'installed. Only valid with --engine docker.'),
+    cfg.StrOpt('buildkit-builder', default=None,
+               help='Name of the docker buildx builder instance to use. '
+                    'If unset the currently active builder is used. '
+                    'Only valid with --buildkit.'),
     cfg.BoolOpt('cache', default=True,
                 help='Use the container engine cache when building'),
     cfg.StrOpt('patches-path', default=None,
@@ -256,7 +264,7 @@
     cfg.StrOpt('image-name-prefix', default='',
                help='Prefix prepended to image names'),
     cfg.StrOpt('repos-yaml', default='',
-               help='Path to alternative repos.yaml file'),
+               help='Path to repos.yaml override file'),
     cfg.StrOpt('engine', default='docker', choices=['docker', 'podman'],
                help='Container engine to build images on.'),
     cfg.StrOpt('podman_base_url', default='unix:///run/podman/podman.sock',

kolla/common/sources.py

@@ -388,11 +388,11 @@
                      'openstack-network-exporter'
                      '-linux-${debian_arch}')},
     'prometheus-server': {
-        'version': '3.5.3',
+        'version': '3.5.4',
         'type': 'url',
         'sha256': {
-            'amd64': '8c30b9d99664e39b0363c0ba54fab30a7958e9d3de27246bf26ed85e6cfb8946',  # noqa: E501
-            'arm64': '11457bc76cab34f5ac05ba05fb80cfca1e8be7e4b31ae7c054879ce1066cb9a5'},  # noqa: E501
+            'amd64': 'be64e1cf657e6e7d132aca022f6135abedd660db053952e3f69e8affa3b9cc9e',  # noqa: E501
+            'arm64': '97634b9b6ed9ef18689e3b3572f1b277714650657516494fd164dcc53c1d3f48'},  # noqa: E501
         'location': ('https://github.com/'
                      'prometheus/prometheus/'
                      'releases/download/v${version}/'

kolla/common/utils.py

@@ -54,6 +54,21 @@ def make_a_logger(conf=None, image_name=None):
 LOG = make_a_logger()
 
 
+def check_docker_buildx():
+    try:
+        subprocess.check_output(  # nosec
+            ['docker', 'buildx', 'version'], stderr=subprocess.STDOUT)
+    except OSError as ex:
+        if ex.errno == 2:
+            LOG.error('"docker" command is not found.')
+        raise
+    except subprocess.CalledProcessError:
+        LOG.error('"docker buildx" is not available. '
+                  'Install the docker-buildx-plugin or disable BuildKit '
+                  'with "--nobuildkit".')
+        raise
+
+
 def get_docker_squash_version():
 
     try:

kolla/image/build.py

@@ -120,6 +120,10 @@ def run_build():
                       "Try running 'pip install docker'\n"
                       "Python error: %s", e)
             sys.exit(1)
+        if conf.buildkit and conf.squash:
+            LOG.error('--buildkit and --squash are mutually exclusive: '
+                      'docker buildx build does not support squashing.')
+            sys.exit(1)
         if conf.squash:
             squash_version = utils.get_docker_squash_version()
             LOG.info('Image squash is enabled and "docker-squash" version '
@@ -171,6 +175,12 @@ def run_build():
         kolla.list_dependencies()
         return
 
+    if (conf.engine == engine.Engine.DOCKER.value and conf.buildkit):
+        try:
+            utils.check_docker_buildx()
+        except Exception:
+            sys.exit(1)
+
     push_queue = queue.Queue()
     build_queue = kolla.build_queue(push_queue)
     workers = []