Synchronise master with upstream

doc/source/admin/image-building.rst

@@ -372,9 +372,22 @@ verification purposes.
 Python packages build options
 -----------------------------
 
-The block ``base_pip_conf`` in the ``base`` Dockerfile can be used to provide
-the PyPI build customisation options via the standard environment variables
-like ``PIP_INDEX_URL``, ``PIP_TRUSTED_HOST``, etc.
+The PyPI mirror used during image builds can be configured directly in
+``kolla-build.conf``:
+
+.. path /etc/kolla/kolla-build.conf
+.. code-block:: ini
+
+   [DEFAULT]
+   pip_index_url = https://pypi.example.com/simple
+   pip_trusted_host = pypi.example.com
+   pip_extra_index_url = https://wheels.example.com/simple
+
+These options set the ``PIP_INDEX_URL``, ``PIP_TRUSTED_HOST``, and
+``PIP_EXTRA_INDEX_URL`` ARG variables in the base image via the
+``base_pip_conf`` Dockerfile block. The ``pip_extra_index_url`` option is
+optional. For further customisation, the block ``base_pip_conf`` can be
+overridden via the template_override mechanism.
 
 To override PYPI upper-constraints of all OpenStack images, you can
 define the source location of openstack-base. in ``kolla-build.conf``.

kolla/common/config.py

@@ -298,6 +298,14 @@
                 help='Clean all package cache.'),
     cfg.ListOpt('allowed-to-fail', default=[],
                 help='Images which are allowed to fail'),
+    cfg.StrOpt('pip_index_url',
+               help='URL of the primary pip index (sets ARG PIP_INDEX_URL)'),
+    cfg.StrOpt('pip_trusted_host',
+               help='Hostname to mark as trusted for pip '
+                    '(sets ARG PIP_TRUSTED_HOST)'),
+    cfg.StrOpt('pip_extra_index_url',
+               help='URL of an extra pip index '
+                    '(sets ARG PIP_EXTRA_INDEX_URL)'),
 ]
 
 

kolla/docker/base/Dockerfile.j2

@@ -341,9 +341,9 @@ RUN touch /usr/local/bin/kolla_extend_start \
     && rm -f /tmp/kolla_bashrc
 
 {% block base_pip_conf %}
-# the variables like PIP_INDEX_URL, PIP_EXTRA_INDEX_URL, PIP_TRUSTED_HOST etc. should be defined here.
-# ENV PIP_INDEX_URL=https://pypi.python.org/simple
-# ENV PIP_TRUSTED_HOST=pypi.python.org
+# pip build-time settings (PIP_INDEX_URL, PIP_TRUSTED_HOST, PIP_EXTRA_INDEX_URL) are
+# injected as ARG instructions by kolla-build and passed via --build-arg, so they do
+# not persist in the final image. Override this block to add further pip configuration.
 {% endblock %}
 
 {{ macros.kolla_patch_sources() }}

kolla/docker/ironic/ironic-pxe/extend_start.sh

@@ -46,15 +46,13 @@ function prepare_ipxe {
         # NOTE(m-anson): ipxe-arm64.efi is not symlinked from /boot to
         # /usr/lib/ipxe by the Ubuntu ipxe package, so fix that here.
         if [[ -e /boot/ipxe-arm64.efi ]]; then
-            ln -s /boot/ipxe-arm64.efi /usr/lib/ipxe/
+            ln -sf /boot/ipxe-arm64.efi /usr/lib/ipxe/
         fi
         cp /usr/lib/ipxe/{undionly.kpxe,ipxe*.efi,snponly.efi} ${TFTPBOOT_PATH}/
     elif [[ "${KOLLA_BASE_DISTRO}" =~ centos|rocky ]]; then
         cp /usr/share/ipxe/{undionly.kpxe,ipxe-snponly-x86_64.efi} ${TFTPBOOT_PATH}/
         cp /usr/share/ipxe/arm64-efi/snponly.efi ${TFTPBOOT_PATH}/ipxe-snponly-aarch64.efi
-        if [[ ! -e ${TFTPBOOT_PATH}/snponly.efi ]]; then
-            ln -s ${TFTPBOOT_PATH}/ipxe-snponly-${KOLLA_BASE_ARCH}.efi ${TFTPBOOT_PATH}/snponly.efi
-        fi
+        ln -sf ${TFTPBOOT_PATH}/ipxe-snponly-${KOLLA_BASE_ARCH}.efi ${TFTPBOOT_PATH}/snponly.efi
     fi
 }
 

kolla/image/kolla_worker.py

@@ -411,6 +411,21 @@ def create_dockerfiles(self):
                 env.globals.update(self._get_methods())
                 template = env.get_template(template_name)
             content = template.render(values, env=os.environ)
+            pip_args = [
+                'ARG {}'.format(k)
+                for k, v in (
+                    ('PIP_INDEX_URL', self.conf.pip_index_url),
+                    ('PIP_TRUSTED_HOST', self.conf.pip_trusted_host),
+                    ('PIP_EXTRA_INDEX_URL', self.conf.pip_extra_index_url),
+                ) if v
+            ]
+            if pip_args:
+                lines = content.split('\n')
+                for i, line in enumerate(lines):
+                    if line.startswith('FROM '):
+                        lines[i + 1:i + 1] = pip_args
+                        break
+                content = '\n'.join(lines)
             content_path = os.path.join(path, 'Dockerfile')
             with open(content_path, 'w') as f:
                 LOG.debug("Rendered %s into:", tpl_path)

kolla/image/tasks.py

@@ -304,6 +304,15 @@ def update_buildargs(self):
             if proxy_var in os.environ and proxy_var not in buildargs:
                 buildargs[proxy_var] = os.environ.get(proxy_var)
 
+        pip_vars = (
+            ('PIP_INDEX_URL', self.conf.pip_index_url),
+            ('PIP_TRUSTED_HOST', self.conf.pip_trusted_host),
+            ('PIP_EXTRA_INDEX_URL', self.conf.pip_extra_index_url),
+        )
+        for pip_var, value in pip_vars:
+            if value and pip_var not in buildargs:
+                buildargs[pip_var] = value
+
         if not buildargs:
             return None
         return buildargs

releasenotes/notes/ironic-pxe-symlinks-18d09b19373f16c4.yaml

@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    Fixes ``ironic_pxe`` launch failures when symlinks to EFI files have
+    previously been created.
+    `LP#2160317 <https://launchpad.net/bugs/2160317>`__

releasenotes/notes/pip-build-args-3e222f6a67c57407.yaml

@@ -0,0 +1,19 @@
+---
+features:
+  - |
+    Added ``pip_index_url``, ``pip_trusted_host`` and ``pip_extra_index_url``
+    options to ``kolla-build.conf``. These set the corresponding
+    ``PIP_INDEX_URL``, ``PIP_TRUSTED_HOST`` and ``PIP_EXTRA_INDEX_URL``
+    build-time ``ARG`` instructions in the base image, passed via
+    ``--build-arg`` so the values do not persist in the final image.
+upgrade:
+  - |
+    Pip index configuration for image builds should now be set via the new
+    ``pip_index_url``, ``pip_trusted_host`` and ``pip_extra_index_url``
+    options in ``kolla-build.conf``, instead of ``ENV PIP_*`` lines in a
+    ``base_pip_conf`` template override. If your ``template_overrides.j2``
+    still sets ``ENV PIP_INDEX_URL``/``ENV PIP_TRUSTED_HOST``/
+    ``ENV PIP_EXTRA_INDEX_URL`` in that block, it will continue to take
+    precedence over the new build ``ARG`` values (``ENV`` persists in the
+    image and is evaluated after ``ARG``). Remove those ``ENV`` overrides
+    and use the new config options instead.

tests/playbooks/run.yml

@@ -62,6 +62,15 @@
       when:
         - not publisher
 
+    - name: Set pip mirror config
+      vars:
+        kolla_pip_mirror_config:
+          DEFAULT:
+            pip_index_url: "{{ nodepool_pypi_mirror }}"
+            pip_trusted_host: "{{ nodepool_mirror_host }}"
+      ansible.builtin.set_fact:
+        kolla_build_config: "{{ kolla_build_config | combine(kolla_pip_mirror_config, recursive=True) }}"
+
     - name: Set up base repo overrides
       vars:
         kolla_repos_yaml_config:

tests/templates/template_overrides.j2

@@ -1,31 +1,3 @@
 {% raw %}
 {% extends parent_template %}
-
-{% block base_pip_conf %}
-{% endraw %}
-
-ENV PIP_INDEX_URL {{ nodepool_pypi_mirror }}
-ENV PIP_TRUSTED_HOST {{ nodepool_mirror_host }}
-{% if use_infra_wheels_mirror | default(true) %}
-ENV PIP_EXTRA_INDEX_URL {{ nodepool_wheel_mirror }}
-{% endif %}
-
-{% raw %}
-{% endblock %}
-{% endraw %}
-
-{# Revert to upstream mirrors after build is complete #}
-
-{% raw %}
-{% block footer %}
-{% endraw %}
-
-ENV PIP_INDEX_URL=
-ENV PIP_TRUSTED_HOST=
-{% if use_infra_wheels_mirror | default(true) %}
-ENV PIP_EXTRA_INDEX_URL=
-{% endif %}
-
-{% raw %}
-{% endblock %}
 {% endraw %}

zuul.d/project.yaml

@@ -19,6 +19,7 @@
       - kolla-scenario-masakari
       - kolla-scenario-octavia
       - kolla-scenario-ovn
+      - kolla-scenario-prometheus-opensearch
       - kolla-scenario-telemetry
       - kolla-tox-genconfig
       - openstack-python3-jobs

zuul.d/scenarios/prometheus-opensearch.yaml

@@ -0,0 +1,12 @@
+---
+- project-template:
+    name: kolla-scenario-prometheus-opensearch
+    check:
+      jobs:
+        - kolla-ansible-debian-trixie-prometheus-opensearch: &files
+            files: ^kolla/docker/(base|fluentd|grafana|opensearch|prometheus)/
+        - kolla-ansible-debian-trixie-prometheus-opensearch-upgrade: *files
+        - kolla-ansible-rocky-10-prometheus-opensearch: *files
+        - kolla-ansible-rocky-10-prometheus-opensearch-upgrade: *files
+        - kolla-ansible-ubuntu-noble-prometheus-opensearch: *files
+        - kolla-ansible-ubuntu-noble-prometheus-opensearch-upgrade: *files