Update CI scripts from master (3.27)

build-scripts/unpack-tarballs

@@ -62,9 +62,11 @@ cd "$BASEDIR"
 gzip -dc $SOURCE_TARBALL | tar -xf -
 ln -s cfengine-3* core
 
-log_debug "UNPACKING MASTERFILES TARBALL AND SYMLINKING masterfiles/"
+log_debug "UNPACKING MASTERFILES TARBALL: $MASTERFILES_TARBALL AND SYMLINKING masterfiles/"
 # shellcheck disable=SC2086
 # > Double quote to prevent globbing and word splitting.
 # We want globbing here
 gzip -dc $MASTERFILES_TARBALL | tar -xf -
-ln -s cfengine-masterfiles-* masterfiles
+# the masterfiles tarball would match cfengine-masterfiles-* so find the unpacked dir specifically
+_cfmpfdir=$(find . -type d -name 'cfengine-masterfiles-*')
+ln -s "$_cfmpfdir" masterfiles

ci/centos-7-setup-devtoolset-11.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -ex
+sudo yum install -y centos-release-scl
+sudo rm -f /etc/yum.repos.d/CentOS-SCLo-scl.repo
+sudo sed -i 's,^#baseurl.*$,baseurl=https://vault.centos.org/7.9.2009/sclo/x86_64/rh/,' /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo
+sudo sed -i '/mirrorlist/d' /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo
+sudo yum update -y
+sudo yum install -y devtoolset-11
+if ! grep "source /opt/rh/devtoolset-11/enable" /usr/lib/rpm/find-debuginfo.sh; then
+  sudo sed -i '1a\source /opt/rh/devtoolset-11/enable' /usr/lib/rpm/find-debuginfo.sh
+fi
+source /opt/rh/devtoolset-11/enable

ci/cfengine-build-host-setup.cf

@@ -86,6 +86,15 @@ bundle agent cfengine_build_host_setup
     mingw_build_host::
       "mingw-w64";
 
+      # build-scripts/package-msi builds the MSI natively with wixl; msiinfo
+      # (msitools) is used to inspect it. No Wine, no i386. See ENT-13868.
+      "wixl";
+      "msitools";
+
+      # build-scripts/package-msi derives deterministic MSI GUIDs with uuidgen
+      # (uuid-runtime) for reproducible builds. See ENT-13792.
+      "uuid-runtime";
+
       "binfmt-support"
         comment => "update-binfmts command needed for build-scripts/package-msi script";
 
@@ -120,8 +129,8 @@ bundle agent cfengine_build_host_setup
       "gcc";
 
       "ncurses-devel"
-        if => not("sles_15"),
-        comment => "sles 15 requires a downgrade to install ncurses-devel as of July 25, 2025";
+        if => not("sles_15|sles_12"),
+        comment => "sles 12/15 require a downgrade to install ncurses-devel; handled by the commands: promise below";
 
       "pam-devel";
       "rsync";
@@ -269,7 +278,8 @@ bundle agent cfengine_build_host_setup
         expression => not(fileexists("/etc/cfengine-in-container.flag")),
         comment => "We use an explicit flag file that we control to avoid ambiguity about whether we are in a container or not.";
 
-    # Rust is build dependency for leech2 (gate on ubuntu>=20, debian>=12, redhat>=8)
+@if minimum_version(3.23)
+    # Rust is build dependency for leech2 (gate on ubuntu>=20, debian>=12, redhat>=7)
     ubuntu::
       "leech2_build_toolchain_host"
         expression => version_compare("$(sys.os_version_major)", ">=", "20");
@@ -280,7 +290,8 @@ bundle agent cfengine_build_host_setup
 
     (redhat|centos)::
       "leech2_build_toolchain_host"
-        expression => version_compare("$(sys.os_version_major)", ">=", "8");
+        expression => version_compare("$(sys.os_version_major)", ">=", "7");
+@endif
 
     any::
       "have_rust" expression => fileexists("/opt/rust/bin/rustc");
@@ -301,7 +312,6 @@ bundle agent cfengine_build_host_setup
       "java_ok"
         expression => version_compare("${java_version}", ">=", "21.0.0");
 @endif
-
       "missing_groovy" expression => not(fileexists("/usr/bin/groovy"));
 
     (redhat|centos).!(redhat_6|centos_6|redhat_7|centos_7)::
@@ -316,9 +326,7 @@ bundle agent cfengine_build_host_setup
     # rhel/centos-6 and 7 do not support --nobest or best property in yum.conf
     redhat_8|centos_8::
       "have_fakeroot"
-        expression => returnszero(
-          "command -v fakeroot >/dev/null", "useshell"
-        );
+        expression => returnszero("command -v fakeroot >/dev/null", "useshell");
 
     (redhat|centos)::
       "epel_release_ok"
@@ -370,6 +378,11 @@ bundle agent cfengine_build_host_setup
         comment => "note: centos-7 has installed instead of --installed argument, and that works on rhel-8 and rhel-9 so go with the sub-command instead of option";
 
   commands:
+    sshd_hardened::
+      'kill -1 $(pgrep -f "sshd -D")'
+        handle => "sshd_restarted",
+        contain => in_shell,
+        comment => "Reload sshd config with SIGHUP(1) to apply hardened configuration";
     have_tmp_mount::
       "mount -o remount,size=5G /tmp"
         comment => "We could check if /tmp was size 5G but not worth the trouble since this remount call just sets the maximum size of the tmpfs in virtual memory.",
@@ -386,7 +399,7 @@ bundle agent cfengine_build_host_setup
       "sh $(this.promise_dirname)/linux-install-groovy.sh" contain => in_shell;
 
     missing_java|insufficient_java_version::
-      "sh $(this.promise_dirname)/linux-install-jdk21.sh"
+      "sh $(this.promise_dirname)/linux-install-jdk.sh"
         contain => in_shell,
         classes => results("bundle", "java");
 
@@ -462,8 +475,7 @@ bundle agent cfengine_build_host_setup
     opensuse|suse|sles::
       "have_$(suse_users_and_groups)_group"
         expression => returnszero(
-          "grep '^$(suse_users_and_groups):' /etc/group >/dev/null",
-          "useshell"
+          "grep '^$(suse_users_and_groups):' /etc/group >/dev/null", "useshell"
         );
 
       "have_$(suse_users_and_groups)_user"
@@ -584,9 +596,9 @@ jenkins ALL=NOPASSWD: /usr/bin/podman
         depends_on => { "rpm_build_installed" };
 
   commands:
-    sles_15::
+    sles_15|sles_12::
       "zypper --non-interactive install --allow-downgrade ncurses-devel"
-        comment => "Special case mentioned elsewhere in this policy. ncurses-devel requires a downgrade as of July 25 2025",
+        comment => "Special case mentioned elsewhere in this policy. ncurses-devel requires a downgrade (sles 15 since July 25 2025; sles 12 SP5 since 2026)",
         contain => in_shell;
 
     (redhat_8|centos_8|redhat_9|redhat_10).(!have_perl_package_installed).(yum_dnf_conf_ok)::
@@ -644,12 +656,6 @@ jenkins ALL=NOPASSWD: /usr/bin/podman
         comment => "Verify KbdInteractiveAuthentication (OpenSSH 8.7+) or ChallengeResponseAuthentication (older) is disabled";
 
   services:
-    sshd_hardened::
-      "$(sshd_service_name)"
-        service_policy => "restart",
-        handle => "sshd_restarted",
-        comment => "Restart sshd to apply hardened configuration";
-
     any::
       "fail2ban"
         service_policy => "start",
@@ -659,10 +665,12 @@ jenkins ALL=NOPASSWD: /usr/bin/podman
       "fail2ban"
         service_policy => "restart",
         comment => "Restart fail2ban to apply jail configuration";
-  # skip /etc/hosts change for now, seems kind of wrong and corrupts ip6 entries like `::1 ip6-ip6-loopback`
-  # maybe the following is needed to silence such errors as:     ubuntu-16-mingw-j1: sudo: unable to resolve host localhost.localdomain
-  #    ubuntu::
-  #      "${paths.sed} -ri 's/localhost //' /etc/hosts";
+
+        # skip /etc/hosts change for now, seems kind of wrong and corrupts ip6 entries like `::1 ip6-ip6-loopback`
+        # maybe the following is needed to silence such errors as:     ubuntu-16-mingw-j1: sudo: unable to resolve host localhost.localdomain
+        #    ubuntu::
+        #      "${paths.sed} -ri 's/localhost //' /etc/hosts";
 }
+
 # todo, maybe need
 # ubuntu16-mingw: echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | sudo debconf-set-selections

ci/create_swap_file.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# bash is needed in order to use the "time" built-in and avoid needing
+# an external utility.
+
+set -e                                                   # exit on error
+
+# Argument $1 is the size in megabytes
+if [ x"$1" = x ]  ||  echo "$1" | grep -q '[^0-9]'
+then
+    exit 2
+fi
+SIZE="$1"
+
+if swapon | grep /swapfile >/dev/null; then
+  echo "/swapfile already configured and setup. Exiting."
+  exit 0
+fi
+
+time dd if=/dev/zero of=/swapfile bs=1M count=$SIZE
+chmod 0600 /swapfile
+
+PATH=$PATH:/sbin:/usr/sbin
+mkswap /swapfile
+swapon /swapfile

ci/fix-buildhost.sh

@@ -1,10 +1,37 @@
+#!/usr/bin/env bash
+# it is expected that this file is sourced, not executed directly
+set -ex
+
+if [ -f /etc/os-release ]; then
+  source /etc/os-release
+  if [ "$ID" = "centos" ] && [ "$VERSION_ID" = "7" ]; then
+    if command -v realpath >/dev/null; then
+      my_path="$(realpath "${BASH_SOURCE[0]}")"
+      my_dir="$(dirname "$my_path")"
+      source "$my_dir"/centos-7-setup-devtoolset-11.sh
+    else
+      echo "FAIL: could not find realpath command on rhel/centos-7 to source needed centos-7-setup-devtoolset-11.sh"
+      exit 1
+    fi
+  fi
+fi
+
 if [ "$(uname)" = "HP-UX" ]; then
   # /etc/profile contains tty code that won't work well when sourced and this VUE env var guards against running those bits
   # https://ftp.mirrorservice.org/sites/www.bitsavers.org/pdf/hp/9000_hpux/9.x/B1171-90044_HP_Visual_User_Environment_System_Administration_Manual_Nov91.pdf
   VUE=true
   export VUE
 fi
 
+if [ -f /etc/profile ]; then
+  # running on the proxied host or not we want to make sure local customizations are taken
+  # e.g. ent-14014: custom build of ssh needed for build-artifacts-cache needed and /etc/profile has PATH=/opt/craig/bin:$PATH
+  . /etc/profile
+fi
+
+mkdir -p ~/.ssh
+echo "build-artifacts-cache.cloud.cfengine.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGahpsY8Phk2+isBmuJQjjQVlh6BNL/Qetc14g26gowV" >> ~/.ssh/known_hosts
+
 # /etc/profile can contain tricky things, on suse for example it includes a call to tty which will fail in CI
 # so only source /etc/profile where we absolutely need it.
 if [ "$(uname)" = "HP-UX" ] || [ "$(uname)" = "SunOS" ]; then
@@ -14,11 +41,22 @@ if [ "$(uname)" = "HP-UX" ] || [ "$(uname)" = "SunOS" ]; then
     . /etc/profile
   fi
 fi
-
 # ENT-13750 we return to vendored openssl on rpm platforms so remove possibly installed development packages
 if command -v zypper >/dev/null 2>/dev/null; then
   sudo zypper remove -y libopenssl-devel || true
 fi
 if command -v yum >/dev/null 2>/dev/null; then
   sudo yum erase -y openssl-devel || true
 fi
+
+# MinGW hosts build the MSI with wixl (build-scripts/package-msi) and inspect it
+# with msiinfo (msitools). uuidgen (uuid-runtime) derives deterministic MSI
+# GUIDs for reproducible builds (ENT-13792). Installed by the build-host-setup
+# policy at image time; install here too so not-yet-reimaged mingw hosts get
+# them without a reimage. See ENT-13868.
+if [ -f /etc/cfengine-mingw-build-host.flag ]; then
+  if ! command -v wixl >/dev/null 2>&1 || ! command -v msiinfo >/dev/null 2>&1 || ! command -v uuidgen >/dev/null 2>&1; then
+    sudo apt-get update
+    sudo DEBIAN_FRONTEND=noninteractive apt-get install -y wixl msitools uuid-runtime
+  fi
+fi

ci/initialize-build-host.sh

@@ -136,9 +136,10 @@ do
     sleep 10
 done
 
-echo '========================================= PRINTING CLOUD-INIT LOG ==================================================='
-sudo sed 's/^.*/>>> &/' /var/log/cloud-init-output.log || true
-echo '======================================= DONE PRINTING CLOUD-INIT LOG ================================================'
+# TODO, instead of printing this out ALWAYS, print it out only in case of errors ENT-14372
+#echo '========================================= PRINTING CLOUD-INIT LOG ==================================================='
+#sudo sed 's/^.*/>>> &/' /var/log/cloud-init-output.log || true
+#echo '======================================= DONE PRINTING CLOUD-INIT LOG ================================================'
 
 if [ $attempts -le 0 ]
 then
@@ -147,9 +148,10 @@ then
     exit 1
 fi
 
-echo '=========================================== CURRENT ENVIRONMENT ====================================================='
-export
-echo '========================================= CURRENT ENVIRONMENT END ==================================================='
+# TODO only print current environment on errors, maybe save the environment NOW and then show a diff at ERROR
+#echo '=========================================== CURRENT ENVIRONMENT ====================================================='
+#export
+#echo '========================================= CURRENT ENVIRONMENT END ==================================================='
 
 # Disable TTY requirement. This normally happens in initialize-user-data.sh, but
 # for hosts that do not support cloud user data, it may not have happened
@@ -204,9 +206,9 @@ reset_nested_vm() {
     if sudo dmesg | grep -q "BIOS Google"
     then
 	# We're in Google Cloud, so just need to run nested-vm script again
-        if [ ! -d $HOME/mender-qa ]
+        if [ ! -d $HOME/buildscripts ]
 	then
-            echo "Where is mender-qa repo gone?"
+            echo "Where is buildscripts repo gone?"
 	    sudo ls -lap $HOME
 	    exit 1
         fi
@@ -231,6 +233,7 @@ reset_nested_vm() {
                 sudo arp -d $ip
             fi
 	fi
+        # TODO, remove this, we don't need or use or test nested-vms
 	$HOME/mender-qa/scripts/nested-vm.sh $HOME/*.qcow2
         login="`cat $HOME/proxy-target.txt`"
         if $RSH $login true
@@ -363,9 +366,9 @@ then
     # the repository in provisioning. Permanent hosts don't keep it in HOME,
     # in order to avoid it getting stale, and will have it in the WORKSPACE
     # instead, synced separately below.
-    if [ -d $HOME/mender-qa ]
+    if [ -d $HOME/buildscripts ]
     then
-        $RSYNC -e "$RSH"    $HOME/mender-qa  $login:.
+        $RSYNC -e "$RSH"    $HOME/buildscripts  $login:.
     fi
 
     # Copy the workspace. If there is no workspace defined, we are not in the

ci/initialize-user-data.sh

@@ -0,0 +1,83 @@
+#!/bin/false
+
+# This file should be sourced, not run.
+
+# This script will do build slave setup, including creating credentials for the
+# jenkins user, based on root's credentials (will copy its keys). The script is
+# expected to be sourced early in the user-data phase after provisioning.
+
+# Make sure error detection and verbose output is on, if they aren't already.
+set -x -e
+
+# Add jenkins user and copy credentials.
+useradd -m -u 1010 jenkins || true
+mkdir -p /home/jenkins/.ssh
+# copy /root/.ssh/authorized_keys to /home/jenkins/.ssh, removing everything
+# before 'ssh-rsa'. Some platforms have forcecommand='echo "root access disabled"'
+# there.
+sed 's/.*ssh-rsa/ssh-rsa/' /root/.ssh/authorized_keys >/home/jenkins/.ssh/authorized_keys || true
+
+# Enable sudo access for jenkins.
+echo "jenkins ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers
+
+# Disable TTY requirement.
+sed -i -e 's/^\( *Defaults *requiretty *\)$/# \1/' /etc/sudoers
+
+# Copy the buildscripts repository to jenkins user.
+cp -r /root/buildscripts /home/jenkins
+# was copying build-artifacts-cache known host entry
+#cp /root/mender-qa/data/known_hosts                    /home/jenkins/.ssh/known_hosts
+
+
+# add authorized_keys file before chowning, so that initialize-build-host.sh can manage
+touch /home/jenkins/.ssh/authorized_keys
+
+# Make sure everything in jenkins' folder has right owner.
+chown -R jenkins:jenkins /home/jenkins
+
+groupadd -r kvm || true # In case it already exists.
+usermod -a -G kvm jenkins
+
+# change hostname to localhost
+# it will fix sudo complaining "unable to resolve host digitalocean",
+# and some tests
+hostname localhost
+# Ensure reverse hostname resolution is correct and 127.0.0.1 is always 'localhost'.
+# There's no nice shell command to test it but this one:
+# python -c 'import socket;print socket.gethostbyaddr("127.0.0.1")'
+if test -f /etc/hosts; then
+    sed -i -e '1s/^/127.0.0.1 localhost localhost.localdomain\n/' /etc/hosts
+else
+    echo '127.0.0.1 localhost localhost.localdomain' >/etc/hosts
+fi
+
+apt_get() {
+    # Work around apt-get not waiting for a lock if it's taken. We want to wait
+    # for it instead of bailing out. No good return code to check unfortunately,
+    # so we just have to look inside the log.
+
+    pid=$$
+    # Maximum five minute wait (30 * 10 seconds)
+    attempts=30
+
+    while true
+    do
+        ( /usr/bin/apt-get "$@" 2>&1 ; echo $? > /tmp/apt-get-return-code.$pid.txt ) | tee /tmp/apt-get.$pid.log
+        if [ $attempts -gt 0 ] && \
+               [ "$(cat /tmp/apt-get-return-code.$pid.txt)" -ne 0 ] && \
+               fgrep "Could not get lock" /tmp/apt-get.$pid.log > /dev/null
+        then
+            attempts=$(expr $attempts - 1 || true)
+            sleep 10
+        else
+            break
+        fi
+    done
+
+    ret_code=$(cat /tmp/apt-get-return-code.$pid.txt)
+    rm -f /tmp/apt-get-return-code.$pid.txt /tmp/apt-get.$pid.log
+
+    return $ret_code
+}
+alias apt=apt_get
+alias apt-get=apt_get