diff --git a/build-scripts/unpack-tarballs b/build-scripts/unpack-tarballs index 9130c973b..bb67aa9af 100755 --- a/build-scripts/unpack-tarballs +++ b/build-scripts/unpack-tarballs @@ -62,9 +62,11 @@ cd "$BASEDIR" gzip -dc $SOURCE_TARBALL | tar -xf - ln -s cfengine-3* core -log_debug "UNPACKING MASTERFILES TARBALL AND SYMLINKING masterfiles/" +log_debug "UNPACKING MASTERFILES TARBALL: $MASTERFILES_TARBALL AND SYMLINKING masterfiles/" # shellcheck disable=SC2086 # > Double quote to prevent globbing and word splitting. # We want globbing here gzip -dc $MASTERFILES_TARBALL | tar -xf - -ln -s cfengine-masterfiles-* masterfiles +# the masterfiles tarball would match cfengine-masterfiles-* so find the unpacked dir specifically +_cfmpfdir=$(find . -type d -name 'cfengine-masterfiles-*') +ln -s "$_cfmpfdir" masterfiles diff --git a/ci/centos-7-setup-devtoolset-11.sh b/ci/centos-7-setup-devtoolset-11.sh new file mode 100755 index 000000000..66250e3cf --- /dev/null +++ b/ci/centos-7-setup-devtoolset-11.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -ex +sudo yum install -y centos-release-scl +sudo rm -f /etc/yum.repos.d/CentOS-SCLo-scl.repo +sudo sed -i 's,^#baseurl.*$,baseurl=https://vault.centos.org/7.9.2009/sclo/x86_64/rh/,' /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo +sudo sed -i '/mirrorlist/d' /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo +sudo yum update -y +sudo yum install -y devtoolset-11 +if ! grep "source /opt/rh/devtoolset-11/enable" /usr/lib/rpm/find-debuginfo.sh; then + sudo sed -i '1a\source /opt/rh/devtoolset-11/enable' /usr/lib/rpm/find-debuginfo.sh +fi +source /opt/rh/devtoolset-11/enable diff --git a/ci/cfengine-build-host-setup.cf b/ci/cfengine-build-host-setup.cf index c4e148787..a8205af36 100644 --- a/ci/cfengine-build-host-setup.cf +++ b/ci/cfengine-build-host-setup.cf @@ -86,6 +86,15 @@ bundle agent cfengine_build_host_setup mingw_build_host:: "mingw-w64"; + # build-scripts/package-msi builds the MSI natively with wixl; msiinfo + # (msitools) is used to inspect it. No Wine, no i386. See ENT-13868. + "wixl"; + "msitools"; + + # build-scripts/package-msi derives deterministic MSI GUIDs with uuidgen + # (uuid-runtime) for reproducible builds. See ENT-13792. + "uuid-runtime"; + "binfmt-support" comment => "update-binfmts command needed for build-scripts/package-msi script"; @@ -120,8 +129,8 @@ bundle agent cfengine_build_host_setup "gcc"; "ncurses-devel" - if => not("sles_15"), - comment => "sles 15 requires a downgrade to install ncurses-devel as of July 25, 2025"; + if => not("sles_15|sles_12"), + comment => "sles 12/15 require a downgrade to install ncurses-devel; handled by the commands: promise below"; "pam-devel"; "rsync"; @@ -269,7 +278,8 @@ bundle agent cfengine_build_host_setup expression => not(fileexists("/etc/cfengine-in-container.flag")), comment => "We use an explicit flag file that we control to avoid ambiguity about whether we are in a container or not."; - # Rust is build dependency for leech2 (gate on ubuntu>=20, debian>=12, redhat>=8) +@if minimum_version(3.23) + # Rust is build dependency for leech2 (gate on ubuntu>=20, debian>=12, redhat>=7) ubuntu:: "leech2_build_toolchain_host" expression => version_compare("$(sys.os_version_major)", ">=", "20"); @@ -280,7 +290,8 @@ bundle agent cfengine_build_host_setup (redhat|centos):: "leech2_build_toolchain_host" - expression => version_compare("$(sys.os_version_major)", ">=", "8"); + expression => version_compare("$(sys.os_version_major)", ">=", "7"); +@endif any:: "have_rust" expression => fileexists("/opt/rust/bin/rustc"); @@ -301,7 +312,6 @@ bundle agent cfengine_build_host_setup "java_ok" expression => version_compare("${java_version}", ">=", "21.0.0"); @endif - "missing_groovy" expression => not(fileexists("/usr/bin/groovy")); (redhat|centos).!(redhat_6|centos_6|redhat_7|centos_7):: @@ -316,9 +326,7 @@ bundle agent cfengine_build_host_setup # rhel/centos-6 and 7 do not support --nobest or best property in yum.conf redhat_8|centos_8:: "have_fakeroot" - expression => returnszero( - "command -v fakeroot >/dev/null", "useshell" - ); + expression => returnszero("command -v fakeroot >/dev/null", "useshell"); (redhat|centos):: "epel_release_ok" @@ -370,6 +378,11 @@ bundle agent cfengine_build_host_setup comment => "note: centos-7 has installed instead of --installed argument, and that works on rhel-8 and rhel-9 so go with the sub-command instead of option"; commands: + sshd_hardened:: + 'kill -1 $(pgrep -f "sshd -D")' + handle => "sshd_restarted", + contain => in_shell, + comment => "Reload sshd config with SIGHUP(1) to apply hardened configuration"; have_tmp_mount:: "mount -o remount,size=5G /tmp" comment => "We could check if /tmp was size 5G but not worth the trouble since this remount call just sets the maximum size of the tmpfs in virtual memory.", @@ -386,7 +399,7 @@ bundle agent cfengine_build_host_setup "sh $(this.promise_dirname)/linux-install-groovy.sh" contain => in_shell; missing_java|insufficient_java_version:: - "sh $(this.promise_dirname)/linux-install-jdk21.sh" + "sh $(this.promise_dirname)/linux-install-jdk.sh" contain => in_shell, classes => results("bundle", "java"); @@ -462,8 +475,7 @@ bundle agent cfengine_build_host_setup opensuse|suse|sles:: "have_$(suse_users_and_groups)_group" expression => returnszero( - "grep '^$(suse_users_and_groups):' /etc/group >/dev/null", - "useshell" + "grep '^$(suse_users_and_groups):' /etc/group >/dev/null", "useshell" ); "have_$(suse_users_and_groups)_user" @@ -584,9 +596,9 @@ jenkins ALL=NOPASSWD: /usr/bin/podman depends_on => { "rpm_build_installed" }; commands: - sles_15:: + sles_15|sles_12:: "zypper --non-interactive install --allow-downgrade ncurses-devel" - comment => "Special case mentioned elsewhere in this policy. ncurses-devel requires a downgrade as of July 25 2025", + comment => "Special case mentioned elsewhere in this policy. ncurses-devel requires a downgrade (sles 15 since July 25 2025; sles 12 SP5 since 2026)", contain => in_shell; (redhat_8|centos_8|redhat_9|redhat_10).(!have_perl_package_installed).(yum_dnf_conf_ok):: @@ -644,12 +656,6 @@ jenkins ALL=NOPASSWD: /usr/bin/podman comment => "Verify KbdInteractiveAuthentication (OpenSSH 8.7+) or ChallengeResponseAuthentication (older) is disabled"; services: - sshd_hardened:: - "$(sshd_service_name)" - service_policy => "restart", - handle => "sshd_restarted", - comment => "Restart sshd to apply hardened configuration"; - any:: "fail2ban" service_policy => "start", @@ -659,10 +665,12 @@ jenkins ALL=NOPASSWD: /usr/bin/podman "fail2ban" service_policy => "restart", comment => "Restart fail2ban to apply jail configuration"; - # skip /etc/hosts change for now, seems kind of wrong and corrupts ip6 entries like `::1 ip6-ip6-loopback` - # maybe the following is needed to silence such errors as: ubuntu-16-mingw-j1: sudo: unable to resolve host localhost.localdomain - # ubuntu:: - # "${paths.sed} -ri 's/localhost //' /etc/hosts"; + + # skip /etc/hosts change for now, seems kind of wrong and corrupts ip6 entries like `::1 ip6-ip6-loopback` + # maybe the following is needed to silence such errors as: ubuntu-16-mingw-j1: sudo: unable to resolve host localhost.localdomain + # ubuntu:: + # "${paths.sed} -ri 's/localhost //' /etc/hosts"; } + # todo, maybe need # ubuntu16-mingw: echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | sudo debconf-set-selections diff --git a/ci/create_swap_file.sh b/ci/create_swap_file.sh new file mode 100755 index 000000000..1339d323d --- /dev/null +++ b/ci/create_swap_file.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# bash is needed in order to use the "time" built-in and avoid needing +# an external utility. + +set -e # exit on error + +# Argument $1 is the size in megabytes +if [ x"$1" = x ] || echo "$1" | grep -q '[^0-9]' +then + exit 2 +fi +SIZE="$1" + +if swapon | grep /swapfile >/dev/null; then + echo "/swapfile already configured and setup. Exiting." + exit 0 +fi + +time dd if=/dev/zero of=/swapfile bs=1M count=$SIZE +chmod 0600 /swapfile + +PATH=$PATH:/sbin:/usr/sbin +mkswap /swapfile +swapon /swapfile diff --git a/ci/fix-buildhost.sh b/ci/fix-buildhost.sh index af7cfcdcc..bff94d5c7 100755 --- a/ci/fix-buildhost.sh +++ b/ci/fix-buildhost.sh @@ -1,3 +1,21 @@ +#!/usr/bin/env bash +# it is expected that this file is sourced, not executed directly +set -ex + +if [ -f /etc/os-release ]; then + source /etc/os-release + if [ "$ID" = "centos" ] && [ "$VERSION_ID" = "7" ]; then + if command -v realpath >/dev/null; then + my_path="$(realpath "${BASH_SOURCE[0]}")" + my_dir="$(dirname "$my_path")" + source "$my_dir"/centos-7-setup-devtoolset-11.sh + else + echo "FAIL: could not find realpath command on rhel/centos-7 to source needed centos-7-setup-devtoolset-11.sh" + exit 1 + fi + fi +fi + if [ "$(uname)" = "HP-UX" ]; then # /etc/profile contains tty code that won't work well when sourced and this VUE env var guards against running those bits # https://ftp.mirrorservice.org/sites/www.bitsavers.org/pdf/hp/9000_hpux/9.x/B1171-90044_HP_Visual_User_Environment_System_Administration_Manual_Nov91.pdf @@ -5,6 +23,15 @@ if [ "$(uname)" = "HP-UX" ]; then export VUE fi +if [ -f /etc/profile ]; then + # running on the proxied host or not we want to make sure local customizations are taken + # e.g. ent-14014: custom build of ssh needed for build-artifacts-cache needed and /etc/profile has PATH=/opt/craig/bin:$PATH + . /etc/profile +fi + +mkdir -p ~/.ssh +echo "build-artifacts-cache.cloud.cfengine.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGahpsY8Phk2+isBmuJQjjQVlh6BNL/Qetc14g26gowV" >> ~/.ssh/known_hosts + # /etc/profile can contain tricky things, on suse for example it includes a call to tty which will fail in CI # so only source /etc/profile where we absolutely need it. if [ "$(uname)" = "HP-UX" ] || [ "$(uname)" = "SunOS" ]; then @@ -14,7 +41,6 @@ if [ "$(uname)" = "HP-UX" ] || [ "$(uname)" = "SunOS" ]; then . /etc/profile fi fi - # ENT-13750 we return to vendored openssl on rpm platforms so remove possibly installed development packages if command -v zypper >/dev/null 2>/dev/null; then sudo zypper remove -y libopenssl-devel || true @@ -22,3 +48,15 @@ fi if command -v yum >/dev/null 2>/dev/null; then sudo yum erase -y openssl-devel || true fi + +# MinGW hosts build the MSI with wixl (build-scripts/package-msi) and inspect it +# with msiinfo (msitools). uuidgen (uuid-runtime) derives deterministic MSI +# GUIDs for reproducible builds (ENT-13792). Installed by the build-host-setup +# policy at image time; install here too so not-yet-reimaged mingw hosts get +# them without a reimage. See ENT-13868. +if [ -f /etc/cfengine-mingw-build-host.flag ]; then + if ! command -v wixl >/dev/null 2>&1 || ! command -v msiinfo >/dev/null 2>&1 || ! command -v uuidgen >/dev/null 2>&1; then + sudo apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y wixl msitools uuid-runtime + fi +fi diff --git a/ci/initialize-build-host.sh b/ci/initialize-build-host.sh index a06cf1cd8..eefbbc937 100644 --- a/ci/initialize-build-host.sh +++ b/ci/initialize-build-host.sh @@ -136,9 +136,10 @@ do sleep 10 done -echo '========================================= PRINTING CLOUD-INIT LOG ===================================================' -sudo sed 's/^.*/>>> &/' /var/log/cloud-init-output.log || true -echo '======================================= DONE PRINTING CLOUD-INIT LOG ================================================' +# TODO, instead of printing this out ALWAYS, print it out only in case of errors ENT-14372 +#echo '========================================= PRINTING CLOUD-INIT LOG ===================================================' +#sudo sed 's/^.*/>>> &/' /var/log/cloud-init-output.log || true +#echo '======================================= DONE PRINTING CLOUD-INIT LOG ================================================' if [ $attempts -le 0 ] then @@ -147,9 +148,10 @@ then exit 1 fi -echo '=========================================== CURRENT ENVIRONMENT =====================================================' -export -echo '========================================= CURRENT ENVIRONMENT END ===================================================' +# TODO only print current environment on errors, maybe save the environment NOW and then show a diff at ERROR +#echo '=========================================== CURRENT ENVIRONMENT =====================================================' +#export +#echo '========================================= CURRENT ENVIRONMENT END ===================================================' # Disable TTY requirement. This normally happens in initialize-user-data.sh, but # for hosts that do not support cloud user data, it may not have happened @@ -204,9 +206,9 @@ reset_nested_vm() { if sudo dmesg | grep -q "BIOS Google" then # We're in Google Cloud, so just need to run nested-vm script again - if [ ! -d $HOME/mender-qa ] + if [ ! -d $HOME/buildscripts ] then - echo "Where is mender-qa repo gone?" + echo "Where is buildscripts repo gone?" sudo ls -lap $HOME exit 1 fi @@ -231,6 +233,7 @@ reset_nested_vm() { sudo arp -d $ip fi fi + # TODO, remove this, we don't need or use or test nested-vms $HOME/mender-qa/scripts/nested-vm.sh $HOME/*.qcow2 login="`cat $HOME/proxy-target.txt`" if $RSH $login true @@ -363,9 +366,9 @@ then # the repository in provisioning. Permanent hosts don't keep it in HOME, # in order to avoid it getting stale, and will have it in the WORKSPACE # instead, synced separately below. - if [ -d $HOME/mender-qa ] + if [ -d $HOME/buildscripts ] then - $RSYNC -e "$RSH" $HOME/mender-qa $login:. + $RSYNC -e "$RSH" $HOME/buildscripts $login:. fi # Copy the workspace. If there is no workspace defined, we are not in the diff --git a/ci/initialize-user-data.sh b/ci/initialize-user-data.sh new file mode 100644 index 000000000..605251874 --- /dev/null +++ b/ci/initialize-user-data.sh @@ -0,0 +1,83 @@ +#!/bin/false + +# This file should be sourced, not run. + +# This script will do build slave setup, including creating credentials for the +# jenkins user, based on root's credentials (will copy its keys). The script is +# expected to be sourced early in the user-data phase after provisioning. + +# Make sure error detection and verbose output is on, if they aren't already. +set -x -e + +# Add jenkins user and copy credentials. +useradd -m -u 1010 jenkins || true +mkdir -p /home/jenkins/.ssh +# copy /root/.ssh/authorized_keys to /home/jenkins/.ssh, removing everything +# before 'ssh-rsa'. Some platforms have forcecommand='echo "root access disabled"' +# there. +sed 's/.*ssh-rsa/ssh-rsa/' /root/.ssh/authorized_keys >/home/jenkins/.ssh/authorized_keys || true + +# Enable sudo access for jenkins. +echo "jenkins ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers + +# Disable TTY requirement. +sed -i -e 's/^\( *Defaults *requiretty *\)$/# \1/' /etc/sudoers + +# Copy the buildscripts repository to jenkins user. +cp -r /root/buildscripts /home/jenkins +# was copying build-artifacts-cache known host entry +#cp /root/mender-qa/data/known_hosts /home/jenkins/.ssh/known_hosts + + +# add authorized_keys file before chowning, so that initialize-build-host.sh can manage +touch /home/jenkins/.ssh/authorized_keys + +# Make sure everything in jenkins' folder has right owner. +chown -R jenkins:jenkins /home/jenkins + +groupadd -r kvm || true # In case it already exists. +usermod -a -G kvm jenkins + +# change hostname to localhost +# it will fix sudo complaining "unable to resolve host digitalocean", +# and some tests +hostname localhost +# Ensure reverse hostname resolution is correct and 127.0.0.1 is always 'localhost'. +# There's no nice shell command to test it but this one: +# python -c 'import socket;print socket.gethostbyaddr("127.0.0.1")' +if test -f /etc/hosts; then + sed -i -e '1s/^/127.0.0.1 localhost localhost.localdomain\n/' /etc/hosts +else + echo '127.0.0.1 localhost localhost.localdomain' >/etc/hosts +fi + +apt_get() { + # Work around apt-get not waiting for a lock if it's taken. We want to wait + # for it instead of bailing out. No good return code to check unfortunately, + # so we just have to look inside the log. + + pid=$$ + # Maximum five minute wait (30 * 10 seconds) + attempts=30 + + while true + do + ( /usr/bin/apt-get "$@" 2>&1 ; echo $? > /tmp/apt-get-return-code.$pid.txt ) | tee /tmp/apt-get.$pid.log + if [ $attempts -gt 0 ] && \ + [ "$(cat /tmp/apt-get-return-code.$pid.txt)" -ne 0 ] && \ + fgrep "Could not get lock" /tmp/apt-get.$pid.log > /dev/null + then + attempts=$(expr $attempts - 1 || true) + sleep 10 + else + break + fi + done + + ret_code=$(cat /tmp/apt-get-return-code.$pid.txt) + rm -f /tmp/apt-get-return-code.$pid.txt /tmp/apt-get.$pid.log + + return $ret_code +} +alias apt=apt_get +alias apt-get=apt_get diff --git a/ci/linux-install-composer.sh b/ci/linux-install-composer.sh new file mode 100755 index 000000000..7f9fc8e56 --- /dev/null +++ b/ci/linux-install-composer.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +if command -v composer >/dev/null; then + exit +fi +VERSION="${COMPOSER_VERSION:-2.9.5}" +INSTALL_DIR="${COMPOSER_INSTALL_DIR:-/usr/local/bin}" +PHP_PATH="${PHP_BIN:-php}" +INSTALLER="composer-installer.php" + +trap 'rm -f "$INSTALLER"' EXIT + +curl -fsSL https://getcomposer.org/installer -o "$INSTALLER" + +# Verify checksum +EXPECTED_SIG="$(curl -fsSL https://composer.github.io/installer.sig)" +ACTUAL_SIG="$("$PHP_PATH" -r "echo hash_file('sha384', '$INSTALLER');")" +if [[ "$ACTUAL_SIG" != "$EXPECTED_SIG" ]]; then + echo "Error: Composer installer checksum mismatch" >&2 + exit 1 +fi + +# Install Composer +"$PHP_PATH" "$INSTALLER" --install-dir="$INSTALL_DIR" --filename=composer --version="$VERSION" diff --git a/ci/linux-install-groovy.sh b/ci/linux-install-groovy.sh new file mode 100755 index 000000000..3858c76d0 --- /dev/null +++ b/ci/linux-install-groovy.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +groovy_version=5.0.3 +groovy_zip="apache-groovy-binary-$groovy_version.zip" +cd /opt +rm -rf apache-groovy-binary*zip +wget https://groovy.jfrog.io/artifactory/dist-release-local/groovy-zips/"$groovy_zip" +echo "9d711dcb1dea94df9119558365beb6ac2909a22e30b58ae31de8bcb0dcf33698" "$groovy_zip" > "$groovy_zip".sha256 +sha256sum -c "$groovy_zip".sha256 +unzip -o "$groovy_zip" +ln -sf /opt/groovy-"$groovy_version"/bin/groovy /usr/bin/ +echo "export JAVA_HOME=/usr" >> /etc/profile +. /etc/profile +groovy -v diff --git a/ci/linux-install-jdk.sh b/ci/linux-install-jdk.sh new file mode 100755 index 000000000..849afd737 --- /dev/null +++ b/ci/linux-install-jdk.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -e + +install_jdk() { + # install jdk "manually" + # depending on os, might want to do something like `apt remove default-jre openjdk-*-jre-*` + cd /opt + # in order to have a stable download we must use the latest-1 version as that is the most recent in the "archive" + baseurl=https://download.oracle.com/java/21/archive/ + major_version=21 + baseurl="https://download.oracle.com/java/${major_version}/archive/" + version=21.0.11 + if uname -m | grep aarch64; then + tarball=jdk-${version}_linux-aarch64_bin.tar.gz + # checksum from https://download.oracle.com/java/${major_version}/archive/jdk-${version}_linux-aarch64_bin.tar.gz.sha256 + sha=2ebe89cad767abba83fb0b8cedd2d2d9bcbf947315fde78f7263a57a24f43b96 + else + tarball=jdk-${version}_linux-x64_bin.tar.gz + # checksum from https://download.oracle.com/java/${major_version}/latest/jdk-${version}_linux-x64_bin.tar.gz.sha256 + sha=e1c25a83f9e2e374c93e0c29cc3d98a947621ae0fefa4a8d932951eb160c47c3 + fi + wget --quiet "$baseurl$tarball" + echo "$sha $tarball" | sha256sum --check - + tar xf "$tarball" + tee /etc/profile.d/jdk.sh << EOF +export JAVA_HOME="/opt/jdk-$version" +export PATH=\$PATH:\$JAVA_HOME/bin +EOF + chown -R root:root "/opt/jdk-$version" + chmod -R 755 "/opt/jdk-$version" + if command -v update-alternatives; then + update-alternatives --install /usr/bin/java java "/opt/jdk-$version/bin/java" 9999 + else + ln -s "/opt/jdk-$version/bin/java" /usr/bin/java + fi + cd - +} + +if command -v java; then + echo "java already installed, will skip install." + exit +fi + +# TODO check version +if [ "$(whoami)" = "root" ]; then + install_jdk +else + sudo bash -c install_jdk +fi diff --git a/ci/linux-install-node.sh b/ci/linux-install-node.sh new file mode 100755 index 000000000..930dde304 --- /dev/null +++ b/ci/linux-install-node.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +node_version=20.10.0 + +npm install compare-versions + +node -e "const cv = require('compare-versions'); process.exit(cv.compareVersions('$node_version', process.version));" +result=$? +echo "result: $result" +if [ "$result" != "1" ]; then + echo "node version is >= $node_version, no install will be performed." + exit +fi + +cd /opt +sudo curl -k -O https://unofficial-builds.nodejs.org/download/release/v$node_version/node-v$node_version-linux-x64-glibc-217.tar.xz +echo "dcfc5dfcdea4d0883bb35a4f2e09bc4745b6e689d88f4ad09d9ccc252024049b node-v$node_version-linux-x64-glibc-217.tar.xz" | sudo tee node-v$node_version-linux-x64-glibc-217.tar.xz.sha256 +sudo sha256sum --check node-v$node_version-linux-x64-glibc-217.tar.xz.sha256 +sudo tar xf node-v$node_version-linux-x64-glibc-217.tar.xz +sudo tee /etc/profile.d/nodejs.sh << EOF +export NODE_HOME=/opt/node-v$node_version-linux-x64-glibc-217 +export PATH=\$PATH:\$NODE_HOME/bin +EOF +sudo update-alternatives --install /usr/bin/node node /opt/node-v$node_version-linux-x64-glibc-217/bin/node 1 +sudo update-alternatives --install /usr/bin/npm npm /opt/node-v$node_version-linux-x64-glibc-217/bin/npm 1 +sudo update-alternatives --install /usr/bin/npx npx /opt/node-v$node_version-linux-x64-glibc-217/bin/npx 1 +sudo update-alternatives --install /usr/bin/corepack corepack /opt/node-v$node_version-linux-x64-glibc-217/bin/corepack 1 +cd /root + diff --git a/ci/linux-install-php.sh b/ci/linux-install-php.sh new file mode 100755 index 000000000..9012238ee --- /dev/null +++ b/ci/linux-install-php.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -e + +php_version=8.3.20 + +if command -v php; then + if php -r "version_compare(PHP_VERSION, \""$php_version"\") >= 0 || exit(1);"; then + echo "host has php version >= $php_version. will skip install." + exit + fi +fi + +# install PHP 8.3 from source +sudo apt-get -y install gcc g++ make pkg-config libxml2-dev libsqlite3-dev \ +libcurl4-openssl-dev libpng-dev libonig-dev libldap2-dev libzip-dev \ +zlib1g-dev libssl-dev + +cd /tmp +wget https://www.php.net/distributions/php-$php_version.tar.gz +tar xzf php-$php_version.tar.gz +cd php-$php_version + +# --enable-bcmath is needed by tecnickcom/tc-lib-barcode and tecnickcom/tc-lib-pdf +./configure --prefix=/usr/local --with-curl --enable-bcmath --enable-gd --enable-mbstring --with-ldap --with-openssl --with-zlib + +make -j"$(nproc)" +sudo make install + +php -v +# END of PHP installation diff --git a/ci/linux-install-protobuf.sh b/ci/linux-install-protobuf.sh new file mode 100755 index 000000000..c92cf58a5 --- /dev/null +++ b/ci/linux-install-protobuf.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -e + +install_protobuf() { + # Install the protoc compiler "manually" from the official prebuilt + # release, verifying the SHA256 checksum of the zip. protoc is needed to + # build the cargo-based leech2 dependency. + # + # The release archives do not ship .sha256 files, so the checksums below are + # computed by us (and refreshed by the dependency update script). + cd /opt + version=35.1 + baseurl="https://github.com/protocolbuffers/protobuf/releases/download/v${version}" + + if uname -m | grep aarch64; then + arch=linux-aarch_64 + # sha256sum of protoc-${version}-linux-aarch_64.zip + sha=01bf9d08808c7f96678b63f4bd8efa559bb4f83d5a7a270d5edaf507f9d5d9cf + else + arch=linux-x86_64 + # sha256sum of protoc-${version}-linux-x86_64.zip + sha=6930ebf62bd4ea607b98fff052596c6ee564b9835b4ce172c75a3f53ae9d91b7 + fi + + zipfile="protoc-${version}-${arch}.zip" + wget --quiet "$baseurl/$zipfile" + echo "$sha $zipfile" | sha256sum --check - + # Installs bin/protoc and include/ under /usr/local. + unzip -o "$zipfile" -d /usr/local + rm "$zipfile" + + chmod a+rx /usr/local/bin/protoc + cd - +} + +# Re-exec under sudo when not root (e.g. when sourced from fix-buildhost.sh as +# the build user). +if [ "$(id -u)" -ne 0 ]; then + exec sudo bash "$0" +fi +install_protobuf diff --git a/ci/linux-install-rust.sh b/ci/linux-install-rust.sh new file mode 100755 index 000000000..a3811a556 --- /dev/null +++ b/ci/linux-install-rust.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +set -e + +install_rust() { + # Install the Rust toolchain "manually" from the official standalone + # installers, verifying the SHA256 checksum of each tarball. This is the + # cargo-based build dependency needed to build leech2. + # + # We install the individual component tarballs (rustc, cargo, rust-std) + # rather than the combined "rust" archive: the combined one is ~360 MB and + # extracts to ~1.4 GB of docs/clippy/llvm-tools we never install. The build + # hosts are tight on disk, so we also delete each tarball and its extracted + # tree right after installing it to keep peak disk usage low. + # + # Linux builds are native (x86_64 packages are built on x86_64 hosts, + # aarch64 on aarch64 hosts), so we only install the host's own Linux std. + # Windows is the only cross-compilation target, and only on MinGW build + # hosts, so the caller passes "x86_64-pc-windows-gnu" as an argument there. + baseurl="https://static.rust-lang.org/dist" + version=1.97.1 + prefix=/opt/rust + extra_targets="$@" + + workdir="$(mktemp -d)" + trap 'rm -rf "$workdir"' EXIT + cd "$workdir" + + if uname -m | grep aarch64; then + host=aarch64-unknown-linux-gnu + # checksum from $baseurl/rustc-${version}-aarch64-unknown-linux-gnu.tar.gz.sha256 + rustc_sha=8d9f3095a3108e38232f123645e2a0368bca67c975f8b91088004f8ffd9a6d39 + # checksum from $baseurl/cargo-${version}-aarch64-unknown-linux-gnu.tar.gz.sha256 + cargo_sha=cafde243518b016f825d13a5f2ca4ffd7d80f5966dfbec3539cd092c7c4f9871 + else + host=x86_64-unknown-linux-gnu + # checksum from $baseurl/rustc-${version}-x86_64-unknown-linux-gnu.tar.gz.sha256 + rustc_sha=1c441e430c1cca49dff54a8d59c41038bf6f79f7b8756596cb2f36511a015eba + # checksum from $baseurl/cargo-${version}-x86_64-unknown-linux-gnu.tar.gz.sha256 + cargo_sha=90cbeb8adfe8ca0fcbe01a18bd9b99d3e5e98fd29c003032828ebf3ffba0e4ed + fi + + # rust-std checksums per target. These are host-architecture independent. + # checksum from $baseurl/rust-std-${version}-x86_64-unknown-linux-gnu.tar.gz.sha256 + std_x86_64_linux_sha=45b927ecf776b9645ca2ed5b287fc6814824a073c30c7c5d5c0ad4425295ecee + # checksum from $baseurl/rust-std-${version}-aarch64-unknown-linux-gnu.tar.gz.sha256 + std_aarch64_linux_sha=ceffa428fe573507418858d59add342b351730725f36f7679b251e8b68103dec + # checksum from $baseurl/rust-std-${version}-x86_64-pc-windows-gnu.tar.gz.sha256 + std_x86_64_windows_sha=0e240192a91dace872d72fd2a5ad21b4be5d278b0d0281d58013e11f908fb5b6 + + # Download, verify, extract and install a single component tarball, then + # remove both the tarball and its extracted tree before moving on. + install_component() { + name="$1" + sha="$2" + tarball="$name.tar.gz" + wget --quiet "$baseurl/$tarball" + echo "$sha $tarball" | sha256sum --check - + tar xf "$tarball" + rm "$tarball" + "$name/install.sh" --prefix="$prefix" + rm -rf "$name" + } + + # Install the rust-std for a given target triple, looking up its checksum. + install_std() { + case "$1" in + x86_64-unknown-linux-gnu) sha="$std_x86_64_linux_sha" ;; + aarch64-unknown-linux-gnu) sha="$std_aarch64_linux_sha" ;; + x86_64-pc-windows-gnu) sha="$std_x86_64_windows_sha" ;; + *) + echo "No pinned checksum for rust-std target '$1'" >&2 + exit 1 + ;; + esac + install_component "rust-std-${version}-$1" "$sha" + } + + install_component "rustc-${version}-${host}" "$rustc_sha" + install_component "cargo-${version}-${host}" "$cargo_sha" + + # The host's own native std, plus any cross-compilation targets requested. + install_std "$host" + for target in $extra_targets; do + install_std "$target" + done + + tee /etc/profile.d/rust.sh << EOF +export PATH=\$PATH:$prefix/bin +EOF + + chown -R root:root "$prefix" + # Make sure it's readable by the build user. + chmod -R a+rX "$prefix" +} + +# Re-exec under sudo when not root (e.g. when sourced from fix-buildhost.sh as +# the build user), preserving any cross-compilation target arguments. +if [ "$(id -u)" -ne 0 ]; then + exec sudo bash "$0" "$@" +fi +install_rust "$@" diff --git a/ci/quick-install-cfengine-enterprise.sh.sha256 b/ci/quick-install-cfengine-enterprise.sh.sha256 index e6af0f5bf..600dbbb57 100644 --- a/ci/quick-install-cfengine-enterprise.sh.sha256 +++ b/ci/quick-install-cfengine-enterprise.sh.sha256 @@ -1 +1 @@ -e52748ded116a80f3d50310c74c7c2892ea4bc74a10ca5955d591a42664c890f quick-install-cfengine-enterprise.sh +6dceb3b29976d86b60cb493f58633217db3ba1b4681f5c4277a6dc137c221b0c quick-install-cfengine-enterprise.sh diff --git a/ci/setup-bootstrap-host.sh b/ci/setup-bootstrap-host.sh new file mode 100755 index 000000000..dfd681ab7 --- /dev/null +++ b/ci/setup-bootstrap-host.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -e +thisdir="$(dirname "$0")" + +# current standard of debian-9 for bootstrap requires using archive.debian.org +rm -f /etc/apt/sources.list.d/*; echo 'deb http://archive.debian.org/debian/ stretch main contrib non-free' >/etc/apt/sources.list + +apt-get -qy update +apt-get -qy upgrade +apt-get -y install git autogen autoconf automake m4 make bison flex binutils libtool gcc g++ libc-dev \ + liblmdb-dev libpam0g-dev python libssl-dev libpcre3-dev psmisc curl jq unzip \ + librsync-dev \ + autoconf-archive \ + pigz parallel libpcre2-dev php-zip + +bash "$thisdir"/linux-install-php.sh + +bash "$thisdir"/linux-install-node.sh + +bash "$thisdir"/linux-install-composer.sh + +bash "$thisdir"/linux-install-jdk.sh diff --git a/ci/setup-cfengine-build-host.sh b/ci/setup-cfengine-build-host.sh index 15343f6c9..5c785d83d 100755 --- a/ci/setup-cfengine-build-host.sh +++ b/ci/setup-cfengine-build-host.sh @@ -2,100 +2,83 @@ shopt -s expand_aliases thisdir="$(dirname "$0")" -# Use the newest CFEngine version we can -CFE_VERSION=3.26.0 -if [ -f /etc/centos-release ]; then - _version=$(cat /etc/centos-release | cut -d' ' -f3 | cut -d. -f1) - if [ "$_version" = "6" ]; then - CFE_VERSION=3.24.2 - fi -elif [ -f /etc/os-release ]; then - source /etc/os-release - if [ "$ID" = "debian" ]; then - if [ "$VERSION_ID" -lt "9" ]; then - echo "Platform $ID $VERSION_ID is too old." - exit 9 - fi - if [ "$VERSION_ID" -lt "11" ]; then - CFE_VERSION=3.21.7 - fi - fi - if [ "$ID" = "redhat" ] || [ "$ID" = "centos" ]; then - if [ "$VERSION_ID" -lt "6" ]; then - echo "Platform $ID $VERSION_ID is too old." - exit 9 - fi - if [ "$VERSION_ID" -lt "7" ]; then - CFE_VERSION=3.24.2 - fi - fi - if [ "$ID" = "ubuntu" ]; then - _version="$(echo "$VERSION_ID" | cut -d. -f1)" - if [ "$_version" -lt "16" ]; then - echo "Platform $ID $VERSION_ID is too old." - exit 9 - fi - if [ "$_version" -lt "20" ]; then - CFE_VERSION=3.21.7 - fi - fi +# handle env cfengine_role +if [ -n "$cfengine_role" ]; then + touch /etc/cfengine-"$cfengine_role".flag fi # install needed packages and software for a build host -set -ex +set -e if [ "$(id -u)" != "0" ]; then - echo "$0 must be run as root" - exit 1 + echo "$0 must be run as root" + exit 1 fi ls -la /home/ if ! id -u jenkins; then - useradd jenkins -p jenkins + useradd jenkins -p jenkins fi mkdir -p /home/jenkins -chown -R jenkins /home/jenkins -function cleanup() -{ - set -ex - if command -v apt 2>/dev/null; then - # workaround for CFE-4544, remove scriptlets call systemctl even when systemctl is-system-running returns false +# The following is copied from prepare-testmachine-chroot +CHROOT_ROOT=/home/jenkins/testmachine-chroot/ +fuser -k "$CHROOT_ROOT" >/dev/null 2>&1 || true +# Unmount the /proc filesystem if it was previously mounted inside the chroot. +umount "${CHROOT_ROOT}proc" >/dev/null 2>&1 || true + +echo "Show running processes in case a race condition occurs when chowning files (happened once 2026-07-28)" +ps -efl + +chown -R jenkins /home/jenkins || true + +# cleanup any previous runs cfengine-masterfiles tar balls +rm -rf cfengine-masterfiles* + +function cleanup() { + set -e + set -x + [ -f /var/log/messages ] && tail /var/log/messages + command -v journalctl >/dev/null && journalctl | grep -P '(error|fail)' + if command -v apt >/dev/null 2>&1; then + # workaround for CFE-4544, remove scriptlets call systemctl even when systemctl is-system-running returns false # Replace systemctl with a no-op stub that always succeeds. We can't # symlink to /bin/echo (or /bin/true): on Ubuntu 26 coreutils is the # single multi-call uutils binary that dispatches on argv[0], so it # would fail with "coreutils: unknown program 'systemctl'". - rm -f /bin/systemctl - printf '#!/bin/sh\nexit 0\n' > /bin/systemctl + rm -f /bin/systemctl + printf '#!/bin/sh\nexit 0\n' >/bin/systemctl chmod +x /bin/systemctl - apt remove -y cfengine-nova || true - elif command -v yum 2>/dev/null; then - yum erase -y cfengine-nova || true - elif command -v zypper 2>/dev/null; then - zypper remove -y cfengine-nova || true - else - echo "No supported package manager to uninstall cfengine." - exit 1 - fi - echo "Cleaning up CFEngine install by moving to /var/bak.cfengine and /opt/bak.cfengine" - rm -rf /var/bak.cfengine - mv /var/cfengine /var/bak.cfengine || true - rm -rf /opt/bak.cfengine - mv /opt/cfengine /opt/bak.cfengine || true - mv /var/log/CFE* /var/bak.cfengine/ || true - mv /var/log/postgresql.log /var/bak.cfengine || true - - if command -v pkill; then - pkill -9 cf-agent || true - pkill -9 cf-serverd || true - pkill -9 cf-monitord || true - pkill -9 cf-execd || true - else - echo "No pkill available. Maybe some cf procs left over?" - ps -efl | grep cf - fi - ls -l /home - chown -R jenkins /home/jenkins - echo "Done with cleanup()" + apt remove -y cfengine-nova || true + elif command -v yum >/dev/null 2>&1; then + yum erase -y cfengine-nova || true + elif command -v zypper >/dev/null 2>&1; then + zypper remove -y cfengine-nova || true + else + echo "No supported package manager to uninstall cfengine." + exit 1 + fi + echo "Cleaning up CFEngine install by moving to /var/bak.cfengine and /opt/bak.cfengine" + rm -rf /var/bak.cfengine + mv /var/cfengine /var/bak.cfengine 2>/dev/null || true + rm -rf /opt/bak.cfengine + mv /opt/cfengine /opt/bak.cfengine 2>/dev/null || true + mv /var/log/CFE* /var/bak.cfengine/ 2>/dev/null || true + mv /var/log/postgresql.log /var/bak.cfengine 2>/dev/null || true + + if command -v pkill >/dev/null 2>&1; then + pkill -9 cf-agent || true + pkill -9 cf-serverd || true + pkill -9 cf-monitord || true + pkill -9 cf-execd || true + else + echo "No pkill available. Maybe some cf procs left over?" + # pgrep suggestion is ignored due to wanting POSIX compatability + # shellcheck disable=SC2009 + ps -efl | grep cf + fi + ls -l /home + chown -R jenkins /home/jenkins + echo "Done with cleanup()" } trap cleanup ERR @@ -105,136 +88,221 @@ trap cleanup SIGTERM echo "Using buildscripts commit:" # we have very old platforms with old git that doesn't understand -C option so cd/cd .. it is ( - cd "$thisdir"/.. - # buildscripts is owned by jenkins so in order to run rev-parse command as root (this script is run with sudo) we must make it safe if git is used - if [ -d /home/jenkins/buildscripts/.git ]; then - if command -v git >/dev/null; then - git config --global --add safe.directory /home/jenkins/buildscripts - # show what version of buildscripts we are using - git rev-parse HEAD - else - echo "buildscripts/.git is present but git is not installed" - exit 1 + cd "$thisdir"/.. + # buildscripts is owned by jenkins so in order to run rev-parse command as root (this script is run with sudo) we must make it safe if git is used + if [ -d /home/jenkins/buildscripts/.git ]; then + if command -v git >/dev/null; then + git config --global --add safe.directory /home/jenkins/buildscripts + # show what version of buildscripts we are using + git rev-parse HEAD + else + echo "buildscripts/.git is present but git is not installed" + exit 1 + fi fi - fi ) echo "Install distribution upgrades and set software alias for platform" if [ -f /etc/os-release ]; then - if grep rhel /etc/os-release; then + if grep -q rhel /etc/os-release; then + yum update --assumeyes + alias software='yum install --assumeyes' + alias erase-packages='yum erase --assumeyes' + elif grep -q debian /etc/os-release; then + DEBIAN_FRONTEND=noninteractive apt update + + # sometimes the /boot partition is too small to handle kernel upgrade regenerations of initrd and related files on ubuntu, so allow failure first + DEBIAN_FRONTEND=noninteractive apt upgrade --yes || true + DEBIAN_FRONTEND=noninteractive apt autoremove --yes + + # and now perform the upgrade a second time after hopefully autoremove cleans up /boot partition of kernel files that cause failure + DEBIAN_FRONTEND=noninteractive apt upgrade --yes + DEBIAN_FRONTEND=noninteractive apt autoremove --yes + alias software='DEBIAN_FRONTEND=noninteractive apt install --yes' + alias erase-packages='DEBIAN_FRONTEND=noninteractive apt purge --yes' + elif grep -q suse /etc/os-release; then + zypper -n update + alias software='zypper install -y' + alias erase-packages='zypper uninstall -y' + else + echo "Unknown platform ID $ID. Need this information in order to update/upgrade distribution packages." + exit 1 + fi +elif [ -f /etc/redhat-release ]; then yum update --assumeyes alias software='yum install --assumeyes' - elif grep debian /etc/os-release; then - DEBIAN_FRONTEND=noninteractive apt upgrade --yes && DEBIAN_FRONTEND=noninteractive apt autoremove --yes - alias software='DEBIAN_FRONTEND=noninteractive apt install --yes' - elif grep suse /etc/os-release; then - zypper -n update - alias software='zypper install -y' - else - echo "Unknown platform ID $ID. Need this information in order to update/upgrade distribution packages." - exit 1 - fi -elif [ -f /etc/redhat-release ]; then - yum update --assumeyes - alias software='yum install --assumeyes' else - echo "No /etc/os-release or /etc/redhat-release so cant determine platform." - exit 1 + echo "No /etc/os-release or /etc/redhat-release so cant determine platform." + exit 1 fi -if command -v wget; then - alias urlget=wget -elif command -v curl; then - alias urlget='curl -O' +if command -v wget >/dev/null 2>&1; then + alias urlget=wget +elif command -v curl >/dev/null 2>&1; then + alias urlget='curl -O' else - echo "Error: need something to fetch URLs. Didn't find either wget or curl." - exit 1 + echo "Error: need something to fetch URLs. Didn't find either wget or curl." + exit 1 fi -echo "Checking for pre-installed CFEngine (chicken/egg problem)" -# We need a cf-agent to run build host setup policy and redhat-10-arm has not previous package to install. -# solution: install from source and make a custom AWS AMI image -if ! /var/cfengine/bin/cf-agent -V; then - echo "No existing CFEngine install found, try quickinstall script..." - if grep -i suse /etc/os-release; then - # need to add our public key first otherwise zypper install fails - rpm --import https://cfengine-package-repos.s3.amazonaws.com/pub/gpg.key - if grep 'VERSION.*12' /etc/os-release; then - urlget https://cfengine-package-repos.s3.amazonaws.com/enterprise/Enterprise-"$CFE_VERSION"/agent/agent_suse12_x86_64/cfengine-nova-"$CFE_VERSION"-1.suse12.x86_64.rpm - zypper install -y cfengine-nova-"$CFE_VERSION"-1.suse12.x86_64.rpm - elif grep 'VERSION.*15' /etc/os-release; then - urlget https://cfengine-package-repos.s3.amazonaws.com/enterprise/Enterprise-"$CFE_VERSION"/agent/agent_suse15_x86_64/cfengine-nova-"$CFE_VERSION"-1.suse15.x86_64.rpm - zypper install -y cfengine-nova-"$CFE_VERSION"-1.suse15.x86_64.rpm - else - echo "Unsupported suse version:" - grep VERSION /etc/os-release - exit 1 +if grep -q 6.10 /etc/issue 2>/dev/null; then + # special case of centos-6, cf-remote depends on urllib3 which depends on openssl 1.1.1+ that is not available + # generally we rely on cf-remote to install cfengine-nova and download masterfiles so here we must provide for both of those + echo "Found CentOS 6.10 so installing via hard-coded package URL..." + + if [ ! -x /var/cfengine/bin/cf-agent ]; then + rm -rf cfengine-nova*rpm + urlget https://cfengine-package-repos.s3.amazonaws.com/enterprise/Enterprise-3.24.3/agent/agent_rhel6_x86_64/cfengine-nova-3.24.3-1.el6.x86_64.rpm + rpm -i cfengine-nova-3.24.3-1.el6.x86_64.rpm fi - else - urlget https://s3.amazonaws.com/cfengine.packages/quick-install-cfengine-enterprise.sh - # log sha256 checksum expected and actuall for debugging purposes - echo "Expected quick install checksum: " - cat "$thisdir"/quick-install-cfengine-enterprise.sh.sha256 - echo "Actual quick install checksum: " - sha256sum quick-install-cfengine-enterprise.sh - - sha256sum --check "$thisdir"/quick-install-cfengine-enterprise.sh.sha256 - chmod +x quick-install-cfengine-enterprise.sh - export CFEngine_Enterprise_Package_Version="$CFE_VERSION" - bash ./quick-install-cfengine-enterprise.sh agent - fi + urlget https://cfengine-package-repos.s3.amazonaws.com/enterprise/Enterprise-3.24.3/misc/cfengine-masterfiles-3.24.3-1.pkg.tar.gz fi -if ! /var/cfengine/bin/cf-agent -V; then - echo "quickinstall script didn't install CFEngine, try cf-remote..." - # try pipx first for debian as pip won't work. - # If that fails to install CFEngine then try python3-pip for redhats. - if software pipx; then - pipx install cf-remote - export PATH=$HOME/.local/bin:$PATH - elif software python3-pip; then - pip install cf-remote - fi - if command -v cf-remote >/dev/null; then - cf-remote --log-level info --version master install --clients localhost || true - fi +# Here we start replacing the use of CFEngine policy with scripts. See ENT-14330 +if [ -f /etc/cfengine-bootstrap-pr-host.flag ]; then + "$thisdir"/setup-bootstrap-host.sh + exit fi -if [ ! -x /var/cfengine/bin/cf-agent ]; then - echo "cf-remote didn't install CFEngine, build from source..." - software git - echo "quickinstall and cf-remote didn't install cf-agent, try from source" - CFE_VERSION=3.26.0 # need to use an actualy release which has a checksum for masterfiles download - rm -rf core # just in case we are repeating the script - git clone --recursive --depth 1 https://github.com/cfengine/core - ( - cd core - ./ci/install.sh - ) +if [ -f /etc/cfengine-containers-host.flag ]; then + "$thisdir"/setup-ci-host.sh + exit +fi + +if grep -q ubuntu /etc/os-release; then + if grep -qi version=\"16 /etc/os-release; then + urlget https://cfengine-package-repos.s3.amazonaws.com/enterprise/Enterprise-3.21.8/agent/agent_ubuntu16_x86_64/cfengine-nova_3.21.8-1.ubuntu16_amd64.deb + dpkg -i cfengine-nova_3.21.8-1.ubuntu16_amd64.deb + urlget https://cfengine-package-repos.s3.amazonaws.com/enterprise/Enterprise-3.21.8/misc/cfengine-masterfiles-3.21.8-1.pkg.tar.gz + fi +fi + +if grep -q suse /etc/os-release; then + urlget https://cfengine-package-repos.s3.amazonaws.com/pub/gpg.key + rpm --import gpg.key + + if grep -qi version=\"12 /etc/os-release; then + echo "SUSE-12 found, cf-remote cannot be installed here so download directly similar to CentOS-6." + if [ ! -x /var/cfengine/bin/cf-agent ]; then + rm -rf cfengine-nova*rpm + urlget https://cfengine-package-repos.s3.amazonaws.com/enterprise/Enterprise-3.24.3/agent/agent_suse12_x86_64/cfengine-nova-3.24.3-1.suse12.x86_64.rpm + zypper in -y cfengine-nova-3.24.3-1.suse12.x86_64.rpm + fi + urlget https://cfengine-package-repos.s3.amazonaws.com/enterprise/Enterprise-3.24.3/misc/cfengine-masterfiles-3.24.3-1.pkg.tar.gz + fi fi -# get masterfiles -urlget https://cfengine-package-repos.s3.amazonaws.com/enterprise/Enterprise-"$CFE_VERSION"/misc/cfengine-masterfiles-"$CFE_VERSION"-1.pkg.tar.gz +if [ ! -x /var/cfengine/bin/cf-agent ]; then + if ! ls cfengine-masterfiles*tar.gz >/dev/null 2>&1; then + echo "Installing cf-remote for possible package install and masterfiles download" + # try pipx first for debian as pip won't work. + # If that fails to install CFEngine then try python3-pip for redhats. + PIP="" + software python3-venv || true # on ubuntu-20 this is needed, debian-12 it is not but won't hurt + if software pipx; then + PIP=pipx + export PATH=$HOME/.local/bin:$PATH + elif software python3-pip; then + if command -v pip >/dev/null 2>&1; then + PIP=pip + elif command -v pip3 >/dev/null 2>&1; then + PIP=pip3 + fi + elif software python-pip; then + if command -v pip >/dev/null 2>&1; then + PIP=pip + fi + else + echo "Tried installing pipx, python3-pip and python-pip, none of which resulted in pipx, pip3 or pip being available. Exiting." + exit 23 + fi + export PATH=/usr/local/bin:$PATH # some pip/pipx use /usr/local/bin + + if [ "$PIP" = "pipx" ]; then + $PIP uninstall cf-remote || true # no -y option in pipx + else + $PIP uninstall -y cf-remote || true + fi # just in case a previous is there and would cause the install to fail + $PIP install cf-remote || true # if this fails we will try to install from source + fi # no masterfiles downloaded +fi # no cf-agent installed + +echo "Checking for pre-installed CFEngine (chicken/egg problem)" +# We need a cf-agent to run build host setup policy and redhat-10-arm did not have a previous package to install. +if ! /var/cfengine/bin/cf-agent -V 2>/dev/null; then + echo "No existing CFEngine install found, try cf-remote..." + if grep -qi stretch /etc/os-release || grep -qi buster /etc/os-release || grep -qi bionic /etc/os-release; then + _VERSION="--version 3.21.8" # 3.27.0 and 3.24.x do not have debian 9 (stretch) or debian 10 (buster) or ubuntu 18 (bionic) + elif grep -qi bullseye /etc/os-release; then + _VERSION="--version 3.24.3" # 3.27.0 has only debian > 11 (bullseye) + elif grep -q suse /etc/os-release; then + # here we must use 3.24.2 instead of 3.24.3 because 3.24.3 has libcurl 4 which depends on unavailable OPENSSL_3.2.0 + _VERSION="--version 3.24.2" # we removed suse platforms in 3.27.0 + else + _VERSION="" + fi + + # ENT-14373, migrate any cf-remote cache/config files to avoid dirs_exist_ok problems on old pythons + if [ -d "$HOME"/.cfengine/cf-remote/json ]; then + mkdir -p "$HOME"/.cache/cfengine/cf-remote + mv "$HOME"/.cfengine/cf-remote/json "$HOME"/.cache/cfengine/cf-remote/ + fi + if [ -d "$HOME"/.cfengine/cf-remote/packages ]; then + mkdir -p "$HOME"/.cache/cfengine/cf-remote + mv "$HOME"/.cfengine/cf-remote/packages "$HOME"/.cache/cfengine/cf-remote/ + fi + if [ -d "$HOME"/.cfengine/cf-remote ]; then + mkdir -p "$HOME"/.config/cfengine/cf-remote + mv "$HOME"/.cfengine/cf-remote/* "$HOME"/.config/cfengine/cf-remote/ + fi + + + erase-packages cfbuild-* || true # in case a dirty build was left on a long-living build host -sha256sum --check "$thisdir"/cfengine-masterfiles-"$CFE_VERSION"-1.pkg.tar.gz.sha256 + # We are passing a two-token string and need it to stay two tokens for proper argument parsing in $_VERSION + # shellcheck disable=SC2086 + cf-remote --log-level info $_VERSION install --clients localhost || true +fi + +if [ ! -x /var/cfengine/bin/cf-agent ]; then + [ -f /var/log/CFEngine-Install.log ] && tail /var/log/CFEngine-Install.log + echo "cf-remote didn't install CFEngine, build from source..." + software git + echo "cf-remote didn't install cf-agent, try from source" + rm -rf core # just in case we are repeating the script + git clone --recursive --depth 1 https://github.com/cfengine/core + ( + cd core + ./ci/install.sh + ) +fi -tar xf cfengine-masterfiles-"$CFE_VERSION"-1.pkg.tar.gz +# download masterfiles if not already present (such as in case of centos-6 above, hard-coded 3.24.3 download) +if ! ls cfengine-masterfiles*gz >/dev/null 2>&1; then + # if we are using a CFEngine pre-installed (chicken/egg) image we would skip cf-remote install so need to download directly + if ! command -v cf-remote >/dev/null 2>&1; then + urlget https://cfengine-package-repos.s3.amazonaws.com/enterprise/Enterprise-3.27.0/misc/cfengine-masterfiles-3.27.0-1.pkg.tar.gz + else + cf-remote download masterfiles --output-dir . + fi +fi +tar xf cfengine-masterfiles-*tar.gz cp -a masterfiles/* /var/cfengine/inputs/ # run three times to ensure all is done ( - cd "$thisdir" - policy=./cfengine-build-host-setup.cf - # just to be sure, make policy read/write for our user only to avoid errors when running - chmod 600 "$policy" - /var/cfengine/bin/cf-agent -KIf "$policy" -b cfengine_build_host_setup | tee promises.log - grep -i error: promises.log && exit 1 - /var/cfengine/bin/cf-agent -KIf "$policy" -b cfengine_build_host_setup | tee -a promises.log - grep -i error: promises.log && exit 1 - /var/cfengine/bin/cf-agent -KIf "$policy" -b cfengine_build_host_setup | tee -a promises.log - grep -i error: promises.log && exit 1 - echo "Done evaluating policy. End of promies.log:" - tail promises.log + cd "$thisdir" + policy=./cfengine-build-host-setup.cf + # just to be sure, make policy read/write for our user only to avoid errors when running + chmod 600 "$policy" + /var/cfengine/bin/cf-agent -KIf "$policy" -b cfengine_build_host_setup | tee promises.log + grep -i error: promises.log && exit 1 + /var/cfengine/bin/cf-agent -KIf "$policy" -b cfengine_build_host_setup | tee -a promises.log + grep -i error: promises.log && exit 1 + /var/cfengine/bin/cf-agent -KIf "$policy" -b cfengine_build_host_setup | tee -a promises.log + grep -i error: promises.log && exit 1 + echo "Done evaluating policy. End of promises.log:" + tail promises.log ) cleanup diff --git a/ci/setup-ci-host.sh b/ci/setup-ci-host.sh new file mode 100755 index 000000000..c88f5d7a3 --- /dev/null +++ b/ci/setup-ci-host.sh @@ -0,0 +1,207 @@ +#!/usr/bin/env bash +set -e +shopt -s expand_aliases +thisdir="$(dirname "$0")" + +packages="" # a space separated list of packages to install +function add-pkg() { + packages+=" $*" +} + +# we setup some vars for platform versions to make it easier to make choice later +# default version is 0 so that a check can be [ "$debian" -gt "12" ] and that will skip non-debians and such +redhat=0 +debian=0 +ubuntu=0 +suse=0 +# shellcheck disable=SC2034 +solaris=0 +# shellcheck disable=SC2034 +hpux=0 +# shellcheck disable=SC2034 +aix=0 + +if [ -f /etc/os-release ]; then + source /etc/os-release + if grep -q rhel /etc/os-release; then + yum update --assumeyes + alias packages='yum install --assumeyes' + redhat="$VERSION_ID" + elif grep -q debian /etc/os-release; then + alias packages='DEBIAN_FRONTEND=noninteractive apt install --yes' + debian="$VERSION_ID" + elif grep -q suse /etc/os-release; then + alias packages='zypper install -y' + # shellcheck disable=SC2034 + suse="$VERSION_ID" + else + echo "Unknown platform ID $ID. Need this information in order to update/upgrade distribution packages." + exit 1 + fi +elif [ -f /etc/redhat-release ]; then + alias packages='yum install --assumeyes' + # shellcheck disable=SC1091 + source /etc/redhat-release + redhat="$VERSION_ID" +else + echo "No /etc/os-release or /etc/redhat-release so cant determine platform." + exit 1 +fi + +if [ -f /etc/cfengine-containers-host.flag ]; then + if [ "$debian" -ge "12" ]; then + # in jenkins, CONTAINER labeled nodes are capable of running container builds like valgrind-check and static-check + add-pkg unzip # linux-install-groovy.sh needs unzip to unpack the groovy distribution archive + add-pkg buildah + add-pkg jq + add-pkg make + add-pkg parallel + add-pkg podman + if ! command -v groovy; then + bash "$thisdir"/linux-install-groovy.sh + fi + + # NOPASSWD is needed for various tools related to container jobs + rm -rf /etc/sudoers.d/999-local + cat >/etc/sudoers.d/999-local </dev/null; then + if [ ! -f /etc/cfengine-containers-host.flag ]; then + sysctl kernel.core_pattern='|/lib/systemd/systemd-coredump %p %u %g %s %t %e' + fi +fi + +"$thisdir"/linux-install-jdk.sh # the script should skip if sufficient java is already installed + +# leech2 build toolchain host +if [ "$ubuntu" -ge 20 ] || [ "$debian" -ge 12 ] || [ "$redhat" -ge 7 ]; then + "$thisdir"/linux-install-protobuf.sh + # TODO if mingw then pass along x86_64-pc-windows-gnu as an arg to install rust + "$thisdir"/linux-install-rust.sh +fi + +if [ "$redhat" -ge 7 ]; then + yum groups install -y 'Development Tools' +fi + +if [ "$redhat" = 8 ]; then + sudo rpm -iv https://kojipkgs.fedoraproject.org//packages/fakeroot/1.23/1.fc29/x86_64/fakeroot-1.23-1.fc29.x86_64.rpm https://kojipkgs.fedoraproject.org//packages/fakeroot/1.23/1.fc29/x86_64/fakeroot-libs-1.23-1.fc29.x86_64.rpm +fi + +if [ "$redhat" -gt 7 ]; then + yum install --assumeyes https://dl.fedoraproject.org/pub/epel/epel-release-latest-"$redhat".noarch.rpm +fi