From 33cd1b49cc27e09f0e364f6aad16e725ee72d895 Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Mon, 27 Jul 2026 17:45:43 -0500 Subject: [PATCH 1/4] Update CI scripts from master (3.24) There have been many changes and all of them should likely be cherry picked but I don't want to bother finding them all. The ci directory is pretty safe to just pull en-masse from master. In ENT-11241 and ENT-14330 we migrated to running setup-cfengine-build-host.sh in several important CI jobs so the updates need to be in all LTS branches. Ticket: ENT-14330 Changelog: none Conflicts: ci/cfengine-build-host-setup.cf ci/fix-buildhost.sh --- ci/buildhost-image.sh | 29 + ci/centos-7-setup-devtoolset-11.sh | 12 + ci/cfengine-build-host-setup.cf | 618 +++++++++++++++--- ...ine-masterfiles-3.21.7-1.pkg.tar.gz.sha256 | 1 + ...ine-masterfiles-3.24.2-1.pkg.tar.gz.sha256 | 1 + ...ine-masterfiles-3.26.0-1.pkg.tar.gz.sha256 | 1 + ci/clean-buildhost-images.sh | 1 + ci/create-revisions-file.sh | 90 +++ ci/create_swap_file.sh | 24 + ci/distribution-patched.sh | 39 ++ ci/fix-buildhost.sh | 29 +- ci/images.sh | 9 + ci/initialize-build-host.sh | 64 +- ci/initialize-user-data.sh | 83 +++ ci/lint.sh | 14 + ci/linux-install-composer.sh | 24 + ci/linux-install-groovy.sh | 13 + ci/linux-install-jdk.sh | 49 ++ ci/linux-install-node.sh | 29 + ci/linux-install-php.sh | 30 + ci/linux-install-protobuf.sh | 41 ++ ci/linux-install-rust.sh | 101 +++ ci/patched-image.sh | 17 + ci/platform-container-image.list | 5 + ...uick-install-cfengine-enterprise.sh.sha256 | 2 +- ci/setup-bootstrap-host.sh | 22 + ci/setup-cfengine-build-host.sh | 375 ++++++++--- ci/setup-ci-host.sh | 207 ++++++ 28 files changed, 1671 insertions(+), 259 deletions(-) create mode 100755 ci/buildhost-image.sh create mode 100755 ci/centos-7-setup-devtoolset-11.sh create mode 100644 ci/cfengine-masterfiles-3.21.7-1.pkg.tar.gz.sha256 create mode 100644 ci/cfengine-masterfiles-3.24.2-1.pkg.tar.gz.sha256 create mode 100644 ci/cfengine-masterfiles-3.26.0-1.pkg.tar.gz.sha256 create mode 100755 ci/clean-buildhost-images.sh create mode 100755 ci/create-revisions-file.sh create mode 100755 ci/create_swap_file.sh create mode 100755 ci/distribution-patched.sh create mode 100644 ci/images.sh create mode 100644 ci/initialize-user-data.sh create mode 100755 ci/lint.sh create mode 100755 ci/linux-install-composer.sh create mode 100755 ci/linux-install-groovy.sh create mode 100755 ci/linux-install-jdk.sh create mode 100755 ci/linux-install-node.sh create mode 100755 ci/linux-install-php.sh create mode 100755 ci/linux-install-protobuf.sh create mode 100755 ci/linux-install-rust.sh create mode 100755 ci/patched-image.sh create mode 100644 ci/platform-container-image.list create mode 100755 ci/setup-bootstrap-host.sh create mode 100755 ci/setup-ci-host.sh diff --git a/ci/buildhost-image.sh b/ci/buildhost-image.sh new file mode 100755 index 000000000..27ee08bdb --- /dev/null +++ b/ci/buildhost-image.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -ex + +platform=$1 +c=$platform +base=$platform-patched +built=$platform-buildhost +if buildah images --format '{{.Name}}:{{.Tag}}' | grep $built; then + echo "container image $built already exists, remove if you want to rebuild" + continue +fi + +buildah rm $c || true +buildah --name $c from $base +buildah copy $c . /buildscripts/ci + +# This section is debian specific for now. TODO: add alternatives when we add more platforms. +buildah run $c apt install -y procps wget sudo +buildah run $c apt remove -y cfengine-nova || true + +buildah run $c rm -rf /var/cfengine || true +# touch flag file for policy to know it is in a container and avoid some aspects of configuration +buildah run $c touch /etc/cfengine-in-container.flag +buildah run $c /buildscripts/ci/setup-cfengine-build-host.sh | tee setup-cfengine-build-host.log +# the above, if errored out, is not causing an error, need to fix that. +grep -i error setup-cfengine-build-host.log && exit 1 +buildah tag $c $c-$(date +%F) +buildah commit $c $c-buildhost +buildah rm $c diff --git a/ci/centos-7-setup-devtoolset-11.sh b/ci/centos-7-setup-devtoolset-11.sh new file mode 100755 index 000000000..66250e3cf --- /dev/null +++ b/ci/centos-7-setup-devtoolset-11.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -ex +sudo yum install -y centos-release-scl +sudo rm -f /etc/yum.repos.d/CentOS-SCLo-scl.repo +sudo sed -i 's,^#baseurl.*$,baseurl=https://vault.centos.org/7.9.2009/sclo/x86_64/rh/,' /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo +sudo sed -i '/mirrorlist/d' /etc/yum.repos.d/CentOS-SCLo-scl-rh.repo +sudo yum update -y +sudo yum install -y devtoolset-11 +if ! grep "source /opt/rh/devtoolset-11/enable" /usr/lib/rpm/find-debuginfo.sh; then + sudo sed -i '1a\source /opt/rh/devtoolset-11/enable' /usr/lib/rpm/find-debuginfo.sh +fi +source /opt/rh/devtoolset-11/enable diff --git a/ci/cfengine-build-host-setup.cf b/ci/cfengine-build-host-setup.cf index 115bd1e68..a8205af36 100644 --- a/ci/cfengine-build-host-setup.cf +++ b/ci/cfengine-build-host-setup.cf @@ -6,27 +6,39 @@ body file control bundle agent cfengine_build_host_setup { meta: - "assumptions" string => "The operating system has working repository lists and has been updated and upgraded recently."; + "assumptions" + string => "The operating system has working repository lists and has been updated and upgraded recently."; packages: - debian_9|debian_10|ubuntu_16:: - "python-psycopg2"; - debian_11|debian_12|ubuntu_20|ubuntu_22|ubuntu_24:: - "python3-psycopg2"; ubuntu_16:: - "systemd-coredump" comment => "ubuntu_16 doesn't have systemd-coredump by default?"; + "systemd-coredump" + comment => "ubuntu_16 doesn't have systemd-coredump by default?"; + ubuntu_20:: - "autoconf" comment => "because on arm ubuntu-20 we need to reconfigure the debian-9 bootstrapped configure scripts."; + "autoconf" + comment => "because on arm ubuntu-20 we need to reconfigure the debian-9 bootstrapped configure scripts."; + "shellcheck" comment => "not sure why only ubuntu-20 needed this."; - debian.(!debian_12.!ubuntu_22.!ubuntu_26):: - "python" comment => "debian-12 has only python3"; - !(debian_9|ubuntu_16).(debian|ubuntu):: - "default-jre" comment => "on debian10+ and ubuntu18+ this will be jdk11, good enough for jenkins 2.426.1 https://www.jenkins.io/doc/book/platform-information/support-policy-java/index.html"; + + debian.(!debian_13.!debian_12.!ubuntu_22.!ubuntu_24.!ubuntu_25.!ubuntu_26):: + "python" comment => "debian>=12 and ubuntu>=22 only has python3"; + + debian.(!debian_9.!debian_10.!debian_11.!ubuntu_20.!ubuntu_18.!ubuntu_16):: + "python3"; + + "python-is-python3" + comment => "pipeline hosts need plain old python for buildscripts/build-scripts/get_labels_expr.py"; + + debian_13|ubuntu_25|ubuntu_26:: + "ntpsec"; + + debian.(!debian_13.!ubuntu_25.!ubuntu_26):: + "ntp"; debian|ubuntu:: + "fail2ban" comment => "Ban IPs with repeated failed SSH auth attempts"; "libltdl7" package_policy => "delete"; "libltdl-dev" package_policy => "delete"; - "binutils"; "bison"; "build-essential"; @@ -37,27 +49,89 @@ bundle agent cfengine_build_host_setup "fakeroot"; "flex"; "gdb"; - "libncurses5" comment => "added for debian-10"; - "libncurses5-dev" comment => "added for debian-10"; "libexpat1-dev"; "libmodule-load-conditional-perl"; "libpam0g-dev"; - "ntp"; "pkg-config"; "psmisc"; - "python3-pip"; + + "python3-pip" + comment => "this will bring in python3 if needed on debian>=12 and ubuntu>=22"; + + "python3-psycopg2"; "rsync" comment => "added for debian-10"; - ubuntu_16.mingw_build_host:: # for now, only ubu16 hosts mingw builds - "wine:i386"; + "systemd-coredump" + comment => "added step to jenkins testing-pr job to query for coredumps on failures"; + + # core/ci/dependencies.sh is run on some systems where quickinstall and cf-remote cannot install an agent to run this policy so we must remove some system packages that we also build since both install to /usr as a prefix. + # we do need these still installed on bootstrap-pr hosts though, so guard against that class + debian.!bootstrap_pr_host:: + "libattr1-dev" package_policy => "delete"; + "libssl-dev" package_policy => "delete"; + "libpcre2-dev" package_policy => "delete"; + "libacl1-dev" package_policy => "delete"; + "libyaml-dev" package_policy => "delete"; + "libxml2-dev" package_policy => "delete"; + "librsync-dev" package_policy => "delete"; + + ubuntu_20:: + "libncurses5"; + "libncurses5-dev"; + + (debian|ubuntu).!(debian_9|ubuntu_16|ubuntu_18|ubuntu_20):: + "libncurses6"; + "libncurses-dev"; + + mingw_build_host:: "mingw-w64"; + + # build-scripts/package-msi builds the MSI natively with wixl; msiinfo + # (msitools) is used to inspect it. No Wine, no i386. See ENT-13868. + "wixl"; + "msitools"; + + # build-scripts/package-msi derives deterministic MSI GUIDs with uuidgen + # (uuid-runtime) for reproducible builds. See ENT-13792. + "uuid-runtime"; + + "binfmt-support" + comment => "update-binfmts command needed for build-scripts/package-msi script"; + + mingw_build_host.have_i386_architecture:: + "wine:i386"; + (debian_10|debian_11).systemssl_build_host:: "libssl-dev"; -# I attempted to arrange these packages in order of: generic (all versions) and then as if we gradually added them through time: rhel-6, 7, 8, 9... + debian.bootstrap_pr_host:: + "libssl-dev"; + + # bootstrap_pr host needs this to configure before we build openssl ourselves + "librsync-dev"; + + # bootstrap_pr host needs this to run configure and make dist + "autoconf-archive" comment => "Required to resolve the AX_PTHREAD macro"; + + debian.containers_host:: + # in jenkins, CONTAINER labeled nodes are capable of running container builds like valgrind-check and static-check + "unzip" + comment => "linux-install-groovy.sh needs unzip to unpack the groovy distribution archive."; + + "buildah"; + "jq"; + "make"; + "parallel"; + "podman"; + + # I attempted to arrange these packages in order of: generic (all versions) and then as if we gradually added them through time: rhel-6, 7, 8, 9... suse|opensuse|sles|redhat|centos:: "gcc"; - "ncurses-devel"; + + "ncurses-devel" + if => not("sles_15|sles_12"), + comment => "sles 12/15 require a downgrade to install ncurses-devel; handled by the commands: promise below"; + "pam-devel"; "rsync"; "make"; @@ -77,11 +151,16 @@ bundle agent cfengine_build_host_setup (redhat_6|centos_6).(yum_dnf_conf_ok):: "rpm-build" handle => "rpm_build_installed"; - "python-psycopg2" comment => "centos-6 provides python2 and psycopg2 for python2 as a package"; - "perl-IO-Compress-Zlib" comment => "provides perl(IO::Uncompress::Gunzip) needed by lcov dependency package"; + + "python-psycopg2" + comment => "centos-6 provides python2 and psycopg2 for python2 as a package"; + + "perl-IO-Compress-Zlib" + comment => "provides perl(IO::Uncompress::Gunzip) needed by lcov dependency package"; + "perl-JSON"; -# perl-Digest-MD5 and perl-Data-Dumper are included in perl for centos-6 + # perl-Digest-MD5 and perl-Data-Dumper are included in perl for centos-6 (redhat_6|centos_6|redhat_7|centos_7).(yum_dnf_conf_ok):: "gdb"; "ntp"; @@ -90,50 +169,59 @@ bundle agent cfengine_build_host_setup "perl-devel"; "xfsprogs"; - (redhat_6|centos_6).(yum_dnf_conf_ok):: - "python-psycopg2" comment => "centos-6 provides python2 and psycopg2 for python2 as a package"; - (redhat_6|centos_6).(yum_dnf_conf_ok):: - "python-psycopg2" comment => "centos-6 provides python2 and psycopg2 for python2 as a package"; - !(redhat_6|centos_6).(yum_dnf_conf_ok):: - "python3-psycopg2"; - -# note that shellcheck, fakeroot and ccache require epel-release to be installed + # note that shellcheck, fakeroot and ccache require epel-release to be installed + # epel-release is installed by distribution package in rhel-7 and by URL for rhel-8+ later in commands section (redhat_7|centos_7).(yum_dnf_conf_ok):: - "epel-release"; + "epel-release" classes => results("bundle", "epel_release"); + + !(redhat_7|centos_7).(redhat|centos).(yum_dnf_conf_ok).epel_release_ok:: + "fail2ban-server" + comment => "Ban IPs with repeated failed SSH auth attempts. On centos/rhel 8+ we must specify individual packages instead of just fail2ban as package method will append -*.* which would include conflicting shorewall and shorewall-lite packages."; + + "fail2ban-sendmail"; + "fail2ban-firewalld"; "ccache"; "fakeroot"; - "java-11-openjdk" comment => "ok for jenkins 2.426.1 https://www.jenkins.io/doc/book/platform-information/support-policy-java/index.html"; "perl-JSON-PP"; - "perl-Data-Dumper"; "perl-Digest-MD5"; - (redhat_7|centos_7|redhat_9).(yum_dnf_conf_ok):: + (redhat_7|centos_7|redhat_9|redhat_10).(yum_dnf_conf_ok):: "python3-pip"; - (redhat_7|centos_7|redhat_8|centos_8|redhat_9).(yum_dnf_conf_ok):: + (redhat_7|centos_7|redhat_8|centos_8|redhat_9|redhat_10).(yum_dnf_conf_ok):: "perl-ExtUtils-MakeMaker"; - "perl-IO-Compress" comment => "provides perl(IO::Uncompress::Gunzip) needed by lcov dependency package"; + + "perl-IO-Compress" + comment => "provides perl(IO::Uncompress::Gunzip) needed by lcov dependency package"; + "psmisc"; "which"; (redhat_8|centos_8).(yum_dnf_conf_ok):: - "python2-psycopg2" comment => "This will bring in python2"; - "python3-rpm-macros" -> { "provides macro py3_shebang_fix needed in rhel-8 for /var/cfengine/bin/cfbs", "ENT-11338" } + "python3-rpm-macros" -> { + "provides macro py3_shebang_fix needed in rhel-8 for /var/cfengine/bin/cfbs", + "ENT-11338", + } comment => "There are several versions of python(x)-rpm-macros. We choose this one to get platform-python which is guaranteed to be installed in rhel-8."; + "platform-python-devel" -> { "cfbs shebang", "ENT-11338" } comment => "py3_shebang_fix macro needs /usr/bin/pathfix.py from platform-python-devel package"; - - (redhat_8|centos_8|redhat_9).(yum_dnf_conf_ok):: - "java-1.8.0-openjdk-headless" package_policy => "delete", + (redhat_8|centos_8|redhat_9|redhat_10).(yum_dnf_conf_ok):: + "java-1.8.0-openjdk-headless" + package_policy => "delete", comment => "Installing Development Tools includes this jdk1.8 which we do not want."; - "java-17-openjdk"; + "pkgconf" comment => "pkgconfig renamed to pkgconf in rhel8"; "selinux-policy-devel" comment => "maybe add to _7 and _6?"; - (redhat_9).(yum_dnf_conf_ok):: - "perl-Sys-Hostname" comment => "Needed by __04_examples_outputs_check_outputs_cf"; - "python3-psycopg2"; + (redhat_9|redhat_10).(yum_dnf_conf_ok):: + "perl-Sys-Hostname" + comment => "Needed by __04_examples_outputs_check_outputs_cf"; + + redhat_10.(yum_dnf_conf_ok):: + "patch"; + "perl-FindBin" comment => "Needed by postgresql 18"; suse|opensuse|sles:: "binutils"; @@ -143,130 +231,446 @@ bundle agent cfengine_build_host_setup "pkg-config"; "rpm-build"; - suse_12|opensuse_12|sles_12:: - "java-11-openjdk"; - suse_15|opensuse_15|sles_15:: - "java-17-openjdk"; + vars: + "java_version_raw" + string => execresult("java -version 2>&1 | grep version", "useshell"), + unless => "missing_java"; + "java_version" + string => nth(splitstring("${java_version_raw}", '"', 20), 1); - vars: "suse_users_and_groups" slist => { "daemon", "bin", "sys" }; + "sshd_hardening_directives" + slist => { + "PermitRootLogin", + "PasswordAuthentication", + "KbdInteractiveAuthentication", + "ChallengeResponseAuthentication", + }; + + "sshd_config_files" + slist => findfiles( + "/etc/ssh/sshd_config", "/etc/ssh/sshd_config.d/*.conf" + ); + + debian|ubuntu:: + "sshd_service_name" string => "ssh"; + + !(debian|ubuntu):: + "sshd_service_name" string => "sshd"; + classes: any:: - "mingw_build_host" expression => fileexists("/etc/cfengine-mingw-build-host.flag"); - "systemssl_build_host" expression => fileexists("/etc/cfengine-systemssl-build-host.flag"); - debian_9|ubuntu_16|redhat_6|centos_6:: - "have_opt_jdk21" expression => fileexists("/opt/jdk-21.0.1"); + "mingw_build_host" + expression => fileexists("/etc/cfengine-mingw-build-host.flag"); + + "systemssl_build_host" + expression => fileexists("/etc/cfengine-systemssl-build-host.flag"); + + "bootstrap_pr_host" + expression => fileexists("/etc/cfengine-bootstrap-pr-host.flag"); + + "containers_host" + expression => fileexists("/etc/cfengine-containers-host.flag"); + + "not_in_container" + expression => not(fileexists("/etc/cfengine-in-container.flag")), + comment => "We use an explicit flag file that we control to avoid ambiguity about whether we are in a container or not."; + +@if minimum_version(3.23) + # Rust is build dependency for leech2 (gate on ubuntu>=20, debian>=12, redhat>=7) + ubuntu:: + "leech2_build_toolchain_host" + expression => version_compare("$(sys.os_version_major)", ">=", "20"); + + debian:: + "leech2_build_toolchain_host" + expression => version_compare("$(sys.os_version_major)", ">=", "12"); + + (redhat|centos):: + "leech2_build_toolchain_host" + expression => version_compare("$(sys.os_version_major)", ">=", "7"); +@endif + + any:: + "have_rust" expression => fileexists("/opt/rust/bin/rustc"); + "have_protoc" expression => fileexists("/usr/local/bin/protoc"); + + linux:: + "have_tmp_mount" + expression => returnszero("mount | grep '/tmp'", "useshell"); + + "have_coredumpctl" + expression => returnszero("command -v coredumpctl", "useshell"); + + "missing_java" expression => not(fileexists("/usr/bin/java")); +@if minimum_version(3.23) + "insufficient_java_version" + expression => version_compare("${java_version}", "<", "21.0.0"); + + "java_ok" + expression => version_compare("${java_version}", ">=", "21.0.0"); +@endif + "missing_groovy" expression => not(fileexists("/usr/bin/groovy")); + (redhat|centos).!(redhat_6|centos_6|redhat_7|centos_7):: - "yum_conf_ok" expression => returnszero("grep best=False /etc/yum.conf", "useshell"); + "yum_conf_ok" + expression => returnszero( + "grep best=False /etc/yum.conf >/dev/null", "useshell" + ); + redhat_6|centos_6|redhat_7|centos_7:: - "yum_conf_ok" expression => "any"; # rhel/centos-6 and 7 do not support --nobest or best property in yum.conf + "yum_conf_ok" expression => "any"; + + # rhel/centos-6 and 7 do not support --nobest or best property in yum.conf redhat_8|centos_8:: - "have_fakeroot" expression => returnszero("command -v fakeroot >/dev/null", "useshell"); - redhat_8|centos_8|redhat_9:: - "redhat_has_python3" expression => returnszero("command -v python3 >/dev/null", "useshell"); - "dnf_conf_ok" expression => returnszero("grep best=False /etc/dnf/dnf.conf", "useshell"); - redhat_8|centos_8|redhat_9:: - "have_perl_package_installed" expression => returnszero("rpm -q perl >/dev/null", "useshell"); - redhat_9:: - "have_python3_pip_package_installed" expression => returnszero("rpm -q python3-pip >/dev/null", "useshell"); - (redhat_8|centos_8|redhat_9).(yum_conf_ok.dnf_conf_ok):: + "have_fakeroot" + expression => returnszero("command -v fakeroot >/dev/null", "useshell"); + + (redhat|centos):: + "epel_release_ok" + expression => returnszero( + "yum info installed epel-release", "useshell" + ); + + (redhat_8|centos_8).have_perl:: + "have_data_dumper" + expression => returnszero("cpan -l | grep Data::Dumper", "useshell"); + + "have_data_dumper_names" + expression => returnszero( + "cpan -l | grep Data::Dumper::Names", "useshell" + ); + + "have_padwalker" + expression => returnszero("cpan -l | grep PadWalker", "useshell"); + + redhat_8|centos_8|redhat_9|redhat_10:: + "redhat_has_python3" + expression => returnszero("command -v python3 >/dev/null", "useshell"); + + "dnf_conf_ok" + expression => returnszero( + "grep best=False /etc/dnf/dnf.conf >/dev/null", "useshell" + ); + + redhat_8|centos_8|redhat_9|redhat_10:: + "have_perl_package_installed" + expression => returnszero("rpm -q perl >/dev/null", "useshell"); + + redhat_9|redhat_10:: + "have_python3_pip_package_installed" + expression => returnszero("rpm -q python3-pip >/dev/null", "useshell"); + + !(redhat_6|centos_6|redhat_7|centos_7).(yum_conf_ok.dnf_conf_ok):: "yum_dnf_conf_ok" expression => "any"; + (redhat_6|centos_6|redhat_7|centos_7).(yum_conf_ok):: "yum_dnf_conf_ok" expression => "any"; - (redhat_7|centos_7|redhat_8|centos_8|redhat_9).(yum_dnf_conf_ok):: - "have_development_tools" expression => returnszero("yum groups list installed | grep 'Development Tools' >/dev/null", "useshell"), + + (redhat_7|centos_7|redhat_8|centos_8|redhat_9|redhat_10).(yum_dnf_conf_ok):: + "have_development_tools" + expression => returnszero( + "yum groups list installed | grep 'Development Tools' >/dev/null", + "useshell" + ), comment => "note: centos-7 has installed instead of --installed argument, and that works on rhel-8 and rhel-9 so go with the sub-command instead of option"; - ubuntu_20:: - "have_python2_pip" expression => fileexists("/usr/local/bin/pip"); - ubuntu_20.have_python2_pip:: - "have_python2_psycopg2" expression => returnszero("/usr/local/bin/pip list psycopg2", "useshell"); commands: - !have_opt_jdk21.(debian_9|ubuntu_16|redhat_6|centos_6):: - "sh $(this.promise_dirname)/linux-install-jdk21.sh" contain => in_shell; - (redhat_7|centos_7|redhat_8|centos_8|redhat_9).(!have_development_tools).(yum_dnf_conf_ok):: + sshd_hardened:: + 'kill -1 $(pgrep -f "sshd -D")' + handle => "sshd_restarted", + contain => in_shell, + comment => "Reload sshd config with SIGHUP(1) to apply hardened configuration"; + have_tmp_mount:: + "mount -o remount,size=5G /tmp" + comment => "We could check if /tmp was size 5G but not worth the trouble since this remount call just sets the maximum size of the tmpfs in virtual memory.", + contain => in_shell; + + have_coredumpctl.not_in_container:: + "sysctl kernel.core_pattern='|/lib/systemd/systemd-coredump %p %u %g %s %t %e'" -> { + "ENT-12669" + } + comment => "Ensure that core_pattern is proper for systemd-coredump if coredumpctl is present.", + contain => in_shell; + + containers_host.missing_groovy.java_ok:: + "sh $(this.promise_dirname)/linux-install-groovy.sh" contain => in_shell; + + missing_java|insufficient_java_version:: + "sh $(this.promise_dirname)/linux-install-jdk.sh" + contain => in_shell, + classes => results("bundle", "java"); + + # leech2 build toolchain: protoc and the Rust toolchain. Both installers + # pin a version and verify the SHA256 checksum of the downloaded tarball. + leech2_build_toolchain_host.!have_protoc:: + "sh $(this.promise_dirname)/linux-install-protobuf.sh" + contain => in_shell, + comment => "Install pinned protoc; required to build the cargo-based leech2 dependency."; + + # Linux builds are native, so the installer only adds the host's own Linux + # std. Windows is the only cross-compilation target, and only MinGW build + # hosts cross-compile it, so we pass that target there alone. + leech2_build_toolchain_host.!have_rust:: + "sh $(this.promise_dirname)/linux-install-rust.sh" + args => ifelse("mingw_build_host", "x86_64-pc-windows-gnu", ""), + contain => in_shell, + comment => "Install the Rust toolchain system-wide under /opt/rust for building the cargo-based leech2 dependency."; + + (redhat_7|centos_7|redhat_8|centos_8|redhat_9|redhat_10).(!have_development_tools).(yum_dnf_conf_ok):: "yum groups install -y 'Development Tools'" contain => in_shell; - (redhat_8|centos_8).!have_fakeroot:: # special fakeroot, missing from _8 an d up? + + (redhat_8|centos_8).!have_fakeroot:: + # special fakeroot, missing from _8 an d up? "sudo rpm -iv https://kojipkgs.fedoraproject.org//packages/fakeroot/1.23/1.fc29/x86_64/fakeroot-1.23-1.fc29.x86_64.rpm https://kojipkgs.fedoraproject.org//packages/fakeroot/1.23/1.fc29/x86_64/fakeroot-libs-1.23-1.fc29.x86_64.rpm" contain => in_shell; - (redhat_8|centos_8|redhat_9).!redhat_has_python3:: + + (redhat|centos).!(redhat_7|centos_7).!(redhat_6|centos_6).!epel_release_ok:: + "yum install --assumeyes https://dl.fedoraproject.org/pub/epel/epel-release-latest-${sys.os_version_major}.noarch.rpm" + comment => "rhel-7 installs this with a packages promise. TODO: check a sha for the download URL somehow?", + classes => results("bundle", "epel_release"), + contain => in_shell; + + (redhat_8|centos_8).!have_data_dumper.have_perl:: + "cpan Data::Dumper" contain => in_shell; + + (redhat_8|centos_8).!have_data_dumper_names.have_perl:: + "cpan Data::Dumper::Names" contain => in_shell; + + (redhat_8|centos_8).!have_padwalker.have_perl:: + "cpan PadWalker" contain => in_shell; + + (redhat_8|centos_8|redhat_9|redhat_10).!redhat_has_python3:: "yum install -y python3" -> { "CFE-4313" } contain => in_shell, comment => "workaround for yum package_method trying to install python3-*.* which causes conflicts."; - (redhat_8|centos_8|redhat_9).!yum_conf_ok:: + + (redhat_8|centos_8|redhat_9|redhat_10).!yum_conf_ok:: "sed -i '/best=True/s/True/False/' /etc/yum.conf" contain => in_shell; - (redhat_8|centos_8|redhat_9).!dnf_conf_ok:: - "sed -i '/best=True/s/True/False/' /etc/dnf/dnf.conf" contain => in_shell; + (redhat_8|centos_8|redhat_9|redhat_10).!dnf_conf_ok:: + "sed -i '/best=True/s/True/False/' /etc/dnf/dnf.conf" contain => in_shell; classes: debian_11:: "have_pip2" expression => fileexists("/usr/local/bin/pip"); - ubuntu_16:: - "have_i386_architecture" expression => strcmp(execresult("${paths.dpkg} --print-foreign-architectures", "noshell"), "i386"); + ubuntu:: - "have_localhost_hostname" expression => strcmp(execresult("${paths.hostname}", "noshell"), "localhost.localdomain"); + "have_i386_architecture" + expression => strcmp( + execresult("${paths.dpkg} --print-foreign-architectures", "noshell"), + "i386" + ); + + "localhost_localdomain_hostname_missing" + expression => not( + strcmp( + execresult("${paths.hostname} -f", "useshell"), + "localhost.localdomain" + ) + ); + opensuse|suse|sles:: - "have_$(suse_users_and_groups)_group" expression => returnszero("grep '^$(suse_users_and_groups):' /etc/group >/dev/null", "useshell"); - "have_$(suse_users_and_groups)_user" expression => returnszero("grep '^$(suse_users_and_groups):' /etc/passwd >/dev/null", "useshell"); + "have_$(suse_users_and_groups)_group" + expression => returnszero( + "grep '^$(suse_users_and_groups):' /etc/group >/dev/null", "useshell" + ); + + "have_$(suse_users_and_groups)_user" + expression => returnszero( + "grep '^$(suse_users_and_groups):' /etc/passwd >/dev/null", + "useshell" + ); files: - ubuntu_16|redhat_9:: - "/etc/hosts" - edit_line => regex_replace("127.0.0.1 localhost localhost.localdomain","127.0.0.1 localhost.localdomain"); + redhat:: + "/etc/environment" + edit_line => contains_literal_string("LC_ALL=C"), + comment => "It seems that centos-7 at least needs LC_ALL=C now, maybe due to different way we access: ProxyJump ssh commands from jenkins instead of java ssh client. 2026-04-03"; + + linux:: + "/home/jenkins/.ssh/known_hosts" + create => "true", + perms => mog("644", "jenkins", "jenkins"), + content => "github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl +github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg= +github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk= +build-artifacts-cache.cloud.cfengine.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGahpsY8Phk2+isBmuJQjjQVlh6BNL/Qetc14g26gowV +build-artifacts-cache.cloud.cfengine.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCdv55BmDmwEjN3izaL8intrSRJQglkv++TaQppopKCh5VkYuSpNj/W+x0vjxwL3+NNp4CAhOLYuGTFuUL8zz/H0kwWE06c3WjghpHS3NS1usiamVZN6uaJMGwDaq8fPB3WiiI/L5lLM8/ubXxfmta0UlVufHCiBH8pBQarAY5rdPDtITXzu56qIa3k9Ou7Si2r/1n37espzwsPxoBqVJg7BvzMV28MzxdmQpGP6puuPfi9tvYbtnF788le3jvGOc+3GOiwtXsv44y/PCJNhi7sFeUfxocuNbWXZ3x/UEfbN8IiUVZsUAuLQFeqr3pv4w28D+SvJHBMCFudygenmLc3th+typiFRmqam7UQif9Pa3e2FxX4ghTp1KNXBoCQluIk2j7wX9zXppSyUG6d7tPDzfu81lImuW34+bsZvYq+s+25vbAhSDdQe1lqG0Fdvvi+zbqrMYQuMfnInDrK52xNSZcfATWjudhmY6wiwdSS0XsBADmZsy3qf3ErdEabqQk= +build-artifacts-cache.cloud.cfengine.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIzU5+SoC4gbtV3Wfw4oB6oMs5RYKGFCiS0lVeN4XQlAM8UjvyUUSflytf/vQEANv1OJs5vicslRn/iPlrvF8Mk="; + + "/etc/security/limits.conf" + edit_line => lines_present( + " +root - core unlimited +* - core unlimited +" + ); + + "/etc/fail2ban/jail.local" + create => "true", + content => "[sshd] +enabled = true +port = ssh +maxretry = 5 +bantime = 3600 +findtime = 600", + classes => if_repaired("fail2ban_config_changed"), + comment => "Configure fail2ban to ban IPs after 5 failed SSH attempts within 10 minutes"; + + "$(sshd_config_files)" + edit_line => comment_lines_matching( + "^$(sshd_hardening_directives)\s+(?!no\s*$).*", "#" + ), + classes => if_repaired("sshd_hardened"), + comment => "Comment out insecure SSH auth directives in sshd_config and drop-ins"; + + "/etc/ssh/sshd_config" + edit_line => prepend_if_no_line("$(sshd_hardening_directives) no"), + if => fileexists("/etc/ssh/sshd_config"), + classes => if_repaired("sshd_hardened"), + comment => "Ensure SSH hardening directives are at the top of sshd_config, before any Include"; + + ubuntu_16|ubuntu_18|redhat_9|redhat_10:: + "/etc/hosts" -> { "ENT-12437" } + edit_line => regex_replace( + "127.0.0.1 localhost localhost.localdomain", + "127.0.0.1 localhost.localdomain" + ), + comment => "In order for some check_outputs peers related tests to work, hostname -f must match sys.fqhost so remove localhost and leave localhost.localdomain"; + debian_9:: - "/etc/apt/sources.list.d/*" - delete => tidy; + "/etc/apt/sources.list.d/*" delete => tidy; + + # Note: apt-transport-https is not available on Debian 9, so we cannot use + # HTTPS here yet. "/etc/apt/sources.list" - content => "deb https://archive.debian.org/debian/ stretch main contrib non-free"; + content => "deb http://archive.debian.org/debian/ stretch main contrib non-free"; + suse_15|opensuse_15|sles_15:: "/home/jenkins/.rpmmacros" content => "%dist .suse15", comment => "ensure %dist works in RPM .spec files - needed to add OS name/version to rpm filename"; + suse_12|opensuse_12|sles_12:: "/home/jenkins/.rpmmacros" content => "%dist .suse12", comment => "ensure %dist works in RPM .spec files - needed to add OS name/version to rpm filename"; + suse_11|opensuse_11|sles_11:: "/home/jenkins/.rpmmacros" content => "%dist .suse11", comment => "ensure %dist works in RPM .spec files - needed to add OS name/version to rpm filename"; + debian.containers_host:: + "/etc/sudoers.d/999-local" + comment => "NOPASSWD is needed for various tools related to container jobs", + content => "%wheel ALL=NOPASSWD: /usr/bin/lvm-cache-stats +%wheel ALL=NOPASSWD: /usr/bin/podman +%sudo ALL=NOPASSWD: /usr/bin/lvm-cache-stats +%sudo ALL=NOPASSWD: /usr/bin/podman +%sudo ALL=NOPASSWD: /usr/sbin/lvs +%sudo ALL=NOPASSWD: /usr/bin/journalctl +jenkins ALL=NOPASSWD: /usr/bin/podman +", + create => "true", + perms => mog("400", "root", "root"); + + centos_7:: + "/etc/environment" + comment => "centos-7 vagrant VMs have unconfigured locale that breaks some tests so force LC_ALL=C", + content => "LC_ALL=C", + create => "true", + perms => mog("0644", "root", "root"); + + redhat_10:: + "/usr/lib/rpm/redhat/macros" + edit_line => insert_lines("%_empty_manifest_terminate_build 0"), + depends_on => { "rpm_build_installed" }; + + redhat_8|centos_8|redhat_9|centos_9|redhat_10:: + "/usr/lib/rpm/redhat/macros" + edit_line => comment_lines_matching("%_enable_debug_packages 0", "#"), + depends_on => { "rpm_build_installed" }; + commands: - (redhat_8|centos_8|redhat_9).(!have_perl_package_installed).(yum_dnf_conf_ok):: - "yum install -y perl" contain => in_shell, + sles_15|sles_12:: + "zypper --non-interactive install --allow-downgrade ncurses-devel" + comment => "Special case mentioned elsewhere in this policy. ncurses-devel requires a downgrade (sles 15 since July 25 2025; sles 12 SP5 since 2026)", + contain => in_shell; + + (redhat_8|centos_8|redhat_9|redhat_10).(!have_perl_package_installed).(yum_dnf_conf_ok):: + "yum install -y perl" + contain => in_shell, + classes => results("bundle", "have_perl"), comment => "even though rhel8/9 come with /bin/perl perl >= 5.8.8 is needed by cfbuild-lcov-1.16-1.noarch. So the package must be installed."; - redhat_9.!have_python3_pip_package_installed.(yum_dnf_conf_ok):: + + redhat_9|redhat_10.!have_python3_pip_package_installed.(yum_dnf_conf_ok):: "yum install -y python3-pip" contain => in_shell; - redhat_8|centos_8|redhat_9|centos_9:: - "sudo sed -ri 's/^%_enable_debug_packages/#\0/' /usr/lib/rpm/redhat/macros" contain => in_shell, - depends_on => { "rpm_build_installed" }; - ubuntu_16.!have_i386_architecture:: # mingw build host - "${paths.dpkg} --add-architecture i386"; + mingw_build_host.!have_i386_architecture:: + "${paths.dpkg} --add-architecture i386" handle => "i386_arch_added"; + + "DEBIAN_FRONTEND=noninteractive apt-get update" + depends_on => { "i386_arch_added" }, + contain => in_shell; - ubuntu.!have_localhost_hostname:: + ubuntu.not_in_container.localhost_localdomain_hostname_missing:: "/usr/bin/hostnamectl set-hostname localhost.localdomain" comment => "hack for aws ubuntu hosts having unique ip-n-n-n-n hostnames, we need localhost.localdomain"; + !have_daemon_group.(suse|sles|opensuse):: "groupadd -g 1 daemon" contain => in_shell; + !have_bin_group.(suse|sles|opensuse):: "groupadd -g 2 bin" contain => in_shell; + !have_sys_group.(suse|sles|opensuse):: "groupadd -g 3 sys" contain => in_shell; + !have_daemon_user.(suse|sles|opensuse):: - "useradd -u 1 daemon" contain => in_shell; + "useradd -u 1 daemon -g daemon" contain => in_shell; + !have_bin_user.(suse|sles|opensuse):: - "useradd -u 2 bin" contain => in_shell; + "useradd -u 2 bin -g bin" contain => in_shell; + !have_sys_user.(suse|sles|opensuse):: - "useradd -u 3 sys" contain => in_shell; + "useradd -u 3 sys -g sys" contain => in_shell; -# skip /etc/hosts change for now, seems kind of wrong and corrupts ip6 entries like `::1 ip6-ip6-loopback` -# maybe the following is needed to silence such errors as: ubuntu-16-mingw-j1: sudo: unable to resolve host localhost.localdomain -# ubuntu:: -# "${paths.sed} -ri 's/localhost //' /etc/hosts"; + linux:: + "sshd -T 2>/dev/null | grep -qiE '^PermitRootLogin no'" + depends_on => { "sshd_restarted" }, + contain => in_shell, + comment => "Verify PermitRootLogin is disabled"; + + "sshd -T 2>/dev/null | grep -qiE '^PasswordAuthentication no'" + depends_on => { "sshd_restarted" }, + contain => in_shell, + comment => "Verify PasswordAuthentication is disabled"; + + "sshd -T 2>/dev/null | grep -qiE '^(KbdInteractive|ChallengeResponse)Authentication no'" + depends_on => { "sshd_restarted" }, + contain => in_shell, + comment => "Verify KbdInteractiveAuthentication (OpenSSH 8.7+) or ChallengeResponseAuthentication (older) is disabled"; + + services: + any:: + "fail2ban" + service_policy => "start", + comment => "Ensure fail2ban is running"; + + fail2ban_config_changed:: + "fail2ban" + service_policy => "restart", + comment => "Restart fail2ban to apply jail configuration"; + + # skip /etc/hosts change for now, seems kind of wrong and corrupts ip6 entries like `::1 ip6-ip6-loopback` + # maybe the following is needed to silence such errors as: ubuntu-16-mingw-j1: sudo: unable to resolve host localhost.localdomain + # ubuntu:: + # "${paths.sed} -ri 's/localhost //' /etc/hosts"; } + # todo, maybe need # ubuntu16-mingw: echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | sudo debconf-set-selections diff --git a/ci/cfengine-masterfiles-3.21.7-1.pkg.tar.gz.sha256 b/ci/cfengine-masterfiles-3.21.7-1.pkg.tar.gz.sha256 new file mode 100644 index 000000000..9037e2d2b --- /dev/null +++ b/ci/cfengine-masterfiles-3.21.7-1.pkg.tar.gz.sha256 @@ -0,0 +1 @@ +b9afe61210c43880aada212eb6119506c00bcc030b17c2cbdd0bf88a1514bd82 cfengine-masterfiles-3.21.7-1.pkg.tar.gz diff --git a/ci/cfengine-masterfiles-3.24.2-1.pkg.tar.gz.sha256 b/ci/cfengine-masterfiles-3.24.2-1.pkg.tar.gz.sha256 new file mode 100644 index 000000000..42a7a234e --- /dev/null +++ b/ci/cfengine-masterfiles-3.24.2-1.pkg.tar.gz.sha256 @@ -0,0 +1 @@ +8ced84602c14da5b6736b3b0bfda4be22f574c2702f9af5a788fae7245ff02e1 cfengine-masterfiles-3.24.2-1.pkg.tar.gz diff --git a/ci/cfengine-masterfiles-3.26.0-1.pkg.tar.gz.sha256 b/ci/cfengine-masterfiles-3.26.0-1.pkg.tar.gz.sha256 new file mode 100644 index 000000000..3a99eea98 --- /dev/null +++ b/ci/cfengine-masterfiles-3.26.0-1.pkg.tar.gz.sha256 @@ -0,0 +1 @@ +1217b72455f09776442c5abb40b66f49629175b298c0b7baf0d5ba5ffe94e5ed cfengine-masterfiles-3.26.0-1.pkg.tar.gz diff --git a/ci/clean-buildhost-images.sh b/ci/clean-buildhost-images.sh new file mode 100755 index 000000000..731a9aeba --- /dev/null +++ b/ci/clean-buildhost-images.sh @@ -0,0 +1 @@ +buildah images | grep buildhost | awk '{print $3}' | xargs buildah rmi diff --git a/ci/create-revisions-file.sh b/ci/create-revisions-file.sh new file mode 100755 index 000000000..f881e7365 --- /dev/null +++ b/ci/create-revisions-file.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Look for pull-request-title and pull-request-body files in the current directory +# these files should be generated by the environment in which we find ourselves: for now either github actions/workflows or jenkins pipelines + +# expected inputs via environment variables +# REPOS - space separated names of repositories to include, if a forward slash (/) included then the format is / otherwise the default org is cfengine + +# helper functions +function rev_env_name() +{ + local repo=$1 + echo "${repo}_rev" | tr '[:lower:]' '[:upper:]' | tr '/-' '_' +} + +# process revisions +# replace numeric revisions by pull/REVISION/merge + +# blank out revisions file +echo > revisions + +# process pull-request-body file added by execution environment like Jenkinsfile or github workflow +if [ -f pull-request-body ]; then + awk 'BEGIN { RS="https://" } {print $0}' pull-request-body | while IFS= read -r line + do + org_repo_rev=$(echo "$line" | awk 'match($0,/github.com\/([a-zA-Z-]*)\/([a-zA-Z-]*)\/pull\/([0-9]*)/,a) {print a[1] ":" a[2] ":" a[3]}') + _org=$(echo "$org_repo_rev" | cut -d: -f1) + _repo=$(echo "$org_repo_rev" | cut -d: -f2) + _rev=$(echo "$org_repo_rev" | cut -d: -f3) + if [ -n "$_repo" ]; then + varname=$(rev_env_name "$_repo") + declare "$varname"="$_rev" + fi + done +fi + +# figure out what our upstream branch (master, 3.24.x, etc) is from jenkins environment variables +# PR_BASE - from jenkins ${pullRequest.base} +# BRANCH_NAME - from jenkins, only usable for non-pull requests +if [ -n "$PR_BASE" ]; then + _TMP="$PR_BASE" +else + _TMP="$BRANCH_NAME" +fi +if [ "$(expr "$_TMP" : ".*.x")" = 0 ]; then + if [ "$_TMP" = "master" ]; then + _DOCS_BRANCH=master + _BRANCH=master + else + _DOCS_BRANCH="$_TMP" + _BRANCH="$_TMP".x + fi +else + _DOCS_BRANCH=${_TMP%.x} # % removes suffix of .x from _TMP + _BRANCH="$_TMP" +fi + +for reponame in $REPOS; do + if [ "$(expr "$reponame" : ".*/.*")" = 0 ]; then + org=cfengine + repo="$reponame" + else + org="$(echo "$reponame" | cut -d/ -f1)" + repo="$(echo "$reponame" | cut -d/ -f2)" + fi + varname="$(rev_env_name "$repo")" + + # if _REV is empty, set to the default of either _DOCS_BRANCH (documentation repo) or _BRANCH (all others) + if [ -z "${!varname}" ]; then + if [ "$varname" = "NT_DOCS_REV" ]; then + declare "$varname"="main" + else + if [ "$varname" = "DOCUMENTATION_REV" ]; then + declare "$varname"="$_DOCS_BRANCH" + else + declare "$varname"="$_BRANCH" + fi + fi + fi + + declare "$varname"="$(echo "${!varname}"|sed -r 's|^([0-9]+)$|pull/\1/merge|')" + # do nothing if empty, remove tag: if found, add origin/ otherwise + declare "$varname"="$(echo "${!varname}"|awk 'BEGIN {FS= ":"} /^$/ {next} /^tag:.*/ {print $2} !/^tag:/ {print "origin/"$1}')" + + # remove origin/ (if any) and add refs/ if necessary + declare "$varname"="${!varname##origin/}" + if [ -n "${!varname}" ] && [ "$(expr "${!varname}" : "pull/")" != 0 ]; then + declare "$varname"="refs/${!varname}" + fi + echo "$repo git@github.com:${org}/${repo} ${!varname}" >> revisions +done diff --git a/ci/create_swap_file.sh b/ci/create_swap_file.sh new file mode 100755 index 000000000..1339d323d --- /dev/null +++ b/ci/create_swap_file.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# bash is needed in order to use the "time" built-in and avoid needing +# an external utility. + +set -e # exit on error + +# Argument $1 is the size in megabytes +if [ x"$1" = x ] || echo "$1" | grep -q '[^0-9]' +then + exit 2 +fi +SIZE="$1" + +if swapon | grep /swapfile >/dev/null; then + echo "/swapfile already configured and setup. Exiting." + exit 0 +fi + +time dd if=/dev/zero of=/swapfile bs=1M count=$SIZE +chmod 0600 /swapfile + +PATH=$PATH:/sbin:/usr/sbin +mkswap /swapfile +swapon /swapfile diff --git a/ci/distribution-patched.sh b/ci/distribution-patched.sh new file mode 100755 index 000000000..fc8e582d1 --- /dev/null +++ b/ci/distribution-patched.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -ex +for debian_release in stretch buster bullseye; do + if grep "CODENAME=$debian_release" /etc/os-release; then + echo "deb http://archive.debian.org/debian-archive/debian ${debian_release} main" >/etc/apt/sources.list + echo "deb http://archive.debian.org/debian-archive/debian ${debian_release}-backports main" >>/etc/apt/sources.list + fi +done +if [ -f /etc/centos-release ]; then + _version=$(cat /etc/centos-release | cut -d' ' -f3 | cut -d. -f1) + if [ "$_version" = "6" ] || [ "$_version" = "7" ]; then + sed -i 's/mirror.centos.org/vault.centos.org/;/^mirrorlist/d;s/^#baseurl/baseurl/' /etc/yum.repos.d/CentOS-Base.repo + fi +fi +if command -v yum; then + yum -e 0 -d 0 -y update + yum -e 0 -d 0 -y install git rsync +fi +if command -v apt; then + DEBIAN_FRONTEND=noninteractive apt -yqq update + DEBIAN_FRONTEND=noninteractive apt -yqq upgrade + DEBIAN_FRONTEND=noninteractive apt install -yqq git rsync +fi +if command -v zypper; then + source /etc/os-release + rpm --import https://download.opensuse.org/distribution/leap/$VERSION_ID/repo/oss/repodata/repomd.xml.key + zypper ar -cfp 90 https://download.opensuse.org/distribution/leap/$VERSION_ID/repo/oss/ oss + for repo in oss sle backports; do + rpm --import https://download.opensuse.org/update/leap/$VERSION_ID/$repo/repodata/repomd.xml.key + zypper ar -cfp 70 https://download.opensuse.org/update/leap/$VERSION_ID/$repo/ update-$repo + done + zypper -qn ref + zypper lr # diagnostic to see what repos are enabled + zypper -qn update + zypper -qn rm libsnmp15 + zypper -qn install git rsync + groupadd jenkins || true + useradd -m -u 1010 -g jenkins jenkins || true +fi diff --git a/ci/fix-buildhost.sh b/ci/fix-buildhost.sh index af7cfcdcc..3077104e5 100755 --- a/ci/fix-buildhost.sh +++ b/ci/fix-buildhost.sh @@ -1,3 +1,16 @@ +#!/usr/bin/env bash +# it is expected that this file is sourced, not executed directly +set -ex +my_path="$(realpath "${BASH_SOURCE[0]}")" +my_dir="$(dirname "$my_path")" + +if [ -f /etc/os-release ]; then + source /etc/os-release + if [ "$ID" = "centos" ] && [ "$VERSION_ID" = "7" ]; then + source "$my_dir"/centos-7-setup-devtoolset-11.sh + fi +fi + if [ "$(uname)" = "HP-UX" ]; then # /etc/profile contains tty code that won't work well when sourced and this VUE env var guards against running those bits # https://ftp.mirrorservice.org/sites/www.bitsavers.org/pdf/hp/9000_hpux/9.x/B1171-90044_HP_Visual_User_Environment_System_Administration_Manual_Nov91.pdf @@ -5,6 +18,9 @@ if [ "$(uname)" = "HP-UX" ]; then export VUE fi +mkdir -p ~/.ssh +echo "build-artifacts-cache.cloud.cfengine.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGahpsY8Phk2+isBmuJQjjQVlh6BNL/Qetc14g26gowV" >> ~/.ssh/known_hosts + # /etc/profile can contain tricky things, on suse for example it includes a call to tty which will fail in CI # so only source /etc/profile where we absolutely need it. if [ "$(uname)" = "HP-UX" ] || [ "$(uname)" = "SunOS" ]; then @@ -14,7 +30,6 @@ if [ "$(uname)" = "HP-UX" ] || [ "$(uname)" = "SunOS" ]; then . /etc/profile fi fi - # ENT-13750 we return to vendored openssl on rpm platforms so remove possibly installed development packages if command -v zypper >/dev/null 2>/dev/null; then sudo zypper remove -y libopenssl-devel || true @@ -22,3 +37,15 @@ fi if command -v yum >/dev/null 2>/dev/null; then sudo yum erase -y openssl-devel || true fi + +# MinGW hosts build the MSI with wixl (build-scripts/package-msi) and inspect it +# with msiinfo (msitools). uuidgen (uuid-runtime) derives deterministic MSI +# GUIDs for reproducible builds (ENT-13792). Installed by the build-host-setup +# policy at image time; install here too so not-yet-reimaged mingw hosts get +# them without a reimage. See ENT-13868. +if [ -f /etc/cfengine-mingw-build-host.flag ]; then + if ! command -v wixl >/dev/null 2>&1 || ! command -v msiinfo >/dev/null 2>&1 || ! command -v uuidgen >/dev/null 2>&1; then + sudo apt-get update + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y wixl msitools uuid-runtime + fi +fi diff --git a/ci/images.sh b/ci/images.sh new file mode 100644 index 000000000..3af9c2157 --- /dev/null +++ b/ci/images.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -ex + +# todo: centos7, opensuse/leap:15 (no :12), registry.access.redhat.com/ubi9 (-minimal, -init, -micro (standard)) +# run this on x86 and arm hardware to cover "all the bases" :) +for platform in $(cat platform-container-image.list); do + ./patched-image.sh $platform + ./buildhost-image.sh $platform +done diff --git a/ci/initialize-build-host.sh b/ci/initialize-build-host.sh index 0349e5e28..7080d2dd4 100644 --- a/ci/initialize-build-host.sh +++ b/ci/initialize-build-host.sh @@ -8,7 +8,7 @@ # serial execution of initialization steps. It will post the output when # finished, if any. # -# 2. If $HOME/proxy-target.txt exists, it means this is a proxy host, and the +# 2. If $(pwd)/proxy-target.txt exists, it means this is a proxy host, and the # real build machine is on the host specified by the login details inside # that file. If the file does not exist, we are on the build slave itself. # After figuring that stuff out, the script will run the rest of the original @@ -101,29 +101,6 @@ command -v /sbin/ip 2>/dev/null && /sbin/ip addr || true RSYNC="rsync --delete -zrlpt -T /tmp" RSH="ssh -o BatchMode=yes" -# Retry a command up to 5 times with a 10 second pause between attempts. -# Used for the early proxy-target setup commands (rsync/ssh) where a brief -# network blip to an exotic build host should not fail the entire build. -try_run() { - max_tries=5 - i=1 - ret=0 - while [ "$i" -le "$max_tries" ]; do - ret=0 - "$@" || ret=$? - if [ "$ret" -eq 0 ]; then - return 0 - fi - if [ "$i" -lt "$max_tries" ]; then - echo "try_run: attempt $i/$max_tries failed (exit $ret) for: $*; retrying in 10s..." >&2 - sleep 10 - fi - i=$((i + 1)) - done - echo "try_run: command failed after $max_tries attempts: $*" >&2 - return "$ret" -} - # Support launching scripts that were initially launched under bash. if [ -n "$BASH_VERSION" ] then @@ -159,9 +136,10 @@ do sleep 10 done -echo '========================================= PRINTING CLOUD-INIT LOG ===================================================' -sudo sed 's/^.*/>>> &/' /var/log/cloud-init-output.log || true -echo '======================================= DONE PRINTING CLOUD-INIT LOG ================================================' +# TODO, instead of printing this out ALWAYS, print it out only in case of errors ENT-14372 +#echo '========================================= PRINTING CLOUD-INIT LOG ===================================================' +#sudo sed 's/^.*/>>> &/' /var/log/cloud-init-output.log || true +#echo '======================================= DONE PRINTING CLOUD-INIT LOG ================================================' if [ $attempts -le 0 ] then @@ -170,9 +148,10 @@ then exit 1 fi -echo '=========================================== CURRENT ENVIRONMENT =====================================================' -export -echo '========================================= CURRENT ENVIRONMENT END ===================================================' +# TODO only print current environment on errors, maybe save the environment NOW and then show a diff at ERROR +#echo '=========================================== CURRENT ENVIRONMENT =====================================================' +#export +#echo '========================================= CURRENT ENVIRONMENT END ===================================================' # Disable TTY requirement. This normally happens in initialize-user-data.sh, but # for hosts that do not support cloud user data, it may not have happened @@ -227,9 +206,9 @@ reset_nested_vm() { if sudo dmesg | grep -q "BIOS Google" then # We're in Google Cloud, so just need to run nested-vm script again - if [ ! -d $HOME/mender-qa ] + if [ ! -d $HOME/buildscripts ] then - echo "Where is mender-qa repo gone?" + echo "Where is buildscripts repo gone?" sudo ls -lap $HOME exit 1 fi @@ -248,14 +227,15 @@ reset_nested_vm() { fi if [ ! -z "$login" ] then - ip=`sed 's/.*@//' $HOME/proxy-target.txt` + ip=`sed 's/.*@//' $(pwd)/proxy-target.txt` if sudo arp | grep -q $ip then sudo arp -d $ip fi fi + # TODO, remove this, we don't need or use or test nested-vms $HOME/mender-qa/scripts/nested-vm.sh $HOME/*.qcow2 - login="`cat $HOME/proxy-target.txt`" + login="`cat $(pwd)/proxy-target.txt`" if $RSH $login true then echo "Nested VM is back up, it seems. Happily continuing!" @@ -297,13 +277,13 @@ reset_nested_vm() { fi } -if [ -f $HOME/proxy-target.txt ] +if [ -f $(pwd)/proxy-target.txt ] then # -------------------------------------------------------------------------- # Check target machine health. # -------------------------------------------------------------------------- - login="$(cat $HOME/proxy-target.txt)" + login="$(cat $(pwd)/proxy-target.txt)" if [ ! -z "$login" ] && $RSH $login true then @@ -327,7 +307,7 @@ then # -------------------------------------------------------------------------- # Put our currently executing script on the proxy target. - try_run $RSYNC -e "$RSH" "$0" $login:commands-from-proxy.sh + $RSYNC -e "$RSH" "$0" $login:commands-from-proxy.sh # And the important parts of the environment. for var in \ @@ -379,16 +359,16 @@ then echo "PROXIED=1" >> env.sh echo "export PROXIED" >> env.sh - try_run $RSYNC -e "$RSH" env.sh $login:. + $RSYNC -e "$RSH" env.sh $login:. # And the helper tools, including this script. # Note that only provisioned hosts will have this in HOME, since they use # the repository in provisioning. Permanent hosts don't keep it in HOME, # in order to avoid it getting stale, and will have it in the WORKSPACE # instead, synced separately below. - if [ -d $HOME/mender-qa ] + if [ -d $HOME/buildscripts ] then - try_run $RSYNC -e "$RSH" $HOME/mender-qa $login:. + $RSYNC -e "$RSH" $HOME/buildscripts $login:. fi # Copy the workspace. If there is no workspace defined, we are not in the @@ -402,8 +382,8 @@ then echo "$WORKSPACE_REMOTE is not removed on build host." exit 2 fi - try_run $RSH $login mkdir -p "$WORKSPACE_REMOTE" - try_run $RSYNC -e "$RSH" "$WORKSPACE"/ $login:"$WORKSPACE_REMOTE"/ + $RSH $login mkdir -p "$WORKSPACE_REMOTE" + $RSYNC -e "$RSH" "$WORKSPACE"/ $login:"$WORKSPACE_REMOTE"/ fi # -------------------------------------------------------------------------- diff --git a/ci/initialize-user-data.sh b/ci/initialize-user-data.sh new file mode 100644 index 000000000..605251874 --- /dev/null +++ b/ci/initialize-user-data.sh @@ -0,0 +1,83 @@ +#!/bin/false + +# This file should be sourced, not run. + +# This script will do build slave setup, including creating credentials for the +# jenkins user, based on root's credentials (will copy its keys). The script is +# expected to be sourced early in the user-data phase after provisioning. + +# Make sure error detection and verbose output is on, if they aren't already. +set -x -e + +# Add jenkins user and copy credentials. +useradd -m -u 1010 jenkins || true +mkdir -p /home/jenkins/.ssh +# copy /root/.ssh/authorized_keys to /home/jenkins/.ssh, removing everything +# before 'ssh-rsa'. Some platforms have forcecommand='echo "root access disabled"' +# there. +sed 's/.*ssh-rsa/ssh-rsa/' /root/.ssh/authorized_keys >/home/jenkins/.ssh/authorized_keys || true + +# Enable sudo access for jenkins. +echo "jenkins ALL=(ALL:ALL) NOPASSWD:ALL" >> /etc/sudoers + +# Disable TTY requirement. +sed -i -e 's/^\( *Defaults *requiretty *\)$/# \1/' /etc/sudoers + +# Copy the buildscripts repository to jenkins user. +cp -r /root/buildscripts /home/jenkins +# was copying build-artifacts-cache known host entry +#cp /root/mender-qa/data/known_hosts /home/jenkins/.ssh/known_hosts + + +# add authorized_keys file before chowning, so that initialize-build-host.sh can manage +touch /home/jenkins/.ssh/authorized_keys + +# Make sure everything in jenkins' folder has right owner. +chown -R jenkins:jenkins /home/jenkins + +groupadd -r kvm || true # In case it already exists. +usermod -a -G kvm jenkins + +# change hostname to localhost +# it will fix sudo complaining "unable to resolve host digitalocean", +# and some tests +hostname localhost +# Ensure reverse hostname resolution is correct and 127.0.0.1 is always 'localhost'. +# There's no nice shell command to test it but this one: +# python -c 'import socket;print socket.gethostbyaddr("127.0.0.1")' +if test -f /etc/hosts; then + sed -i -e '1s/^/127.0.0.1 localhost localhost.localdomain\n/' /etc/hosts +else + echo '127.0.0.1 localhost localhost.localdomain' >/etc/hosts +fi + +apt_get() { + # Work around apt-get not waiting for a lock if it's taken. We want to wait + # for it instead of bailing out. No good return code to check unfortunately, + # so we just have to look inside the log. + + pid=$$ + # Maximum five minute wait (30 * 10 seconds) + attempts=30 + + while true + do + ( /usr/bin/apt-get "$@" 2>&1 ; echo $? > /tmp/apt-get-return-code.$pid.txt ) | tee /tmp/apt-get.$pid.log + if [ $attempts -gt 0 ] && \ + [ "$(cat /tmp/apt-get-return-code.$pid.txt)" -ne 0 ] && \ + fgrep "Could not get lock" /tmp/apt-get.$pid.log > /dev/null + then + attempts=$(expr $attempts - 1 || true) + sleep 10 + else + break + fi + done + + ret_code=$(cat /tmp/apt-get-return-code.$pid.txt) + rm -f /tmp/apt-get-return-code.$pid.txt /tmp/apt-get.$pid.log + + return $ret_code +} +alias apt=apt_get +alias apt-get=apt_get diff --git a/ci/lint.sh b/ci/lint.sh new file mode 100755 index 000000000..610bfcd7a --- /dev/null +++ b/ci/lint.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +shellcheck_dirs() { + grep -Erl '^(#!/(bin|usr/bin)/(env )?(sh|bash))' "$1" | while read -r file; do + shellcheck --external-sources --source-path=build-scripts "$file" + done +} + +shellcheck_dirs build-scripts/ + +# some dirs are "dirty" aka need some work so don't fail on those yet +shellcheck_dirs ci/ packaging/ || true + +cfengine format --check +cfengine lint --strict no ./ diff --git a/ci/linux-install-composer.sh b/ci/linux-install-composer.sh new file mode 100755 index 000000000..7f9fc8e56 --- /dev/null +++ b/ci/linux-install-composer.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +if command -v composer >/dev/null; then + exit +fi +VERSION="${COMPOSER_VERSION:-2.9.5}" +INSTALL_DIR="${COMPOSER_INSTALL_DIR:-/usr/local/bin}" +PHP_PATH="${PHP_BIN:-php}" +INSTALLER="composer-installer.php" + +trap 'rm -f "$INSTALLER"' EXIT + +curl -fsSL https://getcomposer.org/installer -o "$INSTALLER" + +# Verify checksum +EXPECTED_SIG="$(curl -fsSL https://composer.github.io/installer.sig)" +ACTUAL_SIG="$("$PHP_PATH" -r "echo hash_file('sha384', '$INSTALLER');")" +if [[ "$ACTUAL_SIG" != "$EXPECTED_SIG" ]]; then + echo "Error: Composer installer checksum mismatch" >&2 + exit 1 +fi + +# Install Composer +"$PHP_PATH" "$INSTALLER" --install-dir="$INSTALL_DIR" --filename=composer --version="$VERSION" diff --git a/ci/linux-install-groovy.sh b/ci/linux-install-groovy.sh new file mode 100755 index 000000000..3858c76d0 --- /dev/null +++ b/ci/linux-install-groovy.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +groovy_version=5.0.3 +groovy_zip="apache-groovy-binary-$groovy_version.zip" +cd /opt +rm -rf apache-groovy-binary*zip +wget https://groovy.jfrog.io/artifactory/dist-release-local/groovy-zips/"$groovy_zip" +echo "9d711dcb1dea94df9119558365beb6ac2909a22e30b58ae31de8bcb0dcf33698" "$groovy_zip" > "$groovy_zip".sha256 +sha256sum -c "$groovy_zip".sha256 +unzip -o "$groovy_zip" +ln -sf /opt/groovy-"$groovy_version"/bin/groovy /usr/bin/ +echo "export JAVA_HOME=/usr" >> /etc/profile +. /etc/profile +groovy -v diff --git a/ci/linux-install-jdk.sh b/ci/linux-install-jdk.sh new file mode 100755 index 000000000..849afd737 --- /dev/null +++ b/ci/linux-install-jdk.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -e + +install_jdk() { + # install jdk "manually" + # depending on os, might want to do something like `apt remove default-jre openjdk-*-jre-*` + cd /opt + # in order to have a stable download we must use the latest-1 version as that is the most recent in the "archive" + baseurl=https://download.oracle.com/java/21/archive/ + major_version=21 + baseurl="https://download.oracle.com/java/${major_version}/archive/" + version=21.0.11 + if uname -m | grep aarch64; then + tarball=jdk-${version}_linux-aarch64_bin.tar.gz + # checksum from https://download.oracle.com/java/${major_version}/archive/jdk-${version}_linux-aarch64_bin.tar.gz.sha256 + sha=2ebe89cad767abba83fb0b8cedd2d2d9bcbf947315fde78f7263a57a24f43b96 + else + tarball=jdk-${version}_linux-x64_bin.tar.gz + # checksum from https://download.oracle.com/java/${major_version}/latest/jdk-${version}_linux-x64_bin.tar.gz.sha256 + sha=e1c25a83f9e2e374c93e0c29cc3d98a947621ae0fefa4a8d932951eb160c47c3 + fi + wget --quiet "$baseurl$tarball" + echo "$sha $tarball" | sha256sum --check - + tar xf "$tarball" + tee /etc/profile.d/jdk.sh << EOF +export JAVA_HOME="/opt/jdk-$version" +export PATH=\$PATH:\$JAVA_HOME/bin +EOF + chown -R root:root "/opt/jdk-$version" + chmod -R 755 "/opt/jdk-$version" + if command -v update-alternatives; then + update-alternatives --install /usr/bin/java java "/opt/jdk-$version/bin/java" 9999 + else + ln -s "/opt/jdk-$version/bin/java" /usr/bin/java + fi + cd - +} + +if command -v java; then + echo "java already installed, will skip install." + exit +fi + +# TODO check version +if [ "$(whoami)" = "root" ]; then + install_jdk +else + sudo bash -c install_jdk +fi diff --git a/ci/linux-install-node.sh b/ci/linux-install-node.sh new file mode 100755 index 000000000..930dde304 --- /dev/null +++ b/ci/linux-install-node.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +node_version=20.10.0 + +npm install compare-versions + +node -e "const cv = require('compare-versions'); process.exit(cv.compareVersions('$node_version', process.version));" +result=$? +echo "result: $result" +if [ "$result" != "1" ]; then + echo "node version is >= $node_version, no install will be performed." + exit +fi + +cd /opt +sudo curl -k -O https://unofficial-builds.nodejs.org/download/release/v$node_version/node-v$node_version-linux-x64-glibc-217.tar.xz +echo "dcfc5dfcdea4d0883bb35a4f2e09bc4745b6e689d88f4ad09d9ccc252024049b node-v$node_version-linux-x64-glibc-217.tar.xz" | sudo tee node-v$node_version-linux-x64-glibc-217.tar.xz.sha256 +sudo sha256sum --check node-v$node_version-linux-x64-glibc-217.tar.xz.sha256 +sudo tar xf node-v$node_version-linux-x64-glibc-217.tar.xz +sudo tee /etc/profile.d/nodejs.sh << EOF +export NODE_HOME=/opt/node-v$node_version-linux-x64-glibc-217 +export PATH=\$PATH:\$NODE_HOME/bin +EOF +sudo update-alternatives --install /usr/bin/node node /opt/node-v$node_version-linux-x64-glibc-217/bin/node 1 +sudo update-alternatives --install /usr/bin/npm npm /opt/node-v$node_version-linux-x64-glibc-217/bin/npm 1 +sudo update-alternatives --install /usr/bin/npx npx /opt/node-v$node_version-linux-x64-glibc-217/bin/npx 1 +sudo update-alternatives --install /usr/bin/corepack corepack /opt/node-v$node_version-linux-x64-glibc-217/bin/corepack 1 +cd /root + diff --git a/ci/linux-install-php.sh b/ci/linux-install-php.sh new file mode 100755 index 000000000..9012238ee --- /dev/null +++ b/ci/linux-install-php.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +set -e + +php_version=8.3.20 + +if command -v php; then + if php -r "version_compare(PHP_VERSION, \""$php_version"\") >= 0 || exit(1);"; then + echo "host has php version >= $php_version. will skip install." + exit + fi +fi + +# install PHP 8.3 from source +sudo apt-get -y install gcc g++ make pkg-config libxml2-dev libsqlite3-dev \ +libcurl4-openssl-dev libpng-dev libonig-dev libldap2-dev libzip-dev \ +zlib1g-dev libssl-dev + +cd /tmp +wget https://www.php.net/distributions/php-$php_version.tar.gz +tar xzf php-$php_version.tar.gz +cd php-$php_version + +# --enable-bcmath is needed by tecnickcom/tc-lib-barcode and tecnickcom/tc-lib-pdf +./configure --prefix=/usr/local --with-curl --enable-bcmath --enable-gd --enable-mbstring --with-ldap --with-openssl --with-zlib + +make -j"$(nproc)" +sudo make install + +php -v +# END of PHP installation diff --git a/ci/linux-install-protobuf.sh b/ci/linux-install-protobuf.sh new file mode 100755 index 000000000..c92cf58a5 --- /dev/null +++ b/ci/linux-install-protobuf.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -e + +install_protobuf() { + # Install the protoc compiler "manually" from the official prebuilt + # release, verifying the SHA256 checksum of the zip. protoc is needed to + # build the cargo-based leech2 dependency. + # + # The release archives do not ship .sha256 files, so the checksums below are + # computed by us (and refreshed by the dependency update script). + cd /opt + version=35.1 + baseurl="https://github.com/protocolbuffers/protobuf/releases/download/v${version}" + + if uname -m | grep aarch64; then + arch=linux-aarch_64 + # sha256sum of protoc-${version}-linux-aarch_64.zip + sha=01bf9d08808c7f96678b63f4bd8efa559bb4f83d5a7a270d5edaf507f9d5d9cf + else + arch=linux-x86_64 + # sha256sum of protoc-${version}-linux-x86_64.zip + sha=6930ebf62bd4ea607b98fff052596c6ee564b9835b4ce172c75a3f53ae9d91b7 + fi + + zipfile="protoc-${version}-${arch}.zip" + wget --quiet "$baseurl/$zipfile" + echo "$sha $zipfile" | sha256sum --check - + # Installs bin/protoc and include/ under /usr/local. + unzip -o "$zipfile" -d /usr/local + rm "$zipfile" + + chmod a+rx /usr/local/bin/protoc + cd - +} + +# Re-exec under sudo when not root (e.g. when sourced from fix-buildhost.sh as +# the build user). +if [ "$(id -u)" -ne 0 ]; then + exec sudo bash "$0" +fi +install_protobuf diff --git a/ci/linux-install-rust.sh b/ci/linux-install-rust.sh new file mode 100755 index 000000000..a3811a556 --- /dev/null +++ b/ci/linux-install-rust.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +set -e + +install_rust() { + # Install the Rust toolchain "manually" from the official standalone + # installers, verifying the SHA256 checksum of each tarball. This is the + # cargo-based build dependency needed to build leech2. + # + # We install the individual component tarballs (rustc, cargo, rust-std) + # rather than the combined "rust" archive: the combined one is ~360 MB and + # extracts to ~1.4 GB of docs/clippy/llvm-tools we never install. The build + # hosts are tight on disk, so we also delete each tarball and its extracted + # tree right after installing it to keep peak disk usage low. + # + # Linux builds are native (x86_64 packages are built on x86_64 hosts, + # aarch64 on aarch64 hosts), so we only install the host's own Linux std. + # Windows is the only cross-compilation target, and only on MinGW build + # hosts, so the caller passes "x86_64-pc-windows-gnu" as an argument there. + baseurl="https://static.rust-lang.org/dist" + version=1.97.1 + prefix=/opt/rust + extra_targets="$@" + + workdir="$(mktemp -d)" + trap 'rm -rf "$workdir"' EXIT + cd "$workdir" + + if uname -m | grep aarch64; then + host=aarch64-unknown-linux-gnu + # checksum from $baseurl/rustc-${version}-aarch64-unknown-linux-gnu.tar.gz.sha256 + rustc_sha=8d9f3095a3108e38232f123645e2a0368bca67c975f8b91088004f8ffd9a6d39 + # checksum from $baseurl/cargo-${version}-aarch64-unknown-linux-gnu.tar.gz.sha256 + cargo_sha=cafde243518b016f825d13a5f2ca4ffd7d80f5966dfbec3539cd092c7c4f9871 + else + host=x86_64-unknown-linux-gnu + # checksum from $baseurl/rustc-${version}-x86_64-unknown-linux-gnu.tar.gz.sha256 + rustc_sha=1c441e430c1cca49dff54a8d59c41038bf6f79f7b8756596cb2f36511a015eba + # checksum from $baseurl/cargo-${version}-x86_64-unknown-linux-gnu.tar.gz.sha256 + cargo_sha=90cbeb8adfe8ca0fcbe01a18bd9b99d3e5e98fd29c003032828ebf3ffba0e4ed + fi + + # rust-std checksums per target. These are host-architecture independent. + # checksum from $baseurl/rust-std-${version}-x86_64-unknown-linux-gnu.tar.gz.sha256 + std_x86_64_linux_sha=45b927ecf776b9645ca2ed5b287fc6814824a073c30c7c5d5c0ad4425295ecee + # checksum from $baseurl/rust-std-${version}-aarch64-unknown-linux-gnu.tar.gz.sha256 + std_aarch64_linux_sha=ceffa428fe573507418858d59add342b351730725f36f7679b251e8b68103dec + # checksum from $baseurl/rust-std-${version}-x86_64-pc-windows-gnu.tar.gz.sha256 + std_x86_64_windows_sha=0e240192a91dace872d72fd2a5ad21b4be5d278b0d0281d58013e11f908fb5b6 + + # Download, verify, extract and install a single component tarball, then + # remove both the tarball and its extracted tree before moving on. + install_component() { + name="$1" + sha="$2" + tarball="$name.tar.gz" + wget --quiet "$baseurl/$tarball" + echo "$sha $tarball" | sha256sum --check - + tar xf "$tarball" + rm "$tarball" + "$name/install.sh" --prefix="$prefix" + rm -rf "$name" + } + + # Install the rust-std for a given target triple, looking up its checksum. + install_std() { + case "$1" in + x86_64-unknown-linux-gnu) sha="$std_x86_64_linux_sha" ;; + aarch64-unknown-linux-gnu) sha="$std_aarch64_linux_sha" ;; + x86_64-pc-windows-gnu) sha="$std_x86_64_windows_sha" ;; + *) + echo "No pinned checksum for rust-std target '$1'" >&2 + exit 1 + ;; + esac + install_component "rust-std-${version}-$1" "$sha" + } + + install_component "rustc-${version}-${host}" "$rustc_sha" + install_component "cargo-${version}-${host}" "$cargo_sha" + + # The host's own native std, plus any cross-compilation targets requested. + install_std "$host" + for target in $extra_targets; do + install_std "$target" + done + + tee /etc/profile.d/rust.sh << EOF +export PATH=\$PATH:$prefix/bin +EOF + + chown -R root:root "$prefix" + # Make sure it's readable by the build user. + chmod -R a+rX "$prefix" +} + +# Re-exec under sudo when not root (e.g. when sourced from fix-buildhost.sh as +# the build user), preserving any cross-compilation target arguments. +if [ "$(id -u)" -ne 0 ]; then + exec sudo bash "$0" "$@" +fi +install_rust "$@" diff --git a/ci/patched-image.sh b/ci/patched-image.sh new file mode 100755 index 000000000..3eb91bafe --- /dev/null +++ b/ci/patched-image.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -ex + +platform=$1 +patched=$platform-patched +if buildah images --format '{{.Name}}:{{.Tag}}' | grep $patched; then + echo "container image $patched already exists. buildah rm $patched if you want to regenerate" + continue +fi +if ! buildah ps | grep $platform; then + buildah --name $platform from $platform +fi +buildah copy $platform . /buildscripts/ci +buildah run $platform /buildscripts/ci/distribution-patched.sh +#buildah tag $platform $platform-$(date +%F) +#buildah commit $platform $patched +#buildah rm $platform diff --git a/ci/platform-container-image.list b/ci/platform-container-image.list new file mode 100644 index 000000000..8aa2bc9f7 --- /dev/null +++ b/ci/platform-container-image.list @@ -0,0 +1,5 @@ +debian:11 +debian:12 +ubuntu:20.04 +ubuntu:22.04 +ubuntu:24.04 diff --git a/ci/quick-install-cfengine-enterprise.sh.sha256 b/ci/quick-install-cfengine-enterprise.sh.sha256 index c083b5243..600dbbb57 100644 --- a/ci/quick-install-cfengine-enterprise.sh.sha256 +++ b/ci/quick-install-cfengine-enterprise.sh.sha256 @@ -1 +1 @@ -a29732a4a6c89ddebce925aa213b460bd6273ff776b510a938a541ebfc49520a quick-install-cfengine-enterprise.sh +6dceb3b29976d86b60cb493f58633217db3ba1b4681f5c4277a6dc137c221b0c quick-install-cfengine-enterprise.sh diff --git a/ci/setup-bootstrap-host.sh b/ci/setup-bootstrap-host.sh new file mode 100755 index 000000000..dfd681ab7 --- /dev/null +++ b/ci/setup-bootstrap-host.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -e +thisdir="$(dirname "$0")" + +# current standard of debian-9 for bootstrap requires using archive.debian.org +rm -f /etc/apt/sources.list.d/*; echo 'deb http://archive.debian.org/debian/ stretch main contrib non-free' >/etc/apt/sources.list + +apt-get -qy update +apt-get -qy upgrade +apt-get -y install git autogen autoconf automake m4 make bison flex binutils libtool gcc g++ libc-dev \ + liblmdb-dev libpam0g-dev python libssl-dev libpcre3-dev psmisc curl jq unzip \ + librsync-dev \ + autoconf-archive \ + pigz parallel libpcre2-dev php-zip + +bash "$thisdir"/linux-install-php.sh + +bash "$thisdir"/linux-install-node.sh + +bash "$thisdir"/linux-install-composer.sh + +bash "$thisdir"/linux-install-jdk.sh diff --git a/ci/setup-cfengine-build-host.sh b/ci/setup-cfengine-build-host.sh index 0f4324451..795d5f567 100755 --- a/ci/setup-cfengine-build-host.sh +++ b/ci/setup-cfengine-build-host.sh @@ -1,146 +1,305 @@ #!/usr/bin/env bash shopt -s expand_aliases +thisdir="$(dirname "$0")" -# TODO get latest LTS dynamically -CFE_VERSION=3.21.4 +# handle env cfengine_role +if [ -n "$cfengine_role" ]; then + touch /etc/cfengine-"$cfengine_role".flag +fi # install needed packages and software for a build host -set -ex +set -e if [ "$(id -u)" != "0" ]; then - echo "$0 must be run as root" - exit 1 + echo "$0 must be run as root" + exit 1 fi ls -la /home/ +if ! id -u jenkins; then + useradd jenkins -p jenkins +fi +mkdir -p /home/jenkins + +# The following is copied from prepare-testmachine-chroot +CHROOT_ROOT=/home/jenkins/testmachine-chroot/ +fuser -k "$CHROOT_ROOT" >/dev/null 2>&1 || true +# Unmount the /proc filesystem if it was previously mounted inside the chroot. +umount "${CHROOT_ROOT}proc" >/dev/null 2>&1 || true + chown -R jenkins /home/jenkins -if [ -d /var/cfengine ]; then - echo "Error: CFEngine already installed on this host. Will not proceed trying to setup build host with CFEngine temporary install." - exit 1 -fi +# cleanup any previous runs cfengine-masterfiles tar balls +rm -rf cfengine-masterfiles* +function cleanup() { + set -e + set -x + [ -f /var/log/messages ] && tail /var/log/messages + command -v journalctl >/dev/null && journalctl | grep -P '(error|fail)' + if command -v apt >/dev/null 2>&1; then + # workaround for CFE-4544, remove scriptlets call systemctl even when systemctl is-system-running returns false + # Replace systemctl with a no-op stub that always succeeds. We can't + # symlink to /bin/echo (or /bin/true): on Ubuntu 26 coreutils is the + # single multi-call uutils binary that dispatches on argv[0], so it + # would fail with "coreutils: unknown program 'systemctl'". + rm -f /bin/systemctl + printf '#!/bin/sh\nexit 0\n' >/bin/systemctl + chmod +x /bin/systemctl + apt remove -y cfengine-nova || true + elif command -v yum >/dev/null 2>&1; then + yum erase -y cfengine-nova || true + elif command -v zypper >/dev/null 2>&1; then + zypper remove -y cfengine-nova || true + else + echo "No supported package manager to uninstall cfengine." + exit 1 + fi + echo "Cleaning up CFEngine install by moving to /var/bak.cfengine and /opt/bak.cfengine" + rm -rf /var/bak.cfengine + mv /var/cfengine /var/bak.cfengine 2>/dev/null || true + rm -rf /opt/bak.cfengine + mv /opt/cfengine /opt/bak.cfengine 2>/dev/null || true + mv /var/log/CFE* /var/bak.cfengine/ 2>/dev/null || true + mv /var/log/postgresql.log /var/bak.cfengine 2>/dev/null || true -function cleanup() -{ - set -ex - if command -v apt 2>/dev/null; then - # workaround for CFE-4544, remove scriptlets call systemctl even when systemctl is-system-running returns false - # Replace systemctl with a no-op stub that always succeeds. We can't - # symlink to /bin/echo (or /bin/true): on Ubuntu 26 coreutils is the - # single multi-call uutils binary that dispatches on argv[0], so it - # would fail with "coreutils: unknown program 'systemctl'". - rm -f /bin/systemctl - printf '#!/bin/sh\nexit 0\n' > /bin/systemctl - chmod +x /bin/systemctl - apt remove -y cfengine-nova || true - elif command -v yum 2>/dev/null; then - yum erase -y cfengine-nova || true - elif command -v zypper 2>/dev/null; then - zypper remove -y cfengine-nova || true - else - echo "No supported package manager to uninstall cfengine." - exit 1 - fi - echo "Ensuring CFEngine fully uninstalled/cleaned up" - rm -rf /var/cfengine /opt/cfengine /var/log/CFE* /var/log/postgresql.log || true - if command -v pkill; then - pkill -9 cf-agent || true - pkill -9 cf-serverd || true - pkill -9 cf-monitord || true - pkill -9 cf-execd || true - else - echo "No pkill available. Maybe some cf procs left over?" - ps -efl | grep cf - fi - ls -l /home - chown -R jenkins /home/jenkins + if command -v pkill >/dev/null 2>&1; then + pkill -9 cf-agent || true + pkill -9 cf-serverd || true + pkill -9 cf-monitord || true + pkill -9 cf-execd || true + else + echo "No pkill available. Maybe some cf procs left over?" + # pgrep suggestion is ignored due to wanting POSIX compatability + # shellcheck disable=SC2009 + ps -efl | grep cf + fi + ls -l /home + chown -R jenkins /home/jenkins + echo "Done with cleanup()" } trap cleanup ERR trap cleanup SIGINT trap cleanup SIGTERM - echo "Using buildscripts commit:" # we have very old platforms with old git that doesn't understand -C option so cd/cd .. it is -cd buildscripts -# buildscripts is owned by jenkins so in order to run rev-parse command as root (this script is run with sudo) we must make it safe -git config --global --add safe.directory /home/jenkins/buildscripts -git rev-parse HEAD -cd .. +( + cd "$thisdir"/.. + # buildscripts is owned by jenkins so in order to run rev-parse command as root (this script is run with sudo) we must make it safe if git is used + if [ -d /home/jenkins/buildscripts/.git ]; then + if command -v git >/dev/null; then + git config --global --add safe.directory /home/jenkins/buildscripts + # show what version of buildscripts we are using + git rev-parse HEAD + else + echo "buildscripts/.git is present but git is not installed" + exit 1 + fi + fi +) -echo "Install any distribution upgrades" +echo "Install distribution upgrades and set software alias for platform" if [ -f /etc/os-release ]; then - if grep rhel /etc/os-release; then - yum upgrade --assumeyes - elif grep debian /etc/os-release; then - DEBIAN_FRONTEND=noninteractive apt upgrade --yes && DEBIAN_FRONTEND=noninteractive apt autoremove --yes - elif grep suse /etc/os-release; then - zypper -n update - else - echo "Unknown platform ID $ID. Need this information in order to update/upgrade distribution packages." - exit 1 - fi + if grep -q rhel /etc/os-release; then + yum update --assumeyes + alias software='yum install --assumeyes' + alias erase-packages='yum erase --assumeyes' + elif grep -q debian /etc/os-release; then + DEBIAN_FRONTEND=noninteractive apt update + + # sometimes the /boot partition is too small to handle kernel upgrade regenerations of initrd and related files on ubuntu, so allow failure first + DEBIAN_FRONTEND=noninteractive apt upgrade --yes || true + DEBIAN_FRONTEND=noninteractive apt autoremove --yes + + # and now perform the upgrade a second time after hopefully autoremove cleans up /boot partition of kernel files that cause failure + DEBIAN_FRONTEND=noninteractive apt upgrade --yes + DEBIAN_FRONTEND=noninteractive apt autoremove --yes + alias software='DEBIAN_FRONTEND=noninteractive apt install --yes' + alias erase-packages='DEBIAN_FRONTEND=noninteractive apt purge --yes' + elif grep -q suse /etc/os-release; then + zypper -n update + alias software='zypper install -y' + alias erase-packages='zypper uninstall -y' + else + echo "Unknown platform ID $ID. Need this information in order to update/upgrade distribution packages." + exit 1 + fi elif [ -f /etc/redhat-release ]; then - yum upgrade --assumeyes + yum update --assumeyes + alias software='yum install --assumeyes' else - echo "No /etc/os-release or /etc/redhat-release so cant determine platform." - exit 1 + echo "No /etc/os-release or /etc/redhat-release so cant determine platform." + exit 1 fi -if command -v wget; then - alias urlget=wget -elif command -v curl; then - alias urlget='curl -O' +if command -v wget >/dev/null 2>&1; then + alias urlget=wget +elif command -v curl >/dev/null 2>&1; then + alias urlget='curl -O' else - echo "Error: need something to fetch URLs. Didn't find either wget or curl." - exit 1 -fi -if grep -i suse /etc/os-release; then - # need to add our public key first otherwise zypper install fails - rpm --import https://cfengine-package-repos.s3.amazonaws.com/pub/gpg.key - if grep 'VERSION.*12' /etc/os-release; then - urlget https://cfengine-package-repos.s3.amazonaws.com/enterprise/Enterprise-"$CFE_VERSION"/agent/agent_suse12_x86_64/cfengine-nova-"$CFE_VERSION"-1.suse12.x86_64.rpm - zypper install -y cfengine-nova-"$CFE_VERSION"-1.suse12.x86_64.rpm - elif grep 'VERSION.*15' /etc/os-release; then - urlget https://cfengine-package-repos.s3.amazonaws.com/enterprise/Enterprise-"$CFE_VERSION"/agent/agent_suse15_x86_64/cfengine-nova-"$CFE_VERSION"-1.suse15.x86_64.rpm - zypper install -y cfengine-nova-"$CFE_VERSION"-1.suse15.x86_64.rpm - else - echo "Unsupported suse version:" - grep VERSION /etc/os-release + echo "Error: need something to fetch URLs. Didn't find either wget or curl." exit 1 - fi -else - urlget https://s3.amazonaws.com/cfengine.packages/quick-install-cfengine-enterprise.sh - # log sha256 checksum expected and actuall for debugging purposes - echo "Expected quick install checksum: " - cat ./buildscripts/ci/quick-install-cfengine-enterprise.sh.sha256 - echo "Actual quick install checksum: " - sha256sum quick-install-cfengine-enterprise.sh - - sha256sum --check ./buildscripts/ci/quick-install-cfengine-enterprise.sh.sha256 - chmod +x quick-install-cfengine-enterprise.sh - export CFEngine_Enterprise_Package_Version="$CFE_VERSION" - bash ./quick-install-cfengine-enterprise.sh agent fi -# get masterfiles -urlget https://cfengine-package-repos.s3.amazonaws.com/enterprise/Enterprise-"$CFE_VERSION"/misc/cfengine-masterfiles-"$CFE_VERSION"-1.pkg.tar.gz +if grep -q 6.10 /etc/issue 2>/dev/null; then + # special case of centos-6, cf-remote depends on urllib3 which depends on openssl 1.1.1+ that is not available + # generally we rely on cf-remote to install cfengine-nova and download masterfiles so here we must provide for both of those + echo "Found CentOS 6.10 so installing via hard-coded package URL..." -sha256sum --check ./buildscripts/ci/cfengine-masterfiles-"$CFE_VERSION"-1.pkg.tar.gz.sha256 + if [ ! -x /var/cfengine/bin/cf-agent ]; then + rm -rf cfengine-nova*rpm + urlget https://cfengine-package-repos.s3.amazonaws.com/enterprise/Enterprise-3.24.3/agent/agent_rhel6_x86_64/cfengine-nova-3.24.3-1.el6.x86_64.rpm + rpm -i cfengine-nova-3.24.3-1.el6.x86_64.rpm + fi + urlget https://cfengine-package-repos.s3.amazonaws.com/enterprise/Enterprise-3.24.3/misc/cfengine-masterfiles-3.24.3-1.pkg.tar.gz +fi + +# Here we start replacing the use of CFEngine policy with scripts. See ENT-14330 +if [ -f /etc/cfengine-bootstrap-pr-host.flag ]; then + "$thisdir"/setup-bootstrap-host.sh + exit +fi -tar xf cfengine-masterfiles-"$CFE_VERSION"-1.pkg.tar.gz +if [ -f /etc/cfengine-containers-host.flag ]; then + "$thisdir"/setup-ci-host.sh + exit +fi + +if grep -q ubuntu /etc/os-release; then + if grep -qi version=\"16 /etc/os-release; then + urlget https://cfengine-package-repos.s3.amazonaws.com/enterprise/Enterprise-3.21.8/agent/agent_ubuntu16_x86_64/cfengine-nova_3.21.8-1.ubuntu16_amd64.deb + dpkg -i cfengine-nova_3.21.8-1.ubuntu16_amd64.deb + urlget https://cfengine-package-repos.s3.amazonaws.com/enterprise/Enterprise-3.21.8/misc/cfengine-masterfiles-3.21.8-1.pkg.tar.gz + fi +fi + +if grep -q suse /etc/os-release; then + urlget https://cfengine-package-repos.s3.amazonaws.com/pub/gpg.key + rpm --import gpg.key + + if grep -qi version=\"12 /etc/os-release; then + echo "SUSE-12 found, cf-remote cannot be installed here so download directly similar to CentOS-6." + if [ ! -x /var/cfengine/bin/cf-agent ]; then + rm -rf cfengine-nova*rpm + urlget https://cfengine-package-repos.s3.amazonaws.com/enterprise/Enterprise-3.24.3/agent/agent_suse12_x86_64/cfengine-nova-3.24.3-1.suse12.x86_64.rpm + zypper in -y cfengine-nova-3.24.3-1.suse12.x86_64.rpm + fi + urlget https://cfengine-package-repos.s3.amazonaws.com/enterprise/Enterprise-3.24.3/misc/cfengine-masterfiles-3.24.3-1.pkg.tar.gz + fi +fi + +if [ ! -x /var/cfengine/bin/cf-agent ]; then + if ! ls cfengine-masterfiles*tar.gz >/dev/null 2>&1; then + echo "Installing cf-remote for possible package install and masterfiles download" + # try pipx first for debian as pip won't work. + # If that fails to install CFEngine then try python3-pip for redhats. + PIP="" + software python3-venv || true # on ubuntu-20 this is needed, debian-12 it is not but won't hurt + if software pipx; then + PIP=pipx + export PATH=$HOME/.local/bin:$PATH + elif software python3-pip; then + if command -v pip >/dev/null 2>&1; then + PIP=pip + elif command -v pip3 >/dev/null 2>&1; then + PIP=pip3 + fi + elif software python-pip; then + if command -v pip >/dev/null 2>&1; then + PIP=pip + fi + else + echo "Tried installing pipx, python3-pip and python-pip, none of which resulted in pipx, pip3 or pip being available. Exiting." + exit 23 + fi + export PATH=/usr/local/bin:$PATH # some pip/pipx use /usr/local/bin + + if [ "$PIP" = "pipx" ]; then + $PIP uninstall cf-remote || true # no -y option in pipx + else + $PIP uninstall -y cf-remote || true + fi # just in case a previous is there and would cause the install to fail + $PIP install cf-remote || true # if this fails we will try to install from source + fi # no masterfiles downloaded +fi # no cf-agent installed + +echo "Checking for pre-installed CFEngine (chicken/egg problem)" +# We need a cf-agent to run build host setup policy and redhat-10-arm did not have a previous package to install. +if ! /var/cfengine/bin/cf-agent -V 2>/dev/null; then + echo "No existing CFEngine install found, try cf-remote..." + if grep -qi stretch /etc/os-release || grep -qi buster /etc/os-release || grep -qi bionic /etc/os-release; then + _VERSION="--version 3.21.8" # 3.27.0 and 3.24.x do not have debian 9 (stretch) or debian 10 (buster) or ubuntu 18 (bionic) + elif grep -qi bullseye /etc/os-release; then + _VERSION="--version 3.24.3" # 3.27.0 has only debian > 11 (bullseye) + elif grep -q suse /etc/os-release; then + # here we must use 3.24.2 instead of 3.24.3 because 3.24.3 has libcurl 4 which depends on unavailable OPENSSL_3.2.0 + _VERSION="--version 3.24.2" # we removed suse platforms in 3.27.0 + else + _VERSION="" + fi + + # ENT-14373, migrate any cf-remote cache/config files to avoid dirs_exist_ok problems on old pythons + if [ -d "$HOME"/.cfengine/cf-remote/json ]; then + mkdir -p "$HOME"/.cache/cfengine/cf-remote + mv "$HOME"/.cfengine/cf-remote/json "$HOME"/.cache/cfengine/cf-remote/ + fi + if [ -d "$HOME"/.cfengine/cf-remote/packages ]; then + mkdir -p "$HOME"/.cache/cfengine/cf-remote + mv "$HOME"/.cfengine/cf-remote/packages "$HOME"/.cache/cfengine/cf-remote/ + fi + if [ -d "$HOME"/.cfengine/cf-remote ]; then + mkdir -p "$HOME"/.config/cfengine/cf-remote + mv "$HOME"/.cfengine/cf-remote/* "$HOME"/.config/cfengine/cf-remote/ + fi + + + erase-packages cfbuild-* || true # in case a dirty build was left on a long-living build host + + # We are passing a two-token string and need it to stay two tokens for proper argument parsing in $_VERSION + # shellcheck disable=SC2086 + cf-remote --log-level info $_VERSION install --clients localhost || true +fi + +if [ ! -x /var/cfengine/bin/cf-agent ]; then + [ -f /var/log/CFEngine-Install.log ] && tail /var/log/CFEngine-Install.log + echo "cf-remote didn't install CFEngine, build from source..." + software git + echo "cf-remote didn't install cf-agent, try from source" + rm -rf core # just in case we are repeating the script + git clone --recursive --depth 1 https://github.com/cfengine/core + ( + cd core + ./ci/install.sh + ) +fi + +# download masterfiles if not already present (such as in case of centos-6 above, hard-coded 3.24.3 download) +if ! ls cfengine-masterfiles*gz >/dev/null 2>&1; then + # if we are using a CFEngine pre-installed (chicken/egg) image we would skip cf-remote install so need to download directly + if ! command -v cf-remote >/dev/null 2>&1; then + urlget https://cfengine-package-repos.s3.amazonaws.com/enterprise/Enterprise-3.27.0/misc/cfengine-masterfiles-3.27.0-1.pkg.tar.gz + else + cf-remote download masterfiles --output-dir . + fi +fi +tar xf cfengine-masterfiles-*tar.gz cp -a masterfiles/* /var/cfengine/inputs/ # run three times to ensure all is done -policy="$(dirname "$0")"/cfengine-build-host-setup.cf -# just to be sure, make policy read/write for our user only to avoid errors when running -chmod 600 "$policy" -/var/cfengine/bin/cf-agent -KIf "$policy" -b cfengine_build_host_setup | tee promises.log -grep -i error: promises.log && exit 1 -/var/cfengine/bin/cf-agent -KIf "$policy" -b cfengine_build_host_setup | tee -a promises.log -grep -i error: promises.log && exit 1 -/var/cfengine/bin/cf-agent -KIf "$policy" -b cfengine_build_host_setup | tee -a promises.log -grep -i error: promises.log && exit 1 +( + cd "$thisdir" + policy=./cfengine-build-host-setup.cf + # just to be sure, make policy read/write for our user only to avoid errors when running + chmod 600 "$policy" + /var/cfengine/bin/cf-agent -KIf "$policy" -b cfengine_build_host_setup | tee promises.log + grep -i error: promises.log && exit 1 + /var/cfengine/bin/cf-agent -KIf "$policy" -b cfengine_build_host_setup | tee -a promises.log + grep -i error: promises.log && exit 1 + /var/cfengine/bin/cf-agent -KIf "$policy" -b cfengine_build_host_setup | tee -a promises.log + grep -i error: promises.log && exit 1 + echo "Done evaluating policy. End of promises.log:" + tail promises.log +) cleanup diff --git a/ci/setup-ci-host.sh b/ci/setup-ci-host.sh new file mode 100755 index 000000000..c88f5d7a3 --- /dev/null +++ b/ci/setup-ci-host.sh @@ -0,0 +1,207 @@ +#!/usr/bin/env bash +set -e +shopt -s expand_aliases +thisdir="$(dirname "$0")" + +packages="" # a space separated list of packages to install +function add-pkg() { + packages+=" $*" +} + +# we setup some vars for platform versions to make it easier to make choice later +# default version is 0 so that a check can be [ "$debian" -gt "12" ] and that will skip non-debians and such +redhat=0 +debian=0 +ubuntu=0 +suse=0 +# shellcheck disable=SC2034 +solaris=0 +# shellcheck disable=SC2034 +hpux=0 +# shellcheck disable=SC2034 +aix=0 + +if [ -f /etc/os-release ]; then + source /etc/os-release + if grep -q rhel /etc/os-release; then + yum update --assumeyes + alias packages='yum install --assumeyes' + redhat="$VERSION_ID" + elif grep -q debian /etc/os-release; then + alias packages='DEBIAN_FRONTEND=noninteractive apt install --yes' + debian="$VERSION_ID" + elif grep -q suse /etc/os-release; then + alias packages='zypper install -y' + # shellcheck disable=SC2034 + suse="$VERSION_ID" + else + echo "Unknown platform ID $ID. Need this information in order to update/upgrade distribution packages." + exit 1 + fi +elif [ -f /etc/redhat-release ]; then + alias packages='yum install --assumeyes' + # shellcheck disable=SC1091 + source /etc/redhat-release + redhat="$VERSION_ID" +else + echo "No /etc/os-release or /etc/redhat-release so cant determine platform." + exit 1 +fi + +if [ -f /etc/cfengine-containers-host.flag ]; then + if [ "$debian" -ge "12" ]; then + # in jenkins, CONTAINER labeled nodes are capable of running container builds like valgrind-check and static-check + add-pkg unzip # linux-install-groovy.sh needs unzip to unpack the groovy distribution archive + add-pkg buildah + add-pkg jq + add-pkg make + add-pkg parallel + add-pkg podman + if ! command -v groovy; then + bash "$thisdir"/linux-install-groovy.sh + fi + + # NOPASSWD is needed for various tools related to container jobs + rm -rf /etc/sudoers.d/999-local + cat >/etc/sudoers.d/999-local </dev/null; then + if [ ! -f /etc/cfengine-containers-host.flag ]; then + sysctl kernel.core_pattern='|/lib/systemd/systemd-coredump %p %u %g %s %t %e' + fi +fi + +"$thisdir"/linux-install-jdk.sh # the script should skip if sufficient java is already installed + +# leech2 build toolchain host +if [ "$ubuntu" -ge 20 ] || [ "$debian" -ge 12 ] || [ "$redhat" -ge 7 ]; then + "$thisdir"/linux-install-protobuf.sh + # TODO if mingw then pass along x86_64-pc-windows-gnu as an arg to install rust + "$thisdir"/linux-install-rust.sh +fi + +if [ "$redhat" -ge 7 ]; then + yum groups install -y 'Development Tools' +fi + +if [ "$redhat" = 8 ]; then + sudo rpm -iv https://kojipkgs.fedoraproject.org//packages/fakeroot/1.23/1.fc29/x86_64/fakeroot-1.23-1.fc29.x86_64.rpm https://kojipkgs.fedoraproject.org//packages/fakeroot/1.23/1.fc29/x86_64/fakeroot-libs-1.23-1.fc29.x86_64.rpm +fi + +if [ "$redhat" -gt 7 ]; then + yum install --assumeyes https://dl.fedoraproject.org/pub/epel/epel-release-latest-"$redhat".noarch.rpm +fi From a02195d61416efdc66d63f45e934190983fa1aaa Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Sun, 26 Jul 2026 10:17:02 -0400 Subject: [PATCH 2/4] Fixed unpack-tarballs handling of masterfiles tarballs The symlink step was failing due to the tarball itself matching the glob Ticket: none Changelog: none (cherry picked from commit f3c42b256df4e7c4f9434c2dc497526b516f28f3) Conflicts: build-scripts/unpack-tarballs major refactors in build-scripts in master/3.27 are not here so adjusted accordingly --- build-scripts/unpack-tarballs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/build-scripts/unpack-tarballs b/build-scripts/unpack-tarballs index 7f39ae4d8..c12a3a8e8 100755 --- a/build-scripts/unpack-tarballs +++ b/build-scripts/unpack-tarballs @@ -36,6 +36,8 @@ cd $BASEDIR gzip -dc $SOURCE_TARBALL | tar -xf - ln -s cfengine-3* core -echo "UNPACKING MASTERFILES TARBALL AND SYMLINKING masterfiles/" +echo "UNPACKING MASTERFILES TARBALL: $MASTERFILES_TARBALL AND SYMLINKING masterfiles/" gzip -dc $MASTERFILES_TARBALL | tar -xf - -ln -s cfengine-masterfiles-* masterfiles +# the masterfiles tarball would match cfengine-masterfiles-* so find the unpacked dir specifically +_cfmpfdir=$(find -type d -name 'cfengine-masterfiles-*') +ln -s "$_cfmpfdir" masterfiles From facd15c3aa34d4900585d325dfe2c334de3db627 Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Tue, 28 Jul 2026 13:19:47 -0500 Subject: [PATCH 3/4] Fixed unpack-tarballs to use more portable find command arguments Some find commands require a path to search in. Ticket: ENT-11241 Changelog: none --- build-scripts/unpack-tarballs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-scripts/unpack-tarballs b/build-scripts/unpack-tarballs index c12a3a8e8..35c110c20 100755 --- a/build-scripts/unpack-tarballs +++ b/build-scripts/unpack-tarballs @@ -39,5 +39,5 @@ ln -s cfengine-3* core echo "UNPACKING MASTERFILES TARBALL: $MASTERFILES_TARBALL AND SYMLINKING masterfiles/" gzip -dc $MASTERFILES_TARBALL | tar -xf - # the masterfiles tarball would match cfengine-masterfiles-* so find the unpacked dir specifically -_cfmpfdir=$(find -type d -name 'cfengine-masterfiles-*') +_cfmpfdir=$(find . -type d -name 'cfengine-masterfiles-*') ln -s "$_cfmpfdir" masterfiles From 29e2e7520d282602576cb33fd7bc30c361e63aa2 Mon Sep 17 00:00:00 2001 From: Craig Comstock Date: Tue, 28 Jul 2026 13:14:03 -0500 Subject: [PATCH 4/4] Fixed ci scripts to work properly on proxied hosts $(pwd) was the jenkins workspace directory which does not have proxy-target.txt so revert back to $HOME rhel/centos-7 need centos-7-setup-devtoolset-11.sh and also realpath to find the right place to source. Limit that to hosts which have realpath and fail fast on rhel/centos-7 hosts that do not have realpath available. Ticket: ENT-11241 Changelog: none (cherry picked from commit dbc1efbc9ee19c6bd30a421373fdf1116ea351da) --- ci/fix-buildhost.sh | 11 ++++++++--- ci/initialize-build-host.sh | 10 +++++----- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/ci/fix-buildhost.sh b/ci/fix-buildhost.sh index 3077104e5..520dd19fb 100755 --- a/ci/fix-buildhost.sh +++ b/ci/fix-buildhost.sh @@ -1,13 +1,18 @@ #!/usr/bin/env bash # it is expected that this file is sourced, not executed directly set -ex -my_path="$(realpath "${BASH_SOURCE[0]}")" -my_dir="$(dirname "$my_path")" if [ -f /etc/os-release ]; then source /etc/os-release if [ "$ID" = "centos" ] && [ "$VERSION_ID" = "7" ]; then - source "$my_dir"/centos-7-setup-devtoolset-11.sh + if command -v realpath >/dev/null; then + my_path="$(realpath "${BASH_SOURCE[0]}")" + my_dir="$(dirname "$my_path")" + source "$my_dir"/centos-7-setup-devtoolset-11.sh + else + echo "FAIL: could not find realpath command on rhel/centos-7 to source needed centos-7-setup-devtoolset-11.sh" + exit 1 + fi fi fi diff --git a/ci/initialize-build-host.sh b/ci/initialize-build-host.sh index 7080d2dd4..eefbbc937 100644 --- a/ci/initialize-build-host.sh +++ b/ci/initialize-build-host.sh @@ -8,7 +8,7 @@ # serial execution of initialization steps. It will post the output when # finished, if any. # -# 2. If $(pwd)/proxy-target.txt exists, it means this is a proxy host, and the +# 2. If $HOME/proxy-target.txt exists, it means this is a proxy host, and the # real build machine is on the host specified by the login details inside # that file. If the file does not exist, we are on the build slave itself. # After figuring that stuff out, the script will run the rest of the original @@ -227,7 +227,7 @@ reset_nested_vm() { fi if [ ! -z "$login" ] then - ip=`sed 's/.*@//' $(pwd)/proxy-target.txt` + ip=`sed 's/.*@//' $HOME/proxy-target.txt` if sudo arp | grep -q $ip then sudo arp -d $ip @@ -235,7 +235,7 @@ reset_nested_vm() { fi # TODO, remove this, we don't need or use or test nested-vms $HOME/mender-qa/scripts/nested-vm.sh $HOME/*.qcow2 - login="`cat $(pwd)/proxy-target.txt`" + login="`cat $HOME/proxy-target.txt`" if $RSH $login true then echo "Nested VM is back up, it seems. Happily continuing!" @@ -277,13 +277,13 @@ reset_nested_vm() { fi } -if [ -f $(pwd)/proxy-target.txt ] +if [ -f $HOME/proxy-target.txt ] then # -------------------------------------------------------------------------- # Check target machine health. # -------------------------------------------------------------------------- - login="$(cat $(pwd)/proxy-target.txt)" + login="$(cat $HOME/proxy-target.txt)" if [ ! -z "$login" ] && $RSH $login true then