diff --git a/testcase/command-execute-chroot.sh b/testcase/command-execute-chroot.sh new file mode 100755 index 0000000000000000000000000000000000000000..e31bca11cbaadc2da1086299c33906e0ce5bc323 --- /dev/null +++ b/testcase/command-execute-chroot.sh @@ -0,0 +1,1058 @@ +#!/usr/bin/env bash +############################################################################### +# @用例ID: IJV5MW +# @用例名称: command-execute-chroot +# @用例级别: 3 +# @用例标签: +# @用例类型: 功能测试 +# @自动化: 1 +############################################################################### +set -u +set -o pipefail + +export LANG=C +export LANGUAGE=C +export LC_ALL=C + +[ -z "${TST_TS_TOPDIR:-}" ] && { + TST_TS_TOPDIR="$(realpath "$(dirname "$0")/..")" + export TST_TS_TOPDIR +} + +set +u +source "${TST_TS_TOPDIR}/tst_common/lib/common.sh" || exit 1 +set -u + +JOBS=1 +TIMEOUT_SEC=600 +INSTALL_TIMEOUT_SEC=300 +COMMAND_TIMEOUT_SEC=300 +SERVICE_TIMEOUT_SEC=120 +REMOVE_TIMEOUT_SEC=300 + +g_run_stamp="$(date '+%Y%m%d-%H%M%S')-$$" +g_log_root="${TST_TS_TOPDIR}/logs/chroot-command-execute-${g_run_stamp}" +g_result_file="${TST_TS_TOPDIR}/package-command-result-chroot-${g_run_stamp}.txt" +g_tmpdir="" +g_work_root="" +g_base_root="" +g_default_tc_sysdir="" +g_cleanup_failed=0 +g_cleaned=0 +g_worker_pids=() +g_main_child_pid="" +WORKER_PHASE_PID="" +SUDO=() + +usage() { + cat <<'EOF' +Usage: bash testcase/command-execute-chroot.sh [options] + +Options: + -j N Number of concurrent chroot workers, default: 1 + --timeout SEC Per-package total timeout, default: 600 + --install-timeout SEC Install stage timeout, default: 300 + --command-timeout SEC Command stage timeout, default: 300 + --service-timeout SEC Service stage timeout, default: 120 + --remove-timeout SEC Remove stage timeout, default: 300 + -h, --help Show this help +EOF +} + +die() { + err "$*" + return 1 +} + +is_positive_integer() { + case "$1" in + ''|*[!0-9]*) return 1 ;; + *) [ "$1" -gt 0 ] ;; + esac +} + +safe_name() { + local index="$1" + local name="$2" + local safe + + safe="$(printf '%s' "$name" | sed 's/[^A-Za-z0-9_.-]/_/g' | cut -c 1-120)" + [ -n "$safe" ] || safe="pkg" + printf '%04d-%s' "$index" "$safe" +} + +clean_field() { + printf '%s' "$1" | tr '\t\r\n' ' ' +} + +append_reason() { + local old_reason="$1" + local new_reason="$2" + + [ -n "$new_reason" ] || { + printf '%s' "$old_reason" + return 0 + } + if [ -z "$old_reason" ]; then + printf '%s' "$new_reason" + else + printf '%s; %s' "$old_reason" "$new_reason" + fi +} + +write_tsv_result() { + local output_file="$1" + local src_pkg="$2" + local bin_pkg="$3" + local install_status="$4" + local command_status="$5" + local service_status="$6" + local remove_status="$7" + local result_status="$8" + local reason="$9" + local log_file="${10}" + + reason="$(clean_field "$reason")" + log_file="$(clean_field "$log_file")" + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \ + "$src_pkg" "$bin_pkg" "$install_status" "$command_status" \ + "$service_status" "$remove_status" "$result_status" "$reason" \ + "$log_file" >"$output_file" +} + +parse_args() { + while [ "$#" -gt 0 ]; do + case "$1" in + -j) + [ "$#" -ge 2 ] || return 1 + JOBS="$2" + shift 2 + ;; + --timeout|--install-timeout|--command-timeout|--service-timeout|--remove-timeout) + [ "$#" -ge 2 ] || return 1 + case "$1" in + --timeout) TIMEOUT_SEC="$2" ;; + --install-timeout) INSTALL_TIMEOUT_SEC="$2" ;; + --command-timeout) COMMAND_TIMEOUT_SEC="$2" ;; + --service-timeout) SERVICE_TIMEOUT_SEC="$2" ;; + --remove-timeout) REMOVE_TIMEOUT_SEC="$2" ;; + esac + shift 2 + ;; + --timeout=*) TIMEOUT_SEC="${1#*=}"; shift ;; + --install-timeout=*) INSTALL_TIMEOUT_SEC="${1#*=}"; shift ;; + --command-timeout=*) COMMAND_TIMEOUT_SEC="${1#*=}"; shift ;; + --service-timeout=*) SERVICE_TIMEOUT_SEC="${1#*=}"; shift ;; + --remove-timeout=*) REMOVE_TIMEOUT_SEC="${1#*=}"; shift ;; + -h|--help) + usage + exit 0 + ;; + --) + shift + break + ;; + -*) + err "unknown option: $1" + return 1 + ;; + *) break ;; + esac + done + + [ "$#" -eq 3 ] || return 1 + PKG_LIST="$(realpath "$1" 2>/dev/null || true)" + DESKTOP_WHITELIST="$(realpath "$2" 2>/dev/null || true)" + SERVICE_PKG_LIST="$(realpath "$3" 2>/dev/null || true)" + export PKG_LIST DESKTOP_WHITELIST SERVICE_PKG_LIST +} + +_show_case_attr() { + echo "Test case: command-execute-chroot" +} + +_get_tst_result_file() { + g_default_tc_sysdir="$TST_TC_SYSDIR" + TST_RESULT_FILE="$g_log_root/report.result" + TST_TC_SYSDIR="$g_log_root/testcase/.tc.${TST_TC_PID}.sysdir" + ALLURE_DATA_DIR="$g_log_root/allure_data" + export TST_RESULT_FILE TST_TC_SYSDIR ALLURE_DATA_DIR + + mkdir -p "$TST_TC_SYSDIR" "$ALLURE_DATA_DIR" + _set_tcstat "$TST_INIT" + if [ ! -f "$TST_RESULT_FILE" ]; then + { + echo "TESTSUITE" + echo "suite-name: $TST_TS_NAME" + echo "suite-start-time: $(get_timestamp_ms)" + echo "suite-end-time:" + echo + } >"$TST_RESULT_FILE" + fi + echo "$TST_RESULT_FILE" +} + +path_is_safe() { + local path="$1" + local resolved + + [ -n "$path" ] || return 1 + resolved="$(realpath -m "$path")" || return 1 + case "$resolved" in + /var/tmp/yum-ci-pkg-test-chroot.*) return 0 ;; + *) return 1 ;; + esac +} + +unmount_tree() { + local root="$1" + local targets=() + local target + + path_is_safe "$root" || return 1 + mapfile -t targets < <(findmnt -rn -o TARGET 2>/dev/null | \ + awk -v r="${root}/" 'index($0, r) == 1 { print length($0) "\t" $0 }' | \ + sort -rn | cut -f2-) + for target in "${targets[@]}"; do + [ "$target" = "$root" ] && continue + "${SUDO[@]}" umount "$target" 2>/dev/null || true + done + + if findmnt -rn -o TARGET 2>/dev/null | awk -v r="${root}/" 'index($0, r) == 1 { found=1 } END { exit !found }'; then + sleep 1 + for target in "${targets[@]}"; do + [ "$target" = "$root" ] && continue + "${SUDO[@]}" umount "$target" 2>/dev/null || \ + "${SUDO[@]}" umount -l "$target" 2>/dev/null || true + done + fi + + ! findmnt -rn -o TARGET 2>/dev/null | awk -v r="${root}/" 'index($0, r) == 1 { found=1 } END { exit !found }' +} + +safe_remove_tree() { + local path="$1" + + path_is_safe "$path" || return 1 + if findmnt -rn -o TARGET 2>/dev/null | awk -v r="${path}/" 'index($0, r) == 1 { found=1 } END { exit !found }'; then + return 1 + fi + "${SUDO[@]}" rm -rf --one-file-system -- "$path" +} + +safe_remove_local_tree() { + local path="$1" + local resolved + + [ -n "$path" ] || return 0 + resolved="$(realpath -m "$path")" || return 1 + case "$resolved" in + /tmp/yum-ci-pkg-test-chroot.*|"${TST_TS_TOPDIR}"/logs/testcase/.tc.*.sysdir) ;; + *) return 1 ;; + esac + rm -rf -- "$resolved" +} + +cleanup_main() { + local pid + local worker_root + + [ "$g_cleaned" -eq 0 ] || return 0 + g_cleaned=1 + + if [ -n "$g_main_child_pid" ]; then + "${SUDO[@]}" kill -TERM -- "-$g_main_child_pid" >/dev/null 2>&1 || true + sleep 1 + "${SUDO[@]}" kill -KILL -- "-$g_main_child_pid" >/dev/null 2>&1 || true + wait "$g_main_child_pid" 2>/dev/null || true + g_main_child_pid="" + fi + for pid in "${g_worker_pids[@]}"; do + kill "$pid" 2>/dev/null || true + done + for pid in "${g_worker_pids[@]}"; do + wait "$pid" 2>/dev/null || true + done + + if [ -n "$g_work_root" ] && path_is_safe "$g_work_root"; then + while IFS= read -r worker_root; do + [ -n "$worker_root" ] || continue + unmount_tree "$worker_root" || g_cleanup_failed=1 + done < <(find "$g_work_root" -type d -name root -path '*/workers/*/root' 2>/dev/null || true) + if [ "$g_cleanup_failed" -eq 0 ]; then + safe_remove_tree "$g_work_root" || g_cleanup_failed=1 + fi + fi + + safe_remove_local_tree "$g_tmpdir" || g_cleanup_failed=1 + safe_remove_local_tree "$g_default_tc_sysdir" || g_cleanup_failed=1 + if [ "$g_cleanup_failed" -ne 0 ]; then + err "cleanup failed; inspect mounts under ${g_work_root:-unknown}" + return 1 + fi + return 0 +} + +run_tracked_command() { + local rc + + setsid "$@" & + g_main_child_pid=$! + wait "$g_main_child_pid" + rc=$? + g_main_child_pid="" + return "$rc" +} + +create_phase_runner() { + local output_file="$1" + + cat >"$output_file" <<'PHASE_RUNNER' +#!/usr/bin/env bash +set -u +set -o pipefail +export LANG=C +export LANGUAGE=C +export LC_ALL=C +export TST_TS_TOPDIR=/opt/yum-ci +ulimit -c 0 + +phase="${1:-}" +pkg_name="${2:-}" +log_dir="/var/log/yum-ci" +desktop_whitelist="/opt/yum-ci/desktop-whitelist" +service_pkg_list="/opt/yum-ci/service-pkg-list" + +msg() { printf '%s\n' "$*"; } +err() { printf 'ERROR: %s\n' "$*" >&2; } + +source /opt/yum-ci/tst_lib/ts_common.sh || exit 2 + +write_reason() { + printf '%s\n' "$2" >"$log_dir/$1.reason" +} + +pkg_base_name() { + local name + name="$(cat "$log_dir/binary-name" 2>/dev/null || true)" + if [ -z "$name" ]; then + name="$(rpm -q --qf '%{NAME}\n' "$pkg_name" 2>/dev/null | head -n 1 || true)" + fi + [ -n "$name" ] || name="$(printf '%s' "$pkg_name" | sed -E 's/\.(x86_64|aarch64|noarch|i[0-9]86)$//')" + printf '%s' "$name" +} + +list_has_exact_name() { + local list_file="$1" + local candidate="$2" + [ -s "$list_file" ] || return 1 + awk -v p="$candidate" ' + /^[[:space:]]*#/ || /^[[:space:]]*$/ { next } + { + line = $0 + sub(/^[[:space:]]+/, "", line) + sub(/[[:space:]]+$/, "", line) + if (line == p) found = 1 + } + END { exit(found ? 0 : 1) } + ' "$list_file" +} + +bus_error_in() { + grep -Eiq 'failed to connect to bus|system has not been booted with systemd|host is down|no such file or directory' "$1" 2>/dev/null +} + +systemd_available() { + command -v systemctl >/dev/null 2>&1 || return 1 + [ "$(cat /proc/1/comm 2>/dev/null)" = "systemd" ] || return 1 + systemctl --no-pager show --property=Version --value >"$log_dir/systemd-probe.log" 2>&1 + [ -s "$log_dir/systemd-probe.log" ] || return 1 + ! bus_error_in "$log_dir/systemd-probe.log" +} + +test_commands_safely() { + local command_list="$1" + local command_dir="$log_dir/command-logs" + local index=0 + local test_command + local help_rc + local version_rc + local rc + local file_output + local failed=0 + + mkdir -p "$command_dir" + : >"$log_dir/command-failures.list" + while IFS= read -r test_command; do + [ -n "$test_command" ] || continue + index=$((index + 1)) + : >"$command_dir/${index}.log" + + timeout -s SIGKILL 1 "$test_command" --help >"$command_dir/${index}.log" 2>&1 + help_rc=$? + [ "$help_rc" -eq 0 ] && continue + + timeout -s SIGKILL 1 "$test_command" --version >>"$command_dir/${index}.log" 2>&1 + version_rc=$? + [ "$version_rc" -eq 0 ] && continue + + for rc in "$help_rc" "$version_rc"; do + case "$rc" in + 132|133|134|135|136|138|139) + printf '%s\n' "$test_command" >>"$log_dir/command-failures.list" + failed=1 + continue 2 + ;; + esac + done + + if grep -qi 'error while loading shared libraries' "$command_dir/${index}.log"; then + printf '%s\n' "$test_command" >>"$log_dir/command-failures.list" + failed=1 + continue + fi + if ! realpath "$test_command" >/dev/null 2>&1; then + printf '%s\n' "$test_command" >>"$log_dir/command-failures.list" + failed=1 + continue + fi + file_output="$(file "$(realpath "$test_command")" 2>&1)" + printf '%s\n' "$file_output" >>"$command_dir/${index}.log" + if ! printf '%s\n' "$file_output" | grep -qw ELF; then + printf '%s\n' "$test_command" >>"$log_dir/command-failures.list" + failed=1 + continue + fi + timeout -s SIGKILL 1 strace -f -e trace=execve "$test_command" --help \ + >"$command_dir/${index}.strace" 2>&1 || true + if grep -Eq '^execve\(.*\) += 0$' "$command_dir/${index}.strace"; then + continue + fi + timeout -s SIGKILL 1 "$test_command" >>"$command_dir/${index}.log" 2>&1 + rc=$? + [ "$rc" -eq 0 ] && continue + case "$rc" in + 132|133|134|135|136|138|139) + printf '%s\n' "$test_command" >>"$log_dir/command-failures.list" + failed=1 + continue + ;; + esac + if grep -Eiqw 'invalid|argument|unknown|option|usage|version' "$command_dir/${index}.log" && \ + ! grep -Eiq 'segmentation fault|core dump|error while loading shared libraries' \ + "$command_dir/${index}.log"; then + continue + fi + + printf '%s\n' "$test_command" >>"$log_dir/command-failures.list" + failed=1 + done <"$command_list" + return "$failed" +} + +install_phase() { + if rpm -q "$pkg_name" >/dev/null 2>&1; then + echo 1 >"$log_dir/preinstalled" + else + echo 0 >"$log_dir/preinstalled" + fi + + if test_single_rpm_install "" "$pkg_name" "$log_dir/install"; then + cp -f "$log_dir/install.install" "$log_dir/install.log" 2>/dev/null || : >"$log_dir/install.log" + if rpm -q "$pkg_name" >/dev/null 2>&1; then + rpm -q --qf '%{NAME}\n' "$pkg_name" 2>/dev/null | head -n 1 >"$log_dir/binary-name" + get_src_pkg_by_bin "$pkg_name" 2>/dev/null | head -n 1 >"$log_dir/source-package" + [ -s "$log_dir/source-package" ] || printf '%s\n' "$pkg_name" >"$log_dir/source-package" + write_reason install "install passed" + return 0 + fi + fi + + cp -f "$log_dir/install.install" "$log_dir/install.log" 2>/dev/null || : >"$log_dir/install.log" + get_src_pkg_by_bin "$pkg_name" 2>/dev/null | head -n 1 >"$log_dir/source-package" + [ -s "$log_dir/source-package" ] || printf '%s\n' "$pkg_name" >"$log_dir/source-package" + write_reason install "install failed" + return 1 +} + +command_phase() { + local base_name + base_name="$(pkg_base_name)" + : >"$log_dir/command.log" + + if list_has_exact_name "$desktop_whitelist" "$base_name"; then + write_reason command "desktop whitelist" + return 20 + fi + + rpm -ql "$pkg_name" 2>"$log_dir/rpm-ql-command.err" | \ + grep -E '^/usr/(s)?bin/' | grep -v '\.so' | grep -v '/usr/bin/nde-' \ + >"$log_dir/command-files.list" || true + : >"$log_dir/commands.list" + if [ -s "$log_dir/command-files.list" ]; then + get_elf_executable "$log_dir/command-files.list" "$log_dir/commands.list" \ + >>"$log_dir/command.log" 2>&1 || true + fi + if [ ! -s "$log_dir/commands.list" ]; then + write_reason command "no executable command" + return 20 + fi + if test_commands_safely "$log_dir/commands.list" >>"$log_dir/command.log" 2>&1; then + write_reason command "command passed" + return 0 + fi + write_reason command "command failed" + return 1 +} + +service_phase() { + local base_name + local unit_name + local operation_log + local skipped=0 + local failed=0 + + base_name="$(pkg_base_name)" + : >"$log_dir/service.log" + rpm -ql "$pkg_name" 2>"$log_dir/rpm-ql-service.err" | \ + grep -E '\.(service|socket|target)$' | awk -F/ '{print $NF}' | sort -u \ + >"$log_dir/systemd-units.list" || true + + if [ ! -s "$log_dir/systemd-units.list" ]; then + write_reason service "no systemd unit" + return 0 + fi + if list_has_exact_name "$service_pkg_list" "$base_name"; then + write_reason service "special service package requires configuration" + return 20 + fi + + : >"$log_dir/systemd-testable-units.list" + while IFS= read -r unit_name; do + [ -n "$unit_name" ] || continue + if printf '%s' "$unit_name" | grep -q '@\.'; then + printf 'SKIP template unit %s\n' "$unit_name" >>"$log_dir/service.log" + skipped=1 + else + printf '%s\n' "$unit_name" >>"$log_dir/systemd-testable-units.list" + fi + done <"$log_dir/systemd-units.list" + + if [ ! -s "$log_dir/systemd-testable-units.list" ]; then + write_reason service "template systemd unit" + return 20 + fi + if ! systemd_available; then + write_reason service "systemd is not available in plain chroot" + return 20 + fi + + while IFS= read -r unit_name; do + [ -n "$unit_name" ] || continue + operation_log="$log_dir/service-${unit_name}.log" + : >"$operation_log" + + if ! timeout -s SIGKILL 30 systemctl --no-pager start "$unit_name" >>"$operation_log" 2>&1; then + if bus_error_in "$operation_log"; then + skipped=1 + continue + fi + failed=1 + elif ! timeout -s SIGKILL 30 systemctl --no-pager is-active --quiet "$unit_name" >>"$operation_log" 2>&1; then + if bus_error_in "$operation_log"; then + skipped=1 + else + failed=1 + fi + fi + + if ! timeout -s SIGKILL 30 systemctl --no-pager stop "$unit_name" >>"$operation_log" 2>&1; then + if bus_error_in "$operation_log"; then + skipped=1 + elif ! grep -iq 'configured to refuse manual start/stop' "$operation_log"; then + failed=1 + fi + fi + cat "$operation_log" >>"$log_dir/service.log" + done <"$log_dir/systemd-testable-units.list" + + if [ "$failed" -ne 0 ]; then + write_reason service "service operation failed" + return 1 + fi + if [ "$skipped" -ne 0 ]; then + write_reason service "systemd unit skipped" + return 20 + fi + write_reason service "service passed" + return 0 +} + +remove_phase() { + local base_name + base_name="$(pkg_base_name)" + + if [ "$(cat "$log_dir/preinstalled" 2>/dev/null || echo 0)" = "1" ]; then + : >"$log_dir/remove.log" + write_reason remove "package present in base rootfs" + return 20 + fi + + if test_single_rpm_remove "$pkg_name" "$pkg_name" "$log_dir/remove" \ + /opt/yum-ci/base-installed.list; then + cp -f "$log_dir/remove.remove" "$log_dir/remove.log" 2>/dev/null || : >"$log_dir/remove.log" + if ! rpm -q "$base_name" >/dev/null 2>&1; then + write_reason remove "remove passed" + return 0 + fi + fi + cp -f "$log_dir/remove.remove" "$log_dir/remove.log" 2>/dev/null || : >"$log_dir/remove.log" + write_reason remove "remove failed" + return 1 +} + +case "$phase" in + install) install_phase ;; + command) command_phase ;; + service) service_phase ;; + remove) remove_phase ;; + *) err "unknown phase: $phase"; exit 2 ;; +esac +PHASE_RUNNER + chmod +x "$output_file" +} + +prepare_base_rootfs() { + local runner_file="$g_tmpdir/phase-runner" + local releasever + + releasever="$(. /etc/os-release; printf '%s' "${VERSION_ID%%.*}")" + mkdir -p "$g_base_root" + msg "preparing base rootfs: $g_base_root" + "${SUDO[@]}" mkdir -p "$g_base_root/etc/yum.repos.d" "$g_base_root/etc/dnf/vars" + "${SUDO[@]}" cp -a /etc/yum.repos.d/. "$g_base_root/etc/yum.repos.d/" + [ ! -f /etc/dnf/dnf.conf ] || \ + "${SUDO[@]}" cp -f /etc/dnf/dnf.conf "$g_base_root/etc/dnf/dnf.conf" + [ ! -f /etc/yum/vars/releasever ] || \ + "${SUDO[@]}" cp -f /etc/yum/vars/releasever "$g_base_root/etc/dnf/vars/releasever" + "${SUDO[@]}" cp -L /etc/resolv.conf "$g_base_root/etc/resolv.conf" + if ! run_tracked_command "${SUDO[@]}" dnf -y --installroot="$g_base_root" --releasever="$releasever" \ + --setopt=install_weak_deps=False install \ + opencloudos-release opencloudos-repos epol-release \ + bash coreutils dnf yum rpm findutils grep gawk sed file strace util-linux \ + procps-ng systemd >"$g_log_root/rootfs-build.log" 2>&1; then + err "base rootfs preparation failed: $g_log_root/rootfs-build.log" + return 1 + fi + + create_phase_runner "$runner_file" + "${SUDO[@]}" mkdir -p "$g_base_root/opt/yum-ci/tst_lib" "$g_base_root/var/log/yum-ci" \ + "$g_base_root/etc/yum.repos.d" + "${SUDO[@]}" cp -f "$runner_file" "$g_base_root/opt/yum-ci/phase-runner" + "${SUDO[@]}" cp -f "$TST_TS_TOPDIR/tst_lib/ts_common.sh" \ + "$g_base_root/opt/yum-ci/tst_lib/ts_common.sh" + "${SUDO[@]}" cp -f "$TST_TS_TOPDIR/tst_lib/other_common.sh" \ + "$g_base_root/opt/yum-ci/tst_lib/other_common.sh" + "${SUDO[@]}" cp -f "$DESKTOP_WHITELIST" "$g_base_root/opt/yum-ci/desktop-whitelist" + "${SUDO[@]}" cp -f "$SERVICE_PKG_LIST" "$g_base_root/opt/yum-ci/service-pkg-list" + "${SUDO[@]}" chmod 0755 "$g_base_root/opt/yum-ci/phase-runner" + run_tracked_command "${SUDO[@]}" chroot "$g_base_root" /usr/bin/env -i HOME=/root PATH=/usr/sbin:/usr/bin:/sbin:/bin \ + LANG=C LC_ALL=C yum list installed >"$g_tmpdir/base-installed.list" 2>"$g_log_root/rootfs-query.log" || return 1 + "${SUDO[@]}" cp -f "$g_tmpdir/base-installed.list" "$g_base_root/opt/yum-ci/base-installed.list" + run_tracked_command "${SUDO[@]}" chroot "$g_base_root" /usr/bin/env -i HOME=/root PATH=/usr/sbin:/usr/bin:/sbin:/bin \ + LANG=C LC_ALL=C dnf -q repolist --enabled >>"$g_log_root/rootfs-query.log" 2>&1 +} + +mount_worker_root() { + local root="$1" + local pkg_log_dir="$2" + + "${SUDO[@]}" mount -t proc -o nosuid,nodev,noexec proc "$root/proc" || return 1 + "${SUDO[@]}" mount -t sysfs -o ro,nosuid,nodev,noexec sysfs "$root/sys" || return 1 + "${SUDO[@]}" mount --rbind /dev "$root/dev" || return 1 + "${SUDO[@]}" mount --make-rslave "$root/dev" || return 1 + "${SUDO[@]}" mount -t tmpfs -o nosuid,nodev,mode=755,size=64m tmpfs "$root/run" || return 1 + "${SUDO[@]}" mount --bind "$pkg_log_dir" "$root/var/log/yum-ci" || return 1 +} + +phase_reason() { + local pkg_log_dir="$1" + local phase="$2" + local fallback="$3" + + if [ -s "$pkg_log_dir/$phase.reason" ]; then + head -n 1 "$pkg_log_dir/$phase.reason" + else + printf '%s' "$fallback" + fi +} + +run_chroot_phase() { + local root="$1" + local pkg_name="$2" + local phase="$3" + local configured_timeout="$4" + local deadline="$5" + local pkg_log_dir="$6" + local remaining + local effective_timeout + local rc + + remaining=$((deadline - SECONDS)) + [ "$remaining" -gt 0 ] || return 124 + effective_timeout="$configured_timeout" + [ "$remaining" -ge "$effective_timeout" ] || effective_timeout="$remaining" + + setsid "${SUDO[@]}" timeout --kill-after=10s "$effective_timeout" \ + chroot "$root" /usr/bin/env -i HOME=/root PATH=/usr/sbin:/usr/bin:/sbin:/bin \ + LANG=C LANGUAGE=C LC_ALL=C /opt/yum-ci/phase-runner "$phase" "$pkg_name" \ + >>"$pkg_log_dir/package.log" 2>&1 & + WORKER_PHASE_PID=$! + wait "$WORKER_PHASE_PID" + rc=$? + WORKER_PHASE_PID="" + return "$rc" +} + +status_from_phase_rc() { + case "$1" in + 0) printf PASS ;; + 20) printf SKIP ;; + *) printf FAIL ;; + esac +} + +overall_status() { + local install_status="$1" + local command_status="$2" + local service_status="$3" + local remove_status="$4" + + if [ "$install_status" = FAIL ] || [ "$command_status" = FAIL ] || \ + [ "$service_status" = FAIL ] || [ "$remove_status" = FAIL ]; then + printf FAIL + elif [ "$install_status" = SKIP ] || [ "$command_status" = SKIP ] || \ + [ "$service_status" = SKIP ] || [ "$remove_status" = SKIP ]; then + printf SKIP + else + printf PASS + fi +} + +run_one_package() { + local index="$1" + local pkg_name="$2" + local safe_pkg + local worker_dir + local root + local pkg_log_dir + local log_file + local deadline + local install_status=SKIP + local command_status=SKIP + local service_status=SKIP + local remove_status=SKIP + local result_status=FAIL + local source_pkg="$pkg_name" + local reason="" + local rc + local cleanup_failed=0 + + safe_pkg="$(safe_name "$index" "$pkg_name")" + worker_dir="$g_work_root/workers/$safe_pkg" + root="$worker_dir/root" + pkg_log_dir="$g_log_root/$safe_pkg" + log_file="${pkg_log_dir#"$TST_TS_TOPDIR/"}/package.log" + mkdir -p "$root" "$pkg_log_dir" + : >"$pkg_log_dir/package.log" + + worker_cleanup() { + if [ -n "$WORKER_PHASE_PID" ]; then + "${SUDO[@]}" kill -TERM -- "-$WORKER_PHASE_PID" >/dev/null 2>&1 || true + sleep 1 + "${SUDO[@]}" kill -KILL -- "-$WORKER_PHASE_PID" >/dev/null 2>&1 || true + wait "$WORKER_PHASE_PID" 2>/dev/null || true + WORKER_PHASE_PID="" + fi + unmount_tree "$root" || cleanup_failed=1 + if [ "$cleanup_failed" -eq 0 ]; then + safe_remove_tree "$worker_dir" || cleanup_failed=1 + fi + if [ "$cleanup_failed" -ne 0 ]; then + : >"$pkg_log_dir/cleanup-failed" + fi + } + trap worker_cleanup EXIT + trap 'exit 130' INT TERM + + if printf '%s\n' "$pkg_name" | grep -Eq '(^livepatch-kernel-|(-debuginfo|-debugsource)(\.|-|$))'; then + write_tsv_result "$pkg_log_dir/result.tsv" "$pkg_name" "$pkg_name" \ + SKIP SKIP SKIP SKIP SKIP "filtered package" "$log_file" + return 0 + fi + + if ! "${SUDO[@]}" cp -a --reflink=auto "$g_base_root/." "$root/" \ + >>"$pkg_log_dir/package.log" 2>&1; then + write_tsv_result "$pkg_log_dir/result.tsv" "$pkg_name" "$pkg_name" \ + FAIL SKIP SKIP SKIP FAIL "rootfs clone failed" "$log_file" + return 0 + fi + if ! mount_worker_root "$root" "$pkg_log_dir" >>"$pkg_log_dir/package.log" 2>&1; then + write_tsv_result "$pkg_log_dir/result.tsv" "$pkg_name" "$pkg_name" \ + FAIL SKIP SKIP SKIP FAIL "chroot mount failed" "$log_file" + return 0 + fi + + deadline=$((SECONDS + TIMEOUT_SEC)) + run_chroot_phase "$root" "$pkg_name" install "$INSTALL_TIMEOUT_SEC" "$deadline" "$pkg_log_dir" + rc=$? + install_status="$(status_from_phase_rc "$rc")" + if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then + reason="$(append_reason "$reason" "install timeout")" + else + reason="$(append_reason "$reason" "$(phase_reason "$pkg_log_dir" install "install failed")")" + fi + + if [ "$install_status" = PASS ]; then + source_pkg="$(head -n 1 "$pkg_log_dir/source-package" 2>/dev/null || printf '%s' "$pkg_name")" + + run_chroot_phase "$root" "$pkg_name" command "$COMMAND_TIMEOUT_SEC" "$deadline" "$pkg_log_dir" + rc=$? + command_status="$(status_from_phase_rc "$rc")" + if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then + reason="$(append_reason "$reason" "command timeout")" + else + reason="$(append_reason "$reason" "$(phase_reason "$pkg_log_dir" command "command failed")")" + fi + + run_chroot_phase "$root" "$pkg_name" service "$SERVICE_TIMEOUT_SEC" "$deadline" "$pkg_log_dir" + rc=$? + service_status="$(status_from_phase_rc "$rc")" + if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then + reason="$(append_reason "$reason" "service timeout")" + else + reason="$(append_reason "$reason" "$(phase_reason "$pkg_log_dir" service "service failed")")" + fi + + run_chroot_phase "$root" "$pkg_name" remove "$REMOVE_TIMEOUT_SEC" "$deadline" "$pkg_log_dir" + rc=$? + remove_status="$(status_from_phase_rc "$rc")" + if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then + reason="$(append_reason "$reason" "remove timeout")" + else + reason="$(append_reason "$reason" "$(phase_reason "$pkg_log_dir" remove "remove failed")")" + fi + else + reason="$(append_reason "$reason" "command skipped after install failure")" + reason="$(append_reason "$reason" "service skipped after install failure")" + reason="$(append_reason "$reason" "remove skipped after install failure")" + fi + + result_status="$(overall_status "$install_status" "$command_status" "$service_status" "$remove_status")" + write_tsv_result "$pkg_log_dir/result.tsv" "$source_pkg" "$pkg_name" \ + "$install_status" "$command_status" "$service_status" "$remove_status" \ + "$result_status" "$reason" "$log_file" +} + +wait_for_slot() { + while [ "$(jobs -rp | wc -l)" -ge "$JOBS" ]; do + wait -n || true + done +} + +wait_for_all_jobs() { + while [ "$(jobs -rp | wc -l)" -gt 0 ]; do + wait -n || true + done +} + +build_package_queue() { + local queue_file="$1" + + awk ' + /^[[:space:]]*$/ { next } + /^[[:space:]]*#/ { next } + $1 == "Installed" || $1 == "Available" { next } + { print $1 } + ' "$PKG_LIST" >"$queue_file" +} + +validate_result_row() { + local result_file="$1" + + [ -s "$result_file" ] || return 1 + awk -F '\t' ' + NR != 1 || NF != 9 { bad = 1 } + $3 !~ /^(PASS|FAIL|SKIP)$/ || $4 !~ /^(PASS|FAIL|SKIP)$/ || + $5 !~ /^(PASS|FAIL|SKIP)$/ || $6 !~ /^(PASS|FAIL|SKIP)$/ || + $7 !~ /^(PASS|FAIL|SKIP)$/ { bad = 1 } + END { exit(bad ? 1 : 0) } + ' "$result_file" +} + +merge_results() { + local queue_file="$1" + local data_file="$g_tmpdir/all-results.tsv" + local index=0 + local pkg_name + local safe_pkg + local pkg_log_dir + local result_file + local log_file + + : >"$data_file" + while IFS= read -r pkg_name; do + index=$((index + 1)) + safe_pkg="$(safe_name "$index" "$pkg_name")" + pkg_log_dir="$g_log_root/$safe_pkg" + result_file="$pkg_log_dir/result.tsv" + log_file="${pkg_log_dir#"$TST_TS_TOPDIR/"}/package.log" + + if [ -f "$pkg_log_dir/cleanup-failed" ]; then + write_tsv_result "$result_file" "$pkg_name" "$pkg_name" FAIL SKIP SKIP SKIP FAIL \ + "worker cleanup failed" "$log_file" + elif ! validate_result_row "$result_file"; then + write_tsv_result "$result_file" "$pkg_name" "$pkg_name" FAIL SKIP SKIP SKIP FAIL \ + "missing or invalid worker result" "$log_file" + fi + cat "$result_file" >>"$data_file" + done <"$queue_file" + + { + printf 'source_package\tbinary_package\tinstall\tcommand\tservice\tremove\tresult\treason\tlog_file\n' + cat "$data_file" + } >"$g_result_file" +} + +write_summary() { + local data_file="$g_tmpdir/all-results.tsv" + local expected_count="$1" + local total_count + local pass_count + local fail_count + local skip_count + + total_count="$(wc -l <"$data_file")" + pass_count="$(awk -F '\t' '$7 == "PASS" { n++ } END { print n + 0 }' "$data_file")" + fail_count="$(awk -F '\t' '$7 == "FAIL" { n++ } END { print n + 0 }' "$data_file")" + skip_count="$(awk -F '\t' '$7 == "SKIP" { n++ } END { print n + 0 }' "$data_file")" + + echo "Environment: chroot" + echo "Concurrency: $JOBS" + echo "Total: $total_count" + echo "PASS: $pass_count" + echo "FAIL: $fail_count" + echo "SKIP: $skip_count" + echo "Result: $g_result_file" + echo "Logs: $g_log_root" + + [ "$total_count" -eq "$expected_count" ] || return 1 + [ $((pass_count + fail_count + skip_count)) -eq "$expected_count" ] || return 1 + [ "$fail_count" -eq 0 ] +} + +preflight() { + local command_name + + is_positive_integer "$JOBS" || die "-j must be a positive integer: $JOBS" || return 1 + is_positive_integer "$TIMEOUT_SEC" || die "--timeout must be a positive integer: $TIMEOUT_SEC" || return 1 + is_positive_integer "$INSTALL_TIMEOUT_SEC" || die "--install-timeout must be a positive integer: $INSTALL_TIMEOUT_SEC" || return 1 + is_positive_integer "$COMMAND_TIMEOUT_SEC" || die "--command-timeout must be a positive integer: $COMMAND_TIMEOUT_SEC" || return 1 + is_positive_integer "$SERVICE_TIMEOUT_SEC" || die "--service-timeout must be a positive integer: $SERVICE_TIMEOUT_SEC" || return 1 + is_positive_integer "$REMOVE_TIMEOUT_SEC" || die "--remove-timeout must be a positive integer: $REMOVE_TIMEOUT_SEC" || return 1 + [ -f "$PKG_LIST" ] || die "pkg_list not found: $PKG_LIST" || return 1 + [ -f "$DESKTOP_WHITELIST" ] || die "desktop_whitelist not found: $DESKTOP_WHITELIST" || return 1 + [ -f "$SERVICE_PKG_LIST" ] || die "service_pkg_list not found: $SERVICE_PKG_LIST" || return 1 + + for command_name in dnf chroot mount umount findmnt timeout setsid cp realpath awk; do + command -v "$command_name" >/dev/null 2>&1 || die "$command_name not found" || return 1 + done + if [ "$(id -u)" -eq 0 ]; then + SUDO=() + else + command -v sudo >/dev/null 2>&1 || die "sudo not found" || return 1 + sudo -n true >/dev/null 2>&1 || die "passwordless sudo is required" || return 1 + SUDO=(sudo -n) + fi +} + +do_test_impl() { + local queue_file + local expected_count + local index=0 + local pkg_name + local pid + + parse_args "$@" || { + usage + return 1 + } + preflight || return 1 + + mkdir -p "$g_log_root" + g_tmpdir="$(mktemp -d /tmp/yum-ci-pkg-test-chroot.XXXXXX)" + g_work_root="$(mktemp -d /var/tmp/yum-ci-pkg-test-chroot.XXXXXX)" + path_is_safe "$g_work_root" || return 1 + g_base_root="$g_work_root/base" + queue_file="$g_tmpdir/pkg.queue" + + build_package_queue "$queue_file" + expected_count="$(wc -l <"$queue_file")" + [ "$expected_count" -gt 0 ] || die "package list is empty" || return 1 + + msg "log root: $g_log_root" + msg "result: $g_result_file" + msg "packages: $expected_count" + prepare_base_rootfs || return 1 + + mkdir -p "$g_work_root/workers" + while IFS= read -r pkg_name; do + [ -n "$pkg_name" ] || continue + index=$((index + 1)) + wait_for_slot + run_one_package "$index" "$pkg_name" & + pid=$! + g_worker_pids+=("$pid") + done <"$queue_file" + wait_for_all_jobs + g_worker_pids=() + + merge_results "$queue_file" + write_summary "$expected_count" +} + +do_test() { + if do_test_impl "$@"; then + _tc_pass + return 0 + fi + _tc_fail + return 1 +} + +tc_setup() { + msg "test setup" + return 0 +} + +tc_teardown() { + msg "test teardown" + cleanup_main +} + +############################################################################### +main_signal_cleanup() { + trap - EXIT INT TERM + cleanup_main || true + exit 130 +} + +trap cleanup_main EXIT +trap main_signal_cleanup INT TERM +set +u +tst_main "$@" +tst_ret=$? +cleanup_main || tst_ret=1 +trap - EXIT INT TERM +exit "$tst_ret" +###############################################################################