231 lines
7.5 KiB
Bash
Executable file
231 lines
7.5 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
export PATH="/run/current-system/sw/bin:/usr/bin:/bin:${PATH}"
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
|
WORK_ROOT="${ULTRACLOUD_WORK_ROOT:-${REPO_ROOT}/work}"
|
|
LOG_DIR="${1:-${ULTRACLOUD_KVM_PUBLISHABLE_LOG_DIR:-${WORK_ROOT}/publishable-kvm-suite}}"
|
|
|
|
mkdir -p "${LOG_DIR}"
|
|
|
|
log() {
|
|
printf '[publishable-kvm-suite] %s\n' "$*"
|
|
}
|
|
|
|
die() {
|
|
printf '[publishable-kvm-suite] ERROR: %s\n' "$*" >&2
|
|
exit 1
|
|
}
|
|
|
|
host_cpu_count() {
|
|
local count
|
|
count="$(getconf _NPROCESSORS_ONLN 2>/dev/null || nproc 2>/dev/null || echo 1)"
|
|
if [[ ! "${count}" =~ ^[0-9]+$ ]] || (( count < 1 )); then
|
|
count=1
|
|
fi
|
|
printf '%s\n' "${count}"
|
|
}
|
|
|
|
default_local_nix_max_jobs() {
|
|
local cpu_count="$1"
|
|
if (( cpu_count <= 2 )); then
|
|
printf '1\n'
|
|
return 0
|
|
fi
|
|
|
|
printf '%s\n' "$(( (cpu_count + 1) / 2 ))"
|
|
}
|
|
|
|
default_local_nix_build_cores() {
|
|
local cpu_count="$1"
|
|
local max_jobs="$2"
|
|
local build_cores=1
|
|
|
|
if (( max_jobs > 0 )); then
|
|
build_cores="$(( cpu_count / max_jobs ))"
|
|
fi
|
|
if (( build_cores < 1 )); then
|
|
build_cores=1
|
|
fi
|
|
|
|
printf '%s\n' "${build_cores}"
|
|
}
|
|
|
|
append_nix_config_line() {
|
|
local line="$1"
|
|
if [[ -n "${NIX_CONFIG:-}" ]]; then
|
|
NIX_CONFIG+=$'\n'
|
|
fi
|
|
NIX_CONFIG+="${line}"
|
|
}
|
|
|
|
host_nested_param_path() {
|
|
if [[ -f /sys/module/kvm_intel/parameters/nested ]]; then
|
|
printf '%s\n' /sys/module/kvm_intel/parameters/nested
|
|
elif [[ -f /sys/module/kvm_amd/parameters/nested ]]; then
|
|
printf '%s\n' /sys/module/kvm_amd/parameters/nested
|
|
fi
|
|
}
|
|
|
|
require_publishable_kvm_host() {
|
|
[[ -e /dev/kvm ]] || die "/dev/kvm is missing"
|
|
[[ -r /dev/kvm && -w /dev/kvm ]] || die "/dev/kvm is not readable and writable for $(id -un)"
|
|
|
|
local nested_path nested_value
|
|
nested_path="$(host_nested_param_path || true)"
|
|
if [[ -z "${nested_path}" ]]; then
|
|
return 0
|
|
fi
|
|
|
|
nested_value="$(<"${nested_path}")"
|
|
case "${nested_value}" in
|
|
1|Y|y)
|
|
;;
|
|
*)
|
|
die "nested virtualization is disabled on the host (${nested_path}=${nested_value})"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
choose_runtime_root() {
|
|
if [[ -n "${ULTRACLOUD_KVM_RUNTIME_ROOT:-}" ]]; then
|
|
printf '%s\n' "${ULTRACLOUD_KVM_RUNTIME_ROOT}"
|
|
return 0
|
|
fi
|
|
|
|
printf '%s\n' "${WORK_ROOT}/publishable-kvm-runtime"
|
|
}
|
|
|
|
get_hostname() {
|
|
if command -v hostname >/dev/null 2>&1; then
|
|
hostname
|
|
else
|
|
uname -n
|
|
fi
|
|
}
|
|
|
|
prepare_runtime_dirs() {
|
|
local runtime_root cpu_count default_max_jobs default_build_cores
|
|
|
|
runtime_root="$(choose_runtime_root)"
|
|
cpu_count="$(host_cpu_count)"
|
|
default_max_jobs="$(default_local_nix_max_jobs "${cpu_count}")"
|
|
default_build_cores="$(default_local_nix_build_cores "${cpu_count}" "${default_max_jobs}")"
|
|
export ULTRACLOUD_KVM_RUNTIME_ROOT="${runtime_root}"
|
|
export TMPDIR="${TMPDIR:-${WORK_ROOT}/tmp}"
|
|
export XDG_CACHE_HOME="${XDG_CACHE_HOME:-${runtime_root}/xdg-cache}"
|
|
export PHOTON_CLUSTER_WORK_ROOT="${PHOTON_CLUSTER_WORK_ROOT:-${WORK_ROOT}/test-cluster}"
|
|
export PHOTON_VM_DIR="${PHOTON_VM_DIR:-${PHOTON_CLUSTER_WORK_ROOT}/state}"
|
|
export PHOTON_CLUSTER_VDE_SWITCH_DIR="${PHOTON_CLUSTER_VDE_SWITCH_DIR:-${PHOTON_CLUSTER_WORK_ROOT}/vde-switch}"
|
|
export ULTRACLOUD_LOCAL_NIX_MAX_JOBS="${ULTRACLOUD_LOCAL_NIX_MAX_JOBS:-${default_max_jobs}}"
|
|
export ULTRACLOUD_LOCAL_NIX_BUILD_CORES="${ULTRACLOUD_LOCAL_NIX_BUILD_CORES:-${default_build_cores}}"
|
|
export PHOTON_CLUSTER_NIX_MAX_JOBS="${PHOTON_CLUSTER_NIX_MAX_JOBS:-${ULTRACLOUD_LOCAL_NIX_MAX_JOBS}}"
|
|
export PHOTON_CLUSTER_NIX_BUILD_CORES="${PHOTON_CLUSTER_NIX_BUILD_CORES:-${ULTRACLOUD_LOCAL_NIX_BUILD_CORES}}"
|
|
export PHOTON_VM_SKIP_NESTED_KVM_VALIDATE=0
|
|
export PHOTON_VM_FORCE_TCG=0
|
|
append_nix_config_line "builders ="
|
|
append_nix_config_line "max-jobs = ${ULTRACLOUD_LOCAL_NIX_MAX_JOBS}"
|
|
append_nix_config_line "cores = ${ULTRACLOUD_LOCAL_NIX_BUILD_CORES}"
|
|
append_nix_config_line "experimental-features = nix-command flakes"
|
|
append_nix_config_line "warn-dirty = false"
|
|
export NIX_CONFIG
|
|
mkdir -p "${TMPDIR}" "${XDG_CACHE_HOME}" "${PHOTON_CLUSTER_WORK_ROOT}" "${PHOTON_CLUSTER_VDE_SWITCH_DIR}" "${runtime_root}"
|
|
}
|
|
|
|
capture_environment() {
|
|
{
|
|
printf 'started_at=%s\n' "$(date -Is)"
|
|
printf 'hostname=%s\n' "$(get_hostname)"
|
|
printf 'kernel=%s\n' "$(uname -a)"
|
|
printf 'pwd=%s\n' "$(pwd)"
|
|
printf 'user=%s\n' "$(id -un)"
|
|
printf 'uid=%s\n' "$(id -u)"
|
|
printf 'gid=%s\n' "$(id -g)"
|
|
printf 'branch=%s\n' "$(git -C "${REPO_ROOT}" branch --show-current)"
|
|
printf 'commit=%s\n' "$(git -C "${REPO_ROOT}" rev-parse HEAD)"
|
|
printf 'nix_version=%s\n' "$(nix --version)"
|
|
printf 'runtime_root=%s\n' "${ULTRACLOUD_KVM_RUNTIME_ROOT:-}"
|
|
printf 'tmpdir=%s\n' "${TMPDIR:-}"
|
|
printf 'xdg_cache_home=%s\n' "${XDG_CACHE_HOME:-}"
|
|
printf 'photon_cluster_work_root=%s\n' "${PHOTON_CLUSTER_WORK_ROOT:-}"
|
|
printf 'photon_vm_dir=%s\n' "${PHOTON_VM_DIR:-}"
|
|
printf 'photon_cluster_vde_switch_dir=%s\n' "${PHOTON_CLUSTER_VDE_SWITCH_DIR:-}"
|
|
printf 'host_cpu_count=%s\n' "$(host_cpu_count)"
|
|
printf 'local_nix_max_jobs=%s\n' "${ULTRACLOUD_LOCAL_NIX_MAX_JOBS:-}"
|
|
printf 'local_nix_build_cores=%s\n' "${ULTRACLOUD_LOCAL_NIX_BUILD_CORES:-}"
|
|
printf 'photon_cluster_nix_max_jobs=%s\n' "${PHOTON_CLUSTER_NIX_MAX_JOBS:-}"
|
|
printf 'photon_cluster_nix_build_cores=%s\n' "${PHOTON_CLUSTER_NIX_BUILD_CORES:-}"
|
|
printf 'nested_kvm_validate_skipped=%s\n' "${PHOTON_VM_SKIP_NESTED_KVM_VALIDATE:-0}"
|
|
printf 'vm_accelerator_mode=%s\n' "$([[ "${PHOTON_VM_FORCE_TCG:-0}" == "1" ]] && echo tcg || echo kvm)"
|
|
printf 'photon_vm_console_wait_timeout=%s\n' "${PHOTON_VM_CONSOLE_WAIT_TIMEOUT:-}"
|
|
printf 'kvm_present=%s\n' "$([[ -e /dev/kvm ]] && echo yes || echo no)"
|
|
printf 'kvm_access=%s\n' "$([[ -r /dev/kvm && -w /dev/kvm ]] && echo rw || echo no)"
|
|
if [[ -e /dev/kvm ]]; then
|
|
printf 'kvm_stat=%s\n' "$(stat -c '%A %U %G %t:%T' /dev/kvm)"
|
|
fi
|
|
if [[ -f /sys/module/kvm_intel/parameters/nested ]]; then
|
|
printf 'kvm_intel_nested=%s\n' "$(cat /sys/module/kvm_intel/parameters/nested)"
|
|
fi
|
|
if [[ -f /sys/module/kvm_amd/parameters/nested ]]; then
|
|
printf 'kvm_amd_nested=%s\n' "$(cat /sys/module/kvm_amd/parameters/nested)"
|
|
fi
|
|
printf 'nix_builders=%s\n' "$(nix config show builders 2>/dev/null | awk -F' = ' 'NR==1 { print $2 }')"
|
|
} >"${LOG_DIR}/environment.txt"
|
|
}
|
|
|
|
finish_environment() {
|
|
local rc="${1:-0}"
|
|
{
|
|
printf 'finished_at=%s\n' "$(date -Is)"
|
|
printf 'exit_status=%s\n' "${rc}"
|
|
} >>"${LOG_DIR}/environment.txt"
|
|
}
|
|
|
|
run_case() {
|
|
local name="$1"
|
|
shift
|
|
local logfile="${LOG_DIR}/${name}.log"
|
|
local metafile="${LOG_DIR}/${name}.meta"
|
|
local started_at ended_at rc
|
|
|
|
started_at="$(date -Is)"
|
|
printf 'command=%s\n' "$*" >"${metafile}"
|
|
printf 'started_at=%s\n' "${started_at}" >>"${metafile}"
|
|
|
|
log "running ${name}: $*"
|
|
set +e
|
|
(
|
|
cd "${REPO_ROOT}"
|
|
"$@"
|
|
) 2>&1 | tee "${logfile}"
|
|
rc=${PIPESTATUS[0]}
|
|
set -e
|
|
|
|
ended_at="$(date -Is)"
|
|
printf 'ended_at=%s\n' "${ended_at}" >>"${metafile}"
|
|
printf 'exit_code=%s\n' "${rc}" >>"${metafile}"
|
|
|
|
if (( rc != 0 )); then
|
|
log "${name} failed; see ${logfile}"
|
|
return "${rc}"
|
|
fi
|
|
|
|
log "${name} passed"
|
|
}
|
|
|
|
main() {
|
|
trap 'finish_environment "$?"' EXIT
|
|
prepare_runtime_dirs
|
|
require_publishable_kvm_host
|
|
capture_environment
|
|
|
|
run_case fresh-smoke nix run ./nix/test-cluster#cluster -- fresh-smoke
|
|
run_case fresh-demo-vm-webapp nix run ./nix/test-cluster#cluster -- fresh-demo-vm-webapp
|
|
run_case fresh-matrix nix run ./nix/test-cluster#cluster -- fresh-matrix
|
|
|
|
log "publishable KVM suite passed; logs in ${LOG_DIR}"
|
|
}
|
|
|
|
main "$@"
|