Coredesktop-Dev/lib/vm.sh

#!/bin/bash
#
# Purpose : Start, reach and stop the smoketest's QEMU virtual machines. Defines
#           what CoreDesktop's smoketest/lib/scenario.sh expects of a driver:
#           vm_exec, vm_put, vm_reboot, VM_GIT_SSH and VM_GIT_URL.

SMOKETEST_STATE="${XDG_CACHE_HOME:-$HOME/.cache}/coredesktop-dev/smoketest"
# IMAGES and KVM_DEVICE are used by smoketest.sh
# shellcheck disable=SC2034
IMAGES="$SMOKETEST_STATE/images"
# Shared with every VM as its /var/cache/pacman/pkg, so packages download once
PKG_CACHE="$SMOKETEST_STATE/pkg"
SSH_KEY="$SMOKETEST_STATE/id_ed25519"
# shellcheck disable=SC2034
KVM_DEVICE="${KVM_DEVICE:-/dev/kvm}"

VM_USER=tester
VM_MEMORY="${SMOKETEST_MEMORY:-8G}"
VM_CPUS="${SMOKETEST_CPUS:-4}"
# Longest a single command in the VM may run (an install takes an hour or more)
VM_EXEC_TIMEOUT="${SMOKETEST_EXEC_TIMEOUT:-4h}"
# Seconds to wait for the VM to answer on SSH, and for GNOME to start
VM_SSH_TIMEOUT="${SMOKETEST_SSH_TIMEOUT:-900}"
VM_DESKTOP_TIMEOUT="${SMOKETEST_DESKTOP_TIMEOUT:-300}"

SSH_OPTS=(
  -i "$SSH_KEY"
  -o IdentitiesOnly=yes
  -o StrictHostKeyChecking=no
  -o UserKnownHostsFile=/dev/null
  -o LogLevel=ERROR
  -o BatchMode=yes
  -o ConnectTimeout=10
  -o ServerAliveInterval=15
  -o ServerAliveCountMax=8
)

# Commands in the VM run as they would in a GNOME Terminal: with the GNOME
# session's environment, in an interactive bash that has read ~/.bashrc
VM_SESSION_ENV='eval "$(systemctl --user show-environment 2> /dev/null | sed "s/^/export /")"
export TERM="${TERM:-dumb}"'

ensure_ssh_key() {
  mkdir -p "$SMOKETEST_STATE"
  [ -f "$SSH_KEY" ] || ssh-keygen -q -t ed25519 -N '' -C coredesktop-smoketest -f "$SSH_KEY"
}

free_port() {
  local port
  for _ in $(seq 50); do
    port=$((20000 + RANDOM % 20000))
    [ -z "$(ss -Hltn "( sport = :$port )" 2> /dev/null)" ] && { echo "$port"; return 0; }
  done
  return 1
}

# vm_start DIR DISK [QEMU ARGS...]: boot DISK in the background. DIR gets the
# pid file and the serial console's log.
vm_start() {
  VM_DIR="$1"
  local disk="$2"
  shift 2
  local display=(-display none)
  [ -n "${SMOKETEST_VNC:-}" ] && display+=(-vnc "127.0.0.1:$SMOKETEST_VNC")

  ensure_ssh_key
  mkdir -p "$VM_DIR" "$PKG_CACHE"
  VM_PORT="$(free_port)" || die "Couldn't find a free port for the VM's SSH."

  # -vga std, not virtio: on Arch, virtio needs qemu-hw-display-virtio-vga
  # and qemu-hw-display-virtio-gpu, and with only the first QEMU segfaults
  # without a message. The VMs are headless (or VNC), so std is enough.
  qemu-system-x86_64 \
    -name coredesktop-smoketest \
    -machine q35,accel=kvm \
    -cpu host \
    -smp "$VM_CPUS" \
    -m "$VM_MEMORY" \
    -drive "file=$disk,if=virtio,format=qcow2,discard=unmap" \
    -nic "user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:$VM_PORT-:22" \
    -virtfs "local,path=$PKG_CACHE,mount_tag=pkgcache,security_model=mapped-xattr" \
    -vga std \
    "${display[@]}" \
    -serial "file:$VM_DIR/serial.log" \
    -pidfile "$VM_DIR/qemu.pid" \
    -daemonize \
    "$@" || die "QEMU didn't start."

  VM_GIT_SSH="ssh$(printf ' %q' "${SSH_OPTS[@]}")"
  VM_GIT_URL="ssh://$VM_USER@127.0.0.1:$VM_PORT"
  export VM_GIT_SSH VM_GIT_URL
}

vm_pid() {
  cat "$VM_DIR/qemu.pid" 2> /dev/null
}

vm_running() {
  local pid
  pid="$(vm_pid)" && [ -n "$pid" ] && kill -0 "$pid" 2> /dev/null
}

vm_ssh() {
  ssh "${SSH_OPTS[@]}" -p "$VM_PORT" "$VM_USER@127.0.0.1" "$@"
}

# The command to SSH in by hand
vm_ssh_command() {
  echo "ssh -i $SSH_KEY -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p $VM_PORT $VM_USER@127.0.0.1"
}

# Progress for the long waits: wait_begin says what it's waiting for, and
# wait_tick, called in the wait's loop, says so again once a minute
wait_begin() {
  WAIT_WHAT="$1"
  WAIT_START=$SECONDS
  WAIT_NEXT=$((SECONDS + 60))
  echo "$1..."
}

wait_tick() {
  (( SECONDS >= WAIT_NEXT )) || return 0
  echo "  still $WAIT_WHAT ($(( (SECONDS - WAIT_START) / 60 )) min)"
  WAIT_NEXT=$((WAIT_NEXT + 60))
}

wait_done() {
  echo "  done after $(( SECONDS - WAIT_START ))s"
}

wait_for_ssh() {
  local deadline=$((SECONDS + VM_SSH_TIMEOUT))
  wait_begin "waiting for the VM to boot and answer on SSH (its console: $VM_DIR/serial.log)"
  until vm_ssh true < /dev/null 2> /dev/null; do
    vm_running || die "The VM stopped. Its console log: $VM_DIR/serial.log"
    (( SECONDS < deadline )) || die "The VM didn't answer on SSH. Its console log: $VM_DIR/serial.log"
    wait_tick
    sleep 5
  done
  wait_done
}

# Wait for the automatic GNOME login. Not fatal: the health checks catch a
# desktop that didn't start.
wait_for_desktop() {
  local deadline=$((SECONDS + VM_DESKTOP_TIMEOUT))
  wait_begin "waiting for GNOME to start in the VM"
  until vm_ssh "pgrep -u $VM_USER -x gnome-shell" < /dev/null &> /dev/null; do
    if (( SECONDS >= deadline )); then
      echo "Warning: GNOME didn't start in the VM within ${VM_DESKTOP_TIMEOUT}s."
      return 0
    fi
    wait_tick
    sleep 5
  done
  wait_done
}

vm_wait_up() {
  wait_for_ssh
  wait_for_desktop
}

# vm_exec SCRIPT: run SCRIPT with bash in the VM, as tester, in the GNOME
# session's environment. Interactive bash's complaints about having no
# terminal are dropped.
vm_exec() {
  timeout "$VM_EXEC_TIMEOUT" ssh "${SSH_OPTS[@]}" -p "$VM_PORT" -T "$VM_USER@127.0.0.1" \
    "bash -ic $(printf '%q' "$VM_SESSION_ENV
$1")" < /dev/null \
    2> >(grep -v -e '^bash: cannot set terminal process group' -e '^bash: no job control in this shell' >&2)
}

# vm_put LOCAL REMOTE
vm_put() {
  scp -q "${SSH_OPTS[@]}" -P "$VM_PORT" "$1" "$VM_USER@127.0.0.1:$2"
}

vm_boot_id() {
  vm_ssh 'cat /proc/sys/kernel/random/boot_id' < /dev/null 2> /dev/null
}

vm_reboot() {
  local before deadline=$((SECONDS + VM_SSH_TIMEOUT))
  echo "== reboot"
  before="$(vm_boot_id)"
  vm_ssh 'sudo systemctl reboot' < /dev/null &> /dev/null || true
  wait_begin "waiting for the VM to come back"
  until [ "$(vm_boot_id)" != "$before" ] && [ -n "$(vm_boot_id)" ]; do
    vm_running || die "The VM stopped while rebooting. Its console log: $VM_DIR/serial.log"
    (( SECONDS < deadline )) || die "The VM didn't come back after rebooting. Its console log: $VM_DIR/serial.log"
    wait_tick
    sleep 5
  done
  wait_done
  wait_for_desktop
}

# Shut the VM down cleanly (so its disk can be kept), killing it if it hangs
vm_stop() {
  vm_running || return 0
  echo "Shutting down the VM..."
  vm_ssh 'sudo systemctl poweroff' < /dev/null &> /dev/null || true
  local deadline=$((SECONDS + 120))
  while vm_running && (( SECONDS < deadline )); do
    sleep 2
  done
  vm_kill
}

vm_kill() {
  local pid
  pid="$(vm_pid)" || return 0
  [ -n "$pid" ] && kill "$pid" 2> /dev/null
  rm -f "$VM_DIR/qemu.pid"
  return 0
}