Coredesktop-Dev/tests/smoketest.bats

#!/usr/bin/env bats

load test_helper

# The driver with fake QEMU: "VMs" are sleep processes, and ssh runs its command
# right here. The working copy has a small fake smoketest of its own.

setup() {
  common_setup
  make_working_copy

  export KVM_DEVICE="$BATS_TEST_TMPDIR/kvm"
  touch "$KVM_DEVICE"
  export SMOKETEST_DESKTOP_TIMEOUT=0 SMOKETEST_SSH_TIMEOUT=5
  STATE="$HOME/.cache/coredesktop-dev/smoketest"
  mkdir -p "$STATE/images"
  touch "$STATE/images/base.qcow2" "$STATE/images/installed.qcow2"
  git -C "$WC" rev-parse HEAD > "$STATE/images/installed.ref"

  # A VM is a sleep that lets go of bats' output pipes
  stub qemu-system-x86_64 'while [ $# -gt 0 ]; do [ "$1" = -pidfile ] && pidfile="$2"; shift; done
    /usr/bin/sleep 300 > /dev/null 2>&1 3>&- &
    echo $! > "$pidfile"'
  stub qemu-img 'touch "${@: -1}"'
  stub xorriso
  stub curl
  stub ss
  stub ssh-keygen 'touch "${@: -1}" "${@: -1}.pub"'
  stub systemctl
  # Like a real SSH session, the command gets no controlling terminal: the
  # driver runs bash -i, which would otherwise try to take over the one bats
  # runs in, and be stopped for it
  stub ssh 'while [ $# -gt 0 ]; do
      case "$1" in -i | -o | -p) shift 2 ;; *@*) shift; break ;; *) shift ;; esac
    done
    [ $# -gt 0 ] || exit 0
    cd "$HOME" && exec setsid -w bash -c "$*"'
  stub scp 'dest="${*: -1}"; cp "${@: -2:1}" "${dest#*:}"'

  SC="$WC/smoketest/scenarios"
  mkdir -p "$SC" "$WC/smoketest/lib"
  cat > "$WC/smoketest/lib/scenario.sh" << 'EOF'
SKIP_STATUS=3
smoketest_prepare() { echo "prepare $IMAGE $SMOKETEST_REF $IMAGE_REF"; }
fail() { echo "FAIL: $*"; exit 1; }
skip() { echo "SKIP: $*"; exit 3; }
EOF
  scenario_file 10-passes pr installed 'vm_exec "echo in the vm"; echo "ref=$SMOKETEST_REF"'
  scenario_file 20-fails pr base 'false; echo "still going"'
  scenario_file 30-variants nightly base 'echo "variant $VARIANT"' 'VARIANTS=(one two)'
  scenario_file 40-skips pr base 'skip "not today"'
}

# scenario_file NAME TIER IMAGE BODY [EXTRA]
scenario_file() {
  printf '%s\n' "DESCRIPTION=\"$1 does things\"" "TIER=$2" "IMAGE=$3" "${5:-}" \
    "scenario() {" "$4" "}" > "$SC/$1.sh"
}

last_run() {
  ls -d "$STATE/runs/"* | tail -n 1
}

@test "list shows each scenario's tier, image and description" {
  run cdd smoketest list
  [ "$status" -eq 0 ]
  [[ "$output" == *"10-passes"*"pr"*"installed"*"10-passes does things"* ]]
  [[ "$output" == *"30-variants"*"nightly"*"base"*"(2 variants)"* ]]
}

@test "run runs the pr scenarios against HEAD and reports each" {
  run cdd smoketest run
  [ "$status" -eq 1 ]
  [[ "$output" == *"PASS  10-passes"* ]]
  [[ "$output" == *"FAIL  20-fails"* ]]
  [[ "$output" == *"SKIP  40-skips"* ]]
  [[ "$output" != *"30-variants"* ]]
  [[ "$output" == *"in the vm"* ]]
  [[ "$output" == *"ref=$(git -C "$WC" rev-parse HEAD)"* ]]
  [ -f "$(last_run)/summary" ]
}

@test "a scenario stops at its first failing command" {
  run cdd smoketest run 20
  [ "$status" -eq 1 ]
  [[ "$output" != *"still going"* ]]
}

@test "run picks scenarios by name or number" {
  run cdd smoketest run 10 40-skips
  [ "$status" -eq 0 ]
  [[ "$output" == *"PASS  10-passes"* ]]
  [[ "$output" == *"SKIP  40-skips"* ]]
  [[ "$output" != *"20-fails"* ]]
}

@test "run stops on an unknown scenario before starting any VM" {
  run cdd smoketest run 10 nosuch
  [ "$status" -eq 1 ]
  [[ "$output" == *"No scenario called nosuch"* ]]
  not_called qemu-system-x86_64
}

@test "each variant runs in its own VM" {
  run cdd smoketest run 30
  [ "$status" -eq 0 ]
  [[ "$output" == *"PASS  30-variants @ one"* ]]
  [[ "$output" == *"PASS  30-variants @ two"* ]]
  [ "$(grep -c '^qemu-system-x86_64 ' "$STUB_LOG")" -eq 2 ]
  grep -q "variant two" "$(last_run)/30-variants@two/scenario.log"
}

@test "--tier all runs every scenario" {
  run cdd smoketest run --tier all
  [[ "$output" == *"30-variants @ one"* ]]
  [[ "$output" == *"10-passes"* ]]
}

@test "the scenario boots a copy of its image, and gets the installed image's commit" {
  run cdd smoketest run 10
  [ "$status" -eq 0 ]
  grep -q "^qemu-img create -q -f qcow2 -b $STATE/images/installed.qcow2 " "$STUB_LOG"
  [[ "$output" == *"prepare installed $(git -C "$WC" rev-parse HEAD) $(cat "$STATE/images/installed.ref")"* ]]
}

@test "the VM is stopped and its disk deleted after each scenario" {
  run cdd smoketest run 10 20
  for dir in "$(last_run)"/*/; do
    [ ! -e "$dir/qemu.pid" ]
    [ ! -e "$dir/disk.qcow2" ]
  done
}

@test "--keep leaves a failed scenario's VM running, and stops the run" {
  run cdd smoketest run --keep 20 10
  [ "$status" -eq 1 ]
  [[ "$output" == *"The VM is still running"* ]]
  [[ "$output" != *"PASS  10-passes"* ]]
  pid="$(cat "$(last_run)/20-fails/qemu.pid")"
  kill -0 "$pid"
  kill "$pid"
}

@test "run --ref tests another commit" {
  first="$(git -C "$WC" rev-parse HEAD)"
  git -C "$WC" commit -q --allow-empty -m second
  run cdd smoketest run --ref "$first" 10
  [ "$status" -eq 0 ]
  [[ "$output" == *"ref=$first"* ]]
}

@test "run stops without KVM" {
  rm "$KVM_DEVICE"
  run cdd smoketest run 10
  [ "$status" -eq 1 ]
  [[ "$output" == *"Can't use $KVM_DEVICE"* ]]
}

@test "smoketest needs a working copy with a smoketest folder" {
  rm -rf "$WC/smoketest"
  run cdd smoketest list
  [ "$status" -eq 1 ]
  [[ "$output" == *"has no smoketest/scenarios"* ]]
}

@test "images shows what's built" {
  rm "$STATE/images/installed.qcow2"
  run cdd smoketest images
  [ "$status" -eq 0 ]
  [[ "$output" == *"base       built"* ]]
  [[ "$output" == *"installed  not built"* ]]
}