#!/usr/bin/env bats load test_helper # The driver with fake QEMU: "VMs" are sleep processes, and ssh runs its command # right here. The working copy has a small fake smoketest of its own. setup() { common_setup make_working_copy export KVM_DEVICE="$BATS_TEST_TMPDIR/kvm" touch "$KVM_DEVICE" export SMOKETEST_DESKTOP_TIMEOUT=0 SMOKETEST_SSH_TIMEOUT=5 STATE="$HOME/.cache/coredesktop-dev/smoketest" mkdir -p "$STATE/images" touch "$STATE/images/base.qcow2" "$STATE/images/installed.qcow2" git -C "$WC" rev-parse HEAD > "$STATE/images/installed.ref" # A VM is a sleep that lets go of bats' output pipes stub qemu-system-x86_64 'while [ $# -gt 0 ]; do [ "$1" = -pidfile ] && pidfile="$2"; shift; done /usr/bin/sleep 300 > /dev/null 2>&1 3>&- & echo $! > "$pidfile"' stub qemu-img 'touch "${@: -1}"' stub xorriso stub curl stub ss stub ssh-keygen 'touch "${@: -1}" "${@: -1}.pub"' stub systemctl stub ssh 'while [ $# -gt 0 ]; do case "$1" in -i | -o | -p) shift 2 ;; *@*) shift; break ;; *) shift ;; esac done [ $# -gt 0 ] || exit 0 cd "$HOME" && exec bash -c "$*"' stub scp 'dest="${*: -1}"; cp "${@: -2:1}" "${dest#*:}"' SC="$WC/smoketest/scenarios" mkdir -p "$SC" "$WC/smoketest/lib" cat > "$WC/smoketest/lib/scenario.sh" << 'EOF' SKIP_STATUS=3 smoketest_prepare() { echo "prepare $IMAGE $SMOKETEST_REF $IMAGE_REF"; } fail() { echo "FAIL: $*"; exit 1; } skip() { echo "SKIP: $*"; exit 3; } EOF scenario_file 10-passes pr installed 'vm_exec "echo in the vm"; echo "ref=$SMOKETEST_REF"' scenario_file 20-fails pr base 'false; echo "still going"' scenario_file 30-variants nightly base 'echo "variant $VARIANT"' 'VARIANTS=(one two)' scenario_file 40-skips pr base 'skip "not today"' } # scenario_file NAME TIER IMAGE BODY [EXTRA] scenario_file() { printf '%s\n' "DESCRIPTION=\"$1 does things\"" "TIER=$2" "IMAGE=$3" "${5:-}" \ "scenario() {" "$4" "}" > "$SC/$1.sh" } last_run() { ls -d "$STATE/runs/"* | tail -n 1 } @test "list shows each scenario's tier, image and description" { run cdd smoketest list [ "$status" -eq 0 ] [[ "$output" == *"10-passes"*"pr"*"installed"*"10-passes does things"* ]] [[ "$output" == *"30-variants"*"nightly"*"base"*"(2 variants)"* ]] } @test "run runs the pr scenarios against HEAD and reports each" { run cdd smoketest run [ "$status" -eq 1 ] [[ "$output" == *"PASS 10-passes"* ]] [[ "$output" == *"FAIL 20-fails"* ]] [[ "$output" == *"SKIP 40-skips"* ]] [[ "$output" != *"30-variants"* ]] [[ "$output" == *"in the vm"* ]] [[ "$output" == *"ref=$(git -C "$WC" rev-parse HEAD)"* ]] [ -f "$(last_run)/summary" ] } @test "a scenario stops at its first failing command" { run cdd smoketest run 20 [ "$status" -eq 1 ] [[ "$output" != *"still going"* ]] } @test "run picks scenarios by name or number" { run cdd smoketest run 10 40-skips [ "$status" -eq 0 ] [[ "$output" == *"PASS 10-passes"* ]] [[ "$output" == *"SKIP 40-skips"* ]] [[ "$output" != *"20-fails"* ]] } @test "run stops on an unknown scenario before starting any VM" { run cdd smoketest run 10 nosuch [ "$status" -eq 1 ] [[ "$output" == *"No scenario called nosuch"* ]] not_called qemu-system-x86_64 } @test "each variant runs in its own VM" { run cdd smoketest run 30 [ "$status" -eq 0 ] [[ "$output" == *"PASS 30-variants @ one"* ]] [[ "$output" == *"PASS 30-variants @ two"* ]] [ "$(grep -c '^qemu-system-x86_64 ' "$STUB_LOG")" -eq 2 ] grep -q "variant two" "$(last_run)/30-variants@two/scenario.log" } @test "--tier all runs every scenario" { run cdd smoketest run --tier all [[ "$output" == *"30-variants @ one"* ]] [[ "$output" == *"10-passes"* ]] } @test "the scenario boots a copy of its image, and gets the installed image's commit" { run cdd smoketest run 10 [ "$status" -eq 0 ] grep -q "^qemu-img create -q -f qcow2 -b $STATE/images/installed.qcow2 " "$STUB_LOG" [[ "$output" == *"prepare installed $(git -C "$WC" rev-parse HEAD) $(cat "$STATE/images/installed.ref")"* ]] } @test "the VM is stopped and its disk deleted after each scenario" { run cdd smoketest run 10 20 for dir in "$(last_run)"/*/; do [ ! -e "$dir/qemu.pid" ] [ ! -e "$dir/disk.qcow2" ] done } @test "--keep leaves a failed scenario's VM running, and stops the run" { run cdd smoketest run --keep 20 10 [ "$status" -eq 1 ] [[ "$output" == *"The VM is still running"* ]] [[ "$output" != *"PASS 10-passes"* ]] pid="$(cat "$(last_run)/20-fails/qemu.pid")" kill -0 "$pid" kill "$pid" } @test "run --ref tests another commit" { first="$(git -C "$WC" rev-parse HEAD)" git -C "$WC" commit -q --allow-empty -m second run cdd smoketest run --ref "$first" 10 [ "$status" -eq 0 ] [[ "$output" == *"ref=$first"* ]] } @test "run stops without KVM" { rm "$KVM_DEVICE" run cdd smoketest run 10 [ "$status" -eq 1 ] [[ "$output" == *"Can't use $KVM_DEVICE"* ]] } @test "smoketest needs a working copy with a smoketest folder" { rm -rf "$WC/smoketest" run cdd smoketest list [ "$status" -eq 1 ] [[ "$output" == *"has no smoketest/scenarios"* ]] } @test "images shows what's built" { rm "$STATE/images/installed.qcow2" run cdd smoketest images [ "$status" -eq 0 ] [[ "$output" == *"base built"* ]] [[ "$output" == *"installed not built"* ]] }