testing harness, coredesktop-dev

smoketest/scenarios/10-fresh-install.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# shellcheck disable=SC2034
+DESCRIPTION="install.sh on a fresh Arch + GNOME machine; saves the reference fingerprint"
+IMAGE=base
+TIER=nightly
+
+scenario() {
+  cd_install
+  assert_status 0
+  assert_vm "install.sh left no local changes in the checkout" \
+    '[ -z "$(git -C ~/.local/share/coredesktop status --porcelain --untracked-files=no)" ]'
+
+  # The README says to reboot after installing
+  vm_reboot
+  health
+
+  fingerprint "$SMOKETEST_RESULTS/installed.fingerprint"
+  cp "$SMOKETEST_RESULTS/installed.fingerprint" "$(reference_fingerprint)"
+}

smoketest/scenarios/11-refresh.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+# shellcheck disable=SC2034
+DESCRIPTION="refresh (re-running install.sh) on an installed machine changes nothing"
+IMAGE=installed
+TIER=pr
+
+scenario() {
+  fingerprint "$SMOKETEST_RESULTS/before.fingerprint"
+
+  run_in_vm refresh 'TERM=dumb ~/.local/bin/refresh'
+  assert_status 0
+  refute_output "Running migration"
+
+  fingerprint "$SMOKETEST_RESULTS/after.fingerprint"
+  assert_same_fingerprint "$SMOKETEST_RESULTS/before.fingerprint" "$SMOKETEST_RESULTS/after.fingerprint"
+  health
+}

smoketest/scenarios/20-noop-update.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# shellcheck disable=SC2034
+DESCRIPTION="update with nothing new to do installs nothing and changes nothing"
+IMAGE=installed
+TIER=pr
+
+# smoketest_prepare has already updated the machine to the commit under test
+scenario() {
+  fingerprint "$SMOKETEST_RESULTS/before.fingerprint"
+
+  cd_update
+  assert_status 0
+  assert_output "Done."
+  refute_output "Running migration"
+  # install_packages prints this only when it installs something
+  refute_output "Installing:"
+
+  fingerprint "$SMOKETEST_RESULTS/after.fingerprint"
+  assert_same_fingerprint "$SMOKETEST_RESULTS/before.fingerprint" "$SMOKETEST_RESULTS/after.fingerprint"
+  health
+}

smoketest/scenarios/30-upgrade-from-baseline.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+# shellcheck disable=SC2034
+DESCRIPTION="install an older commit, update to the commit under test: same machine as a fresh install"
+IMAGE=base
+TIER=nightly
+mapfile -t VARIANTS < <(sed 's/#.*//' "$(dirname "${BASH_SOURCE[0]}")/../baselines" | awk 'NF { print $1 }')
+
+scenario() {
+  [ -f "$(reference_fingerprint)" ] \
+    || skip "needs the fingerprint 10-fresh-install saves; run both in the same run"
+
+  upstream_set "$VARIANT"
+  cd_install
+  assert_status 0
+  vm_reboot
+
+  upstream_set "$SMOKETEST_REF"
+  cd_update
+  assert_status 0
+  vm_reboot
+  health
+
+  fingerprint "$SMOKETEST_RESULTS/updated.fingerprint"
+  assert_same_fingerprint "$(reference_fingerprint)" "$SMOKETEST_RESULTS/updated.fingerprint"
+}

smoketest/scenarios/31-update-replaces-itself.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# shellcheck disable=SC2034
+DESCRIPTION="a release that changes update and adds a hook: the new update and hook run in the same update"
+IMAGE=installed
+TIER=pr
+
+scenario() {
+  upstream_commit "Change update and add a hook" '
+    sed -i "s/^FAILED=()\$/echo \"smoketest: the new update ran\"\nFAILED=()/" update
+    grep -q "smoketest: the new update ran" update
+    printf "%s\n" "#!/bin/bash" "run_update() { touch ~/smoketest-hook-ran; }" > updates.d/50-smoketest.sh'
+
+  cd_update
+  assert_status 0
+  assert_output "smoketest: the new update ran"
+  assert_vm "the new hook ran" 'test -f ~/smoketest-hook-ran'
+  assert_vm "~/.local/bin/update is the new one" \
+    'cmp ~/.local/bin/update ~/.local/share/coredesktop/update'
+}

smoketest/scenarios/32-new-migration.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# shellcheck disable=SC2034
+DESCRIPTION="a release with a new migration: it runs once, and is recorded"
+IMAGE=installed
+TIER=pr
+
+scenario() {
+  upstream_commit "Add a migration" '
+    echo "echo ran >> ~/smoketest-migration" > "migrations/$(date +%s).sh"'
+
+  cd_update
+  assert_status 0
+  assert_output "Running migration"
+  cd_update
+  assert_status 0
+  refute_output "Running migration"
+
+  assert_vm "the migration ran exactly once" '[ "$(cat ~/smoketest-migration)" = ran ]'
+  assert_vm "the migration is recorded" '
+    cd ~/.local/share/coredesktop/migrations
+    for m in *.sh; do test -f ~/.local/state/coredesktop/migrations/$m; done'
+}

smoketest/scenarios/40-failing-migration.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# shellcheck disable=SC2034
+DESCRIPTION="a failing migration stops later ones, fails the update, and runs again next time"
+IMAGE=installed
+TIER=pr
+
+scenario() {
+  # The first fails while ~/smoketest-block exists; the second must wait for it
+  upstream_commit "Add two migrations" '
+    now=$(date +%s)
+    echo "[ ! -e ~/smoketest-block ] && echo first >> ~/smoketest-ran" > "migrations/$now.sh"
+    echo "echo second >> ~/smoketest-ran" > "migrations/$((now + 1)).sh"'
+  assert_vm "block the first migration" 'touch ~/smoketest-block'
+
+  cd_update
+  assert_status 1
+  assert_output "failed; it will run again on the next update"
+  assert_output "Done, with errors in: 12-migrations.sh"
+  refute_vm "neither migration ran" 'test -e ~/smoketest-ran'
+
+  assert_vm "unblock the first migration" 'rm ~/smoketest-block'
+  cd_update
+  assert_status 0
+  assert_vm "both ran, in order" '[ "$(cat ~/smoketest-ran)" = "$(printf "first\nsecond")" ]'
+
+  cd_update
+  assert_status 0
+  assert_vm "neither ran again" '[ "$(wc -l < ~/smoketest-ran)" -eq 2 ]'
+}

smoketest/scenarios/41-failing-hook.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+# shellcheck disable=SC2034
+DESCRIPTION="a failing update hook fails the update but not the hooks after it"
+IMAGE=installed
+TIER=pr
+
+scenario() {
+  upstream_commit "Add a failing hook" '
+    printf "%s\n" "#!/bin/bash" "run_update() { echo smoketest: failing; return 1; }" > updates.d/13-smoketest-fail.sh
+    printf "%s\n" "#!/bin/bash" "run_update() { touch ~/smoketest-after; }" > updates.d/96-smoketest-after.sh'
+
+  cd_update
+  assert_status 1
+  assert_output "smoketest: failing"
+  assert_output "Done, with errors in: 13-smoketest-fail.sh"
+  assert_vm "the hook after it ran" 'test -f ~/smoketest-after'
+}

smoketest/scenarios/42-upstream-unreachable.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# shellcheck disable=SC2034
+DESCRIPTION="update when CoreDesktop's repo can't be reached runs the hooks on disk and reports it"
+IMAGE=installed
+TIER=pr
+
+scenario() {
+  assert_vm "take the upstream away" "sudo mv $UPSTREAM $UPSTREAM.away"
+  cd_update
+  assert_status 1
+  assert_output "could not pull the latest CoreDesktop"
+  assert_output "Done, with errors in: git pull"
+  assert_output "Updating Arch..."
+
+  assert_vm "bring it back" "sudo mv $UPSTREAM.away $UPSTREAM"
+  cd_update
+  assert_status 0
+}

smoketest/scenarios/43-local-edits.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# shellcheck disable=SC2034
+DESCRIPTION="local edits to the checkout: kept if upstream doesn't touch the file, reported if it does"
+IMAGE=installed
+TIER=pr
+
+# Expanded in the VM, not here
+# shellcheck disable=SC2088
+CD="~/.local/share/coredesktop"
+
+scenario() {
+  assert_vm "edit README.md in the checkout" "echo '# local edit' >> $CD/README.md"
+  upstream_commit "Change packages.conf" 'echo "# upstream change" >> packages.conf'
+  cd_update
+  assert_status 0
+  assert_vm "the edit is kept" "tail -n 1 $CD/README.md | grep -qx '# local edit'"
+  assert_vm "the upstream change arrived" "tail -n 1 $CD/packages.conf | grep -qx '# upstream change'"
+
+  upstream_commit "Change README.md" 'echo "# upstream readme change" >> README.md'
+  cd_update
+  assert_status 1
+  assert_output "Done, with errors in: git pull"
+
+  assert_vm "throw the edit away" "git -C $CD checkout -- README.md"
+  cd_update
+  assert_status 0
+}

smoketest/scenarios/44-low-disk.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+# shellcheck disable=SC2034
+DESCRIPTION="update refuses to start with less than 10 GiB free, unless UPDATE_FORCE=1"
+IMAGE=installed
+TIER=pr
+
+scenario() {
+  # Leave 5 GiB free on /
+  assert_vm "fill /" '
+    avail=$(df -k --output=avail / | tail -n 1 | tr -d " ")
+    sudo fallocate -l $(( (avail - 5 * 1024 * 1024) * 1024 )) /smoketest-fill'
+
+  cd_update
+  assert_status 1
+  assert_output "updating needs at least 10 GiB"
+  refute_output "Updating Arch..."
+
+  cd_update UPDATE_FORCE=1
+  assert_status 0
+}

smoketest/scenarios/50-stale-system.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+# shellcheck disable=SC2034
+DESCRIPTION="a machine a month behind on a stale mirror: update picks fresh mirrors, upgrades the kernel, asks for a reboot"
+IMAGE=installed
+TIER=nightly
+
+scenario() {
+  local date kernel_before
+  date="$(date -d '30 days ago' +%Y/%m/%d)"
+
+  # Go back to Arch as it was a month ago, from the Arch Linux Archive, which
+  # is also a mirror that stopped syncing a month ago
+  run_in_vm downgrade "set -e
+    echo 'Server = https://archive.archlinux.org/repos/$date/\$repo/os/\$arch' \
+      | sudo tee /etc/pacman.d/mirrorlist
+    sudo pacman -Syyuu --noconfirm"
+  [ "$STATUS" -eq 0 ] || skip "couldn't downgrade to Arch as of $date (see $LAST_LOG)"
+  vm_reboot
+  kernel_before="$(vm_value 'uname -r')"
+
+  run_in_vm version '"$HOME/.local/bin/version"'
+  assert_output "last synced with Arch"
+
+  local snapshots_before
+  snapshots_before="$(vm_value 'sudo snapper -c root list 2> /dev/null | wc -l')"
+
+  cd_update
+  assert_status 0
+  assert_output "Please reboot when convenient"
+  refute_vm "the first mirror isn't the archive any more" \
+    'grep -m1 "^Server" /etc/pacman.d/mirrorlist | grep -q archive.archlinux.org'
+  assert_vm "update took a snapshot first (if / is btrfs)" "
+    [ \"\$(findmnt -no FSTYPE /)\" != btrfs ] \
+      || [ \"\$(sudo snapper -c root list | wc -l)\" -gt $snapshots_before ]"
+
+  vm_reboot
+  refute_vm "a newer kernel is running" "[ \"\$(uname -r)\" = $(q "$kernel_before") ]"
+  health
+}

smoketest/scenarios/51-skip-packages.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# shellcheck disable=SC2034
+DESCRIPTION="update doesn't reinstall a package or Flatpak listed in skip-packages"
+IMAGE=installed
+TIER=pr
+
+scenario() {
+  assert_vm "remove Zoom and LibreOffice, and skip them" '
+    set -e
+    sudo flatpak uninstall --system --noninteractive us.zoom.Zoom
+    sudo pacman -Rns --noconfirm libreoffice-fresh
+    mkdir -p ~/.config/coredesktop
+    printf "%s\n" us.zoom.Zoom libreoffice-fresh >> ~/.config/coredesktop/skip-packages'
+
+  cd_update
+  assert_status 0
+  refute_vm "Zoom is still gone" 'flatpak info --system us.zoom.Zoom'
+  refute_vm "LibreOffice is still gone" 'pacman -Qi libreoffice-fresh'
+}

smoketest/scenarios/52-siblings.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+# shellcheck disable=SC2034
+DESCRIPTION="update pulls a sibling project and runs its update.sh; a failing one is reported"
+IMAGE=installed
+TIER=pr
+
+SIB="$SMOKETEST_VM_DIR/smoketest-sibling"
+
+scenario() {
+  assert_vm "make a sibling project" "set -e
+    git init -q $SIB
+    cd $SIB
+    echo 'touch ~/sibling-installed' > install.sh
+    echo 'echo v1 >> ~/sibling-updated' > update.sh
+    git add -A
+    git commit -qm v1"
+
+  run_in_vm sibling-add "\"\$HOME/.local/bin/sibling\" add $SIB"
+  assert_status 0
+  assert_vm "its install.sh ran" 'test -f ~/sibling-installed'
+
+  assert_vm "release v2" "set -e
+    cd $SIB
+    echo 'echo v2 >> ~/sibling-updated' > update.sh
+    git commit -qam v2"
+  cd_update
+  assert_status 0
+  assert_vm "v2's update.sh ran" 'tail -n 1 ~/sibling-updated | grep -qx v2'
+
+  assert_vm "release a v3 whose update.sh fails" "set -e
+    cd $SIB
+    echo 'exit 1' > update.sh
+    git commit -qam v3"
+  cd_update
+  assert_status 1
+  assert_output "smoketest-sibling's update.sh failed"
+  assert_output "Done, with errors in: 71-siblings.sh"
+}