coredesktop/tests/smoketest.bats

#!/usr/bin/env bats

load test_helper

# smoketest/lib/scenario.sh with a "VM" that is just a local shell in the
# test's HOME, and a small repo of its own standing in for CoreDesktop

setup() {
  common_setup
  export GIT_AUTHOR_NAME=Test GIT_AUTHOR_EMAIL=test@localhost
  export GIT_COMMITTER_NAME=Test GIT_COMMITTER_EMAIL=test@localhost

  # Two commits: install.sh writes which one ran, and what it read from stdin
  export SMOKETEST_REPO="$BATS_TEST_TMPDIR/repo"
  git init -q "$SMOKETEST_REPO"
  cd "$SMOKETEST_REPO"
  printf '%s\n' 'echo v1 > ~/installed' 'read -r line; echo "read: $line" >> ~/installed' > install.sh
  git add install.sh
  git commit -qm v1
  V1="$(git rev-parse HEAD)"
  sed -i 's/v1/v2/' install.sh
  git commit -qam v2
  V2="$(git rev-parse HEAD)"
  cd "$BATS_TEST_TMPDIR"

  export SMOKETEST_VM_DIR="$BATS_TEST_TMPDIR/vm"
  mkdir -p "$SMOKETEST_VM_DIR"
  export SMOKETEST_RESULTS="$BATS_TEST_TMPDIR/results"
  export SMOKETEST_SHARED="$BATS_TEST_TMPDIR/shared"
  mkdir -p "$SMOKETEST_RESULTS" "$SMOKETEST_SHARED"
  export SMOKETEST_REF="$V2"

  vm_exec() { bash -c "$1" < /dev/null; }
  vm_put() { cp "$1" "$2"; }
  source "$REPO/smoketest/lib/scenario.sh"
}

upstream_main() {
  git --git-dir="$UPSTREAM" rev-parse main
}

@test "prepare makes the fake upstream, at the commit under test" {
  smoketest_prepare
  [ "$(upstream_main)" = "$V2" ]
}

@test "prepare sends CoreDesktop's URLs to the fake upstream" {
  smoketest_prepare
  for url in "${COREDESKTOP_URLS[@]}"; do
    git clone -q "$url" "$BATS_TEST_TMPDIR/clone"
    [ "$(git -C "$BATS_TEST_TMPDIR/clone" rev-parse HEAD)" = "$V2" ]
    rm -rf "$BATS_TEST_TMPDIR/clone"
  done
}

@test "prepare can run twice" {
  smoketest_prepare
  smoketest_prepare
  [ "$(git config --global --get-all "url.$UPSTREAM.insteadOf" | wc -l)" -eq "${#COREDESKTOP_URLS[@]}" ]
}

@test "prepare on the installed image updates it to the commit under test" {
  export IMAGE=installed IMAGE_REF="$V1"
  mkdir -p "$HOME/.local/bin"
  printf '%s\n' '#!/bin/bash' 'echo "update ran with $(git --git-dir="$UPSTREAM" rev-parse main)"' \
    > "$HOME/.local/bin/update"
  chmod +x "$HOME/.local/bin/update"
  export UPSTREAM

  run smoketest_prepare
  [ "$status" -eq 0 ]
  grep -qx "update ran with $V2" "$SMOKETEST_RESULTS/01-update.log"
}

@test "prepare on the installed image fails if the image isn't behind the commit under test" {
  export IMAGE=installed IMAGE_REF="$V2" SMOKETEST_REF="$V1"

  run smoketest_prepare
  [ "$status" -eq 1 ]
  [[ "$output" == *"isn't in the history of the commit under test"* ]]
}

@test "upstream_set moves main back and forth" {
  smoketest_prepare
  upstream_set "$V1"
  [ "$(upstream_main)" = "$V1" ]
  upstream_set "$V2"
  [ "$(upstream_main)" = "$V2" ]
}

@test "upstream_set fails on an unknown commit" {
  smoketest_prepare
  run upstream_set nosuchcommit
  [ "$status" -eq 1 ]
  [[ "$output" == *"no such commit: nosuchcommit"* ]]
}

@test "upstream_commit releases a commit made by its script" {
  smoketest_prepare
  upstream_commit "Add a file" 'echo hello > new-file'

  [ "$(git --git-dir="$UPSTREAM" rev-parse main^)" = "$V2" ]
  [ "$(git --git-dir="$UPSTREAM" show main:new-file)" = hello ]
  [ "$(git --git-dir="$UPSTREAM" log -1 --format=%s main)" = "Add a file" ]
}

@test "upstream_commit fails the scenario if its script fails" {
  smoketest_prepare
  run upstream_commit "Broken" 'false'
  [ "$status" -eq 1 ]
  [ "$(upstream_main)" = "$V2" ]
}

@test "cd_install runs main's install.sh, with nothing on stdin" {
  smoketest_prepare
  upstream_set "$V1"
  cd_install
  [ "$STATUS" -eq 0 ]
  [ "$(cat "$HOME/installed")" = "$(printf 'v1\nread: ')" ]
}

@test "run_in_vm keeps the output, status and a numbered log, and doesn't fail" {
  run_in_vm first 'echo one; exit 3'
  [ "$STATUS" -eq 3 ]
  [ "$OUTPUT" = one ]
  [ "$(cat "$SMOKETEST_RESULTS/01-first.log")" = one ]

  run_in_vm second 'echo two >&2'
  [ "$STATUS" -eq 0 ]
  [ "$OUTPUT" = two ]
  [ -f "$SMOKETEST_RESULTS/02-second.log" ]
}

@test "run_in_vm writes its log while the script runs" {
  run_in_vm slow 'echo early; cat "$SMOKETEST_RESULTS/01-slow.log" > "$HOME/seen"'
  [ "$STATUS" -eq 0 ]
  [ "$(cat "$HOME/seen")" = early ]
}

@test "cd_update passes its variables to update" {
  mkdir -p "$HOME/.local/bin"
  printf '%s\n' '#!/bin/bash' 'echo "force=$UPDATE_FORCE"' > "$HOME/.local/bin/update"
  chmod +x "$HOME/.local/bin/update"

  cd_update UPDATE_FORCE=1
  [ "$STATUS" -eq 0 ]
  [ "$OUTPUT" = "force=1" ]
}

@test "assert_status and assert_output fail with the log's path" {
  run_in_vm thing 'echo hello; exit 1'

  run assert_status 0
  [ "$status" -eq 1 ]
  [[ "$output" == *"thing exited 1, expected 0"*"01-thing.log"* ]]

  run assert_output goodbye
  [ "$status" -eq 1 ]
  run assert_output hello
  [ "$status" -eq 0 ]
  run refute_output hello
  [ "$status" -eq 1 ]
}

@test "assert_vm and refute_vm" {
  run assert_vm "true passes" true
  [ "$status" -eq 0 ]
  run assert_vm "false fails" 'echo why; false'
  [ "$status" -eq 1 ]
  [[ "$output" == *why*"FAIL: false fails"* ]]
  run refute_vm "false passes" false
  [ "$status" -eq 0 ]
  run refute_vm "true fails" true
  [ "$status" -eq 1 ]
}

@test "assert_same_fingerprint passes on a match, and saves the diff when they differ" {
  printf 'a\nb\n' > "$BATS_TEST_TMPDIR/one"
  printf 'a\nb\n' > "$BATS_TEST_TMPDIR/two"
  printf 'a\nc\n' > "$BATS_TEST_TMPDIR/three"

  run assert_same_fingerprint "$BATS_TEST_TMPDIR/one" "$BATS_TEST_TMPDIR/two"
  [ "$status" -eq 0 ]

  run assert_same_fingerprint "$BATS_TEST_TMPDIR/one" "$BATS_TEST_TMPDIR/three"
  [ "$status" -eq 1 ]
  grep -qx -- '-b' "$SMOKETEST_RESULTS/fingerprint.diff"
  grep -qx -- '+c' "$SMOKETEST_RESULTS/fingerprint.diff"
}

@test "skip exits with the skip status" {
  run skip "not today"
  [ "$status" -eq "$SKIP_STATUS" ]
  [[ "$output" == *"SKIP: not today"* ]]
}

@test "every scenario sets a description, image and tier, and defines scenario" {
  for file in "$REPO"/smoketest/scenarios/*.sh; do
    (
      unset DESCRIPTION IMAGE TIER
      unset -f scenario
      source "$file"
      [ -n "$DESCRIPTION" ] || { echo "$file: no DESCRIPTION"; exit 1; }
      [[ "$IMAGE" == base || "$IMAGE" == installed ]] || { echo "$file: IMAGE is '$IMAGE'"; exit 1; }
      [[ "$TIER" == pr || "$TIER" == nightly ]] || { echo "$file: TIER is '$TIER'"; exit 1; }
      declare -f scenario > /dev/null || { echo "$file: no scenario function"; exit 1; }
    )
  done
}

@test "every baseline is a commit in this repo" {
  sed 's/#.*//' "$REPO/smoketest/baselines" | awk 'NF { print $1 }' | while read -r ref; do
    git -C "$REPO" rev-parse --verify --quiet "$ref^{commit}" > /dev/null || { echo "unknown: $ref"; exit 1; }
  done
}