diff --git a/bats/compatibility/.gitignore b/bats/compatibility/.gitignore new file mode 100644 index 00000000000..f5c7eac3d9e --- /dev/null +++ b/bats/compatibility/.gitignore @@ -0,0 +1 @@ +env_test/ diff --git a/bats/compatibility/README.md b/bats/compatibility/README.md new file mode 100644 index 00000000000..4de34a299b4 --- /dev/null +++ b/bats/compatibility/README.md @@ -0,0 +1,23 @@ +## Compatibility Tests + +These tests attempt to ensure forward and backward compatibility for Dolt versions. + +The testing script `runner.sh` checks out and builds older Dolt release versions to ensure that they can read data from +newer repositories and vice-versa. +A test directory `/env_test` is created outside of source control to preserve the environment across +`git checkout` commands. + +For each Dolt release version listed in `versions.txt`, `runner.sh` creates a legacy Dolt repository using the +`/test_files/setup_repo.sh` script in a directory named with the corresponding version. +An additional Dolt repository is created using Dolt built from the initial git branch. +BATS tests, located in `test_files/bats/`, are used to verify the forward and backward compatibility of all Dolt versions +and the repositories created with those versions. + +### Updating + +The BATS tests used to verify compatibility are inherently fragile. +Our primary integration tests in `/dolt/bats/` setup and tear down their environment for each test. +Because the tests rely on creating a repo with one version of Dolt and running BATS tests with a different version, +we cannot isolate their environment without building Dolt twice per test or setting up a different Dolt repo per test. +The initial version of these tests does all write operations in the `setup_repo.sh` script, and limits state modifications +within the BATS test to `dolt checkout` branch changes. Take care when editing the BATS tests to follow this pattern. \ No newline at end of file diff --git a/bats/compatibility/runner.sh b/bats/compatibility/runner.sh new file mode 100755 index 00000000000..8df2b9ae3fe --- /dev/null +++ b/bats/compatibility/runner.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +function build_dolt() { + pushd "$dolt_dir" > /dev/null || exit + git checkout "$1" > /dev/null + go install . + popd > /dev/null || exit +} + +function setup_dir() { + if [ -d "$1" ]; then rm -r "$1"; fi + mkdir "$1" + pushd "$1" > /dev/null || exit + "$top_dir/setup_repo.sh" > setup_repo.log + cp -r "$top_dir"/bats/* . + popd > /dev/null || exit +} + +function run_bats_tests() { + pushd "$1" > /dev/null || exit + cwd=$(pwd) + hash=$(git rev-parse HEAD) + echo "testing dolt @ $hash against repo in $cwd" + bats . + echo + popd > /dev/null || exit +} + +# ensure that we have a clean working change set before we begin +if [[ $(git diff --stat) != '' ]]; then + echo "cannot run compatibility tests with git working changes" + exit +fi + +# copy all the test files to take them out of source control +# when we checkout different Dolt releases we don't want to +# delete our environment +test_env="env_test" +rm -r $test_env +mkdir $test_env +cp -r test_files/* $test_env +pushd $test_env > /dev/null || exit + +top_dir=$(pwd) +starting_branch=$(git rev-parse --abbrev-ref HEAD) +dolt_dir="../../../go/cmd/dolt/" + +# setup a repository with dolt built +# from the current branch +build_dolt "$starting_branch" +setup_dir "head" + +while IFS= read -r ver +do + + build_dolt "$ver" + setup_dir "$ver" + + # run compatibility.bats to ensure dolt @ $ver can + # read a repo created with dolt @ HEAD + ver_hash=$(git rev-parse HEAD) + echo "hash for dolt @ $ver: $ver_hash" + run_bats_tests head + +done < <(grep -v '^ *#' < dolt_versions.txt) + +# now build dolt @ HEAD and make sure we can read +# all of the legacy repositories we created +build_dolt "$starting_branch" + +while IFS= read -r ver +do + head_hash=$(git rev-parse HEAD) + echo "hash for dolt @ head: $head_hash" + run_bats_tests "$ver" + +done < <(grep -v '^ *#' < dolt_versions.txt) + + +popd > /dev/null || exit \ No newline at end of file diff --git a/bats/compatibility/test_files/bats/compatibility.bats b/bats/compatibility/test_files/bats/compatibility.bats new file mode 100755 index 00000000000..6410cfe2abd --- /dev/null +++ b/bats/compatibility/test_files/bats/compatibility.bats @@ -0,0 +1,158 @@ +#!/usr/bin/env bats +load $BATS_TEST_DIRNAME/helper/common.bash + +setup() { + setup_common +} + +teardown() { + teardown_common +} +clear +@test "dolt version" { + run dolt version + [ "$status" -eq 0 ] + regex='dolt version [0-9]+.[0-9]+.[0-9]+' + [[ "$output" =~ $regex ]] || false +} + +@test "dolt status" { + run dolt status + [ "$status" -eq 0 ] + [[ "$output" =~ "On branch master" ]] || false + [[ "$output" =~ "nothing to commit, working tree clean" ]] || false +} + +@test "dolt ls" { + run dolt ls + [ "$status" -eq 0 ] + [[ "${lines[0]}" =~ "Tables in working set:" ]] || false +} + +@test "dolt branch" { + run dolt branch + [ "$status" -eq 0 ] +} + +@test "dolt diff" { + run dolt diff + [ "$status" -eq 0 ] +} + +@test "dolt schema show on branch init" { + run dolt checkout init + [ "$status" -eq 0 ] + + dolt schema show + run dolt schema show abc + [ "$status" -eq 0 ] + [[ "${lines[0]}" =~ "abc @ working" ]] || false + [[ "${lines[1]}" =~ "CREATE TABLE \`abc\` (" ]] || false + [[ "${lines[2]}" =~ " \`pk\` BIGINT NOT NULL COMMENT 'tag:0'," ]] || false + [[ "${lines[3]}" =~ " \`a\` LONGTEXT COMMENT 'tag:100'," ]] || false + [[ "${lines[4]}" =~ " \`b\` DOUBLE COMMENT 'tag:101'," ]] || false + [[ "${lines[5]}" =~ " \`w\` BIGINT COMMENT 'tag:102'," ]] || false + [[ "${lines[6]}" =~ " \`x\` BIGINT COMMENT 'tag:103'," ]] || false + [[ "${lines[7]}" =~ " PRIMARY KEY (\`pk\`)" ]] || false + [[ "${lines[8]}" =~ ");" ]] || false +} + +@test "dolt sql 'select * from abc' on branch init" { + # checkout we're on the right branch + run dolt status + [ "$status" -eq 0 ] + [[ "$output" =~ "On branch init" ]] || false + [[ "$output" =~ "nothing to commit, working tree clean" ]] || false + + dolt sql -q 'select * from abc;' + run dolt sql -q 'select * from abc;' + [ "$status" -eq 0 ] + + + [[ "${lines[1]}" =~ "| pk | a | b | w | x |" ]] || false + [[ "${lines[2]}" =~ "+----+------+-----+---+---+" ]] || false + [[ "${lines[3]}" =~ "| 0 | asdf | 1.1 | 0 | 0 |" ]] || false + [[ "${lines[4]}" =~ "| 1 | asdf | 1.1 | 0 | 0 |" ]] || false + [[ "${lines[5]}" =~ "| 2 | asdf | 1.1 | 0 | 0 |" ]] || false +} + +@test "dolt schema show on branch master" { + run dolt checkout master + [ "$status" -eq 0 ] + + dolt schema show + run dolt schema show abc + [ "$status" -eq 0 ] + [[ "${lines[0]}" =~ "abc @ working" ]] || false + [[ "${lines[1]}" =~ "CREATE TABLE \`abc\` (" ]] || false + [[ "${lines[2]}" =~ "\`pk\` BIGINT NOT NULL COMMENT 'tag:0'," ]] || false + [[ "${lines[3]}" =~ "\`a\` LONGTEXT COMMENT 'tag:100'," ]] || false + [[ "${lines[4]}" =~ "\`b\` DOUBLE COMMENT 'tag:101'," ]] || false + [[ "${lines[5]}" =~ "\`x\` BIGINT COMMENT 'tag:103'," ]] || false + [[ "${lines[6]}" =~ "\`y\` BIGINT COMMENT 'tag:104'," ]] || false + [[ "${lines[7]}" =~ "PRIMARY KEY (\`pk\`)" ]] || false + [[ "${lines[8]}" =~ ");" ]] || false +} + + +@test "dolt sql 'select * from abc' on branch master" { + # checkout we're on the right branch + run dolt status + [ "$status" -eq 0 ] + [[ "$output" =~ "On branch master" ]] || false + [[ "$output" =~ "nothing to commit, working tree clean" ]] || false + + dolt sql -q 'select * from abc;' + run dolt sql -q 'select * from abc;' + [ "$status" -eq 0 ] + [[ "${lines[1]}" =~ "| pk | a | b | x | y |" ]] || false + [[ "${lines[2]}" =~ "+----+------+-----+---+--------+" ]] || false + [[ "${lines[3]}" =~ "| 0 | asdf | 1.1 | 0 | |" ]] || false + [[ "${lines[4]}" =~ "| 2 | asdf | 1.1 | 0 | |" ]] || false + [[ "${lines[5]}" =~ "| 3 | data | 1.1 | 0 | |" ]] || false +} + +@test "dolt schema show on branch other" { + run dolt checkout other + [ "$status" -eq 0 ] + + dolt schema show + run dolt schema show abc + [ "$status" -eq 0 ] + [[ "${lines[0]}" =~ "abc @ working" ]] || false + [[ "${lines[1]}" =~ "CREATE TABLE \`abc\` (" ]] || false + [[ "${lines[2]}" =~ "\`pk\` BIGINT NOT NULL COMMENT 'tag:0'," ]] || false + [[ "${lines[3]}" =~ "\`a\` LONGTEXT COMMENT 'tag:100'," ]] || false + [[ "${lines[4]}" =~ "\`b\` DOUBLE COMMENT 'tag:101'," ]] || false + [[ "${lines[5]}" =~ "\`w\` BIGINT COMMENT 'tag:102'," ]] || false + [[ "${lines[6]}" =~ "\`z\` BIGINT COMMENT 'tag:105'," ]] || false + [[ "${lines[7]}" =~ "PRIMARY KEY (\`pk\`)" ]] || false + [[ "${lines[8]}" =~ ");" ]] || false +} + +@test "dolt sql 'select * from abc' on branch other" { + # checkout we're on the right branch + run dolt status + [ "$status" -eq 0 ] + [[ "$output" =~ "On branch other" ]] || false + [[ "$output" =~ "nothing to commit, working tree clean" ]] || false + + dolt sql -q 'select * from abc;' + run dolt sql -q 'select * from abc;' + [ "$status" -eq 0 ] + [[ "${lines[1]}" =~ "| pk | a | b | w | z |" ]] || false + [[ "${lines[2]}" =~ "+----+------+-----+---+--------+" ]] || false + [[ "${lines[3]}" =~ "| 0 | asdf | 1.1 | 0 | |" ]] || false + [[ "${lines[4]}" =~ "| 1 | asdf | 1.1 | 0 | |" ]] || false + [[ "${lines[5]}" =~ "| 4 | data | 1.1 | 0 | |" ]] || false + + dolt checkout master +} + +@test "dolt table import" { + run dolt table import -c -s abc_schema.json abc2 abc.csv + [ "$status" -eq 0 ] + [[ "$output" =~ "Import completed successfully." ]] || false + + dolt sql -q 'drop table abc2' +} \ No newline at end of file diff --git a/bats/compatibility/test_files/bats/helper/common.bash b/bats/compatibility/test_files/bats/helper/common.bash new file mode 100644 index 00000000000..b2130f32dcb --- /dev/null +++ b/bats/compatibility/test_files/bats/helper/common.bash @@ -0,0 +1,16 @@ +load helper/windows-compat + +if [ -z "$BATS_TMPDIR" ]; then + export BATS_TMPDIR=$HOME/batstmp/ + mkdir $BATS_TMPDIR +fi + +setup_common() { + echo "setup" > /dev/null +} + +teardown_common() { + echo "teardown" > /dev/null +} + +dolt config --global --add metrics.disabled true > /dev/null 2>&1 diff --git a/bats/compatibility/test_files/bats/helper/windows-compat.bash b/bats/compatibility/test_files/bats/helper/windows-compat.bash new file mode 100644 index 00000000000..3a7049eccc9 --- /dev/null +++ b/bats/compatibility/test_files/bats/helper/windows-compat.bash @@ -0,0 +1,24 @@ +nativepath() { echo "$1"; } +nativevar() { eval export "$1"="$2"; } +skiponwindows() { :; } + +IS_WINDOWS=false + +if [ -d /mnt/c/Windows/System32 ]; then + IS_WINDOWS=true + if [ ! -d /mnt/c/batstmp ]; then + mkdir /mnt/c/batstmp + fi + BATS_TMPDIR=`TMPDIR=/mnt/c/batstmp mktemp -d -t dolt-bats-tests-XXXXXX` + export BATS_TMPDIR + nativepath() { + wslpath -w "$1" + } + nativevar() { + eval export "$1"="$2" + export WSLENV="$1$3" + } + skiponwindows() { + skip "$1" + } +fi diff --git a/bats/compatibility/test_files/dolt_versions.txt b/bats/compatibility/test_files/dolt_versions.txt new file mode 100644 index 00000000000..dcd458281f2 --- /dev/null +++ b/bats/compatibility/test_files/dolt_versions.txt @@ -0,0 +1,2 @@ +v0.13.0 +v0.14.0 \ No newline at end of file diff --git a/bats/compatibility/test_files/setup_repo.sh b/bats/compatibility/test_files/setup_repo.sh new file mode 100755 index 00000000000..6660192d8ff --- /dev/null +++ b/bats/compatibility/test_files/setup_repo.sh @@ -0,0 +1,62 @@ +#!/bin/bash + +dolt init + +dolt sql <