Merge pull request #429 from liquidata-inc/andy/back_compat

andy/Test compatibility between Dolt versions

bats/compatibility/.gitignore

@@ -0,0 +1 @@
+env_test/

bats/compatibility/README.md

@@ -0,0 +1,23 @@
+## Compatibility Tests
+
+These tests attempt to ensure forward and backward compatibility for Dolt versions.
+
+The testing script `runner.sh` checks out and builds older Dolt release versions to ensure that they can read data from
+newer repositories and vice-versa.
+A test directory `/env_test` is created outside of source control to preserve the environment across
+`git checkout` commands.
+
+For each Dolt release version listed in `versions.txt`, `runner.sh` creates a legacy Dolt repository using the
+`/test_files/setup_repo.sh` script in a directory named with the corresponding version.
+An additional Dolt repository is created using Dolt built from the initial git branch.
+BATS tests, located in `test_files/bats/`, are used to verify the forward and backward compatibility of all Dolt versions
+and the repositories created with those versions.
+
+### Updating
+
+The BATS tests used to verify compatibility are inherently fragile.
+Our primary integration tests in `/dolt/bats/` setup and tear down their environment for each test.
+Because the tests rely on creating a repo with one version of Dolt and running BATS tests with a different version, 
+we cannot isolate their environment without building Dolt twice per test or setting up a different Dolt repo per test.
+The initial version of these tests does all write operations in the `setup_repo.sh` script, and limits state modifications
+within the BATS test to `dolt checkout` branch changes. Take care when editing the BATS tests to follow this pattern.

bats/compatibility/runner.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+function build_dolt() {
+  pushd "$dolt_dir" > /dev/null || exit
+  git checkout "$1" > /dev/null
+  go install .
+  popd > /dev/null || exit
+}
+
+function setup_dir() {
+  if [ -d "$1" ]; then rm -r "$1"; fi
+  mkdir "$1"
+  pushd "$1" > /dev/null || exit
+  "$top_dir/setup_repo.sh" > setup_repo.log
+  cp -r "$top_dir"/bats/* .
+  popd > /dev/null || exit
+}
+
+function run_bats_tests() {
+  pushd "$1" > /dev/null || exit
+  cwd=$(pwd)
+  hash=$(git rev-parse HEAD)
+  echo "testing dolt @ $hash against repo in $cwd"
+  bats .
+  echo
+  popd > /dev/null || exit
+}
+
+# ensure that we have a clean working change set before we begin
+if [[ $(git diff --stat) != '' ]]; then
+  echo "cannot run compatibility tests with git working changes"
+  exit
+fi
+
+# copy all the test files to take them out of source control
+# when we checkout different Dolt releases we don't want to
+# delete our environment
+test_env="env_test"
+rm -r $test_env
+mkdir $test_env
+cp -r test_files/* $test_env
+pushd $test_env > /dev/null || exit
+
+top_dir=$(pwd)
+starting_branch=$(git rev-parse --abbrev-ref HEAD)
+dolt_dir="../../../go/cmd/dolt/"
+
+# setup a repository with dolt built
+# from the current branch
+build_dolt "$starting_branch"
+setup_dir "head"
+
+while IFS= read -r ver
+do
+
+  build_dolt "$ver"
+  setup_dir "$ver"
+
+  # run compatibility.bats to ensure dolt @ $ver can
+  # read a repo created with dolt @ HEAD
+  ver_hash=$(git rev-parse HEAD)
+  echo "hash for dolt @ $ver: $ver_hash"
+  run_bats_tests head
+
+done < <(grep -v '^ *#' < dolt_versions.txt)
+
+# now build dolt @ HEAD and make sure we can read
+# all of the legacy repositories we created
+build_dolt "$starting_branch"
+
+while IFS= read -r ver
+do
+  head_hash=$(git rev-parse HEAD)
+  echo "hash for dolt @ head: $head_hash"
+  run_bats_tests "$ver"
+
+done < <(grep -v '^ *#' < dolt_versions.txt)
+
+
+popd > /dev/null || exit

bats/compatibility/test_files/bats/compatibility.bats

@@ -0,0 +1,158 @@
+#!/usr/bin/env bats
+load $BATS_TEST_DIRNAME/helper/common.bash
+
+setup() {
+    setup_common
+}
+
+teardown() {
+    teardown_common
+}
+clear
+@test "dolt version" {
+    run dolt version
+    [ "$status" -eq 0 ]
+    regex='dolt version [0-9]+.[0-9]+.[0-9]+'
+    [[ "$output" =~ $regex ]] || false
+}
+
+@test "dolt status" {
+    run dolt status
+    [ "$status" -eq 0 ]
+    [[ "$output" =~ "On branch master" ]] || false
+    [[ "$output" =~ "nothing to commit, working tree clean" ]] || false
+}
+
+@test "dolt ls" {
+    run dolt ls
+    [ "$status" -eq 0 ]
+    [[ "${lines[0]}" =~ "Tables in working set:" ]] || false
+}
+
+@test "dolt branch" {
+    run dolt branch
+    [ "$status" -eq 0 ]
+}
+
+@test "dolt diff" {
+    run dolt diff
+    [ "$status" -eq 0 ]
+}
+
+@test "dolt schema show on branch init" {
+    run dolt checkout init
+    [ "$status" -eq 0 ]
+
+    dolt schema show
+    run dolt schema show abc
+    [ "$status" -eq 0 ]
+    [[ "${lines[0]}" =~ "abc @ working" ]] || false
+    [[ "${lines[1]}" =~ "CREATE TABLE \`abc\` (" ]] || false
+    [[ "${lines[2]}" =~ " \`pk\` BIGINT NOT NULL COMMENT 'tag:0'," ]] || false
+    [[ "${lines[3]}" =~ " \`a\` LONGTEXT COMMENT 'tag:100'," ]] || false
+    [[ "${lines[4]}" =~ " \`b\` DOUBLE COMMENT 'tag:101'," ]] || false
+    [[ "${lines[5]}" =~ " \`w\` BIGINT COMMENT 'tag:102'," ]] || false
+    [[ "${lines[6]}" =~ " \`x\` BIGINT COMMENT 'tag:103'," ]] || false
+    [[ "${lines[7]}" =~ " PRIMARY KEY (\`pk\`)" ]] || false
+    [[ "${lines[8]}" =~ ");" ]] || false
+}
+
+@test "dolt sql 'select * from abc' on branch init" {
+    # checkout we're on the right branch
+    run dolt status
+    [ "$status" -eq 0 ]
+    [[ "$output" =~ "On branch init" ]] || false
+    [[ "$output" =~ "nothing to commit, working tree clean" ]] || false
+
+    dolt sql -q 'select * from abc;'
+    run dolt sql -q 'select * from abc;'
+    [ "$status" -eq 0 ]
+
+
+    [[ "${lines[1]}" =~ "| pk | a    | b   | w | x |" ]] || false
+    [[ "${lines[2]}" =~ "+----+------+-----+---+---+" ]] || false
+    [[ "${lines[3]}" =~ "| 0  | asdf | 1.1 | 0 | 0 |" ]] || false
+    [[ "${lines[4]}" =~ "| 1  | asdf | 1.1 | 0 | 0 |" ]] || false
+    [[ "${lines[5]}" =~ "| 2  | asdf | 1.1 | 0 | 0 |" ]] || false
+}
+
+@test "dolt schema show on branch master" {
+    run dolt checkout master
+    [ "$status" -eq 0 ]
+
+    dolt schema show
+    run dolt schema show abc
+    [ "$status" -eq 0 ]
+    [[ "${lines[0]}" =~ "abc @ working" ]] || false
+    [[ "${lines[1]}" =~ "CREATE TABLE \`abc\` (" ]] || false
+    [[ "${lines[2]}" =~ "\`pk\` BIGINT NOT NULL COMMENT 'tag:0'," ]] || false
+    [[ "${lines[3]}" =~ "\`a\` LONGTEXT COMMENT 'tag:100'," ]] || false
+    [[ "${lines[4]}" =~ "\`b\` DOUBLE COMMENT 'tag:101'," ]] || false
+    [[ "${lines[5]}" =~ "\`x\` BIGINT COMMENT 'tag:103'," ]] || false
+    [[ "${lines[6]}" =~ "\`y\` BIGINT COMMENT 'tag:104'," ]] || false
+    [[ "${lines[7]}" =~ "PRIMARY KEY (\`pk\`)" ]] || false
+    [[ "${lines[8]}" =~ ");" ]] || false
+}
+
+
+@test "dolt sql 'select * from abc' on branch master" {
+    # checkout we're on the right branch
+    run dolt status
+    [ "$status" -eq 0 ]
+    [[ "$output" =~ "On branch master" ]] || false
+    [[ "$output" =~ "nothing to commit, working tree clean" ]] || false
+
+    dolt sql -q 'select * from abc;'
+    run dolt sql -q 'select * from abc;'
+    [ "$status" -eq 0 ]
+    [[ "${lines[1]}" =~ "| pk | a    | b   | x | y      |" ]] || false
+    [[ "${lines[2]}" =~ "+----+------+-----+---+--------+" ]] || false
+    [[ "${lines[3]}" =~ "| 0  | asdf | 1.1 | 0 | <NULL> |" ]] || false
+    [[ "${lines[4]}" =~ "| 2  | asdf | 1.1 | 0 | <NULL> |" ]] || false
+    [[ "${lines[5]}" =~ "| 3  | data | 1.1 | 0 | <NULL> |" ]] || false
+}
+
+@test "dolt schema show on branch other" {
+    run dolt checkout other
+    [ "$status" -eq 0 ]
+
+    dolt schema show
+    run dolt schema show abc
+    [ "$status" -eq 0 ]
+    [[ "${lines[0]}" =~ "abc @ working" ]] || false
+    [[ "${lines[1]}" =~ "CREATE TABLE \`abc\` (" ]] || false
+    [[ "${lines[2]}" =~ "\`pk\` BIGINT NOT NULL COMMENT 'tag:0'," ]] || false
+    [[ "${lines[3]}" =~ "\`a\` LONGTEXT COMMENT 'tag:100'," ]] || false
+    [[ "${lines[4]}" =~ "\`b\` DOUBLE COMMENT 'tag:101'," ]] || false
+    [[ "${lines[5]}" =~ "\`w\` BIGINT COMMENT 'tag:102'," ]] || false
+    [[ "${lines[6]}" =~ "\`z\` BIGINT COMMENT 'tag:105'," ]] || false
+    [[ "${lines[7]}" =~ "PRIMARY KEY (\`pk\`)" ]] || false
+    [[ "${lines[8]}" =~ ");" ]] || false
+}
+
+@test "dolt sql 'select * from abc' on branch other" {
+    # checkout we're on the right branch
+    run dolt status
+    [ "$status" -eq 0 ]
+    [[ "$output" =~ "On branch other" ]] || false
+    [[ "$output" =~ "nothing to commit, working tree clean" ]] || false
+
+    dolt sql -q 'select * from abc;'
+    run dolt sql -q 'select * from abc;'
+    [ "$status" -eq 0 ]
+    [[ "${lines[1]}" =~ "| pk | a    | b   | w | z      |" ]] || false
+    [[ "${lines[2]}" =~ "+----+------+-----+---+--------+" ]] || false
+    [[ "${lines[3]}" =~ "| 0  | asdf | 1.1 | 0 | <NULL> |" ]] || false
+    [[ "${lines[4]}" =~ "| 1  | asdf | 1.1 | 0 | <NULL> |" ]] || false
+    [[ "${lines[5]}" =~ "| 4  | data | 1.1 | 0 | <NULL> |" ]] || false
+
+    dolt checkout master
+}
+
+@test "dolt table import" {
+    run dolt table import -c -s abc_schema.json abc2 abc.csv
+    [ "$status" -eq 0 ]
+    [[ "$output" =~ "Import completed successfully." ]] || false
+
+    dolt sql -q 'drop table abc2'
+}

bats/compatibility/test_files/bats/helper/common.bash

@@ -0,0 +1,16 @@
+load helper/windows-compat
+
+if [ -z "$BATS_TMPDIR" ]; then
+    export BATS_TMPDIR=$HOME/batstmp/
+    mkdir $BATS_TMPDIR
+fi
+
+setup_common() {
+    echo "setup" > /dev/null
+}
+
+teardown_common() {
+    echo "teardown" > /dev/null
+}
+
+dolt config --global --add metrics.disabled true > /dev/null 2>&1

bats/compatibility/test_files/bats/helper/windows-compat.bash

@@ -0,0 +1,24 @@
+nativepath() { echo "$1"; }
+nativevar() { eval export "$1"="$2"; }
+skiponwindows() { :; }
+
+IS_WINDOWS=false
+
+if [ -d /mnt/c/Windows/System32 ]; then
+    IS_WINDOWS=true
+    if [ ! -d /mnt/c/batstmp ]; then
+        mkdir /mnt/c/batstmp
+    fi
+    BATS_TMPDIR=`TMPDIR=/mnt/c/batstmp mktemp -d -t dolt-bats-tests-XXXXXX`
+    export BATS_TMPDIR
+    nativepath() {
+        wslpath -w "$1"
+    }
+    nativevar() {
+        eval export "$1"="$2"
+        export WSLENV="$1$3"
+    }
+    skiponwindows() {
+        skip "$1"
+    }
+fi

bats/compatibility/test_files/dolt_versions.txt

@@ -0,0 +1,2 @@
+v0.13.0
+v0.14.0

bats/compatibility/test_files/setup_repo.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+dolt init
+
+dolt sql <<SQL
+CREATE TABLE abc (
+  pk BIGINT NOT NULL COMMENT 'tag:0',
+  a LONGTEXT COMMENT 'tag:100',
+  b DOUBLE COMMENT 'tag:101',
+  w BIGINT COMMENT 'tag:102',
+  x BIGINT COMMENT 'tag:103',
+  PRIMARY KEY (pk)
+);
+INSERT INTO abc VALUES (0, 'asdf', 1.1, 0, 0);
+INSERT INTO abc VALUES (1, 'asdf', 1.1, 0, 0);
+INSERT INTO abc VALUES (2, 'asdf', 1.1, 0, 0);
+SQL
+dolt add .
+dolt commit -m "initialized data"
+dolt branch init
+
+
+dolt branch other
+dolt sql <<SQL
+DELETE FROM abc WHERE pk=1;
+INSERT INTO abc VALUES (3, 'data', 1.1, 0, 0);
+ALTER TABLE abc DROP COLUMN w;
+ALTER TABLE abc ADD COLUMN y BIGINT COMMENT 'tag:104';
+SQL
+dolt add .
+dolt commit -m "made changes to master"
+
+dolt checkout other
+dolt sql <<SQL
+DELETE FROM abc WHERE pk=2;
+INSERT INTO abc VALUES (4, 'data', 1.1, 0, 0);
+ALTER TABLE abc DROP COLUMN x;
+ALTER TABLE abc ADD COLUMN z BIGINT COMMENT 'tag:105';
+SQL
+dolt add .
+dolt commit -m "made changes to other"
+
+dolt checkout master
+dolt table export abc abc.csv
+dolt schema export abc abc_schema.json
+
+# add info to the log
+echo
+echo "dolt status"
+dolt status
+
+echo
+echo "dolt branch"
+dolt branch
+
+echo
+echo "dolt schema show"
+dolt schema show
+
+echo
+echo "dolt sql -q 'select * from abc;'"
+dolt sql -q 'select * from abc;'