update

.check-build

@@ -7,9 +7,9 @@ REFERENCE_FILES=(
     src/mrapps/crash.go
     src/mrapps/indexer.go
     src/mrapps/mtiming.go
-    src/mrapps/rtiming.go
-    src/mrapps/crash.go
     src/mrapps/nocrash.go
+    src/mrapps/rtiming.go
+    src/mrapps/wc.go
     src/main/mrsequential.go
     src/main/mrmaster.go
     src/main/mrworker.go

src/.gitignore

@@ -5,3 +5,4 @@ mrtmp.*
 /mapreduce/x.txt
 /pbservice/x.txt
 /kvpaxos/x.txt
+*.so

src/kvraft/test_test.go

@@ -1,7 +1,7 @@
 package kvraft
 
-import "../linearizability"
-
+import "../porcupine"
+import "../models"
 import "testing"
 import "strconv"
 import "time"
@@ -10,6 +10,8 @@ import "log"
 import "strings"
 import "sync"
 import "sync/atomic"
+import "fmt"
+import "io/ioutil"
 
 // The tester generously allows solutions to complete elections in one second
 // (much more than the paper's range of timeouts).
@@ -322,7 +324,7 @@ func GenericTestLinearizability(t *testing.T, part string, nclients int, nserver
 	cfg.begin(title)
 
 	begin := time.Now()
-	var operations []linearizability.Operation
+	var operations []porcupine.Operation
 	var opMu sync.Mutex
 
 	done_partitioner := int32(0)
@@ -344,24 +346,24 @@ func GenericTestLinearizability(t *testing.T, part string, nclients int, nserver
 			for atomic.LoadInt32(&done_clients) == 0 {
 				key := strconv.Itoa(rand.Int() % nclients)
 				nv := "x " + strconv.Itoa(cli) + " " + strconv.Itoa(j) + " y"
-				var inp linearizability.KvInput
-				var out linearizability.KvOutput
+				var inp models.KvInput
+				var out models.KvOutput
 				start := int64(time.Since(begin))
 				if (rand.Int() % 1000) < 500 {
 					Append(cfg, myck, key, nv)
-					inp = linearizability.KvInput{Op: 2, Key: key, Value: nv}
+					inp = models.KvInput{Op: 2, Key: key, Value: nv}
 					j++
 				} else if (rand.Int() % 1000) < 100 {
 					Put(cfg, myck, key, nv)
-					inp = linearizability.KvInput{Op: 1, Key: key, Value: nv}
+					inp = models.KvInput{Op: 1, Key: key, Value: nv}
 					j++
 				} else {
 					v := Get(cfg, myck, key)
-					inp = linearizability.KvInput{Op: 0, Key: key}
-					out = linearizability.KvOutput{Value: v}
+					inp = models.KvInput{Op: 0, Key: key}
+					out = models.KvOutput{Value: v}
 				}
 				end := int64(time.Since(begin))
-				op := linearizability.Operation{Input: inp, Call: start, Output: out, Return: end}
+				op := porcupine.Operation{Input: inp, Call: start, Output: out, Return: end, ClientId: cli}
 				opMu.Lock()
 				operations = append(operations, op)
 				opMu.Unlock()
@@ -423,13 +425,23 @@ func GenericTestLinearizability(t *testing.T, part string, nclients int, nserver
 
 	cfg.end()
 
-	// log.Printf("Checking linearizability of %d operations", len(operations))
-	// start := time.Now()
-	ok := linearizability.CheckOperationsTimeout(linearizability.KvModel(), operations, linearizabilityCheckTimeout)
-	// dur := time.Since(start)
-	// log.Printf("Linearizability check done in %s; result: %t", time.Since(start).String(), ok)
-	if !ok {
+	res, info := porcupine.CheckOperationsVerbose(models.KvModel, operations, linearizabilityCheckTimeout)
+	if res == porcupine.Illegal {
+		file, err := ioutil.TempFile("", "*.html")
+		if err != nil {
+			fmt.Printf("info: failed to create temp file for visualization")
+		} else {
+			err = porcupine.Visualize(models.KvModel, info, file)
+			if err != nil {
+				fmt.Printf("info: failed to write history visualization to %s\n", file.Name())
+			} else {
+				fmt.Printf("info: wrote history visualization to %s\n", file.Name())
+			}
+		}
+		t.Fatal("history is not linearizable")
 		t.Fatal("history is not linearizable")
+	} else if res == porcupine.Unknown {
+		fmt.Println("info: linearizability check timed out, assuming history is ok")
 	}
 }
 

src/main/test-mr.sh

@@ -1,10 +1,10 @@
 #!/bin/sh
 
-# 
+#
 # basic map-reduce test
 #
 
-RACE= 
+RACE=
 
 # uncomment this to run the tests with the Go race detector.
 #RACE=-race
@@ -26,6 +26,8 @@ rm -f mr-*
 (cd .. && go build $RACE mrworker.go) || exit 1
 (cd .. && go build $RACE mrsequential.go) || exit 1
 
+failed_any=0
+
 # first word-count
 
 # generate the correct output
@@ -35,15 +37,15 @@ rm -f mr-out*
 
 echo '***' Starting wc test.
 
-../mrmaster ../pg*txt &
+timeout -k 2s 180s ../mrmaster ../pg*txt &
 
 # give the master time to create the sockets.
 sleep 1
 
 # start multiple workers.
-../mrworker ../../mrapps/wc.so &
-../mrworker ../../mrapps/wc.so &
-../mrworker ../../mrapps/wc.so &
+timeout -k 2s 180s ../mrworker ../../mrapps/wc.so &
+timeout -k 2s 180s ../mrworker ../../mrapps/wc.so &
+timeout -k 2s 180s ../mrworker ../../mrapps/wc.so &
 
 # wait for one of the processes to exit.
 # under bash, this waits for all processes,
@@ -61,7 +63,7 @@ then
 else
   echo '---' wc output is not the same as mr-correct-wc.txt
   echo '---' wc test: FAIL
-  exit 1
+  failed_any=1
 fi
 
 # wait for remaining workers and master to exit.
@@ -77,12 +79,12 @@ rm -f mr-out*
 
 echo '***' Starting indexer test.
 
-../mrmaster ../pg*txt &
+timeout -k 2s 180s ../mrmaster ../pg*txt &
 sleep 1
 
 # start multiple workers
-../mrworker ../../mrapps/indexer.so &
-../mrworker ../../mrapps/indexer.so
+timeout -k 2s 180s ../mrworker ../../mrapps/indexer.so &
+timeout -k 2s 180s ../mrworker ../../mrapps/indexer.so
 
 sort mr-out* | grep . > mr-indexer-all
 if cmp mr-indexer-all mr-correct-indexer.txt
@@ -91,7 +93,7 @@ then
 else
   echo '---' indexer output is not the same as mr-correct-indexer.txt
   echo '---' indexer test: FAIL
-  exit 1
+  failed_any=1
 fi
 
 wait ; wait
@@ -101,18 +103,18 @@ echo '***' Starting map parallelism test.
 
 rm -f mr-out* mr-worker*
 
-../mrmaster ../pg*txt &
+timeout -k 2s 180s ../mrmaster ../pg*txt &
 sleep 1
 
-../mrworker ../../mrapps/mtiming.so &
-../mrworker ../../mrapps/mtiming.so
+timeout -k 2s 180s ../mrworker ../../mrapps/mtiming.so &
+timeout -k 2s 180s ../mrworker ../../mrapps/mtiming.so
 
 NT=`cat mr-out* | grep '^times-' | wc -l | sed 's/ //g'`
 if [ "$NT" != "2" ]
 then
   echo '---' saw "$NT" workers rather than 2
   echo '---' map parallelism test: FAIL
-  exit 1
+  failed_any=1
 fi
 
 if cat mr-out* | grep '^parallel.* 2' > /dev/null
@@ -121,7 +123,7 @@ then
 else
   echo '---' map workers did not run in parallel
   echo '---' map parallelism test: FAIL
-  exit 1
+  failed_any=1
 fi
 
 wait ; wait
@@ -131,18 +133,18 @@ echo '***' Starting reduce parallelism test.
 
 rm -f mr-out* mr-worker*
 
-../mrmaster ../pg*txt &
+timeout -k 2s 180s ../mrmaster ../pg*txt &
 sleep 1
 
-../mrworker ../../mrapps/rtiming.so &
-../mrworker ../../mrapps/rtiming.so
+timeout -k 2s 180s ../mrworker ../../mrapps/rtiming.so &
+timeout -k 2s 180s ../mrworker ../../mrapps/rtiming.so
 
 NT=`cat mr-out* | grep '^[a-z] 2' | wc -l | sed 's/ //g'`
 if [ "$NT" -lt "2" ]
 then
   echo '---' too few parallel reduces.
   echo '---' reduce parallelism test: FAIL
-  exit 1
+  failed_any=1
 else
   echo '---' reduce parallelism test: PASS
 fi
@@ -158,30 +160,30 @@ rm -f mr-out*
 echo '***' Starting crash test.
 
 rm -f mr-done
-(../mrmaster ../pg*txt ; touch mr-done ) &
+(timeout -k 2s 180s ../mrmaster ../pg*txt ; touch mr-done ) &
 sleep 1
 
 # start multiple workers
-../mrworker ../../mrapps/crash.so &
+timeout -k 2s 180s ../mrworker ../../mrapps/crash.so &
 
 # mimic rpc.go's masterSock()
 SOCKNAME=/var/tmp/824-mr-`id -u`
 
 ( while [ -e $SOCKNAME -a ! -f mr-done ]
   do
-    ../mrworker ../../mrapps/crash.so 
+    timeout -k 2s 180s ../mrworker ../../mrapps/crash.so
     sleep 1
   done ) &
 
 ( while [ -e $SOCKNAME -a ! -f mr-done ]
   do
-    ../mrworker ../../mrapps/crash.so 
+    timeout -k 2s 180s ../mrworker ../../mrapps/crash.so
     sleep 1
   done ) &
 
 while [ -e $SOCKNAME -a ! -f mr-done ]
 do
-  ../mrworker ../../mrapps/crash.so 
+  timeout -k 2s 180s ../mrworker ../../mrapps/crash.so
   sleep 1
 done
 
@@ -197,7 +199,12 @@ then
 else
   echo '---' crash output is not the same as mr-correct-crash.txt
   echo '---' crash test: FAIL
-  exit 1
+  failed_any=1
 fi
 
-echo '***' PASSED ALL TESTS
+if [ $failed_any -eq 0 ]; then
+    echo '***' PASSED ALL TESTS
+else
+    echo '***' FAILED SOME TESTS
+    exit 1
+fi