From 74b741bad20397c2bcbe59abe9ba96f956cc9b2d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Moritz=20=27Morty=27=20Str=C3=BCbe?= <morty@gmx.net>
Date: Wed, 8 Oct 2014 09:22:37 +0200
Subject: [PATCH] Support running multiple random seeds in case one fails

---
 regression-tests/Makefile.simulation-test |   6 +-
 regression-tests/simexec.sh               | 114 ++++++++++++----------
 2 files changed, 68 insertions(+), 52 deletions(-)

diff --git a/regression-tests/Makefile.simulation-test b/regression-tests/Makefile.simulation-test
index 5c8b99e40..bc14c8b24 100644
--- a/regression-tests/Makefile.simulation-test
+++ b/regression-tests/Makefile.simulation-test
@@ -30,8 +30,8 @@ TESTS=$(wildcard ??-*.csc)
 TESTLOGS=$(patsubst %.csc,%.testlog,$(TESTS))
 LOGS=$(patsubst %.csc,%.log,$(TESTS))
 FAILLOGS=$(patsubst %.csc,%.faillog,$(TESTS))
-#Set random seed to create reproduceable results.
-RANDOMSEED=1
+#Set random seeds to create reproduceable results.
+RANDOMSEED=1 5
 
 CONTIKI=../..
 
@@ -57,7 +57,7 @@ RUNALL=false
 endif
 
 %.testlog: %.csc cooja	
-	@$(CONTIKI)/regression-tests/simexec.sh "$(RUNALL)" "$<" "$(CONTIKI)" "$(basename $@)" "$(RANDOMSEED)"
+	@$(CONTIKI)/regression-tests/simexec.sh "$(RUNALL)" "$<" "$(CONTIKI)" "$(basename $@)" $(RANDOMSEED)
 
 clean:
 	@rm -f $(TESTLOGS) $(LOGS) $(FAILLOGS) COOJA.log COOJA.testlog \
diff --git a/regression-tests/simexec.sh b/regression-tests/simexec.sh
index 1d1e46b7f..cb6d5e5b0 100755
--- a/regression-tests/simexec.sh
+++ b/regression-tests/simexec.sh
@@ -1,66 +1,82 @@
 #!/bin/bash
+# Do not return an error
 RUNALL=$1
-CSC=$2
-CONTIKI=$3
-BASENAME=$4
-RANDOMSEED=$5
+shift
+# The simulation to run
+CSC=$1
+shift
+#Contiki directory
+CONTIKI=$1
+shift
+#The basename of the experiment
+BASENAME=$1
+shift
+# The test will end on the first successfull run
 
 #set -x
 
-echo -n "Running test $BASENAME "
+while (( "$#" )); do
+	RANDOMSEED=$1
+	echo -n "Running test $BASENAME with random Seed $RANDOMSEED: "
 
-java -Xshare:on -jar $CONTIKI/tools/cooja/dist/cooja.jar -nogui=$CSC -contiki=$CONTIKI -random-seed=$RANDOMSEED > $BASENAME.log &
-JPID=$!
+	java -Xshare:on -jar $CONTIKI/tools/cooja/dist/cooja.jar -nogui=$CSC -contiki=$CONTIKI -random-seed=$RANDOMSEED > $BASENAME.log &
+	JPID=$!
 
-# Copy the log and only print "." if it changed
-touch $BASENAME.log.prog
-while kill -0 $JPID 2> /dev/null
-do
-  sleep 1
-  diff $BASENAME.log $BASENAME.log.prog > /dev/null
-  if [ $? -ne 0 ] 
-  then
-    echo -n "."
-    cp $BASENAME.log $BASENAME.log.prog
-  fi
+	# Copy the log and only print "." if it changed
+	touch $BASENAME.log.prog
+	while kill -0 $JPID 2> /dev/null
+	do
+		sleep 1
+		diff $BASENAME.log $BASENAME.log.prog > /dev/null
+		if [ $? -ne 0 ] 
+		then
+		  echo -n "."
+		  cp $BASENAME.log $BASENAME.log.prog
+		fi
+	done
+	rm $BASENAME.log.prog
+
+
+	wait $JPID
+	JRV=$?
+
+	if [ $JRV -eq 0 ] ; then
+		touch COOJA.testlog; 
+		mv COOJA.testlog $BASENAME.testlog 
+		echo " OK"
+		exit 0
+	fi
+
+
+
+	# In case of failure
+
+
+
+	#Verbose output when using CI
+	if [ "$CI" = "true" ];  then 
+		echo "==== COOJA.log ====" ; cat COOJA.log; 
+		echo "==== COOJA.testlog ====" ; cat COOJA.testlog; 
+	else  
+		tail -50 COOJA.log ; 
+	fi; 
+
+	mv COOJA.testlog $BASENAME.$RANDOMSEED.faillog
+
+	shift
 done
-rm $BASENAME.log.prog
 
-
-wait $JPID
-JRV=$?
-
-if [ $JRV -eq 0 ] ; then
-  touch COOJA.testlog; 
-  mv COOJA.testlog $BASENAME.testlog 
-  echo " OK"
-  exit 0
-fi
-
-
-
-# In case of failure
-
-echo " FAIL ಠ_ಠ" | tee -a COOJA.testlog; 
-
-#Verbose output when using CI
-if [ "$CI" = "true" ];  then 
-  echo "==== COOJA.log ====" ; cat COOJA.log; 
-  echo "==== COOJA.testlog ====" ; cat COOJA.testlog; 
-else  
-  tail -50 COOJA.log ; 
-fi; 
-
-mv COOJA.testlog $BASENAME.faillog
+#All seeds failed
+	echo " FAIL ಠ_ಠ" | tee -a $BASENAME.$RANDOMSEED.faillog;
 
 # We do not want Make to stop -> Return 0
 if [ "$RUNALL" = "true" ] ; then
-  touch COOJA.testlog; 
-  mv COOJA.testlog $BASENAME.testlog; 
-  exit 0
+	touch COOJA.testlog; 
+	mv COOJA.testlog $BASENAME.testlog; 
+	exit 0
 fi
 
-#This is a failure
+
 exit 1