#!/bin/bash
set -euo pipefail

dir=`dirname $0`

updateoracle=false
forceupdateoracle=false
files=""
success=true
suffix=""

while test $# != 0; do
case "$1" in
  "-suffix")
      suffix="$2"
      shift 2;;
  "-update-oracle")
      updateoracle=true
      shift 1;;
  "-force-update-oracle")
      forceupdateoracle=true
      shift 1;;
  "-"*)
      printf "unknown option: %s\n" "$1"
      printf "usage: check-ce-bench [-update-oracle] [-suffix <suffix>] <files>\n"
      printf "  <files> must be given without the '.mlw' suffix\n"
      printf "  if <files> empty, use all files from directory 'ce'\n"
      exit 2;;
  *)
      files="$files $1"
      shift 1;;
esac
done

if test "$files" = "" ; then
    files="$dir/check-ce/*.mlw"
fi

failed=""

whydata=$(bin/why3"$suffix" --print-datadir)
whylib=$(bin/why3"$suffix" --print-libdir)

colorize() {
    if command -v pygmentize &> /dev/null; then
        pygmentize -ldiff
    else
        cat
    fi
}

# types for which the current bench is non deterministic
nondet_types=""

remove_solver_details () {
    sed 's/ ([0-9.]\+s, [0-9])\.$/./' \
  | sed 's/[0-9]\+\.[0-9]\+s//g' \
  | sed 's/[0-9]\+ steps//g' \
  | sed 's/Prover result is: Timeout.*$\|Unknown (\(unknown\|incomplete\|unknown + \(incomplete\|interrupted\)\)).*$\|Out of memory\.$\|Step limit exceeded.*$/Prover result is: Unknown or time\/memory\/step limit./'
}

# $1 = prover, $2 = dir, $3 = filename, $4 = true for WP; false for SP, $5 = steps
run () {
    printf "  $2 ($1)... "
    file_path="$2/$3"
    if $4; then
        f="${file_path}_$1_WP"
        oracle_file="$2/oracles/$3_$1_WP.oracle"
        wp_sp=""
        echo "Weakest Precondition" > "$f.out"
        printf "Weakest Precondition  ${file_path} ($1)... "
    else
        f="${file_path}_$1_SP"
        oracle_file="$2/oracles/$3_$1_SP.oracle"
        wp_sp=" --debug=vc_sp"
        echo "Strongest Postcondition" > "$f.out"
        printf "Strongest Postcondition  ${file_path} ($1)... "
    fi
    prover="$1,counterexamples"
    steps=$5
    # Call why3 prove to create CE (and accept return value of 2 for unproven goals)
    ($dir/../bin/why3"$suffix" prove --apply-transform=split_vc \
       --prover="$prover" --stepslimit=$steps  --timelimit=0 \
       --check-ce --verbosity=5 \
       --library="$2" "${file_path}.mlw" \
       --debug=check-ce,check-ce-summary,rac-values \
       || [ $? -eq 2 ]) 2>&1 \
     | sed -e "s|$whydata|WHY3DATA|g" -e "s|$whylib|WHY3LIB|g" \
     > "$f.out"
    str_out=$(cat "$f.out"|remove_solver_details)
    if [ -e "$oracle_file" ]; then
        str_oracle=$(cat "$oracle_file"|remove_solver_details)
    else
        str_oracle=""
    fi
    if [ "$str_oracle" = "$str_out" ] ; then
        echo "ok"
    else
	if $updateoracle; then
	    echo "Updating oracle"
	    mv "$f.out" "${oracle_file}"
	else
	    echo "FAILED!"
	    echo "diff is the following:"
	    echo "$f"
	    (diff -u <(echo "$str_oracle") <(echo "$str_out") \
        || [ $? -eq 1 ])|colorize
	    failed="$failed$f\n"
	    success=false
	fi
    fi
    if $forceupdateoracle; then
	echo "Forcing update oracle for ${file_path}, prover $1"
        mv "$f.out" "${oracle_file}"
    fi
}

for file in $files; do
    filedir=`dirname $file`
    filebase=`basename $file .mlw`
    printf "Running provers on $filedir/$filebase.mlw\n";
    cvc4steps=150000
    z3steps=3000000
    run CVC4,1.8 $filedir $filebase true   $cvc4steps
    run CVC4,1.8 $filedir $filebase false  $cvc4steps
    run Z3,4.8.10 $filedir $filebase true  $z3steps
    run Z3,4.8.10 $filedir $filebase false $z3steps
done
if [ "$success" = true ]; then
    echo "Check-ce bench: success"
    exitvalue=0
else
    printf "\nCheck-ce bench: failed\n$failed\n"
    exitvalue=1
fi

# Check the reproduction of the experiments by Petiot (2018).
if $updateoracle; then
  updatearg="--updateoracle"
else
  updatearg=""
fi
bench/check-ce/petiot2018/experiments.sh --prover CVC4,1.8 \
  --ce-prover CVC4,1.8,counterexamples --stepslimit 10000 --rac-prover CVC4,1.8 \
  $updatearg
if [ $? = 1 ]; then exitvalue=1; fi
bench/check-ce/petiot2018/experiments.sh --prover Z3,4.8.10 \
  --ce-prover Z3,4.8.10,counterexamples --stepslimit 500000 --rac-prover Z3,4.8.10 \
  $updatearg
if [ $? = 1 ]; then exitvalue=1; fi

exit $exitvalue
