diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go
index ebb3e04461..82e7bdff3d 100644
--- a/cmd/evm/runner.go
+++ b/cmd/evm/runner.go
@@ -166,8 +166,11 @@ func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) ([]byte, exe
 				if haveGasUsed != gasUsed {
 					panic(fmt.Sprintf("gas differs, have %v want %v", haveGasUsed, gasUsed))
 				}
-				if haveErr != err {
-					panic(fmt.Sprintf("err differs, have %v want %v", haveErr, err))
+				if (haveErr == nil) != (err == nil) {
+					panic(fmt.Sprintf("err differs in nil-ness, have %v want %v", haveErr, err))
+				}
+				if haveErr != nil && err != nil && haveErr.Error() != err.Error() {
+					panic(fmt.Sprintf("err differs, have %q want %q", haveErr.Error(), err.Error()))
 				}
 			}
 		})