cmd/evm: per-type result schema and error reporting for consume direct

- Add lastBlockHash to blocktest/enginetest, lastPayloadStatus to enginetest
- Remove stateRoot from blocktest/enginetest (only statetest has it)
- Report validation/rejection error in `error` even when test passes,
  for negative tests (expected exceptions)
- Enables EELS consume direct to map errors through ExceptionMapper
  and verify correct exception for every invalid test

cmd/evm/blockrunner.go

@@ -173,17 +173,16 @@ func runBlockTest(ctx *cli.Context, fname string) ([]testResult, error) {
 		}
 		test := tests[name]
 		result := &testResult{Name: name, Pass: true}
-		var finalRoot *common.Hash
+		var finalHash *common.Hash
 		if err := test.Run(false, rawdb.PathScheme, ctx.Bool(WitnessCrossCheckFlag.Name), tracer, func(res error, chain *core.BlockChain) {
 			if ctx.Bool(DumpFlag.Name) {
 				if s, _ := chain.State(); s != nil {
 					result.State = dump(s)
 				}
 			}
-			// Capture final state root for end marker
 			if chain != nil {
-				root := chain.CurrentBlock().Root
-				finalRoot = &root
+				hash := chain.CurrentBlock().Hash()
+				finalHash = &hash
 			}
 		}); err != nil {
 			result.Pass, result.Error = false, err.Error()
@@ -191,9 +190,11 @@ func runBlockTest(ctx *cli.Context, fname string) ([]testResult, error) {
 
 		// Always assign fork (regardless of pass/fail or tracer)
 		result.Fork = test.Network()
-		// Assign root if test succeeded
-		if result.Pass && finalRoot != nil {
-			result.Root = finalRoot
+		if finalHash != nil {
+			result.BlockHash = finalHash
+		}
+		if result.Pass && test.LastBlockError != "" {
+			result.Error = test.LastBlockError
 		}
 
 		// When fuzzing, write results after every block

cmd/evm/enginerunner.go

@@ -192,7 +192,7 @@ func runEngineTest(ctx *cli.Context, fname string) ([]testResult, error) {
 		}
 		test := testsByName[name]
 		result := &testResult{Name: name, Pass: true}
-		var finalRoot *common.Hash
+		var finalHash *common.Hash
 		if err := test.Run(rawdb.PathScheme, tracer, func(res error, chain *core.BlockChain) {
 			if ctx.Bool(DumpFlag.Name) {
 				if s, _ := chain.State(); s != nil {
@@ -200,16 +200,20 @@ func runEngineTest(ctx *cli.Context, fname string) ([]testResult, error) {
 				}
 			}
 			if chain != nil {
-				root := chain.CurrentBlock().Root
-				finalRoot = &root
+				hash := chain.CurrentBlock().Hash()
+				finalHash = &hash
 			}
 		}); err != nil {
 			result.Pass, result.Error = false, err.Error()
 		}
 
 		result.Fork = test.Network()
-		if result.Pass && finalRoot != nil {
-			result.Root = finalRoot
+		if finalHash != nil {
+			result.BlockHash = finalHash
+		}
+		result.PayloadStatus = test.LastPayloadStatus
+		if result.Pass && test.LastValidationError != "" {
+			result.Error = test.LastValidationError
 		}
 
 		if ctx.IsSet(FuzzFlag.Name) {

cmd/evm/reporter.go

@@ -33,13 +33,15 @@ const (
 // testResult contains the execution status after running a state test, any
 // error that might have occurred and a dump of the final state if requested.
 type testResult struct {
-	Name  string       `json:"name"`
-	Pass  bool         `json:"pass"`
-	Root  *common.Hash `json:"stateRoot,omitempty"`
-	Fork  string       `json:"fork"`
-	Error string       `json:"error"`
-	State *state.Dump  `json:"state,omitempty"`
-	Stats *execStats   `json:"benchStats,omitempty"`
+	Name          string       `json:"name"`
+	Pass          bool         `json:"pass"`
+	Root          *common.Hash `json:"stateRoot,omitempty"`
+	Fork          string       `json:"fork"`
+	Error         string       `json:"error"`
+	BlockHash     *common.Hash `json:"lastBlockHash,omitempty"`
+	PayloadStatus string       `json:"lastPayloadStatus,omitempty"`
+	State         *state.Dump  `json:"state,omitempty"`
+	Stats         *execStats   `json:"benchStats,omitempty"`
 }
 
 func (r testResult) String() string {

cmd/evm/staterunner.go

@@ -210,6 +210,9 @@ func runStateTest(ctx *cli.Context, fname string) ([]testResult, error) {
 					result.Pass, result.Error = false, err.Error()
 					return
 				}
+				if test.LastTxError != "" {
+					result.Error = test.LastTxError
+				}
 			})
 			results = append(results, *result)
 		}

tests/block_test_util.go

@@ -48,7 +48,8 @@ import (
 
 // A BlockTest checks handling of entire blocks.
 type BlockTest struct {
-	json btJSON
+	json           btJSON
+	LastBlockError string // actual error from rejected blocks, for result reporting
 }
 
 // UnmarshalJSON implements json.Unmarshaler interface.
@@ -250,6 +251,7 @@ func (t *BlockTest) insertBlocks(blockchain *core.BlockChain) ([]btBlock, error)
 		cb, err := b.decode()
 		if err != nil {
 			if b.BlockHeader == nil {
+				t.LastBlockError = err.Error()
 				log.Info("Block decoding failed", "index", bi, "err", err)
 				continue // OK - block is supposed to be invalid, continue with next block
 			} else {
@@ -261,6 +263,7 @@ func (t *BlockTest) insertBlocks(blockchain *core.BlockChain) ([]btBlock, error)
 		i, err := blockchain.InsertChain(blocks)
 		if err != nil {
 			if b.BlockHeader == nil {
+				t.LastBlockError = err.Error()
 				continue // OK - block is supposed to be invalid, continue with next block
 			} else {
 				return nil, fmt.Errorf("block #%v insertion into chain failed: %v", blocks[i].Number(), err)

tests/engine_test_util.go

@@ -46,7 +46,9 @@ import (
 
 // EngineTest checks processing of engine API payloads.
 type EngineTest struct {
-	json etJSON
+	json                etJSON
+	LastPayloadStatus   string // set during Run, exposed for the runner
+	LastValidationError string // actual validation error from engine
 }
 
 func (t *EngineTest) UnmarshalJSON(in []byte) error {
@@ -235,6 +237,11 @@ func (t *EngineTest) Run(scheme string, tracer *tracing.Hooks, postCheck func(er
 		if err != nil {
 			return fmt.Errorf("payload %d: unexpected error: %v", i, err)
 		}
+		// Track last payload status and validation error for result reporting
+		t.LastPayloadStatus = status.Status
+		if status.ValidationError != nil {
+			t.LastValidationError = *status.ValidationError
+		}
 		// Check validation error expectation
 		if payload.ValidationError != "" {
 			if status.Status != engine.INVALID {

tests/state_test_util.go

@@ -51,7 +51,8 @@ import (
 // StateTest checks transaction processing without block context.
 // See https://github.com/ethereum/EIPs/issues/176 for the test format specification.
 type StateTest struct {
-	json stJSON
+	json       stJSON
+	LastTxError string // actual tx error, for result reporting
 }
 
 // StateSubtest selects a specific configuration of a General State Test.
@@ -211,7 +212,7 @@ func (t *StateTest) checkError(subtest StateSubtest, err error) error {
 		return fmt.Errorf("unexpected error: %w", err)
 	}
 	if err != nil && expectedError != "" {
-		// Ignore expected errors (TODO MariusVanDerWijden check error string)
+		t.LastTxError = err.Error()
 		return nil
 	}
 	return nil