feat: enable invalidating txs (#208)

## Why this should be merged

In case a tx's execution (specifically a registered precompile) should
be invalidated, this allows the EVM to find this error.

## How this works

Adds a setter and getter that can be called from a precompile.

## How this was tested

UT

core/state_transition.go

@@ -364,10 +364,7 @@ func (st *StateTransition) preCheck() error {
 //
 // However if any consensus issue encountered, return the error directly with
 // nil evm execution result.
-func (st *StateTransition) TransitionDb() (*ExecutionResult, error) {
+func (st *StateTransition) transitionDb() (*ExecutionResult, error) {
-	if err := st.canExecuteTransaction(); err != nil {
-		return nil, err
-	}
 	// First check this message satisfies all consensus rules before
 	// applying the message. The rules include these clauses
 	//

core/state_transition.libevm.go

@@ -17,6 +17,9 @@
 package core
 
 import (
+	"fmt"
+
+	"github.com/ava-labs/libevm/core/vm"
 	"github.com/ava-labs/libevm/log"
 	"github.com/ava-labs/libevm/params"
 )
@@ -27,6 +30,50 @@ func (st *StateTransition) rulesHooks() params.RulesHooks {
 	return rules.Hooks()
 }
 
+// NOTE: other than the final paragraph, the comment on
+// [StateTransition.TransitionDb] is copied, verbatim, from the upstream
+// version, which has been changed to [StateTransition.transitionDb] to allow
+// its behaviour to be augmented.
+
+// Keeps the vm package imported by this specific file so VS Code can support
+// comments like [vm.EVM].
+var _ = (*vm.EVM)(nil)
+
+// TransitionDb will transition the state by applying the current message and
+// returning the evm execution result with following fields.
+//
+//   - used gas: total gas used (including gas being refunded)
+//   - returndata: the returned data from evm
+//   - concrete execution error: various EVM errors which abort the execution, e.g.
+//     ErrOutOfGas, ErrExecutionReverted
+//
+// However if any consensus issue encountered, return the error directly with
+// nil evm execution result.
+//
+// libevm-specific behaviour: if, during execution, [vm.EVM.InvalidateExecution]
+// is called with a non-nil error then said error will be returned, wrapped. All
+// state transitions (e.g. nonce incrementing) will be reverted to a snapshot
+// taken before execution.
+func (st *StateTransition) TransitionDb() (*ExecutionResult, error) {
+	if err := st.canExecuteTransaction(); err != nil {
+		return nil, err
+	}
+
+	snap := st.state.Snapshot()   // computationally cheap operation
+	res, err := st.transitionDb() // original geth implementation
+
+	// [NOTE]: At the time of implementation of this libevm override, non-nil
+	// values of `err` and `invalid` (below) are mutually exclusive. However, as
+	// a defensive measure, we don't return early on non-nil `err` in case an
+	// upstream update breaks this invariant.
+
+	if invalid := st.evm.ExecutionInvalidated(); invalid != nil {
+		st.state.RevertToSnapshot(snap)
+		err = fmt.Errorf("execution invalidated: %w", invalid)
+	}
+	return res, err
+}
+
 // canExecuteTransaction is a convenience wrapper for calling the
 // [params.RulesHooks.CanExecuteTransaction] hook.
 func (st *StateTransition) canExecuteTransaction() error {

core/vm/contracts.libevm.go

@@ -187,6 +187,9 @@ type PrecompileEnvironment interface {
 	BlockNumber() *big.Int
 	BlockTime() uint64
 
+	// Invalidate invalidates the transaction calling this precompile.
+	InvalidateExecution(error)
+
 	// Call is equivalent to [EVM.Call] except that the `caller` argument is
 	// removed and automatically determined according to the type of call that
 	// invoked the precompile.

core/vm/contracts.libevm_test.go

@@ -18,7 +18,9 @@ package vm_test
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"math"
 	"math/big"
 	"reflect"
 	"strings"
@@ -302,6 +304,79 @@ func TestNewStatefulPrecompile(t *testing.T) {
 	}
 }
 
+func TestPrecompileInvalidatesExecution(t *testing.T) {
+	errIfInvalidated := errors.New("execution invalidated")
+	inputToInvalidate := []byte("invalidate")
+	run := func(env vm.PrecompileEnvironment, input []byte) ([]byte, error) {
+		if bytes.Equal(input, inputToInvalidate) {
+			env.InvalidateExecution(errIfInvalidated)
+		}
+		return []byte{}, nil
+	}
+
+	precompile := common.HexToAddress("60C0DE") // GO CODE
+	hooks := &hookstest.Stub{
+		PrecompileOverrides: map[common.Address]libevm.PrecompiledContract{
+			precompile: vm.NewStatefulPrecompile(run),
+		},
+	}
+	hooks.Register(t)
+
+	// The EVM instance MUST be reused across all tests to ensure that
+	// [vm.EVM.Reset] undoes any invalidation.
+	stateDB, evm := ethtest.NewZeroEVM(t)
+
+	tests := []struct {
+		name    string
+		nonce   uint64
+		input   []byte
+		wantErr error
+	}{
+		{
+			name:    "not_invalidating",
+			input:   []byte{},
+			nonce:   0,
+			wantErr: nil,
+		},
+		{
+			name:    "invalidating",
+			nonce:   1,
+			input:   inputToInvalidate,
+			wantErr: errIfInvalidated,
+		},
+		{
+			// Tests that:
+			// (a) [vm.EVM.Reset] undoes the previous invalidation; and
+			// (b) Invalidation reverted state changes, as seen by the nonce.
+			name:    "evm_reset_not_invalidating_after_invalid",
+			input:   []byte{},
+			nonce:   1, // unchanged because the last was invalidated
+			wantErr: nil,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			msg := &core.Message{
+				Nonce: tt.nonce,
+				Data:  tt.input,
+
+				// Common across all txs
+				To:       &precompile,
+				GasLimit: 1e6, // arbitrary but sufficiently high
+				GasPrice: big.NewInt(0),
+				Value:    big.NewInt(0),
+			}
+
+			evm.Reset(core.NewEVMTxContext(msg), stateDB)
+
+			gas := core.GasPool(math.MaxUint64)
+			_, err := core.ApplyMessage(evm, msg, &gas)
+			require.ErrorIs(t, err, tt.wantErr, "core.ApplyMessage()")
+		})
+	}
+}
+
 func TestInheritReadOnly(t *testing.T) {
 	// The regular test of stateful precompiles only checks the read-only state
 	// when called directly via vm.EVM.*Call*() methods. That approach will not

core/vm/environment.libevm.go

@@ -49,6 +49,8 @@ func (e *environment) IncomingCallType() CallType        { return e.callType }
 func (e *environment) BlockNumber() *big.Int             { return new(big.Int).Set(e.evm.Context.BlockNumber) }
 func (e *environment) BlockTime() uint64                 { return e.evm.Context.Time }
 
+func (e *environment) InvalidateExecution(err error) { e.evm.InvalidateExecution(err) }
+
 func (e *environment) refundGas(add uint64) error {
 	gas, overflow := math.SafeAdd(e.self.Gas, add)
 	if overflow {

core/vm/evm.go

@@ -128,6 +128,9 @@ type EVM struct {
 	// available gas is calculated in gasCall* according to the 63/64 rule and later
 	// applied in opCall*.
 	callGasTemp uint64
+
+	// libevm
+	executionInvalidated error // see [EVM.InvalidateExecution]
 }
 
 // NewEVM returns a new EVM. The returned EVM is not thread safe and should
@@ -160,6 +163,7 @@ func NewEVM(blockCtx BlockContext, txCtx TxContext, statedb StateDB, chainConfig
 // Reset resets the EVM with a new transaction context.Reset
 // This is not threadsafe and should only be done very cautiously.
 func (evm *EVM) Reset(txCtx TxContext, statedb StateDB) {
+	evm.executionInvalidated = nil // see [EVM.InvalidateExecution]
 	evm.TxContext, evm.StateDB = evm.overrideEVMResetArgs(txCtx, statedb)
 }
 

core/vm/evm.libevm.go

@@ -43,3 +43,21 @@ func (evm *EVM) canCreateContract(caller ContractRef, contractToCreate common.Ad
 
 	return gas, err
 }
+
+// InvalidateExecution sets the error that will be returned by
+// [EVM.ExecutionInvalidated] for the length of the current transaction; i.e.
+// until [EVM.Reset] is called. This is honoured by state-transition logic to
+// render the execution itself void (as against reverted).
+//
+// This method MUST NOT be exposed in a manner that allows contracts to set
+// the error; it MAY be exposed to precompiles.
+func (evm *EVM) InvalidateExecution(err error) {
+	evm.executionInvalidated = err
+}
+
+// ExecutionInvalidated returns the last value passed to
+// [EVM.InvalidateExecution] or nil if no such call has occurred or if
+// [EVM.Reset] has been called.
+func (evm *EVM) ExecutionInvalidated() error {
+	return evm.executionInvalidated
+}