go-ethereum/eth/tracers/firehose_test.go

package tracers

import (
	"encoding/json"
	"fmt"
	"math"
	"math/big"
	"os"
	"reflect"
	"regexp"
	"slices"
	"testing"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/core/tracing"
	"github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/core/vm"
	"github.com/ethereum/go-ethereum/params"
	pbeth "github.com/streamingfast/firehose-ethereum/types/pb/sf/ethereum/type/v2"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"golang.org/x/exp/maps"
	"google.golang.org/protobuf/encoding/protojson"
	"google.golang.org/protobuf/proto"
	"google.golang.org/protobuf/types/known/timestamppb"
)

func TestFirehoseCallStack_Push(t *testing.T) {
	type actionRunner func(t *testing.T, s *CallStack)

	push := func(call *pbeth.Call) actionRunner { return func(_ *testing.T, s *CallStack) { s.Push(call) } }
	pop := func() actionRunner { return func(_ *testing.T, s *CallStack) { s.Pop() } }
	check := func(r actionRunner) actionRunner { return func(t *testing.T, s *CallStack) { r(t, s) } }

	tests := []struct {
		name    string
		actions []actionRunner
	}{
		{
			"push/pop emtpy", []actionRunner{
				push(&pbeth.Call{}),
				pop(),
				check(func(t *testing.T, s *CallStack) {
					require.Len(t, s.stack, 0)
				}),
			},
		},
		{
			"push/push/push", []actionRunner{
				push(&pbeth.Call{}),
				push(&pbeth.Call{}),
				push(&pbeth.Call{}),
				check(func(t *testing.T, s *CallStack) {
					require.Len(t, s.stack, 3)

					require.Equal(t, 1, int(s.stack[0].Index))
					require.Equal(t, 0, int(s.stack[0].ParentIndex))

					require.Equal(t, 2, int(s.stack[1].Index))
					require.Equal(t, 1, int(s.stack[1].ParentIndex))

					require.Equal(t, 3, int(s.stack[2].Index))
					require.Equal(t, 2, int(s.stack[2].ParentIndex))
				}),
			},
		},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			s := NewCallStack()

			for _, action := range tt.actions {
				action(t, s)
			}
		})
	}
}

func Test_validateKnownTransactionTypes(t *testing.T) {
	tests := []struct {
		name      string
		txType    byte
		knownType bool
		want      error
	}{
		{"legacy", 0, true, nil},
		{"access_list", 1, true, nil},
		{"inexistant", 255, false, nil},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			err := validateFirehoseKnownTransactionType(tt.txType, tt.knownType)
			if tt.want == nil && err != nil {
				t.Fatalf("Transaction of type %d expected to validate properly but received error %q", tt.txType, err)
			} else if tt.want != nil && err == nil {
				t.Fatalf("Transaction of type %d expected to validate improperly but generated no error", tt.txType)
			} else if tt.want != nil && err != nil && tt.want.Error() != err.Error() {
				t.Fatalf("Transaction of type %d expected to validate improperly but generated error %q does not match expected error %q", tt.txType, err, tt.want)
			}
		})
	}
}

var ignorePbFieldNames = map[string]bool{
	"Hash":            true,
	"TotalDifficulty": true,
	"state":           true,
	"unknownFields":   true,
	"sizeCache":       true,

	// This was a Polygon specific field that existed for a while and has since been
	// removed. It can be safely ignored in all protocols now.
	"TxDependency": true,
}

var pbFieldNameToGethMapping = map[string]string{
	"WithdrawalsRoot":  "WithdrawalsHash",
	"MixHash":          "MixDigest",
	"BaseFeePerGas":    "BaseFee",
	"StateRoot":        "Root",
	"ExtraData":        "Extra",
	"Timestamp":        "Time",
	"ReceiptRoot":      "ReceiptHash",
	"TransactionsRoot": "TxHash",
	"LogsBloom":        "Bloom",
}

var (
	pbHeaderType   = reflect.TypeFor[pbeth.BlockHeader]()
	gethHeaderType = reflect.TypeFor[types.Header]()
)

func Test_TypesHeader_AllConsensusFieldsAreKnown(t *testing.T) {
	// This exact hash varies from protocol to protocol and also sometimes from one version to the other.
	// When adding support for a new hard-fork that adds new block header fields, it's normal that this value
	// changes. If you are sure the two struct are the same, then you can update the expected hash below
	// to the new value.
	expectedHash := common.HexToHash("4ced4916132bbf6a7819a310bbac4abf354062a00efc980ea4f0bab406546ac5")

	gethHeaderValue := reflect.New(gethHeaderType)
	fillAllFieldsWithNonEmptyValues(t, gethHeaderValue, reflect.VisibleFields(gethHeaderType))
	gethHeader := gethHeaderValue.Interface().(*types.Header)

	// If you hit this assertion, it means that the fields `types.Header` of go-ethereum differs now
	// versus last time this test was edited.
	//
	// It's important to understand that in Ethereum Block Header (e.g. `*types.Header`), the `Hash` is
	// actually a computed value based on the other fields in the struct, so if you change any field,
	// the hash will change also.
	//
	// On hard-fork, it happens that new fields are added, this test serves as a way to "detect" in code
	// that the expected fields of `types.Header` changed
	require.Equal(t, expectedHash, gethHeader.Hash(),
		"Geth Header Hash mismatch, got %q but expecting %q on *types.Header:\n\nGeth Header (from fillNonDefault(new(*types.Header)))\n%s",
		gethHeader.Hash().Hex(),
		expectedHash,
		asIndentedJSON(t, gethHeader),
	)
}

func Test_FirehoseAndGethHeaderFieldMatches(t *testing.T) {
	pbFields := filter(reflect.VisibleFields(pbHeaderType), func(f reflect.StructField) bool {
		return !ignorePbFieldNames[f.Name]
	})

	gethFields := reflect.VisibleFields(gethHeaderType)

	pbFieldCount := len(pbFields)
	gethFieldCount := len(gethFields)

	pbFieldNames := extractStructFieldNames(pbFields)
	gethFieldNames := extractStructFieldNames(gethFields)

	// If you reach this assertion, it means that the fields count in the protobuf and go-ethereum are different.
	// It is super important that you properly update the mapping from pbeth.BlockHeader to go-ethereum/core/types.Header
	// that is done in `codecHeaderToGethHeader` function in `executor/provider_statedb.go`.
	require.Equal(
		t,
		pbFieldCount,
		gethFieldCount,
		fieldsCountMismatchMessage(t, pbFieldNames, gethFieldNames))

	for pbFieldName := range pbFieldNames {
		pbFieldRenamedName, found := pbFieldNameToGethMapping[pbFieldName]
		if !found {
			pbFieldRenamedName = pbFieldName
		}

		assert.Contains(t, gethFieldNames, pbFieldRenamedName, "pbField.Name=%q (original %q) not found in gethFieldNames", pbFieldRenamedName, pbFieldName)
	}
}

var endsWithUnknownConstant = regexp.MustCompile(`.*\(\d+\)$`)

func TestFirehose_BalanceChangeAllMappedCorrectly(t *testing.T) {

	for i := 0; i <= math.MaxUint8; i++ {
		tracingReason := tracing.BalanceChangeReason(i)
		if tracingReason == tracing.BalanceChangeUnspecified || tracingReason == tracing.BalanceChangeRevert {
			// Should never happen in Firehose tracer, only if tracer is wrapped with [tracing.WrapWithJournal]
			continue
		}

		// Here, we leverage the fact that the `tracing.BalanceChangeReason` Stringer will render the String
		// as `<EnumName>(<indexValue>)` if the index is not mapped to a constant in the enum. If this happens,
		// we know it's not a defined constant in the Geth tracing package.
		//
		// Otherwise, it's defined and we should have some mapping for it in the `balanceChangeReasonFromChain` function.
		//
		// There is a loophole of this technique and it's that if the code generator defining the enum Stringer is
		// not run, we will think it's an undefined constant and will miss it.
		if !endsWithUnknownConstant.MatchString(tracingReason.String()) {
			require.NotPanics(t, func() {
				balanceChangeReasonFromChain(tracingReason)
			}, "BalanceChangeReason panicked for value %v", tracingReason)
		}
	}
}

func TestFirehose_GasChangeAllMappedCorrectly(t *testing.T) {
	for i := 0; i <= math.MaxUint8; i++ {
		tracingReason := tracing.GasChangeReason(i)

		// Those are ignored and never mapped
		if tracingReason == tracing.GasChangeUnspecified || tracingReason == tracing.GasChangeCallOpCode || tracingReason == tracing.GasChangeIgnored {
			continue
		}

		// Here, we leverage the fact that the `tracing.GasChangeReason` Stringer will render the String
		// as `<EnumName>(<indexValue>)` if the index is not mapped to a constant in the enum. If this happens,
		// we know it's not a defined constant in the Geth tracing package.
		//
		// Otherwise, it's defined and we should have some mapping for it in the `gasChangeReasonFromChain` function.
		//
		// There is a loophole of this technique and it's that if the code generator defining the enum Stringer is
		// not run, we will think it's an undefined constant and will miss it.
		if !endsWithUnknownConstant.MatchString(tracingReason.String()) {
			require.NotPanics(t, func() {
				gasChangeReasonFromChain(tracingReason)
			}, "GasChangeReason panicked for value %v", tracingReason)
		}
	}
}

func fillAllFieldsWithNonEmptyValues(t *testing.T, structValue reflect.Value, fields []reflect.StructField) {
	t.Helper()

	for _, field := range fields {
		fieldValue := structValue.Elem().FieldByName(field.Name)
		require.True(t, fieldValue.IsValid(), "field %q not found", field.Name)

		switch fieldValue.Interface().(type) {
		case []byte:
			fieldValue.Set(reflect.ValueOf([]byte{1}))
		case uint64:
			fieldValue.Set(reflect.ValueOf(uint64(1)))
		case *uint64:
			var mockValue uint64 = 1
			fieldValue.Set(reflect.ValueOf(&mockValue))
		case *common.Hash:
			var mockValue common.Hash = common.HexToHash("0x01")
			fieldValue.Set(reflect.ValueOf(&mockValue))
		case common.Hash:
			fieldValue.Set(reflect.ValueOf(common.HexToHash("0x01")))
		case common.Address:
			fieldValue.Set(reflect.ValueOf(common.HexToAddress("0x01")))
		case types.Bloom:
			fieldValue.Set(reflect.ValueOf(types.BytesToBloom([]byte{1})))
		case types.BlockNonce:
			fieldValue.Set(reflect.ValueOf(types.EncodeNonce(1)))
		case *big.Int:
			fieldValue.Set(reflect.ValueOf(big.NewInt(1)))
		case *pbeth.BigInt:
			fieldValue.Set(reflect.ValueOf(&pbeth.BigInt{Bytes: []byte{1}}))
		case *timestamppb.Timestamp:
			fieldValue.Set(reflect.ValueOf(&timestamppb.Timestamp{Seconds: 1}))
		default:
			// If you reach this panic in test, simply add a case above with a sane non-default
			// value for the type in question.
			t.Fatalf("unsupported type %T", fieldValue.Interface())
		}
	}
}

func fieldsCountMismatchMessage(t *testing.T, pbFieldNames map[string]bool, gethFieldNames map[string]bool) string {
	t.Helper()

	pbRemappedFieldNames := make(map[string]bool, len(pbFieldNames))
	for pbFieldName := range pbFieldNames {
		pbFieldRenamedName, found := pbFieldNameToGethMapping[pbFieldName]
		if !found {
			pbFieldRenamedName = pbFieldName
		}

		pbRemappedFieldNames[pbFieldRenamedName] = true
	}

	return fmt.Sprintf(
		"Field count mistmatch between `pbeth.BlockHeader` (has %d fields) and `*types.Header` (has %d fields)\n\n"+
			"Fields in `pbeth.Blockheader`:\n%s\n\n"+
			"Fields in `*types.Header`:\n%s\n\n"+
			"Missing in `pbeth.BlockHeader`:\n%s\n\n"+
			"Missing in `*types.Header`:\n%s",
		len(pbRemappedFieldNames),
		len(gethFieldNames),
		asIndentedJSON(t, maps.Keys(pbRemappedFieldNames)),
		asIndentedJSON(t, maps.Keys(gethFieldNames)),
		asIndentedJSON(t, missingInSet(gethFieldNames, pbRemappedFieldNames)),
		asIndentedJSON(t, missingInSet(pbRemappedFieldNames, gethFieldNames)),
	)
}

func asIndentedJSON(t *testing.T, v any) string {
	t.Helper()
	out, err := json.MarshalIndent(v, "", "  ")
	require.NoError(t, err)

	return string(out)
}

func missingInSet(a, b map[string]bool) []string {
	missing := make([]string, 0)
	for name := range a {
		if !b[name] {
			missing = append(missing, name)
		}
	}

	return missing
}

func extractStructFieldNames(fields []reflect.StructField) map[string]bool {
	result := make(map[string]bool, len(fields))
	for _, field := range fields {
		result[field.Name] = true
	}
	return result
}

func filter[S ~[]T, T any](s S, f func(T) bool) (out S) {
	out = make(S, 0, len(s)/4)
	for i, v := range s {
		if f(v) {
			out = append(out, s[i])
		}
	}

	return out
}

func TestFirehose_reorderIsolatedTransactionsAndOrdinals(t *testing.T) {
	tests := []struct {
		name              string
		populate          func(t *Firehose)
		expectedBlockFile string
	}{
		{
			name: "empty",
			populate: func(t *Firehose) {
				t.OnBlockStart(blockEvent(1))

				// Simulated GetTxTracer being called
				t.blockReorderOrdinalOnce.Do(func() {
					t.blockReorderOrdinal = true
					t.blockReorderOrdinalSnapshot = t.blockOrdinal.value
				})

				t.blockOrdinal.Reset()
				t.onTxStart(txEvent(), hex2Hash("CC"), from, to)
				t.OnCallEnter(0, byte(vm.CALL), from, to, nil, 0, nil)
				t.OnBalanceChange(empty, b(1), b(2), 0)
				t.OnCallExit(0, nil, 0, nil, false)
				t.OnTxEnd(txReceiptEvent(2), nil)

				t.blockOrdinal.Reset()
				t.onTxStart(txEvent(), hex2Hash("AA"), from, to)
				t.OnCallEnter(0, byte(vm.CALL), from, to, nil, 0, nil)
				t.OnBalanceChange(empty, b(1), b(2), 0)
				t.OnCallExit(0, nil, 0, nil, false)
				t.OnTxEnd(txReceiptEvent(0), nil)

				t.blockOrdinal.Reset()
				t.onTxStart(txEvent(), hex2Hash("BB"), from, to)
				t.OnCallEnter(0, byte(vm.CALL), from, to, nil, 0, nil)
				t.OnBalanceChange(empty, b(1), b(2), 0)
				t.OnCallExit(0, nil, 0, nil, false)
				t.OnTxEnd(txReceiptEvent(1), nil)
			},
			expectedBlockFile: "testdata/firehose/reorder-ordinals-empty.golden.json",
		},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			f := NewFirehose(&FirehoseConfig{
				ApplyBackwardCompatibility: ptr(false),
			})
			f.OnBlockchainInit(params.AllEthashProtocolChanges)

			tt.populate(f)

			f.reorderIsolatedTransactionsAndOrdinals()

			goldenUpdate := os.Getenv("GOLDEN_UPDATE") == "true"
			goldenPath := tt.expectedBlockFile

			if !goldenUpdate && !fileExits(t, goldenPath) {
				t.Fatalf("the golden file %q does not exist, re-run with 'GOLDEN_UPDATE=true go test ./... -run %q' to generate the intial version", goldenPath, t.Name())
			}

			content, err := protojson.MarshalOptions{Indent: "  "}.Marshal(f.block)
			require.NoError(t, err)

			if goldenUpdate {
				require.NoError(t, os.WriteFile(goldenPath, content, os.ModePerm))
			}

			expected, err := os.ReadFile(goldenPath)
			require.NoError(t, err)

			expectedBlock := &pbeth.Block{}
			protojson.Unmarshal(expected, expectedBlock)

			if !proto.Equal(expectedBlock, f.block) {
				assert.Equal(t, expectedBlock, f.block, "Run 'GOLDEN_UPDATE=true go test ./... -run %q' to update golden file", t.Name())
			}

			seenOrdinals := make(map[uint64]int)

			walkChanges(f.block.BalanceChanges, seenOrdinals)
			walkChanges(f.block.CodeChanges, seenOrdinals)
			walkCalls(f.block.SystemCalls, seenOrdinals)

			for _, trx := range f.block.TransactionTraces {
				seenOrdinals[trx.BeginOrdinal] = seenOrdinals[trx.BeginOrdinal] + 1
				seenOrdinals[trx.EndOrdinal] = seenOrdinals[trx.EndOrdinal] + 1
				walkCalls(trx.Calls, seenOrdinals)
			}

			// No ordinal should be seen more than once
			for ordinal, count := range seenOrdinals {
				assert.Equal(t, 1, count, "Ordinal %d seen %d times", ordinal, count)
			}

			ordinals := maps.Keys(seenOrdinals)
			slices.Sort(ordinals)

			// All ordinals should be in stricly increasing order
			prev := -1
			for _, ordinal := range ordinals {
				if prev != -1 {
					assert.Equal(t, prev+1, int(ordinal), "Ordinal %d is not in sequence", ordinal)
				}
			}
		})
	}
}

func walkCalls(calls []*pbeth.Call, ordinals map[uint64]int) {
	for _, call := range calls {
		walkCall(call, ordinals)
	}
}

func walkCall(call *pbeth.Call, ordinals map[uint64]int) {
	ordinals[call.BeginOrdinal] = ordinals[call.BeginOrdinal] + 1
	ordinals[call.EndOrdinal] = ordinals[call.EndOrdinal] + 1

	walkChanges(call.BalanceChanges, ordinals)
	walkChanges(call.CodeChanges, ordinals)
	walkChanges(call.Logs, ordinals)
	walkChanges(call.StorageChanges, ordinals)
	walkChanges(call.NonceChanges, ordinals)
	walkChanges(call.GasChanges, ordinals)
}

func walkChanges[T any](changes []T, ordinals map[uint64]int) {
	for _, change := range changes {
		var x any = change
		if v, ok := x.(interface{ GetOrdinal() uint64 }); ok {
			ordinals[v.GetOrdinal()] = ordinals[v.GetOrdinal()] + 1
		}
	}
}

var b = big.NewInt
var empty, from, to = common.HexToAddress("00"), common.HexToAddress("01"), common.HexToAddress("02")
var hex2Hash = common.HexToHash

func fileExits(t *testing.T, path string) bool {
	t.Helper()
	stat, err := os.Stat(path)
	return err == nil && !stat.IsDir()
}

func txEvent() *types.Transaction {
	return types.NewTx(&types.LegacyTx{
		Nonce:    0,
		GasPrice: big.NewInt(1),
		Gas:      1,
		To:       &to,
		Value:    big.NewInt(1),
		Data:     nil,
		V:        big.NewInt(1),
		R:        big.NewInt(1),
		S:        big.NewInt(1),
	})
}

func txReceiptEvent(txIndex uint) *types.Receipt {
	return &types.Receipt{
		Status:           1,
		TransactionIndex: txIndex,
	}
}

func blockEvent(height uint64) tracing.BlockEvent {
	return tracing.BlockEvent{
		Block: types.NewBlock(&types.Header{
			Number: big.NewInt(int64(height)),
		}, nil, nil, nil),
	}
}

func TestMemory_GetPtr(t *testing.T) {
	type args struct {
		offset int64
		size   int64
	}
	tests := []struct {
		name string
		m    Memory
		args args
		want []byte
	}{
		{"memory is just a bit too small", Memory([]byte{1, 2, 3}), args{0, 4}, []byte{1, 2, 3, 0}},
		{"memory is flushed with request", Memory([]byte{1, 2, 3, 4}), args{0, 4}, []byte{1, 2, 3, 4}},
		{"memory is just a bit too big", Memory([]byte{1, 2, 3, 4, 5}), args{0, 4}, []byte{1, 2, 3, 4}},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			assert.Equal(t, tt.want, tt.m.GetPtr(tt.args.offset, tt.args.size))
		})
	}
}