Added Firehose 3.0 backport tracer

buf.gen.yaml

@@ -0,0 +1,6 @@
+version: v1
+plugins:
+  - plugin: buf.build/protocolbuffers/go:v1.31.0
+    out: pb
+    opt: paths=source_relative
+

core/state/statedb.go

@@ -645,6 +645,9 @@ func (s *StateDB) GetOrNewStateObject(addr common.Address) *stateObject {
 func (s *StateDB) createObject(addr common.Address) (newobj, prev *stateObject) {
 	prev = s.getDeletedStateObject(addr) // Note, prev might have been deleted, we need that!
 	newobj = newObject(s, addr, nil)
+	if s.logger != nil && s.logger.OnNewAccount != nil {
+		s.logger.OnNewAccount(addr, prev != nil)
+	}
 	if prev == nil {
 		s.journal.append(createObjectChange{account: &addr})
 	} else {

core/tracing/hooks.go

@@ -163,6 +163,13 @@ type Hooks struct {
 	OnCodeChange    CodeChangeHook
 	OnStorageChange StorageChangeHook
 	OnLog           LogHook
+
+	// Firehose backward compatibility
+	// This hook exist because some current Firehose supported chains requires it
+	// but this field is going to be deprecated and newer chains will not produced
+	// those events anymore. The hook is registered conditionally based on the
+	// tracer configuration.
+	OnNewAccount func(address common.Address, previousExisted bool)
 }
 
 // BalanceChangeReason is used to indicate the reason for a balance change, useful

eth/tracers/firehose_test.go

@@ -0,0 +1,288 @@
+package tracers
+
+import (
+	"encoding/json"
+	"fmt"
+	"math/big"
+	"reflect"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	pbeth "github.com/ethereum/go-ethereum/pb/sf/ethereum/type/v2"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/exp/maps"
+	"google.golang.org/protobuf/types/known/timestamppb"
+)
+
+func TestFirehoseCallStack_Push(t *testing.T) {
+	type actionRunner func(t *testing.T, s *CallStack)
+
+	push := func(call *pbeth.Call) actionRunner { return func(_ *testing.T, s *CallStack) { s.Push(call) } }
+	pop := func() actionRunner { return func(_ *testing.T, s *CallStack) { s.Pop() } }
+	check := func(r actionRunner) actionRunner { return func(t *testing.T, s *CallStack) { r(t, s) } }
+
+	tests := []struct {
+		name    string
+		actions []actionRunner
+	}{
+		{
+			"push/pop emtpy", []actionRunner{
+				push(&pbeth.Call{}),
+				pop(),
+				check(func(t *testing.T, s *CallStack) {
+					require.Len(t, s.stack, 0)
+				}),
+			},
+		},
+		{
+			"push/push/push", []actionRunner{
+				push(&pbeth.Call{}),
+				push(&pbeth.Call{}),
+				push(&pbeth.Call{}),
+				check(func(t *testing.T, s *CallStack) {
+					require.Len(t, s.stack, 3)
+
+					require.Equal(t, 1, int(s.stack[0].Index))
+					require.Equal(t, 0, int(s.stack[0].ParentIndex))
+
+					require.Equal(t, 2, int(s.stack[1].Index))
+					require.Equal(t, 1, int(s.stack[1].ParentIndex))
+
+					require.Equal(t, 3, int(s.stack[2].Index))
+					require.Equal(t, 2, int(s.stack[2].ParentIndex))
+				}),
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			s := NewCallStack()
+
+			for _, action := range tt.actions {
+				action(t, s)
+			}
+		})
+	}
+}
+
+func Test_validateKnownTransactionTypes(t *testing.T) {
+	tests := []struct {
+		name      string
+		txType    byte
+		knownType bool
+		want      error
+	}{
+		{"legacy", 0, true, nil},
+		{"access_list", 1, true, nil},
+		{"inexistant", 255, false, nil},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := validateFirehoseKnownTransactionType(tt.txType, tt.knownType)
+			if tt.want == nil && err != nil {
+				t.Fatalf("Transaction of type %d expected to validate properly but received error %q", tt.txType, err)
+			} else if tt.want != nil && err == nil {
+				t.Fatalf("Transaction of type %d expected to validate improperly but generated no error", tt.txType)
+			} else if tt.want != nil && err != nil && tt.want.Error() != err.Error() {
+				t.Fatalf("Transaction of type %d expected to validate improperly but generated error %q does not match expected error %q", tt.txType, err, tt.want)
+			}
+		})
+	}
+}
+
+var ignorePbFieldNames = map[string]bool{
+	"Hash":            true,
+	"TotalDifficulty": true,
+	"state":           true,
+	"unknownFields":   true,
+	"sizeCache":       true,
+
+	// This was a Polygon specific field that existed for a while and has since been
+	// removed. It can be safely ignored in all protocols now.
+	"TxDependency": true,
+}
+
+var pbFieldNameToGethMapping = map[string]string{
+	"WithdrawalsRoot":  "WithdrawalsHash",
+	"MixHash":          "MixDigest",
+	"BaseFeePerGas":    "BaseFee",
+	"StateRoot":        "Root",
+	"ExtraData":        "Extra",
+	"Timestamp":        "Time",
+	"ReceiptRoot":      "ReceiptHash",
+	"TransactionsRoot": "TxHash",
+	"LogsBloom":        "Bloom",
+}
+
+var (
+	pbHeaderType   = reflect.TypeFor[pbeth.BlockHeader]()
+	gethHeaderType = reflect.TypeFor[types.Header]()
+)
+
+func Test_TypesHeader_AllConsensusFieldsAreKnown(t *testing.T) {
+	// This exact hash varies from protocol to protocol and also sometimes from one version to the other.
+	// When adding support for a new hard-fork that adds new block header fields, it's normal that this value
+	// changes. If you are sure the two struct are the same, then you can update the expected hash below
+	// to the new value.
+	expectedHash := common.HexToHash("5341947c531e5c9cf38202784b16ac66484fe1838aa6e825436b22321b927296")
+
+	gethHeaderValue := reflect.New(gethHeaderType)
+	fillAllFieldsWithNonEmptyValues(t, gethHeaderValue, reflect.VisibleFields(gethHeaderType))
+	gethHeader := gethHeaderValue.Interface().(*types.Header)
+
+	// If you hit this assertion, it means that the fields `types.Header` of go-ethereum differs now
+	// versus last time this test was edited.
+	//
+	// It's important to understand that in Ethereum Block Header (e.g. `*types.Header`), the `Hash` is
+	// actually a computed value based on the other fields in the struct, so if you change any field,
+	// the hash will change also.
+	//
+	// On hard-fork, it happens that new fields are added, this test serves as a way to "detect" in codde
+	// that the expected fields of `types.Header` changed
+	require.Equal(t, expectedHash, gethHeader.Hash(),
+		"Geth Header Hash mistmatch, got %q but expecting %q on *types.Header:\n\nGeth Header (from fillNonDefault(new(*types.Header)))\n%s",
+		gethHeader.Hash().Hex(),
+		expectedHash,
+		asIndentedJSON(t, gethHeader),
+	)
+}
+
+func Test_FirehoseAndGethHeaderFieldMatches(t *testing.T) {
+	pbFields := filter(reflect.VisibleFields(pbHeaderType), func(f reflect.StructField) bool {
+		return !ignorePbFieldNames[f.Name]
+	})
+
+	gethFields := reflect.VisibleFields(gethHeaderType)
+
+	pbFieldCount := len(pbFields)
+	gethFieldCount := len(gethFields)
+
+	pbFieldNames := extractStructFieldNames(pbFields)
+	gethFieldNames := extractStructFieldNames(gethFields)
+
+	// If you reach this assertion, it means that the fields count in the protobuf and go-ethereum are different.
+	// It is super important that you properly update the mapping from pbeth.BlockHeader to go-ethereum/core/types.Header
+	// that is done in `codecHeaderToGethHeader` function in `executor/provider_statedb.go`.
+	require.Equal(
+		t,
+		pbFieldCount,
+		gethFieldCount,
+		fieldsCountMistmatchMessage(t, pbFieldNames, gethFieldNames))
+
+	for pbFieldName := range pbFieldNames {
+		pbFieldRenamedName, found := pbFieldNameToGethMapping[pbFieldName]
+		if !found {
+			pbFieldRenamedName = pbFieldName
+		}
+
+		assert.Contains(t, gethFieldNames, pbFieldRenamedName, "pbField.Name=%q (original %q) not found in gethFieldNames", pbFieldRenamedName, pbFieldName)
+	}
+}
+
+func fillAllFieldsWithNonEmptyValues(t *testing.T, structValue reflect.Value, fields []reflect.StructField) {
+	t.Helper()
+
+	for _, field := range fields {
+		fieldValue := structValue.Elem().FieldByName(field.Name)
+		require.True(t, fieldValue.IsValid(), "field %q not found", field.Name)
+
+		switch fieldValue.Interface().(type) {
+		case []byte:
+			fieldValue.Set(reflect.ValueOf([]byte{1}))
+		case uint64:
+			fieldValue.Set(reflect.ValueOf(uint64(1)))
+		case *uint64:
+			var mockValue uint64 = 1
+			fieldValue.Set(reflect.ValueOf(&mockValue))
+		case *common.Hash:
+			var mockValue common.Hash = common.HexToHash("0x01")
+			fieldValue.Set(reflect.ValueOf(&mockValue))
+		case common.Hash:
+			fieldValue.Set(reflect.ValueOf(common.HexToHash("0x01")))
+		case common.Address:
+			fieldValue.Set(reflect.ValueOf(common.HexToAddress("0x01")))
+		case types.Bloom:
+			fieldValue.Set(reflect.ValueOf(types.BytesToBloom([]byte{1})))
+		case types.BlockNonce:
+			fieldValue.Set(reflect.ValueOf(types.EncodeNonce(1)))
+		case *big.Int:
+			fieldValue.Set(reflect.ValueOf(big.NewInt(1)))
+		case *pbeth.BigInt:
+			fieldValue.Set(reflect.ValueOf(&pbeth.BigInt{Bytes: []byte{1}}))
+		case *timestamppb.Timestamp:
+			fieldValue.Set(reflect.ValueOf(&timestamppb.Timestamp{Seconds: 1}))
+		default:
+			// If you reach this panic in test, simply add a case above with a sane non-default
+			// value for the type in question.
+			t.Fatalf("unsupported type %T", fieldValue.Interface())
+		}
+	}
+}
+
+func fieldsCountMistmatchMessage(t *testing.T, pbFieldNames map[string]bool, gethFieldNames map[string]bool) string {
+	t.Helper()
+
+	pbRemappedFieldNames := make(map[string]bool, len(pbFieldNames))
+	for pbFieldName := range pbFieldNames {
+		pbFieldRenamedName, found := pbFieldNameToGethMapping[pbFieldName]
+		if !found {
+			pbFieldRenamedName = pbFieldName
+		}
+
+		pbRemappedFieldNames[pbFieldRenamedName] = true
+	}
+
+	return fmt.Sprintf(
+		"Field count mistmatch between `pbeth.BlockHeader` (has %d fields) and `*types.Header` (has %d fields)\n\n"+
+			"Fields in `pbeth.Blockheader`:\n%s\n\n"+
+			"Fields in `*types.Header`:\n%s\n\n"+
+			"Missing in `pbeth.BlockHeader`:\n%s\n\n"+
+			"Missing in `*types.Header`:\n%s",
+		len(pbRemappedFieldNames),
+		len(gethFieldNames),
+		asIndentedJSON(t, maps.Keys(pbRemappedFieldNames)),
+		asIndentedJSON(t, maps.Keys(gethFieldNames)),
+		asIndentedJSON(t, missingInSet(gethFieldNames, pbRemappedFieldNames)),
+		asIndentedJSON(t, missingInSet(pbRemappedFieldNames, gethFieldNames)),
+	)
+}
+
+func asIndentedJSON(t *testing.T, v any) string {
+	t.Helper()
+	out, err := json.MarshalIndent(v, "", "  ")
+	require.NoError(t, err)
+
+	return string(out)
+}
+
+func missingInSet(a, b map[string]bool) []string {
+	missing := make([]string, 0)
+	for name := range a {
+		if !b[name] {
+			missing = append(missing, name)
+		}
+	}
+
+	return missing
+}
+
+func extractStructFieldNames(fields []reflect.StructField) map[string]bool {
+	result := make(map[string]bool, len(fields))
+	for _, field := range fields {
+		result[field.Name] = true
+	}
+	return result
+}
+
+func filter[S ~[]T, T any](s S, f func(T) bool) (out S) {
+	out = make(S, 0, len(s)/4)
+	for i, v := range s {
+		if f(v) {
+			out = append(out, s[i])
+		}
+	}
+
+	return out
+}

go.mod

@@ -70,6 +70,7 @@ require (
 	golang.org/x/text v0.13.0
 	golang.org/x/time v0.3.0
 	golang.org/x/tools v0.13.0
+	google.golang.org/protobuf v1.27.1
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -138,7 +139,6 @@ require (
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
-	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	rsc.io/tmplfunc v0.0.3 // indirect
 )

pb/sf/ethereum/type/v2/type.go

@@ -0,0 +1,27 @@
+package pbeth
+
+import (
+	"encoding/hex"
+	"math/big"
+	"time"
+)
+
+var b0 = big.NewInt(0)
+
+func (b *Block) PreviousID() string {
+	return hex.EncodeToString(b.Header.ParentHash)
+}
+
+func (b *Block) Time() time.Time {
+	return b.Header.Timestamp.AsTime()
+}
+
+func (m *BigInt) Native() *big.Int {
+	if m == nil {
+		return b0
+	}
+
+	z := new(big.Int)
+	z.SetBytes(m.Bytes)
+	return z
+}