DEV: Added configuration

This commit is contained in:
David Zhou 2025-05-12 09:36:55 -04:00
parent bd85c18970
commit e8ddd05284
3 changed files with 34 additions and 19 deletions

View file

@ -109,8 +109,7 @@ func NewTracingHooksFromFirehose(tracer *Firehose) *tracing.Hooks {
type FirehoseConfig struct { type FirehoseConfig struct {
ApplyBackwardCompatibility *bool `json:"applyBackwardCompatibility"` ApplyBackwardCompatibility *bool `json:"applyBackwardCompatibility"`
// TODO: Add LOGGING on init ConcurrentBlockFlushing bool `json:"concurrentBlockFlushing"`
ConcurrentBlockFlushing bool `json:"concurrentBlockFlushing"`
// Only used for testing, only possible through JSON configuration // Only used for testing, only possible through JSON configuration
private *privateFirehoseConfig private *privateFirehoseConfig
@ -131,6 +130,7 @@ func (c *FirehoseConfig) LogKeyValues() []any {
return []any{ return []any{
"config.applyBackwardCompatibility", applyBackwardCompatibility, "config.applyBackwardCompatibility", applyBackwardCompatibility,
"config.concurrentBlockFlushing", c.ConcurrentBlockFlushing,
} }
} }
@ -160,6 +160,7 @@ type Firehose struct {
// here. If not set in the config, then we inspect `OnBlockchainInit` the chain config to determine // here. If not set in the config, then we inspect `OnBlockchainInit` the chain config to determine
// if it's a network for which we must reproduce the legacy bugs. // if it's a network for which we must reproduce the legacy bugs.
applyBackwardCompatibility *bool applyBackwardCompatibility *bool
concurrentBlockFlushing bool
// Block state // Block state
block *pbeth.Block block *pbeth.Block
@ -190,7 +191,7 @@ type Firehose struct {
testingBuffer *bytes.Buffer testingBuffer *bytes.Buffer
testingIgnoreGenesisBlock bool testingIgnoreGenesisBlock bool
// Worker queuefor blockprinting // Worker queue for block printing
blockPrintQueue chan *blockPrintJob blockPrintQueue chan *blockPrintJob
flushDone sync.WaitGroup flushDone sync.WaitGroup
closeOnce sync.Once closeOnce sync.Once
@ -233,6 +234,7 @@ func NewFirehose(config *FirehoseConfig) *Firehose {
hasher: crypto.NewKeccakState(), hasher: crypto.NewKeccakState(),
tracerID: "global", tracerID: "global",
applyBackwardCompatibility: config.ApplyBackwardCompatibility, applyBackwardCompatibility: config.ApplyBackwardCompatibility,
concurrentBlockFlushing: config.ConcurrentBlockFlushing,
// Block state // Block state
blockOrdinal: &Ordinal{}, blockOrdinal: &Ordinal{},
@ -246,8 +248,6 @@ func NewFirehose(config *FirehoseConfig) *Firehose {
callStack: NewCallStack(), callStack: NewCallStack(),
deferredCallState: NewDeferredCallState(), deferredCallState: NewDeferredCallState(),
latestCallEnterSuicided: false, latestCallEnterSuicided: false,
blockPrintQueue: make(chan *blockPrintJob, 100),
} }
if config.private != nil { if config.private != nil {
@ -257,8 +257,12 @@ func NewFirehose(config *FirehoseConfig) *Firehose {
} }
} }
firehose.flushDone.Add(1) if config.ConcurrentBlockFlushing {
go firehose.blockPrintWorker() log.Info("Concurrent block flushing enabled: starting goroutine...")
firehose.blockPrintQueue = make(chan *blockPrintJob, 100)
firehose.flushDone.Add(1)
go firehose.blockPrintWorker()
}
return firehose return firehose
} }
@ -496,12 +500,17 @@ func (f *Firehose) OnBlockEnd(err error) {
} }
f.ensureInBlockAndNotInTrx() f.ensureInBlockAndNotInTrx()
// f.printBlockToFirehose(f.block, f.blockFinality)
job := &blockPrintJob{ // Flush block to firehose and optionally use goroutine
block: f.block, if f.concurrentBlockFlushing {
finality: f.blockFinality, job := &blockPrintJob{
block: f.block,
finality: f.blockFinality,
}
f.blockPrintQueue <- job
} else {
f.printBlockToFirehose(f.block, f.blockFinality)
} }
f.blockPrintQueue <- job
} else { } else {
// An error occurred, could have happen in transaction/call context, we must not check if in trx/call, only check in block // An error occurred, could have happen in transaction/call context, we must not check if in trx/call, only check in block
@ -621,8 +630,11 @@ func (f *Firehose) reorderCallOrdinals(call *pbeth.Call, ordinalBase uint64) (or
} }
func (f *Firehose) OnClose() { func (f *Firehose) OnClose() {
log.Info("Firehose closing: waiting for worker goroutines to finish and shutting down channels") log.Info("Firehose closing...")
f.CloseBlockPrintQueue() if f.concurrentBlockFlushing {
log.Info("waiting for worker goroutines to finish and shutting down channels")
f.CloseBlockPrintQueue()
}
} }
func (f *Firehose) OnSystemCallStart() { func (f *Firehose) OnSystemCallStart() {
@ -2829,8 +2841,11 @@ func (f *Firehose) blockPrintWorker() {
} }
func (f *Firehose) CloseBlockPrintQueue() { func (f *Firehose) CloseBlockPrintQueue() {
f.closeOnce.Do(func() { if f.concurrentBlockFlushing {
close(f.blockPrintQueue) f.closeOnce.Do(func() {
f.flushDone.Wait() log.Info("Closing channel: flushing the remaining blocks to firehose")
}) close(f.blockPrintQueue)
f.flushDone.Wait()
})
}
} }

View file

@ -56,7 +56,6 @@ func TestFirehosePrestate(t *testing.T) {
blockLines.assertOnlyBlockEquals(t, filepath.Join(folder, string(model)), 1) blockLines.assertOnlyBlockEquals(t, filepath.Join(folder, string(model)), 1)
}) })
} }
} }
} }

View file

@ -33,6 +33,7 @@ func newFirehoseTestTracer(t *testing.T, model tracingModel) (*tracers.Firehose,
t.Helper() t.Helper()
tracer, err := tracers.NewFirehoseFromRawJSON([]byte(fmt.Sprintf(`{ tracer, err := tracers.NewFirehoseFromRawJSON([]byte(fmt.Sprintf(`{
"concurrentBlockFlushing": true,
"_private": { "_private": {
"flushToTestBuffer": true, "flushToTestBuffer": true,
"ignoreGenesisBlock": true, "ignoreGenesisBlock": true,