diff --git a/beacon/light/api/sync_server.go b/beacon/light/api/sync_server.go index a2fda4b6a1..e156b4b6b6 100755 --- a/beacon/light/api/sync_server.go +++ b/beacon/light/api/sync_server.go @@ -18,12 +18,12 @@ package api import ( "sync" - "time" "github.com/ethereum/go-ethereum/beacon/light" "github.com/ethereum/go-ethereum/beacon/light/types" "github.com/ethereum/go-ethereum/beacon/merkle" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/log" "github.com/protolambda/zrnt/eth2/beacon/capella" ) @@ -71,7 +71,7 @@ func (s *SyncServer) UnsubscribeHeads() { s.lock.Unlock() } -func (s *SyncServer) Delay() time.Duration { return 0 } //TODO +func (s *SyncServer) DelayUntil() mclock.AbsTime { return 0 } //TODO func (s *SyncServer) Fail(desc string) { log.Warn("API endpoint failure", "URL", s.api.url, "error", desc) diff --git a/beacon/light/request/scheduler.go b/beacon/light/request/scheduler.go index 9bd4122e46..775142e523 100644 --- a/beacon/light/request/scheduler.go +++ b/beacon/light/request/scheduler.go @@ -20,8 +20,7 @@ import ( "sync" "time" - "github.com/ethereum/go-ethereum/beacon/light/types" - "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/mclock" ) const softRequestTimeout = time.Second @@ -49,15 +48,6 @@ type Module interface { Process(env *Environment) } -// RequestServer is a general server interface that can be extended by modules -// with specific request types. -type RequestServer interface { - SubscribeHeads(newHead func(uint64, common.Hash), newSignedHead func(types.SignedHead)) - UnsubscribeHeads() - Delay() time.Duration // if non-zero then no requests should be sent for the given duration - Fail(string) // report server failure -} - // ModuleTrigger allows modules to trigger themselves or each other when changes // in their underlying data structures could have made further operations possible. type ModuleTrigger struct { @@ -87,25 +77,33 @@ type Scheduler struct { headTracker *HeadTracker lock sync.Mutex + clock mclock.Clock modules []Module // first has highest priority servers []*Server triggers map[string]*ModuleTrigger triggeredBy map[Module][]*ModuleTrigger stopCh chan chan struct{} - triggerCh chan struct{} - triggerLock sync.Mutex - processing, triggered bool - trModules map[Module]struct{} - trServers map[*Server]struct{} + triggerCh chan struct{} // restarts waiting sync loop + testWaitCh chan struct{} // accepts sends when sync loop is waiting + testTimerCh chan bool // sends true when simulated timer is processed; false when stopped + triggerLock sync.Mutex + waiting, triggered bool + trModules map[Module]struct{} + trServers map[*Server]struct{} } // NewScheduler creates a new Scheduler. -func NewScheduler(headTracker *HeadTracker) *Scheduler { +func NewScheduler(headTracker *HeadTracker, clock mclock.Clock) *Scheduler { s := &Scheduler{ headTracker: headTracker, + clock: clock, stopCh: make(chan chan struct{}), - triggerCh: make(chan struct{}, 1), + // Note: triggerCh and testWaitCh should not have capacity in order to ensure + // that after a trigger happens testWaitCh will block until the resulting + // processing round has been finished + triggerCh: make(chan struct{}), + testWaitCh: make(chan struct{}), triggers: make(map[string]*ModuleTrigger), triggeredBy: make(map[Module][]*ModuleTrigger), } @@ -160,6 +158,7 @@ func (s *Scheduler) RegisterServer(requestServer RequestServer) { server := s.newServer(requestServer) s.servers = append(s.servers, server) s.headTracker.registerServer(server) + s.triggerServer(server) } // UnregisterServer removes a registered server. @@ -204,7 +203,6 @@ func (s *Scheduler) Stop() { func (s *Scheduler) syncLoop() { s.lock.Lock() s.triggerLock.Lock() - s.processing = true for { trModules, trServers := s.trModules, s.trServers s.trModules, s.trServers = nil, nil @@ -213,19 +211,24 @@ func (s *Scheduler) syncLoop() { s.processModules(trModules, trServers) s.triggerLock.Lock() } else { - s.processing = false + s.waiting = true s.triggerLock.Unlock() s.lock.Unlock() - select { - case stop := <-s.stopCh: - close(stop) - return - case <-s.triggerCh: + loop: + for { + select { + case stop := <-s.stopCh: + close(stop) + return + case <-s.triggerCh: + break loop + case <-s.testWaitCh: + } } s.lock.Lock() s.triggerLock.Lock() s.triggered = false - s.processing = true + s.waiting = false } } } @@ -275,7 +278,7 @@ func (s *Scheduler) triggerServer(server *Server) { s.trServers = make(map[*Server]struct{}) } s.trServers[server] = struct{}{} - if !s.processing && !s.triggered { + if s.waiting && !s.triggered { s.triggerCh <- struct{}{} s.triggered = true } @@ -291,7 +294,7 @@ func (s *Scheduler) triggerModule(module Module) { s.trModules = make(map[Module]struct{}) } s.trModules[module] = struct{}{} - if !s.processing && !s.triggered { + if s.waiting && !s.triggered { s.triggerCh <- struct{}{} s.triggered = true } diff --git a/beacon/light/request/scheduler_test.go b/beacon/light/request/scheduler_test.go new file mode 100644 index 0000000000..a7f0d924b2 --- /dev/null +++ b/beacon/light/request/scheduler_test.go @@ -0,0 +1,307 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package request + +import ( + "fmt" + "testing" + "time" + + "github.com/ethereum/go-ethereum/beacon/light/types" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/mclock" +) + +type testRequestServer struct { + newHead func(uint64, common.Hash) + newSignedHead func(types.SignedHead) + clock *mclock.Simulated + delayUntil mclock.AbsTime + failed bool +} + +func (s *testRequestServer) SubscribeHeads(newHead func(uint64, common.Hash), newSignedHead func(types.SignedHead)) { + s.newHead, s.newSignedHead = newHead, newSignedHead +} + +func (s *testRequestServer) UnsubscribeHeads() {} + +func (s *testRequestServer) DelayUntil() mclock.AbsTime { + return s.delayUntil +} + +func (s *testRequestServer) Fail(string) { + s.failed = true +} + +type testModule struct { + processCh chan testProcess + triggers []int + index int +} + +func (m *testModule) SetupModuleTriggers(trigger func(id string, subscribe bool) *ModuleTrigger) { + for _, tr := range m.triggers { + trigger(fmt.Sprintf("trigger %d", tr), true) + } +} + +type testProcess struct { + m *testModule + env *Environment + done chan struct{} +} + +func (m *testModule) Process(env *Environment) { + done := make(chan struct{}) + m.processCh <- testProcess{m, env, done} + <-done +} + +type schedulerTest struct { + t *testing.T + clock *mclock.Simulated + scheduler *Scheduler + modules []*testModule + servers []*testRequestServer + triggers []*ModuleTrigger + processCh chan testProcess + + env *Environment + done chan struct{} +} + +func newSchedulerTest(t *testing.T, clock *mclock.Simulated, serverCount, triggerCount int, moduleTriggers [][]int) *schedulerTest { + st := &schedulerTest{ + t: t, + clock: clock, + scheduler: NewScheduler(NewHeadTracker(func(*Server, types.SignedHead) {}), clock), + modules: make([]*testModule, len(moduleTriggers)), + triggers: make([]*ModuleTrigger, triggerCount), + processCh: make(chan testProcess), + } + for i, tr := range moduleTriggers { + st.modules[i] = &testModule{ + processCh: st.processCh, + triggers: tr, + index: i, + } + st.scheduler.RegisterModule(st.modules[i]) + } + for i := range st.triggers { + st.triggers[i] = st.scheduler.GetModuleTrigger(fmt.Sprintf("trigger %d", i)) + } + st.scheduler.Start() + st.expectProcess(-1) // expect initial waiting state + for i := 0; i < serverCount; i++ { + st.addServer() + } + return st +} + +func (st *schedulerTest) addServer() *testRequestServer { + server := &testRequestServer{ + clock: st.clock, + } + st.scheduler.RegisterServer(server) + st.servers = append(st.servers, server) + st.expectServerTrigger() + return server +} + +func (st *schedulerTest) expectServerTrigger() { + for i := range st.modules { + st.expectProcess(i) // expect all modules to be triggered + st.processDone() + } + st.expectProcess(-1) // expect waiting state +} + +func (st *schedulerTest) expectProcess(expIndex int) { + var ( + tp testProcess + index int + ) + select { + case tp = <-st.processCh: + index = tp.m.index + case st.scheduler.testWaitCh <- struct{}{}: + index = -1 + } + if index != expIndex { + st.t.Fatalf("Incorrect processed module index (got %d, expected %d)", index, expIndex) + } + st.env, st.done = tp.env, tp.done +} + +func (st *schedulerTest) processDone() { + close(st.done) +} + +func TestModuleTrigger(t *testing.T) { + st := newSchedulerTest(t, &mclock.Simulated{}, 0, 6, [][]int{{0, 1, 2, 4}, {1, 3}, {0, 2, 4}, {2, 3, 4}}) + st.triggers[0].Trigger() // triggers modules 0, 2 + // round 1 + st.expectProcess(0) + st.triggers[1].Trigger() // triggers modules 0, 1 + st.triggers[2].Trigger() // triggers modules 0, 2, 3 + st.triggers[3].Trigger() // triggers modules 1, 3 + st.triggers[4].Trigger() // triggers modules 0, 2, 3 + st.processDone() + st.expectProcess(2) + st.processDone() + // round 2 + st.expectProcess(0) + st.processDone() + st.expectProcess(1) + st.triggers[3].Trigger() // triggers modules 1, 3 + st.processDone() + st.expectProcess(2) + st.processDone() + st.expectProcess(3) + st.triggers[1].Trigger() // triggers modules 0, 1 + st.processDone() + // round 3 + st.expectProcess(0) + st.processDone() + st.expectProcess(1) + st.triggers[5].Trigger() // doesn't trigger anything + st.processDone() + st.expectProcess(3) + st.processDone() + st.expectProcess(-1) + // waiting for a next trigger + st.triggers[2].Trigger() // triggers modules 0, 2, 3 + // round 4 + st.expectProcess(0) + st.processDone() + st.expectProcess(2) + st.processDone() + st.expectProcess(3) + st.processDone() + st.expectProcess(-1) +} + +type testRequest struct { + reqLock SingleLock + returnFns []func() +} + +func (r *testRequest) CanSendTo(server *Server) (canSend bool, priority uint64) { + return r.reqLock.CanRequest(), 0 +} + +func (r *testRequest) SendTo(server *Server) { + reqId := r.reqLock.Send(server) + r.returnFns = append(r.returnFns, func() { + r.reqLock.Returned(server, reqId) + }) +} + +func (r *testRequest) returned() { + r.returnFns[0]() + r.returnFns = r.returnFns[1:] +} + +func TestServerTrigger(t *testing.T) { + st := newSchedulerTest(t, &mclock.Simulated{}, 3, 2, [][]int{{0}, {}}) + st.scheduler.testTimerCh = make(chan bool) + req := &testRequest{} + req.reqLock.Trigger = st.triggers[0] + + tryRequest := func(expCanRequest, expSuccess, serverTrigger bool) { + st.expectProcess(0) + c := st.env.CanRequestNow() + if c != expCanRequest { + t.Fatalf("Environment.CanRequestNow() returned wrong result (got: %v expected: %v)", c, expCanRequest) + } + if c { + if s := st.env.TryRequest(req); s != expSuccess { + t.Fatalf("Environment.TryRequest(req) returned wrong result (got: %v expected: %v)", s, expSuccess) + } + } + st.processDone() + if serverTrigger { + for i := 1; i < len(st.modules); i++ { + st.expectProcess(i) + st.processDone() + } + } + st.expectProcess(-1) + } + + expectActiveTimers := func(exp int) { + if timers := st.clock.ActiveTimers(); timers != exp { + t.Fatalf("Invalid number of simulated clock timers (got %v, expected %v)", timers, exp) + } + } + + expectTimerFinished := func(exp bool) { + if processed := <-st.scheduler.testTimerCh; processed != exp { + t.Fatalf("Invalid simulated timer result (got processed == %v, expected %v)", processed, exp) + } + } + + st.servers[2].delayUntil = mclock.AbsTime(time.Second * 5) + st.triggers[0].Trigger() + tryRequest(true, true, false) + expectActiveTimers(2) // delay, timeout + st.triggers[0].Trigger() + // there are available servers but first request not timed out yet + tryRequest(true, false, false) + expectActiveTimers(2) // delay, timeout + st.clock.Run(softRequestTimeout) + expectActiveTimers(1) // delay + expectTimerFinished(true) // timeout timer processed + // expect module trigger by Server (timeout), should be able to send again + tryRequest(true, true, false) + expectActiveTimers(2) // delay, timeout + st.clock.Run(softRequestTimeout) + expectActiveTimers(1) // delay + expectTimerFinished(true) // timeout timer processed + // two servers timed out and one delayed; no one to request from + tryRequest(false, false, false) + st.clock.Run(time.Second*5 - softRequestTimeout*2) + expectActiveTimers(0) + expectTimerFinished(true) // timeout timer processed + // expect server trigger by Server (expired delay), should be able to send again + tryRequest(true, true, true) + expectActiveTimers(1) // timeout + st.clock.Run(softRequestTimeout) + expectActiveTimers(0) + expectTimerFinished(true) // timeout timer processed + // expect module trigger by Server; all servers timed out now + tryRequest(false, false, false) + req.returned() + st.expectServerTrigger() // server triggered because not blocked by timeout anymore + req.returned() + st.expectServerTrigger() + req.returned() + st.expectServerTrigger() + // now send a request that does not time out + st.triggers[0].Trigger() + tryRequest(true, true, false) + expectActiveTimers(1) // timeout + st.clock.Run(softRequestTimeout / 2) + expectActiveTimers(1) // timeout + req.returned() + expectTimerFinished(false) // timeout timer stopped + expectActiveTimers(0) + st.expectProcess(0) // module triggered by request lock + st.processDone() + st.expectProcess(-1) // no server trigger expected because server was not blocked by timeout +} diff --git a/beacon/light/request/server.go b/beacon/light/request/server.go index b6b78b96a8..eb185a5567 100644 --- a/beacon/light/request/server.go +++ b/beacon/light/request/server.go @@ -21,9 +21,20 @@ import ( "sync" "time" + "github.com/ethereum/go-ethereum/beacon/light/types" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/mclock" ) +// RequestServer is a general server interface that can be extended by modules +// with specific request types. +type RequestServer interface { + SubscribeHeads(newHead func(uint64, common.Hash), newSignedHead func(types.SignedHead)) + UnsubscribeHeads() + DelayUntil() mclock.AbsTime // no requests should be sent before this + Fail(string) // report server failure +} + // Server is a wrapper around RequestServer that handles request timeouts, delays // and keeps track of the server's latest reported (not necessarily validated) head. type Server struct { @@ -38,8 +49,8 @@ type Server struct { lock sync.Mutex sent map[uint64]chan struct{} // closed when returned; nil when timed out timeoutCount int - delayed bool - delayChecked bool + delayUntil mclock.AbsTime + delayTimer mclock.ChanTimer // if non-nil then expires at delayUntil needTrigger bool lastReqId uint64 stopCh chan struct{} @@ -95,29 +106,53 @@ func (s *Server) canRequestNow() (bool, uint64) { // delayed. In this case it also starts a timer to ensure that a server trigger // can be emitted when the server becomes available again. func (s *Server) isDelayed() bool { - if s.delayChecked { - return s.delayed + delayUntil := s.RequestServer.DelayUntil() + if delayUntil == s.delayUntil { + return s.delayTimer != nil } - s.delayChecked = true - delay := s.RequestServer.Delay() - if s.delayed = delay > 0; s.delayed { - go func() { - timer := time.NewTimer(delay) - select { - case <-timer.C: - s.lock.Lock() - s.delayed = false + s.delayUntil = delayUntil + if s.delayTimer != nil { + if s.delayTimer.Stop() { + if s.scheduler.testTimerCh != nil { + s.scheduler.testTimerCh <- false // simulated timer stopped + } + } else { + s.delayTimer = nil + } + } + delay := time.Duration(delayUntil - s.scheduler.clock.Now()) + if delay <= 0 { + s.delayTimer = nil + return false + } + if s.delayTimer == nil { + s.delayTimer = s.scheduler.clock.NewTimer(delay) + } else { + s.delayTimer.Reset(delay) + } + timer := s.delayTimer + go func() { + select { + case <-timer.C(): + s.lock.Lock() + if s.delayTimer == timer { + s.delayTimer = nil if s.needTrigger && s.timeoutCount == 0 { s.needTrigger = false s.scheduler.triggerServer(s) } - s.lock.Unlock() - case <-s.stopCh: - timer.Stop() } - }() - } - return s.delayed + s.lock.Unlock() + if s.scheduler.testTimerCh != nil { + s.scheduler.testTimerCh <- true // simulated timer processed + } + case <-s.stopCh: + if timer.Stop() && s.scheduler.testTimerCh != nil { + s.scheduler.testTimerCh <- false // simulated timer stopped + } + } + }() + return true } // sendRequest generates a request ID and starts a timeout timer. If the timeout @@ -130,11 +165,10 @@ func (s *Server) sendRequest(timeoutTrigger *ModuleTrigger) uint64 { reqId := s.lastReqId returnCh := make(chan struct{}) s.sent[reqId] = returnCh - s.delayChecked = false + timer := s.scheduler.clock.NewTimer(softRequestTimeout) go func() { - timer := time.NewTimer(softRequestTimeout) select { - case <-timer.C: + case <-timer.C(): s.lock.Lock() if _, ok := s.sent[reqId]; ok { s.sent[reqId] = nil @@ -144,10 +178,17 @@ func (s *Server) sendRequest(timeoutTrigger *ModuleTrigger) uint64 { if timeoutTrigger != nil { timeoutTrigger.Trigger() } + if s.scheduler.testTimerCh != nil { + s.scheduler.testTimerCh <- true // simulated timer processed + } case <-returnCh: - timer.Stop() + if timer.Stop() && s.scheduler.testTimerCh != nil { + s.scheduler.testTimerCh <- false // simulated timer stopped + } case <-s.stopCh: - timer.Stop() + if timer.Stop() && s.scheduler.testTimerCh != nil { + s.scheduler.testTimerCh <- false // simulated timer stopped + } } }() return reqId diff --git a/cmd/blsync/main.go b/cmd/blsync/main.go index 597c7cac22..8f73e2888a 100644 --- a/cmd/blsync/main.go +++ b/cmd/blsync/main.go @@ -127,7 +127,7 @@ func blsync(ctx *cli.Context) error { } // set up sync modules and triggers - scheduler := request.NewScheduler(headTracker) + scheduler := request.NewScheduler(headTracker, &mclock.System{}) scheduler.RegisterModule(checkpointInit) scheduler.RegisterModule(forwardSync) scheduler.RegisterModule(headUpdater)