diff --git a/eth/handler.go b/eth/handler.go
index 0ef98e454f..9a02f1f20f 100644
--- a/eth/handler.go
+++ b/eth/handler.go
@@ -275,7 +275,6 @@ func (pm *ProtocolManager) Stop() {
 	// Quit chainSync and txsync64.
 	// After this is done, no new peers will be accepted.
 	close(pm.quitSync)
-	pm.downloader.Cancel()
 	pm.wg.Wait()
 
 	// Disconnect existing sessions.
diff --git a/eth/sync.go b/eth/sync.go
index 0b9506c720..1fecba4fd3 100644
--- a/eth/sync.go
+++ b/eth/sync.go
@@ -220,6 +220,7 @@ func (cs *chainSyncer) loop() {
 
 		case <-cs.pm.quitSync:
 			if cs.doneCh != nil {
+				cs.pm.downloader.Cancel()
 				<-cs.doneCh
 			}
 			return