mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-05-13 11:36:37 +00:00
accounts/keystore: use map instead of slice to keep all accounts
After profiling geth startup times for #16874 it turned out that most of the time was spent resizing accounts slice for ordered insertion. This change uses map to track all accounts instead of slice to improve performance. geth startup time using keystore with one million accounts reduced from 23 minutes to 2.5 minutes. Benchmarks show relatively-small overhead increase for small keystores and large decrease for huge: ``` goos: linux goarch: amd64 pkg: github.com/ethereum/go-ethereum/accounts/keystore │ HEAD~1 │ HEAD │ │ sec/op │ sec/op vs base │ Add/preload=10-8 1.030µ ± 5% 1.447µ ± 7% +40.51% (p=0.000 n=10) Add/preload=100-8 1.109µ ± 4% 1.463µ ± 3% +31.88% (p=0.000 n=10) Add/preload=1000-8 1.860µ ± 3% 1.477µ ± 5% -20.57% (p=0.000 n=10) Add/preload=1000000-8 5177.640µ ± 2% 1.654µ ± 11% -99.97% (p=0.000 n=10) Find/preload=10/by_address-8 23.70n ± 1% 23.88n ± 3% ~ (p=0.271 n=10) Find/preload=10/by_path-8 50.43n ± 2% 39.88n ± 5% -20.94% (p=0.000 n=10) Find/preload=10/ambiguous-8 323.6n ± 1% 1049.0n ± 3% +224.17% (p=0.000 n=10) Find/preload=100/by_address-8 23.69n ± 1% 23.63n ± 6% ~ (p=0.739 n=10) Find/preload=100/by_path-8 362.70n ± 1% 37.84n ± 3% -89.57% (p=0.000 n=10) Find/preload=100/ambiguous-8 2.683µ ± 2% 19.235µ ± 2% +617.05% (p=0.000 n=10) Find/preload=1000/by_address-8 26.45n ± 1% 27.73n ± 2% +4.82% (p=0.000 n=10) Find/preload=1000/by_path-8 3211.00n ± 3% 38.22n ± 8% -98.81% (p=0.000 n=10) Find/preload=1000/ambiguous-8 26.14µ ± 2% 263.59µ ± 1% +908.41% (p=0.000 n=10) Find/preload=1000000/by_address-8 26.47n ± 4% 26.41n ± 1% ~ (p=0.566 n=10) Find/preload=1000000/by_path-8 3683325.50n ± 4% 44.09n ± 45% -100.00% (p=0.000 n=10) Find/preload=1000000/ambiguous-8 39.68m ± 14% 819.48m ± 7% +1965.01% (p=0.000 n=10) geomean 2.346µ 791.4n -66.27% │ HEAD~1 │ HEAD │ │ B/op │ B/op vs base │ Add/preload=10-8 643.0 ± 0% 662.0 ± 0% +2.95% (p=0.000 n=10) Add/preload=100-8 643.0 ± 0% 662.0 ± 0% +2.95% (p=0.000 n=10) Add/preload=1000-8 584.0 ± 5% 662.0 ± 0% +13.36% (p=0.000 n=10) Add/preload=1000000-8 88.00 ± 0% 662.00 ± 17% +652.27% (p=0.000 n=10) Find/preload=10/by_address-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Find/preload=10/by_path-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Find/preload=10/ambiguous-8 624.0 ± 0% 1200.0 ± 0% +92.31% (p=0.000 n=10) Find/preload=100/by_address-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Find/preload=100/by_path-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Find/preload=100/ambiguous-8 6.047Ki ± 0% 12.047Ki ± 0% +99.22% (p=0.000 n=10) Find/preload=1000/by_address-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Find/preload=1000/by_path-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Find/preload=1000/ambiguous-8 56.05Ki ± 0% 112.05Ki ± 0% +99.92% (p=0.000 n=10) Find/preload=1000000/by_address-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Find/preload=1000000/by_path-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Find/preload=1000000/ambiguous-8 53.41Mi ± 0% 106.81Mi ± 0% +100.00% (p=0.000 n=10) geomean ² +36.09% ² ¹ all samples are equal ² summaries must be >0 to compute geomean │ HEAD~1 │ HEAD │ │ allocs/op │ allocs/op vs base │ Add/preload=10-8 3.000 ± 0% 4.000 ± 0% +33.33% (p=0.000 n=10) Add/preload=100-8 3.000 ± 0% 4.000 ± 0% +33.33% (p=0.000 n=10) Add/preload=1000-8 3.000 ± 0% 4.000 ± 0% +33.33% (p=0.000 n=10) Add/preload=1000000-8 2.000 ± 0% 4.000 ± 0% +100.00% (p=0.000 n=10) Find/preload=10/by_address-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Find/preload=10/by_path-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Find/preload=10/ambiguous-8 2.000 ± 0% 3.000 ± 0% +50.00% (p=0.000 n=10) Find/preload=100/by_address-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Find/preload=100/by_path-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Find/preload=100/ambiguous-8 2.000 ± 0% 3.000 ± 0% +50.00% (p=0.000 n=10) Find/preload=1000/by_address-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Find/preload=1000/by_path-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Find/preload=1000/ambiguous-8 2.000 ± 0% 3.000 ± 0% +50.00% (p=0.000 n=10) Find/preload=1000000/by_address-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Find/preload=1000000/by_path-8 0.000 ± 0% 0.000 ± 0% ~ (p=1.000 n=10) ¹ Find/preload=1000000/ambiguous-8 2.000 ± 0% 3.000 ± 0% +50.00% (p=0.000 n=10) geomean ² +21.97% ² ¹ all samples are equal ² summaries must be >0 to compute geomean ``` Updates #16874
This commit is contained in:
Alexander Yastrebov
parent
b46a319870
commit
a00430d4aa
1 changed files with 43 additions and 24 deletions
|
|
@ -66,7 +66,7 @@ type accountCache struct {
|
||||||
keydir string
|
keydir string
|
||||||
watcher *watcher
|
watcher *watcher
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
all []accounts.Account
|
byURL map[accounts.URL][]accounts.Account
|
||||||
byAddr map[common.Address][]accounts.Account
|
byAddr map[common.Address][]accounts.Account
|
||||||
throttle *time.Timer
|
throttle *time.Timer
|
||||||
notify chan struct{}
|
notify chan struct{}
|
||||||
|
|
@ -76,6 +76,7 @@ type accountCache struct {
|
||||||
func newAccountCache(keydir string) (*accountCache, chan struct{}) {
|
func newAccountCache(keydir string) (*accountCache, chan struct{}) {
|
||||||
ac := &accountCache{
|
ac := &accountCache{
|
||||||
keydir: keydir,
|
keydir: keydir,
|
||||||
|
byURL: make(map[accounts.URL][]accounts.Account),
|
||||||
byAddr: make(map[common.Address][]accounts.Account),
|
byAddr: make(map[common.Address][]accounts.Account),
|
||||||
notify: make(chan struct{}, 1),
|
notify: make(chan struct{}, 1),
|
||||||
fileC: fileCache{all: mapset.NewThreadUnsafeSet[string]()},
|
fileC: fileCache{all: mapset.NewThreadUnsafeSet[string]()},
|
||||||
|
|
@ -88,8 +89,11 @@ func (ac *accountCache) accounts() []accounts.Account {
|
||||||
ac.maybeReload()
|
ac.maybeReload()
|
||||||
ac.mu.Lock()
|
ac.mu.Lock()
|
||||||
defer ac.mu.Unlock()
|
defer ac.mu.Unlock()
|
||||||
cpy := make([]accounts.Account, len(ac.all))
|
cpy := make([]accounts.Account, 0, len(ac.byURL))
|
||||||
copy(cpy, ac.all)
|
for _, accs := range ac.byURL {
|
||||||
|
cpy = append(cpy, accs...)
|
||||||
|
}
|
||||||
|
sort.SliceStable(cpy, func(i, j int) bool { return cpy[i].URL.Cmp(cpy[j].URL) < 0 })
|
||||||
return cpy
|
return cpy
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -104,14 +108,11 @@ func (ac *accountCache) add(newAccount accounts.Account) {
|
||||||
ac.mu.Lock()
|
ac.mu.Lock()
|
||||||
defer ac.mu.Unlock()
|
defer ac.mu.Unlock()
|
||||||
|
|
||||||
i := sort.Search(len(ac.all), func(i int) bool { return ac.all[i].URL.Cmp(newAccount.URL) >= 0 })
|
if accs, ok := ac.byURL[newAccount.URL]; ok && slices.Contains(accs, newAccount) {
|
||||||
if i < len(ac.all) && ac.all[i] == newAccount {
|
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
// newAccount is not in the cache.
|
// newAccount is not in the cache.
|
||||||
ac.all = append(ac.all, accounts.Account{})
|
ac.byURL[newAccount.URL] = append(ac.byURL[newAccount.URL], newAccount)
|
||||||
copy(ac.all[i+1:], ac.all[i:])
|
|
||||||
ac.all[i] = newAccount
|
|
||||||
ac.byAddr[newAccount.Address] = append(ac.byAddr[newAccount.Address], newAccount)
|
ac.byAddr[newAccount.Address] = append(ac.byAddr[newAccount.Address], newAccount)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -120,7 +121,12 @@ func (ac *accountCache) delete(removed accounts.Account) {
|
||||||
ac.mu.Lock()
|
ac.mu.Lock()
|
||||||
defer ac.mu.Unlock()
|
defer ac.mu.Unlock()
|
||||||
|
|
||||||
ac.all = removeAccount(ac.all, removed)
|
if bu := removeAccount(ac.byURL[removed.URL], removed); len(bu) == 0 {
|
||||||
|
delete(ac.byURL, removed.URL)
|
||||||
|
} else {
|
||||||
|
ac.byURL[removed.URL] = bu
|
||||||
|
}
|
||||||
|
|
||||||
if ba := removeAccount(ac.byAddr[removed.Address], removed); len(ba) == 0 {
|
if ba := removeAccount(ac.byAddr[removed.Address], removed); len(ba) == 0 {
|
||||||
delete(ac.byAddr, removed.Address)
|
delete(ac.byAddr, removed.Address)
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -132,11 +138,14 @@ func (ac *accountCache) delete(removed accounts.Account) {
|
||||||
func (ac *accountCache) deleteByFile(path string) {
|
func (ac *accountCache) deleteByFile(path string) {
|
||||||
ac.mu.Lock()
|
ac.mu.Lock()
|
||||||
defer ac.mu.Unlock()
|
defer ac.mu.Unlock()
|
||||||
i := sort.Search(len(ac.all), func(i int) bool { return ac.all[i].URL.Path >= path })
|
url := accounts.URL{Scheme: KeyStoreScheme, Path: path}
|
||||||
|
if accs, ok := ac.byURL[url]; ok {
|
||||||
if i < len(ac.all) && ac.all[i].URL.Path == path {
|
removed := accs[0]
|
||||||
removed := ac.all[i]
|
if len(accs) == 1 {
|
||||||
ac.all = append(ac.all[:i], ac.all[i+1:]...)
|
delete(ac.byURL, url)
|
||||||
|
} else {
|
||||||
|
ac.byURL[url] = accs[1:]
|
||||||
|
}
|
||||||
if ba := removeAccount(ac.byAddr[removed.Address], removed); len(ba) == 0 {
|
if ba := removeAccount(ac.byAddr[removed.Address], removed); len(ba) == 0 {
|
||||||
delete(ac.byAddr, removed.Address)
|
delete(ac.byAddr, removed.Address)
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -166,24 +175,34 @@ func removeAccount(slice []accounts.Account, elem accounts.Account) []accounts.A
|
||||||
// The exact matching rules are explained by the documentation of accounts.Account.
|
// The exact matching rules are explained by the documentation of accounts.Account.
|
||||||
// Callers must hold ac.mu.
|
// Callers must hold ac.mu.
|
||||||
func (ac *accountCache) find(a accounts.Account) (accounts.Account, error) {
|
func (ac *accountCache) find(a accounts.Account) (accounts.Account, error) {
|
||||||
// Limit search to address candidates if possible.
|
|
||||||
matches := ac.all
|
|
||||||
if (a.Address != common.Address{}) {
|
|
||||||
matches = ac.byAddr[a.Address]
|
|
||||||
}
|
|
||||||
if a.URL.Path != "" {
|
if a.URL.Path != "" {
|
||||||
// If only the basename is specified, complete the path.
|
// If only the basename is specified, complete the path.
|
||||||
if !strings.ContainsRune(a.URL.Path, filepath.Separator) {
|
if !strings.ContainsRune(a.URL.Path, filepath.Separator) {
|
||||||
a.URL.Path = filepath.Join(ac.keydir, a.URL.Path)
|
a.URL.Path = filepath.Join(ac.keydir, a.URL.Path)
|
||||||
}
|
}
|
||||||
for i := range matches {
|
}
|
||||||
if matches[i].URL == a.URL {
|
// Limit search to address candidates if possible.
|
||||||
return matches[i], nil
|
var matches []accounts.Account
|
||||||
|
if (a.Address != common.Address{}) {
|
||||||
|
matches = ac.byAddr[a.Address]
|
||||||
|
if a.URL.Path != "" {
|
||||||
|
for i := range matches {
|
||||||
|
if matches[i].URL == a.URL {
|
||||||
|
return matches[i], nil
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (a.Address == common.Address{}) {
|
} else {
|
||||||
|
if a.URL.Path != "" {
|
||||||
|
if accs, ok := ac.byURL[a.URL]; ok {
|
||||||
|
return accs[0], nil
|
||||||
|
}
|
||||||
return accounts.Account{}, ErrNoMatch
|
return accounts.Account{}, ErrNoMatch
|
||||||
}
|
}
|
||||||
|
matches = make([]accounts.Account, 0, len(ac.byURL))
|
||||||
|
for _, accs := range ac.byURL {
|
||||||
|
matches = append(matches, accs...)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
switch len(matches) {
|
switch len(matches) {
|
||||||
case 1:
|
case 1:
|
||||||
|
|
@ -193,7 +212,7 @@ func (ac *accountCache) find(a accounts.Account) (accounts.Account, error) {
|
||||||
default:
|
default:
|
||||||
err := &AmbiguousAddrError{Addr: a.Address, Matches: make([]accounts.Account, len(matches))}
|
err := &AmbiguousAddrError{Addr: a.Address, Matches: make([]accounts.Account, len(matches))}
|
||||||
copy(err.Matches, matches)
|
copy(err.Matches, matches)
|
||||||
slices.SortFunc(err.Matches, byURL)
|
slices.SortStableFunc(err.Matches, byURL)
|
||||||
return accounts.Account{}, err
|
return accounts.Account{}, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue