From a377eb3238b256ae3ad569f1c2985cd3f5d5d152 Mon Sep 17 00:00:00 2001 From: Willian Paixao Date: Fri, 23 Jan 2026 13:28:56 +0100 Subject: [PATCH] accounts/keystore, cmd/clef, signer/core: add database-backed keystore This commit introduces a database-backed keystore implementation using Pebble as an alternative to the file-based keystore. This addresses the scalability issues described in #20363 Key changes: - Add DBKeyStore type backed by Pebble database for O(1) address lookups - Add --keystore-type flag to clef (options: "file" or "db") - Add migrate-keystore command to convert file-based keystores to DB - Update StartClefAccountManager to support both keystore backends The database keystore uses the same Web3 Secret Storage encryption format as the file-based keystore, ensuring compatibility. The file-based keystore remains the default for backwards compatibility. Database schema: - k
-> encrypted keystore JSON - m:count -> total key count (uint64) Fixes #20363 Signed-off-by: Willian Paixao --- accounts/keystore/keystore_db.go | 765 ++++++++++++++++++++++++++ accounts/keystore/keystore_db_test.go | 674 +++++++++++++++++++++++ cmd/clef/main.go | 26 +- cmd/clef/migrate.go | 243 ++++++++ signer/core/api.go | 23 +- 5 files changed, 1724 insertions(+), 7 deletions(-) create mode 100644 accounts/keystore/keystore_db.go create mode 100644 accounts/keystore/keystore_db_test.go create mode 100644 cmd/clef/migrate.go diff --git a/accounts/keystore/keystore_db.go b/accounts/keystore/keystore_db.go new file mode 100644 index 0000000000..36d178e1f3 --- /dev/null +++ b/accounts/keystore/keystore_db.go @@ -0,0 +1,765 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package keystore + +import ( + "crypto/ecdsa" + crand "crypto/rand" + "encoding/binary" + "errors" + "fmt" + "math/big" + "runtime" + "sync" + "time" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/pebble" + "github.com/ethereum/go-ethereum/event" +) + +// Database key prefixes for the keystore database. +var ( + keyPrefix = []byte("k") // k
-> encrypted keystore JSON + countKey = []byte("m:count") +) + +// dbKeyStorePassphrase implements the keyStore interface using a database backend. +type dbKeyStorePassphrase struct { + db ethdb.KeyValueStore + scryptN int + scryptP int +} + +// makeKeyDBKey creates the database key for a given address. +func makeKeyDBKey(addr common.Address) []byte { + return append(keyPrefix, addr.Bytes()...) +} + +// GetKey retrieves and decrypts the key from the database. +func (ks *dbKeyStorePassphrase) GetKey(addr common.Address, filename string, auth string) (*Key, error) { + // In DB mode, filename is ignored - we look up by address + dbKey := makeKeyDBKey(addr) + keyjson, err := ks.db.Get(dbKey) + if err != nil { + return nil, ErrNoMatch + } + key, err := DecryptKey(keyjson, auth) + if err != nil { + return nil, err + } + // Verify the decrypted key matches the requested address + if key.Address != addr { + return nil, fmt.Errorf("key content mismatch: have account %x, want %x", key.Address, addr) + } + return key, nil +} + +// StoreKey encrypts and stores the key in the database. +func (ks *dbKeyStorePassphrase) StoreKey(filename string, key *Key, auth string) error { + // In DB mode, filename is derived from address + keyjson, err := EncryptKey(key, auth, ks.scryptN, ks.scryptP) + if err != nil { + return err + } + dbKey := makeKeyDBKey(key.Address) + + // Check if this is a new key or an update + isNew := false + if has, _ := ks.db.Has(dbKey); !has { + isNew = true + } + + // Store the encrypted key + if err := ks.db.Put(dbKey, keyjson); err != nil { + return err + } + + // Update count if this is a new key + if isNew { + count := ks.getCount() + ks.setCount(count + 1) + } + + return nil +} + +// JoinPath returns the "path" for the key (the address in hex). +// In DB mode, this is used as an identifier rather than a file path. +func (ks *dbKeyStorePassphrase) JoinPath(filename string) string { + // For DB mode, we just return the filename as-is + // The actual storage uses the address, not the filename + return filename +} + +// DeleteKey removes a key from the database. +func (ks *dbKeyStorePassphrase) DeleteKey(addr common.Address) error { + dbKey := makeKeyDBKey(addr) + + // Check if the key exists + if has, _ := ks.db.Has(dbKey); !has { + return ErrNoMatch + } + + // Delete the key + if err := ks.db.Delete(dbKey); err != nil { + return err + } + + // Update count + count := ks.getCount() + if count > 0 { + ks.setCount(count - 1) + } + + return nil +} + +// getCount returns the total number of keys in the database. +func (ks *dbKeyStorePassphrase) getCount() uint64 { + data, err := ks.db.Get(countKey) + if err != nil { + return 0 + } + if len(data) < 8 { + return 0 + } + return binary.BigEndian.Uint64(data) +} + +// setCount updates the total number of keys in the database. +func (ks *dbKeyStorePassphrase) setCount(count uint64) { + data := make([]byte, 8) + binary.BigEndian.PutUint64(data, count) + ks.db.Put(countKey, data) +} + +// dbAccountCache implements account caching backed by a database. +type dbAccountCache struct { + db ethdb.KeyValueStore + mu sync.Mutex + notify chan struct{} +} + +// newDBAccountCache creates a new database-backed account cache. +func newDBAccountCache(db ethdb.KeyValueStore) (*dbAccountCache, chan struct{}) { + ac := &dbAccountCache{ + db: db, + notify: make(chan struct{}, 1), + } + return ac, ac.notify +} + +// accounts returns all accounts in the database. +func (ac *dbAccountCache) accounts() []accounts.Account { + ac.mu.Lock() + defer ac.mu.Unlock() + + var accs []accounts.Account + iter := ac.db.NewIterator(keyPrefix, nil) + defer iter.Release() + + for iter.Next() { + key := iter.Key() + if len(key) != len(keyPrefix)+common.AddressLength { + continue + } + addr := common.BytesToAddress(key[len(keyPrefix):]) + accs = append(accs, accounts.Account{ + Address: addr, + URL: accounts.URL{Scheme: KeyStoreScheme, Path: addr.Hex()}, + }) + } + return accs +} + +// hasAddress checks if an address exists in the database. +func (ac *dbAccountCache) hasAddress(addr common.Address) bool { + ac.mu.Lock() + defer ac.mu.Unlock() + + dbKey := makeKeyDBKey(addr) + has, _ := ac.db.Has(dbKey) + return has +} + +// add adds an account to the cache (notifies watchers). +func (ac *dbAccountCache) add(newAccount accounts.Account) { + ac.mu.Lock() + defer ac.mu.Unlock() + + // Notify watchers of the change + select { + case ac.notify <- struct{}{}: + default: + } +} + +// delete removes an account from the cache (notifies watchers). +func (ac *dbAccountCache) delete(removed accounts.Account) { + ac.mu.Lock() + defer ac.mu.Unlock() + + // Notify watchers of the change + select { + case ac.notify <- struct{}{}: + default: + } +} + +// find returns the account for a given address. +// Caller must hold ac.mu. +func (ac *dbAccountCache) find(a accounts.Account) (accounts.Account, error) { + dbKey := makeKeyDBKey(a.Address) + has, err := ac.db.Has(dbKey) + if err != nil || !has { + return accounts.Account{}, ErrNoMatch + } + return accounts.Account{ + Address: a.Address, + URL: accounts.URL{Scheme: KeyStoreScheme, Path: a.Address.Hex()}, + }, nil +} + +// maybeReload is a no-op for DB cache since DB is always up-to-date. +func (ac *dbAccountCache) maybeReload() {} + +// close closes the notify channel. +func (ac *dbAccountCache) close() { + ac.mu.Lock() + defer ac.mu.Unlock() + + if ac.notify != nil { + close(ac.notify) + ac.notify = nil + } +} + +// DBKeyStore is a keystore backed by a database for scalability. +type DBKeyStore struct { + db ethdb.KeyValueStore + storage *dbKeyStorePassphrase + cache *dbAccountCache + changes chan struct{} + unlocked map[common.Address]*unlocked + + wallets []accounts.Wallet + updateFeed event.Feed + updateScope event.SubscriptionScope + updating bool + + mu sync.RWMutex + importMu sync.Mutex +} + +// NewDBKeyStore creates a keystore backed by a database for scalability. +// This is designed for use cases with millions of keys where the file-based +// keystore becomes impractical due to filesystem scanning overhead. +func NewDBKeyStore(dbPath string, scryptN, scryptP int) (*DBKeyStore, error) { + db, err := pebble.New(dbPath, 16, 16, "keystore", false) + if err != nil { + return nil, fmt.Errorf("failed to open keystore database: %w", err) + } + + storage := &dbKeyStorePassphrase{ + db: db, + scryptN: scryptN, + scryptP: scryptP, + } + + cache, changes := newDBAccountCache(db) + + ks := &DBKeyStore{ + db: db, + storage: storage, + cache: cache, + changes: changes, + unlocked: make(map[common.Address]*unlocked), + } + + // Create the initial list of wallets from the cache + accs := cache.accounts() + ks.wallets = make([]accounts.Wallet, len(accs)) + for i := 0; i < len(accs); i++ { + ks.wallets[i] = &dbKeystoreWallet{account: accs[i], keystore: ks} + } + + // Set up a cleanup to close the cache (and notify channel) when the keystore is garbage collected + runtime.AddCleanup(ks, func(c *dbAccountCache) { + c.close() + }, cache) + + return ks, nil +} + +// Close closes the database. +func (ks *DBKeyStore) Close() error { + ks.mu.Lock() + defer ks.mu.Unlock() + + ks.cache.close() + if ks.db != nil { + return ks.db.Close() + } + return nil +} + +// Wallets returns all wallets managed by the keystore. +func (ks *DBKeyStore) Wallets() []accounts.Wallet { + ks.refreshWallets() + + ks.mu.RLock() + defer ks.mu.RUnlock() + + cpy := make([]accounts.Wallet, len(ks.wallets)) + copy(cpy, ks.wallets) + return cpy +} + +// refreshWallets updates the wallet list from the database. +func (ks *DBKeyStore) refreshWallets() { + ks.mu.Lock() + accs := ks.cache.accounts() + + var ( + wallets = make([]accounts.Wallet, 0, len(accs)) + events []accounts.WalletEvent + ) + + // Build a map of existing wallets for comparison + existing := make(map[common.Address]accounts.Wallet) + for _, w := range ks.wallets { + accts := w.Accounts() + if len(accts) > 0 { + existing[accts[0].Address] = w + } + } + + // Process accounts from DB + seen := make(map[common.Address]bool) + for _, account := range accs { + seen[account.Address] = true + if w, ok := existing[account.Address]; ok { + wallets = append(wallets, w) + } else { + wallet := &dbKeystoreWallet{account: account, keystore: ks} + events = append(events, accounts.WalletEvent{Wallet: wallet, Kind: accounts.WalletArrived}) + wallets = append(wallets, wallet) + } + } + + // Find removed wallets + for addr, w := range existing { + if !seen[addr] { + events = append(events, accounts.WalletEvent{Wallet: w, Kind: accounts.WalletDropped}) + } + } + + ks.wallets = wallets + ks.mu.Unlock() + + // Fire wallet events + for _, event := range events { + ks.updateFeed.Send(event) + } +} + +// Subscribe implements accounts.Backend. +func (ks *DBKeyStore) Subscribe(sink chan<- accounts.WalletEvent) event.Subscription { + ks.mu.Lock() + defer ks.mu.Unlock() + + sub := ks.updateScope.Track(ks.updateFeed.Subscribe(sink)) + + if !ks.updating { + ks.updating = true + go ks.updater() + } + return sub +} + +// updater listens for account changes and refreshes wallets. +func (ks *DBKeyStore) updater() { + for { + select { + case <-ks.changes: + } + ks.refreshWallets() + + ks.mu.Lock() + if ks.updateScope.Count() == 0 { + ks.updating = false + ks.mu.Unlock() + return + } + ks.mu.Unlock() + } +} + +// HasAddress reports whether a key with the given address is present. +func (ks *DBKeyStore) HasAddress(addr common.Address) bool { + return ks.cache.hasAddress(addr) +} + +// Accounts returns all key files present in the keystore. +func (ks *DBKeyStore) Accounts() []accounts.Account { + return ks.cache.accounts() +} + +// NewAccount generates a new key and stores it in the database. +func (ks *DBKeyStore) NewAccount(passphrase string) (accounts.Account, error) { + _, account, err := storeNewKey(ks.storage, crand.Reader, passphrase) + if err != nil { + return accounts.Account{}, err + } + ks.cache.add(account) + ks.refreshWallets() + return account, nil +} + +// Delete deletes the key matched by account if the passphrase is correct. +func (ks *DBKeyStore) Delete(a accounts.Account, passphrase string) error { + // Decrypt to verify password + a, key, err := ks.getDecryptedKey(a, passphrase) + if key != nil { + zeroKey(key.PrivateKey) + } + if err != nil { + return err + } + + // Delete from database + if err := ks.storage.DeleteKey(a.Address); err != nil { + return err + } + + ks.cache.delete(a) + ks.refreshWallets() + return nil +} + +// Find resolves the given account into a unique entry in the keystore. +func (ks *DBKeyStore) Find(a accounts.Account) (accounts.Account, error) { + ks.cache.mu.Lock() + a, err := ks.cache.find(a) + ks.cache.mu.Unlock() + return a, err +} + +// getDecryptedKey retrieves and decrypts the key for the given account. +func (ks *DBKeyStore) getDecryptedKey(a accounts.Account, auth string) (accounts.Account, *Key, error) { + a, err := ks.Find(a) + if err != nil { + return a, nil, err + } + key, err := ks.storage.GetKey(a.Address, a.URL.Path, auth) + return a, key, err +} + +// Export exports as a JSON key, encrypted with newPassphrase. +func (ks *DBKeyStore) Export(a accounts.Account, passphrase, newPassphrase string) (keyJSON []byte, err error) { + _, key, err := ks.getDecryptedKey(a, passphrase) + if err != nil { + return nil, err + } + defer zeroKey(key.PrivateKey) + return EncryptKey(key, newPassphrase, ks.storage.scryptN, ks.storage.scryptP) +} + +// Import stores the given encrypted JSON key into the database. +func (ks *DBKeyStore) Import(keyJSON []byte, passphrase, newPassphrase string) (accounts.Account, error) { + key, err := DecryptKey(keyJSON, passphrase) + if key != nil && key.PrivateKey != nil { + defer zeroKey(key.PrivateKey) + } + if err != nil { + return accounts.Account{}, err + } + ks.importMu.Lock() + defer ks.importMu.Unlock() + + if ks.cache.hasAddress(key.Address) { + return accounts.Account{Address: key.Address}, ErrAccountAlreadyExists + } + return ks.importKey(key, newPassphrase) +} + +// importKey imports a key into the database. +func (ks *DBKeyStore) importKey(key *Key, passphrase string) (accounts.Account, error) { + a := accounts.Account{ + Address: key.Address, + URL: accounts.URL{Scheme: KeyStoreScheme, Path: key.Address.Hex()}, + } + if err := ks.storage.StoreKey(a.URL.Path, key, passphrase); err != nil { + return accounts.Account{}, err + } + ks.cache.add(a) + ks.refreshWallets() + return a, nil +} + +// Update changes the passphrase of an existing account. +func (ks *DBKeyStore) Update(a accounts.Account, passphrase, newPassphrase string) error { + a, key, err := ks.getDecryptedKey(a, passphrase) + if err != nil { + return err + } + defer zeroKey(key.PrivateKey) + return ks.storage.StoreKey(a.URL.Path, key, newPassphrase) +} + +// Count returns the number of keys in the database. +func (ks *DBKeyStore) Count() uint64 { + return ks.storage.getCount() +} + +// dbKeystoreWallet implements accounts.Wallet for DB-backed keystore. +type dbKeystoreWallet struct { + account accounts.Account + keystore *DBKeyStore +} + +// URL implements accounts.Wallet, returning the URL of the account. +func (w *dbKeystoreWallet) URL() accounts.URL { + return w.account.URL +} + +// Status implements accounts.Wallet, returning whether the wallet is locked. +func (w *dbKeystoreWallet) Status() (string, error) { + w.keystore.mu.RLock() + defer w.keystore.mu.RUnlock() + + if _, ok := w.keystore.unlocked[w.account.Address]; ok { + return "Unlocked", nil + } + return "Locked", nil +} + +// Open implements accounts.Wallet, but is a no-op for DB keystores. +func (w *dbKeystoreWallet) Open(passphrase string) error { + return nil +} + +// Close implements accounts.Wallet, but is a no-op for DB keystores. +func (w *dbKeystoreWallet) Close() error { + return nil +} + +// Accounts implements accounts.Wallet, returning the account this wallet holds. +func (w *dbKeystoreWallet) Accounts() []accounts.Account { + return []accounts.Account{w.account} +} + +// Contains implements accounts.Wallet, returning whether a particular account is or is not part of this wallet. +func (w *dbKeystoreWallet) Contains(account accounts.Account) bool { + return account.Address == w.account.Address +} + +// Derive implements accounts.Wallet, but is a no-op for DB keystores. +func (w *dbKeystoreWallet) Derive(path accounts.DerivationPath, pin bool) (accounts.Account, error) { + return accounts.Account{}, errors.New("not supported") +} + +// SelfDerive implements accounts.Wallet, but is a no-op for DB keystores. +func (w *dbKeystoreWallet) SelfDerive(bases []accounts.DerivationPath, chain ethereum.ChainStateReader) {} + +// SignData implements accounts.Wallet. +func (w *dbKeystoreWallet) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) { + return w.signHash(account, crypto.Keccak256(data)) +} + +// SignDataWithPassphrase implements accounts.Wallet. +func (w *dbKeystoreWallet) SignDataWithPassphrase(account accounts.Account, passphrase, mimeType string, data []byte) ([]byte, error) { + return w.signHashWithPassphrase(account, passphrase, crypto.Keccak256(data)) +} + +// SignText implements accounts.Wallet. +func (w *dbKeystoreWallet) SignText(account accounts.Account, text []byte) ([]byte, error) { + return w.signHash(account, accounts.TextHash(text)) +} + +// SignTextWithPassphrase implements accounts.Wallet. +func (w *dbKeystoreWallet) SignTextWithPassphrase(account accounts.Account, passphrase string, text []byte) ([]byte, error) { + return w.signHashWithPassphrase(account, passphrase, accounts.TextHash(text)) +} + +// signHash signs the given hash with the account. +func (w *dbKeystoreWallet) signHash(account accounts.Account, hash []byte) ([]byte, error) { + if !w.Contains(account) { + return nil, accounts.ErrUnknownAccount + } + return w.keystore.SignHash(account, hash) +} + +// signHashWithPassphrase signs the given hash with the account using the passphrase. +func (w *dbKeystoreWallet) signHashWithPassphrase(account accounts.Account, passphrase string, hash []byte) ([]byte, error) { + if !w.Contains(account) { + return nil, accounts.ErrUnknownAccount + } + return w.keystore.SignHashWithPassphrase(account, passphrase, hash) +} + +// SignHash signs the given hash. +func (ks *DBKeyStore) SignHash(a accounts.Account, hash []byte) ([]byte, error) { + ks.mu.RLock() + defer ks.mu.RUnlock() + + unlockedKey, found := ks.unlocked[a.Address] + if !found { + return nil, ErrLocked + } + return signHash(hash, unlockedKey.PrivateKey) +} + +// SignHashWithPassphrase signs hash if the private key matching the given address +// can be decrypted with the given passphrase. +func (ks *DBKeyStore) SignHashWithPassphrase(a accounts.Account, passphrase string, hash []byte) (signature []byte, err error) { + _, key, err := ks.getDecryptedKey(a, passphrase) + if err != nil { + return nil, err + } + defer zeroKey(key.PrivateKey) + return crypto.Sign(hash, key.PrivateKey) +} + +// signHash is a helper to sign a hash with a private key. +func signHash(hash []byte, priv *ecdsa.PrivateKey) ([]byte, error) { + return crypto.Sign(hash, priv) +} + +// SignTx signs the given transaction with the requested account. +func (ks *DBKeyStore) SignTx(a accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) { + ks.mu.RLock() + defer ks.mu.RUnlock() + + unlockedKey, found := ks.unlocked[a.Address] + if !found { + return nil, ErrLocked + } + signer := types.LatestSignerForChainID(chainID) + return types.SignTx(tx, signer, unlockedKey.PrivateKey) +} + +// SignTxWithPassphrase signs the transaction if the private key matching the +// given address can be decrypted with the given passphrase. +func (ks *DBKeyStore) SignTxWithPassphrase(a accounts.Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) { + _, key, err := ks.getDecryptedKey(a, passphrase) + if err != nil { + return nil, err + } + defer zeroKey(key.PrivateKey) + signer := types.LatestSignerForChainID(chainID) + return types.SignTx(tx, signer, key.PrivateKey) +} + +// Unlock unlocks the given account indefinitely. +func (ks *DBKeyStore) Unlock(a accounts.Account, passphrase string) error { + return ks.TimedUnlock(a, passphrase, 0) +} + +// Lock removes the private key with the given address from memory. +func (ks *DBKeyStore) Lock(addr common.Address) error { + ks.mu.Lock() + unl, found := ks.unlocked[addr] + ks.mu.Unlock() + if found { + ks.expire(addr, unl, time.Duration(0)*time.Nanosecond) + } + return nil +} + +// TimedUnlock unlocks the given account with the passphrase. The account +// stays unlocked for the duration of timeout. A timeout of 0 unlocks the account +// until the program exits. +func (ks *DBKeyStore) TimedUnlock(a accounts.Account, passphrase string, timeout time.Duration) error { + a, key, err := ks.getDecryptedKey(a, passphrase) + if err != nil { + return err + } + + ks.mu.Lock() + defer ks.mu.Unlock() + u, found := ks.unlocked[a.Address] + if found { + if u.abort == nil { + // The address was unlocked indefinitely, so unlocking + // it with a timeout would be confusing. + zeroKey(key.PrivateKey) + return nil + } + // Terminate the expire goroutine and replace it below. + close(u.abort) + } + if timeout > 0 { + u = &unlocked{Key: key, abort: make(chan struct{})} + go ks.expire(a.Address, u, timeout) + } else { + u = &unlocked{Key: key} + } + ks.unlocked[a.Address] = u + return nil +} + +// expire removes an unlocked key after the timeout. +func (ks *DBKeyStore) expire(addr common.Address, u *unlocked, timeout time.Duration) { + t := time.NewTimer(timeout) + defer t.Stop() + select { + case <-u.abort: + // just quit + case <-t.C: + ks.mu.Lock() + if ks.unlocked[addr] == u { + zeroKey(u.PrivateKey) + delete(ks.unlocked, addr) + } + ks.mu.Unlock() + } +} + +// SignTx for dbKeystoreWallet implements accounts.Wallet. +func (w *dbKeystoreWallet) SignTx(account accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) { + if !w.Contains(account) { + return nil, accounts.ErrUnknownAccount + } + return w.keystore.SignTx(account, tx, chainID) +} + +// SignTxWithPassphrase for dbKeystoreWallet implements accounts.Wallet. +func (w *dbKeystoreWallet) SignTxWithPassphrase(account accounts.Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) { + if !w.Contains(account) { + return nil, accounts.ErrUnknownAccount + } + return w.keystore.SignTxWithPassphrase(account, passphrase, tx, chainID) +} + +// ImportECDSA stores the given key into the database, encrypting it with the passphrase. +func (ks *DBKeyStore) ImportECDSA(priv *ecdsa.PrivateKey, passphrase string) (accounts.Account, error) { + ks.importMu.Lock() + defer ks.importMu.Unlock() + + key := newKeyFromECDSA(priv) + if ks.cache.hasAddress(key.Address) { + return accounts.Account{Address: key.Address}, ErrAccountAlreadyExists + } + return ks.importKey(key, passphrase) +} diff --git a/accounts/keystore/keystore_db_test.go b/accounts/keystore/keystore_db_test.go new file mode 100644 index 0000000000..dafba030e3 --- /dev/null +++ b/accounts/keystore/keystore_db_test.go @@ -0,0 +1,674 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package keystore + +import ( + "math/big" + "testing" + "time" + + "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" +) + +func TestDBKeyStore_NewAccount(t *testing.T) { + dir := t.TempDir() + ks, err := NewDBKeyStore(dir, LightScryptN, LightScryptP) + if err != nil { + t.Fatalf("Failed to create DB keystore: %v", err) + } + defer ks.Close() + + // Create a new account + a, err := ks.NewAccount("password123") + if err != nil { + t.Fatalf("Failed to create account: %v", err) + } + + if a.Address == (common.Address{}) { + t.Fatal("Account address is zero") + } + + // Verify the account exists + if !ks.HasAddress(a.Address) { + t.Fatal("Account not found in keystore") + } + + // Verify the account is in the list + accounts := ks.Accounts() + if len(accounts) != 1 { + t.Fatalf("Expected 1 account, got %d", len(accounts)) + } + if accounts[0].Address != a.Address { + t.Fatalf("Account address mismatch") + } +} + +func TestDBKeyStore_MultipleAccounts(t *testing.T) { + dir := t.TempDir() + ks, err := NewDBKeyStore(dir, LightScryptN, LightScryptP) + if err != nil { + t.Fatalf("Failed to create DB keystore: %v", err) + } + defer ks.Close() + + // Create multiple accounts + numAccounts := 10 + addresses := make(map[common.Address]bool) + + for i := 0; i < numAccounts; i++ { + a, err := ks.NewAccount("password123") + if err != nil { + t.Fatalf("Failed to create account %d: %v", i, err) + } + addresses[a.Address] = true + } + + // Verify all accounts exist + accounts := ks.Accounts() + if len(accounts) != numAccounts { + t.Fatalf("Expected %d accounts, got %d", numAccounts, len(accounts)) + } + + for _, acc := range accounts { + if !addresses[acc.Address] { + t.Fatalf("Unexpected account in list: %s", acc.Address.Hex()) + } + } + + // Check count + if ks.Count() != uint64(numAccounts) { + t.Fatalf("Expected count %d, got %d", numAccounts, ks.Count()) + } +} + +func TestDBKeyStore_Delete(t *testing.T) { + dir := t.TempDir() + ks, err := NewDBKeyStore(dir, LightScryptN, LightScryptP) + if err != nil { + t.Fatalf("Failed to create DB keystore: %v", err) + } + defer ks.Close() + + // Create an account + a, err := ks.NewAccount("password123") + if err != nil { + t.Fatalf("Failed to create account: %v", err) + } + + // Delete with wrong password should fail + if err := ks.Delete(a, "wrongpassword"); err == nil { + t.Fatal("Delete with wrong password should fail") + } + + // Delete with correct password should succeed + if err := ks.Delete(a, "password123"); err != nil { + t.Fatalf("Failed to delete account: %v", err) + } + + // Verify the account is gone + if ks.HasAddress(a.Address) { + t.Fatal("Account should not exist after deletion") + } + + accounts := ks.Accounts() + if len(accounts) != 0 { + t.Fatalf("Expected 0 accounts, got %d", len(accounts)) + } +} + +func TestDBKeyStore_UnlockAndSign(t *testing.T) { + dir := t.TempDir() + ks, err := NewDBKeyStore(dir, LightScryptN, LightScryptP) + if err != nil { + t.Fatalf("Failed to create DB keystore: %v", err) + } + defer ks.Close() + + // Create an account + a, err := ks.NewAccount("password123") + if err != nil { + t.Fatalf("Failed to create account: %v", err) + } + + // Try to sign without unlocking - should fail + hash := crypto.Keccak256([]byte("test message")) + _, err = ks.SignHash(a, hash) + if err != ErrLocked { + t.Fatal("SignHash should fail when account is locked") + } + + // Unlock the account + if err := ks.Unlock(a, "password123"); err != nil { + t.Fatalf("Failed to unlock account: %v", err) + } + + // Now signing should work + sig, err := ks.SignHash(a, hash) + if err != nil { + t.Fatalf("Failed to sign hash: %v", err) + } + + if len(sig) != 65 { + t.Fatalf("Unexpected signature length: %d", len(sig)) + } + + // Lock the account + if err := ks.Lock(a.Address); err != nil { + t.Fatalf("Failed to lock account: %v", err) + } + + // Signing should fail again + _, err = ks.SignHash(a, hash) + if err != ErrLocked { + t.Fatal("SignHash should fail when account is locked") + } +} + +func TestDBKeyStore_SignWithPassphrase(t *testing.T) { + dir := t.TempDir() + ks, err := NewDBKeyStore(dir, LightScryptN, LightScryptP) + if err != nil { + t.Fatalf("Failed to create DB keystore: %v", err) + } + defer ks.Close() + + // Create an account + a, err := ks.NewAccount("password123") + if err != nil { + t.Fatalf("Failed to create account: %v", err) + } + + hash := crypto.Keccak256([]byte("test message")) + + // Sign with wrong passphrase should fail + _, err = ks.SignHashWithPassphrase(a, "wrongpassword", hash) + if err == nil { + t.Fatal("SignHashWithPassphrase with wrong password should fail") + } + + // Sign with correct passphrase should succeed + sig, err := ks.SignHashWithPassphrase(a, "password123", hash) + if err != nil { + t.Fatalf("Failed to sign hash: %v", err) + } + + if len(sig) != 65 { + t.Fatalf("Unexpected signature length: %d", len(sig)) + } +} + +func TestDBKeyStore_TimedUnlock(t *testing.T) { + dir := t.TempDir() + ks, err := NewDBKeyStore(dir, LightScryptN, LightScryptP) + if err != nil { + t.Fatalf("Failed to create DB keystore: %v", err) + } + defer ks.Close() + + // Create an account + a, err := ks.NewAccount("password123") + if err != nil { + t.Fatalf("Failed to create account: %v", err) + } + + // Unlock for a short duration + if err := ks.TimedUnlock(a, "password123", 100*time.Millisecond); err != nil { + t.Fatalf("Failed to timed unlock account: %v", err) + } + + // Signing should work immediately + hash := crypto.Keccak256([]byte("test message")) + _, err = ks.SignHash(a, hash) + if err != nil { + t.Fatalf("Failed to sign hash: %v", err) + } + + // Wait for unlock to expire + time.Sleep(200 * time.Millisecond) + + // Signing should fail after timeout + _, err = ks.SignHash(a, hash) + if err != ErrLocked { + t.Fatal("SignHash should fail after timed unlock expires") + } +} + +func TestDBKeyStore_SignTx(t *testing.T) { + dir := t.TempDir() + ks, err := NewDBKeyStore(dir, LightScryptN, LightScryptP) + if err != nil { + t.Fatalf("Failed to create DB keystore: %v", err) + } + defer ks.Close() + + // Create an account + a, err := ks.NewAccount("password123") + if err != nil { + t.Fatalf("Failed to create account: %v", err) + } + + // Create a transaction + to := common.HexToAddress("0x1234567890123456789012345678901234567890") + tx := types.NewTransaction(0, to, big.NewInt(1000), 21000, big.NewInt(1), nil) + chainID := big.NewInt(1) + + // Sign without unlocking should fail + _, err = ks.SignTx(a, tx, chainID) + if err != ErrLocked { + t.Fatal("SignTx should fail when account is locked") + } + + // Unlock and sign + if err := ks.Unlock(a, "password123"); err != nil { + t.Fatalf("Failed to unlock account: %v", err) + } + + signedTx, err := ks.SignTx(a, tx, chainID) + if err != nil { + t.Fatalf("Failed to sign transaction: %v", err) + } + + // Verify the signature + signer := types.LatestSignerForChainID(chainID) + from, err := types.Sender(signer, signedTx) + if err != nil { + t.Fatalf("Failed to recover sender: %v", err) + } + + if from != a.Address { + t.Fatalf("Sender mismatch: got %s, want %s", from.Hex(), a.Address.Hex()) + } +} + +func TestDBKeyStore_ImportExport(t *testing.T) { + dir := t.TempDir() + ks, err := NewDBKeyStore(dir, LightScryptN, LightScryptP) + if err != nil { + t.Fatalf("Failed to create DB keystore: %v", err) + } + defer ks.Close() + + // Create an account + a, err := ks.NewAccount("password123") + if err != nil { + t.Fatalf("Failed to create account: %v", err) + } + + // Export the account + keyJSON, err := ks.Export(a, "password123", "newpassword456") + if err != nil { + t.Fatalf("Failed to export account: %v", err) + } + + // Create a new keystore + dir2 := t.TempDir() + ks2, err := NewDBKeyStore(dir2, LightScryptN, LightScryptP) + if err != nil { + t.Fatalf("Failed to create second DB keystore: %v", err) + } + defer ks2.Close() + + // Import the account + importedAcc, err := ks2.Import(keyJSON, "newpassword456", "finalpassword789") + if err != nil { + t.Fatalf("Failed to import account: %v", err) + } + + if importedAcc.Address != a.Address { + t.Fatalf("Imported account address mismatch") + } + + // Verify the imported account works + if err := ks2.Unlock(importedAcc, "finalpassword789"); err != nil { + t.Fatalf("Failed to unlock imported account: %v", err) + } +} + +func TestDBKeyStore_ImportECDSA(t *testing.T) { + dir := t.TempDir() + ks, err := NewDBKeyStore(dir, LightScryptN, LightScryptP) + if err != nil { + t.Fatalf("Failed to create DB keystore: %v", err) + } + defer ks.Close() + + // Generate a private key + privateKey, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("Failed to generate key: %v", err) + } + + expectedAddr := crypto.PubkeyToAddress(privateKey.PublicKey) + + // Import the key + a, err := ks.ImportECDSA(privateKey, "password123") + if err != nil { + t.Fatalf("Failed to import ECDSA key: %v", err) + } + + if a.Address != expectedAddr { + t.Fatalf("Imported address mismatch: got %s, want %s", a.Address.Hex(), expectedAddr.Hex()) + } + + // Verify the account works + if err := ks.Unlock(a, "password123"); err != nil { + t.Fatalf("Failed to unlock imported account: %v", err) + } + + hash := crypto.Keccak256([]byte("test")) + sig, err := ks.SignHash(a, hash) + if err != nil { + t.Fatalf("Failed to sign with imported key: %v", err) + } + + // Verify the signature + pubKey, err := crypto.SigToPub(hash, sig) + if err != nil { + t.Fatalf("Failed to recover public key: %v", err) + } + + recoveredAddr := crypto.PubkeyToAddress(*pubKey) + if recoveredAddr != expectedAddr { + t.Fatalf("Recovered address mismatch") + } +} + +func TestDBKeyStore_Update(t *testing.T) { + dir := t.TempDir() + ks, err := NewDBKeyStore(dir, LightScryptN, LightScryptP) + if err != nil { + t.Fatalf("Failed to create DB keystore: %v", err) + } + defer ks.Close() + + // Create an account + a, err := ks.NewAccount("password123") + if err != nil { + t.Fatalf("Failed to create account: %v", err) + } + + // Update password with wrong old password should fail + if err := ks.Update(a, "wrongpassword", "newpassword456"); err == nil { + t.Fatal("Update with wrong password should fail") + } + + // Update password with correct old password + if err := ks.Update(a, "password123", "newpassword456"); err != nil { + t.Fatalf("Failed to update password: %v", err) + } + + // Old password should no longer work + if err := ks.Unlock(a, "password123"); err == nil { + t.Fatal("Old password should not work after update") + } + + // New password should work + if err := ks.Unlock(a, "newpassword456"); err != nil { + t.Fatalf("New password should work: %v", err) + } +} + +func TestDBKeyStore_DuplicateImport(t *testing.T) { + dir := t.TempDir() + ks, err := NewDBKeyStore(dir, LightScryptN, LightScryptP) + if err != nil { + t.Fatalf("Failed to create DB keystore: %v", err) + } + defer ks.Close() + + // Generate and import a key + privateKey, err := crypto.GenerateKey() + if err != nil { + t.Fatalf("Failed to generate key: %v", err) + } + + _, err = ks.ImportECDSA(privateKey, "password123") + if err != nil { + t.Fatalf("Failed to import key: %v", err) + } + + // Try to import the same key again - should fail + _, err = ks.ImportECDSA(privateKey, "password123") + if err != ErrAccountAlreadyExists { + t.Fatalf("Expected ErrAccountAlreadyExists, got: %v", err) + } +} + +func TestDBKeyStore_Wallets(t *testing.T) { + dir := t.TempDir() + ks, err := NewDBKeyStore(dir, LightScryptN, LightScryptP) + if err != nil { + t.Fatalf("Failed to create DB keystore: %v", err) + } + defer ks.Close() + + // Create accounts + a1, err := ks.NewAccount("password123") + if err != nil { + t.Fatalf("Failed to create account: %v", err) + } + + a2, err := ks.NewAccount("password456") + if err != nil { + t.Fatalf("Failed to create account: %v", err) + } + + // Get wallets + wallets := ks.Wallets() + if len(wallets) != 2 { + t.Fatalf("Expected 2 wallets, got %d", len(wallets)) + } + + // Find wallet for each account + foundA1, foundA2 := false, false + for _, w := range wallets { + accs := w.Accounts() + if len(accs) != 1 { + t.Fatal("Each wallet should have exactly 1 account") + } + if accs[0].Address == a1.Address { + foundA1 = true + } + if accs[0].Address == a2.Address { + foundA2 = true + } + } + + if !foundA1 || !foundA2 { + t.Fatal("Not all accounts found in wallets") + } +} + +func TestDBKeyStore_WalletSignTx(t *testing.T) { + dir := t.TempDir() + ks, err := NewDBKeyStore(dir, LightScryptN, LightScryptP) + if err != nil { + t.Fatalf("Failed to create DB keystore: %v", err) + } + defer ks.Close() + + // Create an account + a, err := ks.NewAccount("password123") + if err != nil { + t.Fatalf("Failed to create account: %v", err) + } + + // Get wallet + wallets := ks.Wallets() + if len(wallets) != 1 { + t.Fatalf("Expected 1 wallet, got %d", len(wallets)) + } + wallet := wallets[0] + + // Create a transaction + to := common.HexToAddress("0x1234567890123456789012345678901234567890") + tx := types.NewTransaction(0, to, big.NewInt(1000), 21000, big.NewInt(1), nil) + chainID := big.NewInt(1) + + // Sign transaction with passphrase through wallet + signedTx, err := wallet.SignTxWithPassphrase(a, "password123", tx, chainID) + if err != nil { + t.Fatalf("Failed to sign transaction: %v", err) + } + + // Verify the signature + signer := types.LatestSignerForChainID(chainID) + from, err := types.Sender(signer, signedTx) + if err != nil { + t.Fatalf("Failed to recover sender: %v", err) + } + + if from != a.Address { + t.Fatalf("Sender mismatch: got %s, want %s", from.Hex(), a.Address.Hex()) + } +} + +func TestDBKeyStore_Persistence(t *testing.T) { + dir := t.TempDir() + + // Create keystore and add an account + ks, err := NewDBKeyStore(dir, LightScryptN, LightScryptP) + if err != nil { + t.Fatalf("Failed to create DB keystore: %v", err) + } + + a, err := ks.NewAccount("password123") + if err != nil { + t.Fatalf("Failed to create account: %v", err) + } + + addr := a.Address + ks.Close() + + // Reopen keystore and verify account persists + ks2, err := NewDBKeyStore(dir, LightScryptN, LightScryptP) + if err != nil { + t.Fatalf("Failed to reopen DB keystore: %v", err) + } + defer ks2.Close() + + if !ks2.HasAddress(addr) { + t.Fatal("Account should persist after reopening keystore") + } + + accounts := ks2.Accounts() + if len(accounts) != 1 { + t.Fatalf("Expected 1 account, got %d", len(accounts)) + } + + if accounts[0].Address != addr { + t.Fatal("Account address mismatch after reopen") + } + + // Verify the account still works + if err := ks2.Unlock(accounts[0], "password123"); err != nil { + t.Fatalf("Failed to unlock account after reopen: %v", err) + } +} + +func BenchmarkDBKeyStore_NewAccount(b *testing.B) { + dir := b.TempDir() + ks, err := NewDBKeyStore(dir, LightScryptN, LightScryptP) + if err != nil { + b.Fatalf("Failed to create DB keystore: %v", err) + } + defer ks.Close() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := ks.NewAccount("password123") + if err != nil { + b.Fatalf("Failed to create account: %v", err) + } + } +} + +func BenchmarkDBKeyStore_HasAddress(b *testing.B) { + dir := b.TempDir() + ks, err := NewDBKeyStore(dir, LightScryptN, LightScryptP) + if err != nil { + b.Fatalf("Failed to create DB keystore: %v", err) + } + defer ks.Close() + + // Pre-populate with accounts + var addrs []common.Address + for i := 0; i < 1000; i++ { + a, err := ks.NewAccount("password123") + if err != nil { + b.Fatalf("Failed to create account: %v", err) + } + addrs = append(addrs, a.Address) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + addr := addrs[i%len(addrs)] + ks.HasAddress(addr) + } +} + +func BenchmarkDBKeyStore_Accounts(b *testing.B) { + dir := b.TempDir() + ks, err := NewDBKeyStore(dir, LightScryptN, LightScryptP) + if err != nil { + b.Fatalf("Failed to create DB keystore: %v", err) + } + defer ks.Close() + + // Pre-populate with accounts + for i := 0; i < 1000; i++ { + _, err := ks.NewAccount("password123") + if err != nil { + b.Fatalf("Failed to create account: %v", err) + } + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + accounts := ks.Accounts() + if len(accounts) != 1000 { + b.Fatalf("Expected 1000 accounts, got %d", len(accounts)) + } + } +} + +// Helper function to check if account implements wallet interface +func TestDBKeystoreWallet_Interface(t *testing.T) { + dir := t.TempDir() + ks, err := NewDBKeyStore(dir, LightScryptN, LightScryptP) + if err != nil { + t.Fatalf("Failed to create DB keystore: %v", err) + } + defer ks.Close() + + _, err = ks.NewAccount("password123") + if err != nil { + t.Fatalf("Failed to create account: %v", err) + } + + wallets := ks.Wallets() + if len(wallets) != 1 { + t.Fatalf("Expected 1 wallet, got %d", len(wallets)) + } + + // Check the wallet implements accounts.Wallet + var _ accounts.Wallet = wallets[0] +} diff --git a/cmd/clef/main.go b/cmd/clef/main.go index dde4ae853f..5bc4ee021f 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -90,7 +90,12 @@ var ( keystoreFlag = &cli.StringFlag{ Name: "keystore", Value: filepath.Join(node.DefaultDataDir(), "keystore"), - Usage: "Directory for the keystore", + Usage: "Directory for the keystore (file mode) or path to the keystore database (db mode)", + } + keystoreTypeFlag = &cli.StringFlag{ + Name: "keystore-type", + Value: "file", + Usage: "Keystore backend type: file (default) or db", } configdirFlag = &cli.StringFlag{ Name: "configdir", @@ -200,6 +205,7 @@ The delpw command removes a password for a given address (keyfile). Flags: []cli.Flag{ logLevelFlag, keystoreFlag, + keystoreTypeFlag, utils.LightKDFFlag, acceptFlag, }, @@ -221,6 +227,7 @@ The gendoc generates example structures of the json-rpc communication types. Flags: []cli.Flag{ logLevelFlag, keystoreFlag, + keystoreTypeFlag, utils.LightKDFFlag, acceptFlag, }, @@ -234,6 +241,7 @@ The gendoc generates example structures of the json-rpc communication types. Flags: []cli.Flag{ logLevelFlag, keystoreFlag, + keystoreTypeFlag, utils.LightKDFFlag, acceptFlag, }, @@ -248,6 +256,7 @@ The gendoc generates example structures of the json-rpc communication types. Flags: []cli.Flag{ logLevelFlag, keystoreFlag, + keystoreTypeFlag, utils.LightKDFFlag, acceptFlag, }, @@ -266,6 +275,7 @@ func init() { app.Flags = []cli.Flag{ logLevelFlag, keystoreFlag, + keystoreTypeFlag, configdirFlag, chainIdFlag, utils.LightKDFFlag, @@ -406,9 +416,13 @@ func initInternalApi(c *cli.Context) (*core.UIServerAPI, core.UIClientAPI, error ui = core.NewCommandlineUI() pwStorage storage.Storage = &storage.NoStorage{} ksLoc = c.String(keystoreFlag.Name) + ksType = c.String(keystoreTypeFlag.Name) lightKdf = c.Bool(utils.LightKDFFlag.Name) ) - am := core.StartClefAccountManager(ksLoc, true, lightKdf, "") + am, err := core.StartClefAccountManager(ksLoc, ksType, true, lightKdf, "") + if err != nil { + return nil, nil, err + } api := core.NewSignerAPI(am, 0, true, ui, nil, false, pwStorage) internalApi := core.NewUIServerAPI(api) return internalApi, ui, nil @@ -696,14 +710,18 @@ func signer(c *cli.Context) error { var ( chainId = c.Int64(chainIdFlag.Name) ksLoc = c.String(keystoreFlag.Name) + ksType = c.String(keystoreTypeFlag.Name) lightKdf = c.Bool(utils.LightKDFFlag.Name) advanced = c.Bool(advancedMode.Name) nousb = c.Bool(utils.NoUSBFlag.Name) scpath = c.String(utils.SmartCardDaemonPathFlag.Name) ) log.Info("Starting signer", "chainid", chainId, "keystore", ksLoc, - "light-kdf", lightKdf, "advanced", advanced) - am := core.StartClefAccountManager(ksLoc, nousb, lightKdf, scpath) + "keystore-type", ksType, "light-kdf", lightKdf, "advanced", advanced) + am, err := core.StartClefAccountManager(ksLoc, ksType, nousb, lightKdf, scpath) + if err != nil { + utils.Fatalf("Failed to start account manager: %v", err) + } defer am.Close() apiImpl := core.NewSignerAPI(am, chainId, nousb, ui, db, advanced, pwStorage) diff --git a/cmd/clef/migrate.go b/cmd/clef/migrate.go new file mode 100644 index 0000000000..883eafa0f2 --- /dev/null +++ b/cmd/clef/migrate.go @@ -0,0 +1,243 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/ethereum/go-ethereum/accounts/keystore" + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/log" + "github.com/urfave/cli/v2" +) + +var ( + migrateFromFlag = &cli.StringFlag{ + Name: "from", + Usage: "Source file-based keystore directory", + Required: true, + } + migrateToFlag = &cli.StringFlag{ + Name: "to", + Usage: "Destination database keystore path", + Required: true, + } + migrateKeystoreCommand = &cli.Command{ + Action: migrateKeystore, + Name: "migrate-keystore", + Usage: "Migrate keys from file-based keystore to database keystore", + ArgsUsage: "", + Flags: []cli.Flag{ + logLevelFlag, + migrateFromFlag, + migrateToFlag, + utils.LightKDFFlag, + acceptFlag, + }, + Description: ` +The migrate-keystore command migrates all keys from a file-based keystore +to a database-backed keystore for improved scalability. + +Example: + clef migrate-keystore --from ~/.ethereum/keystore --to ~/.ethereum/keystore.db + +Note: This operation requires entering the password for each key to decrypt +and re-encrypt it into the database. The original keystore files are not +modified or deleted. +`, + } +) + +func init() { + app.Commands = append(app.Commands, migrateKeystoreCommand) +} + +func migrateKeystore(c *cli.Context) error { + if err := initialize(c); err != nil { + return err + } + + fromPath := c.String(migrateFromFlag.Name) + toPath := c.String(migrateToFlag.Name) + lightKdf := c.Bool(utils.LightKDFFlag.Name) + + // Validate source path + info, err := os.Stat(fromPath) + if err != nil { + return fmt.Errorf("source keystore not found: %w", err) + } + if !info.IsDir() { + return fmt.Errorf("source keystore must be a directory: %s", fromPath) + } + + // Check if destination already exists + if _, err := os.Stat(toPath); err == nil { + return fmt.Errorf("destination already exists: %s (remove it first if you want to start fresh)", toPath) + } + + // Set scrypt parameters + var n, p int + if lightKdf { + n, p = keystore.LightScryptN, keystore.LightScryptP + } else { + n, p = keystore.StandardScryptN, keystore.StandardScryptP + } + + // Create source keystore (file-based) + srcKs := keystore.NewKeyStore(fromPath, n, p) + srcAccounts := srcKs.Accounts() + + if len(srcAccounts) == 0 { + log.Info("No accounts found in source keystore", "path", fromPath) + return nil + } + + log.Info("Found accounts to migrate", "count", len(srcAccounts), "from", fromPath, "to", toPath) + + // Create destination keystore (database-backed) + dstKs, err := keystore.NewDBKeyStore(toPath, n, p) + if err != nil { + return fmt.Errorf("failed to create destination keystore: %w", err) + } + defer dstKs.Close() + + // Migrate each account + migrated := 0 + skipped := 0 + failed := 0 + + for i, account := range srcAccounts { + log.Info("Migrating account", "index", i+1, "total", len(srcAccounts), "address", account.Address.Hex()) + + // Check if account already exists in destination + if dstKs.HasAddress(account.Address) { + log.Warn("Account already exists in destination, skipping", "address", account.Address.Hex()) + skipped++ + continue + } + + // Get password for this account + prompt := fmt.Sprintf("Enter password for account %s", account.Address.Hex()) + password := utils.GetPassPhrase(prompt, false) + + // Export from source + keyJSON, err := srcKs.Export(account, password, password) + if err != nil { + log.Error("Failed to export account", "address", account.Address.Hex(), "err", err) + failed++ + continue + } + + // Import to destination + _, err = dstKs.Import(keyJSON, password, password) + if err != nil { + log.Error("Failed to import account", "address", account.Address.Hex(), "err", err) + failed++ + continue + } + + log.Info("Successfully migrated account", "address", account.Address.Hex()) + migrated++ + } + + // Print summary + fmt.Println() + log.Info("Migration complete", + "migrated", migrated, + "skipped", skipped, + "failed", failed, + "total", len(srcAccounts)) + + if failed > 0 { + return fmt.Errorf("%d account(s) failed to migrate", failed) + } + + // Verify migration + dstAccounts := dstKs.Accounts() + log.Info("Verification", "accounts_in_destination", len(dstAccounts)) + + fmt.Println() + fmt.Println("Migration successful!") + fmt.Println() + fmt.Printf("To use the database keystore, run clef with:\n") + fmt.Printf(" clef --keystore-type db --keystore %s\n", toPath) + fmt.Println() + fmt.Println("Your original keystore files have NOT been modified.") + fmt.Println("Once you verify everything works, you may optionally backup and remove the old keystore.") + + return nil +} + +// MigrateKeystoreBatch migrates keys without prompting for passwords. +// This is useful for programmatic migration when passwords are known. +func MigrateKeystoreBatch(fromPath, toPath string, passwords map[string]string, lightKdf bool) error { + // Set scrypt parameters + var n, p int + if lightKdf { + n, p = keystore.LightScryptN, keystore.LightScryptP + } else { + n, p = keystore.StandardScryptN, keystore.StandardScryptP + } + + // Create source keystore + srcKs := keystore.NewKeyStore(fromPath, n, p) + srcAccounts := srcKs.Accounts() + + if len(srcAccounts) == 0 { + return nil + } + + // Create destination keystore + dstKs, err := keystore.NewDBKeyStore(toPath, n, p) + if err != nil { + return fmt.Errorf("failed to create destination keystore: %w", err) + } + defer dstKs.Close() + + // Migrate each account + for _, account := range srcAccounts { + addrHex := account.Address.Hex() + password, ok := passwords[addrHex] + if !ok { + // Try lowercase + password, ok = passwords[filepath.Base(account.URL.Path)] + if !ok { + return fmt.Errorf("no password provided for account %s", addrHex) + } + } + + // Skip if already exists + if dstKs.HasAddress(account.Address) { + continue + } + + // Export and import + keyJSON, err := srcKs.Export(account, password, password) + if err != nil { + return fmt.Errorf("failed to export account %s: %w", addrHex, err) + } + + _, err = dstKs.Import(keyJSON, password, password) + if err != nil { + return fmt.Errorf("failed to import account %s: %w", addrHex, err) + } + } + + return nil +} diff --git a/signer/core/api.go b/signer/core/api.go index 12acf925f0..ab4ecd783f 100644 --- a/signer/core/api.go +++ b/signer/core/api.go @@ -130,7 +130,9 @@ type Metadata struct { Origin string `json:"Origin"` } -func StartClefAccountManager(ksLocation string, nousb, lightKDF bool, scpath string) *accounts.Manager { +// StartClefAccountManager creates an account manager for Clef with the specified keystore type. +// ksType can be "file" (default) for file-based keystore or "db" for database-backed keystore. +func StartClefAccountManager(ksLocation, ksType string, nousb, lightKDF bool, scpath string) (*accounts.Manager, error) { var ( backends []accounts.Backend n, p = keystore.StandardScryptN, keystore.StandardScryptP @@ -140,7 +142,22 @@ func StartClefAccountManager(ksLocation string, nousb, lightKDF bool, scpath str } // support password based accounts if len(ksLocation) > 0 { - backends = append(backends, keystore.NewKeyStore(ksLocation, n, p)) + switch ksType { + case "db": + // Use database-backed keystore for scalability + dbks, err := keystore.NewDBKeyStore(ksLocation, n, p) + if err != nil { + return nil, fmt.Errorf("failed to open DB keystore: %w", err) + } + backends = append(backends, dbks) + log.Info("Using database-backed keystore", "path", ksLocation) + case "file", "": + // Use traditional file-based keystore (default) + backends = append(backends, keystore.NewKeyStore(ksLocation, n, p)) + log.Info("Using file-based keystore", "path", ksLocation) + default: + return nil, fmt.Errorf("unknown keystore type: %s (supported: file, db)", ksType) + } } if !nousb { // Start a USB hub for Ledger hardware wallets @@ -184,7 +201,7 @@ func StartClefAccountManager(ksLocation string, nousb, lightKDF bool, scpath str } } } - return accounts.NewManager(nil, backends...) + return accounts.NewManager(nil, backends...), nil } // MetadataFromContext extracts Metadata from a given context.Context