accounts/keystore, finish db backed keystore

accounts/keystore/db_keystore.go

@@ -1,9 +1,327 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
 package keystore
 
-import "reflect"
+import (
+	"crypto/ecdsa"
+	crand "crypto/rand"
+	"fmt"
+	"math/big"
+	"reflect"
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/event"
+	"github.com/pborman/uuid"
+)
 
 // DBKeyStoreType is the reflect type of a keystore backend.
 var DBKeyStoreType = reflect.TypeOf(&keyStoreDB{})
 
 type keyStoreDB struct {
+	storage  *keyStorePassphraseDB        // storage backend, might be mysql or postgres
+	unlocked map[common.Address]*unlocked // Currently unlocked account (decrypted private keys)
+
+	mu sync.RWMutex
+}
+
+// Wallets implements accounts.Backend, returning all single-key wallets from the KeyStore.
+func (ks *keyStoreDB) Wallets() []accounts.Wallet {
+	accs := ks.storage.All()
+	wallets := make([]accounts.Wallet, len(accs))
+	for idx, acc := range accs {
+		wallets[idx] = &keystoreWalletDB{account: acc, keystore: ks}
+	}
+	return wallets
+}
+
+// Subscribe implements accounts.Backend, creating an async subscription to
+// receive notifications on the addition or removal of KeyStore wallets.
+func (ks *keyStoreDB) Subscribe(sink chan<- accounts.WalletEvent) event.Subscription {
+	// Since this is a database backend, we don't need a in-memory cache to hold all the wallets
+	// so notifications on actions of wallets can be ignored
+	return nil
+}
+
+// HasAddress reports whether a key with the given address is present.
+func (ks *keyStoreDB) HasAddress(addr common.Address) bool {
+	return ks.storage.Exists(addr)
+}
+
+// Accounts returns all key files present in the KeyStore.
+func (ks *keyStoreDB) Accounts() []accounts.Account {
+	return ks.storage.All()
+}
+
+// Delete deletes the key matched by account if the passphrase is correct.
+// If the account contains no filename, the address must match a unique key.
+func (ks *keyStoreDB) Delete(a accounts.Account, passphrase string) error {
+	// Decrypting the key isn't really necessary, but we do
+	// it anyway to check the password and zero out the key
+	// immediately afterwards.
+	a, key, err := ks.getDecryptedKey(a, passphrase)
+	if key != nil {
+		zeroKey(key.PrivateKey)
+	}
+	if err != nil {
+		return err
+	}
+	ks.storage.db.Del(a.Address.Hex())
+	return nil
+}
+
+// SignHash calculates an ECDSA signature for the given hash. The produced
+// signature is in the [R || S || V] format where V is 0 or 1.
+func (ks *keyStoreDB) SignHash(a accounts.Account, hash []byte) ([]byte, error) {
+	// Look up the key to sign with and abort if it cannot be found
+	ks.mu.RLock()
+	defer ks.mu.RUnlock()
+
+	unlockedKey, found := ks.unlocked[a.Address]
+	if !found {
+		return nil, ErrLocked
+	}
+	// Sign the hash using plain ECDSA operations
+	return crypto.Sign(hash, unlockedKey.PrivateKey)
+}
+
+// SignTx signs the given transaction with the requested account.
+func (ks *keyStoreDB) SignTx(a accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	// Look up the key to sign with and abort if it cannot be found
+	ks.mu.RLock()
+	defer ks.mu.RUnlock()
+
+	unlockedKey, found := ks.unlocked[a.Address]
+	if !found {
+		return nil, ErrLocked
+	}
+	// Depending on the presence of the chain ID, sign with EIP155 or homestead
+	if chainID != nil {
+		return types.SignTx(tx, types.NewEIP155Signer(chainID), unlockedKey.PrivateKey)
+	}
+	return types.SignTx(tx, types.HomesteadSigner{}, unlockedKey.PrivateKey)
+}
+
+// SignHashWithPassphrase signs hash if the private key matching the given address
+// can be decrypted with the given passphrase. The produced signature is in the
+// [R || S || V] format where V is 0 or 1.
+func (ks *keyStoreDB) SignHashWithPassphrase(a accounts.Account, passphrase string, hash []byte) (signature []byte, err error) {
+	_, key, err := ks.getDecryptedKey(a, passphrase)
+	if err != nil {
+		return nil, err
+	}
+	defer zeroKey(key.PrivateKey)
+	return crypto.Sign(hash, key.PrivateKey)
+}
+
+// SignTxWithPassphrase signs the transaction if the private key matching the
+// given address can be decrypted with the given passphrase.
+func (ks *keyStoreDB) SignTxWithPassphrase(a accounts.Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	_, key, err := ks.getDecryptedKey(a, passphrase)
+	if err != nil {
+		return nil, err
+	}
+	defer zeroKey(key.PrivateKey)
+
+	// Depending on the presence of the chain ID, sign with EIP155 or homestead
+	if chainID != nil {
+		return types.SignTx(tx, types.NewEIP155Signer(chainID), key.PrivateKey)
+	}
+	return types.SignTx(tx, types.HomesteadSigner{}, key.PrivateKey)
+}
+
+// Unlock unlocks the given account indefinitely.
+func (ks *keyStoreDB) Unlock(a accounts.Account, passphrase string) error {
+	return ks.TimedUnlock(a, passphrase, 0)
+}
+
+// Lock removes the private key with the given address from memory.
+func (ks *keyStoreDB) Lock(addr common.Address) error {
+	ks.mu.Lock()
+	if unl, found := ks.unlocked[addr]; found {
+		ks.mu.Unlock()
+		ks.expire(addr, unl, time.Duration(0)*time.Nanosecond)
+	} else {
+		ks.mu.Unlock()
+	}
+	return nil
+}
+
+// TimedUnlock unlocks the given account with the passphrase. The account
+// stays unlocked for the duration of timeout. A timeout of 0 unlocks the account
+// until the program exits. The account must match a unique key file.
+//
+// If the account address is already unlocked for a duration, TimedUnlock extends or
+// shortens the active unlock timeout. If the address was previously unlocked
+// indefinitely the timeout is not altered.
+func (ks *keyStoreDB) TimedUnlock(a accounts.Account, passphrase string, timeout time.Duration) error {
+	a, key, err := ks.getDecryptedKey(a, passphrase)
+	if err != nil {
+		return err
+	}
+
+	ks.mu.Lock()
+	defer ks.mu.Unlock()
+
+	u, found := ks.unlocked[a.Address]
+	if found {
+		if u.abort == nil {
+			// The address was unlocked indefinitely, so unlocking
+			// it with a timeout would be confusing.
+			zeroKey(key.PrivateKey)
+			return nil
+		}
+		// Terminate the expire goroutine and replace it below.
+		close(u.abort)
+	}
+	if timeout > 0 {
+		u = &unlocked{Key: key, abort: make(chan struct{})}
+		go ks.expire(a.Address, u, timeout)
+	} else {
+		u = &unlocked{Key: key}
+	}
+	ks.unlocked[a.Address] = u
+	return nil
+}
+
+func (ks *keyStoreDB) expire(addr common.Address, u *unlocked, timeout time.Duration) {
+	t := time.NewTimer(timeout)
+	defer t.Stop()
+	select {
+	case <-u.abort:
+		// just quit
+	case <-t.C:
+		ks.mu.Lock()
+		// only drop if it's still the same key instance that dropLater
+		// was launched with. we can check that using pointer equality
+		// because the map stores a new pointer every time the key is
+		// unlocked.
+		if ks.unlocked[addr] == u {
+			zeroKey(u.PrivateKey)
+			delete(ks.unlocked, addr)
+		}
+		ks.mu.Unlock()
+	}
+}
+
+// Find resolves the given account into a unique entry in the KeyStore.
+func (ks *keyStoreDB) Find(a accounts.Account) (accounts.Account, error) {
+	return ks.storage.Find(a)
+}
+
+// NewAccount generates a new key and stores it into the KeyStore,
+// encrypting it with the passphrase.
+func (ks *keyStoreDB) NewAccount(passphrase string) (accounts.Account, error) {
+	key, err := newKey(crand.Reader)
+	if err != nil {
+		return accounts.Account{}, err
+	}
+	a := accounts.Account{
+		Address: key.Address,
+		URL:     accounts.URL{Scheme: KeyStoreScheme, Path: ks.storage.JoinPath(key.Address.Hex())},
+	}
+	if err := ks.storage.StoreKey(key, passphrase); err != nil {
+		zeroKey(key.PrivateKey)
+		return accounts.Account{}, err
+	}
+	return a, err
+}
+
+// Export exports as a JSON key, encrypted with newPassphrase.
+func (ks *keyStoreDB) Export(a accounts.Account, passphrase, newPassphrase string) (keyJSON []byte, err error) {
+	_, key, err := ks.getDecryptedKey(a, passphrase)
+	if err != nil {
+		return nil, err
+	}
+	N, P := ks.storage.scryptN, ks.storage.scryptP
+	return EncryptKey(key, newPassphrase, N, P)
+}
+
+// Import stores the given encrypted JSON key into the KeyStore.
+func (ks *keyStoreDB) Import(keyJSON []byte, passphrase, newPassphrase string) (accounts.Account, error) {
+	key, err := DecryptKey(keyJSON, passphrase)
+	if key != nil && key.PrivateKey != nil {
+		defer zeroKey(key.PrivateKey)
+	}
+	if err != nil {
+		return accounts.Account{}, err
+	}
+	if ks.storage.Exists(key.Address) {
+		return accounts.Account{}, fmt.Errorf("account already exists")
+	}
+	return ks.importKey(key, newPassphrase)
+}
+
+// ImportECDSA stores the given key into the KeyStore, encrypting it with the passphrase.
+func (ks *keyStoreDB) ImportECDSA(priv *ecdsa.PrivateKey, passphrase string) (accounts.Account, error) {
+	key := newKeyFromECDSA(priv)
+	if ks.storage.Exists(key.Address) {
+		return accounts.Account{}, fmt.Errorf("account already exists")
+	}
+	return ks.importKey(key, passphrase)
+}
+
+func (ks *keyStoreDB) importKey(key *Key, passphrase string) (accounts.Account, error) {
+	a := accounts.Account{
+		Address: key.Address,
+		URL:     accounts.URL{Scheme: KeyStoreScheme, Path: ks.storage.JoinPath(key.Address.Hex())},
+	}
+	if err := ks.storage.StoreKey(key, passphrase); err != nil {
+		return accounts.Account{}, err
+	}
+	return a, nil
+}
+
+// Update changes the passphrase of an existing account.
+func (ks *keyStoreDB) Update(a accounts.Account, passphrase, newPassphrase string) error {
+	_, key, err := ks.getDecryptedKey(a, passphrase)
+	if err != nil {
+		return err
+	}
+	return ks.storage.StoreKey(key, newPassphrase)
+}
+
+// ImportPreSaleKey decrypts the given Ethereum presale wallet and stores
+// a key file in the KeyStore. The key file is encrypted with the same passphrase.
+func (ks *keyStoreDB) ImportPreSaleKey(keyJSON []byte, passphrase string) (accounts.Account, error) {
+	key, err := decryptPreSaleKey(keyJSON, passphrase)
+	if err != nil {
+		return accounts.Account{}, nil
+	}
+	key.Id = uuid.NewRandom()
+	if err := ks.storage.StoreKey(key, passphrase); err != nil {
+		return accounts.Account{}, err
+	}
+	a := accounts.Account{
+		Address: key.Address,
+		URL:     accounts.URL{Scheme: KeyStoreScheme, Path: ks.storage.JoinPath(key.Address.Hex())},
+	}
+	return a, nil
+}
+
+func (ks *keyStoreDB) getDecryptedKey(a accounts.Account, auth string) (accounts.Account, *Key, error) {
+	a, err := ks.Find(a)
+	if err != nil {
+		return a, nil, err
+	}
+	key, err := ks.storage.GetKey(a.Address, auth)
+	return a, key, err
 }

accounts/keystore/db_passphrase.go

@@ -0,0 +1,117 @@
+// Copyright 2014 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package keystore
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/cmd/clef/dbutil"
+	"github.com/ethereum/go-ethereum/common"
+)
+
+type keyStorePassphraseDB struct {
+	db      *dbutil.KVStore
+	scryptN int
+	scryptP int
+
+	// skipKeyFileVerification disables the security-feature which does
+	// reads and decrypts any newly created keyfiles. This should be 'false' in all
+	// cases except tests -- setting this to 'true' is not recommended.
+	skipKeyFileVerification bool
+}
+
+func (ks keyStorePassphraseDB) GetKey(addr common.Address, auth string) (*Key, error) {
+	keyjson, err := ks.db.Get(addr.Hex())
+	if err != nil {
+		return nil, err
+	}
+
+	key, err := DecryptKey([]byte(keyjson), auth)
+	if err != nil {
+		return nil, err
+	}
+	// Make sure we're really operating on the requested key (no swap attacks)
+	if key.Address != addr {
+		return nil, fmt.Errorf("key content mismatch: have account %x, want %x", key.Address, addr)
+	}
+	return key, nil
+}
+
+func (ks keyStorePassphraseDB) StoreKey(key *Key, auth string) error {
+	keyjson, err := EncryptKey(key, auth, ks.scryptN, ks.scryptP)
+	if err != nil {
+		return err
+	}
+	// Write into database
+	err = ks.db.Put(key.Address.Hex(), string(keyjson))
+	if err != nil {
+		return err
+	}
+	if !ks.skipKeyFileVerification {
+		// Verify that we can decrypt the file with the given password.
+		_, err = ks.GetKey(key.Address, auth)
+		if err != nil {
+			msg := "An error was encountered when saving and verifying the keystore file. \n" +
+				"This indicates that the keystore is corrupted. \n" +
+				"The corrupted key is stored at \n%v\n" +
+				"Please file a ticket at:\n\n" +
+				"https://github.com/ethereum/go-ethereum/issues." +
+				"The error was : %s"
+			//lint:ignore ST1005 This is a message for the user
+			return fmt.Errorf(msg, ks.db.Conf.DataSourceName(), err)
+		}
+	}
+	return nil
+}
+
+// JoinPath returns Path (custom database related) for creating accounts.Account
+func (ks keyStorePassphraseDB) JoinPath(key string) string {
+	return ks.db.Conf.Adapter + "/" + ks.db.Table + "/" + key
+}
+
+func (ks keyStorePassphraseDB) Exists(addr common.Address) bool {
+	return ks.db.Exists(addr.Hex())
+}
+
+func (ks keyStorePassphraseDB) All() []accounts.Account {
+	keys := ks.db.All()
+	accs := make([]accounts.Account, len(keys))
+	for idx, key := range keys {
+		accs[idx] = accounts.Account{
+			Address: common.HexToAddress(key),
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: ks.JoinPath(key)},
+		}
+	}
+	return accs
+}
+
+// Find returns the account with the correct URL for database backed account
+func (ks keyStorePassphraseDB) Find(a accounts.Account) (accounts.Account, error) {
+	found := ks.Exists(a.Address)
+	if found {
+		return accounts.Account{
+			Address: a.Address,
+			URL:     accounts.URL{Scheme: KeyStoreScheme, Path: ks.JoinPath(a.Address.Hex())},
+		}, nil
+	}
+	return accounts.Account{}, ErrNoMatch
+}
+
+func (ks keyStorePassphraseDB) Size() int {
+	return ks.db.Size()
+}

accounts/keystore/db_wallet.go

@@ -0,0 +1,145 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package keystore
+
+import (
+	"math/big"
+
+	ethereum "github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+)
+
+// keystoreWalletDB implements the accounts.Wallet interface for the original
+// keystore.
+type keystoreWalletDB struct {
+	account  accounts.Account // Single account contained in this wallet
+	keystore *keyStoreDB      // Keystore where the account originates from
+}
+
+// URL implements accounts.Wallet, returning the URL of the account within.
+func (w *keystoreWalletDB) URL() accounts.URL {
+	return w.account.URL
+}
+
+// Status implements accounts.Wallet, returning whether the account held by the
+// keystore wallet is unlocked or not.
+func (w *keystoreWalletDB) Status() (string, error) {
+	if _, ok := w.keystore.unlocked[w.account.Address]; ok {
+		return "Unlocked", nil
+	}
+	return "Locked", nil
+}
+
+// Open implements accounts.Wallet, but is a noop for plain wallets since there
+// is no connection or decryption step necessary to access the list of accounts.
+func (w *keystoreWalletDB) Open(passphrase string) error { return nil }
+
+// Close implements accounts.Wallet, but is a noop for plain wallets since there
+// is no meaningful open operation.
+func (w *keystoreWalletDB) Close() error { return nil }
+
+// Accounts implements accounts.Wallet, returning an account list consisting of
+// a single account that the plain kestore wallet contains.
+func (w *keystoreWalletDB) Accounts() []accounts.Account {
+	return []accounts.Account{w.account}
+}
+
+// Contains implements accounts.Wallet, returning whether a particular account is
+// or is not wrapped by this wallet instance.
+func (w *keystoreWalletDB) Contains(account accounts.Account) bool {
+	return account.Address == w.account.Address && (account.URL == (accounts.URL{}) || account.URL == w.account.URL)
+}
+
+// Derive implements accounts.Wallet, but is a noop for plain wallets since there
+// is no notion of hierarchical account derivation for plain keystore accounts.
+func (w *keystoreWalletDB) Derive(path accounts.DerivationPath, pin bool) (accounts.Account, error) {
+	return accounts.Account{}, accounts.ErrNotSupported
+}
+
+// SelfDerive implements accounts.Wallet, but is a noop for plain wallets since
+// there is no notion of hierarchical account derivation for plain keystore accounts.
+func (w *keystoreWalletDB) SelfDerive(bases []accounts.DerivationPath, chain ethereum.ChainStateReader) {
+}
+
+// signHash attempts to sign the given hash with
+// the given account. If the wallet does not wrap this particular account, an
+// error is returned to avoid account leakage (even though in theory we may be
+// able to sign via our shared keystore backend).
+func (w *keystoreWalletDB) signHash(account accounts.Account, hash []byte) ([]byte, error) {
+	// Make sure the requested account is contained within
+	if !w.Contains(account) {
+		return nil, accounts.ErrUnknownAccount
+	}
+	// Account seems valid, request the keystore to sign
+	return w.keystore.SignHash(account, hash)
+}
+
+// SignData signs keccak256(data). The mimetype parameter describes the type of data being signed
+func (w *keystoreWalletDB) SignData(account accounts.Account, mimeType string, data []byte) ([]byte, error) {
+	return w.signHash(account, crypto.Keccak256(data))
+}
+
+// SignDataWithPassphrase signs keccak256(data). The mimetype parameter describes the type of data being signed
+func (w *keystoreWalletDB) SignDataWithPassphrase(account accounts.Account, passphrase, mimeType string, data []byte) ([]byte, error) {
+	// Make sure the requested account is contained within
+	if !w.Contains(account) {
+		return nil, accounts.ErrUnknownAccount
+	}
+	// Account seems valid, request the keystore to sign
+	return w.keystore.SignHashWithPassphrase(account, passphrase, crypto.Keccak256(data))
+}
+
+func (w *keystoreWalletDB) SignText(account accounts.Account, text []byte) ([]byte, error) {
+	return w.signHash(account, accounts.TextHash(text))
+}
+
+// SignTextWithPassphrase implements accounts.Wallet, attempting to sign the
+// given hash with the given account using passphrase as extra authentication.
+func (w *keystoreWalletDB) SignTextWithPassphrase(account accounts.Account, passphrase string, text []byte) ([]byte, error) {
+	// Make sure the requested account is contained within
+	if !w.Contains(account) {
+		return nil, accounts.ErrUnknownAccount
+	}
+	// Account seems valid, request the keystore to sign
+	return w.keystore.SignHashWithPassphrase(account, passphrase, accounts.TextHash(text))
+}
+
+// SignTx implements accounts.Wallet, attempting to sign the given transaction
+// with the given account. If the wallet does not wrap this particular account,
+// an error is returned to avoid account leakage (even though in theory we may
+// be able to sign via our shared keystore backend).
+func (w *keystoreWalletDB) SignTx(account accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	// Make sure the requested account is contained within
+	if !w.Contains(account) {
+		return nil, accounts.ErrUnknownAccount
+	}
+	// Account seems valid, request the keystore to sign
+	return w.keystore.SignTx(account, tx, chainID)
+}
+
+// SignTxWithPassphrase implements accounts.Wallet, attempting to sign the given
+// transaction with the given account using passphrase as extra authentication.
+func (w *keystoreWalletDB) SignTxWithPassphrase(account accounts.Account, passphrase string, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) {
+	// Make sure the requested account is contained within
+	if !w.Contains(account) {
+		return nil, accounts.ErrUnknownAccount
+	}
+	// Account seems valid, request the keystore to sign
+	return w.keystore.SignTxWithPassphrase(account, passphrase, tx, chainID)
+}

accounts/keystore/key.go

@@ -52,8 +52,10 @@ type keyStore interface {
 	// Loads and decrypts the key from disk.
 	GetKey(addr common.Address, filename string, auth string) (*Key, error)
 	// Writes and encrypts the key.
+	// If it is a database backed keyStore, the input filename will be ignored
 	StoreKey(filename string, k *Key, auth string) error
-	// Joins filename with the key directory unless it is already absolute.
+	// JoinPath forms the Account URL Path
+	// If it is a database backed keyStore, the input should be the key string
 	JoinPath(filename string) string
 }
 

accounts/keystore/passphrase.go

@@ -26,48 +26,14 @@ The crypto is documented at https://github.com/ethereum/wiki/wiki/Web3-Secret-St
 package keystore
 
 import (
-	"bytes"
-	"crypto/aes"
 	"crypto/rand"
-	"crypto/sha256"
-	"encoding/hex"
-	"encoding/json"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 
 	"github.com/ethereum/go-ethereum/accounts"
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/common/math"
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/pborman/uuid"
-	"golang.org/x/crypto/pbkdf2"
-	"golang.org/x/crypto/scrypt"
-)
-
-const (
-	keyHeaderKDF = "scrypt"
-
-	// StandardScryptN is the N parameter of Scrypt encryption algorithm, using 256MB
-	// memory and taking approximately 1s CPU time on a modern processor.
-	StandardScryptN = 1 << 18
-
-	// StandardScryptP is the P parameter of Scrypt encryption algorithm, using 256MB
-	// memory and taking approximately 1s CPU time on a modern processor.
-	StandardScryptP = 1
-
-	// LightScryptN is the N parameter of Scrypt encryption algorithm, using 4MB
-	// memory and taking approximately 100ms CPU time on a modern processor.
-	LightScryptN = 1 << 12
-
-	// LightScryptP is the P parameter of Scrypt encryption algorithm, using 4MB
-	// memory and taking approximately 100ms CPU time on a modern processor.
-	LightScryptP = 6
-
-	scryptR     = 8
-	scryptDKLen = 32
 )
 
 type keyStorePassphrase struct {
@@ -136,222 +102,3 @@ func (ks keyStorePassphrase) JoinPath(filename string) string {
 	}
 	return filepath.Join(ks.keysDirPath, filename)
 }
-
-// Encryptdata encrypts the data given as 'data' with the password 'auth'.
-func EncryptDataV3(data, auth []byte, scryptN, scryptP int) (CryptoJSON, error) {
-
-	salt := make([]byte, 32)
-	if _, err := io.ReadFull(rand.Reader, salt); err != nil {
-		panic("reading from crypto/rand failed: " + err.Error())
-	}
-	derivedKey, err := scrypt.Key(auth, salt, scryptN, scryptR, scryptP, scryptDKLen)
-	if err != nil {
-		return CryptoJSON{}, err
-	}
-	encryptKey := derivedKey[:16]
-
-	iv := make([]byte, aes.BlockSize) // 16
-	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
-		panic("reading from crypto/rand failed: " + err.Error())
-	}
-	cipherText, err := aesCTRXOR(encryptKey, data, iv)
-	if err != nil {
-		return CryptoJSON{}, err
-	}
-	mac := crypto.Keccak256(derivedKey[16:32], cipherText)
-
-	scryptParamsJSON := make(map[string]interface{}, 5)
-	scryptParamsJSON["n"] = scryptN
-	scryptParamsJSON["r"] = scryptR
-	scryptParamsJSON["p"] = scryptP
-	scryptParamsJSON["dklen"] = scryptDKLen
-	scryptParamsJSON["salt"] = hex.EncodeToString(salt)
-	cipherParamsJSON := cipherparamsJSON{
-		IV: hex.EncodeToString(iv),
-	}
-
-	cryptoStruct := CryptoJSON{
-		Cipher:       "aes-128-ctr",
-		CipherText:   hex.EncodeToString(cipherText),
-		CipherParams: cipherParamsJSON,
-		KDF:          keyHeaderKDF,
-		KDFParams:    scryptParamsJSON,
-		MAC:          hex.EncodeToString(mac),
-	}
-	return cryptoStruct, nil
-}
-
-// EncryptKey encrypts a key using the specified scrypt parameters into a json
-// blob that can be decrypted later on.
-func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
-	keyBytes := math.PaddedBigBytes(key.PrivateKey.D, 32)
-	cryptoStruct, err := EncryptDataV3(keyBytes, []byte(auth), scryptN, scryptP)
-	if err != nil {
-		return nil, err
-	}
-	encryptedKeyJSONV3 := encryptedKeyJSONV3{
-		hex.EncodeToString(key.Address[:]),
-		cryptoStruct,
-		key.Id.String(),
-		version,
-	}
-	return json.Marshal(encryptedKeyJSONV3)
-}
-
-// DecryptKey decrypts a key from a json blob, returning the private key itself.
-func DecryptKey(keyjson []byte, auth string) (*Key, error) {
-	// Parse the json into a simple map to fetch the key version
-	m := make(map[string]interface{})
-	if err := json.Unmarshal(keyjson, &m); err != nil {
-		return nil, err
-	}
-	// Depending on the version try to parse one way or another
-	var (
-		keyBytes, keyId []byte
-		err             error
-	)
-	if version, ok := m["version"].(string); ok && version == "1" {
-		k := new(encryptedKeyJSONV1)
-		if err := json.Unmarshal(keyjson, k); err != nil {
-			return nil, err
-		}
-		keyBytes, keyId, err = decryptKeyV1(k, auth)
-	} else {
-		k := new(encryptedKeyJSONV3)
-		if err := json.Unmarshal(keyjson, k); err != nil {
-			return nil, err
-		}
-		keyBytes, keyId, err = decryptKeyV3(k, auth)
-	}
-	// Handle any decryption errors and return the key
-	if err != nil {
-		return nil, err
-	}
-	key := crypto.ToECDSAUnsafe(keyBytes)
-
-	return &Key{
-		Id:         uuid.UUID(keyId),
-		Address:    crypto.PubkeyToAddress(key.PublicKey),
-		PrivateKey: key,
-	}, nil
-}
-
-func DecryptDataV3(cryptoJson CryptoJSON, auth string) ([]byte, error) {
-	if cryptoJson.Cipher != "aes-128-ctr" {
-		return nil, fmt.Errorf("cipher not supported: %v", cryptoJson.Cipher)
-	}
-	mac, err := hex.DecodeString(cryptoJson.MAC)
-	if err != nil {
-		return nil, err
-	}
-
-	iv, err := hex.DecodeString(cryptoJson.CipherParams.IV)
-	if err != nil {
-		return nil, err
-	}
-
-	cipherText, err := hex.DecodeString(cryptoJson.CipherText)
-	if err != nil {
-		return nil, err
-	}
-
-	derivedKey, err := getKDFKey(cryptoJson, auth)
-	if err != nil {
-		return nil, err
-	}
-
-	calculatedMAC := crypto.Keccak256(derivedKey[16:32], cipherText)
-	if !bytes.Equal(calculatedMAC, mac) {
-		return nil, ErrDecrypt
-	}
-
-	plainText, err := aesCTRXOR(derivedKey[:16], cipherText, iv)
-	if err != nil {
-		return nil, err
-	}
-	return plainText, err
-}
-
-func decryptKeyV3(keyProtected *encryptedKeyJSONV3, auth string) (keyBytes []byte, keyId []byte, err error) {
-	if keyProtected.Version != version {
-		return nil, nil, fmt.Errorf("version not supported: %v", keyProtected.Version)
-	}
-	keyId = uuid.Parse(keyProtected.Id)
-	plainText, err := DecryptDataV3(keyProtected.Crypto, auth)
-	if err != nil {
-		return nil, nil, err
-	}
-	return plainText, keyId, err
-}
-
-func decryptKeyV1(keyProtected *encryptedKeyJSONV1, auth string) (keyBytes []byte, keyId []byte, err error) {
-	keyId = uuid.Parse(keyProtected.Id)
-	mac, err := hex.DecodeString(keyProtected.Crypto.MAC)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	iv, err := hex.DecodeString(keyProtected.Crypto.CipherParams.IV)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	cipherText, err := hex.DecodeString(keyProtected.Crypto.CipherText)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	derivedKey, err := getKDFKey(keyProtected.Crypto, auth)
-	if err != nil {
-		return nil, nil, err
-	}
-
-	calculatedMAC := crypto.Keccak256(derivedKey[16:32], cipherText)
-	if !bytes.Equal(calculatedMAC, mac) {
-		return nil, nil, ErrDecrypt
-	}
-
-	plainText, err := aesCBCDecrypt(crypto.Keccak256(derivedKey[:16])[:16], cipherText, iv)
-	if err != nil {
-		return nil, nil, err
-	}
-	return plainText, keyId, err
-}
-
-func getKDFKey(cryptoJSON CryptoJSON, auth string) ([]byte, error) {
-	authArray := []byte(auth)
-	salt, err := hex.DecodeString(cryptoJSON.KDFParams["salt"].(string))
-	if err != nil {
-		return nil, err
-	}
-	dkLen := ensureInt(cryptoJSON.KDFParams["dklen"])
-
-	if cryptoJSON.KDF == keyHeaderKDF {
-		n := ensureInt(cryptoJSON.KDFParams["n"])
-		r := ensureInt(cryptoJSON.KDFParams["r"])
-		p := ensureInt(cryptoJSON.KDFParams["p"])
-		return scrypt.Key(authArray, salt, n, r, p, dkLen)
-
-	} else if cryptoJSON.KDF == "pbkdf2" {
-		c := ensureInt(cryptoJSON.KDFParams["c"])
-		prf := cryptoJSON.KDFParams["prf"].(string)
-		if prf != "hmac-sha256" {
-			return nil, fmt.Errorf("unsupported PBKDF2 PRF: %s", prf)
-		}
-		key := pbkdf2.Key(authArray, salt, c, dkLen, sha256.New)
-		return key, nil
-	}
-
-	return nil, fmt.Errorf("unsupported KDF: %s", cryptoJSON.KDF)
-}
-
-// TODO: can we do without this when unmarshalling dynamic JSON?
-// why do integers in KDF params end up as float64 and not int after
-// unmarshal?
-func ensureInt(x interface{}) int {
-	res, ok := x.(int)
-	if !ok {
-		res = int(x.(float64))
-	}
-	return res
-}

accounts/keystore/passphrase_util.go

@@ -0,0 +1,260 @@
+package keystore
+
+import (
+	"bytes"
+	"crypto/aes"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+
+	"github.com/ethereum/go-ethereum/common/math"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/pborman/uuid"
+	"golang.org/x/crypto/pbkdf2"
+	"golang.org/x/crypto/scrypt"
+)
+
+const (
+	keyHeaderKDF = "scrypt"
+
+	// StandardScryptN is the N parameter of Scrypt encryption algorithm, using 256MB
+	// memory and taking approximately 1s CPU time on a modern processor.
+	StandardScryptN = 1 << 18
+
+	// StandardScryptP is the P parameter of Scrypt encryption algorithm, using 256MB
+	// memory and taking approximately 1s CPU time on a modern processor.
+	StandardScryptP = 1
+
+	// LightScryptN is the N parameter of Scrypt encryption algorithm, using 4MB
+	// memory and taking approximately 100ms CPU time on a modern processor.
+	LightScryptN = 1 << 12
+
+	// LightScryptP is the P parameter of Scrypt encryption algorithm, using 4MB
+	// memory and taking approximately 100ms CPU time on a modern processor.
+	LightScryptP = 6
+
+	scryptR     = 8
+	scryptDKLen = 32
+)
+
+// Encryptdata encrypts the data given as 'data' with the password 'auth'.
+func EncryptDataV3(data, auth []byte, scryptN, scryptP int) (CryptoJSON, error) {
+
+	salt := make([]byte, 32)
+	if _, err := io.ReadFull(rand.Reader, salt); err != nil {
+		panic("reading from crypto/rand failed: " + err.Error())
+	}
+	derivedKey, err := scrypt.Key(auth, salt, scryptN, scryptR, scryptP, scryptDKLen)
+	if err != nil {
+		return CryptoJSON{}, err
+	}
+	encryptKey := derivedKey[:16]
+
+	iv := make([]byte, aes.BlockSize) // 16
+	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
+		panic("reading from crypto/rand failed: " + err.Error())
+	}
+	cipherText, err := aesCTRXOR(encryptKey, data, iv)
+	if err != nil {
+		return CryptoJSON{}, err
+	}
+	mac := crypto.Keccak256(derivedKey[16:32], cipherText)
+
+	scryptParamsJSON := make(map[string]interface{}, 5)
+	scryptParamsJSON["n"] = scryptN
+	scryptParamsJSON["r"] = scryptR
+	scryptParamsJSON["p"] = scryptP
+	scryptParamsJSON["dklen"] = scryptDKLen
+	scryptParamsJSON["salt"] = hex.EncodeToString(salt)
+	cipherParamsJSON := cipherparamsJSON{
+		IV: hex.EncodeToString(iv),
+	}
+
+	cryptoStruct := CryptoJSON{
+		Cipher:       "aes-128-ctr",
+		CipherText:   hex.EncodeToString(cipherText),
+		CipherParams: cipherParamsJSON,
+		KDF:          keyHeaderKDF,
+		KDFParams:    scryptParamsJSON,
+		MAC:          hex.EncodeToString(mac),
+	}
+	return cryptoStruct, nil
+}
+
+// EncryptKey encrypts a key using the specified scrypt parameters into a json
+// blob that can be decrypted later on.
+func EncryptKey(key *Key, auth string, scryptN, scryptP int) ([]byte, error) {
+	keyBytes := math.PaddedBigBytes(key.PrivateKey.D, 32)
+	cryptoStruct, err := EncryptDataV3(keyBytes, []byte(auth), scryptN, scryptP)
+	if err != nil {
+		return nil, err
+	}
+	encryptedKeyJSONV3 := encryptedKeyJSONV3{
+		hex.EncodeToString(key.Address[:]),
+		cryptoStruct,
+		key.Id.String(),
+		version,
+	}
+	return json.Marshal(encryptedKeyJSONV3)
+}
+
+// DecryptKey decrypts a key from a json blob, returning the private key itself.
+func DecryptKey(keyjson []byte, auth string) (*Key, error) {
+	// Parse the json into a simple map to fetch the key version
+	m := make(map[string]interface{})
+	if err := json.Unmarshal(keyjson, &m); err != nil {
+		return nil, err
+	}
+	// Depending on the version try to parse one way or another
+	var (
+		keyBytes, keyId []byte
+		err             error
+	)
+	if version, ok := m["version"].(string); ok && version == "1" {
+		k := new(encryptedKeyJSONV1)
+		if err := json.Unmarshal(keyjson, k); err != nil {
+			return nil, err
+		}
+		keyBytes, keyId, err = decryptKeyV1(k, auth)
+	} else {
+		k := new(encryptedKeyJSONV3)
+		if err := json.Unmarshal(keyjson, k); err != nil {
+			return nil, err
+		}
+		keyBytes, keyId, err = decryptKeyV3(k, auth)
+	}
+	// Handle any decryption errors and return the key
+	if err != nil {
+		return nil, err
+	}
+	key := crypto.ToECDSAUnsafe(keyBytes)
+
+	return &Key{
+		Id:         uuid.UUID(keyId),
+		Address:    crypto.PubkeyToAddress(key.PublicKey),
+		PrivateKey: key,
+	}, nil
+}
+
+func DecryptDataV3(cryptoJson CryptoJSON, auth string) ([]byte, error) {
+	if cryptoJson.Cipher != "aes-128-ctr" {
+		return nil, fmt.Errorf("cipher not supported: %v", cryptoJson.Cipher)
+	}
+	mac, err := hex.DecodeString(cryptoJson.MAC)
+	if err != nil {
+		return nil, err
+	}
+
+	iv, err := hex.DecodeString(cryptoJson.CipherParams.IV)
+	if err != nil {
+		return nil, err
+	}
+
+	cipherText, err := hex.DecodeString(cryptoJson.CipherText)
+	if err != nil {
+		return nil, err
+	}
+
+	derivedKey, err := getKDFKey(cryptoJson, auth)
+	if err != nil {
+		return nil, err
+	}
+
+	calculatedMAC := crypto.Keccak256(derivedKey[16:32], cipherText)
+	if !bytes.Equal(calculatedMAC, mac) {
+		return nil, ErrDecrypt
+	}
+
+	plainText, err := aesCTRXOR(derivedKey[:16], cipherText, iv)
+	if err != nil {
+		return nil, err
+	}
+	return plainText, err
+}
+
+func decryptKeyV3(keyProtected *encryptedKeyJSONV3, auth string) (keyBytes []byte, keyId []byte, err error) {
+	if keyProtected.Version != version {
+		return nil, nil, fmt.Errorf("version not supported: %v", keyProtected.Version)
+	}
+	keyId = uuid.Parse(keyProtected.Id)
+	plainText, err := DecryptDataV3(keyProtected.Crypto, auth)
+	if err != nil {
+		return nil, nil, err
+	}
+	return plainText, keyId, err
+}
+
+func decryptKeyV1(keyProtected *encryptedKeyJSONV1, auth string) (keyBytes []byte, keyId []byte, err error) {
+	keyId = uuid.Parse(keyProtected.Id)
+	mac, err := hex.DecodeString(keyProtected.Crypto.MAC)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	iv, err := hex.DecodeString(keyProtected.Crypto.CipherParams.IV)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	cipherText, err := hex.DecodeString(keyProtected.Crypto.CipherText)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	derivedKey, err := getKDFKey(keyProtected.Crypto, auth)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	calculatedMAC := crypto.Keccak256(derivedKey[16:32], cipherText)
+	if !bytes.Equal(calculatedMAC, mac) {
+		return nil, nil, ErrDecrypt
+	}
+
+	plainText, err := aesCBCDecrypt(crypto.Keccak256(derivedKey[:16])[:16], cipherText, iv)
+	if err != nil {
+		return nil, nil, err
+	}
+	return plainText, keyId, err
+}
+
+func getKDFKey(cryptoJSON CryptoJSON, auth string) ([]byte, error) {
+	authArray := []byte(auth)
+	salt, err := hex.DecodeString(cryptoJSON.KDFParams["salt"].(string))
+	if err != nil {
+		return nil, err
+	}
+	dkLen := ensureInt(cryptoJSON.KDFParams["dklen"])
+
+	if cryptoJSON.KDF == keyHeaderKDF {
+		n := ensureInt(cryptoJSON.KDFParams["n"])
+		r := ensureInt(cryptoJSON.KDFParams["r"])
+		p := ensureInt(cryptoJSON.KDFParams["p"])
+		return scrypt.Key(authArray, salt, n, r, p, dkLen)
+
+	} else if cryptoJSON.KDF == "pbkdf2" {
+		c := ensureInt(cryptoJSON.KDFParams["c"])
+		prf := cryptoJSON.KDFParams["prf"].(string)
+		if prf != "hmac-sha256" {
+			return nil, fmt.Errorf("unsupported PBKDF2 PRF: %s", prf)
+		}
+		key := pbkdf2.Key(authArray, salt, c, dkLen, sha256.New)
+		return key, nil
+	}
+
+	return nil, fmt.Errorf("unsupported KDF: %s", cryptoJSON.KDF)
+}
+
+// TODO: can we do without this when unmarshalling dynamic JSON?
+// why do integers in KDF params end up as float64 and not int after
+// unmarshal?
+func ensureInt(x interface{}) int {
+	res, ok := x.(int)
+	if !ok {
+		res = int(x.(float64))
+	}
+	return res
+}

cmd/clef/dbutil/dbutil.go

@@ -66,9 +66,9 @@ func NewKVStore(path, table string) (*KVStore, error) {
 	}
 
 	return &KVStore{
-		conf:  conf,
+		Conf:  conf,
 		db:    db,
-		table: table,
+		Table: table,
 	}, nil
 }
 
@@ -124,8 +124,8 @@ CREATE TABLE IF NOT EXISTS %s (
 
 // KVStore is used for abstracting a generic database as a simple key value storage
 type KVStore struct {
-	table string
+	Table string
-	conf  *DBConf
+	Conf  *DBConf
 	db    *sql.DB
 }
 
@@ -145,8 +145,7 @@ func (kvstore *KVStore) Put(key, value string) error {
 	if len(key) == 0 {
 		return errors.New("0-length key")
 	}
-	_, err := kvstore.Get(key)
+	if !kvstore.Exists(key) {
-	if err != nil || err == sql.ErrNoRows {
 		return kvstore.insertRow(key, value)
 	} else {
 		return kvstore.updateRow(key, value)
@@ -161,15 +160,8 @@ func (kvstore *KVStore) Del(key string) {
 
 // All returns all keys in the database
 func (kvstore *KVStore) All() []string {
-	var size int
+	size := kvstore.Size()
-	sql := kvstore.adjustSQLPlaceholder(countSQL)
+	sql := kvstore.adjustSQLPlaceholder(allSQL)
-	err := kvstore.db.QueryRow(sql).Scan(&size)
-	if err != nil {
-		log.Error("Error counting key numbers: ", err)
-		return nil
-	}
-
-	sql = kvstore.adjustSQLPlaceholder(allSQL)
 	rows, err := kvstore.db.Query(sql)
 	if err != nil {
 		log.Error("Error retrieving all keys: ", err)
@@ -192,6 +184,24 @@ func (kvstore *KVStore) All() []string {
 	return result
 }
 
+// Exists returns a boolean indicates if the key exists or not
+func (kvstore *KVStore) Exists(key string) bool {
+	v, err := kvstore.Get(key)
+	return err == nil && v != ""
+}
+
+// Size returns number of entries that exists in the kvstore
+func (kvstore *KVStore) Size() int {
+	var size int
+	sql := kvstore.adjustSQLPlaceholder(countSQL)
+	err := kvstore.db.QueryRow(sql).Scan(&size)
+	if err != nil {
+		log.Error("Error counting key numbers: ", err)
+		return 0
+	}
+	return size
+}
+
 func (kvstore *KVStore) insertRow(key, value string) error {
 	sql := kvstore.adjustSQLPlaceholder(insertSQL)
 	return kvstore.exec(sql, key, value)
@@ -203,7 +213,7 @@ func (kvstore *KVStore) updateRow(key, value string) error {
 }
 
 func (kvstore *KVStore) adjustSQLPlaceholder(sql string) string {
-	switch kvstore.conf.Adapter {
+	switch kvstore.Conf.Adapter {
 	case "postgres":
 		params := strings.Count(sql, "?")
 		for i := 1; i <= params; i++ {
@@ -214,7 +224,7 @@ func (kvstore *KVStore) adjustSQLPlaceholder(sql string) string {
 		// since they're already using ? as placeholder, do nothing
 	}
 
-	return strings.ReplaceAll(sql, "tableName", kvstore.table)
+	return strings.ReplaceAll(sql, "tableName", kvstore.Table)
 }
 
 func (kvstore *KVStore) queryRow(query string, args ...interface{}) (string, error) {