forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-08-20 10:52:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

signer/storage abstract aes-gcm encryption into a layer above all storages

Browse source
This commit is contained in:
Huiyi Li 2020-01-31 16:30:00 -08:00
parent 9b649bcc09
commit 17c81b5e64
7 changed files with 240 additions and 383 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

217
signer/storage/db_storage.go
View file

@ -17,219 +17,26 @@
package storage
import (
"database/sql"
"encoding/json"
"fmt"
"strings"
"github.com/ethereum/go-ethereum/log"
// here we are adding multiple default supported db drivers
_ "github.com/go-sql-driver/mysql"
_ "github.com/lib/pq"
"github.com/ethereum/go-ethereum/cmd/clef/dbutil"
)
// DBStorage is a storage type which is backed by a general purpose database
type DBStorage struct {
driverName string
dataSourceName string
tableName string
db *sql.DB
key []byte
// DBStorageAPI is a storage api which is backed by a general purpose database
type DBStorageAPI struct {
kvstore *dbutil.KVStore
}
// DBRow is the structure to hold a row of our configuration database
// table schemas for all three tables (kps, js, config) are the same
type DBRow struct {
id int
key string
val string
}
// Default table name for storages
const (
PasswordTable = "kps"
ConfigTable = "config"
JsTable = "js"
)
// NewDBStorage create new database backed storage
func NewDBStorage(key []byte, driverName, dataSourceName, tableName string) (*DBStorage, error) {
// sql.Open only validates the input, but didn't create a connection
db, err := sql.Open(driverName, dataSourceName)
if err != nil {
log.Error("failed to validate driver: #{driverName}, #{dataSourceName}")
db.Close()
return nil, err
}
// Connects to the database and make sure it is ok, connection will be closed shortly since default MaxIdle is 0
err = db.Ping()
if err != nil {
log.Error("failed to connect to database: #{dataSourceName}")
db.Close()
return nil, err
}
// set connection limits
db.SetMaxOpenConns(5)
// init table
initTable(driverName, tableName, db)
return &DBStorage{
driverName: driverName,
dataSourceName: dataSourceName,
tableName: tableName,
db: db,
key: key,
}, nil
}
func initTable(driverName, tableName string, db *sql.DB) error {
var err error
switch driverName {
case "postgres":
_, err = db.Exec(fmt.Sprintf(`
CREATE TABLE IF NOT EXISTS %s (
id SERIAL PRIMARY KEY,
k VARCHAR(255) UNIQUE NOT NULL,
v TEXT NOT NULL
)
`, tableName))
case "mysql":
_, err = db.Exec(fmt.Sprintf(`
CREATE TABLE IF NOT EXISTS %s (
id INT AUTO_INCREMENT PRIMARY KEY,
k VARCHAR(255) UNIQUE NOT NULL,
v TEXT NOT NULL
)
`, tableName))
case "sqlite3":
_, err = db.Exec(fmt.Sprintf(`
CREATE TABLE IF NOT EXISTS %s (
id INTEGER PRIMARY KEY,
k TEXT,
v TEXT
)
`, tableName))
}
return err
// Get returns the previously stored value, or an error if the key is 0-length
// or unknown.
func (api *DBStorageAPI) Get(key string) (string, error) {
return api.kvstore.Get(key)
}
// Put stores a value by key. 0-length keys results in noop.
func (s *DBStorage) Put(key, value string) {
if len(key) == 0 {
return
}
ciphertext, iv, err := Encrypt(s.key, []byte(value), []byte(key))
if err != nil {
log.Warn("Failed to encrypt entry", "err", err)
return
}
creds := StoredCredential{Iv: iv, CipherText: ciphertext}
sql := s.formatSQL(getSQL)
_, exist, err := s.queryRow(sql, key)
if err != nil {
log.Warn("Failed to execute SQL", "err", err)
return
}
raw, err := json.Marshal(creds)
if err != nil {
log.Warn("Failed to marshal StoredCredential data")
return
}
if !exist {
sql = s.formatSQL(insertSQL)
s.exec(sql, key, raw)
} else {
sql = s.formatSQL(updateSQL)
s.exec(sql, raw, key)
}
}
// Get returns the previously stored value, or an error if it does not exist or
// key is of 0-length.
func (s *DBStorage) Get(key string) (string, error) {
sql := s.formatSQL(getSQL)
row, exist, err := s.queryRow(sql, key)
if err != nil {
log.Warn("Failed to execute SQL", "err", err)
return "", err
}
if !exist {
log.Warn("Key does not exist", "key", key)
return "", ErrNotFound
}
cred := StoredCredential{}
if err = json.Unmarshal([]byte(row.val), &cred); err != nil {
log.Warn("Failed to unmarshall stored json", "err", err)
return "", err
}
entry, err := Decrypt(s.key, cred.Iv, cred.CipherText, []byte(key))
if err != nil {
log.Warn("Failed to decrypt key", "key", key)
return "", err
}
return string(entry), nil
func (api *DBStorageAPI) Put(key, value string) {
api.kvstore.Put(key, value)
}
// Del removes a key-value pair. If the key doesn't exist, the method is a noop.
func (s *DBStorage) Del(key string) {
sql := s.formatSQL(deleteSQL)
s.exec(sql, key)
}
func (s *DBStorage) exec(query string, args ...interface{}) {
_, err := s.db.Exec(query, args...)
if err != nil {
log.Warn("Failed to execute sql", query, args)
}
}
func (s *DBStorage) queryRow(query string, args ...interface{}) (*DBRow, bool, error) {
row := DBRow{}
err := s.db.QueryRow(query, args...).Scan(&row.id, &row.key, &row.val)
if err != nil && err != sql.ErrNoRows {
return nil, false, err
}
if row.id == 0 {
return nil, false, nil
}
return &row, true, nil
}
var (
getSQL string = "SELECT * FROM tableName WHERE k = ?"
updateSQL string = "UPDATE tableName SET v = ? WHERE k = ?"
insertSQL string = "INSERT INTO tableName (k, v) VALUES (?, ?)"
deleteSQL string = "DELETE FROM tableName WHERE k = ?"
)
func (s *DBStorage) formatSQL(sql string) string {
switch s.driverName {
case "postgres":
params := strings.Count(sql, "?")
for i := 1; i <= params; i++ {
sql = strings.Replace(sql, "?", fmt.Sprintf("$%d", i), 1)
}
default:
// for MS SQL Server / MySQL / SQLite
// since they're already using ? as placeholder, do nothing
}
return strings.ReplaceAll(sql, "tableName", s.tableName)
}
// Close sql.DB
func (s *DBStorage) Close() {
s.db.Close()
func (api *DBStorageAPI) Del(key string) {
api.kvstore.Del(key)
}

69
signer/storage/db_storage_test.go
View file

@ -1,69 +0,0 @@
package storage
import (
"io/ioutil"
"path/filepath"
"testing"
_ "github.com/mattn/go-sqlite3"
)
var (
ds *DBStorage
key string
)
func init() {
key = "AES256Key-32Characters1234567890"
tmpDir, _ := ioutil.TempDir("", "eth-encrypted-db-storge-test")
ds, _ = NewDBStorage([]byte(key), "sqlite3", filepath.Join(tmpDir, "test.db"), "kps")
}
func TestDBStorage(t *testing.T) {
// test Put
k1, v1 := "k1", "v1"
ds.Put(k1, v1)
// test Get
ret, err := ds.Get(k1)
if err != nil || ret != v1 {
t.Fatal("Get didn't return correct result")
}
// test Put when there's duplicate
v2 := "v2"
ds.Put(k1, v2)
ret, err = ds.Get(k1)
if err != nil || ret != v2 {
t.Fatal("Get didn't return correct result")
}
// test Del
ds.Del(k1)
ret, err = ds.Get(k1)
if err != ErrNotFound {
t.Fatal("Del didn't work as expected")
}
}
func TestSwappedKeysForDBStorage(t *testing.T) {
ds.Put("k1", "v1")
ds.Put("k2", "v2")
// now make a modified copy
swap := func() {
creds1, _, _ := ds.queryRow("SELECT * FROM kps WHERE k = 'k1'")
creds2, _, _ := ds.queryRow("SELECT * FROM kps WHERE k = 'k2'")
ds.exec("UPDATE kps SET v = ? WHERE k = ?", creds1.val, "k2")
ds.exec("UPDATE kps SET v = ? WHERE k = ?", creds2.val, "k1")
}
swap()
if v, _ := ds.Get("k1"); v != "" {
t.Errorf("swapped value should return empty")
}
swap()
if v, _ := ds.Get("k1"); v != "v1" {
t.Errorf(v)
t.Errorf("double-swapped value should work fine")
}
}

32
signer/storage/ephemeral_storage.go Normal file
View file

@ -0,0 +1,32 @@
package storage
// EphemeralStorageAPI is an in-memory storage api that does
// not persist values to disk. Mainly used for testing
type EphemeralStorageAPI struct {
data map[string]string
}
// Put stores a value by key. 0-length keys results in noop.
func (s *EphemeralStorageAPI) Put(key, value string) {
if len(key) == 0 {
return
}
s.data[key] = value
}
// Get returns the previously stored value, or an error if the key is 0-length
// or unknown.
func (s *EphemeralStorageAPI) Get(key string) (string, error) {
if len(key) == 0 {
return "", ErrZeroKey
}
if v, ok := s.data[key]; ok {
return v, nil
}
return "", ErrNotFound
}
// Del removes a key-value pair. If the key doesn't exist, the method is a noop.
func (s *EphemeralStorageAPI) Del(key string) {
delete(s.data, key)
}

89
signer/storage/aes_gcm_storage.go → signer/storage/file_storage.go
View file

@ -24,85 +24,82 @@ import (
"github.com/ethereum/go-ethereum/log"
)
// AESEncryptedStorage is a storage type which is backed by a json-file. The json-file contains
// key-value mappings, where the keys are _not_ encrypted, only the values are.
type AESEncryptedStorage struct {
// FileStorageAPI is a storage type which is backed by a json-file.
type FileStorageAPI struct {
// File to read/write credentials
filename string
// Key stored in base64
key []byte
}
// NewAESEncryptedStorage creates a new encrypted storage backed by the given file/key
func NewAESEncryptedStorage(filename string, key []byte) *AESEncryptedStorage {
return &AESEncryptedStorage{
filename: filename,
key: key,
}
}
// Put stores a value by key. 0-length keys results in noop.
func (s *AESEncryptedStorage) Put(key, value string) {
func (s *FileStorageAPI) Put(key, value string) {
if len(key) == 0 {
return
}
data, err := s.readEncryptedStorage()
data, err := s.readStorage()
if err != nil {
log.Warn("Failed to read encrypted storage", "err", err, "file", s.filename)
return
}
ciphertext, iv, err := Encrypt(s.key, []byte(value), []byte(key))
if err != nil {
log.Warn("Failed to encrypt entry", "err", err)
return
}
encrypted := StoredCredential{Iv: iv, CipherText: ciphertext}
data[key] = encrypted
if err = s.writeEncryptedStorage(data); err != nil {
data[key] = value
if err = s.writeStorage(data); err != nil {
log.Warn("Failed to write entry", "err", err)
}
}
// Get returns the previously stored value, or an error if it does not exist or
// key is of 0-length.
func (s *AESEncryptedStorage) Get(key string) (string, error) {
func (s *FileStorageAPI) Get(key string) (string, error) {
if len(key) == 0 {
return "", ErrZeroKey
}
data, err := s.readEncryptedStorage()
data, err := s.readStorage()
if err != nil {
log.Warn("Failed to read encrypted storage", "err", err, "file", s.filename)
return "", err
}
encrypted, exist := data[key]
value, exist := data[key]
if !exist {
log.Warn("Key does not exist", "key", key)
return "", ErrNotFound
}
entry, err := Decrypt(s.key, encrypted.Iv, encrypted.CipherText, []byte(key))
if err != nil {
log.Warn("Failed to decrypt key", "key", key)
return "", err
}
return string(entry), nil
return value, nil
}
// Del removes a key-value pair. If the key doesn't exist, the method is a noop.
func (s *AESEncryptedStorage) Del(key string) {
data, err := s.readEncryptedStorage()
func (s *FileStorageAPI) Del(key string) {
data, err := s.readStorage()
if err != nil {
log.Warn("Failed to read encrypted storage", "err", err, "file", s.filename)
return
}
delete(data, key)
if err = s.writeEncryptedStorage(data); err != nil {
if err = s.writeStorage(data); err != nil {
log.Warn("Failed to write entry", "err", err)
}
}
// readEncryptedStorage reads the file with encrypted creds
func (s *AESEncryptedStorage) readEncryptedStorage() (map[string]StoredCredential, error) {
creds := make(map[string]StoredCredential)
// func (s *FileStorageAPI) readEncryptedStorage() (map[string]StoredCredential, error) {
// creds := make(map[string]StoredCredential)
// raw, err := ioutil.ReadFile(s.filename)
// if err != nil {
// if os.IsNotExist(err) {
// // Doesn't exist yet
// return creds, nil
// }
// log.Warn("Failed to read encrypted storage", "err", err, "file", s.filename)
// }
// if err = json.Unmarshal(raw, &creds); err != nil {
// log.Warn("Failed to unmarshal encrypted storage", "err", err, "file", s.filename)
// return nil, err
// }
// return creds, nil
// }
// readStorage reads the file with encrypted creds and return them as is
func (s *FileStorageAPI) readStorage() (map[string]string, error) {
creds := make(map[string]string)
raw, err := ioutil.ReadFile(s.filename)
if err != nil {
@ -110,17 +107,29 @@ func (s *AESEncryptedStorage) readEncryptedStorage() (map[string]StoredCredentia
// Doesn't exist yet
return creds, nil
}
log.Warn("Failed to read encrypted storage", "err", err, "file", s.filename)
log.Warn("Failed to read file storage", "err", err, "file", s.filename)
}
if err = json.Unmarshal(raw, &creds); err != nil {
log.Warn("Failed to unmarshal encrypted storage", "err", err, "file", s.filename)
log.Warn("Failed to unmarshal file storage", "err", err, "file", s.filename)
return nil, err
}
return creds, nil
}
// writeEncryptedStorage write the file with encrypted creds
func (s *AESEncryptedStorage) writeEncryptedStorage(creds map[string]StoredCredential) error {
// func (s *FileStorageAPI) writeEncryptedStorage(creds map[string]StoredCredential) error {
// raw, err := json.Marshal(creds)
// if err != nil {
// return err
// }
// if err = ioutil.WriteFile(s.filename, raw, 0600); err != nil {
// return err
// }
// return nil
// }
func (s *FileStorageAPI) writeStorage(creds map[string]string) error {
raw, err := json.Marshal(creds)
if err != nil {
return err

68
signer/storage/aes_gcm_storage_test.go → signer/storage/file_storage_test.go
View file

@ -17,42 +17,32 @@
package storage
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/log"
"github.com/mattn/go-colorable"
)
func TestFileStorage(t *testing.T) {
a := map[string]StoredCredential{
"secret": {
Iv: common.Hex2Bytes("cdb30036279601aeee60f16b"),
CipherText: common.Hex2Bytes("f311ac49859d7260c2c464c28ffac122daf6be801d3cfd3edcbde7e00c9ff74f"),
},
"secret2": {
Iv: common.Hex2Bytes("afb8a7579bf971db9f8ceeed"),
CipherText: common.Hex2Bytes("2df87baf86b5073ef1f03e3cc738de75b511400f5465bb0ddeacf47ae4dc267d"),
},
func TestFileStorageAPI(t *testing.T) {
a := map[string]string{
"secret": "value1",
"secret2": "value2",
}
d, err := ioutil.TempDir("", "eth-encrypted-storage-test")
if err != nil {
t.Fatal(err)
}
stored := &AESEncryptedStorage{
stored := &FileStorageAPI{
filename: fmt.Sprintf("%v/vault.json", d),
key: []byte("AES256Key-32Characters1234567890"),
}
stored.writeEncryptedStorage(a)
read := &AESEncryptedStorage{
stored.writeStorage(a)
read := &FileStorageAPI{
filename: fmt.Sprintf("%v/vault.json", d),
key: []byte("AES256Key-32Characters1234567890"),
}
creds, err := read.readEncryptedStorage()
creds, err := read.readStorage()
if err != nil {
t.Fatal(err)
}
@ -60,15 +50,16 @@ func TestFileStorage(t *testing.T) {
if v2, exist := creds[k]; !exist {
t.Errorf("Missing entry %v", k)
} else {
if !bytes.Equal(v.CipherText, v2.CipherText) {
t.Errorf("Wrong ciphertext, expected %x got %x", v.CipherText, v2.CipherText)
if v != v2 {
t.Errorf("Wrong ciphertext, expected %x got %x", v, v2)
}
if !bytes.Equal(v.Iv, v2.Iv) {
if v != v2 {
t.Errorf("Wrong iv")
}
}
}
}
func TestEnd2End(t *testing.T) {
log.Root().SetHandler(log.LvlFilterHandler(log.Lvl(3), log.StreamHandler(colorable.NewColorableStderr(), log.TerminalFormat(true))))
@ -77,17 +68,21 @@ func TestEnd2End(t *testing.T) {
t.Fatal(err)
}
s1 := &AESEncryptedStorage{
filename: fmt.Sprintf("%v/vault.json", d),
key: []byte("AES256Key-32Characters1234567890"),
}
s2 := &AESEncryptedStorage{
filename: fmt.Sprintf("%v/vault.json", d),
key: []byte("AES256Key-32Characters1234567890"),
filename := fmt.Sprintf("%v/vault.json", d)
key := []byte("AES256Key-32Characters1234567890")
fs := NewFileStorage(filename, key)
fs.Put("bazonk", "foobar")
// make sure intermediate result is encrypted correctly
encrypted, err := fs.api.Get("bazonk")
cred := StoredCredential{}
if err = json.Unmarshal([]byte(encrypted), &cred); err != nil {
t.Error("Failed to unmarshal encrypted credential", "err", err)
}
s1.Put("bazonk", "foobar")
if v, err := s2.Get("bazonk"); v != "foobar" || err != nil {
// make sure return is correct
if v, err := fs.Get("bazonk"); v != "foobar" || err != nil {
t.Errorf("Expected bazonk->foobar (nil error), got '%v' (%v error)", v, err)
}
}
@ -102,16 +97,15 @@ func TestSwappedKeys(t *testing.T) {
t.Fatal(err)
}
s1 := &AESEncryptedStorage{
filename: fmt.Sprintf("%v/vault.json", d),
key: []byte("AES256Key-32Characters1234567890"),
}
filename := fmt.Sprintf("%v/vault.json", d)
key := []byte("AES256Key-32Characters1234567890")
s1 := NewFileStorage(filename, key)
s1.Put("k1", "v1")
s1.Put("k2", "v2")
// Now make a modified copy
creds := make(map[string]StoredCredential)
raw, err := ioutil.ReadFile(s1.filename)
creds := make(map[string]string)
raw, err := ioutil.ReadFile(filename)
if err != nil {
t.Fatal(err)
}
@ -126,7 +120,7 @@ func TestSwappedKeys(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if err = ioutil.WriteFile(s1.filename, raw, 0600); err != nil {
if err = ioutil.WriteFile(filename, raw, 0600); err != nil {
t.Fatal(err)
}
}

17
signer/storage/no_storage.go Normal file
View file

@ -0,0 +1,17 @@
package storage
import "errors"
// NoStorageAPI is a dummy construct which doesn't remember anything you tell it
type NoStorageAPI struct{}
// Put is a dummy function that do nothing
func (s *NoStorageAPI) Put(key, value string) {}
// Del is a dummy function that do nothing
func (s *NoStorageAPI) Del(key string) {}
// Get is a dummy function that do nothing
func (s *NoStorageAPI) Get(key string) (string, error) {
return "", errors.New("missing key, I probably forgot")
}

131
signer/storage/storage.go
View file

@ -16,7 +16,13 @@
package storage
import "errors"
import (
"encoding/json"
"errors"
"github.com/ethereum/go-ethereum/cmd/clef/dbutil"
"github.com/ethereum/go-ethereum/log"
)
var (
// ErrZeroKey is returned if an attempt was made to inset a 0-length key.
@ -26,7 +32,8 @@ var (
ErrNotFound = errors.New("not found")
)
type Storage interface {
// storageAPI is the interface that defines interactions with backend storage client
type storageAPI interface {
// Put stores a value by key. 0-length keys results in noop.
Put(key, value string)
@ -38,49 +45,109 @@ type Storage interface {
Del(key string)
}
// EphemeralStorage is an in-memory storage that does
// not persist values to disk. Mainly used for testing
type EphemeralStorage struct {
data map[string]string
// Storage is the storage client which is used by client to store key/value mappings.
// The keys are _not_ encrypted, only the values are.
type Storage struct {
api storageAPI
key []byte
}
// Put stores a value by key. 0-length keys results in noop.
func (s *EphemeralStorage) Put(key, value string) {
if len(key) == 0 {
return
}
s.data[key] = value
}
// Get returns the previously stored value, or an error if the key is 0-length
// or unknown.
func (s *EphemeralStorage) Get(key string) (string, error) {
// Get calls the underlying storageApi's Get function and then decrypts the value field
func (s *Storage) Get(key string) (string, error) {
if len(key) == 0 {
return "", ErrZeroKey
}
if v, ok := s.data[key]; ok {
return v, nil
data, err := s.api.Get(key)
if err != nil {
return "", err
}
return "", ErrNotFound
cred := StoredCredential{}
if err = json.Unmarshal([]byte(data), &cred); err != nil {
log.Warn("Failed to unmarshal encrypted credential", "err", err)
return "", err
}
entry, err := Decrypt(s.key, cred.Iv, cred.CipherText, []byte(key))
if err != nil {
log.Warn("Failed to decrypt key", "key", key)
return "", err
}
return string(entry), nil
}
// Del removes a key-value pair. If the key doesn't exist, the method is a noop.
func (s *EphemeralStorage) Del(key string) {
delete(s.data, key)
// Put encrypts the value field with key as additionalData to prevent value swap attack.
// Then calls the underlying storageApi's Put function to persist the key/value pair
func (s *Storage) Put(key, value string) {
if len(key) == 0 {
return
}
ciphertext, iv, err := Encrypt(s.key, []byte(value), []byte(key))
if err != nil {
log.Warn("Failed to encrypt entry", "err", err)
return
}
encrypted := StoredCredential{Iv: iv, CipherText: ciphertext}
raw, err := json.Marshal(encrypted)
if err != nil {
log.Warn("Failed to marshal credential", "err", err)
return
}
s.api.Put(key, string(raw))
}
// Del calls the underlying storageApi's Del function to delete the key/value pair
func (s *Storage) Del(key string) {
s.api.Del(key)
}
// NewEphemeralStorage creates an in-memory storage that does
// not persist values to disk. Mainly used for testing
func NewEphemeralStorage() Storage {
s := &EphemeralStorage{
api := &EphemeralStorageAPI{
data: make(map[string]string),
}
return s
return Storage{
api: api,
key: []byte(""),
}
}
// NoStorage is a dummy construct which doesn't remember anything you tell it
type NoStorage struct{}
func (s *NoStorage) Put(key, value string) {}
func (s *NoStorage) Del(key string) {}
func (s *NoStorage) Get(key string) (string, error) {
return "", errors.New("missing key, I probably forgot")
// NewNoStorage creates an dummy storage which didn't remember anything you tell it
func NewNoStorage() Storage {
api := &NoStorageAPI{}
return Storage{
api: api,
key: []byte(""),
}
}
// NewDBStorage creates a database storage
func NewDBStorage(path, table string, key []byte) (*Storage, error) {
kvstore, err := dbutil.NewKVStore(path, table)
if err != nil {
return nil, err
}
api := &DBStorageAPI{
kvstore: kvstore,
}
return &Storage{
api: api,
key: key,
}, nil
}
// NewFileStorage creates a storage type which is backed by a json-file.
func NewFileStorage(filename string, key []byte) *Storage {
api := &FileStorageAPI{
filename: filename,
}
return &Storage{
api: api,
key: key,
}
}